YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: 合并远程 ft_wyt 分支，解决冲突

Browse Source
This commit is contained in:
wangjiming
2026-08-19 17:39:18 +08:00
parent 78680a5c5c 080ef6ab00
commit 8455d99d49
64 changed files with 4616 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

301
backend/app/api/v1/endpoints/data_process.py
View File

@@ -35,7 +35,12 @@ from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.core.logging import get_structured_logger
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -61,6 +66,10 @@ from app.modules.data_process.document_chunking import (
chunk_semantic_text,
merge_short_chunks,
)
from app.modules.data_process.evaluation import (
evaluate_result_record,
reevaluate_edited_record,
)
from app.modules.data_process.generation import generate_model_records
from app.modules.data_process.office_preview import (
MAX_XLSX_PREVIEW_ROWS,
@@ -82,6 +91,8 @@ from app.modules.data_process.store import (
new_id,
repeat_task_id,
)
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessRepeatRequest,
@@ -97,6 +108,7 @@ from app.schemas.data_process import (
PreviewItemUpdate,
ProcessType,
PublishRequest,
ResultBatchEvaluateRequest,
ResultBatchRegenerateRequest,
ResultRegenerateRequest,
ResultUpdate,
@@ -221,9 +233,38 @@ def _commit_source_batch(
staged: list[StagedSourceObject],
) -> list[dict[str, Any]]:
storage.publish(staged)
storage_object_ids: list[str] = []
try:
# The source reference remains in the task schema for compatibility,
# while storage_objects provides the authoritative MinIO index.
if get_settings().minio_enabled:
for item in prepared:
reference = str(item.get("storage_object_id") or "")
if not reference.startswith("minio://"):
continue
object_key = storage.object_key(reference)
metadata = get_object_storage().stat(object_key)
object_row = get_platform_store().create_storage_object({
"resource_type": "data_process_source",
"resource_id": task_id,
"version_id": str(item.get("id") or new_id("dpsf")),
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": item.get("name"),
"content_type": (item.get("metadata") or {}).get("content_type", "application/octet-stream"),
"byte_size": metadata.get("byte_size") or item.get("raw_size") or 0,
"checksum_sha256": item.get("checksum_sha256"),
"status": "available",
"created_by": item.get("created_by"),
})
storage_object_ids.append(str(object_row["id"]))
return store.add_source_files(task_id, prepared)
except Exception:
for object_id in storage_object_ids:
try:
get_platform_store().update_storage_object(object_id, {"status": "deleted"})
except Exception:
pass
for item in staged:
try:
storage.delete(item.reference)
@@ -771,6 +812,25 @@ def _run_generation(
duplicate_count=duplicate_count,
error_count=error_count,
)
result_bytes = "".join(
structured_json_dumps(item) + "\n" for item in accepted
).encode("utf-8")
if should_store_in_minio(len(result_bytes)):
result_key = f"data-process/{task_id}/results/{generation_run_id}.jsonl"
uploaded = get_object_storage().put_bytes(result_key, result_bytes, "application/jsonl")
get_platform_store().create_storage_object({
"resource_type": "data_process_result",
"resource_id": task_id,
"version_id": generation_run_id,
"bucket": uploaded["bucket"],
"object_key": result_key,
"file_name": f"{generation_run_id}.jsonl",
"content_type": "application/jsonl",
"byte_size": len(result_bytes),
"checksum_sha256": hashlib.sha256(result_bytes).hexdigest(),
"status": "available",
"created_by": (store.get_task(task_id) or {}).get("created_by"),
})
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
@@ -930,7 +990,7 @@ def _repeat_file_copies(
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
new_file_id = new_id("dpsf")
old_reference = str(source.get("storage_object_id") or "")
if old_reference.startswith("local://data-process/"):
if old_reference.startswith(("local://data-process/", "minio://data-process/")):
staged_object = storage.stage_copy(
batch_id=batch_id,
source_reference=old_reference,
@@ -2044,12 +2104,13 @@ def update_result(
or 20
),
)
quality = score_quality(
# 编辑后内容已变化：重算规则与语义层，旧的评审分不再可信直接丢弃。
update["quality_score"] = reevaluate_edited_record(
merged,
min_output_length=minimum,
source_content=source_content,
previous_quality=current.get("quality_score"),
min_output_length=minimum,
)
update["quality_score"] = asdict(quality)
result = store.update_result(
task_id,
result_id,
@@ -2094,11 +2155,6 @@ def restore_result(
or 20
),
)
quality = score_quality(
restored,
min_output_length=minimum,
source_content=source_content,
)
restored = store.update_result(
task_id,
result_id,
@@ -2108,7 +2164,12 @@ def restore_result(
"output": restored["output"],
"chosen": restored["chosen"],
"rejected": restored["rejected"],
"quality_score": asdict(quality),
"quality_score": reevaluate_edited_record(
restored,
source_content=source_content,
previous_quality=current.get("quality_score"),
min_output_length=minimum,
),
"expected_updated_at": current.get("updated_at"),
},
)
@@ -2271,6 +2332,46 @@ def _safe_regeneration_error(exc: Exception) -> str:
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
def _evaluate_result_in_place(
task_id: str,
current: dict[str, Any],
source_content: str,
config: dict[str, Any],
evaluation_model: dict[str, Any] | None,
store: DataProcessStore,
*,
expected_updated_at: str,
model_client: httpx.Client | None = None,
minimum: int = 20,
) -> dict[str, Any]:
"""评测单条结果并落库；复用逐结果互斥锁避免与重生成并发写冲突。"""
result_id = str(current["id"])
with _claim_result_regeneration(task_id, result_id):
quality = evaluate_result_record(
{
"instruction": current.get("instruction"),
"input": current.get("input"),
"output": current.get("output"),
"chosen": current.get("chosen"),
"rejected": current.get("rejected"),
},
source_content=source_content,
model=evaluation_model,
config=config,
client=model_client,
min_output_length=minimum,
)
return store.update_result(
task_id,
result_id,
{
"quality_score": quality,
"expected_updated_at": expected_updated_at,
},
)
@router.post("/{task_id}/results/regenerate-batch")
def regenerate_results_batch(
task_id: str,
@@ -2444,6 +2545,186 @@ def regenerate_results_batch(
)
@router.post("/{task_id}/results/evaluate-batch")
def evaluate_results_batch(
task_id: str,
payload: ResultBatchEvaluateRequest,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
"""对一批结果执行三层质量评测（规则+语义+评审），允许部分成功。"""
started_at = time.perf_counter()
batch_id = new_id("dpeb")
with api_errors():
task = store.get_task(task_id)
if task.get("status") == "running":
raise ConflictError("data process task is running")
if task.get("output_dataset_id"):
raise InvalidStateError("published results cannot be evaluated")
config = task.get("config") or {}
evaluation_model: dict[str, Any] | None = None
model_id = _value(config, "generation_model_id", "generationModelId", None)
if model_id:
try:
evaluation_model = store.get_generation_model(str(model_id))
except NotFoundError:
logger.warning(
"data process evaluation model unavailable, judge layer "
"skipped task_id=%s model_id=%s",
task_id,
model_id,
)
evaluation_config = {
**config,
"output_type": str(
_value(config, "output_type", "outputType", "standard")
).strip().lower(),
}
minimum = max(
1,
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
)
prepared: list[tuple[int, dict[str, Any], str, str]] = []
failures: list[tuple[int, dict[str, str]]] = []
for index, requested in enumerate(payload.items):
try:
current = store.get_result(task_id, requested.result_id)
if requested.expected_updated_at != str(current.get("updated_at") or ""):
raise ConflictError("data process result was modified by another request")
source_content = ""
preview_id = current.get("preview_item_id")
if preview_id:
preview = store.get_preview_item(task_id, str(preview_id))
source_content = str(
preview.get("edited_content")
or preview.get("original_content")
or ""
)
prepared.append(
(index, current, source_content, requested.expected_updated_at)
)
except ConflictError as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "conflict",
"message": _safe_regeneration_error(exc),
}))
except (NotFoundError, InvalidStateError) as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "skipped",
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation started batch_id=%s task_id=%s "
"requested=%s prepared=%s judge_enabled=%s",
batch_id,
task_id,
len(payload.items),
len(prepared),
evaluation_model is not None,
)
successes: list[tuple[int, dict[str, Any]]] = []
if prepared:
try:
from app.modules.data_process.algorithms.embedding import (
semantic_embedding_model,
)
semantic_embedding_model()
except Exception:
logger.warning(
"data process semantic embedding unavailable, semantic layer "
"will be skipped batch_id=%s",
batch_id,
)
request_timeout = _result_regeneration_timeout(config)
model_timeout = httpx.Timeout(
request_timeout,
connect=min(10.0, request_timeout),
)
model_limits = httpx.Limits(
max_connections=RESULT_REGENERATION_CONCURRENCY,
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-evaluation",
) as executor:
futures = {
executor.submit(
_evaluate_result_in_place,
task_id,
current,
source_content,
evaluation_config,
evaluation_model,
store,
expected_updated_at=expected_updated_at,
model_client=model_client if evaluation_model else None,
minimum=minimum,
): (index, str(current["id"]), time.perf_counter())
for index, current, source_content, expected_updated_at in prepared
}
for future in as_completed(futures):
index, result_id, item_started_at = futures[future]
try:
evaluated = future.result()
successes.append((index, evaluated))
outcome = "succeeded"
except ConflictError as exc:
outcome = "conflict"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except Exception as exc:
outcome = "evaluation_failed"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation item finished "
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
outcome,
(time.perf_counter() - item_started_at) * 1000,
)
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch evaluation completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
len(failure_items),
duration_ms,
)
return ok(
{
"batch_id": batch_id,
"total": len(payload.items),
"succeeded": len(success_items),
"failed": len(failure_items),
"duration_ms": round(duration_ms, 2),
"items": success_items,
"failures": failure_items,
},
"data process results evaluated",
)
@router.post("/{task_id}/results/{result_id}/regenerate")
def regenerate_result(
task_id: str,

417
backend/app/api/v1/endpoints/platform.py
View File

@@ -23,8 +23,9 @@ from app.core.audit import audit_log, AuditActions
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
from app.modules.compute_gateway.sync import _archive_node_directory, fetch_eval_result_content, poll_compute_jobs_once
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
@@ -57,6 +58,33 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
return None
def _normalize_gpu_indices(payload: dict[str, Any], *, allow_primary: bool = True) -> list[int]:
"""Normalize all frontend GPU selection shapes to sorted integer indexes."""
raw = payload.get("gpu_indices")
if raw is None:
raw = payload.get("gpus")
if raw is None and allow_primary and payload.get("gpu_id") is not None:
raw = [payload.get("gpu_id")]
if raw is None or raw == "":
return []
if isinstance(raw, str):
raw = [item.strip() for item in raw.split(",") if item.strip()]
if not isinstance(raw, (list, tuple, set)):
raw = [raw]
result: set[int] = set()
for item in raw:
if isinstance(item, str) and ":" in item:
item = item.rsplit(":", 1)[-1]
try:
index = int(item)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid GPU index: {item}") from exc
if index < 0:
raise ValueError("GPU index must be non-negative")
result.add(index)
return sorted(result)
async def _wait_for_object_storage() -> None:
"""Wait for MinIO before starting a resource task."""
settings = get_settings()
@@ -73,6 +101,106 @@ async def _wait_for_object_storage() -> None:
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
def _store_json_snapshot(
store: Any,
resource_type: str,
resource_id: str,
version_id: str,
object_key: str,
payload: dict[str, Any],
created_by: str | None = None,
) -> dict[str, Any]:
"""Persist non-secret task/model parameters as an auditable MinIO snapshot."""
sanitized = {
key: value
for key, value in payload.items()
if key not in {"api_key", "secret_key", "password", "token", "access_token"}
}
raw = json.dumps(sanitized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
# Task/evaluation payloads are already persisted in PostgreSQL. Avoid an
# extra MinIO round trip for small non-secret parameter snapshots.
if not should_store_in_minio(len(raw), content_type="application/json", file_format="json"):
return {}
uploaded = get_object_storage().put_bytes(object_key, raw, "application/json")
return store.create_storage_object({
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": Path(object_key).name,
"content_type": "application/json",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
"created_by": created_by,
})
def _dataset_file_bytes(store: Any, file_id: str) -> bytes:
"""Return the canonical dataset bytes, lazily indexing legacy DB content."""
row = store.dataset_file(file_id)
if not get_settings().minio_enabled:
return str(row.get("content") or "").encode("utf-8")
storage_object = None
object_id = str(row.get("storage_object_id") or "")
if object_id:
try:
storage_object = store.storage_object(object_id)
except KeyError:
storage_object = None
if storage_object and storage_object.get("status") == "available":
try:
return get_object_storage().get_bytes(storage_object["object_key"])
except Exception as exc: # noqa: BLE001 - expose storage outage to callers
raise RuntimeError(f"dataset object is unavailable in MinIO: {exc}") from exc
# Compatibility migration for files created before MinIO was enabled.
raw = str(row.get("content") or "").encode("utf-8")
if not raw:
raise RuntimeError(f"dataset file has no MinIO object or legacy content: {file_id}")
# Small files intentionally remain database-backed. They can still be
# copied to a compute node directly when a task needs them.
if not should_store_in_minio(len(raw), file_format=row.get("file_format")):
return raw
object_key = (
f"datasets/{row['dataset_id']}/versions/"
f"{row.get('active_version_id') or row['id']}/{Path(str(row.get('name') or row['id'])).name}"
)
uploaded = get_object_storage().put_bytes(object_key, raw, "application/octet-stream")
created = store.create_storage_object({
"resource_type": "dataset",
"resource_id": str(row["dataset_id"]),
"version_id": str(row.get("active_version_id") or row["id"]),
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": row.get("name"),
"content_type": "application/octet-stream",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(row["id"]), created["id"])
return raw
def _dataset_version_bytes(store: Any, file_id: str, version_id: str) -> bytes:
row = store.dataset_file(file_id)
try:
version = next(item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id)
except StopIteration as exc:
raise KeyError(version_id) from exc
object_id = str(version.get("storage_object_id") or "")
if get_settings().minio_enabled and object_id:
try:
obj = store.storage_object(object_id)
return get_object_storage().get_bytes(obj["object_key"])
except KeyError:
pass
return _dataset_file_bytes(store, file_id)
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
"""Select the compute node for an eval job.
@@ -103,18 +231,63 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id:
if not get_settings().minio_enabled or not resource_id:
return None
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects and resource_type in {"model", "trained_model"}:
resource = None
if resource_type == "model":
try:
resource = store.model(resource_id)
except KeyError:
try:
resource = store.model_by_name(resource_id)
except KeyError:
resource = next((item for item in store.models() if item.get("path") == resource_id), None)
else:
resource = next(
(item for item in store.trained_models() if item.get("id") == resource_id or item.get("name") == resource_id),
None,
)
source_path = str(
(resource or {}).get("path")
or (resource or {}).get("merged_path")
or (resource or {}).get("artifact_dir")
or ""
)
resolved_id = str((resource or {}).get("id") or resource_id)
if source_path and resolved_id:
client = ComputeNodeClient(node["api_base_url"], timeout=900)
await _archive_node_directory(
store,
client,
node,
source_path,
resource_type,
resolved_id,
"legacy-import",
f"models/{resolved_id}" if resource_type == "model" else f"trained_models/{resolved_id}",
)
resource_id = resolved_id
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
return None
client = ComputeNodeClient(node["api_base_url"], timeout=900)
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
for obj in objects:
object_key = str(obj["object_key"])
marker = f"{root_name}/{resource_id}/versions/"
relative_name = Path(str(obj.get("file_name") or object_key)).name
if marker in object_key:
suffix = object_key.split(marker, 1)[1]
if "/" in suffix:
suffix = suffix.split("/", 1)[1]
if suffix:
relative_name = suffix
await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": get_object_storage().presigned_get(obj["object_key"]),
"download_url": get_object_storage().presigned_get(object_key),
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
"relative_path": f"{root_name}/{resource_id}/{relative_name}",
})
return f"/data/yg-ft/{root_name}/{resource_id}"
@@ -370,8 +543,23 @@ async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[st
if not preflight["valid"]:
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
raise RuntimeError(f"preflight failed: {errors}")
payload = {**payload, "compute_node_id": preflight["node"]["id"]}
prepared_job_payload = preflight.get("job_payload") or {}
payload = {
**payload,
"compute_node_id": preflight["node"]["id"],
"prepared_base_model_path": prepared_job_payload.get("model_name_or_path") or payload.get("prepared_base_model_path"),
}
task = store.start_task(payload)
if get_settings().minio_enabled:
_store_json_snapshot(
store,
"fine_tune",
str(task["id"]),
str(task["id"]),
f"training/{task['id']}/versions/{task['id']}/training-config.json",
task,
task.get("created_by"),
)
if get_settings().compute_mode == "simulator":
return task
node, job_payload = store.build_compute_job_payload(task["id"])
@@ -427,6 +615,20 @@ async def _fine_tune_preflight_with_job_payload(
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
base_model_path = str(job_payload.get("model_name_or_path") or job_payload.get("base_model") or "")
base_model_id = str(job_payload.get("base_model_id") or job_payload.get("model_id") or "")
base_model = None
if base_model_id:
try:
base_model = store.model(base_model_id)
except KeyError:
base_model = None
if base_model is None:
base_model = next((item for item in store.models() if item.get("path") == base_model_path), None)
if base_model:
prepared_model = await _prepare_resource_on_node(store, "model", str(base_model["id"]), node)
if prepared_model:
job_payload = {**job_payload, "base_model": prepared_model, "model_name_or_path": prepared_model}
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
sync_errors.append(f"shared storage health check failed: {exc}")
if get_settings().compute_mode == "simulator":
@@ -1076,8 +1278,9 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict =
@router.get("/dataset-manage/preview/{file_id}")
async def dataset_preview(file_id: str) -> dict[str, Any]:
try:
row = get_platform_store().dataset_file(file_id)
return ok({"content": row["content"]})
store = get_platform_store()
content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace")
return ok({"content": content})
except KeyError:
raise fail(404, "dataset file not found")
@@ -1106,7 +1309,8 @@ async def dataset_version_content(file_id: str, version_id: str) -> dict[str, An
version = next((item for item in versions if item["id"] == version_id), None)
if not version:
raise KeyError(version_id)
return ok({"version": version, "content": row["content"]})
content = _dataset_version_bytes(get_platform_store(), file_id, version_id)
return ok({"version": version, "content": content.decode("utf-8", errors="replace")})
except KeyError:
raise fail(404, "dataset version not found")
@@ -1197,7 +1401,7 @@ async def _sync_training_dataset_to_compute_node(
) -> list[dict[str, Any]]:
if get_settings().minio_enabled:
files = store.training_dataset_files(dataset_id)
object_by_resource_name: dict[tuple[str, str], dict[str, Any]] = {}
object_by_resource_name: dict[tuple[str, str, str], dict[str, Any]] = {}
resource_ids = {str(dataset_id)} | {
str(item.get("dataset_id"))
for item in files
@@ -1206,18 +1410,19 @@ async def _sync_training_dataset_to_compute_node(
for resource_id in resource_ids:
for obj in store.storage_objects_for_resource("dataset", resource_id):
file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
object_by_resource_name[(resource_id, file_name)] = obj
object_by_resource_name[(resource_id, file_name, str(obj.get("version_id") or ""))] = obj
results: list[dict[str, Any]] = []
client = ComputeNodeClient(node["api_base_url"])
for item in files:
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
item_dataset_id = str(item.get("dataset_id") or dataset_id)
obj = object_by_resource_name.get((item_dataset_id, target_name))
if not obj and item.get("content"):
# 兼容 MinIO 接入前已经发布的数据处理数据集：
# 预检时用数据库正文补建对象，避免要求用户重新处理数据集。
raw = str(item.get("content") or "").encode("utf-8")
version_id = str(item.get("active_version_id") or item["id"])
obj = object_by_resource_name.get((item_dataset_id, target_name, version_id))
if not obj and item.get("content"):
# 兼容 MinIO 接入前已经发布的数据处理数据集。大文件补建
# MinIO 对象，小文件直接从数据库正文同步到目标节点。
raw = str(item.get("content") or "").encode("utf-8")
if should_store_in_minio(len(raw)):
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
obj = store.create_storage_object({
@@ -1233,8 +1438,30 @@ async def _sync_training_dataset_to_compute_node(
"status": "available",
})
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
else:
result = await client.upload_file(
target_name,
raw,
f"datasets/{dataset_id}/{target_name}",
resource_type="dataset",
resource_id=dataset_id,
)
store.upsert_resource_replica(
node["id"], "dataset", dataset_id, str(result.get("local_path") or "")
)
results.append({
"node_id": node["id"],
"node_code": node.get("code"),
"file_id": item.get("id"),
"name": target_name,
"local_path": result.get("local_path"),
"byte_size": result.get("byte_size"),
"checksum_sha256": result.get("checksum_sha256"),
"storage_backend": "database",
})
continue
if not obj:
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
raise RuntimeError(f"dataset file is not available: {target_name}")
url = get_object_storage().presigned_get(obj["object_key"])
result = await client.prepare_cache({
"resource_id": dataset_id,
@@ -1250,7 +1477,13 @@ async def _sync_training_dataset_to_compute_node(
dataset_id,
str(result.get("local_path") or ""),
)
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
results.append({
**result,
"file_id": item.get("id"),
"name": target_name,
"node_id": node["id"],
"storage_backend": "minio",
})
return results
if not dataset_id:
raise RuntimeError("train_dataset_id is required")
@@ -1316,7 +1549,11 @@ async def upload_dataset_files(
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
created.append(created_file)
pending_sync.append((created_file["id"], created_file["name"], raw))
if get_settings().minio_enabled:
if should_store_in_minio(
len(raw),
content_type=file.content_type,
file_format=Path(created_file["name"]).suffix,
):
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
storage_object = get_platform_store().create_storage_object({
@@ -1360,7 +1597,10 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
full_file = store.dataset_file(str(item["id"]))
except KeyError:
continue
files.append({**item, "content": full_file.get("content") or ""})
files.append({
**item,
"content": _dataset_file_bytes(store, str(item["id"])).decode("utf-8", errors="replace"),
})
if not files:
raise fail(404, "dataset has no downloadable files")
@@ -1401,8 +1641,12 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
row = get_platform_store().dataset_file(file_id)
return PlainTextResponse(row["content"], media_type="text/plain")
store = get_platform_store()
row = store.dataset_file(file_id)
if str(row.get("dataset_id")) != str(dataset_id):
raise fail(404, "dataset file not found")
content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id)
return PlainTextResponse(content.decode("utf-8", errors="replace"), media_type="text/plain")
@router.get("/dataset-manage")
@@ -1545,10 +1789,10 @@ async def start_fine_tune(
if node_id and gpu_indices:
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
raise fail(403, "无权使用所选 GPU，请联系管理员分配")
# 记录创建者
if node_id and not gpu_indices:
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
payload["strict_node_selection"] = bool(node_id)
# 页面明确选择节点时，调度器必须保持节点约束；否则可能落到其它节点。
payload["strict_node_selection"] = bool(payload.get("compute_node_id") or payload.get("node_id"))
payload.setdefault("created_by", current_user.get("id"))
try:
return ok(await _submit_fine_tune_task(store, payload))
@@ -1679,6 +1923,42 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
)
@router.get("/fine-tune/{task_id}/gpu-status")
async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
"""Return live GPU metrics for the task's selected node and cards."""
store = get_platform_store()
try:
task = store.task(task_id)
except KeyError:
raise fail(404, "fine tune task not found")
if not has_resource_access("fine-tune", task_id, current_user, "read"):
raise fail(403, "no permission to access this task")
node = _node_for_task(task)
selected = set(_normalize_gpu_indices({"gpus": task.get("gpus") or []}, allow_primary=False))
if not node:
return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected)})
try:
if get_settings().compute_mode == "simulator":
live_items = store.gpus()
else:
live_items = await ComputeNodeClient(node["api_base_url"]).gpu_resources()
items = []
for item in live_items:
index = int(item.get("gpu_index", item.get("id", -1)))
if selected and index not in selected:
continue
items.append({
**item,
"id": index,
"node_id": node["id"],
"node_code": node.get("code"),
"node_name": node.get("name"),
})
return ok({"source": "compute", "items": items, "selected_gpus": sorted(selected)})
except Exception as exc: # noqa: BLE001 - let UI retain last good snapshot
return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected), "error": str(exc)})
@router.put("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
@@ -1828,15 +2108,35 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Start an evaluation task: submit eval job to compute node."""
store = get_platform_store()
try:
gpu_indices = _normalize_gpu_indices(payload)
except ValueError as exc:
raise fail(400, str(exc))
if not gpu_indices:
raise fail(400, "请选择至少一张 GPU")
# 1. Create eval task record
payload.setdefault("created_by", current_user.get("id"))
task = store.create_eval_task({**payload, "status": "pending"})
if get_settings().minio_enabled:
_store_json_snapshot(
store,
"eval",
str(task["id"]),
str(task["id"]),
f"evaluations/{task['id']}/versions/{task['id']}/evaluation-config.json",
payload,
current_user.get("id"),
)
# 2. Resolve model path (supports both regular models and trained models)
model_id = str(payload.get("model_id", ""))
model_path = ""
adapter_path = payload.get("adapter_path", "")
model_node_id = ""
ds_files: list[dict[str, Any]] = []
model_resource_type = "model"
model_resource_id = model_id
adapter_resource_id = ""
try:
db_model = store.model(model_id)
model_path = db_model.get("path", "")
@@ -1845,6 +2145,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
# Try trained_models table (IDs prefixed with tm_)
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
if trained:
model_resource_type = "trained_model" if trained.get("merged") else "model"
model_resource_id = trained.get("id") or model_id
adapter_resource_id = trained.get("id") or ""
model_node_id = trained.get("compute_node_id") or ""
merged_path = trained.get("merged_path", "")
base_path = trained.get("base_model_path", "")
@@ -1858,6 +2161,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
adapter_path = merged_path
else:
model_path = merged_path or base_path
if model_resource_type == "model":
base_model = next((item for item in store.models() if item.get("path") == base_path), None)
model_resource_id = str((base_model or {}).get("id") or base_path)
if not model_path:
store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})
@@ -1930,6 +2236,33 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
store.update_eval_task(task["id"], {"status": "failed", "error": message})
return ok({"task_id": task["id"], "status": "failed", "error": message})
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node)
if prepared_model:
model_path = prepared_model
if adapter_resource_id and model_resource_type == "model":
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", adapter_resource_id, node)
if prepared_adapter:
adapter_path = prepared_adapter
dataset_sync = await _sync_training_dataset_to_compute_node(store, node, dataset_id)
if dataset_sync:
dataset_path = str(dataset_sync[0].get("local_path") or dataset_path)
except Exception as exc:
store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"})
return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})
node_gpus = {
int(item.get("id", item.get("gpu_index", -1))): item
for item in store.gpus()
if item.get("node_id") == node["id"]
}
unavailable = [index for index in gpu_indices if node_gpus.get(index, {}).get("status") != "idle"]
if unavailable:
message = f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}"
store.update_eval_task(task["id"], {"status": "failed", "error": message})
return ok({"task_id": task["id"], "status": "failed", "error": message})
# 6. Build eval job payload
output_dir = f"/data/yg-ft/outputs/{task['id']}"
job_payload = {
@@ -1943,7 +2276,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
"output_dir": output_dir,
"basic_metrics": payload.get("basic_metrics", {}),
"dimension": dimension_cfg,
"gpus": [int(payload.get("gpu_id", 0))],
"gpu_id": gpu_indices[0],
"gpu_indices": gpu_indices,
"gpus": gpu_indices,
"temperature": payload.get("temperature", 0.1),
"max_new_tokens": payload.get("max_new_tokens", 512),
"compute_node_id": node["id"],
@@ -1991,7 +2326,10 @@ async def model_eval_delete(task_id: str, current_user: dict = Depends(get_curre
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
if pending:
return pending
try:
get_platform_store().delete_eval_task(task_id)
except KeyError:
raise fail(404, "eval task not found")
return ok({"deleted": task_id})
@@ -2247,6 +2585,15 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr
"model_name_or_path": model_path,
"template": item.get("template", "qwen"),
}
try:
item_gpu_indices = _normalize_gpu_indices(item)
except ValueError as exc:
loaded_models.append({**item, "status": "error", "error": str(exc)})
continue
if not item_gpu_indices:
loaded_models.append({**item, "status": "error", "error": "no GPU selected"})
continue
load_payload["gpu_indices"] = item_gpu_indices
if item.get("adapter_path"):
load_payload["adapter_name_or_path"] = item["adapter_path"]
if get_settings().compute_mode == "simulator":
@@ -2255,14 +2602,28 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr
# 只派发：HTTP 响应成功即视为已接受（节点会异步加载），loaded 字段忽略
item_dispatched = False
errors = []
for node in _candidate_online_nodes(store, preferred_node_id):
candidate_nodes = _candidate_online_nodes(store, preferred_node_id)
if preferred_node_id:
candidate_nodes = candidate_nodes[:1]
for node in candidate_nodes:
try:
node_gpu_map = {
int(gpu.get("id", gpu.get("gpu_index", -1))): gpu
for gpu in store.gpus()
if gpu.get("node_id") == node["id"]
}
unavailable = [
index for index in item_gpu_indices
if node_gpu_map.get(index, {}).get("status") != "idle"
]
if unavailable:
raise RuntimeError(f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}")
if get_settings().minio_enabled:
await _wait_for_object_storage()
client = ComputeNodeClient(node["api_base_url"])
await client.inference_load(load_payload)
store.mark_inference_loaded(node["id"])
loaded_models.append({**item, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
store.mark_inference_loaded(node["id"], item_gpu_indices)
loaded_models.append({**item, "gpu_indices": item_gpu_indices, "gpus": item_gpu_indices, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
item_dispatched = True
break
except Exception as exc: # noqa: BLE001 - try next candidate node
@@ -2365,7 +2726,7 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[
# 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
result = await client.inference_load(payload)
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
store.mark_inference_loaded(node["id"])
store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})
@@ -2445,7 +2806,7 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), curren
# 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
result = await client.inference_load({**payload, "compute_node_id": node["id"]})
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
store.mark_inference_loaded(node["id"])
store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})

13
backend/app/core/audit.py
View File

@@ -61,6 +61,7 @@ def audit_log(
detail = _build_detail(detail_template, kwargs)
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=target_id,
detail=detail,
@@ -87,6 +88,7 @@ def audit_log(
detail = _build_detail(detail_template, kwargs)
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=target_id,
detail=detail,
@@ -136,6 +138,7 @@ def _build_detail(template: str, kwargs: dict) -> str:
def _record_audit(
action: str,
actor_id: Optional[str],
target_type: str,
target_id: Optional[str],
detail: str,
@@ -149,6 +152,7 @@ def _record_audit(
store = get_platform_store()
store.record_audit(
action=action,
actor_id=actor_id,
target_type=target_type or None,
target_id=target_id,
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
@@ -157,6 +161,15 @@ def _record_audit(
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
def _extract_actor_id(kwargs: dict) -> Optional[str]:
"""从 FastAPI 注入的当前用户中提取操作人 ID。"""
for key in ("current_user", "user"):
value = kwargs.get(key)
if isinstance(value, dict) and value.get("id"):
return str(value["id"])
return None
# ==================== 预定义的审计操作常量 ====================
class AuditActions:

3
backend/app/core/config.py
View File

@@ -68,6 +68,9 @@ class Settings:
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
minio_secure: bool = _bool_env("MINIO_SECURE", False)
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
# MinIO round trips. Larger files remain the shared canonical objects.
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")

140
backend/app/db/platform_store.py
View File

@@ -454,15 +454,21 @@ class PlatformStore:
self.ensure_seed_data()
# Track which compute nodes have an active inference model loaded
self._inference_nodes: set[str] = set()
self._inference_gpu_indexes: dict[str, set[int]] = {}
self._last_runtime_refresh = 0.0
# ── inference node tracking ────────────────────────────────────
def mark_inference_loaded(self, node_id: str) -> None:
def mark_inference_loaded(self, node_id: str, gpu_indexes: list[int] | None = None) -> None:
self._inference_nodes.add(node_id)
if gpu_indexes is not None:
self._inference_gpu_indexes[node_id] = {int(item) for item in gpu_indexes}
else:
self._inference_gpu_indexes.pop(node_id, None)
def mark_inference_unloaded(self, node_id: str) -> None:
self._inference_nodes.discard(node_id)
self._inference_gpu_indexes.pop(node_id, None)
def is_inference_loaded(self, node_id: str) -> bool:
return node_id in self._inference_nodes
@@ -545,6 +551,16 @@ class PlatformStore:
"last_error": "TEXT",
},
)
self._ensure_columns(
conn,
"storage_objects",
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
)
self._ensure_columns(
conn,
"model_artifacts",
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
)
schema_dir = Path(__file__).with_name("sql")
for extra in (
"002_governance.sql",
@@ -556,7 +572,17 @@ class PlatformStore:
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
# data_convert_tasks 表补充 created_by 字段（用于数据隔离）
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
self._ensure_columns(
conn,
"data_convert_tasks",
{
"created_by": "TEXT",
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
"output_storage_object_id": "TEXT",
"output_content": "TEXT",
},
)
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
# 修复历史数据：将 data_convert_tasks.created_by 回填到关联的 datasets 记录
try:
conn.execute("""
@@ -1254,6 +1280,13 @@ class PlatformStore:
raise KeyError(artifact_id)
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
with self.connect() as conn:
conn.execute(
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
(storage_object_id, artifact_id),
)
def model_lineage(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
parents = conn.execute(
@@ -2221,7 +2254,7 @@ class PlatformStore:
(str(validation_dataset_id),),
).fetchall(),
]
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
"dataset_split"
@@ -2482,12 +2515,18 @@ class PlatformStore:
def eval_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
# 评测任务采用软删除，普通列表不得再次返回已删除记录。
rows = conn.execute(
"SELECT * FROM eval_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC"
).fetchall()
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
def eval_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
row = conn.execute(
"SELECT * FROM eval_tasks WHERE id=? AND deleted_at IS NULL",
(task_id,),
).fetchone()
if not row:
raise KeyError(task_id)
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
@@ -2558,7 +2597,13 @@ class PlatformStore:
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
result = conn.execute(
"UPDATE eval_tasks SET deleted_at=?, deleted_by=? "
"WHERE id=? AND deleted_at IS NULL RETURNING id",
(utcnow(), "system", task_id),
)
if not result.fetchone():
raise KeyError(task_id)
def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running."""
@@ -2766,7 +2811,35 @@ class PlatformStore:
"SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')",
(node_id,),
).fetchall()
return {int(row["gpu_index"]) for row in rows}
active = {int(row["gpu_index"]) for row in rows}
# Evaluation jobs use the same Compute ProcessManager GPU lock but do
# not have fine-tune allocation rows; derive their selected cards here
# so a training task cannot race onto an evaluation GPU.
for row in conn.execute(
"SELECT payload FROM eval_tasks WHERE status IN ('syncing','queued','running')"
).fetchall():
payload = json_loads(row["payload"], {})
if payload.get("compute_node_id") != node_id:
continue
selected = payload.get("gpu_indices") or payload.get("gpus")
if selected is None and payload.get("gpu_id") is not None:
selected = [payload.get("gpu_id")]
active.update(int(item) for item in selected or [])
# Loaded inference models also reserve only their selected cards.
active.update(self._inference_gpu_indexes.get(node_id, set()))
for row in conn.execute("SELECT payload FROM compare_tasks").fetchall():
payload = json_loads(row["payload"], {})
load_status = payload.get("load_status") or {}
if isinstance(load_status, str):
load_status = json_loads(load_status, {})
for item in load_status.get("loaded_models") or []:
if item.get("node_id") != node_id or item.get("status") not in {"starting", "ready", "running"}:
continue
selected = item.get("gpu_indices") or item.get("gpus")
if selected is None and item.get("gpu_id") is not None:
selected = [item.get("gpu_id")]
active.update(int(gpu) for gpu in selected or [])
return active
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
@@ -3000,17 +3073,18 @@ class PlatformStore:
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
json_dumps(payload.get("metadata") or {}),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
@@ -3049,6 +3123,13 @@ class PlatformStore:
).fetchall()
return [dict(row) for row in rows]
def storage_object(self, object_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
@@ -3195,6 +3276,9 @@ class PlatformStore:
# 推理模型占用算力节点同样计入：优先从 compare_tasks 持久化状态派生
# （重启后仍准确），并用内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3204,8 +3288,13 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
for nid in inference_node_ids:
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
for nid in set(inference_node_ids) | set(inference_gpu_indexes):
running_map[nid] = running_map.get(nid, 0) + 1
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
return [
@@ -3434,6 +3523,9 @@ class PlatformStore:
# 推理模型占用的节点：优先从 compare_tasks 持久化状态派生（重启后仍准确），
# 内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3443,7 +3535,12 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
items = []
for row in rows:
task = next(
@@ -3459,7 +3556,10 @@ class PlatformStore:
t
for t in eval_running
if t.get("compute_node_id") == row["node_id"]
and row["gpu_index"] == (int(t["gpu_id"]) if t.get("gpu_id") is not None else -1)
and row["gpu_index"] in {
int(item)
for item in (t.get("gpu_indices") or t.get("gpus") or ([t["gpu_id"]] if t.get("gpu_id") is not None else []))
}
),
None,
)
@@ -3468,7 +3568,8 @@ class PlatformStore:
eval_task is not None and eval_task.get("status") in {"syncing", "queued"}
)
# Also mark GPU as busy if an inference model is loaded on this node
if row["node_id"] in inference_node_ids and not busy:
inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set())
if inference_on_gpu and not busy:
busy = True
reserved = False
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
@@ -3704,6 +3805,8 @@ class PlatformStore:
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
keyword: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
@@ -3726,6 +3829,13 @@ class PlatformStore:
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if target_id:
clauses.append("target_id=?")
params.append(target_id)
if keyword:
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
pattern = f"%{keyword}%"
params.extend([pattern, pattern])
if start_time:
clauses.append("time>=?")
params.append(start_time)

11
backend/app/db/sql/000_full_init.sql
View File

@@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts (
path TEXT NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
checksum_sha256 TEXT,
storage_object_id TEXT,
storage_backend TEXT NOT NULL DEFAULT 'minio',
metadata TEXT NOT NULL,
compute_job_id TEXT,
create_time TEXT NOT NULL
@@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects (
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
metadata TEXT NOT NULL DEFAULT '{}',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
@@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
@@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks (
input_count INTEGER NOT NULL DEFAULT 0,
output_count INTEGER NOT NULL DEFAULT 0,
error_message TEXT,
output_content TEXT,
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
deleted_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员

4
backend/app/modules/compute_gateway/client.py
View File

@@ -240,6 +240,10 @@ class ComputeNodeClient:
async def inference_status(self) -> dict[str, Any]:
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
async def gpu_resources(self) -> list[dict[str, Any]]:
"""Read live per-GPU metrics from this compute node."""
return await self.gpus()
async def inference_unload(self) -> dict[str, Any]:
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)

131
backend/app/modules/compute_gateway/sync.py
View File

@@ -2,15 +2,75 @@ from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any
from app.db.platform_store import get_platform_store
from app.core.config import get_settings
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.storage.minio_store import get_object_storage
# starting 状态允许的最大轮询次数（约 40 * 3s ≈ 2 分钟），超过即判定节点不可达
MAX_STARTING_ATTEMPTS = 40
async def _archive_node_directory(
store: Any,
client: ComputeNodeClient,
node: dict[str, Any],
source_path: str,
resource_type: str,
resource_id: str,
version_id: str,
object_prefix: str,
) -> list[dict[str, Any]]:
"""Archive a completed node directory to MinIO, preserving subdirectories."""
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
source = Path(source_path).resolve()
try:
relative_root = source.relative_to(data_root).as_posix()
except ValueError as exc:
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
queue = [relative_root]
archived: list[dict[str, Any]] = []
while queue:
relative = queue.pop(0)
listing = await client.list_files(root="data", relative_path=relative)
for item in listing.get("items") or []:
item_relative = str(item.get("relative_path") or "")
if item.get("type") == "directory":
queue.append(item_relative)
continue
path = str(item.get("path") or "")
if not path:
continue
try:
relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix()
except ValueError:
relative_file = Path(str(item.get("name") or Path(path).name)).name
object_key = f"{object_prefix}/{version_id}/{relative_file}"
upload_url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, upload_url, object_key)
metadata = get_object_storage().stat(object_key)
archived.append(
store.create_storage_object(
{
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": relative_file,
"content_type": "application/octet-stream",
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0,
"checksum_sha256": result.get("checksum_sha256") or "",
"status": "available",
}
)
)
return archived
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
@@ -73,7 +133,8 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
if node_status == "ready":
item["status"] = "ready"
item.pop("error", None)
store.mark_inference_loaded(node["id"])
selected_gpus = item.get("gpu_indices") or item.get("gpus")
store.mark_inference_loaded(node["id"], selected_gpus)
elif node_status == "error":
item["status"] = "error"
item["error"] = status.get("error") or "model load failed on compute node"
@@ -138,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
updated_task = store.apply_compute_job(task["id"], job)
if (
get_settings().minio_enabled
and job.get("status") == "completed"
and job.get("output_dir")
):
trained_model = next(
(
item
for item in store.trained_models()
if item.get("name")
== (task.get("output_model_name") or f"{task.get('name')}-lora")
),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
synced.append(updated_task)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
@@ -150,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
try:
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
payload = (store.compute_job(record["id"]).get("payload") or {})
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "")
if trained_model_id:
trained_model = next(
(item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
ComputeNodeClient(node["api_base_url"], timeout=900),
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or record["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"job_id": record["id"], "error": str(exc)})
@@ -174,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"eval",
str(eval_task["id"]),
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
f"evaluations/{eval_task['id']}",
)
# 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记
eval_synced += 1
except Exception as exc: # noqa: BLE001

287
backend/app/modules/data_convert/router.py
View File

@@ -1,17 +1,21 @@
from __future__ import annotations
import json
import hashlib
import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi.responses import FileResponse
from fastapi.responses import FileResponse, Response
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.config import get_settings
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, new_id
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
@@ -56,6 +60,142 @@ def _output_dir(task_id: str) -> Path:
return _task_dir(task_id) / "output"
def _minio_enabled() -> bool:
return bool(get_settings().minio_enabled)
def _input_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/input/{Path(name).name}"
def _output_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/output/{Path(name).name}"
def _task_objects(task_id: str) -> list[dict[str, Any]]:
return get_platform_store().storage_objects_for_resource("data_convert", task_id)
def _register_object(
task_id: str,
*,
version_id: str,
object_key: str,
file_name: str,
content_type: str,
content: bytes,
created_by: str | None,
) -> dict[str, Any]:
storage = get_object_storage()
uploaded = storage.put_bytes(object_key, content, content_type)
return get_platform_store().create_storage_object(
{
"resource_type": "data_convert",
"resource_id": task_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": file_name,
"content_type": content_type,
"byte_size": len(content),
"checksum_sha256": hashlib.sha256(content).hexdigest(),
"status": "available",
"created_by": created_by,
}
)
def _input_objects(task_id: str) -> list[dict[str, Any]]:
prefix = f"data-convert/{task_id}/input/"
return sorted(
[item for item in _task_objects(task_id) if str(item.get("object_key") or "").startswith(prefix)],
key=lambda item: str(item.get("file_name") or item.get("object_key") or ""),
)
def _output_object(task: dict[str, Any]) -> dict[str, Any] | None:
key = _output_object_key(task["id"], _safe_output_filename(task.get("output_filename")))
return next((item for item in _task_objects(task["id"]) if item.get("object_key") == key), None)
def _read_output(task: dict[str, Any]) -> bytes | None:
if _minio_enabled():
item = _output_object(task)
if item:
return get_object_storage().get_bytes(item["object_key"])
inline = task.get("output_content")
return str(inline).encode("utf-8") if inline is not None else None
path = _task_output_path(task)
return path.read_bytes() if path.exists() else None
def _convert_from_minio(task: dict[str, Any], created_by: str | None) -> tuple[int, int, bytes]:
output_name = _safe_output_filename(task.get("output_filename"))
output_lines: list[str] = []
input_count = 0
output_count = 0
for item in _input_objects(task["id"]):
input_count += 1
raw = get_object_storage().get_bytes(item["object_key"])
data = json.loads(raw.decode("utf-8"))
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {item.get('file_name')}")
for record in records:
output_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
output = "".join(output_lines).encode("utf-8")
store = get_platform_store()
with store.connect() as conn:
if should_store_in_minio(len(output)):
output_object = _register_object(
task["id"],
version_id="output",
object_key=_output_object_key(task["id"], output_name),
file_name=output_name,
content_type="application/jsonl",
content=output,
created_by=created_by,
)
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=%s, output_content=NULL, storage_backend='minio' WHERE id=%s",
(output_object["id"], task["id"]),
)
else:
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=NULL, output_content=%s, storage_backend='database' WHERE id=%s",
(output.decode("utf-8"), task["id"]),
)
return input_count, output_count, output
def _convert_from_local(task: dict[str, Any]) -> tuple[int, int, bytes]:
input_dir = _input_dir(task["id"])
output_dir = _output_dir(task["id"])
input_dir.mkdir(parents=True, exist_ok=True)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
output_path.unlink(missing_ok=True)
input_count = 0
output_count = 0
with output_path.open("w", encoding="utf-8") as output_file:
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
data = json.loads(json_file.read_text(encoding="utf-8"))
records = data if isinstance(data, list) else [data] if isinstance(data, dict) else None
if records is None:
raise ValueError(f"JSON must be object or array: {json_file.name}")
for record in records:
output_file.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
return input_count, output_count, output_path.read_bytes()
@router.get("")
def list_tasks(
page: int = 1,
@@ -109,7 +249,8 @@ def create_task(
"VALUES (%s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id),
)
# 创建目录
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
if not _minio_enabled():
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
return ok(_get_task(task_id))
@@ -123,9 +264,15 @@ def get_task(
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
# 附加输入文件列表
input_dir = _input_dir(task_id)
# 附加输入文件列表；旧任务没有对象记录时继续读取本地兼容目录。
files = []
if _minio_enabled():
files = [
{"name": item.get("file_name") or Path(item["object_key"]).name, "size": item.get("byte_size") or 0}
for item in _input_objects(task_id)
]
else:
input_dir = _input_dir(task_id)
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
@@ -146,16 +293,26 @@ async def upload_source_files(
raise fail(404, "task not found")
if task["status"] not in ("pending", "uploaded"):
raise fail(400, "task is not editable")
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
staged = []
for upload in files:
name = Path(upload.filename or "input.json").name
if not name.lower().endswith(".json"):
raise fail(415, f"only JSON files are supported: {name}")
target = input_dir / name
content = await upload.read()
target.write_bytes(content)
if _minio_enabled():
_register_object(
task_id,
version_id=f"input-{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}",
object_key=_input_object_key(task_id, name),
file_name=name,
content_type=upload.content_type or "application/json",
content=content,
created_by=task.get("created_by") or current_user.get("id"),
)
else:
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
(input_dir / name).write_bytes(content)
staged.append({"name": name, "size": len(content)})
store = get_platform_store()
# 标记上传完成
@@ -166,30 +323,12 @@ async def upload_source_files(
)
# 自动转换并导入数据集
try:
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
# 清空旧输出（如果重新上传）
if output_path.exists():
output_path.unlink()
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
if _minio_enabled():
input_count, output_count, output = _convert_from_minio(
task, task.get("created_by") or current_user.get("id")
)
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
input_count, output_count, output = _convert_from_local(task)
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
@@ -197,12 +336,12 @@ async def upload_source_files(
(input_count, output_count, task_id),
)
# 自动导入数据集
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
size_bytes = len(content.encode("utf-8"))
dataset = store.create_dataset({
"name": task["name"],
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -212,7 +351,20 @@ async def upload_source_files(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or current_user.get("id"),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({
"staged_files": staged,
"auto_converted": True,
@@ -248,28 +400,10 @@ def run_convert(
(task_id,),
)
try:
input_dir = _input_dir(task_id)
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
if _minio_enabled():
input_count, output_count, _ = _convert_from_minio(task, task.get("created_by") or current_user.get("id"))
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
input_count, output_count, _ = _convert_from_local(task)
# 更新任务状态
with store.connect() as conn:
conn.execute(
@@ -297,11 +431,15 @@ def download_result(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
if _minio_enabled():
return Response(content=output, media_type="application/octet-stream", headers={
"Content-Disposition": f"attachment; filename={_safe_output_filename(task.get('output_filename'))}"
})
return FileResponse(
str(output_path),
str(_task_output_path(task)),
media_type="application/octet-stream",
filename=_safe_output_filename(task.get("output_filename")),
)
@@ -319,10 +457,10 @@ def import_as_dataset(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
dataset_name = str(payload.get("name") or task["name"]).strip()
description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip()
size_bytes = len(content.encode("utf-8"))
@@ -331,7 +469,7 @@ def import_as_dataset(
dataset = store.create_dataset({
"name": dataset_name,
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -341,7 +479,20 @@ def import_as_dataset(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or (current_user.get("id") if current_user else None),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({"dataset_id": dataset_id, "name": dataset_name})
@@ -360,7 +511,15 @@ def delete_task(
"UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s",
(task_id,),
)
# 清理文件
if _minio_enabled():
for item in _task_objects(task_id):
try:
get_object_storage().delete(item["object_key"])
store.update_storage_object(item["id"], {"status": "deleted"})
except Exception:
pass
else:
# 旧兼容数据仍清理本地目录。
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():

24
backend/app/modules/data_process/algorithms/embedding.py Normal file
View File

@@ -0,0 +1,24 @@
"""数据处理算法 - 本地语义嵌入模型共享单例。"""
from __future__ import annotations
import os
from functools import lru_cache
from typing import Any
@lru_cache(maxsize=1)
def semantic_embedding_model() -> Any:
"""加载本地嵌入模型，供语义分块与语义质量评分共用。
模型可在部署环境覆盖；默认模型体积较小且适合中英文语义判断。
返回 LlamaIndex BaseEmbedding，通过 ``get_text_embedding`` 使用。
"""
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
return HuggingFaceEmbedding(
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
trust_remote_code=False,
)

20
backend/app/modules/data_process/algorithms/parsers/office.py
View File

@@ -7,12 +7,13 @@ import re
import unicodedata
import zipfile
import xml.etree.ElementTree as ET
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from pathlib import PurePosixPath
from typing import Any
from urllib.parse import unquote, urlsplit
from docx import Document
from docx.oxml.ns import qn
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
@@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
except zipfile.BadZipFile as exc:
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
def iter_document_blocks(parent: Any) -> Iterator[Any]:
"""按文档顺序产出正文段落与表格，并下钻 SDT 内容控件。
Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里，只遍历 body
直接子级会把这些段落整段丢掉。
"""
for child in parent.iterchildren():
if isinstance(child, (CT_P, CT_Tbl)):
yield child
elif child.tag == qn("w:sdt"):
content = child.find(qn("w:sdtContent"))
if content is not None:
yield from iter_document_blocks(content)
def _extract_docx_text(raw: bytes) -> str:
_validate_office_archive(raw, "docx")
try:
@@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str:
parts: list[str] = []
total = 0
for child in document.element.body.iterchildren():
for child in iter_document_blocks(document.element.body):
if isinstance(child, CT_P):
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
continue

85
backend/app/modules/data_process/algorithms/quality.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
import math
import re
import unicodedata
from collections import Counter
@@ -341,3 +342,87 @@ def score_quality(
flags=tuple(flags),
fingerprint=fingerprint,
)
def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
if not left or not right or len(left) != len(right):
return 0.0
dot = math.fsum(a * b for a, b in zip(left, right))
norm_left = math.sqrt(math.fsum(a * a for a in left))
norm_right = math.sqrt(math.fsum(b * b for b in right))
if not norm_left or not norm_right:
return 0.0
return dot / (norm_left * norm_right)
def semantic_quality_scores(
record: Mapping[str, Any],
*,
source_content: str = "",
embed_model: Any = None,
) -> dict[str, Any] | None:
"""用本地嵌入向量计算语义相关性（0-100）。
返回 ``question_answer``（问题↔答案）、``answer_source``（答案↔来源，
无来源时缺省）与 ``overall``；嵌入模型不可用时返回 None 降级，不阻断流程。
"""
try:
if embed_model is None:
from .embedding import semantic_embedding_model
embed_model = semantic_embedding_model()
if embed_model is None:
return None
question = normalize_text(
" ".join(
str(record.get(field) or "")
for field in ("instruction", "input")
)
)
answer = normalize_text(
str(record.get("output") or "") or str(record.get("chosen") or "")
)
source = normalize_text(source_content)
texts = [text for text in {question, answer, source} if text]
if not texts:
return None
vectors = {text: embed_model.get_text_embedding(text) for text in texts}
except Exception:
return None
scores: dict[str, Any] = {}
if question and answer:
scores["question_answer"] = round(
100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2
)
if answer and source:
scores["answer_source"] = round(
100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2
)
if not scores:
return None
scores["overall"] = round(sum(scores.values()) / len(scores), 2)
return scores
def composite_overall(
*,
rule: float | None,
semantic: float | None = None,
judge: float | None = None,
) -> float:
"""三层加权组合：规则 35% + 语义 20% + 评审 45%，缺失层自动重归一。"""
if rule is None:
rule = 0.0
if judge is not None and semantic is not None:
overall = rule * 0.35 + semantic * 0.20 + judge * 0.45
elif semantic is not None:
overall = rule * 0.60 + semantic * 0.40
elif judge is not None:
overall = rule * 0.55 + judge * 0.45
else:
overall = rule
return round(max(0.0, min(100.0, overall)), 2)

122
backend/app/modules/data_process/document_chunking.py
View File

@@ -18,12 +18,19 @@ from llama_index.core.base.embeddings.base import BaseEmbedding
from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
from app.modules.data_process.algorithms import normalize_text
from app.modules.data_process.algorithms.embedding import semantic_embedding_model
ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
_PAGE_FURNITURE = re.compile(
r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$"
)
# Docling 的 markdown 序列化会给列表项补上自动编号，而 Word 的编号存放在
# numbering.xml 中，python-docx 抽取的正文不含这些编号；紧凑匹配前剥掉
# 行首编号，否则带列表的切片会整体定位失败。
_LIST_MARKER_PREFIX = re.compile(
r"(?m)^[ \t>]*(?:(?:\d{1,3}[.)])+|\([a-zA-Z0-9]{1,3}\)|[a-zA-Z][.)]|[-*+•·])[ \t]+"
)
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
_CONVERTER_LOCK = threading.Lock()
@@ -149,18 +156,6 @@ def chunk_fixed_text(
return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
@lru_cache(maxsize=1)
def _semantic_embedding_model() -> BaseEmbedding:
# 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
return HuggingFaceEmbedding(
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
trust_remote_code=False,
)
def chunk_semantic_text(
text: str,
*,
@@ -175,7 +170,7 @@ def chunk_semantic_text(
if not normalized:
return []
splitter = SemanticSplitterNodeParser.from_defaults(
embed_model=embed_model or _semantic_embedding_model(),
embed_model=embed_model or semantic_embedding_model(),
breakpoint_percentile_threshold=breakpoint_percentile_threshold,
buffer_size=1,
sentence_splitter=_sentence_chunks,
@@ -193,6 +188,7 @@ def chunk_semantic_text(
if start is None:
start = _locate_text(normalized, content, 0)
if start is None:
result.append(_unlocated_chunk(content))
continue
if len(_tokenizer().encode(content)) <= chunk_size:
result.append(_make_text_chunk(normalized, start, start + len(content)))
@@ -203,6 +199,7 @@ def chunk_semantic_text(
chunk_overlap=chunk_overlap,
):
if child.source_start is None or child.source_end is None:
result.append(_unlocated_chunk(child.original_content))
continue
result.append(
_make_text_chunk(
@@ -235,6 +232,7 @@ def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
if start is None:
start = _locate_text(source_text, content, 0)
if start is None:
chunks.append(_unlocated_chunk(content))
continue
end = start + len(content)
chunks.append(_make_text_chunk(source_text, start, end))
@@ -247,6 +245,20 @@ def _locate_text(source: str, content: str, start: int) -> int | None:
return position if position >= 0 else None
def _unlocated_chunk(content: str) -> DocumentChunk:
"""正文在源文本中定位失败时保底保留切片，只放弃行号信息。"""
return DocumentChunk(
original_content=content,
contextualized_content=content,
source_start=None,
source_end=None,
source_start_line=None,
source_end_line=None,
token_count=len(_tokenizer().encode(content)),
)
def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
content = source[start:end]
return DocumentChunk(
@@ -316,6 +328,14 @@ def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
return "".join(compact), offsets
def _expand_to_line_boundaries(source_text: str, start: int, end: int) -> tuple[int, int]:
while start > 0 and source_text[start - 1] not in "\r\n":
start -= 1
while end < len(source_text) and source_text[end] not in "\r\n":
end += 1
return start, end
def _project_layout_span(
source_text: str,
content: str,
@@ -324,21 +344,79 @@ def _project_layout_span(
source_offsets: list[int],
compact_start: int,
) -> tuple[int | None, int | None, int]:
compact_content, _ = _compact_with_offsets(content)
for candidate in (content, _LIST_MARKER_PREFIX.sub("", content)):
compact_content, _ = _compact_with_offsets(candidate)
if len(compact_content) < 4:
return None, None, compact_start
continue
position = compact_source.find(compact_content, compact_start)
if position < 0:
position = compact_source.find(compact_content)
if position < 0:
continue
start, end = _expand_to_line_boundaries(
source_text,
source_offsets[position],
source_offsets[position + len(compact_content) - 1] + 1,
)
# 重复内容回退匹配可能命中已消费的更早位置，游标只进不退，
# 避免后续切片跟着错位。
return start, end, max(compact_start, position + len(compact_content))
return _project_layout_span_by_anchors(
source_text,
content,
compact_source=compact_source,
source_offsets=source_offsets,
compact_start=compact_start,
)
def _project_layout_span_by_anchors(
source_text: str,
content: str,
*,
compact_source: str,
source_offsets: list[int],
compact_start: int,
) -> tuple[int | None, int | None, int]:
"""按行锚点顺序匹配，容忍切片里插入的重复表头等非连续内容。"""
segments = [
compact
for compact in (
_compact_with_offsets(line)[0]
for line in _LIST_MARKER_PREFIX.sub("", content).split("\n")
)
if len(compact) >= 6
]
if not segments:
return None, None, compact_start
start = source_offsets[position]
end = source_offsets[position + len(compact_content) - 1] + 1
while start > 0 and source_text[start - 1] not in "\r\n":
start -= 1
while end < len(source_text) and source_text[end] not in "\r\n":
end += 1
return start, end, position + len(compact_content)
total = sum(len(segment) for segment in segments)
def match_from(cursor: int) -> tuple[list[tuple[int, int]], int]:
matched: list[tuple[int, int]] = []
position = cursor
for segment in segments:
found = compact_source.find(segment, position)
if found < 0:
continue
matched.append((found, found + len(segment)))
position = found + len(segment)
return matched, sum(end - start for start, end in matched)
matched, covered = match_from(compact_start)
if covered * 2 < total:
retried, retry_covered = match_from(0)
if retry_covered > covered:
matched, covered = retried, retry_covered
# 覆盖不足一半时宁可不定位，也不能给出错误的行号。
if not matched or covered * 2 < total:
return None, None, compact_start
start, end = _expand_to_line_boundaries(
source_text,
source_offsets[matched[0][0]],
source_offsets[matched[-1][1] - 1] + 1,
)
return start, end, max(compact_start, matched[-1][1])
def chunk_layout_document(

313
backend/app/modules/data_process/evaluation.py Normal file
View File

@@ -0,0 +1,313 @@
"""数据处理 - 生成结果的多层质量评测。
三层体系：规则层（确定性规则分）+ 语义层（本地嵌入向量）+ 评审层
（复用生成模型按 rubric 打分的 LLM-as-judge）。任一层失败自动降级，
评测永远返回可用结果，不阻断调用方流程。
"""
from __future__ import annotations
import logging
from collections.abc import Mapping
from dataclasses import asdict
from datetime import UTC, datetime
from typing import Any
import httpx
from .algorithms import normalize_text, score_quality
from .algorithms.quality import composite_overall, semantic_quality_scores
from .generation import (
ModelGenerationError,
_is_retryable_generation_error,
_json_payload,
_message_content,
chat_completions_url,
)
logger = logging.getLogger(__name__)
# 送入评审提示词的来源正文上限，避免超长切片挤占评分输出空间。
_MAX_JUDGE_SOURCE_CHARS = 6000
_JUDGE_DIMENSIONS: dict[str, tuple[str, ...]] = {
"standard": (
"faithfulness",
"correctness",
"clarity",
"completeness",
"alignment",
),
"reasoning": (
"faithfulness",
"correctness",
"clarity",
"completeness",
"alignment",
"reasoning_validity",
),
"dpo": (
"clarity",
"chosen_quality",
"rejected_quality",
"preference_reasonableness",
"faithfulness",
),
}
_DIMENSION_LABELS: dict[str, str] = {
"faithfulness": "忠实度",
"correctness": "正确性",
"clarity": "问题清晰度",
"completeness": "回答完整性",
"alignment": "指令对齐",
"reasoning_validity": "推理有效性",
"chosen_quality": "chosen 回答质量",
"rejected_quality": "rejected 回答质量",
"preference_reasonableness": "偏好区分合理性",
}
_DIMENSION_RULES: dict[str, str] = {
"faithfulness": "忠实度：答案的全部陈述是否被参考资料支持，没有编造、没有引入资料之外的信息；未提供参考资料时按答案内部自洽性评估",
"correctness": "正确性：答案中的事实、概念与计算是否正确",
"clarity": "问题清晰度：问题是否清晰、自包含、无歧义，脱离上下文也能理解",
"completeness": "回答完整性：答案是否充分、直接地回应了问题的全部要点",
"alignment": "指令对齐：答案的形式与范围是否符合问题的要求（如格式、语言、范围限定）",
"reasoning_validity": "推理有效性：思维链步骤是否逻辑连贯、无跳步或循环论证，结论是否由推理过程自然得出",
"chosen_quality": "chosen 回答质量：更优回答的正确性、完整性与表述质量",
"rejected_quality": "rejected 回答质量：较差回答是否仍具备基本可读性，使对比训练有意义",
"preference_reasonableness": "偏好区分合理性：chosen 是否明显优于 rejected，且优劣差异与问题直接相关",
}
def _judge_system_prompt(output_type: str) -> str:
dimensions = _JUDGE_DIMENSIONS[output_type]
rules = "\n".join(f"- {_DIMENSION_RULES[name]}" for name in dimensions)
scores_schema = ", ".join(f'"{name}": 1-5' for name in dimensions)
return (
"你是大模型训练数据质量评审员。严格依据用户消息中的【参考资料】评审这条训练数据，逐维度按 1-5 分打分：\n"
f"{rules}\n"
"评分锚点：5 分=完全符合维度描述；3 分=基本符合但有明显不足；1 分=严重不符合。\n"
"忠实度只依据参考资料与公认常识判断，无法得到支持的陈述必须扣分；不要因为答案冗长而加分。\n"
"只输出一个 JSON 对象，不要输出 JSON 之外的任何文字。\n"
'输出格式：{"scores": {' + scores_schema + '}, "reason": "一句话总评", "issues": ["具体问题，没有则为空数组"]}'
)
def _judge_user_prompt(record: Mapping[str, Any], source_content: str) -> str:
source = normalize_text(source_content)[:_MAX_JUDGE_SOURCE_CHARS] or "（无参考资料）"
instruction = normalize_text(str(record.get("instruction") or "")) or "（空）"
input_text = normalize_text(str(record.get("input") or ""))
sections = [f"【参考资料】\n{source}", f"【问题】\n{instruction}"]
if input_text:
sections.append(f"【输入】\n{input_text}")
if record.get("chosen") or record.get("rejected"):
sections.append(f"【更优回答 chosen】\n{normalize_text(str(record.get('chosen') or '')) or '（空）'}")
sections.append(f"【较差回答 rejected】\n{normalize_text(str(record.get('rejected') or '')) or '（空）'}")
else:
output = normalize_text(str(record.get("output") or ""))
sections.append(f"【回答】\n{output or '（空）'}")
return "\n\n".join(sections)
def _validated_judge_payload(payload: Any, output_type: str) -> dict[str, Any]:
if not isinstance(payload, Mapping):
raise ModelGenerationError("评审响应不是 JSON 对象")
raw_scores = payload.get("scores")
if not isinstance(raw_scores, Mapping):
raise ModelGenerationError("评审响应缺少 scores 对象")
expected = _JUDGE_DIMENSIONS[output_type]
scores: dict[str, float] = {}
for name in expected:
value = raw_scores.get(name)
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ModelGenerationError(f"评审响应缺少维度 {name} 的有效分数")
scores[name] = round(max(1.0, min(5.0, float(value))), 1)
issues = payload.get("issues")
if not isinstance(issues, list):
issues = []
issues = [str(item)[:200] for item in issues if str(item).strip()][:8]
reason = normalize_text(str(payload.get("reason") or ""))[:300]
return {
"scores": scores,
"overall": round(sum(scores.values()) / len(scores) * 20, 2),
"reason": reason,
"issues": issues,
}
def _judge_record(
record: Mapping[str, Any],
source_content: str,
*,
model: Mapping[str, Any],
config: Mapping[str, Any],
client: httpx.Client | None,
) -> dict[str, Any] | None:
output_type = str(config.get("output_type") or "standard").strip().lower()
if output_type not in _JUDGE_DIMENSIONS:
output_type = "standard"
endpoint = chat_completions_url(str(model.get("api_url") or ""))
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
if not model_name:
raise ModelGenerationError("generation model name is required")
temperature = 0.1
max_tokens = max(256, min(2048, int(config.get("max_tokens", 1024) or 1024)))
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60) or 60)))
retries = max(0, min(5, int(config.get("generation_retries", 2) or 2)))
headers = {"Content-Type": "application/json"}
api_key = str(model.get("api_key") or "").strip()
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
request_payload: dict[str, Any] = {
"model": model_name,
"messages": [
{"role": "system", "content": _judge_system_prompt(output_type)},
{"role": "user", "content": _judge_user_prompt(record, source_content)},
],
"temperature": temperature,
"max_tokens": max_tokens,
}
if bool(config.get("json_mode", False)):
request_payload["response_format"] = {"type": "json_object"}
owns_client = client is None
http_client = client or httpx.Client(timeout=timeout)
try:
last_error: Exception | None = None
for _ in range(retries + 1):
try:
response = http_client.post(endpoint, headers=headers, json=request_payload)
response.raise_for_status()
body = response.json()
if not isinstance(body, Mapping):
raise ModelGenerationError("model response body must be a JSON object")
judged = _validated_judge_payload(
_json_payload(_message_content(body)),
output_type,
)
judged["model"] = model_name
judged["output_type"] = output_type
return judged
except Exception as exc:
last_error = exc
if not _is_retryable_generation_error(exc):
break
raise ModelGenerationError(f"质量评审调用失败: {last_error}")
finally:
if owns_client:
http_client.close()
def evaluate_result_record(
record: Mapping[str, Any],
*,
source_content: str = "",
model: Mapping[str, Any] | None = None,
config: Mapping[str, Any] | None = None,
client: httpx.Client | None = None,
embed_model: Any = None,
min_output_length: int = 20,
) -> dict[str, Any]:
"""对一条生成结果执行三层评测，返回可直接落库的 quality_score 字典。
规则层字段保持原样平铺（向后兼容既有读取方）；新增 ``semantic``、
``judge``、``layers``、``evaluated`` 与组合 ``overall``。
"""
config_dict = dict(config or {})
rule = score_quality(
record,
min_output_length=min_output_length,
source_content=source_content,
)
quality: dict[str, Any] = asdict(rule)
semantic = semantic_quality_scores(
record,
source_content=source_content,
embed_model=embed_model,
)
judge: dict[str, Any] | None = None
if model is not None:
try:
judge = _judge_record(
record,
source_content,
model=model,
config=config_dict,
client=client,
)
except Exception as exc:
logger.warning(
"data process judge evaluation degraded: %s",
exc,
)
layers = {
"rule": rule.overall,
"semantic": semantic.get("overall") if semantic else None,
"judge": judge.get("overall") if judge else None,
}
quality.update(
semantic=semantic,
judge=judge,
layers=layers,
evaluated=True,
evaluated_at=datetime.now(UTC).isoformat(),
overall=composite_overall(
rule=layers["rule"],
semantic=layers["semantic"],
judge=layers["judge"],
),
)
return quality
def reevaluate_edited_record(
record: Mapping[str, Any],
*,
source_content: str = "",
previous_quality: Mapping[str, Any] | None = None,
embed_model: Any = None,
min_output_length: int = 20,
) -> dict[str, Any]:
"""手动编辑/恢复后重算规则与语义层，丢弃已过期的评审层。
编辑会改变内容，旧的评审分不再可信；规则与语义层本地重算零成本。
``evaluated`` 标记沿用原值，保证已评测过的结果编辑后仍有可用分数。
"""
rule = score_quality(
record,
min_output_length=min_output_length,
source_content=source_content,
)
quality: dict[str, Any] = asdict(rule)
semantic = semantic_quality_scores(
record,
source_content=source_content,
embed_model=embed_model,
)
previous = dict(previous_quality or {})
evaluated = bool(previous.get("evaluated"))
layers = {
"rule": rule.overall,
"semantic": semantic.get("overall") if semantic else None,
"judge": None,
}
quality.update(
semantic=semantic,
judge=None,
layers=layers,
evaluated=evaluated,
evaluated_at=(
datetime.now(UTC).isoformat() if evaluated else None
),
overall=composite_overall(
rule=layers["rule"],
semantic=layers["semantic"],
),
)
return quality

23
backend/app/modules/data_process/office_preview.py
View File

@@ -11,6 +11,7 @@ import re
from typing import Any
from docx import Document
from docx.oxml.ns import qn
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
@@ -27,6 +28,7 @@ from app.modules.data_process.algorithms import (
_xlsx_sheet_merge_ranges,
normalize_text,
)
from app.modules.data_process.algorithms.parsers.office import iter_document_blocks
MAX_DOCX_PREVIEW_BLOCKS = 2_000
MAX_XLSX_PREVIEW_ROWS = 200
@@ -49,12 +51,21 @@ def _docx_alignment(paragraph: Paragraph) -> str:
def _docx_heading_level(paragraph: Paragraph) -> int | None:
style = paragraph.style
if style is None:
return None
style_name = str(style.name or "")
style_id = str(style.style_id or "")
style_name = str(style.name or "") if style is not None else ""
style_id = str(style.style_id or "") if style is not None else ""
match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
return int(match.group(1)) if match else None
if match:
return int(match.group(1))
# Word 的目录和导航窗格依据大纲级别识别标题；未套标题样式但带
# outlineLvl 的段落（如手工排版的编号小节）同样是标题。
outline = paragraph._p.find(f"{qn('w:pPr')}/{qn('w:outlineLvl')}")
if outline is not None:
value = outline.get(qn("w:val"))
if value is not None and value.isdigit():
level = int(value)
if 0 <= level <= 5:
return level + 1
return None
def build_docx_preview(raw: bytes) -> dict[str, Any]:
@@ -84,7 +95,7 @@ def build_docx_preview(raw: bytes) -> dict[str, Any]:
has_source_content = True
return text, start, source_cursor
for child in document.element.body.iterchildren():
for child in iter_document_blocks(document.element.body):
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
truncated = True
break

128
backend/app/modules/data_process/storage.py
View File

@@ -1,4 +1,4 @@
"""数据处理原始源文件的受控本地对象存储。"""
"""数据处理源文件的受控暂存与分层对象存储。"""
from __future__ import annotations
@@ -13,6 +13,9 @@ from pathlib import Path, PurePosixPath
from typing import Iterable, Iterator
from urllib.parse import quote, unquote, urlsplit
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
class DataProcessStorageError(ValueError):
"""本地对象引用或文件系统状态不安全。"""
@@ -68,7 +71,7 @@ def _safe_basename(value: str) -> str:
class LocalDataProcessStorage:
"""只允许访问配置根目录下的版本化原始文件。"""
"""Stage locally, but publish and read authoritative source files from MinIO."""
def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None:
configured = Path(root) if root is not None else _configured_storage_root()
@@ -132,10 +135,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -168,6 +168,18 @@ class LocalDataProcessStorage:
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
if self._is_minio_reference(source_reference):
content = self.read(source_reference)
if content is None:
raise DataProcessStorageError("original source object is not available")
return self.stage_bytes(
batch_id=batch_id,
task_id=task_id,
source_file_id=source_file_id,
version=version,
name=basename,
content=content,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
@@ -193,10 +205,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -212,14 +221,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("duplicate staged source object")
seen_temporary_paths.add(item._temporary_path)
for item in staged:
if self._is_minio_reference(item.reference):
content = item._temporary_path.read_bytes()
get_object_storage().put_bytes(
self.object_key(item.reference),
content,
"application/octet-stream",
)
elif not item.reference.startswith("db://data-process/"):
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
self._fsync_directory(final_path.parent)
published.append(item)
item._temporary_path.unlink()
self._fsync_directory(final_path.parent)
except Exception:
for item in reversed(published):
try:
@@ -258,7 +275,13 @@ class LocalDataProcessStorage:
raise first_error
def read(self, reference: str) -> bytes | None:
"""读取 local 引用；旧 ``db://`` 对象返回 ``None`` 由数据库正文兜底。"""
"""Read a MinIO object or legacy local reference."""
if self._is_minio_reference(reference):
try:
return get_object_storage().get_bytes(self.object_key(reference))
except Exception as exc: # noqa: BLE001 - normalize object-not-found for callers
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
@@ -276,6 +299,11 @@ class LocalDataProcessStorage:
) -> int | None:
"""返回受控 local 对象大小；旧 ``db://`` 对象没有原始文件。"""
if self._is_minio_reference(reference):
try:
return int(get_object_storage().stat(self.object_key(reference)).get("byte_size") or 0)
except Exception as exc: # noqa: BLE001
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return None
@@ -301,6 +329,15 @@ class LocalDataProcessStorage:
) -> Iterator[bytes]:
"""按范围流式读取原始文件，避免 PDF 预览把大文件整体载入内存。"""
if self._is_minio_reference(reference):
content = self.read(reference) or b""
if start < 0 or expected_size != len(content) or start > expected_size:
raise DataProcessStorageError("source object size does not match metadata")
remaining = expected_size - start if length is None else length
if remaining < 0 or start + remaining > expected_size:
raise DataProcessStorageError("invalid source byte range")
yield content[start : start + remaining]
return
relative_path = self._relative_from_reference(reference)
if relative_path is None:
raise DataProcessStorageError("original source object is not available")
@@ -337,6 +374,12 @@ class LocalDataProcessStorage:
) -> bool:
"""校验 local 引用归属；旧 ``db://`` 引用无需文件系统处理。"""
if self._is_minio_reference(reference):
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
return True
if str(reference or "").startswith("db://data-process/"):
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return True
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -382,6 +425,19 @@ class LocalDataProcessStorage:
) -> bool:
"""删除受控 local 对象；旧 ``db://`` 引用保持不变。"""
if self._is_minio_reference(reference):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
get_object_storage().delete(self.object_key(reference))
return True
if str(reference or "").startswith("db://data-process/"):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return False
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -426,7 +482,7 @@ class LocalDataProcessStorage:
if reference.startswith("db://"):
return None
parsed = urlsplit(reference)
if parsed.scheme != "local" or parsed.netloc != "data-process":
if parsed.scheme not in {"local", "minio"} or parsed.netloc != "data-process":
raise DataProcessStorageError("unsupported source storage reference")
if parsed.query or parsed.fragment or "\\" in parsed.path:
raise DataProcessStorageError("unsafe source storage reference")
@@ -460,6 +516,39 @@ class LocalDataProcessStorage:
basename = _safe_basename(decoded[3])
return PurePosixPath(task_id, source_file_id, f"v{version}", basename)
@staticmethod
def _is_minio_reference(reference: str) -> bool:
return str(reference or "").startswith("minio://data-process/")
@staticmethod
def _reference(task_id: str, source_file_id: str, version: int, basename: str) -> str:
scheme = "minio" if get_settings().minio_enabled else "local"
return f"{scheme}://data-process/{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
@staticmethod
def object_key(reference: str) -> str:
parsed = urlsplit(reference)
if parsed.scheme != "minio" or parsed.netloc != "data-process":
raise DataProcessStorageError("reference is not a MinIO source object")
return "data-process/" + parsed.path.lstrip("/")
def _assert_minio_owner(self, reference: str, task_id: str, source_file_id: str) -> None:
relative = self._relative_from_reference(reference)
if relative is None:
raise DataProcessStorageError("invalid MinIO source reference")
self._assert_expected_owner(relative, expected_task_id=task_id, expected_source_file_id=source_file_id)
@staticmethod
def _assert_database_owner(reference: str, task_id: str, source_file_id: str) -> None:
parsed = urlsplit(reference)
parts = parsed.path.lstrip("/").split("/")
if parsed.netloc != "data-process" or len(parts) != 3:
raise DataProcessStorageError("invalid database source reference")
expected_task_id = _safe_component(task_id, "expected task id")
expected_source_file_id = _safe_component(source_file_id, "expected source file id")
if tuple(parts[:2]) != (expected_task_id, expected_source_file_id) or parts[2] != "v1":
raise DataProcessStorageError("source storage object owner mismatch")
def _path_for_relative(self, relative_path: PurePosixPath) -> Path:
if relative_path.is_absolute() or any(
part in {"", ".", ".."} for part in relative_path.parts
@@ -479,6 +568,19 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("invalid staged source object")
if self._issued_staged_objects.get(item._temporary_path) is not item:
raise DataProcessStorageError("staged source object was not issued by this storage")
if item.reference.startswith("db://data-process/"):
parsed = urlsplit(item.reference)
parts = parsed.path.lstrip("/").split("/")
expected = item._relative_path.parts[:3]
if (
parsed.netloc != "data-process"
or parsed.query
or parsed.fragment
or len(parts) != 3
or tuple(parts) != expected
):
raise DataProcessStorageError("staged source object reference mismatch")
else:
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")

7
backend/app/modules/data_process/store/base.py
View File

@@ -247,14 +247,19 @@ def _source_storage_descriptor(
or f"db://data-process/{task_id}/{file_id}/v1"
)
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/"
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
expected_local_prefix
):
storage_backend = "local"
elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len(
expected_minio_prefix
):
storage_backend = "minio"
elif storage_object_id == expected_database_reference:
storage_backend = "database"
elif storage_object_id.startswith(("local://data-process/", "db://data-process/")):
elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")):
raise DataProcessStoreError("source storage object owner mismatch")
else:
raise DataProcessStoreError("unsupported source storage object reference")

24
backend/app/modules/data_process/store/datasets.py
View File

@@ -11,6 +11,7 @@ import psycopg
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from .base import (
StoreBase,
@@ -168,7 +169,11 @@ class DatasetsMixin:
split_name: assignments.count(split_name) for split_name in split_order
}
split_specs: list[dict[str, Any]] = []
use_minio = bool(get_settings().minio_enabled)
# Explicit local is retained for old callers/tests that request the
# legacy backend; all normal platform requests default to MinIO.
allow_minio = bool(get_settings().minio_enabled) and str(
payload.get("storage_type") or "minio"
).lower() != "local"
for split_name in split_order:
split_records = [
(source_row, record)
@@ -193,12 +198,15 @@ class DatasetsMixin:
"storage_object_id": (
f"db://data-process/{task_id}/{file_id}/v1"
),
"store_in_minio": allow_minio and should_store_in_minio(
len(raw), content_type="application/jsonl", file_format="jsonl"
),
}
)
source_result_ids = [row["id"] for row in rows]
common_metadata = {
"source": "data_process",
"storage_backend": "minio" if use_minio else "database",
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
"source_task_id": task_id,
"output_type": _task_output_type(task),
"reasoning_detail": _task_reasoning_detail(task),
@@ -273,7 +281,7 @@ class DatasetsMixin:
split_name = str(spec["split"])
dataset_id = dataset_ids[split_name]
storage_object_id = str(spec["storage_object_id"])
if use_minio:
if spec["store_in_minio"]:
file_name = f"{base_dataset_name}.{split_name}.jsonl"
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
uploaded = get_object_storage().put_bytes(
@@ -356,7 +364,7 @@ class DatasetsMixin:
(
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
f"{len(spec['raw'])} B",
len(spec["raw"]),
len(spec["records"]),
@@ -386,7 +394,7 @@ class DatasetsMixin:
dataset_id,
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
task_id,
task_id,
f"{len(spec['raw'])} B",
@@ -405,7 +413,11 @@ class DatasetsMixin:
),
).fetchone()
file_metadata = {**dataset_metadata, "file_split": split_name}
file_metadata = {
**dataset_metadata,
"file_split": split_name,
"storage_backend": "minio" if spec["store_in_minio"] else "database",
}
version = {
"id": spec["version_id"],
"version_no": 1,

14
backend/app/modules/data_process/store/source_files.py
View File

@@ -137,7 +137,7 @@ class SourceFilesMixin:
payload["record_count"],
payload["file_format"],
payload["checksum_sha256"],
payload["content"],
"" if str(storage_object_id or "").startswith("minio://") else payload["content"],
str(payload["content"])[:2000],
json_dumps(metadata_payload),
task.get("tenant_id"),
@@ -223,7 +223,17 @@ class SourceFilesMixin:
).fetchone()
if not row:
raise NotFoundError("source file not found")
return _decode_row(row) or {}
decoded = _decode_row(row) or {}
# New source files keep only a preview in PostgreSQL. Load the
# authoritative body from MinIO on demand for existing processing code.
reference = str(decoded.get("storage_object_id") or "")
if include_content and not decoded.get("content") and reference.startswith("minio://"):
from app.modules.data_process.storage import get_data_process_storage
decoded["content"] = (get_data_process_storage().read(reference) or b"").decode(
"utf-8", errors="replace"
)
return decoded
def source_content_window(
self, task_id: str, file_id: str, offset: int, limit: int

59
backend/app/modules/storage/minio_store.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import timedelta
from functools import lru_cache
from io import BytesIO
from collections.abc import Iterator
from typing import Any
from minio import Minio
@@ -53,6 +54,64 @@ class MinioObjectStorage:
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def get_bytes(self, object_key: str) -> bytes:
"""Read an object through the backend for small API responses and workers."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
return response.read()
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
"""Stream an object without loading the complete file into memory."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
while True:
chunk = response.read(chunk_size)
if not chunk:
break
yield chunk
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
self._ensure_enabled()
self.ensure_bucket()
try:
return [
{
"object_key": item.object_name,
"byte_size": item.size or 0,
"etag": item.etag,
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
}
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
]
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def delete(self, object_key: str) -> None:
self._ensure_enabled()
self.ensure_bucket()
try:
self.client.remove_object(self.bucket, object_key)
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
self._ensure_enabled()
self.ensure_bucket()

54
backend/app/modules/storage/policy.py Normal file
View File

@@ -0,0 +1,54 @@
"""Storage placement rules shared by dataset and data-processing flows."""
from __future__ import annotations
from app.core.config import get_settings
_INLINE_TEXT_FORMATS = {
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
"yaml", "yml", "xml", "html", "text/plain", "application/json",
"application/jsonl", "text/csv",
}
def should_store_in_minio(
size_bytes: int | None,
*,
content_type: str | None = None,
file_format: str | None = None,
) -> bool:
"""Return whether a file is large enough to use the shared object store.
Small files remain inline in PostgreSQL so page previews and metadata reads
do not pay an object-storage round trip. MinIO is still mandatory for
large files when it is enabled.
"""
if not get_settings().minio_enabled:
return False
try:
size = max(0, int(size_bytes or 0))
except (TypeError, ValueError):
size = 0
if size > get_settings().minio_inline_max_bytes:
return True
# Binary office/document files remain in MinIO even when small because
# their original bytes cannot be safely represented by a text DB column.
normalized_format = str(file_format or "").strip().lower().lstrip(".")
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
if normalized_format or normalized_type:
return not (
normalized_format in _INLINE_TEXT_FORMATS
or normalized_type in _INLINE_TEXT_FORMATS
or normalized_type.startswith("text/")
)
return False
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
"""Return ``minio`` or ``database`` for a managed file."""
if str(requested or "").strip().lower() == "local":
return "database"
return "minio" if should_store_in_minio(size_bytes) else "database"

10
backend/app/modules/system/router.py
View File

@@ -56,13 +56,15 @@ def audit_logs(
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
target_id: str | None = Query(default=None, description="目标 ID"),
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict:
"""审计日志查询：按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
"""审计日志查询：按组织、操作人、动作、资源、关键字和时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
@@ -73,6 +75,8 @@ def audit_logs(
actor_id=actor_id,
action=action,
target_type=target_type,
target_id=target_id,
keyword=keyword,
start_time=start_time,
end_time=end_time,
limit=limit,
@@ -88,6 +92,8 @@ def audit_logs_export(
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
target_id: str | None = Query(default=None, description="目标 ID"),
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
@@ -103,6 +109,8 @@ def audit_logs_export(
actor_id=actor_id,
action=action,
target_type=target_type,
target_id=target_id,
keyword=keyword,
start_time=start_time,
end_time=end_time,
limit=10000,

20
backend/app/schemas/data_process.py
View File

@@ -384,6 +384,26 @@ class ResultBatchRegenerateRequest(BaseModel):
return self
class ResultBatchEvaluateItem(BaseModel):
model_config = ConfigDict(extra="forbid")
result_id: str = Field(min_length=1, max_length=100)
expected_updated_at: str = Field(min_length=1, max_length=100)
class ResultBatchEvaluateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
items: list[ResultBatchEvaluateItem] = Field(min_length=1, max_length=50)
@model_validator(mode="after")
def validate_unique_results(self) -> ResultBatchEvaluateRequest:
result_ids = [item.result_id for item in self.items]
if len(result_ids) != len(set(result_ids)):
raise ValueError("result_id values must be unique")
return self
class DatasetSplit(BaseModel):
model_config = ConfigDict(extra="forbid")

71
backend/tests/test_data_process_algorithms.py
View File

@@ -9,6 +9,8 @@ from decimal import Decimal
import pytest
from docx import Document
from docx.oxml import parse_xml
from docx.oxml.ns import nsdecls, qn
from openpyxl import Workbook
from pptx import Presentation
from pptx.util import Inches
@@ -38,6 +40,7 @@ from app.modules.data_process.algorithms import (
stable_split_assignments,
structured_json_dumps,
)
from app.modules.data_process.office_preview import build_docx_preview
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
@@ -318,6 +321,74 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def _docx_with_sdt_bytes() -> bytes:
"""构造带 SDT 目录内容控件的 docx，段落顺序为正文、SDT、正文。"""
document = Document()
document.add_paragraph("正文开头。")
sdt = parse_xml(
"<w:sdt %s><w:sdtPr><w:id w:val='1'/></w:sdtPr>"
"<w:sdtContent><w:p><w:r><w:t>目录条目 第一章 概述</w:t></w:r></w:p>"
"</w:sdtContent></w:sdt>" % nsdecls("w")
)
body = document.element.body
sect_pr = body.find(qn("w:sectPr"))
if sect_pr is not None:
sect_pr.addprevious(sdt)
else:
body.append(sdt)
document.add_paragraph("正文结尾。")
output = io.BytesIO()
document.save(output)
return output.getvalue()
def test_docx_extraction_and_preview_include_sdt_content() -> None:
raw = _docx_with_sdt_bytes()
parsed = parse_text_content(raw, filename="toc.docx")
assert "目录条目 第一章 概述" in parsed.text
assert (
parsed.text.index("正文开头。")
< parsed.text.index("目录条目 第一章 概述")
< parsed.text.index("正文结尾。")
)
preview = build_docx_preview(raw)
paragraph_texts = [
block["text"] for block in preview["blocks"] if block["type"] == "paragraph"
]
assert "目录条目 第一章 概述" in paragraph_texts
# 预览偏移必须与正文抽取规则一致，否则前端定位会错位。
sdt_block = next(
block
for block in preview["blocks"]
if block.get("text") == "目录条目 第一章 概述"
)
assert parsed.text[sdt_block["source_start"] : sdt_block["source_end"]] == (
"目录条目 第一章 概述"
)
def test_docx_preview_detects_outline_level_headings() -> None:
"""未套标题样式但设了大纲级别的段落（Word 目录按此收录）也按标题渲染。"""
document = Document()
document.add_heading("一级标题", level=1)
plain = document.add_paragraph("4.2.1 数据管理")
p_pr = plain._p.get_or_add_pPr()
p_pr.append(parse_xml("<w:outlineLvl %s w:val='2'/>" % nsdecls("w")))
document.add_paragraph("普通正文段落。")
output = io.BytesIO()
document.save(output)
preview = build_docx_preview(output.getvalue())
blocks = {b["text"]: b for b in preview["blocks"] if b["type"] == "paragraph"}
assert blocks["一级标题"]["heading_level"] == 1
assert blocks["4.2.1 数据管理"]["heading_level"] == 3
assert blocks["普通正文段落。"]["heading_level"] is None
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active

217
backend/tests/test_data_process_api.py
View File

@@ -1691,6 +1691,223 @@ def test_batch_result_regeneration_rejects_locked_tasks_before_model_call(
assert model_calls == 0
def _prepare_evaluation_task(
client: TestClient,
store: Any,
tmp_path: Path,
*,
config: dict[str, Any] | None = None,
) -> str:
task_id = client.post(
"/modelTF/data-process",
json={
"name": "数据评测",
"process_type": "structured",
"config": config or {"generation_model_id": "model-1", "output_type": "standard"},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
progress=100,
workflow_step="results",
results_confirmed=False,
)
store.models["model-1"] = {
"id": "model-1",
"online_model_name": "test-model",
"api_url": "https://model.example/v1",
"api_key": "secret",
}
store.previews[task_id] = [
{
"id": "preview-1",
"status": "original",
"original_content": "申请编号用于唯一标识一笔报销申请。",
"edited_content": "申请编号用于唯一标识一笔报销申请。",
},
{
"id": "preview-2",
"status": "original",
"original_content": "联系电话用于联系申请人。",
"edited_content": "联系电话用于联系申请人。",
},
]
store.results[task_id] = [
{
"id": "result-1",
"preview_item_id": "preview-1",
"instruction": "申请编号有什么作用？",
"input": "",
"output": "申请编号用于唯一标识一笔报销申请。",
"original_instruction": "申请编号有什么作用？",
"original_input": "",
"original_output": "申请编号用于唯一标识一笔报销申请。",
"status": "valid",
"error": None,
"split": "train",
"quality_score": {},
"updated_at": "2026-08-19T09:00:00Z",
},
{
"id": "result-2",
"preview_item_id": "preview-2",
"instruction": "联系电话有什么作用？",
"input": "",
"output": "联系电话用于联系申请人。",
"original_instruction": "联系电话有什么作用？",
"original_input": "",
"original_output": "联系电话用于联系申请人。",
"status": "valid",
"error": None,
"split": "train",
"quality_score": {},
"updated_at": "2026-08-19T09:00:01Z",
},
]
return task_id
def test_results_can_be_evaluated_in_batch_with_partial_success(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
evaluation_calls: list[dict[str, Any]] = []
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
evaluation_calls.append({"record": deepcopy(record), "kwargs": {k: v for k, v in kwargs.items() if k != "client"}})
return {
"overall": 88.0,
"completeness": 100.0,
"length": 100.0,
"readability": 100.0,
"relevance": 90.0,
"duplicate": 100.0,
"is_valid": True,
"flags": [],
"fingerprint": "fp",
"semantic": {"question_answer": 80.0, "answer_source": 90.0, "overall": 85.0},
"judge": {"scores": {"faithfulness": 5}, "overall": 90.0},
"layers": {"rule": 92.0, "semantic": 85.0, "judge": 90.0},
"evaluated": True,
}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={
"items": [
{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"},
# 乐观锁版本不匹配：该条应按冲突失败，另一条仍成功。
{"result_id": "result-2", "expected_updated_at": "2026-08-18T00:00:00Z"},
],
},
)
assert response.status_code == 200
data = response.json()["data"]
assert data["total"] == 2
assert data["succeeded"] == 1
assert data["failed"] == 1
assert [item["id"] for item in data["items"]] == ["result-1"]
assert data["failures"][0]["result_id"] == "result-2"
assert data["failures"][0]["code"] == "conflict"
assert len(evaluation_calls) == 1
assert evaluation_calls[0]["record"]["instruction"] == "申请编号有什么作用？"
assert evaluation_calls[0]["kwargs"]["model"]["online_model_name"] == "test-model"
assert evaluation_calls[0]["kwargs"]["source_content"] == "申请编号用于唯一标识一笔报销申请。"
stored = store.results[task_id][0]["quality_score"]
assert stored["evaluated"] is True
assert stored["layers"]["judge"] == 90.0
assert store.results[task_id][1]["quality_score"] == {}
def test_evaluation_without_generation_model_skips_judge_layer(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path, config={"output_type": "standard"})
seen_models: list[Any] = []
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
seen_models.append(kwargs.get("model"))
return {
"overall": 70.0, "is_valid": True, "flags": [],
"semantic": None, "judge": None,
"layers": {"rule": 70.0, "semantic": None, "judge": None},
"evaluated": True,
}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
)
assert response.status_code == 200
assert response.json()["data"]["succeeded"] == 1
# 任务未配置生成模型时，评审层收到的 model 必须是 None。
assert seen_models == [None]
def test_evaluation_rejects_running_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
store.tasks[task_id]["status"] = "running"
evaluation_calls = 0
def fake_evaluate(*args: Any, **kwargs: Any) -> dict[str, Any]:
nonlocal evaluation_calls
evaluation_calls += 1
return {"overall": 0, "is_valid": True, "flags": []}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
)
assert response.status_code == 409
assert evaluation_calls == 0
def test_result_update_preserves_evaluation_layers_and_drops_stale_judge(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
store.results[task_id][0]["quality_score"] = {
"overall": 90.0,
"is_valid": True,
"flags": [],
"semantic": {"overall": 85.0},
"judge": {"overall": 92.0},
"layers": {"rule": 90.0, "semantic": 85.0, "judge": 92.0},
"evaluated": True,
}
response = client.put(
f"/modelTF/data-process/{task_id}/results/result-1",
json={"output": "人工修正后的答案：申请编号唯一标识一笔报销申请。"},
)
assert response.status_code == 200
stored = store.results[task_id][0]["quality_score"]
# 手动编辑后：规则+语义重算，评审分丢弃，evaluated 标记保留。
assert stored["evaluated"] is True
assert stored["judge"] is None
assert stored["layers"]["judge"] is None
assert stored["layers"]["rule"] is not None
assert stored["overall"] >= 0
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
tmp_path: Path,
) -> None:

284
backend/tests/test_data_process_evaluation.py Normal file
View File

@@ -0,0 +1,284 @@
"""数据评测模块（三层质量评分）的单元测试。"""
from __future__ import annotations
import json
from typing import Any
import httpx
import pytest
from app.modules.data_process.algorithms.quality import (
composite_overall,
semantic_quality_scores,
)
from app.modules.data_process.evaluation import (
_JUDGE_DIMENSIONS,
_judge_system_prompt,
_validated_judge_payload,
evaluate_result_record,
reevaluate_edited_record,
)
from app.modules.data_process.generation import ModelGenerationError
RECORD = {
"instruction": "申请编号有什么作用？",
"input": "",
"output": "申请编号用于唯一标识一笔报销申请，便于跟踪审批状态。",
}
SOURCE = "报销系统中，申请编号用于唯一标识一笔报销申请，并支持跟踪审批状态。"
class _FakeEmbedModel:
"""按关键词返回固定向量，模拟语义嵌入。"""
def get_text_embedding(self, text: str) -> list[float]:
if "作用" in text or "编号" in text and "？" in text:
return [0.9, 0.1, 0.0]
if "申请编号" in text:
return [0.85, 0.2, 0.0]
return [0.0, 0.1, 0.9]
class _FailingEmbedModel:
def get_text_embedding(self, text: str) -> list[float]:
raise RuntimeError("embedding unavailable")
class _FakeResponse:
def __init__(self, payload: dict[str, Any]):
self._payload = payload
def raise_for_status(self) -> None:
return None
def json(self) -> dict[str, Any]:
return self._payload
class _FakeClient:
def __init__(self, content: str):
self._content = content
self.calls: list[dict[str, Any]] = []
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
self.calls.append({"endpoint": endpoint, "payload": json})
return _FakeResponse({
"choices": [{"message": {"content": self._content}, "finish_reason": "stop"}],
})
def close(self) -> None:
return None
class _RaisingClient:
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
raise httpx.ConnectError("model endpoint unreachable")
def close(self) -> None:
return None
def _judge_content(scores: dict[str, float], **extra: Any) -> str:
return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra})
def test_judge_system_prompt_covers_rubric_dimensions() -> None:
standard = _judge_system_prompt("standard")
for name in _JUDGE_DIMENSIONS["standard"]:
assert name in standard
assert "1-5" in standard
dpo = _judge_system_prompt("dpo")
assert "chosen_quality" in dpo
assert "preference_reasonableness" in dpo
reasoning = _judge_system_prompt("reasoning")
assert "reasoning_validity" in reasoning
def test_validated_judge_payload_converts_scores_to_overall() -> None:
judged = _validated_judge_payload(
{
"scores": {
"faithfulness": 5,
"correctness": 4,
"clarity": 4,
"completeness": 3,
"alignment": 4,
},
"reason": "答案可靠",
"issues": ["回答略冗长"],
},
"standard",
)
assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2)
assert judged["issues"] == ["回答略冗长"]
assert judged["reason"] == "答案可靠"
def test_validated_judge_payload_clamps_out_of_range_scores() -> None:
judged = _validated_judge_payload(
{
"scores": {
"faithfulness": 9,
"correctness": 4,
"clarity": 4,
"completeness": 0,
"alignment": 4,
},
},
"standard",
)
assert judged["scores"]["faithfulness"] == 5.0
assert judged["scores"]["completeness"] == 1.0
@pytest.mark.parametrize(
"scores",
[
{"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3},
{
"faithfulness": 5,
"correctness": 4,
"clarity": "high",
"completeness": 3,
"alignment": 4,
},
],
)
def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None:
with pytest.raises(ModelGenerationError):
_validated_judge_payload({"scores": scores}, "standard")
def test_semantic_quality_scores_uses_cosine_similarity() -> None:
scores = semantic_quality_scores(
RECORD,
source_content=SOURCE,
embed_model=_FakeEmbedModel(),
)
assert scores is not None
assert 0 < scores["question_answer"] <= 100
assert 0 < scores["answer_source"] <= 100
assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2)
def test_semantic_quality_scores_degrades_to_none_on_failure() -> None:
assert (
semantic_quality_scores(
RECORD,
source_content=SOURCE,
embed_model=_FailingEmbedModel(),
)
is None
)
def test_composite_overall_weights_available_layers() -> None:
assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2)
assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2)
assert composite_overall(rule=80) == 80.0
assert composite_overall(rule=None, judge=100) == 45.0
def test_evaluate_result_record_combines_three_layers() -> None:
client = _FakeClient(
_judge_content({
"faithfulness": 5,
"correctness": 4,
"clarity": 5,
"completeness": 4,
"alignment": 5,
})
)
quality = evaluate_result_record(
RECORD,
source_content=SOURCE,
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
config={"output_type": "standard", "generation_retries": 0},
client=client,
embed_model=_FakeEmbedModel(),
)
assert quality["evaluated"] is True
assert quality["judge"] is not None
assert quality["judge"]["model"] == "judge-model"
assert quality["semantic"] is not None
assert quality["layers"]["judge"] == quality["judge"]["overall"]
assert quality["overall"] == composite_overall(
rule=quality["layers"]["rule"],
semantic=quality["layers"]["semantic"],
judge=quality["layers"]["judge"],
)
# 评审提示词必须携带来源原文作为评分锚点（正文经 NFKC 归一化）。
user_message = client.calls[0]["payload"]["messages"][1]["content"]
assert "申请编号用于唯一标识一笔报销" in user_message
def test_evaluate_result_record_degrades_when_model_fails() -> None:
quality = evaluate_result_record(
RECORD,
source_content=SOURCE,
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
config={"output_type": "standard", "generation_retries": 0},
client=_RaisingClient(),
embed_model=_FakeEmbedModel(),
)
assert quality["judge"] is None
assert quality["layers"]["judge"] is None
assert quality["semantic"] is not None
assert quality["overall"] == composite_overall(
rule=quality["layers"]["rule"],
semantic=quality["layers"]["semantic"],
)
def test_evaluate_result_record_without_model_runs_two_layers() -> None:
quality = evaluate_result_record(
RECORD,
source_content=SOURCE,
model=None,
embed_model=_FakeEmbedModel(),
)
assert quality["judge"] is None
assert quality["evaluated"] is True
assert quality["overall"] == composite_overall(
rule=quality["layers"]["rule"],
semantic=quality["layers"]["semantic"],
)
def test_reevaluate_edited_record_drops_stale_judge() -> None:
previous = {
"evaluated": True,
"judge": {"overall": 90.0},
}
quality = reevaluate_edited_record(
{**RECORD, "output": "编辑后的新答案内容，用于验证重评逻辑。"},
source_content=SOURCE,
previous_quality=previous,
embed_model=_FakeEmbedModel(),
)
assert quality["evaluated"] is True
assert quality["judge"] is None
assert quality["layers"]["judge"] is None
assert quality["semantic"] is not None
def test_reevaluate_edited_record_keeps_unevaluated_state() -> None:
quality = reevaluate_edited_record(
RECORD,
source_content=SOURCE,
previous_quality={},
embed_model=_FakeEmbedModel(),
)
assert quality["evaluated"] is False
assert quality["evaluated_at"] is None

81
backend/tests/test_document_chunking.py
View File

@@ -9,6 +9,7 @@ from app.modules.data_process.document_chunking import (
DocumentChunk,
_compact_with_offsets,
_document_converter,
_nodes_to_chunks,
_project_layout_span,
chunk_fixed_text,
chunk_semantic_text,
@@ -103,6 +104,86 @@ def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() ->
assert cursor > 0
def test_layout_projection_tolerates_list_numbers_inserted_by_serializer() -> None:
# Word 自动编号存放在 numbering.xml，python-docx 抽取的正文没有编号，
# 而 Docling 序列化切片时会补上 "1. " 前缀，投影不能因此失败。
source = "接入方式说明\n结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。"
compact_source, offsets = _compact_with_offsets(source)
start, end, cursor = _project_layout_span(
source,
"1. 结构化数据接入需要先配置连接地址。\n2. 非结构化接入需要上传文档。",
compact_source=compact_source,
source_offsets=offsets,
compact_start=0,
)
assert start is not None and end is not None
assert source[start:end] == "结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。"
assert cursor > 0
def test_layout_projection_never_moves_cursor_backwards() -> None:
source = "重复段落内容。\n中间正文。\n重复段落内容。"
compact_source, offsets = _compact_with_offsets(source)
# 重复内容回退匹配命中已消费的更早位置时，游标必须保持不退。
_, _, cursor = _project_layout_span(
source,
"重复段落内容。",
compact_source=compact_source,
source_offsets=offsets,
compact_start=compact_source.index("中间正文"),
)
assert cursor >= compact_source.index("中间正文")
def test_layout_projection_falls_back_to_line_anchors_for_inserted_content() -> None:
# 表格跨切片时 Docling 会在续片中重复表头，正文不再是连续子串；
# 按行锚点匹配仍应定位到表头所在行到末行数据之间的连续区间。
source = "表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二"
compact_source, offsets = _compact_with_offsets(source)
start, end, _ = _project_layout_span(
source,
"表头甲 表头乙\n第二行数据 说明二",
compact_source=compact_source,
source_offsets=offsets,
compact_start=0,
)
assert start is not None and end is not None
assert source[start:end] == (
"表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二"
)
def test_layout_projection_refuses_low_coverage_anchor_match() -> None:
source = "完全无关的正文内容甲。\n完全无关的正文内容乙。"
compact_source, offsets = _compact_with_offsets(source)
start, end, cursor = _project_layout_span(
source,
"找不到的数据行内容\n另一条找不到的数据行内容",
compact_source=compact_source,
source_offsets=offsets,
compact_start=0,
)
assert start is None
assert end is None
assert cursor == 0
def test_text_splitter_keeps_chunks_that_cannot_be_located() -> None:
class FakeNode:
def get_content(self) -> str:
return "这段文本在源文本中不存在。"
chunks = _nodes_to_chunks([FakeNode()], "完全不同的源文本。")
assert len(chunks) == 1
assert chunks[0].original_content == "这段文本在源文本中不存在。"
assert chunks[0].source_start is None
assert chunks[0].source_start_line is None
def test_short_layout_chunk_merges_with_neighbor_and_keeps_page_provenance() -> None:
source = "短标题\n这是一段足够长的正文内容，用于测试相邻切片合并。"
chunks = [

49
compute/api/main.py
View File

@@ -221,6 +221,35 @@ def create_app() -> FastAPI:
return fallback_gpu_resources()
items: list[dict[str, Any]] = []
processes_by_uuid: dict[str, list[dict[str, Any]]] = {}
try:
process_result = subprocess.run(
[
"nvidia-smi",
"--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
"--format=csv,noheader,nounits",
],
check=True,
capture_output=True,
text=True,
timeout=5,
)
for process_line in process_result.stdout.splitlines():
process_parts = [part.strip() for part in process_line.split(",")]
if len(process_parts) < 4:
continue
process_uuid, pid, process_name, used_memory = process_parts[:4]
processes_by_uuid.setdefault(process_uuid, []).append(
{
"pid": int(_safe_float(pid)),
"name": process_name,
"memory_used_gb": round(_safe_float(used_memory) / 1024, 2),
}
)
except Exception:
# Some driver/runtime combinations do not expose compute-apps;
# utilization and memory metrics remain useful without processes.
pass
for line in result.stdout.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 9:
@@ -244,7 +273,7 @@ def create_app() -> FastAPI:
"temperature": int(_safe_float(temp)),
"power_w": round(_safe_float(power), 1),
"power_limit_w": round(_safe_float(power_limit), 1),
"processes": [],
"processes": processes_by_uuid.get(uuid, []),
}
)
return items
@@ -747,6 +776,23 @@ def create_app() -> FastAPI:
infer_backend: str (default: "huggingface")
infer_dtype: str (default: "auto")
"""
requested_gpus = payload.get("gpu_indices")
if requested_gpus is None:
requested_gpus = payload.get("gpus") or []
try:
requested_gpus = sorted({int(item) for item in requested_gpus})
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"invalid GPU selection: {exc}") from exc
if any(item < 0 for item in requested_gpus):
raise HTTPException(status_code=400, detail="GPU index must be non-negative")
if requested_gpus:
known_gpus = {int(item.get("gpu_index", item.get("id", -1))) for item in gpu_resources()}
missing = sorted(set(requested_gpus) - known_gpus)
if missing:
raise HTTPException(status_code=409, detail=f"requested GPU not found: {missing}")
conflict = sorted(set(requested_gpus).intersection(process_manager.locked_gpus()))
if conflict:
raise HTTPException(status_code=409, detail=f"GPU already used by another compute job: {conflict}")
session = get_inference_session()
result = session.load(
model_name_or_path=payload.get("model_name_or_path", ""),
@@ -754,6 +800,7 @@ def create_app() -> FastAPI:
template=payload.get("template", "qwen"),
infer_backend=payload.get("infer_backend", "huggingface"),
infer_dtype=payload.get("infer_dtype", "auto"),
gpu_indices=requested_gpus,
)
return result

25
compute/engines/llama_factory/inference.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import os
import threading
import time
import uuid
@@ -36,6 +37,7 @@ class InferenceSession:
self._generating_args: dict[str, Any] = {}
self._model_name: str = ""
self._adapter_path: str = ""
self._gpu_indices: list[int] = []
self._loaded_at: float = 0.0
@property
@@ -53,6 +55,7 @@ class InferenceSession:
"loaded_at": self._loaded_at,
"request_id": self._request_id,
"error": self._error,
"gpu_indices": list(self._gpu_indices),
}
def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]:
@@ -87,8 +90,12 @@ class InferenceSession:
template="qwen",
infer_backend="huggingface",
infer_dtype="auto",
gpu_indices=None,
**kwargs,
) -> dict[str, Any]:
requested_gpus = sorted({int(item) for item in (gpu_indices or [])})
if any(item < 0 for item in requested_gpus):
return {"loaded": False, "status": "error", "error": "GPU index must be non-negative"}
with self._state_lock:
if self._status == "loading":
# A model is already loading — dedupe, reuse the same request id.
@@ -97,6 +104,7 @@ class InferenceSession:
self._status = "loading"
self._error = ""
self._request_id = uuid.uuid4().hex[:12]
self._gpu_indices = requested_gpus
self._cancel_requested = False
self._load_args = {
"model_name_or_path": model_name_or_path,
@@ -115,13 +123,21 @@ class InferenceSession:
def _load_worker(self) -> None:
"""Build the ChatModel off the state lock so info() never blocks."""
with self._state_lock:
requested_gpus = list(self._gpu_indices)
model = None
tokenizer = None
generating_args: dict[str, Any] = {}
error = ""
previous_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
try:
# Set visibility before LLaMA-Factory/PyTorch initializes CUDA.
if requested_gpus:
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(item) for item in requested_gpus)
if self._teardown_old:
self._release_model()
with self._state_lock:
self._gpu_indices = requested_gpus
from llamafactory.chat import ChatModel
from llamafactory.hparams import get_infer_args
@@ -138,6 +154,12 @@ class InferenceSession:
generating_args = dict(generating_args)
except Exception as exc: # noqa: BLE001 - surface load failure via status
error = str(exc)
finally:
if requested_gpus:
if previous_visible_devices is None:
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
else:
os.environ["CUDA_VISIBLE_DEVICES"] = previous_visible_devices
with self._state_lock:
if error:
self._model = None
@@ -152,6 +174,7 @@ class InferenceSession:
self._model = None
self._tokenizer = None
self._status = "idle"
self._gpu_indices = []
return
self._model = model
self._tokenizer = tokenizer
@@ -189,6 +212,7 @@ class InferenceSession:
self._adapter_path = ""
self._loaded_at = 0.0
self._error = ""
self._gpu_indices = []
def unload(self) -> dict[str, Any]:
with self._state_lock:
@@ -208,6 +232,7 @@ class InferenceSession:
self._adapter_path = ""
self._loaded_at = 0.0
self._error = ""
self._gpu_indices = []
return {"unloaded": True, "status": "idle"}
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]:

1
docker/app/.env
View File

@@ -60,5 +60,6 @@ MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=change_me_minio_secret
MINIO_BUCKET=yg-ft-resources
MINIO_SECURE=false
MINIO_INLINE_MAX_BYTES=262144
STORAGE_WAIT_SECONDS=300
STORAGE_CHECK_INTERVAL_SECONDS=10

1
docker/app/docker-compose.yml
View File

@@ -70,6 +70,7 @@ services:
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
MINIO_SECURE: ${MINIO_SECURE:-false}
MINIO_INLINE_MAX_BYTES: ${MINIO_INLINE_MAX_BYTES:-262144}
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process}

89
docs/governance-user-guide.md
View File

@@ -1,9 +1,10 @@
# 平台治理功能使用指南
> 版本：v1.1
> 日期：2026-08-13
> 适用版本：YG Zhilian v1.0+
> 更新说明：移除页面权限码设计，改为基于角色的简化权限模型
> 版本：v1.3
> 日期：2026-08-19
> 适用版本：YG Fine-Tune Platform v1.0+
> 更新说明：合并组织权限、审批和运行日志入口；取消项目空间菜单但保留旧接口兼容
---
@@ -40,28 +41,27 @@
### 1.3 入口在哪里？
所有治理功能集中在左侧导航栏的 **「系统设置」** 和 **「平台治理」** 分组下：
治理、组织和运维功能按职责分布在左侧导航栏的 **「平台治理」**、**「系统设置」** 和 **「算力资源」** 分组下：
```
系统设置
├── 用户设置 ← 用户 CRUD + 角色权限 + 密码管理（仅 admin）
├── 平台性能 ← 系统监控
└── 查看日志 ← 日志查看
平台治理
├── 租户管理 ← 组织/团队（仅 admin）
├── 项目空间 ← 项目级资源隔离（仅 admin）
├── 审批模板 ← 定义哪些操作需要审批（仅 admin）
├── 审批中心 ← 处理待审批请求（仅 admin）
└── 审计日志 ← 查看所有操作记录（仅 admin）
├── 组织与权限 ← 用户与角色、租户与配额（仅 admin）
├── 资源授权 ← 数据集、模型等资源授权（仅 admin）
└── 审批中心 ← 待审批请求与审批策略（仅 admin）
系统设置
├── 平台性能 ← 系统资源监控
└── 运行日志 ← 运行日志、审计记录、操作诊断
算力资源
└── 算力节点 ← GPU 分配与管理（仅 admin）
```
> ⚠️ 以上菜单**只有 admin 用户能看到**。普通用户登录后不会出现这些入口。
> ⚠️ 平台治理、资源授权、审批中心和算力节点菜单仅 admin 用户能看到。运行日志入口继续沿用原权限，普通用户可查看系统/训练日志；审计记录和操作诊断页签仅 admin 可见。
>
> **重要变更（v1.1）**：非 admin 用户**默认可以访问所有业务功能菜单**（模型训练、评测、推理、数据集、数据处理等），无需管理员单独分配权限。
> **重要变更（v1.2）**：非 admin 用户**默认可以访问所有业务功能菜单**（模型训练、评测、推理、数据集、数据处理等），无需管理员单独分配权限；治理和资源管理入口仍仅 admin 可见。
> **当前菜单调整（v1.3）**：平台不再提供项目空间菜单和项目级操作入口。历史项目表、接口和旧地址仅作为兼容层保留，当前资源访问以用户所有权、租户边界（如启用）和资源 ACL 为准；新建业务资源不再要求项目字段。
---
@@ -73,24 +73,25 @@
| 用户类型 | 可见菜单 | 说明 |
|---------|---------|------|
| **admin（管理员）** | **全部菜单** | 包括用户设置、平台治理、算力节点等管理功能 |
| **非 admin 用户** | **除管理功能外的所有业务菜单** | 模型训练/评测/推理、数据集、数据处理、日志等 |
| **admin（管理员）** | **全部菜单** | 包括组织权限、审批、运行日志、平台治理和算力节点等管理功能 |
| **非 admin 用户** | **除管理功能外的所有业务菜单** | 模型训练/评测/推理、数据集、数据处理等 |
> **核心原则**：
> - 非 admin 用户**默认拥有所有业务功能的访问权限**，无需单独分配
> - 仅以下功能**仅管理员可见**：
> - `用户设置`（用户 CRUD、角色管理）
> - `平台治理`（租户管理、项目空间、审批模板/中心、审计日志）
> - `平台治理 - 组织与权限`（用户、角色、租户与配额）
> - `平台治理`（资源授权、审批中心）
> - `算力节点`（GPU 分配）
> - `运行日志`中的审计记录和操作诊断
>
> 资源级别的访问控制通过 **ACL（访问控制列表）** 实现，详见第 4 章。
### 2.2 创建用户
**路径**：`用户设置` → `创建用户`
**路径**：`平台治理` → `组织与权限` → `用户与角色` → `创建用户`
1. 以 admin 身份登录平台
2. 进入「用户设置」页面
2. 进入「组织与权限」页面的「用户与角色」页签
3. 点击右上角「创建用户」按钮
4. 填写信息：
- **账号**：登录用户名（如 `zhangsan`）
@@ -107,12 +108,10 @@
| 功能分组 | 包含菜单 | 路由前缀 |
|---------|---------|----------|
| 系统设置 - 用户设置 | 用户列表、创建用户、重置密码 | `/user-settings` |
| 平台治理 - 租户管理 | 租户列表、配额设置 | `/tenants` |
| 平台治理 - 项目空间 | 项目列表、成员管理、ACL | `/projects` |
| 平台治理 - 审批模板 | 审批流程定义 | `/approval-templates` |
| 平台治理 - 审批中心 | 待审批请求处理 | `/approval-instances` |
| 平台治理 - 审计日志 | 操作记录查询与导出 | `/audit-logs` |
| 平台治理 - 组织与权限 | 用户、角色、租户与配额 | `/organization` |
| 平台治理 - 资源授权 | 数据集、模型等资源 ACL | `/resource-acl` |
| 平台治理 - 审批中心 | 待审批请求、审批历史与策略 | `/approval-instances` |
| 系统设置 - 运行日志 | 系统/训练日志；管理员可查看审计记录、操作诊断 | `/logs` |
| 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` |
### 2.4 重置用户密码
@@ -125,18 +124,18 @@
3. 输入新密码，确认
**方式二：用户自行修改**
1. 用户登录后在「用户设置」页面点击「修改密码」按钮
1. 用户登录后在「组织与权限」页面的「用户与角色」页签点击「修改密码」按钮
2. 输入旧密码 + 新密码（至少 6 位）
3. 确认修改
### 2.5 删除用户
**路径**：`用户设置` → 用户列表 → 操作列「删除」
**路径**：`组织与权限` → `用户与角色` → 用户列表 → 操作列「删除」
> ⚠️ 删除用户时会**级联清理**其所有关联数据：
> - 该用户创建的数据集、基座模型、微调产物、评测任务
> - 该用户的 ACL 授权记录、GPU 分配记录
> - 该用户的审批实例、审计日志、项目成员关系、登录会话
> - 该用户的审批实例、审计日志、历史项目成员关系、登录会话
> - **训练任务保留不删**（避免算力节点上的物理任务数据不一致）
---
@@ -148,7 +147,7 @@
当服务器有多张 GPU 卡（如 8×A800）时，需要指定**哪个用户能用哪张卡**：
- 避免两个人同时选同一张卡导致训练冲突
- 按团队/项目隔离算力资源
- 按用户和租户边界隔离算力资源
- 控制每个用户的 GPU 配额
### 3.2 分配 GPU（仅 admin）
@@ -259,7 +258,6 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
| 删除他人的数据集 | 非 admin 删除别人创建的数据集 | 创建审批实例 或 admin 直接执行 |
| 删除他人的模型 | 非 admin 删除别人创建的模型 | 同上 |
| 停止他人的训练任务 | 非 admin 停止别人发起的任务 | 同上 |
| 归档/删除项目空间 | 存在待审批变更时 | 拒绝执行 |
**核心规则**：admin 做任何操作都直接执行（旁路）；普通用户操作他人资源时进入审批流程。
@@ -305,7 +303,7 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
3. 决策：「通过」或「拒绝」
4. 决策结果自动执行对应操作并记录审计日志
**审批模板**（`平台治理` → `审批模板`）：定义每种操作需要几步审批、每步谁来审。默认模板都是单步（admin 审批即可）。
**审批策略**（`平台治理` → `审批中心` → `审批策略`）：定义每种操作需要几步审批、每步谁来审。默认模板都是单步（admin 审批即可）。
---
@@ -324,24 +322,29 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
### 6.2 查询审计日志
**路径**：`平台治理` → `审计日志`
**路径**：`系统设置` → `运行日志` → `审计记录`
支持筛选条件：
| 筛选项 | 说明 |
|---|---|
| 操作人 | 按用户 ID 过滤 |
| 动作类型 | 如 `user.create`, `dataset.delete`, `gpu.assign` 等 |
| 目标资源类型 | dataset / model / fine_tune_task 等 |
| 租户 | 按租户名称选择 |
| 操作人 | 按用户名称选择，不需要手工填写用户 ID |
| 动作类型 | 使用中文动作选择，例如创建数据集、删除模型、授予资源权限 |
| 目标资源类型 | 使用中文资源类型选择，例如数据集、模型、训练任务 |
| 关键词 | 模糊搜索目标 ID 或审计详情 |
| 目标 ID | 对指定资源 ID 进行精确查询 |
| 时间范围 | 开始时间 ~ 结束时间 |
项目筛选已移除。底层接口仍兼容历史 `project_id` 参数，但当前平台不再提供项目菜单。
### 6.3 导出审计日志
审计日志页面底部有「导出 CSV」按钮，导出的文件包含当前筛选条件下的全部记录，可用于合规审计或问题追溯。
运行日志的「审计记录」页签提供「导出 CSV」按钮；「操作诊断」页签用于检索失败操作和接口耗时，可用于问题追溯。
### 6.4 日志保留策略
审计日志受**留存策略**控制（`平台治理` → 租户管理 → 绑定留存策略）。默认保留 30 天，超期自动清理。
审计日志受**留存策略**控制（`平台治理` → `组织与权限` → `租户与配额`）。默认保留 30 天，超期自动清理。
---
@@ -351,7 +354,7 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
根据 v1.1 权限模型：
1. **业务菜单**（训练、评测、推理、数据集等）：普通用户**默认全部可见**，无需分配
2. **管理菜单**（用户设置、租户管理、算力节点等）：**仅 admin 可见**，这是设计如此
2. **管理菜单**（组织与权限、资源授权、审批中心、算力节点，以及运行日志中的审计/诊断页签）：**仅 admin 可见**，这是设计如此
如果普通用户看不到业务菜单，请检查：
- 用户是否正常登录（token 是否有效）
@@ -402,7 +405,7 @@ curl -H "Authorization: Bearer platform-token-admin" \
### Q6: 用户忘记密码怎么办？
两种方案：
1. **admin 重置**：在「用户设置」→ 用户列表 →「重置密码」
1. **admin 重置**：在「组织与权限」→「用户与角色」→ 用户列表 →「重置密码」
2. **用户自助修改**：用户登录后点击「修改密码」（需知道旧密码）
如果是完全忘记且不是 admin，只能由 admin 重置。

28
docs/menu-functional-requirements.md
View File

@@ -1,6 +1,8 @@
# 菜单与功能需求总览
> 本文根据当前前端侧边栏、路由、需求文档、接口文档、部署文档和 SQL 脚本整理。当前代码和 SQL 均按正式系统开发基线维护；Mock、Simulator 只能作为显式联调能力，不作为默认开发准则。
>
> 当前治理版本取消“项目空间”菜单。历史项目表和接口仅保留兼容，不再作为前端业务入口或新资源的必填隔离层。
## 1. 菜单分层
@@ -17,9 +19,11 @@
| 数据治理 | 数据处理 | `/data-process` | `data-process` | 前端页面已有，后端待完整实现 | 文档上传、切片预览、LLM 生成、结果编辑、发布数据集 |
| 其他工具 | 数据类型转换 | `/data-convert` | `data-convert` | 前端页面已有，后端待实现 | JSON/JSONL/Markdown 等格式转换任务 |
| 算力资源 | 算力节点 | `/compute` | `compute` | 已接入节点管理接口 | 节点地址、权重、标签、启用状态、GPU、队列、资源副本 |
| 系统设置 | 用户设置 | `/user-settings` | `user-settings` | 已接入基础用户接口 | 用户列表、创建用户、启停、页面权限 |
| 平台治理 | 组织与权限 | `/organization` | `user-settings` | 新增合并入口 | 用户与角色、租户与配额、密码管理 |
| 平台治理 | 资源授权 | `/resource-acl` | `user-settings` | 已接入 ACL 接口 | 数据集、模型等资源授权 |
| 平台治理 | 审批中心 | `/approval-instances` | `user-settings` | 新增合并入口 | 待审批请求、审批历史、审批策略 |
| 系统设置 | 平台性能 | `/hardware` | `hardware` | 已有接口，需接真实采集 | CPU、内存、磁盘、GPU、进程、网络监控 |
| 系统设置 | 查看日志 | `/logs` | `logs` | 已有接口，需接真实日志文件 | 后端日志、error 日志、训练日志索引、日志内容查看 |
| 系统设置 | 运行日志 | `/logs` | `logs` | 新增合并入口 | 运行日志、训练日志；管理员可查看审计记录、操作诊断 |
### 1.2 当前二级和隐藏路由
@@ -41,19 +45,17 @@
| 数据集创建/编辑/预览 | `/dataset/create`、`/dataset/:id/edit`、`/dataset/:id/preview` | 数据集管理 | 数据集元数据、文件、版本与内容 |
| 自定义工具 | `/tools`、`/tools/create`、`/tools/:id/edit` | 规划入口 | 路由存在，当前侧边栏未展示，后续可归入“其他工具” |
| 算力子页 | `/compute/gpus`、`/compute/queue`、`/compute/nodes` | 算力节点 | 当前可作为页签或深链 |
| 创建用户/权限设置 | `/user-settings/create`、`/user-settings/:id/permission` | 用户设置 | 用户创建和页面权限 |
| 组织与权限内部页签 | `/user-settings`、`/tenants`、`/user-settings/create`、`/user-settings/:id/permission` | 平台治理 - 组织与权限 | 旧地址兼容，当前通过页签进入 |
| 项目旧地址 | `/projects`、`/projects/:id` | 兼容跳转 | 跳转到组织与权限，不再展示项目管理 |
| 审批策略旧地址 | `/approval-templates` | 兼容跳转 | 跳转到审批中心的策略页签 |
| 日志旧地址 | `/audit-logs`、`/operation-logs` | 兼容跳转 | 跳转到运行日志对应页签 |
| 无权限页 | `/permission-denied` | 系统页 | 路由守卫无权限跳转 |
### 1.3 企业治理待补菜单
### 1.3 后续治理扩展
| 建议菜单分组 | 菜单 | 建议路由 | 优先级 | 必要性 |
| --- | --- | --- | --- | --- |
| 组织与项目 | 租户管理 | `/tenants`、`/tenants/:id` | P0 | 多租户隔离、配额、留存策略入口 |
| 组织与项目 | 项目空间 | `/projects`、`/projects/:id`、`/projects/:id/members` | P0 | 项目级模型/数据集/任务隔离 |
| 组织与项目 | 资源授权 | `/projects/:id/permissions` 或资源详情弹窗 | P0 | 模型/数据集/任务级 ACL |
| 治理中心 | 审批中心 | `/approvals`、`/approvals/:id` | P0 | 删除、发布、导出、停止他人任务等高风险动作 |
| 治理中心 | 审批设置 | `/approval-settings` | P1 | 审批模板、审批人规则、超时策略 |
| 治理中心 | 审计中心 | `/audit-logs`、`/login-logs`、`/download-logs` | P1 | 操作审计、登录审计、下载审计、导出 |
| 系统设置 | 运行日志扩展 | `/logs`、`/login-logs`、`/download-logs` | P1 | 增加登录审计、下载审计、导出审计维度 |
| 运维中心 | 存储管理 | `/storage` | P1 | 本地磁盘占用、临时文件、checkpoint 清理、留存 |
| 运维中心 | 训练引擎管理 | `/training-engines` | P2 | LLaMA-Factory 和后续引擎能力 schema、健康检查 |
| 模型服务 | 模型服务治理 | `/model-services`、`/model-services/:id` | P1 | 测试/生产服务发布、调用统计、下线审批 |
@@ -62,7 +64,7 @@
| 菜单/模块 | 主要接口 | 当前运行 SQL | 目标 SQL |
| --- | --- | --- | --- |
| 登录、用户设置 | `/modelTF/login`、`/modelTF/me`、`/modelTF/users` | `users` | `users`、`login_sessions`、`permissions`、`role_permissions`、`user_permission_overrides` |
| 登录、组织与权限 | `/modelTF/login`、`/modelTF/me`、`/modelTF/users`、`/modelTF/tenants` | `users`、`tenants` | `users`、`login_sessions`、`permissions`、`role_permissions`、`user_permission_overrides`、`tenants` |
| 服务看板 | `/modelTF/dashboard/overview`、`/modelTF/health` | 复用模型/数据集/任务/算力表 | `system_metric_snapshots`、`web_logs`、各业务表聚合 |
| 模型管理 | `/modelTF/model-manage`、`/modelTF/model-manage/trained-models`、`/modelTF/model-manage/merge` | `models`、`trained_models` | `models`、`trained_models`、`storage_objects`、`local_import_jobs`、`resource_acl` |
| 数据集管理 | `/modelTF/dataset-manage`、`/modelTF/dataset-manage/upload/{id}`、`/preview`、`/versions` | `datasets`、`dataset_files` | `datasets`、`dataset_files`、`dataset_file_versions`、`dataset_records`、`storage_objects` |
@@ -70,12 +72,12 @@
| 训练日志 | `/modelTF/training-log-files`、`/modelTF/training-log-content` | 由任务表生成索引 | 日志文件元数据、`fine_tune_metrics`、`audit_logs` |
| 算力节点 | `/modelTF/compute/nodes`、`/compute/gpus`、`/compute/queue`、`/compute/nodes/{id}/replicas` | `compute_nodes`、`gpus`、`resource_replicas`、`resource_sync_jobs` | `compute_nodes`、`gpu_devices`、`compute_node_engines`、`compute_jobs`、`resource_replicas`、`resource_sync_jobs` |
| 平台性能 | `/modelTF/system-info`、`/modelTF/compute/gpus` | `gpus`、任务表 | `system_metric_snapshots`、`gpu_devices`、`compute_jobs` |
| 查看日志 | `/modelTF/log-files`、`/modelTF/log-content`、`/modelTF/web-log` | 文件日志 | `web_logs`、`audit_logs`，大日志进入日志平台 |
| 运行日志 | `/modelTF/log-files`、`/modelTF/log-content`、`/modelTF/web-log`、`/modelTF/audit-logs` | 文件日志 | `web_logs`、`audit_logs`，大日志进入日志平台 |
| 模型评测 | `/modelTF/model-eval`、`/modelTF/dimension` | 当前运行 SQL 未覆盖 | `eval_tasks`、`eval_dimensions`、`eval_sample_results`、`eval_dimension_summaries` |
| 模型推理/对比 | `/modelTF/model-compare`、`/modelTF/model-chat/*` | 当前运行 SQL 未覆盖 | `inference_tasks`、`inference_task_models`、`chat_sessions`、`chat_messages` |
| 数据处理 | `/modelTF/data-process/*` | 当前运行 SQL 未覆盖 | `data_process_tasks`、`data_process_source_files`、`data_process_preview_items`、`data_process_results` |
| 数据转换/自定义工具 | `/modelTF/data-convert/jobs`、`/modelTF/tools` | 当前运行 SQL 未覆盖 | `data_convert_jobs`、`custom_tools` |
| 租户/项目/资源授权 | `/modelTF/tenants`、`/modelTF/projects`、`/modelTF/resources/{type}/{id}/acl` | 当前运行 SQL 未覆盖 | `tenants`、`tenant_users`、`projects`、`project_members`、`resource_acl` |
| 租户/资源授权 | `/modelTF/tenants`、`/modelTF/resources/{type}/{id}/acl` | 当前运行 SQL 未覆盖 | `tenants`、`tenant_users`、`resource_acl`；`projects`、`project_members` 仅作兼容 |
| 审批/审计/留存/配额 | `/modelTF/approvals`、`/modelTF/audit-logs`、`/modelTF/retention-policies`、`/modelTF/quotas/usage` | 当前运行 SQL 未覆盖 | `approval_templates`、`approval_instances`、`approval_steps`、`audit_logs`、`retention_policies`、`quotas`、`quota_usage` |
## 3. 文档和脚本检查结论

4
docs/permissions-design.md
View File

@@ -4,6 +4,8 @@
> 日期：2026-08-02
> 状态：设计基线，供后端实现和前端联调参照
> **当前菜单基线（2026-08-19）**：平台治理已取消“项目空间”作为用户可见菜单和新资源的业务隔离层。当前前端入口为“组织与权限、资源授权、审批中心”，系统设置下的“运行日志”承载运行日志、审计记录和操作诊断。`projects`、`project_members` 表及相关后端接口仅作历史兼容，不删除、不要求新建资源填写 `project_id`。
---
## 目录
@@ -129,7 +131,7 @@
| `compute` | `/compute` | 算力节点 |
| `hardware` | `/hardware` | 平台性能 |
| `logs` | `/logs`, `/training-log/:id` | 查看日志 |
| `user-settings` | `/user-settings`, `/tenants`, `/projects`, `/approvals`, `/audit-logs` | 系统设置与平台治理 |
| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances`, `/logs` | 平台治理和运行日志（管理员） |
---

120
docs/platform-governance-menu-design.md Normal file
View File

@@ -0,0 +1,120 @@
# 平台治理菜单设计与开发计划
> 版本：v1.0
> 日期：2026-08-19
> 状态：按本文档实施
## 1. 设计结论
当前项目已经有用户所有权、资源 ACL、租户接口和审计接口，但项目隔离尚未真正落地。核心资源的 `project_id` 当前没有有效业务数据，训练、评测、推理和数据集创建流程也没有统一的项目上下文。
因此当前版本取消项目层级设计，资源权限统一采用：
```text
用户所有权 + 资源 ACL + 租户边界（可选）
```
项目相关数据库表和后端接口暂不物理删除，仅作为历史兼容能力保留，后续不再新增项目数据，也不在前端提供项目入口。
## 2. 最终菜单
```text
平台治理
├── 组织与权限
├── 资源授权
└── 审批中心
系统设置
├── 平台性能
└── 运行日志
├── 系统日志
├── 训练日志
├── 审计记录
└── 操作诊断
算力资源
└── 算力节点
```
### 2.1 组织与权限
使用页签统一承载：
- 用户与角色：用户 CRUD、启停、密码、角色。
- 租户与配额：租户、GPU 配额、存储配额和资源数量配额。
用户、租户和配额仍使用独立表和接口，不把组织配额字段混入用户表。单租户部署时可默认停留在“用户与角色”页签。
### 2.2 资源授权
保留当前资源 ACL 能力，支持数据集、训练模型等资源的 `read/write/execute/download/delete/admin` 权限。项目不再作为授权前置条件。
### 2.3 审批中心
统一使用页签承载：
- 待审批/审批历史。
- 我的申请。
- 审批策略，仅管理员可见。
“审批策略”不再作为独立一级菜单。
### 2.4 运行日志
在现有系统日志、训练日志基础上增加：
- 审计记录：写操作、授权、审批、删除、导出等敏感操作。
- 操作诊断：失败操作、错误类型和接口耗时。
“审计中心”不再作为独立菜单。旧的 `/audit-logs` 和 `/operation-logs` 地址保留重定向。
## 3. 兼容策略
| 原入口 | 新入口/处理方式 |
|---|---|
| `/user-settings` | 重定向到 `/organization?tab=users` |
| `/tenants` | 重定向到 `/organization?tab=tenants` |
| `/approval-templates` | 重定向到 `/approval-instances?tab=strategies` |
| `/audit-logs` | 重定向到 `/logs?tab=audit` |
| `/operation-logs` | 重定向到 `/logs?tab=operations` |
| `/projects` | 移除前端入口；旧地址重定向到组织与权限 |
后端的租户、项目、审批、ACL、审计 API 暂不删除，保证已有脚本和历史客户端不立即失效。数据库不执行删表操作，也不新增项目字段迁移。
## 4. 开发计划
### 阶段一：导航和页面聚合
1. 新增“组织与权限”聚合页面。
2. 新增“审批中心”聚合页面。
3. 扩展“运行日志”页面，加入审计和操作诊断页签。
4. 调整侧边栏，只展示最终菜单。
### 阶段二：兼容旧入口
1. 旧用户、租户、审批策略、审计和操作日志路由改为重定向。
2. 保留原页面组件、API 和后端路由，避免历史调用失效。
3. 项目路由不再作为业务入口，不再新增项目数据。
### 阶段三：权限和功能检查
1. 管理员可以访问组织、租户、配额、审批、审计和 ACL。
2. 普通用户不能访问平台治理菜单；运行日志基础页签继续保持原有访问权限。
3. 审批策略页签仅管理员可见。
4. 审计和操作诊断仍保留管理员可见能力。
5. 数据集、模型、训练、评测、推理继续使用用户所有权和 ACL，不增加项目选择器。
### 阶段四：验证
- `npm run build`。
- 检查旧路由重定向。
- 检查管理员菜单显示。
- 检查非管理员权限拦截。
- 检查 Backend 健康接口和前端静态资源。
## 5. 暂不处理事项
- 不删除 `projects`、`project_members` 表。
- 不删除后端项目模块，避免历史数据和接口调用中断。
- 不把租户配额字段直接合并到 `users` 表。
- 不改变现有数据集、模型、训练、评测、推理的业务接口格式。

347
docs/当前项目开发进度.md Normal file
View File

@@ -0,0 +1,347 @@
# 当前项目开发进度
> 评估基线：2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。
>
> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容，统一标记为“部分完成”或“未完成”。
## 一、项目定位与总体结论
当前项目是一个面向多用户、多算力节点的模型训练与推理平台，主要链路为：
```text
Vue 前端
|
FastAPI Backend API
|-- PostgreSQL：业务元数据、权限、任务状态、小型内容和预览数据
|-- Redis：会话、限流、短期缓存和任务辅助状态
|-- MinIO：模型、数据集、报告和大文件的统一对象存储
|-- Compute API / Agent：训练、推理、评测、模型合并和 GPU 执行
|
多台算力节点
```
整体判断：
| 范围 | 当前状态 | 结论 |
|---|---|---|
| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 |
| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 |
| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入，跨节点一致性和失败恢复仍需加强 |
| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现，但完整的租户/项目隔离尚未闭环 |
| MinIO 统一存储 | 部分完成 | 大文件和模型已接入，仍存在兼容性的本地路径和部分数据双写/回退路径 |
| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 |
| 前端体验 | 基本可用，需优化 | 构建问题已持续修复，但页面响应等待、首屏体积和部分错误提示仍需优化 |
## 二、已完成的功能
### 2.1 平台基础与部署
- 已建立 Vue 3 + TypeScript + Vite 前端工程。
- 已建立 FastAPI 后端服务，提供登录、平台管理和模型业务接口。
- 已建立 Compute API / Agent，用于连接算力节点并执行训练、推理、评测和模型处理任务。
- 已使用 PostgreSQL 保存核心业务数据，Redis 提供会话、限流和缓存能力。
- 已增加 MinIO 服务及 Backend 的 MinIO 配置，支持和 Compute Agent 分离部署。
- 已提供 `docker/app`、`docker/compute`、`docker/minio` 和 `docker/offline` 部署目录。
- 已考虑后端、算力服务、MinIO 分布在不同服务器时使用独立网络；Compute 节点访问 MinIO 需要配置所有节点都能访问的固定 IP 或 DNS。
- 离线部署目录已经同步后端、算力相关源码和初始化 SQL 的主要改造内容。
### 2.2 登录、用户和权限基础
- 用户登录、退出、当前用户信息和密码修改接口已经存在。
- 已有 Token 会话、Redis 会话记录和登录限流逻辑。
- 已建立用户、角色、权限码和角色权限关系。
- 已实现管理员、普通用户等基础角色分层。
- 已实现页面路由守卫、菜单过滤和前端按钮级权限的基础能力。
- 已建立资源 ACL 管理页面和相关接口，可对用户或角色授予资源级权限。
- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。
- 已建立运行日志、审计日志查询页面及审计记录写入机制。
- 已加入软删除相关字段和部分删除逻辑，避免直接物理删除业务资源。
### 2.3 算力节点与 GPU 资源
- 已实现算力节点的新增、编辑、启用、禁用、维护/删除、连通性测试和健康检查。
- 已实现节点列表、节点详情、节点副本/同步状态和 Compute Agent 连接。
- 已实现 GPU 信息发现、GPU 状态查询和队列查询。
- 已建立 `gpu_allocations`、`gpu_assignments`、调度锁等资源分配表。
- 训练任务已经支持选择调度节点和一张或多张 GPU，并在预检阶段校验资源可用性。
- Compute Agent 已支持训练、评测、推理和缓存准备等任务接口。
- 已存在资源副本和同步任务模型，用于记录节点侧资源同步状态。
### 2.4 数据集管理
- 已实现数据集创建、列表、详情、编辑、删除和文件上传。
- 已实现数据集文件下载、预览、记录列表和数据记录编辑入口。
- 已处理 JSON 与 JSONL 的记录数差异：JSON 数组按元素计数，JSONL 按有效行计数，避免把整个 JSON 文件误按行数统计。
- 已提供数据集版本列表、版本详情、创建版本、切换当前激活版本和删除版本接口。
- 已增加数据集文件、版本、数据记录等初始化表结构。
- 已支持数据集文件在数据库小内容和 MinIO 大文件之间按策略存储。
- 已在训练预检中检查数据集文件是否存在、是否可从 MinIO 获取以及是否能准备到目标算力节点。
### 2.5 数据处理与数据转换
- 已提供结构化数据、非结构化数据、外部数据源的处理创建流程。
- 已实现源文件上传、预览、分片/切分、生成、质量检查、去重和结果管理等数据处理流程。
- 已支持处理结果生成数据集或导入数据集版本。
- 已建立数据处理任务、源文件、预览项、结果等数据表。
- 已提供 JSON、JSONL 等数据格式转换页面和后端任务接口。
- 已将数据转换输出接入 MinIO/数据库分层存储：小型文本结果可存数据库，大文件存 MinIO。
- 已处理输出文件下载和转换结果元数据保存问题。
### 2.6 模型训练
- 已实现训练任务创建、配置预检、命令预览、启动、停止、重试和删除。
- 已接入 LLaMA-Factory 等训练适配逻辑。
- 已支持选择训练数据、基座模型、算力节点和 GPU。
- 已实现训练日志获取、训练任务概览、诊断信息、检查点和训练指标查询。
- 前端训练详情已经具备训练曲线解析和展示逻辑，日志轮询间隔已调整为 3 秒。
- 已增加 GPU 详情展示入口，包括显存和利用率等 Compute Agent 上报信息。
- 已支持训练任务的 MinIO 数据准备和目标算力节点缓存准备。
### 2.7 模型管理与权重合并
- 已实现在线模型/基座模型和训练模型的列表、创建、详情、用途修改和删除。
- 已建立模型、训练模型、模型血缘、模型产物和导出任务相关表。
- 已提供权重合并入口，能够根据训练任务准备基座模型和 Adapter，并提交 Compute Agent 执行合并。
- 已增加模型产物和 MinIO 对象关联字段。
- 合并结果能够在任务完成后归档到 MinIO 的设计和主要代码路径已经建立。
### 2.8 模型推理与模型对比
- 已实现推理模型列表、创建、详情和删除入口。
- 已实现模型加载、卸载、服务启动、服务状态查询和对话调用。
- 已实现模型对比任务及多模型聊天相关接口。
- 已增加推理失败重试、停止和资源释放的处理路径。
- 已支持根据页面选择的算力节点准备模型缓存，兼容训练时所选节点优先的业务要求。
- Compute Agent 已提供本地推理会话和模型缓存状态接口。
### 2.9 模型评测
- 已实现评测任务列表、创建、详情和删除。
- 已实现评测维度管理和评测规则配置页面。
- 已建立评测任务、评测维度、对比任务等数据库表。
- 已支持选择模型、数据集、评测维度、算力节点和 GPU 的基础流程。
- 已接入 Compute Agent 执行评测任务，并保存评测结果和报告相关元数据。
### 2.10 数据存储策略
- 已建立 `storage_objects`、`storage_cache_jobs` 等 MinIO 元数据和缓存任务表。
- 已建立 MinIO 对象上传、下载、预签名 URL 和节点缓存准备的主要接口。
- 已采用分层策略：
- 小型 JSON、JSONL、CSV、任务参数快照、预览数据保留在数据库，降低频繁预览的 MinIO 延迟。
- 模型权重、训练产物、评测报告和大文件使用 MinIO。
- 小型 PDF、DOCX、XLSX 仍优先存 MinIO，以保留原始二进制文件内容。
- 已增加 `data_convert_tasks.output_content`，用于保存小型转换结果，避免所有小结果都依赖 MinIO。
- Backend 和离线包中的 `000_full_init.sql` 已同步，当前两份初始化脚本内容一致。
## 三、部分完成、仍需完善的功能
### 3.1 MinIO 统一数据源尚未完全闭环
当前 MinIO 已成为模型、大文件和跨节点资源的主存储方向，但仍保留以下兼容路径：
- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。
- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段，这是当前已确认的小文件性能策略，不是错误，但必须统一记录来源、大小、校验值和版本。
- 大文件在部分代码路径中仍通过 `read()` 或 `put_bytes()` 一次性读入内存，未完成流式或分片上传。
- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联，数据库没有直接外键，删除和数据一致性需要应用层保证。
- 删除业务资源后，对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。
### 3.2 MinIO 预签名接口的权限边界需要加强
当前预签名接口已经存在，但 PUT 上传场景仍需要重点补强：
- 需要根据资源类型和资源 ID 校验当前用户的写权限，而不应只校验读取权限。
- 需要服务端生成并校验对象 Key，避免客户端任意写入其他用户或其他资源的对象路径。
- 需要增加上传完成确认接口，校验对象实际存在、大小和校验值后再写入业务表。
- 需要限制允许的 Bucket、Content-Type、大小和有效期。
- 需要记录预签名创建、上传完成、失败和过期事件，便于审计。
### 3.3 激活版本和跨节点资源版本仍需加强
数据集已经有 `active_version_id` 和版本表，但以下场景仍需补充：
- 训练、推理和评测必须只使用资源当前激活版本，并在任务创建时固化版本 ID。
- 同一文件名的不同版本不能只依靠文件名同步，应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。
- 已创建任务在后续切换激活版本后，不能被意外切换到新版本。
- 需要为每个准备到算力节点的资源保存版本、对象 ETag/校验值和本地路径清单。
- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。
### 3.4 权限 2.0 尚未完全落地
已有用户、角色、权限码、ACL、审批和审计基础，但仍存在以下差距：
- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。
- 项目空间设计已经讨论过取消，但数据库中仍保留 `projects`、`project_members` 等历史结构，需要明确兼容策略和最终迁移方式。
- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。
- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。
- 前端按钮权限已经有基础实现，但不能替代后端鉴权；仍需要对所有关键动作进行后端默认拒绝校验。
- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。
### 3.5 模型合并、导出和评测报告闭环不足
- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立，但失败时的清理、重试和幂等性仍需加强。
- 合并结果归档到 MinIO 的逻辑主要依赖任务完成轮询，服务重启或轮询中断时可能需要补偿扫描。
- 模型导出任务目前有查询模型和表结构，但完整的创建、执行、进度、失败重试和下载闭环尚未完成。
- 评测结果和报告字段已经存在，但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。
- 评测指标配置和执行器返回指标之间仍需要强类型映射，避免前端显示为通用的 `custom`。
### 3.6 GPU 资源分配需要统一到所有任务类型
- 训练已经有较完整的节点/GPU 选择和预检流程。
- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码，但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。
- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。
- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。
- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报，仍需要校验采样时间、单位、空值和任务对应关系。
## 四、尚未完成的功能
以下功能在当前代码中没有形成可验收的完整闭环，或仍处于设计/基础代码阶段：
1. **完整的租户隔离和资源继承模型**：所有列表、详情、下载、缓存、训练、推理、评测和导出接口都需要统一的租户范围过滤。
2. **项目取消后的正式数据迁移方案**：需要决定历史项目数据如何归属到用户或租户，并提供一次性迁移脚本和回滚方案。
3. **预签名上传完成确认和对象校验**：包括 Key 白名单、ACL、大小限制、哈希/ETag 和状态回写。
4. **MinIO 对象生命周期管理**：软删除后的延迟删除、失败重试、孤儿对象扫描、对象引用检查和管理员清理入口。
5. **Compute Agent 缓存治理**：容量上限、LRU/TTL、运行任务保护、磁盘占用监控、缓存清单和版本校验。
6. **统一的资源归档编排器**：训练、合并、评测和推理相关产物需要支持断点恢复、幂等重试和服务重启补偿。
7. **模型导出完整流程**：导出任务创建、格式/量化参数、进度、失败重试、MinIO 归档和下载权限。
8. **流式和分片文件传输**：避免大文件上传、下载和对象复制时将完整内容读入 Backend 或 Compute Agent 内存。
9. **生产级 MinIO 安全和高可用**：默认密钥替换、TLS、网络访问控制、管理员 Console 隔离、容量监控、备份和恢复。
10. **完整的端到端测试和持续集成**：至少覆盖单节点、多节点、多 GPU、跨用户、跨租户、版本切换、MinIO 不可用和服务重启恢复。
11. **统一数据库迁移体系**：当前初始化 SQL 适合新库初始化，但尚未替代正式的版本化迁移工具；已有数据库更新仍需要明确迁移脚本和执行记录。
## 五、需要优化的功能
### 5.1 后端响应性能
- 页面列表接口需要避免每条记录重复查询用户、资源、MinIO 元数据和 Compute 节点状态。
- MinIO 的 Bucket 检查、对象 Head 和预签名生成应使用连接复用、短期缓存和批量查询。
- 训练、推理、评测页面不应通过过短间隔轮询大量详情接口，应按任务状态动态退避，并在完成后停止轮询。
- 对 dashboard、节点健康、GPU 状态等高频数据应区分实时数据和缓存数据。
- 后端日志轮询和健康检查日志需要继续降噪，仅在状态变化、失败或达到较长周期时输出。
### 5.2 前端加载和交互
- 列表页面应区分首屏 loading、刷新 loading、操作 loading，避免整页长时间无反馈。
- 推理、评测、训练详情应使用统一的任务状态刷新策略和超时提示。
- 前端仍有 FontAwesome 在线资源解析警告，应清理对外部网络文件的依赖，保证离线环境打开速度。
- 应继续拆分首屏大体积 chunk，并减少一次性加载不相关页面组件。
- GPU 选择组件需要明确显示空闲、占用、不可达、预留和已分配状态。
- 错误提示应携带资源名称、节点名称、版本和下一步处理建议，减少只显示 500/404 的情况。
### 5.3 训练、推理和评测可靠性
- 所有任务创建前应执行同一套资源权限、版本存在性、MinIO 可用性和 GPU 原子分配校验。
- 任务创建接口应支持幂等键，避免前端重复点击造成重复任务。
- 节点不可达时应快速失败或进入可见的等待状态，不能让页面长时间无反馈。
- 失败重试应区分网络瞬时失败、资源不足、模型文件缺失、参数错误和执行器失败。
- 任务停止后必须释放 GPU 分配、推理端口、缓存锁和临时目录。
### 5.4 数据和模型一致性
- 每个对象都应保存大小、校验值、版本 ID、来源、创建者、租户和引用状态。
- 数据库中的小文件内容和 MinIO 对象不能同时被当作可独立修改的主副本；需要明确唯一写入入口。
- 数据集激活版本变更需要留下审计记录，并影响后续任务创建但不改变已创建任务。
- 模型权重、Adapter、合并结果和导出结果需要形成完整血缘关系。
## 六、数据库和初始化脚本状态
当前 `backend/app/db/sql/000_full_init.sql` 已包含以下主要类别：
- 用户、模型、训练模型、模型血缘、模型产物、模型导出任务。
- 数据集、数据集文件、数据集版本、数据集记录。
- 算力节点、GPU、GPU 分配、调度锁、Compute Job。
- 资源副本、资源同步任务、MinIO 对象、缓存任务。
- 评测任务、评测维度、模型对比任务。
- 租户、项目兼容表、项目成员、角色、会话、ACL。
- 审批模板、审批实例、审批步骤、审计日志、留存策略。
- 数据处理任务、源文件、预览项、处理结果、数据转换任务。
已确认的近期字段包括：
- `model_artifacts.storage_object_id`
- `model_artifacts.storage_backend`
- `dataset_files.storage_object_id`
- `data_convert_tasks.output_content`
- `data_convert_tasks.output_storage_object_id`
- `data_convert_tasks.storage_backend`
- `eval_tasks.report_storage_object_id`
离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意：
- 初始化 SQL 主要用于新数据库或新数据卷；已有数据库不能仅靠重启容器自动获得全部新字段。
- 生产/测试数据库需要执行可追踪的迁移脚本，并在迁移前备份或生成结构快照。
- `ensure_schema` 类运行时补字段逻辑只能作为兼容兜底，不能替代正式迁移。
- 后续如果正式移除项目设计，需要先完成数据归属迁移，再决定是否删除历史表，不能直接从初始化 SQL 中删除表。
## 七、当前验证结果
已完成的静态和局部验证：
- Backend 和离线 Backend 源码 `compileall` 检查通过。
- MinIO 分层策略冒烟验证通过：小型 JSON/JSONL 可落数据库，小型二进制和超过阈值的内容进入 MinIO。
- 主工程和离线包初始化 SQL 已做同步检查，内容一致。
- 前端此前已完成 `npm run build` 类型错误修复，构建剩余问题主要是非阻断的资源/分包警告。
- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。
当前不能据此宣称“全量功能测试通过”：
- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设，需要按当前分层存储策略更新。
- WSL Docker 运行时验证受当前环境的 `E_ACCESSDENIED` 影响，不能在本次文档生成时完成全部容器健康、数据库字段和跨节点测试。
- 多节点、多 GPU、MinIO 临时不可用、服务重启恢复和跨用户权限测试仍需要在可用运行环境中执行。
## 八、下一阶段开发计划
### P0：安全与数据正确性
1. 完善 MinIO 预签名 PUT 的资源写权限、对象 Key 白名单、大小/类型限制和上传完成确认。
2. 统一任务创建时的资源版本固化，训练、推理、评测只使用已授权的激活版本快照。
3. 逐一补齐评测、推理、模型合并、模型导出、缓存准备的后端权限校验和审计记录。
4. 完成租户隔离查询范围，清理或兼容历史项目字段，补充数据迁移脚本。
### P1：跨节点可靠性
1. 建立资源清单/manifest，记录 MinIO 对象版本、校验值、目标节点路径和缓存状态。
2. 完善训练、合并、评测和推理的准备、执行、归档、失败重试和服务重启补偿。
3. 完善 GPU 原子分配、异常回收、节点重连对账和任务释放。
4. 增加缓存容量、TTL/LRU、运行任务保护和磁盘占用监控。
5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。
### P2：性能与用户体验
1. 优化页面列表接口和高频轮询，采用批量查询、短期缓存和动态退避。
2. 将大文件上传/下载/复制改为流式或分片传输。
3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。
4. 处理前端离线资源警告，继续拆分首屏 chunk。
5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。
### P3：工程化和上线准备
1. 建立正式数据库版本迁移机制和离线升级脚本。
2. 增加 CI：前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。
3. 增加多节点端到端测试和 MinIO 故障注入测试。
4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。
5. 建立生产运行手册，包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。
## 九、阶段验收标准
完成下一阶段后，至少应满足：
- 用户只能看到和操作其所属租户授权的模型、数据集、训练任务、推理服务和评测任务。
- 任何任务创建都能明确记录用户、租户、资源版本、算力节点、GPU 列表和 MinIO 对象版本。
- 同一个节点的同一张 GPU 不能被两个活动任务同时分配。
- MinIO 临时不可用时，任务进入可解释的等待/失败状态，并能按策略重试，页面不会无限等待。
- Backend 或 Compute Agent 重启后，任务、缓存、GPU 分配和归档状态可以对账恢复。
- 训练、合并、评测和推理产物都能在 MinIO 中找到，并且可以通过权限校验后的接口下载或使用。
- 删除资源后不会继续出现在普通列表中，关联对象能够按引用状态延迟清理并留下审计记录。
- 新数据库初始化和已有数据库迁移后，所有业务接口不再因为缺表或缺字段启动失败。
- 离线部署不依赖外部字体、图标或 CDN，前端首屏和核心业务操作在无网络环境下可用。
## 十、相关文件索引
- 平台架构：[platform-architecture-requirements.md](./platform-architecture-requirements.md)
- 权限设计：[permissions-design.md](./permissions-design.md)
- MinIO 与 Compute 缓存方案：[minio-compute-cache-plan.md](./minio-compute-cache-plan.md)
- 平台治理菜单设计：[platform-governance-menu-design.md](./platform-governance-menu-design.md)
- 数据处理设计：[data-process-design.md](./data-process-design.md)
- 数据库初始化脚本：[../backend/app/db/sql/000_full_init.sql](../backend/app/db/sql/000_full_init.sql)
- 离线部署目录：[../docker/offline](../docker/offline)

2
frontend/src/api/modules/audit.ts
View File

@@ -20,6 +20,8 @@ export interface AuditQuery {
actor_id?: string
action?: string
target_type?: string
target_id?: string
keyword?: string
start_time?: string
end_time?: string
limit?: number

11
frontend/src/api/modules/dataProcess.ts
View File

@@ -20,6 +20,8 @@ import type {
DataProcessPublishResult,
DataProcessQualityScore,
DataProcessResult,
DataProcessResultBatchEvaluatePayload,
DataProcessResultBatchEvaluateResult,
DataProcessResultBatchRegeneratePayload,
DataProcessResultBatchRegenerateResult,
DataProcessResultRegeneratePayload,
@@ -320,5 +322,14 @@ export const regenerateDataProcessResults = (
{ timeout: 240_000 },
)
export const evaluateDataProcessResults = (
taskId: string | number,
payload: DataProcessResultBatchEvaluatePayload,
) => post<DataProcessResultBatchEvaluateResult>(
`/data-process/${encodeURIComponent(taskId)}/results/evaluate-batch`,
payload,
{ timeout: 240_000 },
)
export const publishDataProcess = (taskId: string | number, payload: DataProcessPublishPayload) =>
post<DataProcessPublishResult>(`/data-process/${encodeURIComponent(taskId)}/publish`, payload)

11
frontend/src/api/modules/fineTune.ts
View File

@@ -42,12 +42,23 @@ export interface FineTunePreflightResult {
sync_results?: Array<Record<string, unknown>>
}
export interface FineTuneGpuStatus {
source: string
items: Array<Record<string, unknown>>
selected_gpus: number[]
error?: string
}
/** 训练任务列表 */
export const getFineTuneList = () => get<FineTuneTask[]>('/fine-tune')
/** 训练任务详情 */
export const getFineTune = (id: string | number) => get<FineTuneTask>(`/fine-tune/${id}`)
/** 获取任务所在 Compute 节点的实时 GPU 指标 */
export const getFineTuneGpuStatus = (id: string | number) =>
get<FineTuneGpuStatus>(`/fine-tune/${id}/gpu-status`)
/** 任务名查重 */
export const checkFineTuneName = (name: string) =>
get<{ exists: boolean }>('/fine-tune/check-name', { name })

17
frontend/src/components/AppSidebar.vue
View File

@@ -15,6 +15,10 @@ const activeMenu = computed(() => {
if (seg === 'training-log') return 'fine-tune'
// 维度管理归到模型评测
if (route.path.includes('model-eval/dimension')) return 'model-eval'
// 组织与权限承接用户、租户和历史治理入口
if (route.path.startsWith('/organization') || route.path.startsWith('/user-settings') || route.path.startsWith('/tenants')) return 'organization'
// 运行日志承接审计和操作诊断两个历史入口
if (route.path.startsWith('/logs') || route.path.startsWith('/audit-logs') || route.path.startsWith('/operation-logs')) return 'logs'
// 对比对话归到模型推理
if (route.path.startsWith('/model-compare/chat')) return 'model-inference'
// 合并权重归到模型管理
@@ -77,21 +81,16 @@ const menuGroups: MenuGroup[] = [
{
title: '平台治理',
items: [
{ key: 'tenants', label: '租户管理', icon: 'fa-building', to: '/tenants', permission: 'user-settings' },
{ key: 'projects', label: '项目空间', icon: 'fa-folder', to: '/projects', permission: 'user-settings' },
{ key: 'organization', label: '组织与权限', icon: 'fa-users', to: '/organization', permission: 'user-settings' },
{ key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' },
{ key: 'audit-logs', label: '审计日志', icon: 'fa-history', to: '/audit-logs', permission: 'user-settings' },
{ key: 'operation-logs', label: '操作日志', icon: 'fa-list', to: '/operation-logs', permission: 'user-settings' },
{ key: 'approval-templates', label: '审批模板', icon: 'fa-list-alt', to: '/approval-templates', permission: 'user-settings' },
{ key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' },
],
},
{
title: '系统设置',
items: [
{ key: 'user-settings', label: '用户设置', icon: 'fa-users', to: '/user-settings', permission: 'user-settings' },
{ key: 'hardware', label: '平台性能', icon: 'fa-bar-chart', to: '/hardware', permission: 'hardware' },
{ key: 'logs', label: '查看日志', icon: 'fa-file-text', to: '/logs', permission: 'logs' },
{ key: 'logs', label: '运行日志', icon: 'fa-file-text', to: '/logs', permission: 'logs' },
],
},
]
@@ -101,9 +100,9 @@ const menuGroups: MenuGroup[] = [
*
* 1. admin 用户：可以看到所有菜单
* 2. 非 admin 用户：
* - 默认可见所有业务菜单（模型训练、评测、推理、数据集、数据处理、转换、性能、日志等）
* - 默认可见所有业务菜单（模型训练、评测、推理、数据集、数据处理、转换、性能、运行日志等）
* - 仅以下菜单对非 admin 不可见：
* - user-settings（用户设置、租户管理、项目空间、审批模板/中心、审计日志）
* - user-settings（组织与权限、资源授权、审批；运行日志中的审计/诊断页签）
* - compute（算力节点/GPU 分配）
*
* 注意：移除了旧的权限码（permission code）过滤逻辑，

5
frontend/src/plugins/echarts.ts
View File

@@ -3,18 +3,21 @@
*/
import { use } from 'echarts/core'
import { CanvasRenderer } from 'echarts/renderers'
import { BarChart, PieChart } from 'echarts/charts'
import { BarChart, PieChart, RadarChart } from 'echarts/charts'
import {
GridComponent,
TooltipComponent,
LegendComponent,
RadarComponent,
} from 'echarts/components'
use([
CanvasRenderer,
BarChart,
PieChart,
RadarChart,
GridComponent,
TooltipComponent,
LegendComponent,
RadarComponent,
])

45
frontend/src/router/index.ts
View File

@@ -32,11 +32,17 @@ const routes: RouteRecordRaw[] = [
meta: { title: '服务看板' },
},
// 平台治理
{
path: 'organization',
name: 'organization',
component: () => import('@/views/governance/OrganizationPermissionView.vue'),
meta: { title: '组织与权限', pageSurface: 'self', permission: 'user-settings' },
},
{
path: 'tenants',
name: 'tenants',
component: () => import('@/views/tenants/TenantListView.vue'),
meta: { title: '租户管理', permission: 'user-settings' },
redirect: '/organization?tab=tenants',
meta: { title: '租户与配额', permission: 'user-settings' },
},
{
path: 'tenants/:id',
@@ -47,37 +53,37 @@ const routes: RouteRecordRaw[] = [
{
path: 'projects',
name: 'projects',
component: () => import('@/views/projects/ProjectListView.vue'),
meta: { title: '项目空间', permission: 'user-settings' },
redirect: '/organization?tab=users',
meta: { title: '组织与权限', permission: 'user-settings' },
},
{
path: 'projects/:id',
name: 'project-detail',
component: () => import('@/views/projects/ProjectDetailView.vue'),
meta: { title: '项目详情', permission: 'user-settings' },
redirect: '/organization?tab=users',
meta: { title: '组织与权限', permission: 'user-settings' },
},
{
path: 'audit-logs',
name: 'audit-logs',
component: () => import('@/views/audit/AuditLogView.vue'),
meta: { title: '审计日志', permission: 'user-settings' },
redirect: '/logs?tab=audit',
meta: { title: '运行日志', permission: 'user-settings' },
},
{
path: 'operation-logs',
name: 'operation-logs',
component: () => import('@/views/audit/OperationLogView.vue'),
meta: { title: '操作日志', permission: 'user-settings' },
redirect: '/logs?tab=operations',
meta: { title: '运行日志', permission: 'user-settings' },
},
{
path: 'approval-templates',
name: 'approval-templates',
component: () => import('@/views/approvals/ApprovalTemplateView.vue'),
meta: { title: '审批模板', permission: 'user-settings' },
redirect: '/approval-instances?tab=strategies',
meta: { title: '审批中心', permission: 'user-settings' },
},
{
path: 'approval-instances',
name: 'approval-instances',
component: () => import('@/views/approvals/ApprovalInstanceView.vue'),
component: () => import('@/views/approvals/ApprovalCenterView.vue'),
meta: { title: '审批中心', permission: 'user-settings' },
},
{
@@ -302,14 +308,14 @@ const routes: RouteRecordRaw[] = [
{
path: 'logs',
name: 'logs',
component: () => import('@/views/system/LogsView.vue'),
meta: { title: '查看日志' },
component: () => import('@/views/system/RuntimeLogsView.vue'),
meta: { title: '运行日志', pageSurface: 'self', permission: 'logs' },
},
{
path: 'user-settings',
name: 'user-settings',
component: () => import('@/views/system/UserSettingsView.vue'),
meta: { title: '用户设置', pageSurface: 'self', permission: 'user-settings' },
redirect: '/organization?tab=users',
meta: { title: '组织与权限', pageSurface: 'self', permission: 'user-settings' },
},
{
path: 'user-settings/create',
@@ -357,6 +363,7 @@ const permissionBySegment: Record<string, PermissionCode> = {
tools: 'data-convert',
hardware: 'hardware',
logs: 'logs',
organization: 'user-settings',
'user-settings': 'user-settings',
tenants: 'user-settings',
projects: 'user-settings',
@@ -379,7 +386,7 @@ function requiredPermission(path: string, explicit?: unknown) {
// 权限控制规则（基于 governance-user-guide.md 设计）：
// - admin 用户：可以访问所有页面
// - 非 admin 用户：默认可访问所有业务页面（训练、评测、推理、数据等）
// 仅以下页面限制 admin 访问：user-settings、compute（算力节点）
// 仅治理与资源管理页面限制 admin 访问：organization、user-settings、compute
router.beforeEach((to, _from, next) => {
if (!to.meta.public) routeLoading.value = true
const auth = useAuthStore()
@@ -404,7 +411,7 @@ router.beforeEach((to, _from, next) => {
if (!to.meta.skipPermission) {
const permission = requiredPermission(to.path, to.meta.permission)
// 仅限制管理员专属页面的访问权限
// user-settings（用户设置、租户管理、项目空间、审批、审计日志）仅 admin 可访问
// user-settings（组织与权限、资源授权、审批中心、运行日志）仅 admin 可访问
if (permission === 'user-settings' && !auth.isAdmin) {
next({ name: 'permission-denied', replace: true })
return

51
frontend/src/types/dataProcess.ts
View File

@@ -398,6 +398,52 @@ export interface DataProcessResultBatchRegenerateResult {
failures: DataProcessResultBatchRegenerateFailure[]
}
export interface DataProcessResultBatchEvaluateItem {
result_id: string
expected_updated_at: string
}
export interface DataProcessResultBatchEvaluatePayload {
items: DataProcessResultBatchEvaluateItem[]
}
export interface DataProcessResultBatchEvaluateFailure {
result_id: string
code: 'conflict' | 'skipped' | 'evaluation_failed' | 'internal_error'
message: string
}
export interface DataProcessResultBatchEvaluateResult {
batch_id: string
total: number
succeeded: number
failed: number
duration_ms: number
items: DataProcessResult[]
failures: DataProcessResultBatchEvaluateFailure[]
}
export interface DataProcessQualitySemantic {
question_answer?: number
answer_source?: number
overall?: number
}
export interface DataProcessQualityJudge {
scores?: Record<string, number>
overall?: number
reason?: string
issues?: string[]
model?: string
output_type?: string
}
export interface DataProcessQualityLayers {
rule?: number | null
semantic?: number | null
judge?: number | null
}
export interface DataProcessQualityScore {
overall?: number
completeness?: number
@@ -408,6 +454,11 @@ export interface DataProcessQualityScore {
is_valid?: boolean
flags?: string[]
fingerprint?: string
semantic?: DataProcessQualitySemantic | null
judge?: DataProcessQualityJudge | null
layers?: DataProcessQualityLayers | null
evaluated?: boolean
evaluated_at?: string | null
source_pages?: number[]
heading_path?: string[]
source_locator?: DataProcessSourceLocator

8
frontend/src/types/index.ts
View File

@@ -218,6 +218,8 @@ export interface LoadedModel {
port?: number
node_id?: string
node_name?: string
gpu_indices?: number[]
gpus?: number[]
error?: string
}
@@ -239,6 +241,8 @@ export interface CompareModelRef {
gpu_id: number
node_id?: string
node_name?: string
gpu_indices?: number[]
gpus?: number[]
source?: string
port?: number
}
@@ -282,7 +286,9 @@ export interface StartEvalPayload {
eval_task_name: string
eval_type: EvalType
model_id: string | number
gpu_id: string | number
gpu_id: string | number | string[]
gpu_indices?: number[]
gpus?: number[]
compute_node_id?: string
dataset_id: string | number
dimension_id: string | number

66
frontend/src/views/approvals/ApprovalCenterView.vue Normal file
View File

@@ -0,0 +1,66 @@
<script setup lang="ts">
import { computed } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import ApprovalInstanceView from './ApprovalInstanceView.vue'
import ApprovalTemplateView from './ApprovalTemplateView.vue'
const route = useRoute()
const router = useRouter()
const activeTab = computed<'instances' | 'mine' | 'strategies'>({
get: () => route.query.tab === 'strategies' ? 'strategies' : route.query.tab === 'mine' ? 'mine' : 'instances',
set: (value: string) => {
void router.replace({ query: value === 'instances' ? {} : { tab: value } })
},
})
</script>
<template>
<div class="approval-center">
<header class="page-header">
<div>
<h2>审批中心</h2>
<p>集中处理审批申请、审批历史和审批策略。</p>
</div>
</header>
<el-tabs v-model="activeTab">
<el-tab-pane label="审批申请" name="instances">
<ApprovalInstanceView v-if="activeTab === 'instances'" />
</el-tab-pane>
<el-tab-pane label="我的申请" name="mine">
<ApprovalInstanceView v-if="activeTab === 'mine'" mine />
</el-tab-pane>
<el-tab-pane label="审批策略" name="strategies">
<ApprovalTemplateView v-if="activeTab === 'strategies'" />
</el-tab-pane>
</el-tabs>
</div>
</template>
<style scoped lang="scss">
.approval-center {
min-height: 100%;
padding: 20px;
}
.page-header {
margin-bottom: 4px;
h2 {
margin: 0;
color: #1f2937;
font-size: 22px;
}
p {
margin: 6px 0 0;
color: #64748b;
font-size: 13px;
}
}
:deep(.page) {
padding: 16px 0 0;
}
</style>

16
frontend/src/views/approvals/ApprovalInstanceView.vue
View File

@@ -1,11 +1,15 @@
<script setup lang="ts">
import { onMounted, reactive, ref } from 'vue'
import { computed, onMounted, ref } from 'vue'
import { ElMessage } from 'element-plus'
import DataTablePage from '@/components/DataTablePage.vue'
import { getApprovalInstances, decideApproval, type ApprovalInstance } from '@/api/modules/approval'
import { getUsers } from '@/api/modules/system'
import { useAuthStore } from '@/stores/auth'
import type { SystemUser } from '@/types'
const props = defineProps<{ mine?: boolean }>()
const auth = useAuthStore()
const loading = ref(false)
const instances = ref<ApprovalInstance[]>([])
const users = ref<SystemUser[]>([])
@@ -14,6 +18,14 @@ const showDecide = ref(false)
const current = ref<ApprovalInstance | null>(null)
const decision = ref({ step_index: 0, approver_id: '', approved: true, comment: '' })
const visibleInstances = computed(() => {
if (!props.mine) return instances.value
const currentUserId = auth.currentUser?.id
return currentUserId
? instances.value.filter((item) => item.applicant_id === currentUserId)
: []
})
const statusOptions = [
{ label: '待审批', value: 'pending' },
{ label: '已通过', value: 'approved' },
@@ -77,7 +89,7 @@ onMounted(() => {
<template>
<div class="page">
<DataTablePage title="审批实例" :data="instances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
<DataTablePage :title="props.mine ? '我的申请' : '审批申请'" :data="visibleInstances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
<template #toolbar-extra>
<el-select v-model="statusFilter" placeholder="状态" clearable style="width: 140px" @change="load">
<el-option v-for="s in statusOptions" :key="s.value" :label="s.label" :value="s.value" />

157
frontend/src/views/audit/AuditLogView.vue
View File

@@ -2,16 +2,21 @@
import { onMounted, reactive, ref } from 'vue'
import { ElMessage } from 'element-plus'
import { getAuditLogs, exportAuditLogs, type AuditLog, type AuditQuery } from '@/api/modules/audit'
import { getUsers, type SystemUser } from '@/api/modules/system'
import { getTenants, type Tenant } from '@/api/modules/tenant'
const loading = ref(false)
const logs = ref<AuditLog[]>([])
const total = ref(0)
const users = ref<SystemUser[]>([])
const tenants = ref<Tenant[]>([])
const query = reactive<AuditQuery>({
tenant_id: '',
project_id: '',
actor_id: '',
action: '',
target_type: '',
target_id: '',
keyword: '',
start_time: '',
end_time: '',
limit: 50,
@@ -21,6 +26,87 @@ const query = reactive<AuditQuery>({
// 时间范围（el-date-picker 双向绑定数组 [start, end]）
const timeRange = ref<[string, string] | null>(null)
const actionOptions = [
{ value: 'create_dataset', label: '创建数据集' },
{ value: 'update_dataset', label: '修改数据集' },
{ value: 'delete_dataset', label: '删除数据集' },
{ value: 'create_model', label: '创建模型' },
{ value: 'update_model', label: '修改模型' },
{ value: 'delete_model', label: '删除模型' },
{ value: 'create_fine_tune', label: '创建训练任务' },
{ value: 'update_fine_tune', label: '修改训练任务' },
{ value: 'delete_fine_tune', label: '删除训练任务' },
{ value: 'create_inference', label: '创建推理任务' },
{ value: 'update_inference', label: '修改推理任务' },
{ value: 'delete_inference', label: '删除推理任务' },
{ value: 'create_user', label: '创建用户' },
{ value: 'update_user', label: '修改用户' },
{ value: 'delete_user', label: '删除用户' },
{ value: 'tenant.create', label: '创建租户' },
{ value: 'tenant.update', label: '修改租户' },
{ value: 'tenant.delete', label: '删除租户' },
{ value: 'tenant.quota.set', label: '设置租户配额' },
{ value: 'tenant.retention.set', label: '设置留存策略' },
{ value: 'grant_acl', label: '授予资源权限' },
{ value: 'revoke_acl', label: '撤销资源权限' },
{ value: 'gpu.assign', label: '分配算力卡' },
{ value: 'gpu.release', label: '释放算力卡' },
{ value: 'create', label: '创建' },
{ value: 'update', label: '修改' },
{ value: 'delete', label: '删除' },
{ value: 'start', label: '启动' },
{ value: 'stop', label: '停止' },
{ value: 'upload', label: '上传' },
{ value: 'download', label: '下载' },
{ value: 'convert', label: '转换' },
{ value: 'merge', label: '合并权重' },
{ value: 'publish', label: '发布' },
{ value: 'retry', label: '重试' },
{ value: 'login', label: '登录' },
{ value: 'logout', label: '退出登录' },
]
const targetTypeOptions = [
{ value: 'dataset', label: '数据集' },
{ value: 'model', label: '模型' },
{ value: 'fine_tune_task', label: '训练任务' },
{ value: 'fine_tune', label: '训练任务' },
{ value: 'inference_task', label: '推理任务' },
{ value: 'inference', label: '推理任务' },
{ value: 'eval_task', label: '评测任务' },
{ value: 'trained_model', label: '训练模型' },
{ value: 'convert_task', label: '数据转换任务' },
{ value: 'tenant', label: '租户' },
{ value: 'user', label: '用户' },
{ value: 'resource_acl', label: '资源权限' },
{ value: 'gpu', label: '算力卡' },
{ value: 'retention_policy', label: '留存策略' },
{ value: 'module', label: '业务模块' },
{ value: 'api', label: '接口' },
]
const actionLabels = Object.fromEntries(actionOptions.map((item) => [item.value, item.label]))
const targetTypeLabels = Object.fromEntries(targetTypeOptions.map((item) => [item.value, item.label]))
function userName(id?: string) {
if (!id) return '系统'
const user = users.value.find((item) => item.id === id)
return user ? `${user.display_name || user.username}（${user.username}）` : id
}
function tenantName(id?: string) {
if (!id) return '未关联租户'
return tenants.value.find((item) => item.id === id)?.name || id
}
function actionName(action?: string) {
return action ? actionLabels[action] || action : '未记录'
}
function targetTypeName(type?: string) {
return type ? targetTypeLabels[type] || type : '未指定'
}
function applyTimeRange() {
if (timeRange.value && timeRange.value.length === 2) {
query.start_time = timeRange.value[0]
@@ -42,6 +128,25 @@ async function load() {
}
}
async function loadFilterOptions() {
const [userResult, tenantResult] = await Promise.allSettled([getUsers(), getTenants()])
if (userResult.status === 'fulfilled') users.value = userResult.value
if (tenantResult.status === 'fulfilled') tenants.value = tenantResult.value
}
function resetFilters() {
query.tenant_id = ''
query.actor_id = ''
query.action = ''
query.target_type = ''
query.target_id = ''
query.keyword = ''
query.start_time = ''
query.end_time = ''
timeRange.value = null
void load()
}
async function handleExport() {
try {
const blob = await exportAuditLogs({ ...query, limit: 10000, offset: 0 })
@@ -56,7 +161,9 @@ async function handleExport() {
}
}
onMounted(load)
onMounted(() => {
void Promise.all([loadFilterOptions(), load()])
})
</script>
<template>
@@ -66,21 +173,32 @@ onMounted(load)
<el-button @click="handleExport">导出 CSV</el-button>
</div>
<el-card class="filter-card">
<el-form :inline="true">
<el-form :inline="true" class="filter-form">
<el-form-item label="租户">
<el-input v-model="query.tenant_id" placeholder="tenant_id" clearable />
</el-form-item>
<el-form-item label="项目">
<el-input v-model="query.project_id" placeholder="project_id" clearable />
<el-select v-model="query.tenant_id" placeholder="全部租户" clearable filterable style="width: 190px">
<el-option v-for="tenant in tenants" :key="tenant.id" :label="tenant.name" :value="tenant.id" />
</el-select>
</el-form-item>
<el-form-item label="操作人">
<el-input v-model="query.actor_id" placeholder="actor_id" clearable />
<el-select v-model="query.actor_id" placeholder="全部用户" clearable filterable style="width: 210px">
<el-option v-for="user in users" :key="user.id" :label="`${user.display_name || user.username}（${user.username}）`" :value="user.id" />
</el-select>
</el-form-item>
<el-form-item label="动作">
<el-input v-model="query.action" placeholder="action" clearable />
<el-select v-model="query.action" placeholder="全部动作" clearable filterable style="width: 180px">
<el-option v-for="item in actionOptions" :key="item.value" :label="item.label" :value="item.value" />
</el-select>
</el-form-item>
<el-form-item label="目标类型">
<el-input v-model="query.target_type" placeholder="target_type" clearable />
<el-select v-model="query.target_type" placeholder="全部资源" clearable filterable style="width: 160px">
<el-option v-for="item in targetTypeOptions" :key="item.value" :label="item.label" :value="item.value" />
</el-select>
</el-form-item>
<el-form-item label="关键词">
<el-input v-model="query.keyword" placeholder="资源 ID 或详情" clearable style="width: 220px" />
</el-form-item>
<el-form-item label="目标 ID">
<el-input v-model="query.target_id" placeholder="精确查询，可选" clearable style="width: 180px" />
</el-form-item>
<el-form-item label="时间范围">
<el-date-picker
@@ -97,16 +215,24 @@ onMounted(load)
</el-form-item>
<el-form-item>
<el-button type="primary" @click="load">查询</el-button>
<el-button @click="resetFilters">重置</el-button>
</el-form-item>
</el-form>
</el-card>
<el-table :data="logs" v-loading="loading" border stripe class="log-table">
<el-table-column prop="time" label="时间" min-width="180" />
<el-table-column prop="tenant_id" label="租户" min-width="120" />
<el-table-column prop="project_id" label="项目" min-width="120" />
<el-table-column prop="actor_id" label="操作人" min-width="120" />
<el-table-column prop="action" label="动作" min-width="140" />
<el-table-column prop="target_type" label="目标类型" min-width="120" />
<el-table-column label="租户" min-width="140">
<template #default="{ row }">{{ tenantName(row.tenant_id) }}</template>
</el-table-column>
<el-table-column label="操作人" min-width="180">
<template #default="{ row }">{{ userName(row.actor_id) }}</template>
</el-table-column>
<el-table-column label="动作" min-width="140">
<template #default="{ row }">{{ actionName(row.action) }}</template>
</el-table-column>
<el-table-column label="目标类型" min-width="120">
<template #default="{ row }">{{ targetTypeName(row.target_type) }}</template>
</el-table-column>
<el-table-column prop="target_id" label="目标 ID" min-width="140" show-overflow-tooltip />
<el-table-column prop="detail" label="详情" min-width="200" show-overflow-tooltip />
<el-table-column prop="client_ip" label="IP" min-width="120" />
@@ -120,6 +246,7 @@ onMounted(load)
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px; }
.page-title { margin: 0; font-size: 18px; }
.filter-card { margin-bottom: 16px; }
.filter-form { display: flex; flex-wrap: wrap; }
.log-table { margin-top: 8px; }
.pager { margin-top: 12px; text-align: right; color: #909399; }
</style>

2
frontend/src/views/audit/OperationLogView.vue
View File

@@ -313,7 +313,7 @@ onMounted(() => {
</el-table-column>
<el-table-column label="操作" width="80" align="center" fixed="right">
<template #default="{ row }">
<el-button link type="primary" size="small" @click="showDetail(row)">详情</el-button>
<el-button link type="primary" size="small" @click="showDetail(row as OperationLog)">详情</el-button>
</template>
</el-table-column>
</el-table>

6
frontend/src/views/data-convert/DataConvertView.vue
View File

@@ -2,7 +2,7 @@
import { onMounted, ref } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus'
import { Plus, Delete, Refresh } from '@element-plus/icons-vue'
import type { TagProps, UploadRequestOptions, UploadFile } from 'element-plus'
import type { TagProps, UploadRequestOptions, UploadFile, UploadRawFile } from 'element-plus'
import PageCard from '@/components/PageCard.vue'
import {
getDataConvertTasks,
@@ -30,9 +30,9 @@ async function load() {
}
// 文件上传前的校验（仅校验文件格式）
function beforeUpload(file: UploadFile) {
function beforeUpload(file: UploadRawFile) {
// 检查文件类型
const isJson = file.name.endsWith('.json') || file.raw?.type === 'application/json'
const isJson = file.name.endsWith('.json') || file.type === 'application/json'
if (!isJson) {
ElMessage.error('只能上传 .json 格式的文件')
return false

16
frontend/src/views/data-process/DataProcessCreateView.vue
View File

@@ -18,6 +18,7 @@ import {
previewAffectingOptionsFor,
} from './create/dataProcessCreateState'
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessEvaluation } from './create/useDataProcessEvaluation'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
@@ -126,6 +127,19 @@ const {
outputType: activeOutputType,
beforeGenerate: beforeStartGeneration,
})
const {
evaluation,
evaluateAllResults,
resetEvaluation,
} = useDataProcessEvaluation({
taskId,
results,
selectedResultId,
})
// 生成结果被重置（重新切分/上传/重新生成配置）时同步清空评测进度。
watch(results, (items) => {
if (!items.length) resetEvaluation()
})
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
taskId,
uploadedFiles,
@@ -1156,10 +1170,12 @@ onMounted(() => {
:preview-items="previewItems"
:regenerating-result-id="regeneratingResultId"
:bulk-regeneration="bulkRegeneration"
:evaluation="evaluation"
:output-type="activeOutputType"
@update:field="updateResultField"
@regenerate:all="regenerateAllResults"
@regenerate:item="regenerateResult"
@evaluate:all="evaluateAllResults"
/>
</div>
</div>

171
frontend/src/views/data-process/DataProcessDetailView.vue
View File

@@ -5,6 +5,7 @@ import { ElMessage, ElMessageBox } from 'element-plus'
import PageCard from '@/components/PageCard.vue'
import { usePolling } from '@/composables/usePolling'
import {
evaluateDataProcessResults,
getDataProcessProgress,
getDataProcessResults,
getDataProcessTask,
@@ -13,6 +14,7 @@ import {
restoreDataProcessResult,
updateDataProcessResult,
} from '@/api/modules/dataProcess'
import QualityRadarPopover from './create/QualityRadarPopover.vue'
import type {
DataProcessDatasetSplit,
DataProcessPublishPayload,
@@ -456,13 +458,101 @@ function resultStatusType(status: DataProcessResultStatus) {
}
function qualityScoreLabel(value: DataProcessResult['quality_score']) {
if (value == null) return '-'
if (value == null || !value.evaluated) return '-'
const score = value.overall
return Number.isFinite(score) ? Number(score).toFixed(1) : '-'
}
function qualityFlagsLabel(value: DataProcessResult['quality_score']) {
return value?.flags?.length ? value.flags.join('、') : '未命中质量规则'
function qualityScoreTone(value: DataProcessResult['quality_score']) {
const score = Number(value?.overall)
if (!value?.evaluated || !Number.isFinite(score)) return ''
return score >= 80 ? 'is-success' : score >= 60 ? 'is-warning' : 'is-danger'
}
function qualityScoreEvaluated(value: DataProcessResult['quality_score']) {
return Boolean(value?.evaluated && Number.isFinite(Number(value?.overall)))
}
const evaluationRunning = ref(false)
const evaluationProgress = reactive({
visible: false,
total: 0,
completed: 0,
succeeded: 0,
failed: 0,
})
// 与批量重生成一致的分块大小，单批在接口 240 秒超时预算内。
const EVALUATION_CHUNK_SIZE = 12
const canEvaluate = computed(() => (
detail.value?.status === 'completed' && !hasCurrentPublishedDataset.value
))
const evaluationPercentage = computed(() => (
evaluationProgress.total
? Math.round((evaluationProgress.completed / evaluationProgress.total) * 100)
: 0
))
async function loadAllResultIds() {
const first = await getDataProcessResults(taskId.value, { page: 1, page_size: 500 })
const items = [...first.items]
const pages = Math.ceil(first.total / first.page_size)
for (let page = 2; page <= pages; page += 1) {
const next = await getDataProcessResults(taskId.value, { page, page_size: 500 })
items.push(...next.items)
}
return items
}
async function runResultEvaluation() {
if (evaluationRunning.value || !canEvaluate.value) return
evaluationRunning.value = true
Object.assign(evaluationProgress, {
visible: true,
total: 0,
completed: 0,
succeeded: 0,
failed: 0,
})
try {
const candidates = (await loadAllResultIds()).filter((item) => item.updated_at)
if (!candidates.length) {
ElMessage.info('当前没有可评测的结果')
return
}
evaluationProgress.total = candidates.length
for (let offset = 0; offset < candidates.length; offset += EVALUATION_CHUNK_SIZE) {
const chunk = candidates.slice(offset, offset + EVALUATION_CHUNK_SIZE)
try {
const evaluated = await evaluateDataProcessResults(taskId.value, {
items: chunk.map((item) => ({
result_id: String(item.id),
expected_updated_at: item.updated_at as string,
})),
})
evaluationProgress.completed += evaluated.total
evaluationProgress.succeeded += evaluated.succeeded
evaluationProgress.failed += evaluated.failed
} catch {
evaluationProgress.completed = evaluationProgress.total
evaluationProgress.failed += candidates.length - offset
break
}
}
await loadResults()
if (evaluationProgress.failed === 0) {
ElMessage.success(`数据评测完成：成功 ${evaluationProgress.succeeded} 条`)
} else if (evaluationProgress.succeeded > 0) {
ElMessage.warning(
`数据评测完成：成功 ${evaluationProgress.succeeded} 条，失败 ${evaluationProgress.failed} 条`,
)
} else {
ElMessage.error(`数据评测失败：${evaluationProgress.failed} 条结果未完成评测`)
}
} catch {
ElMessage.error('数据评测中断，已完成的评分保持不变')
} finally {
evaluationRunning.value = false
}
}
function replaceResult(updated: DataProcessResult) {
@@ -824,10 +914,33 @@ onBeforeUnmount(() => {
<el-option label="已修改" value="modified" />
<el-option label="无效" value="invalid" />
</el-select>
<el-button
v-if="canEvaluate"
type="primary"
plain
:loading="evaluationRunning"
:disabled="resultLoading"
@click="runResultEvaluation"
>
<i v-if="!evaluationRunning" class="fa fa-check-square-o" aria-hidden="true" /> 数据评测
</el-button>
<el-button :loading="resultLoading" @click="loadResults"><i class="fa fa-refresh" /></el-button>
</div>
</div>
<div v-if="evaluationProgress.visible" class="evaluation-progress">
<span>
数据评测 {{ evaluationProgress.completed }} / {{ evaluationProgress.total }}
· 成功 {{ evaluationProgress.succeeded }} · 失败 {{ evaluationProgress.failed }}
</span>
<el-progress
:percentage="evaluationPercentage"
:show-text="false"
:stroke-width="5"
:color="evaluationProgress.failed > 0 ? '#d97706' : '#5b50f2'"
/>
</div>
<el-table
v-if="results.length"
:data="results"
@@ -845,9 +958,26 @@ onBeforeUnmount(() => {
</template>
<el-table-column label="质量分" width="88" align="center">
<template #default="{ row }">
<el-tooltip :content="qualityFlagsLabel((row as DataProcessResult).quality_score)">
<span>{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
</el-tooltip>
<el-popover
v-if="qualityScoreEvaluated((row as DataProcessResult).quality_score)"
placement="top"
:width="296"
trigger="hover"
:show-after="150"
popper-class="quality-radar-popper"
>
<template #reference>
<span
class="detail-quality-score"
:class="qualityScoreTone((row as DataProcessResult).quality_score)"
>{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
</template>
<QualityRadarPopover
:quality="(row as DataProcessResult).quality_score!"
:score="Number((row as DataProcessResult).quality_score?.overall)"
/>
</el-popover>
<span v-else class="detail-quality-empty">{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
</template>
</el-table-column>
<el-table-column label="状态" width="90" align="center">
@@ -1099,6 +1229,35 @@ onBeforeUnmount(() => {
.result-filters :deep(.el-select) { width: 120px; }
.result-section :deep(.el-table) { border-radius: 0; }
.evaluation-progress {
display: grid;
grid-template-columns: 1fr 220px;
align-items: center;
gap: 14px;
padding: 10px 18px;
color: #667085;
background: #f8f9fc;
font-size: 12px;
}
.detail-quality-score {
display: inline-block;
min-width: 44px;
padding: 2px 8px;
border-radius: 10px;
color: #475467;
background: #f2f4f7;
font-weight: 700;
font-variant-numeric: tabular-nums;
cursor: default;
&.is-success { color: #067647; background: #e6f4ee; }
&.is-warning { color: #b54708; background: #fef0c7; }
&.is-danger { color: #b42318; background: #fee4e2; }
}
.detail-quality-empty { color: #98a2b3; }
:global(.data-process-result-tooltip) {
box-sizing: border-box;
max-width: min(520px, calc(100vw - 32px));

252
frontend/src/views/data-process/create/QualityRadarPopover.vue Normal file
View File

@@ -0,0 +1,252 @@
<script setup lang="ts">
import { computed } from 'vue'
import VChart from 'vue-echarts'
import '@/plugins/echarts'
import type { EChartsOption } from 'echarts'
import type { ResultQualityDetails } from './types'
const props = defineProps<{
quality: ResultQualityDetails
score?: number
}>()
// 轴标签按词意预置断行，避免长中文标签把雷达网格挤偏或被机械切词。
const JUDGE_DIMENSION_LABELS: Record<string, string> = {
faithfulness: '忠实度',
correctness: '正确性',
clarity: '问题\n清晰度',
completeness: '回答\n完整性',
alignment: '指令\n对齐',
reasoning_validity: '推理\n有效性',
chosen_quality: 'chosen\n质量',
rejected_quality: 'rejected\n质量',
preference_reasonableness: '偏好\n区分',
}
const SEMANTIC_DIMENSION_LABELS: Record<string, string> = {
question_answer: '问答\n相关',
answer_source: '来源\n覆盖',
}
interface RadarDimension {
name: string
value: number
}
const radarDimensions = computed<RadarDimension[]>(() => {
const dimensions: RadarDimension[] = []
for (const [key, value] of Object.entries(props.quality?.judge?.scores ?? {})) {
dimensions.push({
name: JUDGE_DIMENSION_LABELS[key] ?? key,
value: Math.round(value * 20),
})
}
for (const [key, value] of Object.entries(props.quality?.semantic ?? {})) {
if (key === 'overall' || typeof value !== 'number') continue
dimensions.push({
name: SEMANTIC_DIMENSION_LABELS[key] ?? key,
value: Math.round(value),
})
}
return dimensions
})
// 可用维度太少时雷达图失去意义，降级为分层分数展示。
const showRadar = computed(() => radarDimensions.value.length >= 3)
const radarOption = computed<EChartsOption>(() => ({
radar: {
indicator: radarDimensions.value.map((dimension) => ({
name: dimension.name,
max: 100,
})),
radius: '56%',
center: ['50%', '50%'],
splitNumber: 4,
axisName: {
color: '#667085',
fontSize: 10,
lineHeight: 13,
},
splitArea: { areaStyle: { color: ['#fbfbfd', '#f2f4f8'] } },
splitLine: { lineStyle: { color: '#e4e7ec' } },
axisLine: { lineStyle: { color: '#e4e7ec' } },
},
series: [{
type: 'radar',
symbol: 'circle',
symbolSize: 3,
data: [{
value: radarDimensions.value.map((dimension) => dimension.value),
name: '质量维度',
areaStyle: { color: 'rgba(91, 80, 242, 0.18)' },
lineStyle: { color: '#5b50f2', width: 1.5 },
itemStyle: { color: '#5b50f2' },
}],
}],
}))
const layerScores = computed(() => {
const layers = props.quality?.layers ?? {}
return [
{ label: '规则层', value: layers.rule },
{ label: '语义层', value: layers.semantic },
{ label: '评审层', value: layers.judge },
].filter((layer): layer is { label: string; value: number } => (
typeof layer.value === 'number'
))
})
const displayScore = computed(() => {
if (typeof props.score === 'number' && !isNaN(props.score)) {
return props.score.toFixed(1)
}
return null
})
const scoreTone = computed(() => {
const numScore = props.score ?? 0
return numScore >= 80 ? 'is-success' : numScore >= 60 ? 'is-warning' : 'is-danger'
})
</script>
<template>
<div class="quality-popover">
<div v-if="displayScore !== null" class="popover-header">
<strong>质量评测</strong>
<span class="popover-score" :class="scoreTone">{{ displayScore }}</span>
</div>
<VChart
v-if="showRadar"
class="quality-radar"
:option="radarOption"
autoresize
/>
<div v-else class="radar-fallback">
维度数据不足，已评测维度少于 3 个时以分层分数为准。
</div>
<div class="layer-scores">
<div v-for="layer in layerScores" :key="layer.label" class="layer-item">
<span>{{ layer.label }}</span>
<el-progress
:percentage="Math.round(layer.value)"
:stroke-width="6"
:show-text="false"
:color="layer.value >= 80 ? '#12b76a' : layer.value >= 60 ? '#f0b429' : '#d92d20'"
/>
<em>{{ layer.value.toFixed(0) }}</em>
</div>
</div>
<p v-if="quality?.judge?.reason" class="judge-reason">{{ quality.judge.reason }}</p>
<div v-if="quality?.judge?.issues?.length" class="judge-issues">
<span v-for="issue in quality.judge.issues" :key="issue" class="issue-tag">{{ issue }}</span>
</div>
<div v-if="quality?.judge?.model" class="judge-model">评审模型：{{ quality.judge.model }}</div>
</div>
</template>
<style scoped lang="scss">
.quality-popover {
display: flex;
flex-direction: column;
gap: 10px;
width: 100%;
box-sizing: border-box;
}
.popover-header {
display: flex;
align-items: center;
justify-content: space-between;
padding-bottom: 6px;
border-bottom: 1px solid #f2f4f7;
strong {
color: #344054;
font-size: 13px;
}
}
.popover-score {
color: #344054;
font-size: 18px;
font-weight: 700;
font-variant-numeric: tabular-nums;
&.is-success { color: #12b76a; }
&.is-warning { color: #d99b0b; }
&.is-danger { color: #d92d20; }
}
.quality-radar {
width: 100%;
height: 220px;
box-sizing: border-box;
}
.radar-fallback {
padding: 18px 10px;
color: #98a2b3;
font-size: 12px;
text-align: center;
}
.layer-scores {
display: flex;
flex-direction: column;
gap: 6px;
}
.layer-item {
display: grid;
grid-template-columns: 44px 1fr 28px;
align-items: center;
gap: 8px;
color: #667085;
font-size: 11px;
em {
color: #344054;
font-style: normal;
font-weight: 600;
text-align: right;
font-variant-numeric: tabular-nums;
}
}
.judge-reason {
margin: 0;
color: #475467;
font-size: 12px;
line-height: 1.6;
}
.judge-issues {
display: flex;
flex-wrap: wrap;
gap: 4px;
}
.issue-tag {
padding: 2px 8px;
color: #b54708;
background: #fef0c7;
border-radius: 3px;
font-size: 11px;
}
.judge-model {
color: #98a2b3;
font-size: 11px;
}
</style>
<style lang="scss">
.quality-radar-popper {
padding: 14px 16px !important;
}
</style>

99
frontend/src/views/data-process/create/ResultEditorStep.vue
View File

@@ -1,6 +1,7 @@
<script setup lang="ts">
import { computed, ref } from 'vue'
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
import QualityRadarPopover from './QualityRadarPopover.vue'
import type { BulkResultRegenerationState, PreviewItem, ResultEvaluationState, ResultItem } from './types'
import type { DataProcessOutputType } from '@/types/dataProcess'
const props = defineProps<{
@@ -9,6 +10,7 @@ const props = defineProps<{
selectedId: string | null
regeneratingResultId: string | null
bulkRegeneration: BulkResultRegenerationState
evaluation: ResultEvaluationState
outputType: DataProcessOutputType
}>()
@@ -17,6 +19,7 @@ const emit = defineEmits<{
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
'regenerate:item': [id: string]
'regenerate:all': []
'evaluate:all': []
}>()
const search = ref('')
@@ -30,6 +33,15 @@ const bulkRegenerationActive = computed(() => props.bulkRegeneration.status ===
const bulkRegenerationVisible = computed(() => (
props.bulkRegeneration.status !== 'idle' && props.bulkRegeneration.total > 0
))
const evaluationActive = computed(() => props.evaluation.status === 'running')
const evaluationVisible = computed(() => (
props.evaluation.status !== 'idle' && props.evaluation.total > 0
))
const evaluationPercentage = computed(() => {
if (!props.evaluation.total) return 0
return Math.round((props.evaluation.completed / props.evaluation.total) * 100)
})
const evaluatedCount = computed(() => props.items.filter((item) => item.qualityDetails?.evaluated).length)
const bulkRegenerationPercentage = computed(() => (
props.bulkRegeneration.total > 0
? Math.round((props.bulkRegeneration.completed / props.bulkRegeneration.total) * 100)
@@ -98,22 +110,48 @@ function selectRelative(offset: number) {
<template>
<section class="result-step">
<div class="result-workspace">
<aside class="result-list-pane" :class="{ 'has-bulk-progress': bulkRegenerationVisible }">
<aside class="result-list-pane" :class="{ 'has-bulk-progress': bulkRegenerationVisible || evaluationVisible }">
<div class="pane-header result-list-header">
<div class="result-list-title"><strong>生成结果</strong><span>共 {{ items.length }} 条</span></div>
<div class="result-list-title">
<strong>生成结果</strong><span>共 {{ items.length }} 条<template v-if="evaluatedCount"> · 已评测 {{ evaluatedCount }}</template></span>
</div>
<div class="result-list-actions">
<el-button
size="small"
plain
:loading="evaluationActive"
:disabled="!items.length || bulkRegenerationActive || Boolean(regeneratingResultId)"
@click="emit('evaluate:all')"
>
<i v-if="!evaluationActive" class="fa fa-check-square-o" style="margin-right: 4px;" />
数据评测
</el-button>
<el-button
v-if="invalidCount > 0"
size="small"
plain
type="primary"
:loading="bulkRegenerationActive"
:disabled="Boolean(regeneratingResultId) || bulkRegenerationActive"
:disabled="Boolean(regeneratingResultId) || bulkRegenerationActive || evaluationActive"
@click="emit('regenerate:all')"
>
<i v-if="!bulkRegenerationActive" class="fa fa-refresh" style="margin-right: 4px;" />
{{ bulkRegenerationActive ? '重新生成中' : `全部重新生成（${invalidCount}）` }}
</el-button>
</div>
</div>
<div v-if="evaluationVisible" class="bulk-regeneration-progress">
<div>
<span>数据评测 {{ evaluation.completed }} / {{ evaluation.total }}</span>
<span>成功 {{ evaluation.succeeded }} · 失败 {{ evaluation.failed }}</span>
</div>
<el-progress
:percentage="evaluationPercentage"
:show-text="false"
:stroke-width="5"
:color="evaluation.failed > 0 ? '#d97706' : '#5b50f2'"
/>
</div>
<div v-if="bulkRegenerationVisible" class="bulk-regeneration-progress">
<div>
<span>已处理 {{ bulkRegeneration.completed }} / {{ bulkRegeneration.total }}</span>
@@ -146,6 +184,23 @@ function selectRelative(offset: number) {
<strong>{{ item.instruction || '未填写指令' }}</strong>
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
</span>
<el-popover
v-if="item.qualityScore != null && item.qualityDetails"
placement="right"
:width="296"
trigger="hover"
:show-after="150"
popper-class="quality-radar-popper"
>
<template #reference>
<span
class="result-score"
:class="item.qualityScore >= 80 ? 'is-success' : item.qualityScore >= 60 ? 'is-warning' : 'is-danger'"
@click.stop
>{{ item.qualityScore.toFixed(0) }}</span>
</template>
<QualityRadarPopover :quality="item.qualityDetails" :score="item.qualityScore" />
</el-popover>
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
<i
v-else
@@ -303,6 +358,42 @@ function selectRelative(offset: number) {
}
}
.result-list-actions {
display: flex;
flex: none;
align-items: center;
gap: 8px;
}
.result-score {
flex: none;
min-width: 34px;
padding: 2px 8px;
border-radius: 10px;
color: #475467;
background: #f2f4f7;
font-size: 12px;
font-weight: 700;
font-variant-numeric: tabular-nums;
text-align: center;
cursor: default;
&.is-success {
color: #067647;
background: #e6f4ee;
}
&.is-warning {
color: #b54708;
background: #fef0c7;
}
&.is-danger {
color: #b42318;
background: #fee4e2;
}
}
.pane-header {
display: flex;
align-items: center;

21
frontend/src/views/data-process/create/types.ts
View File

@@ -1,6 +1,9 @@
import type {
DataProcessOutputType,
DataProcessPreviewFileStatus,
DataProcessQualityJudge,
DataProcessQualityLayers,
DataProcessQualitySemantic,
DataProcessReasoningDetail,
} from '@/types/dataProcess'
@@ -165,6 +168,14 @@ export interface GenerationState {
message: string
}
export interface ResultQualityDetails {
semantic?: DataProcessQualitySemantic | null
judge?: DataProcessQualityJudge | null
layers?: DataProcessQualityLayers | null
evaluated?: boolean
flags?: string[]
}
export interface ResultItem {
id: string
previewItemId: string | null
@@ -188,7 +199,7 @@ export interface ResultItem {
error?: string
split?: 'train' | 'validation' | 'test'
qualityScore?: number
qualityDetails?: Record<string, number>
qualityDetails?: ResultQualityDetails
updatedAt?: string
}
@@ -201,3 +212,11 @@ export interface BulkResultRegenerationState {
targetIds: string[]
failedIds: string[]
}
export interface ResultEvaluationState {
status: 'idle' | 'running' | 'completed' | 'partial' | 'failed'
total: number
completed: number
succeeded: number
failed: number
}

126
frontend/src/views/data-process/create/useDataProcessEvaluation.ts Normal file
View File

@@ -0,0 +1,126 @@
import { computed, reactive, type Ref } from 'vue'
import { ElMessage } from 'element-plus'
import { evaluateDataProcessResults } from '@/api/modules/dataProcess'
import { mapResult } from './useDataProcessGeneration'
import type { ResultEvaluationState, ResultItem } from './types'
interface EvaluationBindings {
taskId: Ref<string | null>
results: Ref<ResultItem[]>
selectedResultId: Ref<string | null>
}
// 与批量重新生成一致的分块大小：4 个后端 worker 消费三轮，
// 单条评测最长 60 秒，12 条在批量接口 240 秒超时预算内。
const EVALUATION_CHUNK_SIZE = 12
function hasUnsavedChanges(item: ResultItem) {
return item.instruction !== item.savedInstruction
|| item.input !== item.savedInput
|| item.output !== item.savedOutput
|| item.chosen !== item.savedChosen
|| item.rejected !== item.savedRejected
}
export function useDataProcessEvaluation(bindings: EvaluationBindings) {
const evaluation = reactive<ResultEvaluationState>({
status: 'idle',
total: 0,
completed: 0,
succeeded: 0,
failed: 0,
})
const evaluationBusy = computed(() => evaluation.status === 'running')
function resetEvaluation() {
Object.assign(evaluation, {
status: 'idle',
total: 0,
completed: 0,
succeeded: 0,
failed: 0,
})
}
async function evaluateAllResults() {
const taskId = bindings.taskId.value
if (!taskId) return false
if (evaluationBusy.value) {
ElMessage.warning('请等待当前数据评测完成')
return false
}
const candidates = bindings.results.value.filter((item) => item.updatedAt)
if (!candidates.length) {
ElMessage.info('当前没有可评测的结果')
return false
}
const unsaved = candidates.find(hasUnsavedChanges)
if (unsaved) {
bindings.selectedResultId.value = unsaved.id
ElMessage.warning('存在未保存的修改，请先保存后再进行数据评测')
return false
}
Object.assign(evaluation, {
status: 'running',
total: candidates.length,
completed: 0,
succeeded: 0,
failed: 0,
})
let interrupted = false
try {
for (let offset = 0; offset < candidates.length; offset += EVALUATION_CHUNK_SIZE) {
const chunk = candidates.slice(offset, offset + EVALUATION_CHUNK_SIZE)
try {
const evaluated = await evaluateDataProcessResults(taskId, {
items: chunk.map((item) => ({
result_id: item.id,
expected_updated_at: item.updatedAt as string,
})),
})
for (const item of evaluated.items) {
const index = bindings.results.value.findIndex((entry) => entry.id === String(item.id))
if (index >= 0) bindings.results.value[index] = mapResult(item)
}
evaluation.completed += evaluated.total
evaluation.succeeded += evaluated.succeeded
evaluation.failed += evaluated.failed
} catch {
const remaining = candidates.slice(offset)
evaluation.completed = evaluation.total
evaluation.failed += remaining.length
interrupted = true
break
}
}
if (!interrupted && evaluation.failed === 0) {
evaluation.status = 'completed'
ElMessage.success(`数据评测完成：成功 ${evaluation.succeeded} 条`)
} else if (evaluation.succeeded > 0) {
evaluation.status = 'partial'
ElMessage.warning(
`数据评测完成：成功 ${evaluation.succeeded} 条，失败 ${evaluation.failed} 条`,
)
} else {
evaluation.status = 'failed'
ElMessage.error(`数据评测失败：${evaluation.failed} 条结果未完成评测`)
}
return evaluation.failed === 0
} catch {
evaluation.status = 'failed'
ElMessage.error('批量数据评测意外中断，已完成的评分保持不变')
return false
}
}
return {
evaluation,
evaluationBusy,
evaluateAllResults,
resetEvaluation,
}
}

13
frontend/src/views/data-process/create/useDataProcessGeneration.ts
View File

@@ -28,6 +28,7 @@ const POLL_INTERVAL_MS = 1500
const BULK_REGENERATION_CHUNK_SIZE = 12
function mapResult(item: DataProcessResult): ResultItem {
const quality = item.quality_score
return {
id: String(item.id),
previewItemId: item.preview_item_id == null ? null : String(item.preview_item_id),
@@ -50,11 +51,21 @@ function mapResult(item: DataProcessResult): ResultItem {
status: item.status,
error: item.error || undefined,
split: item.split || undefined,
qualityScore: item.quality_score?.overall,
// 生成阶段只有内部规则分，界面不展示；数据评测完成后才显示组合分。
qualityScore: quality?.evaluated ? quality.overall : undefined,
qualityDetails: {
semantic: quality?.semantic ?? null,
judge: quality?.judge ?? null,
layers: quality?.layers ?? null,
evaluated: Boolean(quality?.evaluated),
flags: quality?.flags ?? [],
},
updatedAt: item.updated_at,
}
}
export { mapResult }
export function useDataProcessGeneration(bindings: GenerationBindings) {
const results = ref<ResultItem[]>([])
const selectedResultId = ref<string | null>(null)

18
frontend/src/views/eval/EvalCreateView.vue
View File

@@ -39,7 +39,7 @@ const createdDimensionId = ref<string | number>('')
const taskForm = ref<EvalTaskSetupDraft>({
eval_task_name: '',
model_id: '',
gpu_id: '',
gpu_id: [],
data_source: 'dataset',
dataset_id: '',
leaderboard: false,
@@ -145,12 +145,24 @@ async function handleSubmit() {
const dimensionId = await resolveDimensionId()
// GPU 选择为「节点:GPU序号」复合值，解析出节点与 GPU 序号，
// 多算力节点时必须把节点信息传给后端，否则会派发到错误的算力节点
const [gpuNodeId, gpuIndex] = String(taskForm.value.gpu_id).split(':')
const selectedGpuKeys = Array.isArray(taskForm.value.gpu_id)
? taskForm.value.gpu_id
: [String(taskForm.value.gpu_id)]
const gpuSelections = selectedGpuKeys
.map((key) => {
const [nodeId, gpuIndex] = String(key).split(':')
return { nodeId, gpuIndex: Number(gpuIndex) }
})
.filter((item) => item.nodeId && Number.isInteger(item.gpuIndex) && item.gpuIndex >= 0)
const gpuNodeId = gpuSelections[0]?.nodeId || ''
const gpuIndices = gpuSelections.map((item) => item.gpuIndex)
const evalResult: any = await startEval({
eval_task_name: taskForm.value.eval_task_name,
eval_type: 'custom',
model_id: taskForm.value.model_id,
gpu_id: Number(gpuIndex) || 0,
gpu_id: gpuIndices[0] ?? 0,
gpu_indices: gpuIndices,
gpus: gpuIndices,
compute_node_id: gpuNodeId || '',
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
dimension_id: dimensionId,

20
frontend/src/views/eval/create/EvalTaskSetupStep.vue
View File

@@ -6,7 +6,7 @@ import type { DatasetItem, GpuInfo, TrainedModel } from '@/types'
export interface EvalTaskSetupDraft {
eval_task_name: string
model_id: string | number
gpu_id: string | number
gpu_id: string | number | string[]
data_source: 'dataset' | 'inference'
dataset_id: string | number
leaderboard: boolean
@@ -23,6 +23,13 @@ defineProps<{
const form = defineModel<EvalTaskSetupDraft>({ required: true })
const formRef = ref<FormInstance>()
function handleGpuChange(value: string | number | string[]) {
const keys = Array.isArray(value) ? value.map(String) : [String(value || '')]
const nodeId = keys[0]?.split(':', 1)[0]
if (!nodeId || !Array.isArray(form.value.gpu_id)) return
form.value.gpu_id = keys.filter((key) => key.split(':', 1)[0] === nodeId)
}
const rules: FormRules<EvalTaskSetupDraft> = {
eval_task_name: [
{ required: true, message: '请输入任务名称', trigger: 'blur' },
@@ -101,7 +108,16 @@ defineExpose({ validate })
</el-form-item>
<el-form-item label="选择 GPU" prop="gpu_id">
<el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
<el-select
v-model="form.gpu_id"
multiple
collapse-tags
collapse-tags-tooltip
placeholder="请选择同一算力节点内的一张或多张 GPU"
style="width: 100%"
:loading="loading"
@change="handleGpuChange"
>
<el-option
v-for="gpu in gpus"
:key="`${gpu.node_id || ''}:${gpu.id ?? 0}`"

2
frontend/src/views/eval/create/StartEvalStep.vue
View File

@@ -21,7 +21,7 @@ function nameOf<T extends { id: string | number; name?: string }>(items: T[], id
/** GPU 选择为「节点:GPU序号」复合值，解析并展示为可读标签 */
const gpuLabel = computed(() => {
const key = String(props.task.gpu_id || '')
const key = Array.isArray(props.task.gpu_id) ? String(props.task.gpu_id[0] || '') : String(props.task.gpu_id || '')
const gpu = props.gpus.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === key)
if (gpu) return `${gpu.node_name || gpu.node_code || '算力节点'} / GPU ${gpu.id ?? 0}`
const [nodeId, idx] = key.split(':')

68
frontend/src/views/governance/OrganizationPermissionView.vue Normal file
View File

@@ -0,0 +1,68 @@
<script setup lang="ts">
import { computed } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import UserSettingsView from '@/views/system/UserSettingsView.vue'
import TenantListView from '@/views/tenants/TenantListView.vue'
const route = useRoute()
const router = useRouter()
const activeTab = computed({
get: () => route.query.tab === 'tenants' ? 'tenants' : 'users',
set: (value: string) => {
void router.replace({ query: { tab: value === 'tenants' ? 'tenants' : 'users' } })
},
})
</script>
<template>
<div class="organization-page">
<header class="page-header">
<div>
<h2>组织与权限</h2>
<p>统一管理平台用户、角色、租户和资源配额。</p>
</div>
</header>
<el-tabs v-model="activeTab">
<el-tab-pane label="用户与角色" name="users">
<UserSettingsView v-if="activeTab === 'users'" />
</el-tab-pane>
<el-tab-pane label="租户与配额" name="tenants">
<TenantListView v-if="activeTab === 'tenants'" />
</el-tab-pane>
</el-tabs>
</div>
</template>
<style scoped lang="scss">
.organization-page {
min-height: 100%;
padding: 20px;
}
.page-header {
margin-bottom: 4px;
h2 {
margin: 0;
color: #1f2937;
font-size: 22px;
}
p {
margin: 6px 0 0;
color: #64748b;
font-size: 13px;
}
}
:deep(.user-settings),
:deep(.page) {
padding: 16px 0 0;
}
:deep(.user-settings .page-header > div:first-child) {
display: none;
}
</style>

57
frontend/src/views/inference/InferenceCreateView.vue
View File

@@ -4,8 +4,7 @@ import { useRouter } from 'vue-router'
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
import PageCard from '@/components/PageCard.vue'
import { getModelList, getTrainedModels } from '@/api/modules/model'
import { getSystemInfo } from '@/api/modules/system'
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
import { getComputeGpus, getComputeNodes, type ComputeNode } from '@/api/modules/compute'
import { createCompare, loadCompare } from '@/api/modules/compare'
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
@@ -83,8 +82,8 @@ const form = reactive({
description: '',
/** 选中的模型 key（单选） */
model_key: '',
/** 使用的 GPU */
gpu_key: '',
/** 使用的 GPU（同一节点内可多选） */
gpu_keys: [] as string[],
})
const rules: FormRules = {
@@ -94,12 +93,26 @@ const rules: FormRules = {
/** 当前选中的模型对象 */
const selectedModel = computed(() => modelMap.value[form.model_key])
const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key))
const selectedGpus = computed(() => idleGpus.value.filter((gpu) => form.gpu_keys.includes(gpuKey(gpu))))
function gpuKey(gpu: GpuInfo) {
return `${gpu.node_id || ''}:${gpu.id ?? 0}`
}
function handleGpuChange(keys: string[]) {
const nodeId = keys[0]?.split(':', 1)[0]
if (!nodeId) return
const filtered = keys.filter((key) => key.split(':', 1)[0] === nodeId)
if (filtered.length !== keys.length) {
ElMessage.info('一次推理只能使用同一算力节点内的 GPU，已忽略其它节点的选择')
}
form.gpu_keys = filtered
}
watch(selectedModel, (model) => {
if (!model?.compute_node_id) return
const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}`
if (gpu) form.gpu_keys = [gpuKey(gpu)]
})
async function handleSubmit() {
@@ -111,6 +124,10 @@ async function handleSubmit() {
ElMessage.warning('请选择模型')
return
}
if (!selectedGpus.value.length) {
ElMessage.warning('请至少选择一张空闲 GPU')
return
}
submitting.value = true
startupStatus.value = '正在创建推理任务...'
try {
@@ -130,9 +147,11 @@ async function handleSubmit() {
model_name: m.name,
model_path: m.model_path,
source: m.source,
gpu_id: selectedGpu.value?.id ?? 0,
node_id: selectedGpu.value?.node_id || m.compute_node_id,
node_name: selectedGpu.value?.node_name || m.compute_node_name,
gpu_id: selectedGpus.value[0]?.id ?? 0,
gpu_indices: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)),
gpus: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)),
node_id: selectedGpus.value[0]?.node_id || m.compute_node_id,
node_name: selectedGpus.value[0]?.node_name || m.compute_node_name,
},
],
})
@@ -169,17 +188,17 @@ async function loadData() {
const [db, trained, sys, nodes] = await Promise.all([
getModelList(),
getTrainedModels(),
getSystemInfo(),
getComputeGpus(),
getComputeNodes(),
])
dbModels.value = db || []
trainedModels.value = trained?.models || []
gpus.value = sys?.gpu || []
gpus.value = (sys || []) as unknown as GpuInfo[]
computeNodes.value = nodes || []
// 默认选中第一个空闲 GPU
if (idleGpus.value.length > 0) {
const firstGpu = idleGpus.value[0]
form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}`
form.gpu_keys = [gpuKey(firstGpu)]
}
} catch {
// ignore
@@ -228,12 +247,20 @@ onMounted(loadData)
</el-form-item>
<el-form-item label="GPU">
<el-select v-model="form.gpu_key" style="width: 400px">
<el-select
v-model="form.gpu_keys"
multiple
collapse-tags
collapse-tags-tooltip
style="width: 400px"
placeholder="请选择同一算力节点内的一张或多张 GPU"
@change="handleGpuChange"
>
<el-option
v-for="g in idleGpus"
:key="`${g.node_id || ''}:${g.id ?? 0}`"
:key="gpuKey(g)"
:label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
:value="`${g.node_id || ''}:${g.id ?? 0}`"
:value="gpuKey(g)"
/>
</el-select>
</el-form-item>

78
frontend/src/views/system/RuntimeLogsView.vue Normal file
View File

@@ -0,0 +1,78 @@
<script setup lang="ts">
import { computed } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { useAuthStore } from '@/stores/auth'
import LogsView from './LogsView.vue'
import AuditLogView from '@/views/audit/AuditLogView.vue'
import OperationLogView from '@/views/audit/OperationLogView.vue'
const route = useRoute()
const router = useRouter()
const auth = useAuthStore()
const activeTab = computed({
get: () => {
if (!auth.isAdmin) return 'runtime'
if (route.query.tab === 'audit') return 'audit'
if (route.query.tab === 'operations') return 'operations'
return 'runtime'
},
set: (value: string) => {
void router.replace({ query: value === 'runtime' ? {} : { tab: value } })
},
})
</script>
<template>
<div class="runtime-logs">
<header class="page-header">
<div>
<h2>运行日志</h2>
<p>查看系统运行、训练任务、审计记录和操作诊断信息。</p>
</div>
</header>
<el-tabs v-model="activeTab">
<el-tab-pane label="运行日志" name="runtime">
<LogsView v-if="activeTab === 'runtime'" />
</el-tab-pane>
<el-tab-pane v-if="auth.isAdmin" label="审计记录" name="audit">
<AuditLogView v-if="activeTab === 'audit'" />
</el-tab-pane>
<el-tab-pane v-if="auth.isAdmin" label="操作诊断" name="operations">
<OperationLogView v-if="activeTab === 'operations'" />
</el-tab-pane>
</el-tabs>
</div>
</template>
<style scoped lang="scss">
.runtime-logs {
min-height: 100%;
padding: 20px;
}
.page-header {
margin-bottom: 4px;
h2 {
margin: 0;
color: #1f2937;
font-size: 22px;
}
p {
margin: 6px 0 0;
color: #64748b;
font-size: 13px;
}
}
:deep(.page) {
padding: 16px 0 0;
}
:deep(.page-header .page-title) {
display: none;
}
</style>

18
frontend/src/views/system/TrainingLogView.vue
View File

@@ -8,10 +8,9 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
import { usePolling } from '@/composables/usePolling'
import '@/plugins/echarts-training-log'
import { useModelsStore } from '@/stores/models'
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
import { getFineTune, getFineTuneDiagnostics, getFineTuneGpuStatus, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
import { getDataset } from '@/api/modules/dataset'
import { getSystemInfo } from '@/api/modules/system'
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
import {
buildMetricChartOption,
@@ -205,15 +204,20 @@ async function loadDataset(datasetId: string | number) {
}
}
async function loadGpuStatus() {
async function loadGpuStatus(currentTask: FineTuneTask) {
try {
const systemInfo = await getSystemInfo()
gpuPool.value = systemInfo.gpu ?? []
const live = await getFineTuneGpuStatus(currentTask.id)
if (live.source === 'compute' && live.items.length) {
gpuPool.value = live.items as unknown as GpuInfo[]
gpuUpdatedAt.value = new Date()
gpuLoadError.value = ''
return
}
gpuPool.value = []
gpuLoadError.value = live.error || 'Compute 节点暂未返回实时 GPU 指标'
} catch {
gpuLoadError.value = 'GPU 监控数据暂时不可用'
if (!gpuUpdatedAt.value) gpuPool.value = []
gpuPool.value = []
}
}
@@ -306,7 +310,7 @@ async function refreshAll() {
const datasetPromise = currentTask.train_dataset_id
? loadDataset(currentTask.train_dataset_id)
: Promise.resolve()
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(currentTask), loadDiagnostics(currentTask)])
await loadMetrics(currentTask)
} finally {
loading.value = false

4
frontend/src/views/system/UserCreateView.vue
View File

@@ -12,7 +12,7 @@ const form = reactive<CreateUserPayload>({
username: '',
display_name: '',
password: 'platform123',
role: 'user',
role: 'operator',
status: 'active',
permissions: [],
})
@@ -45,7 +45,7 @@ async function submit() {
<el-form-item label="角色">
<el-select v-model="form.role" style="width: 100%">
<el-option label="管理员" value="admin" />
<el-option label="普通用户" value="user" />
<el-option label="普通用户" value="operator" />
</el-select>
</el-form-item>
<el-form-item label="状态">