diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 2d7b657..de571a8 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -35,7 +35,12 @@ from fastapi.responses import StreamingResponse from psycopg.rows import dict_row from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin + from app.core.logging import get_structured_logger + +from app.core.config import get_settings +from app.db.platform_store import get_platform_store + from app.modules.data_process.algorithms import ( ParsedText, canonical_record_json, @@ -61,6 +66,10 @@ from app.modules.data_process.document_chunking import ( chunk_semantic_text, merge_short_chunks, ) +from app.modules.data_process.evaluation import ( + evaluate_result_record, + reevaluate_edited_record, +) from app.modules.data_process.generation import generate_model_records from app.modules.data_process.office_preview import ( MAX_XLSX_PREVIEW_ROWS, @@ -82,6 +91,8 @@ from app.modules.data_process.store import ( new_id, repeat_task_id, ) +from app.modules.storage.minio_store import get_object_storage +from app.modules.storage.policy import should_store_in_minio from app.schemas.data_process import ( DataProcessRegenerateRequest, DataProcessRepeatRequest, @@ -97,6 +108,7 @@ from app.schemas.data_process import ( PreviewItemUpdate, ProcessType, PublishRequest, + ResultBatchEvaluateRequest, ResultBatchRegenerateRequest, ResultRegenerateRequest, ResultUpdate, @@ -221,9 +233,38 @@ def _commit_source_batch( staged: list[StagedSourceObject], ) -> list[dict[str, Any]]: storage.publish(staged) + storage_object_ids: list[str] = [] try: + # The source reference remains in the task schema for compatibility, + # while storage_objects provides the authoritative MinIO index. + if get_settings().minio_enabled: + for item in prepared: + reference = str(item.get("storage_object_id") or "") + if not reference.startswith("minio://"): + continue + object_key = storage.object_key(reference) + metadata = get_object_storage().stat(object_key) + object_row = get_platform_store().create_storage_object({ + "resource_type": "data_process_source", + "resource_id": task_id, + "version_id": str(item.get("id") or new_id("dpsf")), + "bucket": get_object_storage().bucket, + "object_key": object_key, + "file_name": item.get("name"), + "content_type": (item.get("metadata") or {}).get("content_type", "application/octet-stream"), + "byte_size": metadata.get("byte_size") or item.get("raw_size") or 0, + "checksum_sha256": item.get("checksum_sha256"), + "status": "available", + "created_by": item.get("created_by"), + }) + storage_object_ids.append(str(object_row["id"])) return store.add_source_files(task_id, prepared) except Exception: + for object_id in storage_object_ids: + try: + get_platform_store().update_storage_object(object_id, {"status": "deleted"}) + except Exception: + pass for item in staged: try: storage.delete(item.reference) @@ -771,6 +812,25 @@ def _run_generation( duplicate_count=duplicate_count, error_count=error_count, ) + result_bytes = "".join( + structured_json_dumps(item) + "\n" for item in accepted + ).encode("utf-8") + if should_store_in_minio(len(result_bytes)): + result_key = f"data-process/{task_id}/results/{generation_run_id}.jsonl" + uploaded = get_object_storage().put_bytes(result_key, result_bytes, "application/jsonl") + get_platform_store().create_storage_object({ + "resource_type": "data_process_result", + "resource_id": task_id, + "version_id": generation_run_id, + "bucket": uploaded["bucket"], + "object_key": result_key, + "file_name": f"{generation_run_id}.jsonl", + "content_type": "application/jsonl", + "byte_size": len(result_bytes), + "checksum_sha256": hashlib.sha256(result_bytes).hexdigest(), + "status": "available", + "created_by": (store.get_task(task_id) or {}).get("created_by"), + }) logger.info( "data process generation completed task_id=%s generation_run_id=%s " "output_count=%s filtered_count=%s duplicate_count=%s error_count=%s " @@ -930,7 +990,7 @@ def _repeat_file_copies( source = store.get_source_file(source_task_id, old_file_id, include_content=True) new_file_id = new_id("dpsf") old_reference = str(source.get("storage_object_id") or "") - if old_reference.startswith("local://data-process/"): + if old_reference.startswith(("local://data-process/", "minio://data-process/")): staged_object = storage.stage_copy( batch_id=batch_id, source_reference=old_reference, @@ -2044,12 +2104,13 @@ def update_result( or 20 ), ) - quality = score_quality( + # 编辑后内容已变化：重算规则与语义层，旧的评审分不再可信直接丢弃。 + update["quality_score"] = reevaluate_edited_record( merged, - min_output_length=minimum, source_content=source_content, + previous_quality=current.get("quality_score"), + min_output_length=minimum, ) - update["quality_score"] = asdict(quality) result = store.update_result( task_id, result_id, @@ -2094,11 +2155,6 @@ def restore_result( or 20 ), ) - quality = score_quality( - restored, - min_output_length=minimum, - source_content=source_content, - ) restored = store.update_result( task_id, result_id, @@ -2108,7 +2164,12 @@ def restore_result( "output": restored["output"], "chosen": restored["chosen"], "rejected": restored["rejected"], - "quality_score": asdict(quality), + "quality_score": reevaluate_edited_record( + restored, + source_content=source_content, + previous_quality=current.get("quality_score"), + min_output_length=minimum, + ), "expected_updated_at": current.get("updated_at"), }, ) @@ -2271,6 +2332,46 @@ def _safe_regeneration_error(exc: Exception) -> str: return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed" +def _evaluate_result_in_place( + task_id: str, + current: dict[str, Any], + source_content: str, + config: dict[str, Any], + evaluation_model: dict[str, Any] | None, + store: DataProcessStore, + *, + expected_updated_at: str, + model_client: httpx.Client | None = None, + minimum: int = 20, +) -> dict[str, Any]: + """评测单条结果并落库；复用逐结果互斥锁避免与重生成并发写冲突。""" + + result_id = str(current["id"]) + with _claim_result_regeneration(task_id, result_id): + quality = evaluate_result_record( + { + "instruction": current.get("instruction"), + "input": current.get("input"), + "output": current.get("output"), + "chosen": current.get("chosen"), + "rejected": current.get("rejected"), + }, + source_content=source_content, + model=evaluation_model, + config=config, + client=model_client, + min_output_length=minimum, + ) + return store.update_result( + task_id, + result_id, + { + "quality_score": quality, + "expected_updated_at": expected_updated_at, + }, + ) + + @router.post("/{task_id}/results/regenerate-batch") def regenerate_results_batch( task_id: str, @@ -2444,6 +2545,186 @@ def regenerate_results_batch( ) +@router.post("/{task_id}/results/evaluate-batch") +def evaluate_results_batch( + task_id: str, + payload: ResultBatchEvaluateRequest, + store: DataProcessStore = Depends(get_data_process_store), +) -> dict[str, Any]: + """对一批结果执行三层质量评测（规则+语义+评审），允许部分成功。""" + + started_at = time.perf_counter() + batch_id = new_id("dpeb") + with api_errors(): + task = store.get_task(task_id) + if task.get("status") == "running": + raise ConflictError("data process task is running") + if task.get("output_dataset_id"): + raise InvalidStateError("published results cannot be evaluated") + config = task.get("config") or {} + evaluation_model: dict[str, Any] | None = None + model_id = _value(config, "generation_model_id", "generationModelId", None) + if model_id: + try: + evaluation_model = store.get_generation_model(str(model_id)) + except NotFoundError: + logger.warning( + "data process evaluation model unavailable, judge layer " + "skipped task_id=%s model_id=%s", + task_id, + model_id, + ) + evaluation_config = { + **config, + "output_type": str( + _value(config, "output_type", "outputType", "standard") + ).strip().lower(), + } + minimum = max( + 1, + int(_value(config, "min_output_length", "minOutputLength", 20) or 20), + ) + + prepared: list[tuple[int, dict[str, Any], str, str]] = [] + failures: list[tuple[int, dict[str, str]]] = [] + for index, requested in enumerate(payload.items): + try: + current = store.get_result(task_id, requested.result_id) + if requested.expected_updated_at != str(current.get("updated_at") or ""): + raise ConflictError("data process result was modified by another request") + source_content = "" + preview_id = current.get("preview_item_id") + if preview_id: + preview = store.get_preview_item(task_id, str(preview_id)) + source_content = str( + preview.get("edited_content") + or preview.get("original_content") + or "" + ) + prepared.append( + (index, current, source_content, requested.expected_updated_at) + ) + except ConflictError as exc: + failures.append((index, { + "result_id": requested.result_id, + "code": "conflict", + "message": _safe_regeneration_error(exc), + })) + except (NotFoundError, InvalidStateError) as exc: + failures.append((index, { + "result_id": requested.result_id, + "code": "skipped", + "message": _safe_regeneration_error(exc), + })) + + logger.info( + "data process result batch evaluation started batch_id=%s task_id=%s " + "requested=%s prepared=%s judge_enabled=%s", + batch_id, + task_id, + len(payload.items), + len(prepared), + evaluation_model is not None, + ) + successes: list[tuple[int, dict[str, Any]]] = [] + if prepared: + try: + from app.modules.data_process.algorithms.embedding import ( + semantic_embedding_model, + ) + + semantic_embedding_model() + except Exception: + logger.warning( + "data process semantic embedding unavailable, semantic layer " + "will be skipped batch_id=%s", + batch_id, + ) + request_timeout = _result_regeneration_timeout(config) + model_timeout = httpx.Timeout( + request_timeout, + connect=min(10.0, request_timeout), + ) + model_limits = httpx.Limits( + max_connections=RESULT_REGENERATION_CONCURRENCY, + max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY, + ) + with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \ + ThreadPoolExecutor( + max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)), + thread_name_prefix="data-result-evaluation", + ) as executor: + futures = { + executor.submit( + _evaluate_result_in_place, + task_id, + current, + source_content, + evaluation_config, + evaluation_model, + store, + expected_updated_at=expected_updated_at, + model_client=model_client if evaluation_model else None, + minimum=minimum, + ): (index, str(current["id"]), time.perf_counter()) + for index, current, source_content, expected_updated_at in prepared + } + for future in as_completed(futures): + index, result_id, item_started_at = futures[future] + try: + evaluated = future.result() + successes.append((index, evaluated)) + outcome = "succeeded" + except ConflictError as exc: + outcome = "conflict" + failures.append((index, { + "result_id": result_id, + "code": outcome, + "message": _safe_regeneration_error(exc), + })) + except Exception as exc: + outcome = "evaluation_failed" + failures.append((index, { + "result_id": result_id, + "code": outcome, + "message": _safe_regeneration_error(exc), + })) + logger.info( + "data process result batch evaluation item finished " + "batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f", + batch_id, + task_id, + result_id, + outcome, + (time.perf_counter() - item_started_at) * 1000, + ) + + success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])] + failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])] + duration_ms = (time.perf_counter() - started_at) * 1000 + logger.info( + "data process result batch evaluation completed batch_id=%s task_id=%s " + "succeeded=%s failed=%s duration_ms=%.2f", + batch_id, + task_id, + len(success_items), + len(failure_items), + duration_ms, + ) + return ok( + { + "batch_id": batch_id, + "total": len(payload.items), + "succeeded": len(success_items), + "failed": len(failure_items), + "duration_ms": round(duration_ms, 2), + "items": success_items, + "failures": failure_items, + }, + "data process results evaluated", + ) + + @router.post("/{task_id}/results/{result_id}/regenerate") def regenerate_result( task_id: str, diff --git a/backend/app/api/v1/endpoints/platform.py b/backend/app/api/v1/endpoints/platform.py index 0bd84f5..257767a 100644 --- a/backend/app/api/v1/endpoints/platform.py +++ b/backend/app/api/v1/endpoints/platform.py @@ -23,8 +23,9 @@ from app.core.audit import audit_log, AuditActions from app.core.op_log import op_log, OpModule, OpAction from app.db.platform_store import get_platform_store from app.modules.compute_gateway.client import ComputeNodeClient -from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once +from app.modules.compute_gateway.sync import _archive_node_directory, fetch_eval_result_content, poll_compute_jobs_once from app.modules.storage.minio_store import ObjectStorageError, get_object_storage +from app.modules.storage.policy import should_store_in_minio router = APIRouter() _LOGIN_FAILURES: dict[str, list[float]] = {} @@ -57,6 +58,33 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None: return None +def _normalize_gpu_indices(payload: dict[str, Any], *, allow_primary: bool = True) -> list[int]: + """Normalize all frontend GPU selection shapes to sorted integer indexes.""" + raw = payload.get("gpu_indices") + if raw is None: + raw = payload.get("gpus") + if raw is None and allow_primary and payload.get("gpu_id") is not None: + raw = [payload.get("gpu_id")] + if raw is None or raw == "": + return [] + if isinstance(raw, str): + raw = [item.strip() for item in raw.split(",") if item.strip()] + if not isinstance(raw, (list, tuple, set)): + raw = [raw] + result: set[int] = set() + for item in raw: + if isinstance(item, str) and ":" in item: + item = item.rsplit(":", 1)[-1] + try: + index = int(item) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid GPU index: {item}") from exc + if index < 0: + raise ValueError("GPU index must be non-negative") + result.add(index) + return sorted(result) + + async def _wait_for_object_storage() -> None: """Wait for MinIO before starting a resource task.""" settings = get_settings() @@ -73,6 +101,106 @@ async def _wait_for_object_storage() -> None: await asyncio.sleep(max(1, settings.storage_check_interval_seconds)) +def _store_json_snapshot( + store: Any, + resource_type: str, + resource_id: str, + version_id: str, + object_key: str, + payload: dict[str, Any], + created_by: str | None = None, +) -> dict[str, Any]: + """Persist non-secret task/model parameters as an auditable MinIO snapshot.""" + sanitized = { + key: value + for key, value in payload.items() + if key not in {"api_key", "secret_key", "password", "token", "access_token"} + } + raw = json.dumps(sanitized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8") + # Task/evaluation payloads are already persisted in PostgreSQL. Avoid an + # extra MinIO round trip for small non-secret parameter snapshots. + if not should_store_in_minio(len(raw), content_type="application/json", file_format="json"): + return {} + uploaded = get_object_storage().put_bytes(object_key, raw, "application/json") + return store.create_storage_object({ + "resource_type": resource_type, + "resource_id": resource_id, + "version_id": version_id, + "bucket": uploaded["bucket"], + "object_key": object_key, + "file_name": Path(object_key).name, + "content_type": "application/json", + "byte_size": len(raw), + "checksum_sha256": hashlib.sha256(raw).hexdigest(), + "status": "available", + "created_by": created_by, + }) + + +def _dataset_file_bytes(store: Any, file_id: str) -> bytes: + """Return the canonical dataset bytes, lazily indexing legacy DB content.""" + row = store.dataset_file(file_id) + if not get_settings().minio_enabled: + return str(row.get("content") or "").encode("utf-8") + + storage_object = None + object_id = str(row.get("storage_object_id") or "") + if object_id: + try: + storage_object = store.storage_object(object_id) + except KeyError: + storage_object = None + if storage_object and storage_object.get("status") == "available": + try: + return get_object_storage().get_bytes(storage_object["object_key"]) + except Exception as exc: # noqa: BLE001 - expose storage outage to callers + raise RuntimeError(f"dataset object is unavailable in MinIO: {exc}") from exc + + # Compatibility migration for files created before MinIO was enabled. + raw = str(row.get("content") or "").encode("utf-8") + if not raw: + raise RuntimeError(f"dataset file has no MinIO object or legacy content: {file_id}") + # Small files intentionally remain database-backed. They can still be + # copied to a compute node directly when a task needs them. + if not should_store_in_minio(len(raw), file_format=row.get("file_format")): + return raw + object_key = ( + f"datasets/{row['dataset_id']}/versions/" + f"{row.get('active_version_id') or row['id']}/{Path(str(row.get('name') or row['id'])).name}" + ) + uploaded = get_object_storage().put_bytes(object_key, raw, "application/octet-stream") + created = store.create_storage_object({ + "resource_type": "dataset", + "resource_id": str(row["dataset_id"]), + "version_id": str(row.get("active_version_id") or row["id"]), + "bucket": uploaded["bucket"], + "object_key": object_key, + "file_name": row.get("name"), + "content_type": "application/octet-stream", + "byte_size": len(raw), + "checksum_sha256": hashlib.sha256(raw).hexdigest(), + "status": "available", + }) + store.link_dataset_file_storage_object(str(row["id"]), created["id"]) + return raw + + +def _dataset_version_bytes(store: Any, file_id: str, version_id: str) -> bytes: + row = store.dataset_file(file_id) + try: + version = next(item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id) + except StopIteration as exc: + raise KeyError(version_id) from exc + object_id = str(version.get("storage_object_id") or "") + if get_settings().minio_enabled and object_id: + try: + obj = store.storage_object(object_id) + return get_object_storage().get_bytes(obj["object_key"]) + except KeyError: + pass + return _dataset_file_bytes(store, file_id) + + def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None: """Select the compute node for an eval job. @@ -103,18 +231,63 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: if not get_settings().minio_enabled or not resource_id: return None objects = store.storage_objects_for_resource(resource_type, resource_id) + if not objects and resource_type in {"model", "trained_model"}: + resource = None + if resource_type == "model": + try: + resource = store.model(resource_id) + except KeyError: + try: + resource = store.model_by_name(resource_id) + except KeyError: + resource = next((item for item in store.models() if item.get("path") == resource_id), None) + else: + resource = next( + (item for item in store.trained_models() if item.get("id") == resource_id or item.get("name") == resource_id), + None, + ) + source_path = str( + (resource or {}).get("path") + or (resource or {}).get("merged_path") + or (resource or {}).get("artifact_dir") + or "" + ) + resolved_id = str((resource or {}).get("id") or resource_id) + if source_path and resolved_id: + client = ComputeNodeClient(node["api_base_url"], timeout=900) + await _archive_node_directory( + store, + client, + node, + source_path, + resource_type, + resolved_id, + "legacy-import", + f"models/{resolved_id}" if resource_type == "model" else f"trained_models/{resolved_id}", + ) + resource_id = resolved_id + objects = store.storage_objects_for_resource(resource_type, resource_id) if not objects: return None client = ComputeNodeClient(node["api_base_url"], timeout=900) root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s" for obj in objects: + object_key = str(obj["object_key"]) + marker = f"{root_name}/{resource_id}/versions/" + relative_name = Path(str(obj.get("file_name") or object_key)).name + if marker in object_key: + suffix = object_key.split(marker, 1)[1] + if "/" in suffix: + suffix = suffix.split("/", 1)[1] + if suffix: + relative_name = suffix await client.prepare_cache({ "resource_id": resource_id, "version_id": obj["version_id"], - "download_url": get_object_storage().presigned_get(obj["object_key"]), + "download_url": get_object_storage().presigned_get(object_key), "checksum_sha256": obj.get("checksum_sha256") or "", "byte_size": obj.get("byte_size") or 0, - "relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}", + "relative_path": f"{root_name}/{resource_id}/{relative_name}", }) return f"/data/yg-ft/{root_name}/{resource_id}" @@ -370,8 +543,23 @@ async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[st if not preflight["valid"]: errors = "; ".join(preflight.get("errors") or ["preflight failed"]) raise RuntimeError(f"preflight failed: {errors}") - payload = {**payload, "compute_node_id": preflight["node"]["id"]} + prepared_job_payload = preflight.get("job_payload") or {} + payload = { + **payload, + "compute_node_id": preflight["node"]["id"], + "prepared_base_model_path": prepared_job_payload.get("model_name_or_path") or payload.get("prepared_base_model_path"), + } task = store.start_task(payload) + if get_settings().minio_enabled: + _store_json_snapshot( + store, + "fine_tune", + str(task["id"]), + str(task["id"]), + f"training/{task['id']}/versions/{task['id']}/training-config.json", + task, + task.get("created_by"), + ) if get_settings().compute_mode == "simulator": return task node, job_payload = store.build_compute_job_payload(task["id"]) @@ -427,6 +615,20 @@ async def _fine_tune_preflight_with_job_payload( if get_settings().minio_enabled and get_settings().compute_mode != "simulator": try: await _wait_for_object_storage() + base_model_path = str(job_payload.get("model_name_or_path") or job_payload.get("base_model") or "") + base_model_id = str(job_payload.get("base_model_id") or job_payload.get("model_id") or "") + base_model = None + if base_model_id: + try: + base_model = store.model(base_model_id) + except KeyError: + base_model = None + if base_model is None: + base_model = next((item for item in store.models() if item.get("path") == base_model_path), None) + if base_model: + prepared_model = await _prepare_resource_on_node(store, "model", str(base_model["id"]), node) + if prepared_model: + job_payload = {**job_payload, "base_model": prepared_model, "model_name_or_path": prepared_model} except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure sync_errors.append(f"shared storage health check failed: {exc}") if get_settings().compute_mode == "simulator": @@ -1076,8 +1278,9 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = @router.get("/dataset-manage/preview/{file_id}") async def dataset_preview(file_id: str) -> dict[str, Any]: try: - row = get_platform_store().dataset_file(file_id) - return ok({"content": row["content"]}) + store = get_platform_store() + content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace") + return ok({"content": content}) except KeyError: raise fail(404, "dataset file not found") @@ -1106,7 +1309,8 @@ async def dataset_version_content(file_id: str, version_id: str) -> dict[str, An version = next((item for item in versions if item["id"] == version_id), None) if not version: raise KeyError(version_id) - return ok({"version": version, "content": row["content"]}) + content = _dataset_version_bytes(get_platform_store(), file_id, version_id) + return ok({"version": version, "content": content.decode("utf-8", errors="replace")}) except KeyError: raise fail(404, "dataset version not found") @@ -1197,7 +1401,7 @@ async def _sync_training_dataset_to_compute_node( ) -> list[dict[str, Any]]: if get_settings().minio_enabled: files = store.training_dataset_files(dataset_id) - object_by_resource_name: dict[tuple[str, str], dict[str, Any]] = {} + object_by_resource_name: dict[tuple[str, str, str], dict[str, Any]] = {} resource_ids = {str(dataset_id)} | { str(item.get("dataset_id")) for item in files @@ -1206,35 +1410,58 @@ async def _sync_training_dataset_to_compute_node( for resource_id in resource_ids: for obj in store.storage_objects_for_resource("dataset", resource_id): file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name - object_by_resource_name[(resource_id, file_name)] = obj + object_by_resource_name[(resource_id, file_name, str(obj.get("version_id") or ""))] = obj results: list[dict[str, Any]] = [] client = ComputeNodeClient(node["api_base_url"]) for item in files: target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name item_dataset_id = str(item.get("dataset_id") or dataset_id) - obj = object_by_resource_name.get((item_dataset_id, target_name)) + version_id = str(item.get("active_version_id") or item["id"]) + obj = object_by_resource_name.get((item_dataset_id, target_name, version_id)) if not obj and item.get("content"): - # 兼容 MinIO 接入前已经发布的数据处理数据集： - # 预检时用数据库正文补建对象，避免要求用户重新处理数据集。 + # 兼容 MinIO 接入前已经发布的数据处理数据集。大文件补建 + # MinIO 对象，小文件直接从数据库正文同步到目标节点。 raw = str(item.get("content") or "").encode("utf-8") - version_id = str(item.get("active_version_id") or item["id"]) - object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}" - uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl") - obj = store.create_storage_object({ - "resource_type": "dataset", - "resource_id": item_dataset_id, - "version_id": version_id, - "bucket": uploaded["bucket"], - "object_key": object_key, - "file_name": target_name, - "content_type": "application/jsonl", - "byte_size": len(raw), - "checksum_sha256": hashlib.sha256(raw).hexdigest(), - "status": "available", - }) - store.link_dataset_file_storage_object(str(item["id"]), obj["id"]) + if should_store_in_minio(len(raw)): + object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}" + uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl") + obj = store.create_storage_object({ + "resource_type": "dataset", + "resource_id": item_dataset_id, + "version_id": version_id, + "bucket": uploaded["bucket"], + "object_key": object_key, + "file_name": target_name, + "content_type": "application/jsonl", + "byte_size": len(raw), + "checksum_sha256": hashlib.sha256(raw).hexdigest(), + "status": "available", + }) + store.link_dataset_file_storage_object(str(item["id"]), obj["id"]) + else: + result = await client.upload_file( + target_name, + raw, + f"datasets/{dataset_id}/{target_name}", + resource_type="dataset", + resource_id=dataset_id, + ) + store.upsert_resource_replica( + node["id"], "dataset", dataset_id, str(result.get("local_path") or "") + ) + results.append({ + "node_id": node["id"], + "node_code": node.get("code"), + "file_id": item.get("id"), + "name": target_name, + "local_path": result.get("local_path"), + "byte_size": result.get("byte_size"), + "checksum_sha256": result.get("checksum_sha256"), + "storage_backend": "database", + }) + continue if not obj: - raise RuntimeError(f"dataset file is not available in MinIO: {target_name}") + raise RuntimeError(f"dataset file is not available: {target_name}") url = get_object_storage().presigned_get(obj["object_key"]) result = await client.prepare_cache({ "resource_id": dataset_id, @@ -1250,7 +1477,13 @@ async def _sync_training_dataset_to_compute_node( dataset_id, str(result.get("local_path") or ""), ) - results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]}) + results.append({ + **result, + "file_id": item.get("id"), + "name": target_name, + "node_id": node["id"], + "storage_backend": "minio", + }) return results if not dataset_id: raise RuntimeError("train_dataset_id is required") @@ -1316,7 +1549,11 @@ async def upload_dataset_files( created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content) created.append(created_file) pending_sync.append((created_file["id"], created_file["name"], raw)) - if get_settings().minio_enabled: + if should_store_in_minio( + len(raw), + content_type=file.content_type, + file_format=Path(created_file["name"]).suffix, + ): object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}" uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream") storage_object = get_platform_store().create_storage_object({ @@ -1360,7 +1597,10 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur full_file = store.dataset_file(str(item["id"])) except KeyError: continue - files.append({**item, "content": full_file.get("content") or ""}) + files.append({ + **item, + "content": _dataset_file_bytes(store, str(item["id"])).decode("utf-8", errors="replace"), + }) if not files: raise fail(404, "dataset has no downloadable files") @@ -1401,8 +1641,12 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur @router.get("/dataset-manage/download/{dataset_id}/{file_id}") async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse: - row = get_platform_store().dataset_file(file_id) - return PlainTextResponse(row["content"], media_type="text/plain") + store = get_platform_store() + row = store.dataset_file(file_id) + if str(row.get("dataset_id")) != str(dataset_id): + raise fail(404, "dataset file not found") + content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id) + return PlainTextResponse(content.decode("utf-8", errors="replace"), media_type="text/plain") @router.get("/dataset-manage") @@ -1545,10 +1789,10 @@ async def start_fine_tune( if node_id and gpu_indices: if not store.check_gpu_access(current_user["id"], node_id, gpu_indices): raise fail(403, "无权使用所选 GPU，请联系管理员分配") - # 记录创建者 if node_id and not gpu_indices: payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id) - payload["strict_node_selection"] = bool(node_id) + # 页面明确选择节点时，调度器必须保持节点约束；否则可能落到其它节点。 + payload["strict_node_selection"] = bool(payload.get("compute_node_id") or payload.get("node_id")) payload.setdefault("created_by", current_user.get("id")) try: return ok(await _submit_fine_tune_task(store, payload)) @@ -1679,6 +1923,42 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]: ) +@router.get("/fine-tune/{task_id}/gpu-status") +async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]: + """Return live GPU metrics for the task's selected node and cards.""" + store = get_platform_store() + try: + task = store.task(task_id) + except KeyError: + raise fail(404, "fine tune task not found") + if not has_resource_access("fine-tune", task_id, current_user, "read"): + raise fail(403, "no permission to access this task") + node = _node_for_task(task) + selected = set(_normalize_gpu_indices({"gpus": task.get("gpus") or []}, allow_primary=False)) + if not node: + return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected)}) + try: + if get_settings().compute_mode == "simulator": + live_items = store.gpus() + else: + live_items = await ComputeNodeClient(node["api_base_url"]).gpu_resources() + items = [] + for item in live_items: + index = int(item.get("gpu_index", item.get("id", -1))) + if selected and index not in selected: + continue + items.append({ + **item, + "id": index, + "node_id": node["id"], + "node_code": node.get("code"), + "node_name": node.get("name"), + }) + return ok({"source": "compute", "items": items, "selected_gpus": sorted(selected)}) + except Exception as exc: # noqa: BLE001 - let UI retain last good snapshot + return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected), "error": str(exc)}) + + @router.put("/fine-tune/{task_id}") @op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id") async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: @@ -1828,15 +2108,35 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """Start an evaluation task: submit eval job to compute node.""" store = get_platform_store() + try: + gpu_indices = _normalize_gpu_indices(payload) + except ValueError as exc: + raise fail(400, str(exc)) + if not gpu_indices: + raise fail(400, "请选择至少一张 GPU") # 1. Create eval task record payload.setdefault("created_by", current_user.get("id")) task = store.create_eval_task({**payload, "status": "pending"}) + if get_settings().minio_enabled: + _store_json_snapshot( + store, + "eval", + str(task["id"]), + str(task["id"]), + f"evaluations/{task['id']}/versions/{task['id']}/evaluation-config.json", + payload, + current_user.get("id"), + ) # 2. Resolve model path (supports both regular models and trained models) model_id = str(payload.get("model_id", "")) model_path = "" adapter_path = payload.get("adapter_path", "") model_node_id = "" + ds_files: list[dict[str, Any]] = [] + model_resource_type = "model" + model_resource_id = model_id + adapter_resource_id = "" try: db_model = store.model(model_id) model_path = db_model.get("path", "") @@ -1845,6 +2145,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di # Try trained_models table (IDs prefixed with tm_) trained = next((m for m in store.trained_models() if m["id"] == model_id), None) if trained: + model_resource_type = "trained_model" if trained.get("merged") else "model" + model_resource_id = trained.get("id") or model_id + adapter_resource_id = trained.get("id") or "" model_node_id = trained.get("compute_node_id") or "" merged_path = trained.get("merged_path", "") base_path = trained.get("base_model_path", "") @@ -1858,6 +2161,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di adapter_path = merged_path else: model_path = merged_path or base_path + if model_resource_type == "model": + base_model = next((item for item in store.models() if item.get("path") == base_path), None) + model_resource_id = str((base_model or {}).get("id") or base_path) if not model_path: store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"}) return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"}) @@ -1930,6 +2236,33 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di store.update_eval_task(task["id"], {"status": "failed", "error": message}) return ok({"task_id": task["id"], "status": "failed", "error": message}) + if get_settings().minio_enabled and get_settings().compute_mode != "simulator": + try: + prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node) + if prepared_model: + model_path = prepared_model + if adapter_resource_id and model_resource_type == "model": + prepared_adapter = await _prepare_resource_on_node(store, "trained_model", adapter_resource_id, node) + if prepared_adapter: + adapter_path = prepared_adapter + dataset_sync = await _sync_training_dataset_to_compute_node(store, node, dataset_id) + if dataset_sync: + dataset_path = str(dataset_sync[0].get("local_path") or dataset_path) + except Exception as exc: + store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"}) + return ok({"task_id": task["id"], "status": "failed", "error": str(exc)}) + + node_gpus = { + int(item.get("id", item.get("gpu_index", -1))): item + for item in store.gpus() + if item.get("node_id") == node["id"] + } + unavailable = [index for index in gpu_indices if node_gpus.get(index, {}).get("status") != "idle"] + if unavailable: + message = f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}" + store.update_eval_task(task["id"], {"status": "failed", "error": message}) + return ok({"task_id": task["id"], "status": "failed", "error": message}) + # 6. Build eval job payload output_dir = f"/data/yg-ft/outputs/{task['id']}" job_payload = { @@ -1943,7 +2276,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di "output_dir": output_dir, "basic_metrics": payload.get("basic_metrics", {}), "dimension": dimension_cfg, - "gpus": [int(payload.get("gpu_id", 0))], + "gpu_id": gpu_indices[0], + "gpu_indices": gpu_indices, + "gpus": gpu_indices, "temperature": payload.get("temperature", 0.1), "max_new_tokens": payload.get("max_new_tokens", 512), "compute_node_id": node["id"], @@ -1991,7 +2326,10 @@ async def model_eval_delete(task_id: str, current_user: dict = Depends(get_curre pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}") if pending: return pending - get_platform_store().delete_eval_task(task_id) + try: + get_platform_store().delete_eval_task(task_id) + except KeyError: + raise fail(404, "eval task not found") return ok({"deleted": task_id}) @@ -2247,6 +2585,15 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr "model_name_or_path": model_path, "template": item.get("template", "qwen"), } + try: + item_gpu_indices = _normalize_gpu_indices(item) + except ValueError as exc: + loaded_models.append({**item, "status": "error", "error": str(exc)}) + continue + if not item_gpu_indices: + loaded_models.append({**item, "status": "error", "error": "no GPU selected"}) + continue + load_payload["gpu_indices"] = item_gpu_indices if item.get("adapter_path"): load_payload["adapter_name_or_path"] = item["adapter_path"] if get_settings().compute_mode == "simulator": @@ -2255,14 +2602,28 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr # 只派发：HTTP 响应成功即视为已接受（节点会异步加载），loaded 字段忽略 item_dispatched = False errors = [] - for node in _candidate_online_nodes(store, preferred_node_id): + candidate_nodes = _candidate_online_nodes(store, preferred_node_id) + if preferred_node_id: + candidate_nodes = candidate_nodes[:1] + for node in candidate_nodes: try: + node_gpu_map = { + int(gpu.get("id", gpu.get("gpu_index", -1))): gpu + for gpu in store.gpus() + if gpu.get("node_id") == node["id"] + } + unavailable = [ + index for index in item_gpu_indices + if node_gpu_map.get(index, {}).get("status") != "idle" + ] + if unavailable: + raise RuntimeError(f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}") if get_settings().minio_enabled: await _wait_for_object_storage() client = ComputeNodeClient(node["api_base_url"]) await client.inference_load(load_payload) - store.mark_inference_loaded(node["id"]) - loaded_models.append({**item, "status": "starting", "node_id": node["id"], "node_name": node.get("name")}) + store.mark_inference_loaded(node["id"], item_gpu_indices) + loaded_models.append({**item, "gpu_indices": item_gpu_indices, "gpus": item_gpu_indices, "status": "starting", "node_id": node["id"], "node_name": node.get("name")}) item_dispatched = True break except Exception as exc: # noqa: BLE001 - try next candidate node @@ -2365,7 +2726,7 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[ # 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功 result = await client.inference_load(payload) if result.get("loaded") or result.get("status") in {"loading", "ready"}: - store.mark_inference_loaded(node["id"]) + store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload)) return ok(result) except Exception as exc: return ok({"loaded": False, "error": str(exc)}) @@ -2445,7 +2806,7 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), curren # 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功 result = await client.inference_load({**payload, "compute_node_id": node["id"]}) if result.get("loaded") or result.get("status") in {"loading", "ready"}: - store.mark_inference_loaded(node["id"]) + store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload)) return ok(result) except Exception as exc: return ok({"loaded": False, "error": str(exc)}) diff --git a/backend/app/core/audit.py b/backend/app/core/audit.py index 66544f2..6601c52 100644 --- a/backend/app/core/audit.py +++ b/backend/app/core/audit.py @@ -61,6 +61,7 @@ def audit_log( detail = _build_detail(detail_template, kwargs) _record_audit( action=action, + actor_id=_extract_actor_id(kwargs), target_type=target_type, target_id=target_id, detail=detail, @@ -87,6 +88,7 @@ def audit_log( detail = _build_detail(detail_template, kwargs) _record_audit( action=action, + actor_id=_extract_actor_id(kwargs), target_type=target_type, target_id=target_id, detail=detail, @@ -136,6 +138,7 @@ def _build_detail(template: str, kwargs: dict) -> str: def _record_audit( action: str, + actor_id: Optional[str], target_type: str, target_id: Optional[str], detail: str, @@ -149,6 +152,7 @@ def _record_audit( store = get_platform_store() store.record_audit( action=action, + actor_id=actor_id, target_type=target_type or None, target_id=target_id, detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}", @@ -157,6 +161,15 @@ def _record_audit( logger.error("写入审计日志失败 action=%s", action, exc_info=True) +def _extract_actor_id(kwargs: dict) -> Optional[str]: + """从 FastAPI 注入的当前用户中提取操作人 ID。""" + for key in ("current_user", "user"): + value = kwargs.get(key) + if isinstance(value, dict) and value.get("id"): + return str(value["id"]) + return None + + # ==================== 预定义的审计操作常量 ==================== class AuditActions: diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8d67558..c73ff23 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -68,6 +68,9 @@ class Settings: minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin") minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources") minio_secure: bool = _bool_env("MINIO_SECURE", False) + # Small text/data files stay inline in PostgreSQL to avoid unnecessary + # MinIO round trips. Larger files remain the shared canonical objects. + minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024) storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300) storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10) compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "") diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index 2a2f5dd..ce35551 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -454,15 +454,21 @@ class PlatformStore: self.ensure_seed_data() # Track which compute nodes have an active inference model loaded self._inference_nodes: set[str] = set() + self._inference_gpu_indexes: dict[str, set[int]] = {} self._last_runtime_refresh = 0.0 # ── inference node tracking ──────────────────────────────────── - def mark_inference_loaded(self, node_id: str) -> None: + def mark_inference_loaded(self, node_id: str, gpu_indexes: list[int] | None = None) -> None: self._inference_nodes.add(node_id) + if gpu_indexes is not None: + self._inference_gpu_indexes[node_id] = {int(item) for item in gpu_indexes} + else: + self._inference_gpu_indexes.pop(node_id, None) def mark_inference_unloaded(self, node_id: str) -> None: self._inference_nodes.discard(node_id) + self._inference_gpu_indexes.pop(node_id, None) def is_inference_loaded(self, node_id: str) -> bool: return node_id in self._inference_nodes @@ -545,6 +551,16 @@ class PlatformStore: "last_error": "TEXT", }, ) + self._ensure_columns( + conn, + "storage_objects", + {"metadata": "TEXT NOT NULL DEFAULT '{}'"}, + ) + self._ensure_columns( + conn, + "model_artifacts", + {"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"}, + ) schema_dir = Path(__file__).with_name("sql") for extra in ( "002_governance.sql", @@ -556,7 +572,17 @@ class PlatformStore: if extra_path.exists(): conn.executescript(extra_path.read_text(encoding="utf-8")) # data_convert_tasks 表补充 created_by 字段（用于数据隔离） - self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"}) + self._ensure_columns( + conn, + "data_convert_tasks", + { + "created_by": "TEXT", + "storage_backend": "TEXT NOT NULL DEFAULT 'minio'", + "output_storage_object_id": "TEXT", + "output_content": "TEXT", + }, + ) + self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"}) # 修复历史数据：将 data_convert_tasks.created_by 回填到关联的 datasets 记录 try: conn.execute(""" @@ -1254,6 +1280,13 @@ class PlatformStore: raise KeyError(artifact_id) return {**dict(row), "metadata": json_loads(row["metadata"], {})} + def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None: + with self.connect() as conn: + conn.execute( + "UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?", + (storage_object_id, artifact_id), + ) + def model_lineage(self, model_id: str) -> dict[str, Any]: with self.connect() as conn: parents = conn.execute( @@ -2221,7 +2254,7 @@ class PlatformStore: (str(validation_dataset_id),), ).fetchall(), ] - model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id + model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {} if not dataset or dataset.get("type") != "train" or dataset_metadata.get( "dataset_split" @@ -2482,12 +2515,18 @@ class PlatformStore: def eval_tasks(self) -> list[dict[str, Any]]: with self.connect() as conn: - rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall() + # 评测任务采用软删除，普通列表不得再次返回已删除记录。 + rows = conn.execute( + "SELECT * FROM eval_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC" + ).fetchall() return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows] def eval_task(self, task_id: str) -> dict[str, Any]: with self.connect() as conn: - row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone() + row = conn.execute( + "SELECT * FROM eval_tasks WHERE id=? AND deleted_at IS NULL", + (task_id,), + ).fetchone() if not row: raise KeyError(task_id) payload = self._enrich_eval_payload(conn, self._json_payload_row(row)) @@ -2558,7 +2597,13 @@ class PlatformStore: def delete_eval_task(self, task_id: str) -> None: with self.connect() as conn: - conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id)) + result = conn.execute( + "UPDATE eval_tasks SET deleted_at=?, deleted_by=? " + "WHERE id=? AND deleted_at IS NULL RETURNING id", + (utcnow(), "system", task_id), + ) + if not result.fetchone(): + raise KeyError(task_id) def running_eval_tasks(self) -> list[dict[str, Any]]: """Return eval tasks that have been submitted to a compute node and are still running.""" @@ -2766,7 +2811,35 @@ class PlatformStore: "SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')", (node_id,), ).fetchall() - return {int(row["gpu_index"]) for row in rows} + active = {int(row["gpu_index"]) for row in rows} + # Evaluation jobs use the same Compute ProcessManager GPU lock but do + # not have fine-tune allocation rows; derive their selected cards here + # so a training task cannot race onto an evaluation GPU. + for row in conn.execute( + "SELECT payload FROM eval_tasks WHERE status IN ('syncing','queued','running')" + ).fetchall(): + payload = json_loads(row["payload"], {}) + if payload.get("compute_node_id") != node_id: + continue + selected = payload.get("gpu_indices") or payload.get("gpus") + if selected is None and payload.get("gpu_id") is not None: + selected = [payload.get("gpu_id")] + active.update(int(item) for item in selected or []) + # Loaded inference models also reserve only their selected cards. + active.update(self._inference_gpu_indexes.get(node_id, set())) + for row in conn.execute("SELECT payload FROM compare_tasks").fetchall(): + payload = json_loads(row["payload"], {}) + load_status = payload.get("load_status") or {} + if isinstance(load_status, str): + load_status = json_loads(load_status, {}) + for item in load_status.get("loaded_models") or []: + if item.get("node_id") != node_id or item.get("status") not in {"starting", "ready", "running"}: + continue + selected = item.get("gpu_indices") or item.get("gpus") + if selected is None and item.get("gpu_id") is not None: + selected = [item.get("gpu_id")] + active.update(int(gpu) for gpu in selected or []) + return active def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]: rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall() @@ -3000,17 +3073,18 @@ class PlatformStore: """ INSERT INTO storage_objects (id, resource_type, resource_id, version_id, bucket, object_key, file_name, - content_type, checksum_sha256, byte_size, status, created_by, create_time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (resource_type, resource_id, version_id, object_key) DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type, checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size, - status=EXCLUDED.status, created_by=EXCLUDED.created_by + status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by """, ( object_id, payload["resource_type"], payload["resource_id"], payload["version_id"], payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"), payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"), + json_dumps(payload.get("metadata") or {}), payload.get("created_by"), payload.get("create_time") or utcnow(), ), ) @@ -3049,6 +3123,13 @@ class PlatformStore: ).fetchall() return [dict(row) for row in rows] + def storage_object(self, object_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone() + if not row: + raise KeyError(object_id) + return dict(row) + def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]: allowed = {"status", "checksum_sha256", "byte_size", "content_type"} fields = {key: value for key, value in payload.items() if key in allowed} @@ -3195,6 +3276,9 @@ class PlatformStore: # 推理模型占用算力节点同样计入：优先从 compare_tasks 持久化状态派生 # （重启后仍准确），并用内存标记兜底（直接 preload 的模型无 compare 记录） inference_node_ids = set(self._inference_nodes) + inference_gpu_indexes: dict[str, set[int]] = { + node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items() + } for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall(): ls = json_loads(ctr["payload"], {}).get("load_status") or {} if isinstance(ls, str): @@ -3204,8 +3288,13 @@ class PlatformStore: ls = {} for m in ls.get("loaded_models") or []: if m.get("status") in {"ready", "running"} and m.get("node_id"): - inference_node_ids.add(m["node_id"]) - for nid in inference_node_ids: + node_id = m["node_id"] + selected = m.get("gpu_indices") or m.get("gpus") + if selected: + inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected) + else: + inference_node_ids.add(node_id) + for nid in set(inference_node_ids) | set(inference_gpu_indexes): running_map[nid] = running_map.get(nid, 0) + 1 rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall() return [ @@ -3434,6 +3523,9 @@ class PlatformStore: # 推理模型占用的节点：优先从 compare_tasks 持久化状态派生（重启后仍准确）， # 内存标记兜底（直接 preload 的模型无 compare 记录） inference_node_ids = set(self._inference_nodes) + inference_gpu_indexes: dict[str, set[int]] = { + node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items() + } for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall(): ls = json_loads(ctr["payload"], {}).get("load_status") or {} if isinstance(ls, str): @@ -3443,7 +3535,12 @@ class PlatformStore: ls = {} for m in ls.get("loaded_models") or []: if m.get("status") in {"ready", "running"} and m.get("node_id"): - inference_node_ids.add(m["node_id"]) + node_id = m["node_id"] + selected = m.get("gpu_indices") or m.get("gpus") + if selected: + inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected) + else: + inference_node_ids.add(node_id) items = [] for row in rows: task = next( @@ -3459,7 +3556,10 @@ class PlatformStore: t for t in eval_running if t.get("compute_node_id") == row["node_id"] - and row["gpu_index"] == (int(t["gpu_id"]) if t.get("gpu_id") is not None else -1) + and row["gpu_index"] in { + int(item) + for item in (t.get("gpu_indices") or t.get("gpus") or ([t["gpu_id"]] if t.get("gpu_id") is not None else [])) + } ), None, ) @@ -3468,7 +3568,8 @@ class PlatformStore: eval_task is not None and eval_task.get("status") in {"syncing", "queued"} ) # Also mark GPU as busy if an inference model is loaded on this node - if row["node_id"] in inference_node_ids and not busy: + inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set()) + if inference_on_gpu and not busy: busy = True reserved = False memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1) @@ -3704,6 +3805,8 @@ class PlatformStore: actor_id: str | None = None, action: str | None = None, target_type: str | None = None, + target_id: str | None = None, + keyword: str | None = None, start_time: str | None = None, end_time: str | None = None, limit: int = 50, @@ -3726,6 +3829,13 @@ class PlatformStore: if target_type: clauses.append("target_type=?") params.append(target_type) + if target_id: + clauses.append("target_id=?") + params.append(target_id) + if keyword: + clauses.append("(target_id LIKE ? OR detail LIKE ?)") + pattern = f"%{keyword}%" + params.extend([pattern, pattern]) if start_time: clauses.append("time>=?") params.append(start_time) diff --git a/backend/app/db/sql/000_full_init.sql b/backend/app/db/sql/000_full_init.sql index 0aaf969..9f98994 100644 --- a/backend/app/db/sql/000_full_init.sql +++ b/backend/app/db/sql/000_full_init.sql @@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts ( path TEXT NOT NULL, size_bytes BIGINT NOT NULL DEFAULT 0, checksum_sha256 TEXT, + storage_object_id TEXT, + storage_backend TEXT NOT NULL DEFAULT 'minio', metadata TEXT NOT NULL, compute_job_id TEXT, create_time TEXT NOT NULL @@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects ( checksum_sha256 TEXT, byte_size BIGINT NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'pending', + metadata TEXT NOT NULL DEFAULT '{}', created_by TEXT, create_time TEXT NOT NULL, UNIQUE (resource_type, resource_id, version_id, object_key) @@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now(); ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now(); ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; +ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT; +ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio'; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}'; -- ---- 数据处理任务 / 源文件 / 预览 / 结果 ---- @@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks ( input_count INTEGER NOT NULL DEFAULT 0, output_count INTEGER NOT NULL DEFAULT 0, error_message TEXT, + output_content TEXT, create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')), update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')), deleted_at TIMESTAMPTZ ); CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status); CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC); +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio'; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT; -- ============================================================================ -- 七、种子数据：初始管理员 / 操作员 diff --git a/backend/app/modules/compute_gateway/client.py b/backend/app/modules/compute_gateway/client.py index 280d617..4287d5a 100644 --- a/backend/app/modules/compute_gateway/client.py +++ b/backend/app/modules/compute_gateway/client.py @@ -240,6 +240,10 @@ class ComputeNodeClient: async def inference_status(self) -> dict[str, Any]: return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT) + async def gpu_resources(self) -> list[dict[str, Any]]: + """Read live per-GPU metrics from this compute node.""" + return await self.gpus() + async def inference_unload(self) -> dict[str, Any]: return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT) diff --git a/backend/app/modules/compute_gateway/sync.py b/backend/app/modules/compute_gateway/sync.py index 1cac998..ce6183d 100644 --- a/backend/app/modules/compute_gateway/sync.py +++ b/backend/app/modules/compute_gateway/sync.py @@ -2,15 +2,75 @@ from __future__ import annotations import json import time +from pathlib import Path from typing import Any from app.db.platform_store import get_platform_store +from app.core.config import get_settings from app.modules.compute_gateway.client import ComputeNodeClient +from app.modules.storage.minio_store import get_object_storage # starting 状态允许的最大轮询次数（约 40 * 3s ≈ 2 分钟），超过即判定节点不可达 MAX_STARTING_ATTEMPTS = 40 +async def _archive_node_directory( + store: Any, + client: ComputeNodeClient, + node: dict[str, Any], + source_path: str, + resource_type: str, + resource_id: str, + version_id: str, + object_prefix: str, +) -> list[dict[str, Any]]: + """Archive a completed node directory to MinIO, preserving subdirectories.""" + data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve() + source = Path(source_path).resolve() + try: + relative_root = source.relative_to(data_root).as_posix() + except ValueError as exc: + raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc + queue = [relative_root] + archived: list[dict[str, Any]] = [] + while queue: + relative = queue.pop(0) + listing = await client.list_files(root="data", relative_path=relative) + for item in listing.get("items") or []: + item_relative = str(item.get("relative_path") or "") + if item.get("type") == "directory": + queue.append(item_relative) + continue + path = str(item.get("path") or "") + if not path: + continue + try: + relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix() + except ValueError: + relative_file = Path(str(item.get("name") or Path(path).name)).name + object_key = f"{object_prefix}/{version_id}/{relative_file}" + upload_url = get_object_storage().presigned_put(object_key) + result = await client.upload_file_to_url(path, upload_url, object_key) + metadata = get_object_storage().stat(object_key) + archived.append( + store.create_storage_object( + { + "resource_type": resource_type, + "resource_id": resource_id, + "version_id": version_id, + "bucket": get_object_storage().bucket, + "object_key": object_key, + "file_name": relative_file, + "content_type": "application/octet-stream", + "byte_size": metadata.get("byte_size") or result.get("byte_size") or 0, + "checksum_sha256": result.get("checksum_sha256") or "", + "status": "available", + } + ) + ) + return archived + + def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None: return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None) @@ -73,7 +133,8 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]: if node_status == "ready": item["status"] = "ready" item.pop("error", None) - store.mark_inference_loaded(node["id"]) + selected_gpus = item.get("gpu_indices") or item.get("gpus") + store.mark_inference_loaded(node["id"], selected_gpus) elif node_status == "error": item["status"] = "error" item["error"] = status.get("error") or "model load failed on compute node" @@ -138,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]: job["log_snippet"] = str(last_logs.get("content") or "")[:8192] except Exception: pass - synced.append(store.apply_compute_job(task["id"], job)) + updated_task = store.apply_compute_job(task["id"], job) + if ( + get_settings().minio_enabled + and job.get("status") == "completed" + and job.get("output_dir") + ): + trained_model = next( + ( + item + for item in store.trained_models() + if item.get("name") + == (task.get("output_model_name") or f"{task.get('name')}-lora") + ), + None, + ) + if trained_model: + archived = await _archive_node_directory( + store, + client, + node, + str(job["output_dir"]), + "trained_model", + str(trained_model["id"]), + str(job.get("id") or task.get("compute_job_id") or task["id"]), + f"trained_models/{trained_model['id']}", + ) + artifacts = store.model_artifacts(str(trained_model["id"])) + if archived and artifacts: + store.link_model_artifact_storage_object( + str(artifacts[0]["id"]), str(archived[0]["id"]) + ) + synced.append(updated_task) except Exception as exc: # noqa: BLE001 - keep polling other jobs failed.append({"task_id": task["id"], "error": str(exc)}) standalone_synced: list[dict[str, Any]] = [] @@ -150,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]: try: job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"]) standalone_synced.append(store.sync_model_merge_job(record["id"], job)) + if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"): + payload = (store.compute_job(record["id"]).get("payload") or {}) + trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "") + if trained_model_id: + trained_model = next( + (item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id), + None, + ) + if trained_model: + archived = await _archive_node_directory( + store, + ComputeNodeClient(node["api_base_url"], timeout=900), + node, + str(job["output_dir"]), + "trained_model", + str(trained_model["id"]), + str(job.get("id") or record["id"]), + f"trained_models/{trained_model['id']}", + ) + artifacts = store.model_artifacts(str(trained_model["id"])) + if archived and artifacts: + store.link_model_artifact_storage_object( + str(artifacts[0]["id"]), str(archived[0]["id"]) + ) except Exception as exc: # noqa: BLE001 - keep polling other jobs failed.append({"job_id": record["id"], "error": str(exc)}) @@ -174,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]: except Exception: pass store.apply_eval_job_result(eval_task["id"], job, result_content) + if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"): + await _archive_node_directory( + store, + client, + node, + str(job["output_dir"]), + "eval", + str(eval_task["id"]), + str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]), + f"evaluations/{eval_task['id']}", + ) # 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记 eval_synced += 1 except Exception as exc: # noqa: BLE001 diff --git a/backend/app/modules/data_convert/router.py b/backend/app/modules/data_convert/router.py index 4ed11e0..2a364b6 100644 --- a/backend/app/modules/data_convert/router.py +++ b/backend/app/modules/data_convert/router.py @@ -1,17 +1,21 @@ from __future__ import annotations import json +import hashlib import os from pathlib import Path from typing import Any from fastapi import APIRouter, Body, Depends, File, UploadFile -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, Response from app.api.v1.endpoints.platform import ok, fail from app.core.auth import get_current_user, is_admin +from app.core.config import get_settings from app.core.op_log import op_log, OpModule, OpAction from app.db.platform_store import get_platform_store, new_id +from app.modules.storage.minio_store import get_object_storage +from app.modules.storage.policy import should_store_in_minio router = APIRouter(prefix="/data-convert", tags=["data-convert"]) @@ -56,6 +60,142 @@ def _output_dir(task_id: str) -> Path: return _task_dir(task_id) / "output" +def _minio_enabled() -> bool: + return bool(get_settings().minio_enabled) + + +def _input_object_key(task_id: str, name: str) -> str: + return f"data-convert/{task_id}/input/{Path(name).name}" + + +def _output_object_key(task_id: str, name: str) -> str: + return f"data-convert/{task_id}/output/{Path(name).name}" + + +def _task_objects(task_id: str) -> list[dict[str, Any]]: + return get_platform_store().storage_objects_for_resource("data_convert", task_id) + + +def _register_object( + task_id: str, + *, + version_id: str, + object_key: str, + file_name: str, + content_type: str, + content: bytes, + created_by: str | None, +) -> dict[str, Any]: + storage = get_object_storage() + uploaded = storage.put_bytes(object_key, content, content_type) + return get_platform_store().create_storage_object( + { + "resource_type": "data_convert", + "resource_id": task_id, + "version_id": version_id, + "bucket": uploaded["bucket"], + "object_key": object_key, + "file_name": file_name, + "content_type": content_type, + "byte_size": len(content), + "checksum_sha256": hashlib.sha256(content).hexdigest(), + "status": "available", + "created_by": created_by, + } + ) + + +def _input_objects(task_id: str) -> list[dict[str, Any]]: + prefix = f"data-convert/{task_id}/input/" + return sorted( + [item for item in _task_objects(task_id) if str(item.get("object_key") or "").startswith(prefix)], + key=lambda item: str(item.get("file_name") or item.get("object_key") or ""), + ) + + +def _output_object(task: dict[str, Any]) -> dict[str, Any] | None: + key = _output_object_key(task["id"], _safe_output_filename(task.get("output_filename"))) + return next((item for item in _task_objects(task["id"]) if item.get("object_key") == key), None) + + +def _read_output(task: dict[str, Any]) -> bytes | None: + if _minio_enabled(): + item = _output_object(task) + if item: + return get_object_storage().get_bytes(item["object_key"]) + inline = task.get("output_content") + return str(inline).encode("utf-8") if inline is not None else None + path = _task_output_path(task) + return path.read_bytes() if path.exists() else None + + +def _convert_from_minio(task: dict[str, Any], created_by: str | None) -> tuple[int, int, bytes]: + output_name = _safe_output_filename(task.get("output_filename")) + output_lines: list[str] = [] + input_count = 0 + output_count = 0 + for item in _input_objects(task["id"]): + input_count += 1 + raw = get_object_storage().get_bytes(item["object_key"]) + data = json.loads(raw.decode("utf-8")) + if isinstance(data, list): + records = data + elif isinstance(data, dict): + records = [data] + else: + raise ValueError(f"JSON must be object or array: {item.get('file_name')}") + for record in records: + output_lines.append(json.dumps(record, ensure_ascii=False) + "\n") + output_count += 1 + output = "".join(output_lines).encode("utf-8") + store = get_platform_store() + with store.connect() as conn: + if should_store_in_minio(len(output)): + output_object = _register_object( + task["id"], + version_id="output", + object_key=_output_object_key(task["id"], output_name), + file_name=output_name, + content_type="application/jsonl", + content=output, + created_by=created_by, + ) + conn.execute( + "UPDATE data_convert_tasks SET output_storage_object_id=%s, output_content=NULL, storage_backend='minio' WHERE id=%s", + (output_object["id"], task["id"]), + ) + else: + conn.execute( + "UPDATE data_convert_tasks SET output_storage_object_id=NULL, output_content=%s, storage_backend='database' WHERE id=%s", + (output.decode("utf-8"), task["id"]), + ) + return input_count, output_count, output + + +def _convert_from_local(task: dict[str, Any]) -> tuple[int, int, bytes]: + input_dir = _input_dir(task["id"]) + output_dir = _output_dir(task["id"]) + input_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = _task_output_path(task) + output_path.unlink(missing_ok=True) + input_count = 0 + output_count = 0 + with output_path.open("w", encoding="utf-8") as output_file: + for json_file in sorted(input_dir.iterdir()): + if not json_file.is_file() or not json_file.name.lower().endswith(".json"): + continue + input_count += 1 + data = json.loads(json_file.read_text(encoding="utf-8")) + records = data if isinstance(data, list) else [data] if isinstance(data, dict) else None + if records is None: + raise ValueError(f"JSON must be object or array: {json_file.name}") + for record in records: + output_file.write(json.dumps(record, ensure_ascii=False) + "\n") + output_count += 1 + return input_count, output_count, output_path.read_bytes() + + @router.get("") def list_tasks( page: int = 1, @@ -109,9 +249,10 @@ def create_task( "VALUES (%s, %s, %s, %s, %s)", (task_id, name, description, output_filename, user_id), ) - # 创建目录 - _input_dir(task_id).mkdir(parents=True, exist_ok=True) - _output_dir(task_id).mkdir(parents=True, exist_ok=True) + # MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。 + if not _minio_enabled(): + _input_dir(task_id).mkdir(parents=True, exist_ok=True) + _output_dir(task_id).mkdir(parents=True, exist_ok=True) return ok(_get_task(task_id)) @@ -123,13 +264,19 @@ def get_task( task = _get_task(task_id) if not task: raise fail(404, "task not found") - # 附加输入文件列表 - input_dir = _input_dir(task_id) + # 附加输入文件列表；旧任务没有对象记录时继续读取本地兼容目录。 files = [] - if input_dir.exists(): - for f in sorted(input_dir.iterdir()): - if f.is_file(): - files.append({"name": f.name, "size": f.stat().st_size}) + if _minio_enabled(): + files = [ + {"name": item.get("file_name") or Path(item["object_key"]).name, "size": item.get("byte_size") or 0} + for item in _input_objects(task_id) + ] + else: + input_dir = _input_dir(task_id) + if input_dir.exists(): + for f in sorted(input_dir.iterdir()): + if f.is_file(): + files.append({"name": f.name, "size": f.stat().st_size}) task["input_files"] = files return ok(task) @@ -146,16 +293,26 @@ async def upload_source_files( raise fail(404, "task not found") if task["status"] not in ("pending", "uploaded"): raise fail(400, "task is not editable") - input_dir = _input_dir(task_id) - input_dir.mkdir(parents=True, exist_ok=True) staged = [] for upload in files: name = Path(upload.filename or "input.json").name if not name.lower().endswith(".json"): raise fail(415, f"only JSON files are supported: {name}") - target = input_dir / name content = await upload.read() - target.write_bytes(content) + if _minio_enabled(): + _register_object( + task_id, + version_id=f"input-{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}", + object_key=_input_object_key(task_id, name), + file_name=name, + content_type=upload.content_type or "application/json", + content=content, + created_by=task.get("created_by") or current_user.get("id"), + ) + else: + input_dir = _input_dir(task_id) + input_dir.mkdir(parents=True, exist_ok=True) + (input_dir / name).write_bytes(content) staged.append({"name": name, "size": len(content)}) store = get_platform_store() # 标记上传完成 @@ -166,30 +323,12 @@ async def upload_source_files( ) # 自动转换并导入数据集 try: - output_dir = _output_dir(task_id) - output_dir.mkdir(parents=True, exist_ok=True) - output_path = _task_output_path(task) - # 清空旧输出（如果重新上传） - if output_path.exists(): - output_path.unlink() - input_count = 0 - output_count = 0 - for json_file in sorted(input_dir.iterdir()): - if not json_file.is_file() or not json_file.name.lower().endswith(".json"): - continue - input_count += 1 - with open(json_file, "r", encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, list): - records = data - elif isinstance(data, dict): - records = [data] - else: - raise ValueError(f"JSON must be object or array: {json_file.name}") - with open(output_path, "a", encoding="utf-8") as f: - for record in records: - f.write(json.dumps(record, ensure_ascii=False) + "\n") - output_count += 1 + if _minio_enabled(): + input_count, output_count, output = _convert_from_minio( + task, task.get("created_by") or current_user.get("id") + ) + else: + input_count, output_count, output = _convert_from_local(task) with store.connect() as conn: conn.execute( "UPDATE data_convert_tasks SET status='completed', " @@ -197,12 +336,12 @@ async def upload_source_files( (input_count, output_count, task_id), ) # 自动导入数据集 - content = output_path.read_text(encoding="utf-8") + content = output.decode("utf-8") size_bytes = len(content.encode("utf-8")) dataset = store.create_dataset({ "name": task["name"], "type": "train", - "storage_type": "local", + "storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"), "source": "upload", "task_id": task_id, "size": f"{size_bytes} B", @@ -212,7 +351,20 @@ async def upload_source_files( }) dataset_id = dataset["id"] with store.connect() as conn: - store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content) + dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content) + if should_store_in_minio(len(output)): + output_name = _safe_output_filename(task.get("output_filename")) + object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}" + uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl") + storage_object = store.create_storage_object({ + "resource_type": "dataset", "resource_id": dataset_id, + "version_id": dataset_file.get("active_version_id") or dataset_file["id"], + "bucket": uploaded["bucket"], "object_key": object_key, + "file_name": output_name, "content_type": "application/jsonl", + "byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(), + "status": "available", "created_by": task.get("created_by") or current_user.get("id"), + }) + store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"]) return ok({ "staged_files": staged, "auto_converted": True, @@ -248,28 +400,10 @@ def run_convert( (task_id,), ) try: - input_dir = _input_dir(task_id) - output_dir = _output_dir(task_id) - output_dir.mkdir(parents=True, exist_ok=True) - output_path = _task_output_path(task) - input_count = 0 - output_count = 0 - for json_file in sorted(input_dir.iterdir()): - if not json_file.is_file() or not json_file.name.lower().endswith(".json"): - continue - input_count += 1 - with open(json_file, "r", encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, list): - records = data - elif isinstance(data, dict): - records = [data] - else: - raise ValueError(f"JSON must be object or array: {json_file.name}") - with open(output_path, "a", encoding="utf-8") as f: - for record in records: - f.write(json.dumps(record, ensure_ascii=False) + "\n") - output_count += 1 + if _minio_enabled(): + input_count, output_count, _ = _convert_from_minio(task, task.get("created_by") or current_user.get("id")) + else: + input_count, output_count, _ = _convert_from_local(task) # 更新任务状态 with store.connect() as conn: conn.execute( @@ -297,11 +431,15 @@ def download_result( raise fail(404, "task not found") if task["status"] != "completed": raise fail(400, "task is not completed") - output_path = _task_output_path(task) - if not output_path.exists(): + output = _read_output(task) + if output is None: raise fail(404, "output file not found") + if _minio_enabled(): + return Response(content=output, media_type="application/octet-stream", headers={ + "Content-Disposition": f"attachment; filename={_safe_output_filename(task.get('output_filename'))}" + }) return FileResponse( - str(output_path), + str(_task_output_path(task)), media_type="application/octet-stream", filename=_safe_output_filename(task.get("output_filename")), ) @@ -319,10 +457,10 @@ def import_as_dataset( raise fail(404, "task not found") if task["status"] != "completed": raise fail(400, "task is not completed") - output_path = _task_output_path(task) - if not output_path.exists(): + output = _read_output(task) + if output is None: raise fail(404, "output file not found") - content = output_path.read_text(encoding="utf-8") + content = output.decode("utf-8") dataset_name = str(payload.get("name") or task["name"]).strip() description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip() size_bytes = len(content.encode("utf-8")) @@ -331,7 +469,7 @@ def import_as_dataset( dataset = store.create_dataset({ "name": dataset_name, "type": "train", - "storage_type": "local", + "storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"), "source": "upload", "task_id": task_id, "size": f"{size_bytes} B", @@ -341,7 +479,20 @@ def import_as_dataset( }) dataset_id = dataset["id"] with store.connect() as conn: - store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content) + dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content) + if should_store_in_minio(len(output)): + output_name = _safe_output_filename(task.get("output_filename")) + object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}" + uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl") + storage_object = store.create_storage_object({ + "resource_type": "dataset", "resource_id": dataset_id, + "version_id": dataset_file.get("active_version_id") or dataset_file["id"], + "bucket": uploaded["bucket"], "object_key": object_key, + "file_name": output_name, "content_type": "application/jsonl", + "byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(), + "status": "available", "created_by": task.get("created_by") or (current_user.get("id") if current_user else None), + }) + store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"]) return ok({"dataset_id": dataset_id, "name": dataset_name}) @@ -360,11 +511,19 @@ def delete_task( "UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s", (task_id,), ) - # 清理文件 - import shutil - task_dir = _task_dir(task_id) - if task_dir.exists(): - shutil.rmtree(task_dir, ignore_errors=True) + if _minio_enabled(): + for item in _task_objects(task_id): + try: + get_object_storage().delete(item["object_key"]) + store.update_storage_object(item["id"], {"status": "deleted"}) + except Exception: + pass + else: + # 旧兼容数据仍清理本地目录。 + import shutil + task_dir = _task_dir(task_id) + if task_dir.exists(): + shutil.rmtree(task_dir, ignore_errors=True) return ok({"deleted": task_id}) diff --git a/backend/app/modules/data_process/algorithms/embedding.py b/backend/app/modules/data_process/algorithms/embedding.py new file mode 100644 index 0000000..eb21d03 --- /dev/null +++ b/backend/app/modules/data_process/algorithms/embedding.py @@ -0,0 +1,24 @@ +"""数据处理算法 - 本地语义嵌入模型共享单例。""" + +from __future__ import annotations + +import os +from functools import lru_cache +from typing import Any + + +@lru_cache(maxsize=1) +def semantic_embedding_model() -> Any: + """加载本地嵌入模型，供语义分块与语义质量评分共用。 + + 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义判断。 + 返回 LlamaIndex BaseEmbedding，通过 ``get_text_embedding`` 使用。 + """ + + from llama_index.embeddings.huggingface import HuggingFaceEmbedding + + return HuggingFaceEmbedding( + model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"), + device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"), + trust_remote_code=False, + ) diff --git a/backend/app/modules/data_process/algorithms/parsers/office.py b/backend/app/modules/data_process/algorithms/parsers/office.py index ad3a00b..7cda3de 100644 --- a/backend/app/modules/data_process/algorithms/parsers/office.py +++ b/backend/app/modules/data_process/algorithms/parsers/office.py @@ -7,12 +7,13 @@ import re import unicodedata import zipfile import xml.etree.ElementTree as ET -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from pathlib import PurePosixPath from typing import Any from urllib.parse import unquote, urlsplit from docx import Document +from docx.oxml.ns import qn from docx.oxml.table import CT_Tbl from docx.oxml.text.paragraph import CT_P from docx.table import Table @@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None: except zipfile.BadZipFile as exc: raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc +def iter_document_blocks(parent: Any) -> Iterator[Any]: + """按文档顺序产出正文段落与表格，并下钻 SDT 内容控件。 + + Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里，只遍历 body + 直接子级会把这些段落整段丢掉。 + """ + + for child in parent.iterchildren(): + if isinstance(child, (CT_P, CT_Tbl)): + yield child + elif child.tag == qn("w:sdt"): + content = child.find(qn("w:sdtContent")) + if content is not None: + yield from iter_document_blocks(content) + def _extract_docx_text(raw: bytes) -> str: _validate_office_archive(raw, "docx") try: @@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str: parts: list[str] = [] total = 0 - for child in document.element.body.iterchildren(): + for child in iter_document_blocks(document.element.body): if isinstance(child, CT_P): total = _append_bounded_text(parts, Paragraph(child, document).text, total) continue diff --git a/backend/app/modules/data_process/algorithms/quality.py b/backend/app/modules/data_process/algorithms/quality.py index 78b2fca..308c244 100644 --- a/backend/app/modules/data_process/algorithms/quality.py +++ b/backend/app/modules/data_process/algorithms/quality.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import json +import math import re import unicodedata from collections import Counter @@ -341,3 +342,87 @@ def score_quality( flags=tuple(flags), fingerprint=fingerprint, ) + + +def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float: + if not left or not right or len(left) != len(right): + return 0.0 + dot = math.fsum(a * b for a, b in zip(left, right)) + norm_left = math.sqrt(math.fsum(a * a for a in left)) + norm_right = math.sqrt(math.fsum(b * b for b in right)) + if not norm_left or not norm_right: + return 0.0 + return dot / (norm_left * norm_right) + + +def semantic_quality_scores( + record: Mapping[str, Any], + *, + source_content: str = "", + embed_model: Any = None, +) -> dict[str, Any] | None: + """用本地嵌入向量计算语义相关性（0-100）。 + + 返回 ``question_answer``（问题↔答案）、``answer_source``（答案↔来源， + 无来源时缺省）与 ``overall``；嵌入模型不可用时返回 None 降级，不阻断流程。 + """ + + try: + if embed_model is None: + from .embedding import semantic_embedding_model + + embed_model = semantic_embedding_model() + if embed_model is None: + return None + + question = normalize_text( + " ".join( + str(record.get(field) or "") + for field in ("instruction", "input") + ) + ) + answer = normalize_text( + str(record.get("output") or "") or str(record.get("chosen") or "") + ) + source = normalize_text(source_content) + texts = [text for text in {question, answer, source} if text] + if not texts: + return None + vectors = {text: embed_model.get_text_embedding(text) for text in texts} + except Exception: + return None + + scores: dict[str, Any] = {} + if question and answer: + scores["question_answer"] = round( + 100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2 + ) + if answer and source: + scores["answer_source"] = round( + 100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2 + ) + if not scores: + return None + scores["overall"] = round(sum(scores.values()) / len(scores), 2) + return scores + + +def composite_overall( + *, + rule: float | None, + semantic: float | None = None, + judge: float | None = None, +) -> float: + """三层加权组合：规则 35% + 语义 20% + 评审 45%，缺失层自动重归一。""" + + if rule is None: + rule = 0.0 + if judge is not None and semantic is not None: + overall = rule * 0.35 + semantic * 0.20 + judge * 0.45 + elif semantic is not None: + overall = rule * 0.60 + semantic * 0.40 + elif judge is not None: + overall = rule * 0.55 + judge * 0.45 + else: + overall = rule + return round(max(0.0, min(100.0, overall)), 2) diff --git a/backend/app/modules/data_process/document_chunking.py b/backend/app/modules/data_process/document_chunking.py index 830c70a..b36816c 100644 --- a/backend/app/modules/data_process/document_chunking.py +++ b/backend/app/modules/data_process/document_chunking.py @@ -18,12 +18,19 @@ from llama_index.core.base.embeddings.base import BaseEmbedding from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter from app.modules.data_process.algorithms import normalize_text +from app.modules.data_process.algorithms.embedding import semantic_embedding_model ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"] _PAGE_FURNITURE = re.compile( r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$" ) +# Docling 的 markdown 序列化会给列表项补上自动编号，而 Word 的编号存放在 +# numbering.xml 中，python-docx 抽取的正文不含这些编号；紧凑匹配前剥掉 +# 行首编号，否则带列表的切片会整体定位失败。 +_LIST_MARKER_PREFIX = re.compile( + r"(?m)^[ \t>]*(?:(?:\d{1,3}[.)])+|\([a-zA-Z0-9]{1,3}\)|[a-zA-Z][.)]|[-*+•·])[ \t]+" +) _COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE) _CONVERTER_LOCK = threading.Lock() @@ -149,18 +156,6 @@ def chunk_fixed_text( return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap) -@lru_cache(maxsize=1) -def _semantic_embedding_model() -> BaseEmbedding: - # 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。 - from llama_index.embeddings.huggingface import HuggingFaceEmbedding - - return HuggingFaceEmbedding( - model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"), - device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"), - trust_remote_code=False, - ) - - def chunk_semantic_text( text: str, *, @@ -175,7 +170,7 @@ def chunk_semantic_text( if not normalized: return [] splitter = SemanticSplitterNodeParser.from_defaults( - embed_model=embed_model or _semantic_embedding_model(), + embed_model=embed_model or semantic_embedding_model(), breakpoint_percentile_threshold=breakpoint_percentile_threshold, buffer_size=1, sentence_splitter=_sentence_chunks, @@ -193,6 +188,7 @@ def chunk_semantic_text( if start is None: start = _locate_text(normalized, content, 0) if start is None: + result.append(_unlocated_chunk(content)) continue if len(_tokenizer().encode(content)) <= chunk_size: result.append(_make_text_chunk(normalized, start, start + len(content))) @@ -203,6 +199,7 @@ def chunk_semantic_text( chunk_overlap=chunk_overlap, ): if child.source_start is None or child.source_end is None: + result.append(_unlocated_chunk(child.original_content)) continue result.append( _make_text_chunk( @@ -235,6 +232,7 @@ def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]: if start is None: start = _locate_text(source_text, content, 0) if start is None: + chunks.append(_unlocated_chunk(content)) continue end = start + len(content) chunks.append(_make_text_chunk(source_text, start, end)) @@ -247,6 +245,20 @@ def _locate_text(source: str, content: str, start: int) -> int | None: return position if position >= 0 else None +def _unlocated_chunk(content: str) -> DocumentChunk: + """正文在源文本中定位失败时保底保留切片，只放弃行号信息。""" + + return DocumentChunk( + original_content=content, + contextualized_content=content, + source_start=None, + source_end=None, + source_start_line=None, + source_end_line=None, + token_count=len(_tokenizer().encode(content)), + ) + + def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk: content = source[start:end] return DocumentChunk( @@ -316,6 +328,14 @@ def _compact_with_offsets(value: str) -> tuple[str, list[int]]: return "".join(compact), offsets +def _expand_to_line_boundaries(source_text: str, start: int, end: int) -> tuple[int, int]: + while start > 0 and source_text[start - 1] not in "\r\n": + start -= 1 + while end < len(source_text) and source_text[end] not in "\r\n": + end += 1 + return start, end + + def _project_layout_span( source_text: str, content: str, @@ -324,21 +344,79 @@ def _project_layout_span( source_offsets: list[int], compact_start: int, ) -> tuple[int | None, int | None, int]: - compact_content, _ = _compact_with_offsets(content) - if len(compact_content) < 4: + for candidate in (content, _LIST_MARKER_PREFIX.sub("", content)): + compact_content, _ = _compact_with_offsets(candidate) + if len(compact_content) < 4: + continue + position = compact_source.find(compact_content, compact_start) + if position < 0: + position = compact_source.find(compact_content) + if position < 0: + continue + start, end = _expand_to_line_boundaries( + source_text, + source_offsets[position], + source_offsets[position + len(compact_content) - 1] + 1, + ) + # 重复内容回退匹配可能命中已消费的更早位置，游标只进不退， + # 避免后续切片跟着错位。 + return start, end, max(compact_start, position + len(compact_content)) + return _project_layout_span_by_anchors( + source_text, + content, + compact_source=compact_source, + source_offsets=source_offsets, + compact_start=compact_start, + ) + + +def _project_layout_span_by_anchors( + source_text: str, + content: str, + *, + compact_source: str, + source_offsets: list[int], + compact_start: int, +) -> tuple[int | None, int | None, int]: + """按行锚点顺序匹配，容忍切片里插入的重复表头等非连续内容。""" + + segments = [ + compact + for compact in ( + _compact_with_offsets(line)[0] + for line in _LIST_MARKER_PREFIX.sub("", content).split("\n") + ) + if len(compact) >= 6 + ] + if not segments: return None, None, compact_start - position = compact_source.find(compact_content, compact_start) - if position < 0: - position = compact_source.find(compact_content) - if position < 0: + total = sum(len(segment) for segment in segments) + + def match_from(cursor: int) -> tuple[list[tuple[int, int]], int]: + matched: list[tuple[int, int]] = [] + position = cursor + for segment in segments: + found = compact_source.find(segment, position) + if found < 0: + continue + matched.append((found, found + len(segment))) + position = found + len(segment) + return matched, sum(end - start for start, end in matched) + + matched, covered = match_from(compact_start) + if covered * 2 < total: + retried, retry_covered = match_from(0) + if retry_covered > covered: + matched, covered = retried, retry_covered + # 覆盖不足一半时宁可不定位，也不能给出错误的行号。 + if not matched or covered * 2 < total: return None, None, compact_start - start = source_offsets[position] - end = source_offsets[position + len(compact_content) - 1] + 1 - while start > 0 and source_text[start - 1] not in "\r\n": - start -= 1 - while end < len(source_text) and source_text[end] not in "\r\n": - end += 1 - return start, end, position + len(compact_content) + start, end = _expand_to_line_boundaries( + source_text, + source_offsets[matched[0][0]], + source_offsets[matched[-1][1] - 1] + 1, + ) + return start, end, max(compact_start, matched[-1][1]) def chunk_layout_document( diff --git a/backend/app/modules/data_process/evaluation.py b/backend/app/modules/data_process/evaluation.py new file mode 100644 index 0000000..c9051f3 --- /dev/null +++ b/backend/app/modules/data_process/evaluation.py @@ -0,0 +1,313 @@ +"""数据处理 - 生成结果的多层质量评测。 + +三层体系：规则层（确定性规则分）+ 语义层（本地嵌入向量）+ 评审层 +（复用生成模型按 rubric 打分的 LLM-as-judge）。任一层失败自动降级， +评测永远返回可用结果，不阻断调用方流程。 +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from dataclasses import asdict +from datetime import UTC, datetime +from typing import Any + +import httpx + +from .algorithms import normalize_text, score_quality +from .algorithms.quality import composite_overall, semantic_quality_scores +from .generation import ( + ModelGenerationError, + _is_retryable_generation_error, + _json_payload, + _message_content, + chat_completions_url, +) + +logger = logging.getLogger(__name__) + +# 送入评审提示词的来源正文上限，避免超长切片挤占评分输出空间。 +_MAX_JUDGE_SOURCE_CHARS = 6000 + +_JUDGE_DIMENSIONS: dict[str, tuple[str, ...]] = { + "standard": ( + "faithfulness", + "correctness", + "clarity", + "completeness", + "alignment", + ), + "reasoning": ( + "faithfulness", + "correctness", + "clarity", + "completeness", + "alignment", + "reasoning_validity", + ), + "dpo": ( + "clarity", + "chosen_quality", + "rejected_quality", + "preference_reasonableness", + "faithfulness", + ), +} + +_DIMENSION_LABELS: dict[str, str] = { + "faithfulness": "忠实度", + "correctness": "正确性", + "clarity": "问题清晰度", + "completeness": "回答完整性", + "alignment": "指令对齐", + "reasoning_validity": "推理有效性", + "chosen_quality": "chosen 回答质量", + "rejected_quality": "rejected 回答质量", + "preference_reasonableness": "偏好区分合理性", +} + +_DIMENSION_RULES: dict[str, str] = { + "faithfulness": "忠实度：答案的全部陈述是否被参考资料支持，没有编造、没有引入资料之外的信息；未提供参考资料时按答案内部自洽性评估", + "correctness": "正确性：答案中的事实、概念与计算是否正确", + "clarity": "问题清晰度：问题是否清晰、自包含、无歧义，脱离上下文也能理解", + "completeness": "回答完整性：答案是否充分、直接地回应了问题的全部要点", + "alignment": "指令对齐：答案的形式与范围是否符合问题的要求（如格式、语言、范围限定）", + "reasoning_validity": "推理有效性：思维链步骤是否逻辑连贯、无跳步或循环论证，结论是否由推理过程自然得出", + "chosen_quality": "chosen 回答质量：更优回答的正确性、完整性与表述质量", + "rejected_quality": "rejected 回答质量：较差回答是否仍具备基本可读性，使对比训练有意义", + "preference_reasonableness": "偏好区分合理性：chosen 是否明显优于 rejected，且优劣差异与问题直接相关", +} + + +def _judge_system_prompt(output_type: str) -> str: + dimensions = _JUDGE_DIMENSIONS[output_type] + rules = "\n".join(f"- {_DIMENSION_RULES[name]}" for name in dimensions) + scores_schema = ", ".join(f'"{name}": 1-5' for name in dimensions) + return ( + "你是大模型训练数据质量评审员。严格依据用户消息中的【参考资料】评审这条训练数据，逐维度按 1-5 分打分：\n" + f"{rules}\n" + "评分锚点：5 分=完全符合维度描述；3 分=基本符合但有明显不足；1 分=严重不符合。\n" + "忠实度只依据参考资料与公认常识判断，无法得到支持的陈述必须扣分；不要因为答案冗长而加分。\n" + "只输出一个 JSON 对象，不要输出 JSON 之外的任何文字。\n" + '输出格式：{"scores": {' + scores_schema + '}, "reason": "一句话总评", "issues": ["具体问题，没有则为空数组"]}' + ) + + +def _judge_user_prompt(record: Mapping[str, Any], source_content: str) -> str: + source = normalize_text(source_content)[:_MAX_JUDGE_SOURCE_CHARS] or "（无参考资料）" + instruction = normalize_text(str(record.get("instruction") or "")) or "（空）" + input_text = normalize_text(str(record.get("input") or "")) + sections = [f"【参考资料】\n{source}", f"【问题】\n{instruction}"] + if input_text: + sections.append(f"【输入】\n{input_text}") + if record.get("chosen") or record.get("rejected"): + sections.append(f"【更优回答 chosen】\n{normalize_text(str(record.get('chosen') or '')) or '（空）'}") + sections.append(f"【较差回答 rejected】\n{normalize_text(str(record.get('rejected') or '')) or '（空）'}") + else: + output = normalize_text(str(record.get("output") or "")) + sections.append(f"【回答】\n{output or '（空）'}") + return "\n\n".join(sections) + + +def _validated_judge_payload(payload: Any, output_type: str) -> dict[str, Any]: + if not isinstance(payload, Mapping): + raise ModelGenerationError("评审响应不是 JSON 对象") + raw_scores = payload.get("scores") + if not isinstance(raw_scores, Mapping): + raise ModelGenerationError("评审响应缺少 scores 对象") + expected = _JUDGE_DIMENSIONS[output_type] + scores: dict[str, float] = {} + for name in expected: + value = raw_scores.get(name) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ModelGenerationError(f"评审响应缺少维度 {name} 的有效分数") + scores[name] = round(max(1.0, min(5.0, float(value))), 1) + issues = payload.get("issues") + if not isinstance(issues, list): + issues = [] + issues = [str(item)[:200] for item in issues if str(item).strip()][:8] + reason = normalize_text(str(payload.get("reason") or ""))[:300] + return { + "scores": scores, + "overall": round(sum(scores.values()) / len(scores) * 20, 2), + "reason": reason, + "issues": issues, + } + + +def _judge_record( + record: Mapping[str, Any], + source_content: str, + *, + model: Mapping[str, Any], + config: Mapping[str, Any], + client: httpx.Client | None, +) -> dict[str, Any] | None: + output_type = str(config.get("output_type") or "standard").strip().lower() + if output_type not in _JUDGE_DIMENSIONS: + output_type = "standard" + endpoint = chat_completions_url(str(model.get("api_url") or "")) + model_name = str(model.get("online_model_name") or model.get("name") or "").strip() + if not model_name: + raise ModelGenerationError("generation model name is required") + temperature = 0.1 + max_tokens = max(256, min(2048, int(config.get("max_tokens", 1024) or 1024))) + timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60) or 60))) + retries = max(0, min(5, int(config.get("generation_retries", 2) or 2))) + headers = {"Content-Type": "application/json"} + api_key = str(model.get("api_key") or "").strip() + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + request_payload: dict[str, Any] = { + "model": model_name, + "messages": [ + {"role": "system", "content": _judge_system_prompt(output_type)}, + {"role": "user", "content": _judge_user_prompt(record, source_content)}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + } + if bool(config.get("json_mode", False)): + request_payload["response_format"] = {"type": "json_object"} + + owns_client = client is None + http_client = client or httpx.Client(timeout=timeout) + try: + last_error: Exception | None = None + for _ in range(retries + 1): + try: + response = http_client.post(endpoint, headers=headers, json=request_payload) + response.raise_for_status() + body = response.json() + if not isinstance(body, Mapping): + raise ModelGenerationError("model response body must be a JSON object") + judged = _validated_judge_payload( + _json_payload(_message_content(body)), + output_type, + ) + judged["model"] = model_name + judged["output_type"] = output_type + return judged + except Exception as exc: + last_error = exc + if not _is_retryable_generation_error(exc): + break + raise ModelGenerationError(f"质量评审调用失败: {last_error}") + finally: + if owns_client: + http_client.close() + + +def evaluate_result_record( + record: Mapping[str, Any], + *, + source_content: str = "", + model: Mapping[str, Any] | None = None, + config: Mapping[str, Any] | None = None, + client: httpx.Client | None = None, + embed_model: Any = None, + min_output_length: int = 20, +) -> dict[str, Any]: + """对一条生成结果执行三层评测，返回可直接落库的 quality_score 字典。 + + 规则层字段保持原样平铺（向后兼容既有读取方）；新增 ``semantic``、 + ``judge``、``layers``、``evaluated`` 与组合 ``overall``。 + """ + + config_dict = dict(config or {}) + rule = score_quality( + record, + min_output_length=min_output_length, + source_content=source_content, + ) + quality: dict[str, Any] = asdict(rule) + + semantic = semantic_quality_scores( + record, + source_content=source_content, + embed_model=embed_model, + ) + judge: dict[str, Any] | None = None + if model is not None: + try: + judge = _judge_record( + record, + source_content, + model=model, + config=config_dict, + client=client, + ) + except Exception as exc: + logger.warning( + "data process judge evaluation degraded: %s", + exc, + ) + + layers = { + "rule": rule.overall, + "semantic": semantic.get("overall") if semantic else None, + "judge": judge.get("overall") if judge else None, + } + quality.update( + semantic=semantic, + judge=judge, + layers=layers, + evaluated=True, + evaluated_at=datetime.now(UTC).isoformat(), + overall=composite_overall( + rule=layers["rule"], + semantic=layers["semantic"], + judge=layers["judge"], + ), + ) + return quality + + +def reevaluate_edited_record( + record: Mapping[str, Any], + *, + source_content: str = "", + previous_quality: Mapping[str, Any] | None = None, + embed_model: Any = None, + min_output_length: int = 20, +) -> dict[str, Any]: + """手动编辑/恢复后重算规则与语义层，丢弃已过期的评审层。 + + 编辑会改变内容，旧的评审分不再可信；规则与语义层本地重算零成本。 + ``evaluated`` 标记沿用原值，保证已评测过的结果编辑后仍有可用分数。 + """ + + rule = score_quality( + record, + min_output_length=min_output_length, + source_content=source_content, + ) + quality: dict[str, Any] = asdict(rule) + semantic = semantic_quality_scores( + record, + source_content=source_content, + embed_model=embed_model, + ) + previous = dict(previous_quality or {}) + evaluated = bool(previous.get("evaluated")) + layers = { + "rule": rule.overall, + "semantic": semantic.get("overall") if semantic else None, + "judge": None, + } + quality.update( + semantic=semantic, + judge=None, + layers=layers, + evaluated=evaluated, + evaluated_at=( + datetime.now(UTC).isoformat() if evaluated else None + ), + overall=composite_overall( + rule=layers["rule"], + semantic=layers["semantic"], + ), + ) + return quality diff --git a/backend/app/modules/data_process/office_preview.py b/backend/app/modules/data_process/office_preview.py index fcfd0ed..e496b7a 100644 --- a/backend/app/modules/data_process/office_preview.py +++ b/backend/app/modules/data_process/office_preview.py @@ -11,6 +11,7 @@ import re from typing import Any from docx import Document +from docx.oxml.ns import qn from docx.oxml.table import CT_Tbl from docx.oxml.text.paragraph import CT_P from docx.table import Table @@ -27,6 +28,7 @@ from app.modules.data_process.algorithms import ( _xlsx_sheet_merge_ranges, normalize_text, ) +from app.modules.data_process.algorithms.parsers.office import iter_document_blocks MAX_DOCX_PREVIEW_BLOCKS = 2_000 MAX_XLSX_PREVIEW_ROWS = 200 @@ -49,12 +51,21 @@ def _docx_alignment(paragraph: Paragraph) -> str: def _docx_heading_level(paragraph: Paragraph) -> int | None: style = paragraph.style - if style is None: - return None - style_name = str(style.name or "") - style_id = str(style.style_id or "") + style_name = str(style.name or "") if style is not None else "" + style_id = str(style.style_id or "") if style is not None else "" match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE) - return int(match.group(1)) if match else None + if match: + return int(match.group(1)) + # Word 的目录和导航窗格依据大纲级别识别标题；未套标题样式但带 + # outlineLvl 的段落（如手工排版的编号小节）同样是标题。 + outline = paragraph._p.find(f"{qn('w:pPr')}/{qn('w:outlineLvl')}") + if outline is not None: + value = outline.get(qn("w:val")) + if value is not None and value.isdigit(): + level = int(value) + if 0 <= level <= 5: + return level + 1 + return None def build_docx_preview(raw: bytes) -> dict[str, Any]: @@ -84,7 +95,7 @@ def build_docx_preview(raw: bytes) -> dict[str, Any]: has_source_content = True return text, start, source_cursor - for child in document.element.body.iterchildren(): + for child in iter_document_blocks(document.element.body): if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS: truncated = True break diff --git a/backend/app/modules/data_process/storage.py b/backend/app/modules/data_process/storage.py index ce44a24..513f32a 100644 --- a/backend/app/modules/data_process/storage.py +++ b/backend/app/modules/data_process/storage.py @@ -1,4 +1,4 @@ -"""数据处理原始源文件的受控本地对象存储。""" +"""数据处理源文件的受控暂存与分层对象存储。""" from __future__ import annotations @@ -13,6 +13,9 @@ from pathlib import Path, PurePosixPath from typing import Iterable, Iterator from urllib.parse import quote, unquote, urlsplit +from app.core.config import get_settings +from app.modules.storage.minio_store import get_object_storage + class DataProcessStorageError(ValueError): """本地对象引用或文件系统状态不安全。""" @@ -68,7 +71,7 @@ def _safe_basename(value: str) -> str: class LocalDataProcessStorage: - """只允许访问配置根目录下的版本化原始文件。""" + """Stage locally, but publish and read authoritative source files from MinIO.""" def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None: configured = Path(root) if root is not None else _configured_storage_root() @@ -132,10 +135,7 @@ class LocalDataProcessStorage: f"v{version}", basename, ) - reference = ( - "local://data-process/" - f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}" - ) + reference = self._reference(task_id, source_file_id, version, basename) staged = StagedSourceObject(reference, temporary_path, relative_path) self._issued_staged_objects[temporary_path] = staged return staged @@ -168,6 +168,18 @@ class LocalDataProcessStorage: expected_task_id=expected_source_task_id, expected_source_file_id=expected_source_file_id, ) + if self._is_minio_reference(source_reference): + content = self.read(source_reference) + if content is None: + raise DataProcessStorageError("original source object is not available") + return self.stage_bytes( + batch_id=batch_id, + task_id=task_id, + source_file_id=source_file_id, + version=version, + name=basename, + content=content, + ) descriptor, source_info = self._open_read_descriptor(source_relative) os.close(descriptor) @@ -193,10 +205,7 @@ class LocalDataProcessStorage: f"v{version}", basename, ) - reference = ( - "local://data-process/" - f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}" - ) + reference = self._reference(task_id, source_file_id, version, basename) staged = StagedSourceObject(reference, temporary_path, relative_path) self._issued_staged_objects[temporary_path] = staged return staged @@ -212,14 +221,22 @@ class LocalDataProcessStorage: raise DataProcessStorageError("duplicate staged source object") seen_temporary_paths.add(item._temporary_path) for item in staged: - final_path = self._path_for_relative(item._relative_path) - self._ensure_directory(final_path.parent) - if final_path.exists() or final_path.is_symlink(): - raise DataProcessStorageError("source storage object already exists") - os.link(item._temporary_path, final_path, follow_symlinks=False) + if self._is_minio_reference(item.reference): + content = item._temporary_path.read_bytes() + get_object_storage().put_bytes( + self.object_key(item.reference), + content, + "application/octet-stream", + ) + elif not item.reference.startswith("db://data-process/"): + final_path = self._path_for_relative(item._relative_path) + self._ensure_directory(final_path.parent) + if final_path.exists() or final_path.is_symlink(): + raise DataProcessStorageError("source storage object already exists") + os.link(item._temporary_path, final_path, follow_symlinks=False) + self._fsync_directory(final_path.parent) published.append(item) item._temporary_path.unlink() - self._fsync_directory(final_path.parent) except Exception: for item in reversed(published): try: @@ -258,7 +275,13 @@ class LocalDataProcessStorage: raise first_error def read(self, reference: str) -> bytes | None: - """读取 local 引用；旧 ``db://`` 对象返回 ``None`` 由数据库正文兜底。""" + """Read a MinIO object or legacy local reference.""" + + if self._is_minio_reference(reference): + try: + return get_object_storage().get_bytes(self.object_key(reference)) + except Exception as exc: # noqa: BLE001 - normalize object-not-found for callers + raise DataProcessStorageError("source storage object does not exist") from exc relative_path = self._relative_from_reference(reference) if relative_path is None: @@ -276,6 +299,11 @@ class LocalDataProcessStorage: ) -> int | None: """返回受控 local 对象大小；旧 ``db://`` 对象没有原始文件。""" + if self._is_minio_reference(reference): + try: + return int(get_object_storage().stat(self.object_key(reference)).get("byte_size") or 0) + except Exception as exc: # noqa: BLE001 + raise DataProcessStorageError("source storage object does not exist") from exc relative_path = self._relative_from_reference(reference) if relative_path is None: return None @@ -301,6 +329,15 @@ class LocalDataProcessStorage: ) -> Iterator[bytes]: """按范围流式读取原始文件，避免 PDF 预览把大文件整体载入内存。""" + if self._is_minio_reference(reference): + content = self.read(reference) or b"" + if start < 0 or expected_size != len(content) or start > expected_size: + raise DataProcessStorageError("source object size does not match metadata") + remaining = expected_size - start if length is None else length + if remaining < 0 or start + remaining > expected_size: + raise DataProcessStorageError("invalid source byte range") + yield content[start : start + remaining] + return relative_path = self._relative_from_reference(reference) if relative_path is None: raise DataProcessStorageError("original source object is not available") @@ -337,6 +374,12 @@ class LocalDataProcessStorage: ) -> bool: """校验 local 引用归属；旧 ``db://`` 引用无需文件系统处理。""" + if self._is_minio_reference(reference): + self._assert_minio_owner(reference, expected_task_id, expected_source_file_id) + return True + if str(reference or "").startswith("db://data-process/"): + self._assert_database_owner(reference, expected_task_id, expected_source_file_id) + return True relative_path = self._relative_from_reference(reference) if relative_path is None: return False @@ -382,6 +425,19 @@ class LocalDataProcessStorage: ) -> bool: """删除受控 local 对象；旧 ``db://`` 引用保持不变。""" + if self._is_minio_reference(reference): + if (expected_task_id is None) != (expected_source_file_id is None): + raise DataProcessStorageError("both expected storage owner fields are required") + if expected_task_id is not None and expected_source_file_id is not None: + self._assert_minio_owner(reference, expected_task_id, expected_source_file_id) + get_object_storage().delete(self.object_key(reference)) + return True + if str(reference or "").startswith("db://data-process/"): + if (expected_task_id is None) != (expected_source_file_id is None): + raise DataProcessStorageError("both expected storage owner fields are required") + if expected_task_id is not None and expected_source_file_id is not None: + self._assert_database_owner(reference, expected_task_id, expected_source_file_id) + return False relative_path = self._relative_from_reference(reference) if relative_path is None: return False @@ -426,7 +482,7 @@ class LocalDataProcessStorage: if reference.startswith("db://"): return None parsed = urlsplit(reference) - if parsed.scheme != "local" or parsed.netloc != "data-process": + if parsed.scheme not in {"local", "minio"} or parsed.netloc != "data-process": raise DataProcessStorageError("unsupported source storage reference") if parsed.query or parsed.fragment or "\\" in parsed.path: raise DataProcessStorageError("unsafe source storage reference") @@ -460,6 +516,39 @@ class LocalDataProcessStorage: basename = _safe_basename(decoded[3]) return PurePosixPath(task_id, source_file_id, f"v{version}", basename) + @staticmethod + def _is_minio_reference(reference: str) -> bool: + return str(reference or "").startswith("minio://data-process/") + + @staticmethod + def _reference(task_id: str, source_file_id: str, version: int, basename: str) -> str: + scheme = "minio" if get_settings().minio_enabled else "local" + return f"{scheme}://data-process/{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}" + + @staticmethod + def object_key(reference: str) -> str: + parsed = urlsplit(reference) + if parsed.scheme != "minio" or parsed.netloc != "data-process": + raise DataProcessStorageError("reference is not a MinIO source object") + return "data-process/" + parsed.path.lstrip("/") + + def _assert_minio_owner(self, reference: str, task_id: str, source_file_id: str) -> None: + relative = self._relative_from_reference(reference) + if relative is None: + raise DataProcessStorageError("invalid MinIO source reference") + self._assert_expected_owner(relative, expected_task_id=task_id, expected_source_file_id=source_file_id) + + @staticmethod + def _assert_database_owner(reference: str, task_id: str, source_file_id: str) -> None: + parsed = urlsplit(reference) + parts = parsed.path.lstrip("/").split("/") + if parsed.netloc != "data-process" or len(parts) != 3: + raise DataProcessStorageError("invalid database source reference") + expected_task_id = _safe_component(task_id, "expected task id") + expected_source_file_id = _safe_component(source_file_id, "expected source file id") + if tuple(parts[:2]) != (expected_task_id, expected_source_file_id) or parts[2] != "v1": + raise DataProcessStorageError("source storage object owner mismatch") + def _path_for_relative(self, relative_path: PurePosixPath) -> Path: if relative_path.is_absolute() or any( part in {"", ".", ".."} for part in relative_path.parts @@ -479,9 +568,22 @@ class LocalDataProcessStorage: raise DataProcessStorageError("invalid staged source object") if self._issued_staged_objects.get(item._temporary_path) is not item: raise DataProcessStorageError("staged source object was not issued by this storage") - expected_relative = self._relative_from_reference(item.reference) - if expected_relative is None or expected_relative != item._relative_path: - raise DataProcessStorageError("staged source object reference mismatch") + if item.reference.startswith("db://data-process/"): + parsed = urlsplit(item.reference) + parts = parsed.path.lstrip("/").split("/") + expected = item._relative_path.parts[:3] + if ( + parsed.netloc != "data-process" + or parsed.query + or parsed.fragment + or len(parts) != 3 + or tuple(parts) != expected + ): + raise DataProcessStorageError("staged source object reference mismatch") + else: + expected_relative = self._relative_from_reference(item.reference) + if expected_relative is None or expected_relative != item._relative_path: + raise DataProcessStorageError("staged source object reference mismatch") staging_root = self._root / ".staging" try: relative_temporary = item._temporary_path.relative_to(staging_root) diff --git a/backend/app/modules/data_process/store/base.py b/backend/app/modules/data_process/store/base.py index e7618bf..4784a7a 100644 --- a/backend/app/modules/data_process/store/base.py +++ b/backend/app/modules/data_process/store/base.py @@ -247,14 +247,19 @@ def _source_storage_descriptor( or f"db://data-process/{task_id}/{file_id}/v1" ) expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/" + expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/" expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1" if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len( expected_local_prefix ): storage_backend = "local" + elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len( + expected_minio_prefix + ): + storage_backend = "minio" elif storage_object_id == expected_database_reference: storage_backend = "database" - elif storage_object_id.startswith(("local://data-process/", "db://data-process/")): + elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")): raise DataProcessStoreError("source storage object owner mismatch") else: raise DataProcessStoreError("unsupported source storage object reference") diff --git a/backend/app/modules/data_process/store/datasets.py b/backend/app/modules/data_process/store/datasets.py index 4950549..edce977 100644 --- a/backend/app/modules/data_process/store/datasets.py +++ b/backend/app/modules/data_process/store/datasets.py @@ -11,6 +11,7 @@ import psycopg from app.core.config import get_settings from app.modules.storage.minio_store import get_object_storage +from app.modules.storage.policy import should_store_in_minio from .base import ( StoreBase, @@ -168,7 +169,11 @@ class DatasetsMixin: split_name: assignments.count(split_name) for split_name in split_order } split_specs: list[dict[str, Any]] = [] - use_minio = bool(get_settings().minio_enabled) + # Explicit local is retained for old callers/tests that request the + # legacy backend; all normal platform requests default to MinIO. + allow_minio = bool(get_settings().minio_enabled) and str( + payload.get("storage_type") or "minio" + ).lower() != "local" for split_name in split_order: split_records = [ (source_row, record) @@ -193,12 +198,15 @@ class DatasetsMixin: "storage_object_id": ( f"db://data-process/{task_id}/{file_id}/v1" ), + "store_in_minio": allow_minio and should_store_in_minio( + len(raw), content_type="application/jsonl", file_format="jsonl" + ), } ) source_result_ids = [row["id"] for row in rows] common_metadata = { "source": "data_process", - "storage_backend": "minio" if use_minio else "database", + "storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database", "source_task_id": task_id, "output_type": _task_output_type(task), "reasoning_detail": _task_reasoning_detail(task), @@ -273,7 +281,7 @@ class DatasetsMixin: split_name = str(spec["split"]) dataset_id = dataset_ids[split_name] storage_object_id = str(spec["storage_object_id"]) - if use_minio: + if spec["store_in_minio"]: file_name = f"{base_dataset_name}.{split_name}.jsonl" object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}" uploaded = get_object_storage().put_bytes( @@ -356,7 +364,7 @@ class DatasetsMixin: ( dataset_name, dataset_types[split_name], - "minio" if use_minio else (payload.get("storage_type") or "local"), + "minio" if spec["store_in_minio"] else "database", f"{len(spec['raw'])} B", len(spec["raw"]), len(spec["records"]), @@ -386,7 +394,7 @@ class DatasetsMixin: dataset_id, dataset_name, dataset_types[split_name], - "minio" if use_minio else (payload.get("storage_type") or "local"), + "minio" if spec["store_in_minio"] else "database", task_id, task_id, f"{len(spec['raw'])} B", @@ -405,7 +413,11 @@ class DatasetsMixin: ), ).fetchone() - file_metadata = {**dataset_metadata, "file_split": split_name} + file_metadata = { + **dataset_metadata, + "file_split": split_name, + "storage_backend": "minio" if spec["store_in_minio"] else "database", + } version = { "id": spec["version_id"], "version_no": 1, diff --git a/backend/app/modules/data_process/store/source_files.py b/backend/app/modules/data_process/store/source_files.py index 19d01ac..d27f4f3 100644 --- a/backend/app/modules/data_process/store/source_files.py +++ b/backend/app/modules/data_process/store/source_files.py @@ -137,7 +137,7 @@ class SourceFilesMixin: payload["record_count"], payload["file_format"], payload["checksum_sha256"], - payload["content"], + "" if str(storage_object_id or "").startswith("minio://") else payload["content"], str(payload["content"])[:2000], json_dumps(metadata_payload), task.get("tenant_id"), @@ -223,7 +223,17 @@ class SourceFilesMixin: ).fetchone() if not row: raise NotFoundError("source file not found") - return _decode_row(row) or {} + decoded = _decode_row(row) or {} + # New source files keep only a preview in PostgreSQL. Load the + # authoritative body from MinIO on demand for existing processing code. + reference = str(decoded.get("storage_object_id") or "") + if include_content and not decoded.get("content") and reference.startswith("minio://"): + from app.modules.data_process.storage import get_data_process_storage + + decoded["content"] = (get_data_process_storage().read(reference) or b"").decode( + "utf-8", errors="replace" + ) + return decoded def source_content_window( self, task_id: str, file_id: str, offset: int, limit: int diff --git a/backend/app/modules/storage/minio_store.py b/backend/app/modules/storage/minio_store.py index 89fc43c..25de9ba 100644 --- a/backend/app/modules/storage/minio_store.py +++ b/backend/app/modules/storage/minio_store.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import timedelta from functools import lru_cache from io import BytesIO +from collections.abc import Iterator from typing import Any from minio import Minio @@ -53,6 +54,64 @@ class MinioObjectStorage: except S3Error as exc: raise ObjectStorageError(str(exc)) from exc + def get_bytes(self, object_key: str) -> bytes: + """Read an object through the backend for small API responses and workers.""" + self._ensure_enabled() + self.ensure_bucket() + response = None + try: + response = self.client.get_object(self.bucket, object_key) + return response.read() + except S3Error as exc: + raise ObjectStorageError(str(exc)) from exc + finally: + if response is not None: + response.close() + response.release_conn() + + def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]: + """Stream an object without loading the complete file into memory.""" + self._ensure_enabled() + self.ensure_bucket() + response = None + try: + response = self.client.get_object(self.bucket, object_key) + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + except S3Error as exc: + raise ObjectStorageError(str(exc)) from exc + finally: + if response is not None: + response.close() + response.release_conn() + + def list_objects(self, prefix: str) -> list[dict[str, Any]]: + self._ensure_enabled() + self.ensure_bucket() + try: + return [ + { + "object_key": item.object_name, + "byte_size": item.size or 0, + "etag": item.etag, + "last_modified": item.last_modified.isoformat() if item.last_modified else None, + } + for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True) + ] + except S3Error as exc: + raise ObjectStorageError(str(exc)) from exc + + def delete(self, object_key: str) -> None: + self._ensure_enabled() + self.ensure_bucket() + try: + self.client.remove_object(self.bucket, object_key) + except S3Error as exc: + raise ObjectStorageError(str(exc)) from exc + def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]: self._ensure_enabled() self.ensure_bucket() diff --git a/backend/app/modules/storage/policy.py b/backend/app/modules/storage/policy.py new file mode 100644 index 0000000..d56e062 --- /dev/null +++ b/backend/app/modules/storage/policy.py @@ -0,0 +1,54 @@ +"""Storage placement rules shared by dataset and data-processing flows.""" + +from __future__ import annotations + +from app.core.config import get_settings + + +_INLINE_TEXT_FORMATS = { + "txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv", + "yaml", "yml", "xml", "html", "text/plain", "application/json", + "application/jsonl", "text/csv", +} + + +def should_store_in_minio( + size_bytes: int | None, + *, + content_type: str | None = None, + file_format: str | None = None, +) -> bool: + """Return whether a file is large enough to use the shared object store. + + Small files remain inline in PostgreSQL so page previews and metadata reads + do not pay an object-storage round trip. MinIO is still mandatory for + large files when it is enabled. + """ + + if not get_settings().minio_enabled: + return False + try: + size = max(0, int(size_bytes or 0)) + except (TypeError, ValueError): + size = 0 + if size > get_settings().minio_inline_max_bytes: + return True + # Binary office/document files remain in MinIO even when small because + # their original bytes cannot be safely represented by a text DB column. + normalized_format = str(file_format or "").strip().lower().lstrip(".") + normalized_type = str(content_type or "").strip().lower().split(";", 1)[0] + if normalized_format or normalized_type: + return not ( + normalized_format in _INLINE_TEXT_FORMATS + or normalized_type in _INLINE_TEXT_FORMATS + or normalized_type.startswith("text/") + ) + return False + + +def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str: + """Return ``minio`` or ``database`` for a managed file.""" + + if str(requested or "").strip().lower() == "local": + return "database" + return "minio" if should_store_in_minio(size_bytes) else "database" diff --git a/backend/app/modules/system/router.py b/backend/app/modules/system/router.py index 7ebd5aa..d6d57f4 100644 --- a/backend/app/modules/system/router.py +++ b/backend/app/modules/system/router.py @@ -56,13 +56,15 @@ def audit_logs( actor_id: str | None = Query(default=None, description="操作人 ID"), action: str | None = Query(default=None, description="动作类型"), target_type: str | None = Query(default=None, description="目标类型"), + target_id: str | None = Query(default=None, description="目标 ID"), + keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"), start_time: str | None = Query(default=None, description="ISO8601 起始时间"), end_time: str | None = Query(default=None, description="ISO8601 结束时间"), limit: int = Query(default=50, ge=1, le=200), offset: int = Query(default=0, ge=0), current_user: dict = Depends(get_current_user), ) -> dict: - """审计日志查询：按租户/项目/操作人/动作/目标类型/时间范围分页过滤。""" + """审计日志查询：按组织、操作人、动作、资源、关键字和时间范围分页过滤。""" if not is_admin(current_user): from app.api.v1.endpoints.platform import fail raise fail(403, "admin permission required") @@ -73,6 +75,8 @@ def audit_logs( actor_id=actor_id, action=action, target_type=target_type, + target_id=target_id, + keyword=keyword, start_time=start_time, end_time=end_time, limit=limit, @@ -88,6 +92,8 @@ def audit_logs_export( actor_id: str | None = Query(default=None, description="操作人 ID"), action: str | None = Query(default=None, description="动作类型"), target_type: str | None = Query(default=None, description="目标类型"), + target_id: str | None = Query(default=None, description="目标 ID"), + keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"), start_time: str | None = Query(default=None, description="ISO8601 起始时间"), end_time: str | None = Query(default=None, description="ISO8601 结束时间"), current_user: dict = Depends(get_current_user), @@ -103,6 +109,8 @@ def audit_logs_export( actor_id=actor_id, action=action, target_type=target_type, + target_id=target_id, + keyword=keyword, start_time=start_time, end_time=end_time, limit=10000, diff --git a/backend/app/schemas/data_process.py b/backend/app/schemas/data_process.py index aa3671f..4778ba0 100644 --- a/backend/app/schemas/data_process.py +++ b/backend/app/schemas/data_process.py @@ -384,6 +384,26 @@ class ResultBatchRegenerateRequest(BaseModel): return self +class ResultBatchEvaluateItem(BaseModel): + model_config = ConfigDict(extra="forbid") + + result_id: str = Field(min_length=1, max_length=100) + expected_updated_at: str = Field(min_length=1, max_length=100) + + +class ResultBatchEvaluateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[ResultBatchEvaluateItem] = Field(min_length=1, max_length=50) + + @model_validator(mode="after") + def validate_unique_results(self) -> ResultBatchEvaluateRequest: + result_ids = [item.result_id for item in self.items] + if len(result_ids) != len(set(result_ids)): + raise ValueError("result_id values must be unique") + return self + + class DatasetSplit(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/backend/tests/test_data_process_algorithms.py b/backend/tests/test_data_process_algorithms.py index cda83f5..cff6132 100644 --- a/backend/tests/test_data_process_algorithms.py +++ b/backend/tests/test_data_process_algorithms.py @@ -9,6 +9,8 @@ from decimal import Decimal import pytest from docx import Document +from docx.oxml import parse_xml +from docx.oxml.ns import nsdecls, qn from openpyxl import Workbook from pptx import Presentation from pptx.util import Inches @@ -38,6 +40,7 @@ from app.modules.data_process.algorithms import ( stable_split_assignments, structured_json_dumps, ) +from app.modules.data_process.office_preview import build_docx_preview def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]: @@ -318,6 +321,74 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None: assert parsed_pptx.records == () +def _docx_with_sdt_bytes() -> bytes: + """构造带 SDT 目录内容控件的 docx，段落顺序为正文、SDT、正文。""" + + document = Document() + document.add_paragraph("正文开头。") + sdt = parse_xml( + "" + "目录条目 第一章 概述" + "" % nsdecls("w") + ) + body = document.element.body + sect_pr = body.find(qn("w:sectPr")) + if sect_pr is not None: + sect_pr.addprevious(sdt) + else: + body.append(sdt) + document.add_paragraph("正文结尾。") + output = io.BytesIO() + document.save(output) + return output.getvalue() + + +def test_docx_extraction_and_preview_include_sdt_content() -> None: + raw = _docx_with_sdt_bytes() + + parsed = parse_text_content(raw, filename="toc.docx") + assert "目录条目 第一章 概述" in parsed.text + assert ( + parsed.text.index("正文开头。") + < parsed.text.index("目录条目 第一章 概述") + < parsed.text.index("正文结尾。") + ) + + preview = build_docx_preview(raw) + paragraph_texts = [ + block["text"] for block in preview["blocks"] if block["type"] == "paragraph" + ] + assert "目录条目 第一章 概述" in paragraph_texts + # 预览偏移必须与正文抽取规则一致，否则前端定位会错位。 + sdt_block = next( + block + for block in preview["blocks"] + if block.get("text") == "目录条目 第一章 概述" + ) + assert parsed.text[sdt_block["source_start"] : sdt_block["source_end"]] == ( + "目录条目 第一章 概述" + ) + + +def test_docx_preview_detects_outline_level_headings() -> None: + """未套标题样式但设了大纲级别的段落（Word 目录按此收录）也按标题渲染。""" + + document = Document() + document.add_heading("一级标题", level=1) + plain = document.add_paragraph("4.2.1 数据管理") + p_pr = plain._p.get_or_add_pPr() + p_pr.append(parse_xml("" % nsdecls("w"))) + document.add_paragraph("普通正文段落。") + output = io.BytesIO() + document.save(output) + + preview = build_docx_preview(output.getvalue()) + blocks = {b["text"]: b for b in preview["blocks"] if b["type"] == "paragraph"} + assert blocks["一级标题"]["heading_level"] == 1 + assert blocks["4.2.1 数据管理"]["heading_level"] == 3 + assert blocks["普通正文段落。"]["heading_level"] is None + + def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None: workbook = Workbook() first = workbook.active diff --git a/backend/tests/test_data_process_api.py b/backend/tests/test_data_process_api.py index 2170935..1e354ac 100644 --- a/backend/tests/test_data_process_api.py +++ b/backend/tests/test_data_process_api.py @@ -1691,6 +1691,223 @@ def test_batch_result_regeneration_rejects_locked_tasks_before_model_call( assert model_calls == 0 +def _prepare_evaluation_task( + client: TestClient, + store: Any, + tmp_path: Path, + *, + config: dict[str, Any] | None = None, +) -> str: + task_id = client.post( + "/modelTF/data-process", + json={ + "name": "数据评测", + "process_type": "structured", + "config": config or {"generation_model_id": "model-1", "output_type": "standard"}, + }, + ).json()["data"]["id"] + store.tasks[task_id].update( + status="completed", + progress=100, + workflow_step="results", + results_confirmed=False, + ) + store.models["model-1"] = { + "id": "model-1", + "online_model_name": "test-model", + "api_url": "https://model.example/v1", + "api_key": "secret", + } + store.previews[task_id] = [ + { + "id": "preview-1", + "status": "original", + "original_content": "申请编号用于唯一标识一笔报销申请。", + "edited_content": "申请编号用于唯一标识一笔报销申请。", + }, + { + "id": "preview-2", + "status": "original", + "original_content": "联系电话用于联系申请人。", + "edited_content": "联系电话用于联系申请人。", + }, + ] + store.results[task_id] = [ + { + "id": "result-1", + "preview_item_id": "preview-1", + "instruction": "申请编号有什么作用？", + "input": "", + "output": "申请编号用于唯一标识一笔报销申请。", + "original_instruction": "申请编号有什么作用？", + "original_input": "", + "original_output": "申请编号用于唯一标识一笔报销申请。", + "status": "valid", + "error": None, + "split": "train", + "quality_score": {}, + "updated_at": "2026-08-19T09:00:00Z", + }, + { + "id": "result-2", + "preview_item_id": "preview-2", + "instruction": "联系电话有什么作用？", + "input": "", + "output": "联系电话用于联系申请人。", + "original_instruction": "联系电话有什么作用？", + "original_input": "", + "original_output": "联系电话用于联系申请人。", + "status": "valid", + "error": None, + "split": "train", + "quality_score": {}, + "updated_at": "2026-08-19T09:00:01Z", + }, + ] + return task_id + + +def test_results_can_be_evaluated_in_batch_with_partial_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, store, _ = make_client(tmp_path) + task_id = _prepare_evaluation_task(client, store, tmp_path) + evaluation_calls: list[dict[str, Any]] = [] + + def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]: + evaluation_calls.append({"record": deepcopy(record), "kwargs": {k: v for k, v in kwargs.items() if k != "client"}}) + return { + "overall": 88.0, + "completeness": 100.0, + "length": 100.0, + "readability": 100.0, + "relevance": 90.0, + "duplicate": 100.0, + "is_valid": True, + "flags": [], + "fingerprint": "fp", + "semantic": {"question_answer": 80.0, "answer_source": 90.0, "overall": 85.0}, + "judge": {"scores": {"faithfulness": 5}, "overall": 90.0}, + "layers": {"rule": 92.0, "semantic": 85.0, "judge": 90.0}, + "evaluated": True, + } + + monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate) + response = client.post( + f"/modelTF/data-process/{task_id}/results/evaluate-batch", + json={ + "items": [ + {"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}, + # 乐观锁版本不匹配：该条应按冲突失败，另一条仍成功。 + {"result_id": "result-2", "expected_updated_at": "2026-08-18T00:00:00Z"}, + ], + }, + ) + + assert response.status_code == 200 + data = response.json()["data"] + assert data["total"] == 2 + assert data["succeeded"] == 1 + assert data["failed"] == 1 + assert [item["id"] for item in data["items"]] == ["result-1"] + assert data["failures"][0]["result_id"] == "result-2" + assert data["failures"][0]["code"] == "conflict" + + assert len(evaluation_calls) == 1 + assert evaluation_calls[0]["record"]["instruction"] == "申请编号有什么作用？" + assert evaluation_calls[0]["kwargs"]["model"]["online_model_name"] == "test-model" + assert evaluation_calls[0]["kwargs"]["source_content"] == "申请编号用于唯一标识一笔报销申请。" + + stored = store.results[task_id][0]["quality_score"] + assert stored["evaluated"] is True + assert stored["layers"]["judge"] == 90.0 + assert store.results[task_id][1]["quality_score"] == {} + + +def test_evaluation_without_generation_model_skips_judge_layer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, store, _ = make_client(tmp_path) + task_id = _prepare_evaluation_task(client, store, tmp_path, config={"output_type": "standard"}) + seen_models: list[Any] = [] + + def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]: + seen_models.append(kwargs.get("model")) + return { + "overall": 70.0, "is_valid": True, "flags": [], + "semantic": None, "judge": None, + "layers": {"rule": 70.0, "semantic": None, "judge": None}, + "evaluated": True, + } + + monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate) + response = client.post( + f"/modelTF/data-process/{task_id}/results/evaluate-batch", + json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]}, + ) + + assert response.status_code == 200 + assert response.json()["data"]["succeeded"] == 1 + # 任务未配置生成模型时，评审层收到的 model 必须是 None。 + assert seen_models == [None] + + +def test_evaluation_rejects_running_task( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, store, _ = make_client(tmp_path) + task_id = _prepare_evaluation_task(client, store, tmp_path) + store.tasks[task_id]["status"] = "running" + evaluation_calls = 0 + + def fake_evaluate(*args: Any, **kwargs: Any) -> dict[str, Any]: + nonlocal evaluation_calls + evaluation_calls += 1 + return {"overall": 0, "is_valid": True, "flags": []} + + monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate) + response = client.post( + f"/modelTF/data-process/{task_id}/results/evaluate-batch", + json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]}, + ) + + assert response.status_code == 409 + assert evaluation_calls == 0 + + +def test_result_update_preserves_evaluation_layers_and_drops_stale_judge( + tmp_path: Path, +) -> None: + client, store, _ = make_client(tmp_path) + task_id = _prepare_evaluation_task(client, store, tmp_path) + store.results[task_id][0]["quality_score"] = { + "overall": 90.0, + "is_valid": True, + "flags": [], + "semantic": {"overall": 85.0}, + "judge": {"overall": 92.0}, + "layers": {"rule": 90.0, "semantic": 85.0, "judge": 92.0}, + "evaluated": True, + } + + response = client.put( + f"/modelTF/data-process/{task_id}/results/result-1", + json={"output": "人工修正后的答案：申请编号唯一标识一笔报销申请。"}, + ) + + assert response.status_code == 200 + stored = store.results[task_id][0]["quality_score"] + # 手动编辑后：规则+语义重算，评审分丢弃，evaluated 标记保留。 + assert stored["evaluated"] is True + assert stored["judge"] is None + assert stored["layers"]["judge"] is None + assert stored["layers"]["rule"] is not None + assert stored["overall"] >= 0 + + def test_preview_build_replaces_only_selected_files_and_reports_file_counts( tmp_path: Path, ) -> None: diff --git a/backend/tests/test_data_process_evaluation.py b/backend/tests/test_data_process_evaluation.py new file mode 100644 index 0000000..eb974bf --- /dev/null +++ b/backend/tests/test_data_process_evaluation.py @@ -0,0 +1,284 @@ +"""数据评测模块（三层质量评分）的单元测试。""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest + +from app.modules.data_process.algorithms.quality import ( + composite_overall, + semantic_quality_scores, +) +from app.modules.data_process.evaluation import ( + _JUDGE_DIMENSIONS, + _judge_system_prompt, + _validated_judge_payload, + evaluate_result_record, + reevaluate_edited_record, +) +from app.modules.data_process.generation import ModelGenerationError + +RECORD = { + "instruction": "申请编号有什么作用？", + "input": "", + "output": "申请编号用于唯一标识一笔报销申请，便于跟踪审批状态。", +} +SOURCE = "报销系统中，申请编号用于唯一标识一笔报销申请，并支持跟踪审批状态。" + + +class _FakeEmbedModel: + """按关键词返回固定向量，模拟语义嵌入。""" + + def get_text_embedding(self, text: str) -> list[float]: + if "作用" in text or "编号" in text and "？" in text: + return [0.9, 0.1, 0.0] + if "申请编号" in text: + return [0.85, 0.2, 0.0] + return [0.0, 0.1, 0.9] + + +class _FailingEmbedModel: + def get_text_embedding(self, text: str) -> list[float]: + raise RuntimeError("embedding unavailable") + + +class _FakeResponse: + def __init__(self, payload: dict[str, Any]): + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return self._payload + + +class _FakeClient: + def __init__(self, content: str): + self._content = content + self.calls: list[dict[str, Any]] = [] + + def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse: + self.calls.append({"endpoint": endpoint, "payload": json}) + return _FakeResponse({ + "choices": [{"message": {"content": self._content}, "finish_reason": "stop"}], + }) + + def close(self) -> None: + return None + + +class _RaisingClient: + def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse: + raise httpx.ConnectError("model endpoint unreachable") + + def close(self) -> None: + return None + + +def _judge_content(scores: dict[str, float], **extra: Any) -> str: + return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra}) + + +def test_judge_system_prompt_covers_rubric_dimensions() -> None: + standard = _judge_system_prompt("standard") + for name in _JUDGE_DIMENSIONS["standard"]: + assert name in standard + assert "1-5" in standard + + dpo = _judge_system_prompt("dpo") + assert "chosen_quality" in dpo + assert "preference_reasonableness" in dpo + + reasoning = _judge_system_prompt("reasoning") + assert "reasoning_validity" in reasoning + + +def test_validated_judge_payload_converts_scores_to_overall() -> None: + judged = _validated_judge_payload( + { + "scores": { + "faithfulness": 5, + "correctness": 4, + "clarity": 4, + "completeness": 3, + "alignment": 4, + }, + "reason": "答案可靠", + "issues": ["回答略冗长"], + }, + "standard", + ) + + assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2) + assert judged["issues"] == ["回答略冗长"] + assert judged["reason"] == "答案可靠" + + +def test_validated_judge_payload_clamps_out_of_range_scores() -> None: + judged = _validated_judge_payload( + { + "scores": { + "faithfulness": 9, + "correctness": 4, + "clarity": 4, + "completeness": 0, + "alignment": 4, + }, + }, + "standard", + ) + + assert judged["scores"]["faithfulness"] == 5.0 + assert judged["scores"]["completeness"] == 1.0 + + +@pytest.mark.parametrize( + "scores", + [ + {"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3}, + { + "faithfulness": 5, + "correctness": 4, + "clarity": "high", + "completeness": 3, + "alignment": 4, + }, + ], +) +def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None: + with pytest.raises(ModelGenerationError): + _validated_judge_payload({"scores": scores}, "standard") + + +def test_semantic_quality_scores_uses_cosine_similarity() -> None: + scores = semantic_quality_scores( + RECORD, + source_content=SOURCE, + embed_model=_FakeEmbedModel(), + ) + + assert scores is not None + assert 0 < scores["question_answer"] <= 100 + assert 0 < scores["answer_source"] <= 100 + assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2) + + +def test_semantic_quality_scores_degrades_to_none_on_failure() -> None: + assert ( + semantic_quality_scores( + RECORD, + source_content=SOURCE, + embed_model=_FailingEmbedModel(), + ) + is None + ) + + +def test_composite_overall_weights_available_layers() -> None: + assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2) + assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2) + assert composite_overall(rule=80) == 80.0 + assert composite_overall(rule=None, judge=100) == 45.0 + + +def test_evaluate_result_record_combines_three_layers() -> None: + client = _FakeClient( + _judge_content({ + "faithfulness": 5, + "correctness": 4, + "clarity": 5, + "completeness": 4, + "alignment": 5, + }) + ) + quality = evaluate_result_record( + RECORD, + source_content=SOURCE, + model={"api_url": "https://model.example", "online_model_name": "judge-model"}, + config={"output_type": "standard", "generation_retries": 0}, + client=client, + embed_model=_FakeEmbedModel(), + ) + + assert quality["evaluated"] is True + assert quality["judge"] is not None + assert quality["judge"]["model"] == "judge-model" + assert quality["semantic"] is not None + assert quality["layers"]["judge"] == quality["judge"]["overall"] + assert quality["overall"] == composite_overall( + rule=quality["layers"]["rule"], + semantic=quality["layers"]["semantic"], + judge=quality["layers"]["judge"], + ) + # 评审提示词必须携带来源原文作为评分锚点（正文经 NFKC 归一化）。 + user_message = client.calls[0]["payload"]["messages"][1]["content"] + assert "申请编号用于唯一标识一笔报销" in user_message + + +def test_evaluate_result_record_degrades_when_model_fails() -> None: + quality = evaluate_result_record( + RECORD, + source_content=SOURCE, + model={"api_url": "https://model.example", "online_model_name": "judge-model"}, + config={"output_type": "standard", "generation_retries": 0}, + client=_RaisingClient(), + embed_model=_FakeEmbedModel(), + ) + + assert quality["judge"] is None + assert quality["layers"]["judge"] is None + assert quality["semantic"] is not None + assert quality["overall"] == composite_overall( + rule=quality["layers"]["rule"], + semantic=quality["layers"]["semantic"], + ) + + +def test_evaluate_result_record_without_model_runs_two_layers() -> None: + quality = evaluate_result_record( + RECORD, + source_content=SOURCE, + model=None, + embed_model=_FakeEmbedModel(), + ) + + assert quality["judge"] is None + assert quality["evaluated"] is True + assert quality["overall"] == composite_overall( + rule=quality["layers"]["rule"], + semantic=quality["layers"]["semantic"], + ) + + +def test_reevaluate_edited_record_drops_stale_judge() -> None: + previous = { + "evaluated": True, + "judge": {"overall": 90.0}, + } + quality = reevaluate_edited_record( + {**RECORD, "output": "编辑后的新答案内容，用于验证重评逻辑。"}, + source_content=SOURCE, + previous_quality=previous, + embed_model=_FakeEmbedModel(), + ) + + assert quality["evaluated"] is True + assert quality["judge"] is None + assert quality["layers"]["judge"] is None + assert quality["semantic"] is not None + + +def test_reevaluate_edited_record_keeps_unevaluated_state() -> None: + quality = reevaluate_edited_record( + RECORD, + source_content=SOURCE, + previous_quality={}, + embed_model=_FakeEmbedModel(), + ) + + assert quality["evaluated"] is False + assert quality["evaluated_at"] is None diff --git a/backend/tests/test_document_chunking.py b/backend/tests/test_document_chunking.py index f705104..29afe6d 100644 --- a/backend/tests/test_document_chunking.py +++ b/backend/tests/test_document_chunking.py @@ -9,6 +9,7 @@ from app.modules.data_process.document_chunking import ( DocumentChunk, _compact_with_offsets, _document_converter, + _nodes_to_chunks, _project_layout_span, chunk_fixed_text, chunk_semantic_text, @@ -103,6 +104,86 @@ def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() -> assert cursor > 0 +def test_layout_projection_tolerates_list_numbers_inserted_by_serializer() -> None: + # Word 自动编号存放在 numbering.xml，python-docx 抽取的正文没有编号， + # 而 Docling 序列化切片时会补上 "1. " 前缀，投影不能因此失败。 + source = "接入方式说明\n结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。" + compact_source, offsets = _compact_with_offsets(source) + start, end, cursor = _project_layout_span( + source, + "1. 结构化数据接入需要先配置连接地址。\n2. 非结构化接入需要上传文档。", + compact_source=compact_source, + source_offsets=offsets, + compact_start=0, + ) + + assert start is not None and end is not None + assert source[start:end] == "结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。" + assert cursor > 0 + + +def test_layout_projection_never_moves_cursor_backwards() -> None: + source = "重复段落内容。\n中间正文。\n重复段落内容。" + compact_source, offsets = _compact_with_offsets(source) + # 重复内容回退匹配命中已消费的更早位置时，游标必须保持不退。 + _, _, cursor = _project_layout_span( + source, + "重复段落内容。", + compact_source=compact_source, + source_offsets=offsets, + compact_start=compact_source.index("中间正文"), + ) + assert cursor >= compact_source.index("中间正文") + + +def test_layout_projection_falls_back_to_line_anchors_for_inserted_content() -> None: + # 表格跨切片时 Docling 会在续片中重复表头，正文不再是连续子串； + # 按行锚点匹配仍应定位到表头所在行到末行数据之间的连续区间。 + source = "表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二" + compact_source, offsets = _compact_with_offsets(source) + start, end, _ = _project_layout_span( + source, + "表头甲 表头乙\n第二行数据 说明二", + compact_source=compact_source, + source_offsets=offsets, + compact_start=0, + ) + + assert start is not None and end is not None + assert source[start:end] == ( + "表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二" + ) + + +def test_layout_projection_refuses_low_coverage_anchor_match() -> None: + source = "完全无关的正文内容甲。\n完全无关的正文内容乙。" + compact_source, offsets = _compact_with_offsets(source) + start, end, cursor = _project_layout_span( + source, + "找不到的数据行内容\n另一条找不到的数据行内容", + compact_source=compact_source, + source_offsets=offsets, + compact_start=0, + ) + + assert start is None + assert end is None + assert cursor == 0 + + +def test_text_splitter_keeps_chunks_that_cannot_be_located() -> None: + class FakeNode: + def get_content(self) -> str: + return "这段文本在源文本中不存在。" + + chunks = _nodes_to_chunks([FakeNode()], "完全不同的源文本。") + + assert len(chunks) == 1 + assert chunks[0].original_content == "这段文本在源文本中不存在。" + assert chunks[0].source_start is None + assert chunks[0].source_start_line is None + + def test_short_layout_chunk_merges_with_neighbor_and_keeps_page_provenance() -> None: source = "短标题\n这是一段足够长的正文内容，用于测试相邻切片合并。" chunks = [ diff --git a/compute/api/main.py b/compute/api/main.py index 6b13470..7931b9a 100644 --- a/compute/api/main.py +++ b/compute/api/main.py @@ -221,6 +221,35 @@ def create_app() -> FastAPI: return fallback_gpu_resources() items: list[dict[str, Any]] = [] + processes_by_uuid: dict[str, list[dict[str, Any]]] = {} + try: + process_result = subprocess.run( + [ + "nvidia-smi", + "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + for process_line in process_result.stdout.splitlines(): + process_parts = [part.strip() for part in process_line.split(",")] + if len(process_parts) < 4: + continue + process_uuid, pid, process_name, used_memory = process_parts[:4] + processes_by_uuid.setdefault(process_uuid, []).append( + { + "pid": int(_safe_float(pid)), + "name": process_name, + "memory_used_gb": round(_safe_float(used_memory) / 1024, 2), + } + ) + except Exception: + # Some driver/runtime combinations do not expose compute-apps; + # utilization and memory metrics remain useful without processes. + pass for line in result.stdout.splitlines(): parts = [part.strip() for part in line.split(",")] if len(parts) < 9: @@ -244,7 +273,7 @@ def create_app() -> FastAPI: "temperature": int(_safe_float(temp)), "power_w": round(_safe_float(power), 1), "power_limit_w": round(_safe_float(power_limit), 1), - "processes": [], + "processes": processes_by_uuid.get(uuid, []), } ) return items @@ -747,6 +776,23 @@ def create_app() -> FastAPI: infer_backend: str (default: "huggingface") infer_dtype: str (default: "auto") """ + requested_gpus = payload.get("gpu_indices") + if requested_gpus is None: + requested_gpus = payload.get("gpus") or [] + try: + requested_gpus = sorted({int(item) for item in requested_gpus}) + except (TypeError, ValueError) as exc: + raise HTTPException(status_code=400, detail=f"invalid GPU selection: {exc}") from exc + if any(item < 0 for item in requested_gpus): + raise HTTPException(status_code=400, detail="GPU index must be non-negative") + if requested_gpus: + known_gpus = {int(item.get("gpu_index", item.get("id", -1))) for item in gpu_resources()} + missing = sorted(set(requested_gpus) - known_gpus) + if missing: + raise HTTPException(status_code=409, detail=f"requested GPU not found: {missing}") + conflict = sorted(set(requested_gpus).intersection(process_manager.locked_gpus())) + if conflict: + raise HTTPException(status_code=409, detail=f"GPU already used by another compute job: {conflict}") session = get_inference_session() result = session.load( model_name_or_path=payload.get("model_name_or_path", ""), @@ -754,6 +800,7 @@ def create_app() -> FastAPI: template=payload.get("template", "qwen"), infer_backend=payload.get("infer_backend", "huggingface"), infer_dtype=payload.get("infer_dtype", "auto"), + gpu_indices=requested_gpus, ) return result diff --git a/compute/engines/llama_factory/inference.py b/compute/engines/llama_factory/inference.py index 72cad82..018ec0b 100644 --- a/compute/engines/llama_factory/inference.py +++ b/compute/engines/llama_factory/inference.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import threading import time import uuid @@ -36,6 +37,7 @@ class InferenceSession: self._generating_args: dict[str, Any] = {} self._model_name: str = "" self._adapter_path: str = "" + self._gpu_indices: list[int] = [] self._loaded_at: float = 0.0 @property @@ -53,6 +55,7 @@ class InferenceSession: "loaded_at": self._loaded_at, "request_id": self._request_id, "error": self._error, + "gpu_indices": list(self._gpu_indices), } def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]: @@ -87,8 +90,12 @@ class InferenceSession: template="qwen", infer_backend="huggingface", infer_dtype="auto", + gpu_indices=None, **kwargs, ) -> dict[str, Any]: + requested_gpus = sorted({int(item) for item in (gpu_indices or [])}) + if any(item < 0 for item in requested_gpus): + return {"loaded": False, "status": "error", "error": "GPU index must be non-negative"} with self._state_lock: if self._status == "loading": # A model is already loading — dedupe, reuse the same request id. @@ -97,6 +104,7 @@ class InferenceSession: self._status = "loading" self._error = "" self._request_id = uuid.uuid4().hex[:12] + self._gpu_indices = requested_gpus self._cancel_requested = False self._load_args = { "model_name_or_path": model_name_or_path, @@ -115,13 +123,21 @@ class InferenceSession: def _load_worker(self) -> None: """Build the ChatModel off the state lock so info() never blocks.""" + with self._state_lock: + requested_gpus = list(self._gpu_indices) model = None tokenizer = None generating_args: dict[str, Any] = {} error = "" + previous_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES") try: + # Set visibility before LLaMA-Factory/PyTorch initializes CUDA. + if requested_gpus: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(item) for item in requested_gpus) if self._teardown_old: self._release_model() + with self._state_lock: + self._gpu_indices = requested_gpus from llamafactory.chat import ChatModel from llamafactory.hparams import get_infer_args @@ -138,6 +154,12 @@ class InferenceSession: generating_args = dict(generating_args) except Exception as exc: # noqa: BLE001 - surface load failure via status error = str(exc) + finally: + if requested_gpus: + if previous_visible_devices is None: + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + else: + os.environ["CUDA_VISIBLE_DEVICES"] = previous_visible_devices with self._state_lock: if error: self._model = None @@ -152,6 +174,7 @@ class InferenceSession: self._model = None self._tokenizer = None self._status = "idle" + self._gpu_indices = [] return self._model = model self._tokenizer = tokenizer @@ -189,6 +212,7 @@ class InferenceSession: self._adapter_path = "" self._loaded_at = 0.0 self._error = "" + self._gpu_indices = [] def unload(self) -> dict[str, Any]: with self._state_lock: @@ -208,6 +232,7 @@ class InferenceSession: self._adapter_path = "" self._loaded_at = 0.0 self._error = "" + self._gpu_indices = [] return {"unloaded": True, "status": "idle"} def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]: diff --git a/docker/app/.env b/docker/app/.env index c5bc101..f2cf2c4 100644 --- a/docker/app/.env +++ b/docker/app/.env @@ -60,5 +60,6 @@ MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=change_me_minio_secret MINIO_BUCKET=yg-ft-resources MINIO_SECURE=false +MINIO_INLINE_MAX_BYTES=262144 STORAGE_WAIT_SECONDS=300 STORAGE_CHECK_INTERVAL_SECONDS=10 diff --git a/docker/app/docker-compose.yml b/docker/app/docker-compose.yml index 59279d1..341f8dc 100644 --- a/docker/app/docker-compose.yml +++ b/docker/app/docker-compose.yml @@ -70,6 +70,7 @@ services: MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources} MINIO_SECURE: ${MINIO_SECURE:-false} + MINIO_INLINE_MAX_BYTES: ${MINIO_INLINE_MAX_BYTES:-262144} STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300} STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10} DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process} diff --git a/docs/governance-user-guide.md b/docs/governance-user-guide.md index 18afbf8..1706626 100644 --- a/docs/governance-user-guide.md +++ b/docs/governance-user-guide.md @@ -1,9 +1,10 @@ # 平台治理功能使用指南 -> 版本：v1.1 -> 日期：2026-08-13 -> 适用版本：YG Zhilian v1.0+ -> 更新说明：移除页面权限码设计，改为基于角色的简化权限模型 + +> 版本：v1.3 +> 日期：2026-08-19 +> 适用版本：YG Fine-Tune Platform v1.0+ +> 更新说明：合并组织权限、审批和运行日志入口；取消项目空间菜单但保留旧接口兼容 --- @@ -40,28 +41,27 @@ ### 1.3 入口在哪里？ -所有治理功能集中在左侧导航栏的 **「系统设置」** 和 **「平台治理」** 分组下： +治理、组织和运维功能按职责分布在左侧导航栏的 **「平台治理」**、**「系统设置」** 和 **「算力资源」** 分组下： ``` -系统设置 -├── 用户设置 ← 用户 CRUD + 角色权限 + 密码管理（仅 admin） -├── 平台性能 ← 系统监控 -└── 查看日志 ← 日志查看 - 平台治理 -├── 租户管理 ← 组织/团队（仅 admin） -├── 项目空间 ← 项目级资源隔离（仅 admin） -├── 审批模板 ← 定义哪些操作需要审批（仅 admin） -├── 审批中心 ← 处理待审批请求（仅 admin） -└── 审计日志 ← 查看所有操作记录（仅 admin） +├── 组织与权限 ← 用户与角色、租户与配额（仅 admin） +├── 资源授权 ← 数据集、模型等资源授权（仅 admin） +└── 审批中心 ← 待审批请求与审批策略（仅 admin） + +系统设置 +├── 平台性能 ← 系统资源监控 +└── 运行日志 ← 运行日志、审计记录、操作诊断 算力资源 └── 算力节点 ← GPU 分配与管理（仅 admin） ``` -> ⚠️ 以上菜单**只有 admin 用户能看到**。普通用户登录后不会出现这些入口。 +> ⚠️ 平台治理、资源授权、审批中心和算力节点菜单仅 admin 用户能看到。运行日志入口继续沿用原权限，普通用户可查看系统/训练日志；审计记录和操作诊断页签仅 admin 可见。 > -> **重要变更（v1.1）**：非 admin 用户**默认可以访问所有业务功能菜单**（模型训练、评测、推理、数据集、数据处理等），无需管理员单独分配权限。 +> **重要变更（v1.2）**：非 admin 用户**默认可以访问所有业务功能菜单**（模型训练、评测、推理、数据集、数据处理等），无需管理员单独分配权限；治理和资源管理入口仍仅 admin 可见。 + +> **当前菜单调整（v1.3）**：平台不再提供项目空间菜单和项目级操作入口。历史项目表、接口和旧地址仅作为兼容层保留，当前资源访问以用户所有权、租户边界（如启用）和资源 ACL 为准；新建业务资源不再要求项目字段。 --- @@ -73,24 +73,25 @@ | 用户类型 | 可见菜单 | 说明 | |---------|---------|------| -| **admin（管理员）** | **全部菜单** | 包括用户设置、平台治理、算力节点等管理功能 | -| **非 admin 用户** | **除管理功能外的所有业务菜单** | 模型训练/评测/推理、数据集、数据处理、日志等 | +| **admin（管理员）** | **全部菜单** | 包括组织权限、审批、运行日志、平台治理和算力节点等管理功能 | +| **非 admin 用户** | **除管理功能外的所有业务菜单** | 模型训练/评测/推理、数据集、数据处理等 | > **核心原则**： > - 非 admin 用户**默认拥有所有业务功能的访问权限**，无需单独分配 > - 仅以下功能**仅管理员可见**： -> - `用户设置`（用户 CRUD、角色管理） -> - `平台治理`（租户管理、项目空间、审批模板/中心、审计日志） +> - `平台治理 - 组织与权限`（用户、角色、租户与配额） +> - `平台治理`（资源授权、审批中心） > - `算力节点`（GPU 分配） +> - `运行日志`中的审计记录和操作诊断 > > 资源级别的访问控制通过 **ACL（访问控制列表）** 实现，详见第 4 章。 ### 2.2 创建用户 -**路径**：`用户设置` → `创建用户` +**路径**：`平台治理` → `组织与权限` → `用户与角色` → `创建用户` 1. 以 admin 身份登录平台 -2. 进入「用户设置」页面 +2. 进入「组织与权限」页面的「用户与角色」页签 3. 点击右上角「创建用户」按钮 4. 填写信息： - **账号**：登录用户名（如 `zhangsan`） @@ -107,12 +108,10 @@ | 功能分组 | 包含菜单 | 路由前缀 | |---------|---------|----------| -| 系统设置 - 用户设置 | 用户列表、创建用户、重置密码 | `/user-settings` | -| 平台治理 - 租户管理 | 租户列表、配额设置 | `/tenants` | -| 平台治理 - 项目空间 | 项目列表、成员管理、ACL | `/projects` | -| 平台治理 - 审批模板 | 审批流程定义 | `/approval-templates` | -| 平台治理 - 审批中心 | 待审批请求处理 | `/approval-instances` | -| 平台治理 - 审计日志 | 操作记录查询与导出 | `/audit-logs` | +| 平台治理 - 组织与权限 | 用户、角色、租户与配额 | `/organization` | +| 平台治理 - 资源授权 | 数据集、模型等资源 ACL | `/resource-acl` | +| 平台治理 - 审批中心 | 待审批请求、审批历史与策略 | `/approval-instances` | +| 系统设置 - 运行日志 | 系统/训练日志；管理员可查看审计记录、操作诊断 | `/logs` | | 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` | ### 2.4 重置用户密码 @@ -125,18 +124,18 @@ 3. 输入新密码，确认 **方式二：用户自行修改** -1. 用户登录后在「用户设置」页面点击「修改密码」按钮 +1. 用户登录后在「组织与权限」页面的「用户与角色」页签点击「修改密码」按钮 2. 输入旧密码 + 新密码（至少 6 位） 3. 确认修改 ### 2.5 删除用户 -**路径**：`用户设置` → 用户列表 → 操作列「删除」 +**路径**：`组织与权限` → `用户与角色` → 用户列表 → 操作列「删除」 > ⚠️ 删除用户时会**级联清理**其所有关联数据： > - 该用户创建的数据集、基座模型、微调产物、评测任务 > - 该用户的 ACL 授权记录、GPU 分配记录 -> - 该用户的审批实例、审计日志、项目成员关系、登录会话 +> - 该用户的审批实例、审计日志、历史项目成员关系、登录会话 > - **训练任务保留不删**（避免算力节点上的物理任务数据不一致） --- @@ -148,7 +147,7 @@ 当服务器有多张 GPU 卡（如 8×A800）时，需要指定**哪个用户能用哪张卡**： - 避免两个人同时选同一张卡导致训练冲突 -- 按团队/项目隔离算力资源 +- 按用户和租户边界隔离算力资源 - 控制每个用户的 GPU 配额 ### 3.2 分配 GPU（仅 admin） @@ -259,7 +258,6 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \ | 删除他人的数据集 | 非 admin 删除别人创建的数据集 | 创建审批实例 或 admin 直接执行 | | 删除他人的模型 | 非 admin 删除别人创建的模型 | 同上 | | 停止他人的训练任务 | 非 admin 停止别人发起的任务 | 同上 | -| 归档/删除项目空间 | 存在待审批变更时 | 拒绝执行 | **核心规则**：admin 做任何操作都直接执行（旁路）；普通用户操作他人资源时进入审批流程。 @@ -305,7 +303,7 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \ 3. 决策：「通过」或「拒绝」 4. 决策结果自动执行对应操作并记录审计日志 -**审批模板**（`平台治理` → `审批模板`）：定义每种操作需要几步审批、每步谁来审。默认模板都是单步（admin 审批即可）。 +**审批策略**（`平台治理` → `审批中心` → `审批策略`）：定义每种操作需要几步审批、每步谁来审。默认模板都是单步（admin 审批即可）。 --- @@ -324,24 +322,29 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \ ### 6.2 查询审计日志 -**路径**：`平台治理` → `审计日志` +**路径**：`系统设置` → `运行日志` → `审计记录` 支持筛选条件： | 筛选项 | 说明 | |---|---| -| 操作人 | 按用户 ID 过滤 | -| 动作类型 | 如 `user.create`, `dataset.delete`, `gpu.assign` 等 | -| 目标资源类型 | dataset / model / fine_tune_task 等 | +| 租户 | 按租户名称选择 | +| 操作人 | 按用户名称选择，不需要手工填写用户 ID | +| 动作类型 | 使用中文动作选择，例如创建数据集、删除模型、授予资源权限 | +| 目标资源类型 | 使用中文资源类型选择，例如数据集、模型、训练任务 | +| 关键词 | 模糊搜索目标 ID 或审计详情 | +| 目标 ID | 对指定资源 ID 进行精确查询 | | 时间范围 | 开始时间 ~ 结束时间 | +项目筛选已移除。底层接口仍兼容历史 `project_id` 参数，但当前平台不再提供项目菜单。 + ### 6.3 导出审计日志 -审计日志页面底部有「导出 CSV」按钮，导出的文件包含当前筛选条件下的全部记录，可用于合规审计或问题追溯。 +运行日志的「审计记录」页签提供「导出 CSV」按钮；「操作诊断」页签用于检索失败操作和接口耗时，可用于问题追溯。 ### 6.4 日志保留策略 -审计日志受**留存策略**控制（`平台治理` → 租户管理 → 绑定留存策略）。默认保留 30 天，超期自动清理。 +审计日志受**留存策略**控制（`平台治理` → `组织与权限` → `租户与配额`）。默认保留 30 天，超期自动清理。 --- @@ -351,7 +354,7 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \ 根据 v1.1 权限模型： 1. **业务菜单**（训练、评测、推理、数据集等）：普通用户**默认全部可见**，无需分配 -2. **管理菜单**（用户设置、租户管理、算力节点等）：**仅 admin 可见**，这是设计如此 +2. **管理菜单**（组织与权限、资源授权、审批中心、算力节点，以及运行日志中的审计/诊断页签）：**仅 admin 可见**，这是设计如此 如果普通用户看不到业务菜单，请检查： - 用户是否正常登录（token 是否有效） @@ -402,7 +405,7 @@ curl -H "Authorization: Bearer platform-token-admin" \ ### Q6: 用户忘记密码怎么办？ 两种方案： -1. **admin 重置**：在「用户设置」→ 用户列表 →「重置密码」 +1. **admin 重置**：在「组织与权限」→「用户与角色」→ 用户列表 →「重置密码」 2. **用户自助修改**：用户登录后点击「修改密码」（需知道旧密码） 如果是完全忘记且不是 admin，只能由 admin 重置。 diff --git a/docs/menu-functional-requirements.md b/docs/menu-functional-requirements.md index e73fdc3..06209fa 100644 --- a/docs/menu-functional-requirements.md +++ b/docs/menu-functional-requirements.md @@ -1,6 +1,8 @@ # 菜单与功能需求总览 > 本文根据当前前端侧边栏、路由、需求文档、接口文档、部署文档和 SQL 脚本整理。当前代码和 SQL 均按正式系统开发基线维护；Mock、Simulator 只能作为显式联调能力，不作为默认开发准则。 +> +> 当前治理版本取消“项目空间”菜单。历史项目表和接口仅保留兼容，不再作为前端业务入口或新资源的必填隔离层。 ## 1. 菜单分层 @@ -17,9 +19,11 @@ | 数据治理 | 数据处理 | `/data-process` | `data-process` | 前端页面已有，后端待完整实现 | 文档上传、切片预览、LLM 生成、结果编辑、发布数据集 | | 其他工具 | 数据类型转换 | `/data-convert` | `data-convert` | 前端页面已有，后端待实现 | JSON/JSONL/Markdown 等格式转换任务 | | 算力资源 | 算力节点 | `/compute` | `compute` | 已接入节点管理接口 | 节点地址、权重、标签、启用状态、GPU、队列、资源副本 | -| 系统设置 | 用户设置 | `/user-settings` | `user-settings` | 已接入基础用户接口 | 用户列表、创建用户、启停、页面权限 | +| 平台治理 | 组织与权限 | `/organization` | `user-settings` | 新增合并入口 | 用户与角色、租户与配额、密码管理 | +| 平台治理 | 资源授权 | `/resource-acl` | `user-settings` | 已接入 ACL 接口 | 数据集、模型等资源授权 | +| 平台治理 | 审批中心 | `/approval-instances` | `user-settings` | 新增合并入口 | 待审批请求、审批历史、审批策略 | | 系统设置 | 平台性能 | `/hardware` | `hardware` | 已有接口，需接真实采集 | CPU、内存、磁盘、GPU、进程、网络监控 | -| 系统设置 | 查看日志 | `/logs` | `logs` | 已有接口，需接真实日志文件 | 后端日志、error 日志、训练日志索引、日志内容查看 | +| 系统设置 | 运行日志 | `/logs` | `logs` | 新增合并入口 | 运行日志、训练日志；管理员可查看审计记录、操作诊断 | ### 1.2 当前二级和隐藏路由 @@ -41,19 +45,17 @@ | 数据集创建/编辑/预览 | `/dataset/create`、`/dataset/:id/edit`、`/dataset/:id/preview` | 数据集管理 | 数据集元数据、文件、版本与内容 | | 自定义工具 | `/tools`、`/tools/create`、`/tools/:id/edit` | 规划入口 | 路由存在，当前侧边栏未展示，后续可归入“其他工具” | | 算力子页 | `/compute/gpus`、`/compute/queue`、`/compute/nodes` | 算力节点 | 当前可作为页签或深链 | -| 创建用户/权限设置 | `/user-settings/create`、`/user-settings/:id/permission` | 用户设置 | 用户创建和页面权限 | +| 组织与权限内部页签 | `/user-settings`、`/tenants`、`/user-settings/create`、`/user-settings/:id/permission` | 平台治理 - 组织与权限 | 旧地址兼容，当前通过页签进入 | +| 项目旧地址 | `/projects`、`/projects/:id` | 兼容跳转 | 跳转到组织与权限，不再展示项目管理 | +| 审批策略旧地址 | `/approval-templates` | 兼容跳转 | 跳转到审批中心的策略页签 | +| 日志旧地址 | `/audit-logs`、`/operation-logs` | 兼容跳转 | 跳转到运行日志对应页签 | | 无权限页 | `/permission-denied` | 系统页 | 路由守卫无权限跳转 | -### 1.3 企业治理待补菜单 +### 1.3 后续治理扩展 | 建议菜单分组 | 菜单 | 建议路由 | 优先级 | 必要性 | | --- | --- | --- | --- | --- | -| 组织与项目 | 租户管理 | `/tenants`、`/tenants/:id` | P0 | 多租户隔离、配额、留存策略入口 | -| 组织与项目 | 项目空间 | `/projects`、`/projects/:id`、`/projects/:id/members` | P0 | 项目级模型/数据集/任务隔离 | -| 组织与项目 | 资源授权 | `/projects/:id/permissions` 或资源详情弹窗 | P0 | 模型/数据集/任务级 ACL | -| 治理中心 | 审批中心 | `/approvals`、`/approvals/:id` | P0 | 删除、发布、导出、停止他人任务等高风险动作 | -| 治理中心 | 审批设置 | `/approval-settings` | P1 | 审批模板、审批人规则、超时策略 | -| 治理中心 | 审计中心 | `/audit-logs`、`/login-logs`、`/download-logs` | P1 | 操作审计、登录审计、下载审计、导出 | +| 系统设置 | 运行日志扩展 | `/logs`、`/login-logs`、`/download-logs` | P1 | 增加登录审计、下载审计、导出审计维度 | | 运维中心 | 存储管理 | `/storage` | P1 | 本地磁盘占用、临时文件、checkpoint 清理、留存 | | 运维中心 | 训练引擎管理 | `/training-engines` | P2 | LLaMA-Factory 和后续引擎能力 schema、健康检查 | | 模型服务 | 模型服务治理 | `/model-services`、`/model-services/:id` | P1 | 测试/生产服务发布、调用统计、下线审批 | @@ -62,7 +64,7 @@ | 菜单/模块 | 主要接口 | 当前运行 SQL | 目标 SQL | | --- | --- | --- | --- | -| 登录、用户设置 | `/modelTF/login`、`/modelTF/me`、`/modelTF/users` | `users` | `users`、`login_sessions`、`permissions`、`role_permissions`、`user_permission_overrides` | +| 登录、组织与权限 | `/modelTF/login`、`/modelTF/me`、`/modelTF/users`、`/modelTF/tenants` | `users`、`tenants` | `users`、`login_sessions`、`permissions`、`role_permissions`、`user_permission_overrides`、`tenants` | | 服务看板 | `/modelTF/dashboard/overview`、`/modelTF/health` | 复用模型/数据集/任务/算力表 | `system_metric_snapshots`、`web_logs`、各业务表聚合 | | 模型管理 | `/modelTF/model-manage`、`/modelTF/model-manage/trained-models`、`/modelTF/model-manage/merge` | `models`、`trained_models` | `models`、`trained_models`、`storage_objects`、`local_import_jobs`、`resource_acl` | | 数据集管理 | `/modelTF/dataset-manage`、`/modelTF/dataset-manage/upload/{id}`、`/preview`、`/versions` | `datasets`、`dataset_files` | `datasets`、`dataset_files`、`dataset_file_versions`、`dataset_records`、`storage_objects` | @@ -70,12 +72,12 @@ | 训练日志 | `/modelTF/training-log-files`、`/modelTF/training-log-content` | 由任务表生成索引 | 日志文件元数据、`fine_tune_metrics`、`audit_logs` | | 算力节点 | `/modelTF/compute/nodes`、`/compute/gpus`、`/compute/queue`、`/compute/nodes/{id}/replicas` | `compute_nodes`、`gpus`、`resource_replicas`、`resource_sync_jobs` | `compute_nodes`、`gpu_devices`、`compute_node_engines`、`compute_jobs`、`resource_replicas`、`resource_sync_jobs` | | 平台性能 | `/modelTF/system-info`、`/modelTF/compute/gpus` | `gpus`、任务表 | `system_metric_snapshots`、`gpu_devices`、`compute_jobs` | -| 查看日志 | `/modelTF/log-files`、`/modelTF/log-content`、`/modelTF/web-log` | 文件日志 | `web_logs`、`audit_logs`，大日志进入日志平台 | +| 运行日志 | `/modelTF/log-files`、`/modelTF/log-content`、`/modelTF/web-log`、`/modelTF/audit-logs` | 文件日志 | `web_logs`、`audit_logs`，大日志进入日志平台 | | 模型评测 | `/modelTF/model-eval`、`/modelTF/dimension` | 当前运行 SQL 未覆盖 | `eval_tasks`、`eval_dimensions`、`eval_sample_results`、`eval_dimension_summaries` | | 模型推理/对比 | `/modelTF/model-compare`、`/modelTF/model-chat/*` | 当前运行 SQL 未覆盖 | `inference_tasks`、`inference_task_models`、`chat_sessions`、`chat_messages` | | 数据处理 | `/modelTF/data-process/*` | 当前运行 SQL 未覆盖 | `data_process_tasks`、`data_process_source_files`、`data_process_preview_items`、`data_process_results` | | 数据转换/自定义工具 | `/modelTF/data-convert/jobs`、`/modelTF/tools` | 当前运行 SQL 未覆盖 | `data_convert_jobs`、`custom_tools` | -| 租户/项目/资源授权 | `/modelTF/tenants`、`/modelTF/projects`、`/modelTF/resources/{type}/{id}/acl` | 当前运行 SQL 未覆盖 | `tenants`、`tenant_users`、`projects`、`project_members`、`resource_acl` | +| 租户/资源授权 | `/modelTF/tenants`、`/modelTF/resources/{type}/{id}/acl` | 当前运行 SQL 未覆盖 | `tenants`、`tenant_users`、`resource_acl`；`projects`、`project_members` 仅作兼容 | | 审批/审计/留存/配额 | `/modelTF/approvals`、`/modelTF/audit-logs`、`/modelTF/retention-policies`、`/modelTF/quotas/usage` | 当前运行 SQL 未覆盖 | `approval_templates`、`approval_instances`、`approval_steps`、`audit_logs`、`retention_policies`、`quotas`、`quota_usage` | ## 3. 文档和脚本检查结论 diff --git a/docs/permissions-design.md b/docs/permissions-design.md index 3797631..8af20be 100644 --- a/docs/permissions-design.md +++ b/docs/permissions-design.md @@ -4,6 +4,8 @@ > 日期：2026-08-02 > 状态：设计基线，供后端实现和前端联调参照 +> **当前菜单基线（2026-08-19）**：平台治理已取消“项目空间”作为用户可见菜单和新资源的业务隔离层。当前前端入口为“组织与权限、资源授权、审批中心”，系统设置下的“运行日志”承载运行日志、审计记录和操作诊断。`projects`、`project_members` 表及相关后端接口仅作历史兼容，不删除、不要求新建资源填写 `project_id`。 + --- ## 目录 @@ -129,7 +131,7 @@ | `compute` | `/compute` | 算力节点 | | `hardware` | `/hardware` | 平台性能 | | `logs` | `/logs`, `/training-log/:id` | 查看日志 | -| `user-settings` | `/user-settings`, `/tenants`, `/projects`, `/approvals`, `/audit-logs` | 系统设置与平台治理 | +| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances`, `/logs` | 平台治理和运行日志（管理员） | --- diff --git a/docs/platform-governance-menu-design.md b/docs/platform-governance-menu-design.md new file mode 100644 index 0000000..0336aeb --- /dev/null +++ b/docs/platform-governance-menu-design.md @@ -0,0 +1,120 @@ +# 平台治理菜单设计与开发计划 + +> 版本：v1.0 +> 日期：2026-08-19 +> 状态：按本文档实施 + +## 1. 设计结论 + +当前项目已经有用户所有权、资源 ACL、租户接口和审计接口，但项目隔离尚未真正落地。核心资源的 `project_id` 当前没有有效业务数据，训练、评测、推理和数据集创建流程也没有统一的项目上下文。 + +因此当前版本取消项目层级设计，资源权限统一采用： + +```text +用户所有权 + 资源 ACL + 租户边界（可选） +``` + +项目相关数据库表和后端接口暂不物理删除，仅作为历史兼容能力保留，后续不再新增项目数据，也不在前端提供项目入口。 + +## 2. 最终菜单 + +```text +平台治理 +├── 组织与权限 +├── 资源授权 +└── 审批中心 + +系统设置 +├── 平台性能 +└── 运行日志 + ├── 系统日志 + ├── 训练日志 + ├── 审计记录 + └── 操作诊断 + +算力资源 +└── 算力节点 +``` + +### 2.1 组织与权限 + +使用页签统一承载： + +- 用户与角色：用户 CRUD、启停、密码、角色。 +- 租户与配额：租户、GPU 配额、存储配额和资源数量配额。 + +用户、租户和配额仍使用独立表和接口，不把组织配额字段混入用户表。单租户部署时可默认停留在“用户与角色”页签。 + +### 2.2 资源授权 + +保留当前资源 ACL 能力，支持数据集、训练模型等资源的 `read/write/execute/download/delete/admin` 权限。项目不再作为授权前置条件。 + +### 2.3 审批中心 + +统一使用页签承载： + +- 待审批/审批历史。 +- 我的申请。 +- 审批策略，仅管理员可见。 + +“审批策略”不再作为独立一级菜单。 + +### 2.4 运行日志 + +在现有系统日志、训练日志基础上增加： + +- 审计记录：写操作、授权、审批、删除、导出等敏感操作。 +- 操作诊断：失败操作、错误类型和接口耗时。 + +“审计中心”不再作为独立菜单。旧的 `/audit-logs` 和 `/operation-logs` 地址保留重定向。 + +## 3. 兼容策略 + +| 原入口 | 新入口/处理方式 | +|---|---| +| `/user-settings` | 重定向到 `/organization?tab=users` | +| `/tenants` | 重定向到 `/organization?tab=tenants` | +| `/approval-templates` | 重定向到 `/approval-instances?tab=strategies` | +| `/audit-logs` | 重定向到 `/logs?tab=audit` | +| `/operation-logs` | 重定向到 `/logs?tab=operations` | +| `/projects` | 移除前端入口；旧地址重定向到组织与权限 | + +后端的租户、项目、审批、ACL、审计 API 暂不删除，保证已有脚本和历史客户端不立即失效。数据库不执行删表操作，也不新增项目字段迁移。 + +## 4. 开发计划 + +### 阶段一：导航和页面聚合 + +1. 新增“组织与权限”聚合页面。 +2. 新增“审批中心”聚合页面。 +3. 扩展“运行日志”页面，加入审计和操作诊断页签。 +4. 调整侧边栏，只展示最终菜单。 + +### 阶段二：兼容旧入口 + +1. 旧用户、租户、审批策略、审计和操作日志路由改为重定向。 +2. 保留原页面组件、API 和后端路由，避免历史调用失效。 +3. 项目路由不再作为业务入口，不再新增项目数据。 + +### 阶段三：权限和功能检查 + +1. 管理员可以访问组织、租户、配额、审批、审计和 ACL。 +2. 普通用户不能访问平台治理菜单；运行日志基础页签继续保持原有访问权限。 +3. 审批策略页签仅管理员可见。 +4. 审计和操作诊断仍保留管理员可见能力。 +5. 数据集、模型、训练、评测、推理继续使用用户所有权和 ACL，不增加项目选择器。 + +### 阶段四：验证 + +- `npm run build`。 +- 检查旧路由重定向。 +- 检查管理员菜单显示。 +- 检查非管理员权限拦截。 +- 检查 Backend 健康接口和前端静态资源。 + +## 5. 暂不处理事项 + +- 不删除 `projects`、`project_members` 表。 +- 不删除后端项目模块，避免历史数据和接口调用中断。 +- 不把租户配额字段直接合并到 `users` 表。 +- 不改变现有数据集、模型、训练、评测、推理的业务接口格式。 diff --git a/docs/当前项目开发进度.md b/docs/当前项目开发进度.md new file mode 100644 index 0000000..06d9aa0 --- /dev/null +++ b/docs/当前项目开发进度.md @@ -0,0 +1,347 @@ +# 当前项目开发进度 + +> 评估基线：2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。 +> +> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容，统一标记为“部分完成”或“未完成”。 + +## 一、项目定位与总体结论 + +当前项目是一个面向多用户、多算力节点的模型训练与推理平台，主要链路为： + +```text +Vue 前端 + | +FastAPI Backend API + |-- PostgreSQL：业务元数据、权限、任务状态、小型内容和预览数据 + |-- Redis：会话、限流、短期缓存和任务辅助状态 + |-- MinIO：模型、数据集、报告和大文件的统一对象存储 + |-- Compute API / Agent：训练、推理、评测、模型合并和 GPU 执行 + | +多台算力节点 +``` + +整体判断： + +| 范围 | 当前状态 | 结论 | +|---|---|---| +| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 | +| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 | +| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入，跨节点一致性和失败恢复仍需加强 | +| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现，但完整的租户/项目隔离尚未闭环 | +| MinIO 统一存储 | 部分完成 | 大文件和模型已接入，仍存在兼容性的本地路径和部分数据双写/回退路径 | +| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 | +| 前端体验 | 基本可用，需优化 | 构建问题已持续修复，但页面响应等待、首屏体积和部分错误提示仍需优化 | + +## 二、已完成的功能 + +### 2.1 平台基础与部署 + +- 已建立 Vue 3 + TypeScript + Vite 前端工程。 +- 已建立 FastAPI 后端服务，提供登录、平台管理和模型业务接口。 +- 已建立 Compute API / Agent，用于连接算力节点并执行训练、推理、评测和模型处理任务。 +- 已使用 PostgreSQL 保存核心业务数据，Redis 提供会话、限流和缓存能力。 +- 已增加 MinIO 服务及 Backend 的 MinIO 配置，支持和 Compute Agent 分离部署。 +- 已提供 `docker/app`、`docker/compute`、`docker/minio` 和 `docker/offline` 部署目录。 +- 已考虑后端、算力服务、MinIO 分布在不同服务器时使用独立网络；Compute 节点访问 MinIO 需要配置所有节点都能访问的固定 IP 或 DNS。 +- 离线部署目录已经同步后端、算力相关源码和初始化 SQL 的主要改造内容。 + +### 2.2 登录、用户和权限基础 + +- 用户登录、退出、当前用户信息和密码修改接口已经存在。 +- 已有 Token 会话、Redis 会话记录和登录限流逻辑。 +- 已建立用户、角色、权限码和角色权限关系。 +- 已实现管理员、普通用户等基础角色分层。 +- 已实现页面路由守卫、菜单过滤和前端按钮级权限的基础能力。 +- 已建立资源 ACL 管理页面和相关接口，可对用户或角色授予资源级权限。 +- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。 +- 已建立运行日志、审计日志查询页面及审计记录写入机制。 +- 已加入软删除相关字段和部分删除逻辑，避免直接物理删除业务资源。 + +### 2.3 算力节点与 GPU 资源 + +- 已实现算力节点的新增、编辑、启用、禁用、维护/删除、连通性测试和健康检查。 +- 已实现节点列表、节点详情、节点副本/同步状态和 Compute Agent 连接。 +- 已实现 GPU 信息发现、GPU 状态查询和队列查询。 +- 已建立 `gpu_allocations`、`gpu_assignments`、调度锁等资源分配表。 +- 训练任务已经支持选择调度节点和一张或多张 GPU，并在预检阶段校验资源可用性。 +- Compute Agent 已支持训练、评测、推理和缓存准备等任务接口。 +- 已存在资源副本和同步任务模型，用于记录节点侧资源同步状态。 + +### 2.4 数据集管理 + +- 已实现数据集创建、列表、详情、编辑、删除和文件上传。 +- 已实现数据集文件下载、预览、记录列表和数据记录编辑入口。 +- 已处理 JSON 与 JSONL 的记录数差异：JSON 数组按元素计数，JSONL 按有效行计数，避免把整个 JSON 文件误按行数统计。 +- 已提供数据集版本列表、版本详情、创建版本、切换当前激活版本和删除版本接口。 +- 已增加数据集文件、版本、数据记录等初始化表结构。 +- 已支持数据集文件在数据库小内容和 MinIO 大文件之间按策略存储。 +- 已在训练预检中检查数据集文件是否存在、是否可从 MinIO 获取以及是否能准备到目标算力节点。 + +### 2.5 数据处理与数据转换 + +- 已提供结构化数据、非结构化数据、外部数据源的处理创建流程。 +- 已实现源文件上传、预览、分片/切分、生成、质量检查、去重和结果管理等数据处理流程。 +- 已支持处理结果生成数据集或导入数据集版本。 +- 已建立数据处理任务、源文件、预览项、结果等数据表。 +- 已提供 JSON、JSONL 等数据格式转换页面和后端任务接口。 +- 已将数据转换输出接入 MinIO/数据库分层存储：小型文本结果可存数据库，大文件存 MinIO。 +- 已处理输出文件下载和转换结果元数据保存问题。 + +### 2.6 模型训练 + +- 已实现训练任务创建、配置预检、命令预览、启动、停止、重试和删除。 +- 已接入 LLaMA-Factory 等训练适配逻辑。 +- 已支持选择训练数据、基座模型、算力节点和 GPU。 +- 已实现训练日志获取、训练任务概览、诊断信息、检查点和训练指标查询。 +- 前端训练详情已经具备训练曲线解析和展示逻辑，日志轮询间隔已调整为 3 秒。 +- 已增加 GPU 详情展示入口，包括显存和利用率等 Compute Agent 上报信息。 +- 已支持训练任务的 MinIO 数据准备和目标算力节点缓存准备。 + +### 2.7 模型管理与权重合并 + +- 已实现在线模型/基座模型和训练模型的列表、创建、详情、用途修改和删除。 +- 已建立模型、训练模型、模型血缘、模型产物和导出任务相关表。 +- 已提供权重合并入口，能够根据训练任务准备基座模型和 Adapter，并提交 Compute Agent 执行合并。 +- 已增加模型产物和 MinIO 对象关联字段。 +- 合并结果能够在任务完成后归档到 MinIO 的设计和主要代码路径已经建立。 + +### 2.8 模型推理与模型对比 + +- 已实现推理模型列表、创建、详情和删除入口。 +- 已实现模型加载、卸载、服务启动、服务状态查询和对话调用。 +- 已实现模型对比任务及多模型聊天相关接口。 +- 已增加推理失败重试、停止和资源释放的处理路径。 +- 已支持根据页面选择的算力节点准备模型缓存，兼容训练时所选节点优先的业务要求。 +- Compute Agent 已提供本地推理会话和模型缓存状态接口。 + +### 2.9 模型评测 + +- 已实现评测任务列表、创建、详情和删除。 +- 已实现评测维度管理和评测规则配置页面。 +- 已建立评测任务、评测维度、对比任务等数据库表。 +- 已支持选择模型、数据集、评测维度、算力节点和 GPU 的基础流程。 +- 已接入 Compute Agent 执行评测任务，并保存评测结果和报告相关元数据。 + +### 2.10 数据存储策略 + +- 已建立 `storage_objects`、`storage_cache_jobs` 等 MinIO 元数据和缓存任务表。 +- 已建立 MinIO 对象上传、下载、预签名 URL 和节点缓存准备的主要接口。 +- 已采用分层策略： + - 小型 JSON、JSONL、CSV、任务参数快照、预览数据保留在数据库，降低频繁预览的 MinIO 延迟。 + - 模型权重、训练产物、评测报告和大文件使用 MinIO。 + - 小型 PDF、DOCX、XLSX 仍优先存 MinIO，以保留原始二进制文件内容。 +- 已增加 `data_convert_tasks.output_content`，用于保存小型转换结果，避免所有小结果都依赖 MinIO。 +- Backend 和离线包中的 `000_full_init.sql` 已同步，当前两份初始化脚本内容一致。 + +## 三、部分完成、仍需完善的功能 + +### 3.1 MinIO 统一数据源尚未完全闭环 + +当前 MinIO 已成为模型、大文件和跨节点资源的主存储方向，但仍保留以下兼容路径： + +- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。 +- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段，这是当前已确认的小文件性能策略，不是错误，但必须统一记录来源、大小、校验值和版本。 +- 大文件在部分代码路径中仍通过 `read()` 或 `put_bytes()` 一次性读入内存，未完成流式或分片上传。 +- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联，数据库没有直接外键，删除和数据一致性需要应用层保证。 +- 删除业务资源后，对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。 + +### 3.2 MinIO 预签名接口的权限边界需要加强 + +当前预签名接口已经存在，但 PUT 上传场景仍需要重点补强： + +- 需要根据资源类型和资源 ID 校验当前用户的写权限，而不应只校验读取权限。 +- 需要服务端生成并校验对象 Key，避免客户端任意写入其他用户或其他资源的对象路径。 +- 需要增加上传完成确认接口，校验对象实际存在、大小和校验值后再写入业务表。 +- 需要限制允许的 Bucket、Content-Type、大小和有效期。 +- 需要记录预签名创建、上传完成、失败和过期事件，便于审计。 + +### 3.3 激活版本和跨节点资源版本仍需加强 + +数据集已经有 `active_version_id` 和版本表，但以下场景仍需补充： + +- 训练、推理和评测必须只使用资源当前激活版本，并在任务创建时固化版本 ID。 +- 同一文件名的不同版本不能只依靠文件名同步，应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。 +- 已创建任务在后续切换激活版本后，不能被意外切换到新版本。 +- 需要为每个准备到算力节点的资源保存版本、对象 ETag/校验值和本地路径清单。 +- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。 + +### 3.4 权限 2.0 尚未完全落地 + +已有用户、角色、权限码、ACL、审批和审计基础，但仍存在以下差距： + +- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。 +- 项目空间设计已经讨论过取消，但数据库中仍保留 `projects`、`project_members` 等历史结构，需要明确兼容策略和最终迁移方式。 +- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。 +- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。 +- 前端按钮权限已经有基础实现，但不能替代后端鉴权；仍需要对所有关键动作进行后端默认拒绝校验。 +- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。 + +### 3.5 模型合并、导出和评测报告闭环不足 + +- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立，但失败时的清理、重试和幂等性仍需加强。 +- 合并结果归档到 MinIO 的逻辑主要依赖任务完成轮询，服务重启或轮询中断时可能需要补偿扫描。 +- 模型导出任务目前有查询模型和表结构，但完整的创建、执行、进度、失败重试和下载闭环尚未完成。 +- 评测结果和报告字段已经存在，但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。 +- 评测指标配置和执行器返回指标之间仍需要强类型映射，避免前端显示为通用的 `custom`。 + +### 3.6 GPU 资源分配需要统一到所有任务类型 + +- 训练已经有较完整的节点/GPU 选择和预检流程。 +- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码，但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。 +- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。 +- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。 +- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报，仍需要校验采样时间、单位、空值和任务对应关系。 + +## 四、尚未完成的功能 + +以下功能在当前代码中没有形成可验收的完整闭环，或仍处于设计/基础代码阶段： + +1. **完整的租户隔离和资源继承模型**：所有列表、详情、下载、缓存、训练、推理、评测和导出接口都需要统一的租户范围过滤。 +2. **项目取消后的正式数据迁移方案**：需要决定历史项目数据如何归属到用户或租户，并提供一次性迁移脚本和回滚方案。 +3. **预签名上传完成确认和对象校验**：包括 Key 白名单、ACL、大小限制、哈希/ETag 和状态回写。 +4. **MinIO 对象生命周期管理**：软删除后的延迟删除、失败重试、孤儿对象扫描、对象引用检查和管理员清理入口。 +5. **Compute Agent 缓存治理**：容量上限、LRU/TTL、运行任务保护、磁盘占用监控、缓存清单和版本校验。 +6. **统一的资源归档编排器**：训练、合并、评测和推理相关产物需要支持断点恢复、幂等重试和服务重启补偿。 +7. **模型导出完整流程**：导出任务创建、格式/量化参数、进度、失败重试、MinIO 归档和下载权限。 +8. **流式和分片文件传输**：避免大文件上传、下载和对象复制时将完整内容读入 Backend 或 Compute Agent 内存。 +9. **生产级 MinIO 安全和高可用**：默认密钥替换、TLS、网络访问控制、管理员 Console 隔离、容量监控、备份和恢复。 +10. **完整的端到端测试和持续集成**：至少覆盖单节点、多节点、多 GPU、跨用户、跨租户、版本切换、MinIO 不可用和服务重启恢复。 +11. **统一数据库迁移体系**：当前初始化 SQL 适合新库初始化，但尚未替代正式的版本化迁移工具；已有数据库更新仍需要明确迁移脚本和执行记录。 + +## 五、需要优化的功能 + +### 5.1 后端响应性能 + +- 页面列表接口需要避免每条记录重复查询用户、资源、MinIO 元数据和 Compute 节点状态。 +- MinIO 的 Bucket 检查、对象 Head 和预签名生成应使用连接复用、短期缓存和批量查询。 +- 训练、推理、评测页面不应通过过短间隔轮询大量详情接口，应按任务状态动态退避，并在完成后停止轮询。 +- 对 dashboard、节点健康、GPU 状态等高频数据应区分实时数据和缓存数据。 +- 后端日志轮询和健康检查日志需要继续降噪，仅在状态变化、失败或达到较长周期时输出。 + +### 5.2 前端加载和交互 + +- 列表页面应区分首屏 loading、刷新 loading、操作 loading，避免整页长时间无反馈。 +- 推理、评测、训练详情应使用统一的任务状态刷新策略和超时提示。 +- 前端仍有 FontAwesome 在线资源解析警告，应清理对外部网络文件的依赖，保证离线环境打开速度。 +- 应继续拆分首屏大体积 chunk，并减少一次性加载不相关页面组件。 +- GPU 选择组件需要明确显示空闲、占用、不可达、预留和已分配状态。 +- 错误提示应携带资源名称、节点名称、版本和下一步处理建议，减少只显示 500/404 的情况。 + +### 5.3 训练、推理和评测可靠性 + +- 所有任务创建前应执行同一套资源权限、版本存在性、MinIO 可用性和 GPU 原子分配校验。 +- 任务创建接口应支持幂等键，避免前端重复点击造成重复任务。 +- 节点不可达时应快速失败或进入可见的等待状态，不能让页面长时间无反馈。 +- 失败重试应区分网络瞬时失败、资源不足、模型文件缺失、参数错误和执行器失败。 +- 任务停止后必须释放 GPU 分配、推理端口、缓存锁和临时目录。 + +### 5.4 数据和模型一致性 + +- 每个对象都应保存大小、校验值、版本 ID、来源、创建者、租户和引用状态。 +- 数据库中的小文件内容和 MinIO 对象不能同时被当作可独立修改的主副本；需要明确唯一写入入口。 +- 数据集激活版本变更需要留下审计记录，并影响后续任务创建但不改变已创建任务。 +- 模型权重、Adapter、合并结果和导出结果需要形成完整血缘关系。 + +## 六、数据库和初始化脚本状态 + +当前 `backend/app/db/sql/000_full_init.sql` 已包含以下主要类别： + +- 用户、模型、训练模型、模型血缘、模型产物、模型导出任务。 +- 数据集、数据集文件、数据集版本、数据集记录。 +- 算力节点、GPU、GPU 分配、调度锁、Compute Job。 +- 资源副本、资源同步任务、MinIO 对象、缓存任务。 +- 评测任务、评测维度、模型对比任务。 +- 租户、项目兼容表、项目成员、角色、会话、ACL。 +- 审批模板、审批实例、审批步骤、审计日志、留存策略。 +- 数据处理任务、源文件、预览项、处理结果、数据转换任务。 + +已确认的近期字段包括： + +- `model_artifacts.storage_object_id` +- `model_artifacts.storage_backend` +- `dataset_files.storage_object_id` +- `data_convert_tasks.output_content` +- `data_convert_tasks.output_storage_object_id` +- `data_convert_tasks.storage_backend` +- `eval_tasks.report_storage_object_id` + +离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意： + +- 初始化 SQL 主要用于新数据库或新数据卷；已有数据库不能仅靠重启容器自动获得全部新字段。 +- 生产/测试数据库需要执行可追踪的迁移脚本，并在迁移前备份或生成结构快照。 +- `ensure_schema` 类运行时补字段逻辑只能作为兼容兜底，不能替代正式迁移。 +- 后续如果正式移除项目设计，需要先完成数据归属迁移，再决定是否删除历史表，不能直接从初始化 SQL 中删除表。 + +## 七、当前验证结果 + +已完成的静态和局部验证： + +- Backend 和离线 Backend 源码 `compileall` 检查通过。 +- MinIO 分层策略冒烟验证通过：小型 JSON/JSONL 可落数据库，小型二进制和超过阈值的内容进入 MinIO。 +- 主工程和离线包初始化 SQL 已做同步检查，内容一致。 +- 前端此前已完成 `npm run build` 类型错误修复，构建剩余问题主要是非阻断的资源/分包警告。 +- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。 + +当前不能据此宣称“全量功能测试通过”： + +- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设，需要按当前分层存储策略更新。 +- WSL Docker 运行时验证受当前环境的 `E_ACCESSDENIED` 影响，不能在本次文档生成时完成全部容器健康、数据库字段和跨节点测试。 +- 多节点、多 GPU、MinIO 临时不可用、服务重启恢复和跨用户权限测试仍需要在可用运行环境中执行。 + +## 八、下一阶段开发计划 + +### P0：安全与数据正确性 + +1. 完善 MinIO 预签名 PUT 的资源写权限、对象 Key 白名单、大小/类型限制和上传完成确认。 +2. 统一任务创建时的资源版本固化，训练、推理、评测只使用已授权的激活版本快照。 +3. 逐一补齐评测、推理、模型合并、模型导出、缓存准备的后端权限校验和审计记录。 +4. 完成租户隔离查询范围，清理或兼容历史项目字段，补充数据迁移脚本。 + +### P1：跨节点可靠性 + +1. 建立资源清单/manifest，记录 MinIO 对象版本、校验值、目标节点路径和缓存状态。 +2. 完善训练、合并、评测和推理的准备、执行、归档、失败重试和服务重启补偿。 +3. 完善 GPU 原子分配、异常回收、节点重连对账和任务释放。 +4. 增加缓存容量、TTL/LRU、运行任务保护和磁盘占用监控。 +5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。 + +### P2：性能与用户体验 + +1. 优化页面列表接口和高频轮询，采用批量查询、短期缓存和动态退避。 +2. 将大文件上传/下载/复制改为流式或分片传输。 +3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。 +4. 处理前端离线资源警告，继续拆分首屏 chunk。 +5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。 + +### P3：工程化和上线准备 + +1. 建立正式数据库版本迁移机制和离线升级脚本。 +2. 增加 CI：前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。 +3. 增加多节点端到端测试和 MinIO 故障注入测试。 +4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。 +5. 建立生产运行手册，包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。 + +## 九、阶段验收标准 + +完成下一阶段后，至少应满足： + +- 用户只能看到和操作其所属租户授权的模型、数据集、训练任务、推理服务和评测任务。 +- 任何任务创建都能明确记录用户、租户、资源版本、算力节点、GPU 列表和 MinIO 对象版本。 +- 同一个节点的同一张 GPU 不能被两个活动任务同时分配。 +- MinIO 临时不可用时，任务进入可解释的等待/失败状态，并能按策略重试，页面不会无限等待。 +- Backend 或 Compute Agent 重启后，任务、缓存、GPU 分配和归档状态可以对账恢复。 +- 训练、合并、评测和推理产物都能在 MinIO 中找到，并且可以通过权限校验后的接口下载或使用。 +- 删除资源后不会继续出现在普通列表中，关联对象能够按引用状态延迟清理并留下审计记录。 +- 新数据库初始化和已有数据库迁移后，所有业务接口不再因为缺表或缺字段启动失败。 +- 离线部署不依赖外部字体、图标或 CDN，前端首屏和核心业务操作在无网络环境下可用。 + +## 十、相关文件索引 + +- 平台架构：[platform-architecture-requirements.md](./platform-architecture-requirements.md) +- 权限设计：[permissions-design.md](./permissions-design.md) +- MinIO 与 Compute 缓存方案：[minio-compute-cache-plan.md](./minio-compute-cache-plan.md) +- 平台治理菜单设计：[platform-governance-menu-design.md](./platform-governance-menu-design.md) +- 数据处理设计：[data-process-design.md](./data-process-design.md) +- 数据库初始化脚本：[../backend/app/db/sql/000_full_init.sql](../backend/app/db/sql/000_full_init.sql) +- 离线部署目录：[../docker/offline](../docker/offline) + diff --git a/frontend/src/api/modules/audit.ts b/frontend/src/api/modules/audit.ts index 9999b1e..511d4db 100644 --- a/frontend/src/api/modules/audit.ts +++ b/frontend/src/api/modules/audit.ts @@ -20,6 +20,8 @@ export interface AuditQuery { actor_id?: string action?: string target_type?: string + target_id?: string + keyword?: string start_time?: string end_time?: string limit?: number diff --git a/frontend/src/api/modules/dataProcess.ts b/frontend/src/api/modules/dataProcess.ts index a9c35e0..71faaff 100644 --- a/frontend/src/api/modules/dataProcess.ts +++ b/frontend/src/api/modules/dataProcess.ts @@ -20,6 +20,8 @@ import type { DataProcessPublishResult, DataProcessQualityScore, DataProcessResult, + DataProcessResultBatchEvaluatePayload, + DataProcessResultBatchEvaluateResult, DataProcessResultBatchRegeneratePayload, DataProcessResultBatchRegenerateResult, DataProcessResultRegeneratePayload, @@ -320,5 +322,14 @@ export const regenerateDataProcessResults = ( { timeout: 240_000 }, ) +export const evaluateDataProcessResults = ( + taskId: string | number, + payload: DataProcessResultBatchEvaluatePayload, +) => post( + `/data-process/${encodeURIComponent(taskId)}/results/evaluate-batch`, + payload, + { timeout: 240_000 }, +) + export const publishDataProcess = (taskId: string | number, payload: DataProcessPublishPayload) => post(`/data-process/${encodeURIComponent(taskId)}/publish`, payload) diff --git a/frontend/src/api/modules/fineTune.ts b/frontend/src/api/modules/fineTune.ts index 374ed49..aefc53e 100644 --- a/frontend/src/api/modules/fineTune.ts +++ b/frontend/src/api/modules/fineTune.ts @@ -42,12 +42,23 @@ export interface FineTunePreflightResult { sync_results?: Array> } +export interface FineTuneGpuStatus { + source: string + items: Array> + selected_gpus: number[] + error?: string +} + /** 训练任务列表 */ export const getFineTuneList = () => get('/fine-tune') /** 训练任务详情 */ export const getFineTune = (id: string | number) => get(`/fine-tune/${id}`) +/** 获取任务所在 Compute 节点的实时 GPU 指标 */ +export const getFineTuneGpuStatus = (id: string | number) => + get(`/fine-tune/${id}/gpu-status`) + /** 任务名查重 */ export const checkFineTuneName = (name: string) => get<{ exists: boolean }>('/fine-tune/check-name', { name }) diff --git a/frontend/src/components/AppSidebar.vue b/frontend/src/components/AppSidebar.vue index 62df235..1844d48 100644 --- a/frontend/src/components/AppSidebar.vue +++ b/frontend/src/components/AppSidebar.vue @@ -15,6 +15,10 @@ const activeMenu = computed(() => { if (seg === 'training-log') return 'fine-tune' // 维度管理归到模型评测 if (route.path.includes('model-eval/dimension')) return 'model-eval' + // 组织与权限承接用户、租户和历史治理入口 + if (route.path.startsWith('/organization') || route.path.startsWith('/user-settings') || route.path.startsWith('/tenants')) return 'organization' + // 运行日志承接审计和操作诊断两个历史入口 + if (route.path.startsWith('/logs') || route.path.startsWith('/audit-logs') || route.path.startsWith('/operation-logs')) return 'logs' // 对比对话归到模型推理 if (route.path.startsWith('/model-compare/chat')) return 'model-inference' // 合并权重归到模型管理 @@ -77,21 +81,16 @@ const menuGroups: MenuGroup[] = [ { title: '平台治理', items: [ - { key: 'tenants', label: '租户管理', icon: 'fa-building', to: '/tenants', permission: 'user-settings' }, - { key: 'projects', label: '项目空间', icon: 'fa-folder', to: '/projects', permission: 'user-settings' }, + { key: 'organization', label: '组织与权限', icon: 'fa-users', to: '/organization', permission: 'user-settings' }, { key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' }, - { key: 'audit-logs', label: '审计日志', icon: 'fa-history', to: '/audit-logs', permission: 'user-settings' }, - { key: 'operation-logs', label: '操作日志', icon: 'fa-list', to: '/operation-logs', permission: 'user-settings' }, - { key: 'approval-templates', label: '审批模板', icon: 'fa-list-alt', to: '/approval-templates', permission: 'user-settings' }, { key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' }, ], }, { title: '系统设置', items: [ - { key: 'user-settings', label: '用户设置', icon: 'fa-users', to: '/user-settings', permission: 'user-settings' }, { key: 'hardware', label: '平台性能', icon: 'fa-bar-chart', to: '/hardware', permission: 'hardware' }, - { key: 'logs', label: '查看日志', icon: 'fa-file-text', to: '/logs', permission: 'logs' }, + { key: 'logs', label: '运行日志', icon: 'fa-file-text', to: '/logs', permission: 'logs' }, ], }, ] @@ -101,9 +100,9 @@ const menuGroups: MenuGroup[] = [ * * 1. admin 用户：可以看到所有菜单 * 2. 非 admin 用户： - * - 默认可见所有业务菜单（模型训练、评测、推理、数据集、数据处理、转换、性能、日志等） + * - 默认可见所有业务菜单（模型训练、评测、推理、数据集、数据处理、转换、性能、运行日志等） * - 仅以下菜单对非 admin 不可见： - * - user-settings（用户设置、租户管理、项目空间、审批模板/中心、审计日志） + * - user-settings（组织与权限、资源授权、审批；运行日志中的审计/诊断页签） * - compute（算力节点/GPU 分配） * * 注意：移除了旧的权限码（permission code）过滤逻辑， diff --git a/frontend/src/plugins/echarts.ts b/frontend/src/plugins/echarts.ts index ce1737c..3508f4b 100644 --- a/frontend/src/plugins/echarts.ts +++ b/frontend/src/plugins/echarts.ts @@ -3,18 +3,21 @@ */ import { use } from 'echarts/core' import { CanvasRenderer } from 'echarts/renderers' -import { BarChart, PieChart } from 'echarts/charts' +import { BarChart, PieChart, RadarChart } from 'echarts/charts' import { GridComponent, TooltipComponent, LegendComponent, + RadarComponent, } from 'echarts/components' use([ CanvasRenderer, BarChart, PieChart, + RadarChart, GridComponent, TooltipComponent, LegendComponent, + RadarComponent, ]) diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts index 973a062..91e8b25 100644 --- a/frontend/src/router/index.ts +++ b/frontend/src/router/index.ts @@ -32,11 +32,17 @@ const routes: RouteRecordRaw[] = [ meta: { title: '服务看板' }, }, // 平台治理 + { + path: 'organization', + name: 'organization', + component: () => import('@/views/governance/OrganizationPermissionView.vue'), + meta: { title: '组织与权限', pageSurface: 'self', permission: 'user-settings' }, + }, { path: 'tenants', name: 'tenants', - component: () => import('@/views/tenants/TenantListView.vue'), - meta: { title: '租户管理', permission: 'user-settings' }, + redirect: '/organization?tab=tenants', + meta: { title: '租户与配额', permission: 'user-settings' }, }, { path: 'tenants/:id', @@ -47,37 +53,37 @@ const routes: RouteRecordRaw[] = [ { path: 'projects', name: 'projects', - component: () => import('@/views/projects/ProjectListView.vue'), - meta: { title: '项目空间', permission: 'user-settings' }, + redirect: '/organization?tab=users', + meta: { title: '组织与权限', permission: 'user-settings' }, }, { path: 'projects/:id', name: 'project-detail', - component: () => import('@/views/projects/ProjectDetailView.vue'), - meta: { title: '项目详情', permission: 'user-settings' }, + redirect: '/organization?tab=users', + meta: { title: '组织与权限', permission: 'user-settings' }, }, { path: 'audit-logs', name: 'audit-logs', - component: () => import('@/views/audit/AuditLogView.vue'), - meta: { title: '审计日志', permission: 'user-settings' }, + redirect: '/logs?tab=audit', + meta: { title: '运行日志', permission: 'user-settings' }, }, { path: 'operation-logs', name: 'operation-logs', - component: () => import('@/views/audit/OperationLogView.vue'), - meta: { title: '操作日志', permission: 'user-settings' }, + redirect: '/logs?tab=operations', + meta: { title: '运行日志', permission: 'user-settings' }, }, { path: 'approval-templates', name: 'approval-templates', - component: () => import('@/views/approvals/ApprovalTemplateView.vue'), - meta: { title: '审批模板', permission: 'user-settings' }, + redirect: '/approval-instances?tab=strategies', + meta: { title: '审批中心', permission: 'user-settings' }, }, { path: 'approval-instances', name: 'approval-instances', - component: () => import('@/views/approvals/ApprovalInstanceView.vue'), + component: () => import('@/views/approvals/ApprovalCenterView.vue'), meta: { title: '审批中心', permission: 'user-settings' }, }, { @@ -302,14 +308,14 @@ const routes: RouteRecordRaw[] = [ { path: 'logs', name: 'logs', - component: () => import('@/views/system/LogsView.vue'), - meta: { title: '查看日志' }, + component: () => import('@/views/system/RuntimeLogsView.vue'), + meta: { title: '运行日志', pageSurface: 'self', permission: 'logs' }, }, { path: 'user-settings', name: 'user-settings', - component: () => import('@/views/system/UserSettingsView.vue'), - meta: { title: '用户设置', pageSurface: 'self', permission: 'user-settings' }, + redirect: '/organization?tab=users', + meta: { title: '组织与权限', pageSurface: 'self', permission: 'user-settings' }, }, { path: 'user-settings/create', @@ -357,6 +363,7 @@ const permissionBySegment: Record = { tools: 'data-convert', hardware: 'hardware', logs: 'logs', + organization: 'user-settings', 'user-settings': 'user-settings', tenants: 'user-settings', projects: 'user-settings', @@ -379,7 +386,7 @@ function requiredPermission(path: string, explicit?: unknown) { // 权限控制规则（基于 governance-user-guide.md 设计）： // - admin 用户：可以访问所有页面 // - 非 admin 用户：默认可访问所有业务页面（训练、评测、推理、数据等） -// 仅以下页面限制 admin 访问：user-settings、compute（算力节点） +// 仅治理与资源管理页面限制 admin 访问：organization、user-settings、compute router.beforeEach((to, _from, next) => { if (!to.meta.public) routeLoading.value = true const auth = useAuthStore() @@ -404,7 +411,7 @@ router.beforeEach((to, _from, next) => { if (!to.meta.skipPermission) { const permission = requiredPermission(to.path, to.meta.permission) // 仅限制管理员专属页面的访问权限 - // user-settings（用户设置、租户管理、项目空间、审批、审计日志）仅 admin 可访问 + // user-settings（组织与权限、资源授权、审批中心、运行日志）仅 admin 可访问 if (permission === 'user-settings' && !auth.isAdmin) { next({ name: 'permission-denied', replace: true }) return diff --git a/frontend/src/types/dataProcess.ts b/frontend/src/types/dataProcess.ts index 921387f..63e86b9 100644 --- a/frontend/src/types/dataProcess.ts +++ b/frontend/src/types/dataProcess.ts @@ -398,6 +398,52 @@ export interface DataProcessResultBatchRegenerateResult { failures: DataProcessResultBatchRegenerateFailure[] } +export interface DataProcessResultBatchEvaluateItem { + result_id: string + expected_updated_at: string +} + +export interface DataProcessResultBatchEvaluatePayload { + items: DataProcessResultBatchEvaluateItem[] +} + +export interface DataProcessResultBatchEvaluateFailure { + result_id: string + code: 'conflict' | 'skipped' | 'evaluation_failed' | 'internal_error' + message: string +} + +export interface DataProcessResultBatchEvaluateResult { + batch_id: string + total: number + succeeded: number + failed: number + duration_ms: number + items: DataProcessResult[] + failures: DataProcessResultBatchEvaluateFailure[] +} + +export interface DataProcessQualitySemantic { + question_answer?: number + answer_source?: number + overall?: number +} + +export interface DataProcessQualityJudge { + scores?: Record + overall?: number + reason?: string + issues?: string[] + model?: string + output_type?: string +} + +export interface DataProcessQualityLayers { + rule?: number | null + semantic?: number | null + judge?: number | null +} + export interface DataProcessQualityScore { overall?: number completeness?: number @@ -408,6 +454,11 @@ export interface DataProcessQualityScore { is_valid?: boolean flags?: string[] fingerprint?: string + semantic?: DataProcessQualitySemantic | null + judge?: DataProcessQualityJudge | null + layers?: DataProcessQualityLayers | null + evaluated?: boolean + evaluated_at?: string | null source_pages?: number[] heading_path?: string[] source_locator?: DataProcessSourceLocator diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 4c93efa..722b186 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -218,6 +218,8 @@ export interface LoadedModel { port?: number node_id?: string node_name?: string + gpu_indices?: number[] + gpus?: number[] error?: string } @@ -239,6 +241,8 @@ export interface CompareModelRef { gpu_id: number node_id?: string node_name?: string + gpu_indices?: number[] + gpus?: number[] source?: string port?: number } @@ -282,7 +286,9 @@ export interface StartEvalPayload { eval_task_name: string eval_type: EvalType model_id: string | number - gpu_id: string | number + gpu_id: string | number | string[] + gpu_indices?: number[] + gpus?: number[] compute_node_id?: string dataset_id: string | number dimension_id: string | number diff --git a/frontend/src/views/approvals/ApprovalCenterView.vue b/frontend/src/views/approvals/ApprovalCenterView.vue new file mode 100644 index 0000000..dd6817b --- /dev/null +++ b/frontend/src/views/approvals/ApprovalCenterView.vue @@ -0,0 +1,66 @@ + + + + + diff --git a/frontend/src/views/approvals/ApprovalInstanceView.vue b/frontend/src/views/approvals/ApprovalInstanceView.vue index 4664758..dcb225d 100644 --- a/frontend/src/views/approvals/ApprovalInstanceView.vue +++ b/frontend/src/views/approvals/ApprovalInstanceView.vue @@ -1,11 +1,15 @@ @@ -1099,6 +1229,35 @@ onBeforeUnmount(() => { .result-filters :deep(.el-select) { width: 120px; } .result-section :deep(.el-table) { border-radius: 0; } +.evaluation-progress { + display: grid; + grid-template-columns: 1fr 220px; + align-items: center; + gap: 14px; + padding: 10px 18px; + color: #667085; + background: #f8f9fc; + font-size: 12px; +} + +.detail-quality-score { + display: inline-block; + min-width: 44px; + padding: 2px 8px; + border-radius: 10px; + color: #475467; + background: #f2f4f7; + font-weight: 700; + font-variant-numeric: tabular-nums; + cursor: default; + + &.is-success { color: #067647; background: #e6f4ee; } + &.is-warning { color: #b54708; background: #fef0c7; } + &.is-danger { color: #b42318; background: #fee4e2; } +} + +.detail-quality-empty { color: #98a2b3; } + :global(.data-process-result-tooltip) { box-sizing: border-box; max-width: min(520px, calc(100vw - 32px)); diff --git a/frontend/src/views/data-process/create/QualityRadarPopover.vue b/frontend/src/views/data-process/create/QualityRadarPopover.vue new file mode 100644 index 0000000..25bcf48 --- /dev/null +++ b/frontend/src/views/data-process/create/QualityRadarPopover.vue @@ -0,0 +1,252 @@ + + + + + + + + diff --git a/frontend/src/views/data-process/create/ResultEditorStep.vue b/frontend/src/views/data-process/create/ResultEditorStep.vue index 6bb6a91..ec218b6 100644 --- a/frontend/src/views/data-process/create/ResultEditorStep.vue +++ b/frontend/src/views/data-process/create/ResultEditorStep.vue @@ -1,6 +1,7 @@ + + + + diff --git a/frontend/src/views/inference/InferenceCreateView.vue b/frontend/src/views/inference/InferenceCreateView.vue index 4224f17..c1b174e 100644 --- a/frontend/src/views/inference/InferenceCreateView.vue +++ b/frontend/src/views/inference/InferenceCreateView.vue @@ -4,8 +4,7 @@ import { useRouter } from 'vue-router' import { ElMessage, type FormInstance, type FormRules } from 'element-plus' import PageCard from '@/components/PageCard.vue' import { getModelList, getTrainedModels } from '@/api/modules/model' -import { getSystemInfo } from '@/api/modules/system' -import { getComputeNodes, type ComputeNode } from '@/api/modules/compute' +import { getComputeGpus, getComputeNodes, type ComputeNode } from '@/api/modules/compute' import { createCompare, loadCompare } from '@/api/modules/compare' import type { ModelItem, TrainedModel, GpuInfo } from '@/types' @@ -83,8 +82,8 @@ const form = reactive({ description: '', /** 选中的模型 key（单选） */ model_key: '', - /** 使用的 GPU */ - gpu_key: '', + /** 使用的 GPU（同一节点内可多选） */ + gpu_keys: [] as string[], }) const rules: FormRules = { @@ -94,12 +93,26 @@ const rules: FormRules = { /** 当前选中的模型对象 */ const selectedModel = computed(() => modelMap.value[form.model_key]) -const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key)) +const selectedGpus = computed(() => idleGpus.value.filter((gpu) => form.gpu_keys.includes(gpuKey(gpu)))) + +function gpuKey(gpu: GpuInfo) { + return `${gpu.node_id || ''}:${gpu.id ?? 0}` +} + +function handleGpuChange(keys: string[]) { + const nodeId = keys[0]?.split(':', 1)[0] + if (!nodeId) return + const filtered = keys.filter((key) => key.split(':', 1)[0] === nodeId) + if (filtered.length !== keys.length) { + ElMessage.info('一次推理只能使用同一算力节点内的 GPU，已忽略其它节点的选择') + } + form.gpu_keys = filtered +} watch(selectedModel, (model) => { if (!model?.compute_node_id) return const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id) - if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}` + if (gpu) form.gpu_keys = [gpuKey(gpu)] }) async function handleSubmit() { @@ -111,6 +124,10 @@ async function handleSubmit() { ElMessage.warning('请选择模型') return } + if (!selectedGpus.value.length) { + ElMessage.warning('请至少选择一张空闲 GPU') + return + } submitting.value = true startupStatus.value = '正在创建推理任务...' try { @@ -130,9 +147,11 @@ async function handleSubmit() { model_name: m.name, model_path: m.model_path, source: m.source, - gpu_id: selectedGpu.value?.id ?? 0, - node_id: selectedGpu.value?.node_id || m.compute_node_id, - node_name: selectedGpu.value?.node_name || m.compute_node_name, + gpu_id: selectedGpus.value[0]?.id ?? 0, + gpu_indices: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)), + gpus: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)), + node_id: selectedGpus.value[0]?.node_id || m.compute_node_id, + node_name: selectedGpus.value[0]?.node_name || m.compute_node_name, }, ], }) @@ -169,17 +188,17 @@ async function loadData() { const [db, trained, sys, nodes] = await Promise.all([ getModelList(), getTrainedModels(), - getSystemInfo(), + getComputeGpus(), getComputeNodes(), ]) dbModels.value = db || [] trainedModels.value = trained?.models || [] - gpus.value = sys?.gpu || [] + gpus.value = (sys || []) as unknown as GpuInfo[] computeNodes.value = nodes || [] // 默认选中第一个空闲 GPU if (idleGpus.value.length > 0) { const firstGpu = idleGpus.value[0] - form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}` + form.gpu_keys = [gpuKey(firstGpu)] } } catch { // ignore @@ -228,12 +247,20 @@ onMounted(loadData) - + diff --git a/frontend/src/views/system/RuntimeLogsView.vue b/frontend/src/views/system/RuntimeLogsView.vue new file mode 100644 index 0000000..6d841e7 --- /dev/null +++ b/frontend/src/views/system/RuntimeLogsView.vue @@ -0,0 +1,78 @@ + + + + + diff --git a/frontend/src/views/system/TrainingLogView.vue b/frontend/src/views/system/TrainingLogView.vue index 80c946e..35e98f2 100644 --- a/frontend/src/views/system/TrainingLogView.vue +++ b/frontend/src/views/system/TrainingLogView.vue @@ -8,10 +8,9 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue' import { usePolling } from '@/composables/usePolling' import '@/plugins/echarts-training-log' import { useModelsStore } from '@/stores/models' -import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune' +import { getFineTune, getFineTuneDiagnostics, getFineTuneGpuStatus, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune' import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log' import { getDataset } from '@/api/modules/dataset' -import { getSystemInfo } from '@/api/modules/system' import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants' import { buildMetricChartOption, @@ -205,15 +204,20 @@ async function loadDataset(datasetId: string | number) { } } -async function loadGpuStatus() { +async function loadGpuStatus(currentTask: FineTuneTask) { try { - const systemInfo = await getSystemInfo() - gpuPool.value = systemInfo.gpu ?? [] - gpuUpdatedAt.value = new Date() - gpuLoadError.value = '' + const live = await getFineTuneGpuStatus(currentTask.id) + if (live.source === 'compute' && live.items.length) { + gpuPool.value = live.items as unknown as GpuInfo[] + gpuUpdatedAt.value = new Date() + gpuLoadError.value = '' + return + } + gpuPool.value = [] + gpuLoadError.value = live.error || 'Compute 节点暂未返回实时 GPU 指标' } catch { gpuLoadError.value = 'GPU 监控数据暂时不可用' - if (!gpuUpdatedAt.value) gpuPool.value = [] + gpuPool.value = [] } } @@ -306,7 +310,7 @@ async function refreshAll() { const datasetPromise = currentTask.train_dataset_id ? loadDataset(currentTask.train_dataset_id) : Promise.resolve() - await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)]) + await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(currentTask), loadDiagnostics(currentTask)]) await loadMetrics(currentTask) } finally { loading.value = false diff --git a/frontend/src/views/system/UserCreateView.vue b/frontend/src/views/system/UserCreateView.vue index c2304d7..b0efe7f 100644 --- a/frontend/src/views/system/UserCreateView.vue +++ b/frontend/src/views/system/UserCreateView.vue @@ -12,7 +12,7 @@ const form = reactive({ username: '', display_name: '', password: 'platform123', - role: 'user', + role: 'operator', status: 'active', permissions: [], }) @@ -45,7 +45,7 @@ async function submit() { - +