feat: 更新后端平台模块、Compute引擎、前端组件及构建产物

- 更新 backend 平台 API、platform_store、compute_gateway sync
- 更新 compute agent/engine/adapter 及 API
- 更新 Docker 部署配置（app/compute）
- 新增 frontend/src/utils/ 工具模块
- 新增 scripts/ops_diagnostics.py 运维诊断脚本
- 新增 docs/2026-07-23-development-summary.md 开发总结
- 重构 frontend/dist 构建产物（新 hash）
- 更新前端多个视图组件及 API 模块

Co-Authored-By: Claude <noreply@anthropic.com>

compute/agent/process_manager.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import json
 import contextlib
+import hashlib
 import signal
 import subprocess
 import time
@@ -127,6 +128,7 @@ class ProcessManager:
 
     def serialize(self, job: ManagedProcess) -> dict[str, Any]:
         code = job.process.poll() if job.process is not None else None
+        checkpoints = self._collect_checkpoints(job.output_dir)
         if job.status not in TERMINAL_STATUSES:
             if job.process is None and job.pid is not None and not self._pid_alive(job.pid):
                 job.status = "failed"
@@ -157,6 +159,7 @@ class ProcessManager:
             "output_dir": job.output_dir,
             "log_file": str(job.log_path),
             "artifacts": job.artifacts,
+            "checkpoints": checkpoints,
             "return_code": code,
         }
 
@@ -175,15 +178,47 @@ class ProcessManager:
         artifacts: list[dict[str, Any]] = []
         for path in root.rglob("*"):
             if path.is_file():
+                digest = hashlib.sha256()
+                with path.open("rb") as handle:
+                    for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                        digest.update(chunk)
+                size = path.stat().st_size
                 artifacts.append(
                     {
                         "path": str(path),
                         "name": path.name,
-                        "size": path.stat().st_size,
+                        "size": size,
+                        "size_bytes": size,
+                        "checksum_sha256": digest.hexdigest(),
                     }
                 )
         return artifacts[:200]
 
+    def _collect_checkpoints(self, output_dir: str) -> list[dict[str, Any]]:
+        root = Path(output_dir)
+        if not root.exists():
+            return []
+        checkpoints: list[dict[str, Any]] = []
+        for path in root.glob("checkpoint-*"):
+            if not path.is_dir():
+                continue
+            step = 0
+            try:
+                step = int(path.name.rsplit("-", 1)[-1])
+            except ValueError:
+                step = 0
+            size_bytes = sum(item.stat().st_size for item in path.rglob("*") if item.is_file())
+            checkpoints.append(
+                {
+                    "step": step,
+                    "name": path.name,
+                    "path": str(path),
+                    "size_bytes": size_bytes,
+                    "create_time": path.stat().st_mtime,
+                }
+            )
+        return sorted(checkpoints, key=lambda item: (int(item.get("step") or 0), str(item.get("name") or "")))
+
     def _save_registry(self) -> None:
         items = []
         for job in self.jobs.values():

compute/api/main.py

@@ -13,7 +13,7 @@ from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFi
 from fastapi.responses import FileResponse, JSONResponse
 
 from compute.agent.process_manager import ProcessManager
-from compute.engines.llama_factory.adapter import build_command, parse_log_line
+from compute.engines.llama_factory.adapter import build_command, parse_log_line, prepare_runtime_files
 
 
 def create_app() -> FastAPI:
@@ -74,6 +74,48 @@ def create_app() -> FastAPI:
                 return output.splitlines()[0][:120]
         return ""
 
+    def torch_cuda_status() -> dict[str, Any]:
+        try:
+            import torch  # type: ignore[import-not-found]
+        except Exception as exc:  # noqa: BLE001 - keep health endpoint resilient
+            return {
+                "available": False,
+                "device_count": 0,
+                "torch_version": "",
+                "torch_cuda_version": "",
+                "error": f"torch import failed: {exc}",
+            }
+        try:
+            available = bool(torch.cuda.is_available())
+            device_count = int(torch.cuda.device_count())
+            devices = []
+            for index in range(device_count):
+                props = torch.cuda.get_device_properties(index)
+                devices.append(
+                    {
+                        "index": index,
+                        "name": props.name,
+                        "memory_total_gb": round(props.total_memory / 1024 / 1024 / 1024, 2),
+                    }
+                )
+            return {
+                "available": available,
+                "device_count": device_count,
+                "torch_version": str(torch.__version__),
+                "torch_cuda_version": str(torch.version.cuda or ""),
+                "devices": devices,
+                "error": "" if available else "torch cuda is not available",
+            }
+        except Exception as exc:  # noqa: BLE001 - expose CUDA initialization failures
+            return {
+                "available": False,
+                "device_count": 0,
+                "torch_version": str(getattr(torch, "__version__", "")),
+                "torch_cuda_version": str(getattr(torch.version, "cuda", "") or ""),
+                "devices": [],
+                "error": str(exc),
+            }
+
     def _slice_log_content(
         content: str,
         tail_lines: int | None = None,
@@ -261,6 +303,35 @@ def create_app() -> FastAPI:
             )
         return gpus
 
+    def _validate_training_accelerator(payload: dict[str, Any]) -> tuple[list[str], list[str], dict[str, Any]]:
+        errors: list[str] = []
+        warnings: list[str] = []
+        if str(payload.get("engine") or payload.get("training_engine") or "llama_factory") == "smoke":
+            return errors, warnings, {}
+        requested_gpus = [int(item) for item in payload.get("gpus") or []]
+        if not requested_gpus:
+            warnings.append("no gpu selected; training will run on CPU")
+            return errors, warnings, {}
+        cuda = torch_cuda_status()
+        if not cuda.get("available"):
+            errors.append(f"torch cuda unavailable on compute node: {cuda.get('error') or 'unknown error'}")
+        device_count = int(cuda.get("device_count") or 0)
+        if device_count and max(requested_gpus) >= device_count:
+            errors.append(f"requested gpu index out of torch device range: requested={requested_gpus}, device_count={device_count}")
+        min_memory_gb = _float_env("MIN_TRAINING_GPU_MEMORY_GB", 4.0)
+        gpus = {int(item["gpu_index"]): item for item in gpu_resources() if "gpu_index" in item}
+        for gpu_index in requested_gpus:
+            gpu = gpus.get(gpu_index)
+            if not gpu:
+                errors.append(f"requested gpu not found by nvidia-smi: {gpu_index}")
+                continue
+            memory_total = float(gpu.get("memory_total_gb") or 0)
+            if memory_total and memory_total < min_memory_gb:
+                errors.append(
+                    f"gpu {gpu_index} memory too small: {memory_total}GB < required {min_memory_gb}GB"
+                )
+        return errors, warnings, cuda
+
     def _check_path_item(item: dict[str, Any]) -> dict[str, Any]:
         path = Path(str(item.get("path") or ""))
         exists = path.exists()
@@ -278,13 +349,32 @@ def create_app() -> FastAPI:
             "exists": exists,
             "is_dir": path.is_dir() if exists else False,
             "is_file": path.is_file() if exists else False,
+            "byte_size": sum(child.stat().st_size for child in path.rglob("*") if child.is_file()) if exists and path.is_dir() else path.stat().st_size if exists and path.is_file() else 0,
             "ok": ok or not item.get("required", True),
         }
 
     def _job_preview(payload: dict[str, Any], check_paths: bool) -> dict[str, Any]:
         warnings: list[str] = []
+        runtime_files: list[dict[str, str]] = []
+        command_payload = {**payload, "require_dataset_files": check_paths}
+        if check_paths:
+            try:
+                runtime_files = prepare_runtime_files(command_payload)
+            except OSError as exc:
+                return {
+                    "valid": False,
+                    "errors": [f"prepare runtime files failed: {exc}"],
+                    "warnings": warnings,
+                    "engine": str(payload.get("engine") or payload.get("training_engine") or "llama_factory"),
+                    "command": [],
+                    "command_text": "",
+                    "work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"),
+                    "env": {},
+                    "runtime_files": [],
+                    "path_checks": [],
+                }
         try:
-            command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
+            command = build_command(command_payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
         except ValueError as exc:
             return {
                 "valid": False,
@@ -295,23 +385,36 @@ def create_app() -> FastAPI:
                 "command_text": "",
                 "work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"),
                 "env": {},
+                "runtime_files": runtime_files,
                 "path_checks": [],
             }
 
         errors: list[str] = []
         engine = str(payload.get("engine") or payload.get("training_engine") or "llama_factory")
         path_checks: list[dict[str, Any]] = []
+        accelerator: dict[str, Any] = {}
         if check_paths and engine != "smoke":
             path_checks = [
                 _check_path_item(
                     {
                         "name": "model_name_or_path",
-                        "path": payload.get("model_name_or_path") or payload.get("base_model") or "",
+                        "path": payload.get("model_name_or_path") or payload.get("base_model") or payload.get("base_model_path") or "",
                         "type": "any",
                         "required": True,
                     }
                 )
             ]
+            if engine in {"merge", "export", "llama_factory_export"} and payload.get("adapter_name_or_path"):
+                path_checks.append(
+                    _check_path_item(
+                        {
+                            "name": "adapter_name_or_path",
+                            "path": payload.get("adapter_name_or_path"),
+                            "type": "any",
+                            "required": True,
+                        }
+                    )
+                )
             if payload.get("dataset_dir"):
                 path_checks.append(
                     _check_path_item(
@@ -341,6 +444,10 @@ def create_app() -> FastAPI:
                 errors.append(f"training command not found: {command.command[0]}")
             if not Path(command.work_dir).exists():
                 errors.append(f"llama_factory_home not found: {command.work_dir}")
+            if engine not in {"merge", "export", "llama_factory_export"}:
+                accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload)
+                errors.extend(accelerator_errors)
+                warnings.extend(accelerator_warnings)
         elif engine == "smoke":
             warnings.append("smoke engine skips model and dataset path checks")
 
@@ -353,6 +460,8 @@ def create_app() -> FastAPI:
             "command_text": " ".join(command.command),
             "work_dir": command.work_dir,
             "env": command.env,
+            "runtime_files": runtime_files,
+            "accelerator": accelerator,
             "path_checks": path_checks,
         }
 
@@ -373,6 +482,8 @@ def create_app() -> FastAPI:
         dataset_root = Path(os.getenv("YG_FT_DATASET_ROOT", str(data_root / "datasets")))
         output_root = Path(os.getenv("YG_FT_OUTPUT_ROOT", str(data_root / "outputs")))
         llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
+        gpu_items = gpu_resources()
+        torch_cuda = torch_cuda_status()
         return {
             "status": "ok",
             "api_version": "v1",
@@ -391,8 +502,10 @@ def create_app() -> FastAPI:
             "llama_factory_version": os.getenv("LLAMA_FACTORY_VERSION", ""),
             "execution_mode": execution_mode(),
             "gpu_count": _int_env("COMPUTE_GPU_COUNT", 0),
+            "nvidia_gpu_count": len(gpu_items),
+            "torch_cuda": torch_cuda,
             "gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus",
-            "capabilities": ["gpu_discovery", "llama_factory", "file_gateway", "job_polling"],
+            "capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling"],
         }
 
     @app.get(f"{route_prefix}/v1/compute/jobs")
@@ -466,6 +579,11 @@ def create_app() -> FastAPI:
 
     @app.post(f"{route_prefix}/compute/jobs")
     async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
+        payload = {**payload, "require_dataset_files": True}
+        try:
+            prepare_runtime_files(payload)
+        except OSError as exc:
+            raise HTTPException(status_code=400, detail=f"prepare runtime files failed: {exc}")
         try:
             command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
         except ValueError as exc:

compute/engines/llama_factory/adapter.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import re
 from dataclasses import dataclass
 from pathlib import Path
@@ -13,6 +14,67 @@ class LlamaFactoryCommand:
     env: dict[str, str]
 
 
+def _load_dataset_preview(path: Path) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+    text = path.read_text(encoding="utf-8", errors="replace").strip()
+    if not text:
+        return []
+    if path.suffix.lower() == ".jsonl":
+        items: list[dict[str, Any]] = []
+        for line in text.splitlines()[:20]:
+            line = line.strip()
+            if not line:
+                continue
+            value = json.loads(line)
+            if isinstance(value, dict):
+                items.append(value)
+        return items
+    value = json.loads(text)
+    if isinstance(value, list):
+        return [item for item in value[:20] if isinstance(item, dict)]
+    if isinstance(value, dict):
+        return [value]
+    return []
+
+
+def _validate_dataset_columns(config: dict[str, Any]) -> list[str]:
+    dataset_dir = config.get("dataset_dir")
+    dataset_info = config.get("dataset_info")
+    if not dataset_dir or not isinstance(dataset_info, dict):
+        return []
+    root = Path(str(dataset_dir))
+    errors: list[str] = []
+    for dataset_key, item in dataset_info.items():
+        if not isinstance(item, dict):
+            continue
+        file_name = item.get("file_name")
+        file_names = file_name if isinstance(file_name, list) else [file_name]
+        columns = item.get("columns") if isinstance(item.get("columns"), dict) else {}
+        required_columns = [str(value) for value in columns.values() if value]
+        for name in file_names:
+            if not name:
+                continue
+            path = root / str(name).lstrip("/\\")
+            if not path.exists():
+                continue
+            try:
+                preview_rows = _load_dataset_preview(path)
+            except Exception as exc:  # noqa: BLE001 - expose malformed data as validation error
+                errors.append(f"dataset file parse failed: {path}: {exc}")
+                continue
+            if not preview_rows:
+                errors.append(f"dataset file has no valid object records: {path}")
+                continue
+            available = set().union(*(row.keys() for row in preview_rows))
+            missing = [column for column in required_columns if column not in available]
+            if missing:
+                errors.append(
+                    f"dataset columns missing in {path.name} for {dataset_key}: {', '.join(sorted(set(missing)))}"
+                )
+    return errors
+
+
 def validate_config(config: dict[str, Any]) -> list[str]:
     errors: list[str] = []
     if not config.get("base_model") and not config.get("model_name_or_path"):
@@ -31,6 +93,24 @@ def validate_config(config: dict[str, Any]) -> list[str]:
         epochs = 0
     if epochs <= 0:
         errors.append("n_epochs must be greater than zero")
+    dataset_dir = config.get("dataset_dir")
+    dataset_info = config.get("dataset_info")
+    if config.get("require_dataset_files") and dataset_dir and isinstance(dataset_info, dict):
+        root = Path(str(dataset_dir))
+        for dataset_key, item in dataset_info.items():
+            if not isinstance(item, dict):
+                errors.append(f"dataset_info entry must be object: {dataset_key}")
+                continue
+            file_name = item.get("file_name")
+            file_names = file_name if isinstance(file_name, list) else [file_name]
+            for name in file_names:
+                if not name:
+                    errors.append(f"dataset_info file_name is required: {dataset_key}")
+                    continue
+                path = root / str(name).lstrip("/\\")
+                if not path.exists():
+                    errors.append(f"dataset file not found: {path}")
+        errors.extend(_validate_dataset_columns(config))
     return errors
 
 
@@ -42,12 +122,92 @@ def _optional_arg(config: dict[str, Any], command: list[str], option: str, *keys
             return
 
 
+def _optional_bool_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None:
+    for key in keys:
+        value = config.get(key)
+        if value is True or str(value).lower() == "true":
+            command.extend([option, "true"])
+            return
+
+
+def _normalize_stage(config: dict[str, Any]) -> str:
+    raw = str(config.get("stage") or config.get("train_type") or "sft").strip().lower()
+    return {
+        "sft": "sft",
+        "dpo": "dpo",
+        "cpt": "pt",
+        "pt": "pt",
+        "pretrain": "pt",
+        "rm": "rm",
+        "ppo": "ppo",
+        "kto": "kto",
+    }.get(raw, raw or "sft")
+
+
+def prepare_runtime_files(config: dict[str, Any]) -> list[dict[str, str]]:
+    dataset_dir = config.get("dataset_dir")
+    dataset_info = config.get("dataset_info")
+    if not dataset_dir or not isinstance(dataset_info, dict):
+        return []
+    root = Path(str(dataset_dir))
+    root.mkdir(parents=True, exist_ok=True)
+    path = root / "dataset_info.json"
+    existing: dict[str, Any] = {}
+    if path.exists():
+        try:
+            loaded = json.loads(path.read_text(encoding="utf-8"))
+            existing = loaded if isinstance(loaded, dict) else {}
+        except json.JSONDecodeError:
+            existing = {}
+    existing.update(dataset_info)
+    path.write_text(json.dumps(existing, ensure_ascii=False, indent=2), encoding="utf-8")
+    return [{"name": "dataset_info", "path": str(path)}]
+
+
 def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand:
+    engine = str(config.get("engine") or config.get("training_engine") or "llama_factory")
+    if engine in {"merge", "export", "llama_factory_export"}:
+        model_path = config.get("base_model") or config.get("model_name_or_path") or config.get("base_model_path")
+        adapter_path = config.get("adapter_name_or_path") or config.get("adapter_path") or config.get("lora_path")
+        output_dir = config.get("output_dir") or config.get("export_dir")
+        errors: list[str] = []
+        if not model_path:
+            errors.append("base_model or model_name_or_path is required")
+        if not adapter_path and engine == "merge":
+            errors.append("adapter_name_or_path or adapter_path is required")
+        if not output_dir:
+            errors.append("output_dir or export_dir is required")
+        if errors:
+            raise ValueError("; ".join(errors))
+        command = [
+            "llamafactory-cli",
+            "export",
+            "--model_name_or_path",
+            str(model_path),
+            "--template",
+            str(config.get("template", "qwen")),
+            "--finetuning_type",
+            str(config.get("train_method", config.get("finetuning_type", "lora"))),
+            "--export_dir",
+            str(output_dir),
+            "--export_size",
+            str(config.get("export_size", 2)),
+            "--export_device",
+            str(config.get("export_device", "cpu")),
+            "--export_legacy_format",
+            str(config.get("export_legacy_format", False)).lower(),
+        ]
+        if adapter_path:
+            command.extend(["--adapter_name_or_path", str(adapter_path)])
+        quantization_bit = int(config.get("export_quantization_bit", config.get("quantization_bit", 0)) or 0)
+        if quantization_bit in {4, 8}:
+            command.extend(["--quantization_bit", str(quantization_bit)])
+        return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={})
+
     errors = validate_config(config)
     if errors:
         raise ValueError("; ".join(errors))
 
-    engine = str(config.get("engine") or config.get("training_engine") or "llama_factory")
     if engine == "smoke":
         output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-smoke')}"
         script = (
@@ -75,7 +235,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
         "llamafactory-cli",
         "train",
         "--stage",
-        str(config.get("stage", "sft")).lower(),
+        _normalize_stage(config),
         "--do_train",
         "true",
         "--model_name_or_path",
@@ -112,6 +272,12 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
     _optional_arg(config, command, "--lora_rank", "lora_rank", "rank")
     _optional_arg(config, command, "--lora_alpha", "lora_alpha")
     _optional_arg(config, command, "--lora_dropout", "lora_dropout")
+    _optional_arg(config, command, "--gradient_accumulation_steps", "gradient_accumulation_steps")
+    _optional_arg(config, command, "--val_size", "val_size")
+    _optional_arg(config, command, "--max_samples", "max_samples")
+    _optional_arg(config, command, "--preprocessing_num_workers", "preprocessing_num_workers")
+    _optional_bool_arg(config, command, "--fp16", "fp16")
+    _optional_bool_arg(config, command, "--bf16", "bf16")
     quantization_bit = int(config.get("quantization_bit", 0) or 0)
     if quantization_bit in {4, 8}:
         command.extend(["--quantization_bit", str(quantization_bit)])