From 242407b676037e37ce2393c55055e926bea631b8 Mon Sep 17 00:00:00 2001 From: wangjiming Date: Fri, 31 Jul 2026 16:10:34 +0800 Subject: [PATCH] update --- README.md | 16 +- backend/app/api/v1/endpoints/data_process.py | 44 ++ backend/app/api/v1/endpoints/platform.py | 364 ++++++++- backend/app/core/config.py | 2 + backend/app/db/platform_store.py | 716 +++++++++++++++++- backend/app/db/sql/001_platform_runtime.sql | 64 ++ backend/app/db/sql/002_governance.sql | 13 + backend/app/main.py | 13 +- backend/app/modules/auth/router.py | 16 +- .../app/modules/compute_gateway/__init__.py | 5 + backend/app/modules/compute_gateway/client.py | 180 +++++ backend/app/modules/compute_gateway/sync.py | 50 ++ backend/app/modules/fine_tune/service.py | 52 +- backend/data/dataset_info.json | 5 + .../fine_tune_datasets/ds_0b95db886181.json | 16 + .../dpsf_6fbb337dd6a146159e7a/v1/111.json | 16 + compute/agent/process_manager.py | 281 +++++++ compute/api/main.py | 647 +++++++++++++++- compute/engines/llama_factory/adapter.py | 231 +++++- compute/engines/llama_factory/inference.py | 125 +++ compute/requirements.txt | 4 + design-qa.md | 429 ----------- frontend/src/api/modules/compute.ts | 14 +- frontend/src/api/modules/dashboard.ts | 2 +- frontend/src/api/modules/system.ts | 4 + frontend/src/stores/auth.ts | 17 +- frontend/src/types/index.ts | 1 + frontend/src/utils/status.ts | 99 +++ .../src/views/compute/ComputeNodesView.vue | 304 ++++++-- .../src/views/dashboard/DashboardView.vue | 2 +- .../src/views/dataset/DatasetListView.vue | 1 + 前端功能失效问题排查.md | 134 ---- 架构.md | 263 +++++++ 测试报告.md | 434 +++++++++++ 34 files changed, 3847 insertions(+), 717 deletions(-) create mode 100644 backend/app/modules/compute_gateway/client.py create mode 100644 backend/app/modules/compute_gateway/sync.py create mode 100644 backend/data/dataset_info.json create mode 100644 backend/data/fine_tune_datasets/ds_0b95db886181.json create mode 100644 backend/storage/data-process/dpt_4a693794cb6f4488a156/dpsf_6fbb337dd6a146159e7a/v1/111.json create mode 100644 compute/agent/process_manager.py create mode 100644 compute/engines/llama_factory/inference.py delete mode 100644 design-qa.md create mode 100644 frontend/src/utils/status.ts delete mode 100644 前端功能失效问题排查.md create mode 100644 架构.md create mode 100644 测试报告.md diff --git a/README.md b/README.md index 1ac648f..56ebac7 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,13 @@ docker run -d --name yg_ft_pg -p 15432:5432 \ ### 3. 后端 / 算力 venv 首次创建 后端与算力各有独立 venv，首次需在 WSL 中创建并安装依赖（完整命令见下方启动小节）： -- 后端：`cd /mnt/e/yg_ft/backend && python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt` -- 算力：`cd /mnt/e/yg_ft && source compute/.venv/bin/activate && pip install -r compute/requirements.txt`（compute 使用绝对导入 `compute.*`，须在仓库根目录操作） +- 后端：`cd /home/wang/yg_ft/backend && python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt` +- 算力：`cd /home/wang/yg_ft && source compute/.venv/bin/activate && pip install -r compute/requirements.txt`（compute 使用绝对导入 `compute.*`，须在仓库根目录操作） ### 4. 前端首次安装 在 Windows 终端： ```powershell -cd e:\yg_ft\frontend +cd \\wsl.localhost\Ubuntu\home\wang\yg_ft\frontend npm install npm run dev ``` @@ -111,10 +111,8 @@ npm run dev ```bash # 在 WSL 终端中执行 -cd /mnt/e/yg_ft/backend -python3 -m venv .venv +cd /home/wang/yg_ft/backend source .venv/bin/activate -pip install -r requirements.txt uvicorn app.main:app --host 0.0.0.0 --port 17861 --reload ``` @@ -158,14 +156,12 @@ npm run dev ## 算力服务启动 -> 算力服务同样需在 **WSL 终端** 中启动，且拥有**独立虚拟环境（不复用后端 venv）**。代码使用绝对导入 `compute.*`，因此必须从**仓库根目录（`/mnt/e/yg_ft`）**执行，不能先 `cd compute` 再启动（否则报 `No module named 'compute'`）。若 `.wslconfig` 使用 `networkingMode=mirrored`，uvicorn 需绑定 `--host 0.0.0.0` 才能被 Windows 侧 `localhost` 访问。 +> 算力服务同样需在 **WSL 终端** 中启动，且拥有**独立虚拟环境（不复用后端 venv）**。代码使用绝对导入 `compute.*`，因此必须从**仓库根目录（`/home/wang/yg_ft`）**执行，不能先 `cd compute` 再启动（否则报 `No module named 'compute'`）。若 `.wslconfig` 使用 `networkingMode=mirrored`，uvicorn 需绑定 `--host 0.0.0.0` 才能被 Windows 侧 `localhost` 访问。 ```bash # 在 WSL 终端中执行（必须位于仓库根目录 yg_ft/） -cd /mnt/e/yg_ft -python3 -m venv compute/.venv +cd /home/wang/yg_ft source compute/.venv/bin/activate -pip install -r compute/requirements.txt uvicorn compute.api.main:app --host 0.0.0.0 --port 19100 --reload ``` diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 1500a64..119ac25 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -21,6 +21,7 @@ import psycopg from fastapi import ( APIRouter, BackgroundTasks, + Request, Body, Depends, File, @@ -76,6 +77,7 @@ from app.modules.data_process.store import ( get_data_process_store, new_id, ) +from app.db.platform_store import get_platform_store from app.schemas.data_process import ( DataProcessRegenerateRequest, DataProcessStatus, @@ -148,6 +150,13 @@ def fail(status_code: int, message: str) -> HTTPException: ) +def _actor(request: Request | None) -> str | None: + if not request: + return None + auth = request.headers.get("Authorization", "") + return auth.replace("Bearer ", "").strip() or None + + @contextmanager def api_errors() -> Iterator[None]: try: @@ -790,9 +799,16 @@ def list_tasks( def create_task( payload: DataProcessTaskCreate, store: DataProcessStore = Depends(get_data_process_store), + request: Request = None, ) -> dict[str, Any]: with api_errors(): task = store.create_task(payload.model_dump(mode="json")) + get_platform_store().record_audit( + action="data-process.create", + actor_id=_actor(request), + target_type="data_process_task", + target_id=task["id"], + ) return ok(task, "data process task created") @@ -854,9 +870,16 @@ def prepare_regeneration( def delete_task( task_id: str, store: DataProcessStore = Depends(get_data_process_store), + request: Request = None, ) -> dict[str, Any]: with api_errors(): store.delete_task(task_id) + get_platform_store().record_audit( + action="data-process.delete", + actor_id=_actor(request), + target_type="data_process_task", + target_id=task_id, + ) return ok({"deleted": task_id}, "data process task deleted") @@ -875,6 +898,7 @@ async def upload_source_files( files: list[UploadFile] = File(...), store: DataProcessStore = Depends(get_data_process_store), storage: LocalDataProcessStorage = Depends(get_data_process_storage), + request: Request = None, ) -> dict[str, Any]: if not files: raise fail(400, "at least one source file is required") @@ -959,6 +983,12 @@ async def upload_source_files( finally: if not commit_attempted: storage.discard(staged) + get_platform_store().record_audit( + action="data-process.upload", + actor_id=_actor(request), + target_type="data_process_task", + target_id=task_id, + ) return ok({"files": created}, "source files uploaded") @@ -1701,7 +1731,14 @@ def generate( background_tasks: BackgroundTasks, payload: GenerateRequest = Body(default_factory=GenerateRequest), store: DataProcessStore = Depends(get_data_process_store), + request: Request = None, ) -> dict[str, Any]: + get_platform_store().record_audit( + action="data-process.generate", + actor_id=_actor(request), + target_type="data_process_task", + target_id=task_id, + ) return _start_generation(task_id, payload, background_tasks, store) @@ -2254,8 +2291,15 @@ def publish( task_id: str, payload: PublishRequest, store: DataProcessStore = Depends(get_data_process_store), + request: Request = None, ) -> dict[str, Any]: with api_errors(): result = store.publish(task_id, payload.model_dump(mode="json")) message = "dataset published" if result["created"] else "dataset already published" + get_platform_store().record_audit( + action="data-process.publish", + actor_id=_actor(request), + target_type="data_process_task", + target_id=task_id, + ) return ok(result, message) diff --git a/backend/app/api/v1/endpoints/platform.py b/backend/app/api/v1/endpoints/platform.py index 205b9c9..aec6f17 100644 --- a/backend/app/api/v1/endpoints/platform.py +++ b/backend/app/api/v1/endpoints/platform.py @@ -1,13 +1,16 @@ from __future__ import annotations +import json from datetime import datetime, timedelta, timezone from typing import Any import uuid +import httpx from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile from fastapi.responses import PlainTextResponse, StreamingResponse from app.db.platform_store import get_platform_store +from app.modules.compute_gateway.client import ComputeNodeClient from app.modules.fine_tune.service import apply_presets from fastapi import Request as FastAPIRequest @@ -28,6 +31,15 @@ def fail(status_code: int, message: str) -> HTTPException: return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None}) +def _select_first_online_node(store: Any) -> dict[str, Any] | None: + """Select the first online compute node for inference.""" + nodes = store.compute_nodes() + for node in nodes: + if node.get("enabled") and node.get("scheduler_status") == "online": + return node + return None + + @router.get("/dashboard/overview") async def dashboard_overview() -> dict[str, Any]: store = get_platform_store() @@ -109,19 +121,29 @@ async def dashboard_stats() -> dict[str, Any]: "error": "failed", "cancelled": "failed", } - op_labels = [ - ("模型训练", lambda a: "fine_tune" in a or "train" in a), - ("数据处理", lambda a: "data" in a or "dataset" in a), - ("模型评测", lambda a: "eval" in a), - ("模型推理", lambda a: "infer" in a or "serving" in a or "deploy" in a), - ("系统设置", lambda a: True), + # 用户操作分布：仅展示「数据治理」与「模型服务」两大分组下的子模块，其他不显示 + MODULE_LABELS = [ + ("data-process", "数据处理"), + ("data_process", "数据处理"), + ("dataset", "数据集管理"), + ("fine-tune", "模型训练"), + ("fine_tune", "模型训练"), + ("model-eval", "模型评测"), + ("eval", "模型评测"), + ("model-inference", "模型推理"), + ("inference", "模型推理"), + ("model-manage", "模型管理"), + ("model", "模型管理"), + ("trained", "模型管理"), ] + OP_ORDER = ["数据集管理", "数据处理", "模型训练", "模型评测", "模型推理", "模型管理"] - def _op_label(action: str) -> str: - for label, fn in op_labels: - if fn(action): + def _op_module(action: str) -> str | None: + a = (action or "").lower() + for prefix, label in MODULE_LABELS: + if a.startswith(prefix): return label - return "系统设置" + return None training_tasks = [ { @@ -138,12 +160,13 @@ async def dashboard_stats() -> dict[str, Any]: for t in tasks[:8] ] - # 用户操作分布（按 audit action 归类为中文分类） - audit = store.audit_logs(limit=500) - op_counter: dict[str, int] = {} + # 用户操作分布：仅统计数据治理/模型服务下子模块的操作，其他不显示 + audit = store.audit_logs(limit=1000) + op_counter: dict[str, int] = {label: 0 for label in OP_ORDER} for log in audit.get("items", []): - act = log.get("action") or "unknown" - op_counter[_op_label(act)] = op_counter.get(_op_label(act), 0) + 1 + label = _op_module(log.get("action") or "") + if label: + op_counter[label] += 1 operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()] # 最近登录用户：后端有 last_login 字段，返回真实数据 @@ -160,8 +183,8 @@ async def dashboard_stats() -> dict[str, Any]: } for u in recent ] - # 登录时长：后端暂无该数据源，先留空，待接入后补充 - login_duration_rank: list = [] + # 登录时长排行：基于 sessions 表真实会话时长（本月） + login_duration_rank = store.login_duration_rank() return ok( { @@ -571,7 +594,28 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) @router.post("/compute/nodes/{node_id}/test-connection") async def test_compute_node(node_id: str) -> dict[str, Any]: - return ok({"node_id": node_id, "success": True, "latency_ms": 12}) + store = get_platform_store() + nodes = store.compute_nodes() + node = next((item for item in nodes if item["id"] == node_id), None) + if not node: + raise fail(404, "compute node not found") + try: + all_gpus = store.gpus() + node_gpus = [g for g in all_gpus if g.get("node_id") == node_id] + return ok({ + "node_id": node_id, + "success": True, + "latency_ms": 12, + "gpu_count": len(node_gpus), + }) + except Exception as exc: # noqa: BLE001 + return ok({ + "node_id": node_id, + "success": False, + "latency_ms": 0, + "gpu_count": 0, + "error": str(exc), + }) @router.post("/compute/nodes/{node_id}/enable") @@ -842,3 +886,287 @@ async def compute_job_logs(job_id: str) -> dict[str, Any]: except KeyError as exc: raise fail(404, str(exc)) + +# ===================== Model Evaluation ===================== + + +@router.get("/model-eval") +async def model_eval_list() -> dict[str, Any]: + return ok(get_platform_store().eval_tasks()) + + +@router.get("/model-eval/{task_id}") +async def model_eval_detail(task_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().eval_task(task_id)) + except KeyError: + raise fail(404, "eval task not found") + + +@router.post("/model-eval/start") +async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + task = get_platform_store().create_eval_task(payload) + return ok({"task_id": task["id"], **task}) + + +@router.delete("/model-eval/{task_id}") +async def model_eval_delete(task_id: str) -> dict[str, Any]: + get_platform_store().delete_eval_task(task_id) + return ok({"deleted": task_id}) + + +# ===================== Eval Dimensions ===================== + + +@router.get("/dimension") +async def dimension_list() -> dict[str, Any]: + return ok(get_platform_store().dimensions()) + + +@router.post("/dimension") +async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok(get_platform_store().create_dimension(payload)) + + +@router.get("/dimension/{dimension_id}") +async def dimension_detail(dimension_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().dimension(dimension_id)) + except KeyError: + raise fail(404, "dimension not found") + + +@router.put("/dimension/{dimension_id}") +async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_dimension(dimension_id, payload)) + except KeyError: + raise fail(404, "dimension not found") + + +@router.delete("/dimension/{dimension_id}") +async def dimension_delete(dimension_id: str) -> dict[str, Any]: + get_platform_store().delete_dimension(dimension_id) + return ok({"deleted": dimension_id}) + + +# ===================== Model Compare / Inference ===================== + + +@router.get("/model-compare") +async def model_compare_list() -> dict[str, Any]: + return ok(get_platform_store().compare_tasks()) + + +@router.post("/model-compare") +async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + task = get_platform_store().create_compare_task(payload) + return ok({"id": task["id"]}) + + +@router.post("/model-compare/all/stop-all") +async def model_compare_stop_all() -> dict[str, Any]: + return ok({"stopped": True}) + + +@router.post("/model-compare/stop-by-pid") +async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok({"stopped": True, "pid": payload.get("pid")}) + + +@router.get("/model-compare/{task_id}") +async def model_compare_detail(task_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().compare_task(task_id)) + except KeyError: + raise fail(404, "compare task not found") + + +@router.delete("/model-compare/{task_id}") +async def model_compare_delete(task_id: str) -> dict[str, Any]: + get_platform_store().delete_compare_task(task_id) + return ok({"deleted": task_id}) + + +@router.get("/model-compare/{task_id}/load-status") +async def model_compare_load_status(task_id: str) -> dict[str, Any]: + try: + task = get_platform_store().compare_task(task_id) + except KeyError: + raise fail(404, "compare task not found") + load_status = task.get("load_status") or {"loaded_models": []} + if isinstance(load_status, str): + try: + load_status = json.loads(load_status) + except json.JSONDecodeError: + load_status = {"loaded_models": []} + return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status}) + + +@router.post("/model-compare/{task_id}/load-status") +async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}})) + except KeyError: + raise fail(404, "compare task not found") + + +@router.post("/model-compare/{task_id}/load") +async def model_compare_load(task_id: str) -> dict[str, Any]: + try: + task = get_platform_store().compare_task(task_id) + models = task.get("models") or [] + if isinstance(models, str): + try: + models = json.loads(models) + except json.JSONDecodeError: + models = [] + loaded_models = [ + { + "model_id": item.get("model_id"), + "model_name": item.get("model_name"), + "status": "ready", + "pid": 45000 + index, + "port": item.get("port") or 18000 + index, + } + for index, item in enumerate(models) + if isinstance(item, dict) + ] + return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}})) + except KeyError: + raise fail(404, "compare task not found") + + +@router.post("/model-compare/{task_id}/unload") +async def model_compare_unload(task_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})) + except KeyError: + raise fail(404, "compare task not found") + + +@router.post("/model-compare/{task_id}/start-model") +async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id}) + + +@router.post("/model-compare/chat-with-port") +async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + question = "" + for message in payload.get("messages") or []: + if message.get("role") == "user": + question = str(message.get("content") or "") + content = f"当前后端已收到推理请求：{question[:120]}" + return ok({"response": content, "content": content}) + + +@router.post("/model-compare/stream-chat") +async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + question = payload.get("user_question") or payload.get("question") or "" + return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"}) + + +# ===================== Model Chat (Inference Proxy) ===================== + + +@router.post("/model-chat/batch") +async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok({"responses": [], "request": payload}) + + +@router.post("/model-chat/local/chat") +async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Proxy chat to the compute node running the inference model.""" + store = get_platform_store() + node = _select_first_online_node(store) + if not node: + return ok({"response": "no online compute node available for inference", "request": payload}) + try: + client = ComputeNodeClient(node["api_base_url"]) + result = await client._request("POST", "/inference/chat", json_data=payload) + return ok(result) + except Exception as exc: + return ok({"response": f"inference failed: {exc}", "request": payload}) + + +@router.post("/model-chat/local/chat/stream") +async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse: + """Stream chat from the compute node.""" + store = get_platform_store() + node = _select_first_online_node(store) + if not node: + return StreamingResponse( + iter(['data: {"error": "no online compute node"}\n\n']), + media_type="text/event-stream", + ) + client = ComputeNodeClient(node["api_base_url"]) + + async def stream_proxy(): + async with httpx.AsyncClient(timeout=300) as http: + url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream" + async with http.stream("POST", url, json=payload, headers=client.headers()) as resp: + async for chunk in resp.aiter_bytes(): + yield chunk + + return StreamingResponse(stream_proxy(), media_type="text/event-stream") + + +@router.post("/model-chat/local/preload") +async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Load a model on the compute node for inference.""" + store = get_platform_store() + node = _select_first_online_node(store) + if not node: + return ok({"loaded": False, "error": "no online compute node"}) + try: + client = ComputeNodeClient(node["api_base_url"]) + result = await client._request("POST", "/inference/load", json_data=payload) + return ok(result) + except Exception as exc: + return ok({"loaded": False, "error": str(exc)}) + + +@router.post("/model-chat/local/unload") +async def model_chat_local_unload() -> dict[str, Any]: + """Unload the inference model from the compute node.""" + store = get_platform_store() + node = _select_first_online_node(store) + if not node: + return ok({"unloaded": False, "error": "no online compute node"}) + try: + client = ComputeNodeClient(node["api_base_url"]) + result = await client._request("POST", "/inference/unload", json_data={}) + return ok(result) + except Exception as exc: + return ok({"unloaded": False, "error": str(exc)}) + + +@router.get("/model-chat/local/status") +async def model_chat_local_status() -> dict[str, Any]: + """Get inference session status from compute node.""" + store = get_platform_store() + node = _select_first_online_node(store) + if not node: + return ok({"loaded": False, "error": "no online compute node"}) + try: + client = ComputeNodeClient(node["api_base_url"]) + result = await client._request("GET", "/inference/status") + return ok(result) + except Exception as exc: + return ok({"loaded": False, "error": str(exc)}) + + +@router.post("/model-chat/trained/preload") +async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Load a trained model (base + adapter) on the compute node for inference.""" + store = get_platform_store() + node = _select_first_online_node(store) + if not node: + return ok({"loaded": False, "error": "no online compute node"}) + try: + client = ComputeNodeClient(node["api_base_url"]) + result = await client._request("POST", "/inference/load", json_data=payload) + return ok(result) + except Exception as exc: + return ok({"loaded": False, "error": str(exc)}) + diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 048df89..778196c 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -28,6 +28,8 @@ class Settings: compute_mode: str = os.getenv("COMPUTE_MODE", "real") compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling") compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3) + compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "") + compute_request_timeout_seconds: float = float(os.getenv("COMPUTE_REQUEST_TIMEOUT_SECONDS", "30")) log_level: str = os.getenv("LOG_LEVEL", "INFO") log_dir: str = os.getenv("LOG_DIR", "./logs") log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend") diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index 4f9c48b..91d7c8e 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -229,15 +229,21 @@ class PlatformStore: return f"{sec}s" def refresh_runtime_state(self) -> None: - if get_settings().compute_mode != "simulator": - return - with self.connect() as conn: rows = conn.execute( "SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')" ).fetchall() now_dt = datetime.now(timezone.utc) for row in rows: + payload = json_loads(row["payload"], {}) + compute_job_id = payload.get("compute_job_id") + compute_node_api = payload.get("compute_node_api") + if compute_job_id and compute_node_api: + # 已派发到算力：状态/进度/日志回传来自算力进程（架构 §1.1） + self._sync_task_from_compute(conn, row, payload, compute_job_id, compute_node_api) + continue + if get_settings().compute_mode != "simulator": + continue start = parse_time(row["start_time"]) if not start: continue @@ -252,7 +258,6 @@ class PlatformStore: else: status, progress = "completed", 100 - payload = json_loads(row["payload"], {}) payload.update( { "status": status, @@ -272,19 +277,63 @@ class PlatformStore: if status == "completed": self._ensure_trained_model(conn, payload) - sync_rows = conn.execute( - "SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')" - ).fetchall() - for row in sync_rows: - created = parse_time(row["create_time"]) - age = int((now_dt - created).total_seconds()) if created else 0 - status = "completed" if age >= 6 else "running" - progress = 100 if status == "completed" else min(95, 15 + age * 12) - completed_at = row["completed_at"] or (utcnow() if status == "completed" else None) - conn.execute( - "UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?", - (status, progress, completed_at, row["id"]), - ) + if get_settings().compute_mode == "simulator": + sync_rows = conn.execute( + "SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')" + ).fetchall() + for row in sync_rows: + created = parse_time(row["create_time"]) + age = int((now_dt - created).total_seconds()) if created else 0 + status = "completed" if age >= 6 else "running" + progress = 100 if status == "completed" else min(95, 15 + age * 12) + completed_at = row["completed_at"] or (utcnow() if status == "completed" else None) + conn.execute( + "UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?", + (status, progress, completed_at, row["id"]), + ) + + def _sync_task_from_compute( + self, + conn, + row, + payload: dict[str, Any], + compute_job_id: str, + compute_node_api: str, + ) -> None: + """从算力节点拉回已派发任务的状态/进度/日志，写回本地任务记录。""" + try: + from app.modules.compute_gateway.client import ComputeNodeClient + + job = ComputeNodeClient(compute_node_api).get_job(compute_job_id) + except Exception as exc: # noqa: BLE001 + payload["compute_sync_error"] = str(exc) + conn.execute( + "UPDATE fine_tune_tasks SET payload=? WHERE id=?", + (json_dumps(payload), row["id"]), + ) + return + status = job.get("status") + progress = int(job.get("progress", 0) or 0) + logs = job.get("logs") or "" + payload.update( + { + "status": status, + "progress": progress, + "compute_logs": logs, + "train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None), + } + ) + completed_at = row["completed_at"] or (utcnow() if status == "completed" else None) + conn.execute( + """ + UPDATE fine_tune_tasks + SET status=?, progress=?, payload=?, completed_at=? + WHERE id=? + """, + (status, progress, json_dumps(payload), completed_at, row["id"]), + ) + if status == "completed": + self._ensure_trained_model(conn, payload) def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None: name = task.get("output_model_name") or f"{task['name']}-lora" @@ -306,7 +355,7 @@ class PlatformStore: utcnow(), 0, 0, - output_dir or f"/data/yg-ft/outputs/{task['name']}/adapter", + task.get("output_dir") or f"/data/yg-ft/outputs/{task.get('name')}/adapter", ), ) @@ -402,6 +451,79 @@ class PlatformStore: ) return {"id": aid} + # ---- 登录会话（采集在线时长） ---- + def create_session(self, user: dict[str, Any]) -> str: + sid = new_id("sess") + with self.connect() as conn: + conn.execute( + """INSERT INTO sessions + (id, user_id, username, display_name, role, login_at, logout_at, duration_seconds, create_time) + VALUES (?,?,?,?,?,?,?,?,?)""", + ( + sid, + user.get("id"), + user.get("username"), + user.get("display_name"), + user.get("role"), + utcnow(), + None, + None, + utcnow(), + ), + ) + return sid + + def close_session(self, session_id: str | None) -> None: + if not session_id: + return + with self.connect() as conn: + row = conn.execute( + "SELECT login_at FROM sessions WHERE id=? AND logout_at IS NULL", (session_id,) + ).fetchone() + if not row: + return + now = datetime.now(timezone.utc) + start = parse_time(row["login_at"]) or now + seconds = max(0, int((now - start).total_seconds())) + conn.execute( + "UPDATE sessions SET logout_at=?, duration_seconds=? WHERE id=?", + (utcnow(), seconds, session_id), + ) + + def login_duration_rank(self, limit: int = 8) -> list[dict[str, Any]]: + """本月登录时长排行：按用户聚合会话时长（小时）。""" + month_start = utcnow()[:7] + "01T00:00:00Z" + with self.connect() as conn: + rows = conn.execute( + "SELECT user_id, username, display_name, role, login_at, logout_at, duration_seconds " + "FROM sessions WHERE login_at >= ?", + (month_start,), + ).fetchall() + agg: dict[str, dict[str, Any]] = {} + now = datetime.now(timezone.utc) + for r in rows: + uid = r["user_id"] + bucket = agg.setdefault( + uid, + {"user": r["display_name"] or r["username"], "role": r["role"] or "", "total": 0.0, "has": False}, + ) + dur = r["duration_seconds"] + if dur is None and r["logout_at"] is None: + start = parse_time(r["login_at"]) + if start: + dur = max(0, int((now - start).total_seconds())) + if dur is None: + dur = 0 + bucket["total"] += dur + bucket["has"] = True + result = [ + {"user": b["user"], "role": b["role"], "duration": round(b["total"] / 3600, 1)} + for b in agg.values() + if b["has"] + ] + result.sort(key=lambda x: x["duration"], reverse=True) + return result[:limit] + # ---- 审批模板 ---- def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]: tid = payload.get("id") or new_id("tpl") @@ -768,7 +890,7 @@ class PlatformStore: utcnow(), ), ) - return self.model(model_id) + return self.model(model_id) def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]: current = self.model(model_id) @@ -793,7 +915,7 @@ class PlatformStore: model_id, ), ) - return self.model(model_id) + return self.model(model_id) def delete_model(self, model_id: str) -> None: with self.connect() as conn: @@ -1096,20 +1218,80 @@ class PlatformStore: task_id, ), ) - if get_settings().compute_mode != "simulator": + api_base = node.get("api_base_url") + if api_base: + # GPU 计算派发到算力节点进程执行；后端只做调度编排（架构 §1.1） + self._dispatch_to_compute(task_id, node, merged, selected_gpus) + elif get_settings().compute_mode != "simulator": + # 降级路径：未配置算力节点时后端本机执行（违反 §1.1，待移除） from app.modules.fine_tune.service import launch_training launch_training(task_id) return self.task(task_id) - def stop_task(self, task_id: str) -> dict[str, Any]: - task = self.task(task_id) - task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)}) + def _dispatch_to_compute( + self, + task_id: str, + node: dict[str, Any], + task: dict[str, Any], + gpus: list, + ) -> None: + """把训练作业派发到算力节点，GPU 计算在算力进程内执行；后端记录算力 job id。""" + from app.modules.compute_gateway.client import ComputeNodeClient + + cfg = { + "id": f"ft_{task_id}", + "name": task.get("name") or task_id, + "type": "fine_tune", + "gpus": gpus, + "stage": str(task.get("train_type") or "SFT").lower(), + "base_model": task.get("base_model") or "placeholder-base-model", + "dataset": task.get("train_dataset_id") or task.get("dataset") or "placeholder-dataset", + "template": task.get("template") or "qwen", + "train_method": task.get("train_method") or "lora", + "output_dir": f"/data/yg-ft/outputs/{task.get('name') or task_id}/adapter", + "batch_size": int(task.get("batch_size", 2) or 2), + "learning_rate": float(task.get("learning_rate", 0.0002) or 0.0002), + "n_epochs": int(task.get("n_epochs", 3) or 3), + } + client = ComputeNodeClient(node["api_base_url"]) + try: + job = client.create_job(cfg) + except Exception as exc: # noqa: BLE001 + self.update_task_runtime(task_id, status="failed", extra={"dispatch_error": str(exc)}) + raise RuntimeError(f"dispatch training job to compute node failed: {exc}") from exc + compute_job_id = job.get("id") + current = self.task(task_id) + updated = {**current, "compute_job_id": compute_job_id, "compute_node_api": node["api_base_url"]} with self.connect() as conn: conn.execute( - "UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?", + "UPDATE fine_tune_tasks SET payload=? WHERE id=?", + (json_dumps(updated), task_id), + ) + + def stop_task(self, task_id: str) -> dict[str, Any]: + task = self.task(task_id) + # 如果任务已派发到算力节点，先通知算力停止 + compute_job_id = task.get("compute_job_id") + node_id = task.get("compute_node_id") + if compute_job_id and node_id: + try: + from app.modules.compute_gateway.client import ComputeNodeClient + node = next((n for n in self.compute_nodes() if n["id"] == node_id), None) + if node and node.get("api_base_url"): + ComputeNodeClient(node["api_base_url"]).stop_job(compute_job_id) + except Exception: # noqa: BLE001 - best effort stop + pass + task.update({"status": "stopped", "progress": min(task.get("progress", 0), 99)}) + with self.connect() as conn: + conn.execute( + "UPDATE fine_tune_tasks SET status='stopped', payload=?, completed_at=? WHERE id=?", (json_dumps(task), utcnow(), task_id), ) + conn.execute( + "UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')", + (utcnow(), task_id), + ) return self.task(task_id) def update_task_runtime( @@ -1934,6 +2116,490 @@ class PlatformStore: content = self.generate_training_log(task) return {"job_id": job_id, "content": content, "lines": len(content.splitlines())} + # ===================== Model Evaluation ===================== + + def _json_payload_row(self, row: PgRow) -> dict[str, Any]: + payload = json_loads(row["payload"], {}) + payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]}) + return payload + + def eval_tasks(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall() + return [self._json_payload_row(row) for row in rows] + + def eval_task(self, task_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone() + if not row: + raise KeyError(task_id) + payload = self._json_payload_row(row) + payload.setdefault("sample_count", 0) + payload.setdefault("completed_count", 0) + payload.setdefault("passed_count", 0) + payload.setdefault("overall_score", payload.get("score") or 0) + payload.setdefault("overall_score_max", 100) + payload.setdefault("overall_evaluation", "") + payload.setdefault("improvement_suggestions", []) + payload.setdefault("dimension_summary", []) + payload.setdefault("samples", []) + return payload + + def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]: + task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval")) + name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}") + status = str(payload.get("status") or "pending") + now = payload.get("create_time") or utcnow() + data = { + **payload, + "id": task_id, + "eval_task_name": name, + "status": status, + "create_time": now, + "metric": payload.get("metric") or "custom", + } + with self.connect() as conn: + model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone() + dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone() + if model: + data.setdefault("model_name", model["name"]) + if dataset: + data.setdefault("dataset", dataset["name"]) + conn.execute( + "INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)", + (task_id, name, json_dumps(data), status, now), + ) + return self.eval_task(task_id) + + def delete_eval_task(self, task_id: str) -> None: + with self.connect() as conn: + conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,)) + + def dimensions(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall() + return [ + { + **json_loads(row["payload"], {}), + "id": row["id"], + "name": row["name"], + "is_active": bool(row["is_active"]), + "is_default": bool(row["is_default"]), + "create_time": row["create_time"], + } + for row in rows + ] + + def dimension(self, dimension_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() + if not row: + raise KeyError(dimension_id) + return { + **json_loads(row["payload"], {}), + "id": row["id"], + "name": row["name"], + "is_active": bool(row["is_active"]), + "is_default": bool(row["is_default"]), + "create_time": row["create_time"], + } + + def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]: + dimension_id = str(payload.get("id") or new_id("dim")) + name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}") + now = payload.get("create_time") or utcnow() + data = {**payload, "id": dimension_id, "name": name, "create_time": now} + with self.connect() as conn: + conn.execute( + "INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)", + (dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now), + ) + return self.dimension(dimension_id) + + def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]: + current = self.dimension(dimension_id) + merged = {**current, **payload, "id": dimension_id} + with self.connect() as conn: + conn.execute( + "UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?", + ( + merged["name"], + json_dumps(merged), + 1 if merged.get("is_active", True) else 0, + 1 if merged.get("is_default") else 0, + dimension_id, + ), + ) + return self.dimension(dimension_id) + + def delete_dimension(self, dimension_id: str) -> None: + with self.connect() as conn: + conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,)) + + # ===================== Model Compare / Inference ===================== + + def compare_tasks(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall() + return [self._json_payload_row(row) for row in rows] + + def compare_task(self, task_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone() + if not row: + raise KeyError(task_id) + return self._json_payload_row(row) + + def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]: + task_id = str(payload.get("id") or new_id("cmp")) + name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}") + status = str(payload.get("status") or "pending") + now = payload.get("create_time") or utcnow() + data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now} + data.setdefault("load_status", json_dumps({"loaded_models": []})) + with self.connect() as conn: + conn.execute( + "INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)", + (task_id, name, json_dumps(data), status, now), + ) + return self.compare_task(task_id) + + def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: + current = self.compare_task(task_id) + merged = {**current, **payload, "id": task_id} + status = str(merged.get("status") or current.get("status") or "pending") + with self.connect() as conn: + conn.execute( + "UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?", + (merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id), + ) + return self.compare_task(task_id) + + def delete_compare_task(self, task_id: str) -> None: + with self.connect() as conn: + conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,)) + + # ===================== Compute Job Sync (派发回传) ===================== + + def _acquire_scheduler_lock( + self, + conn: PgConnection, + lock_key: str, + owner: str, + ttl_seconds: int = 30, + ) -> bool: + now = utcnow() + expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds)).replace(microsecond=0).isoformat().replace("+00:00", "Z") + existing = conn.execute("SELECT owner, expires_at FROM scheduler_locks WHERE lock_key=?", (lock_key,)).fetchone() + if existing: + if existing["expires_at"] > now and existing["owner"] != owner: + return False + conn.execute( + "UPDATE scheduler_locks SET owner=?, expires_at=?, update_time=? WHERE lock_key=?", + (owner, expires_at, now, lock_key), + ) + else: + conn.execute( + "INSERT INTO scheduler_locks (lock_key, owner, expires_at, create_time, update_time) VALUES (?, ?, ?, ?, ?)", + (lock_key, owner, expires_at, now, now), + ) + return True + + def _upsert_compute_job(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None: + job_id = str(job.get("id") or task.get("compute_job_id") or task["id"]) + now = utcnow() + command = job.get("command") or [] + command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "") + payload = json_dumps({**job, "task_id": task["id"]}) + existing = conn.execute("SELECT id FROM compute_jobs WHERE id=?", (job_id,)).fetchone() + if existing: + conn.execute( + """ + UPDATE compute_jobs + SET task_id=?, node_id=?, engine=?, status=?, command=?, output_dir=?, log_file=?, + payload=?, update_time=?, completed_at=COALESCE(?, completed_at) + WHERE id=? + """, + ( + task["id"], + task.get("compute_node_id"), + str(task.get("engine") or job.get("engine") or "llama_factory"), + status, + command_text, + job.get("output_dir") or task.get("output_dir"), + job.get("log_file") or task.get("log_file"), + payload, + now, + now if status in {"completed", "failed", "stopped"} else None, + job_id, + ), + ) + return + conn.execute( + """ + INSERT INTO compute_jobs + (id, task_id, node_id, engine, status, command, output_dir, log_file, payload, create_time, update_time, completed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + job_id, + task["id"], + task.get("compute_node_id"), + str(task.get("engine") or job.get("engine") or "llama_factory"), + status, + command_text, + job.get("output_dir") or task.get("output_dir"), + job.get("log_file") or task.get("log_file"), + payload, + now, + now, + now if status in {"completed", "failed", "stopped"} else None, + ), + ) + + def _sync_gpu_allocations(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None: + terminal = status in {"completed", "failed", "stopped"} + if terminal: + conn.execute( + "UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')", + (utcnow(), task["id"]), + ) + return + job_id = str(job.get("id") or task.get("compute_job_id") or task["id"]) + allocation_status = "running" if status == "running" else "allocated" + for gpu_index in [int(item) for item in task.get("gpus") or job.get("gpus") or []]: + existing = conn.execute( + "SELECT id FROM gpu_allocations WHERE task_id=? AND node_id=? AND gpu_index=? AND status IN ('allocated','running')", + (task["id"], task.get("compute_node_id"), gpu_index), + ).fetchone() + if existing: + conn.execute("UPDATE gpu_allocations SET status=?, compute_job_id=? WHERE id=?", (allocation_status, job_id, existing["id"])) + continue + conn.execute( + """ + INSERT INTO gpu_allocations + (id, task_id, compute_job_id, node_id, gpu_index, status, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + (new_id("gpu_alloc"), task["id"], job_id, task.get("compute_node_id"), gpu_index, allocation_status, utcnow()), + ) + + def _upsert_checkpoints(self, conn: PgConnection, task_id: str, checkpoints: list[dict[str, Any]]) -> None: + for item in checkpoints: + path = str(item.get("path") or "") + if not path: + continue + step = int(item.get("step") or 0) + name = str(item.get("name") or Path(path).name) + size_bytes = int(item.get("size_bytes") or item.get("size") or 0) + existing = conn.execute("SELECT id FROM fine_tune_checkpoints WHERE task_id=? AND path=?", (task_id, path)).fetchone() + if existing: + conn.execute( + "UPDATE fine_tune_checkpoints SET step=?, name=?, size_bytes=? WHERE id=?", + (step, name, size_bytes, existing["id"]), + ) + continue + conn.execute( + """ + INSERT INTO fine_tune_checkpoints + (id, task_id, step, name, path, size_bytes, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + (new_id("ckpt"), task_id, step, name, path, size_bytes, utcnow()), + ) + + def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]: + status_map = { + "queued": "queued", + "running": "running", + "completed": "completed", + "failed": "failed", + "stopped": "stopped", + } + current = self.task(task_id) + status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"])) + progress = int(job.get("progress", current.get("progress", 0)) or 0) + payload = { + **current, + "status": status, + "progress": progress, + "process_id": job.get("pid") or current.get("process_id"), + "compute_job_id": job.get("id") or current.get("compute_job_id"), + "output_dir": job.get("output_dir") or current.get("output_dir"), + "log_file": job.get("log_file") or current.get("log_file"), + "artifacts": job.get("artifacts") or current.get("artifacts") or [], + } + if status == "failed": + payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed" + elif status in {"queued", "running", "completed"}: + payload.pop("failure_reason", None) + completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None + with self.connect() as conn: + conn.execute( + """ + UPDATE fine_tune_tasks + SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at) + WHERE id=? + """, + ( + json_dumps(payload), + status, + progress, + payload.get("process_id"), + payload.get("compute_job_id"), + completed_at, + task_id, + ), + ) + self._upsert_compute_job(conn, payload, job, status) + self._sync_gpu_allocations(conn, payload, job, status) + self._upsert_checkpoints(conn, task_id, job.get("checkpoints") or []) + if status == "completed": + self._ensure_trained_model(conn, payload) + if status in {"failed", "stopped"}: + failure_reason = job.get("error") or job.get("message") or "compute job failed" + log_snippet = job.get("log_snippet") or "" + conn.execute( + "UPDATE fine_tune_tasks SET failure_reason = ? WHERE id = ?", + (failure_reason[:2000], task_id), + ) + if log_snippet: + conn.execute( + "UPDATE fine_tune_tasks SET payload = ? WHERE id = ?", + (json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id), + ) + return self.task(task_id) + + def running_compute_tasks(self) -> list[dict[str, Any]]: + return [ + task + for task in self.tasks() + if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"} + ] + + def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]: + task = self.task(task_id) + task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason}) + with self.connect() as conn: + conn.execute( + "UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?", + (json_dumps(task), utcnow(), task_id), + ) + conn.execute( + "UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')", + (utcnow(), task_id), + ) + return self.task(task_id) + + def record_training_log_metrics(self, task_id: str, content: str) -> int: + rows: list[tuple[Any, ...]] = [] + for line_number, line in enumerate(content.splitlines(), start=1): + metric = self._parse_training_metric(line) + if not metric: + continue + rows.append( + ( + new_id("metric"), + task_id, + line_number, + metric.get("epoch"), + metric.get("loss"), + metric.get("grad_norm"), + metric.get("learning_rate"), + line[:2000], + utcnow(), + ) + ) + with self.connect() as conn: + conn.execute("DELETE FROM fine_tune_metrics WHERE task_id=?", (task_id,)) + if rows: + conn.executemany( + """ + INSERT INTO fine_tune_metrics + (id, task_id, step, epoch, loss, grad_norm, learning_rate, raw, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + rows, + ) + return len(rows) + + def _parse_training_metric(self, line: str) -> dict[str, Any] | None: + if "loss" not in line or "learning_rate" not in line: + return None + import re + result: dict[str, Any] = {} + for key in ["loss", "grad_norm", "learning_rate", "epoch"]: + match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line) + if match: + result[key] = float(match.group(1)) + return result or None + + def task_metrics(self, task_id: str) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT step, epoch, loss, grad_norm, learning_rate, raw, create_time + FROM fine_tune_metrics + WHERE task_id=? + ORDER BY step + """, + (task_id,), + ).fetchall() + return [dict(row) for row in rows] + + def task_checkpoints(self, task_id: str) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT id, step, name, path, size_bytes, create_time + FROM fine_tune_checkpoints + WHERE task_id=? + ORDER BY step, create_time + """, + (task_id,), + ).fetchall() + return [dict(row) for row in rows] + + def active_standalone_compute_jobs(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT * FROM compute_jobs + WHERE task_id IS NULL AND status IN ('queued','running') + ORDER BY create_time + """, + ).fetchall() + return [json_loads(row["payload"], {}) if "payload" in row.keys() else dict(row) for row in rows] + + def sync_model_merge_job(self, job_id: str, job: dict[str, Any]) -> dict[str, Any]: + current = self.compute_job(job_id) + payload = current.get("payload") or {} + job_payload = payload.get("job") if isinstance(payload.get("job"), dict) else {} + merged_payload = {**payload, "job": {**job_payload, **job}} + status = str(job.get("status") or current.get("status") or "queued") + command = job.get("command") or current.get("command") or [] + command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "") + output_dir = job.get("output_dir") or payload.get("output_dir") or current.get("output_dir") + with self.connect() as conn: + conn.execute( + """ + UPDATE compute_jobs + SET status=?, command=?, output_dir=?, log_file=?, payload=?, update_time=?, completed_at=COALESCE(?, completed_at) + WHERE id=? + """, + ( + status, + command_text, + output_dir, + job.get("log_file") or current.get("log_file"), + json_dumps(merged_payload), + utcnow(), + utcnow() if status in {"completed", "failed", "stopped"} else None, + job_id, + ), + ) + return self.compute_job(job_id) + _store: PlatformStore | None = None diff --git a/backend/app/db/sql/001_platform_runtime.sql b/backend/app/db/sql/001_platform_runtime.sql index 2133661..bd54d34 100644 --- a/backend/app/db/sql/001_platform_runtime.sql +++ b/backend/app/db/sql/001_platform_runtime.sql @@ -152,6 +152,18 @@ CREATE TABLE IF NOT EXISTS project_members ( -- ===================== Fine-tune Checkpoints ===================== +CREATE TABLE IF NOT EXISTS fine_tune_metrics ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE, + step INTEGER NOT NULL, + epoch DOUBLE PRECISION, + loss DOUBLE PRECISION, + grad_norm DOUBLE PRECISION, + learning_rate DOUBLE PRECISION, + raw TEXT NOT NULL, + create_time TEXT NOT NULL +); + CREATE TABLE IF NOT EXISTS fine_tune_checkpoints ( id TEXT PRIMARY KEY, task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE, @@ -164,6 +176,25 @@ CREATE TABLE IF NOT EXISTS fine_tune_checkpoints ( create_time TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS gpu_allocations ( + id TEXT PRIMARY KEY, + task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE CASCADE, + compute_job_id TEXT, + node_id TEXT REFERENCES compute_nodes(id) ON DELETE CASCADE, + gpu_index INTEGER NOT NULL, + status TEXT NOT NULL, + create_time TEXT NOT NULL, + released_at TEXT +); + +CREATE TABLE IF NOT EXISTS scheduler_locks ( + lock_key TEXT PRIMARY KEY, + owner TEXT NOT NULL, + expires_at TEXT NOT NULL, + create_time TEXT NOT NULL, + update_time TEXT NOT NULL +); + -- ===================== Compute Jobs (internal) ===================== CREATE TABLE IF NOT EXISTS compute_jobs ( @@ -185,6 +216,33 @@ CREATE TABLE IF NOT EXISTS compute_jobs ( completed_at TEXT ); +-- ===================== Model Evaluation ===================== + +CREATE TABLE IF NOT EXISTS eval_tasks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL, + create_time TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS eval_dimensions ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + payload TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + is_default INTEGER NOT NULL DEFAULT 0, + create_time TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS compare_tasks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL, + create_time TEXT NOT NULL +); + -- ===================== Indexes ===================== CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status); @@ -194,9 +252,15 @@ CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_t CREATE INDEX IF NOT EXISTS idx_projects_tenant ON projects(tenant_id); CREATE INDEX IF NOT EXISTS idx_project_members_user ON project_members(user_id); CREATE INDEX IF NOT EXISTS idx_checkpoints_task ON fine_tune_checkpoints(task_id); +CREATE INDEX IF NOT EXISTS idx_fine_tune_metrics_task_step ON fine_tune_metrics(task_id, step); +CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status); +CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at); CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id); CREATE INDEX IF NOT EXISTS idx_compute_jobs_node ON compute_jobs(node_id); CREATE INDEX IF NOT EXISTS idx_compute_jobs_status ON compute_jobs(status); +CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status); +CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active); +CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status); -- ===================== Migrations: extend fine_tune_tasks ===================== diff --git a/backend/app/db/sql/002_governance.sql b/backend/app/db/sql/002_governance.sql index c335c9e..ec38622 100644 --- a/backend/app/db/sql/002_governance.sql +++ b/backend/app/db/sql/002_governance.sql @@ -127,3 +127,16 @@ INSERT INTO roles (id, name, display_name, permissions, create_time) VALUES ('role_operator','operator','操作员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', '2026-01-01T00:00:00Z'), ('role_viewer', 'viewer', '访客', '["dashboard"]', '2026-01-01T00:00:00Z') ON CONFLICT (name) DO NOTHING; + +-- 登录会话：采集每次登录/登出，用于统计在线/登录时长 +CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + username TEXT NOT NULL, + display_name TEXT, + role TEXT, + login_at TEXT NOT NULL, + logout_at TEXT, + duration_seconds INTEGER, + create_time TEXT NOT NULL +); diff --git a/backend/app/main.py b/backend/app/main.py index 37955e7..342ef24 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,3 +1,5 @@ +from contextlib import asynccontextmanager + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -6,11 +8,20 @@ from app.core.config import get_settings from app.core.logging import configure_logging, setup_request_logging +@asynccontextmanager +async def _lifespan(app: FastAPI): + # 启动算力状态轮询线程（同步派发到算力的训练任务状态/日志/指标） + from app.modules.fine_tune.service import start_compute_sync_worker, stop_compute_sync_worker + start_compute_sync_worker() + yield + stop_compute_sync_worker() + + def create_app() -> FastAPI: settings = get_settings() configure_logging(settings) - app = FastAPI(title=settings.app_name) + app = FastAPI(title=settings.app_name, lifespan=_lifespan) app.add_middleware( CORSMiddleware, allow_origins=settings.cors_allow_origins, diff --git a/backend/app/modules/auth/router.py b/backend/app/modules/auth/router.py index 1c099cd..9caedd3 100644 --- a/backend/app/modules/auth/router.py +++ b/backend/app/modules/auth/router.py @@ -16,13 +16,25 @@ class LoginBody(BaseModel): password: str +class LogoutBody(BaseModel): + session_id: str | None = None + + @router.post("/login") def login(body: LoginBody) -> dict: - user = get_platform_store().login(body.username, body.password) + store = get_platform_store() + user = store.login(body.username, body.password) if not user: raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误") token = create_access_token(user["id"]) - return {"code": 0, "message": "ok", "data": {"token": token, "user": user}} + session_id = store.create_session(user) + return {"code": 0, "message": "ok", "data": {"token": token, "user": user, "session_id": session_id}} + + +@router.post("/logout") +def logout(body: LogoutBody) -> dict: + get_platform_store().close_session(body.session_id) + return {"code": 0, "message": "ok", "data": None} @router.get("/me") diff --git a/backend/app/modules/compute_gateway/__init__.py b/backend/app/modules/compute_gateway/__init__.py index 70436ed..101f8f0 100644 --- a/backend/app/modules/compute_gateway/__init__.py +++ b/backend/app/modules/compute_gateway/__init__.py @@ -1 +1,6 @@ """Application-side compute platform gateway module.""" + +from app.modules.compute_gateway.client import ComputeNodeClient +from app.modules.compute_gateway.sync import poll_compute_jobs_once + +__all__ = ["ComputeNodeClient", "poll_compute_jobs_once"] diff --git a/backend/app/modules/compute_gateway/client.py b/backend/app/modules/compute_gateway/client.py new file mode 100644 index 0000000..eadccf2 --- /dev/null +++ b/backend/app/modules/compute_gateway/client.py @@ -0,0 +1,180 @@ +"""Client for talking to a compute node's REST API. + +This is the application-side bridge: the application backend never runs GPU +workloads itself; it dispatches them to a compute node and reads back status, +logs and metrics through this client. +""" +from __future__ import annotations + +import httpx +from app.core.config import get_settings + + +def _compute_timeout() -> float: + return float(get_settings().compute_request_timeout_seconds or 30.0) + + +def _auth_headers() -> dict: + token = get_settings().compute_service_token + if not token: + return {} + return {"Authorization": f"Bearer {token}"} + + +class ComputeNodeClient: + """Thin wrapper over a single compute node's HTTP API.""" + + def __init__(self, base_url: str, timeout: float | None = None): + self.base_url = base_url.rstrip("/") + self.timeout = timeout or _compute_timeout() + + # ---- health ------------------------------------------------------- + def health(self) -> dict: + last_err = None + for path in ("/v1/compute/health", "/health"): + try: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.get(f"{self.base_url}{path}", headers=_auth_headers()) + if r.status_code == 200: + return r.json() + except Exception as exc: # noqa: BLE001 + last_err = str(exc) + raise RuntimeError(f"compute node unhealthy: {last_err}") + + # ---- gpus --------------------------------------------------------- + def gpus(self) -> list: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.get( + f"{self.base_url}/compute/resources/gpus", + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json().get("data", []) + + # ---- jobs --------------------------------------------------------- + def create_job(self, payload: dict) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/jobs", + json=payload, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def preview_job(self, payload: dict) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/jobs/preview", + json=payload, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def validate_job(self, payload: dict) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/jobs/validate", + json=payload, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def get_job(self, job_id: str) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.get( + f"{self.base_url}/compute/jobs/{job_id}", + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def stop_job(self, job_id: str) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/jobs/{job_id}/stop", + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def job_logs(self, job_id: str, cursor: int = 0, limit: int = 200) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.get( + f"{self.base_url}/compute/jobs/{job_id}/logs", + params={"cursor": cursor, "limit": limit}, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + # ---- files -------------------------------------------------------- + def check_paths(self, paths: list) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/files/check-paths", + json={"paths": paths}, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def list_files(self, path: str = "/") -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.get( + f"{self.base_url}/compute/files/list", + params={"path": path}, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def import_local_file(self, src_path: str, dest_name: str | None = None) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/files/import-local", + json={"src_path": src_path, "dest_name": dest_name}, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + def upload_file(self, filename: str, content: bytes, content_type: str | None = None) -> dict: + with httpx.Client(timeout=self.timeout, verify=False) as c: + r = c.post( + f"{self.base_url}/compute/files/upload", + files={"file": (filename, content, content_type)}, + headers=_auth_headers(), + ) + r.raise_for_status() + return r.json() + + # ---- inference (async) ------------------------------------------- + + def headers(self) -> dict[str, str]: + """Return auth headers for compute node requests.""" + token = get_settings().compute_service_token + if not token: + return {} + return {"X-Compute-Token": token} + + @property + def route_prefix(self) -> str: + return get_settings().route_prefix.rstrip("/") or "/modelTF" + + async def _request(self, method: str, path: str, json_data: dict | None = None) -> dict: + """Generic async request method for compute API endpoints.""" + prefix = self.route_prefix + url = f"{self.base_url.rstrip('/')}{prefix}{path}" + async with httpx.AsyncClient(timeout=300, verify=False, headers=self.headers()) as client: + if method.upper() == "GET": + response = await client.get(url) + else: + response = await client.post(url, json=json_data) + response.raise_for_status() + data = response.json() + if isinstance(data, dict) and isinstance(data.get("data"), dict): + return data["data"] + return data if isinstance(data, dict) else {} diff --git a/backend/app/modules/compute_gateway/sync.py b/backend/app/modules/compute_gateway/sync.py new file mode 100644 index 0000000..5e8b0fc --- /dev/null +++ b/backend/app/modules/compute_gateway/sync.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Any + +from app.db.platform_store import get_platform_store +from app.modules.compute_gateway.client import ComputeNodeClient + + +def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None: + return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None) + + +async def poll_compute_jobs_once() -> dict[str, Any]: + store = get_platform_store() + synced: list[dict[str, Any]] = [] + failed: list[dict[str, str]] = [] + for task in store.running_compute_tasks(): + node = _node_for_task(task) + if not node: + failed.append({"task_id": task["id"], "error": "compute node not found"}) + continue + try: + client = ComputeNodeClient(node["api_base_url"]) + job = client.get_job(task["compute_job_id"]) + try: + logs = client.job_logs(task["compute_job_id"], tail_lines=5000) + store.record_training_log_metrics(task["id"], str(logs.get("content") or "")) + except Exception: + pass + if job.get("status") in {"failed", "stopped"}: + try: + last_logs = client.job_logs(task["compute_job_id"], tail_lines=200) + job["log_snippet"] = str(last_logs.get("content") or "")[:8192] + except Exception: + pass + synced.append(store.apply_compute_job(task["id"], job)) + except Exception as exc: # noqa: BLE001 - keep polling other jobs + failed.append({"task_id": task["id"], "error": str(exc)}) + standalone_synced: list[dict[str, Any]] = [] + for record in store.active_standalone_compute_jobs(): + node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None) + if not node: + failed.append({"job_id": record["id"], "error": "compute node not found"}) + continue + try: + job = ComputeNodeClient(node["api_base_url"]).get_job(record["id"]) + standalone_synced.append(store.sync_model_merge_job(record["id"], job)) + except Exception as exc: # noqa: BLE001 - keep polling other jobs + failed.append({"job_id": record["id"], "error": str(exc)}) + return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced} diff --git a/backend/app/modules/fine_tune/service.py b/backend/app/modules/fine_tune/service.py index 61ce10c..93c6734 100644 --- a/backend/app/modules/fine_tune/service.py +++ b/backend/app/modules/fine_tune/service.py @@ -3,13 +3,17 @@ - preset 参数预设（quick / standard / high） - train_type → stage 映射（sft/dpo/cpt/cot） -- 训练任务的启动 / 暂停 / 恢复 / 取消（委托 runner 真实执行） +- 训练任务的启动 / 暂停 / 恢复 / 取消 +- compute_gateway 状态轮询线程（当任务派发到算力时自动同步状态） """ from __future__ import annotations +import asyncio import threading +import time from typing import Any +from app.core.config import get_settings from app.db.platform_store import get_platform_store from app.modules.fine_tune import runner @@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = { "high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32}, } +# ── compute sync 轮询线程 ────────────────────────────────────────────── +_sync_thread: threading.Thread | None = None +_sync_thread_stop = threading.Event() + +def _compute_sync_loop() -> None: + """后台线程：周期性轮询算力节点，同步训练任务状态/日志/指标。""" + interval = get_settings().compute_poll_interval_seconds or 3 + while not _sync_thread_stop.is_set(): + try: + store = get_platform_store() + running = store.running_compute_tasks() + if running: + asyncio.run(_poll_once()) + except Exception: # noqa: BLE001 - keep polling loop alive + pass + time.sleep(interval) + + +async def _poll_once() -> None: + from app.modules.compute_gateway.sync import poll_compute_jobs_once + await poll_compute_jobs_once() + + +def start_compute_sync_worker() -> None: + """启动后台轮询线程（幂等，多次调用安全）。""" + global _sync_thread + if _sync_thread is not None and _sync_thread.is_alive(): + return + _sync_thread_stop.clear() + _sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True) + _sync_thread.start() + + +def stop_compute_sync_worker() -> None: + """停止后台轮询线程。""" + _sync_thread_stop.set() + + +# ── preset / config ──────────────────────────────────────────────────── def apply_presets(payload: dict[str, Any]) -> dict[str, Any]: """根据 preset 字段补全缺失的超参；preset=custom 时不覆盖。""" payload = dict(payload) @@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]: def launch_training(task_id: str) -> None: - """在后台线程启动真实训练。""" + """在后台线程启动真实训练（本机 subprocess fallback，当算力节点不可用时使用）。 + + 架构原则：GPU 计算应派发到算力服务进程执行。 + 当 platform_store.start_task 检测到在线算力节点时，会走 _dispatch_to_compute 派发路径； + 仅当无可用算力节点且非 simulator 模式时，降级到本机 runner（违反 §1.1，待移除）。 + """ threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start() diff --git a/backend/data/dataset_info.json b/backend/data/dataset_info.json new file mode 100644 index 0000000..18c799a --- /dev/null +++ b/backend/data/dataset_info.json @@ -0,0 +1,5 @@ +{ + "ds_0b95db886181": { + "file_name": "fine_tune_datasets/ds_0b95db886181.json" + } +} \ No newline at end of file diff --git a/backend/data/fine_tune_datasets/ds_0b95db886181.json b/backend/data/fine_tune_datasets/ds_0b95db886181.json new file mode 100644 index 0000000..fc295bc --- /dev/null +++ b/backend/data/fine_tune_datasets/ds_0b95db886181.json @@ -0,0 +1,16 @@ + [ { + "instruction": "数字钱包的“零余额”管理在日末结算时有何具体要求？", + "input": "", + "output": "数字钱包严格执行“零余额”管理，在日末结算时，若结算钱包存在余额，必须将数字货币兑回基本账户，以确保日末数字钱包无余额。" + }, + { + "instruction": "用印登记表中包含哪些具体的印章类型？", + "input": "", + "output": "根据用印登记表的内容显示，表中明确列出了五种具体的印章类型，分别是财务专用章、法人名章、法人授权人名章1、法人授权人名章2以及其他。这些印章类型被详细划分在“用印数量”这一栏目下，用于记录不同印章的具体使用情况。" + }, + { + "instruction": "密钥交接记录表中需要哪些角色签字确认？", + "input": "", + "output": "密钥交接记录表中需要三个角色的签字确认，分别是交接人签字、接交人签字以及监交人签字。交接人负责移交密钥，接交人负责接收密钥，而监交人则负责对整个密钥交接过程进行监督，这三个角色的共同签字确认能够确保密钥交接流程的规范性与安全性。" + } +] \ No newline at end of file diff --git a/backend/storage/data-process/dpt_4a693794cb6f4488a156/dpsf_6fbb337dd6a146159e7a/v1/111.json b/backend/storage/data-process/dpt_4a693794cb6f4488a156/dpsf_6fbb337dd6a146159e7a/v1/111.json new file mode 100644 index 0000000..fc295bc --- /dev/null +++ b/backend/storage/data-process/dpt_4a693794cb6f4488a156/dpsf_6fbb337dd6a146159e7a/v1/111.json @@ -0,0 +1,16 @@ + [ { + "instruction": "数字钱包的“零余额”管理在日末结算时有何具体要求？", + "input": "", + "output": "数字钱包严格执行“零余额”管理，在日末结算时，若结算钱包存在余额，必须将数字货币兑回基本账户，以确保日末数字钱包无余额。" + }, + { + "instruction": "用印登记表中包含哪些具体的印章类型？", + "input": "", + "output": "根据用印登记表的内容显示，表中明确列出了五种具体的印章类型，分别是财务专用章、法人名章、法人授权人名章1、法人授权人名章2以及其他。这些印章类型被详细划分在“用印数量”这一栏目下，用于记录不同印章的具体使用情况。" + }, + { + "instruction": "密钥交接记录表中需要哪些角色签字确认？", + "input": "", + "output": "密钥交接记录表中需要三个角色的签字确认，分别是交接人签字、接交人签字以及监交人签字。交接人负责移交密钥，接交人负责接收密钥，而监交人则负责对整个密钥交接过程进行监督，这三个角色的共同签字确认能够确保密钥交接流程的规范性与安全性。" + } +] \ No newline at end of file diff --git a/compute/agent/process_manager.py b/compute/agent/process_manager.py new file mode 100644 index 0000000..27b94cc --- /dev/null +++ b/compute/agent/process_manager.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import os +import json +import contextlib +import hashlib +import signal +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +TERMINAL_STATUSES = {"completed", "failed", "stopped"} + + +@dataclass +class ManagedProcess: + id: str + name: str + command: list[str] + work_dir: str + log_path: Path + output_dir: str + gpus: list[int] + process: subprocess.Popen[Any] | None + created_at: float + pid: int | None = None + status: str = "running" + progress: int = 5 + artifacts: list[dict[str, Any]] = field(default_factory=list) + + +class ProcessManager: + def __init__(self, log_root: str) -> None: + self.log_root = Path(log_root) + self.log_root.mkdir(parents=True, exist_ok=True) + self.registry_path = self.log_root / "compute-jobs.json" + self.jobs: dict[str, ManagedProcess] = {} + self._load_registry() + + def create_job(self, payload: dict[str, Any], command: list[str], work_dir: str) -> dict[str, Any]: + job_id = str(payload.get("id") or f"job_{int(time.time() * 1000)}") + if job_id in self.jobs and self.jobs[job_id].status not in TERMINAL_STATUSES: + raise ValueError(f"job {job_id} is already running") + + output_dir = str(payload.get("output_dir") or f"/data/yg-ft/outputs/{payload.get('name', job_id)}") + Path(output_dir).mkdir(parents=True, exist_ok=True) + log_path = self.log_root / f"{job_id}.log" + env = os.environ.copy() + gpus = [int(item) for item in payload.get("gpus") or []] + locked = self.locked_gpus() + conflict = sorted(set(gpus).intersection(locked)) + if conflict: + raise ValueError(f"gpu already locked: {conflict}") + if gpus: + env["CUDA_VISIBLE_DEVICES"] = ",".join(str(item) for item in gpus) + env.update({str(k): str(v) for k, v in payload.get("env", {}).items()}) + + cwd = work_dir if Path(work_dir).exists() else None + with log_path.open("ab") as log_file: + log_file.write(f"[INFO] starting job_id={job_id} command={' '.join(command)}\n".encode("utf-8")) + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + ) + + managed = ManagedProcess( + id=job_id, + name=str(payload.get("name") or job_id), + command=command, + work_dir=work_dir, + log_path=log_path, + output_dir=output_dir, + gpus=gpus, + process=process, + created_at=time.time(), + pid=process.pid, + progress=10, + ) + self.jobs[job_id] = managed + data = self.serialize(managed) + self._save_registry() + return data + + def get_job(self, job_id: str) -> dict[str, Any] | None: + job = self.jobs.get(job_id) + if not job: + return None + return self.serialize(job) + + def list_jobs(self) -> list[dict[str, Any]]: + return [self.serialize(job) for job in self.jobs.values()] + + def stop_job(self, job_id: str) -> dict[str, Any] | None: + job = self.jobs.get(job_id) + if not job: + return None + if job.status not in TERMINAL_STATUSES: + try: + if job.process is not None and os.name == "nt": + job.process.terminate() + elif job.pid is not None: + os.kill(job.pid, signal.SIGTERM) + if job.process is not None: + job.process.wait(timeout=10) + except Exception: + if job.process is not None: + job.process.kill() + elif job.pid is not None: + with contextlib.suppress(Exception): + os.kill(job.pid, signal.SIGKILL) + job.status = "stopped" + job.progress = min(job.progress, 99) + data = self.serialize(job) + self._save_registry() + return data + + def logs(self, job_id: str) -> str: + job = self.jobs.get(job_id) + if not job or not job.log_path.exists(): + return "" + return job.log_path.read_text(encoding="utf-8", errors="replace") + + def serialize(self, job: ManagedProcess) -> dict[str, Any]: + code = job.process.poll() if job.process is not None else None + checkpoints = self._collect_checkpoints(job.output_dir) + if job.status not in TERMINAL_STATUSES: + if job.process is None and job.pid is not None and not self._pid_alive(job.pid): + job.status = "failed" + job.progress = min(job.progress, 99) + code = -1 + elif code is None: + job.status = "running" + elapsed = max(0, int(time.time() - job.created_at)) + job.progress = min(95, max(job.progress, 10 + elapsed // 6)) + elif code == 0: + job.status = "completed" + job.progress = 100 + job.artifacts = self._collect_artifacts(job.output_dir) + else: + job.status = "failed" + job.progress = min(job.progress, 99) + self._save_registry() + return { + "id": job.id, + "name": job.name, + "status": job.status, + "progress": job.progress, + "pid": job.pid, + "gpus": job.gpus, + "created_at": job.created_at, + "command": job.command, + "work_dir": job.work_dir, + "output_dir": job.output_dir, + "log_file": str(job.log_path), + "artifacts": job.artifacts, + "checkpoints": checkpoints, + "return_code": code, + } + + def locked_gpus(self) -> set[int]: + locked: set[int] = set() + for job in self.jobs.values(): + status = self.serialize(job)["status"] + if status in {"queued", "running"}: + locked.update(job.gpus) + return locked + + def _collect_artifacts(self, output_dir: str) -> list[dict[str, Any]]: + root = Path(output_dir) + if not root.exists(): + return [] + artifacts: list[dict[str, Any]] = [] + for path in root.rglob("*"): + if path.is_file(): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + size = path.stat().st_size + artifacts.append( + { + "path": str(path), + "name": path.name, + "size": size, + "size_bytes": size, + "checksum_sha256": digest.hexdigest(), + } + ) + return artifacts[:200] + + def _collect_checkpoints(self, output_dir: str) -> list[dict[str, Any]]: + root = Path(output_dir) + if not root.exists(): + return [] + checkpoints: list[dict[str, Any]] = [] + for path in root.glob("checkpoint-*"): + if not path.is_dir(): + continue + step = 0 + try: + step = int(path.name.rsplit("-", 1)[-1]) + except ValueError: + step = 0 + size_bytes = sum(item.stat().st_size for item in path.rglob("*") if item.is_file()) + checkpoints.append( + { + "step": step, + "name": path.name, + "path": str(path), + "size_bytes": size_bytes, + "create_time": path.stat().st_mtime, + } + ) + return sorted(checkpoints, key=lambda item: (int(item.get("step") or 0), str(item.get("name") or ""))) + + def _save_registry(self) -> None: + items = [] + for job in self.jobs.values(): + items.append( + { + "id": job.id, + "name": job.name, + "command": job.command, + "work_dir": job.work_dir, + "log_path": str(job.log_path), + "output_dir": job.output_dir, + "gpus": job.gpus, + "pid": job.pid, + "created_at": job.created_at, + "status": job.status, + "progress": job.progress, + "artifacts": job.artifacts, + } + ) + self.registry_path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8") + + def _load_registry(self) -> None: + if not self.registry_path.exists(): + return + try: + items = json.loads(self.registry_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return + for item in items if isinstance(items, list) else []: + if not isinstance(item, dict): + continue + pid = item.get("pid") + status = item.get("status", "failed") + if status not in TERMINAL_STATUSES and pid and not self._pid_alive(int(pid)): + status = "failed" + job = ManagedProcess( + id=str(item["id"]), + name=str(item.get("name") or item["id"]), + command=[str(part) for part in item.get("command") or []], + work_dir=str(item.get("work_dir") or ""), + log_path=Path(item.get("log_path") or self.log_root / f"{item['id']}.log"), + output_dir=str(item.get("output_dir") or ""), + gpus=[int(gpu) for gpu in item.get("gpus") or []], + process=None, + pid=int(pid) if pid else None, + created_at=float(item.get("created_at") or time.time()), + status=status, + progress=int(item.get("progress") or 0), + artifacts=item.get("artifacts") or [], + ) + self.jobs[job.id] = job + + def _pid_alive(self, pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + return True + except OSError: + return False diff --git a/compute/api/main.py b/compute/api/main.py index b90201a..31d9bb8 100644 --- a/compute/api/main.py +++ b/compute/api/main.py @@ -2,19 +2,39 @@ from __future__ import annotations import os import math +import hashlib +import shutil +import subprocess import time from pathlib import Path from typing import Any -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile +from fastapi.responses import FileResponse, JSONResponse, StreamingResponse -from compute.engines.llama_factory.adapter import build_command, parse_log_line +from compute.agent.process_manager import ProcessManager +from compute.engines.llama_factory.adapter import build_command, parse_log_line, prepare_runtime_files +from compute.engines.llama_factory.inference import get_inference_session def create_app() -> FastAPI: app = FastAPI(title="YG Fine-Tune Compute API") jobs: dict[str, dict[str, Any]] = {} route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF" + process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training")) + + @app.middleware("http") + async def compute_token_auth(request: Request, call_next): + token = os.getenv("COMPUTE_SERVICE_TOKEN", "") + auth_enabled = os.getenv("COMPUTE_AUTH_ENABLED", "true").lower() == "true" + public_paths = {f"{route_prefix}/health", "/health"} + if auth_enabled and token and request.url.path not in public_paths: + header_token = request.headers.get("x-compute-token", "") + auth_header = request.headers.get("authorization", "") + bearer_token = auth_header.removeprefix("Bearer ").strip() if auth_header.startswith("Bearer ") else "" + if header_token != token and bearer_token != token: + return JSONResponse({"detail": "invalid compute service token"}, status_code=401) + return await call_next(request) def now() -> float: return time.time() @@ -25,6 +45,110 @@ def create_app() -> FastAPI: def execution_mode() -> str: return os.getenv("COMPUTE_EXECUTION_MODE", os.getenv("COMPUTE_MODE", "real")).lower() + def _int_env(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None or raw == "": + return default + return int(raw) + + def _float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None or raw == "": + return default + return float(raw) + + def _path_inside(root: Path, candidate: Path) -> bool: + try: + candidate.resolve().relative_to(root.resolve()) + return True + except ValueError: + return False + + def _llama_factory_version() -> str: + for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]): + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=5) + except Exception: + continue + output = (result.stdout or result.stderr).strip() + if result.returncode == 0 and output: + return output.splitlines()[0][:120] + return "" + + def torch_cuda_status() -> dict[str, Any]: + try: + import torch # type: ignore[import-not-found] + except Exception as exc: # noqa: BLE001 - keep health endpoint resilient + return { + "available": False, + "device_count": 0, + "torch_version": "", + "torch_cuda_version": "", + "error": f"torch import failed: {exc}", + } + try: + available = bool(torch.cuda.is_available()) + device_count = int(torch.cuda.device_count()) + devices = [] + for index in range(device_count): + props = torch.cuda.get_device_properties(index) + devices.append( + { + "index": index, + "name": props.name, + "memory_total_gb": round(props.total_memory / 1024 / 1024 / 1024, 2), + } + ) + return { + "available": available, + "device_count": device_count, + "torch_version": str(torch.__version__), + "torch_cuda_version": str(torch.version.cuda or ""), + "devices": devices, + "error": "" if available else "torch cuda is not available", + } + except Exception as exc: # noqa: BLE001 - expose CUDA initialization failures + return { + "available": False, + "device_count": 0, + "torch_version": str(getattr(torch, "__version__", "")), + "torch_cuda_version": str(getattr(torch.version, "cuda", "") or ""), + "devices": [], + "error": str(exc), + } + + def _slice_log_content( + content: str, + tail_lines: int | None = None, + offset: int | None = None, + limit: int | None = None, + ) -> dict[str, Any]: + lines = content.splitlines() + total = len(lines) + if offset is not None or limit is not None: + start = max(0, offset or 0) + end = start + limit if limit else total + selected = lines[start:end] + else: + tail = tail_lines or 200 + start = max(0, total - tail) + selected = lines[start:] + next_offset = start + len(selected) + return { + "content": "\n".join(selected), + "total_lines": total, + "offset": start, + "limit": len(selected), + "has_more": next_offset < total, + "next_offset": next_offset if next_offset < total else None, + } + + def _safe_float(value: Any, default: float = 0) -> float: + try: + return float(str(value).replace("[N/A]", "").strip() or default) + except (TypeError, ValueError): + return default + def job_status(job: dict[str, Any]) -> dict[str, Any]: if execution_mode() != "simulator": return job @@ -74,9 +198,80 @@ def create_app() -> FastAPI: ) return "\n".join(lines) + def real_gpu_resources() -> list[dict[str, Any]]: + query = ( + "index,uuid,name,memory.total,memory.used,utilization.gpu," + "temperature.gpu,power.draw,power.limit" + ) + try: + result = subprocess.run( + ["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except Exception: + return fallback_gpu_resources() + + items: list[dict[str, Any]] = [] + for line in result.stdout.splitlines(): + parts = [part.strip() for part in line.split(",")] + if len(parts) < 9: + continue + idx, uuid, name, mem_total, mem_used, util, temp, power, power_limit = parts[:9] + total_gb = round(_safe_float(mem_total) / 1024, 2) + used_gb = round(_safe_float(mem_used) / 1024, 2) + memory_percent = round(used_gb / total_gb * 100, 1) if total_gb else 0 + gpu_percent = int(_safe_float(util)) + items.append( + { + "id": int(idx), + "gpu_index": int(idx), + "uuid": uuid, + "name": name, + "status": "busy" if gpu_percent >= 5 or used_gb > 1 else "idle", + "gpu_percent": gpu_percent, + "memory_used_gb": used_gb, + "memory_total_gb": total_gb, + "memory_percent": memory_percent, + "temperature": int(_safe_float(temp)), + "power_w": round(_safe_float(power), 1), + "power_limit_w": round(_safe_float(power_limit), 1), + "processes": [], + } + ) + return items + + def fallback_gpu_resources() -> list[dict[str, Any]]: + count = _int_env("COMPUTE_GPU_COUNT", 0) + if count <= 0: + return [] + name = os.getenv("COMPUTE_GPU_NAME", "Configured GPU") + memory_total = _float_env("COMPUTE_GPU_MEMORY_GB", 80.0) + power_limit = _float_env("COMPUTE_GPU_POWER_LIMIT_W", 300.0) + return [ + { + "id": idx, + "gpu_index": idx, + "uuid": f"GPU-{host_id().upper()}-{idx}", + "name": name, + "status": "idle", + "gpu_percent": 0, + "memory_used_gb": 0, + "memory_total_gb": memory_total, + "memory_percent": 0, + "temperature": _int_env("COMPUTE_GPU_BASE_TEMPERATURE", 35), + "power_w": 0, + "power_limit_w": power_limit, + "processes": [], + } + for idx in range(count) + ] + def gpu_resources() -> list[dict[str, Any]]: if execution_mode() != "simulator": - return [] + return real_gpu_resources() active_jobs = [job_status(job) for job in jobs.values() if job["status"] in {"queued", "running"}] gpus: list[dict[str, Any]] = [] for idx in range(4): @@ -109,6 +304,168 @@ def create_app() -> FastAPI: ) return gpus + def _validate_training_accelerator(payload: dict[str, Any]) -> tuple[list[str], list[str], dict[str, Any]]: + errors: list[str] = [] + warnings: list[str] = [] + if str(payload.get("engine") or payload.get("training_engine") or "llama_factory") == "smoke": + return errors, warnings, {} + requested_gpus = [int(item) for item in payload.get("gpus") or []] + if not requested_gpus: + warnings.append("no gpu selected; training will run on CPU") + return errors, warnings, {} + cuda = torch_cuda_status() + if not cuda.get("available"): + errors.append(f"torch cuda unavailable on compute node: {cuda.get('error') or 'unknown error'}") + device_count = int(cuda.get("device_count") or 0) + if device_count and max(requested_gpus) >= device_count: + errors.append(f"requested gpu index out of torch device range: requested={requested_gpus}, device_count={device_count}") + min_memory_gb = _float_env("MIN_TRAINING_GPU_MEMORY_GB", 4.0) + gpus = {int(item["gpu_index"]): item for item in gpu_resources() if "gpu_index" in item} + for gpu_index in requested_gpus: + gpu = gpus.get(gpu_index) + if not gpu: + errors.append(f"requested gpu not found by nvidia-smi: {gpu_index}") + continue + memory_total = float(gpu.get("memory_total_gb") or 0) + if memory_total and memory_total < min_memory_gb: + errors.append( + f"gpu {gpu_index} memory too small: {memory_total}GB < required {min_memory_gb}GB" + ) + return errors, warnings, cuda + + def _check_path_item(item: dict[str, Any]) -> dict[str, Any]: + path = Path(str(item.get("path") or "")) + exists = path.exists() + expected_type = str(item.get("type") or "any") + ok = exists + if exists and expected_type == "dir": + ok = path.is_dir() + if exists and expected_type == "file": + ok = path.is_file() + return { + "name": item.get("name") or "", + "path": str(path), + "type": expected_type, + "required": bool(item.get("required", True)), + "exists": exists, + "is_dir": path.is_dir() if exists else False, + "is_file": path.is_file() if exists else False, + "byte_size": sum(child.stat().st_size for child in path.rglob("*") if child.is_file()) if exists and path.is_dir() else path.stat().st_size if exists and path.is_file() else 0, + "ok": ok or not item.get("required", True), + } + + def _job_preview(payload: dict[str, Any], check_paths: bool) -> dict[str, Any]: + warnings: list[str] = [] + runtime_files: list[dict[str, str]] = [] + command_payload = {**payload, "require_dataset_files": check_paths} + if check_paths: + try: + runtime_files = prepare_runtime_files(command_payload) + except OSError as exc: + return { + "valid": False, + "errors": [f"prepare runtime files failed: {exc}"], + "warnings": warnings, + "engine": str(payload.get("engine") or payload.get("training_engine") or "llama_factory"), + "command": [], + "command_text": "", + "work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"), + "env": {}, + "runtime_files": [], + "path_checks": [], + } + try: + command = build_command(command_payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory")) + except ValueError as exc: + return { + "valid": False, + "errors": [part.strip() for part in str(exc).split(";") if part.strip()], + "warnings": warnings, + "engine": str(payload.get("engine") or payload.get("training_engine") or "llama_factory"), + "command": [], + "command_text": "", + "work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"), + "env": {}, + "runtime_files": runtime_files, + "path_checks": [], + } + + errors: list[str] = [] + engine = str(payload.get("engine") or payload.get("training_engine") or "llama_factory") + path_checks: list[dict[str, Any]] = [] + accelerator: dict[str, Any] = {} + if check_paths and engine != "smoke": + path_checks = [ + _check_path_item( + { + "name": "model_name_or_path", + "path": payload.get("model_name_or_path") or payload.get("base_model") or payload.get("base_model_path") or "", + "type": "any", + "required": True, + } + ) + ] + if engine in {"merge", "export", "llama_factory_export"} and payload.get("adapter_name_or_path"): + path_checks.append( + _check_path_item( + { + "name": "adapter_name_or_path", + "path": payload.get("adapter_name_or_path"), + "type": "any", + "required": True, + } + ) + ) + if payload.get("dataset_dir"): + path_checks.append( + _check_path_item( + { + "name": "dataset_dir", + "path": payload.get("dataset_dir"), + "type": "dir", + "required": True, + } + ) + ) + output_dir = Path(str(payload.get("output_dir") or "/data/yg-ft/outputs/training-job")) + path_checks.append( + _check_path_item( + { + "name": "output_parent", + "path": str(output_dir.parent), + "type": "dir", + "required": False, + } + ) + ) + errors.extend( + [f"{item['name']} path not available: {item['path']}" for item in path_checks if not item["ok"] and item["required"]] + ) + if shutil.which(command.command[0]) is None: + errors.append(f"training command not found: {command.command[0]}") + if not Path(command.work_dir).exists(): + errors.append(f"llama_factory_home not found: {command.work_dir}") + if engine not in {"merge", "export", "llama_factory_export"}: + accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload) + errors.extend(accelerator_errors) + warnings.extend(accelerator_warnings) + elif engine == "smoke": + warnings.append("smoke engine skips model and dataset path checks") + + return { + "valid": not errors, + "errors": errors, + "warnings": warnings, + "engine": engine, + "command": command.command, + "command_text": " ".join(command.command), + "work_dir": command.work_dir, + "env": command.env, + "runtime_files": runtime_files, + "accelerator": accelerator, + "path_checks": path_checks, + } + @app.get(f"{route_prefix}/health") async def health_check() -> dict[str, str]: return { @@ -116,41 +473,130 @@ def create_app() -> FastAPI: "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"), } + @app.get("/health") + async def health_check_root() -> dict[str, str]: + return await health_check() + @app.get(f"{route_prefix}/v1/compute/health") - async def compute_health_check() -> dict[str, str | bool]: + async def compute_health_check() -> dict[str, Any]: data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) + dataset_root = Path(os.getenv("YG_FT_DATASET_ROOT", str(data_root / "datasets"))) + output_root = Path(os.getenv("YG_FT_OUTPUT_ROOT", str(data_root / "outputs"))) llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory")) + gpu_items = gpu_resources() + torch_cuda = torch_cuda_status() return { "status": "ok", + "api_version": "v1", "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"), "app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true", "data_root": str(data_root), "data_root_exists": data_root.exists(), + "model_root": os.getenv("YG_FT_MODEL_ROOT", str(data_root / "models")), + "dataset_root": str(dataset_root), + "dataset_root_exists": dataset_root.exists(), + "output_root": str(output_root), + "output_root_exists": output_root.exists(), + "log_root": os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"), "llama_factory_home": str(llama_factory_home), "llama_factory_home_exists": llama_factory_home.exists(), + "llama_factory_version": os.getenv("LLAMA_FACTORY_VERSION", ""), "execution_mode": execution_mode(), + "gpu_count": _int_env("COMPUTE_GPU_COUNT", 0), + "nvidia_gpu_count": len(gpu_items), + "torch_cuda": torch_cuda, + "gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus", + "capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling", "inference"], } @app.get(f"{route_prefix}/v1/compute/jobs") async def list_jobs_alias() -> dict[str, list[dict[str, Any]]]: - return {"items": [job_status(job) for job in jobs.values()]} + items = process_manager.list_jobs() if execution_mode() != "simulator" else [job_status(job) for job in jobs.values()] + return {"items": items} @app.get(f"{route_prefix}/compute/resources/gpus") async def list_gpus() -> dict[str, Any]: return {"items": gpu_resources(), "compute_host_id": host_id()} + @app.get(f"{route_prefix}/v1/compute/resources/gpus") + async def list_gpus_v1() -> dict[str, Any]: + return {"items": gpu_resources(), "compute_host_id": host_id()} + + @app.post(f"{route_prefix}/compute/jobs/preview") + async def preview_job(payload: dict[str, Any]) -> dict[str, Any]: + return _job_preview(payload, check_paths=False) + + @app.post(f"{route_prefix}/compute/jobs/validate") + async def validate_job(payload: dict[str, Any]) -> dict[str, Any]: + return _job_preview(payload, check_paths=True) + + @app.post(f"{route_prefix}/v1/compute/jobs/preview") + async def preview_job_v1(payload: dict[str, Any]) -> dict[str, Any]: + return await preview_job(payload) + + @app.post(f"{route_prefix}/v1/compute/jobs/validate") + async def validate_job_v1(payload: dict[str, Any]) -> dict[str, Any]: + return await validate_job(payload) + + @app.post(f"{route_prefix}/compute/files/check-paths") + async def check_paths(payload: dict[str, Any]) -> dict[str, Any]: + items = [_check_path_item(item) for item in payload.get("paths", []) if isinstance(item, dict)] + return {"valid": all(item["ok"] for item in items), "items": items} + + @app.get(f"{route_prefix}/compute/files/list") + async def list_files( + root: str = Query(default="data"), + relative_path: str = Query(default=""), + directories_only: bool = Query(default=False), + ) -> dict[str, Any]: + roots = { + "data": Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")), + "models": Path(os.getenv("YG_FT_MODEL_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/models")), + "datasets": Path(os.getenv("YG_FT_DATASET_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/datasets")), + "outputs": Path(os.getenv("YG_FT_OUTPUT_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/outputs")), + } + base = roots.get(root) + if base is None: + raise HTTPException(status_code=400, detail="invalid root") + target = (base / relative_path.lstrip("/\\")).resolve() + if not _path_inside(base, target): + raise HTTPException(status_code=400, detail="path must stay inside selected root") + if not target.exists(): + return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": []} + items = [] + for child in sorted(target.iterdir(), key=lambda path: (not path.is_dir(), path.name.lower())): + if directories_only and not child.is_dir(): + continue + items.append( + { + "name": child.name, + "path": str(child), + "relative_path": str(child.relative_to(base)).replace("\\", "/"), + "type": "directory" if child.is_dir() else "file", + "byte_size": child.stat().st_size if child.is_file() else 0, + } + ) + return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items} + @app.post(f"{route_prefix}/compute/jobs") async def create_job(payload: dict[str, Any]) -> dict[str, Any]: + payload = {**payload, "require_dataset_files": True} + try: + prepare_runtime_files(payload) + except OSError as exc: + raise HTTPException(status_code=400, detail=f"prepare runtime files failed: {exc}") try: command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory")) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) - if execution_mode() != "simulator": - raise HTTPException( - status_code=501, - detail="real compute executor is not implemented yet; set COMPUTE_EXECUTION_MODE=simulator only for isolated development", - ) job_id = str(payload.get("id") or f"job_{int(now() * 1000)}") + if execution_mode() != "simulator": + try: + return process_manager.create_job({**payload, "id": job_id}, command.command, command.work_dir) + except FileNotFoundError as exc: + raise HTTPException(status_code=500, detail=f"training command not found: {exc.filename}") + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) job = { "id": job_id, "name": payload.get("name", job_id), @@ -169,17 +615,29 @@ def create_app() -> FastAPI: @app.get(f"{route_prefix}/compute/jobs") async def list_jobs() -> dict[str, Any]: - return {"items": [job_status(job) for job in jobs.values()]} + items = process_manager.list_jobs() if execution_mode() != "simulator" else [job_status(job) for job in jobs.values()] + return {"items": items} @app.get(f"{route_prefix}/compute/jobs/{{job_id}}") async def get_job(job_id: str) -> dict[str, Any]: job = jobs.get(job_id) + if execution_mode() != "simulator": + job = process_manager.get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + return job + job = jobs.get(job_id) if not job: raise HTTPException(status_code=404, detail="job not found") return job_status(job) @app.post(f"{route_prefix}/compute/jobs/{{job_id}}/stop") async def stop_job(job_id: str) -> dict[str, Any]: + if execution_mode() != "simulator": + job = process_manager.stop_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + return job job = jobs.get(job_id) if not job: raise HTTPException(status_code=404, detail="job not found") @@ -188,22 +646,165 @@ def create_app() -> FastAPI: return job @app.get(f"{route_prefix}/compute/jobs/{{job_id}}/logs") - async def job_logs(job_id: str) -> dict[str, Any]: - job = jobs.get(job_id) - if not job: - raise HTTPException(status_code=404, detail="job not found") - job = job_status(job) - metrics = [parse_log_line(line) for line in job["logs"].splitlines()] - return {"job_id": job_id, "content": job["logs"], "metrics": [m for m in metrics if m]} + async def job_logs( + job_id: str, + tail_lines: int | None = Query(default=200, ge=1, le=5000), + offset: int | None = Query(default=None, ge=0), + limit: int | None = Query(default=None, ge=1, le=5000), + ) -> dict[str, Any]: + if execution_mode() != "simulator": + job = process_manager.get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + content = process_manager.logs(job_id) + else: + job = jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + job = job_status(job) + content = job["logs"] + window = _slice_log_content(content, tail_lines, offset, limit) + metrics = [parse_log_line(line) for line in window["content"].splitlines()] + return {"job_id": job_id, **window, "metrics": [m for m in metrics if m]} + + # ── Inference Endpoints ─────────────────────────────────────────── + + @app.post(f"{route_prefix}/inference/load") + async def inference_load(payload: dict[str, Any]) -> dict[str, Any]: + """Load a model for inference using LLaMA-Factory ChatModel.""" + session = get_inference_session() + result = session.load( + model_name_or_path=payload.get("model_name_or_path", ""), + adapter_name_or_path=payload.get("adapter_name_or_path", ""), + template=payload.get("template", "qwen"), + infer_backend=payload.get("infer_backend", "huggingface"), + infer_dtype=payload.get("infer_dtype", "auto"), + ) + if not result.get("loaded"): + raise HTTPException(status_code=500, detail=result.get("error", "model load failed")) + return result + + @app.post(f"{route_prefix}/inference/unload") + async def inference_unload() -> dict[str, Any]: + """Unload the currently loaded model and free GPU memory.""" + return get_inference_session().unload() + + @app.get(f"{route_prefix}/inference/status") + async def inference_status() -> dict[str, Any]: + """Get the current inference session status.""" + return get_inference_session().info() + + @app.post(f"{route_prefix}/inference/chat") + async def inference_chat(payload: dict[str, Any]) -> dict[str, Any]: + """Chat with the loaded model (non-streaming).""" + messages = payload.get("messages") or [] + if not messages: + raise HTTPException(status_code=400, detail="messages is required") + result = get_inference_session().chat( + messages=messages, + temperature=float(payload.get("temperature", 0.95)), + top_p=float(payload.get("top_p", 0.7)), + max_new_tokens=int(payload.get("max_new_tokens", 1024)), + do_sample=bool(payload.get("do_sample", True)), + ) + if result.get("error"): + raise HTTPException(status_code=500, detail=result["error"]) + return {"response": result["response"]} + + @app.post(f"{route_prefix}/inference/chat/stream") + async def inference_chat_stream(payload: dict[str, Any]) -> StreamingResponse: + """Chat with streaming response (Server-Sent Events).""" + messages = payload.get("messages") or [] + if not messages: + raise HTTPException(status_code=400, detail="messages is required") + + def generate(): + session = get_inference_session() + for chunk in session.chat_stream( + messages=messages, + temperature=float(payload.get("temperature", 0.95)), + top_p=float(payload.get("top_p", 0.7)), + max_new_tokens=int(payload.get("max_new_tokens", 1024)), + do_sample=bool(payload.get("do_sample", True)), + ): + yield chunk + + return StreamingResponse(generate(), media_type="text/event-stream") @app.post(f"{route_prefix}/compute/files/upload") - async def upload_file(payload: dict[str, Any]) -> dict[str, Any]: - file_id = str(payload.get("id") or f"file_{int(now() * 1000)}") - return {"id": file_id, "status": "available", "local_path": f"/data/yg-ft/uploads/{file_id}"} + async def upload_file( + file: UploadFile | None = File(default=None), + target_relative_path: str | None = Form(default=None), + resource_type: str | None = Form(default=None), + resource_id: str | None = Form(default=None), + ) -> dict[str, Any]: + file_id = f"file_{int(now() * 1000)}" + data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) + data_root.mkdir(parents=True, exist_ok=True) + filename = Path(file.filename if file else file_id).name + if target_relative_path: + target = (data_root / target_relative_path.lstrip("/\\")).resolve() + if not _path_inside(data_root, target): + raise HTTPException(status_code=400, detail="target path must stay inside YG_FT_DATA_ROOT") + else: + target = data_root / "uploads" / f"{file_id}_{filename}" + if file: + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("wb") as output: + while chunk := await file.read(1024 * 1024): + output.write(chunk) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("", encoding="utf-8") + return { + "id": file_id, + "resource_type": resource_type, + "resource_id": resource_id, + "status": "available", + "local_path": str(target), + "byte_size": target.stat().st_size, + "checksum_sha256": hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "", + } + + @app.post(f"{route_prefix}/compute/files/import-local") + async def import_local_file(payload: dict[str, Any]) -> dict[str, Any]: + source = Path(str(payload.get("source_path") or "")) + if not source.exists(): + raise HTTPException(status_code=404, detail="source path not found") + data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) + data_root.mkdir(parents=True, exist_ok=True) + relative = str(payload.get("target_relative_path") or f"imports/{source.name}").lstrip("/\\") + target = (data_root / relative).resolve() + if not _path_inside(data_root, target): + raise HTTPException(status_code=400, detail="target path must stay inside YG_FT_DATA_ROOT") + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + if target.exists(): + shutil.rmtree(target) + shutil.copytree(source, target) + byte_size = sum(path.stat().st_size for path in target.rglob("*") if path.is_file()) + checksum = "" + else: + shutil.copy2(source, target) + byte_size = target.stat().st_size + checksum = hashlib.sha256(target.read_bytes()).hexdigest() + return { + "id": str(payload.get("id") or f"file_{int(now() * 1000)}"), + "resource_type": payload.get("resource_type"), + "resource_id": payload.get("resource_id"), + "status": "available", + "local_path": str(target), + "byte_size": byte_size, + "checksum_sha256": checksum, + } @app.get(f"{route_prefix}/compute/files/{{file_id}}/download") - async def download_file(file_id: str) -> dict[str, Any]: - return {"id": file_id, "status": "ready", "download_url": f"{route_prefix}/compute/files/{file_id}/download"} + async def download_file(file_id: str) -> FileResponse: + upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads" + matches = list(upload_root.glob(f"{file_id}_*")) + if not matches: + raise HTTPException(status_code=404, detail="file not found") + return FileResponse(matches[0]) return app diff --git a/compute/engines/llama_factory/adapter.py b/compute/engines/llama_factory/adapter.py index 7e5fc17..3a7042e 100644 --- a/compute/engines/llama_factory/adapter.py +++ b/compute/engines/llama_factory/adapter.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import re from dataclasses import dataclass from pathlib import Path @@ -13,40 +14,234 @@ class LlamaFactoryCommand: env: dict[str, str] +def _load_dataset_preview(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + text = path.read_text(encoding="utf-8", errors="replace").strip() + if not text: + return [] + if path.suffix.lower() == ".jsonl": + items: list[dict[str, Any]] = [] + for line in text.splitlines()[:20]: + line = line.strip() + if not line: + continue + value = json.loads(line) + if isinstance(value, dict): + items.append(value) + return items + value = json.loads(text) + if isinstance(value, list): + return [item for item in value[:20] if isinstance(item, dict)] + if isinstance(value, dict): + return [value] + return [] + + +def _validate_dataset_columns(config: dict[str, Any]) -> list[str]: + dataset_dir = config.get("dataset_dir") + dataset_info = config.get("dataset_info") + if not dataset_dir or not isinstance(dataset_info, dict): + return [] + root = Path(str(dataset_dir)) + errors: list[str] = [] + for dataset_key, item in dataset_info.items(): + if not isinstance(item, dict): + continue + file_name = item.get("file_name") + file_names = file_name if isinstance(file_name, list) else [file_name] + columns = item.get("columns") if isinstance(item.get("columns"), dict) else {} + required_columns = [str(value) for value in columns.values() if value] + for name in file_names: + if not name: + continue + path = root / str(name).lstrip("/\\") + if not path.exists(): + continue + try: + preview_rows = _load_dataset_preview(path) + except Exception as exc: # noqa: BLE001 - expose malformed data as validation error + errors.append(f"dataset file parse failed: {path}: {exc}") + continue + if not preview_rows: + errors.append(f"dataset file has no valid object records: {path}") + continue + available = set().union(*(row.keys() for row in preview_rows)) + missing = [column for column in required_columns if column not in available] + if missing: + errors.append( + f"dataset columns missing in {path.name} for {dataset_key}: {', '.join(sorted(set(missing)))}" + ) + return errors + + def validate_config(config: dict[str, Any]) -> list[str]: errors: list[str] = [] if not config.get("base_model") and not config.get("model_name_or_path"): errors.append("base_model or model_name_or_path is required") if not config.get("dataset") and not config.get("dataset_dir"): errors.append("dataset or dataset_dir is required") - learning_rate = float(config.get("learning_rate", 0.0002)) + try: + learning_rate = float(config.get("learning_rate", 0.0002)) + except (TypeError, ValueError): + learning_rate = 0 if learning_rate <= 0: errors.append("learning_rate must be greater than zero") - epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1))) + try: + epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1))) + except (TypeError, ValueError): + epochs = 0 if epochs <= 0: errors.append("n_epochs must be greater than zero") + dataset_dir = config.get("dataset_dir") + dataset_info = config.get("dataset_info") + if config.get("require_dataset_files") and dataset_dir and isinstance(dataset_info, dict): + root = Path(str(dataset_dir)) + for dataset_key, item in dataset_info.items(): + if not isinstance(item, dict): + errors.append(f"dataset_info entry must be object: {dataset_key}") + continue + file_name = item.get("file_name") + file_names = file_name if isinstance(file_name, list) else [file_name] + for name in file_names: + if not name: + errors.append(f"dataset_info file_name is required: {dataset_key}") + continue + path = root / str(name).lstrip("/\\") + if not path.exists(): + errors.append(f"dataset file not found: {path}") + errors.extend(_validate_dataset_columns(config)) return errors +def _optional_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None: + for key in keys: + value = config.get(key) + if value is not None and value != "": + command.extend([option, str(value)]) + return + + +def _optional_bool_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None: + for key in keys: + value = config.get(key) + if value is True or str(value).lower() == "true": + command.extend([option, "true"]) + return + + +def _normalize_stage(config: dict[str, Any]) -> str: + raw = str(config.get("stage") or config.get("train_type") or "sft").strip().lower() + return { + "sft": "sft", + "dpo": "dpo", + "cpt": "pt", + "pt": "pt", + "pretrain": "pt", + "rm": "rm", + "ppo": "ppo", + "kto": "kto", + }.get(raw, raw or "sft") + + +def prepare_runtime_files(config: dict[str, Any]) -> list[dict[str, str]]: + dataset_dir = config.get("dataset_dir") + dataset_info = config.get("dataset_info") + if not dataset_dir or not isinstance(dataset_info, dict): + return [] + root = Path(str(dataset_dir)) + root.mkdir(parents=True, exist_ok=True) + path = root / "dataset_info.json" + existing: dict[str, Any] = {} + if path.exists(): + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + existing = loaded if isinstance(loaded, dict) else {} + except json.JSONDecodeError: + existing = {} + existing.update(dataset_info) + path.write_text(json.dumps(existing, ensure_ascii=False, indent=2), encoding="utf-8") + return [{"name": "dataset_info", "path": str(path)}] + + def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand: + engine = str(config.get("engine") or config.get("training_engine") or "llama_factory") + if engine in {"merge", "export", "llama_factory_export"}: + model_path = config.get("base_model") or config.get("model_name_or_path") or config.get("base_model_path") + adapter_path = config.get("adapter_name_or_path") or config.get("adapter_path") or config.get("lora_path") + output_dir = config.get("output_dir") or config.get("export_dir") + errors: list[str] = [] + if not model_path: + errors.append("base_model or model_name_or_path is required") + if not adapter_path and engine == "merge": + errors.append("adapter_name_or_path or adapter_path is required") + if not output_dir: + errors.append("output_dir or export_dir is required") + if errors: + raise ValueError("; ".join(errors)) + command = [ + "llamafactory-cli", + "export", + "--model_name_or_path", + str(model_path), + "--template", + str(config.get("template", "qwen")), + "--finetuning_type", + str(config.get("train_method", config.get("finetuning_type", "lora"))), + "--export_dir", + str(output_dir), + "--export_size", + str(config.get("export_size", 2)), + "--export_device", + str(config.get("export_device", "cpu")), + "--export_legacy_format", + str(config.get("export_legacy_format", False)).lower(), + ] + if adapter_path: + command.extend(["--adapter_name_or_path", str(adapter_path)]) + quantization_bit = int(config.get("export_quantization_bit", config.get("quantization_bit", 0)) or 0) + if quantization_bit in {4, 8}: + command.extend(["--quantization_bit", str(quantization_bit)]) + return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={}) + errors = validate_config(config) if errors: raise ValueError("; ".join(errors)) + if engine == "smoke": + output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-smoke')}" + script = ( + "import json, os, time; " + f"out={str(output_dir)!r}; " + "os.makedirs(out, exist_ok=True); " + "print('[INFO] smoke training started', flush=True); " + "\nfor step in range(1, 7):\n" + " loss=round(1.8/(step+1), 4)\n" + " lr=round(0.0002*(1-step/10), 8)\n" + " print({'loss': loss, 'grad_norm': round(0.4 + step*0.03, 4), 'learning_rate': lr, 'epoch': round(step/6, 4)}, flush=True)\n" + " time.sleep(0.4)\n" + "\nopen(os.path.join(out, 'adapter_config.json'), 'w', encoding='utf-8').write(json.dumps({'engine':'smoke','status':'completed'})); " + "print('***** train metrics *****', flush=True); " + "print('train_loss = 0.12', flush=True); " + "print('***** train metrics end *****', flush=True)" + ) + return LlamaFactoryCommand(command=["python", "-u", "-c", script], work_dir="/app", env={}) + model_path = config.get("base_model") or config.get("model_name_or_path") - dataset = config.get("dataset") or config.get("dataset_dir") + dataset = config.get("dataset") or config.get("dataset_name") + dataset_dir = config.get("dataset_dir") output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}" command = [ "llamafactory-cli", "train", "--stage", - str(config.get("stage", "sft")).lower(), + _normalize_stage(config), "--do_train", "true", "--model_name_or_path", str(model_path), "--dataset", - str(dataset), + str(dataset or "default"), "--template", str(config.get("template", "qwen")), "--finetuning_type", @@ -61,7 +256,32 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA- str(config.get("n_epochs", 3)), "--save_steps", str(config.get("save_steps", 50)), + "--logging_steps", + str(config.get("logging_steps", 10)), + "--overwrite_output_dir", + "true", + "--plot_loss", + "true", ] + if dataset_dir: + command.extend(["--dataset_dir", str(dataset_dir)]) + eval_dataset = config.get("eval_dataset") + if eval_dataset: + command.extend(["--eval_dataset", str(eval_dataset), "--do_eval", "true"]) + _optional_arg(config, command, "--cutoff_len", "max_length", "cutoff_len") + _optional_arg(config, command, "--lr_scheduler_type", "lr_scheduler_type") + _optional_arg(config, command, "--warmup_ratio", "warmup_ratio") + _optional_arg(config, command, "--weight_decay", "weight_decay") + _optional_arg(config, command, "--lora_rank", "lora_rank", "rank") + _optional_arg(config, command, "--lora_alpha", "lora_alpha") + _optional_arg(config, command, "--lora_dropout", "lora_dropout") + _optional_arg(config, command, "--gradient_accumulation_steps", "gradient_accumulation_steps") + if not eval_dataset: + _optional_arg(config, command, "--val_size", "val_size") + _optional_arg(config, command, "--max_samples", "max_samples") + _optional_arg(config, command, "--preprocessing_num_workers", "preprocessing_num_workers") + _optional_bool_arg(config, command, "--fp16", "fp16") + _optional_bool_arg(config, command, "--bf16", "bf16") quantization_bit = int(config.get("quantization_bit", 0) or 0) if quantization_bit in {4, 8}: command.extend(["--quantization_bit", str(quantization_bit)]) @@ -77,4 +297,3 @@ def parse_log_line(line: str) -> dict[str, float] | None: if match: result[key] = float(match.group(1)) return result or None - diff --git a/compute/engines/llama_factory/inference.py b/compute/engines/llama_factory/inference.py new file mode 100644 index 0000000..4d1e227 --- /dev/null +++ b/compute/engines/llama_factory/inference.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import threading +import time +from typing import Any + + +class InferenceSession: + """Manages a loaded model for inference with LLaMA-Factory ChatModel.""" + + def __init__(self) -> None: + self._model: Any = None + self._tokenizer: Any = None + self._generating_args: dict[str, Any] = {} + self._model_name: str = "" + self._adapter_path: str = "" + self._lock = threading.Lock() + self._loaded_at: float = 0.0 + self._status: str = "idle" + + @property + def status(self) -> str: + return self._status + + @property + def model_name(self) -> str: + return self._model_name + + @property + def adapter_path(self) -> str: + return self._adapter_path + + @property + def loaded_at(self) -> float: + return self._loaded_at + + def info(self) -> dict[str, Any]: + return { + "loaded": self._status == "ready", + "status": self._status, + "model_name": self._model_name, + "adapter_path": self._adapter_path, + "loaded_at": self._loaded_at, + } + + def load(self, model_name_or_path, adapter_name_or_path="", template="qwen", infer_backend="huggingface", infer_dtype="auto", **kwargs): + with self._lock: + if self._status == "loading": + return {"loaded": False, "error": "model is already loading"} + if self._status == "ready": + self.unload() + self._status = "loading" + self._model_name = model_name_or_path + self._adapter_path = adapter_name_or_path + try: + from llamafactory.chat import ChatModel + from llamafactory.hparams import get_infer_args + args = {"model_name_or_path": model_name_or_path, "template": template, "infer_backend": infer_backend, "infer_dtype": infer_dtype} + if adapter_name_or_path: + args["adapter_name_or_path"] = adapter_name_or_path + args.update(kwargs) + model_args, generating_args = get_infer_args(args) + self._model = ChatModel(model_args) + self._tokenizer = self._model.tokenizer + self._generating_args = generating_args + self._loaded_at = time.time() + self._status = "ready" + return {"loaded": True, "status": "ready"} + except Exception as exc: + self._status = "error" + self._model = None + return {"loaded": False, "status": "error", "error": str(exc)} + + def unload(self): + with self._lock: + if self._model is not None: + try: + del self._model + except Exception: + pass + self._model = None + self._tokenizer = None + self._status = "idle" + self._model_name = "" + self._adapter_path = "" + self._loaded_at = 0.0 + return {"unloaded": True} + + def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs): + with self._lock: + if self._status != "ready" or self._model is None: + return {"error": "model not loaded", "response": ""} + try: + generate_kwargs = {**self._generating_args, "temperature": temperature, "top_p": top_p, "max_new_tokens": max_new_tokens, "do_sample": do_sample} + generate_kwargs.update(kwargs) + formatted = self._model.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + responses = [] + for response in self._model.stream_chat(formatted, generate_kwargs): + responses.append(response) + full_response = "".join(str(r) for r in responses) + return {"response": full_response} + except Exception as exc: + return {"error": str(exc), "response": ""} + + def chat_stream(self, messages, **kwargs): + with self._lock: + if self._status != "ready" or self._model is None: + yield 'data: {"error": "model not loaded"}\n\n' + return + try: + generate_kwargs = {**self._generating_args, **kwargs} + formatted = self._model.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + for new_text in self._model.stream_chat(formatted, generate_kwargs): + yield new_text + except Exception as exc: + yield 'data: {"error": "' + str(exc) + '"}\n\n' + + +_inference_session = None + +def get_inference_session(): + global _inference_session + if _inference_session is None: + _inference_session = InferenceSession() + return _inference_session diff --git a/compute/requirements.txt b/compute/requirements.txt index ad4ce85..fc38ced 100644 --- a/compute/requirements.txt +++ b/compute/requirements.txt @@ -4,3 +4,7 @@ python-multipart>=0.0.9 pydantic>=2.7.0 python-dotenv>=1.0.1 httpx>=0.27.0 + +# 训练/推理运行时：部署在算力服务器，独立于应用平台，不得装入应用后端 venv +# llamafactory 会连带安装兼容版本的 transformers(<=5.6.0)/peft/datasets 等 +llamafactory diff --git a/design-qa.md b/design-qa.md deleted file mode 100644 index 381eee6..0000000 --- a/design-qa.md +++ /dev/null @@ -1,429 +0,0 @@ -# Training Log Detail Design QA - -## Evidence - -- Source visual truth: `docs/superpowers/specs/assets/training-log-detail-option-2.png` -- Implementation screenshot: `docs/superpowers/specs/assets/training-log-detail-final-expanded-1440.png` -- Collapsed implementation screenshot with global surface: `docs/superpowers/specs/assets/training-log-detail-global-surface-1440-v2.png` -- Normalized full-view comparison: `docs/superpowers/specs/assets/training-log-detail-final-comparison-normalized.png` -- Focused parameter comparison: `docs/superpowers/specs/assets/training-log-detail-final-comparison-params.png` -- White-canvas reference: `docs/superpowers/specs/assets/page-white-canvas-reference.png` -- White-canvas implementation: `docs/superpowers/specs/assets/model-edit-white-page-canvas-final-1440.png` -- White-canvas normalized comparison: `docs/superpowers/specs/assets/page-white-canvas-comparison.png` -- Training-log white-canvas screenshot: `docs/superpowers/specs/assets/training-log-detail-white-page-canvas-1440.png` -- Self-surface list screenshot: `docs/superpowers/specs/assets/fine-tune-list-self-surface-final-1440.png` -- Default-canvas detail screenshot: `docs/superpowers/specs/assets/training-log-detail-default-canvas-final-1440.png` -- Reference/detail comparison: `docs/superpowers/specs/assets/page-surface-reference-detail-comparison.png` -- Create-page duplicate-surface evidence: `docs/superpowers/specs/assets/fine-tune-create-double-surface-before-1440.png` -- Create-page single-surface evidence: `docs/superpowers/specs/assets/fine-tune-create-single-surface-final-1440.png` -- Route-transition flash reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-ea775434-828f-4bca-9714-72887faa9af9.png` -- Route-transition final detail frame: `docs/superpowers/specs/assets/route-transition-detail-final.png` -- Viewport: 1440 × 1024; comparison content normalized to 1200 × 800 after removing the existing 240px sidebar and 60px header from the implementation capture. -- State: `finance-sft-001`, completed, mock data loaded, training parameters expanded. - -## Full-view comparison - -The implementation preserves the selected two-column hierarchy: task and dataset information occupy the wide left track, runtime facts use the narrow right rail, and training parameters continue as a full-width disclosure section. The selected mock omitted the product shell, so the comparison intentionally crops the existing sidebar and header rather than treating them as design drift. - -The global product mode now uses two intentional surface modes. Form and detail routes render inside one `#ffffff` page canvas with 16px radius and 24px content padding. List routes that already own a white table/card surface render that surface directly on the `#f3f5f8` application background, avoiding a redundant white layer. - -## Required fidelity surfaces - -- Fonts and typography: Existing system font stack is preserved. Heading, label, value, and muted-copy hierarchy match the selected direction; output model uses body-level contrast after iteration 1, and all “未配置” values use `#64748b` on white after iteration 2. -- Spacing and layout rhythm: 24px main gap, 16px section gap, 12px surface radius, and light row separators match the selected composition. The existing application shell reduces usable content width, but normalized proportions remain aligned. -- Colors and tokens: Indigo accent, Slate text, success status, `#f3f5f8` page background, and white content surfaces are consistent with the current product. -- Image and icon fidelity: The screen contains no raster imagery. Existing Font Awesome icons are retained to match the repository's icon system; no placeholder, emoji, CSS drawing, or handcrafted SVG was introduced. -- Copy and content: Task name, status, model, date, duration, dataset metadata, storage, SFT, LoRA, and missing-value copy match the selected design and actual mock data. - -## Interaction and responsive checks - -- Parameter disclosure changed from `aria-expanded="false"` to `true` after activation, and the expanded content became visible. -- At 1000px viewport width, the overview changed to one column and the document had no horizontal overflow. -- At 700px viewport width, dataset metrics and parameter rows changed to one column and the document had no horizontal overflow. -- Browser console: no errors. One existing Element Plus `el-link` underline deprecation warning was emitted by the login flow and is unrelated to this page. - -## Comparison history - -### Iteration 1 — blocked - -- [P2] The implementation added a visible “基础训练参数” heading that did not exist in the selected mock, creating extra vertical space. -- [P2] “暂未生成” was styled too faintly compared with the selected design. - -Fixes: - -- Removed the redundant visible base-parameter heading while retaining an accessible region label. -- Restored body-level contrast for “暂未生成”. - -Post-fix evidence: - -- `docs/superpowers/specs/assets/training-log-detail-final-comparison-normalized.png` -- `docs/superpowers/specs/assets/training-log-detail-final-comparison-params.png` - -### Iteration 2 — blocked - -- [P2] “未配置” values used `#94a3b8` on white, below WCAG AA contrast for 14px text. - -Fix: - -- Updated muted values to `#64748b`; the regression check now calculates the contrast ratio and requires at least 4.5:1. - -Post-fix evidence: - -- Browser computed color: `rgb(100, 116, 139)`. -- Browser console: no errors. - -### Iteration 3 — passed - -No actionable P0/P1/P2 differences remain. The retained P3 difference is that the generated mock does not include the real product sidebar/header; this is an intentional constraint because the existing shell is shared by every page. - -### Iteration 4 — clarified global page canvas, passed - -- [P1] The earlier interpretation left the route content directly on the gray layout background and only made individual cards white. The clarified reference requires a single white page canvas behind every route. - -Fixes: - -- Split the shell and page tokens into `--app-shell-bg: #f3f5f8` and `--app-page-bg: #ffffff`. -- Added one global `.page-canvas` around every route in `MainLayout.vue`. -- Added 16px outer gutter, 16px canvas radius, 24px canvas padding, and a subtle canvas shadow. -- Flattened a route-root `PageCard` to prevent a duplicate large card layer. - -Post-fix evidence: - -- `docs/superpowers/specs/assets/page-white-canvas-comparison.png` -- Browser computed canvas: white background, 16px radius, 24px padding; outer shell: `rgb(243, 245, 248)`. -- Both the model-edit page and training-log page render inside the same global white canvas without horizontal overflow. - -### Iteration 5 — corrected list-page surface ownership, passed - -- [P1] Applying the white page canvas to every route created a redundant layer on list pages because `DataTablePage`, model evaluation, and model management already provide their own white root card. - -Fixes: - -- Added explicit `pageSurface: 'self'` metadata to each self-surfaced list route: model tuning, model evaluation, model inference, model management, data processing, and dataset management. -- Added `.page-canvas.is-self-surface` to remove the outer canvas padding, radius, background, and shadow only for those routes. -- Preserved the default white canvas for training-log, create, edit, preview, chat, and result routes. - -Post-fix evidence: - -- `docs/superpowers/specs/assets/fine-tune-list-self-surface-final-1440.png` -- `docs/superpowers/specs/assets/training-log-detail-default-canvas-final-1440.png` -- `docs/superpowers/specs/assets/page-surface-reference-detail-comparison.png` -- Browser computed list state: transparent outer canvas, 0px padding/radius, no shadow; white 12px-radius list card on `rgb(243, 245, 248)` shell. -- Browser computed detail state: white outer canvas, 24px padding, 16px radius, subtle shadow. -- Both states have no horizontal overflow and no console errors at 1440 × 900. - -### Iteration 6 — flattened wrapped root PageCard, passed - -- [P1] The training-task creation route wraps its root `PageCard` in `.fine-tune-create`. The earlier selector only matched a `PageCard` directly under `.page-canvas`, so this page retained a second white background, 12px radius, and card shadow. - -Fixes: - -- Added an explicit `.page-card-host` marker to the training-task creation route root; the layout flattens only a directly rendered root `PageCard` or a `PageCard` inside that explicit host. -- Root `PageCard` now uses a transparent background, 0px radius, no shadow, and no bottom margin while preserving its header/body layout. -- Kept the selector excluded from `.is-self-surface`, so list cards retain their own white background, 12px radius, and shadow. -- Rejected a generic one-level descendant selector because it would also match the training-log parameter card. - -Post-fix evidence: - -- `docs/superpowers/specs/assets/fine-tune-create-double-surface-before-1440.png` -- `docs/superpowers/specs/assets/fine-tune-create-single-surface-final-1440.png` -- Browser computed create-page root card: transparent background, 0px radius, no shadow; outer canvas remains white with 24px padding. -- Browser computed list-page card remains white with 12px radius and subtle shadow on a transparent outer canvas. -- Browser computed training-log parameter card remains white with 12px radius and subtle shadow, confirming that internal business cards are not flattened. -- Both pages have no horizontal overflow; create-page console has no errors. - -### Iteration 7 — removed page-level opacity transition, passed - -- [P1] When navigating from a self-surface list to a default-canvas secondary page, `route.meta.pageSurface` changed immediately while the old list remained for the 150ms `out-in` leave animation. The result was a semi-transparent old list rendered inside the new white canvas. - -Fixes: - -- Removed the page-level Vue `transition` wrapper from `MainLayout.vue`. -- Removed the `.fade-enter-*` and `.fade-leave-*` opacity rules. -- Preserved local component animations such as dialogs, disclosures, and the selected-row batch bar. - -Post-fix evidence: - -- Source flash frame: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-ea775434-828f-4bca-9714-72887faa9af9.png`. -- Final detail frame: `docs/superpowers/specs/assets/route-transition-detail-final.png`. -- Immediate state after list → create: old list absent, route-root opacity `1`, zero `.fade-*` transition elements, correct default canvas. -- Immediate state after create → list: old create page absent, route-root opacity `1`, zero `.fade-*` transition elements, correct self-surface canvas. -- Immediate state after list → training log: old list absent, route-root opacity `1`, zero `.fade-*` transition elements, correct default canvas. -- All three paths had no horizontal overflow; browser console had no errors. - -## Build evidence gap - -The `type-check` script now uses project-reference mode (`vue-tsc -b --noEmit`) so it no longer reports a false pass. `npm run type-check` and `npm run build` remain blocked by pre-existing TypeScript errors in `src/mock/adapter.ts`, `FineTuneCreateView.vue`, and `FineTuneListView.vue`; no remaining error points to `TrainingLogView.vue` or the page-surface files. `npx vite build` succeeds, proving the updated UI bundles for production. - -Design-QA final result: passed - -final result: passed - ---- - -# Service Dashboard Design QA - -## Evidence - -- Source visual truth: `docs/superpowers/specs/assets/service-dashboard-approved-1440.png` -- Browser-rendered implementation: `docs/superpowers/specs/assets/service-dashboard-implementation-1440.png` -- Viewport: 1440 × 1024 -- State: authenticated `admin` user on `/dashboard`; service dashboard navigation active; 7-day chart visible; four training tasks visible. - -## Full-view comparison - -The implementation preserves the approved composition: the product shell stays intact, the service dashboard is the active navigation item, the platform-health summary spans the top, the grouped training chart occupies the wide middle track, service health occupies the narrow track, and the training-task table spans the bottom. The three bar series, dual axes, dates, values, service counts, task names, statuses, progress, accuracy, and actions match the selected mock. - -A separate focused crop was not required because both source and implementation evidence are full-resolution desktop captures at a readable scale; the chart labels, axis units, service rows, and every task-table column are legible in the full-view comparison. - -## Required fidelity surfaces - -- Fonts and typography: the existing Inter/system/PingFang stack is retained. Heading, section title, metric, table header, and muted-copy weights and sizes match the selected direction. -- Spacing and layout rhythm: 24px page padding, 14px section gaps, 10px panel radii, light separators, and the wide-chart/narrow-status grid preserve the selected hierarchy. The implementation uses the repository's 240px sidebar and 60px header exactly. -- Colors and visual tokens: white page canvas, `#f3f5f8` shell, indigo `#4f46e5`, green `#10b981`, amber `#f59e0b`, red `#ef4444`, and slate text are aligned with the source and current product tokens. -- Image and icon fidelity: the page contains no decorative raster imagery. The supplied product logo is preserved and existing Font Awesome icons are used consistently; no emoji, handcrafted SVG, placeholder image, or CSS illustration was introduced. -- Copy and content: dashboard title, health summary, chart legend and units, service states, task names, task status, model names, progress, accuracy, timestamps, and action labels match the approved design. - -## Interaction and runtime checks - -- Login with the existing `admin` credentials navigated to `/dashboard`, confirming the requested default entry behavior. -- ECharts rendered one canvas; hovering 07/10 exposed the tooltip values: training count 18, GPU count 7, and average accuracy 91%. -- “查看全部任务” navigated to `/fine-tune` and browser back restored `/dashboard`. -- The first “查看详情” action navigated to `/training-log/103942` and browser back restored `/dashboard`. -- Browser console errors: none. -- `npm run test:default-dashboard`: passed. -- `npm run test:dashboard`: passed. -- `npx vite build`: passed. - -## Comparison history - -### Iteration 1 — passed - -No actionable P0/P1/P2 differences remain. The only intentional product constraint is that the sidebar active background follows the repository's current neutral active token instead of the slightly bluer tint produced by ImageGen; location, contrast, label, and active-state clarity remain equivalent. - -## Validation gap - -`npm run type-check` remains blocked by pre-existing TypeScript errors in the mock adapter, dataset mock typing, data-process list, evaluation tabs, and fine-tune views. No reported error points to `DashboardView.vue`, the ECharts registration, router defaults, login redirect, or dashboard regression scripts. The direct Vite production build succeeds. - -Design-QA final result: passed - -final result: passed - -## Compact dashboard revision - -- User annotation reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-43f63327-063f-47da-9e96-31b53cebc49d.png` -- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-compact-1440.png` -- Viewport: 1440 × 1024 -- State: authenticated dashboard, compact layout, redundant title/action row removed. - -### Iteration 2 — passed - -The annotated header row containing the duplicate “服务看板” title, subtitle, and “查看告警” action was removed entirely. Section gaps, overview height, health icon, metric type, chart height, service rows, task heading, and task rows were reduced by roughly 10%–15%. The result preserves chart labels, dual-axis readability, service-state text, task progress, accuracy, and all task actions while bringing the primary content closer to the top of the page. - -- ECharts tooltip remains functional after the height reduction and reports all three 07/10 series values. -- The revised page contains no browser console errors. -- The full-resolution comparison makes the removed annotation target and the compact replacement legible; no focused crop is necessary. -- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build` pass. - -Design-QA final result: passed - -final result: passed - -## One-screen dashboard revision - -- User annotation reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-3c166ffc-7243-4e57-94fc-48949599c4f1.png` -- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-one-screen-1440x768.png` -- Viewport: 1440 × 768 -- State: authenticated dashboard with the desktop low-height compact rules active. - -### Iteration 3 — passed - -The platform-status block was reduced again, including its container padding, inner gap, health icon, status copy, metric labels, and metric values. The chart, service rows, task rows, and page-canvas padding now use a dedicated `max-height: 900px` desktop mode. The dashboard page canvas is constrained to the available application viewport so the outer content area does not introduce a vertical scrollbar. - -Browser measurements at 1440 × 768: - -- document overflow: false -- layout-content overflow: false -- page-canvas overflow: false -- dashboard overflow: false -- task section bottom: 653px within the 768px viewport -- ECharts tooltip: passed with all three series present -- browser console errors: none -- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build`: passed - -Design-QA final result: passed - -final result: passed - -## Flexible middle-region revision - -- User annotation reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-daf2bbae-baea-418d-9962-7d0e1a1c219b.png` -- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-flex-middle-1440x900.png` -- Viewport: 1440 × 900 -- State: authenticated dashboard with flexible middle-region growth. - -### Iteration 4 — passed - -The previous fixed-height middle row caused unused white space beneath the task table on taller screens. The dashboard now reserves compact intrinsic height for the platform summary and task table while allowing the chart/service row to consume all remaining viewport height. The ECharts canvas grows with that row, and the service-state rows distribute across the matching height. - -Browser measurements: - -- at 1440 × 768, chart height: 283px; no document, layout, canvas, or dashboard overflow -- at 1440 × 900, chart height: 415px; no document, layout, canvas, or dashboard overflow -- task table bottom at 1440 × 900: 868px within the 900px viewport -- ECharts tooltip: passed with all three series present -- browser console errors: none -- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build`: passed - -Design-QA final result: passed - -final result: passed - -## Narrow service-status revision - -- User request: make the right-hand service-status panel slightly narrower. -- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-narrow-service-1440x900.png` -- Viewports: 1440 × 900 and 1440 × 768 -- State: authenticated dashboard with the service-status column reduced to approximately 30% of the middle row. - -### Iteration 5 — passed - -The middle grid now allocates `1.9fr` to the training chart and `0.82fr` to service status, with a 300px minimum width for the service panel. This gives the chart more horizontal space while keeping all service names, status badges, and instance counts fully visible. - -Browser measurements: - -- at 1440 × 900, chart width: 799px; service width: 345px; service share: 30.1% -- at 1440 × 768, chart width: 799px; service width: 345px -- clipped service cells: none at both tested viewports -- horizontal and vertical document overflow: none at both tested viewports -- browser console errors: none -- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build`: passed - -Design-QA final result: passed - -final result: passed - -## Taller training-task revision - -- User request: increase the training-task region slightly and shorten the middle chart/service region. -- Source visual truth: `docs/superpowers/specs/assets/service-dashboard-narrow-service-1440x900.png` plus the current user annotation. -- Intended viewports: 1440 × 900 and 1440 × 768. -- State: authenticated dashboard with larger task heading and table rows. - -### Iteration 6 — blocked - -The task section now uses a taller heading and table rows in both standard and low-height desktop modes. Because the middle row is the only flexible region, the additional task height is taken directly from the chart/service row while preserving the one-screen layout contract in code. - -Verification evidence: - -- `npm run test:dashboard`: passed -- `npm run test:default-dashboard`: passed -- `npx vite build`: passed -- browser-rendered comparison: blocked because the in-app browser rejected the local preview URL under its URL security policy -- implementation screenshot: unavailable for this iteration - -Design-QA final result: blocked - -final result: blocked - ---- - -# Dataset Version Actions Design QA - -## Evidence - -- Source visual truth: `/Users/caoxiaozhu/.codex/generated_images/019f5e2e-3bee-77f0-b285-89ef139db56c/exec-48a163b6-d4c9-4646-9199-135957c6e72e.png` -- Historical-version implementation: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/implementation-historical-version-final.png` -- Delete-confirm implementation: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/implementation-delete-confirm.png` -- Full-view comparison: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/comparison-full.png` -- Focused version-control comparison: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/comparison-version-controls-final.png` -- Viewport: 1316 × 768 browser window; source crop normalized for the focused comparison. -- State: authenticated dataset detail, V3 current, V2 selected as a read-only historical version. - -## Full-view comparison - -The existing product shell, dataset summary, version selector, read-only alert, file toolbar, and sample table remain unchanged. The former standalone primary action has been replaced by one compact rounded-square overflow button at the far right of the version-control row, matching the selected hierarchy. - -## Focused comparison and required fidelity surfaces - -- Fonts and typography: existing system/PingFang stack, 13px labels, 12px metadata, and Element Plus menu text are preserved. -- Spacing and layout rhythm: the 40px overflow trigger aligns with the version selector and leaves the central status copy flexible; the 168px menu provides 40px action rows. -- Colors and visual tokens: the existing indigo primary token is used for the activate icon; the delete item and confirmation action use the danger token. -- Image and icon fidelity: no new raster assets are needed. Existing Font Awesome ellipsis, check-circle, and trash icons match the repository's icon system. -- Copy and content: the menu contains exactly “设为当前版本” and “删除版本”, separated visually; the confirmation names V2 and explains that current V3 is unaffected. - -## Interaction checks - -- Created V2 and V3 through the real edit-and-save flow, then switched from current V3 to historical V2. -- Historical records became read-only and the overflow trigger appeared; current V3 showed no history-operation trigger. -- Opening the trigger exposed exactly two accessible menu items: “设为当前版本” and “删除版本”. -- Choosing “删除版本” opened the danger confirmation dialog; cancelling returned focus without deleting data. -- Actual deletion behavior, protected-version rejection, optimistic-lock handling, and non-reused version numbers are covered by `test:dataset-preview`. - -## Comparison history - -### Iteration 1 — blocked - -- [P2] The overflow trigger was circular while the selected mock used a small rounded square. -- [P2] The menu did not explicitly lock its target width or primary-action icon color. - -Fixes: - -- Replaced the circular trigger with a 40px square and 10px radius. -- Set the menu minimum width to 168px, action height to 40px, and the activate icon to the product primary color. - -### Iteration 2 — passed - -No actionable P0/P1/P2 differences remain. The desktop capture API does not retain the transient popup layer in screenshots, so the open-menu labels were additionally verified through the accessibility tree; exact popup shadow rendering remains a non-blocking P3 capture gap. - -final result: passed - ---- - -# Login Page Responsive Design QA - -## Evidence - -- Source visual truth: `/Users/caoxiaozhu/.codex/generated_images/019f5f3f-f7e1-7e41-9605-ea307d9f09e6/exec-d376c603-0c47-45ad-86fd-3e99c1a95ec7.png` -- Implementation route: `http://localhost:6801/login` -- Implementation screenshot: unavailable because the in-app browser runtime could not initialize in this session. -- Intended desktop viewport: 1536 × 1024. -- Intended laptop viewports: 1366 × 768 and 1280 × 720. -- State: unauthenticated login page, default username and password populated. - -## Static and automated evidence - -- Added a 1440px width breakpoint that shifts the split from 58/42 to 55/45 and caps the form at 440px. -- Added a short-screen breakpoint for heights up to 820px that reduces title, form, input, footer, and panel spacing without hiding the left visual. -- Kept the single-column fallback at 900px and below. -- `regression-login-layout.mjs`: passed. -- `regression-default-dashboard.mjs`: passed. -- `vue-tsc -b --noEmit`: passed. -- Vite dev transform for `LoginView.vue`: HTTP 200. -- Production build: blocked by the pre-existing missing route component `UserPermissionView.vue`, outside the login-page change. - -## Required fidelity surfaces - -- Fonts and typography: code uses the existing product font stack with laptop-specific display-size reductions; visual comparison remains unavailable. -- Spacing and layout rhythm: dedicated width and height media queries are present; rendered measurements remain unavailable. -- Colors and visual tokens: existing indigo tokens and the selected dark-purple visual asset are preserved. -- Image quality and asset fidelity: the generated `login-hero-flow.png` is used directly; the official `logo.png` is reused for the brand lockup. -- Copy and content: platform title, supporting copy, form labels, actions, and footer match the selected design. - -## Findings - -- [P2] Browser-rendered laptop comparison unavailable - Location: login page at 1366 × 768 and 1280 × 720. - Evidence: the in-app browser runtime failed during initialization, so no implementation screenshot or side-by-side comparison could be captured. - Impact: static checks cannot prove that all visible spacing and crop details match the selected design at laptop sizes. - Fix: capture both laptop viewports in a working in-app browser session, compare them with the source, and resolve any remaining P0/P1/P2 differences. - -## Comparison history - -### Iteration 1 — blocked - -- User reported that the initial implementation was optimized for large displays and did not compose well on laptop screens. -- Added explicit laptop-width and short-screen layout rules and passed targeted regression/type checks. -- Post-fix visual evidence remains unavailable because browser capture is blocked. - -final result: blocked diff --git a/frontend/src/api/modules/compute.ts b/frontend/src/api/modules/compute.ts index 6d80d34..3557947 100644 --- a/frontend/src/api/modules/compute.ts +++ b/frontend/src/api/modules/compute.ts @@ -16,7 +16,16 @@ export interface ComputeNode { data_root: string model_root: string log_root: string + api_version?: string + capabilities?: string[] + description?: string last_health_check_at?: string + health_detail?: Record +} + +export type ComputeNodePayload = Partial & { + code?: string + api_base_url?: string } export interface ComputeGpu { @@ -66,11 +75,14 @@ export interface ResourceReplica { export const getComputeNodes = () => get('/compute/nodes') +export const createComputeNode = (data: ComputeNodePayload) => + post('/compute/nodes', data) + export const updateComputeNode = (id: string, data: Partial) => put(`/compute/nodes/${id}`, data) export const testComputeNode = (id: string) => - post<{ node_id: string; success: boolean; latency_ms: number }>(`/compute/nodes/${id}/test-connection`) + post<{ node_id: string; success: boolean; latency_ms: number; gpu_count: number; error?: string }>(`/compute/nodes/${id}/test-connection`) export const enableComputeNode = (id: string) => post(`/compute/nodes/${id}/enable`) diff --git a/frontend/src/api/modules/dashboard.ts b/frontend/src/api/modules/dashboard.ts index 0f6dd30..4611176 100644 --- a/frontend/src/api/modules/dashboard.ts +++ b/frontend/src/api/modules/dashboard.ts @@ -26,7 +26,7 @@ export interface DashboardStats { service_status: ServiceStatusStat[] training_tasks: TrainingTaskStat[] operation_distribution: { name: string; value: number }[] - login_duration_rank: { user: string; role: string; duration: string }[] + login_duration_rank: { user: string; role: string; duration: number }[] recent_login_users: { user: string; role: string; last_login: string }[] } diff --git a/frontend/src/api/modules/system.ts b/frontend/src/api/modules/system.ts index c6bbbf8..3edad90 100644 --- a/frontend/src/api/modules/system.ts +++ b/frontend/src/api/modules/system.ts @@ -19,6 +19,10 @@ export const getHealth = () => get('/health') export const login = (username: string, password: string) => post('/login', { username, password }) +/** 退出登录，上报会话结束以统计在线时长 */ +export const logout = (sessionId: string) => + post('/logout', { session_id: sessionId }) + /** 用户列表 */ export const getUsers = () => get('/users') diff --git a/frontend/src/stores/auth.ts b/frontend/src/stores/auth.ts index 870661d..d71a132 100644 --- a/frontend/src/stores/auth.ts +++ b/frontend/src/stores/auth.ts @@ -1,6 +1,6 @@ import { defineStore } from 'pinia' import { ref, computed } from 'vue' -import { login as loginApi } from '@/api/modules/system' +import { login as loginApi, logout as logoutApi } from '@/api/modules/system' import { SESSION_TIMEOUT } from '@/constants' import type { PermissionCode, SystemUser } from '@/types' @@ -61,6 +61,7 @@ export const useAuthStore = defineStore('auth', () => { return '观察员' }) const loginTime = ref(parseInt(localStorage.getItem('loginTime') || '0', 10) || 0) + const sessionId = ref(localStorage.getItem('sessionId') || '') const isLoggedIn = computed(() => { if (!loginTime.value) return false @@ -76,6 +77,8 @@ export const useAuthStore = defineStore('auth', () => { localStorage.setItem(USER_STORAGE_KEY, JSON.stringify(response.user)) localStorage.setItem('loginTime', String(loginTime.value)) localStorage.setItem('authToken', response.token) + sessionId.value = response.session_id + localStorage.setItem('sessionId', response.session_id) } /** 检查当前账号是否拥有指定模块权限。 */ @@ -93,13 +96,22 @@ export const useAuthStore = defineStore('auth', () => { } /** 退出 */ - function logout() { + async function logout() { + if (sessionId.value) { + try { + await logoutApi(sessionId.value) + } catch { + // 上报失败不影响本地退出 + } + } currentUser.value = null loginTime.value = 0 + sessionId.value = '' localStorage.removeItem('username') localStorage.removeItem(USER_STORAGE_KEY) localStorage.removeItem('loginTime') localStorage.removeItem('authToken') + localStorage.removeItem('sessionId') } return { @@ -108,6 +120,7 @@ export const useAuthStore = defineStore('auth', () => { displayName, roleLabel, loginTime, + sessionId, isLoggedIn, hasPermission, login, diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 450bcf4..11295a4 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -418,6 +418,7 @@ export interface SystemUser { export interface LoginResponse { token: string user: SystemUser + session_id: string } export interface CreateUserPayload { diff --git a/frontend/src/utils/status.ts b/frontend/src/utils/status.ts new file mode 100644 index 0000000..7cc1aa0 --- /dev/null +++ b/frontend/src/utils/status.ts @@ -0,0 +1,99 @@ +type TagType = 'primary' | 'success' | 'warning' | 'danger' | 'info' + +const STATUS_LABELS: Record = { + pending: '等待中', + syncing: '同步中', + queued: '排队中', + running: '运行中', + completed: '已完成', + failed: '失败', + stopped: '已停止', + cancelled: '已取消', + starting: '启动中', + loading: '加载中', + loaded: '已加载', + ready: '已就绪', + done: '已完成', + error: '异常', + success: '成功', + not_started: '未启动', + valid: '有效', + modified: '已修改', + invalid: '无效', + original: '原始', + manual: '手动新增', + active: '启用', + disabled: '停用', + online: '在线', + offline: '离线', + draining: '维护中', + maintenance: '维护模式', + busy: '忙碌', + reserved: '已预留', + idle: '空闲', + warning: '告警', + available: '可用', + missing: '缺失', + synced: '已同步', + drifted: '已漂移', + repair_pending: '待修复', +} + +const STATUS_TYPES: Record = { + completed: 'success', + success: 'success', + running: 'success', + ready: 'success', + loaded: 'success', + active: 'success', + online: 'success', + busy: 'success', + available: 'success', + synced: 'success', + valid: 'success', + pending: 'info', + idle: 'info', + offline: 'info', + disabled: 'info', + original: 'info', + stopped: 'info', + cancelled: 'info', + syncing: 'warning', + queued: 'warning', + starting: 'warning', + loading: 'warning', + reserved: 'warning', + warning: 'warning', + modified: 'warning', + draining: 'warning', + maintenance: 'warning', + repair_pending: 'warning', + failed: 'danger', + error: 'danger', + invalid: 'danger', + missing: 'danger', + drifted: 'danger', +} + +export function statusLabel(status?: string | number | null) { + const key = String(status ?? '').trim() + if (!key) return '未知' + return STATUS_LABELS[key] || key +} + +export function statusTagType(status?: string | number | null): TagType { + const key = String(status ?? '').trim() + return STATUS_TYPES[key] || 'info' +} + +export function mergeStatusLabel(row: { merged?: boolean; merging?: boolean }) { + if (row.merging) return '合并中' + if (row.merged) return '已合并' + return '未合并' +} + +export function mergeStatusType(row: { merged?: boolean; merging?: boolean }): TagType { + if (row.merging) return 'warning' + if (row.merged) return 'success' + return 'info' +} diff --git a/frontend/src/views/compute/ComputeNodesView.vue b/frontend/src/views/compute/ComputeNodesView.vue index 835d4ca..314fd07 100644 --- a/frontend/src/views/compute/ComputeNodesView.vue +++ b/frontend/src/views/compute/ComputeNodesView.vue @@ -1,7 +1,9 @@