update
This commit is contained in:
16
README.md
16
README.md
@@ -92,13 +92,13 @@ docker run -d --name yg_ft_pg -p 15432:5432 \
|
|||||||
|
|
||||||
### 3. 后端 / 算力 venv 首次创建
|
### 3. 后端 / 算力 venv 首次创建
|
||||||
后端与算力各有独立 venv,首次需在 WSL 中创建并安装依赖(完整命令见下方启动小节):
|
后端与算力各有独立 venv,首次需在 WSL 中创建并安装依赖(完整命令见下方启动小节):
|
||||||
- 后端:`cd /mnt/e/yg_ft/backend && python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt`
|
- 后端:`cd /home/wang/yg_ft/backend && python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt`
|
||||||
- 算力:`cd /mnt/e/yg_ft && source compute/.venv/bin/activate && pip install -r compute/requirements.txt`(compute 使用绝对导入 `compute.*`,须在仓库根目录操作)
|
- 算力:`cd /home/wang/yg_ft && source compute/.venv/bin/activate && pip install -r compute/requirements.txt`(compute 使用绝对导入 `compute.*`,须在仓库根目录操作)
|
||||||
|
|
||||||
### 4. 前端首次安装
|
### 4. 前端首次安装
|
||||||
在 Windows 终端:
|
在 Windows 终端:
|
||||||
```powershell
|
```powershell
|
||||||
cd e:\yg_ft\frontend
|
cd \\wsl.localhost\Ubuntu\home\wang\yg_ft\frontend
|
||||||
npm install
|
npm install
|
||||||
npm run dev
|
npm run dev
|
||||||
```
|
```
|
||||||
@@ -111,10 +111,8 @@ npm run dev
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 在 WSL 终端中执行
|
# 在 WSL 终端中执行
|
||||||
cd /mnt/e/yg_ft/backend
|
cd /home/wang/yg_ft/backend
|
||||||
python3 -m venv .venv
|
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
pip install -r requirements.txt
|
|
||||||
uvicorn app.main:app --host 0.0.0.0 --port 17861 --reload
|
uvicorn app.main:app --host 0.0.0.0 --port 17861 --reload
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -158,14 +156,12 @@ npm run dev
|
|||||||
|
|
||||||
## 算力服务启动
|
## 算力服务启动
|
||||||
|
|
||||||
> 算力服务同样需在 **WSL 终端** 中启动,且拥有**独立虚拟环境(不复用后端 venv)**。代码使用绝对导入 `compute.*`,因此必须从**仓库根目录(`/mnt/e/yg_ft`)**执行,不能先 `cd compute` 再启动(否则报 `No module named 'compute'`)。若 `.wslconfig` 使用 `networkingMode=mirrored`,uvicorn 需绑定 `--host 0.0.0.0` 才能被 Windows 侧 `localhost` 访问。
|
> 算力服务同样需在 **WSL 终端** 中启动,且拥有**独立虚拟环境(不复用后端 venv)**。代码使用绝对导入 `compute.*`,因此必须从**仓库根目录(`/home/wang/yg_ft`)**执行,不能先 `cd compute` 再启动(否则报 `No module named 'compute'`)。若 `.wslconfig` 使用 `networkingMode=mirrored`,uvicorn 需绑定 `--host 0.0.0.0` 才能被 Windows 侧 `localhost` 访问。
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 在 WSL 终端中执行(必须位于仓库根目录 yg_ft/)
|
# 在 WSL 终端中执行(必须位于仓库根目录 yg_ft/)
|
||||||
cd /mnt/e/yg_ft
|
cd /home/wang/yg_ft
|
||||||
python3 -m venv compute/.venv
|
|
||||||
source compute/.venv/bin/activate
|
source compute/.venv/bin/activate
|
||||||
pip install -r compute/requirements.txt
|
|
||||||
uvicorn compute.api.main:app --host 0.0.0.0 --port 19100 --reload
|
uvicorn compute.api.main:app --host 0.0.0.0 --port 19100 --reload
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import psycopg
|
|||||||
from fastapi import (
|
from fastapi import (
|
||||||
APIRouter,
|
APIRouter,
|
||||||
BackgroundTasks,
|
BackgroundTasks,
|
||||||
|
Request,
|
||||||
Body,
|
Body,
|
||||||
Depends,
|
Depends,
|
||||||
File,
|
File,
|
||||||
@@ -76,6 +77,7 @@ from app.modules.data_process.store import (
|
|||||||
get_data_process_store,
|
get_data_process_store,
|
||||||
new_id,
|
new_id,
|
||||||
)
|
)
|
||||||
|
from app.db.platform_store import get_platform_store
|
||||||
from app.schemas.data_process import (
|
from app.schemas.data_process import (
|
||||||
DataProcessRegenerateRequest,
|
DataProcessRegenerateRequest,
|
||||||
DataProcessStatus,
|
DataProcessStatus,
|
||||||
@@ -148,6 +150,13 @@ def fail(status_code: int, message: str) -> HTTPException:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _actor(request: Request | None) -> str | None:
|
||||||
|
if not request:
|
||||||
|
return None
|
||||||
|
auth = request.headers.get("Authorization", "")
|
||||||
|
return auth.replace("Bearer ", "").strip() or None
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def api_errors() -> Iterator[None]:
|
def api_errors() -> Iterator[None]:
|
||||||
try:
|
try:
|
||||||
@@ -790,9 +799,16 @@ def list_tasks(
|
|||||||
def create_task(
|
def create_task(
|
||||||
payload: DataProcessTaskCreate,
|
payload: DataProcessTaskCreate,
|
||||||
store: DataProcessStore = Depends(get_data_process_store),
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
|
request: Request = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
task = store.create_task(payload.model_dump(mode="json"))
|
task = store.create_task(payload.model_dump(mode="json"))
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="data-process.create",
|
||||||
|
actor_id=_actor(request),
|
||||||
|
target_type="data_process_task",
|
||||||
|
target_id=task["id"],
|
||||||
|
)
|
||||||
return ok(task, "data process task created")
|
return ok(task, "data process task created")
|
||||||
|
|
||||||
|
|
||||||
@@ -854,9 +870,16 @@ def prepare_regeneration(
|
|||||||
def delete_task(
|
def delete_task(
|
||||||
task_id: str,
|
task_id: str,
|
||||||
store: DataProcessStore = Depends(get_data_process_store),
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
|
request: Request = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
store.delete_task(task_id)
|
store.delete_task(task_id)
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="data-process.delete",
|
||||||
|
actor_id=_actor(request),
|
||||||
|
target_type="data_process_task",
|
||||||
|
target_id=task_id,
|
||||||
|
)
|
||||||
return ok({"deleted": task_id}, "data process task deleted")
|
return ok({"deleted": task_id}, "data process task deleted")
|
||||||
|
|
||||||
|
|
||||||
@@ -875,6 +898,7 @@ async def upload_source_files(
|
|||||||
files: list[UploadFile] = File(...),
|
files: list[UploadFile] = File(...),
|
||||||
store: DataProcessStore = Depends(get_data_process_store),
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||||
|
request: Request = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if not files:
|
if not files:
|
||||||
raise fail(400, "at least one source file is required")
|
raise fail(400, "at least one source file is required")
|
||||||
@@ -959,6 +983,12 @@ async def upload_source_files(
|
|||||||
finally:
|
finally:
|
||||||
if not commit_attempted:
|
if not commit_attempted:
|
||||||
storage.discard(staged)
|
storage.discard(staged)
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="data-process.upload",
|
||||||
|
actor_id=_actor(request),
|
||||||
|
target_type="data_process_task",
|
||||||
|
target_id=task_id,
|
||||||
|
)
|
||||||
return ok({"files": created}, "source files uploaded")
|
return ok({"files": created}, "source files uploaded")
|
||||||
|
|
||||||
|
|
||||||
@@ -1701,7 +1731,14 @@ def generate(
|
|||||||
background_tasks: BackgroundTasks,
|
background_tasks: BackgroundTasks,
|
||||||
payload: GenerateRequest = Body(default_factory=GenerateRequest),
|
payload: GenerateRequest = Body(default_factory=GenerateRequest),
|
||||||
store: DataProcessStore = Depends(get_data_process_store),
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
|
request: Request = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="data-process.generate",
|
||||||
|
actor_id=_actor(request),
|
||||||
|
target_type="data_process_task",
|
||||||
|
target_id=task_id,
|
||||||
|
)
|
||||||
return _start_generation(task_id, payload, background_tasks, store)
|
return _start_generation(task_id, payload, background_tasks, store)
|
||||||
|
|
||||||
|
|
||||||
@@ -2254,8 +2291,15 @@ def publish(
|
|||||||
task_id: str,
|
task_id: str,
|
||||||
payload: PublishRequest,
|
payload: PublishRequest,
|
||||||
store: DataProcessStore = Depends(get_data_process_store),
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
|
request: Request = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
result = store.publish(task_id, payload.model_dump(mode="json"))
|
result = store.publish(task_id, payload.model_dump(mode="json"))
|
||||||
message = "dataset published" if result["created"] else "dataset already published"
|
message = "dataset published" if result["created"] else "dataset already published"
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="data-process.publish",
|
||||||
|
actor_id=_actor(request),
|
||||||
|
target_type="data_process_task",
|
||||||
|
target_id=task_id,
|
||||||
|
)
|
||||||
return ok(result, message)
|
return ok(result, message)
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
|
import httpx
|
||||||
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
|
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
|
||||||
from fastapi.responses import PlainTextResponse, StreamingResponse
|
from fastapi.responses import PlainTextResponse, StreamingResponse
|
||||||
|
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
from app.modules.fine_tune.service import apply_presets
|
from app.modules.fine_tune.service import apply_presets
|
||||||
from fastapi import Request as FastAPIRequest
|
from fastapi import Request as FastAPIRequest
|
||||||
|
|
||||||
@@ -28,6 +31,15 @@ def fail(status_code: int, message: str) -> HTTPException:
|
|||||||
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
||||||
|
|
||||||
|
|
||||||
|
def _select_first_online_node(store: Any) -> dict[str, Any] | None:
|
||||||
|
"""Select the first online compute node for inference."""
|
||||||
|
nodes = store.compute_nodes()
|
||||||
|
for node in nodes:
|
||||||
|
if node.get("enabled") and node.get("scheduler_status") == "online":
|
||||||
|
return node
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@router.get("/dashboard/overview")
|
@router.get("/dashboard/overview")
|
||||||
async def dashboard_overview() -> dict[str, Any]:
|
async def dashboard_overview() -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
@@ -109,19 +121,29 @@ async def dashboard_stats() -> dict[str, Any]:
|
|||||||
"error": "failed",
|
"error": "failed",
|
||||||
"cancelled": "failed",
|
"cancelled": "failed",
|
||||||
}
|
}
|
||||||
op_labels = [
|
# 用户操作分布:仅展示「数据治理」与「模型服务」两大分组下的子模块,其他不显示
|
||||||
("模型训练", lambda a: "fine_tune" in a or "train" in a),
|
MODULE_LABELS = [
|
||||||
("数据处理", lambda a: "data" in a or "dataset" in a),
|
("data-process", "数据处理"),
|
||||||
("模型评测", lambda a: "eval" in a),
|
("data_process", "数据处理"),
|
||||||
("模型推理", lambda a: "infer" in a or "serving" in a or "deploy" in a),
|
("dataset", "数据集管理"),
|
||||||
("系统设置", lambda a: True),
|
("fine-tune", "模型训练"),
|
||||||
|
("fine_tune", "模型训练"),
|
||||||
|
("model-eval", "模型评测"),
|
||||||
|
("eval", "模型评测"),
|
||||||
|
("model-inference", "模型推理"),
|
||||||
|
("inference", "模型推理"),
|
||||||
|
("model-manage", "模型管理"),
|
||||||
|
("model", "模型管理"),
|
||||||
|
("trained", "模型管理"),
|
||||||
]
|
]
|
||||||
|
OP_ORDER = ["数据集管理", "数据处理", "模型训练", "模型评测", "模型推理", "模型管理"]
|
||||||
|
|
||||||
def _op_label(action: str) -> str:
|
def _op_module(action: str) -> str | None:
|
||||||
for label, fn in op_labels:
|
a = (action or "").lower()
|
||||||
if fn(action):
|
for prefix, label in MODULE_LABELS:
|
||||||
|
if a.startswith(prefix):
|
||||||
return label
|
return label
|
||||||
return "系统设置"
|
return None
|
||||||
|
|
||||||
training_tasks = [
|
training_tasks = [
|
||||||
{
|
{
|
||||||
@@ -138,12 +160,13 @@ async def dashboard_stats() -> dict[str, Any]:
|
|||||||
for t in tasks[:8]
|
for t in tasks[:8]
|
||||||
]
|
]
|
||||||
|
|
||||||
# 用户操作分布(按 audit action 归类为中文分类)
|
# 用户操作分布:仅统计数据治理/模型服务下子模块的操作,其他不显示
|
||||||
audit = store.audit_logs(limit=500)
|
audit = store.audit_logs(limit=1000)
|
||||||
op_counter: dict[str, int] = {}
|
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
|
||||||
for log in audit.get("items", []):
|
for log in audit.get("items", []):
|
||||||
act = log.get("action") or "unknown"
|
label = _op_module(log.get("action") or "")
|
||||||
op_counter[_op_label(act)] = op_counter.get(_op_label(act), 0) + 1
|
if label:
|
||||||
|
op_counter[label] += 1
|
||||||
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
|
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
|
||||||
|
|
||||||
# 最近登录用户:后端有 last_login 字段,返回真实数据
|
# 最近登录用户:后端有 last_login 字段,返回真实数据
|
||||||
@@ -160,8 +183,8 @@ async def dashboard_stats() -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
for u in recent
|
for u in recent
|
||||||
]
|
]
|
||||||
# 登录时长:后端暂无该数据源,先留空,待接入后补充
|
# 登录时长排行:基于 sessions 表真实会话时长(本月)
|
||||||
login_duration_rank: list = []
|
login_duration_rank = store.login_duration_rank()
|
||||||
|
|
||||||
return ok(
|
return ok(
|
||||||
{
|
{
|
||||||
@@ -571,7 +594,28 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
|
|||||||
|
|
||||||
@router.post("/compute/nodes/{node_id}/test-connection")
|
@router.post("/compute/nodes/{node_id}/test-connection")
|
||||||
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
||||||
return ok({"node_id": node_id, "success": True, "latency_ms": 12})
|
store = get_platform_store()
|
||||||
|
nodes = store.compute_nodes()
|
||||||
|
node = next((item for item in nodes if item["id"] == node_id), None)
|
||||||
|
if not node:
|
||||||
|
raise fail(404, "compute node not found")
|
||||||
|
try:
|
||||||
|
all_gpus = store.gpus()
|
||||||
|
node_gpus = [g for g in all_gpus if g.get("node_id") == node_id]
|
||||||
|
return ok({
|
||||||
|
"node_id": node_id,
|
||||||
|
"success": True,
|
||||||
|
"latency_ms": 12,
|
||||||
|
"gpu_count": len(node_gpus),
|
||||||
|
})
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
return ok({
|
||||||
|
"node_id": node_id,
|
||||||
|
"success": False,
|
||||||
|
"latency_ms": 0,
|
||||||
|
"gpu_count": 0,
|
||||||
|
"error": str(exc),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
@router.post("/compute/nodes/{node_id}/enable")
|
@router.post("/compute/nodes/{node_id}/enable")
|
||||||
@@ -842,3 +886,287 @@ async def compute_job_logs(job_id: str) -> dict[str, Any]:
|
|||||||
except KeyError as exc:
|
except KeyError as exc:
|
||||||
raise fail(404, str(exc))
|
raise fail(404, str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
# ===================== Model Evaluation =====================
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/model-eval")
|
||||||
|
async def model_eval_list() -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().eval_tasks())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/model-eval/{task_id}")
|
||||||
|
async def model_eval_detail(task_id: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().eval_task(task_id))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "eval task not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-eval/start")
|
||||||
|
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
task = get_platform_store().create_eval_task(payload)
|
||||||
|
return ok({"task_id": task["id"], **task})
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/model-eval/{task_id}")
|
||||||
|
async def model_eval_delete(task_id: str) -> dict[str, Any]:
|
||||||
|
get_platform_store().delete_eval_task(task_id)
|
||||||
|
return ok({"deleted": task_id})
|
||||||
|
|
||||||
|
|
||||||
|
# ===================== Eval Dimensions =====================
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/dimension")
|
||||||
|
async def dimension_list() -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().dimensions())
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/dimension")
|
||||||
|
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().create_dimension(payload))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/dimension/{dimension_id}")
|
||||||
|
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().dimension(dimension_id))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "dimension not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/dimension/{dimension_id}")
|
||||||
|
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().update_dimension(dimension_id, payload))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "dimension not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/dimension/{dimension_id}")
|
||||||
|
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
|
||||||
|
get_platform_store().delete_dimension(dimension_id)
|
||||||
|
return ok({"deleted": dimension_id})
|
||||||
|
|
||||||
|
|
||||||
|
# ===================== Model Compare / Inference =====================
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/model-compare")
|
||||||
|
async def model_compare_list() -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().compare_tasks())
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare")
|
||||||
|
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
task = get_platform_store().create_compare_task(payload)
|
||||||
|
return ok({"id": task["id"]})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/all/stop-all")
|
||||||
|
async def model_compare_stop_all() -> dict[str, Any]:
|
||||||
|
return ok({"stopped": True})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/stop-by-pid")
|
||||||
|
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
return ok({"stopped": True, "pid": payload.get("pid")})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/model-compare/{task_id}")
|
||||||
|
async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().compare_task(task_id))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "compare task not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/model-compare/{task_id}")
|
||||||
|
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
||||||
|
get_platform_store().delete_compare_task(task_id)
|
||||||
|
return ok({"deleted": task_id})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/model-compare/{task_id}/load-status")
|
||||||
|
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
task = get_platform_store().compare_task(task_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "compare task not found")
|
||||||
|
load_status = task.get("load_status") or {"loaded_models": []}
|
||||||
|
if isinstance(load_status, str):
|
||||||
|
try:
|
||||||
|
load_status = json.loads(load_status)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
load_status = {"loaded_models": []}
|
||||||
|
return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/{task_id}/load-status")
|
||||||
|
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "compare task not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/{task_id}/load")
|
||||||
|
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
task = get_platform_store().compare_task(task_id)
|
||||||
|
models = task.get("models") or []
|
||||||
|
if isinstance(models, str):
|
||||||
|
try:
|
||||||
|
models = json.loads(models)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
models = []
|
||||||
|
loaded_models = [
|
||||||
|
{
|
||||||
|
"model_id": item.get("model_id"),
|
||||||
|
"model_name": item.get("model_name"),
|
||||||
|
"status": "ready",
|
||||||
|
"pid": 45000 + index,
|
||||||
|
"port": item.get("port") or 18000 + index,
|
||||||
|
}
|
||||||
|
for index, item in enumerate(models)
|
||||||
|
if isinstance(item, dict)
|
||||||
|
]
|
||||||
|
return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "compare task not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/{task_id}/unload")
|
||||||
|
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "compare task not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/{task_id}/start-model")
|
||||||
|
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/chat-with-port")
|
||||||
|
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
question = ""
|
||||||
|
for message in payload.get("messages") or []:
|
||||||
|
if message.get("role") == "user":
|
||||||
|
question = str(message.get("content") or "")
|
||||||
|
content = f"当前后端已收到推理请求:{question[:120]}"
|
||||||
|
return ok({"response": content, "content": content})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-compare/stream-chat")
|
||||||
|
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
question = payload.get("user_question") or payload.get("question") or ""
|
||||||
|
return ok({"response": f"当前后端已收到流式推理请求:{str(question)[:120]}"})
|
||||||
|
|
||||||
|
|
||||||
|
# ===================== Model Chat (Inference Proxy) =====================
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-chat/batch")
|
||||||
|
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
return ok({"responses": [], "request": payload})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-chat/local/chat")
|
||||||
|
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
"""Proxy chat to the compute node running the inference model."""
|
||||||
|
store = get_platform_store()
|
||||||
|
node = _select_first_online_node(store)
|
||||||
|
if not node:
|
||||||
|
return ok({"response": "no online compute node available for inference", "request": payload})
|
||||||
|
try:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
result = await client._request("POST", "/inference/chat", json_data=payload)
|
||||||
|
return ok(result)
|
||||||
|
except Exception as exc:
|
||||||
|
return ok({"response": f"inference failed: {exc}", "request": payload})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-chat/local/chat/stream")
|
||||||
|
async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
|
||||||
|
"""Stream chat from the compute node."""
|
||||||
|
store = get_platform_store()
|
||||||
|
node = _select_first_online_node(store)
|
||||||
|
if not node:
|
||||||
|
return StreamingResponse(
|
||||||
|
iter(['data: {"error": "no online compute node"}\n\n']),
|
||||||
|
media_type="text/event-stream",
|
||||||
|
)
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
|
||||||
|
async def stream_proxy():
|
||||||
|
async with httpx.AsyncClient(timeout=300) as http:
|
||||||
|
url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream"
|
||||||
|
async with http.stream("POST", url, json=payload, headers=client.headers()) as resp:
|
||||||
|
async for chunk in resp.aiter_bytes():
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
return StreamingResponse(stream_proxy(), media_type="text/event-stream")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-chat/local/preload")
|
||||||
|
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
"""Load a model on the compute node for inference."""
|
||||||
|
store = get_platform_store()
|
||||||
|
node = _select_first_online_node(store)
|
||||||
|
if not node:
|
||||||
|
return ok({"loaded": False, "error": "no online compute node"})
|
||||||
|
try:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
result = await client._request("POST", "/inference/load", json_data=payload)
|
||||||
|
return ok(result)
|
||||||
|
except Exception as exc:
|
||||||
|
return ok({"loaded": False, "error": str(exc)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-chat/local/unload")
|
||||||
|
async def model_chat_local_unload() -> dict[str, Any]:
|
||||||
|
"""Unload the inference model from the compute node."""
|
||||||
|
store = get_platform_store()
|
||||||
|
node = _select_first_online_node(store)
|
||||||
|
if not node:
|
||||||
|
return ok({"unloaded": False, "error": "no online compute node"})
|
||||||
|
try:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
result = await client._request("POST", "/inference/unload", json_data={})
|
||||||
|
return ok(result)
|
||||||
|
except Exception as exc:
|
||||||
|
return ok({"unloaded": False, "error": str(exc)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/model-chat/local/status")
|
||||||
|
async def model_chat_local_status() -> dict[str, Any]:
|
||||||
|
"""Get inference session status from compute node."""
|
||||||
|
store = get_platform_store()
|
||||||
|
node = _select_first_online_node(store)
|
||||||
|
if not node:
|
||||||
|
return ok({"loaded": False, "error": "no online compute node"})
|
||||||
|
try:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
result = await client._request("GET", "/inference/status")
|
||||||
|
return ok(result)
|
||||||
|
except Exception as exc:
|
||||||
|
return ok({"loaded": False, "error": str(exc)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/model-chat/trained/preload")
|
||||||
|
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
|
"""Load a trained model (base + adapter) on the compute node for inference."""
|
||||||
|
store = get_platform_store()
|
||||||
|
node = _select_first_online_node(store)
|
||||||
|
if not node:
|
||||||
|
return ok({"loaded": False, "error": "no online compute node"})
|
||||||
|
try:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
result = await client._request("POST", "/inference/load", json_data=payload)
|
||||||
|
return ok(result)
|
||||||
|
except Exception as exc:
|
||||||
|
return ok({"loaded": False, "error": str(exc)})
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ class Settings:
|
|||||||
compute_mode: str = os.getenv("COMPUTE_MODE", "real")
|
compute_mode: str = os.getenv("COMPUTE_MODE", "real")
|
||||||
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
|
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
|
||||||
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
|
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
|
||||||
|
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||||
|
compute_request_timeout_seconds: float = float(os.getenv("COMPUTE_REQUEST_TIMEOUT_SECONDS", "30"))
|
||||||
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
||||||
log_dir: str = os.getenv("LOG_DIR", "./logs")
|
log_dir: str = os.getenv("LOG_DIR", "./logs")
|
||||||
log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend")
|
log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend")
|
||||||
|
|||||||
@@ -229,15 +229,21 @@ class PlatformStore:
|
|||||||
return f"{sec}s"
|
return f"{sec}s"
|
||||||
|
|
||||||
def refresh_runtime_state(self) -> None:
|
def refresh_runtime_state(self) -> None:
|
||||||
if get_settings().compute_mode != "simulator":
|
|
||||||
return
|
|
||||||
|
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
|
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
now_dt = datetime.now(timezone.utc)
|
now_dt = datetime.now(timezone.utc)
|
||||||
for row in rows:
|
for row in rows:
|
||||||
|
payload = json_loads(row["payload"], {})
|
||||||
|
compute_job_id = payload.get("compute_job_id")
|
||||||
|
compute_node_api = payload.get("compute_node_api")
|
||||||
|
if compute_job_id and compute_node_api:
|
||||||
|
# 已派发到算力:状态/进度/日志回传来自算力进程(架构 §1.1)
|
||||||
|
self._sync_task_from_compute(conn, row, payload, compute_job_id, compute_node_api)
|
||||||
|
continue
|
||||||
|
if get_settings().compute_mode != "simulator":
|
||||||
|
continue
|
||||||
start = parse_time(row["start_time"])
|
start = parse_time(row["start_time"])
|
||||||
if not start:
|
if not start:
|
||||||
continue
|
continue
|
||||||
@@ -252,7 +258,6 @@ class PlatformStore:
|
|||||||
else:
|
else:
|
||||||
status, progress = "completed", 100
|
status, progress = "completed", 100
|
||||||
|
|
||||||
payload = json_loads(row["payload"], {})
|
|
||||||
payload.update(
|
payload.update(
|
||||||
{
|
{
|
||||||
"status": status,
|
"status": status,
|
||||||
@@ -272,6 +277,7 @@ class PlatformStore:
|
|||||||
if status == "completed":
|
if status == "completed":
|
||||||
self._ensure_trained_model(conn, payload)
|
self._ensure_trained_model(conn, payload)
|
||||||
|
|
||||||
|
if get_settings().compute_mode == "simulator":
|
||||||
sync_rows = conn.execute(
|
sync_rows = conn.execute(
|
||||||
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
|
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
@@ -286,6 +292,49 @@ class PlatformStore:
|
|||||||
(status, progress, completed_at, row["id"]),
|
(status, progress, completed_at, row["id"]),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _sync_task_from_compute(
|
||||||
|
self,
|
||||||
|
conn,
|
||||||
|
row,
|
||||||
|
payload: dict[str, Any],
|
||||||
|
compute_job_id: str,
|
||||||
|
compute_node_api: str,
|
||||||
|
) -> None:
|
||||||
|
"""从算力节点拉回已派发任务的状态/进度/日志,写回本地任务记录。"""
|
||||||
|
try:
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
|
||||||
|
job = ComputeNodeClient(compute_node_api).get_job(compute_job_id)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
payload["compute_sync_error"] = str(exc)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
|
||||||
|
(json_dumps(payload), row["id"]),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
status = job.get("status")
|
||||||
|
progress = int(job.get("progress", 0) or 0)
|
||||||
|
logs = job.get("logs") or ""
|
||||||
|
payload.update(
|
||||||
|
{
|
||||||
|
"status": status,
|
||||||
|
"progress": progress,
|
||||||
|
"compute_logs": logs,
|
||||||
|
"train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE fine_tune_tasks
|
||||||
|
SET status=?, progress=?, payload=?, completed_at=?
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
(status, progress, json_dumps(payload), completed_at, row["id"]),
|
||||||
|
)
|
||||||
|
if status == "completed":
|
||||||
|
self._ensure_trained_model(conn, payload)
|
||||||
|
|
||||||
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None:
|
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None:
|
||||||
name = task.get("output_model_name") or f"{task['name']}-lora"
|
name = task.get("output_model_name") or f"{task['name']}-lora"
|
||||||
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
|
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
|
||||||
@@ -306,7 +355,7 @@ class PlatformStore:
|
|||||||
utcnow(),
|
utcnow(),
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
output_dir or f"/data/yg-ft/outputs/{task['name']}/adapter",
|
task.get("output_dir") or f"/data/yg-ft/outputs/{task.get('name')}/adapter",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -402,6 +451,79 @@ class PlatformStore:
|
|||||||
)
|
)
|
||||||
return {"id": aid}
|
return {"id": aid}
|
||||||
|
|
||||||
|
# ---- 登录会话(采集在线时长) ----
|
||||||
|
def create_session(self, user: dict[str, Any]) -> str:
|
||||||
|
sid = new_id("sess")
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""INSERT INTO sessions
|
||||||
|
(id, user_id, username, display_name, role, login_at, logout_at, duration_seconds, create_time)
|
||||||
|
VALUES (?,?,?,?,?,?,?,?,?)""",
|
||||||
|
(
|
||||||
|
sid,
|
||||||
|
user.get("id"),
|
||||||
|
user.get("username"),
|
||||||
|
user.get("display_name"),
|
||||||
|
user.get("role"),
|
||||||
|
utcnow(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
utcnow(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return sid
|
||||||
|
|
||||||
|
def close_session(self, session_id: str | None) -> None:
|
||||||
|
if not session_id:
|
||||||
|
return
|
||||||
|
with self.connect() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT login_at FROM sessions WHERE id=? AND logout_at IS NULL", (session_id,)
|
||||||
|
).fetchone()
|
||||||
|
if not row:
|
||||||
|
return
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
start = parse_time(row["login_at"]) or now
|
||||||
|
seconds = max(0, int((now - start).total_seconds()))
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sessions SET logout_at=?, duration_seconds=? WHERE id=?",
|
||||||
|
(utcnow(), seconds, session_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
def login_duration_rank(self, limit: int = 8) -> list[dict[str, Any]]:
|
||||||
|
"""本月登录时长排行:按用户聚合会话时长(小时)。"""
|
||||||
|
month_start = utcnow()[:7] + "01T00:00:00Z"
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT user_id, username, display_name, role, login_at, logout_at, duration_seconds "
|
||||||
|
"FROM sessions WHERE login_at >= ?",
|
||||||
|
(month_start,),
|
||||||
|
).fetchall()
|
||||||
|
agg: dict[str, dict[str, Any]] = {}
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
for r in rows:
|
||||||
|
uid = r["user_id"]
|
||||||
|
bucket = agg.setdefault(
|
||||||
|
uid,
|
||||||
|
{"user": r["display_name"] or r["username"], "role": r["role"] or "", "total": 0.0, "has": False},
|
||||||
|
)
|
||||||
|
dur = r["duration_seconds"]
|
||||||
|
if dur is None and r["logout_at"] is None:
|
||||||
|
start = parse_time(r["login_at"])
|
||||||
|
if start:
|
||||||
|
dur = max(0, int((now - start).total_seconds()))
|
||||||
|
if dur is None:
|
||||||
|
dur = 0
|
||||||
|
bucket["total"] += dur
|
||||||
|
bucket["has"] = True
|
||||||
|
result = [
|
||||||
|
{"user": b["user"], "role": b["role"], "duration": round(b["total"] / 3600, 1)}
|
||||||
|
for b in agg.values()
|
||||||
|
if b["has"]
|
||||||
|
]
|
||||||
|
result.sort(key=lambda x: x["duration"], reverse=True)
|
||||||
|
return result[:limit]
|
||||||
|
|
||||||
# ---- 审批模板 ----
|
# ---- 审批模板 ----
|
||||||
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
|
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
tid = payload.get("id") or new_id("tpl")
|
tid = payload.get("id") or new_id("tpl")
|
||||||
@@ -1096,20 +1218,80 @@ class PlatformStore:
|
|||||||
task_id,
|
task_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
if get_settings().compute_mode != "simulator":
|
api_base = node.get("api_base_url")
|
||||||
|
if api_base:
|
||||||
|
# GPU 计算派发到算力节点进程执行;后端只做调度编排(架构 §1.1)
|
||||||
|
self._dispatch_to_compute(task_id, node, merged, selected_gpus)
|
||||||
|
elif get_settings().compute_mode != "simulator":
|
||||||
|
# 降级路径:未配置算力节点时后端本机执行(违反 §1.1,待移除)
|
||||||
from app.modules.fine_tune.service import launch_training
|
from app.modules.fine_tune.service import launch_training
|
||||||
|
|
||||||
launch_training(task_id)
|
launch_training(task_id)
|
||||||
return self.task(task_id)
|
return self.task(task_id)
|
||||||
|
|
||||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
def _dispatch_to_compute(
|
||||||
task = self.task(task_id)
|
self,
|
||||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)})
|
task_id: str,
|
||||||
|
node: dict[str, Any],
|
||||||
|
task: dict[str, Any],
|
||||||
|
gpus: list,
|
||||||
|
) -> None:
|
||||||
|
"""把训练作业派发到算力节点,GPU 计算在算力进程内执行;后端记录算力 job id。"""
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
|
||||||
|
cfg = {
|
||||||
|
"id": f"ft_{task_id}",
|
||||||
|
"name": task.get("name") or task_id,
|
||||||
|
"type": "fine_tune",
|
||||||
|
"gpus": gpus,
|
||||||
|
"stage": str(task.get("train_type") or "SFT").lower(),
|
||||||
|
"base_model": task.get("base_model") or "placeholder-base-model",
|
||||||
|
"dataset": task.get("train_dataset_id") or task.get("dataset") or "placeholder-dataset",
|
||||||
|
"template": task.get("template") or "qwen",
|
||||||
|
"train_method": task.get("train_method") or "lora",
|
||||||
|
"output_dir": f"/data/yg-ft/outputs/{task.get('name') or task_id}/adapter",
|
||||||
|
"batch_size": int(task.get("batch_size", 2) or 2),
|
||||||
|
"learning_rate": float(task.get("learning_rate", 0.0002) or 0.0002),
|
||||||
|
"n_epochs": int(task.get("n_epochs", 3) or 3),
|
||||||
|
}
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
try:
|
||||||
|
job = client.create_job(cfg)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
self.update_task_runtime(task_id, status="failed", extra={"dispatch_error": str(exc)})
|
||||||
|
raise RuntimeError(f"dispatch training job to compute node failed: {exc}") from exc
|
||||||
|
compute_job_id = job.get("id")
|
||||||
|
current = self.task(task_id)
|
||||||
|
updated = {**current, "compute_job_id": compute_job_id, "compute_node_api": node["api_base_url"]}
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
|
||||||
|
(json_dumps(updated), task_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||||
|
task = self.task(task_id)
|
||||||
|
# 如果任务已派发到算力节点,先通知算力停止
|
||||||
|
compute_job_id = task.get("compute_job_id")
|
||||||
|
node_id = task.get("compute_node_id")
|
||||||
|
if compute_job_id and node_id:
|
||||||
|
try:
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
node = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
||||||
|
if node and node.get("api_base_url"):
|
||||||
|
ComputeNodeClient(node["api_base_url"]).stop_job(compute_job_id)
|
||||||
|
except Exception: # noqa: BLE001 - best effort stop
|
||||||
|
pass
|
||||||
|
task.update({"status": "stopped", "progress": min(task.get("progress", 0), 99)})
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE fine_tune_tasks SET status='stopped', payload=?, completed_at=? WHERE id=?",
|
||||||
(json_dumps(task), utcnow(), task_id),
|
(json_dumps(task), utcnow(), task_id),
|
||||||
)
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
|
||||||
|
(utcnow(), task_id),
|
||||||
|
)
|
||||||
return self.task(task_id)
|
return self.task(task_id)
|
||||||
|
|
||||||
def update_task_runtime(
|
def update_task_runtime(
|
||||||
@@ -1934,6 +2116,490 @@ class PlatformStore:
|
|||||||
content = self.generate_training_log(task)
|
content = self.generate_training_log(task)
|
||||||
return {"job_id": job_id, "content": content, "lines": len(content.splitlines())}
|
return {"job_id": job_id, "content": content, "lines": len(content.splitlines())}
|
||||||
|
|
||||||
|
# ===================== Model Evaluation =====================
|
||||||
|
|
||||||
|
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
||||||
|
payload = json_loads(row["payload"], {})
|
||||||
|
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
||||||
|
return payload
|
||||||
|
|
||||||
|
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
||||||
|
return [self._json_payload_row(row) for row in rows]
|
||||||
|
|
||||||
|
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
raise KeyError(task_id)
|
||||||
|
payload = self._json_payload_row(row)
|
||||||
|
payload.setdefault("sample_count", 0)
|
||||||
|
payload.setdefault("completed_count", 0)
|
||||||
|
payload.setdefault("passed_count", 0)
|
||||||
|
payload.setdefault("overall_score", payload.get("score") or 0)
|
||||||
|
payload.setdefault("overall_score_max", 100)
|
||||||
|
payload.setdefault("overall_evaluation", "")
|
||||||
|
payload.setdefault("improvement_suggestions", [])
|
||||||
|
payload.setdefault("dimension_summary", [])
|
||||||
|
payload.setdefault("samples", [])
|
||||||
|
return payload
|
||||||
|
|
||||||
|
def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval"))
|
||||||
|
name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}")
|
||||||
|
status = str(payload.get("status") or "pending")
|
||||||
|
now = payload.get("create_time") or utcnow()
|
||||||
|
data = {
|
||||||
|
**payload,
|
||||||
|
"id": task_id,
|
||||||
|
"eval_task_name": name,
|
||||||
|
"status": status,
|
||||||
|
"create_time": now,
|
||||||
|
"metric": payload.get("metric") or "custom",
|
||||||
|
}
|
||||||
|
with self.connect() as conn:
|
||||||
|
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
|
||||||
|
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
||||||
|
if model:
|
||||||
|
data.setdefault("model_name", model["name"])
|
||||||
|
if dataset:
|
||||||
|
data.setdefault("dataset", dataset["name"])
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(task_id, name, json_dumps(data), status, now),
|
||||||
|
)
|
||||||
|
return self.eval_task(task_id)
|
||||||
|
|
||||||
|
def delete_eval_task(self, task_id: str) -> None:
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
|
||||||
|
|
||||||
|
def dimensions(self) -> list[dict[str, Any]]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
**json_loads(row["payload"], {}),
|
||||||
|
"id": row["id"],
|
||||||
|
"name": row["name"],
|
||||||
|
"is_active": bool(row["is_active"]),
|
||||||
|
"is_default": bool(row["is_default"]),
|
||||||
|
"create_time": row["create_time"],
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
def dimension(self, dimension_id: str) -> dict[str, Any]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
raise KeyError(dimension_id)
|
||||||
|
return {
|
||||||
|
**json_loads(row["payload"], {}),
|
||||||
|
"id": row["id"],
|
||||||
|
"name": row["name"],
|
||||||
|
"is_active": bool(row["is_active"]),
|
||||||
|
"is_default": bool(row["is_default"]),
|
||||||
|
"create_time": row["create_time"],
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
dimension_id = str(payload.get("id") or new_id("dim"))
|
||||||
|
name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}")
|
||||||
|
now = payload.get("create_time") or utcnow()
|
||||||
|
data = {**payload, "id": dimension_id, "name": name, "create_time": now}
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)",
|
||||||
|
(dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now),
|
||||||
|
)
|
||||||
|
return self.dimension(dimension_id)
|
||||||
|
|
||||||
|
def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
current = self.dimension(dimension_id)
|
||||||
|
merged = {**current, **payload, "id": dimension_id}
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?",
|
||||||
|
(
|
||||||
|
merged["name"],
|
||||||
|
json_dumps(merged),
|
||||||
|
1 if merged.get("is_active", True) else 0,
|
||||||
|
1 if merged.get("is_default") else 0,
|
||||||
|
dimension_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return self.dimension(dimension_id)
|
||||||
|
|
||||||
|
def delete_dimension(self, dimension_id: str) -> None:
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,))
|
||||||
|
|
||||||
|
# ===================== Model Compare / Inference =====================
|
||||||
|
|
||||||
|
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall()
|
||||||
|
return [self._json_payload_row(row) for row in rows]
|
||||||
|
|
||||||
|
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
raise KeyError(task_id)
|
||||||
|
return self._json_payload_row(row)
|
||||||
|
|
||||||
|
def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
task_id = str(payload.get("id") or new_id("cmp"))
|
||||||
|
name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}")
|
||||||
|
status = str(payload.get("status") or "pending")
|
||||||
|
now = payload.get("create_time") or utcnow()
|
||||||
|
data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now}
|
||||||
|
data.setdefault("load_status", json_dumps({"loaded_models": []}))
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(task_id, name, json_dumps(data), status, now),
|
||||||
|
)
|
||||||
|
return self.compare_task(task_id)
|
||||||
|
|
||||||
|
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
current = self.compare_task(task_id)
|
||||||
|
merged = {**current, **payload, "id": task_id}
|
||||||
|
status = str(merged.get("status") or current.get("status") or "pending")
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?",
|
||||||
|
(merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id),
|
||||||
|
)
|
||||||
|
return self.compare_task(task_id)
|
||||||
|
|
||||||
|
def delete_compare_task(self, task_id: str) -> None:
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,))
|
||||||
|
|
||||||
|
# ===================== Compute Job Sync (派发回传) =====================
|
||||||
|
|
||||||
|
def _acquire_scheduler_lock(
|
||||||
|
self,
|
||||||
|
conn: PgConnection,
|
||||||
|
lock_key: str,
|
||||||
|
owner: str,
|
||||||
|
ttl_seconds: int = 30,
|
||||||
|
) -> bool:
|
||||||
|
now = utcnow()
|
||||||
|
expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds)).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||||
|
existing = conn.execute("SELECT owner, expires_at FROM scheduler_locks WHERE lock_key=?", (lock_key,)).fetchone()
|
||||||
|
if existing:
|
||||||
|
if existing["expires_at"] > now and existing["owner"] != owner:
|
||||||
|
return False
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE scheduler_locks SET owner=?, expires_at=?, update_time=? WHERE lock_key=?",
|
||||||
|
(owner, expires_at, now, lock_key),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO scheduler_locks (lock_key, owner, expires_at, create_time, update_time) VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(lock_key, owner, expires_at, now, now),
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _upsert_compute_job(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
|
||||||
|
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
|
||||||
|
now = utcnow()
|
||||||
|
command = job.get("command") or []
|
||||||
|
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
|
||||||
|
payload = json_dumps({**job, "task_id": task["id"]})
|
||||||
|
existing = conn.execute("SELECT id FROM compute_jobs WHERE id=?", (job_id,)).fetchone()
|
||||||
|
if existing:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE compute_jobs
|
||||||
|
SET task_id=?, node_id=?, engine=?, status=?, command=?, output_dir=?, log_file=?,
|
||||||
|
payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
task["id"],
|
||||||
|
task.get("compute_node_id"),
|
||||||
|
str(task.get("engine") or job.get("engine") or "llama_factory"),
|
||||||
|
status,
|
||||||
|
command_text,
|
||||||
|
job.get("output_dir") or task.get("output_dir"),
|
||||||
|
job.get("log_file") or task.get("log_file"),
|
||||||
|
payload,
|
||||||
|
now,
|
||||||
|
now if status in {"completed", "failed", "stopped"} else None,
|
||||||
|
job_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO compute_jobs
|
||||||
|
(id, task_id, node_id, engine, status, command, output_dir, log_file, payload, create_time, update_time, completed_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
job_id,
|
||||||
|
task["id"],
|
||||||
|
task.get("compute_node_id"),
|
||||||
|
str(task.get("engine") or job.get("engine") or "llama_factory"),
|
||||||
|
status,
|
||||||
|
command_text,
|
||||||
|
job.get("output_dir") or task.get("output_dir"),
|
||||||
|
job.get("log_file") or task.get("log_file"),
|
||||||
|
payload,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
now if status in {"completed", "failed", "stopped"} else None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _sync_gpu_allocations(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
|
||||||
|
terminal = status in {"completed", "failed", "stopped"}
|
||||||
|
if terminal:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
|
||||||
|
(utcnow(), task["id"]),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
|
||||||
|
allocation_status = "running" if status == "running" else "allocated"
|
||||||
|
for gpu_index in [int(item) for item in task.get("gpus") or job.get("gpus") or []]:
|
||||||
|
existing = conn.execute(
|
||||||
|
"SELECT id FROM gpu_allocations WHERE task_id=? AND node_id=? AND gpu_index=? AND status IN ('allocated','running')",
|
||||||
|
(task["id"], task.get("compute_node_id"), gpu_index),
|
||||||
|
).fetchone()
|
||||||
|
if existing:
|
||||||
|
conn.execute("UPDATE gpu_allocations SET status=?, compute_job_id=? WHERE id=?", (allocation_status, job_id, existing["id"]))
|
||||||
|
continue
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO gpu_allocations
|
||||||
|
(id, task_id, compute_job_id, node_id, gpu_index, status, create_time)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(new_id("gpu_alloc"), task["id"], job_id, task.get("compute_node_id"), gpu_index, allocation_status, utcnow()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _upsert_checkpoints(self, conn: PgConnection, task_id: str, checkpoints: list[dict[str, Any]]) -> None:
|
||||||
|
for item in checkpoints:
|
||||||
|
path = str(item.get("path") or "")
|
||||||
|
if not path:
|
||||||
|
continue
|
||||||
|
step = int(item.get("step") or 0)
|
||||||
|
name = str(item.get("name") or Path(path).name)
|
||||||
|
size_bytes = int(item.get("size_bytes") or item.get("size") or 0)
|
||||||
|
existing = conn.execute("SELECT id FROM fine_tune_checkpoints WHERE task_id=? AND path=?", (task_id, path)).fetchone()
|
||||||
|
if existing:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE fine_tune_checkpoints SET step=?, name=?, size_bytes=? WHERE id=?",
|
||||||
|
(step, name, size_bytes, existing["id"]),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fine_tune_checkpoints
|
||||||
|
(id, task_id, step, name, path, size_bytes, create_time)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(new_id("ckpt"), task_id, step, name, path, size_bytes, utcnow()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
status_map = {
|
||||||
|
"queued": "queued",
|
||||||
|
"running": "running",
|
||||||
|
"completed": "completed",
|
||||||
|
"failed": "failed",
|
||||||
|
"stopped": "stopped",
|
||||||
|
}
|
||||||
|
current = self.task(task_id)
|
||||||
|
status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"]))
|
||||||
|
progress = int(job.get("progress", current.get("progress", 0)) or 0)
|
||||||
|
payload = {
|
||||||
|
**current,
|
||||||
|
"status": status,
|
||||||
|
"progress": progress,
|
||||||
|
"process_id": job.get("pid") or current.get("process_id"),
|
||||||
|
"compute_job_id": job.get("id") or current.get("compute_job_id"),
|
||||||
|
"output_dir": job.get("output_dir") or current.get("output_dir"),
|
||||||
|
"log_file": job.get("log_file") or current.get("log_file"),
|
||||||
|
"artifacts": job.get("artifacts") or current.get("artifacts") or [],
|
||||||
|
}
|
||||||
|
if status == "failed":
|
||||||
|
payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed"
|
||||||
|
elif status in {"queued", "running", "completed"}:
|
||||||
|
payload.pop("failure_reason", None)
|
||||||
|
completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE fine_tune_tasks
|
||||||
|
SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at)
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
json_dumps(payload),
|
||||||
|
status,
|
||||||
|
progress,
|
||||||
|
payload.get("process_id"),
|
||||||
|
payload.get("compute_job_id"),
|
||||||
|
completed_at,
|
||||||
|
task_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._upsert_compute_job(conn, payload, job, status)
|
||||||
|
self._sync_gpu_allocations(conn, payload, job, status)
|
||||||
|
self._upsert_checkpoints(conn, task_id, job.get("checkpoints") or [])
|
||||||
|
if status == "completed":
|
||||||
|
self._ensure_trained_model(conn, payload)
|
||||||
|
if status in {"failed", "stopped"}:
|
||||||
|
failure_reason = job.get("error") or job.get("message") or "compute job failed"
|
||||||
|
log_snippet = job.get("log_snippet") or ""
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE fine_tune_tasks SET failure_reason = ? WHERE id = ?",
|
||||||
|
(failure_reason[:2000], task_id),
|
||||||
|
)
|
||||||
|
if log_snippet:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE fine_tune_tasks SET payload = ? WHERE id = ?",
|
||||||
|
(json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id),
|
||||||
|
)
|
||||||
|
return self.task(task_id)
|
||||||
|
|
||||||
|
def running_compute_tasks(self) -> list[dict[str, Any]]:
|
||||||
|
return [
|
||||||
|
task
|
||||||
|
for task in self.tasks()
|
||||||
|
if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"}
|
||||||
|
]
|
||||||
|
|
||||||
|
def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]:
|
||||||
|
task = self.task(task_id)
|
||||||
|
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason})
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
||||||
|
(json_dumps(task), utcnow(), task_id),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
|
||||||
|
(utcnow(), task_id),
|
||||||
|
)
|
||||||
|
return self.task(task_id)
|
||||||
|
|
||||||
|
def record_training_log_metrics(self, task_id: str, content: str) -> int:
|
||||||
|
rows: list[tuple[Any, ...]] = []
|
||||||
|
for line_number, line in enumerate(content.splitlines(), start=1):
|
||||||
|
metric = self._parse_training_metric(line)
|
||||||
|
if not metric:
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
(
|
||||||
|
new_id("metric"),
|
||||||
|
task_id,
|
||||||
|
line_number,
|
||||||
|
metric.get("epoch"),
|
||||||
|
metric.get("loss"),
|
||||||
|
metric.get("grad_norm"),
|
||||||
|
metric.get("learning_rate"),
|
||||||
|
line[:2000],
|
||||||
|
utcnow(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute("DELETE FROM fine_tune_metrics WHERE task_id=?", (task_id,))
|
||||||
|
if rows:
|
||||||
|
conn.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO fine_tune_metrics
|
||||||
|
(id, task_id, step, epoch, loss, grad_norm, learning_rate, raw, create_time)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
return len(rows)
|
||||||
|
|
||||||
|
def _parse_training_metric(self, line: str) -> dict[str, Any] | None:
|
||||||
|
if "loss" not in line or "learning_rate" not in line:
|
||||||
|
return None
|
||||||
|
import re
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
|
||||||
|
match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
|
||||||
|
if match:
|
||||||
|
result[key] = float(match.group(1))
|
||||||
|
return result or None
|
||||||
|
|
||||||
|
def task_metrics(self, task_id: str) -> list[dict[str, Any]]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT step, epoch, loss, grad_norm, learning_rate, raw, create_time
|
||||||
|
FROM fine_tune_metrics
|
||||||
|
WHERE task_id=?
|
||||||
|
ORDER BY step
|
||||||
|
""",
|
||||||
|
(task_id,),
|
||||||
|
).fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
def task_checkpoints(self, task_id: str) -> list[dict[str, Any]]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, step, name, path, size_bytes, create_time
|
||||||
|
FROM fine_tune_checkpoints
|
||||||
|
WHERE task_id=?
|
||||||
|
ORDER BY step, create_time
|
||||||
|
""",
|
||||||
|
(task_id,),
|
||||||
|
).fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
def active_standalone_compute_jobs(self) -> list[dict[str, Any]]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM compute_jobs
|
||||||
|
WHERE task_id IS NULL AND status IN ('queued','running')
|
||||||
|
ORDER BY create_time
|
||||||
|
""",
|
||||||
|
).fetchall()
|
||||||
|
return [json_loads(row["payload"], {}) if "payload" in row.keys() else dict(row) for row in rows]
|
||||||
|
|
||||||
|
def sync_model_merge_job(self, job_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
current = self.compute_job(job_id)
|
||||||
|
payload = current.get("payload") or {}
|
||||||
|
job_payload = payload.get("job") if isinstance(payload.get("job"), dict) else {}
|
||||||
|
merged_payload = {**payload, "job": {**job_payload, **job}}
|
||||||
|
status = str(job.get("status") or current.get("status") or "queued")
|
||||||
|
command = job.get("command") or current.get("command") or []
|
||||||
|
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
|
||||||
|
output_dir = job.get("output_dir") or payload.get("output_dir") or current.get("output_dir")
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE compute_jobs
|
||||||
|
SET status=?, command=?, output_dir=?, log_file=?, payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
status,
|
||||||
|
command_text,
|
||||||
|
output_dir,
|
||||||
|
job.get("log_file") or current.get("log_file"),
|
||||||
|
json_dumps(merged_payload),
|
||||||
|
utcnow(),
|
||||||
|
utcnow() if status in {"completed", "failed", "stopped"} else None,
|
||||||
|
job_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return self.compute_job(job_id)
|
||||||
|
|
||||||
|
|
||||||
_store: PlatformStore | None = None
|
_store: PlatformStore | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -152,6 +152,18 @@ CREATE TABLE IF NOT EXISTS project_members (
|
|||||||
|
|
||||||
-- ===================== Fine-tune Checkpoints =====================
|
-- ===================== Fine-tune Checkpoints =====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||||
|
step INTEGER NOT NULL,
|
||||||
|
epoch DOUBLE PRECISION,
|
||||||
|
loss DOUBLE PRECISION,
|
||||||
|
grad_norm DOUBLE PRECISION,
|
||||||
|
learning_rate DOUBLE PRECISION,
|
||||||
|
raw TEXT NOT NULL,
|
||||||
|
create_time TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||||
@@ -164,6 +176,25 @@ CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
|||||||
create_time TEXT NOT NULL
|
create_time TEXT NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS gpu_allocations (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||||
|
compute_job_id TEXT,
|
||||||
|
node_id TEXT REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||||
|
gpu_index INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
released_at TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS scheduler_locks (
|
||||||
|
lock_key TEXT PRIMARY KEY,
|
||||||
|
owner TEXT NOT NULL,
|
||||||
|
expires_at TEXT NOT NULL,
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
update_time TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
-- ===================== Compute Jobs (internal) =====================
|
-- ===================== Compute Jobs (internal) =====================
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS compute_jobs (
|
CREATE TABLE IF NOT EXISTS compute_jobs (
|
||||||
@@ -185,6 +216,33 @@ CREATE TABLE IF NOT EXISTS compute_jobs (
|
|||||||
completed_at TEXT
|
completed_at TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- ===================== Model Evaluation =====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS eval_tasks (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
payload TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
create_time TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS eval_dimensions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
payload TEXT NOT NULL,
|
||||||
|
is_active INTEGER NOT NULL DEFAULT 1,
|
||||||
|
is_default INTEGER NOT NULL DEFAULT 0,
|
||||||
|
create_time TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS compare_tasks (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
payload TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
create_time TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
-- ===================== Indexes =====================
|
-- ===================== Indexes =====================
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
||||||
@@ -194,9 +252,15 @@ CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_t
|
|||||||
CREATE INDEX IF NOT EXISTS idx_projects_tenant ON projects(tenant_id);
|
CREATE INDEX IF NOT EXISTS idx_projects_tenant ON projects(tenant_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_project_members_user ON project_members(user_id);
|
CREATE INDEX IF NOT EXISTS idx_project_members_user ON project_members(user_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_checkpoints_task ON fine_tune_checkpoints(task_id);
|
CREATE INDEX IF NOT EXISTS idx_checkpoints_task ON fine_tune_checkpoints(task_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_metrics_task_step ON fine_tune_metrics(task_id, step);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
|
||||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
|
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node ON compute_jobs(node_id);
|
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node ON compute_jobs(node_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_status ON compute_jobs(status);
|
CREATE INDEX IF NOT EXISTS idx_compute_jobs_status ON compute_jobs(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
|
||||||
|
|
||||||
-- ===================== Migrations: extend fine_tune_tasks =====================
|
-- ===================== Migrations: extend fine_tune_tasks =====================
|
||||||
|
|
||||||
|
|||||||
@@ -127,3 +127,16 @@ INSERT INTO roles (id, name, display_name, permissions, create_time) VALUES
|
|||||||
('role_operator','operator','操作员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', '2026-01-01T00:00:00Z'),
|
('role_operator','operator','操作员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', '2026-01-01T00:00:00Z'),
|
||||||
('role_viewer', 'viewer', '访客', '["dashboard"]', '2026-01-01T00:00:00Z')
|
('role_viewer', 'viewer', '访客', '["dashboard"]', '2026-01-01T00:00:00Z')
|
||||||
ON CONFLICT (name) DO NOTHING;
|
ON CONFLICT (name) DO NOTHING;
|
||||||
|
|
||||||
|
-- 登录会话:采集每次登录/登出,用于统计在线/登录时长
|
||||||
|
CREATE TABLE IF NOT EXISTS sessions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
username TEXT NOT NULL,
|
||||||
|
display_name TEXT,
|
||||||
|
role TEXT,
|
||||||
|
login_at TEXT NOT NULL,
|
||||||
|
logout_at TEXT,
|
||||||
|
duration_seconds INTEGER,
|
||||||
|
create_time TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
@@ -6,11 +8,20 @@ from app.core.config import get_settings
|
|||||||
from app.core.logging import configure_logging, setup_request_logging
|
from app.core.logging import configure_logging, setup_request_logging
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _lifespan(app: FastAPI):
|
||||||
|
# 启动算力状态轮询线程(同步派发到算力的训练任务状态/日志/指标)
|
||||||
|
from app.modules.fine_tune.service import start_compute_sync_worker, stop_compute_sync_worker
|
||||||
|
start_compute_sync_worker()
|
||||||
|
yield
|
||||||
|
stop_compute_sync_worker()
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
configure_logging(settings)
|
configure_logging(settings)
|
||||||
|
|
||||||
app = FastAPI(title=settings.app_name)
|
app = FastAPI(title=settings.app_name, lifespan=_lifespan)
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
allow_origins=settings.cors_allow_origins,
|
allow_origins=settings.cors_allow_origins,
|
||||||
|
|||||||
@@ -16,13 +16,25 @@ class LoginBody(BaseModel):
|
|||||||
password: str
|
password: str
|
||||||
|
|
||||||
|
|
||||||
|
class LogoutBody(BaseModel):
|
||||||
|
session_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
@router.post("/login")
|
@router.post("/login")
|
||||||
def login(body: LoginBody) -> dict:
|
def login(body: LoginBody) -> dict:
|
||||||
user = get_platform_store().login(body.username, body.password)
|
store = get_platform_store()
|
||||||
|
user = store.login(body.username, body.password)
|
||||||
if not user:
|
if not user:
|
||||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
|
||||||
token = create_access_token(user["id"])
|
token = create_access_token(user["id"])
|
||||||
return {"code": 0, "message": "ok", "data": {"token": token, "user": user}}
|
session_id = store.create_session(user)
|
||||||
|
return {"code": 0, "message": "ok", "data": {"token": token, "user": user, "session_id": session_id}}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/logout")
|
||||||
|
def logout(body: LogoutBody) -> dict:
|
||||||
|
get_platform_store().close_session(body.session_id)
|
||||||
|
return {"code": 0, "message": "ok", "data": None}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/me")
|
@router.get("/me")
|
||||||
|
|||||||
@@ -1 +1,6 @@
|
|||||||
"""Application-side compute platform gateway module."""
|
"""Application-side compute platform gateway module."""
|
||||||
|
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||||
|
|
||||||
|
__all__ = ["ComputeNodeClient", "poll_compute_jobs_once"]
|
||||||
|
|||||||
180
backend/app/modules/compute_gateway/client.py
Normal file
180
backend/app/modules/compute_gateway/client.py
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
"""Client for talking to a compute node's REST API.
|
||||||
|
|
||||||
|
This is the application-side bridge: the application backend never runs GPU
|
||||||
|
workloads itself; it dispatches them to a compute node and reads back status,
|
||||||
|
logs and metrics through this client.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from app.core.config import get_settings
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_timeout() -> float:
|
||||||
|
return float(get_settings().compute_request_timeout_seconds or 30.0)
|
||||||
|
|
||||||
|
|
||||||
|
def _auth_headers() -> dict:
|
||||||
|
token = get_settings().compute_service_token
|
||||||
|
if not token:
|
||||||
|
return {}
|
||||||
|
return {"Authorization": f"Bearer {token}"}
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeNodeClient:
|
||||||
|
"""Thin wrapper over a single compute node's HTTP API."""
|
||||||
|
|
||||||
|
def __init__(self, base_url: str, timeout: float | None = None):
|
||||||
|
self.base_url = base_url.rstrip("/")
|
||||||
|
self.timeout = timeout or _compute_timeout()
|
||||||
|
|
||||||
|
# ---- health -------------------------------------------------------
|
||||||
|
def health(self) -> dict:
|
||||||
|
last_err = None
|
||||||
|
for path in ("/v1/compute/health", "/health"):
|
||||||
|
try:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.get(f"{self.base_url}{path}", headers=_auth_headers())
|
||||||
|
if r.status_code == 200:
|
||||||
|
return r.json()
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
last_err = str(exc)
|
||||||
|
raise RuntimeError(f"compute node unhealthy: {last_err}")
|
||||||
|
|
||||||
|
# ---- gpus ---------------------------------------------------------
|
||||||
|
def gpus(self) -> list:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.get(
|
||||||
|
f"{self.base_url}/compute/resources/gpus",
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json().get("data", [])
|
||||||
|
|
||||||
|
# ---- jobs ---------------------------------------------------------
|
||||||
|
def create_job(self, payload: dict) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/jobs",
|
||||||
|
json=payload,
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def preview_job(self, payload: dict) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/jobs/preview",
|
||||||
|
json=payload,
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def validate_job(self, payload: dict) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/jobs/validate",
|
||||||
|
json=payload,
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def get_job(self, job_id: str) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.get(
|
||||||
|
f"{self.base_url}/compute/jobs/{job_id}",
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def stop_job(self, job_id: str) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/jobs/{job_id}/stop",
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def job_logs(self, job_id: str, cursor: int = 0, limit: int = 200) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.get(
|
||||||
|
f"{self.base_url}/compute/jobs/{job_id}/logs",
|
||||||
|
params={"cursor": cursor, "limit": limit},
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
# ---- files --------------------------------------------------------
|
||||||
|
def check_paths(self, paths: list) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/files/check-paths",
|
||||||
|
json={"paths": paths},
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def list_files(self, path: str = "/") -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.get(
|
||||||
|
f"{self.base_url}/compute/files/list",
|
||||||
|
params={"path": path},
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def import_local_file(self, src_path: str, dest_name: str | None = None) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/files/import-local",
|
||||||
|
json={"src_path": src_path, "dest_name": dest_name},
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def upload_file(self, filename: str, content: bytes, content_type: str | None = None) -> dict:
|
||||||
|
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||||
|
r = c.post(
|
||||||
|
f"{self.base_url}/compute/files/upload",
|
||||||
|
files={"file": (filename, content, content_type)},
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
# ---- inference (async) -------------------------------------------
|
||||||
|
|
||||||
|
def headers(self) -> dict[str, str]:
|
||||||
|
"""Return auth headers for compute node requests."""
|
||||||
|
token = get_settings().compute_service_token
|
||||||
|
if not token:
|
||||||
|
return {}
|
||||||
|
return {"X-Compute-Token": token}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def route_prefix(self) -> str:
|
||||||
|
return get_settings().route_prefix.rstrip("/") or "/modelTF"
|
||||||
|
|
||||||
|
async def _request(self, method: str, path: str, json_data: dict | None = None) -> dict:
|
||||||
|
"""Generic async request method for compute API endpoints."""
|
||||||
|
prefix = self.route_prefix
|
||||||
|
url = f"{self.base_url.rstrip('/')}{prefix}{path}"
|
||||||
|
async with httpx.AsyncClient(timeout=300, verify=False, headers=self.headers()) as client:
|
||||||
|
if method.upper() == "GET":
|
||||||
|
response = await client.get(url)
|
||||||
|
else:
|
||||||
|
response = await client.post(url, json=json_data)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
if isinstance(data, dict) and isinstance(data.get("data"), dict):
|
||||||
|
return data["data"]
|
||||||
|
return data if isinstance(data, dict) else {}
|
||||||
50
backend/app/modules/compute_gateway/sync.py
Normal file
50
backend/app/modules/compute_gateway/sync.py
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.db.platform_store import get_platform_store
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
|
||||||
|
|
||||||
|
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
|
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||||
|
|
||||||
|
|
||||||
|
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||||
|
store = get_platform_store()
|
||||||
|
synced: list[dict[str, Any]] = []
|
||||||
|
failed: list[dict[str, str]] = []
|
||||||
|
for task in store.running_compute_tasks():
|
||||||
|
node = _node_for_task(task)
|
||||||
|
if not node:
|
||||||
|
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
|
job = client.get_job(task["compute_job_id"])
|
||||||
|
try:
|
||||||
|
logs = client.job_logs(task["compute_job_id"], tail_lines=5000)
|
||||||
|
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if job.get("status") in {"failed", "stopped"}:
|
||||||
|
try:
|
||||||
|
last_logs = client.job_logs(task["compute_job_id"], tail_lines=200)
|
||||||
|
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
synced.append(store.apply_compute_job(task["id"], job))
|
||||||
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||||
|
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||||
|
standalone_synced: list[dict[str, Any]] = []
|
||||||
|
for record in store.active_standalone_compute_jobs():
|
||||||
|
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
|
||||||
|
if not node:
|
||||||
|
failed.append({"job_id": record["id"], "error": "compute node not found"})
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
job = ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
||||||
|
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||||
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||||
|
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||||
|
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
|
||||||
@@ -3,13 +3,17 @@
|
|||||||
|
|
||||||
- preset 参数预设(quick / standard / high)
|
- preset 参数预设(quick / standard / high)
|
||||||
- train_type → stage 映射(sft/dpo/cpt/cot)
|
- train_type → stage 映射(sft/dpo/cpt/cot)
|
||||||
- 训练任务的启动 / 暂停 / 恢复 / 取消(委托 runner 真实执行)
|
- 训练任务的启动 / 暂停 / 恢复 / 取消
|
||||||
|
- compute_gateway 状态轮询线程(当任务派发到算力时自动同步状态)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
from app.modules.fine_tune import runner
|
from app.modules.fine_tune import runner
|
||||||
|
|
||||||
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
|
|||||||
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
|
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── compute sync 轮询线程 ──────────────────────────────────────────────
|
||||||
|
_sync_thread: threading.Thread | None = None
|
||||||
|
_sync_thread_stop = threading.Event()
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_sync_loop() -> None:
|
||||||
|
"""后台线程:周期性轮询算力节点,同步训练任务状态/日志/指标。"""
|
||||||
|
interval = get_settings().compute_poll_interval_seconds or 3
|
||||||
|
while not _sync_thread_stop.is_set():
|
||||||
|
try:
|
||||||
|
store = get_platform_store()
|
||||||
|
running = store.running_compute_tasks()
|
||||||
|
if running:
|
||||||
|
asyncio.run(_poll_once())
|
||||||
|
except Exception: # noqa: BLE001 - keep polling loop alive
|
||||||
|
pass
|
||||||
|
time.sleep(interval)
|
||||||
|
|
||||||
|
|
||||||
|
async def _poll_once() -> None:
|
||||||
|
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||||
|
await poll_compute_jobs_once()
|
||||||
|
|
||||||
|
|
||||||
|
def start_compute_sync_worker() -> None:
|
||||||
|
"""启动后台轮询线程(幂等,多次调用安全)。"""
|
||||||
|
global _sync_thread
|
||||||
|
if _sync_thread is not None and _sync_thread.is_alive():
|
||||||
|
return
|
||||||
|
_sync_thread_stop.clear()
|
||||||
|
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
|
||||||
|
_sync_thread.start()
|
||||||
|
|
||||||
|
|
||||||
|
def stop_compute_sync_worker() -> None:
|
||||||
|
"""停止后台轮询线程。"""
|
||||||
|
_sync_thread_stop.set()
|
||||||
|
|
||||||
|
|
||||||
|
# ── preset / config ────────────────────────────────────────────────────
|
||||||
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
|
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
"""根据 preset 字段补全缺失的超参;preset=custom 时不覆盖。"""
|
"""根据 preset 字段补全缺失的超参;preset=custom 时不覆盖。"""
|
||||||
payload = dict(payload)
|
payload = dict(payload)
|
||||||
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
def launch_training(task_id: str) -> None:
|
def launch_training(task_id: str) -> None:
|
||||||
"""在后台线程启动真实训练。"""
|
"""在后台线程启动真实训练(本机 subprocess fallback,当算力节点不可用时使用)。
|
||||||
|
|
||||||
|
架构原则:GPU 计算应派发到算力服务进程执行。
|
||||||
|
当 platform_store.start_task 检测到在线算力节点时,会走 _dispatch_to_compute 派发路径;
|
||||||
|
仅当无可用算力节点且非 simulator 模式时,降级到本机 runner(违反 §1.1,待移除)。
|
||||||
|
"""
|
||||||
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()
|
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
5
backend/data/dataset_info.json
Normal file
5
backend/data/dataset_info.json
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"ds_0b95db886181": {
|
||||||
|
"file_name": "fine_tune_datasets/ds_0b95db886181.json"
|
||||||
|
}
|
||||||
|
}
|
||||||
16
backend/data/fine_tune_datasets/ds_0b95db886181.json
Normal file
16
backend/data/fine_tune_datasets/ds_0b95db886181.json
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
[ {
|
||||||
|
"instruction": "数字钱包的“零余额”管理在日末结算时有何具体要求?",
|
||||||
|
"input": "",
|
||||||
|
"output": "数字钱包严格执行“零余额”管理,在日末结算时,若结算钱包存在余额,必须将数字货币兑回基本账户,以确保日末数字钱包无余额。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"instruction": "用印登记表中包含哪些具体的印章类型?",
|
||||||
|
"input": "",
|
||||||
|
"output": "根据用印登记表的内容显示,表中明确列出了五种具体的印章类型,分别是财务专用章、法人名章、法人授权人名章1、法人授权人名章2以及其他。这些印章类型被详细划分在“用印数量”这一栏目下,用于记录不同印章的具体使用情况。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"instruction": "密钥交接记录表中需要哪些角色签字确认?",
|
||||||
|
"input": "",
|
||||||
|
"output": "密钥交接记录表中需要三个角色的签字确认,分别是交接人签字、接交人签字以及监交人签字。交接人负责移交密钥,接交人负责接收密钥,而监交人则负责对整个密钥交接过程进行监督,这三个角色的共同签字确认能够确保密钥交接流程的规范性与安全性。"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
[ {
|
||||||
|
"instruction": "数字钱包的“零余额”管理在日末结算时有何具体要求?",
|
||||||
|
"input": "",
|
||||||
|
"output": "数字钱包严格执行“零余额”管理,在日末结算时,若结算钱包存在余额,必须将数字货币兑回基本账户,以确保日末数字钱包无余额。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"instruction": "用印登记表中包含哪些具体的印章类型?",
|
||||||
|
"input": "",
|
||||||
|
"output": "根据用印登记表的内容显示,表中明确列出了五种具体的印章类型,分别是财务专用章、法人名章、法人授权人名章1、法人授权人名章2以及其他。这些印章类型被详细划分在“用印数量”这一栏目下,用于记录不同印章的具体使用情况。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"instruction": "密钥交接记录表中需要哪些角色签字确认?",
|
||||||
|
"input": "",
|
||||||
|
"output": "密钥交接记录表中需要三个角色的签字确认,分别是交接人签字、接交人签字以及监交人签字。交接人负责移交密钥,接交人负责接收密钥,而监交人则负责对整个密钥交接过程进行监督,这三个角色的共同签字确认能够确保密钥交接流程的规范性与安全性。"
|
||||||
|
}
|
||||||
|
]
|
||||||
281
compute/agent/process_manager.py
Normal file
281
compute/agent/process_manager.py
Normal file
@@ -0,0 +1,281 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import contextlib
|
||||||
|
import hashlib
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
TERMINAL_STATUSES = {"completed", "failed", "stopped"}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ManagedProcess:
|
||||||
|
id: str
|
||||||
|
name: str
|
||||||
|
command: list[str]
|
||||||
|
work_dir: str
|
||||||
|
log_path: Path
|
||||||
|
output_dir: str
|
||||||
|
gpus: list[int]
|
||||||
|
process: subprocess.Popen[Any] | None
|
||||||
|
created_at: float
|
||||||
|
pid: int | None = None
|
||||||
|
status: str = "running"
|
||||||
|
progress: int = 5
|
||||||
|
artifacts: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessManager:
|
||||||
|
def __init__(self, log_root: str) -> None:
|
||||||
|
self.log_root = Path(log_root)
|
||||||
|
self.log_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.registry_path = self.log_root / "compute-jobs.json"
|
||||||
|
self.jobs: dict[str, ManagedProcess] = {}
|
||||||
|
self._load_registry()
|
||||||
|
|
||||||
|
def create_job(self, payload: dict[str, Any], command: list[str], work_dir: str) -> dict[str, Any]:
|
||||||
|
job_id = str(payload.get("id") or f"job_{int(time.time() * 1000)}")
|
||||||
|
if job_id in self.jobs and self.jobs[job_id].status not in TERMINAL_STATUSES:
|
||||||
|
raise ValueError(f"job {job_id} is already running")
|
||||||
|
|
||||||
|
output_dir = str(payload.get("output_dir") or f"/data/yg-ft/outputs/{payload.get('name', job_id)}")
|
||||||
|
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
log_path = self.log_root / f"{job_id}.log"
|
||||||
|
env = os.environ.copy()
|
||||||
|
gpus = [int(item) for item in payload.get("gpus") or []]
|
||||||
|
locked = self.locked_gpus()
|
||||||
|
conflict = sorted(set(gpus).intersection(locked))
|
||||||
|
if conflict:
|
||||||
|
raise ValueError(f"gpu already locked: {conflict}")
|
||||||
|
if gpus:
|
||||||
|
env["CUDA_VISIBLE_DEVICES"] = ",".join(str(item) for item in gpus)
|
||||||
|
env.update({str(k): str(v) for k, v in payload.get("env", {}).items()})
|
||||||
|
|
||||||
|
cwd = work_dir if Path(work_dir).exists() else None
|
||||||
|
with log_path.open("ab") as log_file:
|
||||||
|
log_file.write(f"[INFO] starting job_id={job_id} command={' '.join(command)}\n".encode("utf-8"))
|
||||||
|
process = subprocess.Popen(
|
||||||
|
command,
|
||||||
|
cwd=cwd,
|
||||||
|
env=env,
|
||||||
|
stdout=log_file,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
managed = ManagedProcess(
|
||||||
|
id=job_id,
|
||||||
|
name=str(payload.get("name") or job_id),
|
||||||
|
command=command,
|
||||||
|
work_dir=work_dir,
|
||||||
|
log_path=log_path,
|
||||||
|
output_dir=output_dir,
|
||||||
|
gpus=gpus,
|
||||||
|
process=process,
|
||||||
|
created_at=time.time(),
|
||||||
|
pid=process.pid,
|
||||||
|
progress=10,
|
||||||
|
)
|
||||||
|
self.jobs[job_id] = managed
|
||||||
|
data = self.serialize(managed)
|
||||||
|
self._save_registry()
|
||||||
|
return data
|
||||||
|
|
||||||
|
def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||||
|
job = self.jobs.get(job_id)
|
||||||
|
if not job:
|
||||||
|
return None
|
||||||
|
return self.serialize(job)
|
||||||
|
|
||||||
|
def list_jobs(self) -> list[dict[str, Any]]:
|
||||||
|
return [self.serialize(job) for job in self.jobs.values()]
|
||||||
|
|
||||||
|
def stop_job(self, job_id: str) -> dict[str, Any] | None:
|
||||||
|
job = self.jobs.get(job_id)
|
||||||
|
if not job:
|
||||||
|
return None
|
||||||
|
if job.status not in TERMINAL_STATUSES:
|
||||||
|
try:
|
||||||
|
if job.process is not None and os.name == "nt":
|
||||||
|
job.process.terminate()
|
||||||
|
elif job.pid is not None:
|
||||||
|
os.kill(job.pid, signal.SIGTERM)
|
||||||
|
if job.process is not None:
|
||||||
|
job.process.wait(timeout=10)
|
||||||
|
except Exception:
|
||||||
|
if job.process is not None:
|
||||||
|
job.process.kill()
|
||||||
|
elif job.pid is not None:
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
os.kill(job.pid, signal.SIGKILL)
|
||||||
|
job.status = "stopped"
|
||||||
|
job.progress = min(job.progress, 99)
|
||||||
|
data = self.serialize(job)
|
||||||
|
self._save_registry()
|
||||||
|
return data
|
||||||
|
|
||||||
|
def logs(self, job_id: str) -> str:
|
||||||
|
job = self.jobs.get(job_id)
|
||||||
|
if not job or not job.log_path.exists():
|
||||||
|
return ""
|
||||||
|
return job.log_path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
|
||||||
|
def serialize(self, job: ManagedProcess) -> dict[str, Any]:
|
||||||
|
code = job.process.poll() if job.process is not None else None
|
||||||
|
checkpoints = self._collect_checkpoints(job.output_dir)
|
||||||
|
if job.status not in TERMINAL_STATUSES:
|
||||||
|
if job.process is None and job.pid is not None and not self._pid_alive(job.pid):
|
||||||
|
job.status = "failed"
|
||||||
|
job.progress = min(job.progress, 99)
|
||||||
|
code = -1
|
||||||
|
elif code is None:
|
||||||
|
job.status = "running"
|
||||||
|
elapsed = max(0, int(time.time() - job.created_at))
|
||||||
|
job.progress = min(95, max(job.progress, 10 + elapsed // 6))
|
||||||
|
elif code == 0:
|
||||||
|
job.status = "completed"
|
||||||
|
job.progress = 100
|
||||||
|
job.artifacts = self._collect_artifacts(job.output_dir)
|
||||||
|
else:
|
||||||
|
job.status = "failed"
|
||||||
|
job.progress = min(job.progress, 99)
|
||||||
|
self._save_registry()
|
||||||
|
return {
|
||||||
|
"id": job.id,
|
||||||
|
"name": job.name,
|
||||||
|
"status": job.status,
|
||||||
|
"progress": job.progress,
|
||||||
|
"pid": job.pid,
|
||||||
|
"gpus": job.gpus,
|
||||||
|
"created_at": job.created_at,
|
||||||
|
"command": job.command,
|
||||||
|
"work_dir": job.work_dir,
|
||||||
|
"output_dir": job.output_dir,
|
||||||
|
"log_file": str(job.log_path),
|
||||||
|
"artifacts": job.artifacts,
|
||||||
|
"checkpoints": checkpoints,
|
||||||
|
"return_code": code,
|
||||||
|
}
|
||||||
|
|
||||||
|
def locked_gpus(self) -> set[int]:
|
||||||
|
locked: set[int] = set()
|
||||||
|
for job in self.jobs.values():
|
||||||
|
status = self.serialize(job)["status"]
|
||||||
|
if status in {"queued", "running"}:
|
||||||
|
locked.update(job.gpus)
|
||||||
|
return locked
|
||||||
|
|
||||||
|
def _collect_artifacts(self, output_dir: str) -> list[dict[str, Any]]:
|
||||||
|
root = Path(output_dir)
|
||||||
|
if not root.exists():
|
||||||
|
return []
|
||||||
|
artifacts: list[dict[str, Any]] = []
|
||||||
|
for path in root.rglob("*"):
|
||||||
|
if path.is_file():
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
size = path.stat().st_size
|
||||||
|
artifacts.append(
|
||||||
|
{
|
||||||
|
"path": str(path),
|
||||||
|
"name": path.name,
|
||||||
|
"size": size,
|
||||||
|
"size_bytes": size,
|
||||||
|
"checksum_sha256": digest.hexdigest(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return artifacts[:200]
|
||||||
|
|
||||||
|
def _collect_checkpoints(self, output_dir: str) -> list[dict[str, Any]]:
|
||||||
|
root = Path(output_dir)
|
||||||
|
if not root.exists():
|
||||||
|
return []
|
||||||
|
checkpoints: list[dict[str, Any]] = []
|
||||||
|
for path in root.glob("checkpoint-*"):
|
||||||
|
if not path.is_dir():
|
||||||
|
continue
|
||||||
|
step = 0
|
||||||
|
try:
|
||||||
|
step = int(path.name.rsplit("-", 1)[-1])
|
||||||
|
except ValueError:
|
||||||
|
step = 0
|
||||||
|
size_bytes = sum(item.stat().st_size for item in path.rglob("*") if item.is_file())
|
||||||
|
checkpoints.append(
|
||||||
|
{
|
||||||
|
"step": step,
|
||||||
|
"name": path.name,
|
||||||
|
"path": str(path),
|
||||||
|
"size_bytes": size_bytes,
|
||||||
|
"create_time": path.stat().st_mtime,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return sorted(checkpoints, key=lambda item: (int(item.get("step") or 0), str(item.get("name") or "")))
|
||||||
|
|
||||||
|
def _save_registry(self) -> None:
|
||||||
|
items = []
|
||||||
|
for job in self.jobs.values():
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"id": job.id,
|
||||||
|
"name": job.name,
|
||||||
|
"command": job.command,
|
||||||
|
"work_dir": job.work_dir,
|
||||||
|
"log_path": str(job.log_path),
|
||||||
|
"output_dir": job.output_dir,
|
||||||
|
"gpus": job.gpus,
|
||||||
|
"pid": job.pid,
|
||||||
|
"created_at": job.created_at,
|
||||||
|
"status": job.status,
|
||||||
|
"progress": job.progress,
|
||||||
|
"artifacts": job.artifacts,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.registry_path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
|
def _load_registry(self) -> None:
|
||||||
|
if not self.registry_path.exists():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
items = json.loads(self.registry_path.read_text(encoding="utf-8"))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return
|
||||||
|
for item in items if isinstance(items, list) else []:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
pid = item.get("pid")
|
||||||
|
status = item.get("status", "failed")
|
||||||
|
if status not in TERMINAL_STATUSES and pid and not self._pid_alive(int(pid)):
|
||||||
|
status = "failed"
|
||||||
|
job = ManagedProcess(
|
||||||
|
id=str(item["id"]),
|
||||||
|
name=str(item.get("name") or item["id"]),
|
||||||
|
command=[str(part) for part in item.get("command") or []],
|
||||||
|
work_dir=str(item.get("work_dir") or ""),
|
||||||
|
log_path=Path(item.get("log_path") or self.log_root / f"{item['id']}.log"),
|
||||||
|
output_dir=str(item.get("output_dir") or ""),
|
||||||
|
gpus=[int(gpu) for gpu in item.get("gpus") or []],
|
||||||
|
process=None,
|
||||||
|
pid=int(pid) if pid else None,
|
||||||
|
created_at=float(item.get("created_at") or time.time()),
|
||||||
|
status=status,
|
||||||
|
progress=int(item.get("progress") or 0),
|
||||||
|
artifacts=item.get("artifacts") or [],
|
||||||
|
)
|
||||||
|
self.jobs[job.id] = job
|
||||||
|
|
||||||
|
def _pid_alive(self, pid: int) -> bool:
|
||||||
|
if pid <= 0:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
os.kill(pid, 0)
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
@@ -2,19 +2,39 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import math
|
import math
|
||||||
|
import hashlib
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
|
||||||
|
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||||
|
|
||||||
from compute.engines.llama_factory.adapter import build_command, parse_log_line
|
from compute.agent.process_manager import ProcessManager
|
||||||
|
from compute.engines.llama_factory.adapter import build_command, parse_log_line, prepare_runtime_files
|
||||||
|
from compute.engines.llama_factory.inference import get_inference_session
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
app = FastAPI(title="YG Fine-Tune Compute API")
|
app = FastAPI(title="YG Fine-Tune Compute API")
|
||||||
jobs: dict[str, dict[str, Any]] = {}
|
jobs: dict[str, dict[str, Any]] = {}
|
||||||
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
|
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
|
||||||
|
process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"))
|
||||||
|
|
||||||
|
@app.middleware("http")
|
||||||
|
async def compute_token_auth(request: Request, call_next):
|
||||||
|
token = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||||
|
auth_enabled = os.getenv("COMPUTE_AUTH_ENABLED", "true").lower() == "true"
|
||||||
|
public_paths = {f"{route_prefix}/health", "/health"}
|
||||||
|
if auth_enabled and token and request.url.path not in public_paths:
|
||||||
|
header_token = request.headers.get("x-compute-token", "")
|
||||||
|
auth_header = request.headers.get("authorization", "")
|
||||||
|
bearer_token = auth_header.removeprefix("Bearer ").strip() if auth_header.startswith("Bearer ") else ""
|
||||||
|
if header_token != token and bearer_token != token:
|
||||||
|
return JSONResponse({"detail": "invalid compute service token"}, status_code=401)
|
||||||
|
return await call_next(request)
|
||||||
|
|
||||||
def now() -> float:
|
def now() -> float:
|
||||||
return time.time()
|
return time.time()
|
||||||
@@ -25,6 +45,110 @@ def create_app() -> FastAPI:
|
|||||||
def execution_mode() -> str:
|
def execution_mode() -> str:
|
||||||
return os.getenv("COMPUTE_EXECUTION_MODE", os.getenv("COMPUTE_MODE", "real")).lower()
|
return os.getenv("COMPUTE_EXECUTION_MODE", os.getenv("COMPUTE_MODE", "real")).lower()
|
||||||
|
|
||||||
|
def _int_env(name: str, default: int) -> int:
|
||||||
|
raw = os.getenv(name)
|
||||||
|
if raw is None or raw == "":
|
||||||
|
return default
|
||||||
|
return int(raw)
|
||||||
|
|
||||||
|
def _float_env(name: str, default: float) -> float:
|
||||||
|
raw = os.getenv(name)
|
||||||
|
if raw is None or raw == "":
|
||||||
|
return default
|
||||||
|
return float(raw)
|
||||||
|
|
||||||
|
def _path_inside(root: Path, candidate: Path) -> bool:
|
||||||
|
try:
|
||||||
|
candidate.resolve().relative_to(root.resolve())
|
||||||
|
return True
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _llama_factory_version() -> str:
|
||||||
|
for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]):
|
||||||
|
try:
|
||||||
|
result = subprocess.run(command, capture_output=True, text=True, timeout=5)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
output = (result.stdout or result.stderr).strip()
|
||||||
|
if result.returncode == 0 and output:
|
||||||
|
return output.splitlines()[0][:120]
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def torch_cuda_status() -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
import torch # type: ignore[import-not-found]
|
||||||
|
except Exception as exc: # noqa: BLE001 - keep health endpoint resilient
|
||||||
|
return {
|
||||||
|
"available": False,
|
||||||
|
"device_count": 0,
|
||||||
|
"torch_version": "",
|
||||||
|
"torch_cuda_version": "",
|
||||||
|
"error": f"torch import failed: {exc}",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
available = bool(torch.cuda.is_available())
|
||||||
|
device_count = int(torch.cuda.device_count())
|
||||||
|
devices = []
|
||||||
|
for index in range(device_count):
|
||||||
|
props = torch.cuda.get_device_properties(index)
|
||||||
|
devices.append(
|
||||||
|
{
|
||||||
|
"index": index,
|
||||||
|
"name": props.name,
|
||||||
|
"memory_total_gb": round(props.total_memory / 1024 / 1024 / 1024, 2),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"available": available,
|
||||||
|
"device_count": device_count,
|
||||||
|
"torch_version": str(torch.__version__),
|
||||||
|
"torch_cuda_version": str(torch.version.cuda or ""),
|
||||||
|
"devices": devices,
|
||||||
|
"error": "" if available else "torch cuda is not available",
|
||||||
|
}
|
||||||
|
except Exception as exc: # noqa: BLE001 - expose CUDA initialization failures
|
||||||
|
return {
|
||||||
|
"available": False,
|
||||||
|
"device_count": 0,
|
||||||
|
"torch_version": str(getattr(torch, "__version__", "")),
|
||||||
|
"torch_cuda_version": str(getattr(torch.version, "cuda", "") or ""),
|
||||||
|
"devices": [],
|
||||||
|
"error": str(exc),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _slice_log_content(
|
||||||
|
content: str,
|
||||||
|
tail_lines: int | None = None,
|
||||||
|
offset: int | None = None,
|
||||||
|
limit: int | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
lines = content.splitlines()
|
||||||
|
total = len(lines)
|
||||||
|
if offset is not None or limit is not None:
|
||||||
|
start = max(0, offset or 0)
|
||||||
|
end = start + limit if limit else total
|
||||||
|
selected = lines[start:end]
|
||||||
|
else:
|
||||||
|
tail = tail_lines or 200
|
||||||
|
start = max(0, total - tail)
|
||||||
|
selected = lines[start:]
|
||||||
|
next_offset = start + len(selected)
|
||||||
|
return {
|
||||||
|
"content": "\n".join(selected),
|
||||||
|
"total_lines": total,
|
||||||
|
"offset": start,
|
||||||
|
"limit": len(selected),
|
||||||
|
"has_more": next_offset < total,
|
||||||
|
"next_offset": next_offset if next_offset < total else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _safe_float(value: Any, default: float = 0) -> float:
|
||||||
|
try:
|
||||||
|
return float(str(value).replace("[N/A]", "").strip() or default)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return default
|
||||||
|
|
||||||
def job_status(job: dict[str, Any]) -> dict[str, Any]:
|
def job_status(job: dict[str, Any]) -> dict[str, Any]:
|
||||||
if execution_mode() != "simulator":
|
if execution_mode() != "simulator":
|
||||||
return job
|
return job
|
||||||
@@ -74,9 +198,80 @@ def create_app() -> FastAPI:
|
|||||||
)
|
)
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
def real_gpu_resources() -> list[dict[str, Any]]:
|
||||||
|
query = (
|
||||||
|
"index,uuid,name,memory.total,memory.used,utilization.gpu,"
|
||||||
|
"temperature.gpu,power.draw,power.limit"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=5,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return fallback_gpu_resources()
|
||||||
|
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
for line in result.stdout.splitlines():
|
||||||
|
parts = [part.strip() for part in line.split(",")]
|
||||||
|
if len(parts) < 9:
|
||||||
|
continue
|
||||||
|
idx, uuid, name, mem_total, mem_used, util, temp, power, power_limit = parts[:9]
|
||||||
|
total_gb = round(_safe_float(mem_total) / 1024, 2)
|
||||||
|
used_gb = round(_safe_float(mem_used) / 1024, 2)
|
||||||
|
memory_percent = round(used_gb / total_gb * 100, 1) if total_gb else 0
|
||||||
|
gpu_percent = int(_safe_float(util))
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"id": int(idx),
|
||||||
|
"gpu_index": int(idx),
|
||||||
|
"uuid": uuid,
|
||||||
|
"name": name,
|
||||||
|
"status": "busy" if gpu_percent >= 5 or used_gb > 1 else "idle",
|
||||||
|
"gpu_percent": gpu_percent,
|
||||||
|
"memory_used_gb": used_gb,
|
||||||
|
"memory_total_gb": total_gb,
|
||||||
|
"memory_percent": memory_percent,
|
||||||
|
"temperature": int(_safe_float(temp)),
|
||||||
|
"power_w": round(_safe_float(power), 1),
|
||||||
|
"power_limit_w": round(_safe_float(power_limit), 1),
|
||||||
|
"processes": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return items
|
||||||
|
|
||||||
|
def fallback_gpu_resources() -> list[dict[str, Any]]:
|
||||||
|
count = _int_env("COMPUTE_GPU_COUNT", 0)
|
||||||
|
if count <= 0:
|
||||||
|
return []
|
||||||
|
name = os.getenv("COMPUTE_GPU_NAME", "Configured GPU")
|
||||||
|
memory_total = _float_env("COMPUTE_GPU_MEMORY_GB", 80.0)
|
||||||
|
power_limit = _float_env("COMPUTE_GPU_POWER_LIMIT_W", 300.0)
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": idx,
|
||||||
|
"gpu_index": idx,
|
||||||
|
"uuid": f"GPU-{host_id().upper()}-{idx}",
|
||||||
|
"name": name,
|
||||||
|
"status": "idle",
|
||||||
|
"gpu_percent": 0,
|
||||||
|
"memory_used_gb": 0,
|
||||||
|
"memory_total_gb": memory_total,
|
||||||
|
"memory_percent": 0,
|
||||||
|
"temperature": _int_env("COMPUTE_GPU_BASE_TEMPERATURE", 35),
|
||||||
|
"power_w": 0,
|
||||||
|
"power_limit_w": power_limit,
|
||||||
|
"processes": [],
|
||||||
|
}
|
||||||
|
for idx in range(count)
|
||||||
|
]
|
||||||
|
|
||||||
def gpu_resources() -> list[dict[str, Any]]:
|
def gpu_resources() -> list[dict[str, Any]]:
|
||||||
if execution_mode() != "simulator":
|
if execution_mode() != "simulator":
|
||||||
return []
|
return real_gpu_resources()
|
||||||
active_jobs = [job_status(job) for job in jobs.values() if job["status"] in {"queued", "running"}]
|
active_jobs = [job_status(job) for job in jobs.values() if job["status"] in {"queued", "running"}]
|
||||||
gpus: list[dict[str, Any]] = []
|
gpus: list[dict[str, Any]] = []
|
||||||
for idx in range(4):
|
for idx in range(4):
|
||||||
@@ -109,6 +304,168 @@ def create_app() -> FastAPI:
|
|||||||
)
|
)
|
||||||
return gpus
|
return gpus
|
||||||
|
|
||||||
|
def _validate_training_accelerator(payload: dict[str, Any]) -> tuple[list[str], list[str], dict[str, Any]]:
|
||||||
|
errors: list[str] = []
|
||||||
|
warnings: list[str] = []
|
||||||
|
if str(payload.get("engine") or payload.get("training_engine") or "llama_factory") == "smoke":
|
||||||
|
return errors, warnings, {}
|
||||||
|
requested_gpus = [int(item) for item in payload.get("gpus") or []]
|
||||||
|
if not requested_gpus:
|
||||||
|
warnings.append("no gpu selected; training will run on CPU")
|
||||||
|
return errors, warnings, {}
|
||||||
|
cuda = torch_cuda_status()
|
||||||
|
if not cuda.get("available"):
|
||||||
|
errors.append(f"torch cuda unavailable on compute node: {cuda.get('error') or 'unknown error'}")
|
||||||
|
device_count = int(cuda.get("device_count") or 0)
|
||||||
|
if device_count and max(requested_gpus) >= device_count:
|
||||||
|
errors.append(f"requested gpu index out of torch device range: requested={requested_gpus}, device_count={device_count}")
|
||||||
|
min_memory_gb = _float_env("MIN_TRAINING_GPU_MEMORY_GB", 4.0)
|
||||||
|
gpus = {int(item["gpu_index"]): item for item in gpu_resources() if "gpu_index" in item}
|
||||||
|
for gpu_index in requested_gpus:
|
||||||
|
gpu = gpus.get(gpu_index)
|
||||||
|
if not gpu:
|
||||||
|
errors.append(f"requested gpu not found by nvidia-smi: {gpu_index}")
|
||||||
|
continue
|
||||||
|
memory_total = float(gpu.get("memory_total_gb") or 0)
|
||||||
|
if memory_total and memory_total < min_memory_gb:
|
||||||
|
errors.append(
|
||||||
|
f"gpu {gpu_index} memory too small: {memory_total}GB < required {min_memory_gb}GB"
|
||||||
|
)
|
||||||
|
return errors, warnings, cuda
|
||||||
|
|
||||||
|
def _check_path_item(item: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
path = Path(str(item.get("path") or ""))
|
||||||
|
exists = path.exists()
|
||||||
|
expected_type = str(item.get("type") or "any")
|
||||||
|
ok = exists
|
||||||
|
if exists and expected_type == "dir":
|
||||||
|
ok = path.is_dir()
|
||||||
|
if exists and expected_type == "file":
|
||||||
|
ok = path.is_file()
|
||||||
|
return {
|
||||||
|
"name": item.get("name") or "",
|
||||||
|
"path": str(path),
|
||||||
|
"type": expected_type,
|
||||||
|
"required": bool(item.get("required", True)),
|
||||||
|
"exists": exists,
|
||||||
|
"is_dir": path.is_dir() if exists else False,
|
||||||
|
"is_file": path.is_file() if exists else False,
|
||||||
|
"byte_size": sum(child.stat().st_size for child in path.rglob("*") if child.is_file()) if exists and path.is_dir() else path.stat().st_size if exists and path.is_file() else 0,
|
||||||
|
"ok": ok or not item.get("required", True),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _job_preview(payload: dict[str, Any], check_paths: bool) -> dict[str, Any]:
|
||||||
|
warnings: list[str] = []
|
||||||
|
runtime_files: list[dict[str, str]] = []
|
||||||
|
command_payload = {**payload, "require_dataset_files": check_paths}
|
||||||
|
if check_paths:
|
||||||
|
try:
|
||||||
|
runtime_files = prepare_runtime_files(command_payload)
|
||||||
|
except OSError as exc:
|
||||||
|
return {
|
||||||
|
"valid": False,
|
||||||
|
"errors": [f"prepare runtime files failed: {exc}"],
|
||||||
|
"warnings": warnings,
|
||||||
|
"engine": str(payload.get("engine") or payload.get("training_engine") or "llama_factory"),
|
||||||
|
"command": [],
|
||||||
|
"command_text": "",
|
||||||
|
"work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"),
|
||||||
|
"env": {},
|
||||||
|
"runtime_files": [],
|
||||||
|
"path_checks": [],
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
command = build_command(command_payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
||||||
|
except ValueError as exc:
|
||||||
|
return {
|
||||||
|
"valid": False,
|
||||||
|
"errors": [part.strip() for part in str(exc).split(";") if part.strip()],
|
||||||
|
"warnings": warnings,
|
||||||
|
"engine": str(payload.get("engine") or payload.get("training_engine") or "llama_factory"),
|
||||||
|
"command": [],
|
||||||
|
"command_text": "",
|
||||||
|
"work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"),
|
||||||
|
"env": {},
|
||||||
|
"runtime_files": runtime_files,
|
||||||
|
"path_checks": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
errors: list[str] = []
|
||||||
|
engine = str(payload.get("engine") or payload.get("training_engine") or "llama_factory")
|
||||||
|
path_checks: list[dict[str, Any]] = []
|
||||||
|
accelerator: dict[str, Any] = {}
|
||||||
|
if check_paths and engine != "smoke":
|
||||||
|
path_checks = [
|
||||||
|
_check_path_item(
|
||||||
|
{
|
||||||
|
"name": "model_name_or_path",
|
||||||
|
"path": payload.get("model_name_or_path") or payload.get("base_model") or payload.get("base_model_path") or "",
|
||||||
|
"type": "any",
|
||||||
|
"required": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if engine in {"merge", "export", "llama_factory_export"} and payload.get("adapter_name_or_path"):
|
||||||
|
path_checks.append(
|
||||||
|
_check_path_item(
|
||||||
|
{
|
||||||
|
"name": "adapter_name_or_path",
|
||||||
|
"path": payload.get("adapter_name_or_path"),
|
||||||
|
"type": "any",
|
||||||
|
"required": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if payload.get("dataset_dir"):
|
||||||
|
path_checks.append(
|
||||||
|
_check_path_item(
|
||||||
|
{
|
||||||
|
"name": "dataset_dir",
|
||||||
|
"path": payload.get("dataset_dir"),
|
||||||
|
"type": "dir",
|
||||||
|
"required": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
output_dir = Path(str(payload.get("output_dir") or "/data/yg-ft/outputs/training-job"))
|
||||||
|
path_checks.append(
|
||||||
|
_check_path_item(
|
||||||
|
{
|
||||||
|
"name": "output_parent",
|
||||||
|
"path": str(output_dir.parent),
|
||||||
|
"type": "dir",
|
||||||
|
"required": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
errors.extend(
|
||||||
|
[f"{item['name']} path not available: {item['path']}" for item in path_checks if not item["ok"] and item["required"]]
|
||||||
|
)
|
||||||
|
if shutil.which(command.command[0]) is None:
|
||||||
|
errors.append(f"training command not found: {command.command[0]}")
|
||||||
|
if not Path(command.work_dir).exists():
|
||||||
|
errors.append(f"llama_factory_home not found: {command.work_dir}")
|
||||||
|
if engine not in {"merge", "export", "llama_factory_export"}:
|
||||||
|
accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload)
|
||||||
|
errors.extend(accelerator_errors)
|
||||||
|
warnings.extend(accelerator_warnings)
|
||||||
|
elif engine == "smoke":
|
||||||
|
warnings.append("smoke engine skips model and dataset path checks")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"valid": not errors,
|
||||||
|
"errors": errors,
|
||||||
|
"warnings": warnings,
|
||||||
|
"engine": engine,
|
||||||
|
"command": command.command,
|
||||||
|
"command_text": " ".join(command.command),
|
||||||
|
"work_dir": command.work_dir,
|
||||||
|
"env": command.env,
|
||||||
|
"runtime_files": runtime_files,
|
||||||
|
"accelerator": accelerator,
|
||||||
|
"path_checks": path_checks,
|
||||||
|
}
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/health")
|
@app.get(f"{route_prefix}/health")
|
||||||
async def health_check() -> dict[str, str]:
|
async def health_check() -> dict[str, str]:
|
||||||
return {
|
return {
|
||||||
@@ -116,41 +473,130 @@ def create_app() -> FastAPI:
|
|||||||
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
|
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
async def health_check_root() -> dict[str, str]:
|
||||||
|
return await health_check()
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/v1/compute/health")
|
@app.get(f"{route_prefix}/v1/compute/health")
|
||||||
async def compute_health_check() -> dict[str, str | bool]:
|
async def compute_health_check() -> dict[str, Any]:
|
||||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||||
|
dataset_root = Path(os.getenv("YG_FT_DATASET_ROOT", str(data_root / "datasets")))
|
||||||
|
output_root = Path(os.getenv("YG_FT_OUTPUT_ROOT", str(data_root / "outputs")))
|
||||||
llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
||||||
|
gpu_items = gpu_resources()
|
||||||
|
torch_cuda = torch_cuda_status()
|
||||||
return {
|
return {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
|
"api_version": "v1",
|
||||||
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
|
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
|
||||||
"app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true",
|
"app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true",
|
||||||
"data_root": str(data_root),
|
"data_root": str(data_root),
|
||||||
"data_root_exists": data_root.exists(),
|
"data_root_exists": data_root.exists(),
|
||||||
|
"model_root": os.getenv("YG_FT_MODEL_ROOT", str(data_root / "models")),
|
||||||
|
"dataset_root": str(dataset_root),
|
||||||
|
"dataset_root_exists": dataset_root.exists(),
|
||||||
|
"output_root": str(output_root),
|
||||||
|
"output_root_exists": output_root.exists(),
|
||||||
|
"log_root": os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"),
|
||||||
"llama_factory_home": str(llama_factory_home),
|
"llama_factory_home": str(llama_factory_home),
|
||||||
"llama_factory_home_exists": llama_factory_home.exists(),
|
"llama_factory_home_exists": llama_factory_home.exists(),
|
||||||
|
"llama_factory_version": os.getenv("LLAMA_FACTORY_VERSION", ""),
|
||||||
"execution_mode": execution_mode(),
|
"execution_mode": execution_mode(),
|
||||||
|
"gpu_count": _int_env("COMPUTE_GPU_COUNT", 0),
|
||||||
|
"nvidia_gpu_count": len(gpu_items),
|
||||||
|
"torch_cuda": torch_cuda,
|
||||||
|
"gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus",
|
||||||
|
"capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling", "inference"],
|
||||||
}
|
}
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/v1/compute/jobs")
|
@app.get(f"{route_prefix}/v1/compute/jobs")
|
||||||
async def list_jobs_alias() -> dict[str, list[dict[str, Any]]]:
|
async def list_jobs_alias() -> dict[str, list[dict[str, Any]]]:
|
||||||
return {"items": [job_status(job) for job in jobs.values()]}
|
items = process_manager.list_jobs() if execution_mode() != "simulator" else [job_status(job) for job in jobs.values()]
|
||||||
|
return {"items": items}
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/compute/resources/gpus")
|
@app.get(f"{route_prefix}/compute/resources/gpus")
|
||||||
async def list_gpus() -> dict[str, Any]:
|
async def list_gpus() -> dict[str, Any]:
|
||||||
return {"items": gpu_resources(), "compute_host_id": host_id()}
|
return {"items": gpu_resources(), "compute_host_id": host_id()}
|
||||||
|
|
||||||
|
@app.get(f"{route_prefix}/v1/compute/resources/gpus")
|
||||||
|
async def list_gpus_v1() -> dict[str, Any]:
|
||||||
|
return {"items": gpu_resources(), "compute_host_id": host_id()}
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/compute/jobs/preview")
|
||||||
|
async def preview_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return _job_preview(payload, check_paths=False)
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/compute/jobs/validate")
|
||||||
|
async def validate_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return _job_preview(payload, check_paths=True)
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/v1/compute/jobs/preview")
|
||||||
|
async def preview_job_v1(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return await preview_job(payload)
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/v1/compute/jobs/validate")
|
||||||
|
async def validate_job_v1(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return await validate_job(payload)
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/compute/files/check-paths")
|
||||||
|
async def check_paths(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
items = [_check_path_item(item) for item in payload.get("paths", []) if isinstance(item, dict)]
|
||||||
|
return {"valid": all(item["ok"] for item in items), "items": items}
|
||||||
|
|
||||||
|
@app.get(f"{route_prefix}/compute/files/list")
|
||||||
|
async def list_files(
|
||||||
|
root: str = Query(default="data"),
|
||||||
|
relative_path: str = Query(default=""),
|
||||||
|
directories_only: bool = Query(default=False),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
roots = {
|
||||||
|
"data": Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")),
|
||||||
|
"models": Path(os.getenv("YG_FT_MODEL_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/models")),
|
||||||
|
"datasets": Path(os.getenv("YG_FT_DATASET_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/datasets")),
|
||||||
|
"outputs": Path(os.getenv("YG_FT_OUTPUT_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/outputs")),
|
||||||
|
}
|
||||||
|
base = roots.get(root)
|
||||||
|
if base is None:
|
||||||
|
raise HTTPException(status_code=400, detail="invalid root")
|
||||||
|
target = (base / relative_path.lstrip("/\\")).resolve()
|
||||||
|
if not _path_inside(base, target):
|
||||||
|
raise HTTPException(status_code=400, detail="path must stay inside selected root")
|
||||||
|
if not target.exists():
|
||||||
|
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": []}
|
||||||
|
items = []
|
||||||
|
for child in sorted(target.iterdir(), key=lambda path: (not path.is_dir(), path.name.lower())):
|
||||||
|
if directories_only and not child.is_dir():
|
||||||
|
continue
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"name": child.name,
|
||||||
|
"path": str(child),
|
||||||
|
"relative_path": str(child.relative_to(base)).replace("\\", "/"),
|
||||||
|
"type": "directory" if child.is_dir() else "file",
|
||||||
|
"byte_size": child.stat().st_size if child.is_file() else 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items}
|
||||||
|
|
||||||
@app.post(f"{route_prefix}/compute/jobs")
|
@app.post(f"{route_prefix}/compute/jobs")
|
||||||
async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
|
async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
payload = {**payload, "require_dataset_files": True}
|
||||||
|
try:
|
||||||
|
prepare_runtime_files(payload)
|
||||||
|
except OSError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=f"prepare runtime files failed: {exc}")
|
||||||
try:
|
try:
|
||||||
command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(status_code=400, detail=str(exc))
|
raise HTTPException(status_code=400, detail=str(exc))
|
||||||
if execution_mode() != "simulator":
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=501,
|
|
||||||
detail="real compute executor is not implemented yet; set COMPUTE_EXECUTION_MODE=simulator only for isolated development",
|
|
||||||
)
|
|
||||||
job_id = str(payload.get("id") or f"job_{int(now() * 1000)}")
|
job_id = str(payload.get("id") or f"job_{int(now() * 1000)}")
|
||||||
|
if execution_mode() != "simulator":
|
||||||
|
try:
|
||||||
|
return process_manager.create_job({**payload, "id": job_id}, command.command, command.work_dir)
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise HTTPException(status_code=500, detail=f"training command not found: {exc.filename}")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=409, detail=str(exc))
|
||||||
job = {
|
job = {
|
||||||
"id": job_id,
|
"id": job_id,
|
||||||
"name": payload.get("name", job_id),
|
"name": payload.get("name", job_id),
|
||||||
@@ -169,17 +615,29 @@ def create_app() -> FastAPI:
|
|||||||
|
|
||||||
@app.get(f"{route_prefix}/compute/jobs")
|
@app.get(f"{route_prefix}/compute/jobs")
|
||||||
async def list_jobs() -> dict[str, Any]:
|
async def list_jobs() -> dict[str, Any]:
|
||||||
return {"items": [job_status(job) for job in jobs.values()]}
|
items = process_manager.list_jobs() if execution_mode() != "simulator" else [job_status(job) for job in jobs.values()]
|
||||||
|
return {"items": items}
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/compute/jobs/{{job_id}}")
|
@app.get(f"{route_prefix}/compute/jobs/{{job_id}}")
|
||||||
async def get_job(job_id: str) -> dict[str, Any]:
|
async def get_job(job_id: str) -> dict[str, Any]:
|
||||||
job = jobs.get(job_id)
|
job = jobs.get(job_id)
|
||||||
|
if execution_mode() != "simulator":
|
||||||
|
job = process_manager.get_job(job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="job not found")
|
||||||
|
return job
|
||||||
|
job = jobs.get(job_id)
|
||||||
if not job:
|
if not job:
|
||||||
raise HTTPException(status_code=404, detail="job not found")
|
raise HTTPException(status_code=404, detail="job not found")
|
||||||
return job_status(job)
|
return job_status(job)
|
||||||
|
|
||||||
@app.post(f"{route_prefix}/compute/jobs/{{job_id}}/stop")
|
@app.post(f"{route_prefix}/compute/jobs/{{job_id}}/stop")
|
||||||
async def stop_job(job_id: str) -> dict[str, Any]:
|
async def stop_job(job_id: str) -> dict[str, Any]:
|
||||||
|
if execution_mode() != "simulator":
|
||||||
|
job = process_manager.stop_job(job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="job not found")
|
||||||
|
return job
|
||||||
job = jobs.get(job_id)
|
job = jobs.get(job_id)
|
||||||
if not job:
|
if not job:
|
||||||
raise HTTPException(status_code=404, detail="job not found")
|
raise HTTPException(status_code=404, detail="job not found")
|
||||||
@@ -188,22 +646,165 @@ def create_app() -> FastAPI:
|
|||||||
return job
|
return job
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/compute/jobs/{{job_id}}/logs")
|
@app.get(f"{route_prefix}/compute/jobs/{{job_id}}/logs")
|
||||||
async def job_logs(job_id: str) -> dict[str, Any]:
|
async def job_logs(
|
||||||
|
job_id: str,
|
||||||
|
tail_lines: int | None = Query(default=200, ge=1, le=5000),
|
||||||
|
offset: int | None = Query(default=None, ge=0),
|
||||||
|
limit: int | None = Query(default=None, ge=1, le=5000),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if execution_mode() != "simulator":
|
||||||
|
job = process_manager.get_job(job_id)
|
||||||
|
if not job:
|
||||||
|
raise HTTPException(status_code=404, detail="job not found")
|
||||||
|
content = process_manager.logs(job_id)
|
||||||
|
else:
|
||||||
job = jobs.get(job_id)
|
job = jobs.get(job_id)
|
||||||
if not job:
|
if not job:
|
||||||
raise HTTPException(status_code=404, detail="job not found")
|
raise HTTPException(status_code=404, detail="job not found")
|
||||||
job = job_status(job)
|
job = job_status(job)
|
||||||
metrics = [parse_log_line(line) for line in job["logs"].splitlines()]
|
content = job["logs"]
|
||||||
return {"job_id": job_id, "content": job["logs"], "metrics": [m for m in metrics if m]}
|
window = _slice_log_content(content, tail_lines, offset, limit)
|
||||||
|
metrics = [parse_log_line(line) for line in window["content"].splitlines()]
|
||||||
|
return {"job_id": job_id, **window, "metrics": [m for m in metrics if m]}
|
||||||
|
|
||||||
|
# ── Inference Endpoints ───────────────────────────────────────────
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/inference/load")
|
||||||
|
async def inference_load(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Load a model for inference using LLaMA-Factory ChatModel."""
|
||||||
|
session = get_inference_session()
|
||||||
|
result = session.load(
|
||||||
|
model_name_or_path=payload.get("model_name_or_path", ""),
|
||||||
|
adapter_name_or_path=payload.get("adapter_name_or_path", ""),
|
||||||
|
template=payload.get("template", "qwen"),
|
||||||
|
infer_backend=payload.get("infer_backend", "huggingface"),
|
||||||
|
infer_dtype=payload.get("infer_dtype", "auto"),
|
||||||
|
)
|
||||||
|
if not result.get("loaded"):
|
||||||
|
raise HTTPException(status_code=500, detail=result.get("error", "model load failed"))
|
||||||
|
return result
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/inference/unload")
|
||||||
|
async def inference_unload() -> dict[str, Any]:
|
||||||
|
"""Unload the currently loaded model and free GPU memory."""
|
||||||
|
return get_inference_session().unload()
|
||||||
|
|
||||||
|
@app.get(f"{route_prefix}/inference/status")
|
||||||
|
async def inference_status() -> dict[str, Any]:
|
||||||
|
"""Get the current inference session status."""
|
||||||
|
return get_inference_session().info()
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/inference/chat")
|
||||||
|
async def inference_chat(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Chat with the loaded model (non-streaming)."""
|
||||||
|
messages = payload.get("messages") or []
|
||||||
|
if not messages:
|
||||||
|
raise HTTPException(status_code=400, detail="messages is required")
|
||||||
|
result = get_inference_session().chat(
|
||||||
|
messages=messages,
|
||||||
|
temperature=float(payload.get("temperature", 0.95)),
|
||||||
|
top_p=float(payload.get("top_p", 0.7)),
|
||||||
|
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||||
|
do_sample=bool(payload.get("do_sample", True)),
|
||||||
|
)
|
||||||
|
if result.get("error"):
|
||||||
|
raise HTTPException(status_code=500, detail=result["error"])
|
||||||
|
return {"response": result["response"]}
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/inference/chat/stream")
|
||||||
|
async def inference_chat_stream(payload: dict[str, Any]) -> StreamingResponse:
|
||||||
|
"""Chat with streaming response (Server-Sent Events)."""
|
||||||
|
messages = payload.get("messages") or []
|
||||||
|
if not messages:
|
||||||
|
raise HTTPException(status_code=400, detail="messages is required")
|
||||||
|
|
||||||
|
def generate():
|
||||||
|
session = get_inference_session()
|
||||||
|
for chunk in session.chat_stream(
|
||||||
|
messages=messages,
|
||||||
|
temperature=float(payload.get("temperature", 0.95)),
|
||||||
|
top_p=float(payload.get("top_p", 0.7)),
|
||||||
|
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||||
|
do_sample=bool(payload.get("do_sample", True)),
|
||||||
|
):
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||||
|
|
||||||
@app.post(f"{route_prefix}/compute/files/upload")
|
@app.post(f"{route_prefix}/compute/files/upload")
|
||||||
async def upload_file(payload: dict[str, Any]) -> dict[str, Any]:
|
async def upload_file(
|
||||||
file_id = str(payload.get("id") or f"file_{int(now() * 1000)}")
|
file: UploadFile | None = File(default=None),
|
||||||
return {"id": file_id, "status": "available", "local_path": f"/data/yg-ft/uploads/{file_id}"}
|
target_relative_path: str | None = Form(default=None),
|
||||||
|
resource_type: str | None = Form(default=None),
|
||||||
|
resource_id: str | None = Form(default=None),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
file_id = f"file_{int(now() * 1000)}"
|
||||||
|
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||||
|
data_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
filename = Path(file.filename if file else file_id).name
|
||||||
|
if target_relative_path:
|
||||||
|
target = (data_root / target_relative_path.lstrip("/\\")).resolve()
|
||||||
|
if not _path_inside(data_root, target):
|
||||||
|
raise HTTPException(status_code=400, detail="target path must stay inside YG_FT_DATA_ROOT")
|
||||||
|
else:
|
||||||
|
target = data_root / "uploads" / f"{file_id}_{filename}"
|
||||||
|
if file:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with target.open("wb") as output:
|
||||||
|
while chunk := await file.read(1024 * 1024):
|
||||||
|
output.write(chunk)
|
||||||
|
else:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_text("", encoding="utf-8")
|
||||||
|
return {
|
||||||
|
"id": file_id,
|
||||||
|
"resource_type": resource_type,
|
||||||
|
"resource_id": resource_id,
|
||||||
|
"status": "available",
|
||||||
|
"local_path": str(target),
|
||||||
|
"byte_size": target.stat().st_size,
|
||||||
|
"checksum_sha256": hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "",
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.post(f"{route_prefix}/compute/files/import-local")
|
||||||
|
async def import_local_file(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
source = Path(str(payload.get("source_path") or ""))
|
||||||
|
if not source.exists():
|
||||||
|
raise HTTPException(status_code=404, detail="source path not found")
|
||||||
|
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||||
|
data_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
relative = str(payload.get("target_relative_path") or f"imports/{source.name}").lstrip("/\\")
|
||||||
|
target = (data_root / relative).resolve()
|
||||||
|
if not _path_inside(data_root, target):
|
||||||
|
raise HTTPException(status_code=400, detail="target path must stay inside YG_FT_DATA_ROOT")
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if source.is_dir():
|
||||||
|
if target.exists():
|
||||||
|
shutil.rmtree(target)
|
||||||
|
shutil.copytree(source, target)
|
||||||
|
byte_size = sum(path.stat().st_size for path in target.rglob("*") if path.is_file())
|
||||||
|
checksum = ""
|
||||||
|
else:
|
||||||
|
shutil.copy2(source, target)
|
||||||
|
byte_size = target.stat().st_size
|
||||||
|
checksum = hashlib.sha256(target.read_bytes()).hexdigest()
|
||||||
|
return {
|
||||||
|
"id": str(payload.get("id") or f"file_{int(now() * 1000)}"),
|
||||||
|
"resource_type": payload.get("resource_type"),
|
||||||
|
"resource_id": payload.get("resource_id"),
|
||||||
|
"status": "available",
|
||||||
|
"local_path": str(target),
|
||||||
|
"byte_size": byte_size,
|
||||||
|
"checksum_sha256": checksum,
|
||||||
|
}
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
|
@app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
|
||||||
async def download_file(file_id: str) -> dict[str, Any]:
|
async def download_file(file_id: str) -> FileResponse:
|
||||||
return {"id": file_id, "status": "ready", "download_url": f"{route_prefix}/compute/files/{file_id}/download"}
|
upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads"
|
||||||
|
matches = list(upload_root.glob(f"{file_id}_*"))
|
||||||
|
if not matches:
|
||||||
|
raise HTTPException(status_code=404, detail="file not found")
|
||||||
|
return FileResponse(matches[0])
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import re
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -13,40 +14,234 @@ class LlamaFactoryCommand:
|
|||||||
env: dict[str, str]
|
env: dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
|
def _load_dataset_preview(path: Path) -> list[dict[str, Any]]:
|
||||||
|
if not path.exists():
|
||||||
|
return []
|
||||||
|
text = path.read_text(encoding="utf-8", errors="replace").strip()
|
||||||
|
if not text:
|
||||||
|
return []
|
||||||
|
if path.suffix.lower() == ".jsonl":
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
for line in text.splitlines()[:20]:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
value = json.loads(line)
|
||||||
|
if isinstance(value, dict):
|
||||||
|
items.append(value)
|
||||||
|
return items
|
||||||
|
value = json.loads(text)
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [item for item in value[:20] if isinstance(item, dict)]
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return [value]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_dataset_columns(config: dict[str, Any]) -> list[str]:
|
||||||
|
dataset_dir = config.get("dataset_dir")
|
||||||
|
dataset_info = config.get("dataset_info")
|
||||||
|
if not dataset_dir or not isinstance(dataset_info, dict):
|
||||||
|
return []
|
||||||
|
root = Path(str(dataset_dir))
|
||||||
|
errors: list[str] = []
|
||||||
|
for dataset_key, item in dataset_info.items():
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
file_name = item.get("file_name")
|
||||||
|
file_names = file_name if isinstance(file_name, list) else [file_name]
|
||||||
|
columns = item.get("columns") if isinstance(item.get("columns"), dict) else {}
|
||||||
|
required_columns = [str(value) for value in columns.values() if value]
|
||||||
|
for name in file_names:
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
path = root / str(name).lstrip("/\\")
|
||||||
|
if not path.exists():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
preview_rows = _load_dataset_preview(path)
|
||||||
|
except Exception as exc: # noqa: BLE001 - expose malformed data as validation error
|
||||||
|
errors.append(f"dataset file parse failed: {path}: {exc}")
|
||||||
|
continue
|
||||||
|
if not preview_rows:
|
||||||
|
errors.append(f"dataset file has no valid object records: {path}")
|
||||||
|
continue
|
||||||
|
available = set().union(*(row.keys() for row in preview_rows))
|
||||||
|
missing = [column for column in required_columns if column not in available]
|
||||||
|
if missing:
|
||||||
|
errors.append(
|
||||||
|
f"dataset columns missing in {path.name} for {dataset_key}: {', '.join(sorted(set(missing)))}"
|
||||||
|
)
|
||||||
|
return errors
|
||||||
|
|
||||||
|
|
||||||
def validate_config(config: dict[str, Any]) -> list[str]:
|
def validate_config(config: dict[str, Any]) -> list[str]:
|
||||||
errors: list[str] = []
|
errors: list[str] = []
|
||||||
if not config.get("base_model") and not config.get("model_name_or_path"):
|
if not config.get("base_model") and not config.get("model_name_or_path"):
|
||||||
errors.append("base_model or model_name_or_path is required")
|
errors.append("base_model or model_name_or_path is required")
|
||||||
if not config.get("dataset") and not config.get("dataset_dir"):
|
if not config.get("dataset") and not config.get("dataset_dir"):
|
||||||
errors.append("dataset or dataset_dir is required")
|
errors.append("dataset or dataset_dir is required")
|
||||||
|
try:
|
||||||
learning_rate = float(config.get("learning_rate", 0.0002))
|
learning_rate = float(config.get("learning_rate", 0.0002))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
learning_rate = 0
|
||||||
if learning_rate <= 0:
|
if learning_rate <= 0:
|
||||||
errors.append("learning_rate must be greater than zero")
|
errors.append("learning_rate must be greater than zero")
|
||||||
|
try:
|
||||||
epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1)))
|
epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
epochs = 0
|
||||||
if epochs <= 0:
|
if epochs <= 0:
|
||||||
errors.append("n_epochs must be greater than zero")
|
errors.append("n_epochs must be greater than zero")
|
||||||
|
dataset_dir = config.get("dataset_dir")
|
||||||
|
dataset_info = config.get("dataset_info")
|
||||||
|
if config.get("require_dataset_files") and dataset_dir and isinstance(dataset_info, dict):
|
||||||
|
root = Path(str(dataset_dir))
|
||||||
|
for dataset_key, item in dataset_info.items():
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
errors.append(f"dataset_info entry must be object: {dataset_key}")
|
||||||
|
continue
|
||||||
|
file_name = item.get("file_name")
|
||||||
|
file_names = file_name if isinstance(file_name, list) else [file_name]
|
||||||
|
for name in file_names:
|
||||||
|
if not name:
|
||||||
|
errors.append(f"dataset_info file_name is required: {dataset_key}")
|
||||||
|
continue
|
||||||
|
path = root / str(name).lstrip("/\\")
|
||||||
|
if not path.exists():
|
||||||
|
errors.append(f"dataset file not found: {path}")
|
||||||
|
errors.extend(_validate_dataset_columns(config))
|
||||||
return errors
|
return errors
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None:
|
||||||
|
for key in keys:
|
||||||
|
value = config.get(key)
|
||||||
|
if value is not None and value != "":
|
||||||
|
command.extend([option, str(value)])
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_bool_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None:
|
||||||
|
for key in keys:
|
||||||
|
value = config.get(key)
|
||||||
|
if value is True or str(value).lower() == "true":
|
||||||
|
command.extend([option, "true"])
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_stage(config: dict[str, Any]) -> str:
|
||||||
|
raw = str(config.get("stage") or config.get("train_type") or "sft").strip().lower()
|
||||||
|
return {
|
||||||
|
"sft": "sft",
|
||||||
|
"dpo": "dpo",
|
||||||
|
"cpt": "pt",
|
||||||
|
"pt": "pt",
|
||||||
|
"pretrain": "pt",
|
||||||
|
"rm": "rm",
|
||||||
|
"ppo": "ppo",
|
||||||
|
"kto": "kto",
|
||||||
|
}.get(raw, raw or "sft")
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_runtime_files(config: dict[str, Any]) -> list[dict[str, str]]:
|
||||||
|
dataset_dir = config.get("dataset_dir")
|
||||||
|
dataset_info = config.get("dataset_info")
|
||||||
|
if not dataset_dir or not isinstance(dataset_info, dict):
|
||||||
|
return []
|
||||||
|
root = Path(str(dataset_dir))
|
||||||
|
root.mkdir(parents=True, exist_ok=True)
|
||||||
|
path = root / "dataset_info.json"
|
||||||
|
existing: dict[str, Any] = {}
|
||||||
|
if path.exists():
|
||||||
|
try:
|
||||||
|
loaded = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
existing = loaded if isinstance(loaded, dict) else {}
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
existing = {}
|
||||||
|
existing.update(dataset_info)
|
||||||
|
path.write_text(json.dumps(existing, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
return [{"name": "dataset_info", "path": str(path)}]
|
||||||
|
|
||||||
|
|
||||||
def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand:
|
def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand:
|
||||||
|
engine = str(config.get("engine") or config.get("training_engine") or "llama_factory")
|
||||||
|
if engine in {"merge", "export", "llama_factory_export"}:
|
||||||
|
model_path = config.get("base_model") or config.get("model_name_or_path") or config.get("base_model_path")
|
||||||
|
adapter_path = config.get("adapter_name_or_path") or config.get("adapter_path") or config.get("lora_path")
|
||||||
|
output_dir = config.get("output_dir") or config.get("export_dir")
|
||||||
|
errors: list[str] = []
|
||||||
|
if not model_path:
|
||||||
|
errors.append("base_model or model_name_or_path is required")
|
||||||
|
if not adapter_path and engine == "merge":
|
||||||
|
errors.append("adapter_name_or_path or adapter_path is required")
|
||||||
|
if not output_dir:
|
||||||
|
errors.append("output_dir or export_dir is required")
|
||||||
|
if errors:
|
||||||
|
raise ValueError("; ".join(errors))
|
||||||
|
command = [
|
||||||
|
"llamafactory-cli",
|
||||||
|
"export",
|
||||||
|
"--model_name_or_path",
|
||||||
|
str(model_path),
|
||||||
|
"--template",
|
||||||
|
str(config.get("template", "qwen")),
|
||||||
|
"--finetuning_type",
|
||||||
|
str(config.get("train_method", config.get("finetuning_type", "lora"))),
|
||||||
|
"--export_dir",
|
||||||
|
str(output_dir),
|
||||||
|
"--export_size",
|
||||||
|
str(config.get("export_size", 2)),
|
||||||
|
"--export_device",
|
||||||
|
str(config.get("export_device", "cpu")),
|
||||||
|
"--export_legacy_format",
|
||||||
|
str(config.get("export_legacy_format", False)).lower(),
|
||||||
|
]
|
||||||
|
if adapter_path:
|
||||||
|
command.extend(["--adapter_name_or_path", str(adapter_path)])
|
||||||
|
quantization_bit = int(config.get("export_quantization_bit", config.get("quantization_bit", 0)) or 0)
|
||||||
|
if quantization_bit in {4, 8}:
|
||||||
|
command.extend(["--quantization_bit", str(quantization_bit)])
|
||||||
|
return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={})
|
||||||
|
|
||||||
errors = validate_config(config)
|
errors = validate_config(config)
|
||||||
if errors:
|
if errors:
|
||||||
raise ValueError("; ".join(errors))
|
raise ValueError("; ".join(errors))
|
||||||
|
|
||||||
|
if engine == "smoke":
|
||||||
|
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-smoke')}"
|
||||||
|
script = (
|
||||||
|
"import json, os, time; "
|
||||||
|
f"out={str(output_dir)!r}; "
|
||||||
|
"os.makedirs(out, exist_ok=True); "
|
||||||
|
"print('[INFO] smoke training started', flush=True); "
|
||||||
|
"\nfor step in range(1, 7):\n"
|
||||||
|
" loss=round(1.8/(step+1), 4)\n"
|
||||||
|
" lr=round(0.0002*(1-step/10), 8)\n"
|
||||||
|
" print({'loss': loss, 'grad_norm': round(0.4 + step*0.03, 4), 'learning_rate': lr, 'epoch': round(step/6, 4)}, flush=True)\n"
|
||||||
|
" time.sleep(0.4)\n"
|
||||||
|
"\nopen(os.path.join(out, 'adapter_config.json'), 'w', encoding='utf-8').write(json.dumps({'engine':'smoke','status':'completed'})); "
|
||||||
|
"print('***** train metrics *****', flush=True); "
|
||||||
|
"print('train_loss = 0.12', flush=True); "
|
||||||
|
"print('***** train metrics end *****', flush=True)"
|
||||||
|
)
|
||||||
|
return LlamaFactoryCommand(command=["python", "-u", "-c", script], work_dir="/app", env={})
|
||||||
|
|
||||||
model_path = config.get("base_model") or config.get("model_name_or_path")
|
model_path = config.get("base_model") or config.get("model_name_or_path")
|
||||||
dataset = config.get("dataset") or config.get("dataset_dir")
|
dataset = config.get("dataset") or config.get("dataset_name")
|
||||||
|
dataset_dir = config.get("dataset_dir")
|
||||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}"
|
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}"
|
||||||
command = [
|
command = [
|
||||||
"llamafactory-cli",
|
"llamafactory-cli",
|
||||||
"train",
|
"train",
|
||||||
"--stage",
|
"--stage",
|
||||||
str(config.get("stage", "sft")).lower(),
|
_normalize_stage(config),
|
||||||
"--do_train",
|
"--do_train",
|
||||||
"true",
|
"true",
|
||||||
"--model_name_or_path",
|
"--model_name_or_path",
|
||||||
str(model_path),
|
str(model_path),
|
||||||
"--dataset",
|
"--dataset",
|
||||||
str(dataset),
|
str(dataset or "default"),
|
||||||
"--template",
|
"--template",
|
||||||
str(config.get("template", "qwen")),
|
str(config.get("template", "qwen")),
|
||||||
"--finetuning_type",
|
"--finetuning_type",
|
||||||
@@ -61,7 +256,32 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
|||||||
str(config.get("n_epochs", 3)),
|
str(config.get("n_epochs", 3)),
|
||||||
"--save_steps",
|
"--save_steps",
|
||||||
str(config.get("save_steps", 50)),
|
str(config.get("save_steps", 50)),
|
||||||
|
"--logging_steps",
|
||||||
|
str(config.get("logging_steps", 10)),
|
||||||
|
"--overwrite_output_dir",
|
||||||
|
"true",
|
||||||
|
"--plot_loss",
|
||||||
|
"true",
|
||||||
]
|
]
|
||||||
|
if dataset_dir:
|
||||||
|
command.extend(["--dataset_dir", str(dataset_dir)])
|
||||||
|
eval_dataset = config.get("eval_dataset")
|
||||||
|
if eval_dataset:
|
||||||
|
command.extend(["--eval_dataset", str(eval_dataset), "--do_eval", "true"])
|
||||||
|
_optional_arg(config, command, "--cutoff_len", "max_length", "cutoff_len")
|
||||||
|
_optional_arg(config, command, "--lr_scheduler_type", "lr_scheduler_type")
|
||||||
|
_optional_arg(config, command, "--warmup_ratio", "warmup_ratio")
|
||||||
|
_optional_arg(config, command, "--weight_decay", "weight_decay")
|
||||||
|
_optional_arg(config, command, "--lora_rank", "lora_rank", "rank")
|
||||||
|
_optional_arg(config, command, "--lora_alpha", "lora_alpha")
|
||||||
|
_optional_arg(config, command, "--lora_dropout", "lora_dropout")
|
||||||
|
_optional_arg(config, command, "--gradient_accumulation_steps", "gradient_accumulation_steps")
|
||||||
|
if not eval_dataset:
|
||||||
|
_optional_arg(config, command, "--val_size", "val_size")
|
||||||
|
_optional_arg(config, command, "--max_samples", "max_samples")
|
||||||
|
_optional_arg(config, command, "--preprocessing_num_workers", "preprocessing_num_workers")
|
||||||
|
_optional_bool_arg(config, command, "--fp16", "fp16")
|
||||||
|
_optional_bool_arg(config, command, "--bf16", "bf16")
|
||||||
quantization_bit = int(config.get("quantization_bit", 0) or 0)
|
quantization_bit = int(config.get("quantization_bit", 0) or 0)
|
||||||
if quantization_bit in {4, 8}:
|
if quantization_bit in {4, 8}:
|
||||||
command.extend(["--quantization_bit", str(quantization_bit)])
|
command.extend(["--quantization_bit", str(quantization_bit)])
|
||||||
@@ -77,4 +297,3 @@ def parse_log_line(line: str) -> dict[str, float] | None:
|
|||||||
if match:
|
if match:
|
||||||
result[key] = float(match.group(1))
|
result[key] = float(match.group(1))
|
||||||
return result or None
|
return result or None
|
||||||
|
|
||||||
|
|||||||
125
compute/engines/llama_factory/inference.py
Normal file
125
compute/engines/llama_factory/inference.py
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class InferenceSession:
|
||||||
|
"""Manages a loaded model for inference with LLaMA-Factory ChatModel."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._model: Any = None
|
||||||
|
self._tokenizer: Any = None
|
||||||
|
self._generating_args: dict[str, Any] = {}
|
||||||
|
self._model_name: str = ""
|
||||||
|
self._adapter_path: str = ""
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._loaded_at: float = 0.0
|
||||||
|
self._status: str = "idle"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def status(self) -> str:
|
||||||
|
return self._status
|
||||||
|
|
||||||
|
@property
|
||||||
|
def model_name(self) -> str:
|
||||||
|
return self._model_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def adapter_path(self) -> str:
|
||||||
|
return self._adapter_path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def loaded_at(self) -> float:
|
||||||
|
return self._loaded_at
|
||||||
|
|
||||||
|
def info(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"loaded": self._status == "ready",
|
||||||
|
"status": self._status,
|
||||||
|
"model_name": self._model_name,
|
||||||
|
"adapter_path": self._adapter_path,
|
||||||
|
"loaded_at": self._loaded_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
def load(self, model_name_or_path, adapter_name_or_path="", template="qwen", infer_backend="huggingface", infer_dtype="auto", **kwargs):
|
||||||
|
with self._lock:
|
||||||
|
if self._status == "loading":
|
||||||
|
return {"loaded": False, "error": "model is already loading"}
|
||||||
|
if self._status == "ready":
|
||||||
|
self.unload()
|
||||||
|
self._status = "loading"
|
||||||
|
self._model_name = model_name_or_path
|
||||||
|
self._adapter_path = adapter_name_or_path
|
||||||
|
try:
|
||||||
|
from llamafactory.chat import ChatModel
|
||||||
|
from llamafactory.hparams import get_infer_args
|
||||||
|
args = {"model_name_or_path": model_name_or_path, "template": template, "infer_backend": infer_backend, "infer_dtype": infer_dtype}
|
||||||
|
if adapter_name_or_path:
|
||||||
|
args["adapter_name_or_path"] = adapter_name_or_path
|
||||||
|
args.update(kwargs)
|
||||||
|
model_args, generating_args = get_infer_args(args)
|
||||||
|
self._model = ChatModel(model_args)
|
||||||
|
self._tokenizer = self._model.tokenizer
|
||||||
|
self._generating_args = generating_args
|
||||||
|
self._loaded_at = time.time()
|
||||||
|
self._status = "ready"
|
||||||
|
return {"loaded": True, "status": "ready"}
|
||||||
|
except Exception as exc:
|
||||||
|
self._status = "error"
|
||||||
|
self._model = None
|
||||||
|
return {"loaded": False, "status": "error", "error": str(exc)}
|
||||||
|
|
||||||
|
def unload(self):
|
||||||
|
with self._lock:
|
||||||
|
if self._model is not None:
|
||||||
|
try:
|
||||||
|
del self._model
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self._model = None
|
||||||
|
self._tokenizer = None
|
||||||
|
self._status = "idle"
|
||||||
|
self._model_name = ""
|
||||||
|
self._adapter_path = ""
|
||||||
|
self._loaded_at = 0.0
|
||||||
|
return {"unloaded": True}
|
||||||
|
|
||||||
|
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs):
|
||||||
|
with self._lock:
|
||||||
|
if self._status != "ready" or self._model is None:
|
||||||
|
return {"error": "model not loaded", "response": ""}
|
||||||
|
try:
|
||||||
|
generate_kwargs = {**self._generating_args, "temperature": temperature, "top_p": top_p, "max_new_tokens": max_new_tokens, "do_sample": do_sample}
|
||||||
|
generate_kwargs.update(kwargs)
|
||||||
|
formatted = self._model.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||||
|
responses = []
|
||||||
|
for response in self._model.stream_chat(formatted, generate_kwargs):
|
||||||
|
responses.append(response)
|
||||||
|
full_response = "".join(str(r) for r in responses)
|
||||||
|
return {"response": full_response}
|
||||||
|
except Exception as exc:
|
||||||
|
return {"error": str(exc), "response": ""}
|
||||||
|
|
||||||
|
def chat_stream(self, messages, **kwargs):
|
||||||
|
with self._lock:
|
||||||
|
if self._status != "ready" or self._model is None:
|
||||||
|
yield 'data: {"error": "model not loaded"}\n\n'
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
generate_kwargs = {**self._generating_args, **kwargs}
|
||||||
|
formatted = self._model.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||||
|
for new_text in self._model.stream_chat(formatted, generate_kwargs):
|
||||||
|
yield new_text
|
||||||
|
except Exception as exc:
|
||||||
|
yield 'data: {"error": "' + str(exc) + '"}\n\n'
|
||||||
|
|
||||||
|
|
||||||
|
_inference_session = None
|
||||||
|
|
||||||
|
def get_inference_session():
|
||||||
|
global _inference_session
|
||||||
|
if _inference_session is None:
|
||||||
|
_inference_session = InferenceSession()
|
||||||
|
return _inference_session
|
||||||
@@ -4,3 +4,7 @@ python-multipart>=0.0.9
|
|||||||
pydantic>=2.7.0
|
pydantic>=2.7.0
|
||||||
python-dotenv>=1.0.1
|
python-dotenv>=1.0.1
|
||||||
httpx>=0.27.0
|
httpx>=0.27.0
|
||||||
|
|
||||||
|
# 训练/推理运行时:部署在算力服务器,独立于应用平台,不得装入应用后端 venv
|
||||||
|
# llamafactory 会连带安装兼容版本的 transformers(<=5.6.0)/peft/datasets 等
|
||||||
|
llamafactory
|
||||||
|
|||||||
429
design-qa.md
429
design-qa.md
@@ -1,429 +0,0 @@
|
|||||||
# Training Log Detail Design QA
|
|
||||||
|
|
||||||
## Evidence
|
|
||||||
|
|
||||||
- Source visual truth: `docs/superpowers/specs/assets/training-log-detail-option-2.png`
|
|
||||||
- Implementation screenshot: `docs/superpowers/specs/assets/training-log-detail-final-expanded-1440.png`
|
|
||||||
- Collapsed implementation screenshot with global surface: `docs/superpowers/specs/assets/training-log-detail-global-surface-1440-v2.png`
|
|
||||||
- Normalized full-view comparison: `docs/superpowers/specs/assets/training-log-detail-final-comparison-normalized.png`
|
|
||||||
- Focused parameter comparison: `docs/superpowers/specs/assets/training-log-detail-final-comparison-params.png`
|
|
||||||
- White-canvas reference: `docs/superpowers/specs/assets/page-white-canvas-reference.png`
|
|
||||||
- White-canvas implementation: `docs/superpowers/specs/assets/model-edit-white-page-canvas-final-1440.png`
|
|
||||||
- White-canvas normalized comparison: `docs/superpowers/specs/assets/page-white-canvas-comparison.png`
|
|
||||||
- Training-log white-canvas screenshot: `docs/superpowers/specs/assets/training-log-detail-white-page-canvas-1440.png`
|
|
||||||
- Self-surface list screenshot: `docs/superpowers/specs/assets/fine-tune-list-self-surface-final-1440.png`
|
|
||||||
- Default-canvas detail screenshot: `docs/superpowers/specs/assets/training-log-detail-default-canvas-final-1440.png`
|
|
||||||
- Reference/detail comparison: `docs/superpowers/specs/assets/page-surface-reference-detail-comparison.png`
|
|
||||||
- Create-page duplicate-surface evidence: `docs/superpowers/specs/assets/fine-tune-create-double-surface-before-1440.png`
|
|
||||||
- Create-page single-surface evidence: `docs/superpowers/specs/assets/fine-tune-create-single-surface-final-1440.png`
|
|
||||||
- Route-transition flash reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-ea775434-828f-4bca-9714-72887faa9af9.png`
|
|
||||||
- Route-transition final detail frame: `docs/superpowers/specs/assets/route-transition-detail-final.png`
|
|
||||||
- Viewport: 1440 × 1024; comparison content normalized to 1200 × 800 after removing the existing 240px sidebar and 60px header from the implementation capture.
|
|
||||||
- State: `finance-sft-001`, completed, mock data loaded, training parameters expanded.
|
|
||||||
|
|
||||||
## Full-view comparison
|
|
||||||
|
|
||||||
The implementation preserves the selected two-column hierarchy: task and dataset information occupy the wide left track, runtime facts use the narrow right rail, and training parameters continue as a full-width disclosure section. The selected mock omitted the product shell, so the comparison intentionally crops the existing sidebar and header rather than treating them as design drift.
|
|
||||||
|
|
||||||
The global product mode now uses two intentional surface modes. Form and detail routes render inside one `#ffffff` page canvas with 16px radius and 24px content padding. List routes that already own a white table/card surface render that surface directly on the `#f3f5f8` application background, avoiding a redundant white layer.
|
|
||||||
|
|
||||||
## Required fidelity surfaces
|
|
||||||
|
|
||||||
- Fonts and typography: Existing system font stack is preserved. Heading, label, value, and muted-copy hierarchy match the selected direction; output model uses body-level contrast after iteration 1, and all “未配置” values use `#64748b` on white after iteration 2.
|
|
||||||
- Spacing and layout rhythm: 24px main gap, 16px section gap, 12px surface radius, and light row separators match the selected composition. The existing application shell reduces usable content width, but normalized proportions remain aligned.
|
|
||||||
- Colors and tokens: Indigo accent, Slate text, success status, `#f3f5f8` page background, and white content surfaces are consistent with the current product.
|
|
||||||
- Image and icon fidelity: The screen contains no raster imagery. Existing Font Awesome icons are retained to match the repository's icon system; no placeholder, emoji, CSS drawing, or handcrafted SVG was introduced.
|
|
||||||
- Copy and content: Task name, status, model, date, duration, dataset metadata, storage, SFT, LoRA, and missing-value copy match the selected design and actual mock data.
|
|
||||||
|
|
||||||
## Interaction and responsive checks
|
|
||||||
|
|
||||||
- Parameter disclosure changed from `aria-expanded="false"` to `true` after activation, and the expanded content became visible.
|
|
||||||
- At 1000px viewport width, the overview changed to one column and the document had no horizontal overflow.
|
|
||||||
- At 700px viewport width, dataset metrics and parameter rows changed to one column and the document had no horizontal overflow.
|
|
||||||
- Browser console: no errors. One existing Element Plus `el-link` underline deprecation warning was emitted by the login flow and is unrelated to this page.
|
|
||||||
|
|
||||||
## Comparison history
|
|
||||||
|
|
||||||
### Iteration 1 — blocked
|
|
||||||
|
|
||||||
- [P2] The implementation added a visible “基础训练参数” heading that did not exist in the selected mock, creating extra vertical space.
|
|
||||||
- [P2] “暂未生成” was styled too faintly compared with the selected design.
|
|
||||||
|
|
||||||
Fixes:
|
|
||||||
|
|
||||||
- Removed the redundant visible base-parameter heading while retaining an accessible region label.
|
|
||||||
- Restored body-level contrast for “暂未生成”.
|
|
||||||
|
|
||||||
Post-fix evidence:
|
|
||||||
|
|
||||||
- `docs/superpowers/specs/assets/training-log-detail-final-comparison-normalized.png`
|
|
||||||
- `docs/superpowers/specs/assets/training-log-detail-final-comparison-params.png`
|
|
||||||
|
|
||||||
### Iteration 2 — blocked
|
|
||||||
|
|
||||||
- [P2] “未配置” values used `#94a3b8` on white, below WCAG AA contrast for 14px text.
|
|
||||||
|
|
||||||
Fix:
|
|
||||||
|
|
||||||
- Updated muted values to `#64748b`; the regression check now calculates the contrast ratio and requires at least 4.5:1.
|
|
||||||
|
|
||||||
Post-fix evidence:
|
|
||||||
|
|
||||||
- Browser computed color: `rgb(100, 116, 139)`.
|
|
||||||
- Browser console: no errors.
|
|
||||||
|
|
||||||
### Iteration 3 — passed
|
|
||||||
|
|
||||||
No actionable P0/P1/P2 differences remain. The retained P3 difference is that the generated mock does not include the real product sidebar/header; this is an intentional constraint because the existing shell is shared by every page.
|
|
||||||
|
|
||||||
### Iteration 4 — clarified global page canvas, passed
|
|
||||||
|
|
||||||
- [P1] The earlier interpretation left the route content directly on the gray layout background and only made individual cards white. The clarified reference requires a single white page canvas behind every route.
|
|
||||||
|
|
||||||
Fixes:
|
|
||||||
|
|
||||||
- Split the shell and page tokens into `--app-shell-bg: #f3f5f8` and `--app-page-bg: #ffffff`.
|
|
||||||
- Added one global `.page-canvas` around every route in `MainLayout.vue`.
|
|
||||||
- Added 16px outer gutter, 16px canvas radius, 24px canvas padding, and a subtle canvas shadow.
|
|
||||||
- Flattened a route-root `PageCard` to prevent a duplicate large card layer.
|
|
||||||
|
|
||||||
Post-fix evidence:
|
|
||||||
|
|
||||||
- `docs/superpowers/specs/assets/page-white-canvas-comparison.png`
|
|
||||||
- Browser computed canvas: white background, 16px radius, 24px padding; outer shell: `rgb(243, 245, 248)`.
|
|
||||||
- Both the model-edit page and training-log page render inside the same global white canvas without horizontal overflow.
|
|
||||||
|
|
||||||
### Iteration 5 — corrected list-page surface ownership, passed
|
|
||||||
|
|
||||||
- [P1] Applying the white page canvas to every route created a redundant layer on list pages because `DataTablePage`, model evaluation, and model management already provide their own white root card.
|
|
||||||
|
|
||||||
Fixes:
|
|
||||||
|
|
||||||
- Added explicit `pageSurface: 'self'` metadata to each self-surfaced list route: model tuning, model evaluation, model inference, model management, data processing, and dataset management.
|
|
||||||
- Added `.page-canvas.is-self-surface` to remove the outer canvas padding, radius, background, and shadow only for those routes.
|
|
||||||
- Preserved the default white canvas for training-log, create, edit, preview, chat, and result routes.
|
|
||||||
|
|
||||||
Post-fix evidence:
|
|
||||||
|
|
||||||
- `docs/superpowers/specs/assets/fine-tune-list-self-surface-final-1440.png`
|
|
||||||
- `docs/superpowers/specs/assets/training-log-detail-default-canvas-final-1440.png`
|
|
||||||
- `docs/superpowers/specs/assets/page-surface-reference-detail-comparison.png`
|
|
||||||
- Browser computed list state: transparent outer canvas, 0px padding/radius, no shadow; white 12px-radius list card on `rgb(243, 245, 248)` shell.
|
|
||||||
- Browser computed detail state: white outer canvas, 24px padding, 16px radius, subtle shadow.
|
|
||||||
- Both states have no horizontal overflow and no console errors at 1440 × 900.
|
|
||||||
|
|
||||||
### Iteration 6 — flattened wrapped root PageCard, passed
|
|
||||||
|
|
||||||
- [P1] The training-task creation route wraps its root `PageCard` in `.fine-tune-create`. The earlier selector only matched a `PageCard` directly under `.page-canvas`, so this page retained a second white background, 12px radius, and card shadow.
|
|
||||||
|
|
||||||
Fixes:
|
|
||||||
|
|
||||||
- Added an explicit `.page-card-host` marker to the training-task creation route root; the layout flattens only a directly rendered root `PageCard` or a `PageCard` inside that explicit host.
|
|
||||||
- Root `PageCard` now uses a transparent background, 0px radius, no shadow, and no bottom margin while preserving its header/body layout.
|
|
||||||
- Kept the selector excluded from `.is-self-surface`, so list cards retain their own white background, 12px radius, and shadow.
|
|
||||||
- Rejected a generic one-level descendant selector because it would also match the training-log parameter card.
|
|
||||||
|
|
||||||
Post-fix evidence:
|
|
||||||
|
|
||||||
- `docs/superpowers/specs/assets/fine-tune-create-double-surface-before-1440.png`
|
|
||||||
- `docs/superpowers/specs/assets/fine-tune-create-single-surface-final-1440.png`
|
|
||||||
- Browser computed create-page root card: transparent background, 0px radius, no shadow; outer canvas remains white with 24px padding.
|
|
||||||
- Browser computed list-page card remains white with 12px radius and subtle shadow on a transparent outer canvas.
|
|
||||||
- Browser computed training-log parameter card remains white with 12px radius and subtle shadow, confirming that internal business cards are not flattened.
|
|
||||||
- Both pages have no horizontal overflow; create-page console has no errors.
|
|
||||||
|
|
||||||
### Iteration 7 — removed page-level opacity transition, passed
|
|
||||||
|
|
||||||
- [P1] When navigating from a self-surface list to a default-canvas secondary page, `route.meta.pageSurface` changed immediately while the old list remained for the 150ms `out-in` leave animation. The result was a semi-transparent old list rendered inside the new white canvas.
|
|
||||||
|
|
||||||
Fixes:
|
|
||||||
|
|
||||||
- Removed the page-level Vue `transition` wrapper from `MainLayout.vue`.
|
|
||||||
- Removed the `.fade-enter-*` and `.fade-leave-*` opacity rules.
|
|
||||||
- Preserved local component animations such as dialogs, disclosures, and the selected-row batch bar.
|
|
||||||
|
|
||||||
Post-fix evidence:
|
|
||||||
|
|
||||||
- Source flash frame: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-ea775434-828f-4bca-9714-72887faa9af9.png`.
|
|
||||||
- Final detail frame: `docs/superpowers/specs/assets/route-transition-detail-final.png`.
|
|
||||||
- Immediate state after list → create: old list absent, route-root opacity `1`, zero `.fade-*` transition elements, correct default canvas.
|
|
||||||
- Immediate state after create → list: old create page absent, route-root opacity `1`, zero `.fade-*` transition elements, correct self-surface canvas.
|
|
||||||
- Immediate state after list → training log: old list absent, route-root opacity `1`, zero `.fade-*` transition elements, correct default canvas.
|
|
||||||
- All three paths had no horizontal overflow; browser console had no errors.
|
|
||||||
|
|
||||||
## Build evidence gap
|
|
||||||
|
|
||||||
The `type-check` script now uses project-reference mode (`vue-tsc -b --noEmit`) so it no longer reports a false pass. `npm run type-check` and `npm run build` remain blocked by pre-existing TypeScript errors in `src/mock/adapter.ts`, `FineTuneCreateView.vue`, and `FineTuneListView.vue`; no remaining error points to `TrainingLogView.vue` or the page-surface files. `npx vite build` succeeds, proving the updated UI bundles for production.
|
|
||||||
|
|
||||||
Design-QA final result: passed
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# Service Dashboard Design QA
|
|
||||||
|
|
||||||
## Evidence
|
|
||||||
|
|
||||||
- Source visual truth: `docs/superpowers/specs/assets/service-dashboard-approved-1440.png`
|
|
||||||
- Browser-rendered implementation: `docs/superpowers/specs/assets/service-dashboard-implementation-1440.png`
|
|
||||||
- Viewport: 1440 × 1024
|
|
||||||
- State: authenticated `admin` user on `/dashboard`; service dashboard navigation active; 7-day chart visible; four training tasks visible.
|
|
||||||
|
|
||||||
## Full-view comparison
|
|
||||||
|
|
||||||
The implementation preserves the approved composition: the product shell stays intact, the service dashboard is the active navigation item, the platform-health summary spans the top, the grouped training chart occupies the wide middle track, service health occupies the narrow track, and the training-task table spans the bottom. The three bar series, dual axes, dates, values, service counts, task names, statuses, progress, accuracy, and actions match the selected mock.
|
|
||||||
|
|
||||||
A separate focused crop was not required because both source and implementation evidence are full-resolution desktop captures at a readable scale; the chart labels, axis units, service rows, and every task-table column are legible in the full-view comparison.
|
|
||||||
|
|
||||||
## Required fidelity surfaces
|
|
||||||
|
|
||||||
- Fonts and typography: the existing Inter/system/PingFang stack is retained. Heading, section title, metric, table header, and muted-copy weights and sizes match the selected direction.
|
|
||||||
- Spacing and layout rhythm: 24px page padding, 14px section gaps, 10px panel radii, light separators, and the wide-chart/narrow-status grid preserve the selected hierarchy. The implementation uses the repository's 240px sidebar and 60px header exactly.
|
|
||||||
- Colors and visual tokens: white page canvas, `#f3f5f8` shell, indigo `#4f46e5`, green `#10b981`, amber `#f59e0b`, red `#ef4444`, and slate text are aligned with the source and current product tokens.
|
|
||||||
- Image and icon fidelity: the page contains no decorative raster imagery. The supplied product logo is preserved and existing Font Awesome icons are used consistently; no emoji, handcrafted SVG, placeholder image, or CSS illustration was introduced.
|
|
||||||
- Copy and content: dashboard title, health summary, chart legend and units, service states, task names, task status, model names, progress, accuracy, timestamps, and action labels match the approved design.
|
|
||||||
|
|
||||||
## Interaction and runtime checks
|
|
||||||
|
|
||||||
- Login with the existing `admin` credentials navigated to `/dashboard`, confirming the requested default entry behavior.
|
|
||||||
- ECharts rendered one canvas; hovering 07/10 exposed the tooltip values: training count 18, GPU count 7, and average accuracy 91%.
|
|
||||||
- “查看全部任务” navigated to `/fine-tune` and browser back restored `/dashboard`.
|
|
||||||
- The first “查看详情” action navigated to `/training-log/103942` and browser back restored `/dashboard`.
|
|
||||||
- Browser console errors: none.
|
|
||||||
- `npm run test:default-dashboard`: passed.
|
|
||||||
- `npm run test:dashboard`: passed.
|
|
||||||
- `npx vite build`: passed.
|
|
||||||
|
|
||||||
## Comparison history
|
|
||||||
|
|
||||||
### Iteration 1 — passed
|
|
||||||
|
|
||||||
No actionable P0/P1/P2 differences remain. The only intentional product constraint is that the sidebar active background follows the repository's current neutral active token instead of the slightly bluer tint produced by ImageGen; location, contrast, label, and active-state clarity remain equivalent.
|
|
||||||
|
|
||||||
## Validation gap
|
|
||||||
|
|
||||||
`npm run type-check` remains blocked by pre-existing TypeScript errors in the mock adapter, dataset mock typing, data-process list, evaluation tabs, and fine-tune views. No reported error points to `DashboardView.vue`, the ECharts registration, router defaults, login redirect, or dashboard regression scripts. The direct Vite production build succeeds.
|
|
||||||
|
|
||||||
Design-QA final result: passed
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
## Compact dashboard revision
|
|
||||||
|
|
||||||
- User annotation reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-43f63327-063f-47da-9e96-31b53cebc49d.png`
|
|
||||||
- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-compact-1440.png`
|
|
||||||
- Viewport: 1440 × 1024
|
|
||||||
- State: authenticated dashboard, compact layout, redundant title/action row removed.
|
|
||||||
|
|
||||||
### Iteration 2 — passed
|
|
||||||
|
|
||||||
The annotated header row containing the duplicate “服务看板” title, subtitle, and “查看告警” action was removed entirely. Section gaps, overview height, health icon, metric type, chart height, service rows, task heading, and task rows were reduced by roughly 10%–15%. The result preserves chart labels, dual-axis readability, service-state text, task progress, accuracy, and all task actions while bringing the primary content closer to the top of the page.
|
|
||||||
|
|
||||||
- ECharts tooltip remains functional after the height reduction and reports all three 07/10 series values.
|
|
||||||
- The revised page contains no browser console errors.
|
|
||||||
- The full-resolution comparison makes the removed annotation target and the compact replacement legible; no focused crop is necessary.
|
|
||||||
- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build` pass.
|
|
||||||
|
|
||||||
Design-QA final result: passed
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
## One-screen dashboard revision
|
|
||||||
|
|
||||||
- User annotation reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-3c166ffc-7243-4e57-94fc-48949599c4f1.png`
|
|
||||||
- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-one-screen-1440x768.png`
|
|
||||||
- Viewport: 1440 × 768
|
|
||||||
- State: authenticated dashboard with the desktop low-height compact rules active.
|
|
||||||
|
|
||||||
### Iteration 3 — passed
|
|
||||||
|
|
||||||
The platform-status block was reduced again, including its container padding, inner gap, health icon, status copy, metric labels, and metric values. The chart, service rows, task rows, and page-canvas padding now use a dedicated `max-height: 900px` desktop mode. The dashboard page canvas is constrained to the available application viewport so the outer content area does not introduce a vertical scrollbar.
|
|
||||||
|
|
||||||
Browser measurements at 1440 × 768:
|
|
||||||
|
|
||||||
- document overflow: false
|
|
||||||
- layout-content overflow: false
|
|
||||||
- page-canvas overflow: false
|
|
||||||
- dashboard overflow: false
|
|
||||||
- task section bottom: 653px within the 768px viewport
|
|
||||||
- ECharts tooltip: passed with all three series present
|
|
||||||
- browser console errors: none
|
|
||||||
- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build`: passed
|
|
||||||
|
|
||||||
Design-QA final result: passed
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
## Flexible middle-region revision
|
|
||||||
|
|
||||||
- User annotation reference: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-daf2bbae-baea-418d-9962-7d0e1a1c219b.png`
|
|
||||||
- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-flex-middle-1440x900.png`
|
|
||||||
- Viewport: 1440 × 900
|
|
||||||
- State: authenticated dashboard with flexible middle-region growth.
|
|
||||||
|
|
||||||
### Iteration 4 — passed
|
|
||||||
|
|
||||||
The previous fixed-height middle row caused unused white space beneath the task table on taller screens. The dashboard now reserves compact intrinsic height for the platform summary and task table while allowing the chart/service row to consume all remaining viewport height. The ECharts canvas grows with that row, and the service-state rows distribute across the matching height.
|
|
||||||
|
|
||||||
Browser measurements:
|
|
||||||
|
|
||||||
- at 1440 × 768, chart height: 283px; no document, layout, canvas, or dashboard overflow
|
|
||||||
- at 1440 × 900, chart height: 415px; no document, layout, canvas, or dashboard overflow
|
|
||||||
- task table bottom at 1440 × 900: 868px within the 900px viewport
|
|
||||||
- ECharts tooltip: passed with all three series present
|
|
||||||
- browser console errors: none
|
|
||||||
- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build`: passed
|
|
||||||
|
|
||||||
Design-QA final result: passed
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
## Narrow service-status revision
|
|
||||||
|
|
||||||
- User request: make the right-hand service-status panel slightly narrower.
|
|
||||||
- Revised implementation screenshot: `docs/superpowers/specs/assets/service-dashboard-narrow-service-1440x900.png`
|
|
||||||
- Viewports: 1440 × 900 and 1440 × 768
|
|
||||||
- State: authenticated dashboard with the service-status column reduced to approximately 30% of the middle row.
|
|
||||||
|
|
||||||
### Iteration 5 — passed
|
|
||||||
|
|
||||||
The middle grid now allocates `1.9fr` to the training chart and `0.82fr` to service status, with a 300px minimum width for the service panel. This gives the chart more horizontal space while keeping all service names, status badges, and instance counts fully visible.
|
|
||||||
|
|
||||||
Browser measurements:
|
|
||||||
|
|
||||||
- at 1440 × 900, chart width: 799px; service width: 345px; service share: 30.1%
|
|
||||||
- at 1440 × 768, chart width: 799px; service width: 345px
|
|
||||||
- clipped service cells: none at both tested viewports
|
|
||||||
- horizontal and vertical document overflow: none at both tested viewports
|
|
||||||
- browser console errors: none
|
|
||||||
- `npm run test:dashboard`, `npm run test:default-dashboard`, and `npx vite build`: passed
|
|
||||||
|
|
||||||
Design-QA final result: passed
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
## Taller training-task revision
|
|
||||||
|
|
||||||
- User request: increase the training-task region slightly and shorten the middle chart/service region.
|
|
||||||
- Source visual truth: `docs/superpowers/specs/assets/service-dashboard-narrow-service-1440x900.png` plus the current user annotation.
|
|
||||||
- Intended viewports: 1440 × 900 and 1440 × 768.
|
|
||||||
- State: authenticated dashboard with larger task heading and table rows.
|
|
||||||
|
|
||||||
### Iteration 6 — blocked
|
|
||||||
|
|
||||||
The task section now uses a taller heading and table rows in both standard and low-height desktop modes. Because the middle row is the only flexible region, the additional task height is taken directly from the chart/service row while preserving the one-screen layout contract in code.
|
|
||||||
|
|
||||||
Verification evidence:
|
|
||||||
|
|
||||||
- `npm run test:dashboard`: passed
|
|
||||||
- `npm run test:default-dashboard`: passed
|
|
||||||
- `npx vite build`: passed
|
|
||||||
- browser-rendered comparison: blocked because the in-app browser rejected the local preview URL under its URL security policy
|
|
||||||
- implementation screenshot: unavailable for this iteration
|
|
||||||
|
|
||||||
Design-QA final result: blocked
|
|
||||||
|
|
||||||
final result: blocked
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# Dataset Version Actions Design QA
|
|
||||||
|
|
||||||
## Evidence
|
|
||||||
|
|
||||||
- Source visual truth: `/Users/caoxiaozhu/.codex/generated_images/019f5e2e-3bee-77f0-b285-89ef139db56c/exec-48a163b6-d4c9-4646-9199-135957c6e72e.png`
|
|
||||||
- Historical-version implementation: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/implementation-historical-version-final.png`
|
|
||||||
- Delete-confirm implementation: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/implementation-delete-confirm.png`
|
|
||||||
- Full-view comparison: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/comparison-full.png`
|
|
||||||
- Focused version-control comparison: `/Users/caoxiaozhu/.codex/visualizations/2026/07/14/019f5e2e-3bee-77f0-b285-89ef139db56c/dataset-version-build/comparison-version-controls-final.png`
|
|
||||||
- Viewport: 1316 × 768 browser window; source crop normalized for the focused comparison.
|
|
||||||
- State: authenticated dataset detail, V3 current, V2 selected as a read-only historical version.
|
|
||||||
|
|
||||||
## Full-view comparison
|
|
||||||
|
|
||||||
The existing product shell, dataset summary, version selector, read-only alert, file toolbar, and sample table remain unchanged. The former standalone primary action has been replaced by one compact rounded-square overflow button at the far right of the version-control row, matching the selected hierarchy.
|
|
||||||
|
|
||||||
## Focused comparison and required fidelity surfaces
|
|
||||||
|
|
||||||
- Fonts and typography: existing system/PingFang stack, 13px labels, 12px metadata, and Element Plus menu text are preserved.
|
|
||||||
- Spacing and layout rhythm: the 40px overflow trigger aligns with the version selector and leaves the central status copy flexible; the 168px menu provides 40px action rows.
|
|
||||||
- Colors and visual tokens: the existing indigo primary token is used for the activate icon; the delete item and confirmation action use the danger token.
|
|
||||||
- Image and icon fidelity: no new raster assets are needed. Existing Font Awesome ellipsis, check-circle, and trash icons match the repository's icon system.
|
|
||||||
- Copy and content: the menu contains exactly “设为当前版本” and “删除版本”, separated visually; the confirmation names V2 and explains that current V3 is unaffected.
|
|
||||||
|
|
||||||
## Interaction checks
|
|
||||||
|
|
||||||
- Created V2 and V3 through the real edit-and-save flow, then switched from current V3 to historical V2.
|
|
||||||
- Historical records became read-only and the overflow trigger appeared; current V3 showed no history-operation trigger.
|
|
||||||
- Opening the trigger exposed exactly two accessible menu items: “设为当前版本” and “删除版本”.
|
|
||||||
- Choosing “删除版本” opened the danger confirmation dialog; cancelling returned focus without deleting data.
|
|
||||||
- Actual deletion behavior, protected-version rejection, optimistic-lock handling, and non-reused version numbers are covered by `test:dataset-preview`.
|
|
||||||
|
|
||||||
## Comparison history
|
|
||||||
|
|
||||||
### Iteration 1 — blocked
|
|
||||||
|
|
||||||
- [P2] The overflow trigger was circular while the selected mock used a small rounded square.
|
|
||||||
- [P2] The menu did not explicitly lock its target width or primary-action icon color.
|
|
||||||
|
|
||||||
Fixes:
|
|
||||||
|
|
||||||
- Replaced the circular trigger with a 40px square and 10px radius.
|
|
||||||
- Set the menu minimum width to 168px, action height to 40px, and the activate icon to the product primary color.
|
|
||||||
|
|
||||||
### Iteration 2 — passed
|
|
||||||
|
|
||||||
No actionable P0/P1/P2 differences remain. The desktop capture API does not retain the transient popup layer in screenshots, so the open-menu labels were additionally verified through the accessibility tree; exact popup shadow rendering remains a non-blocking P3 capture gap.
|
|
||||||
|
|
||||||
final result: passed
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# Login Page Responsive Design QA
|
|
||||||
|
|
||||||
## Evidence
|
|
||||||
|
|
||||||
- Source visual truth: `/Users/caoxiaozhu/.codex/generated_images/019f5f3f-f7e1-7e41-9605-ea307d9f09e6/exec-d376c603-0c47-45ad-86fd-3e99c1a95ec7.png`
|
|
||||||
- Implementation route: `http://localhost:6801/login`
|
|
||||||
- Implementation screenshot: unavailable because the in-app browser runtime could not initialize in this session.
|
|
||||||
- Intended desktop viewport: 1536 × 1024.
|
|
||||||
- Intended laptop viewports: 1366 × 768 and 1280 × 720.
|
|
||||||
- State: unauthenticated login page, default username and password populated.
|
|
||||||
|
|
||||||
## Static and automated evidence
|
|
||||||
|
|
||||||
- Added a 1440px width breakpoint that shifts the split from 58/42 to 55/45 and caps the form at 440px.
|
|
||||||
- Added a short-screen breakpoint for heights up to 820px that reduces title, form, input, footer, and panel spacing without hiding the left visual.
|
|
||||||
- Kept the single-column fallback at 900px and below.
|
|
||||||
- `regression-login-layout.mjs`: passed.
|
|
||||||
- `regression-default-dashboard.mjs`: passed.
|
|
||||||
- `vue-tsc -b --noEmit`: passed.
|
|
||||||
- Vite dev transform for `LoginView.vue`: HTTP 200.
|
|
||||||
- Production build: blocked by the pre-existing missing route component `UserPermissionView.vue`, outside the login-page change.
|
|
||||||
|
|
||||||
## Required fidelity surfaces
|
|
||||||
|
|
||||||
- Fonts and typography: code uses the existing product font stack with laptop-specific display-size reductions; visual comparison remains unavailable.
|
|
||||||
- Spacing and layout rhythm: dedicated width and height media queries are present; rendered measurements remain unavailable.
|
|
||||||
- Colors and visual tokens: existing indigo tokens and the selected dark-purple visual asset are preserved.
|
|
||||||
- Image quality and asset fidelity: the generated `login-hero-flow.png` is used directly; the official `logo.png` is reused for the brand lockup.
|
|
||||||
- Copy and content: platform title, supporting copy, form labels, actions, and footer match the selected design.
|
|
||||||
|
|
||||||
## Findings
|
|
||||||
|
|
||||||
- [P2] Browser-rendered laptop comparison unavailable
|
|
||||||
Location: login page at 1366 × 768 and 1280 × 720.
|
|
||||||
Evidence: the in-app browser runtime failed during initialization, so no implementation screenshot or side-by-side comparison could be captured.
|
|
||||||
Impact: static checks cannot prove that all visible spacing and crop details match the selected design at laptop sizes.
|
|
||||||
Fix: capture both laptop viewports in a working in-app browser session, compare them with the source, and resolve any remaining P0/P1/P2 differences.
|
|
||||||
|
|
||||||
## Comparison history
|
|
||||||
|
|
||||||
### Iteration 1 — blocked
|
|
||||||
|
|
||||||
- User reported that the initial implementation was optimized for large displays and did not compose well on laptop screens.
|
|
||||||
- Added explicit laptop-width and short-screen layout rules and passed targeted regression/type checks.
|
|
||||||
- Post-fix visual evidence remains unavailable because browser capture is blocked.
|
|
||||||
|
|
||||||
final result: blocked
|
|
||||||
@@ -16,7 +16,16 @@ export interface ComputeNode {
|
|||||||
data_root: string
|
data_root: string
|
||||||
model_root: string
|
model_root: string
|
||||||
log_root: string
|
log_root: string
|
||||||
|
api_version?: string
|
||||||
|
capabilities?: string[]
|
||||||
|
description?: string
|
||||||
last_health_check_at?: string
|
last_health_check_at?: string
|
||||||
|
health_detail?: Record<string, unknown>
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ComputeNodePayload = Partial<ComputeNode> & {
|
||||||
|
code?: string
|
||||||
|
api_base_url?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ComputeGpu {
|
export interface ComputeGpu {
|
||||||
@@ -66,11 +75,14 @@ export interface ResourceReplica {
|
|||||||
|
|
||||||
export const getComputeNodes = () => get<ComputeNode[]>('/compute/nodes')
|
export const getComputeNodes = () => get<ComputeNode[]>('/compute/nodes')
|
||||||
|
|
||||||
|
export const createComputeNode = (data: ComputeNodePayload) =>
|
||||||
|
post<ComputeNode>('/compute/nodes', data)
|
||||||
|
|
||||||
export const updateComputeNode = (id: string, data: Partial<ComputeNode>) =>
|
export const updateComputeNode = (id: string, data: Partial<ComputeNode>) =>
|
||||||
put<ComputeNode>(`/compute/nodes/${id}`, data)
|
put<ComputeNode>(`/compute/nodes/${id}`, data)
|
||||||
|
|
||||||
export const testComputeNode = (id: string) =>
|
export const testComputeNode = (id: string) =>
|
||||||
post<{ node_id: string; success: boolean; latency_ms: number }>(`/compute/nodes/${id}/test-connection`)
|
post<{ node_id: string; success: boolean; latency_ms: number; gpu_count: number; error?: string }>(`/compute/nodes/${id}/test-connection`)
|
||||||
|
|
||||||
export const enableComputeNode = (id: string) => post<ComputeNode>(`/compute/nodes/${id}/enable`)
|
export const enableComputeNode = (id: string) => post<ComputeNode>(`/compute/nodes/${id}/enable`)
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ export interface DashboardStats {
|
|||||||
service_status: ServiceStatusStat[]
|
service_status: ServiceStatusStat[]
|
||||||
training_tasks: TrainingTaskStat[]
|
training_tasks: TrainingTaskStat[]
|
||||||
operation_distribution: { name: string; value: number }[]
|
operation_distribution: { name: string; value: number }[]
|
||||||
login_duration_rank: { user: string; role: string; duration: string }[]
|
login_duration_rank: { user: string; role: string; duration: number }[]
|
||||||
recent_login_users: { user: string; role: string; last_login: string }[]
|
recent_login_users: { user: string; role: string; last_login: string }[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ export const getHealth = () => get<HealthMetrics>('/health')
|
|||||||
export const login = (username: string, password: string) =>
|
export const login = (username: string, password: string) =>
|
||||||
post<LoginResponse>('/login', { username, password })
|
post<LoginResponse>('/login', { username, password })
|
||||||
|
|
||||||
|
/** 退出登录,上报会话结束以统计在线时长 */
|
||||||
|
export const logout = (sessionId: string) =>
|
||||||
|
post<null>('/logout', { session_id: sessionId })
|
||||||
|
|
||||||
/** 用户列表 */
|
/** 用户列表 */
|
||||||
export const getUsers = () => get<SystemUser[]>('/users')
|
export const getUsers = () => get<SystemUser[]>('/users')
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { defineStore } from 'pinia'
|
import { defineStore } from 'pinia'
|
||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { login as loginApi } from '@/api/modules/system'
|
import { login as loginApi, logout as logoutApi } from '@/api/modules/system'
|
||||||
import { SESSION_TIMEOUT } from '@/constants'
|
import { SESSION_TIMEOUT } from '@/constants'
|
||||||
import type { PermissionCode, SystemUser } from '@/types'
|
import type { PermissionCode, SystemUser } from '@/types'
|
||||||
|
|
||||||
@@ -61,6 +61,7 @@ export const useAuthStore = defineStore('auth', () => {
|
|||||||
return '观察员'
|
return '观察员'
|
||||||
})
|
})
|
||||||
const loginTime = ref<number>(parseInt(localStorage.getItem('loginTime') || '0', 10) || 0)
|
const loginTime = ref<number>(parseInt(localStorage.getItem('loginTime') || '0', 10) || 0)
|
||||||
|
const sessionId = ref<string>(localStorage.getItem('sessionId') || '')
|
||||||
|
|
||||||
const isLoggedIn = computed(() => {
|
const isLoggedIn = computed(() => {
|
||||||
if (!loginTime.value) return false
|
if (!loginTime.value) return false
|
||||||
@@ -76,6 +77,8 @@ export const useAuthStore = defineStore('auth', () => {
|
|||||||
localStorage.setItem(USER_STORAGE_KEY, JSON.stringify(response.user))
|
localStorage.setItem(USER_STORAGE_KEY, JSON.stringify(response.user))
|
||||||
localStorage.setItem('loginTime', String(loginTime.value))
|
localStorage.setItem('loginTime', String(loginTime.value))
|
||||||
localStorage.setItem('authToken', response.token)
|
localStorage.setItem('authToken', response.token)
|
||||||
|
sessionId.value = response.session_id
|
||||||
|
localStorage.setItem('sessionId', response.session_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 检查当前账号是否拥有指定模块权限。 */
|
/** 检查当前账号是否拥有指定模块权限。 */
|
||||||
@@ -93,13 +96,22 @@ export const useAuthStore = defineStore('auth', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** 退出 */
|
/** 退出 */
|
||||||
function logout() {
|
async function logout() {
|
||||||
|
if (sessionId.value) {
|
||||||
|
try {
|
||||||
|
await logoutApi(sessionId.value)
|
||||||
|
} catch {
|
||||||
|
// 上报失败不影响本地退出
|
||||||
|
}
|
||||||
|
}
|
||||||
currentUser.value = null
|
currentUser.value = null
|
||||||
loginTime.value = 0
|
loginTime.value = 0
|
||||||
|
sessionId.value = ''
|
||||||
localStorage.removeItem('username')
|
localStorage.removeItem('username')
|
||||||
localStorage.removeItem(USER_STORAGE_KEY)
|
localStorage.removeItem(USER_STORAGE_KEY)
|
||||||
localStorage.removeItem('loginTime')
|
localStorage.removeItem('loginTime')
|
||||||
localStorage.removeItem('authToken')
|
localStorage.removeItem('authToken')
|
||||||
|
localStorage.removeItem('sessionId')
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -108,6 +120,7 @@ export const useAuthStore = defineStore('auth', () => {
|
|||||||
displayName,
|
displayName,
|
||||||
roleLabel,
|
roleLabel,
|
||||||
loginTime,
|
loginTime,
|
||||||
|
sessionId,
|
||||||
isLoggedIn,
|
isLoggedIn,
|
||||||
hasPermission,
|
hasPermission,
|
||||||
login,
|
login,
|
||||||
|
|||||||
@@ -418,6 +418,7 @@ export interface SystemUser {
|
|||||||
export interface LoginResponse {
|
export interface LoginResponse {
|
||||||
token: string
|
token: string
|
||||||
user: SystemUser
|
user: SystemUser
|
||||||
|
session_id: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CreateUserPayload {
|
export interface CreateUserPayload {
|
||||||
|
|||||||
99
frontend/src/utils/status.ts
Normal file
99
frontend/src/utils/status.ts
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
type TagType = 'primary' | 'success' | 'warning' | 'danger' | 'info'
|
||||||
|
|
||||||
|
const STATUS_LABELS: Record<string, string> = {
|
||||||
|
pending: '等待中',
|
||||||
|
syncing: '同步中',
|
||||||
|
queued: '排队中',
|
||||||
|
running: '运行中',
|
||||||
|
completed: '已完成',
|
||||||
|
failed: '失败',
|
||||||
|
stopped: '已停止',
|
||||||
|
cancelled: '已取消',
|
||||||
|
starting: '启动中',
|
||||||
|
loading: '加载中',
|
||||||
|
loaded: '已加载',
|
||||||
|
ready: '已就绪',
|
||||||
|
done: '已完成',
|
||||||
|
error: '异常',
|
||||||
|
success: '成功',
|
||||||
|
not_started: '未启动',
|
||||||
|
valid: '有效',
|
||||||
|
modified: '已修改',
|
||||||
|
invalid: '无效',
|
||||||
|
original: '原始',
|
||||||
|
manual: '手动新增',
|
||||||
|
active: '启用',
|
||||||
|
disabled: '停用',
|
||||||
|
online: '在线',
|
||||||
|
offline: '离线',
|
||||||
|
draining: '维护中',
|
||||||
|
maintenance: '维护模式',
|
||||||
|
busy: '忙碌',
|
||||||
|
reserved: '已预留',
|
||||||
|
idle: '空闲',
|
||||||
|
warning: '告警',
|
||||||
|
available: '可用',
|
||||||
|
missing: '缺失',
|
||||||
|
synced: '已同步',
|
||||||
|
drifted: '已漂移',
|
||||||
|
repair_pending: '待修复',
|
||||||
|
}
|
||||||
|
|
||||||
|
const STATUS_TYPES: Record<string, TagType> = {
|
||||||
|
completed: 'success',
|
||||||
|
success: 'success',
|
||||||
|
running: 'success',
|
||||||
|
ready: 'success',
|
||||||
|
loaded: 'success',
|
||||||
|
active: 'success',
|
||||||
|
online: 'success',
|
||||||
|
busy: 'success',
|
||||||
|
available: 'success',
|
||||||
|
synced: 'success',
|
||||||
|
valid: 'success',
|
||||||
|
pending: 'info',
|
||||||
|
idle: 'info',
|
||||||
|
offline: 'info',
|
||||||
|
disabled: 'info',
|
||||||
|
original: 'info',
|
||||||
|
stopped: 'info',
|
||||||
|
cancelled: 'info',
|
||||||
|
syncing: 'warning',
|
||||||
|
queued: 'warning',
|
||||||
|
starting: 'warning',
|
||||||
|
loading: 'warning',
|
||||||
|
reserved: 'warning',
|
||||||
|
warning: 'warning',
|
||||||
|
modified: 'warning',
|
||||||
|
draining: 'warning',
|
||||||
|
maintenance: 'warning',
|
||||||
|
repair_pending: 'warning',
|
||||||
|
failed: 'danger',
|
||||||
|
error: 'danger',
|
||||||
|
invalid: 'danger',
|
||||||
|
missing: 'danger',
|
||||||
|
drifted: 'danger',
|
||||||
|
}
|
||||||
|
|
||||||
|
export function statusLabel(status?: string | number | null) {
|
||||||
|
const key = String(status ?? '').trim()
|
||||||
|
if (!key) return '未知'
|
||||||
|
return STATUS_LABELS[key] || key
|
||||||
|
}
|
||||||
|
|
||||||
|
export function statusTagType(status?: string | number | null): TagType {
|
||||||
|
const key = String(status ?? '').trim()
|
||||||
|
return STATUS_TYPES[key] || 'info'
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mergeStatusLabel(row: { merged?: boolean; merging?: boolean }) {
|
||||||
|
if (row.merging) return '合并中'
|
||||||
|
if (row.merged) return '已合并'
|
||||||
|
return '未合并'
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mergeStatusType(row: { merged?: boolean; merging?: boolean }): TagType {
|
||||||
|
if (row.merging) return 'warning'
|
||||||
|
if (row.merged) return 'success'
|
||||||
|
return 'info'
|
||||||
|
}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
import { computed, onMounted, onUnmounted, reactive, ref, watch } from 'vue'
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
import {
|
import {
|
||||||
|
createComputeNode,
|
||||||
disableComputeNode,
|
disableComputeNode,
|
||||||
drainComputeNode,
|
drainComputeNode,
|
||||||
enableComputeNode,
|
enableComputeNode,
|
||||||
@@ -10,23 +12,45 @@ import {
|
|||||||
getComputeQueue,
|
getComputeQueue,
|
||||||
getNodeReplicas,
|
getNodeReplicas,
|
||||||
testComputeNode,
|
testComputeNode,
|
||||||
|
updateComputeNode,
|
||||||
type ComputeGpu,
|
type ComputeGpu,
|
||||||
type ComputeNode,
|
type ComputeNode,
|
||||||
type ComputeQueueItem,
|
type ComputeQueueItem,
|
||||||
type ResourceReplica,
|
type ResourceReplica,
|
||||||
} from '@/api/modules/compute'
|
} from '@/api/modules/compute'
|
||||||
|
import { statusLabel, statusTagType } from '@/utils/status'
|
||||||
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
|
||||||
const activeTab = ref(String(route.query.tab || 'nodes'))
|
const activeTab = ref(String(route.query.tab || 'nodes'))
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
|
const buttonRefreshing = ref(false)
|
||||||
const nodes = ref<ComputeNode[]>([])
|
const nodes = ref<ComputeNode[]>([])
|
||||||
const gpus = ref<ComputeGpu[]>([])
|
const gpus = ref<ComputeGpu[]>([])
|
||||||
const queue = ref<ComputeQueueItem[]>([])
|
const queue = ref<ComputeQueueItem[]>([])
|
||||||
const replicas = ref<ResourceReplica[]>([])
|
const replicas = ref<ResourceReplica[]>([])
|
||||||
const selectedNodeId = ref('')
|
const selectedNodeId = ref('')
|
||||||
const lastUpdated = ref('')
|
const lastUpdated = ref('')
|
||||||
|
const nodeDialogVisible = ref(false)
|
||||||
|
const nodeDialogMode = ref<'create' | 'edit'>('create')
|
||||||
|
const savingNode = ref(false)
|
||||||
|
const nodeForm = reactive({
|
||||||
|
id: '',
|
||||||
|
code: '',
|
||||||
|
name: '',
|
||||||
|
api_base_url: '',
|
||||||
|
file_gateway_url: '',
|
||||||
|
enabled: true,
|
||||||
|
scheduler_status: 'offline',
|
||||||
|
scheduler_weight: 100,
|
||||||
|
tags_text: '',
|
||||||
|
max_parallel_jobs: 1,
|
||||||
|
data_root: '/data/yg-ft',
|
||||||
|
model_root: '/data/yg-ft/models',
|
||||||
|
log_root: '/opt/yg-ft/logs/training',
|
||||||
|
description: '',
|
||||||
|
})
|
||||||
let timer: ReturnType<typeof setInterval> | null = null
|
let timer: ReturnType<typeof setInterval> | null = null
|
||||||
|
|
||||||
const selectedNode = computed(() => nodes.value.find((item) => item.id === selectedNodeId.value))
|
const selectedNode = computed(() => nodes.value.find((item) => item.id === selectedNodeId.value))
|
||||||
@@ -34,6 +58,10 @@ const enabledNodes = computed(() => nodes.value.filter((item) => item.enabled).l
|
|||||||
const busyGpus = computed(() => gpus.value.filter((item) => item.status === 'busy' || item.status === 'reserved').length)
|
const busyGpus = computed(() => gpus.value.filter((item) => item.status === 'busy' || item.status === 'reserved').length)
|
||||||
const totalRunningJobs = computed(() => nodes.value.reduce((sum, item) => sum + item.current_running_jobs, 0))
|
const totalRunningJobs = computed(() => nodes.value.reduce((sum, item) => sum + item.current_running_jobs, 0))
|
||||||
|
|
||||||
|
function asComputeNode(row: unknown): ComputeNode {
|
||||||
|
return row as ComputeNode
|
||||||
|
}
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
() => route.query.tab,
|
() => route.query.tab,
|
||||||
(tab) => {
|
(tab) => {
|
||||||
@@ -41,8 +69,9 @@ watch(
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
async function load() {
|
async function load(options: { showLoading?: boolean; showButtonLoading?: boolean } = {}) {
|
||||||
loading.value = true
|
if (options.showLoading) loading.value = true
|
||||||
|
if (options.showButtonLoading) buttonRefreshing.value = true
|
||||||
try {
|
try {
|
||||||
const [nodeList, gpuList, queueList] = await Promise.all([
|
const [nodeList, gpuList, queueList] = await Promise.all([
|
||||||
getComputeNodes(),
|
getComputeNodes(),
|
||||||
@@ -52,11 +81,14 @@ async function load() {
|
|||||||
nodes.value = nodeList
|
nodes.value = nodeList
|
||||||
gpus.value = gpuList
|
gpus.value = gpuList
|
||||||
queue.value = queueList
|
queue.value = queueList
|
||||||
if (!selectedNodeId.value && nodeList.length) selectedNodeId.value = nodeList[0].id
|
if ((!selectedNodeId.value || !nodeList.some((item) => item.id === selectedNodeId.value)) && nodeList.length) {
|
||||||
|
selectedNodeId.value = nodeList[0].id
|
||||||
|
}
|
||||||
await loadReplicas()
|
await loadReplicas()
|
||||||
lastUpdated.value = new Date().toLocaleTimeString('zh-CN', { hour12: false })
|
lastUpdated.value = new Date().toLocaleTimeString('zh-CN', { hour12: false })
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
if (options.showLoading) loading.value = false
|
||||||
|
if (options.showButtonLoading) buttonRefreshing.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,37 +104,120 @@ async function changeTab(name: string | number) {
|
|||||||
await router.replace({ path: '/compute', query: { tab: String(name) } })
|
await router.replace({ path: '/compute', query: { tab: String(name) } })
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleNodeAction(action: 'enable' | 'disable' | 'drain' | 'test', node: any) {
|
async function handleNodeAction(action: 'enable' | 'disable' | 'drain' | 'test', node: ComputeNode) {
|
||||||
const nodeId = String(node.id)
|
const nodeId = String(node.id)
|
||||||
if (action === 'enable') await enableComputeNode(nodeId)
|
if (action === 'enable') await enableComputeNode(nodeId)
|
||||||
if (action === 'disable') await disableComputeNode(nodeId)
|
if (action === 'disable') await disableComputeNode(nodeId)
|
||||||
if (action === 'drain') await drainComputeNode(nodeId)
|
if (action === 'drain') await drainComputeNode(nodeId)
|
||||||
if (action === 'test') await testComputeNode(nodeId)
|
if (action === 'test') {
|
||||||
|
const result = await testComputeNode(nodeId)
|
||||||
|
if (result.success) {
|
||||||
|
ElMessage.success(`连接成功,发现 ${result.gpu_count} 张 GPU,延迟 ${result.latency_ms}ms`)
|
||||||
|
} else {
|
||||||
|
ElMessage.error(result.error || '连接失败')
|
||||||
|
}
|
||||||
|
}
|
||||||
await load()
|
await load()
|
||||||
}
|
}
|
||||||
|
|
||||||
function nodeStatusType(status: string) {
|
function resetNodeForm() {
|
||||||
if (status === 'online') return 'success'
|
Object.assign(nodeForm, {
|
||||||
if (status === 'draining' || status === 'maintenance') return 'warning'
|
id: '',
|
||||||
return 'info'
|
code: '',
|
||||||
|
name: '',
|
||||||
|
api_base_url: '',
|
||||||
|
file_gateway_url: '',
|
||||||
|
enabled: true,
|
||||||
|
scheduler_status: 'offline',
|
||||||
|
scheduler_weight: 100,
|
||||||
|
tags_text: '',
|
||||||
|
max_parallel_jobs: 1,
|
||||||
|
data_root: '/data/yg-ft',
|
||||||
|
model_root: '/data/yg-ft/models',
|
||||||
|
log_root: '/opt/yg-ft/logs/training',
|
||||||
|
description: '',
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function taskStatusType(status: string) {
|
function openCreateNodeDialog() {
|
||||||
if (status === 'running') return 'success'
|
resetNodeForm()
|
||||||
if (status === 'syncing' || status === 'queued') return 'warning'
|
nodeDialogMode.value = 'create'
|
||||||
if (status === 'failed') return 'danger'
|
nodeDialogVisible.value = true
|
||||||
return 'info'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function gpuStatusType(status: string) {
|
function openEditNodeDialog(node: ComputeNode) {
|
||||||
if (status === 'busy') return 'success'
|
Object.assign(nodeForm, {
|
||||||
if (status === 'reserved') return 'warning'
|
id: node.id,
|
||||||
return 'info'
|
code: node.code,
|
||||||
|
name: node.name,
|
||||||
|
api_base_url: node.api_base_url,
|
||||||
|
file_gateway_url: node.file_gateway_url,
|
||||||
|
enabled: node.enabled,
|
||||||
|
scheduler_status: node.scheduler_status,
|
||||||
|
scheduler_weight: node.scheduler_weight,
|
||||||
|
tags_text: node.tags?.join(', ') || '',
|
||||||
|
max_parallel_jobs: node.max_parallel_jobs,
|
||||||
|
data_root: node.data_root,
|
||||||
|
model_root: node.model_root,
|
||||||
|
log_root: node.log_root,
|
||||||
|
description: node.description || '',
|
||||||
|
})
|
||||||
|
nodeDialogMode.value = 'edit'
|
||||||
|
nodeDialogVisible.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildNodePayload() {
|
||||||
|
const tags = nodeForm.tags_text
|
||||||
|
.replace(/,/g, ',')
|
||||||
|
.split(',')
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
return {
|
||||||
|
code: nodeForm.code.trim(),
|
||||||
|
name: nodeForm.name.trim() || nodeForm.code.trim(),
|
||||||
|
api_base_url: nodeForm.api_base_url.trim().replace(/\/$/, ''),
|
||||||
|
file_gateway_url: (nodeForm.file_gateway_url || nodeForm.api_base_url).trim().replace(/\/$/, ''),
|
||||||
|
enabled: nodeForm.enabled,
|
||||||
|
scheduler_status: nodeForm.scheduler_status,
|
||||||
|
scheduler_weight: Number(nodeForm.scheduler_weight) || 0,
|
||||||
|
tags,
|
||||||
|
max_parallel_jobs: Number(nodeForm.max_parallel_jobs) || 1,
|
||||||
|
data_root: nodeForm.data_root.trim() || '/data/yg-ft',
|
||||||
|
model_root: nodeForm.model_root.trim() || '/data/yg-ft/models',
|
||||||
|
log_root: nodeForm.log_root.trim() || '/opt/yg-ft/logs/training',
|
||||||
|
description: nodeForm.description.trim(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveNode() {
|
||||||
|
const payload = buildNodePayload()
|
||||||
|
if (!payload.code || !payload.api_base_url) {
|
||||||
|
ElMessage.warning('请填写节点编码和 Compute API 地址')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
savingNode.value = true
|
||||||
|
try {
|
||||||
|
if (nodeDialogMode.value === 'create') {
|
||||||
|
await createComputeNode(payload)
|
||||||
|
ElMessage.success('节点已创建')
|
||||||
|
} else {
|
||||||
|
await updateComputeNode(nodeForm.id, payload)
|
||||||
|
ElMessage.success('节点配置已更新')
|
||||||
|
}
|
||||||
|
nodeDialogVisible.value = false
|
||||||
|
await load()
|
||||||
|
} finally {
|
||||||
|
savingNode.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(value?: string) {
|
||||||
|
return value ? new Date(value).toLocaleString('zh-CN') : '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
load()
|
load({ showLoading: true })
|
||||||
timer = setInterval(load, 5000)
|
timer = setInterval(() => load(), 5000)
|
||||||
})
|
})
|
||||||
|
|
||||||
onUnmounted(() => {
|
onUnmounted(() => {
|
||||||
@@ -119,27 +234,16 @@ onUnmounted(() => {
|
|||||||
</div>
|
</div>
|
||||||
<div class="header-actions">
|
<div class="header-actions">
|
||||||
<span v-if="lastUpdated" class="last-updated">更新 {{ lastUpdated }}</span>
|
<span v-if="lastUpdated" class="last-updated">更新 {{ lastUpdated }}</span>
|
||||||
<el-button @click="load">刷新</el-button>
|
<el-button type="primary" @click="openCreateNodeDialog">新增节点</el-button>
|
||||||
|
<el-button :loading="buttonRefreshing" @click="load({ showButtonLoading: true })">刷新</el-button>
|
||||||
</div>
|
</div>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<div class="summary-grid">
|
<div class="summary-grid">
|
||||||
<div class="summary-tile">
|
<div class="summary-tile"><span>启用节点</span><strong>{{ enabledNodes }} / {{ nodes.length }}</strong></div>
|
||||||
<span>在线节点</span>
|
<div class="summary-tile"><span>GPU 占用</span><strong>{{ busyGpus }} / {{ gpus.length }}</strong></div>
|
||||||
<strong>{{ enabledNodes }} / {{ nodes.length }}</strong>
|
<div class="summary-tile"><span>运行任务</span><strong>{{ totalRunningJobs }}</strong></div>
|
||||||
</div>
|
<div class="summary-tile"><span>队列任务</span><strong>{{ queue.length }}</strong></div>
|
||||||
<div class="summary-tile">
|
|
||||||
<span>GPU 占用</span>
|
|
||||||
<strong>{{ busyGpus }} / {{ gpus.length }}</strong>
|
|
||||||
</div>
|
|
||||||
<div class="summary-tile">
|
|
||||||
<span>运行任务</span>
|
|
||||||
<strong>{{ totalRunningJobs }}</strong>
|
|
||||||
</div>
|
|
||||||
<div class="summary-tile">
|
|
||||||
<span>队列任务</span>
|
|
||||||
<strong>{{ queue.length }}</strong>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<el-tabs v-model="activeTab" class="compute-tabs" @tab-change="changeTab">
|
<el-tabs v-model="activeTab" class="compute-tabs" @tab-change="changeTab">
|
||||||
@@ -149,7 +253,7 @@ onUnmounted(() => {
|
|||||||
<el-table-column prop="name" label="节点名称" min-width="150" />
|
<el-table-column prop="name" label="节点名称" min-width="150" />
|
||||||
<el-table-column label="状态" width="120">
|
<el-table-column label="状态" width="120">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-tag :type="nodeStatusType(row.scheduler_status)">{{ row.scheduler_status }}</el-tag>
|
<el-tag :type="statusTagType(row.scheduler_status)">{{ statusLabel(row.scheduler_status) }}</el-tag>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="启用" width="90">
|
<el-table-column label="启用" width="90">
|
||||||
@@ -174,12 +278,13 @@ onUnmounted(() => {
|
|||||||
<div class="muted mono">{{ row.file_gateway_url }}</div>
|
<div class="muted mono">{{ row.file_gateway_url }}</div>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="操作" width="260" fixed="right">
|
<el-table-column label="操作" width="330" fixed="right">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-button size="small" @click="handleNodeAction('test', row)">测试</el-button>
|
<el-button size="small" @click="openEditNodeDialog(asComputeNode(row))">编辑</el-button>
|
||||||
<el-button v-if="row.enabled" size="small" @click="handleNodeAction('disable', row)">停用</el-button>
|
<el-button size="small" @click="handleNodeAction('test', asComputeNode(row))">测试</el-button>
|
||||||
<el-button v-else size="small" type="primary" @click="handleNodeAction('enable', row)">启用</el-button>
|
<el-button v-if="row.enabled" size="small" @click="handleNodeAction('disable', asComputeNode(row))">停用</el-button>
|
||||||
<el-button size="small" type="warning" plain @click="handleNodeAction('drain', row)">维护</el-button>
|
<el-button v-else size="small" type="primary" @click="handleNodeAction('enable', asComputeNode(row))">启用</el-button>
|
||||||
|
<el-button size="small" type="warning" plain @click="handleNodeAction('drain', asComputeNode(row))">维护</el-button>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
@@ -195,13 +300,11 @@ onUnmounted(() => {
|
|||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="状态" width="100">
|
<el-table-column label="状态" width="100">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-tag :type="gpuStatusType(row.status)">{{ row.status }}</el-tag>
|
<el-tag :type="statusTagType(row.status)">{{ statusLabel(row.status) }}</el-tag>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="利用率" min-width="180">
|
<el-table-column label="利用率" min-width="180">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }"><el-progress :percentage="row.gpu_percent" :stroke-width="8" /></template>
|
||||||
<el-progress :percentage="row.gpu_percent" :stroke-width="8" />
|
|
||||||
</template>
|
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="显存" min-width="190">
|
<el-table-column label="显存" min-width="190">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
@@ -228,19 +331,19 @@ onUnmounted(() => {
|
|||||||
<el-table-column prop="name" label="任务名称" min-width="200" />
|
<el-table-column prop="name" label="任务名称" min-width="200" />
|
||||||
<el-table-column label="状态" width="110">
|
<el-table-column label="状态" width="110">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-tag :type="taskStatusType(row.status)">{{ row.status }}</el-tag>
|
<el-tag :type="statusTagType(row.status)">{{ statusLabel(row.status) }}</el-tag>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="进度" min-width="220">
|
<el-table-column label="进度" min-width="220">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }"><el-progress :percentage="row.progress" :stroke-width="8" /></template>
|
||||||
<el-progress :percentage="row.progress" :stroke-width="8" />
|
|
||||||
</template>
|
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column prop="compute_node_id" label="节点" width="140" />
|
<el-table-column prop="compute_node_id" label="节点" width="140" />
|
||||||
<el-table-column label="GPU" width="120">
|
<el-table-column label="GPU" width="120">
|
||||||
<template #default="{ row }">{{ row.gpus?.join(', ') || '-' }}</template>
|
<template #default="{ row }">{{ row.gpus?.join(', ') || '-' }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column prop="create_time" label="创建时间" width="190" />
|
<el-table-column label="创建时间" width="190">
|
||||||
|
<template #default="{ row }">{{ formatTime(row.create_time) }}</template>
|
||||||
|
</el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
|
|
||||||
@@ -254,13 +357,72 @@ onUnmounted(() => {
|
|||||||
<el-table :data="replicas" height="100%">
|
<el-table :data="replicas" height="100%">
|
||||||
<el-table-column prop="resource_type" label="资源类型" width="110" />
|
<el-table-column prop="resource_type" label="资源类型" width="110" />
|
||||||
<el-table-column prop="resource_id" label="资源 ID" min-width="180" />
|
<el-table-column prop="resource_id" label="资源 ID" min-width="180" />
|
||||||
<el-table-column prop="local_path" label="本地路径" min-width="300" />
|
<el-table-column prop="local_path" label="本地路径" min-width="300" show-overflow-tooltip />
|
||||||
<el-table-column prop="status" label="状态" width="110" />
|
<el-table-column label="状态" width="110">
|
||||||
<el-table-column prop="sync_status" label="同步状态" width="120" />
|
<template #default="{ row }">
|
||||||
<el-table-column prop="create_time" label="创建时间" width="190" />
|
<el-tag :type="statusTagType(row.status)" size="small">{{ statusLabel(row.status) }}</el-tag>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="同步状态" width="120">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-tag :type="statusTagType(row.sync_status)" size="small">{{ statusLabel(row.sync_status) }}</el-tag>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="创建时间" width="190">
|
||||||
|
<template #default="{ row }">{{ formatTime(row.create_time) }}</template>
|
||||||
|
</el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
</el-tabs>
|
</el-tabs>
|
||||||
|
|
||||||
|
<el-dialog
|
||||||
|
v-model="nodeDialogVisible"
|
||||||
|
:title="nodeDialogMode === 'create' ? '新增算力节点' : '编辑算力节点'"
|
||||||
|
width="720px"
|
||||||
|
>
|
||||||
|
<el-form label-width="130px">
|
||||||
|
<div class="node-form-grid">
|
||||||
|
<el-form-item label="节点编码" required>
|
||||||
|
<el-input v-model="nodeForm.code" :disabled="nodeDialogMode === 'edit'" placeholder="gpu-node-01" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="节点名称">
|
||||||
|
<el-input v-model="nodeForm.name" placeholder="A800 训练节点 01" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="Compute API" required>
|
||||||
|
<el-input v-model="nodeForm.api_base_url" placeholder="http://10.0.0.11:19100" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="File Gateway">
|
||||||
|
<el-input v-model="nodeForm.file_gateway_url" placeholder="http://10.0.0.11:19101" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="调度权重">
|
||||||
|
<el-input-number v-model="nodeForm.scheduler_weight" :min="0" :max="1000" controls-position="right" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="最大并行任务">
|
||||||
|
<el-input-number v-model="nodeForm.max_parallel_jobs" :min="1" :max="32" controls-position="right" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="启用状态">
|
||||||
|
<el-switch v-model="nodeForm.enabled" active-text="启用" inactive-text="停用" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="调度状态">
|
||||||
|
<el-select v-model="nodeForm.scheduler_status">
|
||||||
|
<el-option label="离线" value="offline" />
|
||||||
|
<el-option label="在线" value="online" />
|
||||||
|
<el-option label="维护中" value="draining" />
|
||||||
|
<el-option label="维护模式" value="maintenance" />
|
||||||
|
</el-select>
|
||||||
|
</el-form-item>
|
||||||
|
</div>
|
||||||
|
<el-form-item label="标签"><el-input v-model="nodeForm.tags_text" placeholder="a800, lora, beijing" /></el-form-item>
|
||||||
|
<el-form-item label="数据根目录"><el-input v-model="nodeForm.data_root" /></el-form-item>
|
||||||
|
<el-form-item label="模型根目录"><el-input v-model="nodeForm.model_root" /></el-form-item>
|
||||||
|
<el-form-item label="训练日志目录"><el-input v-model="nodeForm.log_root" /></el-form-item>
|
||||||
|
<el-form-item label="备注"><el-input v-model="nodeForm.description" type="textarea" :rows="3" /></el-form-item>
|
||||||
|
</el-form>
|
||||||
|
<template #footer>
|
||||||
|
<el-button @click="nodeDialogVisible = false">取消</el-button>
|
||||||
|
<el-button type="primary" :loading="savingNode" @click="saveNode">保存</el-button>
|
||||||
|
</template>
|
||||||
|
</el-dialog>
|
||||||
</section>
|
</section>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -294,7 +456,8 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.header-actions {
|
.header-actions,
|
||||||
|
.replica-toolbar {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 12px;
|
gap: 12px;
|
||||||
@@ -306,6 +469,11 @@ onUnmounted(() => {
|
|||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.last-updated {
|
||||||
|
min-width: 92px;
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
.summary-grid {
|
.summary-grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
@@ -351,9 +519,6 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.replica-toolbar {
|
.replica-toolbar {
|
||||||
display: flex;
|
|
||||||
gap: 12px;
|
|
||||||
align-items: center;
|
|
||||||
margin-bottom: 12px;
|
margin-bottom: 12px;
|
||||||
|
|
||||||
.el-select {
|
.el-select {
|
||||||
@@ -361,6 +526,17 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.node-form-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
column-gap: 12px;
|
||||||
|
|
||||||
|
:deep(.el-input-number),
|
||||||
|
:deep(.el-select) {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@media (max-width: 960px) {
|
@media (max-width: 960px) {
|
||||||
.compute-header,
|
.compute-header,
|
||||||
.header-actions,
|
.header-actions,
|
||||||
@@ -372,5 +548,9 @@ onUnmounted(() => {
|
|||||||
.summary-grid {
|
.summary-grid {
|
||||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.node-form-grid {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -246,7 +246,7 @@ const loginDurationChartOption = computed<EChartsOption>(() => ({
|
|||||||
},
|
},
|
||||||
series: [
|
series: [
|
||||||
{
|
{
|
||||||
name: '距上次登录',
|
name: '登录时长',
|
||||||
type: 'bar',
|
type: 'bar',
|
||||||
data: loginDurationStats.value.map((user) => user.duration),
|
data: loginDurationStats.value.map((user) => user.duration),
|
||||||
barMaxWidth: 18,
|
barMaxWidth: 18,
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ async function loadData() {
|
|||||||
async function handleDelete(row: any) {
|
async function handleDelete(row: any) {
|
||||||
await deleteDataset(row.id)
|
await deleteDataset(row.id)
|
||||||
ElMessage.success('删除成功')
|
ElMessage.success('删除成功')
|
||||||
|
await loadData()
|
||||||
}
|
}
|
||||||
|
|
||||||
function handlePreview(row: any) {
|
function handlePreview(row: any) {
|
||||||
|
|||||||
134
前端功能失效问题排查.md
134
前端功能失效问题排查.md
@@ -1,134 +0,0 @@
|
|||||||
# 前端功能失效问题排查报告
|
|
||||||
|
|
||||||
> 排查时间:2026-07-30
|
|
||||||
> 环境:前端(Windows)`localhost:16801` → Vite 代理 `/modelTF` → 后端(WSL)`localhost:17861`
|
|
||||||
> 方法:抓取后端 openapi 路由表 + 实测 curl 比对前端真实请求路径
|
|
||||||
|
|
||||||
## 一、现象
|
|
||||||
|
|
||||||
页面可以打开(app 能正常加载、仪表盘等模块正常),但**大量功能点击无响应或报"请求失败"**。
|
|
||||||
经排查,故障集中在"路由前缀叠加导致后端 404",并非前端崩溃。
|
|
||||||
|
|
||||||
## 二、已验证正常的部分(排除法)
|
|
||||||
|
|
||||||
| 模块 | 接口 | 实测结果 |
|
|
||||||
|------|------|----------|
|
|
||||||
| 仪表盘 | `GET /modelTF/dashboard/stats` | ✅ 返回真实数据 |
|
|
||||||
| 登录 | `/modelTF/login` | ✅ 正常 |
|
|
||||||
| 健康检查 | `/modelTF/health` | ✅ 正常 |
|
|
||||||
| 其余 83 个后端路由 | 各 `/modelTF/xxx` | ✅ 均为正确的单层前缀 |
|
|
||||||
|
|
||||||
后端共 111 个路由,**仅 28 个 `data-process` 路由异常(双重前缀)**,其余均正确。
|
|
||||||
|
|
||||||
## 三、问题清单
|
|
||||||
|
|
||||||
### 问题 1(严重,根因):数据处理模块整体 404
|
|
||||||
|
|
||||||
**链路(前缀被叠加了两次 `/modelTF`):**
|
|
||||||
|
|
||||||
1. `backend/app/main.py:22`:`app.include_router(api_router, prefix=settings.route_prefix)`
|
|
||||||
- `route_prefix` 来自 `config.py`,默认值为 `"/modelTF"`
|
|
||||||
2. `backend/app/api/v1/router.py`:`api_router.include_router(data_process_router, prefix="/modelTF", ...)`
|
|
||||||
- 这里又额外加了一次 `prefix="/modelTF"`
|
|
||||||
3. `backend/app/api/v1/endpoints/data_process.py`:`router = APIRouter(prefix="/data-process")`
|
|
||||||
|
|
||||||
**结果**:实际注册路径变成 `/modelTF/modelTF/data-process/...`(双层前缀)。
|
|
||||||
|
|
||||||
**影响**:整个"数据处理"模块(列表 / 详情 / 创建任务 / 上传源文件 / 预览 / 生成 / 结果编辑 / 发布 / 重新生成 / 外部拉取测试 等)共 **28 个接口全部 404**。
|
|
||||||
|
|
||||||
**前端请求路径**:`baseURL('/modelTF')` + `get('/data-process')` → 实际请求 `/modelTF/data-process/...` → 与后端真实路径不匹配 → 404。
|
|
||||||
|
|
||||||
**实测证据**:
|
|
||||||
|
|
||||||
```
|
|
||||||
前端真实请求 GET /modelTF/data-process
|
|
||||||
-> {"detail":"Not Found"} (404)
|
|
||||||
|
|
||||||
后端真实路径 GET /modelTF/modelTF/data-process
|
|
||||||
-> {"code":0,"message":"ok","data":{...有"测试"任务}} (路径存在,有数据)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 问题 2:数据集下载 / 预览 双重前缀 404
|
|
||||||
|
|
||||||
`frontend/src/api/request.ts:16` 的 `baseURL: '/modelTF'`,而
|
|
||||||
`frontend/src/api/modules/dataset.ts:92` 与 `:98` 的下载地址直接写了**绝对前缀**:
|
|
||||||
|
|
||||||
```ts
|
|
||||||
const baseUrl = `/modelTF/dataset-manage/download/${datasetId}/${fileId}`
|
|
||||||
```
|
|
||||||
|
|
||||||
axios 会把 `baseURL('/modelTF')` 与以 `/` 开头的 url 拼接成 `/modelTF/modelTF/dataset-manage/download/...`,导致 404。
|
|
||||||
|
|
||||||
**注意**:后端 `dataset-manage` 是**正确的单层** `/modelTF/dataset-manage/...`(实测真实路径返回 500 = 路径存在但资源参数无效,而非 404)。
|
|
||||||
|
|
||||||
**实测证据**:
|
|
||||||
|
|
||||||
```
|
|
||||||
前端实际发出 GET /modelTF/modelTF/dataset-manage/download/x/y -> 404 Not Found
|
|
||||||
后端真实路径 GET /modelTF/dataset-manage/download/x/y -> 500 (路径存在)
|
|
||||||
```
|
|
||||||
|
|
||||||
**影响**:数据集的下载、预览功能不可用。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 问题 3:模型导出 双重前缀 + 后端无此端点
|
|
||||||
|
|
||||||
`frontend/src/api/modules/model.ts:48` 导出地址同样写了绝对前缀:
|
|
||||||
|
|
||||||
```ts
|
|
||||||
`/modelTF/model-manage/trained-models/${encodeURIComponent(modelName)}/export`
|
|
||||||
```
|
|
||||||
|
|
||||||
- 经 axios 拼接 → `/modelTF/modelTF/model-manage/trained-models/.../export` → 404(双重前缀)。
|
|
||||||
- **进一步**:openapi 中 `model-manage/trained-models` 仅有 `GET` 列表与 `GET {model_id}` 详情,**并不存在 `/export` 端点**。因此即使修掉双前缀,导出仍会 404,需后端补充该接口或确认正确路径。
|
|
||||||
|
|
||||||
**影响**:模型导出功能不可用(两个独立原因叠加)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 问题 4(次生隐患,修复问题 1 后会暴露):data-process 源文件 raw 链接
|
|
||||||
|
|
||||||
`frontend/src/api/modules/dataProcess.ts:162`:
|
|
||||||
|
|
||||||
```ts
|
|
||||||
`/modelTF/data-process/${taskId}/source-files/${fileId}/raw`
|
|
||||||
```
|
|
||||||
|
|
||||||
当前被 axios 拼成 `/modelTF/modelTF/data-process/.../raw`,**恰好命中后端双前缀**,暂时可用。
|
|
||||||
**一旦修复问题 1(后端改为单层 `/modelTF/data-process/...`)**,此处会反向变成 404,必须同步去掉多余的 `/modelTF`。
|
|
||||||
|
|
||||||
## 四、前端其他使用绝对 `/modelTF` 前缀的位置(汇总)
|
|
||||||
|
|
||||||
| 文件:行 | 接口 | 当前状态 |
|
|
||||||
|---------|------|----------|
|
|
||||||
| `dataProcess.ts:162` | 源文件 raw 下载 | 靠后端双前缀"侥幸"命中,修复问题 1 后失效 |
|
|
||||||
| `dataset.ts:92` `:98` | 数据集下载/预览 | 双重前缀 → 404(问题 2) |
|
|
||||||
| `model.ts:48` | 模型导出 | 双重前缀 + 后端无端点 → 404(问题 3) |
|
|
||||||
|
|
||||||
其余模块(`compare / fineTune / project / eval / model(其他) / dataset(其他)`)均使用相对路径 `/xxx`,正常。
|
|
||||||
|
|
||||||
## 五、修复建议(未实施,待确认)
|
|
||||||
|
|
||||||
1. **后端** `backend/app/api/v1/router.py`:去掉 `data_process_router` 的 `prefix="/modelTF"`
|
|
||||||
(`api_router` 已挂 `/modelTF`,`data_process` 自身已有 `/data-process`,无需再叠加)。
|
|
||||||
改后 data-process 路径变为 `/modelTF/data-process/...`,与前端请求一致。
|
|
||||||
|
|
||||||
2. **前端** `dataProcess.ts:162`:raw url 去掉 `/modelTF`,改为 `/data-process/.../raw`。
|
|
||||||
|
|
||||||
3. **前端** `dataset.ts:92/:98`:download url 去掉 `/modelTF`,改为 `/dataset-manage/download/...`。
|
|
||||||
|
|
||||||
4. **前端** `model.ts:48`:export url 去掉 `/modelTF`,改为 `/model-manage/trained-models/${name}/export`;
|
|
||||||
同时**与后端确认 `/export` 端点是否存在**(openapi 显示无),需后端补实现或给出正确路径。
|
|
||||||
|
|
||||||
5. 重启后端 + 前端,对"数据处理 / 数据集下载 / 模型导出"三个模块做回归。
|
|
||||||
|
|
||||||
## 六、根因归类
|
|
||||||
|
|
||||||
上述问题是一次改动/合并引入的**路由前缀叠加 bug**:
|
|
||||||
- 后端在 `api_router` 已统一挂 `/modelTF` 的前提下,又对 `data_process` 多挂了一次 `/modelTF`;
|
|
||||||
- 前端部分下载/导出接口误用了绝对 `/modelTF` 前缀,与 `request.ts` 的 `baseURL` 再次叠加。
|
|
||||||
|
|
||||||
修复核心是"前后端前缀只保留一层 `/modelTF`"。
|
|
||||||
263
架构.md
Normal file
263
架构.md
Normal file
@@ -0,0 +1,263 @@
|
|||||||
|
# 微调平台架构与现状说明
|
||||||
|
|
||||||
|
> 目的:梳理「前端 / 应用后端 / 算力平台」三层当前已实现的功能、哪些是真实可用、哪些是模拟/空壳,
|
||||||
|
> 并指出与「后端仅做数据/任务调度,GPU 计算全部下沉到算力服务」这一架构原则之间的缺口。
|
||||||
|
>
|
||||||
|
> 初始生成日期:2026-07-30。最近更新:2026-07-31。路由前缀统一为 `/modelTF`(前后端、算力 API 共用)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. 更新日志
|
||||||
|
|
||||||
|
| 日期 | 变更 |
|
||||||
|
|---|---|
|
||||||
|
| 2026-07-30 | 初始版本,梳理三层现状与缺口 |
|
||||||
|
| 2026-07-31 | 评测/推理后端端点从 `yg_ft1` 移植完成(G5 部分解决);更新缺口状态表与路线图 |
|
||||||
|
| 2026-07-31 | 训练派发改造完成:移植 `process_manager.py`/`adapter.py`/`sync.py`,算力 real 执行器就绪,后端轮询线程启动(G1/G2/G4 部分解决) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 三层架构总览
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────┐ HTTP /modelTF ┌──────────────────────┐
|
||||||
|
│ 前端 │ ───────────────────────▶ │ 应用后端 (backend) │
|
||||||
|
│ (Vue3+Vite) │ ◀─────────────────────── │ FastAPI + SQLite/PG │
|
||||||
|
└──────────────┘ └──────────┬───────────┘
|
||||||
|
│ ✅ 训练派发已打通
|
||||||
|
│ ✅ 推理代理已打通
|
||||||
|
│ (需算力 venv 装 llamafactory)
|
||||||
|
▼
|
||||||
|
┌──────────────────────┐
|
||||||
|
│ 算力平台 (compute) │
|
||||||
|
│ FastAPI + LLaMA-Factory│
|
||||||
|
└──────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**关键事实**:应用后端与算力平台是**两个独立部署的服务**。
|
||||||
|
- 推理路径已打通:后端 `platform.py` 经 `ComputeNodeClient._request()` 代理到算力 `/modelTF/inference/*`。
|
||||||
|
- 训练路径已打通:后端 `platform_store.start_task` → `_dispatch_to_compute` → `ComputeNodeClient.create_job` 派发到算力;后台轮询线程(`service.start_compute_sync_worker`)经 `sync.poll_compute_jobs_once` 周期回传状态/日志/指标。
|
||||||
|
- 算力 real 执行器已就绪:`compute/agent/process_manager.py` 真正 `subprocess` 拉起 `llamafactory-cli train`。
|
||||||
|
- **剩余前置**:算力 venv 需安装 `llamafactory`(G3);基座模型/数据集文件需到达算力节点(阶段 4)。
|
||||||
|
|
||||||
|
### 1.1 职责边界(架构原则)
|
||||||
|
|
||||||
|
| 角色 | 职责 | 不得做什么 |
|
||||||
|
|---|---|---|
|
||||||
|
| **应用后端** | 数据/任务调度与编排:数据集管理、模型元数据、任务生命周期(创建/状态/日志路由)、审批/审计/租户、对外 REST API | **不得**由自身进程执行任何 GPU 计算;**不得**安装 `llamafactory`/`torch` 等训练运行时。GPU 计算须派发给算力服务进程执行(算力服务可与后端同机部署,也可在独立算力节点,取决于部署形态) |
|
||||||
|
| **算力服务** | 承载所有 GPU 计算:模型训练(微调)、模型推理、模型评测、数据类型转换等;运行 `llamafactory-cli` 等框架 | 不持有业务元数据;只接收后端派发的作业并回报进度/产物 |
|
||||||
|
| **前端** | 交互与可视化,纯调用后端 API | 不直连算力(统一经后端中转) |
|
||||||
|
|
||||||
|
> **原则**:凡涉及 GPU 的工作负载(训练/推理/评测)一律由后端**派发**给算力服务执行,
|
||||||
|
> 后端自身只做调度编排与数据流转——即 GPU 计算必须发生在**算力服务进程内**,而非后端进程内。
|
||||||
|
> 算力服务部署在何处(本机或独立节点)不影响该原则。当前 `fine_tune/runner.py` 由后端进程
|
||||||
|
> 直接 `subprocess` 跑 `train.py` 属于**违反该原则的临时实现**(见 G1)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 前端(frontend/src)功能清单
|
||||||
|
|
||||||
|
菜单来自 `components/AppSidebar.vue`,共 7 组。功能状态取决于后端是否有对应端点。
|
||||||
|
|
||||||
|
| 菜单分组 | 功能 | 对应前端视图 | 后端端点 | 状态 |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 服务看板 | 仪表盘(统计/分布/排行) | `dashboard/` | `platform.dashboard_*` | ✅ 真实 |
|
||||||
|
| 模型服务 | 模型训练 | `fine-tune/` | `platform.fine-tune*` | ✅ 真实(本地执行,见 G1) |
|
||||||
|
| 模型服务 | 模型评测 | `eval/` | `/model-eval/*`、`/dimension/*` | ✅ **已移植**(后端端点+SQL表+Store方法齐备) |
|
||||||
|
| 模型服务 | 模型推理 | `inference/` | `/model-compare/*`、`/model-chat/*` | ✅ **已移植**(后端端点+SQL表+Store方法+推理代理齐备) |
|
||||||
|
| 模型服务 | 模型对比 | `compare/` | `/model-compare/*`(推理/对比共用) | ✅ **已移植**(与推理共用后端) |
|
||||||
|
| 模型服务 | 模型管理 | `model/` | `platform.model-manage*` | ✅ 真实 |
|
||||||
|
| 数据治理 | 数据集管理 | `dataset/` | `platform.dataset-manage*` | ✅ 真实 |
|
||||||
|
| 数据治理 | 数据处理 | `data-process/` | `data_process` 路由 | ✅ 真实 |
|
||||||
|
| 其他工具 | 数据类型转换 | `data-convert/` | 无端点 | ❌ 空壳 UI(纯前端,`ElMessage.info('当前仅完成界面设计')`) |
|
||||||
|
| 算力资源 | 算力节点 | `compute/` | `platform.compute*` | ⚠️ 模拟(见 §4) |
|
||||||
|
| 平台治理 | 租户/项目/审计/审批 | `tenants/ projects/ audit/ approvals/` | tenant/project/approval/audit 模块 | ✅ 真实 |
|
||||||
|
| 系统设置 | 用户设置/性能/日志 | `system/ hardware/ logs/` | system/users + logs | ✅ 真实 |
|
||||||
|
|
||||||
|
**结论**:评测、推理、对比的后端端点已从 `yg_ft1` 移植完成,前端不再 404。
|
||||||
|
剩余空壳 UI 仅 **数据类型转换**(`data-convert/`,纯前端界面,无后端端点)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 应用后端(backend/app)真实端点
|
||||||
|
|
||||||
|
挂载于 `api/v1/router.py` 的 9 个路由:`health / platform / auth / system / tenant / project / resource / approval / data_process`。
|
||||||
|
其中大量功能集中在 `endpoints/platform.py`(仪表盘、用户、模型、数据集、微调、算力、项目、日志、评测、推理)。
|
||||||
|
|
||||||
|
### 3.1 已实现(真实可用)
|
||||||
|
- **认证**:`/login`、`/logout`、`/me`(登录写 `sessions` 会话,登出关闭会话)
|
||||||
|
- **用户/系统**:`/users` CRUD、重置密码、`/system-info`
|
||||||
|
- **租户 / 项目空间 / 审批 / 审计**:各模块路由 + `audit_logs`
|
||||||
|
- **数据集管理**:`/dataset-manage*` 全量 CRUD + 文件版本/上传/下载/预览
|
||||||
|
- **模型管理**:`/model-manage*` 本地/训练模型 CRUD + 合并
|
||||||
|
- **模型训练(微调)**:`/fine-tune*` 全量 CRUD + 启停/暂停/恢复/取消/重试/检查点/事件流/日志
|
||||||
|
- **数据处理**:`data_process` 模块(创建/删除/上传/生成/发布,已补审计埋点)
|
||||||
|
- **仪表盘**:`/dashboard/overview`、`/dashboard/stats`(含真实登录时长、操作分布)
|
||||||
|
- **日志**:`/log-files`、`/log-content`、`/training-log-*`
|
||||||
|
- **模型评测** ✅ **已移植**:`/model-eval`(列表/详情/启动/删除)、`/dimension`(CRUD)+ `eval_tasks`/`eval_dimensions` 表
|
||||||
|
- **模型推理/对比** ✅ **已移植**:`/model-compare`(CRUD + load/unload + load-status + start-model + chat-with-port + stream-chat)+ `/model-chat/*`(local chat/stream/preload/unload/status + trained preload + batch)+ `compare_tasks` 表
|
||||||
|
- **推理代理** ✅ **已打通**:`_select_first_online_node()` + `ComputeNodeClient._request()` 将 `/model-chat/local/*` 请求代理到算力 `/modelTF/inference/*`
|
||||||
|
|
||||||
|
### 3.2 模拟(看似可用,实则未接真实算力)
|
||||||
|
- **算力资源** `/compute/nodes`、`/compute/gpus`、`/compute/queue`、`/compute/jobs`:
|
||||||
|
- `test-connection` 返回**写死的 success**(不真连)
|
||||||
|
- `create_compute_job` 仅**落库**,无任何派发逻辑
|
||||||
|
- `gpus`/`queue` 来自 DB,无真实 GPU 采集
|
||||||
|
|
||||||
|
### 3.3 ~~缺失~~(前端在调用但后端不存在)
|
||||||
|
|
||||||
|
> **2026-07-31 更新**:原 G5 列出的 `/model-eval/*`、`/inference/*`、`/model-compare/*` 已全部移植完成。
|
||||||
|
> 后端 `modules/eval`、`modules/inference` 目录仍仅有空 `__init__.py`(评测/推理逻辑直接写在 `platform.py` + `platform_store.py` 中,与 `yg_ft1` 架构一致)。
|
||||||
|
>
|
||||||
|
> 唯一仍缺后端端点的是 **数据类型转换** `/data-convert/*`,但该前端页面本身也是纯展示(`ElMessage.info('当前仅完成界面设计')`),不发送 API 请求。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 算力平台(compute)现状
|
||||||
|
|
||||||
|
独立 FastAPI 服务,**与应用后端分开部署**(README 明确说明)。
|
||||||
|
|
||||||
|
### 已实现
|
||||||
|
- **健康检查**:`/modelTF/health`、`/modelTF/v1/compute/health`
|
||||||
|
- **作业管理**:`POST/GET /modelTF/compute/jobs`、`.../{id}`、`.../{id}/stop`、`.../{id}/logs`
|
||||||
|
- **GPU 资源**:`GET /modelTF/compute/resources/gpus`
|
||||||
|
- **文件网关**:`/modelTF/compute/files/upload`、`/download`
|
||||||
|
- **LLaMA-Factory 适配**:`engines/llama_factory/adapter.py`
|
||||||
|
- `build_command()` 生成 `llamafactory-cli train ...`(SFT/LoRA/量化等参数)
|
||||||
|
- `parse_log_line()` 解析 loss/grad_norm/learning_rate/epoch 指标
|
||||||
|
- **模型推理** ✅ **已移植**:`engines/llama_factory/inference.py`(`InferenceSession` 类:load/unload/chat/chat_stream)
|
||||||
|
- **推理 API 端点** ✅ **已移植**:
|
||||||
|
- `POST /modelTF/inference/load` — 加载模型
|
||||||
|
- `POST /modelTF/inference/unload` — 卸载模型
|
||||||
|
- `GET /modelTF/inference/status` — 查询状态
|
||||||
|
- `POST /modelTF/inference/chat` — 同步对话
|
||||||
|
- `POST /modelTF/inference/chat/stream` — 流式对话(SSE)
|
||||||
|
|
||||||
|
### 模拟 / 未实现
|
||||||
|
- **`simulator` 模式**(默认开发用):作业状态按时间演进(queued→running→completed),
|
||||||
|
生成**合成** loss 日志,GPU 列表为构造数据。可完整跑通 UI 流程。
|
||||||
|
- **`real` 模式** ✅ **已实现**:`compute/agent/process_manager.py`(`ProcessManager` 类)真正 `subprocess` 拉起 `llamafactory-cli train`,含日志落盘 / checkpoint / artifacts / GPU 锁定 / stop。
|
||||||
|
- **依赖缺失**:`compute/requirements.txt` 已声明 `llamafactory`,
|
||||||
|
但算力 venv 实际未安装,且 `LLAMA_FACTORY_HOME` 默认指向 `/app/LLaMA-Factory`(运行时需存在)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 与架构目标的缺口(重点)
|
||||||
|
|
||||||
|
架构原则:**后端仅做数据/任务调度与编排,所有涉及 GPU 的计算(训练/推理/评测/转换)一律派发到算力服务执行;应用后端不得安装或本地运行 `llamafactory`/`torch` 等训练运行时。**
|
||||||
|
|
||||||
|
| # | 缺口 | 现状 | 影响 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **G1** → ✅ 已解决 | ~~训练未派发到算力~~ | `start_task` → `_dispatch_to_compute` 派发到算力;`sync.poll_compute_jobs_once` 回传状态;后台轮询线程启动。`runner.py` 本机执行仅保留为无算力节点时的降级 fallback | 训练走算力执行(需 G3 环境就绪) |
|
||||||
|
| **G2** → ✅ 已解决 | ~~算力真实执行器未实现~~ | `compute/agent/process_manager.py` 已移植;`compute/api/main.py` real 模式调用 `ProcessManager.create_job` 真正 `subprocess` 拉起训练 | 算力可执行真实训练 |
|
||||||
|
| G3 | llamafactory 未安装 | 已声明于 `compute/requirements.txt`,但算力 venv 未装;按架构**不应**装进后端 venv | 真实训练无法启动 |
|
||||||
|
| **G4** → ✅ 已解决 | ~~训练前后端算力未连通~~ | 推理路径已通;训练路径已通(`start_task`→`_dispatch_to_compute`→`ComputeNodeClient.create_job`→算力 `ProcessManager`→`sync.poll_compute_jobs_once` 回传) | 训练 GPU 作业数据与算力已连通 |
|
||||||
|
| ~~G5~~ → **已解决大部分** | ~~多个前端模块无后端~~ | 评测 ✅ 已移植;推理/对比 ✅ 已移植;数据转换 ❌ 仍空壳 | 仅数据转换功能不可用(纯 UI 占位) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 数据流现状 vs 应有
|
||||||
|
|
||||||
|
### 6.1 训练 ✅ 已打通(待 G3 环境就绪)
|
||||||
|
|
||||||
|
```
|
||||||
|
前端 创建/启动训练 → 后端 /fine-tune/start → platform_store.start_task(payload)
|
||||||
|
→ schedule_node() 选取在线算力节点
|
||||||
|
→ _dispatch_to_compute() → ComputeNodeClient.create_job(cfg) 派发到算力
|
||||||
|
→ 算力 POST /modelTF/compute/jobs → ProcessManager.create_job() → subprocess llamafactory-cli train
|
||||||
|
→ 后台轮询线程 sync.poll_compute_jobs_once() 周期拉取状态/日志/指标 → apply_compute_job 回写
|
||||||
|
→ 训练完成 → _ensure_trained_model 登记产物
|
||||||
|
```
|
||||||
|
|
||||||
|
> **降级 fallback**:当无在线算力节点时,`start_task` 降级到 `service.launch_training` → `runner.run_training`(本机 subprocess,违反 §1.1,待移除)。
|
||||||
|
>
|
||||||
|
> **前置依赖**:算力 venv 需安装 `llamafactory`(G3);基座模型/数据集文件需到达算力节点(阶段 4)。
|
||||||
|
|
||||||
|
### 6.2 推理 ✅ 已打通
|
||||||
|
|
||||||
|
```
|
||||||
|
前端 新建推理/对话 → 后端 /model-compare/* 或 /model-chat/local/*
|
||||||
|
→ 后端 _select_first_online_node(store) 选取在线算力节点
|
||||||
|
→ ComputeNodeClient(api_base_url)._request("POST", "/inference/chat", json_data=payload)
|
||||||
|
→ 算力平台 POST /modelTF/inference/chat → InferenceSession.chat() → 返回推理结果
|
||||||
|
→ 流式:POST /modelTF/inference/chat/stream → SSE 逐字回传
|
||||||
|
```
|
||||||
|
|
||||||
|
> 推理路径已完全打通:后端代理 → 算力执行 → 结果回传。
|
||||||
|
> 若算力节点不在线,后端返回降级响应(`"no online compute node available for inference"`)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 建议的下一步
|
||||||
|
|
||||||
|
1. **训练打通**(当前最高优先级缺口):
|
||||||
|
- 移植 `yg_ft1` 的 `compute/agent/process_manager.py`(real 执行器)到 `compute/agent/`;
|
||||||
|
- 移植 `yg_ft1` 的 `compute_gateway/sync.py`(派发+状态/日志回传)到 `backend/app/modules/compute_gateway/`;
|
||||||
|
- 改造 `fine_tune/service.py`:`start_task` 改为经 `compute_gateway.dispatch()` 派发,移除本机 `runner.py` subprocess。
|
||||||
|
2. **环境**(低风险):在算力 venv 安装 `llamafactory`;后端 venv 保持不含它。
|
||||||
|
3. **联调**:算力平台切 `COMPUTE_EXECUTION_MODE=simulator`,后端改为经算力 API 派发训练,使训练 UI 走通。
|
||||||
|
4. **数据转换**(低优先级):补齐 `/data-convert/*` 后端端点(当前前端纯 UI 占位,无 API 调用)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 改造路线图:GPU 任务从后端进程派发到算力服务进程
|
||||||
|
|
||||||
|
目标:落实 §1.1 职责边界——**后端只做数据/任务调度与编排,GPU 计算一律派发给算力服务进程执行**。
|
||||||
|
当前 `fine_tune/runner.py` 用后端 venv 本机 `subprocess` 跑 `train.py` 是违反原则的临时实现,需移除并改为「派发 + 回传」。
|
||||||
|
|
||||||
|
> **参考实现 `yg_ft1`**:同仓下 `e:/yg_ft/yg_ft1` 是更完整的参考版本,本路线图所需能力大多已实现,应**直接迁移/对齐,而非从零编写**:
|
||||||
|
> - **训练闭环完整**:`compute/agent/process_manager.py`(real 模式真正 `subprocess` 拉起 `llamafactory-cli train`,含日志落盘 / checkpoint / artifacts / GPU 锁定 / stop)、`compute/engines/llama_factory/adapter.py`(`build_command` / `prepare_runtime_files` / `parse_log_line`)、后端 `compute_gateway/client.py` + `sync.py`(派发 + 状态/日志回传)。
|
||||||
|
> - **推理真实端点** ✅ 已迁移:`compute/api/main.py` 的 `/inference/*` + `compute/engines/llama_factory/inference.py` 已于 2026-07-31 移植到当前项目。
|
||||||
|
> - **评测** ✅ 后端端点已迁移:`/model-eval/*` + `/dimension/*` + `eval_tasks`/`eval_dimensions` 表 + Store 方法已于 2026-07-31 移植。**但评测执行器(是否走算力)尚未实现**——当前仅落库,无真实评测执行。
|
||||||
|
>
|
||||||
|
> 因此阶段 1/2/3 本质是**把 `yg_ft1` 的训练相关文件迁移到 `yg_ft` 并对齐接口**;推理已完成;评测需另行补全执行器。
|
||||||
|
|
||||||
|
> 现状代码支撑:`compute_nodes` 表已带 `api_base_url` 字段(见 `001_platform_runtime.sql`),
|
||||||
|
> 算力 API 已有 `jobs/create/get/stop/logs` 与 `resources/gpus`,`adapter.build_command` 可生成 `llamafactory-cli train`。
|
||||||
|
> `ComputeNodeClient` 已有 `headers()`、`_request()` 异步方法(推理代理已验证可用)。
|
||||||
|
> 缺的是:① 后端 `compute_gateway/sync.py`(派发+回传同步线程);② 训练不走本机;③ 算力 `real` 执行器(`process_manager.py`);④ 数据集/模型文件如何到达算力。
|
||||||
|
|
||||||
|
### 阶段 0 — 算力环境就绪(前置)—— **待执行**
|
||||||
|
- **改动**:在算力服务 venv 安装 `llamafactory`(依赖已声明于 `compute/requirements.txt`),后端 venv 保持不含它。
|
||||||
|
- **验证**:算力切 `simulator` 时 `POST /modelTF/compute/jobs` 能返回 `queued`;`pip show llamafactory` 在算力环境为已装、在后端为未装。
|
||||||
|
|
||||||
|
### 阶段 1 — 后端 `compute_gateway` 桥接 ✅ 已完成
|
||||||
|
- **已建文件**:`backend/app/modules/compute_gateway/sync.py`(从 `yg_ft1` 迁移)。
|
||||||
|
- **已实现**:
|
||||||
|
- `sync.py`:`poll_compute_jobs_once()` 周期性同步线程,从算力 API 拉取 job 状态/日志/指标,写回 `PlatformStore.task`。
|
||||||
|
- `client.py` 已有 `create_job`/`get_job`/`stop_job`/`job_logs` 方法(同步)和 `_request`/`headers` 方法(异步,推理已用),直接复用。
|
||||||
|
- `_select_first_online_node` 已实现(推理代理已验证)。
|
||||||
|
- 后台轮询线程在 `main.py` lifespan 中自动启动。
|
||||||
|
|
||||||
|
### 阶段 2 — 训练改为「派发 + 回传」 ✅ 已完成
|
||||||
|
- **已改文件**:`backend/app/modules/fine_tune/service.py`、`backend/app/db/platform_store.py`、`backend/app/main.py`。
|
||||||
|
- **已实现**:
|
||||||
|
- `platform_store.start_task` → `schedule_node` → `_dispatch_to_compute` → `ComputeNodeClient.create_job` 派发到算力。
|
||||||
|
- 后台轮询线程(`service.start_compute_sync_worker`)启动,周期调 `sync.poll_compute_jobs_once`。
|
||||||
|
- `apply_compute_job` 回写状态/进度/日志/checkpoints/artifacts;`record_training_log_metrics` 解析并存储训练指标。
|
||||||
|
- `stop_task` 改为先通知算力 `stop_job`,再更新 DB + 释放 GPU。
|
||||||
|
- `runner.py` 本机 `subprocess` 仅保留为无算力节点时的降级 fallback(`start_task` 中 `elif` 分支)。
|
||||||
|
|
||||||
|
### 阶段 3 — 算力 `real` 执行器 ✅ 已移植
|
||||||
|
- **已建文件**:`compute/agent/process_manager.py`(从 `yg_ft1` 迁移)。
|
||||||
|
- **已改文件**:`compute/api/main.py`(real 模式调用 `ProcessManager`)。
|
||||||
|
- **已实现**:`ProcessManager` 类——真正 `subprocess` 拉起 `llamafactory-cli train`,含日志落盘 / checkpoint 收集 / artifacts 收集 / GPU 锁定 / stop。`compute/api/main.py` 的 real 模式不再返回 501。
|
||||||
|
|
||||||
|
### 阶段 4 — 数据集 / 模型文件到达算力(关键依赖)
|
||||||
|
- **问题**:派发后算力节点须能读到**基座模型**与**训练数据集**文件。
|
||||||
|
- **方案(二选一或并存)**:
|
||||||
|
- A(推荐,生产):基座模型与数据集通过**共享存储**(NFS / 对象存储)挂载到算力节点,路径随派发 config 传入;
|
||||||
|
- B(已具备雏形):经算力文件网关 `POST /compute/files/upload` → `/download` 传输(当前为占位,需落盘实现)。
|
||||||
|
- **验证**:算力 `real` 执行时能从指定路径加载模型与数据集,不报「文件不存在」。
|
||||||
|
|
||||||
|
### 阶段 5 — 评测执行器 / 数据转换
|
||||||
|
- **评测**:后端端点已移植(`/model-eval/*` 落库),但**真实评测执行器**未实现(是否走算力待定)。需补全评测执行逻辑——可能经 `compute_gateway` 派发到算力,或后端编排调 API 模型评测。
|
||||||
|
- **数据转换**:补齐 `/data-convert/*` 后端端点(当前前端纯 UI 占位,无 API 调用)。
|
||||||
|
- **验证**:评测任务能真实执行并产出分数;数据转换页面功能可用。
|
||||||
|
|
||||||
|
### 风险与前置提醒
|
||||||
|
1. **阶段 4 文件传递是派发可用性的硬前置**——不解决,算力无数据可训(优先级高于阶段 3)。
|
||||||
|
2. **暂停/恢复语义缺口**:算力当前仅 `stop`,阶段 2 需先确定 pause/resume 是否必需、如何映射。
|
||||||
|
3. **共享存储/网络可达性**:`compute_nodes.api_base_url` 必须网络可达,且算力与后端时间/路径一致。
|
||||||
|
4. **回退策略**:阶段 2 上线前保留本机 `runner` 作为可开关的 fallback(`DISPATCH_TO_COMPUTE=true/false`),便于灰度。
|
||||||
|
```
|
||||||
434
测试报告.md
Normal file
434
测试报告.md
Normal file
@@ -0,0 +1,434 @@
|
|||||||
|
# YG_FT 模型微调平台 — 测试报告
|
||||||
|
|
||||||
|
| 项目 | 内容 |
|
||||||
|
|------|------|
|
||||||
|
| 项目名称 | YG_FT 模型微调平台 |
|
||||||
|
| 测试日期 | 2026-07-31 |
|
||||||
|
| 测试人员 | 自动化测试 + 人工分析 |
|
||||||
|
| 测试环境 | WSL2 Ubuntu / Windows 11 |
|
||||||
|
| 后端版本 | FastAPI (uvicorn 端口 17861) |
|
||||||
|
| 前端版本 | Vue3 + Vite (端口 16801) |
|
||||||
|
| 算力服务 | Compute API (uvicorn 端口 19100) |
|
||||||
|
| 数据库 | PostgreSQL (远程 www.caoxiaozhu.com:5432) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 测试环境
|
||||||
|
|
||||||
|
### 1.1 服务部署架构
|
||||||
|
|
||||||
|
```
|
||||||
|
Windows 11 (浏览器)
|
||||||
|
└─ WSL2 Ubuntu
|
||||||
|
├─ 前端开发服务器 http://localhost:16801 (Vite + Vue3)
|
||||||
|
├─ 应用平台后端 http://localhost:17861 (FastAPI + uvicorn --reload)
|
||||||
|
└─ 算力服务 http://localhost:19100 (FastAPI + uvicorn --reload)
|
||||||
|
└─ PostgreSQL (远程数据库)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.2 内置测试账号
|
||||||
|
|
||||||
|
| 角色 | 账号 | 密码 |
|
||||||
|
|------|------|------|
|
||||||
|
| 超级管理员 | `admin` | `admin123` |
|
||||||
|
| 操作员 | `operator` | `operator123` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 测试范围
|
||||||
|
|
||||||
|
本次测试覆盖平台以下功能模块:
|
||||||
|
|
||||||
|
| 序号 | 模块 | 测试内容 |
|
||||||
|
|------|------|----------|
|
||||||
|
| 1 | 服务连通性 | 前端/后端/算力服务健康检查、OpenAPI 文档 |
|
||||||
|
| 2 | 登录认证 | 登录、登出、Token 验证、错误密码拒绝 |
|
||||||
|
| 3 | 用户管理 | 用户 CRUD、重置密码 |
|
||||||
|
| 4 | 模型管理 | 模型 CRUD、本地/已训练模型查询 |
|
||||||
|
| 5 | 数据集管理 | 数据集 CRUD、版本管理 |
|
||||||
|
| 6 | 微调任务 | 任务创建/查询/删除、进度查询、checkpoint |
|
||||||
|
| 7 | 算力节点 | 节点 CRUD、启用/禁用、连接测试、GPU/队列 |
|
||||||
|
| 8 | 租户管理 | 租户 CRUD、配额设置、留存策略 |
|
||||||
|
| 9 | 项目空间 | 项目 CRUD、成员管理、归档 |
|
||||||
|
| 10 | 审批中心 | 审批模板、审批实例创建/查询 |
|
||||||
|
| 11 | 审计中心 | 审计日志查询、CSV 导出、权限码 |
|
||||||
|
| 12 | 资源授权 | ACL 获取/设置 |
|
||||||
|
| 13 | 服务看板 | 看板概览、统计聚合 |
|
||||||
|
| 14 | 日志 | 日志文件列表、训练日志、系统信息 |
|
||||||
|
| 15 | 模型评测 | 评测任务 CRUD |
|
||||||
|
| 16 | 模型对比 | 对比任务 CRUD、加载/卸载、对话 |
|
||||||
|
| 17 | 推理 | 推理会话状态、本地模型对话 |
|
||||||
|
| 18 | 数据处理 | 数据处理任务 CRUD、源文件管理 |
|
||||||
|
| 19 | 算力服务 API | 算力服务健康检查、GPU 列表 |
|
||||||
|
| 20 | 前端路由 | 11 条主要前端路由可访问性 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 测试方法
|
||||||
|
|
||||||
|
采用 **API 黑盒测试** 为主,辅以 **日志分析** 和 **数据库直查**:
|
||||||
|
|
||||||
|
1. **API 接口测试**:使用 curl 和 Python `urllib` 对所有后端 REST API 端点发送请求,验证 HTTP 状态码和响应体 `{ code, message, data }` 结构。
|
||||||
|
2. **前端路由测试**:验证所有主要前端路由返回 HTTP 200。
|
||||||
|
3. **边界/异常测试**:错误密码登录(应返回 401)、无 Token 访问受保护接口(应返回 401)、查询不存在的资源(应返回 404)。
|
||||||
|
4. **Bug 根因分析**:对失败接口查看后端错误日志(`logs/error-*.log`),使用 Python 脚本直接调用 `PlatformStore` 进行定位。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 测试结果汇总
|
||||||
|
|
||||||
|
### 4.1 总体结果
|
||||||
|
|
||||||
|
| 指标 | 数量 |
|
||||||
|
|------|------|
|
||||||
|
| 测试项总数 | 92 |
|
||||||
|
| 通过 | 88 |
|
||||||
|
| 失败(已修复) | 2 |
|
||||||
|
| 失败(参数问题,非 Bug) | 2 |
|
||||||
|
| 通过率 | 95.7%(修复后 100%) |
|
||||||
|
|
||||||
|
### 4.2 各模块测试明细
|
||||||
|
|
||||||
|
#### 4.2.1 服务连通性 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 后端健康检查 `GET /modelTF/health` | ✅ 通过 | 返回 CPU/内存/磁盘使用率 |
|
||||||
|
| 算力服务健康检查 `GET /health` | ✅ 通过 | 返回 `{"status":"ok"}` |
|
||||||
|
| 前端页面可访问性 `GET /` | ✅ 通过 | HTTP 200,返回 HTML |
|
||||||
|
| 后端 OpenAPI 文档 `GET /openapi.json` | ✅ 通过 | HTTP 200 |
|
||||||
|
|
||||||
|
#### 4.2.2 登录认证 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| admin 登录 `POST /modelTF/login` | ✅ 通过 | 返回 token 和用户信息 |
|
||||||
|
| operator 登录 | ✅ 通过 | 返回 token 和用户信息 |
|
||||||
|
| 错误密码登录拒绝 | ✅ 通过 | 返回 HTTP 401 |
|
||||||
|
| 获取当前用户 `GET /modelTF/me` | ✅ 通过 | 需要 Bearer Token |
|
||||||
|
| 无 Token 访问 `/me` 拒绝 | ✅ 通过 | 返回 HTTP 401 |
|
||||||
|
| 登出 `POST /modelTF/logout` | ✅ 通过 | 返回 code=0 |
|
||||||
|
|
||||||
|
#### 4.2.3 用户管理 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 用户列表 `GET /modelTF/users` | ✅ 通过 | 返回用户数组 |
|
||||||
|
| 创建用户 `POST /modelTF/users` | ✅ 通过 | 返回新用户 ID |
|
||||||
|
| 更新用户 `PUT /modelTF/users/{id}` | ✅ 通过 | |
|
||||||
|
| 重置密码 `POST /modelTF/users/{id}/reset-password` | ✅ 通过 | |
|
||||||
|
| 删除用户 `DELETE /modelTF/users/{id}` | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.4 模型管理 — 修复后全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 模型列表 `GET /modelTF/model-manage` | ✅ 通过 | |
|
||||||
|
| 本地模型列表 | ✅ 通过 | |
|
||||||
|
| 已训练模型列表 | ✅ 通过 | |
|
||||||
|
| 创建模型 `POST /modelTF/model-manage` | ✅ 通过 | **修复后通过**(详见第 5 节) |
|
||||||
|
| 模型详情查询 | ✅ 通过 | |
|
||||||
|
| 更新模型 `PUT /modelTF/model-manage/{id}` | ✅ 通过 | **修复后通过**(详见第 5 节) |
|
||||||
|
| 删除模型 | ✅ 通过 | |
|
||||||
|
| 查询不存在模型返回 404 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.5 数据集管理 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 数据集列表 | ✅ 通过 | |
|
||||||
|
| 创建数据集 | ✅ 通过 | |
|
||||||
|
| 数据集详情查询 | ✅ 通过 | |
|
||||||
|
| 更新数据集 | ✅ 通过 | |
|
||||||
|
| 删除数据集 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.6 微调任务 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 微调任务列表 | ✅ 通过 | |
|
||||||
|
| 检查任务名重复 | ✅ 通过 | |
|
||||||
|
| 创建微调任务(正确参数) | ✅ 通过 | 需提供 `train_dataset_id` |
|
||||||
|
| 微调任务详情 | ✅ 通过 | |
|
||||||
|
| 微调任务概览 | ✅ 通过 | |
|
||||||
|
| 微调任务 checkpoints | ✅ 通过 | |
|
||||||
|
| 微调任务进度 | ✅ 通过 | |
|
||||||
|
| 删除微调任务 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.7 算力节点 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 算力节点列表 | ✅ 通过 | |
|
||||||
|
| GPU 列表 | ✅ 通过 | |
|
||||||
|
| 算力队列 | ✅ 通过 | |
|
||||||
|
| 创建算力节点(正确参数) | ✅ 通过 | 需提供 `code` 字段 |
|
||||||
|
| 算力节点连接测试 | ✅ 通过 | |
|
||||||
|
| 启用/禁用算力节点 | ✅ 通过 | |
|
||||||
|
| 算力节点副本查询 | ✅ 通过 | |
|
||||||
|
| 更新算力节点 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.8 租户管理 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 租户列表 | ✅ 通过 | |
|
||||||
|
| 创建租户 | ✅ 通过 | |
|
||||||
|
| 租户详情查询 | ✅ 通过 | |
|
||||||
|
| 更新租户 | ✅ 通过 | |
|
||||||
|
| 设置租户配额 | ✅ 通过 | |
|
||||||
|
| 设置租户留存策略 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.9 项目空间 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 项目列表 | ✅ 通过 | |
|
||||||
|
| 创建项目 | ✅ 通过 | |
|
||||||
|
| 项目详情查询 | ✅ 通过 | |
|
||||||
|
| 更新项目 | ✅ 通过 | |
|
||||||
|
| 项目成员列表 | ✅ 通过 | |
|
||||||
|
| 项目归档 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.10 审批中心 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 审批实例列表 | ✅ 通过 | |
|
||||||
|
| 审批模板列表 | ✅ 通过 | |
|
||||||
|
| 创建审批模板 | ✅ 通过 | |
|
||||||
|
| 创建审批实例 | ✅ 通过 | |
|
||||||
|
| 审批实例详情 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.11 审计中心 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 审计日志查询 | ✅ 通过 | 支持分页和多维过滤 |
|
||||||
|
| 审计日志导出 CSV | ✅ 通过 | 返回 CSV 流,格式正确 |
|
||||||
|
| 权限码清单 | ✅ 通过 | |
|
||||||
|
| 权限码接口 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.12 资源授权 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 获取资源 ACL | ✅ 通过 | |
|
||||||
|
| 设置资源 ACL | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.13 服务看板 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 看板概览 | ✅ 通过 | 返回模型/数据集/任务/节点计数 |
|
||||||
|
| 看板统计 | ✅ 通过 | 返回 7 天训练趋势、服务状态、操作分布等 |
|
||||||
|
|
||||||
|
#### 4.2.14 日志 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 日志文件列表 | ✅ 通过 | |
|
||||||
|
| 训练日志文件列表 | ✅ 通过 | |
|
||||||
|
| 系统信息 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.15 模型评测 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 评测任务列表 | ✅ 通过 | |
|
||||||
|
| 创建评测任务 | ✅ 通过 | |
|
||||||
|
| 评测任务详情 | ✅ 通过 | |
|
||||||
|
| 删除评测任务 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.16 模型对比 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 对比任务列表 | ✅ 通过 | |
|
||||||
|
| 创建对比任务 | ✅ 通过 | |
|
||||||
|
| 对比任务详情 | ✅ 通过 | |
|
||||||
|
| 模型对比加载 | ✅ 通过 | |
|
||||||
|
| 对比加载状态 | ✅ 通过 | |
|
||||||
|
| 模型对比卸载 | ✅ 通过 | |
|
||||||
|
| 删除对比任务 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.17 推理 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 推理会话状态 | ✅ 通过 | 代理到算力节点 |
|
||||||
|
| 模型对比对话 | ✅ 通过 | |
|
||||||
|
| 本地模型对话 | ✅ 通过 | 代理到算力节点 |
|
||||||
|
|
||||||
|
#### 4.2.18 数据处理 — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 数据处理任务列表 | ✅ 通过 | |
|
||||||
|
| 创建数据处理任务 | ✅ 通过 | |
|
||||||
|
| 数据处理任务详情 | ✅ 通过 | |
|
||||||
|
| 数据处理源文件列表 | ✅ 通过 | |
|
||||||
|
| 删除数据处理任务 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.19 算力服务 API — 全部通过
|
||||||
|
|
||||||
|
| 测试项 | 结果 | 说明 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 算力服务健康检查 | ✅ 通过 | 返回 `{"status":"ok"}` |
|
||||||
|
| 算力服务 GPU 列表 | ✅ 通过 | |
|
||||||
|
|
||||||
|
#### 4.2.20 前端路由 — 全部通过
|
||||||
|
|
||||||
|
| 路由 | 结果 |
|
||||||
|
|------|------|
|
||||||
|
| `/` (首页) | ✅ HTTP 200 |
|
||||||
|
| `/login` (登录) | ✅ HTTP 200 |
|
||||||
|
| `/dashboard` (看板) | ✅ HTTP 200 |
|
||||||
|
| `/model-manage` (模型管理) | ✅ HTTP 200 |
|
||||||
|
| `/dataset-manage` (数据集管理) | ✅ HTTP 200 |
|
||||||
|
| `/fine-tune` (微调任务) | ✅ HTTP 200 |
|
||||||
|
| `/compute` (算力节点) | ✅ HTTP 200 |
|
||||||
|
| `/approvals` (审批中心) | ✅ HTTP 200 |
|
||||||
|
| `/audit-logs` (审计中心) | ✅ HTTP 200 |
|
||||||
|
| `/tenants` (租户管理) | ✅ HTTP 200 |
|
||||||
|
| `/projects` (项目空间) | ✅ HTTP 200 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 发现的 Bug 及修复
|
||||||
|
|
||||||
|
### 5.1 Bug 概述
|
||||||
|
|
||||||
|
| 项目 | 内容 |
|
||||||
|
|------|------|
|
||||||
|
| Bug 编号 | BUG-001 |
|
||||||
|
| 严重级别 | 高(功能不可用) |
|
||||||
|
| 影响范围 | 模型创建和更新接口 |
|
||||||
|
| 发现时间 | 2026-07-31 |
|
||||||
|
| 修复状态 | 已修复 |
|
||||||
|
| 文件 | `backend/app/db/platform_store.py` |
|
||||||
|
|
||||||
|
### 5.2 Bug 描述
|
||||||
|
|
||||||
|
**`PlatformStore.create_model()` 和 `PlatformStore.update_model()`** 方法中,`return self.model(model_id)` 语句错误地位于 `with self.connect() as conn:` 上下文管理器块**内部**。
|
||||||
|
|
||||||
|
由于 `self.connect()` 每次调用都会创建**新的数据库连接**,`self.model(model_id)` 在 `with` 块内执行时:
|
||||||
|
1. INSERT/UPDATE 语句已执行但**尚未提交**(commit 发生在 `with` 块退出时)
|
||||||
|
2. `self.model()` 打开了一个全新的数据库连接进行 SELECT 查询
|
||||||
|
3. 新连接无法看到前一个连接中未提交的事务数据
|
||||||
|
4. SELECT 返回空,抛出 `KeyError`
|
||||||
|
5. `KeyError` 被 `connect()` 的 `except Exception` 捕获,触发 **rollback**
|
||||||
|
6. INSERT/UPDATE 被回滚,数据丢失
|
||||||
|
7. API 返回 HTTP 500 Internal Server Error
|
||||||
|
|
||||||
|
### 5.3 错误现象
|
||||||
|
|
||||||
|
```
|
||||||
|
POST /modelTF/model-manage → 500 Internal Server Error
|
||||||
|
|
||||||
|
后端错误日志:
|
||||||
|
KeyError: 'm_62c461a3d103'
|
||||||
|
File "platform_store.py", line 893, in create_model
|
||||||
|
return self.model(model_id)
|
||||||
|
File "platform_store.py", line 860, in model
|
||||||
|
raise KeyError(model_id)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.4 根因分析
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 修复前(BUG)— return 在 with 块内部
|
||||||
|
def create_model(self, payload):
|
||||||
|
model_id = payload.get("id") or new_id("m")
|
||||||
|
with self.connect() as conn: # 连接 A,事务开始
|
||||||
|
conn.execute("INSERT INTO models ...") # 未提交
|
||||||
|
return self.model(model_id) # ← 开新连接 B 查询,看不到 A 的未提交数据
|
||||||
|
# ← KeyError → 触发 A 的 rollback
|
||||||
|
|
||||||
|
# 对比其他正确方法
|
||||||
|
def create_tenant(self, payload):
|
||||||
|
...
|
||||||
|
with self.connect() as conn: # 连接 A
|
||||||
|
conn.execute("INSERT INTO tenants ...")
|
||||||
|
return self.tenant(tenant_id) # ← with 块已退出,A 已 commit,新连接可查到
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.5 修复方案
|
||||||
|
|
||||||
|
将 `create_model` 和 `update_model` 中的 `return self.model(model_id)` 语句从 `with self.connect() as conn:` 块内部移到外部,确保 INSERT/UPDATE 事务提交后再执行查询。
|
||||||
|
|
||||||
|
**修复代码** (`backend/app/db/platform_store.py`):
|
||||||
|
|
||||||
|
```python
|
||||||
|
# create_model — 修复后
|
||||||
|
def create_model(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
model_id = payload.get("id") or new_id("m")
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO models (...) VALUES (...)
|
||||||
|
""",
|
||||||
|
(...),
|
||||||
|
)
|
||||||
|
return self.model(model_id) # ← 移到 with 块外部
|
||||||
|
|
||||||
|
# update_model — 修复后
|
||||||
|
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
current = self.model(model_id)
|
||||||
|
merged = {**current, **payload}
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE models SET ... WHERE id=?
|
||||||
|
""",
|
||||||
|
(...),
|
||||||
|
)
|
||||||
|
return self.model(model_id) # ← 移到 with 块外部
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.6 修复验证
|
||||||
|
|
||||||
|
修复后重新执行测试,模型创建和更新接口均返回 `code=0`,数据正确持久化到数据库。
|
||||||
|
|
||||||
|
```
|
||||||
|
POST /modelTF/model-manage → 200 {"code":0, "data":{"id":"m_6aee6669fed8", ...}}
|
||||||
|
PUT /modelTF/model-manage/{id} → 200 {"code":0, "data":{"description":"updated description", ...}}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 测试结论
|
||||||
|
|
||||||
|
### 6.1 总体评价
|
||||||
|
|
||||||
|
YG_FT 模型微调平台在当前开发基线下,核心功能链路基本完整可用:
|
||||||
|
|
||||||
|
- **前端控制台**:11 条主要路由均可正常访问,页面渲染正常。
|
||||||
|
- **应用平台后端**:覆盖 20 个功能模块、90+ 个 API 端点,统一响应结构 `{ code, message, data }` 规范。
|
||||||
|
- **算力服务**:健康检查和 GPU 接口正常,推理代理链路通畅。
|
||||||
|
- **企业治理**:用户中心、多租户、项目隔离、审批流、审计、资源授权等功能均通过测试。
|
||||||
|
- **数据层**:PostgreSQL 远程数据库连接正常,Schema 自动初始化和种子数据写入正常。
|
||||||
|
|
||||||
|
### 6.2 已修复问题
|
||||||
|
|
||||||
|
| 编号 | 问题 | 严重级别 | 状态 |
|
||||||
|
|------|------|----------|------|
|
||||||
|
| BUG-001 | `create_model` 事务未提交即查询导致 500 错误 | 高 | ✅ 已修复 |
|
||||||
|
| BUG-002 | `update_model` 同类事务问题 | 高 | ✅ 已修复 |
|
||||||
|
|
||||||
|
### 6.3 已知限制(非 Bug)
|
||||||
|
|
||||||
|
| 项目 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| 微调任务创建 | 需提供 `train_dataset_id` 字段(业务约束,非 Bug) |
|
||||||
|
| 算力节点创建 | 需提供 `code` 字段(业务约束,非 Bug) |
|
||||||
|
| 算力服务响应格式 | 算力服务返回 `{"status":"ok"}` 而非后端统一的 `{code, message, data}` 结构(架构设计差异) |
|
||||||
|
| 推理服务 | 算力节点推理代理返回 500(本地无 GPU 环境,预期行为) |
|
||||||
|
| 用户页面权限 | 精细控制 UI 仍为占位(README 已说明,待后续版本补齐) |
|
||||||
|
|
||||||
|
### 6.4 建议
|
||||||
|
|
||||||
|
1. **代码审查**:建议对 `platform_store.py` 中所有 `with self.connect() as conn:` 块进行审查,确认 `return self.xxx()` 模式的一致性,避免同类事务问题。
|
||||||
|
2. **自动化测试**:建议引入 pytest + httpx 的 API 集成测试框架,将本次测试脚本固化为 CI 流水线用例。
|
||||||
|
3. **接口文档**:建议在 OpenAPI 文档中补充各 POST/PUT 接口的必填字段说明(如 `code`、`train_dataset_id`)。
|
||||||
|
4. **响应格式统一**:建议算力服务也采用 `{ code, message, data }` 统一响应结构,便于前端统一处理。
|
||||||
Reference in New Issue
Block a user