feat: 更新后端平台模块、前端组件及构建产物,新增工作计划文档

- 更新 backend 平台 API endpoints 及 platform_store
- 更新前端 ComputeNodesView、DataProcessCreateView、FineTuneCreateView 等组件
- 更新前端 API 模块(compute、fineTune)
- 重构 frontend/dist 构建产物(新 hash)
- 新增 docs/2026-07-24-work-plan.md 工作计划文档

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-07-24 10:27:52 +08:00
parent b28cfbc6fa
commit 2b10c013ce
151 changed files with 925 additions and 181 deletions

View File

@@ -5,7 +5,7 @@ import json
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
from fastapi import APIRouter, BackgroundTasks, Body, File, HTTPException, Query, UploadFile
from fastapi.responses import PlainTextResponse
from app.core.config import get_settings
@@ -41,6 +41,63 @@ def _node_for_compute_job_record(job_id: str) -> dict[str, Any] | None:
return next((node for node in store.compute_nodes() if node["id"] == record.get("node_id")), None)
def _training_diagnostics(errors: list[str], warnings: list[str] | None = None, log_text: str = "") -> list[dict[str, str]]:
source_items = [*errors, *(warnings or [])]
if log_text:
source_items.append(log_text)
text = "\n".join(source_items).lower()
diagnostics: list[dict[str, str]] = []
rules = [
(
["dataset columns missing", "keyerror", "history", "instruction", "input", "output", "messages"],
"训练数据字段不匹配",
"请检查所选数据集格式是否与训练模板一致。Alpaca 格式通常需要 instruction/input/outputShareGPT 格式通常需要 messages。",
),
(
["dataset file not found", "dataset_dir", "no uploaded file"],
"训练数据文件不可用",
"请确认数据集已上传文件,并且应用服务可以将数据同步到目标算力节点的数据目录。",
),
(
["model_name_or_path path not available", "base_model", "model path", "no such file"],
"基座模型路径不可用",
"请在模型管理中检查本地模型路径,确保该路径在算力服务器或 Compute 容器挂载目录内真实存在。",
),
(
["cuda out of memory", "outofmemoryerror", "显存", "memory"],
"GPU 显存不足",
"请降低 batch_size、cutoff_len、LoRA rank启用 4bit 量化,或选择更高显存的算力节点。",
),
(
["training command not found", "llamafactory-cli"],
"训练框架命令不可用",
"请检查 Compute 镜像是否包含 LLaMA-Factory或确认 llamafactory-cli 已在容器 PATH 中。",
),
(
["llama_factory_home not found"],
"LLaMA-Factory 目录不可用",
"请检查 Compute 服务的 LLAMA_FACTORY_HOME 配置和宿主机挂载路径。",
),
(
["no available compute node", "not schedulable", "disabled", "capacity full"],
"暂无可调度算力节点",
"请检查算力节点是否启用、状态是否在线、并行任务数是否已满,或手动调整节点权重/标签。",
),
]
for keywords, title, suggestion in rules:
if any(keyword in text for keyword in keywords):
diagnostics.append({"level": "error", "title": title, "suggestion": suggestion})
if not diagnostics and (errors or log_text):
diagnostics.append(
{
"level": "error",
"title": "训练任务异常",
"suggestion": "请查看预检错误和训练日志原文优先确认模型路径、数据集格式、GPU 显存和 LLaMA-Factory 参数。",
}
)
return diagnostics
async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("task_id") or payload.get("id") or "")
if task_id and get_settings().compute_mode != "simulator":
@@ -68,6 +125,25 @@ async def _fine_tune_preflight(
sync_resources: bool = False,
) -> dict[str, Any]:
node, job_payload = store.prepare_compute_job_payload(task_id, payload or {})
return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=sync_resources, store=store)
async def _fine_tune_preflight_payload(
store: Any,
payload: dict[str, Any],
validate: bool = True,
) -> dict[str, Any]:
node, job_payload = store.prepare_compute_job_payload_from_payload(payload)
return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=False, store=store)
async def _fine_tune_preflight_with_job_payload(
node: dict[str, Any],
job_payload: dict[str, Any],
validate: bool,
sync_resources: bool,
store: Any,
) -> dict[str, Any]:
sync_results: list[dict[str, Any]] = []
sync_errors: list[str] = []
if sync_resources and get_settings().compute_mode != "simulator":
@@ -105,6 +181,7 @@ async def _fine_tune_preflight(
"valid": bool(preview.get("valid", not errors)) and not errors,
"errors": errors,
"warnings": warnings,
"diagnostics": _training_diagnostics(errors, warnings),
"node": {
"id": node.get("id"),
"code": node.get("code"),
@@ -618,6 +695,26 @@ async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]
raise fail(502, f"submit compute job failed: {exc}")
@router.post("/fine-tune/preflight")
async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True))
except RuntimeError as exc:
return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
except Exception as exc: # noqa: BLE001 - expose compute validation errors to training create page
raise fail(502, f"compute preflight failed: {exc}")
@router.post("/fine-tune/command-preview")
async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False))
except RuntimeError as exc:
return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
except Exception as exc: # noqa: BLE001
raise fail(502, f"compute command preview failed: {exc}")
@router.post("/fine-tune/{task_id}/preflight")
async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
try:
@@ -685,6 +782,32 @@ async def fine_tune_logs(
return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
@router.get("/fine-tune/{task_id}/diagnostics")
async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
store = get_platform_store()
try:
task = store.task(task_id)
except KeyError:
raise fail(404, "fine tune task not found")
log_text = ""
node = _node_for_task(task)
if node and task.get("compute_job_id"):
try:
logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], 1000, None, None)
log_text = str(logs.get("content") or "")
except Exception:
log_text = ""
errors = [str(task.get("failure_reason") or "")] if task.get("failure_reason") else []
return ok(
{
"task_id": task_id,
"status": task.get("status"),
"failure_reason": task.get("failure_reason") or "",
"diagnostics": _training_diagnostics(errors, [], log_text),
}
)
@router.put("/fine-tune/{task_id}")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
@@ -1055,6 +1178,14 @@ async def compute_node_replicas(node_id: str) -> dict[str, Any]:
return ok(get_platform_store().replicas(node_id))
@router.get("/compute/sync-jobs/{sync_id}")
async def compute_sync_job_detail(sync_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().sync_job(sync_id))
except KeyError:
raise fail(404, "sync job not found")
@router.get("/compute/nodes/{node_id}/replicas/drift")
async def compute_node_replica_drift(node_id: str) -> dict[str, Any]:
store = get_platform_store()
@@ -1091,35 +1222,18 @@ async def compute_node_replica_drift(node_id: str) -> dict[str, Any]:
return ok({"node_id": node_id, "items": items, "drifted": len([item for item in items if item.get("sync_status") == "drifted"])})
@router.post("/compute/nodes/{node_id}/replicas/repair")
async def compute_node_replica_repair(node_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
async def _run_resource_replica_repair(sync_id: str, node_id: str, payload: dict[str, Any], replicas_to_repair: list[dict[str, Any]]) -> None:
store = get_platform_store()
store.update_sync_job(sync_id, "running", 5)
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
payload = payload or {}
replica_ids = payload.get("replica_ids") or [
item["id"] for item in store.replicas(node_id) if item.get("sync_status") in {"drifted", "failed", "repair_pending"}
]
updated = store.mark_resource_replica_repair_pending([str(item) for item in replica_ids])
sync_id = store.create_sync_job(
node_id,
{
"resources": [
{
"resource_type": item.get("resource_type"),
"resource_id": item.get("resource_id"),
"replica_id": item.get("id"),
"target_path": item.get("local_path"),
}
for item in updated
]
},
)
store.update_sync_job(sync_id, "failed", 100, completed=True)
return
client = ComputeNodeClient(node["api_base_url"])
repaired = []
failures = []
for replica in updated:
total = max(len(replicas_to_repair), 1)
for index, replica in enumerate(replicas_to_repair, start=1):
replica_id = str(replica["id"])
resource_type = str(replica.get("resource_type") or "")
resource_id = str(replica.get("resource_id") or "")
@@ -1187,8 +1301,46 @@ async def compute_node_replica_repair(node_id: str, payload: dict[str, Any] | No
error = str(exc)
failures.append({"replica_id": replica_id, "resource_type": resource_type, "resource_id": resource_id, "error": error})
repaired.append(store.update_resource_replica_sync_result(replica_id, False, None, int(replica.get("byte_size") or 0), "", error))
progress = min(95, 5 + int(index / total * 90))
store.update_sync_job(sync_id, "running", progress)
store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)
return ok({"sync": store.sync_job(sync_id), "replicas": repaired, "failed": failures})
@router.post("/compute/nodes/{node_id}/replicas/repair")
async def compute_node_replica_repair(
node_id: str,
background_tasks: BackgroundTasks,
payload: dict[str, Any] | None = Body(default=None),
) -> dict[str, Any]:
store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
payload = payload or {}
replica_ids = payload.get("replica_ids") or [
item["id"] for item in store.replicas(node_id) if item.get("sync_status") in {"drifted", "failed", "repair_pending"}
]
updated = store.mark_resource_replica_repair_pending([str(item) for item in replica_ids])
sync_id = store.create_sync_job(
node_id,
{
"operation": "repair",
"resources": [
{
"resource_type": item.get("resource_type"),
"resource_id": item.get("resource_id"),
"replica_id": item.get("id"),
"target_path": item.get("local_path"),
}
for item in updated
],
},
)
if not updated:
store.update_sync_job(sync_id, "completed", 100, completed=True)
return ok({"sync": store.sync_job(sync_id), "replicas": [], "failed": [], "async": False})
background_tasks.add_task(_run_resource_replica_repair, sync_id, node_id, payload, updated)
return ok({"sync": store.sync_job(sync_id), "replicas": updated, "failed": [], "async": True})
@router.get("/compute/nodes/{node_id}/engines")

View File

@@ -1474,25 +1474,40 @@ class PlatformStore:
def prepare_compute_job_payload(self, task_id: str, payload: dict[str, Any] | None = None) -> tuple[dict[str, Any], dict[str, Any]]:
task = self.task(task_id)
merged = {**task, **(payload or {}), "id": task_id}
node = self.schedule_node(merged)
node = self.select_compute_node(merged)
selected_gpus = merged.get("gpus") or [0]
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
def prepare_compute_job_payload_from_payload(self, payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
task_id = str(payload.get("task_id") or payload.get("id") or new_id("ft_preview"))
name = str(payload.get("name") or task_id)
transient_task = {
**payload,
"id": task_id,
"name": name,
"status": str(payload.get("status") or "pending"),
"progress": int(payload.get("progress") or 0),
}
node = self.select_compute_node(transient_task)
selected_gpus = transient_task.get("gpus") or [0]
return node, self._compute_job_payload_from_task_node(transient_task, node, selected_gpus)
def _compute_job_payload_from_task_node(
self,
task: dict[str, Any],
node: dict[str, Any],
selected_gpus: list[int] | list[Any] | None = None,
) -> dict[str, Any]:
base_model_id = task.get("base_model") or task.get("model_id")
dataset_id = str(task.get("train_dataset_id") or task.get("dataset_id") or "")
with self.connect() as conn:
model = conn.execute("SELECT * FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
dataset = conn.execute("SELECT * FROM datasets WHERE id=?", (task.get("train_dataset_id"),)).fetchone()
model = conn.execute("SELECT * FROM models WHERE id=?", (base_model_id,)).fetchone()
dataset = conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()
files = conn.execute(
"SELECT id, name, size, active_version_id, create_time FROM dataset_files WHERE dataset_id=? ORDER BY create_time",
(task.get("train_dataset_id"),),
(dataset_id,),
).fetchall()
model_path = (model and model.get("path")) or task.get("base_model")
dataset_id = str(task.get("train_dataset_id") or "")
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
if not files:
raise RuntimeError(f"dataset has no uploaded file: {dataset_id}")
dataset_key = str(task.get("dataset_key") or llama_dataset_key(dataset_id))
@@ -1897,6 +1912,10 @@ class PlatformStore:
raise RuntimeError("compute scheduler is busy, please retry")
return self._schedule_node_locked(conn, payload)
def select_compute_node(self, payload: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
return self._schedule_node_locked(conn, payload)
def _reserve_gpu_allocations(
self,
conn: PgConnection,