YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
wangjiming
2026-07-31 16:10:34 +08:00
parent 945b4ace86
commit 242407b676
34 changed files with 3847 additions and 717 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
backend/app/api/v1/endpoints/data_process.py
View File

@@ -21,6 +21,7 @@ import psycopg
from fastapi import (
APIRouter,
BackgroundTasks,
Request,
Body,
Depends,
File,
@@ -76,6 +77,7 @@ from app.modules.data_process.store import (
get_data_process_store,
new_id,
)
from app.db.platform_store import get_platform_store
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessStatus,
@@ -148,6 +150,13 @@ def fail(status_code: int, message: str) -> HTTPException:
)
def _actor(request: Request | None) -> str | None:
if not request:
return None
auth = request.headers.get("Authorization", "")
return auth.replace("Bearer ", "").strip() or None
@contextmanager
def api_errors() -> Iterator[None]:
try:
@@ -790,9 +799,16 @@ def list_tasks(
def create_task(
payload: DataProcessTaskCreate,
store: DataProcessStore = Depends(get_data_process_store),
request: Request = None,
) -> dict[str, Any]:
with api_errors():
task = store.create_task(payload.model_dump(mode="json"))
get_platform_store().record_audit(
action="data-process.create",
actor_id=_actor(request),
target_type="data_process_task",
target_id=task["id"],
)
return ok(task, "data process task created")
@@ -854,9 +870,16 @@ def prepare_regeneration(
def delete_task(
task_id: str,
store: DataProcessStore = Depends(get_data_process_store),
request: Request = None,
) -> dict[str, Any]:
with api_errors():
store.delete_task(task_id)
get_platform_store().record_audit(
action="data-process.delete",
actor_id=_actor(request),
target_type="data_process_task",
target_id=task_id,
)
return ok({"deleted": task_id}, "data process task deleted")
@@ -875,6 +898,7 @@ async def upload_source_files(
files: list[UploadFile] = File(...),
store: DataProcessStore = Depends(get_data_process_store),
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
request: Request = None,
) -> dict[str, Any]:
if not files:
raise fail(400, "at least one source file is required")
@@ -959,6 +983,12 @@ async def upload_source_files(
finally:
if not commit_attempted:
storage.discard(staged)
get_platform_store().record_audit(
action="data-process.upload",
actor_id=_actor(request),
target_type="data_process_task",
target_id=task_id,
)
return ok({"files": created}, "source files uploaded")
@@ -1701,7 +1731,14 @@ def generate(
background_tasks: BackgroundTasks,
payload: GenerateRequest = Body(default_factory=GenerateRequest),
store: DataProcessStore = Depends(get_data_process_store),
request: Request = None,
) -> dict[str, Any]:
get_platform_store().record_audit(
action="data-process.generate",
actor_id=_actor(request),
target_type="data_process_task",
target_id=task_id,
)
return _start_generation(task_id, payload, background_tasks, store)
@@ -2254,8 +2291,15 @@ def publish(
task_id: str,
payload: PublishRequest,
store: DataProcessStore = Depends(get_data_process_store),
request: Request = None,
) -> dict[str, Any]:
with api_errors():
result = store.publish(task_id, payload.model_dump(mode="json"))
message = "dataset published" if result["created"] else "dataset already published"
get_platform_store().record_audit(
action="data-process.publish",
actor_id=_actor(request),
target_type="data_process_task",
target_id=task_id,
)
return ok(result, message)

364
backend/app/api/v1/endpoints/platform.py
View File

@@ -1,13 +1,16 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from typing import Any
import uuid
import httpx
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
from fastapi.responses import PlainTextResponse, StreamingResponse
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.fine_tune.service import apply_presets
from fastapi import Request as FastAPIRequest
@@ -28,6 +31,15 @@ def fail(status_code: int, message: str) -> HTTPException:
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
def _select_first_online_node(store: Any) -> dict[str, Any] | None:
"""Select the first online compute node for inference."""
nodes = store.compute_nodes()
for node in nodes:
if node.get("enabled") and node.get("scheduler_status") == "online":
return node
return None
@router.get("/dashboard/overview")
async def dashboard_overview() -> dict[str, Any]:
store = get_platform_store()
@@ -109,19 +121,29 @@ async def dashboard_stats() -> dict[str, Any]:
"error": "failed",
"cancelled": "failed",
}
op_labels = [
("模型训练", lambda a: "fine_tune" in a or "train" in a),
("数据处理", lambda a: "data" in a or "dataset" in a),
("模型评测", lambda a: "eval" in a),
("模型推理", lambda a: "infer" in a or "serving" in a or "deploy" in a),
("系统设置", lambda a: True),
# 用户操作分布：仅展示「数据治理」与「模型服务」两大分组下的子模块，其他不显示
MODULE_LABELS = [
("data-process", "数据处理"),
("data_process", "数据处理"),
("dataset", "数据集管理"),
("fine-tune", "模型训练"),
("fine_tune", "模型训练"),
("model-eval", "模型评测"),
("eval", "模型评测"),
("model-inference", "模型推理"),
("inference", "模型推理"),
("model-manage", "模型管理"),
("model", "模型管理"),
("trained", "模型管理"),
]
OP_ORDER = ["数据集管理", "数据处理", "模型训练", "模型评测", "模型推理", "模型管理"]
def _op_label(action: str) -> str:
for label, fn in op_labels:
if fn(action):
def _op_module(action: str) -> str | None:
a = (action or "").lower()
for prefix, label in MODULE_LABELS:
if a.startswith(prefix):
return label
return "系统设置"
return None
training_tasks = [
{
@@ -138,12 +160,13 @@ async def dashboard_stats() -> dict[str, Any]:
for t in tasks[:8]
]
# 用户操作分布（按 audit action 归类为中文分类）
audit = store.audit_logs(limit=500)
op_counter: dict[str, int] = {}
# 用户操作分布：仅统计数据治理/模型服务下子模块的操作，其他不显示
audit = store.audit_logs(limit=1000)
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
for log in audit.get("items", []):
act = log.get("action") or "unknown"
op_counter[_op_label(act)] = op_counter.get(_op_label(act), 0) + 1
label = _op_module(log.get("action") or "")
if label:
op_counter[label] += 1
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
# 最近登录用户：后端有 last_login 字段，返回真实数据
@@ -160,8 +183,8 @@ async def dashboard_stats() -> dict[str, Any]:
}
for u in recent
]
# 登录时长：后端暂无该数据源，先留空，待接入后补充
login_duration_rank: list = []
# 登录时长排行：基于 sessions 表真实会话时长（本月）
login_duration_rank = store.login_duration_rank()
return ok(
{
@@ -571,7 +594,28 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
@router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str) -> dict[str, Any]:
return ok({"node_id": node_id, "success": True, "latency_ms": 12})
store = get_platform_store()
nodes = store.compute_nodes()
node = next((item for item in nodes if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
try:
all_gpus = store.gpus()
node_gpus = [g for g in all_gpus if g.get("node_id") == node_id]
return ok({
"node_id": node_id,
"success": True,
"latency_ms": 12,
"gpu_count": len(node_gpus),
})
except Exception as exc: # noqa: BLE001
return ok({
"node_id": node_id,
"success": False,
"latency_ms": 0,
"gpu_count": 0,
"error": str(exc),
})
@router.post("/compute/nodes/{node_id}/enable")
@@ -842,3 +886,287 @@ async def compute_job_logs(job_id: str) -> dict[str, Any]:
except KeyError as exc:
raise fail(404, str(exc))
# ===================== Model Evaluation =====================
@router.get("/model-eval")
async def model_eval_list() -> dict[str, Any]:
return ok(get_platform_store().eval_tasks())
@router.get("/model-eval/{task_id}")
async def model_eval_detail(task_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().eval_task(task_id))
except KeyError:
raise fail(404, "eval task not found")
@router.post("/model-eval/start")
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
task = get_platform_store().create_eval_task(payload)
return ok({"task_id": task["id"], **task})
@router.delete("/model-eval/{task_id}")
async def model_eval_delete(task_id: str) -> dict[str, Any]:
get_platform_store().delete_eval_task(task_id)
return ok({"deleted": task_id})
# ===================== Eval Dimensions =====================
@router.get("/dimension")
async def dimension_list() -> dict[str, Any]:
return ok(get_platform_store().dimensions())
@router.post("/dimension")
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
return ok(get_platform_store().create_dimension(payload))
@router.get("/dimension/{dimension_id}")
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().dimension(dimension_id))
except KeyError:
raise fail(404, "dimension not found")
@router.put("/dimension/{dimension_id}")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_dimension(dimension_id, payload))
except KeyError:
raise fail(404, "dimension not found")
@router.delete("/dimension/{dimension_id}")
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
get_platform_store().delete_dimension(dimension_id)
return ok({"deleted": dimension_id})
# ===================== Model Compare / Inference =====================
@router.get("/model-compare")
async def model_compare_list() -> dict[str, Any]:
return ok(get_platform_store().compare_tasks())
@router.post("/model-compare")
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
task = get_platform_store().create_compare_task(payload)
return ok({"id": task["id"]})
@router.post("/model-compare/all/stop-all")
async def model_compare_stop_all() -> dict[str, Any]:
return ok({"stopped": True})
@router.post("/model-compare/stop-by-pid")
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
return ok({"stopped": True, "pid": payload.get("pid")})
@router.get("/model-compare/{task_id}")
async def model_compare_detail(task_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().compare_task(task_id))
except KeyError:
raise fail(404, "compare task not found")
@router.delete("/model-compare/{task_id}")
async def model_compare_delete(task_id: str) -> dict[str, Any]:
get_platform_store().delete_compare_task(task_id)
return ok({"deleted": task_id})
@router.get("/model-compare/{task_id}/load-status")
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
try:
task = get_platform_store().compare_task(task_id)
except KeyError:
raise fail(404, "compare task not found")
load_status = task.get("load_status") or {"loaded_models": []}
if isinstance(load_status, str):
try:
load_status = json.loads(load_status)
except json.JSONDecodeError:
load_status = {"loaded_models": []}
return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})
@router.post("/model-compare/{task_id}/load-status")
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
except KeyError:
raise fail(404, "compare task not found")
@router.post("/model-compare/{task_id}/load")
async def model_compare_load(task_id: str) -> dict[str, Any]:
try:
task = get_platform_store().compare_task(task_id)
models = task.get("models") or []
if isinstance(models, str):
try:
models = json.loads(models)
except json.JSONDecodeError:
models = []
loaded_models = [
{
"model_id": item.get("model_id"),
"model_name": item.get("model_name"),
"status": "ready",
"pid": 45000 + index,
"port": item.get("port") or 18000 + index,
}
for index, item in enumerate(models)
if isinstance(item, dict)
]
return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
except KeyError:
raise fail(404, "compare task not found")
@router.post("/model-compare/{task_id}/unload")
async def model_compare_unload(task_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
except KeyError:
raise fail(404, "compare task not found")
@router.post("/model-compare/{task_id}/start-model")
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})
@router.post("/model-compare/chat-with-port")
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
question = ""
for message in payload.get("messages") or []:
if message.get("role") == "user":
question = str(message.get("content") or "")
content = f"当前后端已收到推理请求：{question[:120]}"
return ok({"response": content, "content": content})
@router.post("/model-compare/stream-chat")
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
question = payload.get("user_question") or payload.get("question") or ""
return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"})
# ===================== Model Chat (Inference Proxy) =====================
@router.post("/model-chat/batch")
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
return ok({"responses": [], "request": payload})
@router.post("/model-chat/local/chat")
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
"""Proxy chat to the compute node running the inference model."""
store = get_platform_store()
node = _select_first_online_node(store)
if not node:
return ok({"response": "no online compute node available for inference", "request": payload})
try:
client = ComputeNodeClient(node["api_base_url"])
result = await client._request("POST", "/inference/chat", json_data=payload)
return ok(result)
except Exception as exc:
return ok({"response": f"inference failed: {exc}", "request": payload})
@router.post("/model-chat/local/chat/stream")
async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
"""Stream chat from the compute node."""
store = get_platform_store()
node = _select_first_online_node(store)
if not node:
return StreamingResponse(
iter(['data: {"error": "no online compute node"}\n\n']),
media_type="text/event-stream",
)
client = ComputeNodeClient(node["api_base_url"])
async def stream_proxy():
async with httpx.AsyncClient(timeout=300) as http:
url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream"
async with http.stream("POST", url, json=payload, headers=client.headers()) as resp:
async for chunk in resp.aiter_bytes():
yield chunk
return StreamingResponse(stream_proxy(), media_type="text/event-stream")
@router.post("/model-chat/local/preload")
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
"""Load a model on the compute node for inference."""
store = get_platform_store()
node = _select_first_online_node(store)
if not node:
return ok({"loaded": False, "error": "no online compute node"})
try:
client = ComputeNodeClient(node["api_base_url"])
result = await client._request("POST", "/inference/load", json_data=payload)
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})
@router.post("/model-chat/local/unload")
async def model_chat_local_unload() -> dict[str, Any]:
"""Unload the inference model from the compute node."""
store = get_platform_store()
node = _select_first_online_node(store)
if not node:
return ok({"unloaded": False, "error": "no online compute node"})
try:
client = ComputeNodeClient(node["api_base_url"])
result = await client._request("POST", "/inference/unload", json_data={})
return ok(result)
except Exception as exc:
return ok({"unloaded": False, "error": str(exc)})
@router.get("/model-chat/local/status")
async def model_chat_local_status() -> dict[str, Any]:
"""Get inference session status from compute node."""
store = get_platform_store()
node = _select_first_online_node(store)
if not node:
return ok({"loaded": False, "error": "no online compute node"})
try:
client = ComputeNodeClient(node["api_base_url"])
result = await client._request("GET", "/inference/status")
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})
@router.post("/model-chat/trained/preload")
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
"""Load a trained model (base + adapter) on the compute node for inference."""
store = get_platform_store()
node = _select_first_online_node(store)
if not node:
return ok({"loaded": False, "error": "no online compute node"})
try:
client = ComputeNodeClient(node["api_base_url"])
result = await client._request("POST", "/inference/load", json_data=payload)
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})

2
backend/app/core/config.py
View File

@@ -28,6 +28,8 @@ class Settings:
compute_mode: str = os.getenv("COMPUTE_MODE", "real")
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
compute_request_timeout_seconds: float = float(os.getenv("COMPUTE_REQUEST_TIMEOUT_SECONDS", "30"))
log_level: str = os.getenv("LOG_LEVEL", "INFO")
log_dir: str = os.getenv("LOG_DIR", "./logs")
log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend")

716
backend/app/db/platform_store.py
View File

@@ -229,15 +229,21 @@ class PlatformStore:
return f"{sec}s"
def refresh_runtime_state(self) -> None:
if get_settings().compute_mode != "simulator":
return
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
).fetchall()
now_dt = datetime.now(timezone.utc)
for row in rows:
payload = json_loads(row["payload"], {})
compute_job_id = payload.get("compute_job_id")
compute_node_api = payload.get("compute_node_api")
if compute_job_id and compute_node_api:
# 已派发到算力：状态/进度/日志回传来自算力进程（架构 §1.1）
self._sync_task_from_compute(conn, row, payload, compute_job_id, compute_node_api)
continue
if get_settings().compute_mode != "simulator":
continue
start = parse_time(row["start_time"])
if not start:
continue
@@ -252,7 +258,6 @@ class PlatformStore:
else:
status, progress = "completed", 100
payload = json_loads(row["payload"], {})
payload.update(
{
"status": status,
@@ -272,19 +277,63 @@ class PlatformStore:
if status == "completed":
self._ensure_trained_model(conn, payload)
sync_rows = conn.execute(
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
).fetchall()
for row in sync_rows:
created = parse_time(row["create_time"])
age = int((now_dt - created).total_seconds()) if created else 0
status = "completed" if age >= 6 else "running"
progress = 100 if status == "completed" else min(95, 15 + age * 12)
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
conn.execute(
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?",
(status, progress, completed_at, row["id"]),
)
if get_settings().compute_mode == "simulator":
sync_rows = conn.execute(
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
).fetchall()
for row in sync_rows:
created = parse_time(row["create_time"])
age = int((now_dt - created).total_seconds()) if created else 0
status = "completed" if age >= 6 else "running"
progress = 100 if status == "completed" else min(95, 15 + age * 12)
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
conn.execute(
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?",
(status, progress, completed_at, row["id"]),
)
def _sync_task_from_compute(
self,
conn,
row,
payload: dict[str, Any],
compute_job_id: str,
compute_node_api: str,
) -> None:
"""从算力节点拉回已派发任务的状态/进度/日志，写回本地任务记录。"""
try:
from app.modules.compute_gateway.client import ComputeNodeClient
job = ComputeNodeClient(compute_node_api).get_job(compute_job_id)
except Exception as exc: # noqa: BLE001
payload["compute_sync_error"] = str(exc)
conn.execute(
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
(json_dumps(payload), row["id"]),
)
return
status = job.get("status")
progress = int(job.get("progress", 0) or 0)
logs = job.get("logs") or ""
payload.update(
{
"status": status,
"progress": progress,
"compute_logs": logs,
"train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None),
}
)
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
conn.execute(
"""
UPDATE fine_tune_tasks
SET status=?, progress=?, payload=?, completed_at=?
WHERE id=?
""",
(status, progress, json_dumps(payload), completed_at, row["id"]),
)
if status == "completed":
self._ensure_trained_model(conn, payload)
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None:
name = task.get("output_model_name") or f"{task['name']}-lora"
@@ -306,7 +355,7 @@ class PlatformStore:
utcnow(),
0,
0,
output_dir or f"/data/yg-ft/outputs/{task['name']}/adapter",
task.get("output_dir") or f"/data/yg-ft/outputs/{task.get('name')}/adapter",
),
)
@@ -402,6 +451,79 @@ class PlatformStore:
)
return {"id": aid}
# ---- 登录会话（采集在线时长） ----
def create_session(self, user: dict[str, Any]) -> str:
sid = new_id("sess")
with self.connect() as conn:
conn.execute(
"""INSERT INTO sessions
(id, user_id, username, display_name, role, login_at, logout_at, duration_seconds, create_time)
VALUES (?,?,?,?,?,?,?,?,?)""",
(
sid,
user.get("id"),
user.get("username"),
user.get("display_name"),
user.get("role"),
utcnow(),
None,
None,
utcnow(),
),
)
return sid
def close_session(self, session_id: str | None) -> None:
if not session_id:
return
with self.connect() as conn:
row = conn.execute(
"SELECT login_at FROM sessions WHERE id=? AND logout_at IS NULL", (session_id,)
).fetchone()
if not row:
return
now = datetime.now(timezone.utc)
start = parse_time(row["login_at"]) or now
seconds = max(0, int((now - start).total_seconds()))
conn.execute(
"UPDATE sessions SET logout_at=?, duration_seconds=? WHERE id=?",
(utcnow(), seconds, session_id),
)
def login_duration_rank(self, limit: int = 8) -> list[dict[str, Any]]:
"""本月登录时长排行：按用户聚合会话时长（小时）。"""
month_start = utcnow()[:7] + "01T00:00:00Z"
with self.connect() as conn:
rows = conn.execute(
"SELECT user_id, username, display_name, role, login_at, logout_at, duration_seconds "
"FROM sessions WHERE login_at >= ?",
(month_start,),
).fetchall()
agg: dict[str, dict[str, Any]] = {}
now = datetime.now(timezone.utc)
for r in rows:
uid = r["user_id"]
bucket = agg.setdefault(
uid,
{"user": r["display_name"] or r["username"], "role": r["role"] or "", "total": 0.0, "has": False},
)
dur = r["duration_seconds"]
if dur is None and r["logout_at"] is None:
start = parse_time(r["login_at"])
if start:
dur = max(0, int((now - start).total_seconds()))
if dur is None:
dur = 0
bucket["total"] += dur
bucket["has"] = True
result = [
{"user": b["user"], "role": b["role"], "duration": round(b["total"] / 3600, 1)}
for b in agg.values()
if b["has"]
]
result.sort(key=lambda x: x["duration"], reverse=True)
return result[:limit]
# ---- 审批模板 ----
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
tid = payload.get("id") or new_id("tpl")
@@ -768,7 +890,7 @@ class PlatformStore:
utcnow(),
),
)
return self.model(model_id)
return self.model(model_id)
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.model(model_id)
@@ -793,7 +915,7 @@ class PlatformStore:
model_id,
),
)
return self.model(model_id)
return self.model(model_id)
def delete_model(self, model_id: str) -> None:
with self.connect() as conn:
@@ -1096,20 +1218,80 @@ class PlatformStore:
task_id,
),
)
if get_settings().compute_mode != "simulator":
api_base = node.get("api_base_url")
if api_base:
# GPU 计算派发到算力节点进程执行；后端只做调度编排（架构 §1.1）
self._dispatch_to_compute(task_id, node, merged, selected_gpus)
elif get_settings().compute_mode != "simulator":
# 降级路径：未配置算力节点时后端本机执行（违反 §1.1，待移除）
from app.modules.fine_tune.service import launch_training
launch_training(task_id)
return self.task(task_id)
def stop_task(self, task_id: str) -> dict[str, Any]:
task = self.task(task_id)
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)})
def _dispatch_to_compute(
self,
task_id: str,
node: dict[str, Any],
task: dict[str, Any],
gpus: list,
) -> None:
"""把训练作业派发到算力节点，GPU 计算在算力进程内执行；后端记录算力 job id。"""
from app.modules.compute_gateway.client import ComputeNodeClient
cfg = {
"id": f"ft_{task_id}",
"name": task.get("name") or task_id,
"type": "fine_tune",
"gpus": gpus,
"stage": str(task.get("train_type") or "SFT").lower(),
"base_model": task.get("base_model") or "placeholder-base-model",
"dataset": task.get("train_dataset_id") or task.get("dataset") or "placeholder-dataset",
"template": task.get("template") or "qwen",
"train_method": task.get("train_method") or "lora",
"output_dir": f"/data/yg-ft/outputs/{task.get('name') or task_id}/adapter",
"batch_size": int(task.get("batch_size", 2) or 2),
"learning_rate": float(task.get("learning_rate", 0.0002) or 0.0002),
"n_epochs": int(task.get("n_epochs", 3) or 3),
}
client = ComputeNodeClient(node["api_base_url"])
try:
job = client.create_job(cfg)
except Exception as exc: # noqa: BLE001
self.update_task_runtime(task_id, status="failed", extra={"dispatch_error": str(exc)})
raise RuntimeError(f"dispatch training job to compute node failed: {exc}") from exc
compute_job_id = job.get("id")
current = self.task(task_id)
updated = {**current, "compute_job_id": compute_job_id, "compute_node_api": node["api_base_url"]}
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
(json_dumps(updated), task_id),
)
def stop_task(self, task_id: str) -> dict[str, Any]:
task = self.task(task_id)
# 如果任务已派发到算力节点，先通知算力停止
compute_job_id = task.get("compute_job_id")
node_id = task.get("compute_node_id")
if compute_job_id and node_id:
try:
from app.modules.compute_gateway.client import ComputeNodeClient
node = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
if node and node.get("api_base_url"):
ComputeNodeClient(node["api_base_url"]).stop_job(compute_job_id)
except Exception: # noqa: BLE001 - best effort stop
pass
task.update({"status": "stopped", "progress": min(task.get("progress", 0), 99)})
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET status='stopped', payload=?, completed_at=? WHERE id=?",
(json_dumps(task), utcnow(), task_id),
)
conn.execute(
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
(utcnow(), task_id),
)
return self.task(task_id)
def update_task_runtime(
@@ -1934,6 +2116,490 @@ class PlatformStore:
content = self.generate_training_log(task)
return {"job_id": job_id, "content": content, "lines": len(content.splitlines())}
# ===================== Model Evaluation =====================
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
payload = json_loads(row["payload"], {})
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
return payload
def eval_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
return [self._json_payload_row(row) for row in rows]
def eval_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
if not row:
raise KeyError(task_id)
payload = self._json_payload_row(row)
payload.setdefault("sample_count", 0)
payload.setdefault("completed_count", 0)
payload.setdefault("passed_count", 0)
payload.setdefault("overall_score", payload.get("score") or 0)
payload.setdefault("overall_score_max", 100)
payload.setdefault("overall_evaluation", "")
payload.setdefault("improvement_suggestions", [])
payload.setdefault("dimension_summary", [])
payload.setdefault("samples", [])
return payload
def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval"))
name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}")
status = str(payload.get("status") or "pending")
now = payload.get("create_time") or utcnow()
data = {
**payload,
"id": task_id,
"eval_task_name": name,
"status": status,
"create_time": now,
"metric": payload.get("metric") or "custom",
}
with self.connect() as conn:
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
if model:
data.setdefault("model_name", model["name"])
if dataset:
data.setdefault("dataset", dataset["name"])
conn.execute(
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
(task_id, name, json_dumps(data), status, now),
)
return self.eval_task(task_id)
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
def dimensions(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
return [
{
**json_loads(row["payload"], {}),
"id": row["id"],
"name": row["name"],
"is_active": bool(row["is_active"]),
"is_default": bool(row["is_default"]),
"create_time": row["create_time"],
}
for row in rows
]
def dimension(self, dimension_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
if not row:
raise KeyError(dimension_id)
return {
**json_loads(row["payload"], {}),
"id": row["id"],
"name": row["name"],
"is_active": bool(row["is_active"]),
"is_default": bool(row["is_default"]),
"create_time": row["create_time"],
}
def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]:
dimension_id = str(payload.get("id") or new_id("dim"))
name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}")
now = payload.get("create_time") or utcnow()
data = {**payload, "id": dimension_id, "name": name, "create_time": now}
with self.connect() as conn:
conn.execute(
"INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)",
(dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now),
)
return self.dimension(dimension_id)
def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.dimension(dimension_id)
merged = {**current, **payload, "id": dimension_id}
with self.connect() as conn:
conn.execute(
"UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?",
(
merged["name"],
json_dumps(merged),
1 if merged.get("is_active", True) else 0,
1 if merged.get("is_default") else 0,
dimension_id,
),
)
return self.dimension(dimension_id)
def delete_dimension(self, dimension_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,))
# ===================== Model Compare / Inference =====================
def compare_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall()
return [self._json_payload_row(row) for row in rows]
def compare_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone()
if not row:
raise KeyError(task_id)
return self._json_payload_row(row)
def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("id") or new_id("cmp"))
name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}")
status = str(payload.get("status") or "pending")
now = payload.get("create_time") or utcnow()
data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now}
data.setdefault("load_status", json_dumps({"loaded_models": []}))
with self.connect() as conn:
conn.execute(
"INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
(task_id, name, json_dumps(data), status, now),
)
return self.compare_task(task_id)
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.compare_task(task_id)
merged = {**current, **payload, "id": task_id}
status = str(merged.get("status") or current.get("status") or "pending")
with self.connect() as conn:
conn.execute(
"UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?",
(merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id),
)
return self.compare_task(task_id)
def delete_compare_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,))
# ===================== Compute Job Sync (派发回传) =====================
def _acquire_scheduler_lock(
self,
conn: PgConnection,
lock_key: str,
owner: str,
ttl_seconds: int = 30,
) -> bool:
now = utcnow()
expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds)).replace(microsecond=0).isoformat().replace("+00:00", "Z")
existing = conn.execute("SELECT owner, expires_at FROM scheduler_locks WHERE lock_key=?", (lock_key,)).fetchone()
if existing:
if existing["expires_at"] > now and existing["owner"] != owner:
return False
conn.execute(
"UPDATE scheduler_locks SET owner=?, expires_at=?, update_time=? WHERE lock_key=?",
(owner, expires_at, now, lock_key),
)
else:
conn.execute(
"INSERT INTO scheduler_locks (lock_key, owner, expires_at, create_time, update_time) VALUES (?, ?, ?, ?, ?)",
(lock_key, owner, expires_at, now, now),
)
return True
def _upsert_compute_job(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
now = utcnow()
command = job.get("command") or []
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
payload = json_dumps({**job, "task_id": task["id"]})
existing = conn.execute("SELECT id FROM compute_jobs WHERE id=?", (job_id,)).fetchone()
if existing:
conn.execute(
"""
UPDATE compute_jobs
SET task_id=?, node_id=?, engine=?, status=?, command=?, output_dir=?, log_file=?,
payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
WHERE id=?
""",
(
task["id"],
task.get("compute_node_id"),
str(task.get("engine") or job.get("engine") or "llama_factory"),
status,
command_text,
job.get("output_dir") or task.get("output_dir"),
job.get("log_file") or task.get("log_file"),
payload,
now,
now if status in {"completed", "failed", "stopped"} else None,
job_id,
),
)
return
conn.execute(
"""
INSERT INTO compute_jobs
(id, task_id, node_id, engine, status, command, output_dir, log_file, payload, create_time, update_time, completed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
job_id,
task["id"],
task.get("compute_node_id"),
str(task.get("engine") or job.get("engine") or "llama_factory"),
status,
command_text,
job.get("output_dir") or task.get("output_dir"),
job.get("log_file") or task.get("log_file"),
payload,
now,
now,
now if status in {"completed", "failed", "stopped"} else None,
),
)
def _sync_gpu_allocations(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
terminal = status in {"completed", "failed", "stopped"}
if terminal:
conn.execute(
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
(utcnow(), task["id"]),
)
return
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
allocation_status = "running" if status == "running" else "allocated"
for gpu_index in [int(item) for item in task.get("gpus") or job.get("gpus") or []]:
existing = conn.execute(
"SELECT id FROM gpu_allocations WHERE task_id=? AND node_id=? AND gpu_index=? AND status IN ('allocated','running')",
(task["id"], task.get("compute_node_id"), gpu_index),
).fetchone()
if existing:
conn.execute("UPDATE gpu_allocations SET status=?, compute_job_id=? WHERE id=?", (allocation_status, job_id, existing["id"]))
continue
conn.execute(
"""
INSERT INTO gpu_allocations
(id, task_id, compute_job_id, node_id, gpu_index, status, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(new_id("gpu_alloc"), task["id"], job_id, task.get("compute_node_id"), gpu_index, allocation_status, utcnow()),
)
def _upsert_checkpoints(self, conn: PgConnection, task_id: str, checkpoints: list[dict[str, Any]]) -> None:
for item in checkpoints:
path = str(item.get("path") or "")
if not path:
continue
step = int(item.get("step") or 0)
name = str(item.get("name") or Path(path).name)
size_bytes = int(item.get("size_bytes") or item.get("size") or 0)
existing = conn.execute("SELECT id FROM fine_tune_checkpoints WHERE task_id=? AND path=?", (task_id, path)).fetchone()
if existing:
conn.execute(
"UPDATE fine_tune_checkpoints SET step=?, name=?, size_bytes=? WHERE id=?",
(step, name, size_bytes, existing["id"]),
)
continue
conn.execute(
"""
INSERT INTO fine_tune_checkpoints
(id, task_id, step, name, path, size_bytes, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(new_id("ckpt"), task_id, step, name, path, size_bytes, utcnow()),
)
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
status_map = {
"queued": "queued",
"running": "running",
"completed": "completed",
"failed": "failed",
"stopped": "stopped",
}
current = self.task(task_id)
status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"]))
progress = int(job.get("progress", current.get("progress", 0)) or 0)
payload = {
**current,
"status": status,
"progress": progress,
"process_id": job.get("pid") or current.get("process_id"),
"compute_job_id": job.get("id") or current.get("compute_job_id"),
"output_dir": job.get("output_dir") or current.get("output_dir"),
"log_file": job.get("log_file") or current.get("log_file"),
"artifacts": job.get("artifacts") or current.get("artifacts") or [],
}
if status == "failed":
payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed"
elif status in {"queued", "running", "completed"}:
payload.pop("failure_reason", None)
completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None
with self.connect() as conn:
conn.execute(
"""
UPDATE fine_tune_tasks
SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at)
WHERE id=?
""",
(
json_dumps(payload),
status,
progress,
payload.get("process_id"),
payload.get("compute_job_id"),
completed_at,
task_id,
),
)
self._upsert_compute_job(conn, payload, job, status)
self._sync_gpu_allocations(conn, payload, job, status)
self._upsert_checkpoints(conn, task_id, job.get("checkpoints") or [])
if status == "completed":
self._ensure_trained_model(conn, payload)
if status in {"failed", "stopped"}:
failure_reason = job.get("error") or job.get("message") or "compute job failed"
log_snippet = job.get("log_snippet") or ""
conn.execute(
"UPDATE fine_tune_tasks SET failure_reason = ? WHERE id = ?",
(failure_reason[:2000], task_id),
)
if log_snippet:
conn.execute(
"UPDATE fine_tune_tasks SET payload = ? WHERE id = ?",
(json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id),
)
return self.task(task_id)
def running_compute_tasks(self) -> list[dict[str, Any]]:
return [
task
for task in self.tasks()
if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"}
]
def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]:
task = self.task(task_id)
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason})
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
(json_dumps(task), utcnow(), task_id),
)
conn.execute(
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
(utcnow(), task_id),
)
return self.task(task_id)
def record_training_log_metrics(self, task_id: str, content: str) -> int:
rows: list[tuple[Any, ...]] = []
for line_number, line in enumerate(content.splitlines(), start=1):
metric = self._parse_training_metric(line)
if not metric:
continue
rows.append(
(
new_id("metric"),
task_id,
line_number,
metric.get("epoch"),
metric.get("loss"),
metric.get("grad_norm"),
metric.get("learning_rate"),
line[:2000],
utcnow(),
)
)
with self.connect() as conn:
conn.execute("DELETE FROM fine_tune_metrics WHERE task_id=?", (task_id,))
if rows:
conn.executemany(
"""
INSERT INTO fine_tune_metrics
(id, task_id, step, epoch, loss, grad_norm, learning_rate, raw, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
rows,
)
return len(rows)
def _parse_training_metric(self, line: str) -> dict[str, Any] | None:
if "loss" not in line or "learning_rate" not in line:
return None
import re
result: dict[str, Any] = {}
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
if match:
result[key] = float(match.group(1))
return result or None
def task_metrics(self, task_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT step, epoch, loss, grad_norm, learning_rate, raw, create_time
FROM fine_tune_metrics
WHERE task_id=?
ORDER BY step
""",
(task_id,),
).fetchall()
return [dict(row) for row in rows]
def task_checkpoints(self, task_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT id, step, name, path, size_bytes, create_time
FROM fine_tune_checkpoints
WHERE task_id=?
ORDER BY step, create_time
""",
(task_id,),
).fetchall()
return [dict(row) for row in rows]
def active_standalone_compute_jobs(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT * FROM compute_jobs
WHERE task_id IS NULL AND status IN ('queued','running')
ORDER BY create_time
""",
).fetchall()
return [json_loads(row["payload"], {}) if "payload" in row.keys() else dict(row) for row in rows]
def sync_model_merge_job(self, job_id: str, job: dict[str, Any]) -> dict[str, Any]:
current = self.compute_job(job_id)
payload = current.get("payload") or {}
job_payload = payload.get("job") if isinstance(payload.get("job"), dict) else {}
merged_payload = {**payload, "job": {**job_payload, **job}}
status = str(job.get("status") or current.get("status") or "queued")
command = job.get("command") or current.get("command") or []
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
output_dir = job.get("output_dir") or payload.get("output_dir") or current.get("output_dir")
with self.connect() as conn:
conn.execute(
"""
UPDATE compute_jobs
SET status=?, command=?, output_dir=?, log_file=?, payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
WHERE id=?
""",
(
status,
command_text,
output_dir,
job.get("log_file") or current.get("log_file"),
json_dumps(merged_payload),
utcnow(),
utcnow() if status in {"completed", "failed", "stopped"} else None,
job_id,
),
)
return self.compute_job(job_id)
_store: PlatformStore | None = None

64
backend/app/db/sql/001_platform_runtime.sql
View File

@@ -152,6 +152,18 @@ CREATE TABLE IF NOT EXISTS project_members (
-- ===================== Fine-tune Checkpoints =====================
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
step INTEGER NOT NULL,
epoch DOUBLE PRECISION,
loss DOUBLE PRECISION,
grad_norm DOUBLE PRECISION,
learning_rate DOUBLE PRECISION,
raw TEXT NOT NULL,
create_time TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
@@ -164,6 +176,25 @@ CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
create_time TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS gpu_allocations (
id TEXT PRIMARY KEY,
task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
compute_job_id TEXT,
node_id TEXT REFERENCES compute_nodes(id) ON DELETE CASCADE,
gpu_index INTEGER NOT NULL,
status TEXT NOT NULL,
create_time TEXT NOT NULL,
released_at TEXT
);
CREATE TABLE IF NOT EXISTS scheduler_locks (
lock_key TEXT PRIMARY KEY,
owner TEXT NOT NULL,
expires_at TEXT NOT NULL,
create_time TEXT NOT NULL,
update_time TEXT NOT NULL
);
-- ===================== Compute Jobs (internal) =====================
CREATE TABLE IF NOT EXISTS compute_jobs (
@@ -185,6 +216,33 @@ CREATE TABLE IF NOT EXISTS compute_jobs (
completed_at TEXT
);
-- ===================== Model Evaluation =====================
CREATE TABLE IF NOT EXISTS eval_tasks (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL,
create_time TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS eval_dimensions (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
payload TEXT NOT NULL,
is_active INTEGER NOT NULL DEFAULT 1,
is_default INTEGER NOT NULL DEFAULT 0,
create_time TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS compare_tasks (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL,
create_time TEXT NOT NULL
);
-- ===================== Indexes =====================
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
@@ -194,9 +252,15 @@ CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_t
CREATE INDEX IF NOT EXISTS idx_projects_tenant ON projects(tenant_id);
CREATE INDEX IF NOT EXISTS idx_project_members_user ON project_members(user_id);
CREATE INDEX IF NOT EXISTS idx_checkpoints_task ON fine_tune_checkpoints(task_id);
CREATE INDEX IF NOT EXISTS idx_fine_tune_metrics_task_step ON fine_tune_metrics(task_id, step);
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node ON compute_jobs(node_id);
CREATE INDEX IF NOT EXISTS idx_compute_jobs_status ON compute_jobs(status);
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
-- ===================== Migrations: extend fine_tune_tasks =====================

13
backend/app/db/sql/002_governance.sql
View File

@@ -127,3 +127,16 @@ INSERT INTO roles (id, name, display_name, permissions, create_time) VALUES
('role_operator','operator','操作员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', '2026-01-01T00:00:00Z'),
('role_viewer', 'viewer', '访客', '["dashboard"]', '2026-01-01T00:00:00Z')
ON CONFLICT (name) DO NOTHING;
-- 登录会话：采集每次登录/登出，用于统计在线/登录时长
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
username TEXT NOT NULL,
display_name TEXT,
role TEXT,
login_at TEXT NOT NULL,
logout_at TEXT,
duration_seconds INTEGER,
create_time TEXT NOT NULL
);

13
backend/app/main.py
View File

@@ -1,3 +1,5 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
@@ -6,11 +8,20 @@ from app.core.config import get_settings
from app.core.logging import configure_logging, setup_request_logging
@asynccontextmanager
async def _lifespan(app: FastAPI):
# 启动算力状态轮询线程（同步派发到算力的训练任务状态/日志/指标）
from app.modules.fine_tune.service import start_compute_sync_worker, stop_compute_sync_worker
start_compute_sync_worker()
yield
stop_compute_sync_worker()
def create_app() -> FastAPI:
settings = get_settings()
configure_logging(settings)
app = FastAPI(title=settings.app_name)
app = FastAPI(title=settings.app_name, lifespan=_lifespan)
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_allow_origins,

16
backend/app/modules/auth/router.py
View File

@@ -16,13 +16,25 @@ class LoginBody(BaseModel):
password: str
class LogoutBody(BaseModel):
session_id: str | None = None
@router.post("/login")
def login(body: LoginBody) -> dict:
user = get_platform_store().login(body.username, body.password)
store = get_platform_store()
user = store.login(body.username, body.password)
if not user:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
token = create_access_token(user["id"])
return {"code": 0, "message": "ok", "data": {"token": token, "user": user}}
session_id = store.create_session(user)
return {"code": 0, "message": "ok", "data": {"token": token, "user": user, "session_id": session_id}}
@router.post("/logout")
def logout(body: LogoutBody) -> dict:
get_platform_store().close_session(body.session_id)
return {"code": 0, "message": "ok", "data": None}
@router.get("/me")

5
backend/app/modules/compute_gateway/__init__.py
View File

@@ -1 +1,6 @@
"""Application-side compute platform gateway module."""
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import poll_compute_jobs_once
__all__ = ["ComputeNodeClient", "poll_compute_jobs_once"]

180
backend/app/modules/compute_gateway/client.py Normal file
View File

@@ -0,0 +1,180 @@
"""Client for talking to a compute node's REST API.
This is the application-side bridge: the application backend never runs GPU
workloads itself; it dispatches them to a compute node and reads back status,
logs and metrics through this client.
"""
from __future__ import annotations
import httpx
from app.core.config import get_settings
def _compute_timeout() -> float:
return float(get_settings().compute_request_timeout_seconds or 30.0)
def _auth_headers() -> dict:
token = get_settings().compute_service_token
if not token:
return {}
return {"Authorization": f"Bearer {token}"}
class ComputeNodeClient:
"""Thin wrapper over a single compute node's HTTP API."""
def __init__(self, base_url: str, timeout: float | None = None):
self.base_url = base_url.rstrip("/")
self.timeout = timeout or _compute_timeout()
# ---- health -------------------------------------------------------
def health(self) -> dict:
last_err = None
for path in ("/v1/compute/health", "/health"):
try:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(f"{self.base_url}{path}", headers=_auth_headers())
if r.status_code == 200:
return r.json()
except Exception as exc: # noqa: BLE001
last_err = str(exc)
raise RuntimeError(f"compute node unhealthy: {last_err}")
# ---- gpus ---------------------------------------------------------
def gpus(self) -> list:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/resources/gpus",
headers=_auth_headers(),
)
r.raise_for_status()
return r.json().get("data", [])
# ---- jobs ---------------------------------------------------------
def create_job(self, payload: dict) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs",
json=payload,
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def preview_job(self, payload: dict) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs/preview",
json=payload,
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def validate_job(self, payload: dict) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs/validate",
json=payload,
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def get_job(self, job_id: str) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/jobs/{job_id}",
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def stop_job(self, job_id: str) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs/{job_id}/stop",
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def job_logs(self, job_id: str, cursor: int = 0, limit: int = 200) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/jobs/{job_id}/logs",
params={"cursor": cursor, "limit": limit},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
# ---- files --------------------------------------------------------
def check_paths(self, paths: list) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/files/check-paths",
json={"paths": paths},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def list_files(self, path: str = "/") -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/files/list",
params={"path": path},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def import_local_file(self, src_path: str, dest_name: str | None = None) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/files/import-local",
json={"src_path": src_path, "dest_name": dest_name},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def upload_file(self, filename: str, content: bytes, content_type: str | None = None) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/files/upload",
files={"file": (filename, content, content_type)},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
# ---- inference (async) -------------------------------------------
def headers(self) -> dict[str, str]:
"""Return auth headers for compute node requests."""
token = get_settings().compute_service_token
if not token:
return {}
return {"X-Compute-Token": token}
@property
def route_prefix(self) -> str:
return get_settings().route_prefix.rstrip("/") or "/modelTF"
async def _request(self, method: str, path: str, json_data: dict | None = None) -> dict:
"""Generic async request method for compute API endpoints."""
prefix = self.route_prefix
url = f"{self.base_url.rstrip('/')}{prefix}{path}"
async with httpx.AsyncClient(timeout=300, verify=False, headers=self.headers()) as client:
if method.upper() == "GET":
response = await client.get(url)
else:
response = await client.post(url, json=json_data)
response.raise_for_status()
data = response.json()
if isinstance(data, dict) and isinstance(data.get("data"), dict):
return data["data"]
return data if isinstance(data, dict) else {}

50
backend/app/modules/compute_gateway/sync.py Normal file
View File

@@ -0,0 +1,50 @@
from __future__ import annotations
from typing import Any
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
async def poll_compute_jobs_once() -> dict[str, Any]:
store = get_platform_store()
synced: list[dict[str, Any]] = []
failed: list[dict[str, str]] = []
for task in store.running_compute_tasks():
node = _node_for_task(task)
if not node:
failed.append({"task_id": task["id"], "error": "compute node not found"})
continue
try:
client = ComputeNodeClient(node["api_base_url"])
job = client.get_job(task["compute_job_id"])
try:
logs = client.job_logs(task["compute_job_id"], tail_lines=5000)
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
except Exception:
pass
if job.get("status") in {"failed", "stopped"}:
try:
last_logs = client.job_logs(task["compute_job_id"], tail_lines=200)
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
for record in store.active_standalone_compute_jobs():
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
if not node:
failed.append({"job_id": record["id"], "error": "compute node not found"})
continue
try:
job = ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"job_id": record["id"], "error": str(exc)})
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}

52
backend/app/modules/fine_tune/service.py
View File

@@ -3,13 +3,17 @@
- preset 参数预设（quick / standard / high）
- train_type → stage 映射（sft/dpo/cpt/cot）
- 训练任务的启动 / 暂停 / 恢复 / 取消（委托 runner 真实执行）
- 训练任务的启动 / 暂停 / 恢复 / 取消
- compute_gateway 状态轮询线程（当任务派发到算力时自动同步状态）
"""
from __future__ import annotations
import asyncio
import threading
import time
from typing import Any
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.fine_tune import runner
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
}
# ── compute sync 轮询线程 ──────────────────────────────────────────────
_sync_thread: threading.Thread | None = None
_sync_thread_stop = threading.Event()
def _compute_sync_loop() -> None:
"""后台线程：周期性轮询算力节点，同步训练任务状态/日志/指标。"""
interval = get_settings().compute_poll_interval_seconds or 3
while not _sync_thread_stop.is_set():
try:
store = get_platform_store()
running = store.running_compute_tasks()
if running:
asyncio.run(_poll_once())
except Exception: # noqa: BLE001 - keep polling loop alive
pass
time.sleep(interval)
async def _poll_once() -> None:
from app.modules.compute_gateway.sync import poll_compute_jobs_once
await poll_compute_jobs_once()
def start_compute_sync_worker() -> None:
"""启动后台轮询线程（幂等，多次调用安全）。"""
global _sync_thread
if _sync_thread is not None and _sync_thread.is_alive():
return
_sync_thread_stop.clear()
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
_sync_thread.start()
def stop_compute_sync_worker() -> None:
"""停止后台轮询线程。"""
_sync_thread_stop.set()
# ── preset / config ────────────────────────────────────────────────────
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
"""根据 preset 字段补全缺失的超参；preset=custom 时不覆盖。"""
payload = dict(payload)
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
def launch_training(task_id: str) -> None:
"""在后台线程启动真实训练。"""
"""在后台线程启动真实训练（本机 subprocess fallback，当算力节点不可用时使用）。
架构原则：GPU 计算应派发到算力服务进程执行。
当 platform_store.start_task 检测到在线算力节点时，会走 _dispatch_to_compute 派发路径；
仅当无可用算力节点且非 simulator 模式时，降级到本机 runner（违反 §1.1，待移除）。
"""
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()

5
backend/data/dataset_info.json Normal file
View File

@@ -0,0 +1,5 @@
{
"ds_0b95db886181": {
"file_name": "fine_tune_datasets/ds_0b95db886181.json"
}
}

16
backend/data/fine_tune_datasets/ds_0b95db886181.json Normal file
View File

@@ -0,0 +1,16 @@
[ {
"instruction": "数字钱包的“零余额”管理在日末结算时有何具体要求？",
"input": "",
"output": "数字钱包严格执行“零余额”管理，在日末结算时，若结算钱包存在余额，必须将数字货币兑回基本账户，以确保日末数字钱包无余额。"
},
{
"instruction": "用印登记表中包含哪些具体的印章类型？",
"input": "",
"output": "根据用印登记表的内容显示，表中明确列出了五种具体的印章类型，分别是财务专用章、法人名章、法人授权人名章1、法人授权人名章2以及其他。这些印章类型被详细划分在“用印数量”这一栏目下，用于记录不同印章的具体使用情况。"
},
{
"instruction": "密钥交接记录表中需要哪些角色签字确认？",
"input": "",
"output": "密钥交接记录表中需要三个角色的签字确认，分别是交接人签字、接交人签字以及监交人签字。交接人负责移交密钥，接交人负责接收密钥，而监交人则负责对整个密钥交接过程进行监督，这三个角色的共同签字确认能够确保密钥交接流程的规范性与安全性。"
}
]

16
backend/storage/data-process/dpt_4a693794cb6f4488a156/dpsf_6fbb337dd6a146159e7a/v1/111.json Normal file
View File

@@ -0,0 +1,16 @@
[ {
"instruction": "数字钱包的“零余额”管理在日末结算时有何具体要求？",
"input": "",
"output": "数字钱包严格执行“零余额”管理，在日末结算时，若结算钱包存在余额，必须将数字货币兑回基本账户，以确保日末数字钱包无余额。"
},
{
"instruction": "用印登记表中包含哪些具体的印章类型？",
"input": "",
"output": "根据用印登记表的内容显示，表中明确列出了五种具体的印章类型，分别是财务专用章、法人名章、法人授权人名章1、法人授权人名章2以及其他。这些印章类型被详细划分在“用印数量”这一栏目下，用于记录不同印章的具体使用情况。"
},
{
"instruction": "密钥交接记录表中需要哪些角色签字确认？",
"input": "",
"output": "密钥交接记录表中需要三个角色的签字确认，分别是交接人签字、接交人签字以及监交人签字。交接人负责移交密钥，接交人负责接收密钥，而监交人则负责对整个密钥交接过程进行监督，这三个角色的共同签字确认能够确保密钥交接流程的规范性与安全性。"
}
]