update
This commit is contained in:
@@ -21,6 +21,7 @@ import psycopg
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
BackgroundTasks,
|
||||
Request,
|
||||
Body,
|
||||
Depends,
|
||||
File,
|
||||
@@ -76,6 +77,7 @@ from app.modules.data_process.store import (
|
||||
get_data_process_store,
|
||||
new_id,
|
||||
)
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.schemas.data_process import (
|
||||
DataProcessRegenerateRequest,
|
||||
DataProcessStatus,
|
||||
@@ -148,6 +150,13 @@ def fail(status_code: int, message: str) -> HTTPException:
|
||||
)
|
||||
|
||||
|
||||
def _actor(request: Request | None) -> str | None:
|
||||
if not request:
|
||||
return None
|
||||
auth = request.headers.get("Authorization", "")
|
||||
return auth.replace("Bearer ", "").strip() or None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def api_errors() -> Iterator[None]:
|
||||
try:
|
||||
@@ -790,9 +799,16 @@ def list_tasks(
|
||||
def create_task(
|
||||
payload: DataProcessTaskCreate,
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
task = store.create_task(payload.model_dump(mode="json"))
|
||||
get_platform_store().record_audit(
|
||||
action="data-process.create",
|
||||
actor_id=_actor(request),
|
||||
target_type="data_process_task",
|
||||
target_id=task["id"],
|
||||
)
|
||||
return ok(task, "data process task created")
|
||||
|
||||
|
||||
@@ -854,9 +870,16 @@ def prepare_regeneration(
|
||||
def delete_task(
|
||||
task_id: str,
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
store.delete_task(task_id)
|
||||
get_platform_store().record_audit(
|
||||
action="data-process.delete",
|
||||
actor_id=_actor(request),
|
||||
target_type="data_process_task",
|
||||
target_id=task_id,
|
||||
)
|
||||
return ok({"deleted": task_id}, "data process task deleted")
|
||||
|
||||
|
||||
@@ -875,6 +898,7 @@ async def upload_source_files(
|
||||
files: list[UploadFile] = File(...),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
if not files:
|
||||
raise fail(400, "at least one source file is required")
|
||||
@@ -959,6 +983,12 @@ async def upload_source_files(
|
||||
finally:
|
||||
if not commit_attempted:
|
||||
storage.discard(staged)
|
||||
get_platform_store().record_audit(
|
||||
action="data-process.upload",
|
||||
actor_id=_actor(request),
|
||||
target_type="data_process_task",
|
||||
target_id=task_id,
|
||||
)
|
||||
return ok({"files": created}, "source files uploaded")
|
||||
|
||||
|
||||
@@ -1701,7 +1731,14 @@ def generate(
|
||||
background_tasks: BackgroundTasks,
|
||||
payload: GenerateRequest = Body(default_factory=GenerateRequest),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
get_platform_store().record_audit(
|
||||
action="data-process.generate",
|
||||
actor_id=_actor(request),
|
||||
target_type="data_process_task",
|
||||
target_id=task_id,
|
||||
)
|
||||
return _start_generation(task_id, payload, background_tasks, store)
|
||||
|
||||
|
||||
@@ -2254,8 +2291,15 @@ def publish(
|
||||
task_id: str,
|
||||
payload: PublishRequest,
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
result = store.publish(task_id, payload.model_dump(mode="json"))
|
||||
message = "dataset published" if result["created"] else "dataset already published"
|
||||
get_platform_store().record_audit(
|
||||
action="data-process.publish",
|
||||
actor_id=_actor(request),
|
||||
target_type="data_process_task",
|
||||
target_id=task_id,
|
||||
)
|
||||
return ok(result, message)
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import PlainTextResponse, StreamingResponse
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.fine_tune.service import apply_presets
|
||||
from fastapi import Request as FastAPIRequest
|
||||
|
||||
@@ -28,6 +31,15 @@ def fail(status_code: int, message: str) -> HTTPException:
|
||||
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
||||
|
||||
|
||||
def _select_first_online_node(store: Any) -> dict[str, Any] | None:
|
||||
"""Select the first online compute node for inference."""
|
||||
nodes = store.compute_nodes()
|
||||
for node in nodes:
|
||||
if node.get("enabled") and node.get("scheduler_status") == "online":
|
||||
return node
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/dashboard/overview")
|
||||
async def dashboard_overview() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
@@ -109,19 +121,29 @@ async def dashboard_stats() -> dict[str, Any]:
|
||||
"error": "failed",
|
||||
"cancelled": "failed",
|
||||
}
|
||||
op_labels = [
|
||||
("模型训练", lambda a: "fine_tune" in a or "train" in a),
|
||||
("数据处理", lambda a: "data" in a or "dataset" in a),
|
||||
("模型评测", lambda a: "eval" in a),
|
||||
("模型推理", lambda a: "infer" in a or "serving" in a or "deploy" in a),
|
||||
("系统设置", lambda a: True),
|
||||
# 用户操作分布:仅展示「数据治理」与「模型服务」两大分组下的子模块,其他不显示
|
||||
MODULE_LABELS = [
|
||||
("data-process", "数据处理"),
|
||||
("data_process", "数据处理"),
|
||||
("dataset", "数据集管理"),
|
||||
("fine-tune", "模型训练"),
|
||||
("fine_tune", "模型训练"),
|
||||
("model-eval", "模型评测"),
|
||||
("eval", "模型评测"),
|
||||
("model-inference", "模型推理"),
|
||||
("inference", "模型推理"),
|
||||
("model-manage", "模型管理"),
|
||||
("model", "模型管理"),
|
||||
("trained", "模型管理"),
|
||||
]
|
||||
OP_ORDER = ["数据集管理", "数据处理", "模型训练", "模型评测", "模型推理", "模型管理"]
|
||||
|
||||
def _op_label(action: str) -> str:
|
||||
for label, fn in op_labels:
|
||||
if fn(action):
|
||||
def _op_module(action: str) -> str | None:
|
||||
a = (action or "").lower()
|
||||
for prefix, label in MODULE_LABELS:
|
||||
if a.startswith(prefix):
|
||||
return label
|
||||
return "系统设置"
|
||||
return None
|
||||
|
||||
training_tasks = [
|
||||
{
|
||||
@@ -138,12 +160,13 @@ async def dashboard_stats() -> dict[str, Any]:
|
||||
for t in tasks[:8]
|
||||
]
|
||||
|
||||
# 用户操作分布(按 audit action 归类为中文分类)
|
||||
audit = store.audit_logs(limit=500)
|
||||
op_counter: dict[str, int] = {}
|
||||
# 用户操作分布:仅统计数据治理/模型服务下子模块的操作,其他不显示
|
||||
audit = store.audit_logs(limit=1000)
|
||||
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
|
||||
for log in audit.get("items", []):
|
||||
act = log.get("action") or "unknown"
|
||||
op_counter[_op_label(act)] = op_counter.get(_op_label(act), 0) + 1
|
||||
label = _op_module(log.get("action") or "")
|
||||
if label:
|
||||
op_counter[label] += 1
|
||||
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
|
||||
|
||||
# 最近登录用户:后端有 last_login 字段,返回真实数据
|
||||
@@ -160,8 +183,8 @@ async def dashboard_stats() -> dict[str, Any]:
|
||||
}
|
||||
for u in recent
|
||||
]
|
||||
# 登录时长:后端暂无该数据源,先留空,待接入后补充
|
||||
login_duration_rank: list = []
|
||||
# 登录时长排行:基于 sessions 表真实会话时长(本月)
|
||||
login_duration_rank = store.login_duration_rank()
|
||||
|
||||
return ok(
|
||||
{
|
||||
@@ -571,7 +594,28 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/test-connection")
|
||||
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
||||
return ok({"node_id": node_id, "success": True, "latency_ms": 12})
|
||||
store = get_platform_store()
|
||||
nodes = store.compute_nodes()
|
||||
node = next((item for item in nodes if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
try:
|
||||
all_gpus = store.gpus()
|
||||
node_gpus = [g for g in all_gpus if g.get("node_id") == node_id]
|
||||
return ok({
|
||||
"node_id": node_id,
|
||||
"success": True,
|
||||
"latency_ms": 12,
|
||||
"gpu_count": len(node_gpus),
|
||||
})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return ok({
|
||||
"node_id": node_id,
|
||||
"success": False,
|
||||
"latency_ms": 0,
|
||||
"gpu_count": 0,
|
||||
"error": str(exc),
|
||||
})
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/enable")
|
||||
@@ -842,3 +886,287 @@ async def compute_job_logs(job_id: str) -> dict[str, Any]:
|
||||
except KeyError as exc:
|
||||
raise fail(404, str(exc))
|
||||
|
||||
|
||||
# ===================== Model Evaluation =====================
|
||||
|
||||
|
||||
@router.get("/model-eval")
|
||||
async def model_eval_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().eval_tasks())
|
||||
|
||||
|
||||
@router.get("/model-eval/{task_id}")
|
||||
async def model_eval_detail(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().eval_task(task_id))
|
||||
except KeyError:
|
||||
raise fail(404, "eval task not found")
|
||||
|
||||
|
||||
@router.post("/model-eval/start")
|
||||
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
task = get_platform_store().create_eval_task(payload)
|
||||
return ok({"task_id": task["id"], **task})
|
||||
|
||||
|
||||
@router.delete("/model-eval/{task_id}")
|
||||
async def model_eval_delete(task_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_eval_task(task_id)
|
||||
return ok({"deleted": task_id})
|
||||
|
||||
|
||||
# ===================== Eval Dimensions =====================
|
||||
|
||||
|
||||
@router.get("/dimension")
|
||||
async def dimension_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().dimensions())
|
||||
|
||||
|
||||
@router.post("/dimension")
|
||||
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok(get_platform_store().create_dimension(payload))
|
||||
|
||||
|
||||
@router.get("/dimension/{dimension_id}")
|
||||
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().dimension(dimension_id))
|
||||
except KeyError:
|
||||
raise fail(404, "dimension not found")
|
||||
|
||||
|
||||
@router.put("/dimension/{dimension_id}")
|
||||
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_dimension(dimension_id, payload))
|
||||
except KeyError:
|
||||
raise fail(404, "dimension not found")
|
||||
|
||||
|
||||
@router.delete("/dimension/{dimension_id}")
|
||||
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_dimension(dimension_id)
|
||||
return ok({"deleted": dimension_id})
|
||||
|
||||
|
||||
# ===================== Model Compare / Inference =====================
|
||||
|
||||
|
||||
@router.get("/model-compare")
|
||||
async def model_compare_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().compare_tasks())
|
||||
|
||||
|
||||
@router.post("/model-compare")
|
||||
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
task = get_platform_store().create_compare_task(payload)
|
||||
return ok({"id": task["id"]})
|
||||
|
||||
|
||||
@router.post("/model-compare/all/stop-all")
|
||||
async def model_compare_stop_all() -> dict[str, Any]:
|
||||
return ok({"stopped": True})
|
||||
|
||||
|
||||
@router.post("/model-compare/stop-by-pid")
|
||||
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"stopped": True, "pid": payload.get("pid")})
|
||||
|
||||
|
||||
@router.get("/model-compare/{task_id}")
|
||||
async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().compare_task(task_id))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.delete("/model-compare/{task_id}")
|
||||
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_compare_task(task_id)
|
||||
return ok({"deleted": task_id})
|
||||
|
||||
|
||||
@router.get("/model-compare/{task_id}/load-status")
|
||||
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
task = get_platform_store().compare_task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
load_status = task.get("load_status") or {"loaded_models": []}
|
||||
if isinstance(load_status, str):
|
||||
try:
|
||||
load_status = json.loads(load_status)
|
||||
except json.JSONDecodeError:
|
||||
load_status = {"loaded_models": []}
|
||||
return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/load-status")
|
||||
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/load")
|
||||
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
task = get_platform_store().compare_task(task_id)
|
||||
models = task.get("models") or []
|
||||
if isinstance(models, str):
|
||||
try:
|
||||
models = json.loads(models)
|
||||
except json.JSONDecodeError:
|
||||
models = []
|
||||
loaded_models = [
|
||||
{
|
||||
"model_id": item.get("model_id"),
|
||||
"model_name": item.get("model_name"),
|
||||
"status": "ready",
|
||||
"pid": 45000 + index,
|
||||
"port": item.get("port") or 18000 + index,
|
||||
}
|
||||
for index, item in enumerate(models)
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/unload")
|
||||
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/start-model")
|
||||
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})
|
||||
|
||||
|
||||
@router.post("/model-compare/chat-with-port")
|
||||
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
question = ""
|
||||
for message in payload.get("messages") or []:
|
||||
if message.get("role") == "user":
|
||||
question = str(message.get("content") or "")
|
||||
content = f"当前后端已收到推理请求:{question[:120]}"
|
||||
return ok({"response": content, "content": content})
|
||||
|
||||
|
||||
@router.post("/model-compare/stream-chat")
|
||||
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
question = payload.get("user_question") or payload.get("question") or ""
|
||||
return ok({"response": f"当前后端已收到流式推理请求:{str(question)[:120]}"})
|
||||
|
||||
|
||||
# ===================== Model Chat (Inference Proxy) =====================
|
||||
|
||||
|
||||
@router.post("/model-chat/batch")
|
||||
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"responses": [], "request": payload})
|
||||
|
||||
|
||||
@router.post("/model-chat/local/chat")
|
||||
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
"""Proxy chat to the compute node running the inference model."""
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return ok({"response": "no online compute node available for inference", "request": payload})
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
result = await client._request("POST", "/inference/chat", json_data=payload)
|
||||
return ok(result)
|
||||
except Exception as exc:
|
||||
return ok({"response": f"inference failed: {exc}", "request": payload})
|
||||
|
||||
|
||||
@router.post("/model-chat/local/chat/stream")
|
||||
async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
|
||||
"""Stream chat from the compute node."""
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return StreamingResponse(
|
||||
iter(['data: {"error": "no online compute node"}\n\n']),
|
||||
media_type="text/event-stream",
|
||||
)
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
|
||||
async def stream_proxy():
|
||||
async with httpx.AsyncClient(timeout=300) as http:
|
||||
url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream"
|
||||
async with http.stream("POST", url, json=payload, headers=client.headers()) as resp:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(stream_proxy(), media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.post("/model-chat/local/preload")
|
||||
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
"""Load a model on the compute node for inference."""
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return ok({"loaded": False, "error": "no online compute node"})
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
result = await client._request("POST", "/inference/load", json_data=payload)
|
||||
return ok(result)
|
||||
except Exception as exc:
|
||||
return ok({"loaded": False, "error": str(exc)})
|
||||
|
||||
|
||||
@router.post("/model-chat/local/unload")
|
||||
async def model_chat_local_unload() -> dict[str, Any]:
|
||||
"""Unload the inference model from the compute node."""
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return ok({"unloaded": False, "error": "no online compute node"})
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
result = await client._request("POST", "/inference/unload", json_data={})
|
||||
return ok(result)
|
||||
except Exception as exc:
|
||||
return ok({"unloaded": False, "error": str(exc)})
|
||||
|
||||
|
||||
@router.get("/model-chat/local/status")
|
||||
async def model_chat_local_status() -> dict[str, Any]:
|
||||
"""Get inference session status from compute node."""
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return ok({"loaded": False, "error": "no online compute node"})
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
result = await client._request("GET", "/inference/status")
|
||||
return ok(result)
|
||||
except Exception as exc:
|
||||
return ok({"loaded": False, "error": str(exc)})
|
||||
|
||||
|
||||
@router.post("/model-chat/trained/preload")
|
||||
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
"""Load a trained model (base + adapter) on the compute node for inference."""
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return ok({"loaded": False, "error": "no online compute node"})
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
result = await client._request("POST", "/inference/load", json_data=payload)
|
||||
return ok(result)
|
||||
except Exception as exc:
|
||||
return ok({"loaded": False, "error": str(exc)})
|
||||
|
||||
|
||||
@@ -28,6 +28,8 @@ class Settings:
|
||||
compute_mode: str = os.getenv("COMPUTE_MODE", "real")
|
||||
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
|
||||
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
|
||||
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||
compute_request_timeout_seconds: float = float(os.getenv("COMPUTE_REQUEST_TIMEOUT_SECONDS", "30"))
|
||||
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
||||
log_dir: str = os.getenv("LOG_DIR", "./logs")
|
||||
log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend")
|
||||
|
||||
@@ -229,15 +229,21 @@ class PlatformStore:
|
||||
return f"{sec}s"
|
||||
|
||||
def refresh_runtime_state(self) -> None:
|
||||
if get_settings().compute_mode != "simulator":
|
||||
return
|
||||
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
|
||||
).fetchall()
|
||||
now_dt = datetime.now(timezone.utc)
|
||||
for row in rows:
|
||||
payload = json_loads(row["payload"], {})
|
||||
compute_job_id = payload.get("compute_job_id")
|
||||
compute_node_api = payload.get("compute_node_api")
|
||||
if compute_job_id and compute_node_api:
|
||||
# 已派发到算力:状态/进度/日志回传来自算力进程(架构 §1.1)
|
||||
self._sync_task_from_compute(conn, row, payload, compute_job_id, compute_node_api)
|
||||
continue
|
||||
if get_settings().compute_mode != "simulator":
|
||||
continue
|
||||
start = parse_time(row["start_time"])
|
||||
if not start:
|
||||
continue
|
||||
@@ -252,7 +258,6 @@ class PlatformStore:
|
||||
else:
|
||||
status, progress = "completed", 100
|
||||
|
||||
payload = json_loads(row["payload"], {})
|
||||
payload.update(
|
||||
{
|
||||
"status": status,
|
||||
@@ -272,19 +277,63 @@ class PlatformStore:
|
||||
if status == "completed":
|
||||
self._ensure_trained_model(conn, payload)
|
||||
|
||||
sync_rows = conn.execute(
|
||||
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
|
||||
).fetchall()
|
||||
for row in sync_rows:
|
||||
created = parse_time(row["create_time"])
|
||||
age = int((now_dt - created).total_seconds()) if created else 0
|
||||
status = "completed" if age >= 6 else "running"
|
||||
progress = 100 if status == "completed" else min(95, 15 + age * 12)
|
||||
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
|
||||
conn.execute(
|
||||
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?",
|
||||
(status, progress, completed_at, row["id"]),
|
||||
)
|
||||
if get_settings().compute_mode == "simulator":
|
||||
sync_rows = conn.execute(
|
||||
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
|
||||
).fetchall()
|
||||
for row in sync_rows:
|
||||
created = parse_time(row["create_time"])
|
||||
age = int((now_dt - created).total_seconds()) if created else 0
|
||||
status = "completed" if age >= 6 else "running"
|
||||
progress = 100 if status == "completed" else min(95, 15 + age * 12)
|
||||
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
|
||||
conn.execute(
|
||||
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?",
|
||||
(status, progress, completed_at, row["id"]),
|
||||
)
|
||||
|
||||
def _sync_task_from_compute(
|
||||
self,
|
||||
conn,
|
||||
row,
|
||||
payload: dict[str, Any],
|
||||
compute_job_id: str,
|
||||
compute_node_api: str,
|
||||
) -> None:
|
||||
"""从算力节点拉回已派发任务的状态/进度/日志,写回本地任务记录。"""
|
||||
try:
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
job = ComputeNodeClient(compute_node_api).get_job(compute_job_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
payload["compute_sync_error"] = str(exc)
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
|
||||
(json_dumps(payload), row["id"]),
|
||||
)
|
||||
return
|
||||
status = job.get("status")
|
||||
progress = int(job.get("progress", 0) or 0)
|
||||
logs = job.get("logs") or ""
|
||||
payload.update(
|
||||
{
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"compute_logs": logs,
|
||||
"train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None),
|
||||
}
|
||||
)
|
||||
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET status=?, progress=?, payload=?, completed_at=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(status, progress, json_dumps(payload), completed_at, row["id"]),
|
||||
)
|
||||
if status == "completed":
|
||||
self._ensure_trained_model(conn, payload)
|
||||
|
||||
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None:
|
||||
name = task.get("output_model_name") or f"{task['name']}-lora"
|
||||
@@ -306,7 +355,7 @@ class PlatformStore:
|
||||
utcnow(),
|
||||
0,
|
||||
0,
|
||||
output_dir or f"/data/yg-ft/outputs/{task['name']}/adapter",
|
||||
task.get("output_dir") or f"/data/yg-ft/outputs/{task.get('name')}/adapter",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -402,6 +451,79 @@ class PlatformStore:
|
||||
)
|
||||
return {"id": aid}
|
||||
|
||||
# ---- 登录会话(采集在线时长) ----
|
||||
def create_session(self, user: dict[str, Any]) -> str:
|
||||
sid = new_id("sess")
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO sessions
|
||||
(id, user_id, username, display_name, role, login_at, logout_at, duration_seconds, create_time)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
sid,
|
||||
user.get("id"),
|
||||
user.get("username"),
|
||||
user.get("display_name"),
|
||||
user.get("role"),
|
||||
utcnow(),
|
||||
None,
|
||||
None,
|
||||
utcnow(),
|
||||
),
|
||||
)
|
||||
return sid
|
||||
|
||||
def close_session(self, session_id: str | None) -> None:
|
||||
if not session_id:
|
||||
return
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT login_at FROM sessions WHERE id=? AND logout_at IS NULL", (session_id,)
|
||||
).fetchone()
|
||||
if not row:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
start = parse_time(row["login_at"]) or now
|
||||
seconds = max(0, int((now - start).total_seconds()))
|
||||
conn.execute(
|
||||
"UPDATE sessions SET logout_at=?, duration_seconds=? WHERE id=?",
|
||||
(utcnow(), seconds, session_id),
|
||||
)
|
||||
|
||||
def login_duration_rank(self, limit: int = 8) -> list[dict[str, Any]]:
|
||||
"""本月登录时长排行:按用户聚合会话时长(小时)。"""
|
||||
month_start = utcnow()[:7] + "01T00:00:00Z"
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT user_id, username, display_name, role, login_at, logout_at, duration_seconds "
|
||||
"FROM sessions WHERE login_at >= ?",
|
||||
(month_start,),
|
||||
).fetchall()
|
||||
agg: dict[str, dict[str, Any]] = {}
|
||||
now = datetime.now(timezone.utc)
|
||||
for r in rows:
|
||||
uid = r["user_id"]
|
||||
bucket = agg.setdefault(
|
||||
uid,
|
||||
{"user": r["display_name"] or r["username"], "role": r["role"] or "", "total": 0.0, "has": False},
|
||||
)
|
||||
dur = r["duration_seconds"]
|
||||
if dur is None and r["logout_at"] is None:
|
||||
start = parse_time(r["login_at"])
|
||||
if start:
|
||||
dur = max(0, int((now - start).total_seconds()))
|
||||
if dur is None:
|
||||
dur = 0
|
||||
bucket["total"] += dur
|
||||
bucket["has"] = True
|
||||
result = [
|
||||
{"user": b["user"], "role": b["role"], "duration": round(b["total"] / 3600, 1)}
|
||||
for b in agg.values()
|
||||
if b["has"]
|
||||
]
|
||||
result.sort(key=lambda x: x["duration"], reverse=True)
|
||||
return result[:limit]
|
||||
|
||||
# ---- 审批模板 ----
|
||||
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
tid = payload.get("id") or new_id("tpl")
|
||||
@@ -768,7 +890,7 @@ class PlatformStore:
|
||||
utcnow(),
|
||||
),
|
||||
)
|
||||
return self.model(model_id)
|
||||
return self.model(model_id)
|
||||
|
||||
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.model(model_id)
|
||||
@@ -793,7 +915,7 @@ class PlatformStore:
|
||||
model_id,
|
||||
),
|
||||
)
|
||||
return self.model(model_id)
|
||||
return self.model(model_id)
|
||||
|
||||
def delete_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
@@ -1096,20 +1218,80 @@ class PlatformStore:
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
if get_settings().compute_mode != "simulator":
|
||||
api_base = node.get("api_base_url")
|
||||
if api_base:
|
||||
# GPU 计算派发到算力节点进程执行;后端只做调度编排(架构 §1.1)
|
||||
self._dispatch_to_compute(task_id, node, merged, selected_gpus)
|
||||
elif get_settings().compute_mode != "simulator":
|
||||
# 降级路径:未配置算力节点时后端本机执行(违反 §1.1,待移除)
|
||||
from app.modules.fine_tune.service import launch_training
|
||||
|
||||
launch_training(task_id)
|
||||
return self.task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)})
|
||||
def _dispatch_to_compute(
|
||||
self,
|
||||
task_id: str,
|
||||
node: dict[str, Any],
|
||||
task: dict[str, Any],
|
||||
gpus: list,
|
||||
) -> None:
|
||||
"""把训练作业派发到算力节点,GPU 计算在算力进程内执行;后端记录算力 job id。"""
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
cfg = {
|
||||
"id": f"ft_{task_id}",
|
||||
"name": task.get("name") or task_id,
|
||||
"type": "fine_tune",
|
||||
"gpus": gpus,
|
||||
"stage": str(task.get("train_type") or "SFT").lower(),
|
||||
"base_model": task.get("base_model") or "placeholder-base-model",
|
||||
"dataset": task.get("train_dataset_id") or task.get("dataset") or "placeholder-dataset",
|
||||
"template": task.get("template") or "qwen",
|
||||
"train_method": task.get("train_method") or "lora",
|
||||
"output_dir": f"/data/yg-ft/outputs/{task.get('name') or task_id}/adapter",
|
||||
"batch_size": int(task.get("batch_size", 2) or 2),
|
||||
"learning_rate": float(task.get("learning_rate", 0.0002) or 0.0002),
|
||||
"n_epochs": int(task.get("n_epochs", 3) or 3),
|
||||
}
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
try:
|
||||
job = client.create_job(cfg)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.update_task_runtime(task_id, status="failed", extra={"dispatch_error": str(exc)})
|
||||
raise RuntimeError(f"dispatch training job to compute node failed: {exc}") from exc
|
||||
compute_job_id = job.get("id")
|
||||
current = self.task(task_id)
|
||||
updated = {**current, "compute_job_id": compute_job_id, "compute_node_api": node["api_base_url"]}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
||||
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
|
||||
(json_dumps(updated), task_id),
|
||||
)
|
||||
|
||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
# 如果任务已派发到算力节点,先通知算力停止
|
||||
compute_job_id = task.get("compute_job_id")
|
||||
node_id = task.get("compute_node_id")
|
||||
if compute_job_id and node_id:
|
||||
try:
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
node = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
||||
if node and node.get("api_base_url"):
|
||||
ComputeNodeClient(node["api_base_url"]).stop_job(compute_job_id)
|
||||
except Exception: # noqa: BLE001 - best effort stop
|
||||
pass
|
||||
task.update({"status": "stopped", "progress": min(task.get("progress", 0), 99)})
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status='stopped', payload=?, completed_at=? WHERE id=?",
|
||||
(json_dumps(task), utcnow(), task_id),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
|
||||
(utcnow(), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def update_task_runtime(
|
||||
@@ -1934,6 +2116,490 @@ class PlatformStore:
|
||||
content = self.generate_training_log(task)
|
||||
return {"job_id": job_id, "content": content, "lines": len(content.splitlines())}
|
||||
|
||||
# ===================== Model Evaluation =====================
|
||||
|
||||
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
||||
payload = json_loads(row["payload"], {})
|
||||
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
||||
return payload
|
||||
|
||||
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
|
||||
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
payload = self._json_payload_row(row)
|
||||
payload.setdefault("sample_count", 0)
|
||||
payload.setdefault("completed_count", 0)
|
||||
payload.setdefault("passed_count", 0)
|
||||
payload.setdefault("overall_score", payload.get("score") or 0)
|
||||
payload.setdefault("overall_score_max", 100)
|
||||
payload.setdefault("overall_evaluation", "")
|
||||
payload.setdefault("improvement_suggestions", [])
|
||||
payload.setdefault("dimension_summary", [])
|
||||
payload.setdefault("samples", [])
|
||||
return payload
|
||||
|
||||
def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval"))
|
||||
name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}")
|
||||
status = str(payload.get("status") or "pending")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {
|
||||
**payload,
|
||||
"id": task_id,
|
||||
"eval_task_name": name,
|
||||
"status": status,
|
||||
"create_time": now,
|
||||
"metric": payload.get("metric") or "custom",
|
||||
}
|
||||
with self.connect() as conn:
|
||||
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
|
||||
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
||||
if model:
|
||||
data.setdefault("model_name", model["name"])
|
||||
if dataset:
|
||||
data.setdefault("dataset", dataset["name"])
|
||||
conn.execute(
|
||||
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
)
|
||||
return self.eval_task(task_id)
|
||||
|
||||
def delete_eval_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def dimensions(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
|
||||
return [
|
||||
{
|
||||
**json_loads(row["payload"], {}),
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"is_active": bool(row["is_active"]),
|
||||
"is_default": bool(row["is_default"]),
|
||||
"create_time": row["create_time"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def dimension(self, dimension_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(dimension_id)
|
||||
return {
|
||||
**json_loads(row["payload"], {}),
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"is_active": bool(row["is_active"]),
|
||||
"is_default": bool(row["is_default"]),
|
||||
"create_time": row["create_time"],
|
||||
}
|
||||
|
||||
def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
dimension_id = str(payload.get("id") or new_id("dim"))
|
||||
name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {**payload, "id": dimension_id, "name": name, "create_time": now}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now),
|
||||
)
|
||||
return self.dimension(dimension_id)
|
||||
|
||||
def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.dimension(dimension_id)
|
||||
merged = {**current, **payload, "id": dimension_id}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?",
|
||||
(
|
||||
merged["name"],
|
||||
json_dumps(merged),
|
||||
1 if merged.get("is_active", True) else 0,
|
||||
1 if merged.get("is_default") else 0,
|
||||
dimension_id,
|
||||
),
|
||||
)
|
||||
return self.dimension(dimension_id)
|
||||
|
||||
def delete_dimension(self, dimension_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,))
|
||||
|
||||
# ===================== Model Compare / Inference =====================
|
||||
|
||||
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
|
||||
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
return self._json_payload_row(row)
|
||||
|
||||
def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("id") or new_id("cmp"))
|
||||
name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}")
|
||||
status = str(payload.get("status") or "pending")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now}
|
||||
data.setdefault("load_status", json_dumps({"loaded_models": []}))
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
)
|
||||
return self.compare_task(task_id)
|
||||
|
||||
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.compare_task(task_id)
|
||||
merged = {**current, **payload, "id": task_id}
|
||||
status = str(merged.get("status") or current.get("status") or "pending")
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?",
|
||||
(merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id),
|
||||
)
|
||||
return self.compare_task(task_id)
|
||||
|
||||
def delete_compare_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,))
|
||||
|
||||
# ===================== Compute Job Sync (派发回传) =====================
|
||||
|
||||
def _acquire_scheduler_lock(
|
||||
self,
|
||||
conn: PgConnection,
|
||||
lock_key: str,
|
||||
owner: str,
|
||||
ttl_seconds: int = 30,
|
||||
) -> bool:
|
||||
now = utcnow()
|
||||
expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds)).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
existing = conn.execute("SELECT owner, expires_at FROM scheduler_locks WHERE lock_key=?", (lock_key,)).fetchone()
|
||||
if existing:
|
||||
if existing["expires_at"] > now and existing["owner"] != owner:
|
||||
return False
|
||||
conn.execute(
|
||||
"UPDATE scheduler_locks SET owner=?, expires_at=?, update_time=? WHERE lock_key=?",
|
||||
(owner, expires_at, now, lock_key),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"INSERT INTO scheduler_locks (lock_key, owner, expires_at, create_time, update_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(lock_key, owner, expires_at, now, now),
|
||||
)
|
||||
return True
|
||||
|
||||
def _upsert_compute_job(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
|
||||
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
|
||||
now = utcnow()
|
||||
command = job.get("command") or []
|
||||
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
|
||||
payload = json_dumps({**job, "task_id": task["id"]})
|
||||
existing = conn.execute("SELECT id FROM compute_jobs WHERE id=?", (job_id,)).fetchone()
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE compute_jobs
|
||||
SET task_id=?, node_id=?, engine=?, status=?, command=?, output_dir=?, log_file=?,
|
||||
payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
task["id"],
|
||||
task.get("compute_node_id"),
|
||||
str(task.get("engine") or job.get("engine") or "llama_factory"),
|
||||
status,
|
||||
command_text,
|
||||
job.get("output_dir") or task.get("output_dir"),
|
||||
job.get("log_file") or task.get("log_file"),
|
||||
payload,
|
||||
now,
|
||||
now if status in {"completed", "failed", "stopped"} else None,
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO compute_jobs
|
||||
(id, task_id, node_id, engine, status, command, output_dir, log_file, payload, create_time, update_time, completed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
job_id,
|
||||
task["id"],
|
||||
task.get("compute_node_id"),
|
||||
str(task.get("engine") or job.get("engine") or "llama_factory"),
|
||||
status,
|
||||
command_text,
|
||||
job.get("output_dir") or task.get("output_dir"),
|
||||
job.get("log_file") or task.get("log_file"),
|
||||
payload,
|
||||
now,
|
||||
now,
|
||||
now if status in {"completed", "failed", "stopped"} else None,
|
||||
),
|
||||
)
|
||||
|
||||
def _sync_gpu_allocations(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
|
||||
terminal = status in {"completed", "failed", "stopped"}
|
||||
if terminal:
|
||||
conn.execute(
|
||||
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
|
||||
(utcnow(), task["id"]),
|
||||
)
|
||||
return
|
||||
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
|
||||
allocation_status = "running" if status == "running" else "allocated"
|
||||
for gpu_index in [int(item) for item in task.get("gpus") or job.get("gpus") or []]:
|
||||
existing = conn.execute(
|
||||
"SELECT id FROM gpu_allocations WHERE task_id=? AND node_id=? AND gpu_index=? AND status IN ('allocated','running')",
|
||||
(task["id"], task.get("compute_node_id"), gpu_index),
|
||||
).fetchone()
|
||||
if existing:
|
||||
conn.execute("UPDATE gpu_allocations SET status=?, compute_job_id=? WHERE id=?", (allocation_status, job_id, existing["id"]))
|
||||
continue
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO gpu_allocations
|
||||
(id, task_id, compute_job_id, node_id, gpu_index, status, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(new_id("gpu_alloc"), task["id"], job_id, task.get("compute_node_id"), gpu_index, allocation_status, utcnow()),
|
||||
)
|
||||
|
||||
def _upsert_checkpoints(self, conn: PgConnection, task_id: str, checkpoints: list[dict[str, Any]]) -> None:
|
||||
for item in checkpoints:
|
||||
path = str(item.get("path") or "")
|
||||
if not path:
|
||||
continue
|
||||
step = int(item.get("step") or 0)
|
||||
name = str(item.get("name") or Path(path).name)
|
||||
size_bytes = int(item.get("size_bytes") or item.get("size") or 0)
|
||||
existing = conn.execute("SELECT id FROM fine_tune_checkpoints WHERE task_id=? AND path=?", (task_id, path)).fetchone()
|
||||
if existing:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_checkpoints SET step=?, name=?, size_bytes=? WHERE id=?",
|
||||
(step, name, size_bytes, existing["id"]),
|
||||
)
|
||||
continue
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO fine_tune_checkpoints
|
||||
(id, task_id, step, name, path, size_bytes, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(new_id("ckpt"), task_id, step, name, path, size_bytes, utcnow()),
|
||||
)
|
||||
|
||||
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
status_map = {
|
||||
"queued": "queued",
|
||||
"running": "running",
|
||||
"completed": "completed",
|
||||
"failed": "failed",
|
||||
"stopped": "stopped",
|
||||
}
|
||||
current = self.task(task_id)
|
||||
status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"]))
|
||||
progress = int(job.get("progress", current.get("progress", 0)) or 0)
|
||||
payload = {
|
||||
**current,
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"process_id": job.get("pid") or current.get("process_id"),
|
||||
"compute_job_id": job.get("id") or current.get("compute_job_id"),
|
||||
"output_dir": job.get("output_dir") or current.get("output_dir"),
|
||||
"log_file": job.get("log_file") or current.get("log_file"),
|
||||
"artifacts": job.get("artifacts") or current.get("artifacts") or [],
|
||||
}
|
||||
if status == "failed":
|
||||
payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed"
|
||||
elif status in {"queued", "running", "completed"}:
|
||||
payload.pop("failure_reason", None)
|
||||
completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at)
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
json_dumps(payload),
|
||||
status,
|
||||
progress,
|
||||
payload.get("process_id"),
|
||||
payload.get("compute_job_id"),
|
||||
completed_at,
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
self._upsert_compute_job(conn, payload, job, status)
|
||||
self._sync_gpu_allocations(conn, payload, job, status)
|
||||
self._upsert_checkpoints(conn, task_id, job.get("checkpoints") or [])
|
||||
if status == "completed":
|
||||
self._ensure_trained_model(conn, payload)
|
||||
if status in {"failed", "stopped"}:
|
||||
failure_reason = job.get("error") or job.get("message") or "compute job failed"
|
||||
log_snippet = job.get("log_snippet") or ""
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET failure_reason = ? WHERE id = ?",
|
||||
(failure_reason[:2000], task_id),
|
||||
)
|
||||
if log_snippet:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET payload = ? WHERE id = ?",
|
||||
(json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def running_compute_tasks(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
task
|
||||
for task in self.tasks()
|
||||
if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"}
|
||||
]
|
||||
|
||||
def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason})
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
||||
(json_dumps(task), utcnow(), task_id),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
|
||||
(utcnow(), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def record_training_log_metrics(self, task_id: str, content: str) -> int:
|
||||
rows: list[tuple[Any, ...]] = []
|
||||
for line_number, line in enumerate(content.splitlines(), start=1):
|
||||
metric = self._parse_training_metric(line)
|
||||
if not metric:
|
||||
continue
|
||||
rows.append(
|
||||
(
|
||||
new_id("metric"),
|
||||
task_id,
|
||||
line_number,
|
||||
metric.get("epoch"),
|
||||
metric.get("loss"),
|
||||
metric.get("grad_norm"),
|
||||
metric.get("learning_rate"),
|
||||
line[:2000],
|
||||
utcnow(),
|
||||
)
|
||||
)
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM fine_tune_metrics WHERE task_id=?", (task_id,))
|
||||
if rows:
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO fine_tune_metrics
|
||||
(id, task_id, step, epoch, loss, grad_norm, learning_rate, raw, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def _parse_training_metric(self, line: str) -> dict[str, Any] | None:
|
||||
if "loss" not in line or "learning_rate" not in line:
|
||||
return None
|
||||
import re
|
||||
result: dict[str, Any] = {}
|
||||
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
|
||||
match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
|
||||
if match:
|
||||
result[key] = float(match.group(1))
|
||||
return result or None
|
||||
|
||||
def task_metrics(self, task_id: str) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT step, epoch, loss, grad_norm, learning_rate, raw, create_time
|
||||
FROM fine_tune_metrics
|
||||
WHERE task_id=?
|
||||
ORDER BY step
|
||||
""",
|
||||
(task_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def task_checkpoints(self, task_id: str) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, step, name, path, size_bytes, create_time
|
||||
FROM fine_tune_checkpoints
|
||||
WHERE task_id=?
|
||||
ORDER BY step, create_time
|
||||
""",
|
||||
(task_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def active_standalone_compute_jobs(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM compute_jobs
|
||||
WHERE task_id IS NULL AND status IN ('queued','running')
|
||||
ORDER BY create_time
|
||||
""",
|
||||
).fetchall()
|
||||
return [json_loads(row["payload"], {}) if "payload" in row.keys() else dict(row) for row in rows]
|
||||
|
||||
def sync_model_merge_job(self, job_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.compute_job(job_id)
|
||||
payload = current.get("payload") or {}
|
||||
job_payload = payload.get("job") if isinstance(payload.get("job"), dict) else {}
|
||||
merged_payload = {**payload, "job": {**job_payload, **job}}
|
||||
status = str(job.get("status") or current.get("status") or "queued")
|
||||
command = job.get("command") or current.get("command") or []
|
||||
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
|
||||
output_dir = job.get("output_dir") or payload.get("output_dir") or current.get("output_dir")
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE compute_jobs
|
||||
SET status=?, command=?, output_dir=?, log_file=?, payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
status,
|
||||
command_text,
|
||||
output_dir,
|
||||
job.get("log_file") or current.get("log_file"),
|
||||
json_dumps(merged_payload),
|
||||
utcnow(),
|
||||
utcnow() if status in {"completed", "failed", "stopped"} else None,
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
return self.compute_job(job_id)
|
||||
|
||||
|
||||
_store: PlatformStore | None = None
|
||||
|
||||
|
||||
@@ -152,6 +152,18 @@ CREATE TABLE IF NOT EXISTS project_members (
|
||||
|
||||
-- ===================== Fine-tune Checkpoints =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
step INTEGER NOT NULL,
|
||||
epoch DOUBLE PRECISION,
|
||||
loss DOUBLE PRECISION,
|
||||
grad_norm DOUBLE PRECISION,
|
||||
learning_rate DOUBLE PRECISION,
|
||||
raw TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
@@ -164,6 +176,25 @@ CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gpu_allocations (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
compute_job_id TEXT,
|
||||
node_id TEXT REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
released_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scheduler_locks (
|
||||
lock_key TEXT PRIMARY KEY,
|
||||
owner TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
update_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- ===================== Compute Jobs (internal) =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compute_jobs (
|
||||
@@ -185,6 +216,33 @@ CREATE TABLE IF NOT EXISTS compute_jobs (
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
-- ===================== Model Evaluation =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_dimensions (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 1,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compare_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- ===================== Indexes =====================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
||||
@@ -194,9 +252,15 @@ CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_t
|
||||
CREATE INDEX IF NOT EXISTS idx_projects_tenant ON projects(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_project_members_user ON project_members(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_checkpoints_task ON fine_tune_checkpoints(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_metrics_task_step ON fine_tune_metrics(task_id, step);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node ON compute_jobs(node_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_status ON compute_jobs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
|
||||
|
||||
-- ===================== Migrations: extend fine_tune_tasks =====================
|
||||
|
||||
|
||||
@@ -127,3 +127,16 @@ INSERT INTO roles (id, name, display_name, permissions, create_time) VALUES
|
||||
('role_operator','operator','操作员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', '2026-01-01T00:00:00Z'),
|
||||
('role_viewer', 'viewer', '访客', '["dashboard"]', '2026-01-01T00:00:00Z')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
-- 登录会话:采集每次登录/登出,用于统计在线/登录时长
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT,
|
||||
role TEXT,
|
||||
login_at TEXT NOT NULL,
|
||||
logout_at TEXT,
|
||||
duration_seconds INTEGER,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
@@ -6,11 +8,20 @@ from app.core.config import get_settings
|
||||
from app.core.logging import configure_logging, setup_request_logging
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(app: FastAPI):
|
||||
# 启动算力状态轮询线程(同步派发到算力的训练任务状态/日志/指标)
|
||||
from app.modules.fine_tune.service import start_compute_sync_worker, stop_compute_sync_worker
|
||||
start_compute_sync_worker()
|
||||
yield
|
||||
stop_compute_sync_worker()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
configure_logging(settings)
|
||||
|
||||
app = FastAPI(title=settings.app_name)
|
||||
app = FastAPI(title=settings.app_name, lifespan=_lifespan)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_allow_origins,
|
||||
|
||||
@@ -16,13 +16,25 @@ class LoginBody(BaseModel):
|
||||
password: str
|
||||
|
||||
|
||||
class LogoutBody(BaseModel):
|
||||
session_id: str | None = None
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
def login(body: LoginBody) -> dict:
|
||||
user = get_platform_store().login(body.username, body.password)
|
||||
store = get_platform_store()
|
||||
user = store.login(body.username, body.password)
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
|
||||
token = create_access_token(user["id"])
|
||||
return {"code": 0, "message": "ok", "data": {"token": token, "user": user}}
|
||||
session_id = store.create_session(user)
|
||||
return {"code": 0, "message": "ok", "data": {"token": token, "user": user, "session_id": session_id}}
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
def logout(body: LogoutBody) -> dict:
|
||||
get_platform_store().close_session(body.session_id)
|
||||
return {"code": 0, "message": "ok", "data": None}
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
"""Application-side compute platform gateway module."""
|
||||
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
|
||||
__all__ = ["ComputeNodeClient", "poll_compute_jobs_once"]
|
||||
|
||||
180
backend/app/modules/compute_gateway/client.py
Normal file
180
backend/app/modules/compute_gateway/client.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""Client for talking to a compute node's REST API.
|
||||
|
||||
This is the application-side bridge: the application backend never runs GPU
|
||||
workloads itself; it dispatches them to a compute node and reads back status,
|
||||
logs and metrics through this client.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
from app.core.config import get_settings
|
||||
|
||||
|
||||
def _compute_timeout() -> float:
|
||||
return float(get_settings().compute_request_timeout_seconds or 30.0)
|
||||
|
||||
|
||||
def _auth_headers() -> dict:
|
||||
token = get_settings().compute_service_token
|
||||
if not token:
|
||||
return {}
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
class ComputeNodeClient:
|
||||
"""Thin wrapper over a single compute node's HTTP API."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float | None = None):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout or _compute_timeout()
|
||||
|
||||
# ---- health -------------------------------------------------------
|
||||
def health(self) -> dict:
|
||||
last_err = None
|
||||
for path in ("/v1/compute/health", "/health"):
|
||||
try:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(f"{self.base_url}{path}", headers=_auth_headers())
|
||||
if r.status_code == 200:
|
||||
return r.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_err = str(exc)
|
||||
raise RuntimeError(f"compute node unhealthy: {last_err}")
|
||||
|
||||
# ---- gpus ---------------------------------------------------------
|
||||
def gpus(self) -> list:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/resources/gpus",
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json().get("data", [])
|
||||
|
||||
# ---- jobs ---------------------------------------------------------
|
||||
def create_job(self, payload: dict) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs",
|
||||
json=payload,
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def preview_job(self, payload: dict) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs/preview",
|
||||
json=payload,
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def validate_job(self, payload: dict) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs/validate",
|
||||
json=payload,
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def get_job(self, job_id: str) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/jobs/{job_id}",
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def stop_job(self, job_id: str) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs/{job_id}/stop",
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def job_logs(self, job_id: str, cursor: int = 0, limit: int = 200) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/jobs/{job_id}/logs",
|
||||
params={"cursor": cursor, "limit": limit},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
# ---- files --------------------------------------------------------
|
||||
def check_paths(self, paths: list) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/files/check-paths",
|
||||
json={"paths": paths},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def list_files(self, path: str = "/") -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/files/list",
|
||||
params={"path": path},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def import_local_file(self, src_path: str, dest_name: str | None = None) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/files/import-local",
|
||||
json={"src_path": src_path, "dest_name": dest_name},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def upload_file(self, filename: str, content: bytes, content_type: str | None = None) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/files/upload",
|
||||
files={"file": (filename, content, content_type)},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
# ---- inference (async) -------------------------------------------
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
"""Return auth headers for compute node requests."""
|
||||
token = get_settings().compute_service_token
|
||||
if not token:
|
||||
return {}
|
||||
return {"X-Compute-Token": token}
|
||||
|
||||
@property
|
||||
def route_prefix(self) -> str:
|
||||
return get_settings().route_prefix.rstrip("/") or "/modelTF"
|
||||
|
||||
async def _request(self, method: str, path: str, json_data: dict | None = None) -> dict:
|
||||
"""Generic async request method for compute API endpoints."""
|
||||
prefix = self.route_prefix
|
||||
url = f"{self.base_url.rstrip('/')}{prefix}{path}"
|
||||
async with httpx.AsyncClient(timeout=300, verify=False, headers=self.headers()) as client:
|
||||
if method.upper() == "GET":
|
||||
response = await client.get(url)
|
||||
else:
|
||||
response = await client.post(url, json=json_data)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if isinstance(data, dict) and isinstance(data.get("data"), dict):
|
||||
return data["data"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
50
backend/app/modules/compute_gateway/sync.py
Normal file
50
backend/app/modules/compute_gateway/sync.py
Normal file
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
synced: list[dict[str, Any]] = []
|
||||
failed: list[dict[str, str]] = []
|
||||
for task in store.running_compute_tasks():
|
||||
node = _node_for_task(task)
|
||||
if not node:
|
||||
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
job = client.get_job(task["compute_job_id"])
|
||||
try:
|
||||
logs = client.job_logs(task["compute_job_id"], tail_lines=5000)
|
||||
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
|
||||
except Exception:
|
||||
pass
|
||||
if job.get("status") in {"failed", "stopped"}:
|
||||
try:
|
||||
last_logs = client.job_logs(task["compute_job_id"], tail_lines=200)
|
||||
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||
except Exception:
|
||||
pass
|
||||
synced.append(store.apply_compute_job(task["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||
standalone_synced: list[dict[str, Any]] = []
|
||||
for record in store.active_standalone_compute_jobs():
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
|
||||
if not node:
|
||||
failed.append({"job_id": record["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
job = ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
||||
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
|
||||
@@ -3,13 +3,17 @@
|
||||
|
||||
- preset 参数预设(quick / standard / high)
|
||||
- train_type → stage 映射(sft/dpo/cpt/cot)
|
||||
- 训练任务的启动 / 暂停 / 恢复 / 取消(委托 runner 真实执行)
|
||||
- 训练任务的启动 / 暂停 / 恢复 / 取消
|
||||
- compute_gateway 状态轮询线程(当任务派发到算力时自动同步状态)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.fine_tune import runner
|
||||
|
||||
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
|
||||
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
|
||||
}
|
||||
|
||||
# ── compute sync 轮询线程 ──────────────────────────────────────────────
|
||||
_sync_thread: threading.Thread | None = None
|
||||
_sync_thread_stop = threading.Event()
|
||||
|
||||
|
||||
def _compute_sync_loop() -> None:
|
||||
"""后台线程:周期性轮询算力节点,同步训练任务状态/日志/指标。"""
|
||||
interval = get_settings().compute_poll_interval_seconds or 3
|
||||
while not _sync_thread_stop.is_set():
|
||||
try:
|
||||
store = get_platform_store()
|
||||
running = store.running_compute_tasks()
|
||||
if running:
|
||||
asyncio.run(_poll_once())
|
||||
except Exception: # noqa: BLE001 - keep polling loop alive
|
||||
pass
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
async def _poll_once() -> None:
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
await poll_compute_jobs_once()
|
||||
|
||||
|
||||
def start_compute_sync_worker() -> None:
|
||||
"""启动后台轮询线程(幂等,多次调用安全)。"""
|
||||
global _sync_thread
|
||||
if _sync_thread is not None and _sync_thread.is_alive():
|
||||
return
|
||||
_sync_thread_stop.clear()
|
||||
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
|
||||
_sync_thread.start()
|
||||
|
||||
|
||||
def stop_compute_sync_worker() -> None:
|
||||
"""停止后台轮询线程。"""
|
||||
_sync_thread_stop.set()
|
||||
|
||||
|
||||
# ── preset / config ────────────────────────────────────────────────────
|
||||
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""根据 preset 字段补全缺失的超参;preset=custom 时不覆盖。"""
|
||||
payload = dict(payload)
|
||||
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
|
||||
def launch_training(task_id: str) -> None:
|
||||
"""在后台线程启动真实训练。"""
|
||||
"""在后台线程启动真实训练(本机 subprocess fallback,当算力节点不可用时使用)。
|
||||
|
||||
架构原则:GPU 计算应派发到算力服务进程执行。
|
||||
当 platform_store.start_task 检测到在线算力节点时,会走 _dispatch_to_compute 派发路径;
|
||||
仅当无可用算力节点且非 simulator 模式时,降级到本机 runner(违反 §1.1,待移除)。
|
||||
"""
|
||||
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user