from __future__ import annotations import json from datetime import datetime, timedelta, timezone from typing import Any import uuid import httpx from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile from fastapi.responses import PlainTextResponse, StreamingResponse from app.db.platform_store import get_platform_store from app.modules.compute_gateway.client import ComputeNodeClient from app.modules.fine_tune.service import apply_presets from fastapi import Request as FastAPIRequest router = APIRouter() def _actor(request: FastAPIRequest) -> str | None: auth = request.headers.get("Authorization", "") token = auth.replace("Bearer ", "").strip() return token or None def ok(data: Any = None, message: str = "ok") -> dict[str, Any]: return {"code": 0, "message": message, "data": data} def fail(status_code: int, message: str) -> HTTPException: return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None}) def _select_first_online_node(store: Any) -> dict[str, Any] | None: """Select the first online compute node for inference.""" nodes = store.compute_nodes() for node in nodes: if node.get("enabled") and node.get("scheduler_status") == "online": return node return None @router.get("/dashboard/overview") async def dashboard_overview() -> dict[str, Any]: store = get_platform_store() tasks = store.tasks() return ok( { "models": len(store.models()), "datasets": len(store.datasets()), "fine_tune_tasks": len(tasks), "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]), "compute_nodes": len(store.compute_nodes()), "gpus": len(store.gpus()), } ) @router.get("/dashboard/stats") async def dashboard_stats() -> dict[str, Any]: """看板聚合数据：基于平台真实数据；缺项做合理近似（见下）。""" store = get_platform_store() tasks = store.tasks() users = store.users() nodes = store.compute_nodes() datasets = store.datasets() running_statuses = {"syncing", "queued", "running"} running_ft = [t for t in tasks if t.get("status") in running_statuses] failed_ft = [t for t in tasks if t.get("status") == "failed"] online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"] # 近 7 天训练统计（按创建日期分桶；准确率为 None，因任务无该字段） now = datetime.now(timezone.utc) train_by_day: dict[str, int] = {} for t in tasks: ct = t.get("create_time") if ct: train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1 training_7d = [] for i in range(6, -1, -1): day = (now - timedelta(days=i)).strftime("%Y-%m-%d") training_7d.append( { "date": day[5:], "train": train_by_day.get(day, 0), "gpu": sum(len(t.get("gpus") or []) for t in running_ft), "accuracy": None, } ) # 服务状态：模型推理用在线计算节点近似；模型评测暂无独立数据源，置 0 service_status = [ { "type": "模型推理", "status": "error" if (nodes and not online_nodes) else ("busy" if (nodes and len(online_nodes) < len(nodes)) else "normal"), "count": len(online_nodes), }, { "type": "模型微调", "status": "error" if failed_ft else ("busy" if running_ft else "normal"), "count": len(running_ft), }, {"type": "模型评测", "status": "normal", "count": 0}, { "type": "数据处理", "status": "normal" if not failed_ft else "busy", "count": len(datasets), }, ] # 训练任务状态归一化（fine_tune 的 syncing/queued 等映射到前端已知状态） status_map = { "syncing": "running", "queued": "running", "running": "running", "pending": "pending", "paused": "pending", "completed": "completed", "failed": "failed", "error": "failed", "cancelled": "failed", } # 用户操作分布：仅展示「数据治理」与「模型服务」两大分组下的子模块，其他不显示 MODULE_LABELS = [ ("data-process", "数据处理"), ("data_process", "数据处理"), ("dataset", "数据集管理"), ("fine-tune", "模型训练"), ("fine_tune", "模型训练"), ("model-eval", "模型评测"), ("eval", "模型评测"), ("model-inference", "模型推理"), ("inference", "模型推理"), ("model-manage", "模型管理"), ("model", "模型管理"), ("trained", "模型管理"), ] OP_ORDER = ["数据集管理", "数据处理", "模型训练", "模型评测", "模型推理", "模型管理"] def _op_module(action: str) -> str | None: a = (action or "").lower() for prefix, label in MODULE_LABELS: if a.startswith(prefix): return label return None training_tasks = [ { "id": t.get("id"), "name": t.get("name"), "status": status_map.get(t.get("status"), "pending"), "train_type": t.get("train_type"), "train_method": t.get("train_method"), "base_model": t.get("base_model"), "progress": t.get("progress", 0), "accuracy": t.get("accuracy"), "started_at": (t.get("create_time") or "")[:16], } for t in tasks[:8] ] # 用户操作分布：仅统计数据治理/模型服务下子模块的操作，其他不显示 audit = store.audit_logs(limit=1000) op_counter: dict[str, int] = {label: 0 for label in OP_ORDER} for log in audit.get("items", []): label = _op_module(log.get("action") or "") if label: op_counter[label] += 1 operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()] # 最近登录用户：后端有 last_login 字段，返回真实数据 recent = sorted( [u for u in users if u.get("last_login")], key=lambda u: u["last_login"], reverse=True, )[:5] recent_login_users = [ { "user": u.get("display_name") or u.get("username"), "role": u.get("role"), "last_login": (u.get("last_login") or "")[:16], } for u in recent ] # 登录时长排行：基于 sessions 表真实会话时长（本月） login_duration_rank = store.login_duration_rank() return ok( { "online_services": sum(s["count"] for s in service_status), "running_tasks": len(running_ft), "pending_alerts": 0, # 平台暂无独立告警数据源，先置 0，待接入后补充 "training_7d": training_7d, "service_status": service_status, "training_tasks": training_tasks, "operation_distribution": operation_distribution, "login_duration_rank": login_duration_rank, "recent_login_users": recent_login_users, } ) @router.get("/system-info") async def system_info() -> dict[str, Any]: return ok(get_platform_store().system_info()) @router.get("/users") async def users() -> dict[str, Any]: return ok(get_platform_store().users()) @router.post("/users") async def create_user(payload: dict[str, Any] = Body(...), request: FastAPIRequest = None) -> dict[str, Any]: store = get_platform_store() user = store.create_user(payload) store.record_audit( action="user.create", actor_id=_actor(request), target_type="user", target_id=user["id"], detail=f"username={user.get('username')}", ) return ok(user) @router.put("/users/{user_id}") async def update_user(user_id: str, payload: dict[str, Any] = Body(...), request: FastAPIRequest = None) -> dict[str, Any]: store = get_platform_store() try: user = store.update_user(user_id, payload) except KeyError: raise fail(404, "user not found") store.record_audit( action="user.update", actor_id=_actor(request), target_type="user", target_id=user_id, detail=f"fields={','.join(payload.keys())}", ) return ok(user) @router.delete("/users/{user_id}") async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]: try: get_platform_store().delete_user(user_id) return ok({"deleted": user_id, "current_username": current_username}) except KeyError: raise fail(404, "user not found") except ValueError as exc: raise fail(400, str(exc)) @router.post("/users/{user_id}/reset-password") async def reset_password( user_id: str, payload: dict[str, Any] = Body(default={}), request: FastAPIRequest = None, ) -> dict[str, Any]: try: user = get_platform_store().reset_password(user_id, payload.get("password") or "") except KeyError: raise fail(404, "user not found") except ValueError as exc: raise fail(400, str(exc)) get_platform_store().record_audit( action="user.reset_password", actor_id=_actor(request), target_type="user", target_id=user_id, detail=f"username={user.get('username')}", ) return ok({"id": user_id}) @router.get("/model-manage/local-models") async def local_models() -> dict[str, Any]: models = [{"path": item.get("path") or "", "name": item["name"]} for item in get_platform_store().models()] return ok({"models": models}) @router.get("/model-manage/trained-models") async def trained_models() -> dict[str, Any]: return ok({"models": get_platform_store().trained_models()}) @router.delete("/model-manage/trained-models/{model_id}") async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]: return ok({"deleted": model_id, "type": type}) @router.get("/model-manage/name/{name}") async def model_by_name(name: str) -> dict[str, Any]: try: return ok(get_platform_store().model_by_name(name)) except KeyError: raise fail(404, "model not found") @router.get("/model-manage") async def model_list() -> dict[str, Any]: return ok(get_platform_store().models()) @router.post("/model-manage") async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok(get_platform_store().create_model(payload)) @router.get("/model-manage/{model_id}") async def model_detail(model_id: str) -> dict[str, Any]: try: return ok(get_platform_store().model(model_id)) except KeyError: raise fail(404, "model not found") @router.put("/model-manage/{model_id}") async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_model(model_id, payload)) except KeyError: raise fail(404, "model not found") @router.put("/model-manage/{model_id}/purpose") async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")})) except KeyError: raise fail(404, "model not found") @router.delete("/model-manage/{model_id}") async def delete_model(model_id: str) -> dict[str, Any]: get_platform_store().delete_model(model_id) return ok({"deleted": model_id}) @router.post("/model-manage/merge") async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"job_id": f"merge_{uuid.uuid4().hex[:12]}", "status": "queued", **payload}) @router.get("/dataset-manage/preview/{file_id}") async def dataset_preview(file_id: str) -> dict[str, Any]: try: row = get_platform_store().dataset_file(file_id) return ok({"content": row["content"]}) except KeyError: raise fail(404, "dataset file not found") @router.get("/dataset-manage/versions/{file_id}") async def dataset_versions(file_id: str) -> dict[str, Any]: try: return ok(get_platform_store().file_versions(file_id)) except KeyError: raise fail(404, "dataset file not found") @router.get("/dataset-manage/versions/{file_id}/{version_id}") async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]: try: row = get_platform_store().dataset_file(file_id) versions = get_platform_store().file_versions(file_id)["versions"] version = next((item for item in versions if item["id"] == version_id), None) if not version: raise KeyError(version_id) return ok({"version": version, "content": row["content"]}) except KeyError: raise fail(404, "dataset version not found") @router.post("/dataset-manage/versions/{file_id}") async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().create_file_version(file_id, payload)) except KeyError: raise fail(404, "dataset file not found") @router.put("/dataset-manage/versions/{file_id}/active") async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().activate_file_version(file_id, payload["version_id"])) except KeyError: raise fail(404, "dataset version not found") @router.delete("/dataset-manage/versions/{file_id}/{version_id}") async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]: return ok(get_platform_store().file_versions(file_id)) @router.post("/dataset-manage/upload/{dataset_id}") async def upload_dataset_files(dataset_id: str, files: list[UploadFile] = File(default=[])) -> dict[str, Any]: created: list[dict[str, Any]] = [] store = get_platform_store() try: store.dataset(dataset_id) except KeyError: raise fail(404, "dataset not found") with store.connect() as conn: for file in files: raw = await file.read() content = raw.decode("utf-8", errors="replace") created.append(store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)) return ok({"files": created}) @router.get("/dataset-manage/download/{dataset_id}") async def download_dataset(dataset_id: str) -> PlainTextResponse: dataset = get_platform_store().dataset(dataset_id) content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])]) return PlainTextResponse(content, media_type="text/plain") @router.get("/dataset-manage/download/{dataset_id}/{file_id}") async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse: row = get_platform_store().dataset_file(file_id) return PlainTextResponse(row["content"], media_type="text/plain") @router.get("/dataset-manage") async def dataset_list() -> dict[str, Any]: return ok(get_platform_store().datasets()) @router.post("/dataset-manage") async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: dataset = get_platform_store().create_dataset(payload) return ok({"id": dataset["id"]}) @router.get("/dataset-manage/{dataset_id}") async def dataset_detail(dataset_id: str) -> dict[str, Any]: try: return ok(get_platform_store().dataset(dataset_id)) except KeyError: raise fail(404, "dataset not found") @router.put("/dataset-manage/{dataset_id}") async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_dataset(dataset_id, payload)) except KeyError: raise fail(404, "dataset not found") @router.delete("/dataset-manage/{dataset_id}") async def delete_dataset(dataset_id: str) -> dict[str, Any]: get_platform_store().delete_dataset(dataset_id) return ok({"deleted": dataset_id}) @router.get("/fine-tune/check-name") async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]: exists = any(task["name"] == name for task in get_platform_store().tasks()) return ok({"exists": exists}) @router.get("/fine-tune/progress/{task_id}") async def fine_tune_progress(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().progress(task_id)) except KeyError: raise fail(404, "fine tune task not found") @router.post("/fine-tune/tensorboard/start") async def tensorboard_start() -> dict[str, Any]: return ok({"status": "running", "url": "http://localhost:6006"}) @router.get("/fine-tune") async def fine_tune_list() -> dict[str, Any]: return ok(get_platform_store().tasks()) @router.post("/fine-tune") async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: task = get_platform_store().create_task(apply_presets(payload)) return ok({"id": task["id"]}) except ValueError as exc: raise fail(400, str(exc)) @router.post("/fine-tune/start") async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().start_task(payload)) except KeyError: raise fail(404, "fine tune task not found") except RuntimeError as exc: raise fail(409, str(exc)) @router.get("/fine-tune/{task_id}") async def fine_tune_detail(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().task(task_id)) except KeyError: raise fail(404, "fine tune task not found") @router.put("/fine-tune/{task_id}") async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_task(task_id, payload)) except KeyError: raise fail(404, "fine tune task not found") @router.post("/fine-tune/stop/{task_id}") async def stop_fine_tune(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().stop_task(task_id)) except KeyError: raise fail(404, "fine tune task not found") @router.post("/fine-tune/{task_id}/stop") async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]: return await stop_fine_tune(task_id) @router.post("/fine-tune/pause/{task_id}") async def pause_fine_tune(task_id: str) -> dict[str, Any]: if not get_platform_store().pause_task(task_id): raise fail(409, "当前没有可暂停的训练进程") return ok({"paused": task_id}) @router.post("/fine-tune/resume/{task_id}") async def resume_fine_tune(task_id: str) -> dict[str, Any]: if not get_platform_store().resume_task_engine(task_id): raise fail(409, "当前没有可恢复的训练进程") return ok({"resumed": task_id}) @router.post("/fine-tune/cancel/{task_id}") async def cancel_fine_tune(task_id: str) -> dict[str, Any]: get_platform_store().cancel_task_engine(task_id) return ok({"canceled": task_id}) @router.delete("/fine-tune/{task_id}") async def delete_fine_tune(task_id: str) -> dict[str, Any]: get_platform_store().delete_task(task_id) return ok({"deleted": task_id}) @router.get("/fine-tune/{task_id}/overview") async def fine_tune_overview(task_id: str) -> dict[str, Any]: task = get_platform_store().task(task_id) return ok({"task": task, "progress": get_platform_store().progress(task_id)}) @router.get("/fine-tune/{task_id}/checkpoints") async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]: task = get_platform_store().task(task_id) checkpoints = [] for step in [50, 100, 150]: if task.get("progress", 0) >= min(100, step // 2): checkpoints.append({"step": step, "path": f"/data/yg-ft/outputs/{task['name']}/checkpoint-{step}"}) return ok(checkpoints) @router.get("/compute/nodes") async def compute_nodes() -> dict[str, Any]: return ok(get_platform_store().compute_nodes()) @router.post("/compute/nodes") async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().create_compute_node(payload)) except KeyError as exc: raise fail(400, f"missing field: {exc}") @router.put("/compute/nodes/{node_id}") async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_compute_node(node_id, payload)) except KeyError: raise fail(404, "compute node not found") @router.post("/compute/nodes/{node_id}/test-connection") async def test_compute_node(node_id: str) -> dict[str, Any]: store = get_platform_store() nodes = store.compute_nodes() node = next((item for item in nodes if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") try: all_gpus = store.gpus() node_gpus = [g for g in all_gpus if g.get("node_id") == node_id] return ok({ "node_id": node_id, "success": True, "latency_ms": 12, "gpu_count": len(node_gpus), }) except Exception as exc: # noqa: BLE001 return ok({ "node_id": node_id, "success": False, "latency_ms": 0, "gpu_count": 0, "error": str(exc), }) @router.post("/compute/nodes/{node_id}/enable") async def enable_compute_node(node_id: str) -> dict[str, Any]: return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"})) @router.post("/compute/nodes/{node_id}/disable") async def disable_compute_node(node_id: str) -> dict[str, Any]: return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"})) @router.post("/compute/nodes/{node_id}/drain") async def drain_compute_node(node_id: str) -> dict[str, Any]: return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"})) @router.get("/compute/nodes/{node_id}/replicas") async def compute_node_replicas(node_id: str) -> dict[str, Any]: return ok(get_platform_store().replicas(node_id)) @router.get("/compute/gpus") async def compute_gpus() -> dict[str, Any]: return ok(get_platform_store().gpus()) @router.get("/compute/queue") async def compute_queue() -> dict[str, Any]: return ok(get_platform_store().queue()) @router.post("/internal/compute-sync/resources") async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: sync_id = get_platform_store().create_sync_job(payload.get("target_node_id", "node_01"), payload) return ok(get_platform_store().sync_job(sync_id)) @router.get("/internal/compute-sync/resources/{sync_id}") async def compute_sync_detail(sync_id: str) -> dict[str, Any]: try: return ok(get_platform_store().sync_job(sync_id)) except KeyError: raise fail(404, "sync job not found") @router.get("/training-log-files") async def training_log_files() -> dict[str, Any]: return ok(get_platform_store().training_log_files()) @router.get("/training-log-content") async def training_log_content(file: str = Query(...)) -> dict[str, Any]: try: return ok(get_platform_store().training_log_content(file)) except KeyError: raise fail(404, "training log not found") @router.get("/log-files") async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]: return ok(get_platform_store().log_files(date)) @router.get("/log-content") async def log_content(file: str = Query(...)) -> dict[str, Any]: return ok(get_platform_store().log_content(file)) @router.post("/web-log") async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"received": True, **payload}) # ===================== Project Management (§13.2) ===================== @router.get("/projects") async def project_list( tenant_id: str = Query(default="default"), status: str | None = Query(default=None), keyword: str | None = Query(default=None), ) -> dict[str, Any]: return ok(get_platform_store().projects(tenant_id, status, keyword)) @router.post("/projects") async def create_project(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: project = get_platform_store().create_project(payload) return ok({"id": project["id"]}) except KeyError as exc: raise fail(400, f"missing required field: {exc}") @router.get("/projects/{project_id}") async def project_detail(project_id: str) -> dict[str, Any]: try: return ok(get_platform_store().project(project_id)) except KeyError: raise fail(404, "project not found") @router.put("/projects/{project_id}") async def update_project(project_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_project(project_id, payload)) except KeyError: raise fail(404, "project not found") @router.post("/projects/{project_id}/activate") async def activate_project(project_id: str) -> dict[str, Any]: try: return ok(get_platform_store().activate_project(project_id)) except KeyError: raise fail(404, "project not found") @router.get("/projects/{project_id}/members") async def project_members(project_id: str) -> dict[str, Any]: try: return ok(get_platform_store().project_members(project_id)) except KeyError: raise fail(404, "project not found") @router.post("/projects/{project_id}/members") async def add_project_member(project_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: user_id = payload.get("user_id") if not user_id: raise fail(400, "user_id is required") try: return ok(get_platform_store().add_project_member(project_id, user_id, payload.get("role", "member"))) except KeyError as exc: raise fail(404, str(exc)) @router.put("/projects/{project_id}/members/{user_id}") async def update_project_member_role(project_id: str, user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_project_member_role(project_id, user_id, payload.get("role", "member"))) except KeyError as exc: raise fail(404, str(exc)) @router.delete("/projects/{project_id}/members/{user_id}") async def remove_project_member(project_id: str, user_id: str) -> dict[str, Any]: try: get_platform_store().remove_project_member(project_id, user_id) return ok({"deleted": user_id}) except KeyError as exc: raise fail(404, str(exc)) # ===================== Fine-tune Events (§7.1) ===================== @router.get("/fine-tune/{task_id}/events") async def fine_tune_events(task_id: str) -> StreamingResponse: import json as _json async def event_stream(): store = get_platform_store() try: events = store.task_events(task_id) for event in events: yield f"data: {_json.dumps(event, default=str)}\n\n" yield f"data: {_json.dumps({'type': 'done', 'data': {}}, default=str)}\n\n" except KeyError: yield f"data: {_json.dumps({'type': 'error', 'data': {'message': 'task not found'}}, default=str)}\n\n" return StreamingResponse(event_stream(), media_type="text/event-stream") # ===================== Fine-tune Retry & Resume (§13.9) ===================== @router.post("/fine-tune/{task_id}/retry") async def retry_fine_tune(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().retry_task(task_id)) except KeyError: raise fail(404, "fine tune task not found") except ValueError as exc: raise fail(400, str(exc)) @router.post("/fine-tune/{task_id}/resume") async def resume_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: checkpoint_id = payload.get("checkpoint_id") if not checkpoint_id: raise fail(400, "checkpoint_id is required") try: return ok(get_platform_store().resume_task(task_id, checkpoint_id)) except KeyError as exc: raise fail(404, str(exc)) except ValueError as exc: raise fail(400, str(exc)) # ===================== Checkpoint Management (§13.9) ===================== @router.delete("/fine-tune/{task_id}/checkpoints/{checkpoint_id}") async def delete_checkpoint(task_id: str, checkpoint_id: str) -> dict[str, Any]: try: get_platform_store().delete_checkpoint(checkpoint_id) return ok({"deleted": checkpoint_id}) except KeyError: raise fail(404, "checkpoint not found") @router.put("/fine-tune/{task_id}/checkpoint-retention") async def set_checkpoint_retention(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().set_checkpoint_retention(task_id, payload)) except KeyError: raise fail(404, "fine tune task not found") @router.get("/fine-tune/{task_id}/checkpoint-retention") async def get_checkpoint_retention(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().get_checkpoint_retention(task_id)) except KeyError: raise fail(404, "fine tune task not found") # ===================== Compute Jobs (§13.6) ===================== @router.post("/compute/jobs") async def create_compute_job(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: job = get_platform_store().create_compute_job(payload) return ok({"id": job["id"]}) except KeyError as exc: raise fail(400, f"missing required field: {exc}") @router.get("/compute/jobs/{job_id}") async def compute_job_detail(job_id: str) -> dict[str, Any]: try: return ok(get_platform_store().compute_job(job_id)) except KeyError: raise fail(404, "compute job not found") @router.get("/compute/jobs") async def compute_jobs(task_id: str = Query(default=None)) -> dict[str, Any]: if task_id: return ok(get_platform_store().compute_jobs_by_task(task_id)) return ok([]) @router.post("/compute/jobs/{job_id}/stop") async def stop_compute_job(job_id: str) -> dict[str, Any]: try: return ok(get_platform_store().stop_compute_job(job_id)) except KeyError: raise fail(404, "compute job not found") @router.get("/compute/jobs/{job_id}/logs") async def compute_job_logs(job_id: str) -> dict[str, Any]: try: return ok(get_platform_store().compute_job_logs(job_id)) except KeyError as exc: raise fail(404, str(exc)) # ===================== Model Evaluation ===================== @router.get("/model-eval") async def model_eval_list() -> dict[str, Any]: return ok(get_platform_store().eval_tasks()) @router.get("/model-eval/{task_id}") async def model_eval_detail(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().eval_task(task_id)) except KeyError: raise fail(404, "eval task not found") @router.post("/model-eval/start") async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: task = get_platform_store().create_eval_task(payload) return ok({"task_id": task["id"], **task}) @router.delete("/model-eval/{task_id}") async def model_eval_delete(task_id: str) -> dict[str, Any]: get_platform_store().delete_eval_task(task_id) return ok({"deleted": task_id}) # ===================== Eval Dimensions ===================== @router.get("/dimension") async def dimension_list() -> dict[str, Any]: return ok(get_platform_store().dimensions()) @router.post("/dimension") async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok(get_platform_store().create_dimension(payload)) @router.get("/dimension/{dimension_id}") async def dimension_detail(dimension_id: str) -> dict[str, Any]: try: return ok(get_platform_store().dimension(dimension_id)) except KeyError: raise fail(404, "dimension not found") @router.put("/dimension/{dimension_id}") async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_dimension(dimension_id, payload)) except KeyError: raise fail(404, "dimension not found") @router.delete("/dimension/{dimension_id}") async def dimension_delete(dimension_id: str) -> dict[str, Any]: get_platform_store().delete_dimension(dimension_id) return ok({"deleted": dimension_id}) # ===================== Model Compare / Inference ===================== @router.get("/model-compare") async def model_compare_list() -> dict[str, Any]: return ok(get_platform_store().compare_tasks()) @router.post("/model-compare") async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: task = get_platform_store().create_compare_task(payload) return ok({"id": task["id"]}) @router.post("/model-compare/all/stop-all") async def model_compare_stop_all() -> dict[str, Any]: return ok({"stopped": True}) @router.post("/model-compare/stop-by-pid") async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"stopped": True, "pid": payload.get("pid")}) @router.get("/model-compare/{task_id}") async def model_compare_detail(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().compare_task(task_id)) except KeyError: raise fail(404, "compare task not found") @router.delete("/model-compare/{task_id}") async def model_compare_delete(task_id: str) -> dict[str, Any]: get_platform_store().delete_compare_task(task_id) return ok({"deleted": task_id}) @router.get("/model-compare/{task_id}/load-status") async def model_compare_load_status(task_id: str) -> dict[str, Any]: try: task = get_platform_store().compare_task(task_id) except KeyError: raise fail(404, "compare task not found") load_status = task.get("load_status") or {"loaded_models": []} if isinstance(load_status, str): try: load_status = json.loads(load_status) except json.JSONDecodeError: load_status = {"loaded_models": []} return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status}) @router.post("/model-compare/{task_id}/load-status") async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}})) except KeyError: raise fail(404, "compare task not found") @router.post("/model-compare/{task_id}/load") async def model_compare_load(task_id: str) -> dict[str, Any]: try: task = get_platform_store().compare_task(task_id) models = task.get("models") or [] if isinstance(models, str): try: models = json.loads(models) except json.JSONDecodeError: models = [] loaded_models = [ { "model_id": item.get("model_id"), "model_name": item.get("model_name"), "status": "ready", "pid": 45000 + index, "port": item.get("port") or 18000 + index, } for index, item in enumerate(models) if isinstance(item, dict) ] return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}})) except KeyError: raise fail(404, "compare task not found") @router.post("/model-compare/{task_id}/unload") async def model_compare_unload(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})) except KeyError: raise fail(404, "compare task not found") @router.post("/model-compare/{task_id}/start-model") async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id}) @router.post("/model-compare/chat-with-port") async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: question = "" for message in payload.get("messages") or []: if message.get("role") == "user": question = str(message.get("content") or "") content = f"当前后端已收到推理请求：{question[:120]}" return ok({"response": content, "content": content}) @router.post("/model-compare/stream-chat") async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: question = payload.get("user_question") or payload.get("question") or "" return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"}) # ===================== Model Chat (Inference Proxy) ===================== @router.post("/model-chat/batch") async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"responses": [], "request": payload}) @router.post("/model-chat/local/chat") async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: """Proxy chat to the compute node running the inference model.""" store = get_platform_store() node = _select_first_online_node(store) if not node: return ok({"response": "no online compute node available for inference", "request": payload}) try: client = ComputeNodeClient(node["api_base_url"]) result = await client._request("POST", "/inference/chat", json_data=payload) return ok(result) except Exception as exc: return ok({"response": f"inference failed: {exc}", "request": payload}) @router.post("/model-chat/local/chat/stream") async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse: """Stream chat from the compute node.""" store = get_platform_store() node = _select_first_online_node(store) if not node: return StreamingResponse( iter(['data: {"error": "no online compute node"}\n\n']), media_type="text/event-stream", ) client = ComputeNodeClient(node["api_base_url"]) async def stream_proxy(): async with httpx.AsyncClient(timeout=300) as http: url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream" async with http.stream("POST", url, json=payload, headers=client.headers()) as resp: async for chunk in resp.aiter_bytes(): yield chunk return StreamingResponse(stream_proxy(), media_type="text/event-stream") @router.post("/model-chat/local/preload") async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: """Load a model on the compute node for inference.""" store = get_platform_store() node = _select_first_online_node(store) if not node: return ok({"loaded": False, "error": "no online compute node"}) try: client = ComputeNodeClient(node["api_base_url"]) result = await client._request("POST", "/inference/load", json_data=payload) return ok(result) except Exception as exc: return ok({"loaded": False, "error": str(exc)}) @router.post("/model-chat/local/unload") async def model_chat_local_unload() -> dict[str, Any]: """Unload the inference model from the compute node.""" store = get_platform_store() node = _select_first_online_node(store) if not node: return ok({"unloaded": False, "error": "no online compute node"}) try: client = ComputeNodeClient(node["api_base_url"]) result = await client._request("POST", "/inference/unload", json_data={}) return ok(result) except Exception as exc: return ok({"unloaded": False, "error": str(exc)}) @router.get("/model-chat/local/status") async def model_chat_local_status() -> dict[str, Any]: """Get inference session status from compute node.""" store = get_platform_store() node = _select_first_online_node(store) if not node: return ok({"loaded": False, "error": "no online compute node"}) try: client = ComputeNodeClient(node["api_base_url"]) result = await client._request("GET", "/inference/status") return ok(result) except Exception as exc: return ok({"loaded": False, "error": str(exc)}) @router.post("/model-chat/trained/preload") async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: """Load a trained model (base + adapter) on the compute node for inference.""" store = get_platform_store() node = _select_first_online_node(store) if not node: return ok({"loaded": False, "error": "no online compute node"}) try: client = ComputeNodeClient(node["api_base_url"]) result = await client._request("POST", "/inference/load", json_data=payload) return ok(result) except Exception as exc: return ok({"loaded": False, "error": str(exc)})