1173 lines
41 KiB
Python
1173 lines
41 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any
|
||
import uuid
|
||
|
||
import httpx
|
||
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
|
||
from fastapi.responses import PlainTextResponse, StreamingResponse
|
||
|
||
from app.db.platform_store import get_platform_store
|
||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||
from app.modules.fine_tune.service import apply_presets
|
||
from fastapi import Request as FastAPIRequest
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
def _actor(request: FastAPIRequest) -> str | None:
|
||
auth = request.headers.get("Authorization", "")
|
||
token = auth.replace("Bearer ", "").strip()
|
||
return token or None
|
||
|
||
|
||
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
|
||
return {"code": 0, "message": message, "data": data}
|
||
|
||
|
||
def fail(status_code: int, message: str) -> HTTPException:
|
||
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
||
|
||
|
||
def _select_first_online_node(store: Any) -> dict[str, Any] | None:
|
||
"""Select the first online compute node for inference."""
|
||
nodes = store.compute_nodes()
|
||
for node in nodes:
|
||
if node.get("enabled") and node.get("scheduler_status") == "online":
|
||
return node
|
||
return None
|
||
|
||
|
||
@router.get("/dashboard/overview")
|
||
async def dashboard_overview() -> dict[str, Any]:
|
||
store = get_platform_store()
|
||
tasks = store.tasks()
|
||
return ok(
|
||
{
|
||
"models": len(store.models()),
|
||
"datasets": len(store.datasets()),
|
||
"fine_tune_tasks": len(tasks),
|
||
"running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]),
|
||
"compute_nodes": len(store.compute_nodes()),
|
||
"gpus": len(store.gpus()),
|
||
}
|
||
)
|
||
|
||
|
||
@router.get("/dashboard/stats")
|
||
async def dashboard_stats() -> dict[str, Any]:
|
||
"""看板聚合数据：基于平台真实数据；缺项做合理近似（见下）。"""
|
||
store = get_platform_store()
|
||
tasks = store.tasks()
|
||
users = store.users()
|
||
nodes = store.compute_nodes()
|
||
datasets = store.datasets()
|
||
|
||
running_statuses = {"syncing", "queued", "running"}
|
||
running_ft = [t for t in tasks if t.get("status") in running_statuses]
|
||
failed_ft = [t for t in tasks if t.get("status") == "failed"]
|
||
online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"]
|
||
|
||
# 近 7 天训练统计（按创建日期分桶；准确率为 None，因任务无该字段）
|
||
now = datetime.now(timezone.utc)
|
||
train_by_day: dict[str, int] = {}
|
||
for t in tasks:
|
||
ct = t.get("create_time")
|
||
if ct:
|
||
train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1
|
||
training_7d = []
|
||
for i in range(6, -1, -1):
|
||
day = (now - timedelta(days=i)).strftime("%Y-%m-%d")
|
||
training_7d.append(
|
||
{
|
||
"date": day[5:],
|
||
"train": train_by_day.get(day, 0),
|
||
"gpu": sum(len(t.get("gpus") or []) for t in running_ft),
|
||
"accuracy": None,
|
||
}
|
||
)
|
||
|
||
# 服务状态：模型推理用在线计算节点近似；模型评测暂无独立数据源，置 0
|
||
service_status = [
|
||
{
|
||
"type": "模型推理",
|
||
"status": "error" if (nodes and not online_nodes) else ("busy" if (nodes and len(online_nodes) < len(nodes)) else "normal"),
|
||
"count": len(online_nodes),
|
||
},
|
||
{
|
||
"type": "模型微调",
|
||
"status": "error" if failed_ft else ("busy" if running_ft else "normal"),
|
||
"count": len(running_ft),
|
||
},
|
||
{"type": "模型评测", "status": "normal", "count": 0},
|
||
{
|
||
"type": "数据处理",
|
||
"status": "normal" if not failed_ft else "busy",
|
||
"count": len(datasets),
|
||
},
|
||
]
|
||
|
||
# 训练任务状态归一化（fine_tune 的 syncing/queued 等映射到前端已知状态）
|
||
status_map = {
|
||
"syncing": "running",
|
||
"queued": "running",
|
||
"running": "running",
|
||
"pending": "pending",
|
||
"paused": "pending",
|
||
"completed": "completed",
|
||
"failed": "failed",
|
||
"error": "failed",
|
||
"cancelled": "failed",
|
||
}
|
||
# 用户操作分布：仅展示「数据治理」与「模型服务」两大分组下的子模块，其他不显示
|
||
MODULE_LABELS = [
|
||
("data-process", "数据处理"),
|
||
("data_process", "数据处理"),
|
||
("dataset", "数据集管理"),
|
||
("fine-tune", "模型训练"),
|
||
("fine_tune", "模型训练"),
|
||
("model-eval", "模型评测"),
|
||
("eval", "模型评测"),
|
||
("model-inference", "模型推理"),
|
||
("inference", "模型推理"),
|
||
("model-manage", "模型管理"),
|
||
("model", "模型管理"),
|
||
("trained", "模型管理"),
|
||
]
|
||
OP_ORDER = ["数据集管理", "数据处理", "模型训练", "模型评测", "模型推理", "模型管理"]
|
||
|
||
def _op_module(action: str) -> str | None:
|
||
a = (action or "").lower()
|
||
for prefix, label in MODULE_LABELS:
|
||
if a.startswith(prefix):
|
||
return label
|
||
return None
|
||
|
||
training_tasks = [
|
||
{
|
||
"id": t.get("id"),
|
||
"name": t.get("name"),
|
||
"status": status_map.get(t.get("status"), "pending"),
|
||
"train_type": t.get("train_type"),
|
||
"train_method": t.get("train_method"),
|
||
"base_model": t.get("base_model"),
|
||
"progress": t.get("progress", 0),
|
||
"accuracy": t.get("accuracy"),
|
||
"started_at": (t.get("create_time") or "")[:16],
|
||
}
|
||
for t in tasks[:8]
|
||
]
|
||
|
||
# 用户操作分布：仅统计数据治理/模型服务下子模块的操作，其他不显示
|
||
audit = store.audit_logs(limit=1000)
|
||
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
|
||
for log in audit.get("items", []):
|
||
label = _op_module(log.get("action") or "")
|
||
if label:
|
||
op_counter[label] += 1
|
||
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
|
||
|
||
# 最近登录用户：后端有 last_login 字段，返回真实数据
|
||
recent = sorted(
|
||
[u for u in users if u.get("last_login")],
|
||
key=lambda u: u["last_login"],
|
||
reverse=True,
|
||
)[:5]
|
||
recent_login_users = [
|
||
{
|
||
"user": u.get("display_name") or u.get("username"),
|
||
"role": u.get("role"),
|
||
"last_login": (u.get("last_login") or "")[:16],
|
||
}
|
||
for u in recent
|
||
]
|
||
# 登录时长排行：基于 sessions 表真实会话时长（本月）
|
||
login_duration_rank = store.login_duration_rank()
|
||
|
||
return ok(
|
||
{
|
||
"online_services": sum(s["count"] for s in service_status),
|
||
"running_tasks": len(running_ft),
|
||
"pending_alerts": 0, # 平台暂无独立告警数据源，先置 0，待接入后补充
|
||
"training_7d": training_7d,
|
||
"service_status": service_status,
|
||
"training_tasks": training_tasks,
|
||
"operation_distribution": operation_distribution,
|
||
"login_duration_rank": login_duration_rank,
|
||
"recent_login_users": recent_login_users,
|
||
}
|
||
)
|
||
|
||
|
||
@router.get("/system-info")
|
||
async def system_info() -> dict[str, Any]:
|
||
return ok(get_platform_store().system_info())
|
||
|
||
|
||
@router.get("/users")
|
||
async def users() -> dict[str, Any]:
|
||
return ok(get_platform_store().users())
|
||
|
||
|
||
@router.post("/users")
|
||
async def create_user(payload: dict[str, Any] = Body(...), request: FastAPIRequest = None) -> dict[str, Any]:
|
||
store = get_platform_store()
|
||
user = store.create_user(payload)
|
||
store.record_audit(
|
||
action="user.create",
|
||
actor_id=_actor(request),
|
||
target_type="user",
|
||
target_id=user["id"],
|
||
detail=f"username={user.get('username')}",
|
||
)
|
||
return ok(user)
|
||
|
||
|
||
@router.put("/users/{user_id}")
|
||
async def update_user(user_id: str, payload: dict[str, Any] = Body(...), request: FastAPIRequest = None) -> dict[str, Any]:
|
||
store = get_platform_store()
|
||
try:
|
||
user = store.update_user(user_id, payload)
|
||
except KeyError:
|
||
raise fail(404, "user not found")
|
||
store.record_audit(
|
||
action="user.update",
|
||
actor_id=_actor(request),
|
||
target_type="user",
|
||
target_id=user_id,
|
||
detail=f"fields={','.join(payload.keys())}",
|
||
)
|
||
return ok(user)
|
||
|
||
|
||
@router.delete("/users/{user_id}")
|
||
async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]:
|
||
try:
|
||
get_platform_store().delete_user(user_id)
|
||
return ok({"deleted": user_id, "current_username": current_username})
|
||
except KeyError:
|
||
raise fail(404, "user not found")
|
||
except ValueError as exc:
|
||
raise fail(400, str(exc))
|
||
|
||
|
||
@router.post("/users/{user_id}/reset-password")
|
||
async def reset_password(
|
||
user_id: str,
|
||
payload: dict[str, Any] = Body(default={}),
|
||
request: FastAPIRequest = None,
|
||
) -> dict[str, Any]:
|
||
try:
|
||
user = get_platform_store().reset_password(user_id, payload.get("password") or "")
|
||
except KeyError:
|
||
raise fail(404, "user not found")
|
||
except ValueError as exc:
|
||
raise fail(400, str(exc))
|
||
get_platform_store().record_audit(
|
||
action="user.reset_password",
|
||
actor_id=_actor(request),
|
||
target_type="user",
|
||
target_id=user_id,
|
||
detail=f"username={user.get('username')}",
|
||
)
|
||
return ok({"id": user_id})
|
||
|
||
|
||
@router.get("/model-manage/local-models")
|
||
async def local_models() -> dict[str, Any]:
|
||
models = [{"path": item.get("path") or "", "name": item["name"]} for item in get_platform_store().models()]
|
||
return ok({"models": models})
|
||
|
||
|
||
@router.get("/model-manage/trained-models")
|
||
async def trained_models() -> dict[str, Any]:
|
||
return ok({"models": get_platform_store().trained_models()})
|
||
|
||
|
||
@router.delete("/model-manage/trained-models/{model_id}")
|
||
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]:
|
||
return ok({"deleted": model_id, "type": type})
|
||
|
||
|
||
@router.get("/model-manage/name/{name}")
|
||
async def model_by_name(name: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().model_by_name(name))
|
||
except KeyError:
|
||
raise fail(404, "model not found")
|
||
|
||
|
||
@router.get("/model-manage")
|
||
async def model_list() -> dict[str, Any]:
|
||
return ok(get_platform_store().models())
|
||
|
||
|
||
@router.post("/model-manage")
|
||
async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok(get_platform_store().create_model(payload))
|
||
|
||
|
||
@router.get("/model-manage/{model_id}")
|
||
async def model_detail(model_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().model(model_id))
|
||
except KeyError:
|
||
raise fail(404, "model not found")
|
||
|
||
|
||
@router.put("/model-manage/{model_id}")
|
||
async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_model(model_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "model not found")
|
||
|
||
|
||
@router.put("/model-manage/{model_id}/purpose")
|
||
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")}))
|
||
except KeyError:
|
||
raise fail(404, "model not found")
|
||
|
||
|
||
@router.delete("/model-manage/{model_id}")
|
||
async def delete_model(model_id: str) -> dict[str, Any]:
|
||
get_platform_store().delete_model(model_id)
|
||
return ok({"deleted": model_id})
|
||
|
||
|
||
@router.post("/model-manage/merge")
|
||
async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok({"job_id": f"merge_{uuid.uuid4().hex[:12]}", "status": "queued", **payload})
|
||
|
||
|
||
@router.get("/dataset-manage/preview/{file_id}")
|
||
async def dataset_preview(file_id: str) -> dict[str, Any]:
|
||
try:
|
||
row = get_platform_store().dataset_file(file_id)
|
||
return ok({"content": row["content"]})
|
||
except KeyError:
|
||
raise fail(404, "dataset file not found")
|
||
|
||
|
||
@router.get("/dataset-manage/versions/{file_id}")
|
||
async def dataset_versions(file_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().file_versions(file_id))
|
||
except KeyError:
|
||
raise fail(404, "dataset file not found")
|
||
|
||
|
||
@router.get("/dataset-manage/versions/{file_id}/{version_id}")
|
||
async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]:
|
||
try:
|
||
row = get_platform_store().dataset_file(file_id)
|
||
versions = get_platform_store().file_versions(file_id)["versions"]
|
||
version = next((item for item in versions if item["id"] == version_id), None)
|
||
if not version:
|
||
raise KeyError(version_id)
|
||
return ok({"version": version, "content": row["content"]})
|
||
except KeyError:
|
||
raise fail(404, "dataset version not found")
|
||
|
||
|
||
@router.post("/dataset-manage/versions/{file_id}")
|
||
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().create_file_version(file_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "dataset file not found")
|
||
|
||
|
||
@router.put("/dataset-manage/versions/{file_id}/active")
|
||
async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().activate_file_version(file_id, payload["version_id"]))
|
||
except KeyError:
|
||
raise fail(404, "dataset version not found")
|
||
|
||
|
||
@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
|
||
async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]:
|
||
return ok(get_platform_store().file_versions(file_id))
|
||
|
||
|
||
@router.post("/dataset-manage/upload/{dataset_id}")
|
||
async def upload_dataset_files(dataset_id: str, files: list[UploadFile] = File(default=[])) -> dict[str, Any]:
|
||
created: list[dict[str, Any]] = []
|
||
store = get_platform_store()
|
||
try:
|
||
store.dataset(dataset_id)
|
||
except KeyError:
|
||
raise fail(404, "dataset not found")
|
||
with store.connect() as conn:
|
||
for file in files:
|
||
raw = await file.read()
|
||
content = raw.decode("utf-8", errors="replace")
|
||
created.append(store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content))
|
||
return ok({"files": created})
|
||
|
||
|
||
@router.get("/dataset-manage/download/{dataset_id}")
|
||
async def download_dataset(dataset_id: str) -> PlainTextResponse:
|
||
dataset = get_platform_store().dataset(dataset_id)
|
||
content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])])
|
||
return PlainTextResponse(content, media_type="text/plain")
|
||
|
||
|
||
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
|
||
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
|
||
row = get_platform_store().dataset_file(file_id)
|
||
return PlainTextResponse(row["content"], media_type="text/plain")
|
||
|
||
|
||
@router.get("/dataset-manage")
|
||
async def dataset_list() -> dict[str, Any]:
|
||
return ok(get_platform_store().datasets())
|
||
|
||
|
||
@router.post("/dataset-manage")
|
||
async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
dataset = get_platform_store().create_dataset(payload)
|
||
return ok({"id": dataset["id"]})
|
||
|
||
|
||
@router.get("/dataset-manage/{dataset_id}")
|
||
async def dataset_detail(dataset_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().dataset(dataset_id))
|
||
except KeyError:
|
||
raise fail(404, "dataset not found")
|
||
|
||
|
||
@router.put("/dataset-manage/{dataset_id}")
|
||
async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_dataset(dataset_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "dataset not found")
|
||
|
||
|
||
@router.delete("/dataset-manage/{dataset_id}")
|
||
async def delete_dataset(dataset_id: str) -> dict[str, Any]:
|
||
get_platform_store().delete_dataset(dataset_id)
|
||
return ok({"deleted": dataset_id})
|
||
|
||
|
||
@router.get("/fine-tune/check-name")
|
||
async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]:
|
||
exists = any(task["name"] == name for task in get_platform_store().tasks())
|
||
return ok({"exists": exists})
|
||
|
||
|
||
@router.get("/fine-tune/progress/{task_id}")
|
||
async def fine_tune_progress(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().progress(task_id))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
|
||
|
||
@router.post("/fine-tune/tensorboard/start")
|
||
async def tensorboard_start() -> dict[str, Any]:
|
||
return ok({"status": "running", "url": "http://localhost:6006"})
|
||
|
||
|
||
@router.get("/fine-tune")
|
||
async def fine_tune_list() -> dict[str, Any]:
|
||
return ok(get_platform_store().tasks())
|
||
|
||
|
||
@router.post("/fine-tune")
|
||
async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
task = get_platform_store().create_task(apply_presets(payload))
|
||
return ok({"id": task["id"]})
|
||
except ValueError as exc:
|
||
raise fail(400, str(exc))
|
||
|
||
|
||
@router.post("/fine-tune/start")
|
||
async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().start_task(payload))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
except RuntimeError as exc:
|
||
raise fail(409, str(exc))
|
||
|
||
|
||
@router.get("/fine-tune/{task_id}")
|
||
async def fine_tune_detail(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().task(task_id))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
|
||
|
||
@router.put("/fine-tune/{task_id}")
|
||
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_task(task_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
|
||
|
||
@router.post("/fine-tune/stop/{task_id}")
|
||
async def stop_fine_tune(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().stop_task(task_id))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
|
||
|
||
@router.post("/fine-tune/{task_id}/stop")
|
||
async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]:
|
||
return await stop_fine_tune(task_id)
|
||
|
||
|
||
@router.post("/fine-tune/pause/{task_id}")
|
||
async def pause_fine_tune(task_id: str) -> dict[str, Any]:
|
||
if not get_platform_store().pause_task(task_id):
|
||
raise fail(409, "当前没有可暂停的训练进程")
|
||
return ok({"paused": task_id})
|
||
|
||
|
||
@router.post("/fine-tune/resume/{task_id}")
|
||
async def resume_fine_tune(task_id: str) -> dict[str, Any]:
|
||
if not get_platform_store().resume_task_engine(task_id):
|
||
raise fail(409, "当前没有可恢复的训练进程")
|
||
return ok({"resumed": task_id})
|
||
|
||
|
||
@router.post("/fine-tune/cancel/{task_id}")
|
||
async def cancel_fine_tune(task_id: str) -> dict[str, Any]:
|
||
get_platform_store().cancel_task_engine(task_id)
|
||
return ok({"canceled": task_id})
|
||
|
||
|
||
@router.delete("/fine-tune/{task_id}")
|
||
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
|
||
get_platform_store().delete_task(task_id)
|
||
return ok({"deleted": task_id})
|
||
|
||
|
||
@router.get("/fine-tune/{task_id}/overview")
|
||
async def fine_tune_overview(task_id: str) -> dict[str, Any]:
|
||
task = get_platform_store().task(task_id)
|
||
return ok({"task": task, "progress": get_platform_store().progress(task_id)})
|
||
|
||
|
||
@router.get("/fine-tune/{task_id}/checkpoints")
|
||
async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]:
|
||
task = get_platform_store().task(task_id)
|
||
checkpoints = []
|
||
for step in [50, 100, 150]:
|
||
if task.get("progress", 0) >= min(100, step // 2):
|
||
checkpoints.append({"step": step, "path": f"/data/yg-ft/outputs/{task['name']}/checkpoint-{step}"})
|
||
return ok(checkpoints)
|
||
|
||
|
||
@router.get("/compute/nodes")
|
||
async def compute_nodes() -> dict[str, Any]:
|
||
return ok(get_platform_store().compute_nodes())
|
||
|
||
|
||
@router.post("/compute/nodes")
|
||
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().create_compute_node(payload))
|
||
except KeyError as exc:
|
||
raise fail(400, f"missing field: {exc}")
|
||
|
||
|
||
@router.put("/compute/nodes/{node_id}")
|
||
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_compute_node(node_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "compute node not found")
|
||
|
||
|
||
@router.post("/compute/nodes/{node_id}/test-connection")
|
||
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
||
store = get_platform_store()
|
||
nodes = store.compute_nodes()
|
||
node = next((item for item in nodes if item["id"] == node_id), None)
|
||
if not node:
|
||
raise fail(404, "compute node not found")
|
||
try:
|
||
all_gpus = store.gpus()
|
||
node_gpus = [g for g in all_gpus if g.get("node_id") == node_id]
|
||
return ok({
|
||
"node_id": node_id,
|
||
"success": True,
|
||
"latency_ms": 12,
|
||
"gpu_count": len(node_gpus),
|
||
})
|
||
except Exception as exc: # noqa: BLE001
|
||
return ok({
|
||
"node_id": node_id,
|
||
"success": False,
|
||
"latency_ms": 0,
|
||
"gpu_count": 0,
|
||
"error": str(exc),
|
||
})
|
||
|
||
|
||
@router.post("/compute/nodes/{node_id}/enable")
|
||
async def enable_compute_node(node_id: str) -> dict[str, Any]:
|
||
return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))
|
||
|
||
|
||
@router.post("/compute/nodes/{node_id}/disable")
|
||
async def disable_compute_node(node_id: str) -> dict[str, Any]:
|
||
return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))
|
||
|
||
|
||
@router.post("/compute/nodes/{node_id}/drain")
|
||
async def drain_compute_node(node_id: str) -> dict[str, Any]:
|
||
return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))
|
||
|
||
|
||
@router.get("/compute/nodes/{node_id}/replicas")
|
||
async def compute_node_replicas(node_id: str) -> dict[str, Any]:
|
||
return ok(get_platform_store().replicas(node_id))
|
||
|
||
|
||
@router.get("/compute/gpus")
|
||
async def compute_gpus() -> dict[str, Any]:
|
||
return ok(get_platform_store().gpus())
|
||
|
||
|
||
@router.get("/compute/queue")
|
||
async def compute_queue() -> dict[str, Any]:
|
||
return ok(get_platform_store().queue())
|
||
|
||
|
||
@router.post("/internal/compute-sync/resources")
|
||
async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
sync_id = get_platform_store().create_sync_job(payload.get("target_node_id", "node_01"), payload)
|
||
return ok(get_platform_store().sync_job(sync_id))
|
||
|
||
|
||
@router.get("/internal/compute-sync/resources/{sync_id}")
|
||
async def compute_sync_detail(sync_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().sync_job(sync_id))
|
||
except KeyError:
|
||
raise fail(404, "sync job not found")
|
||
|
||
|
||
@router.get("/training-log-files")
|
||
async def training_log_files() -> dict[str, Any]:
|
||
return ok(get_platform_store().training_log_files())
|
||
|
||
|
||
@router.get("/training-log-content")
|
||
async def training_log_content(file: str = Query(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().training_log_content(file))
|
||
except KeyError:
|
||
raise fail(404, "training log not found")
|
||
|
||
|
||
@router.get("/log-files")
|
||
async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]:
|
||
return ok(get_platform_store().log_files(date))
|
||
|
||
|
||
@router.get("/log-content")
|
||
async def log_content(file: str = Query(...)) -> dict[str, Any]:
|
||
return ok(get_platform_store().log_content(file))
|
||
|
||
|
||
@router.post("/web-log")
|
||
async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok({"received": True, **payload})
|
||
|
||
|
||
# ===================== Project Management (§13.2) =====================
|
||
|
||
|
||
@router.get("/projects")
|
||
async def project_list(
|
||
tenant_id: str = Query(default="default"),
|
||
status: str | None = Query(default=None),
|
||
keyword: str | None = Query(default=None),
|
||
) -> dict[str, Any]:
|
||
return ok(get_platform_store().projects(tenant_id, status, keyword))
|
||
|
||
|
||
@router.post("/projects")
|
||
async def create_project(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
project = get_platform_store().create_project(payload)
|
||
return ok({"id": project["id"]})
|
||
except KeyError as exc:
|
||
raise fail(400, f"missing required field: {exc}")
|
||
|
||
|
||
@router.get("/projects/{project_id}")
|
||
async def project_detail(project_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().project(project_id))
|
||
except KeyError:
|
||
raise fail(404, "project not found")
|
||
|
||
|
||
@router.put("/projects/{project_id}")
|
||
async def update_project(project_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_project(project_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "project not found")
|
||
|
||
|
||
@router.post("/projects/{project_id}/activate")
|
||
async def activate_project(project_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().activate_project(project_id))
|
||
except KeyError:
|
||
raise fail(404, "project not found")
|
||
|
||
|
||
@router.get("/projects/{project_id}/members")
|
||
async def project_members(project_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().project_members(project_id))
|
||
except KeyError:
|
||
raise fail(404, "project not found")
|
||
|
||
|
||
@router.post("/projects/{project_id}/members")
|
||
async def add_project_member(project_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
user_id = payload.get("user_id")
|
||
if not user_id:
|
||
raise fail(400, "user_id is required")
|
||
try:
|
||
return ok(get_platform_store().add_project_member(project_id, user_id, payload.get("role", "member")))
|
||
except KeyError as exc:
|
||
raise fail(404, str(exc))
|
||
|
||
|
||
@router.put("/projects/{project_id}/members/{user_id}")
|
||
async def update_project_member_role(project_id: str, user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_project_member_role(project_id, user_id, payload.get("role", "member")))
|
||
except KeyError as exc:
|
||
raise fail(404, str(exc))
|
||
|
||
|
||
@router.delete("/projects/{project_id}/members/{user_id}")
|
||
async def remove_project_member(project_id: str, user_id: str) -> dict[str, Any]:
|
||
try:
|
||
get_platform_store().remove_project_member(project_id, user_id)
|
||
return ok({"deleted": user_id})
|
||
except KeyError as exc:
|
||
raise fail(404, str(exc))
|
||
|
||
|
||
# ===================== Fine-tune Events (§7.1) =====================
|
||
|
||
@router.get("/fine-tune/{task_id}/events")
|
||
async def fine_tune_events(task_id: str) -> StreamingResponse:
|
||
import json as _json
|
||
|
||
async def event_stream():
|
||
store = get_platform_store()
|
||
try:
|
||
events = store.task_events(task_id)
|
||
for event in events:
|
||
yield f"data: {_json.dumps(event, default=str)}\n\n"
|
||
yield f"data: {_json.dumps({'type': 'done', 'data': {}}, default=str)}\n\n"
|
||
except KeyError:
|
||
yield f"data: {_json.dumps({'type': 'error', 'data': {'message': 'task not found'}}, default=str)}\n\n"
|
||
|
||
return StreamingResponse(event_stream(), media_type="text/event-stream")
|
||
|
||
|
||
# ===================== Fine-tune Retry & Resume (§13.9) =====================
|
||
|
||
|
||
@router.post("/fine-tune/{task_id}/retry")
|
||
async def retry_fine_tune(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().retry_task(task_id))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
except ValueError as exc:
|
||
raise fail(400, str(exc))
|
||
|
||
|
||
@router.post("/fine-tune/{task_id}/resume")
|
||
async def resume_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
checkpoint_id = payload.get("checkpoint_id")
|
||
if not checkpoint_id:
|
||
raise fail(400, "checkpoint_id is required")
|
||
try:
|
||
return ok(get_platform_store().resume_task(task_id, checkpoint_id))
|
||
except KeyError as exc:
|
||
raise fail(404, str(exc))
|
||
except ValueError as exc:
|
||
raise fail(400, str(exc))
|
||
|
||
|
||
# ===================== Checkpoint Management (§13.9) =====================
|
||
|
||
|
||
@router.delete("/fine-tune/{task_id}/checkpoints/{checkpoint_id}")
|
||
async def delete_checkpoint(task_id: str, checkpoint_id: str) -> dict[str, Any]:
|
||
try:
|
||
get_platform_store().delete_checkpoint(checkpoint_id)
|
||
return ok({"deleted": checkpoint_id})
|
||
except KeyError:
|
||
raise fail(404, "checkpoint not found")
|
||
|
||
|
||
@router.put("/fine-tune/{task_id}/checkpoint-retention")
|
||
async def set_checkpoint_retention(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().set_checkpoint_retention(task_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
|
||
|
||
@router.get("/fine-tune/{task_id}/checkpoint-retention")
|
||
async def get_checkpoint_retention(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().get_checkpoint_retention(task_id))
|
||
except KeyError:
|
||
raise fail(404, "fine tune task not found")
|
||
|
||
|
||
# ===================== Compute Jobs (§13.6) =====================
|
||
|
||
|
||
@router.post("/compute/jobs")
|
||
async def create_compute_job(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
job = get_platform_store().create_compute_job(payload)
|
||
return ok({"id": job["id"]})
|
||
except KeyError as exc:
|
||
raise fail(400, f"missing required field: {exc}")
|
||
|
||
|
||
@router.get("/compute/jobs/{job_id}")
|
||
async def compute_job_detail(job_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().compute_job(job_id))
|
||
except KeyError:
|
||
raise fail(404, "compute job not found")
|
||
|
||
|
||
@router.get("/compute/jobs")
|
||
async def compute_jobs(task_id: str = Query(default=None)) -> dict[str, Any]:
|
||
if task_id:
|
||
return ok(get_platform_store().compute_jobs_by_task(task_id))
|
||
return ok([])
|
||
|
||
|
||
@router.post("/compute/jobs/{job_id}/stop")
|
||
async def stop_compute_job(job_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().stop_compute_job(job_id))
|
||
except KeyError:
|
||
raise fail(404, "compute job not found")
|
||
|
||
|
||
@router.get("/compute/jobs/{job_id}/logs")
|
||
async def compute_job_logs(job_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().compute_job_logs(job_id))
|
||
except KeyError as exc:
|
||
raise fail(404, str(exc))
|
||
|
||
|
||
# ===================== Model Evaluation =====================
|
||
|
||
|
||
@router.get("/model-eval")
|
||
async def model_eval_list() -> dict[str, Any]:
|
||
return ok(get_platform_store().eval_tasks())
|
||
|
||
|
||
@router.get("/model-eval/{task_id}")
|
||
async def model_eval_detail(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().eval_task(task_id))
|
||
except KeyError:
|
||
raise fail(404, "eval task not found")
|
||
|
||
|
||
@router.post("/model-eval/start")
|
||
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
task = get_platform_store().create_eval_task(payload)
|
||
return ok({"task_id": task["id"], **task})
|
||
|
||
|
||
@router.delete("/model-eval/{task_id}")
|
||
async def model_eval_delete(task_id: str) -> dict[str, Any]:
|
||
get_platform_store().delete_eval_task(task_id)
|
||
return ok({"deleted": task_id})
|
||
|
||
|
||
# ===================== Eval Dimensions =====================
|
||
|
||
|
||
@router.get("/dimension")
|
||
async def dimension_list() -> dict[str, Any]:
|
||
return ok(get_platform_store().dimensions())
|
||
|
||
|
||
@router.post("/dimension")
|
||
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok(get_platform_store().create_dimension(payload))
|
||
|
||
|
||
@router.get("/dimension/{dimension_id}")
|
||
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().dimension(dimension_id))
|
||
except KeyError:
|
||
raise fail(404, "dimension not found")
|
||
|
||
|
||
@router.put("/dimension/{dimension_id}")
|
||
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_dimension(dimension_id, payload))
|
||
except KeyError:
|
||
raise fail(404, "dimension not found")
|
||
|
||
|
||
@router.delete("/dimension/{dimension_id}")
|
||
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
|
||
get_platform_store().delete_dimension(dimension_id)
|
||
return ok({"deleted": dimension_id})
|
||
|
||
|
||
# ===================== Model Compare / Inference =====================
|
||
|
||
|
||
@router.get("/model-compare")
|
||
async def model_compare_list() -> dict[str, Any]:
|
||
return ok(get_platform_store().compare_tasks())
|
||
|
||
|
||
@router.post("/model-compare")
|
||
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
task = get_platform_store().create_compare_task(payload)
|
||
return ok({"id": task["id"]})
|
||
|
||
|
||
@router.post("/model-compare/all/stop-all")
|
||
async def model_compare_stop_all() -> dict[str, Any]:
|
||
return ok({"stopped": True})
|
||
|
||
|
||
@router.post("/model-compare/stop-by-pid")
|
||
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok({"stopped": True, "pid": payload.get("pid")})
|
||
|
||
|
||
@router.get("/model-compare/{task_id}")
|
||
async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().compare_task(task_id))
|
||
except KeyError:
|
||
raise fail(404, "compare task not found")
|
||
|
||
|
||
@router.delete("/model-compare/{task_id}")
|
||
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
||
get_platform_store().delete_compare_task(task_id)
|
||
return ok({"deleted": task_id})
|
||
|
||
|
||
@router.get("/model-compare/{task_id}/load-status")
|
||
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
task = get_platform_store().compare_task(task_id)
|
||
except KeyError:
|
||
raise fail(404, "compare task not found")
|
||
load_status = task.get("load_status") or {"loaded_models": []}
|
||
if isinstance(load_status, str):
|
||
try:
|
||
load_status = json.loads(load_status)
|
||
except json.JSONDecodeError:
|
||
load_status = {"loaded_models": []}
|
||
return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})
|
||
|
||
|
||
@router.post("/model-compare/{task_id}/load-status")
|
||
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
|
||
except KeyError:
|
||
raise fail(404, "compare task not found")
|
||
|
||
|
||
@router.post("/model-compare/{task_id}/load")
|
||
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
task = get_platform_store().compare_task(task_id)
|
||
models = task.get("models") or []
|
||
if isinstance(models, str):
|
||
try:
|
||
models = json.loads(models)
|
||
except json.JSONDecodeError:
|
||
models = []
|
||
loaded_models = [
|
||
{
|
||
"model_id": item.get("model_id"),
|
||
"model_name": item.get("model_name"),
|
||
"status": "ready",
|
||
"pid": 45000 + index,
|
||
"port": item.get("port") or 18000 + index,
|
||
}
|
||
for index, item in enumerate(models)
|
||
if isinstance(item, dict)
|
||
]
|
||
return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
|
||
except KeyError:
|
||
raise fail(404, "compare task not found")
|
||
|
||
|
||
@router.post("/model-compare/{task_id}/unload")
|
||
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
||
try:
|
||
return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
|
||
except KeyError:
|
||
raise fail(404, "compare task not found")
|
||
|
||
|
||
@router.post("/model-compare/{task_id}/start-model")
|
||
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})
|
||
|
||
|
||
@router.post("/model-compare/chat-with-port")
|
||
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
question = ""
|
||
for message in payload.get("messages") or []:
|
||
if message.get("role") == "user":
|
||
question = str(message.get("content") or "")
|
||
content = f"当前后端已收到推理请求：{question[:120]}"
|
||
return ok({"response": content, "content": content})
|
||
|
||
|
||
@router.post("/model-compare/stream-chat")
|
||
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
question = payload.get("user_question") or payload.get("question") or ""
|
||
return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"})
|
||
|
||
|
||
# ===================== Model Chat (Inference Proxy) =====================
|
||
|
||
|
||
@router.post("/model-chat/batch")
|
||
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
return ok({"responses": [], "request": payload})
|
||
|
||
|
||
@router.post("/model-chat/local/chat")
|
||
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
"""Proxy chat to the compute node running the inference model."""
|
||
store = get_platform_store()
|
||
node = _select_first_online_node(store)
|
||
if not node:
|
||
return ok({"response": "no online compute node available for inference", "request": payload})
|
||
try:
|
||
client = ComputeNodeClient(node["api_base_url"])
|
||
result = await client._request("POST", "/inference/chat", json_data=payload)
|
||
return ok(result)
|
||
except Exception as exc:
|
||
return ok({"response": f"inference failed: {exc}", "request": payload})
|
||
|
||
|
||
@router.post("/model-chat/local/chat/stream")
|
||
async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
|
||
"""Stream chat from the compute node."""
|
||
store = get_platform_store()
|
||
node = _select_first_online_node(store)
|
||
if not node:
|
||
return StreamingResponse(
|
||
iter(['data: {"error": "no online compute node"}\n\n']),
|
||
media_type="text/event-stream",
|
||
)
|
||
client = ComputeNodeClient(node["api_base_url"])
|
||
|
||
async def stream_proxy():
|
||
async with httpx.AsyncClient(timeout=300) as http:
|
||
url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream"
|
||
async with http.stream("POST", url, json=payload, headers=client.headers()) as resp:
|
||
async for chunk in resp.aiter_bytes():
|
||
yield chunk
|
||
|
||
return StreamingResponse(stream_proxy(), media_type="text/event-stream")
|
||
|
||
|
||
@router.post("/model-chat/local/preload")
|
||
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
"""Load a model on the compute node for inference."""
|
||
store = get_platform_store()
|
||
node = _select_first_online_node(store)
|
||
if not node:
|
||
return ok({"loaded": False, "error": "no online compute node"})
|
||
try:
|
||
client = ComputeNodeClient(node["api_base_url"])
|
||
result = await client._request("POST", "/inference/load", json_data=payload)
|
||
return ok(result)
|
||
except Exception as exc:
|
||
return ok({"loaded": False, "error": str(exc)})
|
||
|
||
|
||
@router.post("/model-chat/local/unload")
|
||
async def model_chat_local_unload() -> dict[str, Any]:
|
||
"""Unload the inference model from the compute node."""
|
||
store = get_platform_store()
|
||
node = _select_first_online_node(store)
|
||
if not node:
|
||
return ok({"unloaded": False, "error": "no online compute node"})
|
||
try:
|
||
client = ComputeNodeClient(node["api_base_url"])
|
||
result = await client._request("POST", "/inference/unload", json_data={})
|
||
return ok(result)
|
||
except Exception as exc:
|
||
return ok({"unloaded": False, "error": str(exc)})
|
||
|
||
|
||
@router.get("/model-chat/local/status")
|
||
async def model_chat_local_status() -> dict[str, Any]:
|
||
"""Get inference session status from compute node."""
|
||
store = get_platform_store()
|
||
node = _select_first_online_node(store)
|
||
if not node:
|
||
return ok({"loaded": False, "error": "no online compute node"})
|
||
try:
|
||
client = ComputeNodeClient(node["api_base_url"])
|
||
result = await client._request("GET", "/inference/status")
|
||
return ok(result)
|
||
except Exception as exc:
|
||
return ok({"loaded": False, "error": str(exc)})
|
||
|
||
|
||
@router.post("/model-chat/trained/preload")
|
||
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||
"""Load a trained model (base + adapter) on the compute node for inference."""
|
||
store = get_platform_store()
|
||
node = _select_first_online_node(store)
|
||
if not node:
|
||
return ok({"loaded": False, "error": "no online compute node"})
|
||
try:
|
||
client = ComputeNodeClient(node["api_base_url"])
|
||
result = await client._request("POST", "/inference/load", json_data=payload)
|
||
return ok(result)
|
||
except Exception as exc:
|
||
return ok({"loaded": False, "error": str(exc)})
|
||
|