836343b29e
- 新增 backend/app/modules/compute_gateway（client/sync）计算网关模块 - 新增 backend/app/workers/compute_poller 计算轮询 worker - 新增 compute/agent/process_manager 进程管理器 - 新增 scripts/ 脚本目录 - 更新 Docker 部署配置（app/compute/nginx） - 更新后端平台 API、数据库 SQL、core 配置 - 更新前端多个视图组件及 API 模块 - 重构 frontend/dist 构建产物（新 hash） - 更新多项文档 Co-Authored-By: Claude <noreply@anthropic.com>
1092 lines
41 KiB
Python
1092 lines
41 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
|
|
from fastapi.responses import PlainTextResponse
|
|
|
|
from app.core.config import get_settings
|
|
from app.db.platform_store import get_platform_store
|
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
|
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
|
|
return {"code": 0, "message": message, "data": data}
|
|
|
|
|
|
def fail(status_code: int, message: str) -> HTTPException:
|
|
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
|
|
|
|
|
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
|
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
|
|
|
|
|
def _task_for_compute_job(job_id: str) -> dict[str, Any] | None:
|
|
return next((task for task in get_platform_store().tasks() if task.get("compute_job_id") == job_id), None)
|
|
|
|
|
|
async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
|
|
task_id = str(payload.get("task_id") or payload.get("id") or "")
|
|
if task_id and get_settings().compute_mode != "simulator":
|
|
try:
|
|
preflight = await _fine_tune_preflight(store, task_id, payload, validate=True)
|
|
except Exception as exc: # noqa: BLE001 - task has not entered running state yet
|
|
raise RuntimeError(f"preflight failed: {exc}") from exc
|
|
if not preflight["valid"]:
|
|
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
|
|
raise RuntimeError(f"preflight failed: {errors}")
|
|
task = store.start_task(payload)
|
|
if get_settings().compute_mode == "simulator":
|
|
return task
|
|
node, job_payload = store.build_compute_job_payload(task["id"])
|
|
job = await ComputeNodeClient(node["api_base_url"]).create_job(job_payload)
|
|
return store.apply_compute_job(task["id"], job)
|
|
|
|
|
|
async def _fine_tune_preflight(
|
|
store: Any,
|
|
task_id: str,
|
|
payload: dict[str, Any] | None = None,
|
|
validate: bool = True,
|
|
) -> dict[str, Any]:
|
|
node, job_payload = store.prepare_compute_job_payload(task_id, payload or {})
|
|
if get_settings().compute_mode == "simulator":
|
|
preview = {
|
|
"valid": True,
|
|
"errors": [],
|
|
"warnings": ["compute_mode=simulator skips remote compute validation"],
|
|
"engine": job_payload.get("engine") or job_payload.get("training_engine") or "llama_factory",
|
|
"command": [],
|
|
"command_text": "",
|
|
"work_dir": "",
|
|
"env": {},
|
|
"path_checks": [],
|
|
}
|
|
else:
|
|
client = ComputeNodeClient(node["api_base_url"])
|
|
preview = await (client.validate_job(job_payload) if validate else client.preview_job(job_payload))
|
|
errors = list(preview.get("errors") or [])
|
|
warnings = list(preview.get("warnings") or [])
|
|
if not node.get("enabled"):
|
|
errors.append(f"compute node disabled: {node.get('code')}")
|
|
if node.get("scheduler_status") not in {"online", "draining"}:
|
|
errors.append(f"compute node not schedulable: {node.get('code')} status={node.get('scheduler_status')}")
|
|
return {
|
|
"valid": bool(preview.get("valid", not errors)) and not errors,
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
"node": {
|
|
"id": node.get("id"),
|
|
"code": node.get("code"),
|
|
"name": node.get("name"),
|
|
"api_base_url": node.get("api_base_url"),
|
|
"scheduler_status": node.get("scheduler_status"),
|
|
"gpu_count": node.get("gpu_count"),
|
|
},
|
|
"job_payload": job_payload,
|
|
"preview": preview,
|
|
}
|
|
|
|
|
|
@router.post("/login")
|
|
async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
user = get_platform_store().login(payload.get("username", ""), payload.get("password", ""))
|
|
if not user:
|
|
raise fail(401, "invalid username or password")
|
|
return ok({"token": f"platform-token-{user['id']}", "user": user})
|
|
|
|
|
|
@router.get("/me")
|
|
async def me() -> dict[str, Any]:
|
|
return ok(get_platform_store().users()[0])
|
|
|
|
|
|
@router.get("/dashboard/overview")
|
|
async def dashboard_overview() -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
tasks = store.tasks()
|
|
return ok(
|
|
{
|
|
"models": len(store.models()),
|
|
"datasets": len(store.datasets()),
|
|
"fine_tune_tasks": len(tasks),
|
|
"running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]),
|
|
"compute_nodes": len(store.compute_nodes()),
|
|
"gpus": len(store.gpus()),
|
|
}
|
|
)
|
|
|
|
|
|
@router.get("/system-info")
|
|
async def system_info() -> dict[str, Any]:
|
|
return ok(get_platform_store().system_info())
|
|
|
|
|
|
@router.get("/users")
|
|
async def users() -> dict[str, Any]:
|
|
return ok(get_platform_store().users())
|
|
|
|
|
|
@router.post("/users")
|
|
async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok(get_platform_store().create_user(payload))
|
|
|
|
|
|
@router.put("/users/{user_id}")
|
|
async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_user(user_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "user not found")
|
|
|
|
|
|
@router.delete("/users/{user_id}")
|
|
async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]:
|
|
try:
|
|
get_platform_store().delete_user(user_id)
|
|
return ok({"deleted": user_id, "current_username": current_username})
|
|
except KeyError:
|
|
raise fail(404, "user not found")
|
|
except ValueError as exc:
|
|
raise fail(400, str(exc))
|
|
|
|
|
|
@router.get("/model-manage/local-models")
|
|
async def local_models() -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
models = [{"path": item.get("path") or "", "name": item["name"], "source": "registered"} for item in store.models()]
|
|
seen = {item["path"] for item in models if item.get("path")}
|
|
if get_settings().compute_mode != "simulator":
|
|
for node in store.compute_nodes():
|
|
if not node.get("enabled"):
|
|
continue
|
|
try:
|
|
result = await ComputeNodeClient(node["api_base_url"]).list_files(root="models", directories_only=True)
|
|
except Exception:
|
|
continue
|
|
for item in result.get("items") or []:
|
|
path = str(item.get("path") or "")
|
|
if not path or path in seen:
|
|
continue
|
|
seen.add(path)
|
|
models.append(
|
|
{
|
|
"path": path,
|
|
"name": item.get("name") or path.rsplit("/", 1)[-1],
|
|
"source": f"compute:{node.get('code')}",
|
|
}
|
|
)
|
|
return ok({"models": models})
|
|
|
|
|
|
@router.get("/model-manage/trained-models")
|
|
async def trained_models() -> dict[str, Any]:
|
|
return ok({"models": get_platform_store().trained_models()})
|
|
|
|
|
|
@router.delete("/model-manage/trained-models/{model_id}")
|
|
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]:
|
|
get_platform_store().delete_trained_model(model_id)
|
|
return ok({"deleted": model_id, "type": type})
|
|
|
|
|
|
@router.get("/model-manage/name/{name}")
|
|
async def model_by_name(name: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().model_by_name(name))
|
|
except KeyError:
|
|
raise fail(404, "model not found")
|
|
|
|
|
|
@router.get("/model-manage")
|
|
async def model_list() -> dict[str, Any]:
|
|
return ok(get_platform_store().models())
|
|
|
|
|
|
@router.post("/model-manage")
|
|
async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().create_model(payload))
|
|
except KeyError as exc:
|
|
raise fail(400, f"missing field: {exc}")
|
|
except ValueError as exc:
|
|
raise fail(400, str(exc))
|
|
except Exception as exc: # noqa: BLE001 - keep API errors visible to deployment smoke checks
|
|
raise fail(500, f"create model failed: {exc}")
|
|
|
|
|
|
@router.get("/model-manage/{model_id}")
|
|
async def model_detail(model_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().model(model_id))
|
|
except KeyError:
|
|
raise fail(404, "model not found")
|
|
|
|
|
|
@router.put("/model-manage/{model_id}")
|
|
async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_model(model_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "model not found")
|
|
|
|
|
|
@router.put("/model-manage/{model_id}/purpose")
|
|
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")}))
|
|
except KeyError:
|
|
raise fail(404, "model not found")
|
|
|
|
|
|
@router.delete("/model-manage/{model_id}")
|
|
async def delete_model(model_id: str) -> dict[str, Any]:
|
|
get_platform_store().delete_model(model_id)
|
|
return ok({"deleted": model_id})
|
|
|
|
|
|
@router.post("/model-manage/merge")
|
|
async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"job_id": f"merge_{uuid.uuid4().hex[:12]}", "status": "queued", **payload})
|
|
|
|
|
|
@router.get("/dataset-manage/preview/{file_id}")
|
|
async def dataset_preview(file_id: str) -> dict[str, Any]:
|
|
try:
|
|
row = get_platform_store().dataset_file(file_id)
|
|
return ok({"content": row["content"]})
|
|
except KeyError:
|
|
raise fail(404, "dataset file not found")
|
|
|
|
|
|
@router.get("/dataset-manage/versions/{file_id}")
|
|
async def dataset_versions(file_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().file_versions(file_id))
|
|
except KeyError:
|
|
raise fail(404, "dataset file not found")
|
|
|
|
|
|
@router.get("/dataset-manage/versions/{file_id}/{version_id}")
|
|
async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]:
|
|
try:
|
|
row = get_platform_store().dataset_file(file_id)
|
|
versions = get_platform_store().file_versions(file_id)["versions"]
|
|
version = next((item for item in versions if item["id"] == version_id), None)
|
|
if not version:
|
|
raise KeyError(version_id)
|
|
return ok({"version": version, "content": row["content"]})
|
|
except KeyError:
|
|
raise fail(404, "dataset version not found")
|
|
|
|
|
|
@router.post("/dataset-manage/versions/{file_id}")
|
|
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().create_file_version(file_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "dataset file not found")
|
|
|
|
|
|
@router.put("/dataset-manage/versions/{file_id}/active")
|
|
async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().activate_file_version(file_id, payload["version_id"]))
|
|
except KeyError:
|
|
raise fail(404, "dataset version not found")
|
|
|
|
|
|
@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
|
|
async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().delete_file_version(file_id, version_id))
|
|
except KeyError:
|
|
raise fail(404, "dataset version not found")
|
|
except ValueError as exc:
|
|
raise fail(400, str(exc))
|
|
|
|
|
|
async def _sync_dataset_file_to_compute_nodes(
|
|
store: Any,
|
|
dataset_id: str,
|
|
file_id: str,
|
|
filename: str,
|
|
content: bytes,
|
|
) -> list[dict[str, Any]]:
|
|
results: list[dict[str, Any]] = []
|
|
if get_settings().compute_mode == "simulator":
|
|
return results
|
|
target_name = Path(filename or f"{file_id}.jsonl").name
|
|
target_relative_path = f"datasets/{dataset_id}/{target_name}"
|
|
for node in store.compute_nodes():
|
|
if not node.get("enabled"):
|
|
continue
|
|
try:
|
|
result = await ComputeNodeClient(node["api_base_url"]).upload_file(
|
|
target_name,
|
|
content,
|
|
target_relative_path,
|
|
resource_type="dataset",
|
|
resource_id=dataset_id,
|
|
)
|
|
store.upsert_resource_replica(
|
|
node["id"],
|
|
"dataset",
|
|
dataset_id,
|
|
str(result.get("local_path") or ""),
|
|
)
|
|
results.append(
|
|
{
|
|
"node_id": node["id"],
|
|
"node_code": node.get("code"),
|
|
"success": True,
|
|
"local_path": result.get("local_path"),
|
|
"byte_size": result.get("byte_size"),
|
|
"checksum_sha256": result.get("checksum_sha256"),
|
|
}
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - keep upload usable while exposing sync failures
|
|
results.append(
|
|
{
|
|
"node_id": node["id"],
|
|
"node_code": node.get("code"),
|
|
"success": False,
|
|
"error": str(exc),
|
|
}
|
|
)
|
|
return results
|
|
|
|
|
|
@router.post("/dataset-manage/upload/{dataset_id}")
|
|
async def upload_dataset_files(
|
|
dataset_id: str,
|
|
files: list[UploadFile] = File(default=[]),
|
|
sync_to_compute: bool = Query(default=True),
|
|
) -> dict[str, Any]:
|
|
created: list[dict[str, Any]] = []
|
|
compute_sync: list[dict[str, Any]] = []
|
|
store = get_platform_store()
|
|
try:
|
|
store.dataset(dataset_id)
|
|
except KeyError:
|
|
raise fail(404, "dataset not found")
|
|
with store.connect() as conn:
|
|
for file in files:
|
|
raw = await file.read()
|
|
content = raw.decode("utf-8", errors="replace")
|
|
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
|
|
created.append(created_file)
|
|
if sync_to_compute:
|
|
compute_sync.extend(
|
|
await _sync_dataset_file_to_compute_nodes(
|
|
store,
|
|
dataset_id,
|
|
created_file["id"],
|
|
created_file["name"],
|
|
raw,
|
|
)
|
|
)
|
|
return ok({"files": created, "compute_sync": compute_sync})
|
|
|
|
|
|
@router.get("/dataset-manage/download/{dataset_id}")
|
|
async def download_dataset(dataset_id: str) -> PlainTextResponse:
|
|
dataset = get_platform_store().dataset(dataset_id)
|
|
content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])])
|
|
return PlainTextResponse(content, media_type="text/plain")
|
|
|
|
|
|
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
|
|
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
|
|
row = get_platform_store().dataset_file(file_id)
|
|
return PlainTextResponse(row["content"], media_type="text/plain")
|
|
|
|
|
|
@router.get("/dataset-manage")
|
|
async def dataset_list() -> dict[str, Any]:
|
|
return ok(get_platform_store().datasets())
|
|
|
|
|
|
@router.post("/dataset-manage")
|
|
async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
dataset = get_platform_store().create_dataset(payload)
|
|
return ok({"id": dataset["id"]})
|
|
|
|
|
|
@router.get("/dataset-manage/{dataset_id}")
|
|
async def dataset_detail(dataset_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().dataset(dataset_id))
|
|
except KeyError:
|
|
raise fail(404, "dataset not found")
|
|
|
|
|
|
@router.put("/dataset-manage/{dataset_id}")
|
|
async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_dataset(dataset_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "dataset not found")
|
|
|
|
|
|
@router.delete("/dataset-manage/{dataset_id}")
|
|
async def delete_dataset(dataset_id: str) -> dict[str, Any]:
|
|
get_platform_store().delete_dataset(dataset_id)
|
|
return ok({"deleted": dataset_id})
|
|
|
|
|
|
@router.get("/fine-tune/check-name")
|
|
async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]:
|
|
exists = any(task["name"] == name for task in get_platform_store().tasks())
|
|
return ok({"exists": exists})
|
|
|
|
|
|
@router.get("/fine-tune/progress/{task_id}")
|
|
async def fine_tune_progress(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().progress(task_id))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
|
|
|
|
@router.post("/fine-tune/tensorboard/start")
|
|
async def tensorboard_start() -> dict[str, Any]:
|
|
return ok({"status": "running", "url": "http://localhost:6006"})
|
|
|
|
|
|
@router.get("/fine-tune")
|
|
async def fine_tune_list() -> dict[str, Any]:
|
|
return ok(get_platform_store().tasks())
|
|
|
|
|
|
@router.post("/fine-tune")
|
|
async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
task = get_platform_store().create_task(payload)
|
|
return ok({"id": task["id"]})
|
|
except ValueError as exc:
|
|
raise fail(400, str(exc))
|
|
|
|
|
|
@router.post("/fine-tune/start")
|
|
async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
try:
|
|
return ok(await _submit_fine_tune_task(store, payload))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
except RuntimeError as exc:
|
|
task_id = str(payload.get("task_id") or payload.get("id") or "")
|
|
if task_id:
|
|
store.mark_task_failed(task_id, str(exc))
|
|
raise fail(409, str(exc))
|
|
except Exception as exc: # noqa: BLE001 - mark task failed when remote submit fails
|
|
task_id = str(payload.get("task_id") or payload.get("id") or "")
|
|
if task_id:
|
|
store.mark_task_failed(task_id, str(exc))
|
|
raise fail(502, f"submit compute job failed: {exc}")
|
|
|
|
|
|
@router.post("/fine-tune/{task_id}/preflight")
|
|
async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
|
try:
|
|
return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
except RuntimeError as exc:
|
|
raise fail(409, str(exc))
|
|
except Exception as exc: # noqa: BLE001 - expose compute validation errors to training create page
|
|
raise fail(502, f"compute preflight failed: {exc}")
|
|
|
|
|
|
@router.post("/fine-tune/{task_id}/command-preview")
|
|
async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
|
try:
|
|
return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
except RuntimeError as exc:
|
|
raise fail(409, str(exc))
|
|
except Exception as exc: # noqa: BLE001
|
|
raise fail(502, f"compute command preview failed: {exc}")
|
|
|
|
|
|
@router.get("/fine-tune/{task_id}")
|
|
async def fine_tune_detail(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().task(task_id))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
|
|
|
|
@router.get("/fine-tune/{task_id}/logs")
|
|
async def fine_tune_logs(
|
|
task_id: str,
|
|
tail_lines: int | None = Query(default=500, ge=1, le=5000),
|
|
offset: int | None = Query(default=None, ge=0),
|
|
limit: int | None = Query(default=None, ge=1, le=5000),
|
|
) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
try:
|
|
task = store.task(task_id)
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
if task.get("compute_job_id"):
|
|
node = _node_for_task(task)
|
|
if node:
|
|
try:
|
|
logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], tail_lines, offset, limit)
|
|
if task.get("status") in {"queued", "running", "failed", "stopped", "completed"}:
|
|
try:
|
|
job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"])
|
|
store.apply_compute_job(task_id, job)
|
|
except Exception:
|
|
pass
|
|
return ok({"source": "compute", **logs})
|
|
except Exception as exc: # noqa: BLE001 - keep failure reason visible even when log fetch fails
|
|
content = task.get("failure_reason") or f"fetch compute log failed: {exc}"
|
|
return ok({"job_id": task.get("compute_job_id"), "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
|
|
content = task.get("failure_reason") or ""
|
|
return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
|
|
|
|
|
|
@router.put("/fine-tune/{task_id}")
|
|
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_task(task_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
|
|
|
|
@router.post("/fine-tune/stop/{task_id}")
|
|
async def stop_fine_tune(task_id: str) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
try:
|
|
task = store.task(task_id)
|
|
node = _node_for_task(task)
|
|
if task.get("compute_job_id") and node and get_settings().compute_mode != "simulator":
|
|
job = await ComputeNodeClient(node["api_base_url"]).stop_job(task["compute_job_id"])
|
|
return ok(store.apply_compute_job(task_id, job))
|
|
return ok(store.stop_task(task_id))
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
|
|
|
|
@router.post("/fine-tune/{task_id}/stop")
|
|
async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]:
|
|
return await stop_fine_tune(task_id)
|
|
|
|
|
|
@router.post("/fine-tune/{task_id}/retry")
|
|
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
payload = payload or {}
|
|
try:
|
|
task = store.task(task_id)
|
|
except KeyError:
|
|
raise fail(404, "fine tune task not found")
|
|
if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
|
|
raise fail(409, "only failed or stopped tasks can be retried without force=true")
|
|
retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
|
|
store.reset_task_for_retry(task_id, retry_payload)
|
|
try:
|
|
return ok(await _submit_fine_tune_task(store, retry_payload))
|
|
except RuntimeError as exc:
|
|
raise fail(409, str(exc))
|
|
except Exception as exc: # noqa: BLE001 - mark retry failed when remote submit fails
|
|
store.mark_task_failed(task_id, str(exc))
|
|
raise fail(502, f"retry fine tune task failed: {exc}")
|
|
|
|
|
|
@router.delete("/fine-tune/{task_id}")
|
|
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
|
|
get_platform_store().delete_task(task_id)
|
|
return ok({"deleted": task_id})
|
|
|
|
|
|
@router.get("/fine-tune/{task_id}/overview")
|
|
async def fine_tune_overview(task_id: str) -> dict[str, Any]:
|
|
task = get_platform_store().task(task_id)
|
|
return ok({"task": task, "progress": get_platform_store().progress(task_id)})
|
|
|
|
|
|
@router.get("/fine-tune/{task_id}/checkpoints")
|
|
async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]:
|
|
task = get_platform_store().task(task_id)
|
|
checkpoints = []
|
|
for step in [50, 100, 150]:
|
|
if task.get("progress", 0) >= min(100, step // 2):
|
|
checkpoints.append({"step": step, "path": f"/data/yg-ft/outputs/{task['name']}/checkpoint-{step}"})
|
|
return ok(checkpoints)
|
|
|
|
|
|
@router.get("/model-eval")
|
|
async def model_eval_list() -> dict[str, Any]:
|
|
return ok(get_platform_store().eval_tasks())
|
|
|
|
|
|
@router.get("/model-eval/{task_id}")
|
|
async def model_eval_detail(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().eval_task(task_id))
|
|
except KeyError:
|
|
raise fail(404, "eval task not found")
|
|
|
|
|
|
@router.post("/model-eval/start")
|
|
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
task = get_platform_store().create_eval_task(payload)
|
|
return ok({"task_id": task["id"], **task})
|
|
|
|
|
|
@router.delete("/model-eval/{task_id}")
|
|
async def model_eval_delete(task_id: str) -> dict[str, Any]:
|
|
get_platform_store().delete_eval_task(task_id)
|
|
return ok({"deleted": task_id})
|
|
|
|
|
|
@router.get("/dimension")
|
|
async def dimension_list() -> dict[str, Any]:
|
|
return ok(get_platform_store().dimensions())
|
|
|
|
|
|
@router.post("/dimension")
|
|
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok(get_platform_store().create_dimension(payload))
|
|
|
|
|
|
@router.get("/dimension/{dimension_id}")
|
|
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().dimension(dimension_id))
|
|
except KeyError:
|
|
raise fail(404, "dimension not found")
|
|
|
|
|
|
@router.put("/dimension/{dimension_id}")
|
|
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_dimension(dimension_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "dimension not found")
|
|
|
|
|
|
@router.delete("/dimension/{dimension_id}")
|
|
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
|
|
get_platform_store().delete_dimension(dimension_id)
|
|
return ok({"deleted": dimension_id})
|
|
|
|
|
|
@router.get("/model-compare")
|
|
async def model_compare_list() -> dict[str, Any]:
|
|
return ok(get_platform_store().compare_tasks())
|
|
|
|
|
|
@router.post("/model-compare")
|
|
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
task = get_platform_store().create_compare_task(payload)
|
|
return ok({"id": task["id"]})
|
|
|
|
|
|
@router.post("/model-compare/all/stop-all")
|
|
async def model_compare_stop_all() -> dict[str, Any]:
|
|
return ok({"stopped": True})
|
|
|
|
|
|
@router.post("/model-compare/stop-by-pid")
|
|
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"stopped": True, "pid": payload.get("pid")})
|
|
|
|
|
|
@router.get("/model-compare/{task_id}")
|
|
async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().compare_task(task_id))
|
|
except KeyError:
|
|
raise fail(404, "compare task not found")
|
|
|
|
|
|
@router.delete("/model-compare/{task_id}")
|
|
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
|
get_platform_store().delete_compare_task(task_id)
|
|
return ok({"deleted": task_id})
|
|
|
|
|
|
@router.get("/model-compare/{task_id}/load-status")
|
|
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
task = get_platform_store().compare_task(task_id)
|
|
except KeyError:
|
|
raise fail(404, "compare task not found")
|
|
load_status = task.get("load_status") or {"loaded_models": []}
|
|
if isinstance(load_status, str):
|
|
try:
|
|
load_status = json.loads(load_status)
|
|
except json.JSONDecodeError:
|
|
load_status = {"loaded_models": []}
|
|
return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})
|
|
|
|
|
|
@router.post("/model-compare/{task_id}/load-status")
|
|
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
|
|
except KeyError:
|
|
raise fail(404, "compare task not found")
|
|
|
|
|
|
@router.post("/model-compare/{task_id}/load")
|
|
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
task = get_platform_store().compare_task(task_id)
|
|
models = task.get("models") or []
|
|
if isinstance(models, str):
|
|
try:
|
|
models = json.loads(models)
|
|
except json.JSONDecodeError:
|
|
models = []
|
|
loaded_models = [
|
|
{
|
|
"model_id": item.get("model_id"),
|
|
"model_name": item.get("model_name"),
|
|
"status": "ready",
|
|
"pid": 45000 + index,
|
|
"port": item.get("port") or 18000 + index,
|
|
}
|
|
for index, item in enumerate(models)
|
|
if isinstance(item, dict)
|
|
]
|
|
return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
|
|
except KeyError:
|
|
raise fail(404, "compare task not found")
|
|
|
|
|
|
@router.post("/model-compare/{task_id}/unload")
|
|
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
|
|
except KeyError:
|
|
raise fail(404, "compare task not found")
|
|
|
|
|
|
@router.post("/model-compare/{task_id}/start-model")
|
|
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})
|
|
|
|
|
|
@router.post("/model-compare/chat-with-port")
|
|
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
question = ""
|
|
for message in payload.get("messages") or []:
|
|
if message.get("role") == "user":
|
|
question = str(message.get("content") or "")
|
|
content = f"当前后端已收到推理请求：{question[:120]}"
|
|
return ok({"response": content, "content": content})
|
|
|
|
|
|
@router.post("/model-compare/stream-chat")
|
|
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
question = payload.get("user_question") or payload.get("question") or ""
|
|
return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"})
|
|
|
|
|
|
@router.post("/model-chat/batch")
|
|
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"responses": [], "request": payload})
|
|
|
|
|
|
@router.post("/model-chat/local/chat")
|
|
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"response": "local chat adapter is not connected yet", "request": payload})
|
|
|
|
|
|
@router.post("/model-chat/local/preload")
|
|
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"loaded": True, "request": payload})
|
|
|
|
|
|
@router.post("/model-chat/trained/preload")
|
|
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"loaded": True, "request": payload})
|
|
|
|
|
|
@router.get("/compute/nodes")
|
|
async def compute_nodes() -> dict[str, Any]:
|
|
return ok(get_platform_store().compute_nodes())
|
|
|
|
|
|
@router.get("/compute/nodes/{node_id}")
|
|
async def compute_node_detail(node_id: str) -> dict[str, Any]:
|
|
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
return ok(node)
|
|
|
|
|
|
@router.post("/compute/nodes")
|
|
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().create_compute_node(payload))
|
|
except KeyError as exc:
|
|
raise fail(400, f"missing field: {exc}")
|
|
except ValueError as exc:
|
|
raise fail(400, str(exc))
|
|
|
|
|
|
@router.put("/compute/nodes/{node_id}")
|
|
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().update_compute_node(node_id, payload))
|
|
except KeyError:
|
|
raise fail(404, "compute node not found")
|
|
except ValueError as exc:
|
|
raise fail(400, str(exc))
|
|
|
|
|
|
@router.post("/compute/nodes/{node_id}/test-connection")
|
|
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
client = ComputeNodeClient(node["api_base_url"])
|
|
try:
|
|
result = await client.test_connection()
|
|
store.replace_node_gpus(node_id, result["gpus"])
|
|
updated = store.update_compute_node_health(node_id, result["health"], True)
|
|
return ok(
|
|
{
|
|
"node_id": node_id,
|
|
"success": True,
|
|
"latency_ms": result["latency_ms"],
|
|
"gpu_count": len(result["gpus"]),
|
|
"health": updated["health_detail"],
|
|
}
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - return the connection error for node maintenance
|
|
updated = store.update_compute_node_health(node_id, {}, False, str(exc))
|
|
return ok(
|
|
{
|
|
"node_id": node_id,
|
|
"success": False,
|
|
"latency_ms": 0,
|
|
"gpu_count": updated.get("gpu_count", 0),
|
|
"error": str(exc),
|
|
"health": updated["health_detail"],
|
|
}
|
|
)
|
|
|
|
|
|
@router.post("/compute/nodes/{node_id}/health-check")
|
|
async def health_check_compute_node(node_id: str) -> dict[str, Any]:
|
|
return await test_compute_node(node_id)
|
|
|
|
|
|
@router.post("/compute/nodes/{node_id}/enable")
|
|
async def enable_compute_node(node_id: str) -> dict[str, Any]:
|
|
return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))
|
|
|
|
|
|
@router.post("/compute/nodes/{node_id}/disable")
|
|
async def disable_compute_node(node_id: str) -> dict[str, Any]:
|
|
return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))
|
|
|
|
|
|
@router.post("/compute/nodes/{node_id}/drain")
|
|
async def drain_compute_node(node_id: str) -> dict[str, Any]:
|
|
return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))
|
|
|
|
|
|
@router.get("/compute/nodes/{node_id}/replicas")
|
|
async def compute_node_replicas(node_id: str) -> dict[str, Any]:
|
|
return ok(get_platform_store().replicas(node_id))
|
|
|
|
|
|
@router.get("/compute/nodes/{node_id}/engines")
|
|
async def compute_node_engines(node_id: str) -> dict[str, Any]:
|
|
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
health = node.get("health_detail") or {}
|
|
live_error = ""
|
|
try:
|
|
health = await ComputeNodeClient(node["api_base_url"]).health()
|
|
except Exception as exc: # noqa: BLE001 - stored health is enough for offline node detail
|
|
live_error = str(exc)
|
|
capabilities = health.get("capabilities") or node.get("capabilities") or []
|
|
return ok(
|
|
{
|
|
"node_id": node_id,
|
|
"items": [
|
|
{
|
|
"engine": "llama_factory",
|
|
"display_name": "LLaMA-Factory",
|
|
"status": "available" if "llama_factory" in capabilities else "unknown",
|
|
"version": health.get("llama_factory_version") or "",
|
|
"home": health.get("llama_factory_home") or "",
|
|
"home_exists": bool(health.get("llama_factory_home_exists")),
|
|
"capabilities": capabilities,
|
|
"execution_mode": health.get("execution_mode") or "",
|
|
"last_error": live_error,
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
|
|
@router.get("/compute/gpus")
|
|
async def compute_gpus() -> dict[str, Any]:
|
|
return ok(get_platform_store().gpus())
|
|
|
|
|
|
@router.get("/compute/queue")
|
|
async def compute_queue() -> dict[str, Any]:
|
|
return ok(get_platform_store().queue())
|
|
|
|
|
|
@router.get("/compute/jobs/{job_id}")
|
|
async def compute_job_detail(job_id: str) -> dict[str, Any]:
|
|
task = _task_for_compute_job(job_id)
|
|
if not task:
|
|
raise fail(404, "compute job not found")
|
|
node = _node_for_task(task)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
return ok(await ComputeNodeClient(node["api_base_url"]).get_job(job_id))
|
|
|
|
|
|
@router.post("/compute/jobs/{job_id}/stop")
|
|
async def compute_job_stop(job_id: str) -> dict[str, Any]:
|
|
task = _task_for_compute_job(job_id)
|
|
if not task:
|
|
raise fail(404, "compute job not found")
|
|
node = _node_for_task(task)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
|
|
get_platform_store().apply_compute_job(task["id"], job)
|
|
return ok(job)
|
|
|
|
|
|
@router.get("/compute/jobs/{job_id}/logs")
|
|
async def compute_job_logs(
|
|
job_id: str,
|
|
tail_lines: int | None = Query(default=200, ge=1, le=5000),
|
|
offset: int | None = Query(default=None, ge=0),
|
|
limit: int | None = Query(default=None, ge=1, le=5000),
|
|
) -> dict[str, Any]:
|
|
task = _task_for_compute_job(job_id)
|
|
if not task:
|
|
raise fail(404, "compute job not found")
|
|
node = _node_for_task(task)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))
|
|
|
|
|
|
@router.post("/compute/jobs/{job_id}/retry")
|
|
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
payload = payload or {}
|
|
task = _task_for_compute_job(job_id)
|
|
if not task:
|
|
raise fail(404, "compute job not found")
|
|
return await retry_fine_tune(task["id"], payload)
|
|
|
|
|
|
@router.post("/compute/jobs/{job_id}/priority")
|
|
async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
task = _task_for_compute_job(job_id)
|
|
if not task:
|
|
raise fail(404, "compute job not found")
|
|
priority = str(payload.get("priority") or "normal")
|
|
return ok(get_platform_store().update_task_priority(task["id"], priority))
|
|
|
|
|
|
@router.post("/internal/compute-sync/jobs/poll")
|
|
async def poll_compute_jobs() -> dict[str, Any]:
|
|
return ok(await poll_compute_jobs_once())
|
|
|
|
|
|
@router.post("/internal/compute-sync/resources")
|
|
async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
node_id = payload.get("target_node_id") or payload.get("target_compute_node_id")
|
|
if not node_id:
|
|
raise fail(400, "target_node_id is required")
|
|
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
|
if not node:
|
|
raise fail(404, "compute node not found")
|
|
sync_id = store.create_sync_job(node_id, payload)
|
|
replicas = []
|
|
failures = []
|
|
resources = payload.get("resources") or []
|
|
for resource in resources:
|
|
if not resource.get("source_path"):
|
|
continue
|
|
try:
|
|
result = await ComputeNodeClient(node["api_base_url"]).import_local_file(
|
|
{
|
|
"source_path": resource["source_path"],
|
|
"target_relative_path": resource.get("target_relative_path"),
|
|
"resource_type": resource.get("resource_type"),
|
|
"resource_id": resource.get("resource_id"),
|
|
}
|
|
)
|
|
replicas.append(
|
|
store.upsert_resource_replica(
|
|
node_id,
|
|
str(resource.get("resource_type") or "file"),
|
|
str(resource.get("resource_id") or result["id"]),
|
|
result["local_path"],
|
|
)
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - collect per-resource failures
|
|
failures.append({"resource_id": str(resource.get("resource_id")), "error": str(exc)})
|
|
store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)
|
|
return ok({"sync": store.sync_job(sync_id), "replicas": replicas, "failed": failures})
|
|
|
|
|
|
@router.get("/internal/compute-sync/resources/{sync_id}")
|
|
async def compute_sync_detail(sync_id: str) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().sync_job(sync_id))
|
|
except KeyError:
|
|
raise fail(404, "sync job not found")
|
|
|
|
|
|
@router.get("/training-log-files")
|
|
async def training_log_files() -> dict[str, Any]:
|
|
return ok(get_platform_store().training_log_files())
|
|
|
|
|
|
@router.get("/training-log-content")
|
|
async def training_log_content(file: str = Query(...)) -> dict[str, Any]:
|
|
try:
|
|
return ok(get_platform_store().training_log_content(file))
|
|
except KeyError:
|
|
raise fail(404, "training log not found")
|
|
|
|
|
|
@router.get("/log-files")
|
|
async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]:
|
|
return ok(get_platform_store().log_files(date))
|
|
|
|
|
|
@router.get("/log-content")
|
|
async def log_content(file: str = Query(...)) -> dict[str, Any]:
|
|
return ok(get_platform_store().log_content(file))
|
|
|
|
|
|
@router.post("/web-log")
|
|
async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
return ok({"received": True, **payload})
|
|
|