from __future__ import annotations import uuid import json from pathlib import Path from typing import Any from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile from fastapi.responses import PlainTextResponse from app.core.config import get_settings from app.db.platform_store import get_platform_store from app.modules.compute_gateway.client import ComputeNodeClient from app.modules.compute_gateway.sync import poll_compute_jobs_once router = APIRouter() def ok(data: Any = None, message: str = "ok") -> dict[str, Any]: return {"code": 0, "message": message, "data": data} def fail(status_code: int, message: str) -> HTTPException: return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None}) def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None: return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None) def _task_for_compute_job(job_id: str) -> dict[str, Any] | None: return next((task for task in get_platform_store().tasks() if task.get("compute_job_id") == job_id), None) def _node_for_compute_job_record(job_id: str) -> dict[str, Any] | None: store = get_platform_store() try: record = store.compute_job(job_id) except KeyError: return None return next((node for node in store.compute_nodes() if node["id"] == record.get("node_id")), None) async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]: task_id = str(payload.get("task_id") or payload.get("id") or "") if task_id and get_settings().compute_mode != "simulator": try: preflight = await _fine_tune_preflight(store, task_id, payload, validate=True, sync_resources=True) except Exception as exc: # noqa: BLE001 - task has not entered running state yet raise RuntimeError(f"preflight failed: {exc}") from exc if not preflight["valid"]: errors = "; ".join(preflight.get("errors") or ["preflight failed"]) raise RuntimeError(f"preflight failed: {errors}") payload = {**payload, "compute_node_id": preflight["node"]["id"]} task = store.start_task(payload) if get_settings().compute_mode == "simulator": return task node, job_payload = store.build_compute_job_payload(task["id"]) job = await ComputeNodeClient(node["api_base_url"]).create_job(job_payload) return store.apply_compute_job(task["id"], job) async def _fine_tune_preflight( store: Any, task_id: str, payload: dict[str, Any] | None = None, validate: bool = True, sync_resources: bool = False, ) -> dict[str, Any]: node, job_payload = store.prepare_compute_job_payload(task_id, payload or {}) sync_results: list[dict[str, Any]] = [] sync_errors: list[str] = [] if sync_resources and get_settings().compute_mode != "simulator": try: sync_results = await _sync_training_dataset_to_compute_node( store, node, str(job_payload.get("train_dataset_id") or ""), ) except Exception as exc: # noqa: BLE001 - return as preflight error for page visibility sync_errors.append(str(exc)) if get_settings().compute_mode == "simulator": preview = { "valid": True, "errors": [], "warnings": ["compute_mode=simulator skips remote compute validation"], "engine": job_payload.get("engine") or job_payload.get("training_engine") or "llama_factory", "command": [], "command_text": "", "work_dir": "", "env": {}, "path_checks": [], } else: client = ComputeNodeClient(node["api_base_url"]) preview = await (client.validate_job(job_payload) if validate else client.preview_job(job_payload)) errors = list(preview.get("errors") or []) errors.extend(sync_errors) warnings = list(preview.get("warnings") or []) if not node.get("enabled"): errors.append(f"compute node disabled: {node.get('code')}") if node.get("scheduler_status") not in {"online", "draining"}: errors.append(f"compute node not schedulable: {node.get('code')} status={node.get('scheduler_status')}") return { "valid": bool(preview.get("valid", not errors)) and not errors, "errors": errors, "warnings": warnings, "node": { "id": node.get("id"), "code": node.get("code"), "name": node.get("name"), "api_base_url": node.get("api_base_url"), "scheduler_status": node.get("scheduler_status"), "gpu_count": node.get("gpu_count"), }, "job_payload": job_payload, "preview": preview, "sync_results": sync_results, } @router.post("/login") async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: user = get_platform_store().login(payload.get("username", ""), payload.get("password", "")) if not user: raise fail(401, "invalid username or password") return ok({"token": f"platform-token-{user['id']}", "user": user}) @router.get("/me") async def me() -> dict[str, Any]: return ok(get_platform_store().users()[0]) @router.get("/dashboard/overview") async def dashboard_overview() -> dict[str, Any]: store = get_platform_store() tasks = store.tasks() return ok( { "models": len(store.models()), "datasets": len(store.datasets()), "fine_tune_tasks": len(tasks), "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]), "compute_nodes": len(store.compute_nodes()), "gpus": len(store.gpus()), } ) @router.get("/system-info") async def system_info() -> dict[str, Any]: return ok(get_platform_store().system_info()) @router.get("/users") async def users() -> dict[str, Any]: return ok(get_platform_store().users()) @router.post("/users") async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok(get_platform_store().create_user(payload)) @router.put("/users/{user_id}") async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_user(user_id, payload)) except KeyError: raise fail(404, "user not found") @router.delete("/users/{user_id}") async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]: try: get_platform_store().delete_user(user_id) return ok({"deleted": user_id, "current_username": current_username}) except KeyError: raise fail(404, "user not found") except ValueError as exc: raise fail(400, str(exc)) @router.get("/model-manage/local-models") async def local_models() -> dict[str, Any]: store = get_platform_store() models = [{"path": item.get("path") or "", "name": item["name"], "source": "registered"} for item in store.models()] seen = {item["path"] for item in models if item.get("path")} if get_settings().compute_mode != "simulator": for node in store.compute_nodes(): if not node.get("enabled"): continue try: result = await ComputeNodeClient(node["api_base_url"]).list_files(root="models", directories_only=True) except Exception: continue for item in result.get("items") or []: path = str(item.get("path") or "") if not path or path in seen: continue seen.add(path) models.append( { "path": path, "name": item.get("name") or path.rsplit("/", 1)[-1], "source": f"compute:{node.get('code')}", } ) return ok({"models": models}) @router.get("/model-manage/trained-models") async def trained_models() -> dict[str, Any]: return ok({"models": get_platform_store().trained_models()}) @router.delete("/model-manage/trained-models/{model_id}") async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]: get_platform_store().delete_trained_model(model_id) return ok({"deleted": model_id, "type": type}) @router.get("/model-manage/trained-models/{model_id}/artifacts") async def trained_model_artifacts(model_id: str) -> dict[str, Any]: return ok(get_platform_store().model_artifacts(model_id)) @router.get("/model-manage/trained-models/{model_id}/lineage") async def trained_model_lineage(model_id: str) -> dict[str, Any]: return ok(get_platform_store().model_lineage(model_id)) @router.get("/model-manage/export-jobs") async def model_export_jobs(trained_model_id: str | None = Query(default=None)) -> dict[str, Any]: return ok(get_platform_store().model_export_jobs(trained_model_id)) @router.get("/model-manage/name/{name}") async def model_by_name(name: str) -> dict[str, Any]: try: return ok(get_platform_store().model_by_name(name)) except KeyError: raise fail(404, "model not found") @router.get("/model-manage") async def model_list() -> dict[str, Any]: return ok(get_platform_store().models()) @router.post("/model-manage") async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().create_model(payload)) except KeyError as exc: raise fail(400, f"missing field: {exc}") except ValueError as exc: raise fail(400, str(exc)) except Exception as exc: # noqa: BLE001 - keep API errors visible to deployment smoke checks raise fail(500, f"create model failed: {exc}") @router.get("/model-manage/{model_id}") async def model_detail(model_id: str) -> dict[str, Any]: try: return ok(get_platform_store().model(model_id)) except KeyError: raise fail(404, "model not found") @router.put("/model-manage/{model_id}") async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_model(model_id, payload)) except KeyError: raise fail(404, "model not found") @router.put("/model-manage/{model_id}/purpose") async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")})) except KeyError: raise fail(404, "model not found") @router.delete("/model-manage/{model_id}") async def delete_model(model_id: str) -> dict[str, Any]: get_platform_store().delete_model(model_id) return ok({"deleted": model_id}) @router.post("/model-manage/merge") async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: store = get_platform_store() trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "") trained_model = next( ( item for item in store.trained_models() if trained_model_id and (item["id"] == trained_model_id or item["name"] == trained_model_id) ), None, ) base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path")) adapter_path = payload.get("adapter_path") or payload.get("adapter_name_or_path") or (trained_model and trained_model.get("merged_path")) if not base_model_path: raise fail(400, "base_model_path is required") if not adapter_path: raise fail(400, "adapter_path is required") node = store.schedule_node({**payload, "gpus": payload.get("gpus") or []}) health = node.get("health_detail") or {} output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs") output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged") output_dir = str(payload.get("output_dir") or f"{output_root.rstrip('/')}/{output_name}") job_payload = { **payload, "id": str(payload.get("job_id") or f"merge_{uuid.uuid4().hex[:12]}"), "name": output_name, "engine": "merge", "base_model": base_model_path, "model_name_or_path": base_model_path, "adapter_name_or_path": adapter_path, "output_dir": output_dir, "template": payload.get("template", "qwen"), "train_method": payload.get("train_method", "lora"), "gpus": payload.get("gpus") or [], "trained_model_id": trained_model["id"] if trained_model else trained_model_id, "model_name": trained_model["name"] if trained_model else payload.get("model_name"), } if get_settings().compute_mode == "simulator": job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir} else: client = ComputeNodeClient(node["api_base_url"]) preview = await client.validate_job(job_payload) if not preview.get("valid", False): raise fail(409, "; ".join(preview.get("errors") or ["merge preflight failed"])) job = await client.create_job(job_payload) return ok(store.record_model_merge_job(node, job_payload, job, trained_model["id"] if trained_model else trained_model_id)) @router.get("/dataset-manage/preview/{file_id}") async def dataset_preview(file_id: str) -> dict[str, Any]: try: row = get_platform_store().dataset_file(file_id) return ok({"content": row["content"]}) except KeyError: raise fail(404, "dataset file not found") @router.get("/dataset-manage/versions/{file_id}") async def dataset_versions(file_id: str) -> dict[str, Any]: try: return ok(get_platform_store().file_versions(file_id)) except KeyError: raise fail(404, "dataset file not found") @router.get("/dataset-manage/versions/{file_id}/{version_id}") async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]: try: row = get_platform_store().dataset_file(file_id) versions = get_platform_store().file_versions(file_id)["versions"] version = next((item for item in versions if item["id"] == version_id), None) if not version: raise KeyError(version_id) return ok({"version": version, "content": row["content"]}) except KeyError: raise fail(404, "dataset version not found") @router.post("/dataset-manage/versions/{file_id}") async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().create_file_version(file_id, payload)) except KeyError: raise fail(404, "dataset file not found") @router.put("/dataset-manage/versions/{file_id}/active") async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().activate_file_version(file_id, payload["version_id"])) except KeyError: raise fail(404, "dataset version not found") @router.delete("/dataset-manage/versions/{file_id}/{version_id}") async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]: try: return ok(get_platform_store().delete_file_version(file_id, version_id)) except KeyError: raise fail(404, "dataset version not found") except ValueError as exc: raise fail(400, str(exc)) async def _sync_dataset_file_to_compute_nodes( store: Any, dataset_id: str, file_id: str, filename: str, content: bytes, ) -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] if get_settings().compute_mode == "simulator": return results target_name = Path(filename or f"{file_id}.jsonl").name target_relative_path = f"datasets/{dataset_id}/{target_name}" for node in store.compute_nodes(): if not node.get("enabled"): continue try: result = await ComputeNodeClient(node["api_base_url"]).upload_file( target_name, content, target_relative_path, resource_type="dataset", resource_id=dataset_id, ) store.upsert_resource_replica( node["id"], "dataset", dataset_id, str(result.get("local_path") or ""), ) results.append( { "node_id": node["id"], "node_code": node.get("code"), "success": True, "local_path": result.get("local_path"), "byte_size": result.get("byte_size"), "checksum_sha256": result.get("checksum_sha256"), } ) except Exception as exc: # noqa: BLE001 - keep upload usable while exposing sync failures results.append( { "node_id": node["id"], "node_code": node.get("code"), "success": False, "error": str(exc), } ) return results async def _sync_training_dataset_to_compute_node( store: Any, node: dict[str, Any], dataset_id: str, ) -> list[dict[str, Any]]: if not dataset_id: raise RuntimeError("train_dataset_id is required") files = store.training_dataset_files(dataset_id) if not files: raise RuntimeError(f"dataset has no uploaded file: {dataset_id}") client = ComputeNodeClient(node["api_base_url"]) results: list[dict[str, Any]] = [] for item in files: target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name result = await client.upload_file( target_name, str(item.get("content") or "").encode("utf-8"), f"datasets/{dataset_id}/{target_name}", resource_type="dataset", resource_id=dataset_id, ) store.upsert_resource_replica( node["id"], "dataset", dataset_id, str(result.get("local_path") or ""), ) results.append( { "node_id": node["id"], "node_code": node.get("code"), "file_id": item.get("id"), "name": target_name, "local_path": result.get("local_path"), "byte_size": result.get("byte_size"), "checksum_sha256": result.get("checksum_sha256"), } ) return results @router.post("/dataset-manage/upload/{dataset_id}") async def upload_dataset_files( dataset_id: str, files: list[UploadFile] = File(default=[]), sync_to_compute: bool = Query(default=True), ) -> dict[str, Any]: created: list[dict[str, Any]] = [] compute_sync: list[dict[str, Any]] = [] store = get_platform_store() try: store.dataset(dataset_id) except KeyError: raise fail(404, "dataset not found") with store.connect() as conn: for file in files: raw = await file.read() content = raw.decode("utf-8", errors="replace") created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content) created.append(created_file) if sync_to_compute: compute_sync.extend( await _sync_dataset_file_to_compute_nodes( store, dataset_id, created_file["id"], created_file["name"], raw, ) ) return ok({"files": created, "compute_sync": compute_sync}) @router.get("/dataset-manage/download/{dataset_id}") async def download_dataset(dataset_id: str) -> PlainTextResponse: dataset = get_platform_store().dataset(dataset_id) content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])]) return PlainTextResponse(content, media_type="text/plain") @router.get("/dataset-manage/download/{dataset_id}/{file_id}") async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse: row = get_platform_store().dataset_file(file_id) return PlainTextResponse(row["content"], media_type="text/plain") @router.get("/dataset-manage") async def dataset_list() -> dict[str, Any]: return ok(get_platform_store().datasets()) @router.post("/dataset-manage") async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: dataset = get_platform_store().create_dataset(payload) return ok({"id": dataset["id"]}) @router.get("/dataset-manage/{dataset_id}") async def dataset_detail(dataset_id: str) -> dict[str, Any]: try: return ok(get_platform_store().dataset(dataset_id)) except KeyError: raise fail(404, "dataset not found") @router.put("/dataset-manage/{dataset_id}") async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_dataset(dataset_id, payload)) except KeyError: raise fail(404, "dataset not found") @router.delete("/dataset-manage/{dataset_id}") async def delete_dataset(dataset_id: str) -> dict[str, Any]: get_platform_store().delete_dataset(dataset_id) return ok({"deleted": dataset_id}) @router.get("/fine-tune/check-name") async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]: exists = any(task["name"] == name for task in get_platform_store().tasks()) return ok({"exists": exists}) @router.get("/fine-tune/progress/{task_id}") async def fine_tune_progress(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().progress(task_id)) except KeyError: raise fail(404, "fine tune task not found") @router.post("/fine-tune/tensorboard/start") async def tensorboard_start() -> dict[str, Any]: return ok({"status": "running", "url": "http://localhost:6006"}) @router.get("/fine-tune") async def fine_tune_list() -> dict[str, Any]: return ok(get_platform_store().tasks()) @router.post("/fine-tune") async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: task = get_platform_store().create_task(payload) return ok({"id": task["id"]}) except ValueError as exc: raise fail(400, str(exc)) @router.post("/fine-tune/start") async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: store = get_platform_store() try: return ok(await _submit_fine_tune_task(store, payload)) except KeyError: raise fail(404, "fine tune task not found") except RuntimeError as exc: task_id = str(payload.get("task_id") or payload.get("id") or "") if task_id: store.mark_task_failed(task_id, str(exc)) raise fail(409, str(exc)) except Exception as exc: # noqa: BLE001 - mark task failed when remote submit fails task_id = str(payload.get("task_id") or payload.get("id") or "") if task_id: store.mark_task_failed(task_id, str(exc)) raise fail(502, f"submit compute job failed: {exc}") @router.post("/fine-tune/{task_id}/preflight") async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: try: return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True)) except KeyError: raise fail(404, "fine tune task not found") except RuntimeError as exc: raise fail(409, str(exc)) except Exception as exc: # noqa: BLE001 - expose compute validation errors to training create page raise fail(502, f"compute preflight failed: {exc}") @router.post("/fine-tune/{task_id}/command-preview") async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: try: return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False)) except KeyError: raise fail(404, "fine tune task not found") except RuntimeError as exc: raise fail(409, str(exc)) except Exception as exc: # noqa: BLE001 raise fail(502, f"compute command preview failed: {exc}") @router.get("/fine-tune/{task_id}") async def fine_tune_detail(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().task(task_id)) except KeyError: raise fail(404, "fine tune task not found") @router.get("/fine-tune/{task_id}/logs") async def fine_tune_logs( task_id: str, tail_lines: int | None = Query(default=500, ge=1, le=5000), offset: int | None = Query(default=None, ge=0), limit: int | None = Query(default=None, ge=1, le=5000), ) -> dict[str, Any]: store = get_platform_store() try: task = store.task(task_id) except KeyError: raise fail(404, "fine tune task not found") if task.get("compute_job_id"): node = _node_for_task(task) if node: try: logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], tail_lines, offset, limit) try: store.record_training_log_metrics(task_id, str(logs.get("content") or "")) except Exception: pass if task.get("status") in {"queued", "running", "failed", "stopped", "completed"}: try: job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"]) store.apply_compute_job(task_id, job) except Exception: pass return ok({"source": "compute", **logs}) except Exception as exc: # noqa: BLE001 - keep failure reason visible even when log fetch fails content = task.get("failure_reason") or f"fetch compute log failed: {exc}" return ok({"job_id": task.get("compute_job_id"), "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"}) content = task.get("failure_reason") or "" return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"}) @router.put("/fine-tune/{task_id}") async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_task(task_id, payload)) except KeyError: raise fail(404, "fine tune task not found") @router.post("/fine-tune/stop/{task_id}") async def stop_fine_tune(task_id: str) -> dict[str, Any]: store = get_platform_store() try: task = store.task(task_id) node = _node_for_task(task) if task.get("compute_job_id") and node and get_settings().compute_mode != "simulator": job = await ComputeNodeClient(node["api_base_url"]).stop_job(task["compute_job_id"]) return ok(store.apply_compute_job(task_id, job)) return ok(store.stop_task(task_id)) except KeyError: raise fail(404, "fine tune task not found") @router.post("/fine-tune/{task_id}/stop") async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]: return await stop_fine_tune(task_id) @router.post("/fine-tune/{task_id}/retry") async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: store = get_platform_store() payload = payload or {} try: task = store.task(task_id) except KeyError: raise fail(404, "fine tune task not found") if task["status"] not in {"failed", "stopped"} and not payload.get("force"): raise fail(409, "only failed or stopped tasks can be retried without force=true") retry_payload = {**task, **payload, "task_id": task_id, "id": task_id} store.reset_task_for_retry(task_id, retry_payload) try: return ok(await _submit_fine_tune_task(store, retry_payload)) except RuntimeError as exc: raise fail(409, str(exc)) except Exception as exc: # noqa: BLE001 - mark retry failed when remote submit fails store.mark_task_failed(task_id, str(exc)) raise fail(502, f"retry fine tune task failed: {exc}") @router.delete("/fine-tune/{task_id}") async def delete_fine_tune(task_id: str) -> dict[str, Any]: get_platform_store().delete_task(task_id) return ok({"deleted": task_id}) @router.get("/fine-tune/{task_id}/overview") async def fine_tune_overview(task_id: str) -> dict[str, Any]: store = get_platform_store() task = store.task(task_id) return ok( { "task": task, "progress": store.progress(task_id), "metrics": store.task_metrics(task_id), "checkpoints": store.task_checkpoints(task_id), } ) @router.get("/fine-tune/{task_id}/checkpoints") async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]: store = get_platform_store() try: store.task(task_id) except KeyError: raise fail(404, "fine tune task not found") return ok(store.task_checkpoints(task_id)) @router.get("/fine-tune/{task_id}/metrics") async def fine_tune_metrics(task_id: str) -> dict[str, Any]: store = get_platform_store() try: store.task(task_id) except KeyError: raise fail(404, "fine tune task not found") return ok(store.task_metrics(task_id)) @router.get("/model-eval") async def model_eval_list() -> dict[str, Any]: return ok(get_platform_store().eval_tasks()) @router.get("/model-eval/{task_id}") async def model_eval_detail(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().eval_task(task_id)) except KeyError: raise fail(404, "eval task not found") @router.post("/model-eval/start") async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: task = get_platform_store().create_eval_task(payload) return ok({"task_id": task["id"], **task}) @router.delete("/model-eval/{task_id}") async def model_eval_delete(task_id: str) -> dict[str, Any]: get_platform_store().delete_eval_task(task_id) return ok({"deleted": task_id}) @router.get("/dimension") async def dimension_list() -> dict[str, Any]: return ok(get_platform_store().dimensions()) @router.post("/dimension") async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok(get_platform_store().create_dimension(payload)) @router.get("/dimension/{dimension_id}") async def dimension_detail(dimension_id: str) -> dict[str, Any]: try: return ok(get_platform_store().dimension(dimension_id)) except KeyError: raise fail(404, "dimension not found") @router.put("/dimension/{dimension_id}") async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_dimension(dimension_id, payload)) except KeyError: raise fail(404, "dimension not found") @router.delete("/dimension/{dimension_id}") async def dimension_delete(dimension_id: str) -> dict[str, Any]: get_platform_store().delete_dimension(dimension_id) return ok({"deleted": dimension_id}) @router.get("/model-compare") async def model_compare_list() -> dict[str, Any]: return ok(get_platform_store().compare_tasks()) @router.post("/model-compare") async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: task = get_platform_store().create_compare_task(payload) return ok({"id": task["id"]}) @router.post("/model-compare/all/stop-all") async def model_compare_stop_all() -> dict[str, Any]: return ok({"stopped": True}) @router.post("/model-compare/stop-by-pid") async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"stopped": True, "pid": payload.get("pid")}) @router.get("/model-compare/{task_id}") async def model_compare_detail(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().compare_task(task_id)) except KeyError: raise fail(404, "compare task not found") @router.delete("/model-compare/{task_id}") async def model_compare_delete(task_id: str) -> dict[str, Any]: get_platform_store().delete_compare_task(task_id) return ok({"deleted": task_id}) @router.get("/model-compare/{task_id}/load-status") async def model_compare_load_status(task_id: str) -> dict[str, Any]: try: task = get_platform_store().compare_task(task_id) except KeyError: raise fail(404, "compare task not found") load_status = task.get("load_status") or {"loaded_models": []} if isinstance(load_status, str): try: load_status = json.loads(load_status) except json.JSONDecodeError: load_status = {"loaded_models": []} return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status}) @router.post("/model-compare/{task_id}/load-status") async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}})) except KeyError: raise fail(404, "compare task not found") @router.post("/model-compare/{task_id}/load") async def model_compare_load(task_id: str) -> dict[str, Any]: try: task = get_platform_store().compare_task(task_id) models = task.get("models") or [] if isinstance(models, str): try: models = json.loads(models) except json.JSONDecodeError: models = [] loaded_models = [ { "model_id": item.get("model_id"), "model_name": item.get("model_name"), "status": "ready", "pid": 45000 + index, "port": item.get("port") or 18000 + index, } for index, item in enumerate(models) if isinstance(item, dict) ] return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}})) except KeyError: raise fail(404, "compare task not found") @router.post("/model-compare/{task_id}/unload") async def model_compare_unload(task_id: str) -> dict[str, Any]: try: return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})) except KeyError: raise fail(404, "compare task not found") @router.post("/model-compare/{task_id}/start-model") async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id}) @router.post("/model-compare/chat-with-port") async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: question = "" for message in payload.get("messages") or []: if message.get("role") == "user": question = str(message.get("content") or "") content = f"当前后端已收到推理请求：{question[:120]}" return ok({"response": content, "content": content}) @router.post("/model-compare/stream-chat") async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: question = payload.get("user_question") or payload.get("question") or "" return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"}) @router.post("/model-chat/batch") async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"responses": [], "request": payload}) @router.post("/model-chat/local/chat") async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"response": "local chat adapter is not connected yet", "request": payload}) @router.post("/model-chat/local/preload") async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"loaded": True, "request": payload}) @router.post("/model-chat/trained/preload") async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"loaded": True, "request": payload}) @router.get("/compute/nodes") async def compute_nodes() -> dict[str, Any]: return ok(get_platform_store().compute_nodes()) @router.get("/compute/nodes/{node_id}") async def compute_node_detail(node_id: str) -> dict[str, Any]: node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") return ok(node) @router.post("/compute/nodes") async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().create_compute_node(payload)) except KeyError as exc: raise fail(400, f"missing field: {exc}") except ValueError as exc: raise fail(400, str(exc)) @router.put("/compute/nodes/{node_id}") async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: try: return ok(get_platform_store().update_compute_node(node_id, payload)) except KeyError: raise fail(404, "compute node not found") except ValueError as exc: raise fail(400, str(exc)) @router.post("/compute/nodes/{node_id}/test-connection") async def test_compute_node(node_id: str) -> dict[str, Any]: store = get_platform_store() node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") client = ComputeNodeClient(node["api_base_url"]) try: result = await client.test_connection() store.replace_node_gpus(node_id, result["gpus"]) updated = store.update_compute_node_health(node_id, result["health"], True) return ok( { "node_id": node_id, "success": True, "latency_ms": result["latency_ms"], "gpu_count": len(result["gpus"]), "health": updated["health_detail"], } ) except Exception as exc: # noqa: BLE001 - return the connection error for node maintenance updated = store.update_compute_node_health(node_id, {}, False, str(exc)) return ok( { "node_id": node_id, "success": False, "latency_ms": 0, "gpu_count": updated.get("gpu_count", 0), "error": str(exc), "health": updated["health_detail"], } ) @router.post("/compute/nodes/{node_id}/health-check") async def health_check_compute_node(node_id: str) -> dict[str, Any]: return await test_compute_node(node_id) @router.post("/compute/nodes/{node_id}/enable") async def enable_compute_node(node_id: str) -> dict[str, Any]: return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"})) @router.post("/compute/nodes/{node_id}/disable") async def disable_compute_node(node_id: str) -> dict[str, Any]: return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"})) @router.post("/compute/nodes/{node_id}/drain") async def drain_compute_node(node_id: str) -> dict[str, Any]: return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"})) @router.get("/compute/nodes/{node_id}/replicas") async def compute_node_replicas(node_id: str) -> dict[str, Any]: return ok(get_platform_store().replicas(node_id)) @router.get("/compute/nodes/{node_id}/replicas/drift") async def compute_node_replica_drift(node_id: str) -> dict[str, Any]: store = get_platform_store() node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") replicas = store.replicas(node_id) if not replicas: return ok({"node_id": node_id, "items": [], "drifted": 0}) paths = [ { "name": replica["id"], "path": replica["local_path"], "type": "any", "required": True, } for replica in replicas ] try: result = await ComputeNodeClient(node["api_base_url"]).check_paths(paths) except Exception as exc: # noqa: BLE001 raise fail(502, f"replica drift check failed: {exc}") check_map = {str(item.get("name")): item for item in result.get("items") or []} items = [] for replica in replicas: check = check_map.get(replica["id"], {}) updated = store.update_resource_replica_check( replica["id"], bool(check.get("ok")), int(check.get("byte_size") or replica.get("byte_size") or 0), "" if check.get("ok") else f"path not available: {replica['local_path']}", ) items.append({**updated, "check": check}) return ok({"node_id": node_id, "items": items, "drifted": len([item for item in items if item.get("sync_status") == "drifted"])}) @router.post("/compute/nodes/{node_id}/replicas/repair") async def compute_node_replica_repair(node_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: store = get_platform_store() node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") payload = payload or {} replica_ids = payload.get("replica_ids") or [ item["id"] for item in store.replicas(node_id) if item.get("sync_status") in {"drifted", "failed", "repair_pending"} ] updated = store.mark_resource_replica_repair_pending([str(item) for item in replica_ids]) sync_id = store.create_sync_job( node_id, { "resources": [ { "resource_type": item.get("resource_type"), "resource_id": item.get("resource_id"), "replica_id": item.get("id"), "target_path": item.get("local_path"), } for item in updated ] }, ) client = ComputeNodeClient(node["api_base_url"]) repaired = [] failures = [] for replica in updated: replica_id = str(replica["id"]) resource_type = str(replica.get("resource_type") or "") resource_id = str(replica.get("resource_id") or "") try: if resource_type == "dataset": files = store.training_dataset_files(resource_id) if not files: raise RuntimeError(f"dataset has no uploaded file: {resource_id}") total_size = 0 checksum = "" local_path = str(replica.get("local_path") or "") for item in files: filename = Path(str(item.get("name") or f"{item['id']}.jsonl")).name result = await client.upload_file( filename, str(item.get("content") or "").encode("utf-8"), f"datasets/{resource_id}/{filename}", resource_type="dataset", resource_id=resource_id, ) total_size += int(result.get("byte_size") or 0) checksum = str(result.get("checksum_sha256") or checksum) local_path = str(result.get("local_path") or local_path) repaired.append(store.update_resource_replica_sync_result(replica_id, True, local_path, total_size, checksum)) continue source_path = "" target_relative_path = "" if resource_type == "model": model = store.model(resource_id) source_path = str(model.get("path") or "") target_relative_path = f"models/{Path(source_path).name}" if source_path else "" elif resource_type in {"trained_model", "model_artifact"}: if resource_type == "trained_model": artifacts = store.model_artifacts(resource_id) artifact = next((item for item in artifacts if item.get("path")), None) else: artifact = store.model_artifact(resource_id) source_path = str((artifact or {}).get("path") or replica.get("local_path") or "") target_relative_path = f"outputs/{Path(source_path).name}" if source_path else "" else: source_path = str(payload.get("source_path") or replica.get("source_path") or replica.get("local_path") or "") target_relative_path = str(payload.get("target_relative_path") or "") if not source_path: raise RuntimeError(f"authoritative source path not found for {resource_type}:{resource_id}") result = await client.import_local_file( { "source_path": source_path, "target_relative_path": target_relative_path, "resource_type": resource_type, "resource_id": resource_id, } ) repaired.append( store.update_resource_replica_sync_result( replica_id, True, str(result.get("local_path") or replica.get("local_path") or ""), int(result.get("byte_size") or 0), str(result.get("checksum_sha256") or ""), ) ) except Exception as exc: # noqa: BLE001 - collect all replica repair failures error = str(exc) failures.append({"replica_id": replica_id, "resource_type": resource_type, "resource_id": resource_id, "error": error}) repaired.append(store.update_resource_replica_sync_result(replica_id, False, None, int(replica.get("byte_size") or 0), "", error)) store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True) return ok({"sync": store.sync_job(sync_id), "replicas": repaired, "failed": failures}) @router.get("/compute/nodes/{node_id}/engines") async def compute_node_engines(node_id: str) -> dict[str, Any]: node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") health = node.get("health_detail") or {} live_error = "" try: health = await ComputeNodeClient(node["api_base_url"]).health() except Exception as exc: # noqa: BLE001 - stored health is enough for offline node detail live_error = str(exc) capabilities = health.get("capabilities") or node.get("capabilities") or [] return ok( { "node_id": node_id, "items": [ { "engine": "llama_factory", "display_name": "LLaMA-Factory", "status": "available" if "llama_factory" in capabilities else "unknown", "version": health.get("llama_factory_version") or "", "home": health.get("llama_factory_home") or "", "home_exists": bool(health.get("llama_factory_home_exists")), "capabilities": capabilities, "execution_mode": health.get("execution_mode") or "", "last_error": live_error, } ], } ) @router.get("/compute/gpus") async def compute_gpus() -> dict[str, Any]: return ok(get_platform_store().gpus()) @router.get("/compute/queue") async def compute_queue() -> dict[str, Any]: return ok(get_platform_store().queue()) @router.get("/compute/jobs/{job_id}") async def compute_job_detail(job_id: str) -> dict[str, Any]: task = _task_for_compute_job(job_id) if not task: node = _node_for_compute_job_record(job_id) if not node: raise fail(404, "compute job not found") job = await ComputeNodeClient(node["api_base_url"]).get_job(job_id) return ok(get_platform_store().sync_model_merge_job(job_id, job)) node = _node_for_task(task) if not node: raise fail(404, "compute node not found") return ok(await ComputeNodeClient(node["api_base_url"]).get_job(job_id)) @router.post("/compute/jobs/{job_id}/stop") async def compute_job_stop(job_id: str) -> dict[str, Any]: task = _task_for_compute_job(job_id) if not task: node = _node_for_compute_job_record(job_id) if not node: raise fail(404, "compute job not found") job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id) return ok(get_platform_store().sync_model_merge_job(job_id, job)) node = _node_for_task(task) if not node: raise fail(404, "compute node not found") job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id) get_platform_store().apply_compute_job(task["id"], job) return ok(job) @router.get("/compute/jobs/{job_id}/logs") async def compute_job_logs( job_id: str, tail_lines: int | None = Query(default=200, ge=1, le=5000), offset: int | None = Query(default=None, ge=0), limit: int | None = Query(default=None, ge=1, le=5000), ) -> dict[str, Any]: task = _task_for_compute_job(job_id) if not task: node = _node_for_compute_job_record(job_id) if not node: raise fail(404, "compute job not found") return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit)) node = _node_for_task(task) if not node: raise fail(404, "compute node not found") return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit)) @router.post("/compute/jobs/{job_id}/retry") async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: store = get_platform_store() payload = payload or {} task = _task_for_compute_job(job_id) if not task: raise fail(404, "compute job not found") return await retry_fine_tune(task["id"], payload) @router.post("/compute/jobs/{job_id}/priority") async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: task = _task_for_compute_job(job_id) if not task: raise fail(404, "compute job not found") priority = str(payload.get("priority") or "normal") return ok(get_platform_store().update_task_priority(task["id"], priority)) @router.post("/internal/compute-sync/jobs/poll") async def poll_compute_jobs() -> dict[str, Any]: return ok(await poll_compute_jobs_once()) @router.post("/internal/compute-sync/resources") async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: store = get_platform_store() node_id = payload.get("target_node_id") or payload.get("target_compute_node_id") if not node_id: raise fail(400, "target_node_id is required") node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") sync_id = store.create_sync_job(node_id, payload) replicas = [] failures = [] resources = payload.get("resources") or [] for resource in resources: if not resource.get("source_path"): continue try: result = await ComputeNodeClient(node["api_base_url"]).import_local_file( { "source_path": resource["source_path"], "target_relative_path": resource.get("target_relative_path"), "resource_type": resource.get("resource_type"), "resource_id": resource.get("resource_id"), } ) replicas.append( store.upsert_resource_replica( node_id, str(resource.get("resource_type") or "file"), str(resource.get("resource_id") or result["id"]), result["local_path"], ) ) except Exception as exc: # noqa: BLE001 - collect per-resource failures failures.append({"resource_id": str(resource.get("resource_id")), "error": str(exc)}) store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True) return ok({"sync": store.sync_job(sync_id), "replicas": replicas, "failed": failures}) @router.get("/internal/compute-sync/resources/{sync_id}") async def compute_sync_detail(sync_id: str) -> dict[str, Any]: try: return ok(get_platform_store().sync_job(sync_id)) except KeyError: raise fail(404, "sync job not found") @router.get("/training-log-files") async def training_log_files() -> dict[str, Any]: return ok(get_platform_store().training_log_files()) @router.get("/training-log-content") async def training_log_content(file: str = Query(...)) -> dict[str, Any]: try: return ok(get_platform_store().training_log_content(file)) except KeyError: raise fail(404, "training log not found") @router.get("/log-files") async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]: return ok(get_platform_store().log_files(date)) @router.get("/log-content") async def log_content(file: str = Query(...)) -> dict[str, Any]: return ok(get_platform_store().log_content(file)) @router.post("/web-log") async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: return ok({"received": True, **payload})