YG_FT/backend/app/api/v1/endpoints/platform.py

from __future__ import annotations

import json
import asyncio
import hashlib
import uuid
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any

from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import PlainTextResponse, StreamingResponse

import httpx

from app.core.auth import filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage

router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
_DASHBOARD_CACHE_TTL = 5.0
_DASHBOARD_CACHE: dict[str, Any] = {}


def _cached_dashboard(key: str) -> dict[str, Any] | None:
    item = _DASHBOARD_CACHE.get(key)
    if not item or time.monotonic() - item["created_at"] >= _DASHBOARD_CACHE_TTL:
        return None
    return item["value"]


def _store_dashboard_cache(key: str, value: dict[str, Any]) -> dict[str, Any]:
    _DASHBOARD_CACHE[key] = {"created_at": time.monotonic(), "value": value}
    return value


def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
    return {"code": 0, "message": message, "data": data}


def _select_first_online_node(store: Any) -> dict[str, Any] | None:
    """Select the first online compute node for inference."""
    nodes = store.compute_nodes()
    for node in nodes:
        if node.get("enabled") and node.get("scheduler_status") == "online":
            return node
    return None


async def _wait_for_object_storage() -> None:
    """Wait for MinIO before starting a resource task."""
    settings = get_settings()
    deadline = datetime.now(timezone.utc).timestamp() + max(0, settings.storage_wait_seconds)
    last_error = "MinIO unavailable"
    while True:
        try:
            get_object_storage().ensure_bucket()
            return
        except Exception as exc:  # noqa: BLE001 - retry until the configured deadline
            last_error = str(exc)
        if datetime.now(timezone.utc).timestamp() >= deadline:
            raise RuntimeError(f"MinIO unavailable after {settings.storage_wait_seconds}s: {last_error}")
        await asyncio.sleep(max(1, settings.storage_check_interval_seconds))


def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
    """Select the compute node for an eval job.

    被评测模型是节点相关的（训练/合并产物只存在于对应算力节点），因此优先使用
    页面选择的节点或模型所在节点；若该节点不可用则明确失败，绝不派发到其它
    可能没有模型路径的节点（多算力节点场景下这是评测失败的主因）。
    """
    if preferred_node_id:
        node = next((n for n in store.compute_nodes() if n.get("id") == preferred_node_id), None)
        if node:
            if node.get("enabled") and node.get("scheduler_status") == "online":
                return node
            return None
    return _select_first_online_node(store)


def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) -> list[dict[str, Any]]:
    nodes = [node for node in store.compute_nodes() if node.get("enabled") and node.get("scheduler_status") == "online"]
    if not preferred_node_id:
        return nodes
    preferred = [node for node in nodes if node.get("id") == preferred_node_id]
    others = [node for node in nodes if node.get("id") != preferred_node_id]
    return preferred + others


async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: str, node: dict[str, Any]) -> str | None:
    """Prepare MinIO resource files on a node and return the local directory."""
    if not get_settings().minio_enabled or not resource_id:
        return None
    objects = store.storage_objects_for_resource(resource_type, resource_id)
    if not objects:
        return None
    client = ComputeNodeClient(node["api_base_url"], timeout=900)
    root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
    for obj in objects:
        await client.prepare_cache({
            "resource_id": resource_id,
            "version_id": obj["version_id"],
            "download_url": get_object_storage().presigned_get(obj["object_key"]),
            "checksum_sha256": obj.get("checksum_sha256") or "",
            "byte_size": obj.get("byte_size") or 0,
            "relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
        })
    return f"/data/yg-ft/{root_name}/{resource_id}"


def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
    """Convert frontend inference payload to compute API messages format.

    Accepts both:
      - OpenAI-style: {messages: [{role, content}, ...], temperature, ...}
      - Frontend-style: {user_question, system_prompt, temperature, ...}
    """
    if payload.get("messages"):
        messages = payload["messages"]
        # messages already in OpenAI format; pass through with optional system prompt
        if payload.get("system_prompt") and not any(m.get("role") == "system" for m in messages):
            messages = [{"role": "system", "content": payload["system_prompt"]}] + list(messages)
    else:
        messages = []
        if payload.get("system_prompt"):
            messages.append({"role": "system", "content": payload["system_prompt"]})
        question = payload.get("user_question") or payload.get("question") or ""
        if question:
            messages.append({"role": "user", "content": question})
    return {
        "messages": messages,
        "temperature": float(payload.get("temperature", 0.7)),
        "top_p": float(payload.get("top_p", 0.95)),
        "max_new_tokens": int(payload.get("max_tokens", 2048)),
        "do_sample": bool(payload.get("do_sample", True)),
    }


def _node_for_inference_payload(store: Any, payload: dict[str, Any]) -> dict[str, Any] | None:
    node_id = payload.get("node_id") or payload.get("compute_node_id")
    task_id = payload.get("task_id") or payload.get("compare_task_id")
    if task_id and not node_id:
        try:
            task = store.compare_task(str(task_id))
            load_status = task.get("load_status") or {}
            if isinstance(load_status, str):
                load_status = json.loads(load_status)
            loaded_models = load_status.get("loaded_models") or []
            ready_model = next((item for item in loaded_models if item.get("status") in {"ready", "running"} and item.get("node_id")), None)
            if ready_model:
                node_id = ready_model.get("node_id")
        except Exception:
            node_id = None
    if node_id:
        return next((node for node in store.compute_nodes() if node.get("id") == node_id), None)
    return _select_first_online_node(store)


async def _stream_chat_proxy(payload: dict[str, Any]) -> StreamingResponse:
    """Common SSE streaming proxy: convert payload → forward to compute node → stream back."""
    store = get_platform_store()
    # 任务仍在加载中时，直接返回明确的加载中提示，避免转发到尚未就绪的节点
    task_id = payload.get("task_id") or payload.get("compare_task_id")
    if task_id:
        try:
            task = store.compare_task(str(task_id))
            load_status = task.get("load_status") or {}
            if isinstance(load_status, str):
                load_status = json.loads(load_status)
            items = load_status.get("loaded_models") or []
            if items and not any(item.get("status") in {"ready", "running"} for item in items):
                if any(item.get("status") == "starting" for item in items):
                    return StreamingResponse(
                        iter(['data: {"error": "模型加载中，请稍候再试"}\n\n']),
                        media_type="text/event-stream",
                    )
        except Exception:  # noqa: BLE001 - fall through to normal routing on lookup errors
            pass
    node = _node_for_inference_payload(store, payload)
    if not node:
        return StreamingResponse(
            iter(['data: {"error": "no online compute node available for inference"}\n\n']),
            media_type="text/event-stream",
        )
    client = ComputeNodeClient(node["api_base_url"])
    compute_payload = _build_messages_payload(payload)

    async def stream_proxy():
        async with httpx.AsyncClient(timeout=300) as http:
            url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream"
            try:
                async with http.stream("POST", url, json=compute_payload, headers=client.headers()) as resp:
                    if resp.status_code >= 400:
                        yield f'data: {{"error": "compute node returned {resp.status_code}"}}\n\n'.encode()
                        return
                    async for chunk in resp.aiter_bytes():
                        yield chunk
            except Exception as exc:
                yield f'data: {{"error": "stream proxy failed: {exc}"}}\n\n'.encode()

    return StreamingResponse(stream_proxy(), media_type="text/event-stream")


def fail(status_code: int, message: str) -> HTTPException:
    return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})


def _require_approval_or_admin(
    resource_type: str,
    resource_id: str,
    current_user: dict[str, Any],
    action_desc: str = "",
) -> dict[str, Any] | None:
    """
    高风险操作审批旁路：
    - admin 用户直接放行（返回 None）
    - 普通用户创建审批实例，返回审批待定响应（code=202，非 None）
      code=202 使前端响应拦截器走业务错误分支，弹提示并 reject，
      避免前端误认为删除成功。
    """
    if is_admin(current_user):
        return None
    store = get_platform_store()
    instance = store.create_approval_instance({
        "resource_type": resource_type,
        "resource_id": resource_id,
        "applicant_id": current_user.get("id"),
        "template_id": None,
    })
    return {
        "code": 202,
        "message": f"操作已提交审批，等待管理员批准：{action_desc}",
        "data": {"approval_required": True, "approval_id": instance["id"]},
    }


def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
    return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)


def _task_for_compute_job(job_id: str) -> dict[str, Any] | None:
    return next((task for task in get_platform_store().tasks() if task.get("compute_job_id") == job_id), None)


def _node_for_compute_job_record(job_id: str) -> dict[str, Any] | None:
    store = get_platform_store()
    try:
        record = store.compute_job(job_id)
    except KeyError:
        return None
    return next((node for node in store.compute_nodes() if node["id"] == record.get("node_id")), None)


def _training_diagnostics(errors: list[str], warnings: list[str] | None = None, log_text: str = "") -> list[dict[str, str]]:
    source_items = [*errors, *(warnings or [])]
    if log_text:
        source_items.append(log_text)
    text = "\n".join(source_items).lower()
    diagnostics: list[dict[str, str]] = []
    rules = [
        (
            ["api 模型", "api模型", "api model"],
            "API 模型不能用于本地训练",
            "当前选择的基座模型为 API 类型，LLaMA-Factory 需要本地可访问的模型路径。请在模型管理中创建或选择模型来源为「本地」且配置了算力节点路径的模型。",
        ),
        (
            ["未配置算力节点", "未配置.*路径", "模型.*路径"],
            "模型缺少算力节点路径",
            "请在模型管理中编辑该模型，设置模型路径为算力节点可访问的本地目录。",
        ),
        (
            ["不支持本地训练", "not trainable"],
            "模型不可用于训练",
            "当前选择的模型不支持作为 LLaMA-Factory 训练基座。请确认模型来源为本地、路径已配置且模型目录在算力节点上存在。",
        ),
        (
            ["dataset columns missing", "keyerror", "history", "instruction", "input", "output", "messages"],
            "训练数据字段不匹配",
            "请检查所选数据集格式是否与训练模板一致。Alpaca 格式通常需要 instruction/input/output；ShareGPT 格式通常需要 messages。",
        ),
        (
            ["dataset file not found", "dataset_dir", "no uploaded file"],
            "训练数据文件不可用",
            "请确认数据集已上传文件，并且应用服务可以将数据同步到目标算力节点的数据目录。",
        ),
        (
            ["model_name_or_path path not available", "base_model", "model path", "no such file"],
            "基座模型路径不可用",
            "请在模型管理中检查本地模型路径，确保该路径在算力服务器或 Compute 容器挂载目录内真实存在。",
        ),
        (
            ["cuda out of memory", "outofmemoryerror", "显存", "memory"],
            "GPU 显存不足",
            "请降低 batch_size、cutoff_len、LoRA rank，启用 4bit 量化，或选择更高显存的算力节点。",
        ),
        (
            ["training command not found", "llamafactory-cli"],
            "训练框架命令不可用",
            "请检查 Compute 镜像是否包含 LLaMA-Factory，或确认 llamafactory-cli 已在容器 PATH 中。",
        ),
        (
            ["llama_factory_home not found"],
            "LLaMA-Factory 目录不可用",
            "请检查 Compute 服务的 LLAMA_FACTORY_HOME 配置和宿主机挂载路径。",
        ),
        (
            ["no available compute node", "not schedulable", "disabled", "capacity full"],
            "暂无可调度算力节点",
            "请检查算力节点是否启用、状态是否在线、并行任务数是否已满，或手动调整节点权重/标签。",
        ),
    ]
    for keywords, title, suggestion in rules:
        if any(keyword in text for keyword in keywords):
            diagnostics.append({"level": "error", "title": title, "suggestion": suggestion})
    if not diagnostics and (errors or log_text):
        diagnostics.append(
            {
                "level": "error",
                "title": "训练任务异常",
                "suggestion": "请查看预检错误和训练日志原文，优先确认模型路径、数据集格式、GPU 显存和 LLaMA-Factory 参数。",
            }
        )
    return diagnostics


async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
    task_id = str(payload.get("task_id") or payload.get("id") or "")
    if task_id and get_settings().compute_mode != "simulator":
        try:
            preflight = await _fine_tune_preflight(store, task_id, payload, validate=True, sync_resources=True)
        except Exception as exc:  # noqa: BLE001 - task has not entered running state yet
            raise RuntimeError(f"preflight failed: {exc}") from exc
        if not preflight["valid"]:
            errors = "; ".join(preflight.get("errors") or ["preflight failed"])
            raise RuntimeError(f"preflight failed: {errors}")
        payload = {**payload, "compute_node_id": preflight["node"]["id"]}
    task = store.start_task(payload)
    if get_settings().compute_mode == "simulator":
        return task
    node, job_payload = store.build_compute_job_payload(task["id"])
    job = await ComputeNodeClient(node["api_base_url"]).create_job(job_payload)
    return store.apply_compute_job(task["id"], job)


async def _fine_tune_preflight(
    store: Any,
    task_id: str,
    payload: dict[str, Any] | None = None,
    validate: bool = True,
    sync_resources: bool = False,
) -> dict[str, Any]:
    node, job_payload = store.prepare_compute_job_payload(task_id, payload or {})
    return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=sync_resources, store=store)


async def _fine_tune_preflight_payload(
    store: Any,
    payload: dict[str, Any],
    validate: bool = True,
) -> dict[str, Any]:
    node, job_payload = store.prepare_compute_job_payload_from_payload(payload)
    return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=False, store=store)


async def _fine_tune_preflight_with_job_payload(
    node: dict[str, Any],
    job_payload: dict[str, Any],
    validate: bool,
    sync_resources: bool,
    store: Any,
) -> dict[str, Any]:
    sync_results: list[dict[str, Any]] = []
    sync_errors: list[str] = []
    if sync_resources and get_settings().compute_mode != "simulator":
        try:
            sync_results = await _sync_training_dataset_to_compute_node(
                store,
                node,
                str(job_payload.get("train_dataset_id") or ""),
            )
        except Exception as exc:  # noqa: BLE001 - return as preflight error for page visibility
            sync_errors.append(str(exc))
    if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
        try:
            await _wait_for_object_storage()
        except Exception as exc:  # noqa: BLE001 - preflight exposes node storage failure
            sync_errors.append(f"shared storage health check failed: {exc}")
    if get_settings().compute_mode == "simulator":
        preview = {
            "valid": True,
            "errors": [],
            "warnings": ["compute_mode=simulator skips remote compute validation"],
            "engine": job_payload.get("engine") or job_payload.get("training_engine") or "llama_factory",
            "command": [],
            "command_text": "",
            "work_dir": "",
            "env": {},
            "path_checks": [],
        }
    else:
        client = ComputeNodeClient(node["api_base_url"])
        preview = await (client.validate_job(job_payload) if validate else client.preview_job(job_payload))
    errors = list(preview.get("errors") or [])
    errors.extend(sync_errors)
    warnings = list(preview.get("warnings") or [])
    if not node.get("enabled"):
        errors.append(f"compute node disabled: {node.get('code')}")
    if node.get("scheduler_status") not in {"online", "draining"}:
        errors.append(f"compute node not schedulable: {node.get('code')} status={node.get('scheduler_status')}")
    return {
        "valid": bool(preview.get("valid", not errors)) and not errors,
        "errors": errors,
        "warnings": warnings,
        "diagnostics": _training_diagnostics(errors, warnings),
        "node": {
            "id": node.get("id"),
            "code": node.get("code"),
            "name": node.get("name"),
            "api_base_url": node.get("api_base_url"),
            "scheduler_status": node.get("scheduler_status"),
            "gpu_count": node.get("gpu_count"),
        },
        "job_payload": job_payload,
        "preview": preview,
        "sync_results": sync_results,
    }


@router.post("/login")
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
    store = get_platform_store()
    ip = request.client.host if request and request.client else "unknown"
    now = time.time()
    recent = [stamp for stamp in _LOGIN_FAILURES.get(ip, []) if now - stamp < 300]
    if len(recent) >= 5:
        raise fail(429, "too many login attempts, retry later")
    user = store.login(payload.get("username", ""), payload.get("password", ""))
    if not user:
        _LOGIN_FAILURES[ip] = [*recent, now]
        raise fail(401, "invalid username or password")
    _LOGIN_FAILURES.pop(ip, None)
    sess = store.create_session(user["id"], ip=None)
    return ok({"token": f"platform-token-{user['id']}.{sess['session_id']}", "user": user, "session_id": sess["session_id"]})


@router.post("/logout")
async def logout(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    store = get_platform_store()
    session_id = payload.get("session_id", "")
    if session_id:
        store.finish_session(session_id)
    return ok(None)


@router.get("/me")
async def me(request: Request) -> dict[str, Any]:
    """根据 Authorization header 中的 token 返回当前登录用户信息"""
    store = get_platform_store()
    auth = request.headers.get("Authorization", "")
    token = auth.replace("Bearer ", "").strip()
    # token 格式: platform-token-{user_id}
    if token.startswith("platform-token-"):
        user_id = token[len("platform-token-"):].split(".", 1)[0]
        for u in store.users():
            if u.get("id") == user_id:
                return ok(u)
    raise fail(401, "invalid or missing token")


@router.get("/dashboard/overview")
async def dashboard_overview() -> dict[str, Any]:
    cached = _cached_dashboard("overview")
    if cached is not None:
        return cached
    store = get_platform_store()
    tasks = store.tasks()
    return _store_dashboard_cache("overview", ok(
        {
            "models": len(store.models()),
            "datasets": len(store.datasets()),
            "fine_tune_tasks": len(tasks),
            "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]),
            "compute_nodes": len(store.compute_nodes()),
            "gpus": len(store.gpus()),
        }
    ))


@router.get("/dashboard/stats")
async def dashboard_stats() -> dict[str, Any]:
    cached = _cached_dashboard("stats")
    if cached is not None:
        return cached
    """看板聚合数据：基于平台真实数据；缺项做合理近似。"""
    store = get_platform_store()
    tasks = store.tasks()
    users = store.users()
    nodes = store.compute_nodes()
    datasets = store.datasets()
    eval_tasks = store.eval_tasks()
    # 数据处理任务总数（来自 data_process 模块）
    try:
        from app.modules.data_process.store import get_data_process_store

        dp_store = get_data_process_store()
        dp_result = dp_store.list_tasks(page=1, page_size=1)
        dp_count = int(dp_result.get("total", 0))
    except Exception:
        dp_count = 0

    running_statuses = {"syncing", "queued", "running"}
    running_ft = [t for t in tasks if t.get("status") in running_statuses]
    online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"]
    # 评测中运行的任务数
    eval_running = 0
    try:
        eval_tasks = store.eval_tasks()
        eval_running = len([e for e in eval_tasks if e.get("status") in running_statuses])
    except Exception:
        eval_running = 0

    # 近 7 天训练统计（按创建日期分桶）
    now = datetime.now(timezone.utc)
    train_by_day: dict[str, int] = {}
    for t in tasks:
        ct = t.get("create_time")
        if ct:
            train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1
    training_7d = []
    for i in range(6, -1, -1):
        day = (now - timedelta(days=i)).strftime("%Y-%m-%d")
        training_7d.append(
            {
                "date": day[5:],
                "train": train_by_day.get(day, 0),
                "gpu": sum(len(t.get("gpus") or []) for t in running_ft),
                "accuracy": None,
            }
        )

    # 服务状态 —— 每个服务的"实例数"含义：
    #   模型训练 → 训练任务总数
    #   模型评测 → 评测任务总数
    #   模型推理 → 推理/对比任务实例数
    #   模型管理 → 基座模型注册总数
    #   数据集管理 → 数据集总数
    #   数据处理 → 数据处理任务总数
    #   数据类型转换 → 数据转换任务总数
    service_checks = [
        ("模型训练", "fine-tune", len(tasks)),
        ("模型评测", "model-eval", len(eval_tasks)),
        ("模型推理", "model-inference", len(store.compare_tasks())),
        ("模型管理", "model-manage", len(store.models())),
        ("数据集管理", "dataset-manage", len(datasets)),
        ("数据处理", "data-process", dp_count),
        ("数据类型转换", "data-convert", dp_count),
    ]
    service_status = []
    for svc_type, _path, svc_count in service_checks:
        service_status.append({
            "type": svc_type,
            "status": "normal",
            "count": svc_count,
        })

    # 训练任务状态归一化
    status_map = {
        "syncing": "running",
        "queued": "running",
        "running": "running",
        "pending": "pending",
        "paused": "pending",
        "completed": "completed",
        "failed": "failed",
        "error": "failed",
        "cancelled": "failed",
        "stopped": "failed",
    }
    training_tasks = [
        {
            "id": t.get("id"),
            "name": t.get("name"),
            "status": status_map.get(t.get("status"), "pending"),
            "train_type": t.get("train_type") or t.get("trainType") or "",
            "train_method": t.get("train_method") or t.get("trainMethod") or "",
            "base_model": t.get("base_model") or t.get("baseModel") or "",
            "progress": t.get("progress", 0),
            "accuracy": t.get("accuracy"),
            "started_at": (t.get("create_time") or "")[:16],
        }
        for t in tasks[:8]
    ]

    # 用户操作分布：仅统计 模型推理 / 模型训练 / 模型评测 / 数据处理 四类
    MODULE_LABELS = [
        ("data-process", "数据处理"),
        ("data_process", "数据处理"),
        ("dataset", "数据处理"),
        ("fine-tune", "模型训练"),
        ("fine_tune", "模型训练"),
        ("model-eval", "模型评测"),
        ("eval", "模型评测"),
        ("model-inference", "模型推理"),
        ("inference", "模型推理"),
    ]
    OP_ORDER = [
        "数据处理",
        "模型训练",
        "模型评测",
        "模型推理",
    ]

    def _op_module(action: str) -> str | None:
        a = (action or "").lower()
        for prefix, label in MODULE_LABELS:
            if a.startswith(prefix):
                return label
        return None

    audit = store.audit_logs(limit=1000)
    op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
    for log in audit.get("items", []):
        label = _op_module(log.get("action") or "")
        if label:
            op_counter[label] += 1
    operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]

    # 最近登录用户
    recent = sorted(
        [u for u in users if u.get("last_login")],
        key=lambda u: u["last_login"],
        reverse=True,
    )[:5]
    recent_login_users = [
        {
            "user": u.get("display_name") or u.get("username"),
            "role": u.get("role"),
            "last_login": (u.get("last_login") or "")[:16],
        }
        for u in recent
    ]

    # 登录时长排行（本月），只取 top 5
    login_duration_rank = []
    try:
        login_duration_rank = store.login_duration_rank(limit=5)
    except Exception:
        pass

    return _store_dashboard_cache("stats", ok(
        {
            "online_services": sum(s["count"] for s in service_status),
            "running_tasks": len(running_ft) + eval_running,
            "pending_alerts": 0,
            "training_7d": training_7d,
            "service_status": service_status,
            "training_tasks": training_tasks,
            "operation_distribution": operation_distribution,
            "login_duration_rank": login_duration_rank,
            "recent_login_users": recent_login_users,
        }
    ))


@router.get("/system-info")
async def system_info() -> dict[str, Any]:
    return ok(get_platform_store().system_info())


@router.get("/users")
async def users() -> dict[str, Any]:
    return ok(get_platform_store().users())


@router.post("/users")
async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok(get_platform_store().create_user(payload))


@router.put("/users/{user_id}")
async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_user(user_id, payload))
    except KeyError:
        raise fail(404, "user not found")


@router.delete("/users/{user_id}")
async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]:
    try:
        get_platform_store().delete_user(user_id)
        return ok({"deleted": user_id, "current_username": current_username})
    except KeyError:
        raise fail(404, "user not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/users/{user_id}/reset-password")
async def reset_user_password(
    user_id: str,
    payload: dict[str, Any] = Body(default={}),
) -> dict[str, Any]:
    new_password = payload.get("password") or "Platform@123"
    try:
        get_platform_store().reset_password(user_id, new_password)
        return ok({"reset": user_id})
    except KeyError:
        raise fail(404, "user not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/users/me/password")
async def change_my_password(
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    """用户自行修改密码：验证旧密码后设置新密码。"""
    old_password = payload.get("old_password") or ""
    new_password = payload.get("new_password") or ""
    if not old_password or not new_password:
        raise fail(400, "old_password and new_password are required")
    if len(new_password) < 6:
        raise fail(400, "new password must be at least 6 characters")
    try:
        success = get_platform_store().change_password(
            current_user["id"], old_password, new_password
        )
    except KeyError:
        raise fail(404, "user not found")
    if not success:
        raise fail(400, "old password is incorrect")
    return ok({"changed": True})


@router.get("/model-manage/local-models")
async def local_models() -> dict[str, Any]:
    store = get_platform_store()
    models = [{"path": item.get("path") or "", "name": item["name"], "source": "registered"} for item in store.models()]
    seen = {item["path"] for item in models if item.get("path")}
    if get_settings().compute_mode != "simulator":
        for node in store.compute_nodes():
            if not node.get("enabled"):
                continue
            try:
                result = await ComputeNodeClient(node["api_base_url"]).list_files(root="models", directories_only=True)
            except Exception:
                continue
            for item in result.get("items") or []:
                path = str(item.get("path") or "")
                if not path or path in seen:
                    continue
                seen.add(path)
                models.append(
                    {
                        "path": path,
                        "name": item.get("name") or path.rsplit("/", 1)[-1],
                        "source": f"compute:{node.get('code')}",
                    }
                )
    return ok({"models": models})


@router.get("/model-manage/trained-models")
async def trained_models(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    all_models = get_platform_store().trained_models()
    if is_admin(current_user):
        return ok({"models": all_models})
    # 普通用户只能看到自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = set(filter_accessible_resource_ids("trained_model", [m["id"] for m in all_models], current_user))
    result = [m for m in all_models if m.get("created_by") == user_id or m["id"] in accessible]
    return ok({"models": result})


@router.delete("/model-manage/trained-models/{model_id}")
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("trained_model", model_id, current_user, "delete"):
        raise fail(403, "no permission to delete this trained model")
    pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}")
    if pending:
        return pending
    get_platform_store().delete_trained_model(model_id)
    return ok({"deleted": model_id, "type": type})


@router.get("/model-manage/trained-models/{model_id}/artifacts")
async def trained_model_artifacts(model_id: str) -> dict[str, Any]:
    return ok(get_platform_store().model_artifacts(model_id))


@router.get("/model-manage/trained-models/{model_id}/lineage")
async def trained_model_lineage(model_id: str) -> dict[str, Any]:
    return ok(get_platform_store().model_lineage(model_id))


@router.get("/model-manage/export-jobs")
async def model_export_jobs(trained_model_id: str | None = Query(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if trained_model_id and not has_resource_access("trained_model", trained_model_id, current_user, "read"):
        raise fail(403, "no permission to access export jobs")
    return ok(get_platform_store().model_export_jobs(trained_model_id))


@router.get("/model-manage/name/{name}")
async def model_by_name(name: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().model_by_name(name))
    except KeyError:
        raise fail(404, "model not found")


@router.get("/model-manage")
async def model_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    # 基座模型是平台共享资源，所有登录用户均可查看
    return ok(get_platform_store().models())


@router.post("/model-manage/test-online")
async def test_online_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """测试在线模型 API 是否可用：发送一个简单的 chat/completions 请求验证连通性。"""
    api_url = (payload.get("api_url") or "").rstrip("/")
    api_key = payload.get("api_key") or ""
    model_name = payload.get("online_model_name") or ""
    if not api_url:
        raise fail(400, "api_url is required")
    if not model_name:
        raise fail(400, "online_model_name is required")
    import httpx
    try:
        async with httpx.AsyncClient(timeout=15) as client:
            headers = {"Content-Type": "application/json"}
            if api_key:
                headers["Authorization"] = f"Bearer {api_key}"
            # 尝试多种 OpenAI 兼容路径
            chat_paths = [
                f"{api_url}/chat/completions",
                f"{api_url}/v1/chat/completions",
                f"{api_url}/modelTF/v1/chat/completions",
            ]
            resp = None
            for path in chat_paths:
                try:
                    r = await client.post(
                        path,
                        json={
                            "model": model_name,
                            "messages": [{"role": "user", "content": "Hi"}],
                            "max_tokens": 5,
                            "temperature": 0,
                        },
                        headers=headers,
                    )
                    if r.status_code in (200, 201):
                        resp = r
                        break
                except Exception:
                    continue
            if resp is None:
                return ok({"success": False, "error": f"无法连接到 {api_url}，请检查地址和端口"})
            body = resp.json()
            usage = body.get("usage", {})
            return ok({
                "success": True,
                "model": body.get("model", model_name),
                "provider": body.get("object", ""),
                "usage": {
                    "prompt_tokens": usage.get("prompt_tokens", 0),
                    "completion_tokens": usage.get("completion_tokens", 0),
                    "total_tokens": usage.get("total_tokens", 0),
                },
                "latency_ms": None,  # 由前端计算
            })
    except httpx.TimeoutException:
        return ok({"success": False, "error": "连接超时（15s），请检查网络或 API 地址是否正确"})
    except Exception as exc:
        return ok({"success": False, "error": str(exc)})


@router.post("/model-manage")
async def create_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    payload.setdefault("created_by", current_user.get("id"))
    try:
        return ok(get_platform_store().create_model(payload))
    except KeyError as exc:
        raise fail(400, f"missing field: {exc}")
    except ValueError as exc:
        raise fail(400, str(exc))
    except Exception as exc:  # noqa: BLE001 - keep API errors visible to deployment smoke checks
        raise fail(500, f"create model failed: {exc}")


@router.get("/model-manage/{model_id}")
async def model_detail(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        model = get_platform_store().model(model_id)
    except KeyError:
        raise fail(404, "model not found")
    if not has_resource_access("model", model_id, current_user, "read"):
        raise fail(403, "no permission to access this model")
    return ok(model)


@router.put("/model-manage/{model_id}")
async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_model(model_id, payload))
    except KeyError:
        raise fail(404, "model not found")


@router.put("/model-manage/{model_id}/purpose")
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")}))
    except KeyError:
        raise fail(404, "model not found")


@router.delete("/model-manage/{model_id}")
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("model", model_id, current_user, "delete"):
        raise fail(403, "no permission to delete this model")
    pending = _require_approval_or_admin("model", model_id, current_user, f"删除模型 {model_id}")
    if pending:
        return pending
    get_platform_store().delete_model(model_id)
    return ok({"deleted": model_id})


@router.post("/model-manage/merge")
async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "")
    trained_model = next(
        (
            item
            for item in store.trained_models()
            if trained_model_id and (item["id"] == trained_model_id or item["name"] == trained_model_id)
        ),
        None,
    )
    if not trained_model:
        raise fail(404, "trained model not found")
    if not has_resource_access("trained_model", trained_model["id"], current_user, "execute"):
        raise fail(403, "no permission to merge this trained model")
    if payload.get("base_model_id") and not has_resource_access("model", str(payload["base_model_id"]), current_user, "execute"):
        raise fail(403, "no permission to use merge base model")
    base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
    adapter_path = (
        payload.get("adapter_path")
        or payload.get("adapter_name_or_path")
        or (trained_model and (trained_model.get("artifact_dir") or trained_model.get("adapter_path") or trained_model.get("merged_path")))
    )
    if not base_model_path:
        raise fail(400, "base_model_path is required")
    if not adapter_path:
        raise fail(400, "adapter_path is required")
    requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id"))
    node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []})
    if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
        try:
            await _wait_for_object_storage()
        except RuntimeError as exc:
            raise fail(503, str(exc))
        try:
            prepared_base = await _prepare_resource_on_node(store, "model", str(payload.get("base_model_id") or base_model_path), node)
            if prepared_base:
                base_model_path = prepared_base
            prepared_adapter = await _prepare_resource_on_node(store, "trained_model", str(payload.get("adapter_model_id") or (trained_model and trained_model.get("id")) or ""), node)
            if prepared_adapter:
                adapter_path = prepared_adapter
        except Exception as exc:
            raise fail(502, f"merge resource preparation failed: {exc}")
    health = node.get("health_detail") or {}
    output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
    output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
    output_dir = str(payload.get("output_dir") or f"{output_root.rstrip('/')}/{output_name}")
    job_payload = {
        **payload,
        "id": str(payload.get("job_id") or f"merge_{uuid.uuid4().hex[:12]}"),
        "name": output_name,
        "engine": "merge",
        "base_model": base_model_path,
        "model_name_or_path": base_model_path,
        "adapter_name_or_path": adapter_path,
        "output_dir": output_dir,
        "template": payload.get("template", "qwen"),
        "train_method": payload.get("train_method", "lora"),
        "gpus": payload.get("gpus") or [],
        "trained_model_id": trained_model["id"] if trained_model else trained_model_id,
        "model_name": trained_model["name"] if trained_model else payload.get("model_name"),
        "compute_node_id": node["id"],
        "compute_node_code": node.get("code"),
    }
    if get_settings().compute_mode == "simulator":
        job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir}
    else:
        client = ComputeNodeClient(node["api_base_url"], timeout=900)
        preview = await client.validate_job(job_payload)
        if not preview.get("valid", False):
            raise fail(409, "; ".join(preview.get("errors") or ["merge preflight failed"]))
        job = await client.create_job(job_payload)
    return ok(store.record_model_merge_job(node, job_payload, job, trained_model["id"] if trained_model else trained_model_id))


@router.get("/dataset-manage/preview/{file_id}")
async def dataset_preview(file_id: str) -> dict[str, Any]:
    try:
        row = get_platform_store().dataset_file(file_id)
        return ok({"content": row["content"]})
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/records/{file_id}/sources")
async def dataset_record_sources(file_id: str) -> dict[str, Any]:
    try:
        return ok({"items": get_platform_store().dataset_file_record_sources(file_id)})
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/versions/{file_id}")
async def dataset_versions(file_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().file_versions(file_id))
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/versions/{file_id}/{version_id}")
async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]:
    try:
        row = get_platform_store().dataset_file(file_id)
        versions = get_platform_store().file_versions(file_id)["versions"]
        version = next((item for item in versions if item["id"] == version_id), None)
        if not version:
            raise KeyError(version_id)
        return ok({"version": version, "content": row["content"]})
    except KeyError:
        raise fail(404, "dataset version not found")


@router.post("/dataset-manage/versions/{file_id}")
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().create_file_version(file_id, payload))
    except KeyError:
        raise fail(404, "dataset file not found")


@router.put("/dataset-manage/versions/{file_id}/active")
async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().activate_file_version(file_id, payload["version_id"]))
    except KeyError:
        raise fail(404, "dataset version not found")


@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().delete_file_version(file_id, version_id))
    except KeyError:
        raise fail(404, "dataset version not found")
    except ValueError as exc:
        raise fail(400, str(exc))


async def _sync_dataset_file_to_compute_nodes(
    store: Any,
    dataset_id: str,
    file_id: str,
    filename: str,
    content: bytes,
) -> list[dict[str, Any]]:
    results: list[dict[str, Any]] = []
    if get_settings().compute_mode == "simulator" or get_settings().minio_enabled:
        return results
    target_name = Path(filename or f"{file_id}.jsonl").name
    target_relative_path = f"datasets/{dataset_id}/{target_name}"
    for node in store.compute_nodes():
        if not node.get("enabled"):
            continue
        try:
            result = await ComputeNodeClient(node["api_base_url"]).upload_file(
                target_name,
                content,
                target_relative_path,
                resource_type="dataset",
                resource_id=dataset_id,
            )
            store.upsert_resource_replica(
                node["id"],
                "dataset",
                dataset_id,
                str(result.get("local_path") or ""),
            )
            results.append(
                {
                    "node_id": node["id"],
                    "node_code": node.get("code"),
                    "success": True,
                    "local_path": result.get("local_path"),
                    "byte_size": result.get("byte_size"),
                    "checksum_sha256": result.get("checksum_sha256"),
                }
            )
        except Exception as exc:  # noqa: BLE001 - keep upload usable while exposing sync failures
            results.append(
                {
                    "node_id": node["id"],
                    "node_code": node.get("code"),
                    "success": False,
                    "error": str(exc),
                }
            )
    return results


async def _sync_training_dataset_to_compute_node(
    store: Any,
    node: dict[str, Any],
    dataset_id: str,
) -> list[dict[str, Any]]:
    if get_settings().minio_enabled:
        files = store.training_dataset_files(dataset_id)
        objects = store.storage_objects_for_resource("dataset", dataset_id)
        object_by_name = {Path(str(item.get("file_name") or item.get("object_key") or "")).name: item for item in objects}
        results: list[dict[str, Any]] = []
        client = ComputeNodeClient(node["api_base_url"])
        for item in files:
            target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
            obj = object_by_name.get(target_name)
            if not obj:
                raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
            url = get_object_storage().presigned_get(obj["object_key"])
            result = await client.prepare_cache({
                "resource_id": dataset_id,
                "version_id": obj["version_id"],
                "download_url": url,
                "checksum_sha256": obj.get("checksum_sha256") or "",
                "byte_size": obj.get("byte_size") or 0,
                "relative_path": f"datasets/{dataset_id}/{target_name}",
            })
            results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
        return results
    if not dataset_id:
        raise RuntimeError("train_dataset_id is required")
    files = store.training_dataset_files(dataset_id)
    if not files:
        raise RuntimeError(f"dataset has no uploaded file: {dataset_id}")
    split_aware = any(item.get("split") for item in files)
    files = [
        item
        for item in files
        if not split_aware or item.get("split") in {"train", "validation"}
    ]
    client = ComputeNodeClient(node["api_base_url"])
    results: list[dict[str, Any]] = []
    for item in files:
        target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
        result = await client.upload_file(
            target_name,
            str(item.get("content") or "").encode("utf-8"),
            f"datasets/{dataset_id}/{target_name}",
            resource_type="dataset",
            resource_id=dataset_id,
        )
        store.upsert_resource_replica(
            node["id"],
            "dataset",
            dataset_id,
            str(result.get("local_path") or ""),
        )
        results.append(
            {
                "node_id": node["id"],
                "node_code": node.get("code"),
                "file_id": item.get("id"),
                "name": target_name,
                "local_path": result.get("local_path"),
                "byte_size": result.get("byte_size"),
                "checksum_sha256": result.get("checksum_sha256"),
            }
        )
    return results


@router.post("/dataset-manage/upload/{dataset_id}")
async def upload_dataset_files(
    dataset_id: str,
    files: list[UploadFile] = File(default=[]),
    sync_to_compute: bool = Query(default=True),
) -> dict[str, Any]:
    created: list[dict[str, Any]] = []
    compute_sync: list[dict[str, Any]] = []
    pending_sync: list[tuple[str, str, bytes]] = []
    store = get_platform_store()
    try:
        store.dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    with store.connect() as conn:
        for file in files:
            raw = await file.read()
            content = raw.decode("utf-8", errors="replace")
            created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
            created.append(created_file)
            pending_sync.append((created_file["id"], created_file["name"], raw))
            if get_settings().minio_enabled:
                object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
                uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
                get_platform_store().create_storage_object({
                    "resource_type": "dataset", "resource_id": dataset_id,
                    "version_id": created_file.get("active_version_id") or created_file["id"],
                    "bucket": uploaded["bucket"], "object_key": object_key,
                    "file_name": created_file["name"], "content_type": file.content_type,
                    "byte_size": len(raw), "checksum_sha256": hashlib.sha256(raw).hexdigest(),
                    "status": "available",
                })
    if sync_to_compute:
        for file_id, file_name, raw in pending_sync:
            compute_sync.extend(
                await _sync_dataset_file_to_compute_nodes(
                    store,
                    dataset_id,
                    file_id,
                    file_name,
                    raw,
                )
            )
    return ok({"files": created, "compute_sync": compute_sync})


@router.get("/dataset-manage/download/{dataset_id}")
async def download_dataset(dataset_id: str) -> PlainTextResponse:
    dataset = get_platform_store().dataset(dataset_id)
    content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])])
    return PlainTextResponse(content, media_type="text/plain")


@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
    row = get_platform_store().dataset_file(file_id)
    return PlainTextResponse(row["content"], media_type="text/plain")


@router.get("/dataset-manage")
async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    datasets = get_platform_store().datasets()
    if is_admin(current_user):
        return ok(datasets)
    # 普通用户可见：自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = set(filter_accessible_resource_ids("dataset", [d["id"] for d in datasets], current_user))
    result = [d for d in datasets if d.get("created_by") == user_id or d["id"] in accessible]
    return ok(result)


@router.post("/dataset-manage")
async def create_dataset(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    payload.setdefault("created_by", current_user.get("id"))
    dataset = get_platform_store().create_dataset(payload)
    return ok({"id": dataset["id"]})


@router.get("/dataset-manage/{dataset_id}")
async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        dataset = get_platform_store().dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    if not has_resource_access("dataset", dataset_id, current_user, "read"):
        raise fail(403, "no permission to access this dataset")
    return ok(dataset)


@router.put("/dataset-manage/{dataset_id}")
async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_dataset(dataset_id, payload))
    except KeyError:
        raise fail(404, "dataset not found")


@router.delete("/dataset-manage/{dataset_id}")
async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("dataset", dataset_id, current_user, "delete"):
        raise fail(403, "no permission to delete this dataset")
    pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}")
    if pending:
        return pending
    get_platform_store().delete_dataset(dataset_id)
    return ok({"deleted": dataset_id})


@router.get("/fine-tune/check-name")
async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]:
    exists = any(task["name"] == name for task in get_platform_store().tasks())
    return ok({"exists": exists})


@router.get("/fine-tune/progress/{task_id}")
async def fine_tune_progress(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().progress(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/tensorboard/start")
async def tensorboard_start() -> dict[str, Any]:
    return ok({"status": "running", "url": "http://localhost:6006"})


@router.get("/fine-tune")
async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    tasks = get_platform_store().tasks()
    if current_user.get("role") == "admin" or current_user.get("protected"):
        return ok(tasks)
    accessible = set(filter_accessible_resource_ids("fine-tune", [t["id"] for t in tasks], current_user))
    return ok([t for t in tasks if t["id"] in accessible])


@router.post("/fine-tune")
async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    payload.setdefault("created_by", current_user.get("id"))
    if not is_admin(current_user):
        model_id = str(payload.get("base_model") or payload.get("base_model_id") or "")
        dataset_id = str(payload.get("train_dataset_id") or "")
        if model_id and not has_resource_access("model", model_id, current_user, "execute"):
            raise fail(403, "no permission to use this base model")
        if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"):
            raise fail(403, "no permission to use this dataset")
    try:
        task = get_platform_store().create_task(payload)
        return ok({"id": task["id"]})
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/fine-tune/start")
async def start_fine_tune(
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    store = get_platform_store()
    # GPU 权限校验：普通用户只能使用被分配的 GPU
    if not is_admin(current_user):
        node_id = payload.get("compute_node_id") or payload.get("node_id")
        gpu_indices = payload.get("gpu_indices")
        if gpu_indices is None:
            gpu_indices = payload.get("gpus") or []
        if node_id and gpu_indices:
            if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
                raise fail(403, "无权使用所选 GPU，请联系管理员分配")
    # 记录创建者
        if node_id and not gpu_indices:
            payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
        payload["strict_node_selection"] = bool(node_id)
    payload.setdefault("created_by", current_user.get("id"))
    try:
        return ok(await _submit_fine_tune_task(store, payload))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            store.mark_task_failed(task_id, str(exc))
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - mark task failed when remote submit fails
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            store.mark_task_failed(task_id, str(exc))
        raise fail(502, f"submit compute job failed: {exc}")


@router.post("/fine-tune/preflight")
async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True))
    except RuntimeError as exc:
        return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
    except Exception as exc:  # noqa: BLE001 - expose compute validation errors to training create page
        raise fail(502, f"compute preflight failed: {exc}")


@router.post("/fine-tune/command-preview")
async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False))
    except RuntimeError as exc:
        return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"compute command preview failed: {exc}")


@router.post("/fine-tune/{task_id}/preflight")
async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - expose compute validation errors to training create page
        raise fail(502, f"compute preflight failed: {exc}")


@router.post("/fine-tune/{task_id}/command-preview")
async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"compute command preview failed: {exc}")


@router.get("/fine-tune/{task_id}")
async def fine_tune_detail(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().task(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.get("/fine-tune/{task_id}/logs")
async def fine_tune_logs(
    task_id: str,
    tail_lines: int | None = Query(default=500, ge=1, le=5000),
    offset: int | None = Query(default=None, ge=0),
    limit: int | None = Query(default=None, ge=1, le=5000),
) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if task.get("compute_job_id"):
        node = _node_for_task(task)
        if node:
            try:
                logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], tail_lines, offset, limit)
                try:
                    store.record_training_log_metrics(task_id, str(logs.get("content") or ""))
                except Exception:
                    pass
                if task.get("status") in {"queued", "running", "failed", "stopped", "completed"}:
                    try:
                        job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"])
                        store.apply_compute_job(task_id, job)
                    except Exception:
                        pass
                return ok({"source": "compute", **logs})
            except Exception as exc:  # noqa: BLE001 - keep failure reason visible even when log fetch fails
                content = task.get("failure_reason") or f"fetch compute log failed: {exc}"
                return ok({"job_id": task.get("compute_job_id"), "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
    content = task.get("failure_reason") or ""
    return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})


@router.get("/fine-tune/{task_id}/diagnostics")
async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    log_text = ""
    node = _node_for_task(task)
    if node and task.get("compute_job_id"):
        try:
            logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], 1000, None, None)
            log_text = str(logs.get("content") or "")
        except Exception:
            log_text = ""
    errors = [str(task.get("failure_reason") or "")] if task.get("failure_reason") else []
    return ok(
        {
            "task_id": task_id,
            "status": task.get("status"),
            "failure_reason": task.get("failure_reason") or "",
            "diagnostics": _training_diagnostics(errors, [], log_text),
        }
    )


@router.put("/fine-tune/{task_id}")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "write"):
        raise fail(403, "no permission to update this task")
    try:
        return ok(get_platform_store().update_task(task_id, payload))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/stop/{task_id}")
async def stop_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
        # 审批拦截：非 admin 停止他人任务需审批
        pending = _require_approval_or_admin("fine_tune_task", task_id, current_user, f"停止训练任务 {task_id}")
        if pending:
            return pending
        node = _node_for_task(task)
        if task.get("compute_job_id") and node and get_settings().compute_mode != "simulator":
            job = await ComputeNodeClient(node["api_base_url"]).stop_job(task["compute_job_id"])
            return ok(store.apply_compute_job(task_id, job))
        return ok(store.stop_task(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/{task_id}/stop")
async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return await stop_fine_tune(task_id, current_user)


@router.post("/fine-tune/{task_id}/retry")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    payload = payload or {}
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if not has_resource_access("fine-tune", task_id, current_user, "execute"):
        raise fail(403, "no permission to retry this task")
    if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
        raise fail(409, "only failed or stopped tasks can be retried without force=true")
    retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
    store.reset_task_for_retry(task_id, retry_payload)
    try:
        return ok(await _submit_fine_tune_task(store, retry_payload))
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - mark retry failed when remote submit fails
        store.mark_task_failed(task_id, str(exc))
        raise fail(502, f"retry fine tune task failed: {exc}")


@router.delete("/fine-tune/{task_id}")
async def delete_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "delete"):
        raise fail(403, "no permission to delete this task")
    pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}")
    if pending:
        return pending
    get_platform_store().delete_task(task_id)
    return ok({"deleted": task_id})


@router.get("/fine-tune/{task_id}/overview")
async def fine_tune_overview(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    task = store.task(task_id)
    return ok(
        {
            "task": task,
            "progress": store.progress(task_id),
            "metrics": store.task_metrics(task_id),
            "checkpoints": store.task_checkpoints(task_id),
        }
    )


@router.get("/fine-tune/{task_id}/checkpoints")
async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    return ok(store.task_checkpoints(task_id))


@router.get("/fine-tune/{task_id}/metrics")
async def fine_tune_metrics(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    return ok(store.task_metrics(task_id))


@router.get("/model-eval")
async def model_eval_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    tasks = get_platform_store().eval_tasks()
    if current_user.get("role") == "admin" or current_user.get("protected"):
        return ok(tasks)
    accessible = set(filter_accessible_resource_ids("eval", [t["id"] for t in tasks], current_user))
    return ok([t for t in tasks if t["id"] in accessible])


@router.get("/model-eval/{task_id}")
async def model_eval_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        task = store.eval_task(task_id)
        if task.get("compute_job_id") and task.get("compute_node_id") and task.get("status") in {"queued", "running", "completed"}:
            node = next(
                (n for n in store.compute_nodes() if n["id"] == task.get("compute_node_id")),
                None,
            )
            if node:
                try:
                    client = ComputeNodeClient(node["api_base_url"])
                    job = await client.get_job(task["compute_job_id"])
                    result_content = None
                    if job.get("status") == "completed" and not task.get("samples"):
                        result_content = await fetch_eval_result_content(client, node, job)
                    task = store.apply_eval_job_result(task_id, job, result_content)
                except Exception:
                    pass
    except KeyError:
        raise fail(404, "eval task not found")
    if not has_resource_access("eval", task_id, current_user, "read"):
        raise fail(403, "no permission to access this eval task")
    return ok(task)


@router.post("/model-eval/start")
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Start an evaluation task: submit eval job to compute node."""
    store = get_platform_store()
    # 1. Create eval task record
    task = store.create_eval_task({**payload, "status": "pending"})

    # 2. Resolve model path (supports both regular models and trained models)
    model_id = str(payload.get("model_id", ""))
    model_path = ""
    adapter_path = payload.get("adapter_path", "")
    model_node_id = ""
    try:
        db_model = store.model(model_id)
        model_path = db_model.get("path", "")
        model_node_id = db_model.get("compute_node_id") or ""
    except KeyError:
        # Try trained_models table (IDs prefixed with tm_)
        trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
        if trained:
            model_node_id = trained.get("compute_node_id") or ""
            merged_path = trained.get("merged_path", "")
            base_path = trained.get("base_model_path", "")
            if trained.get("merged") and merged_path:
                # Merged model: use merged_path as model, no adapter needed
                model_path = merged_path
            elif base_path:
                # Unmerged: use base model + adapter checkpoint
                model_path = base_path
                if merged_path:
                    adapter_path = merged_path
            else:
                model_path = merged_path or base_path
    if not model_path:
        store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
        return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})

    # 3. Resolve dataset file
    dataset_id = str(payload.get("dataset_id", ""))
    dataset_path = ""
    try:
        ds_files = store.training_dataset_files(dataset_id)
        if ds_files:
            dataset_path = ds_files[0].get("local_path") or ds_files[0].get("name", "")
    except Exception:
        pass
    if not dataset_path:
        # Try to get file content and sync to compute
        try:
            ds = store.dataset(dataset_id)
            for f in ds.get("files", []):
                if f.get("content"):
                    dataset_path = f.get("name", f"dataset_{dataset_id}.jsonl")
                    break
        except KeyError:
            pass
    if not dataset_path:
        store.update_eval_task(task["id"], {"status": "failed", "error": "dataset not found or no files"})
        return ok({"task_id": task["id"], "status": "failed", "error": "dataset not found or no files"})

    # 4. Resolve dimension config
    dimension_id = str(payload.get("dimension_id", ""))
    dimension_cfg: dict[str, Any] = {}
    if dimension_id:
        try:
            dim = store.dimension(dimension_id)
            # Resolve eval model API config
            eval_model_name = dim.get("eval_model", "")
            api_url = ""
            api_key = ""
            api_model_name = ""
            if eval_model_name:
                try:
                    eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name)
                    if isinstance(eval_model, dict):
                        api_url = eval_model.get("api_url", "")
                        api_key = eval_model.get("api_key", "")
                        # 模型记录里的 model_name 是真实 API 模型名（如 deepseek-chat），
                        # 优先传给评测器，避免用平台内部名称调用 LLM API
                        api_model_name = eval_model.get("model_name") or ""
                except (KeyError, Exception):
                    pass
            dimension_cfg = {
                "type": dim.get("type", ""),
                "eval_model": eval_model_name,
                "api_model": api_model_name or eval_model_name,
                "eval_method": dim.get("eval_method", ""),
                "eval_prompt": dim.get("eval_prompt", ""),
                "api_url": api_url,
                "api_key": api_key,
                "score_min": dim.get("score_min", 0),
                "score_max": dim.get("score_max", 5),
                "pass_threshold": dim.get("pass_threshold", 3),
            }
        except KeyError:
            pass

    # 5. Select compute node: 优先页面选择的节点 / 模型所在节点，避免多节点时选错
    preferred_node_id = payload.get("compute_node_id") or payload.get("node_id") or model_node_id
    node = _select_eval_node(store, preferred_node_id)
    if not node:
        message = "no online compute node" if not preferred_node_id else f"model compute node not schedulable: {preferred_node_id}"
        store.update_eval_task(task["id"], {"status": "failed", "error": message})
        return ok({"task_id": task["id"], "status": "failed", "error": message})

    # 6. Build eval job payload
    output_dir = f"/data/yg-ft/outputs/{task['id']}"
    job_payload = {
        "id": f"eval_{task['id']}",
        "name": task.get("eval_task_name", task["id"]),
        "engine": "eval",
        "model_name_or_path": model_path,
        "adapter_name_or_path": adapter_path,
        "template": payload.get("template", "qwen"),
        "dataset_path": dataset_path,
        "output_dir": output_dir,
        "basic_metrics": payload.get("basic_metrics", {}),
        "dimension": dimension_cfg,
        "gpus": [int(payload.get("gpu_id", 0))],
        "temperature": payload.get("temperature", 0.1),
        "max_new_tokens": payload.get("max_new_tokens", 512),
        "compute_node_id": node["id"],
    }

    # 7. Submit to compute node via create_job (uses engine="eval" path)
    try:
        client = ComputeNodeClient(node["api_base_url"])
        # Sync dataset file to compute node if needed
        if not dataset_path.startswith("/"):
            try:
                ds_files = store.training_dataset_files(dataset_id)
                if ds_files and ds_files[0].get("content"):
                    upload_result = await client.upload_file(
                        ds_files[0].get("name", "eval_data.jsonl"),
                        ds_files[0]["content"].encode("utf-8"),
                        f"datasets/{dataset_id}/{ds_files[0].get('name', 'eval_data.jsonl')}",
                        resource_type="dataset",
                        resource_id=dataset_id,
                    )
                    job_payload["dataset_path"] = upload_result.get("local_path", dataset_path)
            except Exception:
                pass

        job = await client.create_job(job_payload)
        store.update_eval_task(task["id"], {
            "status": "running",
            "compute_job_id": job.get("id"),
            "compute_node_id": node["id"],
            "output_dir": output_dir,
        })
        # 评测占用 GPU 由 eval_tasks 派生（gpus()/compute_nodes() 直接统计），
        # 不再复用 mark_inference_loaded 内存标记，避免删除评测后 GPU 状态残留 busy
        return ok({"task_id": task["id"], "status": "running", "job": job})
    except Exception as exc:
        store.update_eval_task(task["id"], {"status": "failed", "error": str(exc)})
        return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})


@router.delete("/model-eval/{task_id}")
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("eval", task_id, current_user, "delete"):
        raise fail(403, "no permission to delete this eval task")
    pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
    if pending:
        return pending
    get_platform_store().delete_eval_task(task_id)
    return ok({"deleted": task_id})


@router.get("/dimension")
async def dimension_list() -> dict[str, Any]:
    return ok(get_platform_store().dimensions())


@router.post("/dimension")
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok(get_platform_store().create_dimension(payload))


@router.get("/dimension/{dimension_id}")
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().dimension(dimension_id))
    except KeyError:
        raise fail(404, "dimension not found")


@router.put("/dimension/{dimension_id}")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_dimension(dimension_id, payload))
    except KeyError:
        raise fail(404, "dimension not found")


@router.delete("/dimension/{dimension_id}")
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
    get_platform_store().delete_dimension(dimension_id)
    return ok({"deleted": dimension_id})


@router.get("/model-compare")
async def model_compare_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    tasks = get_platform_store().compare_tasks()
    if is_admin(current_user):
        return ok(tasks)
    accessible = filter_accessible_resource_ids_batch("compare", [item["id"] for item in tasks], current_user)
    return ok([item for item in tasks if item["id"] in accessible])


@router.post("/model-compare")
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    payload.setdefault("created_by", current_user.get("id"))
    model_ids = payload.get("model_ids") or payload.get("models") or []
    if not is_admin(current_user):
        for model_id in model_ids:
            if isinstance(model_id, dict): model_id = model_id.get("id") or model_id.get("model_id")
            if model_id and not has_resource_access("model", str(model_id), current_user, "execute"):
                raise fail(403, "no permission to use inference model")
    task = get_platform_store().create_compare_task(payload)
    return ok({"id": task["id"]})


@router.post("/model-compare/all/stop-all")
async def model_compare_stop_all() -> dict[str, Any]:
    return ok({"stopped": True})


@router.post("/model-compare/stop-by-pid")
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"stopped": True, "pid": payload.get("pid")})


@router.get("/model-compare/{task_id}")
async def model_compare_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        task = get_platform_store().compare_task(task_id)
        if not has_resource_access("compare", task_id, current_user, "read"):
            raise fail(403, "no permission to access inference task")
        return ok(task)
    except KeyError:
        raise fail(404, "compare task not found")


async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = None) -> dict[str, Any]:
    """Best-effort unload the inference model from the node(s) that hold it.

    任务感知：优先卸载 ``task.load_status.loaded_models`` 中记录的节点；
    无任务时回退到平台记录的已加载推理的节点。每个节点使用短超时，
    保证卸载永远不会长时间阻塞调用方（例如删除操作）。
    """
    node_ids: set[str] = set()
    if task:
        load_status = task.get("load_status") or {}
        if isinstance(load_status, str):
            try:
                load_status = json.loads(load_status)
            except json.JSONDecodeError:
                load_status = {}
        node_ids = {item.get("node_id") for item in load_status.get("loaded_models") or [] if item.get("node_id")}
    if not node_ids:
        node_ids = {node["id"] for node in store.compute_nodes() if store.is_inference_loaded(node["id"])}
    nodes = [node for node in store.compute_nodes() if node["id"] in node_ids]
    results: list[dict[str, Any]] = []
    for node in nodes:
        try:
            result = await ComputeNodeClient(node["api_base_url"]).inference_unload()
            results.append({"node_id": node["id"], "node_code": node.get("code"), "success": True, "result": result})
        except Exception as exc:  # noqa: BLE001 - best-effort unload must not raise
            results.append({"node_id": node["id"], "node_code": node.get("code"), "success": False, "error": str(exc)})
        finally:
            store.mark_inference_unloaded(node["id"])
    return {"unloaded": bool(results), "nodes": results}


@router.delete("/model-compare/{task_id}")
async def model_compare_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    # 先删记录（快），再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞
    try:
        task = get_platform_store().compare_task(task_id)
    except KeyError:
        raise fail(404, "compare task not found")
    if not has_resource_access("compare", task_id, current_user, "delete"):
        raise fail(403, "no permission to delete inference task")
    pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}")
    if pending:
        return pending
    get_platform_store().delete_compare_task(task_id)
    try:
        await _unload_from_compute_node(get_platform_store(), task=task)
    except Exception:  # noqa: BLE001 - deletion must succeed even if unload fails
        pass
    return ok({"deleted": task_id})


@router.get("/model-compare/{task_id}/load-status")
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
    try:
        task = get_platform_store().compare_task(task_id)
    except KeyError:
        raise fail(404, "compare task not found")
    load_status = task.get("load_status") or {"loaded_models": []}
    if isinstance(load_status, str):
        try:
            load_status = json.loads(load_status)
        except json.JSONDecodeError:
            load_status = {"loaded_models": []}
    return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})


@router.post("/model-compare/{task_id}/load-status")
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
    except KeyError:
        raise fail(404, "compare task not found")


def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[dict[str, Any]]) -> None:
    """同一计算节点同一时刻只能加载一个推理模型。

    当新任务把模型派发到了某节点后，把其它任务中在该节点上 ready/running
    的模型标记为已被替换，保持平台 DB 与计算节点实际状态一致。
    """
    taken_node_ids = {m.get("node_id") for m in loaded_models if m.get("node_id") and m.get("status") == "starting"}
    if not taken_node_ids:
        return
    for other in store.compare_tasks():
        if str(other.get("id")) == str(task_id):
            continue
        load_status = other.get("load_status") or {}
        if isinstance(load_status, str):
            try:
                load_status = json.loads(load_status)
            except json.JSONDecodeError:
                load_status = {}
        items = load_status.get("loaded_models") or []
        changed = False
        for item in items:
            if item.get("node_id") in taken_node_ids and item.get("status") in {"ready", "running"}:
                item["status"] = "error"
                item["error"] = "模型已被其他推理任务替换"
                changed = True
        if changed:
            new_status = "loaded" if any(i.get("status") in {"ready", "running"} for i in items) else "failed"
            store.update_compare_task(other["id"], {"status": new_status, "load_status": {"loaded_models": items}})


@router.post("/model-compare/{task_id}/load")
async def model_compare_load(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """异步派发模型加载到算力节点，立即返回。

    加载进度由轮询对账器（compute_poller → reconcile_inference_loads）推进：
    任务项先以 status=starting 记录，对账器查询节点 /inference/status 后
    推进到 ready/error。这里只负责把加载请求派发出去，绝不同步等待加载完成。
    """
    try:
        store = get_platform_store()
        task = store.compare_task(task_id)
        if not has_resource_access("compare", task_id, current_user, "execute"):
            raise fail(403, "no permission to load inference task")
        models = task.get("models") or []
        if isinstance(models, str):
            try:
                models = json.loads(models)
            except json.JSONDecodeError:
                models = []
        online_nodes = _candidate_online_nodes(store)
        if not online_nodes:
            return ok({"status": "failed", "error": "no online compute node"})
        loaded_models = []
        for item in models:
            if not isinstance(item, dict):
                continue
            preferred_node_id = item.get("node_id") or item.get("compute_node_id")
            model_path = item.get("model_path", "")
            if not model_path:
                # 尝试从模型库获取路径
                model_id = item.get("model_id", "")
                try:
                    db_model = store.model(model_id)
                    model_path = db_model.get("path", "")
                except KeyError:
                    trained_model = next((m for m in store.trained_models() if str(m.get("id")) == str(model_id)), None)
                    if trained_model:
                        model_path = trained_model.get("merged_path") or trained_model.get("artifact_dir") or ""
                        preferred_node_id = preferred_node_id or trained_model.get("compute_node_id")
            if not model_path:
                loaded_models.append({**item, "status": "error", "error": "model_path not found"})
                continue
            load_payload = {
                "model_name_or_path": model_path,
                "template": item.get("template", "qwen"),
            }
            if item.get("adapter_path"):
                load_payload["adapter_name_or_path"] = item["adapter_path"]
            if get_settings().compute_mode == "simulator":
                loaded_models.append({**item, "status": "ready", "node_id": "", "node_name": ""})
                continue
            # 只派发：HTTP 响应成功即视为已接受（节点会异步加载），loaded 字段忽略
            item_dispatched = False
            errors = []
            for node in _candidate_online_nodes(store, preferred_node_id):
                try:
                    if get_settings().minio_enabled:
                        await _wait_for_object_storage()
                    client = ComputeNodeClient(node["api_base_url"])
                    await client.inference_load(load_payload)
                    store.mark_inference_loaded(node["id"])
                    loaded_models.append({**item, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
                    item_dispatched = True
                    break
                except Exception as exc:  # noqa: BLE001 - try next candidate node
                    errors.append(f"{node.get('name') or node.get('code')}: {exc}")
            if not item_dispatched:
                loaded_models.append({**item, "status": "error", "error": "; ".join(errors) or "load dispatch failed"})
        if any(m.get("status") == "starting" for m in loaded_models):
            status = "starting"
        elif any(m.get("status") == "error" for m in loaded_models):
            status = "failed"
        else:
            status = "loaded"
        updated = store.update_compare_task(task_id, {"status": status, "load_status": {"loaded_models": loaded_models}})
        # 同一节点同一时刻只能有一个推理模型；新任务占用了节点后，把其它任务上该节点的模型标记为已被替换
        _invalidate_superseded_models(store, task_id, loaded_models)
        return ok(updated)
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/unload")
async def model_compare_unload(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        task = store.compare_task(task_id)
        if not has_resource_access("compare", task_id, current_user, "write"):
            raise fail(403, "no permission to unload inference task")
        # 任务感知卸载：只释放该任务实际加载到的节点，短超时快速返回
        unload_result = await _unload_from_compute_node(store, task=task)
        updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
        return ok({"task": updated, "unload": unload_result})
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/start-model")
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})


@router.post("/model-compare/chat-with-port")
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """Proxy non-streaming chat to the compute node running the inference model."""
    store = get_platform_store()
    node = _node_for_inference_payload(store, payload)
    if not node:
        return ok({"response": "no online compute node available for inference", "request": payload})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/chat", json_data=_build_messages_payload(payload))
        return ok(result)
    except Exception as exc:
        return ok({"response": f"inference failed: {exc}", "request": payload})


@router.post("/model-compare/stream-chat")
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
    """Stream chat from the compute node (SSE proxy)."""
    return await _stream_chat_proxy(payload)


@router.post("/model-chat/batch")
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"responses": [], "request": payload})


@router.post("/model-chat/local/chat")
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """Proxy chat to the compute node running the inference model."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"response": "no online compute node available for inference", "request": payload})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/chat", json_data=_build_messages_payload(payload))
        return ok(result)
    except Exception as exc:
        return ok({"response": f"inference failed: {exc}", "request": payload})


@router.post("/model-chat/local/chat/stream")
async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
    """Stream chat from the compute node."""
    return await _stream_chat_proxy(payload)


@router.post("/model-chat/local/preload")
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """Load a model on the compute node for inference."""
    model_path = (payload.get("model_name_or_path") or "").strip()
    if not model_path:
        return ok({"loaded": False, "error": "model_name_or_path is required"})
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        # 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
        result = await client.inference_load(payload)
        if result.get("loaded") or result.get("status") in {"loading", "ready"}:
            store.mark_inference_loaded(node["id"])
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.post("/model-chat/local/unload")
async def model_chat_local_unload() -> dict[str, Any]:
    """Unload the inference model from the compute node."""
    store = get_platform_store()
    # 释放所有已加载推理的节点（短超时，best-effort）
    results: list[dict[str, Any]] = []
    for n in store.compute_nodes():
        if not store.is_inference_loaded(n["id"]):
            continue
        try:
            client = ComputeNodeClient(n["api_base_url"])
            last_error = ""
            result = None
            for attempt in range(3):
                try:
                    result = await client.inference_unload()
                    break
                except Exception as exc:  # noqa: BLE001 - retry node cleanup
                    last_error = str(exc)
                    if attempt < 2:
                        await asyncio.sleep(2 ** attempt)
            if result is None:
                raise RuntimeError(last_error or "inference unload failed")
            results.append({"node_id": n["id"], "success": True, "result": result})
        except Exception as exc:  # noqa: BLE001 - best-effort unload
            results.append({"node_id": n["id"], "success": False, "error": str(exc)})
        finally:
            store.mark_inference_unloaded(n["id"])
    return ok({"unloaded": True, "nodes": results})


@router.get("/model-chat/local/status")
async def model_chat_local_status() -> dict[str, Any]:
    """Get inference session status from compute node."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client.inference_status()
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.post("/model-chat/trained/preload")
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    resource_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or "")
    if resource_id and not has_resource_access("trained_model", resource_id, current_user, "execute") and not has_resource_access("model", resource_id, current_user, "execute"):
        raise fail(403, "no permission to load this model")
    """Load a trained model (base + adapter) on the compute node for inference."""
    model_path = (payload.get("model_name_or_path") or "").strip()
    if not model_path:
        return ok({"loaded": False, "error": "model_name_or_path is required"})
    store = get_platform_store()
    requested_node_id = str(payload.get("compute_node_id") or payload.get("node_id") or "")
    node = next((item for item in store.compute_nodes() if item.get("id") == requested_node_id and item.get("enabled") and item.get("scheduler_status") == "online"), None)
    if not node:
        node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        prepared_path = await _prepare_resource_on_node(store, "trained_model", str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or ""), node)
        if prepared_path:
            payload = {**payload, "model_name_or_path": prepared_path}
        prepared_path = await _prepare_resource_on_node(store, "model", str(payload.get("model_id") or payload.get("resource_id") or ""), node)
        if prepared_path:
            payload = {**payload, "model_name_or_path": prepared_path}
        client = ComputeNodeClient(node["api_base_url"])
        # 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
        result = await client.inference_load({**payload, "compute_node_id": node["id"]})
        if result.get("loaded") or result.get("status") in {"loading", "ready"}:
            store.mark_inference_loaded(node["id"])
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.get("/compute/nodes")
async def compute_nodes(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    nodes = get_platform_store().compute_nodes()
    if is_admin(current_user):
        return ok(nodes)
    # 普通用户只看到自己被分配 GPU 的节点，避免泄露节点拓扑和未授权资源。
    assigned = {item["node_id"] for item in get_platform_store().gpu_assignments_for_user(current_user["id"])}
    return ok([node for node in nodes if node["id"] in assigned])


@router.post("/storage/objects/presign")
async def presign_storage_object(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Create a short-lived MinIO upload/download URL for a platform resource."""
    if not get_settings().minio_enabled:
        raise fail(503, "MinIO object storage is disabled")
    resource_type = str(payload.get("resource_type") or "")
    resource_id = str(payload.get("resource_id") or "")
    version_id = str(payload.get("version_id") or uuid.uuid4().hex)
    object_key = str(payload.get("object_key") or f"{resource_type}/{resource_id}/versions/{version_id}/resource")
    if not resource_type or not resource_id:
        raise fail(400, "resource_type and resource_id are required")
    if payload.get("method", "put").lower() == "get" and not has_resource_access(resource_type, resource_id, current_user, "read"):
        raise fail(403, "no permission to read this resource")
    try:
        storage = get_object_storage()
        url = storage.presigned_get(object_key) if payload.get("method", "put").lower() == "get" else storage.presigned_put(object_key)
        record = get_platform_store().create_storage_object({
            "resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
            "bucket": storage.bucket, "object_key": object_key, "file_name": payload.get("file_name"),
            "content_type": payload.get("content_type"), "created_by": current_user.get("id"),
        })
        return ok({"url": url, "method": payload.get("method", "put").lower(), "expires_seconds": 3600, "object": record})
    except ObjectStorageError as exc:
        raise fail(503, str(exc))


@router.get("/storage/resources/{resource_type}/{resource_id}")
async def storage_resource_objects(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access(resource_type, resource_id, current_user, "read"):
        raise fail(403, "no permission to read this resource")
    return ok(get_platform_store().storage_objects_for_resource(resource_type, resource_id))


@router.post("/storage/resources/{resource_type}/{resource_id}/prepare/{node_id}")
async def prepare_storage_resource(
    resource_type: str,
    resource_id: str,
    node_id: str,
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    if not has_resource_access(resource_type, resource_id, current_user, "execute"):
        raise fail(403, "no permission to execute this resource")
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    if not get_settings().minio_enabled:
        raise fail(503, "MinIO object storage is disabled")
    objects = store.storage_objects_for_resource(resource_type, resource_id)
    if not objects:
        raise fail(404, "resource has no MinIO objects")
    client = ComputeNodeClient(node["api_base_url"])
    prepared = []
    for obj in objects:
        url = get_object_storage().presigned_get(obj["object_key"])
        filename = Path(str(obj.get("file_name") or obj["object_key"])).name
        cache_job = store.create_storage_cache_job({"storage_object_id": obj["id"], "node_id": node_id, "direction": "download"})
        try:
            result = await client.prepare_cache({
                "resource_id": resource_id,
                "version_id": obj["version_id"],
                "download_url": url,
                "checksum_sha256": obj.get("checksum_sha256") or "",
                "byte_size": obj.get("byte_size") or 0,
                "relative_path": f"{resource_type}s/{resource_id}/{filename}",
            })
            store.update_storage_cache_job(cache_job["id"], {"status": "completed", "progress": 100, "local_path": result.get("local_path"), "completed_at": utcnow()})
        except Exception as exc:
            store.update_storage_cache_job(cache_job["id"], {"status": "failed", "error": str(exc), "completed_at": utcnow()})
            raise
        prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
    return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})


@router.get("/storage/cache/jobs/{node_id}")
async def storage_cache_jobs(node_id: str, limit: int = Query(default=100, ge=1, le=500), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return ok(get_platform_store().storage_cache_jobs_for_node(node_id, limit))


@router.post("/storage/resources/{resource_type}/{resource_id}/archive-node/{node_id}")
async def archive_node_files(
    resource_type: str,
    resource_id: str,
    node_id: str,
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    """Archive completed node files to MinIO without proxying file bytes through Backend."""
    if not has_resource_access(resource_type, resource_id, current_user, "execute"):
        raise fail(403, "no permission to archive this resource")
    store = get_platform_store()
    if not is_admin(current_user):
        model_id = str(payload.get("model_id") or "")
        dataset_id = str(payload.get("dataset_id") or "")
        if not model_id or not has_resource_access("model", model_id, current_user, "execute"):
            raise fail(403, "no permission to evaluate this model")
        if not dataset_id or not has_resource_access("dataset", dataset_id, current_user, "execute"):
            raise fail(403, "no permission to evaluate this dataset")
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    files = payload.get("files") or []
    if not isinstance(files, list) or not files:
        raise fail(400, "files is required")
    client = ComputeNodeClient(node["api_base_url"], timeout=900)
    archived = []
    for item in files:
        path = str(item.get("path") or "")
        name = Path(str(item.get("file_name") or Path(path).name)).name
        version_id = str(item.get("version_id") or uuid.uuid4().hex)
        object_key = str(item.get("object_key") or f"{resource_type}s/{resource_id}/versions/{version_id}/{name}")
        url = get_object_storage().presigned_put(object_key)
        result = await client.upload_file_to_url(path, url, object_key, str(item.get("content_type") or "application/octet-stream"))
        metadata = get_object_storage().stat(object_key)
        record = store.create_storage_object({
            "resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
            "bucket": get_object_storage().bucket, "object_key": object_key, "file_name": name,
            "content_type": item.get("content_type"), "checksum_sha256": result.get("checksum_sha256"),
            "byte_size": metadata.get("byte_size") or result.get("byte_size") or 0, "status": "available",
            "created_by": current_user.get("id"),
        })
        archived.append({"object": record, "node_id": node_id})
    return ok({"status": "available", "items": archived})


@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    if not is_admin(current_user) and not any(item["node_id"] == node_id for item in store.gpu_assignments_for_user(current_user["id"])):
        raise fail(403, "no permission to access this compute node")
    return ok(node)


@router.post("/compute/nodes")
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    try:
        return ok(get_platform_store().create_compute_node(payload))
    except KeyError as exc:
        raise fail(400, f"missing field: {exc}")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.put("/compute/nodes/{node_id}")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    try:
        return ok(get_platform_store().update_compute_node(node_id, payload))
    except KeyError:
        raise fail(404, "compute node not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.delete("/compute/nodes/{node_id}")
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    try:
        return ok(get_platform_store().delete_compute_node(node_id))
    except KeyError:
        raise fail(404, "compute node not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    client = ComputeNodeClient(node["api_base_url"])
    try:
        result = await client.test_connection()
        store.replace_node_gpus(node_id, result["gpus"])
        updated = store.update_compute_node_health(node_id, result["health"], True)
        return ok(
            {
                "node_id": node_id,
                "success": True,
                "latency_ms": result["latency_ms"],
                "gpu_count": len(result["gpus"]),
                "health": updated["health_detail"],
            }
        )
    except Exception as exc:  # noqa: BLE001 - return the connection error for node maintenance
        updated = store.update_compute_node_health(node_id, {}, False, str(exc))
        return ok(
            {
                "node_id": node_id,
                "success": False,
                "latency_ms": 0,
                "gpu_count": updated.get("gpu_count", 0),
                "error": str(exc),
                "health": updated["health_detail"],
            }
        )


@router.post("/compute/nodes/{node_id}/health-check")
async def health_check_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return await test_compute_node(node_id, current_user)


@router.post("/compute/nodes/{node_id}/enable")
async def enable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))


@router.post("/compute/nodes/{node_id}/disable")
async def disable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))


@router.post("/compute/nodes/{node_id}/drain")
async def drain_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))


@router.get("/compute/nodes/{node_id}/replicas")
async def compute_node_replicas(node_id: str) -> dict[str, Any]:
    return ok(get_platform_store().replicas(node_id))


@router.get("/compute/sync-jobs/{sync_id}")
async def compute_sync_job_detail(sync_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().sync_job(sync_id))
    except KeyError:
        raise fail(404, "sync job not found")


@router.get("/compute/nodes/{node_id}/replicas/drift")
async def compute_node_replica_drift(node_id: str) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    replicas = store.replicas(node_id)
    if not replicas:
        return ok({"node_id": node_id, "items": [], "drifted": 0})
    paths = [
        {
            "name": replica["id"],
            "path": replica["local_path"],
            "type": "any",
            "required": True,
        }
        for replica in replicas
    ]
    try:
        result = await ComputeNodeClient(node["api_base_url"]).check_paths(paths)
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"replica drift check failed: {exc}")
    check_map = {str(item.get("name")): item for item in result.get("items") or []}
    items = []
    for replica in replicas:
        check = check_map.get(replica["id"], {})
        updated = store.update_resource_replica_check(
            replica["id"],
            bool(check.get("ok")),
            int(check.get("byte_size") or replica.get("byte_size") or 0),
            "" if check.get("ok") else f"path not available: {replica['local_path']}",
        )
        items.append({**updated, "check": check})
    return ok({"node_id": node_id, "items": items, "drifted": len([item for item in items if item.get("sync_status") == "drifted"])})


async def _run_resource_replica_repair(sync_id: str, node_id: str, payload: dict[str, Any], replicas_to_repair: list[dict[str, Any]]) -> None:
    store = get_platform_store()
    store.update_sync_job(sync_id, "running", 5)
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        store.update_sync_job(sync_id, "failed", 100, completed=True)
        return
    client = ComputeNodeClient(node["api_base_url"])
    repaired = []
    failures = []
    total = max(len(replicas_to_repair), 1)
    for index, replica in enumerate(replicas_to_repair, start=1):
        replica_id = str(replica["id"])
        resource_type = str(replica.get("resource_type") or "")
        resource_id = str(replica.get("resource_id") or "")
        try:
            if resource_type == "dataset":
                files = store.training_dataset_files(resource_id)
                if not files:
                    raise RuntimeError(f"dataset has no uploaded file: {resource_id}")
                total_size = 0
                checksum = ""
                local_path = str(replica.get("local_path") or "")
                for item in files:
                    filename = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
                    result = await client.upload_file(
                        filename,
                        str(item.get("content") or "").encode("utf-8"),
                        f"datasets/{resource_id}/{filename}",
                        resource_type="dataset",
                        resource_id=resource_id,
                    )
                    total_size += int(result.get("byte_size") or 0)
                    checksum = str(result.get("checksum_sha256") or checksum)
                    local_path = str(result.get("local_path") or local_path)
                repaired.append(store.update_resource_replica_sync_result(replica_id, True, local_path, total_size, checksum))
                continue

            source_path = ""
            target_relative_path = ""
            if resource_type == "model":
                model = store.model(resource_id)
                source_path = str(model.get("path") or "")
                target_relative_path = f"models/{Path(source_path).name}" if source_path else ""
            elif resource_type in {"trained_model", "model_artifact"}:
                if resource_type == "trained_model":
                    artifacts = store.model_artifacts(resource_id)
                    artifact = next((item for item in artifacts if item.get("path")), None)
                else:
                    artifact = store.model_artifact(resource_id)
                source_path = str((artifact or {}).get("path") or replica.get("local_path") or "")
                target_relative_path = f"outputs/{Path(source_path).name}" if source_path else ""
            else:
                source_path = str(payload.get("source_path") or replica.get("source_path") or replica.get("local_path") or "")
                target_relative_path = str(payload.get("target_relative_path") or "")

            if not source_path:
                raise RuntimeError(f"authoritative source path not found for {resource_type}:{resource_id}")
            result = await client.import_local_file(
                {
                    "source_path": source_path,
                    "target_relative_path": target_relative_path,
                    "resource_type": resource_type,
                    "resource_id": resource_id,
                }
            )
            repaired.append(
                store.update_resource_replica_sync_result(
                    replica_id,
                    True,
                    str(result.get("local_path") or replica.get("local_path") or ""),
                    int(result.get("byte_size") or 0),
                    str(result.get("checksum_sha256") or ""),
                )
            )
        except Exception as exc:  # noqa: BLE001 - collect all replica repair failures
            error = str(exc)
            failures.append({"replica_id": replica_id, "resource_type": resource_type, "resource_id": resource_id, "error": error})
            repaired.append(store.update_resource_replica_sync_result(replica_id, False, None, int(replica.get("byte_size") or 0), "", error))
        progress = min(95, 5 + int(index / total * 90))
        store.update_sync_job(sync_id, "running", progress)
    store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)


@router.post("/compute/nodes/{node_id}/replicas/repair")
async def compute_node_replica_repair(
    node_id: str,
    background_tasks: BackgroundTasks,
    payload: dict[str, Any] | None = Body(default=None),
) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    payload = payload or {}
    replica_ids = payload.get("replica_ids") or [
        item["id"] for item in store.replicas(node_id) if item.get("sync_status") in {"drifted", "failed", "repair_pending"}
    ]
    updated = store.mark_resource_replica_repair_pending([str(item) for item in replica_ids])
    sync_id = store.create_sync_job(
        node_id,
        {
            "operation": "repair",
            "resources": [
                {
                    "resource_type": item.get("resource_type"),
                    "resource_id": item.get("resource_id"),
                    "replica_id": item.get("id"),
                    "target_path": item.get("local_path"),
                }
                for item in updated
            ],
        },
    )
    if not updated:
        store.update_sync_job(sync_id, "completed", 100, completed=True)
        return ok({"sync": store.sync_job(sync_id), "replicas": [], "failed": [], "async": False})
    background_tasks.add_task(_run_resource_replica_repair, sync_id, node_id, payload, updated)
    return ok({"sync": store.sync_job(sync_id), "replicas": updated, "failed": [], "async": True})


@router.get("/compute/nodes/{node_id}/engines")
async def compute_node_engines(node_id: str) -> dict[str, Any]:
    node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    health = node.get("health_detail") or {}
    live_error = ""
    try:
        health = await ComputeNodeClient(node["api_base_url"]).health()
    except Exception as exc:  # noqa: BLE001 - stored health is enough for offline node detail
        live_error = str(exc)
    capabilities = health.get("capabilities") or node.get("capabilities") or []
    return ok(
        {
            "node_id": node_id,
            "items": [
                {
                    "engine": "llama_factory",
                    "display_name": "LLaMA-Factory",
                    "status": "available" if "llama_factory" in capabilities else "unknown",
                    "version": health.get("llama_factory_version") or "",
                    "home": health.get("llama_factory_home") or "",
                    "home_exists": bool(health.get("llama_factory_home_exists")),
                    "capabilities": capabilities,
                    "execution_mode": health.get("execution_mode") or "",
                    "last_error": live_error,
                }
            ],
        }
    )


@router.get("/compute/gpus")
async def compute_gpus(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    gpus = store.gpus()
    if is_admin(current_user):
        return ok(gpus)
    assigned = {(item["node_id"], int(item["gpu_index"])) for item in store.gpu_assignments_for_user(current_user["id"])}
    return ok([gpu for gpu in gpus if (gpu.get("node_id"), int(gpu.get("id", gpu.get("gpu_index", -1)))) in assigned])


@router.get("/compute/queue")
async def compute_queue() -> dict[str, Any]:
    return ok(get_platform_store().queue())


@router.get("/compute/jobs/{job_id}")
async def compute_job_detail(job_id: str) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        job = await ComputeNodeClient(node["api_base_url"]).get_job(job_id)
        return ok(get_platform_store().sync_model_merge_job(job_id, job))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    return ok(await ComputeNodeClient(node["api_base_url"]).get_job(job_id))


@router.post("/compute/jobs/{job_id}/stop")
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
        raise fail(403, "no permission to stop this compute job")
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
        return ok(get_platform_store().sync_model_merge_job(job_id, job))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
    get_platform_store().apply_compute_job(task["id"], job)
    return ok(job)


@router.get("/compute/jobs/{job_id}/logs")
async def compute_job_logs(
    job_id: str,
    tail_lines: int | None = Query(default=200, ge=1, le=5000),
    offset: int | None = Query(default=None, ge=0),
    limit: int | None = Query(default=None, ge=1, le=5000),
) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))


@router.post("/compute/jobs/{job_id}/retry")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    payload = payload or {}
    task = _task_for_compute_job(job_id)
    if not task:
        raise fail(404, "compute job not found")
    return await retry_fine_tune(task["id"], payload, current_user)


@router.post("/compute/jobs/{job_id}/priority")
async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        raise fail(404, "compute job not found")
    priority = str(payload.get("priority") or "normal")
    return ok(get_platform_store().update_task_priority(task["id"], priority))


@router.post("/internal/compute-sync/jobs/poll")
async def poll_compute_jobs() -> dict[str, Any]:
    return ok(await poll_compute_jobs_once())


@router.post("/internal/compute-sync/resources")
async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    store = get_platform_store()
    node_id = payload.get("target_node_id") or payload.get("target_compute_node_id")
    if not node_id:
        raise fail(400, "target_node_id is required")
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    sync_id = store.create_sync_job(node_id, payload)
    replicas = []
    failures = []
    resources = payload.get("resources") or []
    for resource in resources:
        if not resource.get("source_path"):
            continue
        try:
            result = await ComputeNodeClient(node["api_base_url"]).import_local_file(
                {
                    "source_path": resource["source_path"],
                    "target_relative_path": resource.get("target_relative_path"),
                    "resource_type": resource.get("resource_type"),
                    "resource_id": resource.get("resource_id"),
                }
            )
            replicas.append(
                store.upsert_resource_replica(
                    node_id,
                    str(resource.get("resource_type") or "file"),
                    str(resource.get("resource_id") or result["id"]),
                    result["local_path"],
                )
            )
        except Exception as exc:  # noqa: BLE001 - collect per-resource failures
            failures.append({"resource_id": str(resource.get("resource_id")), "error": str(exc)})
    store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)
    return ok({"sync": store.sync_job(sync_id), "replicas": replicas, "failed": failures})


@router.get("/internal/compute-sync/resources/{sync_id}")
async def compute_sync_detail(sync_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().sync_job(sync_id))
    except KeyError:
        raise fail(404, "sync job not found")


@router.get("/training-log-files")
async def training_log_files() -> dict[str, Any]:
    return ok(get_platform_store().training_log_files())


@router.get("/training-log-content")
async def training_log_content(file: str = Query(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().training_log_content(file))
    except KeyError:
        raise fail(404, "training log not found")


@router.get("/log-files")
async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]:
    return ok(get_platform_store().log_files(date))


@router.get("/log-content")
async def log_content(file: str = Query(...)) -> dict[str, Any]:
    return ok(get_platform_store().log_content(file))


@router.post("/web-log")
async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"received": True, **payload})