from __future__ import annotations

import json
import asyncio
import hashlib
import re
import uuid
import time
from io import BytesIO
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from urllib.parse import quote
from zipfile import ZIP_DEFLATED, ZipFile

from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import Response, StreamingResponse

import httpx

from app.core.auth import bind_active_tenant, filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin, require_admin, resource_record
from app.core.config import get_settings
from app.core.audit import audit_log, AuditActions
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, utcnow
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import (
    _archive_node_directory,
    fetch_eval_progress_content,
    fetch_eval_result_content,
    poll_compute_jobs_once,
)
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
from app.modules.storage.policy import should_store_in_minio

router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
_DASHBOARD_CACHE_TTL = 5.0
_DASHBOARD_CACHE: dict[str, Any] = {}

_STORAGE_RESOURCE_ROOTS = {
    "dataset": "datasets",
    "model": "models",
    "trained_model": "trained_models",
    "eval": "evaluations",
    "fine-tune": "fine-tunes",
    "fine_tune_task": "fine-tunes",
    "compare": "comparisons",
    "inference": "comparisons",
    "data_process": "data-process",
    "data_convert": "data-convert",
}
_STORAGE_RESOURCE_TABLES = {
    "dataset": "datasets",
    "model": "models",
    "trained_model": "trained_models",
    "eval": "eval_tasks",
    "fine-tune": "fine_tune_tasks",
    "fine_tune_task": "fine_tune_tasks",
    "compare": "compare_tasks",
    "inference": "compare_tasks",
    "data_process": "data_process_tasks",
    "data_convert": "data_convert_tasks",
}
_STORAGE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
_STORAGE_CONTENT_TYPES = {
    "application/octet-stream",
    "application/json",
    "application/jsonl",
    "text/plain",
    "text/csv",
    "application/pdf",
    "application/zip",
    "application/gzip",
    "application/x-gzip",
}
_MODEL_SECRET_FIELDS = {"api_key", "secret_key", "password", "token", "access_token"}


def _public_model(model: dict[str, Any]) -> dict[str, Any]:
    """Return a model DTO that never exposes provider credentials."""
    result = {
        key: value for key, value in dict(model).items() if key not in _MODEL_SECRET_FIELDS
    }
    result["api_key_configured"] = bool(model.get("api_key"))
    return result


def _public_models(models: list[dict[str, Any]]) -> list[dict[str, Any]]:
    return [_public_model(model) for model in models]


def _storage_resource_row(store: Any, resource_type: str, resource_id: str) -> dict[str, Any] | None:
    table = _STORAGE_RESOURCE_TABLES.get(resource_type)
    if not table:
        return None
    with store.connect() as conn:
        row = conn.execute(f"SELECT * FROM {table} WHERE id=?", (resource_id,)).fetchone()
    if not row:
        return None
    item = dict(row)
    if item.get("deleted_at"):
        return None
    return item


def _validate_storage_key(
    resource_type: str,
    resource_id: str,
    version_id: str,
    object_key: str | None,
    file_name: str | None,
) -> str:
    root = _STORAGE_RESOURCE_ROOTS[resource_type]
    prefix = f"{root}/{resource_id}/versions/{version_id}/"
    requested = str(object_key or "").replace("\\", "/").strip()
    if not requested:
        safe_name = Path(str(file_name or "resource")).name or "resource"
        requested = f"{prefix}{safe_name}"
    if (
        not requested
        or requested.startswith("/")
        or "\x00" in requested
        or any(part in {"", ".", ".."} for part in requested.split("/"))
        or not requested.startswith(prefix)
        or len(requested) > 1024
    ):
        raise ValueError("object_key must stay inside the resource version prefix")
    return requested


def _capture_resource_versions(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
    """Freeze the resource versions used by a task at task-creation time."""
    snapshot: dict[str, Any] = {}
    dataset_id = str(payload.get("dataset_id") or payload.get("train_dataset_id") or "")
    if dataset_id:
        try:
            dataset = store.dataset(dataset_id)
            snapshot["dataset"] = {
                "id": dataset_id,
                "files": [
                    {
                        "id": item.get("id"),
                        "active_version_id": item.get("active_version_id") or item.get("current_version_id") or item.get("id"),
                        "name": item.get("name"),
                    }
                    for item in dataset.get("files") or []
                ],
            }
        except KeyError:
            pass

    raw_model_ids = payload.get("model_ids") or payload.get("models") or [
        payload.get("model_id") or payload.get("base_model_id") or payload.get("base_model"),
        payload.get("trained_model_id"),
    ]
    model_ids = [
        str(item.get("model_id") or item.get("id") or item.get("name")) if isinstance(item, dict) else str(item)
        for item in raw_model_ids
        if item
    ]
    model_snapshots: list[dict[str, Any]] = []
    for model_id in dict.fromkeys(model_ids):
        try:
            model = store.model(model_id)
        except KeyError:
            model = next((item for item in store.trained_models() if str(item.get("id")) == model_id or str(item.get("name")) == model_id), None)
        if not model:
            continue
        objects = store.storage_objects_for_resource(
            "trained_model" if str(model.get("id") or "").startswith("tm_") else "model",
            str(model.get("id") or model_id),
        )
        model_snapshots.append(
            {
                "id": str(model.get("id") or model_id),
                "name": model.get("name"),
                "storage_versions": sorted({str(item.get("version_id")) for item in objects if item.get("version_id")}),
            }
        )
    if model_snapshots:
        snapshot["models"] = model_snapshots
    return snapshot


def _cached_dashboard(key: str) -> dict[str, Any] | None:
    item = _DASHBOARD_CACHE.get(key)
    if not item or time.monotonic() - item["created_at"] >= _DASHBOARD_CACHE_TTL:
        return None
    return item["value"]


def _store_dashboard_cache(key: str, value: dict[str, Any]) -> dict[str, Any]:
    _DASHBOARD_CACHE[key] = {"created_at": time.monotonic(), "value": value}
    return value


def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
    return {"code": 0, "message": message, "data": data}


def _select_first_online_node(store: Any) -> dict[str, Any] | None:
    """Select the first online compute node for inference."""
    nodes = store.compute_nodes()
    for node in nodes:
        if node.get("enabled") and node.get("scheduler_status") == "online":
            return node
    return None


def _normalize_gpu_indices(payload: dict[str, Any], *, allow_primary: bool = True) -> list[int]:
    """Normalize all frontend GPU selection shapes to sorted integer indexes."""
    raw = payload.get("gpu_indices")
    if raw is None:
        raw = payload.get("gpus")
    if raw is None and allow_primary and payload.get("gpu_id") is not None:
        raw = [payload.get("gpu_id")]
    if raw is None or raw == "":
        return []
    if isinstance(raw, str):
        raw = [item.strip() for item in raw.split(",") if item.strip()]
    if not isinstance(raw, (list, tuple, set)):
        raw = [raw]
    result: set[int] = set()
    for item in raw:
        if isinstance(item, str) and ":" in item:
            item = item.rsplit(":", 1)[-1]
        try:
            index = int(item)
        except (TypeError, ValueError) as exc:
            raise ValueError(f"invalid GPU index: {item}") from exc
        if index < 0:
            raise ValueError("GPU index must be non-negative")
        result.add(index)
    return sorted(result)


async def _wait_for_object_storage() -> None:
    """Wait for MinIO before starting a resource task."""
    settings = get_settings()
    deadline = datetime.now(timezone.utc).timestamp() + max(0, settings.storage_wait_seconds)
    last_error = "MinIO unavailable"
    while True:
        try:
            get_object_storage().ensure_bucket()
            return
        except Exception as exc:  # noqa: BLE001 - retry until the configured deadline
            last_error = str(exc)
        if datetime.now(timezone.utc).timestamp() >= deadline:
            raise RuntimeError(f"MinIO unavailable after {settings.storage_wait_seconds}s: {last_error}")
        await asyncio.sleep(max(1, settings.storage_check_interval_seconds))


def _store_json_snapshot(
    store: Any,
    resource_type: str,
    resource_id: str,
    version_id: str,
    object_key: str,
    payload: dict[str, Any],
    created_by: str | None = None,
) -> dict[str, Any]:
    """Persist non-secret task/model parameters as an auditable MinIO snapshot."""
    sanitized = {
        key: value
        for key, value in payload.items()
        if key not in {"api_key", "secret_key", "password", "token", "access_token"}
    }
    raw = json.dumps(sanitized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
    # Task/evaluation payloads are already persisted in PostgreSQL. Avoid an
    # extra MinIO round trip for small non-secret parameter snapshots.
    if not should_store_in_minio(len(raw), content_type="application/json", file_format="json"):
        return {}
    uploaded = get_object_storage().put_bytes(object_key, raw, "application/json")
    return store.create_storage_object({
        "resource_type": resource_type,
        "resource_id": resource_id,
        "version_id": version_id,
        "bucket": uploaded["bucket"],
        "object_key": object_key,
        "file_name": Path(object_key).name,
        "content_type": "application/json",
        "byte_size": len(raw),
        "checksum_sha256": hashlib.sha256(raw).hexdigest(),
        "status": "available",
        "created_by": created_by,
    })


def _dataset_file_bytes(store: Any, file_id: str) -> bytes:
    """Return the canonical dataset bytes, lazily indexing legacy DB content."""
    row = store.dataset_file(file_id)
    if not get_settings().minio_enabled:
        return str(row.get("content") or "").encode("utf-8")

    storage_object = None
    object_id = str(row.get("storage_object_id") or "")
    if object_id:
        try:
            storage_object = store.storage_object(object_id)
        except KeyError:
            storage_object = None
    if storage_object and storage_object.get("status") == "available":
        try:
            return get_object_storage().get_bytes(storage_object["object_key"])
        except Exception as exc:  # noqa: BLE001 - expose storage outage to callers
            raise RuntimeError(f"dataset object is unavailable in MinIO: {exc}") from exc

    # Compatibility migration for files created before MinIO was enabled.
    raw = str(row.get("content") or "").encode("utf-8")
    if not raw:
        raise RuntimeError(f"dataset file has no MinIO object or legacy content: {file_id}")
    # Small files intentionally remain database-backed. They can still be
    # copied to a compute node directly when a task needs them.
    if not should_store_in_minio(len(raw), file_format=row.get("file_format")):
        return raw
    object_key = (
        f"datasets/{row['dataset_id']}/versions/"
        f"{row.get('active_version_id') or row['id']}/{Path(str(row.get('name') or row['id'])).name}"
    )
    uploaded = get_object_storage().put_bytes(object_key, raw, "application/octet-stream")
    created = store.create_storage_object({
        "resource_type": "dataset",
        "resource_id": str(row["dataset_id"]),
        "version_id": str(row.get("active_version_id") or row["id"]),
        "bucket": uploaded["bucket"],
        "object_key": object_key,
        "file_name": row.get("name"),
        "content_type": "application/octet-stream",
        "byte_size": len(raw),
        "checksum_sha256": hashlib.sha256(raw).hexdigest(),
        "status": "available",
    })
    store.link_dataset_file_storage_object(str(row["id"]), created["id"])
    return raw


def _dataset_version_bytes(store: Any, file_id: str, version_id: str) -> bytes:
    row = store.dataset_file(file_id)
    try:
        version = next(item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id)
    except StopIteration as exc:
        raise KeyError(version_id) from exc
    object_id = str(version.get("storage_object_id") or "")
    if get_settings().minio_enabled and object_id:
        try:
            obj = store.storage_object(object_id)
            return get_object_storage().get_bytes(obj["object_key"])
        except KeyError:
            pass
    return _dataset_file_bytes(store, file_id)


def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
    """Select the compute node for an eval job.

    被评测模型是节点相关的（训练/合并产物只存在于对应算力节点），因此优先使用
    页面选择的节点或模型所在节点；若该节点不可用则明确失败，绝不派发到其它
    可能没有模型路径的节点（多算力节点场景下这是评测失败的主因）。
    """
    if preferred_node_id:
        node = next((n for n in store.compute_nodes() if n.get("id") == preferred_node_id), None)
        if node:
            if node.get("enabled") and node.get("scheduler_status") == "online":
                return node
            return None
    return _select_first_online_node(store)


def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) -> list[dict[str, Any]]:
    nodes = [node for node in store.compute_nodes() if node.get("enabled") and node.get("scheduler_status") == "online"]
    if not preferred_node_id:
        return nodes
    preferred = [node for node in nodes if node.get("id") == preferred_node_id]
    others = [node for node in nodes if node.get("id") != preferred_node_id]
    return preferred + others


async def _archive_model_from_source_node(
    store: Any,
    model: dict[str, Any],
    preferred_node_id: str | None = None,
) -> list[dict[str, Any]]:
    """Archive a registered base model from any reachable node into MinIO.

    A model path is a legacy compute-host path, not ownership metadata. The
    source node is discovered once, then every consumer uses the MinIO object
    manifest and its own local cache.
    """
    model_id = str(model.get("id") or "")
    source_path = str(model.get("path") or "")
    if not model_id or not source_path:
        return []
    existing = store.storage_objects_for_resource("model", model_id)
    if existing:
        return existing
    if not get_settings().minio_enabled:
        return []
    nodes = _candidate_online_nodes(store, preferred_node_id)
    last_error = "no reachable source compute node"
    for source_node in nodes:
        client = ComputeNodeClient(source_node["api_base_url"], timeout=30)
        try:
            check = await client.check_paths([{"path": source_path, "type": "dir", "required": True}])
            if not check.get("valid") or not all(item.get("ok") for item in check.get("items") or []):
                continue
            version_id = str(model.get("storage_version_id") or "initial")
            store.update_model_storage(model_id, "archiving", "", version_id)
            archived = await _archive_node_directory(
                store,
                ComputeNodeClient(source_node["api_base_url"], timeout=900),
                source_node,
                source_path,
                "model",
                model_id,
                version_id,
                f"models/{model_id}",
            )
            if archived:
                store.update_model_storage(model_id, "available", "", version_id)
                return store.storage_objects_for_resource("model", model_id)
            last_error = "model directory contains no files"
        except Exception as exc:  # noqa: BLE001 - persist status and try another node
            last_error = str(exc)
            continue
    try:
        store.update_model_storage(model_id, "failed", last_error)
    except Exception:
        pass
    return []


async def _archive_model_in_background(model_id: str) -> None:
    """Best-effort asynchronous import for newly registered local models."""
    store = get_platform_store()
    try:
        model = store.model(model_id)
        await _archive_model_from_source_node(store, model)
    except Exception as exc:  # noqa: BLE001 - status is persisted by helper when possible
        try:
            store.update_model_storage(model_id, "failed", str(exc))
        except Exception:
            pass


async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: str, node: dict[str, Any]) -> str | None:
    """Prepare MinIO resource files on a node and return the local directory."""
    if not get_settings().minio_enabled or not resource_id:
        return None
    objects = store.storage_objects_for_resource(resource_type, resource_id)
    if not objects and resource_type in {"model", "trained_model"}:
        resource = None
        if resource_type == "model":
            try:
                resource = store.model(resource_id)
            except KeyError:
                try:
                    resource = store.model_by_name(resource_id)
                except KeyError:
                    resource = next((item for item in store.models() if item.get("path") == resource_id), None)
        else:
            resource = next(
                (item for item in store.trained_models() if item.get("id") == resource_id or item.get("name") == resource_id),
                None,
            )
        source_path = str(
            (resource or {}).get("path")
            or (resource or {}).get("merged_path")
            or (resource or {}).get("artifact_dir")
            or ""
        )
        resolved_id = str((resource or {}).get("id") or resource_id)
        if source_path and resolved_id and resource_type == "model":
            objects = await _archive_model_from_source_node(store, resource or {"id": resolved_id, "path": source_path}, node.get("id"))
        elif source_path and resolved_id:
            client = ComputeNodeClient(node["api_base_url"], timeout=900)
            await _archive_node_directory(
                store,
                client,
                node,
                source_path,
                resource_type,
                resolved_id,
                "legacy-import",
                f"models/{resolved_id}" if resource_type == "model" else f"trained_models/{resolved_id}",
            )
            resource_id = resolved_id
            objects = store.storage_objects_for_resource(resource_type, resource_id)
    if not objects:
        return None
    client = ComputeNodeClient(node["api_base_url"], timeout=900)
    root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
    for obj in objects:
        object_key = str(obj["object_key"])
        marker = f"{root_name}/{resource_id}/versions/"
        relative_name = Path(str(obj.get("file_name") or object_key)).name
        if marker in object_key:
            suffix = object_key.split(marker, 1)[1]
            if "/" in suffix:
                suffix = suffix.split("/", 1)[1]
            if suffix:
                relative_name = suffix
        result = await client.prepare_cache({
            "resource_id": resource_id,
            "version_id": obj["version_id"],
            "download_url": get_object_storage().presigned_get(object_key),
            "checksum_sha256": obj.get("checksum_sha256") or "",
            "byte_size": obj.get("byte_size") or 0,
            "relative_path": f"{root_name}/{resource_id}/{relative_name}",
        })
        store.upsert_resource_replica(
            node["id"],
            resource_type,
            resource_id,
            str(result.get("local_path") or f"/data/yg-ft/{root_name}/{resource_id}"),
            version_id=str(obj.get("version_id") or ""),
            storage_object_id=str(obj.get("id") or ""),
            status="available",
            sync_status="synced",
        )
    return f"/data/yg-ft/{root_name}/{resource_id}"


def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
    """Convert frontend inference payload to compute API messages format.

    Accepts both:
      - OpenAI-style: {messages: [{role, content}, ...], temperature, ...}
      - Frontend-style: {user_question, system_prompt, temperature, ...}
    """
    if payload.get("messages"):
        messages = payload["messages"]
        # messages already in OpenAI format; pass through with optional system prompt
        if payload.get("system_prompt") and not any(m.get("role") == "system" for m in messages):
            messages = [{"role": "system", "content": payload["system_prompt"]}] + list(messages)
    else:
        messages = []
        if payload.get("system_prompt"):
            messages.append({"role": "system", "content": payload["system_prompt"]})
        question = payload.get("user_question") or payload.get("question") or ""
        if question:
            messages.append({"role": "user", "content": question})
    return {
        "messages": messages,
        "temperature": float(payload.get("temperature", 0.7)),
        "top_p": float(payload.get("top_p", 0.95)),
        "max_new_tokens": int(payload.get("max_tokens", 2048)),
        "do_sample": bool(payload.get("do_sample", True)),
    }


def _node_for_inference_payload(store: Any, payload: dict[str, Any]) -> dict[str, Any] | None:
    node_id = payload.get("node_id") or payload.get("compute_node_id")
    task_id = payload.get("task_id") or payload.get("compare_task_id")
    if task_id and not node_id:
        try:
            task = store.compare_task(str(task_id))
            load_status = task.get("load_status") or {}
            if isinstance(load_status, str):
                load_status = json.loads(load_status)
            loaded_models = load_status.get("loaded_models") or []
            ready_model = next((item for item in loaded_models if item.get("status") in {"ready", "running"} and item.get("node_id")), None)
            if ready_model:
                node_id = ready_model.get("node_id")
        except Exception:
            node_id = None
    if node_id:
        return next((node for node in store.compute_nodes() if node.get("id") == node_id), None)
    return _select_first_online_node(store)


async def _stream_chat_proxy(payload: dict[str, Any]) -> StreamingResponse:
    """Common SSE streaming proxy: convert payload → forward to compute node → stream back."""
    store = get_platform_store()
    # 任务仍在加载中时，直接返回明确的加载中提示，避免转发到尚未就绪的节点
    task_id = payload.get("task_id") or payload.get("compare_task_id")
    if task_id:
        try:
            task = store.compare_task(str(task_id))
            load_status = task.get("load_status") or {}
            if isinstance(load_status, str):
                load_status = json.loads(load_status)
            items = load_status.get("loaded_models") or []
            if items and not any(item.get("status") in {"ready", "running"} for item in items):
                if any(item.get("status") == "starting" for item in items):
                    return StreamingResponse(
                        iter(['data: {"error": "模型加载中，请稍候再试"}\n\n']),
                        media_type="text/event-stream",
                    )
        except Exception:  # noqa: BLE001 - fall through to normal routing on lookup errors
            pass
    node = _node_for_inference_payload(store, payload)
    if not node:
        return StreamingResponse(
            iter(['data: {"error": "no online compute node available for inference"}\n\n']),
            media_type="text/event-stream",
        )
    client = ComputeNodeClient(node["api_base_url"])
    compute_payload = _build_messages_payload(payload)

    async def stream_proxy():
        async with httpx.AsyncClient(timeout=300) as http:
            url = f"{node['api_base_url'].rstrip('/')}{client.route_prefix}/inference/chat/stream"
            try:
                async with http.stream("POST", url, json=compute_payload, headers=client.headers()) as resp:
                    if resp.status_code >= 400:
                        yield f'data: {{"error": "compute node returned {resp.status_code}"}}\n\n'.encode()
                        return
                    async for chunk in resp.aiter_bytes():
                        yield chunk
            except Exception as exc:
                yield f'data: {{"error": "stream proxy failed: {exc}"}}\n\n'.encode()

    return StreamingResponse(stream_proxy(), media_type="text/event-stream")


def fail(status_code: int, message: str) -> HTTPException:
    return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})


def _require_approval_or_admin(
    resource_type: str,
    resource_id: str,
    current_user: dict[str, Any],
    action_desc: str = "",
    action: str = "resource.change",
) -> dict[str, Any] | None:
    """
    高风险操作审批旁路：
    - admin 用户直接放行（返回 None）
    - 资源创建者（Owner）直接放行（返回 None）
    - 其他普通用户创建审批实例，返回审批待定响应（code=202，非 None）
      code=202 使前端响应拦截器走业务错误分支，弹提示并 reject，
      避免前端误认为删除成功。
    """
    if is_admin(current_user):
        return None
    # 资源创建者直接放行，无需审批
    if _check_owner(resource_type, resource_id, current_user.get("id")):
        return None
    store = get_platform_store()
    if store.consume_approved_approval(
        resource_type,
        resource_id,
        str(current_user.get("id") or ""),
        action,
    ):
        return None
    instance = store.create_approval_instance({
        "resource_type": resource_type,
        "resource_id": resource_id,
        "applicant_id": current_user.get("id"),
        "template_id": None,
        "action": action,
        "tenant_id": current_user.get("tenant_id") or "default",
        "reason": action_desc,
    })
    return {
        "code": 202,
        "message": f"操作已提交审批，等待管理员批准：{action_desc}",
        "data": {"approval_required": True, "approval_id": instance["id"]},
    }


def _check_owner(resource_type: str, resource_id: str, user_id: str | None) -> bool:
    """直接查数据库判断 user_id 是否为资源的 created_by。"""
    from app.core.auth import OWNER_TABLES
    table_info = OWNER_TABLES.get(resource_type)
    if not table_info:
        return False
    table, column = table_info
    store = get_platform_store()
    with store.connect() as conn:
        row = conn.execute(f"SELECT {column} FROM {table} WHERE id=?", (resource_id,)).fetchone()
    if not row:
        return False
    owner = row[column]
    if column == "payload":
        try:
            import json
            owner = json.loads(owner or "{}").get("created_by")
        except (TypeError, ValueError):
            owner = None
    return owner == user_id


def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
    return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)


def _task_for_compute_job(job_id: str) -> dict[str, Any] | None:
    return next((task for task in get_platform_store().tasks() if task.get("compute_job_id") == job_id), None)


def _node_for_compute_job_record(job_id: str) -> dict[str, Any] | None:
    store = get_platform_store()
    try:
        record = store.compute_job(job_id)
    except KeyError:
        return None
    return next((node for node in store.compute_nodes() if node["id"] == record.get("node_id")), None)


def _training_diagnostics(errors: list[str], warnings: list[str] | None = None, log_text: str = "") -> list[dict[str, str]]:
    source_items = [*errors, *(warnings or [])]
    if log_text:
        source_items.append(log_text)
    text = "\n".join(source_items).lower()
    diagnostics: list[dict[str, str]] = []
    rules = [
        (
            ["api 模型", "api模型", "api model"],
            "API 模型不能用于本地训练",
            "当前选择的基座模型为 API 类型，LLaMA-Factory 需要本地可访问的模型路径。请在模型管理中创建或选择模型来源为「本地」且配置了算力节点路径的模型。",
        ),
        (
            ["未配置算力节点", "未配置.*路径", "模型.*路径"],
            "模型缺少算力节点路径",
            "请在模型管理中编辑该模型，设置模型路径为算力节点可访问的本地目录。",
        ),
        (
            ["不支持本地训练", "not trainable"],
            "模型不可用于训练",
            "当前选择的模型不支持作为 LLaMA-Factory 训练基座。请确认模型来源为本地、路径已配置且模型目录在算力节点上存在。",
        ),
        (
            ["dataset columns missing", "keyerror", "history", "instruction", "input", "output", "messages"],
            "训练数据字段不匹配",
            "请检查所选数据集格式是否与训练模板一致。Alpaca 格式通常需要 instruction/input/output；ShareGPT 格式通常需要 messages。",
        ),
        (
            ["dataset file not found", "dataset_dir", "no uploaded file"],
            "训练数据文件不可用",
            "请确认数据集已上传文件，并且应用服务可以将数据同步到目标算力节点的数据目录。",
        ),
        (
            ["model_name_or_path path not available", "base_model", "model path", "no such file"],
            "基座模型路径不可用",
            "请在模型管理中检查本地模型路径，确保该路径在算力服务器或 Compute 容器挂载目录内真实存在。",
        ),
        (
            ["cuda out of memory", "outofmemoryerror", "显存", "memory"],
            "GPU 显存不足",
            "请降低 batch_size、cutoff_len、LoRA rank，启用 4bit 量化，或选择更高显存的算力节点。",
        ),
        (
            ["training command not found", "llamafactory-cli"],
            "训练框架命令不可用",
            "请检查 Compute 镜像是否包含 LLaMA-Factory，或确认 llamafactory-cli 已在容器 PATH 中。",
        ),
        (
            ["llama_factory_home not found"],
            "LLaMA-Factory 目录不可用",
            "请检查 Compute 服务的 LLAMA_FACTORY_HOME 配置和宿主机挂载路径。",
        ),
        (
            ["no available compute node", "not schedulable", "disabled", "capacity full"],
            "暂无可调度算力节点",
            "请检查算力节点是否启用、状态是否在线、并行任务数是否已满，或手动调整节点权重/标签。",
        ),
    ]
    for keywords, title, suggestion in rules:
        if any(keyword in text for keyword in keywords):
            diagnostics.append({"level": "error", "title": title, "suggestion": suggestion})
    if not diagnostics and (errors or log_text):
        diagnostics.append(
            {
                "level": "error",
                "title": "训练任务异常",
                "suggestion": "请查看预检错误和训练日志原文，优先确认模型路径、数据集格式、GPU 显存和 LLaMA-Factory 参数。",
            }
        )
    return diagnostics


async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
    task_id = str(payload.get("task_id") or payload.get("id") or "")
    if task_id and get_settings().compute_mode != "simulator":
        try:
            preflight = await _fine_tune_preflight(store, task_id, payload, validate=True, sync_resources=True)
        except Exception as exc:  # noqa: BLE001 - task has not entered running state yet
            raise RuntimeError(f"preflight failed: {exc}") from exc
        if not preflight["valid"]:
            errors = "; ".join(preflight.get("errors") or ["preflight failed"])
            raise RuntimeError(f"preflight failed: {errors}")
        prepared_job_payload = preflight.get("job_payload") or {}
        payload = {
            **payload,
            "compute_node_id": preflight["node"]["id"],
            "prepared_base_model_path": prepared_job_payload.get("model_name_or_path") or payload.get("prepared_base_model_path"),
        }
    task = store.start_task(payload)
    if get_settings().minio_enabled:
        _store_json_snapshot(
            store,
            "fine_tune",
            str(task["id"]),
            str(task["id"]),
            f"training/{task['id']}/versions/{task['id']}/training-config.json",
            task,
            task.get("created_by"),
        )
    if get_settings().compute_mode == "simulator":
        return task
    node, job_payload = store.build_compute_job_payload(task["id"])
    job = await ComputeNodeClient(node["api_base_url"]).create_job(job_payload)
    return store.apply_compute_job(task["id"], job)


async def _fine_tune_preflight(
    store: Any,
    task_id: str,
    payload: dict[str, Any] | None = None,
    validate: bool = True,
    sync_resources: bool = False,
) -> dict[str, Any]:
    node, job_payload = store.prepare_compute_job_payload(task_id, payload or {})
    return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=sync_resources, store=store)


async def _fine_tune_preflight_payload(
    store: Any,
    payload: dict[str, Any],
    validate: bool = True,
    sync_resources: bool = True,
) -> dict[str, Any]:
    node, job_payload = store.prepare_compute_job_payload_from_payload(payload)
    return await _fine_tune_preflight_with_job_payload(
        node,
        job_payload,
        validate=validate,
        sync_resources=sync_resources,
        store=store,
    )


async def _fine_tune_preflight_with_job_payload(
    node: dict[str, Any],
    job_payload: dict[str, Any],
    validate: bool,
    sync_resources: bool,
    store: Any,
) -> dict[str, Any]:
    sync_results: list[dict[str, Any]] = []
    sync_errors: list[str] = []
    if sync_resources and get_settings().compute_mode != "simulator":
        try:
            sync_results = await _sync_training_dataset_to_compute_node(
                store,
                node,
                str(job_payload.get("train_dataset_id") or ""),
            )
        except Exception as exc:  # noqa: BLE001 - return as preflight error for page visibility
            sync_errors.append(str(exc))
    if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
        try:
            await _wait_for_object_storage()
            base_model_path = str(job_payload.get("model_name_or_path") or job_payload.get("base_model") or "")
            base_model_id = str(job_payload.get("base_model_id") or job_payload.get("model_id") or "")
            base_model = None
            if base_model_id:
                try:
                    base_model = store.model(base_model_id)
                except KeyError:
                    base_model = None
            if base_model is None:
                base_model = next((item for item in store.models() if item.get("path") == base_model_path), None)
            if base_model:
                prepared_model = await _prepare_resource_on_node(store, "model", str(base_model["id"]), node)
                if prepared_model:
                    job_payload = {**job_payload, "base_model": prepared_model, "model_name_or_path": prepared_model}
        except Exception as exc:  # noqa: BLE001 - preflight exposes node storage failure
            sync_errors.append(f"shared storage health check failed: {exc}")
    if get_settings().compute_mode == "simulator":
        preview = {
            "valid": True,
            "errors": [],
            "warnings": ["compute_mode=simulator skips remote compute validation"],
            "engine": job_payload.get("engine") or job_payload.get("training_engine") or "llama_factory",
            "command": [],
            "command_text": "",
            "work_dir": "",
            "env": {},
            "path_checks": [],
        }
    else:
        client = ComputeNodeClient(node["api_base_url"])
        preview = await (client.validate_job(job_payload) if validate else client.preview_job(job_payload))
    errors = list(preview.get("errors") or [])
    errors.extend(sync_errors)
    warnings = list(preview.get("warnings") or [])
    if not node.get("enabled"):
        errors.append(f"compute node disabled: {node.get('code')}")
    if node.get("scheduler_status") not in {"online", "draining"}:
        errors.append(f"compute node not schedulable: {node.get('code')} status={node.get('scheduler_status')}")
    return {
        "valid": bool(preview.get("valid", not errors)) and not errors,
        "errors": errors,
        "warnings": warnings,
        "diagnostics": _training_diagnostics(errors, warnings),
        "node": {
            "id": node.get("id"),
            "code": node.get("code"),
            "name": node.get("name"),
            "api_base_url": node.get("api_base_url"),
            "scheduler_status": node.get("scheduler_status"),
            "gpu_count": node.get("gpu_count"),
        },
        "job_payload": job_payload,
        "preview": preview,
        "sync_results": sync_results,
    }


@router.post("/login")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGIN, target_type="user", target_name_param="username")
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
    store = get_platform_store()
    ip = request.client.host if request and request.client else "unknown"
    now = time.time()
    recent = [stamp for stamp in _LOGIN_FAILURES.get(ip, []) if now - stamp < 300]
    if len(recent) >= 5:
        raise fail(429, "too many login attempts, retry later")
    user = store.login(payload.get("username", ""), payload.get("password", ""))
    if not user:
        _LOGIN_FAILURES[ip] = [*recent, now]
        raise fail(401, "invalid username or password")
    _LOGIN_FAILURES.pop(ip, None)
    sess = store.create_session(user["id"], ip=ip)
    return ok({"token": f"platform-token-{user['id']}.{sess['session_id']}", "user": user, "session_id": sess["session_id"]})


@router.post("/logout")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGOUT, target_type="user")
async def logout(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    session_id = payload.get("session_id", "")
    if session_id:
        try:
            store.finish_session(session_id, current_user.get("id"))
        except TypeError:
            store.finish_session(session_id)
    return ok(None)


@router.get("/me")
async def me(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Return the current session user after the normal token/session checks."""
    return ok(current_user)


@router.get("/dashboard/overview")
async def dashboard_overview(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    cache_key = f"overview:{current_user.get('id') or current_user.get('role')}"
    cached = _cached_dashboard(cache_key)
    if cached is not None:
        return cached
    store = get_platform_store()
    tasks = store.tasks()
    datasets = store.datasets(None if is_admin(current_user) else current_user.get("tenant_id") or "default")
    compares = store.compare_tasks()
    if not is_admin(current_user):
        task_ids = filter_accessible_resource_ids_batch("fine-tune", [str(item.get("id") or "") for item in tasks], current_user)
        dataset_ids = filter_accessible_resource_ids_batch("dataset", [str(item.get("id") or "") for item in datasets], current_user)
        compare_ids = filter_accessible_resource_ids_batch("compare", [str(item.get("id") or "") for item in compares], current_user)
        tasks = [item for item in tasks if str(item.get("id") or "") in task_ids]
        datasets = [item for item in datasets if str(item.get("id") or "") in dataset_ids]
        compares = [item for item in compares if str(item.get("id") or "") in compare_ids]
        nodes = [item for item in store.compute_nodes() if any(gpu.get("node_id") == item.get("id") for gpu in store.gpu_assignments_for_user(current_user.get("id")))]
    else:
        nodes = store.compute_nodes()
    return _store_dashboard_cache(cache_key, ok(
        {
            "models": len(store.models()),
            "datasets": len(datasets),
            "fine_tune_tasks": len(tasks),
            "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]),
            "compute_nodes": len(nodes),
            "gpus": len(store.gpus()),
        }
    ))


@router.get("/dashboard/stats")
async def dashboard_stats(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    cache_key = f"stats:{current_user.get('id') or current_user.get('role')}"
    cached = _cached_dashboard(cache_key)
    if cached is not None:
        return cached
    """看板聚合数据：基于平台真实数据；缺项做合理近似。"""
    store = get_platform_store()
    tasks = store.tasks()
    users = store.users()
    nodes = store.compute_nodes()
    datasets = store.datasets()
    compares = store.compare_tasks()
    eval_tasks = store.eval_tasks()
    if not is_admin(current_user):
        task_ids = filter_accessible_resource_ids_batch("fine-tune", [str(item.get("id") or "") for item in tasks], current_user)
        dataset_ids = filter_accessible_resource_ids_batch("dataset", [str(item.get("id") or "") for item in datasets], current_user)
        eval_ids = filter_accessible_resource_ids_batch("eval", [str(item.get("id") or "") for item in eval_tasks], current_user)
        compare_ids = filter_accessible_resource_ids_batch("compare", [str(item.get("id") or "") for item in compares], current_user)
        tasks = [item for item in tasks if str(item.get("id") or "") in task_ids]
        datasets = [item for item in datasets if str(item.get("id") or "") in dataset_ids]
        eval_tasks = [item for item in eval_tasks if str(item.get("id") or "") in eval_ids]
        compares = [item for item in compares if str(item.get("id") or "") in compare_ids]
        users = [user for user in users if user.get("id") == current_user.get("id")]
        nodes = [item for item in nodes if any(gpu.get("node_id") == item.get("id") for gpu in store.gpu_assignments_for_user(current_user.get("id")))]
    # 数据处理任务总数（来自 data_process 模块）
    try:
        from app.modules.data_process.store import get_data_process_store

        dp_store = get_data_process_store()
        dp_result = dp_store.list_tasks(page=1, page_size=1)
        dp_count = int(dp_result.get("total", 0))
    except Exception:
        dp_count = 0

    running_statuses = {"syncing", "queued", "running"}
    running_ft = [t for t in tasks if t.get("status") in running_statuses]
    online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"]
    # 评测中运行的任务数
    eval_running = 0
    try:
        eval_tasks = eval_tasks if not is_admin(current_user) else store.eval_tasks()
        eval_running = len([e for e in eval_tasks if e.get("status") in running_statuses])
    except Exception:
        eval_running = 0

    # 近 7 天训练统计（按创建日期分桶）
    now = datetime.now(timezone.utc)
    train_by_day: dict[str, int] = {}
    for t in tasks:
        ct = t.get("create_time")
        if ct:
            train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1
    training_7d = []
    for i in range(6, -1, -1):
        day = (now - timedelta(days=i)).strftime("%Y-%m-%d")
        training_7d.append(
            {
                "date": day[5:],
                "train": train_by_day.get(day, 0),
                "gpu": sum(len(t.get("gpus") or []) for t in running_ft),
                "accuracy": None,
            }
        )

    # 服务状态 —— 每个服务的"实例数"含义：
    #   模型训练 → 训练任务总数
    #   模型评测 → 评测任务总数
    #   模型推理 → 推理/对比任务实例数
    #   模型管理 → 基座模型注册总数
    #   数据集管理 → 数据集总数
    #   数据处理 → 数据处理任务总数
    #   数据类型转换 → 数据转换任务总数
    service_checks = [
        ("模型训练", "fine-tune", len(tasks)),
        ("模型评测", "model-eval", len(eval_tasks)),
        ("模型推理", "model-inference", len(compares)),
        ("模型管理", "model-manage", len(store.models())),
        ("数据集管理", "dataset-manage", len(datasets)),
        ("数据处理", "data-process", dp_count),
        ("数据类型转换", "data-convert", dp_count),
    ]
    service_status = []
    for svc_type, _path, svc_count in service_checks:
        service_status.append({
            "type": svc_type,
            "status": "normal",
            "count": svc_count,
        })

    # 训练任务状态归一化
    status_map = {
        "syncing": "running",
        "queued": "running",
        "running": "running",
        "pending": "pending",
        "paused": "pending",
        "completed": "completed",
        "failed": "failed",
        "error": "failed",
        "cancelled": "failed",
        "stopped": "failed",
    }
    training_tasks = [
        {
            "id": t.get("id"),
            "name": t.get("name"),
            "status": status_map.get(t.get("status"), "pending"),
            "train_type": t.get("train_type") or t.get("trainType") or "",
            "train_method": t.get("train_method") or t.get("trainMethod") or "",
            "base_model": t.get("base_model") or t.get("baseModel") or "",
            "progress": t.get("progress", 0),
            "accuracy": t.get("accuracy"),
            "started_at": (t.get("create_time") or "")[:16],
        }
        for t in tasks[:8]
    ]

    # 用户操作分布：仅统计 模型推理 / 模型训练 / 模型评测 / 数据处理 四类
    MODULE_LABELS = [
        ("data-process", "数据处理"),
        ("data_process", "数据处理"),
        ("dataset", "数据处理"),
        ("fine-tune", "模型训练"),
        ("fine_tune", "模型训练"),
        ("model-eval", "模型评测"),
        ("eval", "模型评测"),
        ("model-inference", "模型推理"),
        ("inference", "模型推理"),
    ]
    OP_ORDER = [
        "数据处理",
        "模型训练",
        "模型评测",
        "模型推理",
    ]

    def _op_module(action: str) -> str | None:
        a = (action or "").lower()
        for prefix, label in MODULE_LABELS:
            if a.startswith(prefix):
                return label
        return None

    audit = store.audit_logs(limit=1000, actor_id=None if is_admin(current_user) else current_user.get("id"))
    op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
    for log in audit.get("items", []):
        label = _op_module(log.get("action") or "")
        if label:
            op_counter[label] += 1
    operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]

    # 最近登录用户
    recent = sorted(
        [u for u in users if u.get("last_login")],
        key=lambda u: u["last_login"],
        reverse=True,
    )[:5]
    recent_login_users = [
        {
            "user": u.get("display_name") or u.get("username"),
            "role": u.get("role"),
            "last_login": (u.get("last_login") or "")[:16],
        }
        for u in recent
    ]

    # 登录时长排行（本月），只取 top 5
    login_duration_rank = []
    try:
        login_duration_rank = store.login_duration_rank(limit=5)
    except Exception:
        pass

    return _store_dashboard_cache(cache_key, ok(
        {
            "online_services": sum(s["count"] for s in service_status),
            "running_tasks": len(running_ft) + eval_running,
            "pending_alerts": 0,
            "training_7d": training_7d,
            "service_status": service_status,
            "training_tasks": training_tasks,
            "operation_distribution": operation_distribution,
            "login_duration_rank": login_duration_rank,
            "recent_login_users": recent_login_users,
        }
    ))


@router.get("/system-info")
async def system_info() -> dict[str, Any]:
    return ok(get_platform_store().system_info())


@router.get("/users")
async def users(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(get_platform_store().users())


@router.post("/users")
@op_log(module=OpModule.SYSTEM, action=OpAction.CREATE, target_type="user", target_name_param="username")
async def create_user(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    payload = dict(payload)
    payload.setdefault("password", "123456")
    payload.setdefault("tenant_id", current_user.get("tenant_id") or "admin")
    return ok(get_platform_store().create_user(payload))


@router.put("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user", target_name_param="user_id")
async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_user(user_id, payload))
    except KeyError:
        raise fail(404, "user not found")


@router.delete("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.DELETE, target_type="user", target_name_param="user_id")
async def delete_user(user_id: str, current_username: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    try:
        get_platform_store().delete_user(user_id, deleted_by=str(current_user.get("id") or "system"))
        get_platform_store().record_audit(
            action="user.delete", actor_id=current_user.get("id"),
            target_type="user", target_id=user_id,
            tenant_id=current_user.get("tenant_id"),
            detail="user tombstoned; sessions, memberships and grants revoked",
        )
        return ok({"deleted": user_id, "current_username": current_username})
    except KeyError:
        raise fail(404, "user not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/users/{user_id}/reset-password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password", target_name_param="user_id")
async def reset_user_password(
    user_id: str,
    payload: dict[str, Any] = Body(default={}),
    current_user: dict = Depends(require_admin),
) -> dict[str, Any]:
    new_password = payload.get("password") or "Platform@123"
    try:
        get_platform_store().reset_password(user_id, new_password)
        return ok({"reset": user_id})
    except KeyError:
        raise fail(404, "user not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/users/me/password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password")
async def change_my_password(
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    """用户自行修改密码：验证旧密码后设置新密码。"""
    old_password = payload.get("old_password") or ""
    new_password = payload.get("new_password") or ""
    if not old_password or not new_password:
        raise fail(400, "old_password and new_password are required")
    if len(new_password) < 6:
        raise fail(400, "new password must be at least 6 characters")
    try:
        success = get_platform_store().change_password(
            current_user["id"], old_password, new_password
        )
    except KeyError:
        raise fail(404, "user not found")
    if not success:
        raise fail(400, "old password is incorrect")
    return ok({"changed": True})


@router.get("/model-manage/local-models")
async def local_models(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    all_models = [
        item for item in store.models()
        if str(item.get("model_source") or "").lower() not in {"api", "online"}
    ]
    accessible = filter_accessible_resource_ids_batch("model", [str(item["id"]) for item in all_models], current_user)
    models = [
        {
            "path": item.get("path") or "",
            "name": item["name"],
            "storage_status": item.get("storage_status") or ("available" if store.storage_objects_for_resource("model", str(item["id"])) else "pending"),
        }
        for item in all_models
        if str(item.get("id") or "") in accessible
    ]
    # Admins may discover legacy model directories that have not been
    # registered yet. The directory is only an import source: do not return
    # the node id/code, and never persist it as model ownership. Once selected
    # and created, the model is archived to MinIO and future consumers use the
    # shared object manifest plus a local cache on their target node.
    if is_admin(current_user) and get_settings().compute_mode != "simulator":
        seen_paths = {str(item.get("path") or "") for item in models if item.get("path")}
        for node in store.compute_nodes():
            if not node.get("enabled"):
                continue
            try:
                result = await ComputeNodeClient(node["api_base_url"], timeout=10).list_files(
                    root="models",
                    directories_only=True,
                )
            except Exception:
                continue
            for item in result.get("items") or []:
                path = str(item.get("path") or "")
                if not path or path in seen_paths:
                    continue
                seen_paths.add(path)
                models.append({
                    "path": path,
                    "name": item.get("name") or path.rsplit("/", 1)[-1],
                    "storage_status": "pending",
                })
    return ok({"models": models})


@router.get("/model-manage/trained-models")
async def trained_models(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    all_models = get_platform_store().trained_models()
    if is_admin(current_user):
        return ok({"models": all_models})
    # 普通用户只能看到自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = filter_accessible_resource_ids_batch("trained_model", [m["id"] for m in all_models], current_user)
    result = [m for m in all_models if m.get("created_by") == user_id or m["id"] in accessible]
    return ok({"models": result})


@router.delete("/model-manage/trained-models/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="trained_model", target_name_param="model_id")
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("trained_model", model_id, current_user, "delete"):
        raise fail(403, "no permission to delete this trained model")
    pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}", "trained_model.delete")
    if pending:
        return pending
    get_platform_store().delete_trained_model(model_id)
    return ok({"deleted": model_id, "type": type})


@router.get("/model-manage/trained-models/{model_id}/artifacts")
async def trained_model_artifacts(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("trained_model", model_id, current_user, "read"):
        raise fail(403, "no permission to access model artifacts")
    return ok(get_platform_store().model_artifacts(model_id))


@router.get("/model-manage/trained-models/{model_id}/lineage")
async def trained_model_lineage(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("trained_model", model_id, current_user, "read"):
        raise fail(403, "no permission to access model lineage")
    return ok(get_platform_store().model_lineage(model_id))


@router.get("/model-manage/export-jobs")
async def model_export_jobs(trained_model_id: str | None = Query(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if trained_model_id and not has_resource_access("trained_model", trained_model_id, current_user, "read"):
        raise fail(403, "no permission to access export jobs")
    jobs = get_platform_store().model_export_jobs(trained_model_id)
    if not is_admin(current_user):
        jobs = [item for item in jobs if has_resource_access("trained_model", str(item.get("trained_model_id") or ""), current_user, "read")]
    return ok(jobs)


@router.get("/model-manage/name/{name}")
async def model_by_name(name: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        model = get_platform_store().model_by_name(name)
        if not has_resource_access("model", str(model["id"]), current_user, "read"):
            raise fail(403, "no permission to read this model")
        return ok(_public_model(model))
    except KeyError:
        raise fail(404, "model not found")


@router.get("/model-manage")
async def model_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    models = get_platform_store().models()
    accessible = filter_accessible_resource_ids_batch("model", [str(item["id"]) for item in models], current_user)
    return ok(_public_models([item for item in models if str(item.get("id") or "") in accessible]))


@router.post("/model-manage/test-online")
async def test_online_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """测试在线模型 API 是否可用：发送一个简单的 chat/completions 请求验证连通性。"""
    api_url = (payload.get("api_url") or "").rstrip("/")
    api_key = payload.get("api_key") or ""
    model_name = payload.get("online_model_name") or ""
    if not api_url:
        raise fail(400, "api_url is required")
    if not model_name:
        raise fail(400, "online_model_name is required")
    import httpx
    try:
        async with httpx.AsyncClient(timeout=15) as client:
            headers = {"Content-Type": "application/json"}
            if api_key:
                headers["Authorization"] = f"Bearer {api_key}"
            # 尝试多种 OpenAI 兼容路径
            chat_paths = [
                f"{api_url}/chat/completions",
                f"{api_url}/v1/chat/completions",
                f"{api_url}/modelTF/v1/chat/completions",
            ]
            resp = None
            for path in chat_paths:
                try:
                    r = await client.post(
                        path,
                        json={
                            "model": model_name,
                            "messages": [{"role": "user", "content": "Hi"}],
                            "max_tokens": 5,
                            "temperature": 0,
                        },
                        headers=headers,
                    )
                    if r.status_code in (200, 201):
                        resp = r
                        break
                except Exception:
                    continue
            if resp is None:
                return ok({"success": False, "error": f"无法连接到 {api_url}，请检查地址和端口"})
            body = resp.json()
            usage = body.get("usage", {})
            return ok({
                "success": True,
                "model": body.get("model", model_name),
                "provider": body.get("object", ""),
                "usage": {
                    "prompt_tokens": usage.get("prompt_tokens", 0),
                    "completion_tokens": usage.get("completion_tokens", 0),
                    "total_tokens": usage.get("total_tokens", 0),
                },
                "latency_ms": None,  # 由前端计算
            })
    except httpx.TimeoutException:
        return ok({"success": False, "error": "连接超时（15s），请检查网络或 API 地址是否正确"})
    except Exception as exc:
        return ok({"success": False, "error": str(exc)})


@router.post("/model-manage")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.CREATE, target_type="model", target_name_param="name")
@audit_log(
    action=AuditActions.CREATE_MODEL,
    target_type="model",
    detail_template="创建模型: {name}",
)
async def create_model(
    background_tasks: BackgroundTasks,
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    model_source = str(payload.get("model_source") or "local").lower()
    if model_source not in {"api", "online"} and not is_admin(current_user):
        raise fail(403, "only administrators can register local base models")
    payload["created_by"] = current_user.get("id")
    if model_source in {"api", "online"}:
        payload = bind_active_tenant(payload, current_user)
    try:
        model = get_platform_store().create_model(payload)
        if model_source not in {"api", "online"} and model.get("path") and get_settings().minio_enabled:
            background_tasks.add_task(_archive_model_in_background, str(model["id"]))
        return ok(_public_model(model))
    except KeyError as exc:
        raise fail(400, f"missing field: {exc}")
    except ValueError as exc:
        raise fail(400, str(exc))
    except Exception as exc:  # noqa: BLE001 - keep API errors visible to deployment smoke checks
        raise fail(500, f"create model failed: {exc}")


@router.get("/model-manage/{model_id}")
async def model_detail(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        model = get_platform_store().model(model_id)
    except KeyError:
        raise fail(404, "model not found")
    if not has_resource_access("model", model_id, current_user, "read"):
        raise fail(403, "no permission to read this model")
    return ok(_public_model(model))


@router.put("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model", target_name_param="model_id")
@audit_log(
    action=AuditActions.UPDATE_MODEL,
    target_type="model",
    detail_template="更新模型: {model_id}",
)
async def update_model(model_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    # 基座模型（配置模型）只有管理员可以编辑
    if not is_admin(current_user):
        raise fail(403, "只有管理员可以修改模型配置")
    try:
        return ok(_public_model(get_platform_store().update_model(model_id, payload)))
    except KeyError:
        raise fail(404, "model not found")


@router.put("/model-manage/{model_id}/purpose")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model_purpose", target_name_param="model_id")
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    # 基座模型（配置模型）只有管理员可以修改用途
    if not is_admin(current_user):
        raise fail(403, "只有管理员可以修改模型用途")
    try:
        return ok(_public_model(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")})))
    except KeyError:
        raise fail(404, "model not found")


@router.delete("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="model", target_name_param="model_id")
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    # 基座模型（配置模型）只有管理员可以删除
    if not is_admin(current_user):
        raise fail(403, "只有管理员可以删除模型配置")
    get_platform_store().delete_model(model_id)
    return ok({"deleted": model_id})


@router.post("/model-manage/merge")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.MERGE, target_type="trained_model", target_name_param="trained_model_id")
async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    engine = str(payload.get("engine") or "merge").strip().lower()
    if engine not in {"merge", "export", "llama_factory_export"}:
        raise fail(400, "engine must be merge, export or llama_factory_export")
    trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "")
    trained_model = next(
        (
            item
            for item in store.trained_models()
            if trained_model_id and (item["id"] == trained_model_id or item["name"] == trained_model_id)
        ),
        None,
    )
    if not trained_model:
        raise fail(404, "trained model not found")
    if not has_resource_access("trained_model", trained_model["id"], current_user, "execute"):
        raise fail(403, "no permission to merge this trained model")
    if engine != "merge" and not has_resource_access("trained_model", trained_model["id"], current_user, "download"):
        raise fail(403, "no permission to export this trained model")
    store.record_audit(
        action="trained_model.export" if engine != "merge" else "trained_model.merge",
        actor_id=current_user.get("id"), target_type="trained_model", target_id=trained_model["id"],
        tenant_id=trained_model.get("tenant_id"), detail=f"engine={engine}",
    )
    # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权即可使用
    base_model_path = payload.get("base_model_path") or (
        trained_model.get("merged_path") if engine != "merge" and trained_model and trained_model.get("merged") else None
    ) or (trained_model and trained_model.get("base_model_path"))
    adapter_path = (
        payload.get("adapter_path")
        or payload.get("adapter_name_or_path")
        or (
            trained_model
            and not (engine != "merge" and trained_model.get("merged"))
            and (trained_model.get("artifact_dir") or trained_model.get("adapter_path") or trained_model.get("merged_path"))
        )
    )
    if not base_model_path:
        raise fail(400, "base_model_path is required")
    if not adapter_path and engine == "merge":
        raise fail(400, "adapter_path is required")
    requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id"))
    node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []})
    if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
        try:
            await _wait_for_object_storage()
        except RuntimeError as exc:
            raise fail(503, str(exc))
        try:
            prepared_base = await _prepare_resource_on_node(store, "model", str(payload.get("base_model_id") or base_model_path), node)
            if prepared_base:
                base_model_path = prepared_base
            prepared_adapter = await _prepare_resource_on_node(store, "trained_model", str(payload.get("adapter_model_id") or (trained_model and trained_model.get("id")) or ""), node)
            if prepared_adapter:
                adapter_path = prepared_adapter
        except Exception as exc:
            raise fail(502, f"merge resource preparation failed: {exc}")
    health = node.get("health_detail") or {}
    output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
    output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
    output_dir = str(payload.get("output_dir") or f"{output_root.rstrip('/')}/{output_name}")
    job_payload = {
        **payload,
        "id": str(payload.get("job_id") or f"{('export' if engine != 'merge' else 'merge')}_{uuid.uuid4().hex[:12]}"),
        "name": output_name,
        "engine": engine,
        "base_model": base_model_path,
        "model_name_or_path": base_model_path,
        "adapter_name_or_path": adapter_path,
        "output_dir": output_dir,
        "template": payload.get("template", "qwen"),
        "train_method": payload.get("train_method", "lora"),
        "gpus": payload.get("gpus") or [],
        "trained_model_id": trained_model["id"] if trained_model else trained_model_id,
        "model_name": trained_model["name"] if trained_model else payload.get("model_name"),
        "compute_node_id": node["id"],
        "compute_node_code": node.get("code"),
        "resource_versions": payload.get("resource_versions") or _capture_resource_versions(store, {**payload, "model_id": trained_model["id"]}),
        "tenant_id": payload.get("tenant_id") or current_user.get("tenant_id") or "default",
    }
    if get_settings().compute_mode == "simulator":
        job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir}
    else:
        client = ComputeNodeClient(node["api_base_url"], timeout=900)
        preview = await client.validate_job(job_payload)
        if not preview.get("valid", False):
            raise fail(409, "; ".join(preview.get("errors") or ["merge preflight failed"]))
        job = await client.create_job(job_payload)
    return ok(store.record_model_merge_job(node, job_payload, job, trained_model["id"] if trained_model else trained_model_id))


def _authorize_inference_payload(payload: dict[str, Any], current_user: dict[str, Any]) -> None:
    """Require execute permission for an explicitly selected inference task/model."""
    task_id = str(payload.get("task_id") or payload.get("inference_task_id") or "")
    if task_id and not has_resource_access("compare", task_id, current_user, "execute"):
        raise fail(403, "no permission to use this inference task")
    trained_id = str(payload.get("trained_model_id") or "")
    if not trained_id and payload.get("model_id") and resource_record("trained_model", str(payload["model_id"])):
        trained_id = str(payload["model_id"])
    if trained_id and not has_resource_access("trained_model", trained_id, current_user, "execute"):
        raise fail(403, "no permission to use this trained model")
    model_id = str(payload.get("model_id") or "")
    if model_id and not trained_id and resource_record("model", model_id):
        if not has_resource_access("model", model_id, current_user, "execute"):
            raise fail(403, "no permission to use this base model")
    if not task_id and not trained_id and not model_id and not is_admin(current_user):
        raise fail(403, "inference must reference an authorized model or inference task")


def _authorize_compute_node(node_id: str, current_user: dict[str, Any]) -> None:
    if is_admin(current_user):
        return
    assigned = get_platform_store().gpu_assignments_for_user(str(current_user.get("id") or ""))
    if not any(str(item.get("node_id")) == str(node_id) for item in assigned):
        raise fail(403, "no permission to access this compute node")


def _authorize_training_payload(payload: dict[str, Any], current_user: dict[str, Any]) -> None:
    dataset_id = str(payload.get("train_dataset_id") or payload.get("dataset_id") or "")
    if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"):
        raise fail(403, "no permission to use this training dataset")
    base_model_id = str(payload.get("base_model_id") or "")
    if base_model_id and not has_resource_access("model", base_model_id, current_user, "execute"):
        raise fail(403, "no permission to use this base model")


@router.post("/model-manage/export")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.MERGE, target_type="trained_model", target_name_param="trained_model_id")
async def export_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Export a trained model through the same node, cache and archive path as merge."""
    return await merge_model({**payload, "engine": "export"}, current_user)


@router.get("/dataset-manage/preview/{file_id}")
async def dataset_preview(file_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"):
            raise fail(403, "no permission to preview this dataset")
        content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace")
        return ok({"content": content})
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/records/{file_id}/sources")
async def dataset_record_sources(file_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"):
            raise fail(403, "no permission to access dataset records")
        return ok({"items": store.dataset_file_record_sources(file_id)})
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/versions/{file_id}")
async def dataset_versions(file_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"):
            raise fail(403, "no permission to access dataset versions")
        return ok(store.file_versions(file_id))
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/versions/{file_id}/{version_id}")
async def dataset_version_content(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"):
            raise fail(403, "no permission to access dataset version")
        versions = store.file_versions(file_id)["versions"]
        version = next((item for item in versions if item["id"] == version_id), None)
        if not version:
            raise KeyError(version_id)
        content = _dataset_version_bytes(store, file_id, version_id)
        return ok({"version": version, "content": content.decode("utf-8", errors="replace")})
    except KeyError:
        raise fail(404, "dataset version not found")


@router.post("/dataset-manage/versions/{file_id}")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset_version", target_name_param="file_id")
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "write"):
            raise fail(403, "no permission to create dataset version")
        return ok(store.create_file_version(file_id, payload))
    except KeyError:
        raise fail(404, "dataset file not found")


@router.put("/dataset-manage/versions/{file_id}/active")
async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "write"):
            raise fail(403, "no permission to activate dataset version")
        return ok(store.activate_file_version(file_id, payload["version_id"]))
    except KeyError:
        raise fail(404, "dataset version not found")


@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
@op_log(module=OpModule.DATASET, action=OpAction.DELETE, target_type="dataset_version", target_name_param="version_id")
async def delete_dataset_version(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        row = store.dataset_file(file_id)
        if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "write"):
            raise fail(403, "no permission to delete dataset version")
        return ok(store.delete_file_version(file_id, version_id))
    except KeyError:
        raise fail(404, "dataset version not found")
    except ValueError as exc:
        raise fail(400, str(exc))


async def _sync_dataset_file_to_compute_nodes(
    store: Any,
    dataset_id: str,
    file_id: str,
    filename: str,
    content: bytes,
) -> list[dict[str, Any]]:
    results: list[dict[str, Any]] = []
    if get_settings().compute_mode == "simulator" or get_settings().minio_enabled:
        return results
    target_name = Path(filename or f"{file_id}.jsonl").name
    target_relative_path = f"datasets/{dataset_id}/{target_name}"
    for node in store.compute_nodes():
        if not node.get("enabled"):
            continue
        try:
            result = await ComputeNodeClient(node["api_base_url"]).upload_file(
                target_name,
                content,
                target_relative_path,
                resource_type="dataset",
                resource_id=dataset_id,
            )
            store.upsert_resource_replica(
                node["id"],
                "dataset",
                dataset_id,
                str(result.get("local_path") or ""),
            )
            results.append(
                {
                    "node_id": node["id"],
                    "node_code": node.get("code"),
                    "success": True,
                    "local_path": result.get("local_path"),
                    "byte_size": result.get("byte_size"),
                    "checksum_sha256": result.get("checksum_sha256"),
                }
            )
        except Exception as exc:  # noqa: BLE001 - keep upload usable while exposing sync failures
            results.append(
                {
                    "node_id": node["id"],
                    "node_code": node.get("code"),
                    "success": False,
                    "error": str(exc),
                }
            )
    return results


async def _sync_training_dataset_to_compute_node(
    store: Any,
    node: dict[str, Any],
    dataset_id: str,
) -> list[dict[str, Any]]:
    if get_settings().minio_enabled:
        files = store.training_dataset_files(dataset_id)
        object_by_resource_name: dict[tuple[str, str, str], dict[str, Any]] = {}
        resource_ids = {str(dataset_id)} | {
            str(item.get("dataset_id"))
            for item in files
            if item.get("dataset_id")
        }
        for resource_id in resource_ids:
            for obj in store.storage_objects_for_resource("dataset", resource_id):
                file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
                object_by_resource_name[(resource_id, file_name, str(obj.get("version_id") or ""))] = obj
        results: list[dict[str, Any]] = []
        client = ComputeNodeClient(node["api_base_url"])
        for item in files:
            target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
            item_dataset_id = str(item.get("dataset_id") or dataset_id)
            version_id = str(item.get("active_version_id") or item["id"])
            obj = object_by_resource_name.get((item_dataset_id, target_name, version_id))
            if not obj and item.get("content"):
                # 兼容 MinIO 接入前已经发布的数据处理数据集。大文件补建
                # MinIO 对象，小文件直接从数据库正文同步到目标节点。
                raw = str(item.get("content") or "").encode("utf-8")
                if should_store_in_minio(len(raw)):
                    object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
                    uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
                    obj = store.create_storage_object({
                        "resource_type": "dataset",
                        "resource_id": item_dataset_id,
                        "version_id": version_id,
                        "bucket": uploaded["bucket"],
                        "object_key": object_key,
                        "file_name": target_name,
                        "content_type": "application/jsonl",
                        "byte_size": len(raw),
                        "checksum_sha256": hashlib.sha256(raw).hexdigest(),
                        "status": "available",
                    })
                    store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
                else:
                    result = await client.upload_file(
                        target_name,
                        raw,
                        f"datasets/{dataset_id}/{target_name}",
                        resource_type="dataset",
                        resource_id=dataset_id,
                    )
                    store.upsert_resource_replica(
                        node["id"], "dataset", dataset_id, str(result.get("local_path") or "")
                    )
                    results.append({
                        "node_id": node["id"],
                        "node_code": node.get("code"),
                        "file_id": item.get("id"),
                        "name": target_name,
                        "local_path": result.get("local_path"),
                        "byte_size": result.get("byte_size"),
                        "checksum_sha256": result.get("checksum_sha256"),
                        "storage_backend": "database",
                    })
                    continue
            if not obj:
                raise RuntimeError(f"dataset file is not available: {target_name}")
            url = get_object_storage().presigned_get(obj["object_key"])
            result = await client.prepare_cache({
                "resource_id": dataset_id,
                "version_id": obj["version_id"],
                "download_url": url,
                "checksum_sha256": obj.get("checksum_sha256") or "",
                "byte_size": obj.get("byte_size") or 0,
                "relative_path": f"datasets/{dataset_id}/{target_name}",
            })
            store.upsert_resource_replica(
                node["id"],
                "dataset",
                dataset_id,
                str(result.get("local_path") or ""),
            )
            results.append({
                **result,
                "file_id": item.get("id"),
                "name": target_name,
                "node_id": node["id"],
                "storage_backend": "minio",
            })
        return results
    if not dataset_id:
        raise RuntimeError("train_dataset_id is required")
    files = store.training_dataset_files(dataset_id)
    if not files:
        raise RuntimeError(f"dataset has no uploaded file: {dataset_id}")
    split_aware = any(item.get("split") for item in files)
    files = [
        item
        for item in files
        if not split_aware or item.get("split") in {"train", "validation"}
    ]
    client = ComputeNodeClient(node["api_base_url"])
    results: list[dict[str, Any]] = []
    for item in files:
        target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
        result = await client.upload_file(
            target_name,
            str(item.get("content") or "").encode("utf-8"),
            f"datasets/{dataset_id}/{target_name}",
            resource_type="dataset",
            resource_id=dataset_id,
        )
        store.upsert_resource_replica(
            node["id"],
            "dataset",
            dataset_id,
            str(result.get("local_path") or ""),
        )
        results.append(
            {
                "node_id": node["id"],
                "node_code": node.get("code"),
                "file_id": item.get("id"),
                "name": target_name,
                "local_path": result.get("local_path"),
                "byte_size": result.get("byte_size"),
                "checksum_sha256": result.get("checksum_sha256"),
            }
        )
    return results


@router.post("/dataset-manage/upload/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPLOAD, target_type="dataset_file", target_name_param="dataset_id")
async def upload_dataset_files(
    dataset_id: str,
    files: list[UploadFile] = File(default=[]),
    sync_to_compute: bool = Query(default=True),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    created: list[dict[str, Any]] = []
    compute_sync: list[dict[str, Any]] = []
    pending_sync: list[tuple[str, str, bytes]] = []
    store = get_platform_store()
    try:
        store.dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    if not has_resource_access("dataset", dataset_id, current_user, "write"):
        raise fail(403, "no permission to upload files to this dataset")
    with store.connect() as conn:
        for file in files:
            raw = await file.read()
            content = raw.decode("utf-8", errors="replace")
            created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
            created.append(created_file)
            pending_sync.append((created_file["id"], created_file["name"], raw))
            if should_store_in_minio(
                len(raw),
                content_type=file.content_type,
                file_format=Path(created_file["name"]).suffix,
            ):
                object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
                uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
                storage_object = get_platform_store().create_storage_object({
                    "resource_type": "dataset", "resource_id": dataset_id,
                    "version_id": created_file.get("active_version_id") or created_file["id"],
                    "bucket": uploaded["bucket"], "object_key": object_key,
                    "file_name": created_file["name"], "content_type": file.content_type,
                    "byte_size": len(raw), "checksum_sha256": hashlib.sha256(raw).hexdigest(),
                    "status": "available",
                })
                store.link_dataset_file_storage_object(created_file["id"], storage_object["id"])
    if sync_to_compute:
        for file_id, file_name, raw in pending_sync:
            compute_sync.extend(
                await _sync_dataset_file_to_compute_nodes(
                    store,
                    dataset_id,
                    file_id,
                    file_name,
                    raw,
                )
            )
    return ok({"files": created, "compute_sync": compute_sync})


@router.get("/dataset-manage/download/{dataset_id}")
async def download_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> Response:
    store = get_platform_store()
    try:
        dataset = store.dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    if not has_resource_access("dataset", dataset_id, current_user, "download"):
        raise fail(403, "no permission to download this dataset")
    store.record_audit(action="dataset.download", actor_id=current_user.get("id"), target_type="dataset", target_id=dataset_id, tenant_id=dataset.get("tenant_id"), detail="dataset bundle download")

    files = []
    for item in dataset.get("files", []):
        if item.get("deleted_at"):
            continue
        try:
            full_file = store.dataset_file(str(item["id"]))
        except KeyError:
            continue
        files.append({
            **item,
            "content": _dataset_file_bytes(store, str(item["id"])).decode("utf-8", errors="replace"),
        })
    if not files:
        raise fail(404, "dataset has no downloadable files")

    if len(files) == 1:
        item = files[0]
        filename = Path(str(item.get("name") or f"{dataset_id}.jsonl")).name
        encoded_name = quote(filename, safe="")
        return Response(
            content=str(item.get("content") or "").encode("utf-8"),
            media_type="application/octet-stream",
            headers={
                "Content-Disposition": f"attachment; filename=dataset-file; filename*=UTF-8''{encoded_name}",
            },
        )

    archive = BytesIO()
    with ZipFile(archive, "w", compression=ZIP_DEFLATED) as bundle:
        used_names: set[str] = set()
        for index, item in enumerate(files, start=1):
            filename = Path(str(item.get("name") or f"file-{index}.jsonl")).name
            unique_name = filename
            if unique_name in used_names:
                stem = Path(filename).stem
                suffix = Path(filename).suffix
                unique_name = f"{stem}-{index}{suffix}"
            used_names.add(unique_name)
            bundle.writestr(unique_name, str(item.get("content") or ""))
    archive.seek(0)
    archive_name = quote(f"{dataset.get('name') or dataset_id}.zip", safe="")
    return StreamingResponse(
        archive,
        media_type="application/zip",
        headers={
            "Content-Disposition": f"attachment; filename=dataset.zip; filename*=UTF-8''{archive_name}",
        },
    )


@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
async def download_dataset_file(
    dataset_id: str,
    file_id: str,
    version_id: str | None = Query(default=None),
    current_user: dict = Depends(get_current_user),
) -> Response:
    store = get_platform_store()
    row = store.dataset_file(file_id)
    if str(row.get("dataset_id")) != str(dataset_id):
        raise fail(404, "dataset file not found")
    if not has_resource_access("dataset", dataset_id, current_user, "download"):
        raise fail(403, "no permission to download this dataset")
    store.record_audit(action="dataset.file.download", actor_id=current_user.get("id"), target_type="dataset_file", target_id=file_id, tenant_id=row.get("tenant_id"), detail=f"dataset_id={dataset_id}")
    object_id = str(row.get("storage_object_id") or "")
    if version_id:
        version = next((item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id), None)
        if not version:
            raise fail(404, "dataset version not found")
        object_id = str(version.get("storage_object_id") or "")
    if get_settings().minio_enabled and object_id:
        try:
            obj = store.storage_object(object_id)
            filename = Path(str(obj.get("file_name") or row.get("name") or file_id)).name
            return StreamingResponse(
                get_object_storage().iter_bytes(str(obj["object_key"])),
                media_type=str(obj.get("content_type") or "application/octet-stream"),
                headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename, safe='')}"},
            )
        except KeyError:
            pass
    content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id)
    filename = Path(str(row.get("name") or file_id)).name
    return Response(
        content=content,
        media_type="application/octet-stream",
        headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename, safe='')}"},
    )


@router.get("/dataset-manage")
async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    datasets = get_platform_store().datasets(None if is_admin(current_user) else current_user.get("tenant_id") or "default")
    if is_admin(current_user):
        return ok(datasets)
    # 普通用户可见：自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = filter_accessible_resource_ids_batch("dataset", [d["id"] for d in datasets], current_user)
    result = [d for d in datasets if d.get("created_by") == user_id or d["id"] in accessible]
    return ok(result)


@router.post("/dataset-manage")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
@audit_log(
    action=AuditActions.CREATE_DATASET,
    target_type="dataset",
    detail_template="创建数据集: {name}",
)
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
async def create_dataset(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    bind_active_tenant(payload, current_user)
    payload.setdefault("created_by", current_user.get("id"))
    payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
    try:
        get_platform_store().assert_active_tenant(payload["tenant_id"])
        dataset = get_platform_store().create_dataset(payload)
        return ok({"id": dataset["id"]})
    except ValueError as exc:
        raise fail(409, str(exc))


@router.get("/dataset-manage/{dataset_id}")
async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        dataset = get_platform_store().dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    if not has_resource_access("dataset", dataset_id, current_user, "read"):
        raise fail(403, "no permission to access this dataset")
    return ok(dataset)


@router.put("/dataset-manage/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPDATE, target_type="dataset", target_name_param="dataset_id")
@audit_log(
    action=AuditActions.UPDATE_DATASET,
    target_type="dataset",
    detail_template="更新数据集: {dataset_id}",
)
async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("dataset", dataset_id, current_user, "write"):
        raise fail(403, "no permission to update this dataset")
    try:
        return ok(get_platform_store().update_dataset(dataset_id, payload))
    except KeyError:
        raise fail(404, "dataset not found")


@router.delete("/dataset-manage/{dataset_id}")
@audit_log(
    action=AuditActions.DELETE_DATASET,
    target_type="dataset",
    detail_template="删除数据集: {dataset_id}",
)
@op_log(module=OpModule.DATASET, action=OpAction.DELETE, target_type="dataset", target_name_param="dataset_id")
async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("dataset", dataset_id, current_user, "delete"):
        raise fail(403, "no permission to delete this dataset")
    pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}", "dataset.delete")
    if pending:
        return pending
    get_platform_store().delete_dataset(dataset_id)
    return ok({"deleted": dataset_id})


@router.get("/fine-tune/check-name")
async def check_fine_tune_name(name: str = Query(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    exists = any(task["name"] == name for task in get_platform_store().tasks())
    return ok({"exists": exists})


@router.get("/fine-tune/progress/{task_id}")
async def fine_tune_progress(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "read"):
        raise fail(403, "no permission to access training progress")
    try:
        return ok(get_platform_store().progress(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/tensorboard/start")
async def tensorboard_start(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return ok({"status": "running", "url": "http://localhost:6006"})


@router.get("/fine-tune")
async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    tasks = get_platform_store().tasks()
    if is_admin(current_user):
        return ok(tasks)
    # 普通用户可见：自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = filter_accessible_resource_ids_batch("fine-tune", [t["id"] for t in tasks], current_user)
    result = [t for t in tasks if t.get("created_by") == user_id or t["id"] in accessible]
    return ok(result)


@router.post("/fine-tune")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.CREATE, target_type="fine_tune", target_name_param="name")
@audit_log(
    action=AuditActions.CREATE_FINE_TUNE,
    target_type="fine_tune",
    detail_template="创建微调任务: {name}",
)
async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    bind_active_tenant(payload, current_user)
    _authorize_training_payload(payload, current_user)
    payload.setdefault("created_by", current_user.get("id"))
    payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
    try:
        get_platform_store().assert_active_tenant(payload["tenant_id"])
    except ValueError as exc:
        raise fail(400, str(exc))
    if not is_admin(current_user):
        model_id = str(payload.get("base_model") or payload.get("base_model_id") or "")
        dataset_id = str(payload.get("train_dataset_id") or "")
        if payload.get("base_model_id") and not has_resource_access("model", str(payload["base_model_id"]), current_user, "execute"):
            raise fail(403, "no permission to use this base model")
        # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权
        if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"):
            raise fail(403, "no permission to use this dataset")
    try:
        if not payload.get("resource_versions"):
            payload["resource_versions"] = _capture_resource_versions(get_platform_store(), payload)
        task = get_platform_store().create_task(payload)
        return ok({"id": task["id"]})
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/fine-tune/start")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.START, target_type="fine_tune", target_name_param="name", detail_params=["task_id", "base_model", "train_dataset_id"])
async def start_fine_tune(
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    store = get_platform_store()
    bind_active_tenant(payload, current_user)
    _authorize_training_payload(payload, current_user)
    # GPU 权限校验：普通用户只能使用被分配的 GPU
    if not is_admin(current_user):
        node_id = payload.get("compute_node_id") or payload.get("node_id")
        gpu_indices = payload.get("gpu_indices")
        if gpu_indices is None:
            gpu_indices = payload.get("gpus") or []
        if node_id and gpu_indices:
            if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
                raise fail(403, "无权使用所选 GPU，请联系管理员分配")
        if node_id and not gpu_indices:
            payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
    # 页面明确选择节点时，调度器必须保持节点约束；否则可能落到其它节点。
    payload["strict_node_selection"] = bool(payload.get("compute_node_id") or payload.get("node_id"))
    payload.setdefault("created_by", current_user.get("id"))
    payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
    try:
        store.assert_active_tenant(payload["tenant_id"])
    except ValueError as exc:
        raise fail(400, str(exc))
    if not payload.get("resource_versions"):
        snapshot_payload = dict(payload)
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            try:
                snapshot_payload = {**store.task(task_id), **snapshot_payload}
            except KeyError:
                pass
        payload["resource_versions"] = _capture_resource_versions(store, snapshot_payload)
    try:
        return ok(await _submit_fine_tune_task(store, payload))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            store.mark_task_failed(task_id, str(exc))
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - mark task failed when remote submit fails
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            store.mark_task_failed(task_id, str(exc))
        raise fail(502, f"submit compute job failed: {exc}")


@router.post("/fine-tune/preflight")
async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    _authorize_training_payload(payload, current_user)
    try:
        return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True, sync_resources=True))
    except RuntimeError as exc:
        return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
    except Exception as exc:  # noqa: BLE001 - expose compute validation errors to training create page
        raise fail(502, f"compute preflight failed: {exc}")


@router.post("/fine-tune/command-preview")
async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    _authorize_training_payload(payload, current_user)
    try:
        return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False, sync_resources=False))
    except RuntimeError as exc:
        return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"compute command preview failed: {exc}")


@router.post("/fine-tune/{task_id}/preflight")
async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "read"):
        raise fail(403, "no permission to preflight this task")
    try:
        return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - expose compute validation errors to training create page
        raise fail(502, f"compute preflight failed: {exc}")


@router.post("/fine-tune/{task_id}/command-preview")
async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "read"):
        raise fail(403, "no permission to preview this task")
    try:
        return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"compute command preview failed: {exc}")


@router.get("/fine-tune/{task_id}")
async def fine_tune_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "read"):
        raise fail(403, "no permission to access this task")
    try:
        return ok(get_platform_store().task(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.get("/fine-tune/{task_id}/logs")
async def fine_tune_logs(
    task_id: str,
    tail_lines: int | None = Query(default=500, ge=1, le=5000),
    offset: int | None = Query(default=None, ge=0),
    limit: int | None = Query(default=None, ge=1, le=5000),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if task.get("compute_job_id"):
        node = _node_for_task(task)
        if node:
            try:
                logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], tail_lines, offset, limit)
                try:
                    store.record_training_log_metrics(task_id, str(logs.get("content") or ""))
                except Exception:
                    pass
                if task.get("status") in {"queued", "running", "failed", "stopped", "completed"}:
                    try:
                        job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"])
                        store.apply_compute_job(task_id, job)
                    except Exception:
                        pass
                return ok({"source": "compute", **logs})
            except Exception as exc:  # noqa: BLE001 - keep failure reason visible even when log fetch fails
                content = task.get("failure_reason") or f"fetch compute log failed: {exc}"
                return ok({"job_id": task.get("compute_job_id"), "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
    content = task.get("failure_reason") or ""
    return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})


@router.get("/fine-tune/{task_id}/diagnostics")
async def fine_tune_diagnostics(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    log_text = ""
    node = _node_for_task(task)
    if node and task.get("compute_job_id"):
        try:
            logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], 1000, None, None)
            log_text = str(logs.get("content") or "")
        except Exception:
            log_text = ""
    errors = [str(task.get("failure_reason") or "")] if task.get("failure_reason") else []
    return ok(
        {
            "task_id": task_id,
            "status": task.get("status"),
            "failure_reason": task.get("failure_reason") or "",
            "diagnostics": _training_diagnostics(errors, [], log_text),
        }
    )


@router.get("/fine-tune/{task_id}/gpu-status")
async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
    """Return live GPU metrics for the task's selected node and cards."""
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if not has_resource_access("fine-tune", task_id, current_user, "read"):
        raise fail(403, "no permission to access this task")
    node = _node_for_task(task)
    selected = set(_normalize_gpu_indices({"gpus": task.get("gpus") or []}, allow_primary=False))
    if not node:
        return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected)})
    try:
        if get_settings().compute_mode == "simulator":
            live_items = store.gpus()
        else:
            live_items = await ComputeNodeClient(node["api_base_url"]).gpu_resources()
        items = []
        for item in live_items:
            index = int(item.get("gpu_index", item.get("id", -1)))
            if selected and index not in selected:
                continue
            items.append({
                **item,
                "id": index,
                "node_id": node["id"],
                "node_code": node.get("code"),
                "node_name": node.get("name"),
            })
        return ok({"source": "compute", "items": items, "selected_gpus": sorted(selected)})
    except Exception as exc:  # noqa: BLE001 - let UI retain last good snapshot
        return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected), "error": str(exc)})


@router.put("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "write"):
        raise fail(403, "no permission to update this task")
    try:
        return ok(get_platform_store().update_task(task_id, payload))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/stop/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.STOP, target_type="fine_tune", target_name_param="task_id")
async def stop_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
        # 审批拦截：非 admin 停止他人任务需审批
        pending = _require_approval_or_admin("fine_tune_task", task_id, current_user, f"停止训练任务 {task_id}", "fine_tune.stop")
        if pending:
            return pending
        node = _node_for_task(task)
        if task.get("compute_job_id") and node and get_settings().compute_mode != "simulator":
            job = await ComputeNodeClient(node["api_base_url"]).stop_job(task["compute_job_id"])
            return ok(store.apply_compute_job(task_id, job))
        return ok(store.stop_task(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/{task_id}/stop")
async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return await stop_fine_tune(task_id, current_user)


@router.post("/fine-tune/{task_id}/retry")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.RETRY, target_type="fine_tune", target_name_param="task_id")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    payload = payload or {}
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if not has_resource_access("fine-tune", task_id, current_user, "execute"):
        raise fail(403, "no permission to retry this task")
    if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
        raise fail(409, "only failed or stopped tasks can be retried without force=true")
    retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
    store.reset_task_for_retry(task_id, retry_payload)
    try:
        return ok(await _submit_fine_tune_task(store, retry_payload))
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - mark retry failed when remote submit fails
        store.mark_task_failed(task_id, str(exc))
        raise fail(502, f"retry fine tune task failed: {exc}")


@router.delete("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.DELETE, target_type="fine_tune", target_name_param="task_id")
async def delete_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("fine-tune", task_id, current_user, "delete"):
        raise fail(403, "no permission to delete this task")
    pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}", "fine_tune.delete")
    if pending:
        return pending
    get_platform_store().delete_task(task_id)
    return ok({"deleted": task_id})


@router.get("/fine-tune/{task_id}/overview")
async def fine_tune_overview(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    task = store.task(task_id)
    return ok(
        {
            "task": task,
            "progress": store.progress(task_id),
            "metrics": store.task_metrics(task_id),
            "checkpoints": store.task_checkpoints(task_id),
        }
    )


@router.get("/fine-tune/{task_id}/checkpoints")
async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    return ok(store.task_checkpoints(task_id))


@router.get("/fine-tune/{task_id}/metrics")
async def fine_tune_metrics(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    try:
        store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if not has_resource_access("fine-tune", task_id, current_user, "read"):
        raise fail(403, "no permission to access training metrics")
    return ok(store.task_metrics(task_id))


@router.get("/model-eval")
async def model_eval_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    tasks = get_platform_store().eval_tasks(None if is_admin(current_user) else current_user.get("tenant_id") or "default")
    if is_admin(current_user):
        return ok(tasks)
    # 普通用户可见：自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = filter_accessible_resource_ids_batch("eval", [t["id"] for t in tasks], current_user)
    result = [t for t in tasks if t.get("created_by") == user_id or t["id"] in accessible]
    return ok(result)


@router.get("/model-eval/{task_id}/report")
async def model_eval_report(task_id: str, current_user: dict = Depends(get_current_user)) -> Response:
    """Stream the archived evaluation report, with DB JSON as compatibility fallback."""
    store = get_platform_store()
    try:
        task = store.eval_task(task_id)
    except KeyError:
        raise fail(404, "eval task not found")
    if not has_resource_access("eval", task_id, current_user, "download"):
        raise fail(403, "no permission to access this eval report")
    store.record_audit(action="eval.report.download", actor_id=current_user.get("id"), target_type="eval", target_id=task_id, tenant_id=task.get("tenant_id"), detail="evaluation report download")
    report_object_id = str(task.get("report_storage_object_id") or "")
    if get_settings().minio_enabled and report_object_id:
        try:
            obj = store.storage_object(report_object_id)
            return StreamingResponse(
                get_object_storage().iter_bytes(str(obj["object_key"])),
                media_type=str(obj.get("content_type") or "application/json"),
                headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(f'{task_id}-report.json', safe='')}"},
            )
        except (KeyError, ObjectStorageError):
            pass
    payload = {
        "task_id": task_id,
        "status": task.get("status"),
        "overall_score": task.get("overall_score"),
        "overall_score_max": task.get("overall_score_max"),
        "dimension_summary": task.get("dimension_summary") or [],
        "samples": task.get("samples") or [],
        "overall_evaluation": task.get("overall_evaluation") or "",
        "improvement_suggestions": task.get("improvement_suggestions") or [],
    }
    return Response(
        content=json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"),
        media_type="application/json",
        headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(f'{task_id}-report.json', safe='')}"},
    )


@router.get("/model-eval/{task_id}")
async def model_eval_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        task = store.eval_task(task_id)
        if not has_resource_access("eval", task_id, current_user, "read"):
            raise fail(403, "no permission to access this eval task")
        if task.get("compute_job_id") and task.get("compute_node_id") and task.get("status") in {"queued", "running", "completed"}:
            node = next(
                (n for n in store.compute_nodes() if n["id"] == task.get("compute_node_id")),
                None,
            )
            if node:
                try:
                    client = ComputeNodeClient(node["api_base_url"])
                    job = await client.get_job(task["compute_job_id"])
                    result_content = None
                    if job.get("status") == "completed":
                        result_content = await fetch_eval_result_content(client, node, job)
                    elif job.get("status") in {"queued", "running"}:
                        progress_content = await fetch_eval_progress_content(client, node, job)
                        if progress_content:
                            task = store.update_eval_task(
                                task_id,
                                {"progress_detail": progress_content, "progress": progress_content.get("percentage", task.get("progress", 0))},
                            )
                        result_content = await fetch_eval_result_content(client, node, job)
                    task = store.apply_eval_job_result(task_id, job, result_content)
                except Exception:
                    pass
    except KeyError:
        raise fail(404, "eval task not found")
    return ok(task)


@router.post("/model-eval/start")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.START, target_type="eval_task", target_name_param="name", detail_params=["model_id", "dataset_id"])
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Start an evaluation task: submit eval job to compute node."""
    store = get_platform_store()
    bind_active_tenant(payload, current_user)
    model_id = str(payload.get("model_id") or "")
    dataset_id = str(payload.get("dataset_id") or "")
    if not dataset_id:
        raise fail(400, "dataset_id is required")
    try:
        store.dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    # Base models are shared read-only platform resources. Trained models must
    # pass resource-level execute permission before a task record is created.
    trained_model = next((item for item in store.trained_models() if str(item.get("id")) == model_id or str(item.get("name")) == model_id), None)
    if trained_model and not has_resource_access("trained_model", str(trained_model["id"]), current_user, "execute"):
        raise fail(403, "no permission to evaluate this trained model")
    if not is_admin(current_user) and not has_resource_access("dataset", dataset_id, current_user, "execute"):
        raise fail(403, "no permission to evaluate this dataset")
    payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
    try:
        store.assert_active_tenant(payload["tenant_id"])
    except ValueError as exc:
        raise fail(400, str(exc))
    if not payload.get("resource_versions"):
        payload["resource_versions"] = _capture_resource_versions(store, payload)
    try:
        gpu_indices = _normalize_gpu_indices(payload)
    except ValueError as exc:
        raise fail(400, str(exc))
    if not gpu_indices:
        raise fail(400, "请选择至少一张 GPU")
    # 1. Create eval task record
    payload.setdefault("created_by", current_user.get("id"))
    task = store.create_eval_task({**payload, "status": "pending"})
    if get_settings().minio_enabled:
        _store_json_snapshot(
            store,
            "eval",
            str(task["id"]),
            str(task["id"]),
            f"evaluations/{task['id']}/versions/{task['id']}/evaluation-config.json",
            payload,
            current_user.get("id"),
        )

    # 2. Resolve model path (supports both regular models and trained models)
    model_id = str(payload.get("model_id", ""))
    model_path = ""
    adapter_path = payload.get("adapter_path", "")
    model_node_id = ""
    ds_files: list[dict[str, Any]] = []
    model_resource_type = "model"
    model_resource_id = model_id
    adapter_resource_id = ""
    try:
        db_model = store.model(model_id)
        model_path = db_model.get("path", "")
        model_node_id = db_model.get("compute_node_id") or ""
    except KeyError:
        # Try trained_models table (IDs prefixed with tm_)
        trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
        if trained:
            model_resource_type = "trained_model" if trained.get("merged") else "model"
            model_resource_id = trained.get("id") or model_id
            adapter_resource_id = trained.get("id") or ""
            model_node_id = trained.get("compute_node_id") or ""
            merged_path = trained.get("merged_path", "")
            base_path = trained.get("base_model_path", "")
            if trained.get("merged") and merged_path:
                # Merged model: use merged_path as model, no adapter needed
                model_path = merged_path
            elif base_path:
                # Unmerged: use base model + adapter checkpoint
                model_path = base_path
                if merged_path:
                    adapter_path = merged_path
            else:
                model_path = merged_path or base_path
            if model_resource_type == "model":
                base_model = next((item for item in store.models() if item.get("path") == base_path), None)
                model_resource_id = str((base_model or {}).get("id") or base_path)
    if not model_path:
        store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
        return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})

    # 3. Resolve dataset file
    dataset_id = str(payload.get("dataset_id", ""))
    dataset_path = ""
    try:
        ds_files = store.training_dataset_files(dataset_id)
        if ds_files:
            dataset_path = ds_files[0].get("local_path") or ds_files[0].get("name", "")
    except Exception:
        pass
    if not dataset_path:
        # Try to get file content and sync to compute
        try:
            ds = store.dataset(dataset_id)
            for f in ds.get("files", []):
                if f.get("content"):
                    dataset_path = f.get("name", f"dataset_{dataset_id}.jsonl")
                    break
        except KeyError:
            pass
    if not dataset_path:
        store.update_eval_task(task["id"], {"status": "failed", "error": "dataset not found or no files"})
        return ok({"task_id": task["id"], "status": "failed", "error": "dataset not found or no files"})

    # 4. Resolve dimension config. Admin-maintained dimensions are referenced
    # by ID; user-created evaluation rules are carried inline with the task
    # and must not be persisted into the global dimension catalog.
    dimension_id = str(payload.get("dimension_id", ""))
    dimension_cfg: dict[str, Any] = {}
    if dimension_id:
        try:
            dim = store.dimension(dimension_id)
            # Resolve eval model API config
            eval_model_name = dim.get("eval_model", "")
            api_url = ""
            api_key = ""
            api_model_name = ""
            if eval_model_name:
                try:
                    eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name)
                    if isinstance(eval_model, dict):
                        api_url = eval_model.get("api_url", "")
                        api_key = eval_model.get("api_key", "")
                        # 模型记录里的 model_name 是真实 API 模型名（如 deepseek-chat），
                        # 优先传给评测器，避免用平台内部名称调用 LLM API
                        # 在线模型的真实服务名保存在 online_model_name；
                        # model_name 是历史字段，不能回退到平台内部名称。
                        api_model_name = (
                            eval_model.get("model_name")
                            or eval_model.get("online_model_name")
                            or eval_model.get("name")
                            or ""
                        )
                except (KeyError, Exception):
                    pass
            dimension_cfg = {
                "type": dim.get("type", ""),
                "eval_model": eval_model_name,
                "api_model": api_model_name or eval_model_name,
                "eval_method": dim.get("eval_method", ""),
                "eval_prompt": dim.get("eval_prompt", ""),
                "api_url": api_url,
                "api_key": api_key,
                "score_min": dim.get("score_min", 0),
                "score_max": dim.get("score_max", 5),
                "pass_threshold": dim.get("pass_threshold", 3),
            }
        except KeyError:
            pass
    elif isinstance(payload.get("dimension"), dict):
        dimension_cfg = dict(payload["dimension"])
        eval_model_name = str(dimension_cfg.get("eval_model") or "")
        if eval_model_name:
            try:
                eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name)
                if isinstance(eval_model, dict):
                    dimension_cfg["api_url"] = eval_model.get("api_url", "")
                    dimension_cfg["api_key"] = eval_model.get("api_key", "")
                    dimension_cfg["api_model"] = (
                        eval_model.get("model_name")
                        or eval_model.get("online_model_name")
                        or eval_model.get("name")
                        or eval_model_name
                    )
            except KeyError:
                pass

    # 5. Select compute node: 优先页面选择的节点 / 模型所在节点，避免多节点时选错
    preferred_node_id = payload.get("compute_node_id") or payload.get("node_id") or model_node_id
    node = _select_eval_node(store, preferred_node_id)
    if not node:
        message = "no online compute node" if not preferred_node_id else f"model compute node not schedulable: {preferred_node_id}"
        store.update_eval_task(task["id"], {"status": "failed", "error": message})
        return ok({"task_id": task["id"], "status": "failed", "error": message})

    if not is_admin(current_user) and not store.check_gpu_access(current_user["id"], node["id"], gpu_indices):
        message = "selected GPU is not assigned to the current user"
        store.update_eval_task(task["id"], {"status": "failed", "error": message})
        return ok({"task_id": task["id"], "status": "failed", "error": message})

    if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
        try:
            prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node)
            if prepared_model:
                model_path = prepared_model
            if adapter_resource_id and model_resource_type == "model":
                prepared_adapter = await _prepare_resource_on_node(store, "trained_model", adapter_resource_id, node)
                if prepared_adapter:
                    adapter_path = prepared_adapter
            dataset_sync = await _sync_training_dataset_to_compute_node(store, node, dataset_id)
            if dataset_sync:
                dataset_path = str(dataset_sync[0].get("local_path") or dataset_path)
        except Exception as exc:
            store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"})
            return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})

    try:
        store.reserve_external_gpus(
            "eval",
            str(task["id"]),
            node["id"],
            gpu_indices,
            tenant_id=str(task.get("tenant_id") or current_user.get("tenant_id") or "default"),
        )
    except (RuntimeError, ValueError) as exc:
        message = str(exc)
        store.update_eval_task(task["id"], {"status": "failed", "error": message})
        return ok({"task_id": task["id"], "status": "failed", "error": message})

    # 6. Build eval job payload
    output_dir = f"/data/yg-ft/outputs/{task['id']}"
    job_payload = {
        "id": f"eval_{task['id']}",
        "name": task.get("eval_task_name", task["id"]),
        "engine": "eval",
        "model_name_or_path": model_path,
        "adapter_name_or_path": adapter_path,
        "template": payload.get("template", "qwen"),
        "dataset_path": dataset_path,
        "output_dir": output_dir,
        "basic_metrics": payload.get("basic_metrics", {}),
        "dimension": dimension_cfg,
        "gpu_id": gpu_indices[0],
        "gpu_indices": gpu_indices,
        "gpus": gpu_indices,
        "temperature": payload.get("temperature", 0.1),
        "max_new_tokens": payload.get("max_new_tokens", 512),
        "compute_node_id": node["id"],
    }

    # 7. Submit to compute node via create_job (uses engine="eval" path)
    try:
        client = ComputeNodeClient(node["api_base_url"])
        # Sync dataset file to compute node if needed
        if not dataset_path.startswith("/"):
            try:
                ds_files = store.training_dataset_files(dataset_id)
                if ds_files and ds_files[0].get("content"):
                    upload_result = await client.upload_file(
                        ds_files[0].get("name", "eval_data.jsonl"),
                        ds_files[0]["content"].encode("utf-8"),
                        f"datasets/{dataset_id}/{ds_files[0].get('name', 'eval_data.jsonl')}",
                        resource_type="dataset",
                        resource_id=dataset_id,
                    )
                    job_payload["dataset_path"] = upload_result.get("local_path", dataset_path)
            except Exception:
                pass

        job = await client.create_job(job_payload)
        store.update_eval_task(task["id"], {
            "status": "running",
            "compute_job_id": job.get("id"),
            "compute_node_id": node["id"],
            "gpu_indices": gpu_indices,
            "gpus": gpu_indices,
            "output_dir": output_dir,
        })
        # 评测占用 GPU 由 eval_tasks 派生（gpus()/compute_nodes() 直接统计），
        # 不再复用 mark_inference_loaded 内存标记，避免删除评测后 GPU 状态残留 busy
        return ok({"task_id": task["id"], "status": "running", "job": job})
    except Exception as exc:
        store.release_external_gpus("eval", str(task["id"]), node["id"])
        store.update_eval_task(task["id"], {"status": "failed", "error": str(exc)})
        return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})


@router.delete("/model-eval/{task_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="eval_task", target_name_param="task_id")
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("eval", task_id, current_user, "delete"):
        raise fail(403, "no permission to delete this eval task")
    pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}", "eval.delete")
    if pending:
        return pending
    try:
        get_platform_store().delete_eval_task(task_id)
    except KeyError:
        raise fail(404, "eval task not found")
    return ok({"deleted": task_id})


@router.get("/dimension")
async def dimension_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return ok(get_platform_store().dimensions())


@router.post("/dimension")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.CREATE, target_type="dimension", target_name_param="name")
async def dimension_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(get_platform_store().create_dimension(payload))


@router.get("/dimension/{dimension_id}")
async def dimension_detail(dimension_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().dimension(dimension_id))
    except KeyError:
        raise fail(404, "dimension not found")


@router.put("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.UPDATE, target_type="dimension", target_name_param="dimension_id")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_dimension(dimension_id, payload))
    except KeyError:
        raise fail(404, "dimension not found")


@router.delete("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="dimension", target_name_param="dimension_id")
async def dimension_delete(dimension_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    get_platform_store().delete_dimension(dimension_id)
    return ok({"deleted": dimension_id})


@router.get("/model-compare")
async def model_compare_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    tasks = get_platform_store().compare_tasks()
    if is_admin(current_user):
        return ok(tasks)
    tenant_id = current_user.get("tenant_id") or "default"
    tasks = [task for task in tasks if task.get("tenant_id") in {None, "", tenant_id}]
    # 普通用户可见：自己创建的 + ACL 授权的
    user_id = current_user.get("id")
    accessible = filter_accessible_resource_ids_batch("compare", [item["id"] for item in tasks], current_user)
    result = [item for item in tasks if item.get("created_by") == user_id or item["id"] in accessible]
    return ok(result)


@router.post("/model-compare")
@op_log(module=OpModule.INFERENCE, action=OpAction.CREATE, target_type="compare_task", target_name_param="name")
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    bind_active_tenant(payload, current_user)
    payload.setdefault("created_by", current_user.get("id"))
    payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
    try:
        get_platform_store().assert_active_tenant(payload["tenant_id"])
    except ValueError as exc:
        raise fail(400, str(exc))
    model_ids = payload.get("model_ids") or payload.get("models") or []
    # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权即可用于推理
    if not is_admin(current_user):
        for item in model_ids:
            model_id = str(item.get("model_id") if isinstance(item, dict) else item)
            trained_model = next(
                (model for model in get_platform_store().trained_models() if str(model.get("id")) == model_id or str(model.get("name")) == model_id),
                None,
            )
            if trained_model and not has_resource_access("trained_model", str(trained_model["id"]), current_user, "execute"):
                raise fail(403, f"no permission to use trained model: {model_id}")
    if not payload.get("resource_versions"):
        payload["resource_versions"] = _capture_resource_versions(get_platform_store(), payload)
    task = get_platform_store().create_compare_task(payload)
    return ok({"id": task["id"]})


@router.post("/model-compare/all/stop-all")
async def model_compare_stop_all(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok({"stopped": True})


@router.post("/model-compare/stop-by-pid")
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok({"stopped": True, "pid": payload.get("pid")})


@router.get("/model-compare/{task_id}")
async def model_compare_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        task = get_platform_store().compare_task(task_id)
        if not has_resource_access("compare", task_id, current_user, "read"):
            raise fail(403, "no permission to access inference task")
        return ok(task)
    except KeyError:
        raise fail(404, "compare task not found")


async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = None) -> dict[str, Any]:
    """Best-effort unload the inference model from the node(s) that hold it.

    任务感知：优先卸载 ``task.load_status.loaded_models`` 中记录的节点；
    无任务时回退到平台记录的已加载推理的节点。每个节点使用短超时，
    保证卸载永远不会长时间阻塞调用方（例如删除操作）。
    """
    node_ids: set[str] = set()
    if task:
        load_status = task.get("load_status") or {}
        if isinstance(load_status, str):
            try:
                load_status = json.loads(load_status)
            except json.JSONDecodeError:
                load_status = {}
        node_ids = {item.get("node_id") for item in load_status.get("loaded_models") or [] if item.get("node_id")}
    if not node_ids:
        node_ids = {node["id"] for node in store.compute_nodes() if store.is_inference_loaded(node["id"])}
    nodes = [node for node in store.compute_nodes() if node["id"] in node_ids]
    results: list[dict[str, Any]] = []
    for node in nodes:
        try:
            result = await ComputeNodeClient(node["api_base_url"]).inference_unload()
            results.append({"node_id": node["id"], "node_code": node.get("code"), "success": True, "result": result})
        except Exception as exc:  # noqa: BLE001 - best-effort unload must not raise
            results.append({"node_id": node["id"], "node_code": node.get("code"), "success": False, "error": str(exc)})
        finally:
            store.mark_inference_unloaded(node["id"])
            if task:
                store.release_external_gpus("inference", str(task.get("id") or ""), node["id"])
    return {"unloaded": bool(results), "nodes": results}


@router.delete("/model-compare/{task_id}")
@op_log(module=OpModule.INFERENCE, action=OpAction.DELETE, target_type="inference", target_name_param="task_id")
async def model_compare_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    # 先删记录（快），再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞
    try:
        task = get_platform_store().compare_task(task_id)
    except KeyError:
        raise fail(404, "compare task not found")
    # 先判断是否为任务创建者本人：如果是，直接允许删除（不需要 ACL 授权也不需要审批）
    is_owner = False
    payload_obj = task
    if isinstance(payload_obj, dict):
        is_owner = (payload_obj.get("created_by") == current_user.get("id"))
    else:
        try:
            import json
            payload_str = str(payload_obj.get("payload", "{}") or "{}")
            payload_obj = json.loads(payload_str) if payload_str.startswith("{") else {}
            is_owner = (payload_obj.get("created_by") == current_user.get("id"))
        except (TypeError, ValueError):
            pass

    # 管理员或任务创建者：直接删除
    # 其他用户：需要 ACL delete 权限 + 审批流程
    if not is_admin(current_user) and not is_owner:
        if not has_resource_access("compare", task_id, current_user, "delete"):
            raise fail(403, "no permission to delete inference task")
        pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}", "inference.delete")
        if pending:
            return pending
    get_platform_store().delete_compare_task(task_id)
    try:
        await _unload_from_compute_node(get_platform_store(), task=task)
    except Exception:  # noqa: BLE001 - deletion must succeed even if unload fails
        pass
    return ok({"deleted": task_id})


@router.get("/model-compare/{task_id}/load-status")
async def model_compare_load_status(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        task = get_platform_store().compare_task(task_id)
    except KeyError:
        raise fail(404, "compare task not found")
    if not has_resource_access("compare", task_id, current_user, "read"):
        raise fail(403, "no permission to access inference status")
    load_status = task.get("load_status") or {"loaded_models": []}
    if isinstance(load_status, str):
        try:
            load_status = json.loads(load_status)
        except json.JSONDecodeError:
            load_status = {"loaded_models": []}
    return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})


@router.post("/model-compare/{task_id}/load-status")
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("compare", task_id, current_user, "write"):
        raise fail(403, "no permission to update inference status")
    try:
        return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
    except KeyError:
        raise fail(404, "compare task not found")


def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[dict[str, Any]]) -> None:
    """同一计算节点同一时刻只能加载一个推理模型。

    当新任务把模型派发到了某节点后，把其它任务中在该节点上 ready/running
    的模型标记为已被替换，保持平台 DB 与计算节点实际状态一致。
    """
    taken_node_ids = {m.get("node_id") for m in loaded_models if m.get("node_id") and m.get("status") == "starting"}
    if not taken_node_ids:
        return
    for other in store.compare_tasks():
        if str(other.get("id")) == str(task_id):
            continue
        load_status = other.get("load_status") or {}
        if isinstance(load_status, str):
            try:
                load_status = json.loads(load_status)
            except json.JSONDecodeError:
                load_status = {}
        items = load_status.get("loaded_models") or []
        changed = False
        for item in items:
            if item.get("node_id") in taken_node_ids and item.get("status") in {"ready", "running"}:
                item["status"] = "error"
                item["error"] = "模型已被其他推理任务替换"
                store.release_external_gpus("inference", str(other["id"]), item.get("node_id"))
                changed = True
        if changed:
            new_status = "loaded" if any(i.get("status") in {"ready", "running"} for i in items) else "failed"
            store.update_compare_task(other["id"], {"status": new_status, "load_status": {"loaded_models": items}})


@router.post("/model-compare/{task_id}/load")
@op_log(module=OpModule.INFERENCE, action=OpAction.START, target_type="inference", target_name_param="task_id")
async def model_compare_load(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """异步派发模型加载到算力节点，立即返回。

    加载进度由轮询对账器（compute_poller → reconcile_inference_loads）推进：
    任务项先以 status=starting 记录，对账器查询节点 /inference/status 后
    推进到 ready/error。这里只负责把加载请求派发出去，绝不同步等待加载完成。
    """
    try:
        store = get_platform_store()
        task = store.compare_task(task_id)
        # 先判断是否为任务创建者本人或管理员：如果是，直接允许操作
        is_owner = False
        if isinstance(task, dict):
            is_owner = (task.get("created_by") == current_user.get("id"))
        if not is_admin(current_user) and not is_owner:
            if not has_resource_access("compare", task_id, current_user, "execute"):
                raise fail(403, "no permission to load inference task")
        models = task.get("models") or []
        if isinstance(models, str):
            try:
                models = json.loads(models)
            except json.JSONDecodeError:
                models = []
        online_nodes = _candidate_online_nodes(store)
        if not online_nodes:
            return ok({"status": "failed", "error": "no online compute node"})
        loaded_models = []
        for item in models:
            if not isinstance(item, dict):
                continue
            preferred_node_id = item.get("node_id") or item.get("compute_node_id")
            model_path = item.get("model_path", "")
            if not model_path:
                # 尝试从模型库获取路径
                model_id = item.get("model_id", "")
                try:
                    db_model = store.model(model_id)
                    model_path = db_model.get("path", "")
                except KeyError:
                    trained_model = next((m for m in store.trained_models() if str(m.get("id")) == str(model_id)), None)
                    if trained_model:
                        model_path = trained_model.get("merged_path") or trained_model.get("artifact_dir") or ""
                        preferred_node_id = preferred_node_id or trained_model.get("compute_node_id")
            if not model_path:
                loaded_models.append({**item, "status": "error", "error": "model_path not found"})
                continue
            load_payload = {
                "model_name_or_path": model_path,
                "template": item.get("template", "qwen"),
            }
            try:
                item_gpu_indices = _normalize_gpu_indices(item)
            except ValueError as exc:
                loaded_models.append({**item, "status": "error", "error": str(exc)})
                continue
            if not item_gpu_indices:
                loaded_models.append({**item, "status": "error", "error": "no GPU selected"})
                continue
            load_payload["gpu_indices"] = item_gpu_indices
            if item.get("adapter_path"):
                load_payload["adapter_name_or_path"] = item["adapter_path"]
            if get_settings().compute_mode == "simulator":
                loaded_models.append({**item, "status": "ready", "node_id": "", "node_name": ""})
                continue
            # 只派发：HTTP 响应成功即视为已接受（节点会异步加载），loaded 字段忽略
            item_dispatched = False
            errors = []
            candidate_nodes = _candidate_online_nodes(store, preferred_node_id)
            if preferred_node_id:
                candidate_nodes = candidate_nodes[:1]
            for node in candidate_nodes:
                try:
                    if not is_admin(current_user) and not store.check_gpu_access(current_user["id"], node["id"], item_gpu_indices):
                        raise PermissionError("selected GPU is not assigned to the current user")
                    store.reserve_external_gpus(
                        "inference",
                        str(task_id),
                        node["id"],
                        item_gpu_indices,
                        tenant_id=str(current_user.get("tenant_id") or "default"),
                    )
                    if get_settings().minio_enabled:
                        await _wait_for_object_storage()
                    client = ComputeNodeClient(node["api_base_url"])
                    await client.inference_load(load_payload)
                    store.mark_inference_loaded(node["id"], item_gpu_indices)
                    loaded_models.append({**item, "gpu_indices": item_gpu_indices, "gpus": item_gpu_indices, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
                    item_dispatched = True
                    break
                except Exception as exc:  # noqa: BLE001 - try next candidate node
                    store.release_external_gpus("inference", str(task_id), node["id"])
                    errors.append(f"{node.get('name') or node.get('code')}: {exc}")
            if not item_dispatched:
                loaded_models.append({**item, "status": "error", "error": "; ".join(errors) or "load dispatch failed"})
        if any(m.get("status") == "starting" for m in loaded_models):
            status = "starting"
        elif any(m.get("status") == "error" for m in loaded_models):
            status = "failed"
        else:
            status = "loaded"
        updated = store.update_compare_task(task_id, {"status": status, "load_status": {"loaded_models": loaded_models}})
        # 同一节点同一时刻只能有一个推理模型；新任务占用了节点后，把其它任务上该节点的模型标记为已被替换
        _invalidate_superseded_models(store, task_id, loaded_models)
        return ok(updated)
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/unload")
async def model_compare_unload(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    try:
        store = get_platform_store()
        task = store.compare_task(task_id)
        if not has_resource_access("compare", task_id, current_user, "write"):
            raise fail(403, "no permission to unload inference task")
        # 任务感知卸载：只释放该任务实际加载到的节点，短超时快速返回
        unload_result = await _unload_from_compute_node(store, task=task)
        updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
        return ok({"task": updated, "unload": unload_result})
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/start-model")
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access("compare", task_id, current_user, "execute"):
        raise fail(403, "no permission to start inference task")
    return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})


@router.post("/model-compare/chat-with-port")
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Proxy non-streaming chat to the compute node running the inference model."""
    _authorize_inference_payload(payload, current_user)
    store = get_platform_store()
    node = _node_for_inference_payload(store, payload)
    if not node:
        return ok({"response": "no online compute node available for inference", "request": payload})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/chat", json_data=_build_messages_payload(payload))
        return ok(result)
    except Exception as exc:
        return ok({"response": f"inference failed: {exc}", "request": payload})


@router.post("/model-compare/stream-chat")
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> StreamingResponse:
    """Stream chat from the compute node (SSE proxy)."""
    _authorize_inference_payload(payload, current_user)
    return await _stream_chat_proxy(payload)


@router.post("/model-chat/batch")
async def model_chat_batch(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    _authorize_inference_payload(payload, current_user)
    return ok({"responses": [], "request": payload})


@router.post("/model-chat/local/chat")
async def model_chat_local(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Proxy chat to the compute node running the inference model."""
    _authorize_inference_payload(payload, current_user)
    store = get_platform_store()
    node = _node_for_inference_payload(store, payload)
    if not node:
        return ok({"response": "no online compute node available for inference", "request": payload})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/chat", json_data=_build_messages_payload(payload))
        return ok(result)
    except Exception as exc:
        return ok({"response": f"inference failed: {exc}", "request": payload})


@router.post("/model-chat/local/chat/stream")
async def model_chat_local_stream(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> StreamingResponse:
    """Stream chat from the compute node."""
    _authorize_inference_payload(payload, current_user)
    return await _stream_chat_proxy(payload)


@router.post("/model-chat/local/preload")
async def model_chat_local_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Load a model on the compute node for inference."""
    _authorize_inference_payload(payload, current_user)
    model_path = (payload.get("model_name_or_path") or "").strip()
    if not model_path:
        return ok({"loaded": False, "error": "model_name_or_path is required"})
    store = get_platform_store()
    node = _node_for_inference_payload(store, payload)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        # 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
        result = await client.inference_load(payload)
        if result.get("loaded") or result.get("status") in {"loading", "ready"}:
            store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.post("/model-chat/local/unload")
async def model_chat_local_unload(payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Unload the inference model from the compute node."""
    store = get_platform_store()
    task_id = str((payload or {}).get("task_id") or (payload or {}).get("compare_task_id") or "")
    if task_id and not has_resource_access("compare", task_id, current_user, "execute"):
        raise fail(403, "no permission to unload inference task")
    if not task_id and not is_admin(current_user):
        raise fail(403, "task_id is required for non-administrator unload")
    # 释放所有已加载推理的节点（短超时，best-effort）
    results: list[dict[str, Any]] = []
    for n in store.compute_nodes():
        if not store.is_inference_loaded(n["id"]):
            continue
        try:
            client = ComputeNodeClient(n["api_base_url"])
            last_error = ""
            result = None
            for attempt in range(3):
                try:
                    result = await client.inference_unload()
                    break
                except Exception as exc:  # noqa: BLE001 - retry node cleanup
                    last_error = str(exc)
                    if attempt < 2:
                        await asyncio.sleep(2 ** attempt)
            if result is None:
                raise RuntimeError(last_error or "inference unload failed")
            results.append({"node_id": n["id"], "success": True, "result": result})
        except Exception as exc:  # noqa: BLE001 - best-effort unload
            results.append({"node_id": n["id"], "success": False, "error": str(exc)})
        finally:
            store.mark_inference_unloaded(n["id"])
    return ok({"unloaded": True, "nodes": results})


@router.get("/model-chat/local/status")
async def model_chat_local_status(
    task_id: str | None = Query(default=None),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    """Get inference session status from compute node."""
    if not is_admin(current_user):
        if not task_id:
            raise fail(403, "task_id is required for non-administrator status queries")
        if not has_resource_access("compare", task_id, current_user, "read"):
            raise fail(403, "no permission to inspect inference status")
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client.inference_status()
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.post("/model-chat/trained/preload")
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    resource_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or "")
    # 训练模型必须对应一个仍然存在的资源。管理员可以兼容历史的
    # 直接路径调用，普通用户不能绕过资源 ACL 直接加载任意节点路径。
    trained_resource = resource_record("trained_model", resource_id) if resource_id else None
    if resource_id and not trained_resource:
        raise fail(404, "trained model not found")
    if not resource_id and not is_admin(current_user):
        raise fail(403, "trained_model_id is required for non-administrator preload")
    if trained_resource and not has_resource_access("trained_model", resource_id, current_user, "execute"):
        raise fail(403, "no permission to load this trained model")
    """Load a trained model (base + adapter) on the compute node for inference."""
    model_path = (payload.get("model_name_or_path") or "").strip()
    if not model_path:
        return ok({"loaded": False, "error": "model_name_or_path is required"})
    store = get_platform_store()
    requested_node_id = str(payload.get("compute_node_id") or payload.get("node_id") or "")
    node = next((item for item in store.compute_nodes() if item.get("id") == requested_node_id and item.get("enabled") and item.get("scheduler_status") == "online"), None)
    if not node:
        node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        prepared_path = await _prepare_resource_on_node(store, "trained_model", str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or ""), node)
        if prepared_path:
            payload = {**payload, "model_name_or_path": prepared_path}
        prepared_path = await _prepare_resource_on_node(store, "model", str(payload.get("model_id") or payload.get("resource_id") or ""), node)
        if prepared_path:
            payload = {**payload, "model_name_or_path": prepared_path}
        client = ComputeNodeClient(node["api_base_url"])
        # 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
        result = await client.inference_load({**payload, "compute_node_id": node["id"]})
        if result.get("loaded") or result.get("status") in {"loading", "ready"}:
            store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.get("/compute/nodes")
async def compute_nodes(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    nodes = get_platform_store().compute_nodes()
    if is_admin(current_user):
        return ok(nodes)
    # 普通用户只看到自己被分配 GPU 的节点，避免泄露节点拓扑和未授权资源。
    assigned = {item["node_id"] for item in get_platform_store().gpu_assignments_for_user(current_user["id"])}
    return ok([node for node in nodes if node["id"] in assigned])


@router.post("/storage/objects/presign")
async def presign_storage_object(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Create a short-lived MinIO upload/download URL for a platform resource."""
    if not get_settings().minio_enabled:
        raise fail(503, "MinIO object storage is disabled")
    resource_type = str(payload.get("resource_type") or "")
    resource_id = str(payload.get("resource_id") or "")
    method = str(payload.get("method") or "put").lower()
    if resource_type not in _STORAGE_RESOURCE_ROOTS or not resource_id:
        raise fail(400, "unsupported resource_type or missing resource_id")
    if not _STORAGE_ID_PATTERN.fullmatch(resource_id):
        raise fail(400, "invalid resource_id")
    if not _storage_resource_row(get_platform_store(), resource_type, resource_id):
        raise fail(404, "storage resource not found")
    if method not in {"put", "get"}:
        raise fail(400, "method must be put or get")
    if not has_resource_access(resource_type, resource_id, current_user, "download" if method == "get" else "write"):
        raise fail(403, f"no permission to {method} this resource")
    if resource_type == "model" and method == "put" and not is_admin(current_user):
        raise fail(403, "only administrators can upload base model objects")
    version_id = str(payload.get("version_id") or uuid.uuid4().hex)
    if not _STORAGE_ID_PATTERN.fullmatch(version_id):
        raise fail(400, "invalid version_id")
    try:
        storage = get_object_storage()
        store = get_platform_store()
        if method == "get":
            object_key = _validate_storage_key(resource_type, resource_id, version_id, payload.get("object_key"), payload.get("file_name"))
            record = next(
                (item for item in store.storage_objects_for_resource(resource_type, resource_id) if item.get("object_key") == object_key),
                None,
            )
            if not record:
                raise fail(404, "storage object not found")
            url = storage.presigned_get(object_key)
        else:
            object_key = _validate_storage_key(resource_type, resource_id, version_id, payload.get("object_key"), payload.get("file_name"))
            expected_size = int(payload.get("byte_size") or payload.get("size_bytes") or 0)
            if expected_size < 0 or expected_size > get_settings().minio_presign_max_bytes:
                raise fail(400, "invalid byte_size")
            content_type = str(payload.get("content_type") or "application/octet-stream").split(";", 1)[0].strip().lower()
            if content_type not in _STORAGE_CONTENT_TYPES and not content_type.startswith(("application/vnd.", "text/", "image/")):
                raise fail(400, "unsupported content_type")
            url = storage.presigned_put(object_key)
            record = store.create_storage_object({
                "resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
                "bucket": storage.bucket, "object_key": object_key, "file_name": payload.get("file_name"),
                "content_type": content_type, "byte_size": expected_size,
                "metadata": {
                    "expected_size": expected_size,
                    "expected_checksum_sha256": payload.get("checksum_sha256") or "",
                    "upload_created_at": utcnow(),
                    "upload_expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(),
                },
                "created_by": current_user.get("id"),
            })
        return ok({"url": url, "method": method, "expires_seconds": 3600, "object": record})
    except ObjectStorageError as exc:
        raise fail(503, str(exc))
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/storage/objects/{object_id}/complete")
async def complete_storage_object(
    object_id: str,
    payload: dict[str, Any] = Body(default={}),
    current_user: dict[str, Any] = Depends(get_current_user),
) -> dict[str, Any]:
    """Verify a presigned upload and make it visible to resource consumers."""
    if not get_settings().minio_enabled:
        raise fail(503, "MinIO object storage is disabled")
    store = get_platform_store()
    try:
        record = store.storage_object(object_id)
    except KeyError:
        raise fail(404, "storage object not found")
    if not has_resource_access(str(record["resource_type"]), str(record["resource_id"]), current_user, "write"):
        raise fail(403, "no permission to complete this upload")
    if record.get("status") == "available":
        return ok(record)
    try:
        metadata = record.get("metadata") or {}
        if isinstance(metadata, str):
            metadata = json.loads(metadata or "{}")
        expires_at = str(metadata.get("upload_expires_at") or "")
        if expires_at:
            try:
                if datetime.fromisoformat(expires_at.replace("Z", "+00:00")) <= datetime.now(timezone.utc):
                    raise fail(409, "presigned upload has expired")
            except ValueError:
                pass
        stat = get_object_storage().stat(str(record["object_key"]))
    except (ObjectStorageError, ValueError, json.JSONDecodeError) as exc:
        raise fail(409, f"uploaded object is not available: {exc}")
    expected_size = int(metadata.get("expected_size") or record.get("byte_size") or 0)
    actual_size = int(stat.get("byte_size") or 0)
    expected_checksum = str(metadata.get("expected_checksum_sha256") or "").lower()
    supplied_checksum = str(payload.get("checksum_sha256") or "").lower()
    if expected_checksum and supplied_checksum and supplied_checksum != expected_checksum:
        raise fail(409, "uploaded object checksum does not match presign request")
    if expected_checksum:
        digest = hashlib.sha256()
        try:
            for chunk in get_object_storage().iter_bytes(str(record["object_key"])):
                digest.update(chunk)
        except ObjectStorageError as exc:
            raise fail(409, f"uploaded object checksum could not be verified: {exc}")
        if digest.hexdigest().lower() != expected_checksum:
            raise fail(409, "uploaded object content checksum does not match presign request")
    if actual_size > get_settings().minio_presign_max_bytes:
        raise fail(409, "uploaded object exceeds configured size limit")
    if expected_size and actual_size != expected_size:
        raise fail(409, f"uploaded object size mismatch: expected {expected_size}, got {actual_size}")
    supplied_size = payload.get("byte_size")
    if supplied_size is not None and int(supplied_size) != actual_size:
        raise fail(409, "uploaded object size does not match completion payload")
    completed_metadata = {**metadata, "etag": stat.get("etag") or "", "completed_at": utcnow()}
    completed = store.update_storage_object(
        object_id,
        {
            "status": "available",
            "byte_size": actual_size,
            "checksum_sha256": payload.get("checksum_sha256") or record.get("checksum_sha256") or "",
            "metadata": completed_metadata,
        },
    )
    return ok(completed)


@router.get("/storage/resources/{resource_type}/{resource_id}")
async def storage_resource_objects(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not has_resource_access(resource_type, resource_id, current_user, "read"):
        raise fail(403, "no permission to read this resource")
    return ok(get_platform_store().storage_objects_for_resource(resource_type, resource_id))


@router.get("/storage/resources/{resource_type}/{resource_id}/manifest")
async def storage_resource_manifest(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    """Return the immutable object/version manifest used by training and inference."""
    if resource_type not in _STORAGE_RESOURCE_TABLES or not _storage_resource_row(get_platform_store(), resource_type, resource_id):
        raise fail(404, "storage resource not found")
    if not has_resource_access(resource_type, resource_id, current_user, "read"):
        raise fail(403, "no permission to read this resource")
    objects = get_platform_store().storage_objects_for_resource(resource_type, resource_id)
    versions: dict[str, dict[str, Any]] = {}
    for item in objects:
        version_id = str(item.get("version_id") or "legacy")
        entry = versions.setdefault(version_id, {"version_id": version_id, "objects": [], "total_bytes": 0})
        manifest_item = {
            "id": item.get("id"),
            "object_key": item.get("object_key"),
            "file_name": item.get("file_name"),
            "content_type": item.get("content_type"),
            "byte_size": int(item.get("byte_size") or 0),
            "checksum_sha256": item.get("checksum_sha256") or "",
            "status": item.get("status"),
        }
        entry["objects"].append(manifest_item)
        entry["total_bytes"] += manifest_item["byte_size"]
    return ok({"resource_type": resource_type, "resource_id": resource_id, "versions": list(versions.values())})


@router.post("/storage/resources/{resource_type}/{resource_id}/prepare/{node_id}")
async def prepare_storage_resource(
    resource_type: str,
    resource_id: str,
    node_id: str,
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    if not has_resource_access(resource_type, resource_id, current_user, "execute"):
        raise fail(403, "no permission to execute this resource")
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    if not get_settings().minio_enabled:
        raise fail(503, "MinIO object storage is disabled")
    objects = store.storage_objects_for_resource(resource_type, resource_id)
    if not objects:
        raise fail(404, "resource has no MinIO objects")
    client = ComputeNodeClient(node["api_base_url"])
    prepared = []
    for obj in objects:
        url = get_object_storage().presigned_get(obj["object_key"])
        filename = Path(str(obj.get("file_name") or obj["object_key"])).name
        cache_job = store.create_storage_cache_job({
            "storage_object_id": obj["id"],
            "node_id": node_id,
            "direction": "download",
            "version_id": obj.get("version_id"),
            "checksum_sha256": obj.get("checksum_sha256") or "",
            "byte_size": obj.get("byte_size") or 0,
            "protected_until": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(),
        })
        try:
            result = await client.prepare_cache({
                "resource_id": resource_id,
                "version_id": obj["version_id"],
                "download_url": url,
                "checksum_sha256": obj.get("checksum_sha256") or "",
                "byte_size": obj.get("byte_size") or 0,
                "relative_path": f"{resource_type}s/{resource_id}/{filename}",
            })
            store.update_storage_cache_job(cache_job["id"], {
                "status": "completed", "progress": 100, "local_path": result.get("local_path"),
                "last_accessed_at": utcnow(), "completed_at": utcnow(),
            })
        except Exception as exc:
            store.update_storage_cache_job(cache_job["id"], {"status": "failed", "error": str(exc), "completed_at": utcnow()})
            raise
        prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
    return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})


@router.get("/storage/cache/jobs/{node_id}")
async def storage_cache_jobs(node_id: str, limit: int = Query(default=100, ge=1, le=500), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    if not is_admin(current_user):
        assigned_nodes = {item["node_id"] for item in store.gpu_assignments_for_user(str(current_user.get("id") or ""))}
        if node_id not in assigned_nodes:
            raise fail(403, "no permission to inspect this compute node cache")
    jobs = store.storage_cache_jobs_for_node(node_id, limit)
    if not is_admin(current_user):
        visible = []
        for job in jobs:
            try:
                obj = store.storage_object(str(job.get("storage_object_id") or ""))
            except KeyError:
                continue
            if has_resource_access(str(obj.get("resource_type") or ""), str(obj.get("resource_id") or ""), current_user, "read"):
                visible.append(job)
        jobs = visible
    return ok(jobs)


@router.post("/storage/cleanup")
async def cleanup_storage_objects(
    payload: dict[str, Any] = Body(default={}),
    current_user: dict = Depends(require_admin),
) -> dict[str, Any]:
    """Purge MinIO objects whose business resources were soft-deleted."""
    store = get_platform_store()
    limit = max(1, min(int(payload.get("limit") or 100), 1000))
    dry_run = bool(payload.get("dry_run", False))
    candidates = store.storage_objects_pending_cleanup(limit)
    if dry_run:
        return ok({"dry_run": True, "total": len(candidates), "items": candidates})
    results: list[dict[str, Any]] = []
    for item in candidates:
        job = store.create_storage_cleanup_job(str(item["id"]))
        try:
            get_object_storage().delete(str(item["object_key"]))
            store.finish_storage_cleanup_job(job["id"], True)
            results.append({"storage_object_id": item["id"], "status": "purged"})
        except Exception as exc:  # noqa: BLE001 - retain failed cleanup for retry
            store.finish_storage_cleanup_job(job["id"], False, str(exc))
            results.append({"storage_object_id": item["id"], "status": "failed", "error": str(exc)})
    return ok({"dry_run": False, "total": len(results), "items": results})


@router.get("/storage/orphans")
async def storage_orphans(
    cleanup: bool = Query(default=False),
    limit: int = Query(default=500, ge=1, le=5000),
    current_user: dict[str, Any] = Depends(require_admin),
) -> dict[str, Any]:
    """Find MinIO objects that have no database storage_objects reference."""
    store = get_platform_store()
    with store.connect() as conn:
        known_rows = conn.execute("SELECT object_key FROM storage_objects WHERE status <> 'purged'").fetchall()
    known = {str(row["object_key"]) for row in known_rows}
    all_objects: list[dict[str, Any]] = []
    try:
        for root in sorted(set(_STORAGE_RESOURCE_ROOTS.values())):
            all_objects.extend(get_object_storage().list_objects(f"{root}/"))
    except ObjectStorageError as exc:
        raise fail(503, str(exc))
    orphans = [item for item in all_objects if str(item.get("object_key") or "") not in known][:limit]
    deleted: list[str] = []
    if cleanup:
        for item in orphans:
            key = str(item.get("object_key") or "")
            if not key:
                continue
            try:
                get_object_storage().delete(key)
                deleted.append(key)
            except Exception:
                continue
    return ok({"total": len(orphans), "items": orphans, "cleanup": cleanup, "deleted": deleted})


@router.post("/storage/resources/{resource_type}/{resource_id}/archive-node/{node_id}")
async def archive_node_files(
    resource_type: str,
    resource_id: str,
    node_id: str,
    payload: dict[str, Any] = Body(...),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    """Archive completed node files to MinIO without proxying file bytes through Backend."""
    if not has_resource_access(resource_type, resource_id, current_user, "execute"):
        raise fail(403, "no permission to archive this resource")
    store = get_platform_store()
    if not is_admin(current_user):
        model_id = str(payload.get("model_id") or "")
        dataset_id = str(payload.get("dataset_id") or "")
        # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权
        if not model_id:
            raise fail(400, "model_id is required")
        if not dataset_id or not has_resource_access("dataset", dataset_id, current_user, "execute"):
            raise fail(403, "no permission to evaluate this dataset")
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    files = payload.get("files") or []
    if not isinstance(files, list) or not files:
        raise fail(400, "files is required")
    client = ComputeNodeClient(node["api_base_url"], timeout=900)
    archived = []
    for item in files:
        path = str(item.get("path") or "")
        name = Path(str(item.get("file_name") or Path(path).name)).name
        version_id = str(item.get("version_id") or uuid.uuid4().hex)
        object_key = str(item.get("object_key") or f"{resource_type}s/{resource_id}/versions/{version_id}/{name}")
        url = get_object_storage().presigned_put(object_key)
        result = await client.upload_file_to_url(path, url, object_key, str(item.get("content_type") or "application/octet-stream"))
        metadata = get_object_storage().stat(object_key)
        record = store.create_storage_object({
            "resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
            "bucket": get_object_storage().bucket, "object_key": object_key, "file_name": name,
            "content_type": item.get("content_type"), "checksum_sha256": result.get("checksum_sha256"),
            "byte_size": metadata.get("byte_size") or result.get("byte_size") or 0, "status": "available",
            "created_by": current_user.get("id"),
        })
        archived.append({"object": record, "node_id": node_id})
    return ok({"status": "available", "items": archived})


@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    if not is_admin(current_user) and not any(item["node_id"] == node_id for item in store.gpu_assignments_for_user(current_user["id"])):
        raise fail(403, "no permission to access this compute node")
    return ok(node)


@router.post("/compute/nodes")
@op_log(module=OpModule.COMPUTE, action=OpAction.CREATE, target_type="compute_node", target_name_param="name")
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    try:
        return ok(get_platform_store().create_compute_node(payload))
    except KeyError as exc:
        raise fail(400, f"missing field: {exc}")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.put("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.UPDATE, target_type="compute_node", target_name_param="node_id")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    try:
        return ok(get_platform_store().update_compute_node(node_id, payload))
    except KeyError:
        raise fail(404, "compute node not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.delete("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.DELETE, target_type="compute_node", target_name_param="node_id")
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    try:
        return ok(get_platform_store().delete_compute_node(node_id))
    except KeyError:
        raise fail(404, "compute node not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    client = ComputeNodeClient(node["api_base_url"])
    try:
        result = await client.test_connection()
        store.replace_node_gpus(node_id, result["gpus"])
        updated = store.update_compute_node_health(node_id, result["health"], True)
        return ok(
            {
                "node_id": node_id,
                "success": True,
                "latency_ms": result["latency_ms"],
                "gpu_count": len(result["gpus"]),
                "health": updated["health_detail"],
            }
        )
    except Exception as exc:  # noqa: BLE001 - return the connection error for node maintenance
        updated = store.update_compute_node_health(node_id, {}, False, str(exc))
        return ok(
            {
                "node_id": node_id,
                "success": False,
                "latency_ms": 0,
                "gpu_count": updated.get("gpu_count", 0),
                "error": str(exc),
                "health": updated["health_detail"],
            }
        )


@router.post("/compute/nodes/{node_id}/health-check")
async def health_check_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return await test_compute_node(node_id, current_user)


@router.post("/compute/nodes/{node_id}/enable")
async def enable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))


@router.post("/compute/nodes/{node_id}/disable")
async def disable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))


@router.post("/compute/nodes/{node_id}/drain")
async def drain_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    if not is_admin(current_user):
        raise fail(403, "admin permission required")
    return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))


@router.get("/compute/nodes/{node_id}/replicas")
async def compute_node_replicas(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    _authorize_compute_node(node_id, current_user)
    return ok(get_platform_store().replicas(node_id))


@router.get("/compute/sync-jobs/{sync_id}")
async def compute_sync_job_detail(sync_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().sync_job(sync_id))
    except KeyError:
        raise fail(404, "sync job not found")


@router.get("/compute/nodes/{node_id}/replicas/drift")
async def compute_node_replica_drift(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    _authorize_compute_node(node_id, current_user)
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    replicas = store.replicas(node_id)
    if not replicas:
        return ok({"node_id": node_id, "items": [], "drifted": 0})
    paths = [
        {
            "name": replica["id"],
            "path": replica["local_path"],
            "type": "any",
            "required": True,
        }
        for replica in replicas
    ]
    try:
        result = await ComputeNodeClient(node["api_base_url"]).check_paths(paths)
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"replica drift check failed: {exc}")
    check_map = {str(item.get("name")): item for item in result.get("items") or []}
    items = []
    for replica in replicas:
        check = check_map.get(replica["id"], {})
        updated = store.update_resource_replica_check(
            replica["id"],
            bool(check.get("ok")),
            int(check.get("byte_size") or replica.get("byte_size") or 0),
            "" if check.get("ok") else f"path not available: {replica['local_path']}",
        )
        items.append({**updated, "check": check})
    return ok({"node_id": node_id, "items": items, "drifted": len([item for item in items if item.get("sync_status") == "drifted"])})


async def _run_resource_replica_repair(sync_id: str, node_id: str, payload: dict[str, Any], replicas_to_repair: list[dict[str, Any]]) -> None:
    store = get_platform_store()
    store.update_sync_job(sync_id, "running", 5)
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        store.update_sync_job(sync_id, "failed", 100, completed=True)
        return
    client = ComputeNodeClient(node["api_base_url"])
    repaired = []
    failures = []
    total = max(len(replicas_to_repair), 1)
    for index, replica in enumerate(replicas_to_repair, start=1):
        replica_id = str(replica["id"])
        resource_type = str(replica.get("resource_type") or "")
        resource_id = str(replica.get("resource_id") or "")
        try:
            if resource_type == "dataset":
                files = store.training_dataset_files(resource_id)
                if not files:
                    raise RuntimeError(f"dataset has no uploaded file: {resource_id}")
                total_size = 0
                checksum = ""
                local_path = str(replica.get("local_path") or "")
                for item in files:
                    filename = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
                    result = await client.upload_file(
                        filename,
                        str(item.get("content") or "").encode("utf-8"),
                        f"datasets/{resource_id}/{filename}",
                        resource_type="dataset",
                        resource_id=resource_id,
                    )
                    total_size += int(result.get("byte_size") or 0)
                    checksum = str(result.get("checksum_sha256") or checksum)
                    local_path = str(result.get("local_path") or local_path)
                repaired.append(store.update_resource_replica_sync_result(replica_id, True, local_path, total_size, checksum))
                continue

            source_path = ""
            target_relative_path = ""
            if resource_type == "model":
                model = store.model(resource_id)
                source_path = str(model.get("path") or "")
                target_relative_path = f"models/{Path(source_path).name}" if source_path else ""
            elif resource_type in {"trained_model", "model_artifact"}:
                if resource_type == "trained_model":
                    artifacts = store.model_artifacts(resource_id)
                    artifact = next((item for item in artifacts if item.get("path")), None)
                else:
                    artifact = store.model_artifact(resource_id)
                source_path = str((artifact or {}).get("path") or replica.get("local_path") or "")
                target_relative_path = f"outputs/{Path(source_path).name}" if source_path else ""
            else:
                source_path = str(payload.get("source_path") or replica.get("source_path") or replica.get("local_path") or "")
                target_relative_path = str(payload.get("target_relative_path") or "")

            if not source_path:
                raise RuntimeError(f"authoritative source path not found for {resource_type}:{resource_id}")
            result = await client.import_local_file(
                {
                    "source_path": source_path,
                    "target_relative_path": target_relative_path,
                    "resource_type": resource_type,
                    "resource_id": resource_id,
                }
            )
            repaired.append(
                store.update_resource_replica_sync_result(
                    replica_id,
                    True,
                    str(result.get("local_path") or replica.get("local_path") or ""),
                    int(result.get("byte_size") or 0),
                    str(result.get("checksum_sha256") or ""),
                )
            )
        except Exception as exc:  # noqa: BLE001 - collect all replica repair failures
            error = str(exc)
            failures.append({"replica_id": replica_id, "resource_type": resource_type, "resource_id": resource_id, "error": error})
            repaired.append(store.update_resource_replica_sync_result(replica_id, False, None, int(replica.get("byte_size") or 0), "", error))
        progress = min(95, 5 + int(index / total * 90))
        store.update_sync_job(sync_id, "running", progress)
    store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)


@router.post("/compute/nodes/{node_id}/replicas/repair")
async def compute_node_replica_repair(
    node_id: str,
    background_tasks: BackgroundTasks,
    payload: dict[str, Any] | None = Body(default=None),
    current_user: dict = Depends(require_admin),
) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    payload = payload or {}
    replica_ids = payload.get("replica_ids") or [
        item["id"] for item in store.replicas(node_id) if item.get("sync_status") in {"drifted", "failed", "repair_pending"}
    ]
    updated = store.mark_resource_replica_repair_pending([str(item) for item in replica_ids])
    sync_id = store.create_sync_job(
        node_id,
        {
            "operation": "repair",
            "resources": [
                {
                    "resource_type": item.get("resource_type"),
                    "resource_id": item.get("resource_id"),
                    "replica_id": item.get("id"),
                    "target_path": item.get("local_path"),
                }
                for item in updated
            ],
        },
    )
    if not updated:
        store.update_sync_job(sync_id, "completed", 100, completed=True)
        return ok({"sync": store.sync_job(sync_id), "replicas": [], "failed": [], "async": False})
    background_tasks.add_task(_run_resource_replica_repair, sync_id, node_id, payload, updated)
    return ok({"sync": store.sync_job(sync_id), "replicas": updated, "failed": [], "async": True})


@router.get("/compute/nodes/{node_id}/engines")
async def compute_node_engines(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    _authorize_compute_node(node_id, current_user)
    node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    health = node.get("health_detail") or {}
    live_error = ""
    try:
        health = await ComputeNodeClient(node["api_base_url"]).health()
    except Exception as exc:  # noqa: BLE001 - stored health is enough for offline node detail
        live_error = str(exc)
    capabilities = health.get("capabilities") or node.get("capabilities") or []
    return ok(
        {
            "node_id": node_id,
            "items": [
                {
                    "engine": "llama_factory",
                    "display_name": "LLaMA-Factory",
                    "status": "available" if "llama_factory" in capabilities else "unknown",
                    "version": health.get("llama_factory_version") or "",
                    "home": health.get("llama_factory_home") or "",
                    "home_exists": bool(health.get("llama_factory_home_exists")),
                    "capabilities": capabilities,
                    "execution_mode": health.get("execution_mode") or "",
                    "last_error": live_error,
                }
            ],
        }
    )


@router.get("/compute/gpus")
async def compute_gpus(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    gpus = store.gpus()
    if is_admin(current_user):
        return ok(gpus)
    assigned = {(item["node_id"], int(item["gpu_index"])) for item in store.gpu_assignments_for_user(current_user["id"])}
    return ok([gpu for gpu in gpus if (gpu.get("node_id"), int(gpu.get("id", gpu.get("gpu_index", -1)))) in assigned])


@router.get("/compute/queue")
async def compute_queue(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(get_platform_store().queue())


@router.get("/compute/jobs/{job_id}")
async def compute_job_detail(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if task and not has_resource_access("fine-tune", str(task["id"]), current_user, "read"):
        raise fail(403, "no permission to access this compute job")
    if not task and not is_admin(current_user):
        raise fail(403, "only administrators can access this compute job")
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        job = await ComputeNodeClient(node["api_base_url"]).get_job(job_id)
        return ok(get_platform_store().sync_model_merge_job(job_id, job))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    return ok(await ComputeNodeClient(node["api_base_url"]).get_job(job_id))


@router.post("/compute/jobs/{job_id}/stop")
@op_log(module=OpModule.COMPUTE, action=OpAction.STOP, target_type="compute_job", target_name_param="job_id")
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
        raise fail(403, "no permission to stop this compute job")
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
        return ok(get_platform_store().sync_model_merge_job(job_id, job))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
    get_platform_store().apply_compute_job(task["id"], job)
    return ok(job)


@router.get("/compute/jobs/{job_id}/logs")
async def compute_job_logs(
    job_id: str,
    tail_lines: int | None = Query(default=200, ge=1, le=5000),
    offset: int | None = Query(default=None, ge=0),
    limit: int | None = Query(default=None, ge=1, le=5000),
    current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if task and not has_resource_access("fine-tune", str(task["id"]), current_user, "read"):
        raise fail(403, "no permission to access compute job logs")
    if not task and not is_admin(current_user):
        raise fail(403, "only administrators can access compute job logs")
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))


@router.post("/compute/jobs/{job_id}/retry")
@op_log(module=OpModule.COMPUTE, action=OpAction.RETRY, target_type="compute_job", target_name_param="job_id")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    store = get_platform_store()
    payload = payload or {}
    task = _task_for_compute_job(job_id)
    if not task:
        raise fail(404, "compute job not found")
    return await retry_fine_tune(task["id"], payload, current_user)


@router.post("/compute/jobs/{job_id}/priority")
async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        raise fail(404, "compute job not found")
    priority = str(payload.get("priority") or "normal")
    return ok(get_platform_store().update_task_priority(task["id"], priority))


@router.post("/internal/compute-sync/jobs/poll")
async def poll_compute_jobs(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(await poll_compute_jobs_once())


@router.post("/internal/compute-sync/resources")
async def create_compute_sync(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    store = get_platform_store()
    node_id = payload.get("target_node_id") or payload.get("target_compute_node_id")
    if not node_id:
        raise fail(400, "target_node_id is required")
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    sync_id = store.create_sync_job(node_id, payload)
    replicas = []
    failures = []
    resources = payload.get("resources") or []
    for resource in resources:
        if not resource.get("source_path"):
            continue
        try:
            result = await ComputeNodeClient(node["api_base_url"]).import_local_file(
                {
                    "source_path": resource["source_path"],
                    "target_relative_path": resource.get("target_relative_path"),
                    "resource_type": resource.get("resource_type"),
                    "resource_id": resource.get("resource_id"),
                }
            )
            replicas.append(
                store.upsert_resource_replica(
                    node_id,
                    str(resource.get("resource_type") or "file"),
                    str(resource.get("resource_id") or result["id"]),
                    result["local_path"],
                )
            )
        except Exception as exc:  # noqa: BLE001 - collect per-resource failures
            failures.append({"resource_id": str(resource.get("resource_id")), "error": str(exc)})
    store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)
    return ok({"sync": store.sync_job(sync_id), "replicas": replicas, "failed": failures})


@router.get("/internal/compute-sync/resources/{sync_id}")
async def compute_sync_detail(sync_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().sync_job(sync_id))
    except KeyError:
        raise fail(404, "sync job not found")


@router.get("/training-log-files")
async def training_log_files(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(get_platform_store().training_log_files())


@router.get("/training-log-content")
async def training_log_content(file: str = Query(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().training_log_content(file))
    except KeyError:
        raise fail(404, "training log not found")


@router.get("/log-files")
async def log_files(date: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(get_platform_store().log_files(date))


@router.get("/log-content")
async def log_content(file: str = Query(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
    return ok(get_platform_store().log_content(file))


@router.post("/web-log")
async def web_log(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
    return ok({"received": True, **payload})