YG_FT/backend/app/api/v1/endpoints/platform.py

from __future__ import annotations

import json
import uuid
from pathlib import Path
from typing import Any

from fastapi import APIRouter, BackgroundTasks, Body, File, HTTPException, Query, UploadFile
from fastapi.responses import PlainTextResponse, StreamingResponse

import httpx

from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import poll_compute_jobs_once

router = APIRouter()


def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
    return {"code": 0, "message": message, "data": data}


def _select_first_online_node(store: Any) -> dict[str, Any] | None:
    """Select the first online compute node for inference."""
    nodes = store.compute_nodes()
    for node in nodes:
        if node.get("enabled") and node.get("scheduler_status") == "online":
            return node
    return None


def fail(status_code: int, message: str) -> HTTPException:
    return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})


def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
    return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)


def _task_for_compute_job(job_id: str) -> dict[str, Any] | None:
    return next((task for task in get_platform_store().tasks() if task.get("compute_job_id") == job_id), None)


def _node_for_compute_job_record(job_id: str) -> dict[str, Any] | None:
    store = get_platform_store()
    try:
        record = store.compute_job(job_id)
    except KeyError:
        return None
    return next((node for node in store.compute_nodes() if node["id"] == record.get("node_id")), None)


def _training_diagnostics(errors: list[str], warnings: list[str] | None = None, log_text: str = "") -> list[dict[str, str]]:
    source_items = [*errors, *(warnings or [])]
    if log_text:
        source_items.append(log_text)
    text = "\n".join(source_items).lower()
    diagnostics: list[dict[str, str]] = []
    rules = [
        (
            ["api 模型", "api模型", "api model"],
            "API 模型不能用于本地训练",
            "当前选择的基座模型为 API 类型，LLaMA-Factory 需要本地可访问的模型路径。请在模型管理中创建或选择模型来源为「本地」且配置了算力节点路径的模型。",
        ),
        (
            ["未配置算力节点", "未配置.*路径", "模型.*路径"],
            "模型缺少算力节点路径",
            "请在模型管理中编辑该模型，设置模型路径为算力节点可访问的本地目录。",
        ),
        (
            ["不支持本地训练", "not trainable"],
            "模型不可用于训练",
            "当前选择的模型不支持作为 LLaMA-Factory 训练基座。请确认模型来源为本地、路径已配置且模型目录在算力节点上存在。",
        ),
        (
            ["dataset columns missing", "keyerror", "history", "instruction", "input", "output", "messages"],
            "训练数据字段不匹配",
            "请检查所选数据集格式是否与训练模板一致。Alpaca 格式通常需要 instruction/input/output；ShareGPT 格式通常需要 messages。",
        ),
        (
            ["dataset file not found", "dataset_dir", "no uploaded file"],
            "训练数据文件不可用",
            "请确认数据集已上传文件，并且应用服务可以将数据同步到目标算力节点的数据目录。",
        ),
        (
            ["model_name_or_path path not available", "base_model", "model path", "no such file"],
            "基座模型路径不可用",
            "请在模型管理中检查本地模型路径，确保该路径在算力服务器或 Compute 容器挂载目录内真实存在。",
        ),
        (
            ["cuda out of memory", "outofmemoryerror", "显存", "memory"],
            "GPU 显存不足",
            "请降低 batch_size、cutoff_len、LoRA rank，启用 4bit 量化，或选择更高显存的算力节点。",
        ),
        (
            ["training command not found", "llamafactory-cli"],
            "训练框架命令不可用",
            "请检查 Compute 镜像是否包含 LLaMA-Factory，或确认 llamafactory-cli 已在容器 PATH 中。",
        ),
        (
            ["llama_factory_home not found"],
            "LLaMA-Factory 目录不可用",
            "请检查 Compute 服务的 LLAMA_FACTORY_HOME 配置和宿主机挂载路径。",
        ),
        (
            ["no available compute node", "not schedulable", "disabled", "capacity full"],
            "暂无可调度算力节点",
            "请检查算力节点是否启用、状态是否在线、并行任务数是否已满，或手动调整节点权重/标签。",
        ),
    ]
    for keywords, title, suggestion in rules:
        if any(keyword in text for keyword in keywords):
            diagnostics.append({"level": "error", "title": title, "suggestion": suggestion})
    if not diagnostics and (errors or log_text):
        diagnostics.append(
            {
                "level": "error",
                "title": "训练任务异常",
                "suggestion": "请查看预检错误和训练日志原文，优先确认模型路径、数据集格式、GPU 显存和 LLaMA-Factory 参数。",
            }
        )
    return diagnostics


async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
    task_id = str(payload.get("task_id") or payload.get("id") or "")
    if task_id and get_settings().compute_mode != "simulator":
        try:
            preflight = await _fine_tune_preflight(store, task_id, payload, validate=True, sync_resources=True)
        except Exception as exc:  # noqa: BLE001 - task has not entered running state yet
            raise RuntimeError(f"preflight failed: {exc}") from exc
        if not preflight["valid"]:
            errors = "; ".join(preflight.get("errors") or ["preflight failed"])
            raise RuntimeError(f"preflight failed: {errors}")
        payload = {**payload, "compute_node_id": preflight["node"]["id"]}
    task = store.start_task(payload)
    if get_settings().compute_mode == "simulator":
        return task
    node, job_payload = store.build_compute_job_payload(task["id"])
    job = await ComputeNodeClient(node["api_base_url"]).create_job(job_payload)
    return store.apply_compute_job(task["id"], job)


async def _fine_tune_preflight(
    store: Any,
    task_id: str,
    payload: dict[str, Any] | None = None,
    validate: bool = True,
    sync_resources: bool = False,
) -> dict[str, Any]:
    node, job_payload = store.prepare_compute_job_payload(task_id, payload or {})
    return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=sync_resources, store=store)


async def _fine_tune_preflight_payload(
    store: Any,
    payload: dict[str, Any],
    validate: bool = True,
) -> dict[str, Any]:
    node, job_payload = store.prepare_compute_job_payload_from_payload(payload)
    return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=False, store=store)


async def _fine_tune_preflight_with_job_payload(
    node: dict[str, Any],
    job_payload: dict[str, Any],
    validate: bool,
    sync_resources: bool,
    store: Any,
) -> dict[str, Any]:
    sync_results: list[dict[str, Any]] = []
    sync_errors: list[str] = []
    if sync_resources and get_settings().compute_mode != "simulator":
        try:
            sync_results = await _sync_training_dataset_to_compute_node(
                store,
                node,
                str(job_payload.get("train_dataset_id") or ""),
            )
        except Exception as exc:  # noqa: BLE001 - return as preflight error for page visibility
            sync_errors.append(str(exc))
    if get_settings().compute_mode == "simulator":
        preview = {
            "valid": True,
            "errors": [],
            "warnings": ["compute_mode=simulator skips remote compute validation"],
            "engine": job_payload.get("engine") or job_payload.get("training_engine") or "llama_factory",
            "command": [],
            "command_text": "",
            "work_dir": "",
            "env": {},
            "path_checks": [],
        }
    else:
        client = ComputeNodeClient(node["api_base_url"])
        preview = await (client.validate_job(job_payload) if validate else client.preview_job(job_payload))
    errors = list(preview.get("errors") or [])
    errors.extend(sync_errors)
    warnings = list(preview.get("warnings") or [])
    if not node.get("enabled"):
        errors.append(f"compute node disabled: {node.get('code')}")
    if node.get("scheduler_status") not in {"online", "draining"}:
        errors.append(f"compute node not schedulable: {node.get('code')} status={node.get('scheduler_status')}")
    return {
        "valid": bool(preview.get("valid", not errors)) and not errors,
        "errors": errors,
        "warnings": warnings,
        "diagnostics": _training_diagnostics(errors, warnings),
        "node": {
            "id": node.get("id"),
            "code": node.get("code"),
            "name": node.get("name"),
            "api_base_url": node.get("api_base_url"),
            "scheduler_status": node.get("scheduler_status"),
            "gpu_count": node.get("gpu_count"),
        },
        "job_payload": job_payload,
        "preview": preview,
        "sync_results": sync_results,
    }


@router.post("/login")
async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    user = get_platform_store().login(payload.get("username", ""), payload.get("password", ""))
    if not user:
        raise fail(401, "invalid username or password")
    return ok({"token": f"platform-token-{user['id']}", "user": user})


@router.get("/me")
async def me() -> dict[str, Any]:
    return ok(get_platform_store().users()[0])


@router.get("/dashboard/overview")
async def dashboard_overview() -> dict[str, Any]:
    store = get_platform_store()
    tasks = store.tasks()
    return ok(
        {
            "models": len(store.models()),
            "datasets": len(store.datasets()),
            "fine_tune_tasks": len(tasks),
            "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]),
            "compute_nodes": len(store.compute_nodes()),
            "gpus": len(store.gpus()),
        }
    )


@router.get("/system-info")
async def system_info() -> dict[str, Any]:
    return ok(get_platform_store().system_info())


@router.get("/users")
async def users() -> dict[str, Any]:
    return ok(get_platform_store().users())


@router.post("/users")
async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok(get_platform_store().create_user(payload))


@router.put("/users/{user_id}")
async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_user(user_id, payload))
    except KeyError:
        raise fail(404, "user not found")


@router.delete("/users/{user_id}")
async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]:
    try:
        get_platform_store().delete_user(user_id)
        return ok({"deleted": user_id, "current_username": current_username})
    except KeyError:
        raise fail(404, "user not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.get("/model-manage/local-models")
async def local_models() -> dict[str, Any]:
    store = get_platform_store()
    models = [{"path": item.get("path") or "", "name": item["name"], "source": "registered"} for item in store.models()]
    seen = {item["path"] for item in models if item.get("path")}
    if get_settings().compute_mode != "simulator":
        for node in store.compute_nodes():
            if not node.get("enabled"):
                continue
            try:
                result = await ComputeNodeClient(node["api_base_url"]).list_files(root="models", directories_only=True)
            except Exception:
                continue
            for item in result.get("items") or []:
                path = str(item.get("path") or "")
                if not path or path in seen:
                    continue
                seen.add(path)
                models.append(
                    {
                        "path": path,
                        "name": item.get("name") or path.rsplit("/", 1)[-1],
                        "source": f"compute:{node.get('code')}",
                    }
                )
    return ok({"models": models})


@router.get("/model-manage/trained-models")
async def trained_models() -> dict[str, Any]:
    return ok({"models": get_platform_store().trained_models()})


@router.delete("/model-manage/trained-models/{model_id}")
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]:
    get_platform_store().delete_trained_model(model_id)
    return ok({"deleted": model_id, "type": type})


@router.get("/model-manage/trained-models/{model_id}/artifacts")
async def trained_model_artifacts(model_id: str) -> dict[str, Any]:
    return ok(get_platform_store().model_artifacts(model_id))


@router.get("/model-manage/trained-models/{model_id}/lineage")
async def trained_model_lineage(model_id: str) -> dict[str, Any]:
    return ok(get_platform_store().model_lineage(model_id))


@router.get("/model-manage/export-jobs")
async def model_export_jobs(trained_model_id: str | None = Query(default=None)) -> dict[str, Any]:
    return ok(get_platform_store().model_export_jobs(trained_model_id))


@router.get("/model-manage/name/{name}")
async def model_by_name(name: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().model_by_name(name))
    except KeyError:
        raise fail(404, "model not found")


@router.get("/model-manage")
async def model_list() -> dict[str, Any]:
    return ok(get_platform_store().models())


@router.post("/model-manage")
async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().create_model(payload))
    except KeyError as exc:
        raise fail(400, f"missing field: {exc}")
    except ValueError as exc:
        raise fail(400, str(exc))
    except Exception as exc:  # noqa: BLE001 - keep API errors visible to deployment smoke checks
        raise fail(500, f"create model failed: {exc}")


@router.get("/model-manage/{model_id}")
async def model_detail(model_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().model(model_id))
    except KeyError:
        raise fail(404, "model not found")


@router.put("/model-manage/{model_id}")
async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_model(model_id, payload))
    except KeyError:
        raise fail(404, "model not found")


@router.put("/model-manage/{model_id}/purpose")
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")}))
    except KeyError:
        raise fail(404, "model not found")


@router.delete("/model-manage/{model_id}")
async def delete_model(model_id: str) -> dict[str, Any]:
    get_platform_store().delete_model(model_id)
    return ok({"deleted": model_id})


@router.post("/model-manage/merge")
async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    store = get_platform_store()
    trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "")
    trained_model = next(
        (
            item
            for item in store.trained_models()
            if trained_model_id and (item["id"] == trained_model_id or item["name"] == trained_model_id)
        ),
        None,
    )
    base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
    adapter_path = payload.get("adapter_path") or payload.get("adapter_name_or_path") or (trained_model and trained_model.get("merged_path"))
    if not base_model_path:
        raise fail(400, "base_model_path is required")
    if not adapter_path:
        raise fail(400, "adapter_path is required")
    node = store.schedule_node({**payload, "gpus": payload.get("gpus") or []})
    health = node.get("health_detail") or {}
    output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
    output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
    output_dir = str(payload.get("output_dir") or f"{output_root.rstrip('/')}/{output_name}")
    job_payload = {
        **payload,
        "id": str(payload.get("job_id") or f"merge_{uuid.uuid4().hex[:12]}"),
        "name": output_name,
        "engine": "merge",
        "base_model": base_model_path,
        "model_name_or_path": base_model_path,
        "adapter_name_or_path": adapter_path,
        "output_dir": output_dir,
        "template": payload.get("template", "qwen"),
        "train_method": payload.get("train_method", "lora"),
        "gpus": payload.get("gpus") or [],
        "trained_model_id": trained_model["id"] if trained_model else trained_model_id,
        "model_name": trained_model["name"] if trained_model else payload.get("model_name"),
    }
    if get_settings().compute_mode == "simulator":
        job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir}
    else:
        client = ComputeNodeClient(node["api_base_url"])
        preview = await client.validate_job(job_payload)
        if not preview.get("valid", False):
            raise fail(409, "; ".join(preview.get("errors") or ["merge preflight failed"]))
        job = await client.create_job(job_payload)
    return ok(store.record_model_merge_job(node, job_payload, job, trained_model["id"] if trained_model else trained_model_id))


@router.get("/dataset-manage/preview/{file_id}")
async def dataset_preview(file_id: str) -> dict[str, Any]:
    try:
        row = get_platform_store().dataset_file(file_id)
        return ok({"content": row["content"]})
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/records/{file_id}/sources")
async def dataset_record_sources(file_id: str) -> dict[str, Any]:
    try:
        return ok({"items": get_platform_store().dataset_file_record_sources(file_id)})
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/versions/{file_id}")
async def dataset_versions(file_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().file_versions(file_id))
    except KeyError:
        raise fail(404, "dataset file not found")


@router.get("/dataset-manage/versions/{file_id}/{version_id}")
async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]:
    try:
        row = get_platform_store().dataset_file(file_id)
        versions = get_platform_store().file_versions(file_id)["versions"]
        version = next((item for item in versions if item["id"] == version_id), None)
        if not version:
            raise KeyError(version_id)
        return ok({"version": version, "content": row["content"]})
    except KeyError:
        raise fail(404, "dataset version not found")


@router.post("/dataset-manage/versions/{file_id}")
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().create_file_version(file_id, payload))
    except KeyError:
        raise fail(404, "dataset file not found")


@router.put("/dataset-manage/versions/{file_id}/active")
async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().activate_file_version(file_id, payload["version_id"]))
    except KeyError:
        raise fail(404, "dataset version not found")


@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().delete_file_version(file_id, version_id))
    except KeyError:
        raise fail(404, "dataset version not found")
    except ValueError as exc:
        raise fail(400, str(exc))


async def _sync_dataset_file_to_compute_nodes(
    store: Any,
    dataset_id: str,
    file_id: str,
    filename: str,
    content: bytes,
) -> list[dict[str, Any]]:
    results: list[dict[str, Any]] = []
    if get_settings().compute_mode == "simulator":
        return results
    target_name = Path(filename or f"{file_id}.jsonl").name
    target_relative_path = f"datasets/{dataset_id}/{target_name}"
    for node in store.compute_nodes():
        if not node.get("enabled"):
            continue
        try:
            result = await ComputeNodeClient(node["api_base_url"]).upload_file(
                target_name,
                content,
                target_relative_path,
                resource_type="dataset",
                resource_id=dataset_id,
            )
            store.upsert_resource_replica(
                node["id"],
                "dataset",
                dataset_id,
                str(result.get("local_path") or ""),
            )
            results.append(
                {
                    "node_id": node["id"],
                    "node_code": node.get("code"),
                    "success": True,
                    "local_path": result.get("local_path"),
                    "byte_size": result.get("byte_size"),
                    "checksum_sha256": result.get("checksum_sha256"),
                }
            )
        except Exception as exc:  # noqa: BLE001 - keep upload usable while exposing sync failures
            results.append(
                {
                    "node_id": node["id"],
                    "node_code": node.get("code"),
                    "success": False,
                    "error": str(exc),
                }
            )
    return results


async def _sync_training_dataset_to_compute_node(
    store: Any,
    node: dict[str, Any],
    dataset_id: str,
) -> list[dict[str, Any]]:
    if not dataset_id:
        raise RuntimeError("train_dataset_id is required")
    files = store.training_dataset_files(dataset_id)
    if not files:
        raise RuntimeError(f"dataset has no uploaded file: {dataset_id}")
    split_aware = any(item.get("split") for item in files)
    files = [
        item
        for item in files
        if not split_aware or item.get("split") in {"train", "validation"}
    ]
    client = ComputeNodeClient(node["api_base_url"])
    results: list[dict[str, Any]] = []
    for item in files:
        target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
        result = await client.upload_file(
            target_name,
            str(item.get("content") or "").encode("utf-8"),
            f"datasets/{dataset_id}/{target_name}",
            resource_type="dataset",
            resource_id=dataset_id,
        )
        store.upsert_resource_replica(
            node["id"],
            "dataset",
            dataset_id,
            str(result.get("local_path") or ""),
        )
        results.append(
            {
                "node_id": node["id"],
                "node_code": node.get("code"),
                "file_id": item.get("id"),
                "name": target_name,
                "local_path": result.get("local_path"),
                "byte_size": result.get("byte_size"),
                "checksum_sha256": result.get("checksum_sha256"),
            }
        )
    return results


@router.post("/dataset-manage/upload/{dataset_id}")
async def upload_dataset_files(
    dataset_id: str,
    files: list[UploadFile] = File(default=[]),
    sync_to_compute: bool = Query(default=True),
) -> dict[str, Any]:
    created: list[dict[str, Any]] = []
    compute_sync: list[dict[str, Any]] = []
    store = get_platform_store()
    try:
        store.dataset(dataset_id)
    except KeyError:
        raise fail(404, "dataset not found")
    with store.connect() as conn:
        for file in files:
            raw = await file.read()
            content = raw.decode("utf-8", errors="replace")
            created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
            created.append(created_file)
            if sync_to_compute:
                compute_sync.extend(
                    await _sync_dataset_file_to_compute_nodes(
                        store,
                        dataset_id,
                        created_file["id"],
                        created_file["name"],
                        raw,
                    )
                )
    return ok({"files": created, "compute_sync": compute_sync})


@router.get("/dataset-manage/download/{dataset_id}")
async def download_dataset(dataset_id: str) -> PlainTextResponse:
    dataset = get_platform_store().dataset(dataset_id)
    content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])])
    return PlainTextResponse(content, media_type="text/plain")


@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
    row = get_platform_store().dataset_file(file_id)
    return PlainTextResponse(row["content"], media_type="text/plain")


@router.get("/dataset-manage")
async def dataset_list() -> dict[str, Any]:
    return ok(get_platform_store().datasets())


@router.post("/dataset-manage")
async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    dataset = get_platform_store().create_dataset(payload)
    return ok({"id": dataset["id"]})


@router.get("/dataset-manage/{dataset_id}")
async def dataset_detail(dataset_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().dataset(dataset_id))
    except KeyError:
        raise fail(404, "dataset not found")


@router.put("/dataset-manage/{dataset_id}")
async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_dataset(dataset_id, payload))
    except KeyError:
        raise fail(404, "dataset not found")


@router.delete("/dataset-manage/{dataset_id}")
async def delete_dataset(dataset_id: str) -> dict[str, Any]:
    get_platform_store().delete_dataset(dataset_id)
    return ok({"deleted": dataset_id})


@router.get("/fine-tune/check-name")
async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]:
    exists = any(task["name"] == name for task in get_platform_store().tasks())
    return ok({"exists": exists})


@router.get("/fine-tune/progress/{task_id}")
async def fine_tune_progress(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().progress(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/tensorboard/start")
async def tensorboard_start() -> dict[str, Any]:
    return ok({"status": "running", "url": "http://localhost:6006"})


@router.get("/fine-tune")
async def fine_tune_list() -> dict[str, Any]:
    return ok(get_platform_store().tasks())


@router.post("/fine-tune")
async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        task = get_platform_store().create_task(payload)
        return ok({"id": task["id"]})
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/fine-tune/start")
async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    store = get_platform_store()
    try:
        return ok(await _submit_fine_tune_task(store, payload))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            store.mark_task_failed(task_id, str(exc))
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - mark task failed when remote submit fails
        task_id = str(payload.get("task_id") or payload.get("id") or "")
        if task_id:
            store.mark_task_failed(task_id, str(exc))
        raise fail(502, f"submit compute job failed: {exc}")


@router.post("/fine-tune/preflight")
async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True))
    except RuntimeError as exc:
        return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
    except Exception as exc:  # noqa: BLE001 - expose compute validation errors to training create page
        raise fail(502, f"compute preflight failed: {exc}")


@router.post("/fine-tune/command-preview")
async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False))
    except RuntimeError as exc:
        return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"compute command preview failed: {exc}")


@router.post("/fine-tune/{task_id}/preflight")
async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - expose compute validation errors to training create page
        raise fail(502, f"compute preflight failed: {exc}")


@router.post("/fine-tune/{task_id}/command-preview")
async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
    try:
        return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False))
    except KeyError:
        raise fail(404, "fine tune task not found")
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"compute command preview failed: {exc}")


@router.get("/fine-tune/{task_id}")
async def fine_tune_detail(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().task(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.get("/fine-tune/{task_id}/logs")
async def fine_tune_logs(
    task_id: str,
    tail_lines: int | None = Query(default=500, ge=1, le=5000),
    offset: int | None = Query(default=None, ge=0),
    limit: int | None = Query(default=None, ge=1, le=5000),
) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if task.get("compute_job_id"):
        node = _node_for_task(task)
        if node:
            try:
                logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], tail_lines, offset, limit)
                try:
                    store.record_training_log_metrics(task_id, str(logs.get("content") or ""))
                except Exception:
                    pass
                if task.get("status") in {"queued", "running", "failed", "stopped", "completed"}:
                    try:
                        job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"])
                        store.apply_compute_job(task_id, job)
                    except Exception:
                        pass
                return ok({"source": "compute", **logs})
            except Exception as exc:  # noqa: BLE001 - keep failure reason visible even when log fetch fails
                content = task.get("failure_reason") or f"fetch compute log failed: {exc}"
                return ok({"job_id": task.get("compute_job_id"), "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
    content = task.get("failure_reason") or ""
    return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})


@router.get("/fine-tune/{task_id}/diagnostics")
async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    log_text = ""
    node = _node_for_task(task)
    if node and task.get("compute_job_id"):
        try:
            logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], 1000, None, None)
            log_text = str(logs.get("content") or "")
        except Exception:
            log_text = ""
    errors = [str(task.get("failure_reason") or "")] if task.get("failure_reason") else []
    return ok(
        {
            "task_id": task_id,
            "status": task.get("status"),
            "failure_reason": task.get("failure_reason") or "",
            "diagnostics": _training_diagnostics(errors, [], log_text),
        }
    )


@router.put("/fine-tune/{task_id}")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_task(task_id, payload))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/stop/{task_id}")
async def stop_fine_tune(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        task = store.task(task_id)
        node = _node_for_task(task)
        if task.get("compute_job_id") and node and get_settings().compute_mode != "simulator":
            job = await ComputeNodeClient(node["api_base_url"]).stop_job(task["compute_job_id"])
            return ok(store.apply_compute_job(task_id, job))
        return ok(store.stop_task(task_id))
    except KeyError:
        raise fail(404, "fine tune task not found")


@router.post("/fine-tune/{task_id}/stop")
async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]:
    return await stop_fine_tune(task_id)


@router.post("/fine-tune/{task_id}/retry")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
    store = get_platform_store()
    payload = payload or {}
    try:
        task = store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
        raise fail(409, "only failed or stopped tasks can be retried without force=true")
    retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
    store.reset_task_for_retry(task_id, retry_payload)
    try:
        return ok(await _submit_fine_tune_task(store, retry_payload))
    except RuntimeError as exc:
        raise fail(409, str(exc))
    except Exception as exc:  # noqa: BLE001 - mark retry failed when remote submit fails
        store.mark_task_failed(task_id, str(exc))
        raise fail(502, f"retry fine tune task failed: {exc}")


@router.delete("/fine-tune/{task_id}")
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
    get_platform_store().delete_task(task_id)
    return ok({"deleted": task_id})


@router.get("/fine-tune/{task_id}/overview")
async def fine_tune_overview(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    task = store.task(task_id)
    return ok(
        {
            "task": task,
            "progress": store.progress(task_id),
            "metrics": store.task_metrics(task_id),
            "checkpoints": store.task_checkpoints(task_id),
        }
    )


@router.get("/fine-tune/{task_id}/checkpoints")
async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    return ok(store.task_checkpoints(task_id))


@router.get("/fine-tune/{task_id}/metrics")
async def fine_tune_metrics(task_id: str) -> dict[str, Any]:
    store = get_platform_store()
    try:
        store.task(task_id)
    except KeyError:
        raise fail(404, "fine tune task not found")
    return ok(store.task_metrics(task_id))


@router.get("/model-eval")
async def model_eval_list() -> dict[str, Any]:
    return ok(get_platform_store().eval_tasks())


@router.get("/model-eval/{task_id}")
async def model_eval_detail(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().eval_task(task_id))
    except KeyError:
        raise fail(404, "eval task not found")


@router.post("/model-eval/start")
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    task = get_platform_store().create_eval_task(payload)
    return ok({"task_id": task["id"], **task})


@router.delete("/model-eval/{task_id}")
async def model_eval_delete(task_id: str) -> dict[str, Any]:
    get_platform_store().delete_eval_task(task_id)
    return ok({"deleted": task_id})


@router.get("/dimension")
async def dimension_list() -> dict[str, Any]:
    return ok(get_platform_store().dimensions())


@router.post("/dimension")
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok(get_platform_store().create_dimension(payload))


@router.get("/dimension/{dimension_id}")
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().dimension(dimension_id))
    except KeyError:
        raise fail(404, "dimension not found")


@router.put("/dimension/{dimension_id}")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_dimension(dimension_id, payload))
    except KeyError:
        raise fail(404, "dimension not found")


@router.delete("/dimension/{dimension_id}")
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
    get_platform_store().delete_dimension(dimension_id)
    return ok({"deleted": dimension_id})


@router.get("/model-compare")
async def model_compare_list() -> dict[str, Any]:
    return ok(get_platform_store().compare_tasks())


@router.post("/model-compare")
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    task = get_platform_store().create_compare_task(payload)
    return ok({"id": task["id"]})


@router.post("/model-compare/all/stop-all")
async def model_compare_stop_all() -> dict[str, Any]:
    return ok({"stopped": True})


@router.post("/model-compare/stop-by-pid")
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"stopped": True, "pid": payload.get("pid")})


@router.get("/model-compare/{task_id}")
async def model_compare_detail(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().compare_task(task_id))
    except KeyError:
        raise fail(404, "compare task not found")


@router.delete("/model-compare/{task_id}")
async def model_compare_delete(task_id: str) -> dict[str, Any]:
    get_platform_store().delete_compare_task(task_id)
    return ok({"deleted": task_id})


@router.get("/model-compare/{task_id}/load-status")
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
    try:
        task = get_platform_store().compare_task(task_id)
    except KeyError:
        raise fail(404, "compare task not found")
    load_status = task.get("load_status") or {"loaded_models": []}
    if isinstance(load_status, str):
        try:
            load_status = json.loads(load_status)
        except json.JSONDecodeError:
            load_status = {"loaded_models": []}
    return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})


@router.post("/model-compare/{task_id}/load-status")
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/load")
async def model_compare_load(task_id: str) -> dict[str, Any]:
    try:
        task = get_platform_store().compare_task(task_id)
        models = task.get("models") or []
        if isinstance(models, str):
            try:
                models = json.loads(models)
            except json.JSONDecodeError:
                models = []
        loaded_models = [
            {
                "model_id": item.get("model_id"),
                "model_name": item.get("model_name"),
                "status": "ready",
                "pid": 45000 + index,
                "port": item.get("port") or 18000 + index,
            }
            for index, item in enumerate(models)
            if isinstance(item, dict)
        ]
        return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/unload")
async def model_compare_unload(task_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
    except KeyError:
        raise fail(404, "compare task not found")


@router.post("/model-compare/{task_id}/start-model")
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})


@router.post("/model-compare/chat-with-port")
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    question = ""
    for message in payload.get("messages") or []:
        if message.get("role") == "user":
            question = str(message.get("content") or "")
    content = f"当前后端已收到推理请求：{question[:120]}"
    return ok({"response": content, "content": content})


@router.post("/model-compare/stream-chat")
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    question = payload.get("user_question") or payload.get("question") or ""
    return ok({"response": f"当前后端已收到流式推理请求：{str(question)[:120]}"})


@router.post("/model-chat/batch")
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"responses": [], "request": payload})


@router.post("/model-chat/local/chat")
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """Proxy chat to the compute node running the inference model."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"response": "no online compute node available for inference", "request": payload})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/chat", json_data=payload)
        return ok(result)
    except Exception as exc:
        return ok({"response": f"inference failed: {exc}", "request": payload})


@router.post("/model-chat/local/chat/stream")
async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse:
    """Stream chat from the compute node."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return StreamingResponse(
            iter(['data: {"error": "no online compute node"}\n\n']),
            media_type="text/event-stream",
        )
    client = ComputeNodeClient(node["api_base_url"])

    async def stream_proxy():
        async with httpx.AsyncClient(timeout=300) as http:
            url = f"{node['api_base_url'].rstrip('/')}/modelTF/inference/chat/stream"
            async with http.stream("POST", url, json=payload, headers=client.headers()) as resp:
                async for chunk in resp.aiter_bytes():
                    yield chunk

    return StreamingResponse(stream_proxy(), media_type="text/event-stream")


@router.post("/model-chat/local/preload")
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """Load a model on the compute node for inference."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/load", json_data=payload)
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.post("/model-chat/local/unload")
async def model_chat_local_unload() -> dict[str, Any]:
    """Unload the inference model from the compute node."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"unloaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/unload", json_data={})
        return ok(result)
    except Exception as exc:
        return ok({"unloaded": False, "error": str(exc)})


@router.get("/model-chat/local/status")
async def model_chat_local_status() -> dict[str, Any]:
    """Get inference session status from compute node."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("GET", "/inference/status")
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.post("/model-chat/trained/preload")
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    """Load a trained model (base + adapter) on the compute node for inference."""
    store = get_platform_store()
    node = _select_first_online_node(store)
    if not node:
        return ok({"loaded": False, "error": "no online compute node"})
    try:
        client = ComputeNodeClient(node["api_base_url"])
        result = await client._request("POST", "/inference/load", json_data=payload)
        return ok(result)
    except Exception as exc:
        return ok({"loaded": False, "error": str(exc)})


@router.get("/compute/nodes")
async def compute_nodes() -> dict[str, Any]:
    return ok(get_platform_store().compute_nodes())


@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str) -> dict[str, Any]:
    node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    return ok(node)


@router.post("/compute/nodes")
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().create_compute_node(payload))
    except KeyError as exc:
        raise fail(400, f"missing field: {exc}")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.put("/compute/nodes/{node_id}")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().update_compute_node(node_id, payload))
    except KeyError:
        raise fail(404, "compute node not found")
    except ValueError as exc:
        raise fail(400, str(exc))


@router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    client = ComputeNodeClient(node["api_base_url"])
    try:
        result = await client.test_connection()
        store.replace_node_gpus(node_id, result["gpus"])
        updated = store.update_compute_node_health(node_id, result["health"], True)
        return ok(
            {
                "node_id": node_id,
                "success": True,
                "latency_ms": result["latency_ms"],
                "gpu_count": len(result["gpus"]),
                "health": updated["health_detail"],
            }
        )
    except Exception as exc:  # noqa: BLE001 - return the connection error for node maintenance
        updated = store.update_compute_node_health(node_id, {}, False, str(exc))
        return ok(
            {
                "node_id": node_id,
                "success": False,
                "latency_ms": 0,
                "gpu_count": updated.get("gpu_count", 0),
                "error": str(exc),
                "health": updated["health_detail"],
            }
        )


@router.post("/compute/nodes/{node_id}/health-check")
async def health_check_compute_node(node_id: str) -> dict[str, Any]:
    return await test_compute_node(node_id)


@router.post("/compute/nodes/{node_id}/enable")
async def enable_compute_node(node_id: str) -> dict[str, Any]:
    return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))


@router.post("/compute/nodes/{node_id}/disable")
async def disable_compute_node(node_id: str) -> dict[str, Any]:
    return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))


@router.post("/compute/nodes/{node_id}/drain")
async def drain_compute_node(node_id: str) -> dict[str, Any]:
    return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))


@router.get("/compute/nodes/{node_id}/replicas")
async def compute_node_replicas(node_id: str) -> dict[str, Any]:
    return ok(get_platform_store().replicas(node_id))


@router.get("/compute/sync-jobs/{sync_id}")
async def compute_sync_job_detail(sync_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().sync_job(sync_id))
    except KeyError:
        raise fail(404, "sync job not found")


@router.get("/compute/nodes/{node_id}/replicas/drift")
async def compute_node_replica_drift(node_id: str) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    replicas = store.replicas(node_id)
    if not replicas:
        return ok({"node_id": node_id, "items": [], "drifted": 0})
    paths = [
        {
            "name": replica["id"],
            "path": replica["local_path"],
            "type": "any",
            "required": True,
        }
        for replica in replicas
    ]
    try:
        result = await ComputeNodeClient(node["api_base_url"]).check_paths(paths)
    except Exception as exc:  # noqa: BLE001
        raise fail(502, f"replica drift check failed: {exc}")
    check_map = {str(item.get("name")): item for item in result.get("items") or []}
    items = []
    for replica in replicas:
        check = check_map.get(replica["id"], {})
        updated = store.update_resource_replica_check(
            replica["id"],
            bool(check.get("ok")),
            int(check.get("byte_size") or replica.get("byte_size") or 0),
            "" if check.get("ok") else f"path not available: {replica['local_path']}",
        )
        items.append({**updated, "check": check})
    return ok({"node_id": node_id, "items": items, "drifted": len([item for item in items if item.get("sync_status") == "drifted"])})


async def _run_resource_replica_repair(sync_id: str, node_id: str, payload: dict[str, Any], replicas_to_repair: list[dict[str, Any]]) -> None:
    store = get_platform_store()
    store.update_sync_job(sync_id, "running", 5)
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        store.update_sync_job(sync_id, "failed", 100, completed=True)
        return
    client = ComputeNodeClient(node["api_base_url"])
    repaired = []
    failures = []
    total = max(len(replicas_to_repair), 1)
    for index, replica in enumerate(replicas_to_repair, start=1):
        replica_id = str(replica["id"])
        resource_type = str(replica.get("resource_type") or "")
        resource_id = str(replica.get("resource_id") or "")
        try:
            if resource_type == "dataset":
                files = store.training_dataset_files(resource_id)
                if not files:
                    raise RuntimeError(f"dataset has no uploaded file: {resource_id}")
                total_size = 0
                checksum = ""
                local_path = str(replica.get("local_path") or "")
                for item in files:
                    filename = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
                    result = await client.upload_file(
                        filename,
                        str(item.get("content") or "").encode("utf-8"),
                        f"datasets/{resource_id}/{filename}",
                        resource_type="dataset",
                        resource_id=resource_id,
                    )
                    total_size += int(result.get("byte_size") or 0)
                    checksum = str(result.get("checksum_sha256") or checksum)
                    local_path = str(result.get("local_path") or local_path)
                repaired.append(store.update_resource_replica_sync_result(replica_id, True, local_path, total_size, checksum))
                continue

            source_path = ""
            target_relative_path = ""
            if resource_type == "model":
                model = store.model(resource_id)
                source_path = str(model.get("path") or "")
                target_relative_path = f"models/{Path(source_path).name}" if source_path else ""
            elif resource_type in {"trained_model", "model_artifact"}:
                if resource_type == "trained_model":
                    artifacts = store.model_artifacts(resource_id)
                    artifact = next((item for item in artifacts if item.get("path")), None)
                else:
                    artifact = store.model_artifact(resource_id)
                source_path = str((artifact or {}).get("path") or replica.get("local_path") or "")
                target_relative_path = f"outputs/{Path(source_path).name}" if source_path else ""
            else:
                source_path = str(payload.get("source_path") or replica.get("source_path") or replica.get("local_path") or "")
                target_relative_path = str(payload.get("target_relative_path") or "")

            if not source_path:
                raise RuntimeError(f"authoritative source path not found for {resource_type}:{resource_id}")
            result = await client.import_local_file(
                {
                    "source_path": source_path,
                    "target_relative_path": target_relative_path,
                    "resource_type": resource_type,
                    "resource_id": resource_id,
                }
            )
            repaired.append(
                store.update_resource_replica_sync_result(
                    replica_id,
                    True,
                    str(result.get("local_path") or replica.get("local_path") or ""),
                    int(result.get("byte_size") or 0),
                    str(result.get("checksum_sha256") or ""),
                )
            )
        except Exception as exc:  # noqa: BLE001 - collect all replica repair failures
            error = str(exc)
            failures.append({"replica_id": replica_id, "resource_type": resource_type, "resource_id": resource_id, "error": error})
            repaired.append(store.update_resource_replica_sync_result(replica_id, False, None, int(replica.get("byte_size") or 0), "", error))
        progress = min(95, 5 + int(index / total * 90))
        store.update_sync_job(sync_id, "running", progress)
    store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)


@router.post("/compute/nodes/{node_id}/replicas/repair")
async def compute_node_replica_repair(
    node_id: str,
    background_tasks: BackgroundTasks,
    payload: dict[str, Any] | None = Body(default=None),
) -> dict[str, Any]:
    store = get_platform_store()
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    payload = payload or {}
    replica_ids = payload.get("replica_ids") or [
        item["id"] for item in store.replicas(node_id) if item.get("sync_status") in {"drifted", "failed", "repair_pending"}
    ]
    updated = store.mark_resource_replica_repair_pending([str(item) for item in replica_ids])
    sync_id = store.create_sync_job(
        node_id,
        {
            "operation": "repair",
            "resources": [
                {
                    "resource_type": item.get("resource_type"),
                    "resource_id": item.get("resource_id"),
                    "replica_id": item.get("id"),
                    "target_path": item.get("local_path"),
                }
                for item in updated
            ],
        },
    )
    if not updated:
        store.update_sync_job(sync_id, "completed", 100, completed=True)
        return ok({"sync": store.sync_job(sync_id), "replicas": [], "failed": [], "async": False})
    background_tasks.add_task(_run_resource_replica_repair, sync_id, node_id, payload, updated)
    return ok({"sync": store.sync_job(sync_id), "replicas": updated, "failed": [], "async": True})


@router.get("/compute/nodes/{node_id}/engines")
async def compute_node_engines(node_id: str) -> dict[str, Any]:
    node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    health = node.get("health_detail") or {}
    live_error = ""
    try:
        health = await ComputeNodeClient(node["api_base_url"]).health()
    except Exception as exc:  # noqa: BLE001 - stored health is enough for offline node detail
        live_error = str(exc)
    capabilities = health.get("capabilities") or node.get("capabilities") or []
    return ok(
        {
            "node_id": node_id,
            "items": [
                {
                    "engine": "llama_factory",
                    "display_name": "LLaMA-Factory",
                    "status": "available" if "llama_factory" in capabilities else "unknown",
                    "version": health.get("llama_factory_version") or "",
                    "home": health.get("llama_factory_home") or "",
                    "home_exists": bool(health.get("llama_factory_home_exists")),
                    "capabilities": capabilities,
                    "execution_mode": health.get("execution_mode") or "",
                    "last_error": live_error,
                }
            ],
        }
    )


@router.get("/compute/gpus")
async def compute_gpus() -> dict[str, Any]:
    return ok(get_platform_store().gpus())


@router.get("/compute/queue")
async def compute_queue() -> dict[str, Any]:
    return ok(get_platform_store().queue())


@router.get("/compute/jobs/{job_id}")
async def compute_job_detail(job_id: str) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        job = await ComputeNodeClient(node["api_base_url"]).get_job(job_id)
        return ok(get_platform_store().sync_model_merge_job(job_id, job))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    return ok(await ComputeNodeClient(node["api_base_url"]).get_job(job_id))


@router.post("/compute/jobs/{job_id}/stop")
async def compute_job_stop(job_id: str) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
        return ok(get_platform_store().sync_model_merge_job(job_id, job))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
    get_platform_store().apply_compute_job(task["id"], job)
    return ok(job)


@router.get("/compute/jobs/{job_id}/logs")
async def compute_job_logs(
    job_id: str,
    tail_lines: int | None = Query(default=200, ge=1, le=5000),
    offset: int | None = Query(default=None, ge=0),
    limit: int | None = Query(default=None, ge=1, le=5000),
) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        node = _node_for_compute_job_record(job_id)
        if not node:
            raise fail(404, "compute job not found")
        return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))
    node = _node_for_task(task)
    if not node:
        raise fail(404, "compute node not found")
    return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))


@router.post("/compute/jobs/{job_id}/retry")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
    store = get_platform_store()
    payload = payload or {}
    task = _task_for_compute_job(job_id)
    if not task:
        raise fail(404, "compute job not found")
    return await retry_fine_tune(task["id"], payload)


@router.post("/compute/jobs/{job_id}/priority")
async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    task = _task_for_compute_job(job_id)
    if not task:
        raise fail(404, "compute job not found")
    priority = str(payload.get("priority") or "normal")
    return ok(get_platform_store().update_task_priority(task["id"], priority))


@router.post("/internal/compute-sync/jobs/poll")
async def poll_compute_jobs() -> dict[str, Any]:
    return ok(await poll_compute_jobs_once())


@router.post("/internal/compute-sync/resources")
async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    store = get_platform_store()
    node_id = payload.get("target_node_id") or payload.get("target_compute_node_id")
    if not node_id:
        raise fail(400, "target_node_id is required")
    node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
    if not node:
        raise fail(404, "compute node not found")
    sync_id = store.create_sync_job(node_id, payload)
    replicas = []
    failures = []
    resources = payload.get("resources") or []
    for resource in resources:
        if not resource.get("source_path"):
            continue
        try:
            result = await ComputeNodeClient(node["api_base_url"]).import_local_file(
                {
                    "source_path": resource["source_path"],
                    "target_relative_path": resource.get("target_relative_path"),
                    "resource_type": resource.get("resource_type"),
                    "resource_id": resource.get("resource_id"),
                }
            )
            replicas.append(
                store.upsert_resource_replica(
                    node_id,
                    str(resource.get("resource_type") or "file"),
                    str(resource.get("resource_id") or result["id"]),
                    result["local_path"],
                )
            )
        except Exception as exc:  # noqa: BLE001 - collect per-resource failures
            failures.append({"resource_id": str(resource.get("resource_id")), "error": str(exc)})
    store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)
    return ok({"sync": store.sync_job(sync_id), "replicas": replicas, "failed": failures})


@router.get("/internal/compute-sync/resources/{sync_id}")
async def compute_sync_detail(sync_id: str) -> dict[str, Any]:
    try:
        return ok(get_platform_store().sync_job(sync_id))
    except KeyError:
        raise fail(404, "sync job not found")


@router.get("/training-log-files")
async def training_log_files() -> dict[str, Any]:
    return ok(get_platform_store().training_log_files())


@router.get("/training-log-content")
async def training_log_content(file: str = Query(...)) -> dict[str, Any]:
    try:
        return ok(get_platform_store().training_log_content(file))
    except KeyError:
        raise fail(404, "training log not found")


@router.get("/log-files")
async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]:
    return ok(get_platform_store().log_files(date))


@router.get("/log-content")
async def log_content(file: str = Query(...)) -> dict[str, Any]:
    return ok(get_platform_store().log_content(file))


@router.post("/web-log")
async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
    return ok({"received": True, **payload})