a9ab130d43
P0-1 模型路径治理: - 新增 003_model_path_governance.sql 迁移，models 表增加 can_train 字段 - create_model/update_model 自动计算 can_train（非API+有路径=可训练） - _compute_job_payload_from_task_node 拒绝 API 模型和无可训练路径模型 - 平台诊断规则增加 API 模型/路径缺失检测 P0-2 数据集格式校验: - 新增 dataset_format.py，支持 Alpaca/ShareGPT/DPO/CPT 格式校验 - 训练预检时自动根据 train_type 匹配格式并校验内容字段 - llama_dataset_info 增加 DPO/CPT 格式列映射 P0-3 训练完成产物入库: - _ensure_trained_model 使用 compute 节点返回的真实 artifacts - 注册 per-file artifact 记录（含 size_bytes/checksum_sha256） - trained_models 表增加 artifact_dir 字段 P0-4 失败日志拉取: - poll_compute_jobs_once 检测到 failed/stopped 时强制拉取最后 200 行日志 - apply_compute_job 持久化失败日志片段到任务 payload Co-Authored-By: Claude <noreply@anthropic.com>
52 lines
2.5 KiB
Python
52 lines
2.5 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from app.db.platform_store import get_platform_store
|
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
|
|
|
|
|
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
|
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
|
|
|
|
|
async def poll_compute_jobs_once() -> dict[str, Any]:
|
|
store = get_platform_store()
|
|
synced: list[dict[str, Any]] = []
|
|
failed: list[dict[str, str]] = []
|
|
for task in store.running_compute_tasks():
|
|
node = _node_for_task(task)
|
|
if not node:
|
|
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
|
continue
|
|
try:
|
|
client = ComputeNodeClient(node["api_base_url"])
|
|
job = await client.get_job(task["compute_job_id"])
|
|
try:
|
|
logs = await client.job_logs(task["compute_job_id"], tail_lines=5000)
|
|
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
|
|
except Exception:
|
|
pass
|
|
# P0-4: Force-fetch last log snippet when job reaches terminal state
|
|
if job.get("status") in {"failed", "stopped"}:
|
|
try:
|
|
last_logs = await client.job_logs(task["compute_job_id"], tail_lines=200)
|
|
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
|
except Exception:
|
|
pass
|
|
synced.append(store.apply_compute_job(task["id"], job))
|
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
|
failed.append({"task_id": task["id"], "error": str(exc)})
|
|
standalone_synced: list[dict[str, Any]] = []
|
|
for record in store.active_standalone_compute_jobs():
|
|
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
|
|
if not node:
|
|
failed.append({"job_id": record["id"], "error": "compute node not found"})
|
|
continue
|
|
try:
|
|
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
|
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
|
failed.append({"job_id": record["id"], "error": str(exc)})
|
|
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
|