feat: P0 训练闭环核心功能实现

P0-1 模型路径治理:
- 新增 003_model_path_governance.sql 迁移,models 表增加 can_train 字段
- create_model/update_model 自动计算 can_train(非API+有路径=可训练)
- _compute_job_payload_from_task_node 拒绝 API 模型和无可训练路径模型
- 平台诊断规则增加 API 模型/路径缺失检测

P0-2 数据集格式校验:
- 新增 dataset_format.py,支持 Alpaca/ShareGPT/DPO/CPT 格式校验
- 训练预检时自动根据 train_type 匹配格式并校验内容字段
- llama_dataset_info 增加 DPO/CPT 格式列映射

P0-3 训练完成产物入库:
- _ensure_trained_model 使用 compute 节点返回的真实 artifacts
- 注册 per-file artifact 记录(含 size_bytes/checksum_sha256)
- trained_models 表增加 artifact_dir 字段

P0-4 失败日志拉取:
- poll_compute_jobs_once 检测到 failed/stopped 时强制拉取最后 200 行日志
- apply_compute_job 持久化失败日志片段到任务 payload

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-07-28 13:10:53 +08:00
parent 525fc55cef
commit a9ab130d43
5 changed files with 320 additions and 24 deletions

View File

@@ -27,6 +27,13 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
except Exception:
pass
# P0-4: Force-fetch last log snippet when job reaches terminal state
if job.get("status") in {"failed", "stopped"}:
try:
last_logs = await client.job_logs(task["compute_job_id"], tail_lines=200)
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})

View File

@@ -0,0 +1,148 @@
"""Dataset format validation for Alpaca, ShareGPT, DPO, CPT formats.
Used by the training preflight flow to validate that uploaded dataset files
conform to the declared format before submitting to the compute node.
"""
from __future__ import annotations
import json
from typing import Any
def _load_sample(path: str | None, content: str | None = None, max_samples: int = 20) -> list[dict[str, Any]]:
"""Load up to max_samples records from JSONL file path or raw content string."""
try:
if content is not None:
text = content.strip()
elif path:
with open(path, "r", encoding="utf-8") as fh:
text = fh.read().strip()
else:
return []
except Exception:
return []
if not text:
return []
lines = text.splitlines()[:max_samples]
records: list[dict[str, Any]] = []
for line in lines:
line = line.strip()
if not line:
continue
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(record, dict):
records.append(record)
return records
def _check_alpaca(records: list[dict[str, Any]]) -> list[str]:
"""Validate Alpaca format: requires 'instruction' field."""
errors: list[str] = []
if not records:
errors.append("Alpaca 格式数据集无有效记录")
return errors
missing_instruction = sum(1 for r in records if not r.get("instruction"))
if missing_instruction:
errors.append(
f"Alpaca 格式要求每条记录包含 instruction 字段,"
f"{len(records)}条中有{missing_instruction}条缺失"
)
return errors
def _check_sharegpt(records: list[dict[str, Any]]) -> list[str]:
"""Validate ShareGPT format: requires 'messages' (list of dicts with role/content)."""
errors: list[str] = []
if not records:
errors.append("ShareGPT 格式数据集无有效记录")
return errors
bad = 0
for r in records:
messages = r.get("messages")
if not isinstance(messages, list) or not messages:
bad += 1
continue
for msg in messages:
if not isinstance(msg, dict) or "role" not in msg or "content" not in msg:
bad += 1
break
if bad:
errors.append(
f"ShareGPT 格式要求每条记录包含 messages 列表,"
f"每条消息需有 role 和 content 字段,前{len(records)}条中有{bad}条不符合"
)
return errors
def _check_dpo(records: list[dict[str, Any]]) -> list[str]:
"""Validate DPO format: requires 'chosen' and 'rejected' fields."""
errors: list[str] = []
if not records:
errors.append("DPO 格式数据集无有效记录")
return errors
missing_chosen = sum(1 for r in records if not r.get("chosen"))
missing_rejected = sum(1 for r in records if not r.get("rejected"))
if missing_chosen:
errors.append(f"DPO 格式要求 chosen 字段,前{len(records)}条中有{missing_chosen}条缺失")
if missing_rejected:
errors.append(f"DPO 格式要求 rejected 字段,前{len(records)}条中有{missing_rejected}条缺失")
return errors
def _check_cpt(records: list[dict[str, Any]]) -> list[str]:
"""Validate CPT format: requires 'text' field, should NOT have instruction/output."""
errors: list[str] = []
if not records:
errors.append("CPT 格式数据集无有效记录")
return errors
missing_text = sum(1 for r in records if not r.get("text"))
has_instruction = sum(1 for r in records if r.get("instruction") or r.get("output"))
if missing_text:
errors.append(f"CPT 格式要求 text 字段,前{len(records)}条中有{missing_text}条缺失")
if has_instruction:
errors.append(
f"CPT 格式不应包含 instruction/output 字段(疑似 Alpaca 格式),"
f"{len(records)}条中有{has_instruction}条包含此类字段"
)
return errors
FORMAT_VALIDATORS = {
"alpaca": _check_alpaca,
"alpaca_jsonl": _check_alpaca,
"sharegpt": _check_sharegpt,
"dpo": _check_dpo,
"cpt": _check_cpt,
"pt": _check_cpt,
}
def validate_dataset_format(
dataset_format: str,
content: str | None = None,
path: str | None = None,
max_samples: int = 20,
) -> list[str]:
"""Validate dataset content against expected format.
Args:
dataset_format: One of 'alpaca', 'sharegpt', 'dpo', 'cpt'.
content: Raw file content (JSONL text). Mutually exclusive with path.
path: File path to read content from.
max_samples: Maximum records to sample for validation.
Returns:
List of error messages (empty if valid).
"""
fmt = str(dataset_format).lower().strip()
validator = FORMAT_VALIDATORS.get(fmt)
if not validator:
return [f"不支持的数据集格式: {dataset_format},支持的格式: {', '.join(sorted(FORMAT_VALIDATORS))}"]
records = _load_sample(path=path, content=content, max_samples=max_samples)
return validator(records)