YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: P0 训练闭环核心功能实现

Browse Source 
P0-1 模型路径治理:
- 新增 003_model_path_governance.sql 迁移，models 表增加 can_train 字段
- create_model/update_model 自动计算 can_train（非API+有路径=可训练）
- _compute_job_payload_from_task_node 拒绝 API 模型和无可训练路径模型
- 平台诊断规则增加 API 模型/路径缺失检测

P0-2 数据集格式校验:
- 新增 dataset_format.py，支持 Alpaca/ShareGPT/DPO/CPT 格式校验
- 训练预检时自动根据 train_type 匹配格式并校验内容字段
- llama_dataset_info 增加 DPO/CPT 格式列映射

P0-3 训练完成产物入库:
- _ensure_trained_model 使用 compute 节点返回的真实 artifacts
- 注册 per-file artifact 记录（含 size_bytes/checksum_sha256）
- trained_models 表增加 artifact_dir 字段

P0-4 失败日志拉取:
- poll_compute_jobs_once 检测到 failed/stopped 时强制拉取最后 200 行日志
- apply_compute_job 持久化失败日志片段到任务 payload

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-07-28 13:10:53 +08:00
parent 525fc55cef
commit a9ab130d43
5 changed files with 320 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
backend/app/db/platform_store.py
View File

@@ -153,23 +153,40 @@ def llama_dataset_keys(dataset_key: str, file_names: list[str]) -> list[str]:
def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str = "alpaca") -> dict[str, Any]:
result: dict[str, Any] = {}
fmt = str(formatting).lower()
for key, file_name in zip(llama_dataset_keys(dataset_key, file_names), file_names):
if formatting == "sharegpt":
if fmt == "sharegpt":
result[key] = {
"file_name": file_name,
"formatting": "sharegpt",
"columns": {"messages": "messages"},
}
continue
result[key] = {
"file_name": file_name,
"formatting": "alpaca",
"columns": {
"prompt": "instruction",
"query": "input",
"response": "output",
},
}
elif fmt == "dpo":
result[key] = {
"file_name": file_name,
"formatting": "dpo",
"columns": {
"prompt": "system",
"chosen": "chosen",
"rejected": "rejected",
},
}
elif fmt in {"cpt", "pt", "pretrain"}:
result[key] = {
"file_name": file_name,
"formatting": "cpt",
"columns": {"prompt": "text"},
}
else:
result[key] = {
"file_name": file_name,
"formatting": "alpaca",
"columns": {
"prompt": "instruction",
"query": "input",
"response": "output",
},
}
return result
@@ -432,7 +449,7 @@ class PlatformStore:
(status, progress, completed_at, row["id"]),
)
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None:
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any] | None = None) -> None:
name = task.get("output_model_name") or f"{task['name']}-lora"
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
if exists:
@@ -443,8 +460,8 @@ class PlatformStore:
conn.execute(
"""
INSERT INTO trained_models
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
trained_model_id,
@@ -455,20 +472,52 @@ class PlatformStore:
0,
0,
output_dir,
output_dir,
),
)
# Use real artifact data from compute node when available
artifacts = (job or {}).get("artifacts") or []
if artifacts:
total_size = sum(int(a.get("size_bytes") or a.get("size", 0)) for a in artifacts)
checksums = [a.get("checksum_sha256", "") for a in artifacts if a.get("checksum_sha256")]
combined_checksum = checksums[0] if len(checksums) == 1 else ""
# Register individual artifact files
for artifact in artifacts[:50]: # limit to 50 file entries
artifact_path = artifact.get("path") or artifact.get("name", "")
abs_path = artifact_path if artifact_path.startswith("/") else f"{output_dir.rstrip('/')}/{artifact_path.lstrip('/')}"
self._upsert_model_artifact(
conn,
trained_model_id,
"trained_model",
"adapter_file",
abs_path,
int(artifact.get("size_bytes") or artifact.get("size", 0)),
artifact.get("checksum_sha256", ""),
{
"task_id": task.get("id"),
"train_method": task.get("train_method", "lora"),
"base_model": task.get("base_model"),
"artifact_name": artifact.get("name", ""),
},
task.get("compute_job_id"),
)
else:
total_size = 0
combined_checksum = ""
# Register the top-level adapter directory entry
self._upsert_model_artifact(
conn,
trained_model_id,
"trained_model",
"adapter",
output_dir,
0,
"",
total_size,
combined_checksum,
{
"task_id": task.get("id"),
"train_method": task.get("train_method", "lora"),
"base_model": task.get("base_model"),
"file_count": len(artifacts),
},
task.get("compute_job_id"),
)
@@ -1079,24 +1128,29 @@ class PlatformStore:
def create_model(self, payload: dict[str, Any]) -> dict[str, Any]:
model_id = payload.get("id") or new_id("m")
model_source = payload.get("model_source", "local")
path = payload.get("path", "")
# Automatically determine can_train: local models with a path can be trained
can_train = 1 if (model_source != "api" and path and str(path).strip()) else 0
with self.connect() as conn:
conn.execute(
"""
INSERT INTO models
(id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
(id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, can_train, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
model_id,
payload["name"],
payload.get("type", "LLM"),
payload.get("purpose", "training"),
payload.get("model_source", "local"),
model_source,
payload.get("description"),
payload.get("path"),
path,
payload.get("api_url"),
payload.get("api_key"),
payload.get("online_model_name"),
can_train,
utcnow(),
),
)
@@ -1105,23 +1159,28 @@ class PlatformStore:
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.model(model_id)
merged = {**current, **payload}
# Recompute can_train when relevant fields change
model_source = merged.get("model_source", "local")
path = merged.get("path", "")
can_train = 1 if (model_source != "api" and path and str(path).strip()) else 0
with self.connect() as conn:
conn.execute(
"""
UPDATE models
SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=?
SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=?, can_train=?
WHERE id=?
""",
(
merged["name"],
merged.get("type", "LLM"),
merged.get("purpose", "training"),
merged.get("model_source", "local"),
model_source,
merged.get("description"),
merged.get("path"),
path,
merged.get("api_url"),
merged.get("api_key"),
merged.get("online_model_name"),
can_train,
model_id,
),
)
@@ -1717,6 +1776,23 @@ class PlatformStore:
dataset_id = str(task.get("train_dataset_id") or task.get("dataset_id") or "")
with self.connect() as conn:
model = conn.execute("SELECT * FROM models WHERE id=?", (base_model_id,)).fetchone()
if not model:
raise RuntimeError(f"base model not found: {base_model_id}")
# P0-1: Reject non-trainable models (API models or models without local path)
if not model.get("can_train"):
model_source = model.get("model_source") or "unknown"
model_path = model.get("path") or ""
if model_source == "api":
raise RuntimeError(
f"模型 '{model['name']}' 为 API 模型，不能作为 LLaMA-Factory 本地训练基座，请选择本地路径模型"
)
if not model_path or not str(model_path).strip():
raise RuntimeError(
f"模型 '{model['name']}' 未配置算力节点可访问路径，请先在模型管理中设置模型本地路径"
)
raise RuntimeError(
f"模型 '{model['name']}' 不支持本地训练（source={model_source}），请选择其他模型"
)
dataset = conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()
files = conn.execute(
"""SELECT id, name, size, active_version_id, create_time, metadata
@@ -1767,6 +1843,24 @@ class PlatformStore:
training_keys = runtime_keys[: len(training_files)]
validation_keys = runtime_keys[len(training_files) :]
dataset_format = str(task.get("dataset_format") or (dataset and dataset.get("formatting")) or "alpaca").lower()
# P0-2: Validate dataset content against declared format
train_type = str(task.get("train_type", task.get("train_method", ""))).upper()
expected_format = {
"DPO": "dpo",
"CPT": "cpt",
}.get(train_type)
if expected_format:
dataset_format = expected_format
format_errors: list[str] = []
for file_entry in training_files:
content = file_entry.get("content") or ""
if content:
from app.modules.data_process.dataset_format import validate_dataset_format
file_errors = validate_dataset_format(dataset_format, content=content)
if file_errors:
format_errors.extend(file_errors)
if format_errors:
raise RuntimeError("数据集格式校验失败：\n" + "\n".join(f" - {e}" for e in format_errors[:10]))
health_detail = node.get("health_detail") or {}
dataset_root = str(health_detail.get("dataset_root") or f"{node['data_root'].rstrip('/')}/datasets")
output_root = str(health_detail.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
@@ -1860,7 +1954,21 @@ class PlatformStore:
self._sync_gpu_allocations(conn, payload, job, status)
self._upsert_checkpoints(conn, task_id, job.get("checkpoints") or [])
if status == "completed":
self._ensure_trained_model(conn, payload)
self._ensure_trained_model(conn, payload, job)
# P0-4: Persist failure info for diagnosis
if status in {"failed", "stopped"}:
failure_reason = job.get("error") or job.get("message") or "compute job failed"
log_snippet = job.get("log_snippet") or ""
conn.execute(
"UPDATE fine_tune_tasks SET failure_reason = ? WHERE id = ?",
(failure_reason[:2000], task_id),
)
# Store last log snippet if available (max 8KB)
if log_snippet:
conn.execute(
"UPDATE fine_tune_tasks SET payload = ? WHERE id = ?",
(json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id),
)
return self.task(task_id)
def running_compute_tasks(self) -> list[dict[str, Any]]: