feat: 模型评测端到端闭环 — EvalRunner引擎 + 算力节点Job执行 + 结果回写
算力节点 (compute): - 新建 eval_runner.py: 评测执行引擎,作为subprocess运行 - 加载模型 + JSONL数据集 + 逐样本推理 - BLEU/ROUGE/Cosine基础指标计算 - LLM Judge评分(OpenAI兼容API调用) - 结果写入eval_results.json - adapter.py: build_command新增engine=eval分支 - main.py: 新增/json模块导入,新增/compute/files/read端点,eval job校验 后端: - platform.py: 重写startEval提交eval job到算力节点 - 支持models表和trained_models表查找 - 已合并模型不传adapter路径 - platform_store.py: 新增update_eval_task/running_eval_tasks/apply_eval_job_result - sync.py: poller新增eval job同步,异步读取eval_results.json回写结果 前端: - EvalCreateView/DimensionCreateView: eval模型过滤扩展(API类型+api_url) - EvalCreateView: GPU过滤在线节点空闲GPU - EvalTaskSetupStep: GPU value从数组index改为gpu.id - BasicMetricSetupStep: ROUGE方法名修正(rouge_1→rouge1) - EvalView: 新增5秒轮询刷新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -2076,10 +2076,55 @@ class PlatformStore:
|
||||
)
|
||||
return self.eval_task(task_id)
|
||||
|
||||
def update_eval_task(self, task_id: str, updates: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Update fields in an eval task's payload without replacing the whole record."""
|
||||
task = self.eval_task(task_id)
|
||||
merged = {**task, **updates}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE eval_tasks SET payload=?, status=? WHERE id=?",
|
||||
(json_dumps(merged), merged.get("status", task.get("status", "pending")), task_id),
|
||||
)
|
||||
return self.eval_task(task_id)
|
||||
|
||||
def delete_eval_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def running_eval_tasks(self) -> list[dict[str, Any]]:
|
||||
"""Return eval tasks that have been submitted to a compute node and are still running."""
|
||||
return [
|
||||
task for task in self.eval_tasks()
|
||||
if task.get("compute_job_id") and task.get("status") in {"queued", "running"}
|
||||
]
|
||||
|
||||
def apply_eval_job_result(self, task_id: str, job: dict[str, Any], result_content: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Sync a compute job status/result back to an eval task."""
|
||||
task = self.eval_task(task_id)
|
||||
job_status = str(job.get("status", ""))
|
||||
status_map = {"queued": "running", "running": "running", "completed": "completed",
|
||||
"failed": "failed", "stopped": "stopped"}
|
||||
new_status = status_map.get(job_status, job_status or task.get("status", "pending"))
|
||||
updates: dict[str, Any] = {
|
||||
"status": new_status,
|
||||
"progress": int(job.get("progress", 0)),
|
||||
"output_dir": job.get("output_dir", task.get("output_dir", "")),
|
||||
}
|
||||
# On completion, populate results from eval_results.json content
|
||||
if new_status == "completed" and result_content:
|
||||
updates.update({
|
||||
"overall_score": result_content.get("overall_score", 0),
|
||||
"overall_score_max": result_content.get("overall_score_max", 100),
|
||||
"overall_evaluation": result_content.get("overall_evaluation", ""),
|
||||
"improvement_suggestions": result_content.get("improvement_suggestions", []),
|
||||
"dimension_summary": result_content.get("dimension_summary", []),
|
||||
"samples": result_content.get("samples", []),
|
||||
"sample_count": result_content.get("sample_count", 0),
|
||||
"completed_count": result_content.get("completed_count", 0),
|
||||
"passed_count": result_content.get("passed_count", 0),
|
||||
})
|
||||
return self.update_eval_task(task_id, updates)
|
||||
|
||||
def dimensions(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
|
||||
|
||||
Reference in New Issue
Block a user