fix: 推理/评测结果同步、数据集统计与算力节点管理增强

后端:
- 抽取 fetch_eval_result_content 复用函数,model_eval_detail 直接应用评测任务结果
- health 接口移除数据库依赖,返回静态指标
- 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理
- 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout
- 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error

评测引擎:
- _load_dataset 支持 JSON/JSONL 文件
- 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制

前端:
- 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode
- 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
wuyongtao
2026-08-03 15:49:21 +08:00
parent cc08b164d0
commit 5cc306eb0a
15 changed files with 402 additions and 112 deletions

View File

@@ -10,6 +10,24 @@ def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
output_dir = job.get("output_dir")
if not output_dir:
return None
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
data_root = "/data/yg-ft/"
if full_path.startswith(data_root):
full_path = full_path[len(data_root):]
rel_path = full_path.lstrip("/")
import httpx
url = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
response = await http.get(url, params={"path": rel_path})
response.raise_for_status()
payload = response.json()
return payload if isinstance(payload, dict) else None
async def poll_compute_jobs_once() -> dict[str, Any]:
store = get_platform_store()
synced: list[dict[str, Any]] = []
@@ -66,18 +84,7 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
# Try to read eval_results.json from the job output directory
if job.get("status") == "completed" and job.get("output_dir"):
try:
full_path = f"{job['output_dir'].rstrip('/')}/eval_results.json"
# Convert absolute path to relative (strip YG_FT_DATA_ROOT prefix)
data_root = "/data/yg-ft/"
if full_path.startswith(data_root):
full_path = full_path[len(data_root):]
rel_path = full_path.lstrip("/")
import httpx
settings_path = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
read_resp = await http.get(settings_path, params={"path": rel_path})
if read_resp.status_code == 200:
result_content = read_resp.json()
result_content = await fetch_eval_result_content(client, node, job)
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)