fix: 推理/评测结果同步、数据集统计与算力节点管理增强
后端: - 抽取 fetch_eval_result_content 复用函数,model_eval_detail 直接应用评测任务结果 - health 接口移除数据库依赖,返回静态指标 - 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理 - 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout - 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error 评测引擎: - _load_dataset 支持 JSON/JSONL 文件 - 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制 前端: - 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode - 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
@@ -207,7 +207,8 @@ class ComputeNodeClient:
|
||||
"resource_id": resource_id or "",
|
||||
}
|
||||
files = {"file": (filename, content)}
|
||||
async with httpx.AsyncClient(timeout=max(self.timeout, 60), headers=self.headers()) as client:
|
||||
timeout = httpx.Timeout(max(self.timeout, 60), connect=self.timeout)
|
||||
async with httpx.AsyncClient(timeout=timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/upload"),
|
||||
data=data,
|
||||
|
||||
@@ -10,6 +10,24 @@ def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
|
||||
output_dir = job.get("output_dir")
|
||||
if not output_dir:
|
||||
return None
|
||||
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
|
||||
data_root = "/data/yg-ft/"
|
||||
if full_path.startswith(data_root):
|
||||
full_path = full_path[len(data_root):]
|
||||
rel_path = full_path.lstrip("/")
|
||||
import httpx
|
||||
url = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
|
||||
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
|
||||
response = await http.get(url, params={"path": rel_path})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
synced: list[dict[str, Any]] = []
|
||||
@@ -66,18 +84,7 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
# Try to read eval_results.json from the job output directory
|
||||
if job.get("status") == "completed" and job.get("output_dir"):
|
||||
try:
|
||||
full_path = f"{job['output_dir'].rstrip('/')}/eval_results.json"
|
||||
# Convert absolute path to relative (strip YG_FT_DATA_ROOT prefix)
|
||||
data_root = "/data/yg-ft/"
|
||||
if full_path.startswith(data_root):
|
||||
full_path = full_path[len(data_root):]
|
||||
rel_path = full_path.lstrip("/")
|
||||
import httpx
|
||||
settings_path = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
|
||||
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
|
||||
read_resp = await http.get(settings_path, params={"path": rel_path})
|
||||
if read_resp.status_code == 200:
|
||||
result_content = read_resp.json()
|
||||
result_content = await fetch_eval_result_content(client, node, job)
|
||||
except Exception:
|
||||
pass
|
||||
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
||||
|
||||
Reference in New Issue
Block a user