fix: 推理/评测结果同步、数据集统计与算力节点管理增强

后端:
- 抽取 fetch_eval_result_content 复用函数,model_eval_detail 直接应用评测任务结果
- health 接口移除数据库依赖,返回静态指标
- 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理
- 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout
- 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error

评测引擎:
- _load_dataset 支持 JSON/JSONL 文件
- 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制

前端:
- 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode
- 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
wuyongtao
2026-08-03 15:49:21 +08:00
parent cc08b164d0
commit 5cc306eb0a
15 changed files with 402 additions and 112 deletions

View File

@@ -13,30 +13,42 @@ from __future__ import annotations
import json
import math
import re
import sys
import time
from difflib import SequenceMatcher
from pathlib import Path
from typing import Any
def _load_jsonl(path: str) -> list[dict[str, Any]]:
"""Load a JSONL dataset file. Each line must be a JSON object.
def _load_dataset(path: str) -> list[dict[str, Any]]:
"""Load a JSON or JSONL dataset file.
Supports common field names used across the platform:
* ``instruction`` + ``input`` + ``output`` (Alpaca-style)
* ``question`` + ``answer``
* ``messages`` (ShareGPT-style the last assistant message is treated as reference)
"""
file_path = Path(path)
text = file_path.read_text(encoding="utf-8", errors="replace").strip()
if not text:
return []
if file_path.suffix.lower() == ".json":
value = json.loads(text)
if isinstance(value, list):
return [item for item in value if isinstance(item, dict)]
return [value] if isinstance(value, dict) else []
samples: list[dict[str, Any]] = []
with open(path, encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
for line in text.splitlines():
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(obj, dict):
samples.append(obj)
return samples
@@ -115,8 +127,6 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
from sklearn.metrics.pairwise import cosine_similarity
except ImportError:
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
if len(predictions) < 2:
return {"enabled": True, "score": 0, "error": "need at least 2 samples for corpus cosine"}
try:
vectorizer = TfidfVectorizer()
tfidf = vectorizer.fit_transform(references + predictions)
@@ -129,6 +139,32 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
def _normalize_text(value: str) -> str:
return re.sub(r"\s+", " ", str(value or "").strip().lower())
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
total = len(predictions)
if not total:
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
matched = sum(
1
for ref, pred in zip(references, predictions)
if _normalize_text(ref) == _normalize_text(pred)
)
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
if not predictions:
return {"enabled": True, "score": 0}
scores = [
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
for ref, pred in zip(references, predictions)
]
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
# ---------------------------------------------------------------------------
# LLM Judge
# ---------------------------------------------------------------------------
@@ -265,7 +301,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
# ---- 1. Load dataset ----
print(f"[eval] loading dataset: {dataset_path}")
raw_samples = _load_jsonl(dataset_path)
raw_samples = _load_dataset(dataset_path)
print(f"[eval] loaded {len(raw_samples)} samples")
# ---- 2. Load model ----
@@ -356,6 +392,8 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
cosine_cfg = basic_cfg.get("cosine", {})
if cosine_cfg.get("enabled"):
metrics_result["cosine"] = _compute_cosine(references, predictions)
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
# ---- 5. Summarise ----
completed = len(samples)
@@ -375,9 +413,23 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
overall_evaluation = f"评测完成:{completed} 样本,{passed_count} 通过,平均 {avg_score}/{max_score}"
else:
passed_count = 0
overall_score = 0
enabled_scores = [
float(item.get("score") or 0)
for item in metrics_result.values()
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
]
overall_score = round(sum(enabled_scores) / len(enabled_scores), output_precision) if enabled_scores else 0
overall_score_max = 100
dimension_summary = []
dimension_summary = [
{
"name": name,
"score": float(item.get("score") or 0),
"max_score": 100,
"pass_rate": float(item.get("score") or 0),
}
for name, item in metrics_result.items()
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
]
overall_evaluation = f"评测完成:{completed} 样本(未配置 LLM 评委)"
result = {