fix: 推理/评测结果同步、数据集统计与算力节点管理增强
后端: - 抽取 fetch_eval_result_content 复用函数,model_eval_detail 直接应用评测任务结果 - health 接口移除数据库依赖,返回静态指标 - 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理 - 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout - 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error 评测引擎: - _load_dataset 支持 JSON/JSONL 文件 - 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制 前端: - 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode - 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
@@ -13,30 +13,42 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _load_jsonl(path: str) -> list[dict[str, Any]]:
|
||||
"""Load a JSONL dataset file. Each line must be a JSON object.
|
||||
def _load_dataset(path: str) -> list[dict[str, Any]]:
|
||||
"""Load a JSON or JSONL dataset file.
|
||||
|
||||
Supports common field names used across the platform:
|
||||
* ``instruction`` + ``input`` + ``output`` (Alpaca-style)
|
||||
* ``question`` + ``answer``
|
||||
* ``messages`` (ShareGPT-style – the last assistant message is treated as reference)
|
||||
"""
|
||||
file_path = Path(path)
|
||||
text = file_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return []
|
||||
if file_path.suffix.lower() == ".json":
|
||||
value = json.loads(text)
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
return [value] if isinstance(value, dict) else []
|
||||
|
||||
samples: list[dict[str, Any]] = []
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
samples.append(obj)
|
||||
return samples
|
||||
|
||||
@@ -115,8 +127,6 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
|
||||
if len(predictions) < 2:
|
||||
return {"enabled": True, "score": 0, "error": "need at least 2 samples for corpus cosine"}
|
||||
try:
|
||||
vectorizer = TfidfVectorizer()
|
||||
tfidf = vectorizer.fit_transform(references + predictions)
|
||||
@@ -129,6 +139,32 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
|
||||
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
|
||||
|
||||
|
||||
def _normalize_text(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "").strip().lower())
|
||||
|
||||
|
||||
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
total = len(predictions)
|
||||
if not total:
|
||||
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
|
||||
matched = sum(
|
||||
1
|
||||
for ref, pred in zip(references, predictions)
|
||||
if _normalize_text(ref) == _normalize_text(pred)
|
||||
)
|
||||
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
|
||||
|
||||
|
||||
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
if not predictions:
|
||||
return {"enabled": True, "score": 0}
|
||||
scores = [
|
||||
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
|
||||
for ref, pred in zip(references, predictions)
|
||||
]
|
||||
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM Judge
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -265,7 +301,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
# ---- 1. Load dataset ----
|
||||
print(f"[eval] loading dataset: {dataset_path}")
|
||||
raw_samples = _load_jsonl(dataset_path)
|
||||
raw_samples = _load_dataset(dataset_path)
|
||||
print(f"[eval] loaded {len(raw_samples)} samples")
|
||||
|
||||
# ---- 2. Load model ----
|
||||
@@ -356,6 +392,8 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
cosine_cfg = basic_cfg.get("cosine", {})
|
||||
if cosine_cfg.get("enabled"):
|
||||
metrics_result["cosine"] = _compute_cosine(references, predictions)
|
||||
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
|
||||
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
|
||||
|
||||
# ---- 5. Summarise ----
|
||||
completed = len(samples)
|
||||
@@ -375,9 +413,23 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
overall_evaluation = f"评测完成:{completed} 样本,{passed_count} 通过,平均 {avg_score}/{max_score} 分"
|
||||
else:
|
||||
passed_count = 0
|
||||
overall_score = 0
|
||||
enabled_scores = [
|
||||
float(item.get("score") or 0)
|
||||
for item in metrics_result.values()
|
||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
||||
]
|
||||
overall_score = round(sum(enabled_scores) / len(enabled_scores), output_precision) if enabled_scores else 0
|
||||
overall_score_max = 100
|
||||
dimension_summary = []
|
||||
dimension_summary = [
|
||||
{
|
||||
"name": name,
|
||||
"score": float(item.get("score") or 0),
|
||||
"max_score": 100,
|
||||
"pass_rate": float(item.get("score") or 0),
|
||||
}
|
||||
for name, item in metrics_result.items()
|
||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
||||
]
|
||||
overall_evaluation = f"评测完成:{completed} 样本(未配置 LLM 评委)"
|
||||
|
||||
result = {
|
||||
|
||||
Reference in New Issue
Block a user