feat: 评测任务元数据增强与前端展示优化
后端: - 新增评测指标标签生成(_build_eval_metric_label/_basic_metric_labels) - eval 任务 payload 富化(_enrich_eval_payload): 解析模型名/数据集名/维度配置, 生成 metric_label - 任务创建时解析 trained_models/模型名, metric 回退为 metric_label - 运行时状态刷新增加 2s 节流, 避免频繁触发 前端: - EvalTask 类型增加 dataset_id/metric_label - 评测列表与详情展示 metric_label 指标标签, 模型名/指标超长 tooltip 省略 - 创建页改用 getComputeGpus 过滤空闲 GPU, 移除 getSystemInfo 依赖
This commit is contained in:
wuyongtao
@@ -104,6 +104,68 @@ def count_dataset_records(content: str) -> int:
|
||||
return len([line for line in text.splitlines() if line.strip()])
|
||||
|
||||
|
||||
_EVAL_METHOD_LABELS = {
|
||||
"standard": "标准匹配",
|
||||
"metric_standard": "综合评测",
|
||||
"semantic": "语义相似度",
|
||||
"sentiment": "情感分析",
|
||||
"accuracy": "准确性评估",
|
||||
"safety": "安全性评估",
|
||||
"relevance": "相关性评估",
|
||||
"fluency": "流畅性评估",
|
||||
"factuality": "事实性评估",
|
||||
"custom": "自定义评估",
|
||||
}
|
||||
|
||||
|
||||
def _eval_method_label(value: Any) -> str:
|
||||
if isinstance(value, list):
|
||||
return "、".join(_eval_method_label(item) for item in value if item)
|
||||
text = str(value or "").strip()
|
||||
return _EVAL_METHOD_LABELS.get(text, text)
|
||||
|
||||
|
||||
def _basic_metric_labels(config: dict[str, Any] | None) -> list[str]:
|
||||
cfg = config or {}
|
||||
labels: list[str] = []
|
||||
bleu = cfg.get("bleu") or {}
|
||||
if bleu.get("enabled"):
|
||||
labels.append(f"BLEU-{int(bleu.get('ngram') or 4)}")
|
||||
rouge = cfg.get("rouge") or {}
|
||||
if rouge.get("enabled"):
|
||||
methods = rouge.get("methods") or []
|
||||
method_labels = {
|
||||
"rouge1": "ROUGE-1",
|
||||
"rouge2": "ROUGE-2",
|
||||
"rougeL": "ROUGE-L",
|
||||
"rouge_1": "ROUGE-1",
|
||||
"rouge_2": "ROUGE-2",
|
||||
"rouge_l": "ROUGE-L",
|
||||
}
|
||||
labels.extend(method_labels.get(str(item), str(item)) for item in methods)
|
||||
cosine = cfg.get("cosine") or {}
|
||||
if cosine.get("enabled"):
|
||||
labels.append("Cosine")
|
||||
return labels
|
||||
|
||||
|
||||
def _build_eval_metric_label(payload: dict[str, Any], dimension: dict[str, Any] | None = None) -> str:
|
||||
parts: list[str] = []
|
||||
dim = dimension or {}
|
||||
dim_type = str(dim.get("type") or payload.get("dimension_type") or "").strip()
|
||||
method_label = _eval_method_label(dim.get("eval_method") or payload.get("eval_method"))
|
||||
if dim_type in {"classification", "metric"} and method_label:
|
||||
parts.append(f"LLM：{method_label}")
|
||||
elif dim_type == "text_similarity" and method_label:
|
||||
parts.append(method_label)
|
||||
|
||||
parts.extend(_basic_metric_labels(payload.get("basic_metrics") or {}))
|
||||
if not parts:
|
||||
metric = str(payload.get("metric") or payload.get("eval_type") or "").strip()
|
||||
return "自定义评测" if metric == "custom" else (metric or "-")
|
||||
return " + ".join(parts)
|
||||
|
||||
|
||||
def version_number(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
number = int(value)
|
||||
@@ -339,6 +401,7 @@ class PlatformStore:
|
||||
self.ensure_seed_data()
|
||||
# Track which compute nodes have an active inference model loaded
|
||||
self._inference_nodes: set[str] = set()
|
||||
self._last_runtime_refresh = 0.0
|
||||
|
||||
# ── inference node tracking ────────────────────────────────────
|
||||
|
||||
@@ -463,6 +526,10 @@ class PlatformStore:
|
||||
def refresh_runtime_state(self) -> None:
|
||||
if get_settings().compute_mode != "simulator":
|
||||
return
|
||||
now_ts = time.monotonic()
|
||||
if now_ts - self._last_runtime_refresh < 2:
|
||||
return
|
||||
self._last_runtime_refresh = now_ts
|
||||
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
@@ -2146,19 +2213,54 @@ class PlatformStore:
|
||||
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
||||
payload = json_loads(row["payload"], {})
|
||||
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
||||
if not payload.get("metric_label"):
|
||||
payload["metric_label"] = _build_eval_metric_label(payload)
|
||||
if not payload.get("metric") or payload.get("metric") == "custom":
|
||||
payload["metric"] = payload["metric_label"]
|
||||
return payload
|
||||
|
||||
def _enrich_eval_payload(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
data = dict(payload)
|
||||
model_id = str(data.get("model_id") or "")
|
||||
if model_id and not data.get("model_name"):
|
||||
model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone()
|
||||
if not model:
|
||||
model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone()
|
||||
if model:
|
||||
data["model_name"] = model["name"]
|
||||
|
||||
dataset_id = str(data.get("dataset_id") or "")
|
||||
if dataset_id and not data.get("dataset"):
|
||||
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (dataset_id,)).fetchone()
|
||||
if dataset:
|
||||
data["dataset"] = dataset["name"]
|
||||
|
||||
dimension = None
|
||||
dimension_id = str(data.get("dimension_id") or "")
|
||||
if dimension_id:
|
||||
dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||
if dimension_row:
|
||||
dimension = json_loads(dimension_row["payload"], {})
|
||||
data.setdefault("dimension_type", dimension.get("type"))
|
||||
data.setdefault("eval_method", dimension.get("eval_method"))
|
||||
data.setdefault("evaluator_model", dimension.get("eval_model"))
|
||||
|
||||
data["metric_label"] = _build_eval_metric_label(data, dimension)
|
||||
if not data.get("metric") or data.get("metric") in {"custom", "自定义评测"}:
|
||||
data["metric"] = data["metric_label"]
|
||||
return data
|
||||
|
||||
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
|
||||
|
||||
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
payload = self._json_payload_row(row)
|
||||
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
|
||||
payload.setdefault("sample_count", 0)
|
||||
payload.setdefault("completed_count", 0)
|
||||
payload.setdefault("passed_count", 0)
|
||||
@@ -2184,12 +2286,29 @@ class PlatformStore:
|
||||
"metric": payload.get("metric") or "custom",
|
||||
}
|
||||
with self.connect() as conn:
|
||||
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
|
||||
model_id = str(payload.get("model_id") or "")
|
||||
model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone()
|
||||
trained_model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone()
|
||||
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
||||
dimension = None
|
||||
dimension_id = str(payload.get("dimension_id") or "")
|
||||
if dimension_id:
|
||||
dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||
if dimension_row:
|
||||
dimension = json_loads(dimension_row["payload"], {})
|
||||
if model:
|
||||
data.setdefault("model_name", model["name"])
|
||||
elif trained_model:
|
||||
data.setdefault("model_name", trained_model["name"])
|
||||
if dataset:
|
||||
data.setdefault("dataset", dataset["name"])
|
||||
if dimension:
|
||||
data.setdefault("dimension_type", dimension.get("type"))
|
||||
data.setdefault("eval_method", dimension.get("eval_method"))
|
||||
data.setdefault("evaluator_model", dimension.get("eval_model"))
|
||||
data["metric_label"] = _build_eval_metric_label(data, dimension)
|
||||
if data.get("metric") == "custom":
|
||||
data["metric"] = data["metric_label"]
|
||||
conn.execute(
|
||||
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
|
||||
Reference in New Issue
Block a user