From 4e5c43fad51944b67544634c7f3542212fdf8752 Mon Sep 17 00:00:00 2001 From: wuyongtao Date: Mon, 3 Aug 2026 17:34:21 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=AF=84=E6=B5=8B=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=85=83=E6=95=B0=E6=8D=AE=E5=A2=9E=E5=BC=BA=E4=B8=8E=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=E5=B1=95=E7=A4=BA=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 后端: - 新增评测指标标签生成(_build_eval_metric_label/_basic_metric_labels) - eval 任务 payload 富化(_enrich_eval_payload): 解析模型名/数据集名/维度配置, 生成 metric_label - 任务创建时解析 trained_models/模型名, metric 回退为 metric_label - 运行时状态刷新增加 2s 节流, 避免频繁触发 前端: - EvalTask 类型增加 dataset_id/metric_label - 评测列表与详情展示 metric_label 指标标签, 模型名/指标超长 tooltip 省略 - 创建页改用 getComputeGpus 过滤空闲 GPU, 移除 getSystemInfo 依赖 --- backend/app/db/platform_store.py | 125 ++++++++++++++++++++- frontend/src/types/index.ts | 2 + frontend/src/views/eval/EvalCreateView.vue | 20 +--- frontend/src/views/eval/EvalDetailView.vue | 63 ++++------- frontend/src/views/eval/EvalView.vue | 37 +++++- 5 files changed, 182 insertions(+), 65 deletions(-) diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index 0a0d4a9..617c2ef 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -104,6 +104,68 @@ def count_dataset_records(content: str) -> int: return len([line for line in text.splitlines() if line.strip()]) +_EVAL_METHOD_LABELS = { + "standard": "标准匹配", + "metric_standard": "综合评测", + "semantic": "语义相似度", + "sentiment": "情感分析", + "accuracy": "准确性评估", + "safety": "安全性评估", + "relevance": "相关性评估", + "fluency": "流畅性评估", + "factuality": "事实性评估", + "custom": "自定义评估", +} + + +def _eval_method_label(value: Any) -> str: + if isinstance(value, list): + return "、".join(_eval_method_label(item) for item in value if item) + text = str(value or "").strip() + return _EVAL_METHOD_LABELS.get(text, text) + + +def _basic_metric_labels(config: dict[str, Any] | None) -> list[str]: + cfg = config or {} + labels: list[str] = [] + bleu = cfg.get("bleu") or {} + if bleu.get("enabled"): + labels.append(f"BLEU-{int(bleu.get('ngram') or 4)}") + rouge = cfg.get("rouge") or {} + if rouge.get("enabled"): + methods = rouge.get("methods") or [] + method_labels = { + "rouge1": "ROUGE-1", + "rouge2": "ROUGE-2", + "rougeL": "ROUGE-L", + "rouge_1": "ROUGE-1", + "rouge_2": "ROUGE-2", + "rouge_l": "ROUGE-L", + } + labels.extend(method_labels.get(str(item), str(item)) for item in methods) + cosine = cfg.get("cosine") or {} + if cosine.get("enabled"): + labels.append("Cosine") + return labels + + +def _build_eval_metric_label(payload: dict[str, Any], dimension: dict[str, Any] | None = None) -> str: + parts: list[str] = [] + dim = dimension or {} + dim_type = str(dim.get("type") or payload.get("dimension_type") or "").strip() + method_label = _eval_method_label(dim.get("eval_method") or payload.get("eval_method")) + if dim_type in {"classification", "metric"} and method_label: + parts.append(f"LLM：{method_label}") + elif dim_type == "text_similarity" and method_label: + parts.append(method_label) + + parts.extend(_basic_metric_labels(payload.get("basic_metrics") or {})) + if not parts: + metric = str(payload.get("metric") or payload.get("eval_type") or "").strip() + return "自定义评测" if metric == "custom" else (metric or "-") + return " + ".join(parts) + + def version_number(value: Any, default: int = 0) -> int: try: number = int(value) @@ -339,6 +401,7 @@ class PlatformStore: self.ensure_seed_data() # Track which compute nodes have an active inference model loaded self._inference_nodes: set[str] = set() + self._last_runtime_refresh = 0.0 # ── inference node tracking ──────────────────────────────────── @@ -463,6 +526,10 @@ class PlatformStore: def refresh_runtime_state(self) -> None: if get_settings().compute_mode != "simulator": return + now_ts = time.monotonic() + if now_ts - self._last_runtime_refresh < 2: + return + self._last_runtime_refresh = now_ts with self.connect() as conn: rows = conn.execute( @@ -2146,19 +2213,54 @@ class PlatformStore: def _json_payload_row(self, row: PgRow) -> dict[str, Any]: payload = json_loads(row["payload"], {}) payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]}) + if not payload.get("metric_label"): + payload["metric_label"] = _build_eval_metric_label(payload) + if not payload.get("metric") or payload.get("metric") == "custom": + payload["metric"] = payload["metric_label"] return payload + def _enrich_eval_payload(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload) + model_id = str(data.get("model_id") or "") + if model_id and not data.get("model_name"): + model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone() + if not model: + model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone() + if model: + data["model_name"] = model["name"] + + dataset_id = str(data.get("dataset_id") or "") + if dataset_id and not data.get("dataset"): + dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (dataset_id,)).fetchone() + if dataset: + data["dataset"] = dataset["name"] + + dimension = None + dimension_id = str(data.get("dimension_id") or "") + if dimension_id: + dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() + if dimension_row: + dimension = json_loads(dimension_row["payload"], {}) + data.setdefault("dimension_type", dimension.get("type")) + data.setdefault("eval_method", dimension.get("eval_method")) + data.setdefault("evaluator_model", dimension.get("eval_model")) + + data["metric_label"] = _build_eval_metric_label(data, dimension) + if not data.get("metric") or data.get("metric") in {"custom", "自定义评测"}: + data["metric"] = data["metric_label"] + return data + def eval_tasks(self) -> list[dict[str, Any]]: with self.connect() as conn: rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall() - return [self._json_payload_row(row) for row in rows] + return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows] def eval_task(self, task_id: str) -> dict[str, Any]: with self.connect() as conn: row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone() if not row: raise KeyError(task_id) - payload = self._json_payload_row(row) + payload = self._enrich_eval_payload(conn, self._json_payload_row(row)) payload.setdefault("sample_count", 0) payload.setdefault("completed_count", 0) payload.setdefault("passed_count", 0) @@ -2184,12 +2286,29 @@ class PlatformStore: "metric": payload.get("metric") or "custom", } with self.connect() as conn: - model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone() + model_id = str(payload.get("model_id") or "") + model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone() + trained_model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone() dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone() + dimension = None + dimension_id = str(payload.get("dimension_id") or "") + if dimension_id: + dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() + if dimension_row: + dimension = json_loads(dimension_row["payload"], {}) if model: data.setdefault("model_name", model["name"]) + elif trained_model: + data.setdefault("model_name", trained_model["name"]) if dataset: data.setdefault("dataset", dataset["name"]) + if dimension: + data.setdefault("dimension_type", dimension.get("type")) + data.setdefault("eval_method", dimension.get("eval_method")) + data.setdefault("evaluator_model", dimension.get("eval_model")) + data["metric_label"] = _build_eval_metric_label(data, dimension) + if data.get("metric") == "custom": + data["metric"] = data["metric_label"] conn.execute( "INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)", (task_id, name, json_dumps(data), status, now), diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 0bcec31..5da5188 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -245,7 +245,9 @@ export interface EvalTask { model_name?: string model_id?: number | string dataset?: string + dataset_id?: number | string metric?: string + metric_label?: string score?: number status?: string create_time?: string diff --git a/frontend/src/views/eval/EvalCreateView.vue b/frontend/src/views/eval/EvalCreateView.vue index d8c1af6..7c0b75e 100644 --- a/frontend/src/views/eval/EvalCreateView.vue +++ b/frontend/src/views/eval/EvalCreateView.vue @@ -1,4 +1,4 @@ -