diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index 0a0d4a9..617c2ef 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -104,6 +104,68 @@ def count_dataset_records(content: str) -> int: return len([line for line in text.splitlines() if line.strip()]) +_EVAL_METHOD_LABELS = { + "standard": "标准匹配", + "metric_standard": "综合评测", + "semantic": "语义相似度", + "sentiment": "情感分析", + "accuracy": "准确性评估", + "safety": "安全性评估", + "relevance": "相关性评估", + "fluency": "流畅性评估", + "factuality": "事实性评估", + "custom": "自定义评估", +} + + +def _eval_method_label(value: Any) -> str: + if isinstance(value, list): + return "、".join(_eval_method_label(item) for item in value if item) + text = str(value or "").strip() + return _EVAL_METHOD_LABELS.get(text, text) + + +def _basic_metric_labels(config: dict[str, Any] | None) -> list[str]: + cfg = config or {} + labels: list[str] = [] + bleu = cfg.get("bleu") or {} + if bleu.get("enabled"): + labels.append(f"BLEU-{int(bleu.get('ngram') or 4)}") + rouge = cfg.get("rouge") or {} + if rouge.get("enabled"): + methods = rouge.get("methods") or [] + method_labels = { + "rouge1": "ROUGE-1", + "rouge2": "ROUGE-2", + "rougeL": "ROUGE-L", + "rouge_1": "ROUGE-1", + "rouge_2": "ROUGE-2", + "rouge_l": "ROUGE-L", + } + labels.extend(method_labels.get(str(item), str(item)) for item in methods) + cosine = cfg.get("cosine") or {} + if cosine.get("enabled"): + labels.append("Cosine") + return labels + + +def _build_eval_metric_label(payload: dict[str, Any], dimension: dict[str, Any] | None = None) -> str: + parts: list[str] = [] + dim = dimension or {} + dim_type = str(dim.get("type") or payload.get("dimension_type") or "").strip() + method_label = _eval_method_label(dim.get("eval_method") or payload.get("eval_method")) + if dim_type in {"classification", "metric"} and method_label: + parts.append(f"LLM：{method_label}") + elif dim_type == "text_similarity" and method_label: + parts.append(method_label) + + parts.extend(_basic_metric_labels(payload.get("basic_metrics") or {})) + if not parts: + metric = str(payload.get("metric") or payload.get("eval_type") or "").strip() + return "自定义评测" if metric == "custom" else (metric or "-") + return " + ".join(parts) + + def version_number(value: Any, default: int = 0) -> int: try: number = int(value) @@ -339,6 +401,7 @@ class PlatformStore: self.ensure_seed_data() # Track which compute nodes have an active inference model loaded self._inference_nodes: set[str] = set() + self._last_runtime_refresh = 0.0 # ── inference node tracking ──────────────────────────────────── @@ -463,6 +526,10 @@ class PlatformStore: def refresh_runtime_state(self) -> None: if get_settings().compute_mode != "simulator": return + now_ts = time.monotonic() + if now_ts - self._last_runtime_refresh < 2: + return + self._last_runtime_refresh = now_ts with self.connect() as conn: rows = conn.execute( @@ -2146,19 +2213,54 @@ class PlatformStore: def _json_payload_row(self, row: PgRow) -> dict[str, Any]: payload = json_loads(row["payload"], {}) payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]}) + if not payload.get("metric_label"): + payload["metric_label"] = _build_eval_metric_label(payload) + if not payload.get("metric") or payload.get("metric") == "custom": + payload["metric"] = payload["metric_label"] return payload + def _enrich_eval_payload(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload) + model_id = str(data.get("model_id") or "") + if model_id and not data.get("model_name"): + model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone() + if not model: + model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone() + if model: + data["model_name"] = model["name"] + + dataset_id = str(data.get("dataset_id") or "") + if dataset_id and not data.get("dataset"): + dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (dataset_id,)).fetchone() + if dataset: + data["dataset"] = dataset["name"] + + dimension = None + dimension_id = str(data.get("dimension_id") or "") + if dimension_id: + dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() + if dimension_row: + dimension = json_loads(dimension_row["payload"], {}) + data.setdefault("dimension_type", dimension.get("type")) + data.setdefault("eval_method", dimension.get("eval_method")) + data.setdefault("evaluator_model", dimension.get("eval_model")) + + data["metric_label"] = _build_eval_metric_label(data, dimension) + if not data.get("metric") or data.get("metric") in {"custom", "自定义评测"}: + data["metric"] = data["metric_label"] + return data + def eval_tasks(self) -> list[dict[str, Any]]: with self.connect() as conn: rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall() - return [self._json_payload_row(row) for row in rows] + return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows] def eval_task(self, task_id: str) -> dict[str, Any]: with self.connect() as conn: row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone() if not row: raise KeyError(task_id) - payload = self._json_payload_row(row) + payload = self._enrich_eval_payload(conn, self._json_payload_row(row)) payload.setdefault("sample_count", 0) payload.setdefault("completed_count", 0) payload.setdefault("passed_count", 0) @@ -2184,12 +2286,29 @@ class PlatformStore: "metric": payload.get("metric") or "custom", } with self.connect() as conn: - model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone() + model_id = str(payload.get("model_id") or "") + model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone() + trained_model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone() dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone() + dimension = None + dimension_id = str(payload.get("dimension_id") or "") + if dimension_id: + dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() + if dimension_row: + dimension = json_loads(dimension_row["payload"], {}) if model: data.setdefault("model_name", model["name"]) + elif trained_model: + data.setdefault("model_name", trained_model["name"]) if dataset: data.setdefault("dataset", dataset["name"]) + if dimension: + data.setdefault("dimension_type", dimension.get("type")) + data.setdefault("eval_method", dimension.get("eval_method")) + data.setdefault("evaluator_model", dimension.get("eval_model")) + data["metric_label"] = _build_eval_metric_label(data, dimension) + if data.get("metric") == "custom": + data["metric"] = data["metric_label"] conn.execute( "INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)", (task_id, name, json_dumps(data), status, now), diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 0bcec31..5da5188 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -245,7 +245,9 @@ export interface EvalTask { model_name?: string model_id?: number | string dataset?: string + dataset_id?: number | string metric?: string + metric_label?: string score?: number status?: string create_time?: string diff --git a/frontend/src/views/eval/EvalCreateView.vue b/frontend/src/views/eval/EvalCreateView.vue index d8c1af6..7c0b75e 100644 --- a/frontend/src/views/eval/EvalCreateView.vue +++ b/frontend/src/views/eval/EvalCreateView.vue @@ -1,4 +1,4 @@ -