feat: 模型评测端到端闭环 — EvalRunner引擎 + 算力节点Job执行 + 结果回写

算力节点 (compute):
- 新建 eval_runner.py: 评测执行引擎，作为subprocess运行
  - 加载模型 + JSONL数据集 + 逐样本推理
  - BLEU/ROUGE/Cosine基础指标计算
  - LLM Judge评分(OpenAI兼容API调用)
  - 结果写入eval_results.json
- adapter.py: build_command新增engine=eval分支
- main.py: 新增/json模块导入，新增/compute/files/read端点，eval job校验

后端:
- platform.py: 重写startEval提交eval job到算力节点
  - 支持models表和trained_models表查找
  - 已合并模型不传adapter路径
- platform_store.py: 新增update_eval_task/running_eval_tasks/apply_eval_job_result
- sync.py: poller新增eval job同步，异步读取eval_results.json回写结果

前端:
- EvalCreateView/DimensionCreateView: eval模型过滤扩展(API类型+api_url)
- EvalCreateView: GPU过滤在线节点空闲GPU
- EvalTaskSetupStep: GPU value从数组index改为gpu.id
- BasicMetricSetupStep: ROUGE方法名修正(rouge_1→rouge1)
- EvalView: 新增5秒轮询刷新

Co-Authored-By: Claude <noreply@anthropic.com>

backend/app/api/v1/endpoints/platform.py

@@ -1000,15 +1000,175 @@ async def model_eval_list() -> dict[str, Any]:
 @router.get("/model-eval/{task_id}")
 async def model_eval_detail(task_id: str) -> dict[str, Any]:
     try:
-        return ok(get_platform_store().eval_task(task_id))
+        store = get_platform_store()
+        task = store.eval_task(task_id)
+        # If the eval job completed on a compute node, try to load results
+        if task.get("result_artifact_path"):
+            node = next(
+                (n for n in store.compute_nodes() if n["id"] == task.get("compute_node_id")),
+                None,
+            )
+            if node:
+                try:
+                    client = ComputeNodeClient(node["api_base_url"])
+                    job = await client.get_job(task["compute_job_id"])
+                    artifacts = job.get("artifacts") or []
+                    for art in artifacts:
+                        if art.get("name") == "eval_results.json":
+                            task["_result_artifact"] = art
+                            break
+                except Exception:
+                    pass
+        return ok(task)
     except KeyError:
         raise fail(404, "eval task not found")
 
 
 @router.post("/model-eval/start")
 async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
-    task = get_platform_store().create_eval_task(payload)
-    return ok({"task_id": task["id"], **task})
+    """Start an evaluation task: submit eval job to compute node."""
+    store = get_platform_store()
+    # 1. Create eval task record
+    task = store.create_eval_task({**payload, "status": "pending"})
+
+    # 2. Resolve model path (supports both regular models and trained models)
+    model_id = str(payload.get("model_id", ""))
+    model_path = ""
+    adapter_path = payload.get("adapter_path", "")
+    try:
+        db_model = store.model(model_id)
+        model_path = db_model.get("path", "")
+    except KeyError:
+        # Try trained_models table (IDs prefixed with tm_)
+        trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
+        if trained:
+            merged_path = trained.get("merged_path", "")
+            base_path = trained.get("base_model_path", "")
+            if trained.get("merged") and merged_path:
+                # Merged model: use merged_path as model, no adapter needed
+                model_path = merged_path
+            elif base_path:
+                # Unmerged: use base model + adapter checkpoint
+                model_path = base_path
+                if merged_path:
+                    adapter_path = merged_path
+            else:
+                model_path = merged_path or base_path
+    if not model_path:
+        store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
+        return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})
+
+    # 3. Resolve dataset file
+    dataset_id = str(payload.get("dataset_id", ""))
+    dataset_path = ""
+    try:
+        ds_files = store.training_dataset_files(dataset_id)
+        if ds_files:
+            dataset_path = ds_files[0].get("local_path") or ds_files[0].get("name", "")
+    except Exception:
+        pass
+    if not dataset_path:
+        # Try to get file content and sync to compute
+        try:
+            ds = store.dataset(dataset_id)
+            for f in ds.get("files", []):
+                if f.get("content"):
+                    dataset_path = f.get("name", f"dataset_{dataset_id}.jsonl")
+                    break
+        except KeyError:
+            pass
+    if not dataset_path:
+        store.update_eval_task(task["id"], {"status": "failed", "error": "dataset not found or no files"})
+        return ok({"task_id": task["id"], "status": "failed", "error": "dataset not found or no files"})
+
+    # 4. Resolve dimension config
+    dimension_id = str(payload.get("dimension_id", ""))
+    dimension_cfg: dict[str, Any] = {}
+    if dimension_id:
+        try:
+            dim = store.dimension(dimension_id)
+            # Resolve eval model API config
+            eval_model_name = dim.get("eval_model", "")
+            api_url = ""
+            api_key = ""
+            if eval_model_name:
+                try:
+                    eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name)
+                    api_url = eval_model.get("api_url", "")
+                    api_key = eval_model.get("api_key", "")
+                except (KeyError, Exception):
+                    pass
+            dimension_cfg = {
+                "type": dim.get("type", ""),
+                "eval_model": eval_model_name,
+                "eval_method": dim.get("eval_method", ""),
+                "eval_prompt": dim.get("eval_prompt", ""),
+                "api_url": api_url,
+                "api_key": api_key,
+                "score_min": dim.get("score_min", 0),
+                "score_max": dim.get("score_max", 5),
+                "pass_threshold": dim.get("pass_threshold", 3),
+            }
+        except KeyError:
+            pass
+
+    # 5. Select compute node
+    node = _select_first_online_node(store)
+    if not node:
+        store.update_eval_task(task["id"], {"status": "failed", "error": "no online compute node"})
+        return ok({"task_id": task["id"], "status": "failed", "error": "no online compute node"})
+
+    # 6. Build eval job payload
+    output_dir = f"/data/yg-ft/outputs/{task['id']}"
+    job_payload = {
+        "id": f"eval_{task['id']}",
+        "name": task.get("eval_task_name", task["id"]),
+        "engine": "eval",
+        "model_name_or_path": model_path,
+        "adapter_name_or_path": adapter_path,
+        "template": payload.get("template", "qwen"),
+        "dataset_path": dataset_path,
+        "output_dir": output_dir,
+        "basic_metrics": payload.get("basic_metrics", {}),
+        "dimension": dimension_cfg,
+        "gpus": [int(payload.get("gpu_id", 0))],
+        "temperature": payload.get("temperature", 0.1),
+        "max_new_tokens": payload.get("max_new_tokens", 512),
+        "compute_node_id": node["id"],
+    }
+
+    # 7. Submit to compute node via create_job (uses engine="eval" path)
+    try:
+        client = ComputeNodeClient(node["api_base_url"])
+        # Sync dataset file to compute node if needed
+        if not dataset_path.startswith("/"):
+            try:
+                ds_files = store.training_dataset_files(dataset_id)
+                if ds_files and ds_files[0].get("content"):
+                    upload_result = await client.upload_file(
+                        ds_files[0].get("name", "eval_data.jsonl"),
+                        ds_files[0]["content"].encode("utf-8"),
+                        f"datasets/{dataset_id}/{ds_files[0].get('name', 'eval_data.jsonl')}",
+                        resource_type="dataset",
+                        resource_id=dataset_id,
+                    )
+                    job_payload["dataset_path"] = upload_result.get("local_path", dataset_path)
+            except Exception:
+                pass
+
+        job = await client.create_job(job_payload)
+        store.update_eval_task(task["id"], {
+            "status": "running",
+            "compute_job_id": job.get("id"),
+            "compute_node_id": node["id"],
+            "output_dir": output_dir,
+        })
+        if job.get("status") in {"queued", "running"}:
+            store.mark_inference_loaded(node["id"])
+        return ok({"task_id": task["id"], "status": "running", "job": job})
+    except Exception as exc:
+        store.update_eval_task(task["id"], {"status": "failed", "error": str(exc)})
+        return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})
 
 
 @router.delete("/model-eval/{task_id}")

backend/app/db/platform_store.py

@@ -2076,10 +2076,55 @@ class PlatformStore:
             )
         return self.eval_task(task_id)
 
+    def update_eval_task(self, task_id: str, updates: dict[str, Any]) -> dict[str, Any]:
+        """Update fields in an eval task's payload without replacing the whole record."""
+        task = self.eval_task(task_id)
+        merged = {**task, **updates}
+        with self.connect() as conn:
+            conn.execute(
+                "UPDATE eval_tasks SET payload=?, status=? WHERE id=?",
+                (json_dumps(merged), merged.get("status", task.get("status", "pending")), task_id),
+            )
+        return self.eval_task(task_id)
+
     def delete_eval_task(self, task_id: str) -> None:
         with self.connect() as conn:
             conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
 
+    def running_eval_tasks(self) -> list[dict[str, Any]]:
+        """Return eval tasks that have been submitted to a compute node and are still running."""
+        return [
+            task for task in self.eval_tasks()
+            if task.get("compute_job_id") and task.get("status") in {"queued", "running"}
+        ]
+
+    def apply_eval_job_result(self, task_id: str, job: dict[str, Any], result_content: dict[str, Any] | None = None) -> dict[str, Any]:
+        """Sync a compute job status/result back to an eval task."""
+        task = self.eval_task(task_id)
+        job_status = str(job.get("status", ""))
+        status_map = {"queued": "running", "running": "running", "completed": "completed",
+                       "failed": "failed", "stopped": "stopped"}
+        new_status = status_map.get(job_status, job_status or task.get("status", "pending"))
+        updates: dict[str, Any] = {
+            "status": new_status,
+            "progress": int(job.get("progress", 0)),
+            "output_dir": job.get("output_dir", task.get("output_dir", "")),
+        }
+        # On completion, populate results from eval_results.json content
+        if new_status == "completed" and result_content:
+            updates.update({
+                "overall_score": result_content.get("overall_score", 0),
+                "overall_score_max": result_content.get("overall_score_max", 100),
+                "overall_evaluation": result_content.get("overall_evaluation", ""),
+                "improvement_suggestions": result_content.get("improvement_suggestions", []),
+                "dimension_summary": result_content.get("dimension_summary", []),
+                "samples": result_content.get("samples", []),
+                "sample_count": result_content.get("sample_count", 0),
+                "completed_count": result_content.get("completed_count", 0),
+                "passed_count": result_content.get("passed_count", 0),
+            })
+        return self.update_eval_task(task_id, updates)
+
     def dimensions(self) -> list[dict[str, Any]]:
         with self.connect() as conn:
             rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()

backend/app/modules/compute_gateway/sync.py

@@ -48,4 +48,45 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
             standalone_synced.append(store.sync_model_merge_job(record["id"], job))
         except Exception as exc:  # noqa: BLE001 - keep polling other jobs
             failed.append({"job_id": record["id"], "error": str(exc)})
-    return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
+
+    # ── Eval job sync ────────────────────────────────────────────────
+    eval_synced = 0
+    for eval_task in store.running_eval_tasks():
+        node = next(
+            (item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
+            None,
+        )
+        if not node:
+            failed.append({"eval_task_id": eval_task["id"], "error": "compute node not found"})
+            continue
+        try:
+            client = ComputeNodeClient(node["api_base_url"])
+            job = await client.get_job(eval_task["compute_job_id"])
+            result_content = None
+            # Try to read eval_results.json from the job output directory
+            if job.get("status") == "completed" and job.get("output_dir"):
+                try:
+                    full_path = f"{job['output_dir'].rstrip('/')}/eval_results.json"
+                    # Convert absolute path to relative (strip YG_FT_DATA_ROOT prefix)
+                    data_root = "/data/yg-ft/"
+                    if full_path.startswith(data_root):
+                        full_path = full_path[len(data_root):]
+                    rel_path = full_path.lstrip("/")
+                    import httpx
+                    settings_path = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
+                    async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
+                        read_resp = await http.get(settings_path, params={"path": rel_path})
+                        if read_resp.status_code == 200:
+                            result_content = read_resp.json()
+                except Exception:
+                    pass
+            store.apply_eval_job_result(eval_task["id"], job, result_content)
+            # If job completed, un-mark inference loaded
+            if job.get("status") in {"completed", "failed", "stopped"}:
+                store.mark_inference_unloaded(node["id"])
+            eval_synced += 1
+        except Exception as exc:  # noqa: BLE001
+            failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
+
+    return {"synced": len(synced) + len(standalone_synced) + eval_synced, "failed": failed,
+            "items": synced, "standalone": standalone_synced, "eval_synced": eval_synced}

compute/api/main.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import os
 import math
 import hashlib
@@ -449,6 +450,29 @@ def create_app() -> FastAPI:
                 accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload)
                 errors.extend(accelerator_errors)
                 warnings.extend(accelerator_warnings)
+        elif engine == "eval":
+            # Eval engine: validate model path and dataset path
+            if not payload.get("model_name_or_path"):
+                errors.append("model_name_or_path is required for eval")
+            else:
+                path_checks.append(_check_path_item({
+                    "name": "model_name_or_path",
+                    "path": payload.get("model_name_or_path", ""),
+                    "type": "any",
+                    "required": True,
+                }))
+            if payload.get("dataset_path"):
+                path_checks.append(_check_path_item({
+                    "name": "dataset_path",
+                    "path": payload.get("dataset_path", ""),
+                    "type": "file",
+                    "required": True,
+                }))
+            else:
+                errors.append("dataset_path is required for eval")
+            if shutil.which("python") is None:
+                errors.append("python runtime not found")
+
         elif engine == "smoke":
             warnings.append("smoke engine skips model and dataset path checks")
 
@@ -811,6 +835,22 @@ def create_app() -> FastAPI:
             "checksum_sha256": checksum,
         }
 
+    @app.get(f"{route_prefix}/compute/files/read")
+    async def read_file(path: str = Query(...)) -> JSONResponse:
+        """Read a text file from within YG_FT_DATA_ROOT.  Used by the backend
+        to fetch eval results and other job outputs."""
+        data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
+        target = (data_root / path.lstrip("/\\")).resolve()
+        if not _path_inside(data_root, target):
+            raise HTTPException(status_code=400, detail="path must stay inside YG_FT_DATA_ROOT")
+        if not target.is_file():
+            raise HTTPException(status_code=404, detail="file not found")
+        try:
+            content = target.read_text(encoding="utf-8")
+            return JSONResponse(json.loads(content) if content.strip().startswith("{") else {"content": content})
+        except Exception as exc:
+            raise HTTPException(status_code=500, detail=str(exc))
+
     @app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
     async def download_file(file_id: str) -> FileResponse:
         upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads"

compute/engines/llama_factory/adapter.py

@@ -204,6 +204,31 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
             command.extend(["--quantization_bit", str(quantization_bit)])
         return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={})
 
+    if engine == "eval":
+        output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'eval-job')}"
+        eval_config_path = str(Path(output_dir) / "eval_config.json")
+        eval_config = {
+            "model_name_or_path": config.get("model_name_or_path", ""),
+            "adapter_name_or_path": config.get("adapter_name_or_path", ""),
+            "template": config.get("template", "qwen"),
+            "dataset_path": config.get("dataset_path", ""),
+            "output_dir": output_dir,
+            "basic_metrics": config.get("basic_metrics", {}),
+            "dimension": config.get("dimension", {}),
+            "temperature": config.get("temperature", 0.1),
+            "top_p": config.get("top_p", 0.95),
+            "max_new_tokens": config.get("max_new_tokens", 512),
+            "infer_backend": config.get("infer_backend", "huggingface"),
+            "infer_dtype": config.get("infer_dtype", "auto"),
+        }
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        Path(eval_config_path).write_text(json.dumps(eval_config, ensure_ascii=False, indent=2), encoding="utf-8")
+        return LlamaFactoryCommand(
+            command=["python", "-u", "-m", "compute.engines.llama_factory.eval_runner", "--config", eval_config_path],
+            work_dir="/app",
+            env={},
+        )
+
     errors = validate_config(config)
     if errors:
         raise ValueError("; ".join(errors))

compute/engines/llama_factory/eval_runner.py

@@ -0,0 +1,428 @@
+"""
+Evaluation runner — executes model evaluation as a subprocess job.
+
+Usage:
+    python -m compute.engines.llama_factory.eval_runner --config <config_json_path>
+
+The config JSON is written by the compute API before spawning this subprocess.
+Results are written to ``output_dir/eval_results.json`` and progress is printed
+to stdout (captured as job logs).
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+def _load_jsonl(path: str) -> list[dict[str, Any]]:
+    """Load a JSONL dataset file.  Each line must be a JSON object.
+
+    Supports common field names used across the platform:
+    * ``instruction`` + ``input`` + ``output``  (Alpaca-style)
+    * ``question`` + ``answer``
+    * ``messages``  (ShareGPT-style – the last assistant message is treated as reference)
+    """
+    samples: list[dict[str, Any]] = []
+    with open(path, encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            samples.append(obj)
+    return samples
+
+
+def _sample_question(sample: dict[str, Any]) -> str:
+    """Extract the user-facing question / instruction from a sample."""
+    if sample.get("instruction"):
+        text = sample["instruction"]
+        if sample.get("input"):
+            text += "\n" + sample["input"]
+        return text
+    if sample.get("question"):
+        return sample["question"]
+    # ShareGPT-style: use the last user message as question
+    messages = sample.get("messages") or []
+    user_msgs = [m["content"] for m in messages if m.get("role") == "user"]
+    return user_msgs[-1] if user_msgs else ""
+
+
+def _sample_reference(sample: dict[str, Any]) -> str:
+    """Extract the reference answer from a sample."""
+    if sample.get("output"):
+        return sample["output"]
+    if sample.get("answer"):
+        return sample["answer"]
+    messages = sample.get("messages") or []
+    assistant_msgs = [m["content"] for m in messages if m.get("role") == "assistant"]
+    return assistant_msgs[-1] if assistant_msgs else ""
+
+
+# ---------------------------------------------------------------------------
+# Basic metrics
+# ---------------------------------------------------------------------------
+
+def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4) -> dict[str, Any]:
+    """Compute BLEU score via sacrebleu (corpus-level)."""
+    try:
+        from sacrebleu.metrics import BLEU
+    except ImportError:
+        return {"enabled": False, "error": "sacrebleu not installed", "score": 0}
+    bleu = BLEU(max_ngram_order=ngram)
+    # sacrebleu expects list-of-strings; we have one reference per prediction
+    score = bleu.corpus_score(predictions, [references])
+    return {
+        "enabled": True,
+        "score": round(score.score, 2),
+        "bleu": round(score.score, 2),
+    }
+
+
+def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
+    """Compute ROUGE scores via rouge-score."""
+    try:
+        from rouge_score import rouge_scorer
+    except ImportError:
+        return {"enabled": False, "error": "rouge-score not installed", "score": 0}
+    methods = methods or ["rouge1", "rouge2", "rougeL"]
+    # Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
+    _rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
+    methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
+    scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True)
+    totals: dict[str, float] = {}
+    n = max(len(predictions), 1)
+    for ref, pred in zip(references, predictions):
+        result = scorer.score(ref, pred)
+        for key in methods:
+            totals[key] = totals.get(key, 0) + result[key].fmeasure
+    avg = {k: round(v / n, 4) for k, v in totals.items()}
+    return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg}
+
+
+def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
+    """Compute average cosine similarity via sklearn."""
+    try:
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        from sklearn.metrics.pairwise import cosine_similarity
+    except ImportError:
+        return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
+    if len(predictions) < 2:
+        return {"enabled": True, "score": 0, "error": "need at least 2 samples for corpus cosine"}
+    try:
+        vectorizer = TfidfVectorizer()
+        tfidf = vectorizer.fit_transform(references + predictions)
+        n = len(references)
+        ref_vec = tfidf[:n]
+        pred_vec = tfidf[n:]
+        sims = cosine_similarity(ref_vec, pred_vec).diagonal()
+        return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)}
+    except ValueError:
+        return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
+
+
+# ---------------------------------------------------------------------------
+# LLM Judge
+# ---------------------------------------------------------------------------
+
+def _judge_sample(
+    question: str,
+    reference: str,
+    prediction: str,
+    config: dict[str, Any],
+) -> dict[str, Any]:
+    """Call an OpenAI-compatible LLM to judge a single sample.
+
+    Returns a dict with keys:
+        score, max_score, passed, judgement, evaluation_reason, error_type
+    """
+    api_url = (config.get("api_url") or "").strip().rstrip("/")
+    api_key = (config.get("api_key") or "").strip()
+    eval_model = (config.get("eval_model") or "").strip()
+    eval_prompt = (config.get("eval_prompt") or "").strip()
+    score_min = float(config.get("score_min", 0))
+    score_max = float(config.get("score_max", 5))
+    pass_threshold = float(config.get("pass_threshold", 3))
+
+    if not api_url or not eval_model:
+        return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置",
+                "evaluation_reason": "未配置评测模型", "error_type": "其他"}
+
+    system_msg = (
+        eval_prompt
+        or "你是一个专业的评测专家。请根据参考答-案对被测模型的输出进行评分。"
+    )
+    user_msg = (
+        f"## 问题\n{question}\n\n"
+        f"## 参考答案\n{reference}\n\n"
+        f"## 模型输出\n{prediction}\n\n"
+        f"请给出 {score_min}-{score_max} 分的评分，并说明理由。"
+    )
+
+    try:
+        import urllib.request
+        import urllib.error
+
+        body = json.dumps({
+            "model": eval_model,
+            "messages": [
+                {"role": "system", "content": system_msg},
+                {"role": "user", "content": user_msg},
+            ],
+            "temperature": 0.3,
+            "max_tokens": 512,
+        }).encode("utf-8")
+
+        req = urllib.request.Request(
+            f"{api_url}/v1/chat/completions",
+            data=body,
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {api_key}",
+            },
+        )
+        resp = urllib.request.urlopen(req, timeout=120)
+        data = json.loads(resp.read().decode("utf-8"))
+        reply = data["choices"][0]["message"]["content"]
+    except Exception as exc:
+        return {"score": 0, "max_score": score_max, "passed": False,
+                "judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
+                "error_type": "其他"}
+
+    # Parse score from reply — look for patterns like "4分" or "Score: 4"
+    score = 0
+    import re
+    score_patterns = [
+        r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)',
+        r'(\d+(?:\.\d+)?)\s*分',
+        r'(\d+(?:\.\d+)?)\s*/\s*\d+',
+    ]
+    for pat in score_patterns:
+        m = re.search(pat, reply, re.IGNORECASE)
+        if m:
+            try:
+                score = float(m.group(1))
+            except ValueError:
+                continue
+            break
+    score = max(score_min, min(score_max, score))
+    passed = score >= pass_threshold
+
+    # Determine judgement label
+    if score >= pass_threshold + 1:
+        judgement = "正确"
+    elif score >= pass_threshold:
+        judgement = "部分正确"
+    else:
+        judgement = "错误"
+
+    # Guess error type from reply
+    reply_lower = reply.lower()
+    if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
+        error_type = "幻觉"
+    elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
+        error_type = "不完整"
+    elif any(w in reply_lower for w in ["格式", "format"]):
+        error_type = "格式偏差"
+    elif any(w in reply_lower for w in ["混淆", "confusion", "错误"]):
+        error_type = "混淆"
+    else:
+        error_type = "其他"
+
+    return {
+        "score": score,
+        "max_score": score_max,
+        "passed": passed,
+        "judgement": judgement,
+        "evaluation_reason": reply[:2000],
+        "error_type": error_type,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+
+def run_eval(config: dict[str, Any]) -> dict[str, Any]:
+    """Execute a full evaluation run.  Returns the result dict (also written to file)."""
+    model_path = config["model_name_or_path"]
+    adapter_path = config.get("adapter_name_or_path", "")
+    template = config.get("template", "qwen")
+    dataset_path = config["dataset_path"]
+    output_dir = Path(config["output_dir"])
+    output_dir.mkdir(parents=True, exist_ok=True)
+    basic_cfg = config.get("basic_metrics", {})
+    dimension_cfg = config.get("dimension", {}) or {}
+    output_precision = int(basic_cfg.get("output_precision", 2))
+
+    # ---- 1. Load dataset ----
+    print(f"[eval] loading dataset: {dataset_path}")
+    raw_samples = _load_jsonl(dataset_path)
+    print(f"[eval] loaded {len(raw_samples)} samples")
+
+    # ---- 2. Load model ----
+    print(f"[eval] loading model: {model_path}")
+    from compute.engines.llama_factory.inference import InferenceSession
+    session = InferenceSession()
+    load_result = session.load(
+        model_name_or_path=model_path,
+        adapter_name_or_path=adapter_path,
+        template=template,
+        infer_backend=config.get("infer_backend", "huggingface"),
+        infer_dtype=config.get("infer_dtype", "auto"),
+    )
+    if not load_result.get("loaded"):
+        raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
+    print(f"[eval] model loaded OK")
+
+    # ---- 3. Run inference on each sample ----
+    samples: list[dict[str, Any]] = []
+    predictions: list[str] = []
+    references: list[str] = []
+    questions: list[str] = []
+
+    total = len(raw_samples)
+    judge_enabled = bool(dimension_cfg.get("eval_model") and dimension_cfg.get("api_url"))
+    print(f"[eval] starting inference on {total} samples, judge={'enabled' if judge_enabled else 'disabled'}")
+
+    for idx, raw in enumerate(raw_samples, start=1):
+        question = _sample_question(raw)
+        reference = _sample_reference(raw)
+        if not question:
+            print(f"[eval] sample {idx}/{total}: skipped (no question)")
+            continue
+
+        # Inference
+        chat_msgs = [{"role": "user", "content": question}]
+        result = session.chat(
+            chat_msgs,
+            temperature=float(config.get("temperature", 0.1)),
+            top_p=float(config.get("top_p", 0.95)),
+            max_new_tokens=int(config.get("max_new_tokens", 512)),
+            do_sample=False,
+        )
+        prediction = result.get("response", "") if not result.get("error") else f"[ERROR] {result['error']}"
+
+        predictions.append(prediction)
+        references.append(reference)
+        questions.append(question)
+
+        # LLM Judge
+        judge_result: dict[str, Any] = {}
+        if judge_enabled:
+            judge_result = _judge_sample(question, reference, prediction, dimension_cfg)
+
+        samples.append({
+            "index": idx,
+            "input": question,
+            "reference_answer": reference,
+            "model_output": prediction,
+            "score": judge_result.get("score"),
+            "max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5)),
+            "passed": judge_result.get("passed"),
+            "judgement": judge_result.get("judgement"),
+            "evaluation_reason": judge_result.get("evaluation_reason", ""),
+            "error_type": judge_result.get("error_type"),
+            "dimension_scores": [
+                {"name": "judge_score", "score": judge_result.get("score", 0),
+                 "max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5))},
+            ] if judge_result else [],
+            "status": "completed",
+        })
+
+        progress_pct = int(idx / max(total, 1) * 100)
+        print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
+
+    # ---- 4. Compute basic metrics ----
+    print(f"[eval] computing basic metrics on {len(predictions)} predictions")
+    metrics_result: dict[str, Any] = {}
+
+    bleu_cfg = basic_cfg.get("bleu", {})
+    if bleu_cfg.get("enabled"):
+        metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
+
+    rouge_cfg = basic_cfg.get("rouge", {})
+    if rouge_cfg.get("enabled"):
+        metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
+
+    cosine_cfg = basic_cfg.get("cosine", {})
+    if cosine_cfg.get("enabled"):
+        metrics_result["cosine"] = _compute_cosine(references, predictions)
+
+    # ---- 5. Summarise ----
+    completed = len(samples)
+    if judge_enabled:
+        scored = [s for s in samples if s.get("score") is not None]
+        passed_count = len([s for s in scored if s.get("passed")])
+        avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
+        max_score = dimension_cfg.get("score_max", 5)
+        overall_score = round(avg_score / max_score * 100, output_precision)
+        overall_score_max = 100
+        dimension_summary = [{
+            "name": "综合评分",
+            "score": overall_score,
+            "max_score": 100,
+            "pass_rate": round(passed_count / max(completed, 1) * 100, 1),
+        }]
+        overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/{max_score} 分"
+    else:
+        passed_count = 0
+        overall_score = 0
+        overall_score_max = 100
+        dimension_summary = []
+        overall_evaluation = f"评测完成：{completed} 样本（未配置 LLM 评委）"
+
+    result = {
+        "overall_score": overall_score,
+        "overall_score_max": overall_score_max,
+        "overall_evaluation": overall_evaluation,
+        "improvement_suggestions": [],
+        "dimension_summary": dimension_summary,
+        "samples": samples,
+        "sample_count": total,
+        "completed_count": completed,
+        "passed_count": passed_count,
+        "basic_metrics": metrics_result,
+    }
+
+    # ---- 6. Write results ----
+    result_path = output_dir / "eval_results.json"
+    result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"[eval] results written to {result_path}")
+    return result
+
+
+def main() -> None:
+    import argparse
+    parser = argparse.ArgumentParser(description="YG-FT Evaluation Runner")
+    parser.add_argument("--config", required=True, help="Path to eval config JSON file")
+    args = parser.parse_args()
+
+    config_path = Path(args.config)
+    if not config_path.exists():
+        print(f"FATAL: config file not found: {args.config}", file=sys.stderr)
+        sys.exit(1)
+
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    start = time.time()
+    try:
+        run_eval(config)
+        elapsed = time.time() - start
+        print(f"[eval] DONE in {elapsed:.1f}s")
+    except Exception as exc:
+        print(f"[eval] FAILED: {exc}", file=sys.stderr)
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docs/模型评测功能总结.md

@@ -0,0 +1,121 @@
+# 模型评测功能总结
+
+本项目（基于 LLaMA-Factory 的微调训练平台）包含 **4 套相对独立** 的模型评测能力，分别面向不同的使用场景：
+
+| 能力 | 入口/目录 | 评测类型 | 打分方式 |
+| --- | --- | --- | --- |
+| 1. 学术 Benchmark 评测 | `llamafactory/eval/` | 选择题式基准（类 MMLU/C-Eval） | 选项匹配 + few-shot |
+| 2. 评估工作台 | `backend/app/api/v1/eval/` | 生成式问答（指令跟随） | BLEU / ROUGE / ExactMatch + 可选 LLM 评审 |
+| 3. 平台评估系统 | `backend/app/api/v1/evaluation/` | 基于评估数据集的问答 | 判卷模型（judge model）打分（0–5 分） |
+| 4. 训练时验证评估 | `backend/app/services/task_runner.py` | 训练验证集 | loss 指标 |
+
+下面分别说明。
+
+---
+
+## 1. 学术 Benchmark 评测（LLaMA-Factory 原生）
+
+面向标准学术选择题基准（如 MMLU、C-Eval 等），复用 LLaMA-Factory 原生的评测框架。
+
+**核心文件**
+- `llamafactory/eval/evaluator.py`：`Evaluator` 类 + `run_eval()` 入口
+- `llamafactory/eval/template.py`：评测 prompt 模板（中/英，含 few-shot 示例构建）
+- `llamafactory/hparams/evaluation_args.py`：`EvaluationArguments` 配置类
+
+**工作流程**
+1. 按 `task`（benchmark 名称）加载数据集，按科目（subject）拆分。
+2. 每个样本构造 few-shot 提示词（`n_shot` 控制示例数，由 `lang` 决定中/英模板），将题干与候选选项拼入 prompt。
+3. 调用模型推理得到预测，与标准答案比对，统计每个科目及整体的 `accuracy`。
+4. 结果写入 `save_dir`，打印各科目与平均准确率。
+
+**关键参数（`EvaluationArguments`）**
+- `task`：基准数据集名
+- `batch_size` / `n_shot` / `lang` / `save_dir` / `seed`
+- `model_name_or_path`、`template`、`trust_remote_code` 等模型相关参数
+
+> 该能力属于框架底层，本平台前端未直接提供操作入口，主要通过配置文件/脚本调用。
+
+---
+
+## 2. 评估工作台（生成式评测 + 指标计算）
+
+后端路由位于 `backend/app/api/v1/eval/__init__.py`，前端称为「评估工作台」。**适用于评测模型的指令跟随与生成质量**，并支持 LLM 作为裁判（LLM-as-a-Judge）。
+
+**API 端点**
+- `GET  /evaluation/tasks`：列出评测任务（`frontend/src/api/evaluation.ts:listTasks`）
+- `POST /evaluation/run`：提交一次评测（`runEval`）
+- `GET  /evaluation/report/{task_id}`：拉取评测报告（`getReport`）
+- `DELETE /evaluation/tasks/{task_id}`：删除任务（`deleteTask`）
+
+**评测流程（`run_eval`）**
+1. 通过 **LLaMA-Factory 数据管道**（`get_dataset`) 加载数据集，支持 `subset` 与抽样（`eval_sample`）。
+2. 用 **原生 transformers** 加载模型在本地做生成推理（单进程顺序生成，便于展示样本）。
+3. 计算客观指标（`compute_score`）：
+   - `BLEU`（sacrebleu）
+   - `ROUGE-1 / ROUGE-2 / ROUGE-L`（rouge-score）
+   - `Exact Match`
+4. **可选 LLM 评审**（judge）：当配置了 `judge_model` / `judge_api_base` / `judge_api_key` 时，调用 OpenAI 兼容接口对每条样本打分（10 分制），并输出 4 个维度与理由：
+   - 核心事实正确性 `factual`
+   - 信息完整性 `completeness`
+   - 无幻觉 `no_hallucination`
+   - 格式合规性 `format`
+   - 综合分 `score` + `reason`
+5. 任务状态持久化在后端 `eval_tasks.json`（支持 running/completed/failed/stopped），前端轮询进度。
+
+**前端页面**
+- `frontend/src/views/evaluation/EvaluateTask.vue`：任务列表、创建评测对话框（选模型、数据集、指标、可选 judge 配置）
+- `frontend/src/views/evaluation/EvaluateReport.vue`：报告页，展示综合得分、BLEU、ROUGE-L、各维度指标及「参考答案 vs 模型预测 vs LLM 评审」对比样例
+
+---
+
+## 3. 平台评估系统（基于评估数据集 + 判卷模型）
+
+后端路由位于 `backend/app/api/v1/evaluation/__init__.py`，是平台业务层自研的评测体系。通过「评估数据集」组织题目，可一次性对 **多个被测模型 + 指定判卷模型** 进行批量评分。
+
+**核心概念（数据模型 `backend/app/models/models.py`）**
+- `EvalDataset`（`models.py:131`）：评估数据集，从项目问答对（`Question`/`Chunk`）中按 `question_type`（mixed/fact/reasoning）选题构建，状态 `pending/running/completed/failed`。
+- `EvalResult`（`models.py:147`）：单条评测结果，含 `judge_score`（0–5 分）、`is_correct`（true/false/partial）、`feedback`、`expected_answer` 等。
+- `Task`（`models.py:184`）：后台任务，`task_type="model-evaluation"`，记录进度与 `model_info`（存放平均分等汇总）。
+
+**评测流程（`process_evaluation_task`，`backend/app/services/task_processor.py:336` 起）**
+1. 加载评估数据集关联的题目，可选带入 `chunk` 上下文（RAG 场景）。
+2. 对每道题，先用 `build_eval_prompt` 组合「上下文 + 题目 + 参考答案」，调用 **判卷模型**（`call_model`，temperature=0.3）生成评分。
+3. `parse_eval_result` 解析出 `score`(0–5)、`is_correct`、`feedback`，写入 `EvalResult`。
+4. 逐题提交进度（`completed_count` / `progress`），支持中途 `stopped`。
+5. 汇总：`avg_score = 总分/有效数 × 20`（换算百分制），`avg_score_5 = 总分/有效数`（5 分制），存入 `task.model_info`。判定规则：得分 **≥3 视为正确**。
+
+**特点**
+- 判卷与被测模型解耦：被测模型给出答案，判卷模型（judge）独立评分，降低自评偏差。
+- 支持失败隔离：单题异常写入 `evaluation_status: failed` 记录而不中断整体任务。
+
+---
+
+## 4. 训练时验证评估
+
+在微调训练任务执行期间，由 `backend/app/services/task_runner.py` 的 `do_eval` 触发：
+
+- 在训练过程中对验证集（validation set）计算 `eval_loss`，用于监控过拟合。
+- 结果回填到 `Task` 的 `loss_info` / `detail`，前端绘制 loss 曲线。
+- 属于训练配套的轻量评估，不参与上述 1–3 的业务评测。
+
+---
+
+## 附属：前端评测相关页面
+
+| 文件 | 作用 |
+| --- | --- |
+| `frontend/src/views/evaluation/EvaluateTask.vue` | 评估工作台：任务列表 + 创建评测 |
+| `frontend/src/views/evaluation/EvaluateReport.vue` | 评估报告：指标卡 + 维度标签 + 对比样例 |
+| `frontend/src/api/evaluation.ts` | 评估工作台接口封装 |
+| 平台评估系统入口 | 评估数据集管理 + 评估任务（model-evaluation）创建与结果查看 |
+
+---
+
+## 小结
+
+- **想要学术榜单式准确率** → 用能力 1（LLaMA-Factory `eval/`）。
+- **想要开放式生成质量（BLEU/ROUGE + LLM 评审）** → 用能力 2（评估工作台 `/evaluation/run`）。
+- **想要基于自有问答数据、用判卷模型批量打分** → 用能力 3（平台评估系统 `model-evaluation` 任务）。
+- **训练过程监控** → 能力 4（`do_eval` 验证集 loss）。
+
+三种业务评测（1/2/3）相互独立，可并存于同一平台；数据模型（`EvalDataset`/`EvalResult`/`Task`）主要服务于能力 3，而能力 2 使用独立的 `eval_tasks.json` 文件持久化。

frontend/src/views/eval/DimensionCreateView.vue

@@ -79,7 +79,9 @@ async function loadEditData() {
 async function loadModels() {
   try {
     const all = (await getModelList()) || []
-    evalModels.value = all.filter((m) => m.purpose === 'evaluation')
+    evalModels.value = all.filter(
+      (m) => m.purpose === 'evaluation' || (m.model_source === 'api' && !!m.api_url),
+    )
   } catch {
     evalModels.value = []
   }

frontend/src/views/eval/EvalCreateView.vue

@@ -11,6 +11,7 @@ import { createDimension, startEval } from '@/api/modules/eval'
 import { getTrainedModels, getModelList } from '@/api/modules/model'
 import { getDatasetList } from '@/api/modules/dataset'
 import { getSystemInfo } from '@/api/modules/system'
+import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
 import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types'
 
 type StepExposed = { validate: () => Promise<boolean> }
@@ -84,15 +85,26 @@ async function loadData() {
     getDatasetList(),
     getSystemInfo(),
     getModelList(),
+    getComputeNodes(),
   ])
 
   if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
   if (results[1].status === 'fulfilled') {
     evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
   }
-  if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || []
+  if (results[2].status === 'fulfilled') {
+    const allGpus: GpuInfo[] = results[2].value?.gpu || []
+    const nodes: ComputeNode[] = (results[4].status === 'fulfilled' ? results[4].value : []) || []
+    const onlineIds = new Set(nodes.filter((n) => n.enabled && n.scheduler_status === 'online').map((n) => n.id))
+    // Only show idle GPUs from online compute nodes
+    gpus.value = allGpus.filter(
+      (g) => g.status === 'idle' && (!g.node_id || onlineIds.has(g.node_id)),
+    )
+  }
   if (results[3].status === 'fulfilled') {
-    evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation')
+    evalModels.value = (results[3].value || []).filter(
+      (model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url),
+    )
   }
 
   const failedCount = results.filter((result) => result.status === 'rejected').length

frontend/src/views/eval/EvalView.vue

@@ -1,9 +1,10 @@
 <script setup lang="ts">
-import { ref, onMounted } from 'vue'
+import { ref, onMounted, onUnmounted } from 'vue'
 import { useRouter } from 'vue-router'
 import { ElMessage, ElMessageBox } from 'element-plus'
 import DataTablePage from '@/components/DataTablePage.vue'
 import ModelStatusTag from '@/components/ModelStatusTag.vue'
+import { usePolling } from '@/composables/usePolling'
 import {
   getEvalList,
   deleteEval,
@@ -54,8 +55,19 @@ function handleViewDetail(row: any) {
   router.push({ name: 'model-eval-detail', params: { id: row.id } })
 }
 
-onMounted(() => {
-  loadEvalList()
+const { start: startPolling, stop: stopPolling } = usePolling(
+  () => loadEvalList(),
+  5000,
+  { immediate: false },
+)
+
+onMounted(async () => {
+  await loadEvalList()
+  startPolling()
+})
+
+onUnmounted(() => {
+  stopPolling()
 })
 </script>
 

frontend/src/views/eval/create/BasicMetricSetupStep.vue

@@ -56,9 +56,9 @@ defineExpose({ validate })
     </el-form-item>
     <el-form-item v-if="form.rouge_enabled" label="ROUGE methods">
       <el-checkbox-group v-model="form.rouge_methods">
-        <el-checkbox value="rouge_1">ROUGE-1</el-checkbox>
-        <el-checkbox value="rouge_2">ROUGE-2</el-checkbox>
-        <el-checkbox value="rouge_l">ROUGE-L</el-checkbox>
+        <el-checkbox value="rouge1">ROUGE-1</el-checkbox>
+        <el-checkbox value="rouge2">ROUGE-2</el-checkbox>
+        <el-checkbox value="rougeL">ROUGE-L</el-checkbox>
       </el-checkbox-group>
     </el-form-item>
 

frontend/src/views/eval/create/EvalTaskSetupStep.vue

@@ -103,10 +103,10 @@ defineExpose({ validate })
     <el-form-item label="选择 GPU" prop="gpu_id">
       <el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
         <el-option
-          v-for="(gpu, index) in gpus"
-          :key="index"
-          :label="`${gpu.name} (GPU ${index})`"
-          :value="index"
+          v-for="gpu in gpus"
+          :key="gpu.id"
+          :label="`${gpu.name} (GPU ${gpu.id})`"
+          :value="gpu.id ?? 0"
         />
       </el-select>
     </el-form-item>