From 0c39f2f5b9f2371e8dc3de39e742c75e19248793 Mon Sep 17 00:00:00 2001 From: wuyongtao Date: Tue, 28 Jul 2026 19:34:41 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=A8=A1=E5=9E=8B=E8=AF=84=E6=B5=8B?= =?UTF-8?q?=E7=AB=AF=E5=88=B0=E7=AB=AF=E9=97=AD=E7=8E=AF=20=E2=80=94=20Eva?= =?UTF-8?q?lRunner=E5=BC=95=E6=93=8E=20+=20=E7=AE=97=E5=8A=9B=E8=8A=82?= =?UTF-8?q?=E7=82=B9Job=E6=89=A7=E8=A1=8C=20+=20=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E5=9B=9E=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 算力节点 (compute): - 新建 eval_runner.py: 评测执行引擎，作为subprocess运行 - 加载模型 + JSONL数据集 + 逐样本推理 - BLEU/ROUGE/Cosine基础指标计算 - LLM Judge评分(OpenAI兼容API调用) - 结果写入eval_results.json - adapter.py: build_command新增engine=eval分支 - main.py: 新增/json模块导入，新增/compute/files/read端点，eval job校验 后端: - platform.py: 重写startEval提交eval job到算力节点 - 支持models表和trained_models表查找 - 已合并模型不传adapter路径 - platform_store.py: 新增update_eval_task/running_eval_tasks/apply_eval_job_result - sync.py: poller新增eval job同步，异步读取eval_results.json回写结果 前端: - EvalCreateView/DimensionCreateView: eval模型过滤扩展(API类型+api_url) - EvalCreateView: GPU过滤在线节点空闲GPU - EvalTaskSetupStep: GPU value从数组index改为gpu.id - BasicMetricSetupStep: ROUGE方法名修正(rouge_1→rouge1) - EvalView: 新增5秒轮询刷新 Co-Authored-By: Claude --- backend/app/api/v1/endpoints/platform.py | 166 ++++++- backend/app/db/platform_store.py | 45 ++ backend/app/modules/compute_gateway/sync.py | 43 +- compute/api/main.py | 40 ++ compute/engines/llama_factory/adapter.py | 25 + compute/engines/llama_factory/eval_runner.py | 428 ++++++++++++++++++ docs/模型评测功能总结.md | 121 +++++ .../src/views/eval/DimensionCreateView.vue | 4 +- frontend/src/views/eval/EvalCreateView.vue | 16 +- frontend/src/views/eval/EvalView.vue | 18 +- .../eval/create/BasicMetricSetupStep.vue | 6 +- .../views/eval/create/EvalTaskSetupStep.vue | 8 +- 12 files changed, 903 insertions(+), 17 deletions(-) create mode 100644 compute/engines/llama_factory/eval_runner.py create mode 100644 docs/模型评测功能总结.md diff --git a/backend/app/api/v1/endpoints/platform.py b/backend/app/api/v1/endpoints/platform.py index ecaec0b..1689371 100644 --- a/backend/app/api/v1/endpoints/platform.py +++ b/backend/app/api/v1/endpoints/platform.py @@ -1000,15 +1000,175 @@ async def model_eval_list() -> dict[str, Any]: @router.get("/model-eval/{task_id}") async def model_eval_detail(task_id: str) -> dict[str, Any]: try: - return ok(get_platform_store().eval_task(task_id)) + store = get_platform_store() + task = store.eval_task(task_id) + # If the eval job completed on a compute node, try to load results + if task.get("result_artifact_path"): + node = next( + (n for n in store.compute_nodes() if n["id"] == task.get("compute_node_id")), + None, + ) + if node: + try: + client = ComputeNodeClient(node["api_base_url"]) + job = await client.get_job(task["compute_job_id"]) + artifacts = job.get("artifacts") or [] + for art in artifacts: + if art.get("name") == "eval_results.json": + task["_result_artifact"] = art + break + except Exception: + pass + return ok(task) except KeyError: raise fail(404, "eval task not found") @router.post("/model-eval/start") async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: - task = get_platform_store().create_eval_task(payload) - return ok({"task_id": task["id"], **task}) + """Start an evaluation task: submit eval job to compute node.""" + store = get_platform_store() + # 1. Create eval task record + task = store.create_eval_task({**payload, "status": "pending"}) + + # 2. Resolve model path (supports both regular models and trained models) + model_id = str(payload.get("model_id", "")) + model_path = "" + adapter_path = payload.get("adapter_path", "") + try: + db_model = store.model(model_id) + model_path = db_model.get("path", "") + except KeyError: + # Try trained_models table (IDs prefixed with tm_) + trained = next((m for m in store.trained_models() if m["id"] == model_id), None) + if trained: + merged_path = trained.get("merged_path", "") + base_path = trained.get("base_model_path", "") + if trained.get("merged") and merged_path: + # Merged model: use merged_path as model, no adapter needed + model_path = merged_path + elif base_path: + # Unmerged: use base model + adapter checkpoint + model_path = base_path + if merged_path: + adapter_path = merged_path + else: + model_path = merged_path or base_path + if not model_path: + store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"}) + return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"}) + + # 3. Resolve dataset file + dataset_id = str(payload.get("dataset_id", "")) + dataset_path = "" + try: + ds_files = store.training_dataset_files(dataset_id) + if ds_files: + dataset_path = ds_files[0].get("local_path") or ds_files[0].get("name", "") + except Exception: + pass + if not dataset_path: + # Try to get file content and sync to compute + try: + ds = store.dataset(dataset_id) + for f in ds.get("files", []): + if f.get("content"): + dataset_path = f.get("name", f"dataset_{dataset_id}.jsonl") + break + except KeyError: + pass + if not dataset_path: + store.update_eval_task(task["id"], {"status": "failed", "error": "dataset not found or no files"}) + return ok({"task_id": task["id"], "status": "failed", "error": "dataset not found or no files"}) + + # 4. Resolve dimension config + dimension_id = str(payload.get("dimension_id", "")) + dimension_cfg: dict[str, Any] = {} + if dimension_id: + try: + dim = store.dimension(dimension_id) + # Resolve eval model API config + eval_model_name = dim.get("eval_model", "") + api_url = "" + api_key = "" + if eval_model_name: + try: + eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name) + api_url = eval_model.get("api_url", "") + api_key = eval_model.get("api_key", "") + except (KeyError, Exception): + pass + dimension_cfg = { + "type": dim.get("type", ""), + "eval_model": eval_model_name, + "eval_method": dim.get("eval_method", ""), + "eval_prompt": dim.get("eval_prompt", ""), + "api_url": api_url, + "api_key": api_key, + "score_min": dim.get("score_min", 0), + "score_max": dim.get("score_max", 5), + "pass_threshold": dim.get("pass_threshold", 3), + } + except KeyError: + pass + + # 5. Select compute node + node = _select_first_online_node(store) + if not node: + store.update_eval_task(task["id"], {"status": "failed", "error": "no online compute node"}) + return ok({"task_id": task["id"], "status": "failed", "error": "no online compute node"}) + + # 6. Build eval job payload + output_dir = f"/data/yg-ft/outputs/{task['id']}" + job_payload = { + "id": f"eval_{task['id']}", + "name": task.get("eval_task_name", task["id"]), + "engine": "eval", + "model_name_or_path": model_path, + "adapter_name_or_path": adapter_path, + "template": payload.get("template", "qwen"), + "dataset_path": dataset_path, + "output_dir": output_dir, + "basic_metrics": payload.get("basic_metrics", {}), + "dimension": dimension_cfg, + "gpus": [int(payload.get("gpu_id", 0))], + "temperature": payload.get("temperature", 0.1), + "max_new_tokens": payload.get("max_new_tokens", 512), + "compute_node_id": node["id"], + } + + # 7. Submit to compute node via create_job (uses engine="eval" path) + try: + client = ComputeNodeClient(node["api_base_url"]) + # Sync dataset file to compute node if needed + if not dataset_path.startswith("/"): + try: + ds_files = store.training_dataset_files(dataset_id) + if ds_files and ds_files[0].get("content"): + upload_result = await client.upload_file( + ds_files[0].get("name", "eval_data.jsonl"), + ds_files[0]["content"].encode("utf-8"), + f"datasets/{dataset_id}/{ds_files[0].get('name', 'eval_data.jsonl')}", + resource_type="dataset", + resource_id=dataset_id, + ) + job_payload["dataset_path"] = upload_result.get("local_path", dataset_path) + except Exception: + pass + + job = await client.create_job(job_payload) + store.update_eval_task(task["id"], { + "status": "running", + "compute_job_id": job.get("id"), + "compute_node_id": node["id"], + "output_dir": output_dir, + }) + if job.get("status") in {"queued", "running"}: + store.mark_inference_loaded(node["id"]) + return ok({"task_id": task["id"], "status": "running", "job": job}) + except Exception as exc: + store.update_eval_task(task["id"], {"status": "failed", "error": str(exc)}) + return ok({"task_id": task["id"], "status": "failed", "error": str(exc)}) @router.delete("/model-eval/{task_id}") diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index f66789d..4ebeabb 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -2076,10 +2076,55 @@ class PlatformStore: ) return self.eval_task(task_id) + def update_eval_task(self, task_id: str, updates: dict[str, Any]) -> dict[str, Any]: + """Update fields in an eval task's payload without replacing the whole record.""" + task = self.eval_task(task_id) + merged = {**task, **updates} + with self.connect() as conn: + conn.execute( + "UPDATE eval_tasks SET payload=?, status=? WHERE id=?", + (json_dumps(merged), merged.get("status", task.get("status", "pending")), task_id), + ) + return self.eval_task(task_id) + def delete_eval_task(self, task_id: str) -> None: with self.connect() as conn: conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,)) + def running_eval_tasks(self) -> list[dict[str, Any]]: + """Return eval tasks that have been submitted to a compute node and are still running.""" + return [ + task for task in self.eval_tasks() + if task.get("compute_job_id") and task.get("status") in {"queued", "running"} + ] + + def apply_eval_job_result(self, task_id: str, job: dict[str, Any], result_content: dict[str, Any] | None = None) -> dict[str, Any]: + """Sync a compute job status/result back to an eval task.""" + task = self.eval_task(task_id) + job_status = str(job.get("status", "")) + status_map = {"queued": "running", "running": "running", "completed": "completed", + "failed": "failed", "stopped": "stopped"} + new_status = status_map.get(job_status, job_status or task.get("status", "pending")) + updates: dict[str, Any] = { + "status": new_status, + "progress": int(job.get("progress", 0)), + "output_dir": job.get("output_dir", task.get("output_dir", "")), + } + # On completion, populate results from eval_results.json content + if new_status == "completed" and result_content: + updates.update({ + "overall_score": result_content.get("overall_score", 0), + "overall_score_max": result_content.get("overall_score_max", 100), + "overall_evaluation": result_content.get("overall_evaluation", ""), + "improvement_suggestions": result_content.get("improvement_suggestions", []), + "dimension_summary": result_content.get("dimension_summary", []), + "samples": result_content.get("samples", []), + "sample_count": result_content.get("sample_count", 0), + "completed_count": result_content.get("completed_count", 0), + "passed_count": result_content.get("passed_count", 0), + }) + return self.update_eval_task(task_id, updates) + def dimensions(self) -> list[dict[str, Any]]: with self.connect() as conn: rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall() diff --git a/backend/app/modules/compute_gateway/sync.py b/backend/app/modules/compute_gateway/sync.py index 0e2720a..b8abcbc 100644 --- a/backend/app/modules/compute_gateway/sync.py +++ b/backend/app/modules/compute_gateway/sync.py @@ -48,4 +48,45 @@ async def poll_compute_jobs_once() -> dict[str, Any]: standalone_synced.append(store.sync_model_merge_job(record["id"], job)) except Exception as exc: # noqa: BLE001 - keep polling other jobs failed.append({"job_id": record["id"], "error": str(exc)}) - return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced} + + # ── Eval job sync ──────────────────────────────────────────────── + eval_synced = 0 + for eval_task in store.running_eval_tasks(): + node = next( + (item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")), + None, + ) + if not node: + failed.append({"eval_task_id": eval_task["id"], "error": "compute node not found"}) + continue + try: + client = ComputeNodeClient(node["api_base_url"]) + job = await client.get_job(eval_task["compute_job_id"]) + result_content = None + # Try to read eval_results.json from the job output directory + if job.get("status") == "completed" and job.get("output_dir"): + try: + full_path = f"{job['output_dir'].rstrip('/')}/eval_results.json" + # Convert absolute path to relative (strip YG_FT_DATA_ROOT prefix) + data_root = "/data/yg-ft/" + if full_path.startswith(data_root): + full_path = full_path[len(data_root):] + rel_path = full_path.lstrip("/") + import httpx + settings_path = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read" + async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http: + read_resp = await http.get(settings_path, params={"path": rel_path}) + if read_resp.status_code == 200: + result_content = read_resp.json() + except Exception: + pass + store.apply_eval_job_result(eval_task["id"], job, result_content) + # If job completed, un-mark inference loaded + if job.get("status") in {"completed", "failed", "stopped"}: + store.mark_inference_unloaded(node["id"]) + eval_synced += 1 + except Exception as exc: # noqa: BLE001 + failed.append({"eval_task_id": eval_task["id"], "error": str(exc)}) + + return {"synced": len(synced) + len(standalone_synced) + eval_synced, "failed": failed, + "items": synced, "standalone": standalone_synced, "eval_synced": eval_synced} diff --git a/compute/api/main.py b/compute/api/main.py index 3628525..d88d2fd 100644 --- a/compute/api/main.py +++ b/compute/api/main.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import math import hashlib @@ -449,6 +450,29 @@ def create_app() -> FastAPI: accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload) errors.extend(accelerator_errors) warnings.extend(accelerator_warnings) + elif engine == "eval": + # Eval engine: validate model path and dataset path + if not payload.get("model_name_or_path"): + errors.append("model_name_or_path is required for eval") + else: + path_checks.append(_check_path_item({ + "name": "model_name_or_path", + "path": payload.get("model_name_or_path", ""), + "type": "any", + "required": True, + })) + if payload.get("dataset_path"): + path_checks.append(_check_path_item({ + "name": "dataset_path", + "path": payload.get("dataset_path", ""), + "type": "file", + "required": True, + })) + else: + errors.append("dataset_path is required for eval") + if shutil.which("python") is None: + errors.append("python runtime not found") + elif engine == "smoke": warnings.append("smoke engine skips model and dataset path checks") @@ -811,6 +835,22 @@ def create_app() -> FastAPI: "checksum_sha256": checksum, } + @app.get(f"{route_prefix}/compute/files/read") + async def read_file(path: str = Query(...)) -> JSONResponse: + """Read a text file from within YG_FT_DATA_ROOT. Used by the backend + to fetch eval results and other job outputs.""" + data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) + target = (data_root / path.lstrip("/\\")).resolve() + if not _path_inside(data_root, target): + raise HTTPException(status_code=400, detail="path must stay inside YG_FT_DATA_ROOT") + if not target.is_file(): + raise HTTPException(status_code=404, detail="file not found") + try: + content = target.read_text(encoding="utf-8") + return JSONResponse(json.loads(content) if content.strip().startswith("{") else {"content": content}) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) + @app.get(f"{route_prefix}/compute/files/{{file_id}}/download") async def download_file(file_id: str) -> FileResponse: upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads" diff --git a/compute/engines/llama_factory/adapter.py b/compute/engines/llama_factory/adapter.py index 3a7042e..4c0b7b0 100644 --- a/compute/engines/llama_factory/adapter.py +++ b/compute/engines/llama_factory/adapter.py @@ -204,6 +204,31 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA- command.extend(["--quantization_bit", str(quantization_bit)]) return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={}) + if engine == "eval": + output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'eval-job')}" + eval_config_path = str(Path(output_dir) / "eval_config.json") + eval_config = { + "model_name_or_path": config.get("model_name_or_path", ""), + "adapter_name_or_path": config.get("adapter_name_or_path", ""), + "template": config.get("template", "qwen"), + "dataset_path": config.get("dataset_path", ""), + "output_dir": output_dir, + "basic_metrics": config.get("basic_metrics", {}), + "dimension": config.get("dimension", {}), + "temperature": config.get("temperature", 0.1), + "top_p": config.get("top_p", 0.95), + "max_new_tokens": config.get("max_new_tokens", 512), + "infer_backend": config.get("infer_backend", "huggingface"), + "infer_dtype": config.get("infer_dtype", "auto"), + } + Path(output_dir).mkdir(parents=True, exist_ok=True) + Path(eval_config_path).write_text(json.dumps(eval_config, ensure_ascii=False, indent=2), encoding="utf-8") + return LlamaFactoryCommand( + command=["python", "-u", "-m", "compute.engines.llama_factory.eval_runner", "--config", eval_config_path], + work_dir="/app", + env={}, + ) + errors = validate_config(config) if errors: raise ValueError("; ".join(errors)) diff --git a/compute/engines/llama_factory/eval_runner.py b/compute/engines/llama_factory/eval_runner.py new file mode 100644 index 0000000..135fb20 --- /dev/null +++ b/compute/engines/llama_factory/eval_runner.py @@ -0,0 +1,428 @@ +""" +Evaluation runner — executes model evaluation as a subprocess job. + +Usage: + python -m compute.engines.llama_factory.eval_runner --config + +The config JSON is written by the compute API before spawning this subprocess. +Results are written to ``output_dir/eval_results.json`` and progress is printed +to stdout (captured as job logs). +""" + +from __future__ import annotations + +import json +import math +import sys +import time +from pathlib import Path +from typing import Any + + +def _load_jsonl(path: str) -> list[dict[str, Any]]: + """Load a JSONL dataset file. Each line must be a JSON object. + + Supports common field names used across the platform: + * ``instruction`` + ``input`` + ``output`` (Alpaca-style) + * ``question`` + ``answer`` + * ``messages`` (ShareGPT-style – the last assistant message is treated as reference) + """ + samples: list[dict[str, Any]] = [] + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + samples.append(obj) + return samples + + +def _sample_question(sample: dict[str, Any]) -> str: + """Extract the user-facing question / instruction from a sample.""" + if sample.get("instruction"): + text = sample["instruction"] + if sample.get("input"): + text += "\n" + sample["input"] + return text + if sample.get("question"): + return sample["question"] + # ShareGPT-style: use the last user message as question + messages = sample.get("messages") or [] + user_msgs = [m["content"] for m in messages if m.get("role") == "user"] + return user_msgs[-1] if user_msgs else "" + + +def _sample_reference(sample: dict[str, Any]) -> str: + """Extract the reference answer from a sample.""" + if sample.get("output"): + return sample["output"] + if sample.get("answer"): + return sample["answer"] + messages = sample.get("messages") or [] + assistant_msgs = [m["content"] for m in messages if m.get("role") == "assistant"] + return assistant_msgs[-1] if assistant_msgs else "" + + +# --------------------------------------------------------------------------- +# Basic metrics +# --------------------------------------------------------------------------- + +def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4) -> dict[str, Any]: + """Compute BLEU score via sacrebleu (corpus-level).""" + try: + from sacrebleu.metrics import BLEU + except ImportError: + return {"enabled": False, "error": "sacrebleu not installed", "score": 0} + bleu = BLEU(max_ngram_order=ngram) + # sacrebleu expects list-of-strings; we have one reference per prediction + score = bleu.corpus_score(predictions, [references]) + return { + "enabled": True, + "score": round(score.score, 2), + "bleu": round(score.score, 2), + } + + +def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]: + """Compute ROUGE scores via rouge-score.""" + try: + from rouge_score import rouge_scorer + except ImportError: + return {"enabled": False, "error": "rouge-score not installed", "score": 0} + methods = methods or ["rouge1", "rouge2", "rougeL"] + # Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL" + _rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"} + methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods] + scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True) + totals: dict[str, float] = {} + n = max(len(predictions), 1) + for ref, pred in zip(references, predictions): + result = scorer.score(ref, pred) + for key in methods: + totals[key] = totals.get(key, 0) + result[key].fmeasure + avg = {k: round(v / n, 4) for k, v in totals.items()} + return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg} + + +def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]: + """Compute average cosine similarity via sklearn.""" + try: + from sklearn.feature_extraction.text import TfidfVectorizer + from sklearn.metrics.pairwise import cosine_similarity + except ImportError: + return {"enabled": False, "error": "scikit-learn not installed", "score": 0} + if len(predictions) < 2: + return {"enabled": True, "score": 0, "error": "need at least 2 samples for corpus cosine"} + try: + vectorizer = TfidfVectorizer() + tfidf = vectorizer.fit_transform(references + predictions) + n = len(references) + ref_vec = tfidf[:n] + pred_vec = tfidf[n:] + sims = cosine_similarity(ref_vec, pred_vec).diagonal() + return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)} + except ValueError: + return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"} + + +# --------------------------------------------------------------------------- +# LLM Judge +# --------------------------------------------------------------------------- + +def _judge_sample( + question: str, + reference: str, + prediction: str, + config: dict[str, Any], +) -> dict[str, Any]: + """Call an OpenAI-compatible LLM to judge a single sample. + + Returns a dict with keys: + score, max_score, passed, judgement, evaluation_reason, error_type + """ + api_url = (config.get("api_url") or "").strip().rstrip("/") + api_key = (config.get("api_key") or "").strip() + eval_model = (config.get("eval_model") or "").strip() + eval_prompt = (config.get("eval_prompt") or "").strip() + score_min = float(config.get("score_min", 0)) + score_max = float(config.get("score_max", 5)) + pass_threshold = float(config.get("pass_threshold", 3)) + + if not api_url or not eval_model: + return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置", + "evaluation_reason": "未配置评测模型", "error_type": "其他"} + + system_msg = ( + eval_prompt + or "你是一个专业的评测专家。请根据参考答-案对被测模型的输出进行评分。" + ) + user_msg = ( + f"## 问题\n{question}\n\n" + f"## 参考答案\n{reference}\n\n" + f"## 模型输出\n{prediction}\n\n" + f"请给出 {score_min}-{score_max} 分的评分，并说明理由。" + ) + + try: + import urllib.request + import urllib.error + + body = json.dumps({ + "model": eval_model, + "messages": [ + {"role": "system", "content": system_msg}, + {"role": "user", "content": user_msg}, + ], + "temperature": 0.3, + "max_tokens": 512, + }).encode("utf-8") + + req = urllib.request.Request( + f"{api_url}/v1/chat/completions", + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + }, + ) + resp = urllib.request.urlopen(req, timeout=120) + data = json.loads(resp.read().decode("utf-8")) + reply = data["choices"][0]["message"]["content"] + except Exception as exc: + return {"score": 0, "max_score": score_max, "passed": False, + "judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}", + "error_type": "其他"} + + # Parse score from reply — look for patterns like "4分" or "Score: 4" + score = 0 + import re + score_patterns = [ + r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)', + r'(\d+(?:\.\d+)?)\s*分', + r'(\d+(?:\.\d+)?)\s*/\s*\d+', + ] + for pat in score_patterns: + m = re.search(pat, reply, re.IGNORECASE) + if m: + try: + score = float(m.group(1)) + except ValueError: + continue + break + score = max(score_min, min(score_max, score)) + passed = score >= pass_threshold + + # Determine judgement label + if score >= pass_threshold + 1: + judgement = "正确" + elif score >= pass_threshold: + judgement = "部分正确" + else: + judgement = "错误" + + # Guess error type from reply + reply_lower = reply.lower() + if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]): + error_type = "幻觉" + elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]): + error_type = "不完整" + elif any(w in reply_lower for w in ["格式", "format"]): + error_type = "格式偏差" + elif any(w in reply_lower for w in ["混淆", "confusion", "错误"]): + error_type = "混淆" + else: + error_type = "其他" + + return { + "score": score, + "max_score": score_max, + "passed": passed, + "judgement": judgement, + "evaluation_reason": reply[:2000], + "error_type": error_type, + } + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + +def run_eval(config: dict[str, Any]) -> dict[str, Any]: + """Execute a full evaluation run. Returns the result dict (also written to file).""" + model_path = config["model_name_or_path"] + adapter_path = config.get("adapter_name_or_path", "") + template = config.get("template", "qwen") + dataset_path = config["dataset_path"] + output_dir = Path(config["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + basic_cfg = config.get("basic_metrics", {}) + dimension_cfg = config.get("dimension", {}) or {} + output_precision = int(basic_cfg.get("output_precision", 2)) + + # ---- 1. Load dataset ---- + print(f"[eval] loading dataset: {dataset_path}") + raw_samples = _load_jsonl(dataset_path) + print(f"[eval] loaded {len(raw_samples)} samples") + + # ---- 2. Load model ---- + print(f"[eval] loading model: {model_path}") + from compute.engines.llama_factory.inference import InferenceSession + session = InferenceSession() + load_result = session.load( + model_name_or_path=model_path, + adapter_name_or_path=adapter_path, + template=template, + infer_backend=config.get("infer_backend", "huggingface"), + infer_dtype=config.get("infer_dtype", "auto"), + ) + if not load_result.get("loaded"): + raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}") + print(f"[eval] model loaded OK") + + # ---- 3. Run inference on each sample ---- + samples: list[dict[str, Any]] = [] + predictions: list[str] = [] + references: list[str] = [] + questions: list[str] = [] + + total = len(raw_samples) + judge_enabled = bool(dimension_cfg.get("eval_model") and dimension_cfg.get("api_url")) + print(f"[eval] starting inference on {total} samples, judge={'enabled' if judge_enabled else 'disabled'}") + + for idx, raw in enumerate(raw_samples, start=1): + question = _sample_question(raw) + reference = _sample_reference(raw) + if not question: + print(f"[eval] sample {idx}/{total}: skipped (no question)") + continue + + # Inference + chat_msgs = [{"role": "user", "content": question}] + result = session.chat( + chat_msgs, + temperature=float(config.get("temperature", 0.1)), + top_p=float(config.get("top_p", 0.95)), + max_new_tokens=int(config.get("max_new_tokens", 512)), + do_sample=False, + ) + prediction = result.get("response", "") if not result.get("error") else f"[ERROR] {result['error']}" + + predictions.append(prediction) + references.append(reference) + questions.append(question) + + # LLM Judge + judge_result: dict[str, Any] = {} + if judge_enabled: + judge_result = _judge_sample(question, reference, prediction, dimension_cfg) + + samples.append({ + "index": idx, + "input": question, + "reference_answer": reference, + "model_output": prediction, + "score": judge_result.get("score"), + "max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5)), + "passed": judge_result.get("passed"), + "judgement": judge_result.get("judgement"), + "evaluation_reason": judge_result.get("evaluation_reason", ""), + "error_type": judge_result.get("error_type"), + "dimension_scores": [ + {"name": "judge_score", "score": judge_result.get("score", 0), + "max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5))}, + ] if judge_result else [], + "status": "completed", + }) + + progress_pct = int(idx / max(total, 1) * 100) + print(f"[eval] sample {idx}/{total} ({progress_pct}%) done") + + # ---- 4. Compute basic metrics ---- + print(f"[eval] computing basic metrics on {len(predictions)} predictions") + metrics_result: dict[str, Any] = {} + + bleu_cfg = basic_cfg.get("bleu", {}) + if bleu_cfg.get("enabled"): + metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4))) + + rouge_cfg = basic_cfg.get("rouge", {}) + if rouge_cfg.get("enabled"): + metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods")) + + cosine_cfg = basic_cfg.get("cosine", {}) + if cosine_cfg.get("enabled"): + metrics_result["cosine"] = _compute_cosine(references, predictions) + + # ---- 5. Summarise ---- + completed = len(samples) + if judge_enabled: + scored = [s for s in samples if s.get("score") is not None] + passed_count = len([s for s in scored if s.get("passed")]) + avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision) + max_score = dimension_cfg.get("score_max", 5) + overall_score = round(avg_score / max_score * 100, output_precision) + overall_score_max = 100 + dimension_summary = [{ + "name": "综合评分", + "score": overall_score, + "max_score": 100, + "pass_rate": round(passed_count / max(completed, 1) * 100, 1), + }] + overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/{max_score} 分" + else: + passed_count = 0 + overall_score = 0 + overall_score_max = 100 + dimension_summary = [] + overall_evaluation = f"评测完成：{completed} 样本（未配置 LLM 评委）" + + result = { + "overall_score": overall_score, + "overall_score_max": overall_score_max, + "overall_evaluation": overall_evaluation, + "improvement_suggestions": [], + "dimension_summary": dimension_summary, + "samples": samples, + "sample_count": total, + "completed_count": completed, + "passed_count": passed_count, + "basic_metrics": metrics_result, + } + + # ---- 6. Write results ---- + result_path = output_dir / "eval_results.json" + result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"[eval] results written to {result_path}") + return result + + +def main() -> None: + import argparse + parser = argparse.ArgumentParser(description="YG-FT Evaluation Runner") + parser.add_argument("--config", required=True, help="Path to eval config JSON file") + args = parser.parse_args() + + config_path = Path(args.config) + if not config_path.exists(): + print(f"FATAL: config file not found: {args.config}", file=sys.stderr) + sys.exit(1) + + config = json.loads(config_path.read_text(encoding="utf-8")) + start = time.time() + try: + run_eval(config) + elapsed = time.time() - start + print(f"[eval] DONE in {elapsed:.1f}s") + except Exception as exc: + print(f"[eval] FAILED: {exc}", file=sys.stderr) + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/docs/模型评测功能总结.md b/docs/模型评测功能总结.md new file mode 100644 index 0000000..f6ed2d6 --- /dev/null +++ b/docs/模型评测功能总结.md @@ -0,0 +1,121 @@ +# 模型评测功能总结 + +本项目（基于 LLaMA-Factory 的微调训练平台）包含 **4 套相对独立** 的模型评测能力，分别面向不同的使用场景： + +| 能力 | 入口/目录 | 评测类型 | 打分方式 | +| --- | --- | --- | --- | +| 1. 学术 Benchmark 评测 | `llamafactory/eval/` | 选择题式基准（类 MMLU/C-Eval） | 选项匹配 + few-shot | +| 2. 评估工作台 | `backend/app/api/v1/eval/` | 生成式问答（指令跟随） | BLEU / ROUGE / ExactMatch + 可选 LLM 评审 | +| 3. 平台评估系统 | `backend/app/api/v1/evaluation/` | 基于评估数据集的问答 | 判卷模型（judge model）打分（0–5 分） | +| 4. 训练时验证评估 | `backend/app/services/task_runner.py` | 训练验证集 | loss 指标 | + +下面分别说明。 + +--- + +## 1. 学术 Benchmark 评测（LLaMA-Factory 原生） + +面向标准学术选择题基准（如 MMLU、C-Eval 等），复用 LLaMA-Factory 原生的评测框架。 + +**核心文件** +- `llamafactory/eval/evaluator.py`：`Evaluator` 类 + `run_eval()` 入口 +- `llamafactory/eval/template.py`：评测 prompt 模板（中/英，含 few-shot 示例构建） +- `llamafactory/hparams/evaluation_args.py`：`EvaluationArguments` 配置类 + +**工作流程** +1. 按 `task`（benchmark 名称）加载数据集，按科目（subject）拆分。 +2. 每个样本构造 few-shot 提示词（`n_shot` 控制示例数，由 `lang` 决定中/英模板），将题干与候选选项拼入 prompt。 +3. 调用模型推理得到预测，与标准答案比对，统计每个科目及整体的 `accuracy`。 +4. 结果写入 `save_dir`，打印各科目与平均准确率。 + +**关键参数（`EvaluationArguments`）** +- `task`：基准数据集名 +- `batch_size` / `n_shot` / `lang` / `save_dir` / `seed` +- `model_name_or_path`、`template`、`trust_remote_code` 等模型相关参数 + +> 该能力属于框架底层，本平台前端未直接提供操作入口，主要通过配置文件/脚本调用。 + +--- + +## 2. 评估工作台（生成式评测 + 指标计算） + +后端路由位于 `backend/app/api/v1/eval/__init__.py`，前端称为「评估工作台」。**适用于评测模型的指令跟随与生成质量**，并支持 LLM 作为裁判（LLM-as-a-Judge）。 + +**API 端点** +- `GET /evaluation/tasks`：列出评测任务（`frontend/src/api/evaluation.ts:listTasks`） +- `POST /evaluation/run`：提交一次评测（`runEval`） +- `GET /evaluation/report/{task_id}`：拉取评测报告（`getReport`） +- `DELETE /evaluation/tasks/{task_id}`：删除任务（`deleteTask`） + +**评测流程（`run_eval`）** +1. 通过 **LLaMA-Factory 数据管道**（`get_dataset`) 加载数据集，支持 `subset` 与抽样（`eval_sample`）。 +2. 用 **原生 transformers** 加载模型在本地做生成推理（单进程顺序生成，便于展示样本）。 +3. 计算客观指标（`compute_score`）： + - `BLEU`（sacrebleu） + - `ROUGE-1 / ROUGE-2 / ROUGE-L`（rouge-score） + - `Exact Match` +4. **可选 LLM 评审**（judge）：当配置了 `judge_model` / `judge_api_base` / `judge_api_key` 时，调用 OpenAI 兼容接口对每条样本打分（10 分制），并输出 4 个维度与理由： + - 核心事实正确性 `factual` + - 信息完整性 `completeness` + - 无幻觉 `no_hallucination` + - 格式合规性 `format` + - 综合分 `score` + `reason` +5. 任务状态持久化在后端 `eval_tasks.json`（支持 running/completed/failed/stopped），前端轮询进度。 + +**前端页面** +- `frontend/src/views/evaluation/EvaluateTask.vue`：任务列表、创建评测对话框（选模型、数据集、指标、可选 judge 配置） +- `frontend/src/views/evaluation/EvaluateReport.vue`：报告页，展示综合得分、BLEU、ROUGE-L、各维度指标及「参考答案 vs 模型预测 vs LLM 评审」对比样例 + +--- + +## 3. 平台评估系统（基于评估数据集 + 判卷模型） + +后端路由位于 `backend/app/api/v1/evaluation/__init__.py`，是平台业务层自研的评测体系。通过「评估数据集」组织题目，可一次性对 **多个被测模型 + 指定判卷模型** 进行批量评分。 + +**核心概念（数据模型 `backend/app/models/models.py`）** +- `EvalDataset`（`models.py:131`）：评估数据集，从项目问答对（`Question`/`Chunk`）中按 `question_type`（mixed/fact/reasoning）选题构建，状态 `pending/running/completed/failed`。 +- `EvalResult`（`models.py:147`）：单条评测结果，含 `judge_score`（0–5 分）、`is_correct`（true/false/partial）、`feedback`、`expected_answer` 等。 +- `Task`（`models.py:184`）：后台任务，`task_type="model-evaluation"`，记录进度与 `model_info`（存放平均分等汇总）。 + +**评测流程（`process_evaluation_task`，`backend/app/services/task_processor.py:336` 起）** +1. 加载评估数据集关联的题目，可选带入 `chunk` 上下文（RAG 场景）。 +2. 对每道题，先用 `build_eval_prompt` 组合「上下文 + 题目 + 参考答案」，调用 **判卷模型**（`call_model`，temperature=0.3）生成评分。 +3. `parse_eval_result` 解析出 `score`(0–5)、`is_correct`、`feedback`，写入 `EvalResult`。 +4. 逐题提交进度（`completed_count` / `progress`），支持中途 `stopped`。 +5. 汇总：`avg_score = 总分/有效数 × 20`（换算百分制），`avg_score_5 = 总分/有效数`（5 分制），存入 `task.model_info`。判定规则：得分 **≥3 视为正确**。 + +**特点** +- 判卷与被测模型解耦：被测模型给出答案，判卷模型（judge）独立评分，降低自评偏差。 +- 支持失败隔离：单题异常写入 `evaluation_status: failed` 记录而不中断整体任务。 + +--- + +## 4. 训练时验证评估 + +在微调训练任务执行期间，由 `backend/app/services/task_runner.py` 的 `do_eval` 触发： + +- 在训练过程中对验证集（validation set）计算 `eval_loss`，用于监控过拟合。 +- 结果回填到 `Task` 的 `loss_info` / `detail`，前端绘制 loss 曲线。 +- 属于训练配套的轻量评估，不参与上述 1–3 的业务评测。 + +--- + +## 附属：前端评测相关页面 + +| 文件 | 作用 | +| --- | --- | +| `frontend/src/views/evaluation/EvaluateTask.vue` | 评估工作台：任务列表 + 创建评测 | +| `frontend/src/views/evaluation/EvaluateReport.vue` | 评估报告：指标卡 + 维度标签 + 对比样例 | +| `frontend/src/api/evaluation.ts` | 评估工作台接口封装 | +| 平台评估系统入口 | 评估数据集管理 + 评估任务（model-evaluation）创建与结果查看 | + +--- + +## 小结 + +- **想要学术榜单式准确率** → 用能力 1（LLaMA-Factory `eval/`）。 +- **想要开放式生成质量（BLEU/ROUGE + LLM 评审）** → 用能力 2（评估工作台 `/evaluation/run`）。 +- **想要基于自有问答数据、用判卷模型批量打分** → 用能力 3（平台评估系统 `model-evaluation` 任务）。 +- **训练过程监控** → 能力 4（`do_eval` 验证集 loss）。 + +三种业务评测（1/2/3）相互独立，可并存于同一平台；数据模型（`EvalDataset`/`EvalResult`/`Task`）主要服务于能力 3，而能力 2 使用独立的 `eval_tasks.json` 文件持久化。 diff --git a/frontend/src/views/eval/DimensionCreateView.vue b/frontend/src/views/eval/DimensionCreateView.vue index 8de5b53..99ee24b 100644 --- a/frontend/src/views/eval/DimensionCreateView.vue +++ b/frontend/src/views/eval/DimensionCreateView.vue @@ -79,7 +79,9 @@ async function loadEditData() { async function loadModels() { try { const all = (await getModelList()) || [] - evalModels.value = all.filter((m) => m.purpose === 'evaluation') + evalModels.value = all.filter( + (m) => m.purpose === 'evaluation' || (m.model_source === 'api' && !!m.api_url), + ) } catch { evalModels.value = [] } diff --git a/frontend/src/views/eval/EvalCreateView.vue b/frontend/src/views/eval/EvalCreateView.vue index 7da326e..d8c1af6 100644 --- a/frontend/src/views/eval/EvalCreateView.vue +++ b/frontend/src/views/eval/EvalCreateView.vue @@ -11,6 +11,7 @@ import { createDimension, startEval } from '@/api/modules/eval' import { getTrainedModels, getModelList } from '@/api/modules/model' import { getDatasetList } from '@/api/modules/dataset' import { getSystemInfo } from '@/api/modules/system' +import { getComputeNodes, type ComputeNode } from '@/api/modules/compute' import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types' type StepExposed = { validate: () => Promise } @@ -84,15 +85,26 @@ async function loadData() { getDatasetList(), getSystemInfo(), getModelList(), + getComputeNodes(), ]) if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || [] if (results[1].status === 'fulfilled') { evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval') } - if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || [] + if (results[2].status === 'fulfilled') { + const allGpus: GpuInfo[] = results[2].value?.gpu || [] + const nodes: ComputeNode[] = (results[4].status === 'fulfilled' ? results[4].value : []) || [] + const onlineIds = new Set(nodes.filter((n) => n.enabled && n.scheduler_status === 'online').map((n) => n.id)) + // Only show idle GPUs from online compute nodes + gpus.value = allGpus.filter( + (g) => g.status === 'idle' && (!g.node_id || onlineIds.has(g.node_id)), + ) + } if (results[3].status === 'fulfilled') { - evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation') + evalModels.value = (results[3].value || []).filter( + (model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url), + ) } const failedCount = results.filter((result) => result.status === 'rejected').length diff --git a/frontend/src/views/eval/EvalView.vue b/frontend/src/views/eval/EvalView.vue index 9a7c492..f9c1409 100644 --- a/frontend/src/views/eval/EvalView.vue +++ b/frontend/src/views/eval/EvalView.vue @@ -1,9 +1,10 @@ diff --git a/frontend/src/views/eval/create/BasicMetricSetupStep.vue b/frontend/src/views/eval/create/BasicMetricSetupStep.vue index 962a96d..cea35cf 100644 --- a/frontend/src/views/eval/create/BasicMetricSetupStep.vue +++ b/frontend/src/views/eval/create/BasicMetricSetupStep.vue @@ -56,9 +56,9 @@ defineExpose({ validate }) - ROUGE-1 - ROUGE-2 - ROUGE-L + ROUGE-1 + ROUGE-2 + ROUGE-L diff --git a/frontend/src/views/eval/create/EvalTaskSetupStep.vue b/frontend/src/views/eval/create/EvalTaskSetupStep.vue index 718191c..94196d5 100644 --- a/frontend/src/views/eval/create/EvalTaskSetupStep.vue +++ b/frontend/src/views/eval/create/EvalTaskSetupStep.vue @@ -103,10 +103,10 @@ defineExpose({ validate })