YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data_process): 问答对数据评测体系与质量分雷达图

Browse Source 
- 三层评测：规则层沿用原五维规则分，语义层用本地 BGE 向量算问答/来源
  相关性，评审层复用生成模型按 rubric 打分（忠实度/正确性/清晰度等，
  区分 standard/reasoning/dpo 输出类型），任一层失败自动降级
- 组合分 = 规则 35% + 语义 20% + 评审 45%，缺层自动重归一
- 新增 results/evaluate-batch 批量评测接口，镜像批量重生成的并发、
  乐观锁与部分成功语义；生成阶段不再展示质量分
- 详情页与结果编辑页新增"数据评测"按钮和批量进度；质量分列悬停弹出
  雷达图浮窗（评审 5 维 + 语义 2 维、三层分项、评审理由）
- 手动编辑/恢复后重算规则与语义层并丢弃过期评审分，雷达图不再展示
  失效数据
This commit is contained in:
caoxiaozhu
2026-08-19 14:21:54 +08:00
parent f47611020a
commit 81c2f85c3a
16 changed files with 1917 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
backend/tests/test_data_process_api.py
View File

@@ -1691,6 +1691,223 @@ def test_batch_result_regeneration_rejects_locked_tasks_before_model_call(
assert model_calls == 0
def _prepare_evaluation_task(
client: TestClient,
store: Any,
tmp_path: Path,
*,
config: dict[str, Any] | None = None,
) -> str:
task_id = client.post(
"/modelTF/data-process",
json={
"name": "数据评测",
"process_type": "structured",
"config": config or {"generation_model_id": "model-1", "output_type": "standard"},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
progress=100,
workflow_step="results",
results_confirmed=False,
)
store.models["model-1"] = {
"id": "model-1",
"online_model_name": "test-model",
"api_url": "https://model.example/v1",
"api_key": "secret",
}
store.previews[task_id] = [
{
"id": "preview-1",
"status": "original",
"original_content": "申请编号用于唯一标识一笔报销申请。",
"edited_content": "申请编号用于唯一标识一笔报销申请。",
},
{
"id": "preview-2",
"status": "original",
"original_content": "联系电话用于联系申请人。",
"edited_content": "联系电话用于联系申请人。",
},
]
store.results[task_id] = [
{
"id": "result-1",
"preview_item_id": "preview-1",
"instruction": "申请编号有什么作用？",
"input": "",
"output": "申请编号用于唯一标识一笔报销申请。",
"original_instruction": "申请编号有什么作用？",
"original_input": "",
"original_output": "申请编号用于唯一标识一笔报销申请。",
"status": "valid",
"error": None,
"split": "train",
"quality_score": {},
"updated_at": "2026-08-19T09:00:00Z",
},
{
"id": "result-2",
"preview_item_id": "preview-2",
"instruction": "联系电话有什么作用？",
"input": "",
"output": "联系电话用于联系申请人。",
"original_instruction": "联系电话有什么作用？",
"original_input": "",
"original_output": "联系电话用于联系申请人。",
"status": "valid",
"error": None,
"split": "train",
"quality_score": {},
"updated_at": "2026-08-19T09:00:01Z",
},
]
return task_id
def test_results_can_be_evaluated_in_batch_with_partial_success(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
evaluation_calls: list[dict[str, Any]] = []
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
evaluation_calls.append({"record": deepcopy(record), "kwargs": {k: v for k, v in kwargs.items() if k != "client"}})
return {
"overall": 88.0,
"completeness": 100.0,
"length": 100.0,
"readability": 100.0,
"relevance": 90.0,
"duplicate": 100.0,
"is_valid": True,
"flags": [],
"fingerprint": "fp",
"semantic": {"question_answer": 80.0, "answer_source": 90.0, "overall": 85.0},
"judge": {"scores": {"faithfulness": 5}, "overall": 90.0},
"layers": {"rule": 92.0, "semantic": 85.0, "judge": 90.0},
"evaluated": True,
}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={
"items": [
{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"},
# 乐观锁版本不匹配：该条应按冲突失败，另一条仍成功。
{"result_id": "result-2", "expected_updated_at": "2026-08-18T00:00:00Z"},
],
},
)
assert response.status_code == 200
data = response.json()["data"]
assert data["total"] == 2
assert data["succeeded"] == 1
assert data["failed"] == 1
assert [item["id"] for item in data["items"]] == ["result-1"]
assert data["failures"][0]["result_id"] == "result-2"
assert data["failures"][0]["code"] == "conflict"
assert len(evaluation_calls) == 1
assert evaluation_calls[0]["record"]["instruction"] == "申请编号有什么作用？"
assert evaluation_calls[0]["kwargs"]["model"]["online_model_name"] == "test-model"
assert evaluation_calls[0]["kwargs"]["source_content"] == "申请编号用于唯一标识一笔报销申请。"
stored = store.results[task_id][0]["quality_score"]
assert stored["evaluated"] is True
assert stored["layers"]["judge"] == 90.0
assert store.results[task_id][1]["quality_score"] == {}
def test_evaluation_without_generation_model_skips_judge_layer(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path, config={"output_type": "standard"})
seen_models: list[Any] = []
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
seen_models.append(kwargs.get("model"))
return {
"overall": 70.0, "is_valid": True, "flags": [],
"semantic": None, "judge": None,
"layers": {"rule": 70.0, "semantic": None, "judge": None},
"evaluated": True,
}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
)
assert response.status_code == 200
assert response.json()["data"]["succeeded"] == 1
# 任务未配置生成模型时，评审层收到的 model 必须是 None。
assert seen_models == [None]
def test_evaluation_rejects_running_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
store.tasks[task_id]["status"] = "running"
evaluation_calls = 0
def fake_evaluate(*args: Any, **kwargs: Any) -> dict[str, Any]:
nonlocal evaluation_calls
evaluation_calls += 1
return {"overall": 0, "is_valid": True, "flags": []}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
)
assert response.status_code == 409
assert evaluation_calls == 0
def test_result_update_preserves_evaluation_layers_and_drops_stale_judge(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
store.results[task_id][0]["quality_score"] = {
"overall": 90.0,
"is_valid": True,
"flags": [],
"semantic": {"overall": 85.0},
"judge": {"overall": 92.0},
"layers": {"rule": 90.0, "semantic": 85.0, "judge": 92.0},
"evaluated": True,
}
response = client.put(
f"/modelTF/data-process/{task_id}/results/result-1",
json={"output": "人工修正后的答案：申请编号唯一标识一笔报销申请。"},
)
assert response.status_code == 200
stored = store.results[task_id][0]["quality_score"]
# 手动编辑后：规则+语义重算，评审分丢弃，evaluated 标记保留。
assert stored["evaluated"] is True
assert stored["judge"] is None
assert stored["layers"]["judge"] is None
assert stored["layers"]["rule"] is not None
assert stored["overall"] >= 0
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
tmp_path: Path,
) -> None: