feat(data_process): 问答对数据评测体系与质量分雷达图
- 三层评测:规则层沿用原五维规则分,语义层用本地 BGE 向量算问答/来源 相关性,评审层复用生成模型按 rubric 打分(忠实度/正确性/清晰度等, 区分 standard/reasoning/dpo 输出类型),任一层失败自动降级 - 组合分 = 规则 35% + 语义 20% + 评审 45%,缺层自动重归一 - 新增 results/evaluate-batch 批量评测接口,镜像批量重生成的并发、 乐观锁与部分成功语义;生成阶段不再展示质量分 - 详情页与结果编辑页新增"数据评测"按钮和批量进度;质量分列悬停弹出 雷达图浮窗(评审 5 维 + 语义 2 维、三层分项、评审理由) - 手动编辑/恢复后重算规则与语义层并丢弃过期评审分,雷达图不再展示 失效数据
This commit is contained in:
284
backend/tests/test_data_process_evaluation.py
Normal file
284
backend/tests/test_data_process_evaluation.py
Normal file
@@ -0,0 +1,284 @@
|
||||
"""数据评测模块(三层质量评分)的单元测试。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.modules.data_process.algorithms.quality import (
|
||||
composite_overall,
|
||||
semantic_quality_scores,
|
||||
)
|
||||
from app.modules.data_process.evaluation import (
|
||||
_JUDGE_DIMENSIONS,
|
||||
_judge_system_prompt,
|
||||
_validated_judge_payload,
|
||||
evaluate_result_record,
|
||||
reevaluate_edited_record,
|
||||
)
|
||||
from app.modules.data_process.generation import ModelGenerationError
|
||||
|
||||
RECORD = {
|
||||
"instruction": "申请编号有什么作用?",
|
||||
"input": "",
|
||||
"output": "申请编号用于唯一标识一笔报销申请,便于跟踪审批状态。",
|
||||
}
|
||||
SOURCE = "报销系统中,申请编号用于唯一标识一笔报销申请,并支持跟踪审批状态。"
|
||||
|
||||
|
||||
class _FakeEmbedModel:
|
||||
"""按关键词返回固定向量,模拟语义嵌入。"""
|
||||
|
||||
def get_text_embedding(self, text: str) -> list[float]:
|
||||
if "作用" in text or "编号" in text and "?" in text:
|
||||
return [0.9, 0.1, 0.0]
|
||||
if "申请编号" in text:
|
||||
return [0.85, 0.2, 0.0]
|
||||
return [0.0, 0.1, 0.9]
|
||||
|
||||
|
||||
class _FailingEmbedModel:
|
||||
def get_text_embedding(self, text: str) -> list[float]:
|
||||
raise RuntimeError("embedding unavailable")
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict[str, Any]):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, Any]:
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, content: str):
|
||||
self._content = content
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
|
||||
self.calls.append({"endpoint": endpoint, "payload": json})
|
||||
return _FakeResponse({
|
||||
"choices": [{"message": {"content": self._content}, "finish_reason": "stop"}],
|
||||
})
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _RaisingClient:
|
||||
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
|
||||
raise httpx.ConnectError("model endpoint unreachable")
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _judge_content(scores: dict[str, float], **extra: Any) -> str:
|
||||
return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra})
|
||||
|
||||
|
||||
def test_judge_system_prompt_covers_rubric_dimensions() -> None:
|
||||
standard = _judge_system_prompt("standard")
|
||||
for name in _JUDGE_DIMENSIONS["standard"]:
|
||||
assert name in standard
|
||||
assert "1-5" in standard
|
||||
|
||||
dpo = _judge_system_prompt("dpo")
|
||||
assert "chosen_quality" in dpo
|
||||
assert "preference_reasonableness" in dpo
|
||||
|
||||
reasoning = _judge_system_prompt("reasoning")
|
||||
assert "reasoning_validity" in reasoning
|
||||
|
||||
|
||||
def test_validated_judge_payload_converts_scores_to_overall() -> None:
|
||||
judged = _validated_judge_payload(
|
||||
{
|
||||
"scores": {
|
||||
"faithfulness": 5,
|
||||
"correctness": 4,
|
||||
"clarity": 4,
|
||||
"completeness": 3,
|
||||
"alignment": 4,
|
||||
},
|
||||
"reason": "答案可靠",
|
||||
"issues": ["回答略冗长"],
|
||||
},
|
||||
"standard",
|
||||
)
|
||||
|
||||
assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2)
|
||||
assert judged["issues"] == ["回答略冗长"]
|
||||
assert judged["reason"] == "答案可靠"
|
||||
|
||||
|
||||
def test_validated_judge_payload_clamps_out_of_range_scores() -> None:
|
||||
judged = _validated_judge_payload(
|
||||
{
|
||||
"scores": {
|
||||
"faithfulness": 9,
|
||||
"correctness": 4,
|
||||
"clarity": 4,
|
||||
"completeness": 0,
|
||||
"alignment": 4,
|
||||
},
|
||||
},
|
||||
"standard",
|
||||
)
|
||||
|
||||
assert judged["scores"]["faithfulness"] == 5.0
|
||||
assert judged["scores"]["completeness"] == 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scores",
|
||||
[
|
||||
{"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3},
|
||||
{
|
||||
"faithfulness": 5,
|
||||
"correctness": 4,
|
||||
"clarity": "high",
|
||||
"completeness": 3,
|
||||
"alignment": 4,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None:
|
||||
with pytest.raises(ModelGenerationError):
|
||||
_validated_judge_payload({"scores": scores}, "standard")
|
||||
|
||||
|
||||
def test_semantic_quality_scores_uses_cosine_similarity() -> None:
|
||||
scores = semantic_quality_scores(
|
||||
RECORD,
|
||||
source_content=SOURCE,
|
||||
embed_model=_FakeEmbedModel(),
|
||||
)
|
||||
|
||||
assert scores is not None
|
||||
assert 0 < scores["question_answer"] <= 100
|
||||
assert 0 < scores["answer_source"] <= 100
|
||||
assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2)
|
||||
|
||||
|
||||
def test_semantic_quality_scores_degrades_to_none_on_failure() -> None:
|
||||
assert (
|
||||
semantic_quality_scores(
|
||||
RECORD,
|
||||
source_content=SOURCE,
|
||||
embed_model=_FailingEmbedModel(),
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_composite_overall_weights_available_layers() -> None:
|
||||
assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2)
|
||||
assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2)
|
||||
assert composite_overall(rule=80) == 80.0
|
||||
assert composite_overall(rule=None, judge=100) == 45.0
|
||||
|
||||
|
||||
def test_evaluate_result_record_combines_three_layers() -> None:
|
||||
client = _FakeClient(
|
||||
_judge_content({
|
||||
"faithfulness": 5,
|
||||
"correctness": 4,
|
||||
"clarity": 5,
|
||||
"completeness": 4,
|
||||
"alignment": 5,
|
||||
})
|
||||
)
|
||||
quality = evaluate_result_record(
|
||||
RECORD,
|
||||
source_content=SOURCE,
|
||||
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
|
||||
config={"output_type": "standard", "generation_retries": 0},
|
||||
client=client,
|
||||
embed_model=_FakeEmbedModel(),
|
||||
)
|
||||
|
||||
assert quality["evaluated"] is True
|
||||
assert quality["judge"] is not None
|
||||
assert quality["judge"]["model"] == "judge-model"
|
||||
assert quality["semantic"] is not None
|
||||
assert quality["layers"]["judge"] == quality["judge"]["overall"]
|
||||
assert quality["overall"] == composite_overall(
|
||||
rule=quality["layers"]["rule"],
|
||||
semantic=quality["layers"]["semantic"],
|
||||
judge=quality["layers"]["judge"],
|
||||
)
|
||||
# 评审提示词必须携带来源原文作为评分锚点(正文经 NFKC 归一化)。
|
||||
user_message = client.calls[0]["payload"]["messages"][1]["content"]
|
||||
assert "申请编号用于唯一标识一笔报销" in user_message
|
||||
|
||||
|
||||
def test_evaluate_result_record_degrades_when_model_fails() -> None:
|
||||
quality = evaluate_result_record(
|
||||
RECORD,
|
||||
source_content=SOURCE,
|
||||
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
|
||||
config={"output_type": "standard", "generation_retries": 0},
|
||||
client=_RaisingClient(),
|
||||
embed_model=_FakeEmbedModel(),
|
||||
)
|
||||
|
||||
assert quality["judge"] is None
|
||||
assert quality["layers"]["judge"] is None
|
||||
assert quality["semantic"] is not None
|
||||
assert quality["overall"] == composite_overall(
|
||||
rule=quality["layers"]["rule"],
|
||||
semantic=quality["layers"]["semantic"],
|
||||
)
|
||||
|
||||
|
||||
def test_evaluate_result_record_without_model_runs_two_layers() -> None:
|
||||
quality = evaluate_result_record(
|
||||
RECORD,
|
||||
source_content=SOURCE,
|
||||
model=None,
|
||||
embed_model=_FakeEmbedModel(),
|
||||
)
|
||||
|
||||
assert quality["judge"] is None
|
||||
assert quality["evaluated"] is True
|
||||
assert quality["overall"] == composite_overall(
|
||||
rule=quality["layers"]["rule"],
|
||||
semantic=quality["layers"]["semantic"],
|
||||
)
|
||||
|
||||
|
||||
def test_reevaluate_edited_record_drops_stale_judge() -> None:
|
||||
previous = {
|
||||
"evaluated": True,
|
||||
"judge": {"overall": 90.0},
|
||||
}
|
||||
quality = reevaluate_edited_record(
|
||||
{**RECORD, "output": "编辑后的新答案内容,用于验证重评逻辑。"},
|
||||
source_content=SOURCE,
|
||||
previous_quality=previous,
|
||||
embed_model=_FakeEmbedModel(),
|
||||
)
|
||||
|
||||
assert quality["evaluated"] is True
|
||||
assert quality["judge"] is None
|
||||
assert quality["layers"]["judge"] is None
|
||||
assert quality["semantic"] is not None
|
||||
|
||||
|
||||
def test_reevaluate_edited_record_keeps_unevaluated_state() -> None:
|
||||
quality = reevaluate_edited_record(
|
||||
RECORD,
|
||||
source_content=SOURCE,
|
||||
previous_quality={},
|
||||
embed_model=_FakeEmbedModel(),
|
||||
)
|
||||
|
||||
assert quality["evaluated"] is False
|
||||
assert quality["evaluated_at"] is None
|
||||
Reference in New Issue
Block a user