81c2f85c3a
- 三层评测：规则层沿用原五维规则分，语义层用本地 BGE 向量算问答/来源 相关性，评审层复用生成模型按 rubric 打分（忠实度/正确性/清晰度等， 区分 standard/reasoning/dpo 输出类型），任一层失败自动降级 - 组合分 = 规则 35% + 语义 20% + 评审 45%，缺层自动重归一 - 新增 results/evaluate-batch 批量评测接口，镜像批量重生成的并发、 乐观锁与部分成功语义；生成阶段不再展示质量分 - 详情页与结果编辑页新增"数据评测"按钮和批量进度；质量分列悬停弹出 雷达图浮窗（评审 5 维 + 语义 2 维、三层分项、评审理由） - 手动编辑/恢复后重算规则与语义层并丢弃过期评审分，雷达图不再展示 失效数据
285 lines
8.5 KiB
Python
285 lines
8.5 KiB
Python
"""数据评测模块（三层质量评分）的单元测试。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.modules.data_process.algorithms.quality import (
|
||
composite_overall,
|
||
semantic_quality_scores,
|
||
)
|
||
from app.modules.data_process.evaluation import (
|
||
_JUDGE_DIMENSIONS,
|
||
_judge_system_prompt,
|
||
_validated_judge_payload,
|
||
evaluate_result_record,
|
||
reevaluate_edited_record,
|
||
)
|
||
from app.modules.data_process.generation import ModelGenerationError
|
||
|
||
RECORD = {
|
||
"instruction": "申请编号有什么作用？",
|
||
"input": "",
|
||
"output": "申请编号用于唯一标识一笔报销申请，便于跟踪审批状态。",
|
||
}
|
||
SOURCE = "报销系统中，申请编号用于唯一标识一笔报销申请，并支持跟踪审批状态。"
|
||
|
||
|
||
class _FakeEmbedModel:
|
||
"""按关键词返回固定向量，模拟语义嵌入。"""
|
||
|
||
def get_text_embedding(self, text: str) -> list[float]:
|
||
if "作用" in text or "编号" in text and "？" in text:
|
||
return [0.9, 0.1, 0.0]
|
||
if "申请编号" in text:
|
||
return [0.85, 0.2, 0.0]
|
||
return [0.0, 0.1, 0.9]
|
||
|
||
|
||
class _FailingEmbedModel:
|
||
def get_text_embedding(self, text: str) -> list[float]:
|
||
raise RuntimeError("embedding unavailable")
|
||
|
||
|
||
class _FakeResponse:
|
||
def __init__(self, payload: dict[str, Any]):
|
||
self._payload = payload
|
||
|
||
def raise_for_status(self) -> None:
|
||
return None
|
||
|
||
def json(self) -> dict[str, Any]:
|
||
return self._payload
|
||
|
||
|
||
class _FakeClient:
|
||
def __init__(self, content: str):
|
||
self._content = content
|
||
self.calls: list[dict[str, Any]] = []
|
||
|
||
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
|
||
self.calls.append({"endpoint": endpoint, "payload": json})
|
||
return _FakeResponse({
|
||
"choices": [{"message": {"content": self._content}, "finish_reason": "stop"}],
|
||
})
|
||
|
||
def close(self) -> None:
|
||
return None
|
||
|
||
|
||
class _RaisingClient:
|
||
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
|
||
raise httpx.ConnectError("model endpoint unreachable")
|
||
|
||
def close(self) -> None:
|
||
return None
|
||
|
||
|
||
def _judge_content(scores: dict[str, float], **extra: Any) -> str:
|
||
return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra})
|
||
|
||
|
||
def test_judge_system_prompt_covers_rubric_dimensions() -> None:
|
||
standard = _judge_system_prompt("standard")
|
||
for name in _JUDGE_DIMENSIONS["standard"]:
|
||
assert name in standard
|
||
assert "1-5" in standard
|
||
|
||
dpo = _judge_system_prompt("dpo")
|
||
assert "chosen_quality" in dpo
|
||
assert "preference_reasonableness" in dpo
|
||
|
||
reasoning = _judge_system_prompt("reasoning")
|
||
assert "reasoning_validity" in reasoning
|
||
|
||
|
||
def test_validated_judge_payload_converts_scores_to_overall() -> None:
|
||
judged = _validated_judge_payload(
|
||
{
|
||
"scores": {
|
||
"faithfulness": 5,
|
||
"correctness": 4,
|
||
"clarity": 4,
|
||
"completeness": 3,
|
||
"alignment": 4,
|
||
},
|
||
"reason": "答案可靠",
|
||
"issues": ["回答略冗长"],
|
||
},
|
||
"standard",
|
||
)
|
||
|
||
assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2)
|
||
assert judged["issues"] == ["回答略冗长"]
|
||
assert judged["reason"] == "答案可靠"
|
||
|
||
|
||
def test_validated_judge_payload_clamps_out_of_range_scores() -> None:
|
||
judged = _validated_judge_payload(
|
||
{
|
||
"scores": {
|
||
"faithfulness": 9,
|
||
"correctness": 4,
|
||
"clarity": 4,
|
||
"completeness": 0,
|
||
"alignment": 4,
|
||
},
|
||
},
|
||
"standard",
|
||
)
|
||
|
||
assert judged["scores"]["faithfulness"] == 5.0
|
||
assert judged["scores"]["completeness"] == 1.0
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"scores",
|
||
[
|
||
{"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3},
|
||
{
|
||
"faithfulness": 5,
|
||
"correctness": 4,
|
||
"clarity": "high",
|
||
"completeness": 3,
|
||
"alignment": 4,
|
||
},
|
||
],
|
||
)
|
||
def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None:
|
||
with pytest.raises(ModelGenerationError):
|
||
_validated_judge_payload({"scores": scores}, "standard")
|
||
|
||
|
||
def test_semantic_quality_scores_uses_cosine_similarity() -> None:
|
||
scores = semantic_quality_scores(
|
||
RECORD,
|
||
source_content=SOURCE,
|
||
embed_model=_FakeEmbedModel(),
|
||
)
|
||
|
||
assert scores is not None
|
||
assert 0 < scores["question_answer"] <= 100
|
||
assert 0 < scores["answer_source"] <= 100
|
||
assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2)
|
||
|
||
|
||
def test_semantic_quality_scores_degrades_to_none_on_failure() -> None:
|
||
assert (
|
||
semantic_quality_scores(
|
||
RECORD,
|
||
source_content=SOURCE,
|
||
embed_model=_FailingEmbedModel(),
|
||
)
|
||
is None
|
||
)
|
||
|
||
|
||
def test_composite_overall_weights_available_layers() -> None:
|
||
assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2)
|
||
assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2)
|
||
assert composite_overall(rule=80) == 80.0
|
||
assert composite_overall(rule=None, judge=100) == 45.0
|
||
|
||
|
||
def test_evaluate_result_record_combines_three_layers() -> None:
|
||
client = _FakeClient(
|
||
_judge_content({
|
||
"faithfulness": 5,
|
||
"correctness": 4,
|
||
"clarity": 5,
|
||
"completeness": 4,
|
||
"alignment": 5,
|
||
})
|
||
)
|
||
quality = evaluate_result_record(
|
||
RECORD,
|
||
source_content=SOURCE,
|
||
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
|
||
config={"output_type": "standard", "generation_retries": 0},
|
||
client=client,
|
||
embed_model=_FakeEmbedModel(),
|
||
)
|
||
|
||
assert quality["evaluated"] is True
|
||
assert quality["judge"] is not None
|
||
assert quality["judge"]["model"] == "judge-model"
|
||
assert quality["semantic"] is not None
|
||
assert quality["layers"]["judge"] == quality["judge"]["overall"]
|
||
assert quality["overall"] == composite_overall(
|
||
rule=quality["layers"]["rule"],
|
||
semantic=quality["layers"]["semantic"],
|
||
judge=quality["layers"]["judge"],
|
||
)
|
||
# 评审提示词必须携带来源原文作为评分锚点（正文经 NFKC 归一化）。
|
||
user_message = client.calls[0]["payload"]["messages"][1]["content"]
|
||
assert "申请编号用于唯一标识一笔报销" in user_message
|
||
|
||
|
||
def test_evaluate_result_record_degrades_when_model_fails() -> None:
|
||
quality = evaluate_result_record(
|
||
RECORD,
|
||
source_content=SOURCE,
|
||
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
|
||
config={"output_type": "standard", "generation_retries": 0},
|
||
client=_RaisingClient(),
|
||
embed_model=_FakeEmbedModel(),
|
||
)
|
||
|
||
assert quality["judge"] is None
|
||
assert quality["layers"]["judge"] is None
|
||
assert quality["semantic"] is not None
|
||
assert quality["overall"] == composite_overall(
|
||
rule=quality["layers"]["rule"],
|
||
semantic=quality["layers"]["semantic"],
|
||
)
|
||
|
||
|
||
def test_evaluate_result_record_without_model_runs_two_layers() -> None:
|
||
quality = evaluate_result_record(
|
||
RECORD,
|
||
source_content=SOURCE,
|
||
model=None,
|
||
embed_model=_FakeEmbedModel(),
|
||
)
|
||
|
||
assert quality["judge"] is None
|
||
assert quality["evaluated"] is True
|
||
assert quality["overall"] == composite_overall(
|
||
rule=quality["layers"]["rule"],
|
||
semantic=quality["layers"]["semantic"],
|
||
)
|
||
|
||
|
||
def test_reevaluate_edited_record_drops_stale_judge() -> None:
|
||
previous = {
|
||
"evaluated": True,
|
||
"judge": {"overall": 90.0},
|
||
}
|
||
quality = reevaluate_edited_record(
|
||
{**RECORD, "output": "编辑后的新答案内容，用于验证重评逻辑。"},
|
||
source_content=SOURCE,
|
||
previous_quality=previous,
|
||
embed_model=_FakeEmbedModel(),
|
||
)
|
||
|
||
assert quality["evaluated"] is True
|
||
assert quality["judge"] is None
|
||
assert quality["layers"]["judge"] is None
|
||
assert quality["semantic"] is not None
|
||
|
||
|
||
def test_reevaluate_edited_record_keeps_unevaluated_state() -> None:
|
||
quality = reevaluate_edited_record(
|
||
RECORD,
|
||
source_content=SOURCE,
|
||
previous_quality={},
|
||
embed_model=_FakeEmbedModel(),
|
||
)
|
||
|
||
assert quality["evaluated"] is False
|
||
assert quality["evaluated_at"] is None
|