YG_FT/backend/tests/test_data_process_evaluation.py

"""数据评测模块（三层质量评分）的单元测试。"""

from __future__ import annotations

import json
from typing import Any

import httpx
import pytest

from app.modules.data_process.algorithms.quality import (
    composite_overall,
    semantic_quality_scores,
)
from app.modules.data_process.evaluation import (
    _JUDGE_DIMENSIONS,
    _judge_system_prompt,
    _validated_judge_payload,
    evaluate_result_record,
    reevaluate_edited_record,
)
from app.modules.data_process.generation import ModelGenerationError

RECORD = {
    "instruction": "申请编号有什么作用？",
    "input": "",
    "output": "申请编号用于唯一标识一笔报销申请，便于跟踪审批状态。",
}
SOURCE = "报销系统中，申请编号用于唯一标识一笔报销申请，并支持跟踪审批状态。"


class _FakeEmbedModel:
    """按关键词返回固定向量，模拟语义嵌入。"""

    def get_text_embedding(self, text: str) -> list[float]:
        if "作用" in text or "编号" in text and "？" in text:
            return [0.9, 0.1, 0.0]
        if "申请编号" in text:
            return [0.85, 0.2, 0.0]
        return [0.0, 0.1, 0.9]


class _FailingEmbedModel:
    def get_text_embedding(self, text: str) -> list[float]:
        raise RuntimeError("embedding unavailable")


class _FakeResponse:
    def __init__(self, payload: dict[str, Any]):
        self._payload = payload

    def raise_for_status(self) -> None:
        return None

    def json(self) -> dict[str, Any]:
        return self._payload


class _FakeClient:
    def __init__(self, content: str):
        self._content = content
        self.calls: list[dict[str, Any]] = []

    def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
        self.calls.append({"endpoint": endpoint, "payload": json})
        return _FakeResponse({
            "choices": [{"message": {"content": self._content}, "finish_reason": "stop"}],
        })

    def close(self) -> None:
        return None


class _RaisingClient:
    def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
        raise httpx.ConnectError("model endpoint unreachable")

    def close(self) -> None:
        return None


def _judge_content(scores: dict[str, float], **extra: Any) -> str:
    return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra})


def test_judge_system_prompt_covers_rubric_dimensions() -> None:
    standard = _judge_system_prompt("standard")
    for name in _JUDGE_DIMENSIONS["standard"]:
        assert name in standard
    assert "1-5" in standard

    dpo = _judge_system_prompt("dpo")
    assert "chosen_quality" in dpo
    assert "preference_reasonableness" in dpo

    reasoning = _judge_system_prompt("reasoning")
    assert "reasoning_validity" in reasoning


def test_validated_judge_payload_converts_scores_to_overall() -> None:
    judged = _validated_judge_payload(
        {
            "scores": {
                "faithfulness": 5,
                "correctness": 4,
                "clarity": 4,
                "completeness": 3,
                "alignment": 4,
            },
            "reason": "答案可靠",
            "issues": ["回答略冗长"],
        },
        "standard",
    )

    assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2)
    assert judged["issues"] == ["回答略冗长"]
    assert judged["reason"] == "答案可靠"


def test_validated_judge_payload_clamps_out_of_range_scores() -> None:
    judged = _validated_judge_payload(
        {
            "scores": {
                "faithfulness": 9,
                "correctness": 4,
                "clarity": 4,
                "completeness": 0,
                "alignment": 4,
            },
        },
        "standard",
    )

    assert judged["scores"]["faithfulness"] == 5.0
    assert judged["scores"]["completeness"] == 1.0


@pytest.mark.parametrize(
    "scores",
    [
        {"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3},
        {
            "faithfulness": 5,
            "correctness": 4,
            "clarity": "high",
            "completeness": 3,
            "alignment": 4,
        },
    ],
)
def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None:
    with pytest.raises(ModelGenerationError):
        _validated_judge_payload({"scores": scores}, "standard")


def test_semantic_quality_scores_uses_cosine_similarity() -> None:
    scores = semantic_quality_scores(
        RECORD,
        source_content=SOURCE,
        embed_model=_FakeEmbedModel(),
    )

    assert scores is not None
    assert 0 < scores["question_answer"] <= 100
    assert 0 < scores["answer_source"] <= 100
    assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2)


def test_semantic_quality_scores_degrades_to_none_on_failure() -> None:
    assert (
        semantic_quality_scores(
            RECORD,
            source_content=SOURCE,
            embed_model=_FailingEmbedModel(),
        )
        is None
    )


def test_composite_overall_weights_available_layers() -> None:
    assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2)
    assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2)
    assert composite_overall(rule=80) == 80.0
    assert composite_overall(rule=None, judge=100) == 45.0


def test_evaluate_result_record_combines_three_layers() -> None:
    client = _FakeClient(
        _judge_content({
            "faithfulness": 5,
            "correctness": 4,
            "clarity": 5,
            "completeness": 4,
            "alignment": 5,
        })
    )
    quality = evaluate_result_record(
        RECORD,
        source_content=SOURCE,
        model={"api_url": "https://model.example", "online_model_name": "judge-model"},
        config={"output_type": "standard", "generation_retries": 0},
        client=client,
        embed_model=_FakeEmbedModel(),
    )

    assert quality["evaluated"] is True
    assert quality["judge"] is not None
    assert quality["judge"]["model"] == "judge-model"
    assert quality["semantic"] is not None
    assert quality["layers"]["judge"] == quality["judge"]["overall"]
    assert quality["overall"] == composite_overall(
        rule=quality["layers"]["rule"],
        semantic=quality["layers"]["semantic"],
        judge=quality["layers"]["judge"],
    )
    # 评审提示词必须携带来源原文作为评分锚点（正文经 NFKC 归一化）。
    user_message = client.calls[0]["payload"]["messages"][1]["content"]
    assert "申请编号用于唯一标识一笔报销" in user_message


def test_evaluate_result_record_degrades_when_model_fails() -> None:
    quality = evaluate_result_record(
        RECORD,
        source_content=SOURCE,
        model={"api_url": "https://model.example", "online_model_name": "judge-model"},
        config={"output_type": "standard", "generation_retries": 0},
        client=_RaisingClient(),
        embed_model=_FakeEmbedModel(),
    )

    assert quality["judge"] is None
    assert quality["layers"]["judge"] is None
    assert quality["semantic"] is not None
    assert quality["overall"] == composite_overall(
        rule=quality["layers"]["rule"],
        semantic=quality["layers"]["semantic"],
    )


def test_evaluate_result_record_without_model_runs_two_layers() -> None:
    quality = evaluate_result_record(
        RECORD,
        source_content=SOURCE,
        model=None,
        embed_model=_FakeEmbedModel(),
    )

    assert quality["judge"] is None
    assert quality["evaluated"] is True
    assert quality["overall"] == composite_overall(
        rule=quality["layers"]["rule"],
        semantic=quality["layers"]["semantic"],
    )


def test_reevaluate_edited_record_drops_stale_judge() -> None:
    previous = {
        "evaluated": True,
        "judge": {"overall": 90.0},
    }
    quality = reevaluate_edited_record(
        {**RECORD, "output": "编辑后的新答案内容，用于验证重评逻辑。"},
        source_content=SOURCE,
        previous_quality=previous,
        embed_model=_FakeEmbedModel(),
    )

    assert quality["evaluated"] is True
    assert quality["judge"] is None
    assert quality["layers"]["judge"] is None
    assert quality["semantic"] is not None


def test_reevaluate_edited_record_keeps_unevaluated_state() -> None:
    quality = reevaluate_edited_record(
        RECORD,
        source_content=SOURCE,
        previous_quality={},
        embed_model=_FakeEmbedModel(),
    )

    assert quality["evaluated"] is False
    assert quality["evaluated_at"] is None