YG_FT/compute/tests/test_eval_runner.py

from __future__ import annotations

import json

from compute.engines.llama_factory.eval_runner import (
    _compute_exact_match,
    _compute_rouge,
    _compute_text_similarity,
    _normalise_api_url,
    _parse_judge_reply,
    _load_dataset,
)


def _write(tmp_path, name: str, text: str) -> str:
    path = tmp_path / name
    path.write_text(text, encoding="utf-8")
    return str(path)


def test_load_jsonl_multiline(tmp_path) -> None:
    path = _write(
        tmp_path,
        "eval.jsonl",
        '{"question": "q1", "answer": "a1"}\n{"question": "q2", "answer": "a2"}\n',
    )
    assert _load_dataset(path) == [
        {"question": "q1", "answer": "a1"},
        {"question": "q2", "answer": "a2"},
    ]


def test_load_json_array(tmp_path) -> None:
    path = _write(
        tmp_path,
        "eval.json",
        json.dumps([{"question": "x", "answer": "y"}]),
    )
    assert _load_dataset(path) == [{"question": "x", "answer": "y"}]


def test_load_jsonl_with_bom_and_embedded_array(tmp_path) -> None:
    """jsonl 带 BOM 且单行内嵌 JSON 数组，都应正常加载。"""
    path = _write(
        tmp_path,
        "eval.jsonl",
        "" + json.dumps([{"question": "a", "answer": "b"}, {"question": "c", "answer": "d"}]),
    )
    assert len(_load_dataset(path)) == 2


def test_deterministic_metrics_use_percent_scale() -> None:
    references = ["北京是中国的首都"]
    predictions = ["北京是中国的首都"]
    assert _compute_exact_match(references, predictions)["score"] == 100
    assert _compute_text_similarity(references, predictions)["score"] == 100


def test_rouge_supports_chinese_character_tokenization() -> None:
    import pytest
    pytest.importorskip("rouge_score")
    result = _compute_rouge(["北京是中国的首都"], ["北京是中国的首都"])
    assert result["available"] is True
    assert result["score"] == 100


def test_judge_reply_accepts_json_and_normalises_score() -> None:
    score, payload, reason = _parse_judge_reply(
        '{"score": 4, "dimensions": {"正确性": 4}, "reason": "内容正确"}',
        0,
        5,
    )
    assert score == 80
    assert payload["dimensions"]["正确性"] == 4
    assert reason == "内容正确"


def test_judge_reply_accepts_nlp_demo_dimension_format() -> None:
    score, _, _ = _parse_judge_reply(
        '{"语义一致性": 4, "信息完整性": 3, "事实准确性": 5, "语言流畅性": 4, "综合评价": "整体良好，0.8"}',
        0,
        5,
    )
    assert score == 80


def test_judge_reply_keeps_decimal_scores_in_configured_range() -> None:
    score, _, _ = _parse_judge_reply('{"score": 0.5}', 0, 5)
    assert score == 10


def test_openai_url_does_not_duplicate_v1() -> None:
    assert _normalise_api_url("https://example.test/v1") == "https://example.test/v1/chat/completions"
    assert _normalise_api_url("https://example.test") == "https://example.test/v1/chat/completions"