6f0e82f351
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请 - 存储: MinIO 存储进度迁移、对象存储安全加固与测试 - 计算: GPU 资源预留、compute 轮询与同步增强 - 权限: permission v2 迁移、权限安全验收测试 - 日志: 后端运行日志中文说明、操作日志整合 - 数据处理/评测: 数据转换与模型评测优化 Co-Authored-By: Claude <noreply@anthropic.com>
95 lines
2.9 KiB
Python
95 lines
2.9 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
from compute.engines.llama_factory.eval_runner import (
|
||
_compute_exact_match,
|
||
_compute_rouge,
|
||
_compute_text_similarity,
|
||
_normalise_api_url,
|
||
_parse_judge_reply,
|
||
_load_dataset,
|
||
)
|
||
|
||
|
||
def _write(tmp_path, name: str, text: str) -> str:
|
||
path = tmp_path / name
|
||
path.write_text(text, encoding="utf-8")
|
||
return str(path)
|
||
|
||
|
||
def test_load_jsonl_multiline(tmp_path) -> None:
|
||
path = _write(
|
||
tmp_path,
|
||
"eval.jsonl",
|
||
'{"question": "q1", "answer": "a1"}\n{"question": "q2", "answer": "a2"}\n',
|
||
)
|
||
assert _load_dataset(path) == [
|
||
{"question": "q1", "answer": "a1"},
|
||
{"question": "q2", "answer": "a2"},
|
||
]
|
||
|
||
|
||
def test_load_json_array(tmp_path) -> None:
|
||
path = _write(
|
||
tmp_path,
|
||
"eval.json",
|
||
json.dumps([{"question": "x", "answer": "y"}]),
|
||
)
|
||
assert _load_dataset(path) == [{"question": "x", "answer": "y"}]
|
||
|
||
|
||
def test_load_jsonl_with_bom_and_embedded_array(tmp_path) -> None:
|
||
"""jsonl 带 BOM 且单行内嵌 JSON 数组，都应正常加载。"""
|
||
path = _write(
|
||
tmp_path,
|
||
"eval.jsonl",
|
||
"" + json.dumps([{"question": "a", "answer": "b"}, {"question": "c", "answer": "d"}]),
|
||
)
|
||
assert len(_load_dataset(path)) == 2
|
||
|
||
|
||
def test_deterministic_metrics_use_percent_scale() -> None:
|
||
references = ["北京是中国的首都"]
|
||
predictions = ["北京是中国的首都"]
|
||
assert _compute_exact_match(references, predictions)["score"] == 100
|
||
assert _compute_text_similarity(references, predictions)["score"] == 100
|
||
|
||
|
||
def test_rouge_supports_chinese_character_tokenization() -> None:
|
||
import pytest
|
||
pytest.importorskip("rouge_score")
|
||
result = _compute_rouge(["北京是中国的首都"], ["北京是中国的首都"])
|
||
assert result["available"] is True
|
||
assert result["score"] == 100
|
||
|
||
|
||
def test_judge_reply_accepts_json_and_normalises_score() -> None:
|
||
score, payload, reason = _parse_judge_reply(
|
||
'{"score": 4, "dimensions": {"正确性": 4}, "reason": "内容正确"}',
|
||
0,
|
||
5,
|
||
)
|
||
assert score == 80
|
||
assert payload["dimensions"]["正确性"] == 4
|
||
assert reason == "内容正确"
|
||
|
||
|
||
def test_judge_reply_accepts_nlp_demo_dimension_format() -> None:
|
||
score, _, _ = _parse_judge_reply(
|
||
'{"语义一致性": 4, "信息完整性": 3, "事实准确性": 5, "语言流畅性": 4, "综合评价": "整体良好，0.8"}',
|
||
0,
|
||
5,
|
||
)
|
||
assert score == 80
|
||
|
||
|
||
def test_judge_reply_keeps_decimal_scores_in_configured_range() -> None:
|
||
score, _, _ = _parse_judge_reply('{"score": 0.5}', 0, 5)
|
||
assert score == 10
|
||
|
||
|
||
def test_openai_url_does_not_duplicate_v1() -> None:
|
||
assert _normalise_api_url("https://example.test/v1") == "https://example.test/v1/chat/completions"
|
||
assert _normalise_api_url("https://example.test") == "https://example.test/v1/chat/completions"
|