fix(data-process): 发布精确三路数据切分

This commit is contained in:
caoxiaozhu
2026-07-24 20:43:47 +08:00
parent e6a5a36bc0
commit 9cb77c251a
8 changed files with 405 additions and 181 deletions

View File

@@ -35,6 +35,7 @@ from app.modules.data_process.algorithms import (
remove_document_noise,
score_quality,
stable_split,
stable_split_assignments,
)
@@ -859,6 +860,17 @@ def test_stable_split_is_reproducible_and_validates_ratios() -> None:
stable_split("record", {"train": 80, "validation": 10, "test": 9})
def test_stable_split_assignments_use_exact_deterministic_quotas() -> None:
values = [f"record-{index}" for index in range(28)]
first = stable_split_assignments(values, seed="task-1")
second = stable_split_assignments(values, seed="task-1")
assert first == second
assert first.count("train") == 22
assert first.count("validation") == 3
assert first.count("test") == 3
def test_generate_standard_records_supports_json_qa_and_stable_variants() -> None:
previews = [
{