fix(data-process): 发布精确三路数据切分
This commit is contained in:
@@ -35,6 +35,7 @@ from app.modules.data_process.algorithms import (
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
stable_split,
|
||||
stable_split_assignments,
|
||||
)
|
||||
|
||||
|
||||
@@ -859,6 +860,17 @@ def test_stable_split_is_reproducible_and_validates_ratios() -> None:
|
||||
stable_split("record", {"train": 80, "validation": 10, "test": 9})
|
||||
|
||||
|
||||
def test_stable_split_assignments_use_exact_deterministic_quotas() -> None:
|
||||
values = [f"record-{index}" for index in range(28)]
|
||||
first = stable_split_assignments(values, seed="task-1")
|
||||
second = stable_split_assignments(values, seed="task-1")
|
||||
|
||||
assert first == second
|
||||
assert first.count("train") == 22
|
||||
assert first.count("validation") == 3
|
||||
assert first.count("test") == 3
|
||||
|
||||
|
||||
def test_generate_standard_records_supports_json_qa_and_stable_variants() -> None:
|
||||
previews = [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user