feat(data-process): 支持单项生成五十条数据

This commit is contained in:
caoxiaozhu
2026-07-27 09:11:51 +08:00
parent 25d75f40c7
commit 9428c6b785
7 changed files with 400 additions and 86 deletions

View File

@@ -21,7 +21,6 @@ from app.modules.data_process.algorithms import (
detect_document_structure,
detect_pdf_document_noise,
detect_text_format,
estimate_token_count,
extract_pdf_page_texts,
extract_structured_records,
generate_standard_records,
@@ -748,3 +747,26 @@ def test_generate_standard_records_supports_json_qa_and_stable_variants() -> Non
split={"train": 100, "validation": 0, "test": 0},
split_seed="task-1",
)
def test_generate_standard_records_supports_fifty_unique_semantic_variants() -> None:
records = generate_standard_records(
[{"id": "preview-50", "edited_content": "问:如何操作?\n答:按步骤操作。"}],
qa_pairs_per_item=50,
semantic_enrichment=True,
split={"train": 100, "validation": 0, "test": 0},
split_seed="task-50",
)
assert len(records) == 50
assert len({record["id"] for record in records}) == 50
assert len({record["instruction"] for record in records}) == 50
assert all(record["status"] == "valid" for record in records)
@pytest.mark.parametrize("qa_pairs_per_item", [0, 51])
def test_generate_standard_records_rejects_out_of_range_count(
qa_pairs_per_item: int,
) -> None:
with pytest.raises(ValueError, match=r"\[1, 50\]"):
generate_standard_records([], qa_pairs_per_item=qa_pairs_per_item)