feat(data-process): 支持单项生成五十条数据
This commit is contained in:
@@ -21,7 +21,6 @@ from app.modules.data_process.algorithms import (
|
||||
detect_document_structure,
|
||||
detect_pdf_document_noise,
|
||||
detect_text_format,
|
||||
estimate_token_count,
|
||||
extract_pdf_page_texts,
|
||||
extract_structured_records,
|
||||
generate_standard_records,
|
||||
@@ -748,3 +747,26 @@ def test_generate_standard_records_supports_json_qa_and_stable_variants() -> Non
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
split_seed="task-1",
|
||||
)
|
||||
|
||||
|
||||
def test_generate_standard_records_supports_fifty_unique_semantic_variants() -> None:
|
||||
records = generate_standard_records(
|
||||
[{"id": "preview-50", "edited_content": "问:如何操作?\n答:按步骤操作。"}],
|
||||
qa_pairs_per_item=50,
|
||||
semantic_enrichment=True,
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
split_seed="task-50",
|
||||
)
|
||||
|
||||
assert len(records) == 50
|
||||
assert len({record["id"] for record in records}) == 50
|
||||
assert len({record["instruction"] for record in records}) == 50
|
||||
assert all(record["status"] == "valid" for record in records)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qa_pairs_per_item", [0, 51])
|
||||
def test_generate_standard_records_rejects_out_of_range_count(
|
||||
qa_pairs_per_item: int,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match=r"\[1, 50\]"):
|
||||
generate_standard_records([], qa_pairs_per_item=qa_pairs_per_item)
|
||||
|
||||
Reference in New Issue
Block a user