feat(data-process): 接入三种文档切分引擎

backend/tests/test_data_process_algorithms.py

@@ -15,7 +15,6 @@ from pypdf import PdfWriter
 
 from app.modules.data_process.algorithms import (
     PdfPageText,
-    chunk_unstructured,
     content_quality_flags,
     desensitize_pii,
     desensitize_structured_record,
@@ -662,162 +661,6 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
     )
 
 
-@pytest.mark.parametrize("method", ["structure", "fixed", "custom"])
-def test_chunk_methods_preserve_offsets_and_always_advance(method: str) -> None:
-    text = "# 第一章\n" + "甲。" * 18 + "\n# 第二章\n" + "乙。" * 18
-    kwargs = {"custom_delimiter": "\\n"} if method == "custom" else {}
-    chunks = chunk_unstructured(
-        text,
-        method=method,  # type: ignore[arg-type]
-        chunk_size=12,
-        chunk_overlap=2,
-        min_chunk_size=4,
-        **kwargs,
-    )
-    assert len(chunks) > 1
-    assert all(chunk.content == normalize_text(text)[chunk.start : chunk.end] for chunk in chunks)
-    assert all(chunk.end > chunk.start for chunk in chunks)
-    assert all(left.start < right.start for left, right in zip(chunks, chunks[1:]))
-    assert all(chunk.start_line <= chunk.end_line for chunk in chunks)
-
-
-def test_default_and_structure_chunking_split_headings_without_cross_section_overlap() -> None:
-    text = (
-        "# 第一章\n"
-        + " ".join(f"alpha{i}" for i in range(18))
-        + "\n# 第二章\n"
-        + " ".join(f"beta{i}" for i in range(18))
-    )
-    normalized = normalize_text(text)
-    second_chapter_start = normalized.index("# 第二章")
-    kwargs = {"chunk_size": 10, "chunk_overlap": 3, "min_chunk_size": 4}
-
-    default_chunks = chunk_unstructured(text, **kwargs)
-    structure_chunks = chunk_unstructured(text, method="structure", **kwargs)
-
-    assert default_chunks == structure_chunks
-    assert len(structure_chunks) > 2
-    assert all(
-        chunk.content == normalized[chunk.start : chunk.end] for chunk in structure_chunks
-    )
-    assert all(
-        not (chunk.start < second_chapter_start < chunk.end) for chunk in structure_chunks
-    )
-    second_chapter_chunks = [
-        chunk for chunk in structure_chunks if chunk.start >= second_chapter_start
-    ]
-    assert second_chapter_chunks[0].start == second_chapter_start
-    assert second_chapter_chunks[0].content.startswith("# 第二章")
-
-
-def test_fixed_chunk_offsets_and_actual_token_overlap_are_exact() -> None:
-    text = "  ".join(f"token{i}" for i in range(30))
-    normalized = normalize_text(text)
-    chunks = chunk_unstructured(
-        text,
-        method="fixed",
-        chunk_size=10,
-        chunk_overlap=3,
-        min_chunk_size=4,
-    )
-    assert len(chunks) > 2
-    assert all(chunk.content == normalized[chunk.start : chunk.end] for chunk in chunks)
-    assert all(chunk.token_count == estimate_token_count(chunk.content) for chunk in chunks)
-    assert all(chunk.token_count == 10 for chunk in chunks[:-1])
-    for left, right in zip(chunks, chunks[1:]):
-        overlap_text = normalized[right.start : left.end]
-        assert right.start < left.end
-        assert estimate_token_count(overlap_text) == 3
-        assert left.content.endswith(overlap_text)
-        assert right.content.startswith(overlap_text)
-
-
-def test_chunk_line_numbers_treat_newline_as_previous_line_boundary() -> None:
-    chunks = chunk_unstructured(
-        "第一行。\n第二行。\n第三行。",
-        method="custom",
-        chunk_size=8,
-        chunk_overlap=0,
-        min_chunk_size=2,
-        custom_delimiter="\\n",
-    )
-    assert chunks[0].content.endswith("\n")
-    assert chunks[0].start_line == 1
-    assert chunks[0].end_line == 1
-    assert chunks[1].start_line == 2
-
-
-def test_custom_delimiter_is_preserved_as_the_chunk_boundary() -> None:
-    custom_chunks = chunk_unstructured(
-        "a b c d <CUT> e f g h i j",
-        method="custom",
-        chunk_size=8,
-        chunk_overlap=0,
-        min_chunk_size=2,
-        custom_delimiter="<CUT>",
-    )
-    assert custom_chunks[0].content.endswith("<CUT>")
-
-
-@pytest.mark.parametrize(
-    ("field", "block"),
-    [
-        (
-            "preserve_code_blocks",
-            "```python\n" + "\n".join(f"value_{i} = {i}" for i in range(30)) + "\n```",
-        ),
-        (
-            "preserve_tables",
-            "| 字段 | 说明 |\n| --- | --- |\n"
-            + "\n".join(f"| field_{i} | value_{i} |" for i in range(30)),
-        ),
-        (
-            "preserve_lists",
-            "\n".join(f"- 第 {i} 项需要完整保留" for i in range(30)),
-        ),
-    ],
-)
-def test_markdown_protected_blocks_are_not_split(field: str, block: str) -> None:
-    text = "前言。" * 15 + "\n" + block + "\n" + "结尾。" * 40
-    unprotected = chunk_unstructured(
-        text,
-        method="fixed",
-        chunk_size=40,
-        chunk_overlap=0,
-        min_chunk_size=10,
-    )
-    chunks = chunk_unstructured(
-        text,
-        method="fixed",
-        chunk_size=40,
-        chunk_overlap=0,
-        min_chunk_size=10,
-        **{field: True},
-    )
-    assert all(block not in chunk.content for chunk in unprotected)
-    assert any(block in chunk.content for chunk in chunks)
-
-
-@pytest.mark.parametrize(
-    ("kwargs", "message"),
-    [
-        ({"chunk_size": 0}, "chunk_size"),
-        ({"chunk_size": 10, "chunk_overlap": 10}, "chunk_overlap"),
-        ({"chunk_size": 10, "chunk_overlap": 0, "min_chunk_size": 11}, "min_chunk_size"),
-        (
-            {"chunk_size": 10, "chunk_overlap": 5, "min_chunk_size": 6},
-            "cannot exceed",
-        ),
-        ({"method": "custom", "custom_delimiter": ""}, "custom_delimiter"),
-        ({"method": "semantic"}, "unsupported chunk method"),
-        ({"method": "heading"}, "unsupported chunk method"),
-    ],
-)
-def test_chunk_configuration_validation(kwargs: dict[str, object], message: str) -> None:
-    with pytest.raises(ValueError, match=message):
-        chunk_unstructured("some text", **kwargs)  # type: ignore[arg-type]
-
-
 def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
     valid = {
         "instruction": "如何修改收货地址？",