feat(data-process): 清理PDF文档级噪声

backend/tests/test_data_process_api.py

@@ -12,7 +12,7 @@ from openpyxl import Workbook
 
 from app.api.v1.endpoints import data_process as data_process_endpoint
 from app.api.v1.endpoints.data_process import router
-from app.modules.data_process.algorithms import normalize_text
+from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text
 from app.modules.data_process.storage import (
     DataProcessStorageError,
     LocalDataProcessStorage,
@@ -443,20 +443,49 @@ def _stored_files(storage: LocalDataProcessStorage) -> list[Path]:
     return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()]
 
 
-def _minimal_pdf(text: str = "Hello PDF") -> bytes:
-    stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii")
+def _minimal_pdf_pages(*page_texts: str) -> bytes:
+    if not page_texts:
+        raise ValueError("at least one PDF page is required")
+    font_object_number = 3 + len(page_texts) * 2
+    page_object_numbers = [3 + index * 2 for index in range(len(page_texts))]
     objects = [
         b"<< /Type /Catalog /Pages 2 0 R >>",
-        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
         (
-            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
-            b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>"
+            b"<< /Type /Pages /Kids ["
+            + b" ".join(f"{number} 0 R".encode() for number in page_object_numbers)
+            + b"] /Count "
+            + str(len(page_texts)).encode()
+            + b" >>"
         ),
-        b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n"
-        + stream
-        + b"\nendstream",
-        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
     ]
+    for index, text in enumerate(page_texts):
+        content_object_number = page_object_numbers[index] + 1
+        commands = [b"BT /F1 12 Tf 72 720 Td"]
+        for line_index, line in enumerate(text.splitlines()):
+            escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
+            if line_index:
+                commands.append(b"0 -16 Td")
+            commands.append(f"({escaped}) Tj".encode("ascii"))
+        commands.append(b"ET")
+        stream = b" ".join(commands)
+        objects.extend(
+            [
+                (
+                    b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+                    b"/Resources << /Font << /F1 "
+                    + str(font_object_number).encode()
+                    + b" 0 R >> >> /Contents "
+                    + str(content_object_number).encode()
+                    + b" 0 R >>"
+                ),
+                b"<< /Length "
+                + str(len(stream)).encode()
+                + b" >>\nstream\n"
+                + stream
+                + b"\nendstream",
+            ]
+        )
+    objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
     result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
     offsets = [0]
     for object_number, value in enumerate(objects, start=1):
@@ -478,6 +507,10 @@ def _minimal_pdf(text: str = "Hello PDF") -> bytes:
     return bytes(result)
 
 
+def _minimal_pdf(text: str = "Hello PDF") -> bytes:
+    return _minimal_pdf_pages(text)
+
+
 def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
     client, store, _ = make_client(tmp_path)
     created = client.post(
@@ -1127,6 +1160,59 @@ def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Pat
     assert legacy_pages.status_code == 410
 
 
+def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift(
+    tmp_path: Path,
+) -> None:
+    client, _, _ = make_client(tmp_path)
+    task_id = client.post(
+        "/modelTF/data-process",
+        json={
+            "name": "PDF 文档噪声清理",
+            "process_type": "unstructured",
+            "config": {
+                "chunk_method": "fixed",
+                "chunk_size": 200,
+                "chunk_overlap": 0,
+                "min_chunk_size": 20,
+                "preprocess_options": ["clean_invalid_content"],
+            },
+        },
+    ).json()["data"]["id"]
+    raw = _minimal_pdf_pages(
+        "ACME Internal Manual\nBody page one keeps this guidance and explanation.",
+        "ACME Internal Manual\nContents\n"
+        "Chapter One........3\nChapter Two........4\nAppendix........5",
+        "ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8",
+        "ACME Internal Manual\nBody page four keeps operational details and examples.",
+        "ACME Internal Manual\nBody page five keeps the final effective-date clause.",
+    )
+    uploaded = client.post(
+        f"/modelTF/data-process/{task_id}/source-files",
+        files={"files": ("manual.pdf", raw, "application/pdf")},
+    ).json()["data"]["files"][0]
+    source_content = client.get(
+        f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content"
+    ).json()["data"]["content"]
+
+    built = client.post(f"/modelTF/data-process/{task_id}/preview/build")
+
+    assert built.status_code == 200
+    items = built.json()["data"]["items"]
+    assert items
+    edited = "\n".join(item["edited_content"] for item in items)
+    assert "ACME Internal Manual" not in edited
+    assert "Contents" not in edited
+    assert "Chapter One" not in edited
+    assert "1.2 Approval" not in edited
+    assert "Body page one" in edited
+    assert "Body page five" in edited
+    assert all(
+        item["original_content"]
+        == source_content[item["source_start"] : item["source_end"]]
+        for item in items
+    )
+
+
 def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None:
     client, _, _ = make_client(tmp_path)
     task_id = client.post(
@@ -1450,6 +1536,51 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None
     assert "[EMAIL]" in masked["edited_content"]
 
 
+def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None:
+    source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。")
+    source = {
+        "id": "pdf-source",
+        "name": "manual.pdf",
+        "file_format": "pdf",
+        "content": source_text,
+        "document_noise_spans": (
+            DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"),
+        ),
+    }
+    config = {
+        "chunk_method": "fixed",
+        "chunk_size": 200,
+        "chunk_overlap": 0,
+        "min_chunk_size": 1,
+    }
+
+    cleaned_items = data_process_endpoint._build_preview_items(
+        {
+            "process_type": "unstructured",
+            "config": {**config, "preprocess_options": ["clean_invalid_content"]},
+        },
+        [source],
+    )
+    original_items = data_process_endpoint._build_preview_items(
+        {
+            "process_type": "unstructured",
+            "config": {**config, "preprocess_options": []},
+        },
+        [source],
+    )
+
+    assert len(cleaned_items) == 1
+    cleaned = cleaned_items[0]
+    assert cleaned["original_content"] == source_text[
+        cleaned["source_start"] : cleaned["source_end"]
+    ]
+    assert "重复页眉" not in cleaned["edited_content"]
+    assert "PDF 正文内容" in cleaned["edited_content"]
+    assert cleaned["status"] == "modified"
+    assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"]
+    assert "重复页眉" in original_items[0]["edited_content"]
+
+
 def test_merge_short_content_applies_across_adjacent_structure_sections() -> None:
     content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
     items = _preview_task(