feat(data-process): 清理PDF文档级噪声

backend/app/api/v1/endpoints/data_process.py

@@ -37,6 +37,7 @@ from app.modules.data_process.algorithms import (
     desensitize_pii,
     desensitize_structured_record,
     detect_document_structure,
+    detect_pdf_document_noise,
     estimate_token_count,
     extract_pdf_page_texts,
     generate_standard_records,
@@ -44,6 +45,7 @@ from app.modules.data_process.algorithms import (
     near_duplicate_fingerprint,
     parse_text_content,
     preprocess_structured_records,
+    remove_document_noise,
     score_quality,
 )
 from app.modules.data_process.generation import generate_model_records
@@ -453,13 +455,25 @@ def _build_preview_items(
     for source in source_files:
         parsed = _parse_stored_source(source)
         if process_type == "unstructured":
+            document_noise_spans = tuple(source.get("document_noise_spans") or ())
             chunks = _chunk_source_text(parsed.text, config, preprocess_options)
             for chunk, heading_path in chunks:
+                content = (
+                    remove_document_noise(
+                        chunk.content,
+                        document_noise_spans,
+                        source_offset=chunk.start,
+                    )
+                    if should_clean_invalid and document_noise_spans
+                    else chunk.content
+                )
                 preprocess_flags = content_quality_flags(
-                    chunk.content,
+                    content,
                     min_chars=0,
                     min_tokens=0,
                 )
+                if content != chunk.content:
+                    preprocess_flags = (*preprocess_flags, "document_noise_removed")
                 flag_set = set(preprocess_flags)
                 if "clean_invalid_content" in preprocess_options and flag_set & {
                     "empty_content",
@@ -475,20 +489,19 @@ def _build_preview_items(
                 }:
                     continue
                 if "deduplicate_content" in preprocess_options:
-                    band_keys = _near_duplicate_band_keys(chunk.content)
+                    band_keys = _near_duplicate_band_keys(content)
                     candidates = {
                         previous
                         for key in band_keys
                         for previous in seen_near_duplicate_bands.get(key, ())
                     }
                     if any(
-                        _safe_near_duplicate(chunk.content, previous)
+                        _safe_near_duplicate(content, previous)
                         for previous in candidates
                     ):
                         continue
                     for key in band_keys:
-                        seen_near_duplicate_bands.setdefault(key, []).append(chunk.content)
-                content = chunk.content
+                        seen_near_duplicate_bands.setdefault(key, []).append(content)
                 pii_counts: dict[str, int] = {}
                 if should_desensitize:
                     content, pii_counts = desensitize_pii(content)
@@ -512,7 +525,7 @@ def _build_preview_items(
                         "source_end": chunk.end,
                         "source_start_line": chunk.start_line,
                         "source_end_line": chunk.end_line,
-                        "token_count": chunk.token_count,
+                        "token_count": estimate_token_count(content),
                         "status": "modified" if content != chunk.content else "original",
                         "quality_score": quality,
                     }
@@ -1234,6 +1247,7 @@ def pull_external_source(
 def _prepare_preview_items(
     task_id: str,
     store: DataProcessStore,
+    storage: LocalDataProcessStorage,
     source_file_ids: list[str] | None = None,
 ) -> list[dict[str, Any]]:
     task = store.get_task(task_id)
@@ -1251,6 +1265,48 @@ def _prepare_preview_items(
     ]
     if not sources:
         raise InvalidStateError("at least one source file is required")
+    preprocess_options = _preprocess_options(task.get("config") or {})
+    if (
+        task.get("process_type") == "unstructured"
+        and preprocess_options & {"clean_invalid", "clean_invalid_content"}
+    ):
+        for index, source in enumerate(sources):
+            if str(source.get("file_format") or "").lower() != "pdf":
+                continue
+            storage_object_id = str(source.get("storage_object_id") or "")
+            actual_size = storage.file_size(
+                storage_object_id,
+                expected_task_id=task_id,
+                expected_source_file_id=str(source["id"]),
+            )
+            if actual_size is None:
+                logger.info(
+                    "skip PDF document noise detection for unavailable legacy source %s",
+                    source["id"],
+                )
+                continue
+            expected_size = int(source.get("size_bytes") or 0)
+            if expected_size and actual_size != expected_size:
+                raise ValueError("source object size does not match metadata")
+            raw = b"".join(
+                storage.iter_bytes(
+                    storage_object_id,
+                    expected_task_id=task_id,
+                    expected_source_file_id=str(source["id"]),
+                    expected_size=actual_size,
+                )
+            )
+            pages = extract_pdf_page_texts(raw)
+            extracted_text = "\n\n".join(page.text for page in pages if page.text)
+            if extracted_text != str(source.get("content") or ""):
+                logger.warning(
+                    "skip PDF document noise detection because stored offsets differ for %s",
+                    source["id"],
+                )
+                continue
+            enriched = dict(source)
+            enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
+            sources[index] = enriched
     items = _build_preview_items(task, sources)
     if not items and source_file_ids is None:
         raise InvalidStateError("source files did not produce preview items")
@@ -1262,10 +1318,11 @@ def build_preview(
     task_id: str,
     payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest),
     store: DataProcessStore = Depends(get_data_process_store),
+    storage: LocalDataProcessStorage = Depends(get_data_process_storage),
 ) -> dict[str, Any]:
     with api_errors():
         selected_ids = payload.source_file_ids
-        items = _prepare_preview_items(task_id, store, selected_ids)
+        items = _prepare_preview_items(task_id, store, storage, selected_ids)
         created = store.replace_preview_items(
             task_id,
             items,
@@ -1405,9 +1462,10 @@ def start(
     background_tasks: BackgroundTasks,
     payload: GenerateRequest = Body(default_factory=GenerateRequest),
     store: DataProcessStore = Depends(get_data_process_store),
+    storage: LocalDataProcessStorage = Depends(get_data_process_storage),
 ) -> dict[str, Any]:
     with api_errors():
-        items = _prepare_preview_items(task_id, store)
+        items = _prepare_preview_items(task_id, store, storage)
         store.replace_preview_items(task_id, items)
     return _start_generation(task_id, payload, background_tasks, store)
 

backend/app/modules/data_process/algorithms.py

@@ -175,6 +175,15 @@ class PdfPageText:
     source_end: int
 
 
+@dataclass(frozen=True, slots=True)
+class DocumentNoiseSpan:
+    """PDF 中可安全从展示内容移除的文本范围。"""
+
+    start: int
+    end: int
+    kind: Literal["page_number", "repeated_margin", "table_of_contents"]
+
+
 @dataclass(frozen=True, slots=True)
 class TextChunk:
     """带有可追溯来源位置的非结构化文本切片。"""
@@ -496,6 +505,240 @@ def extract_pdf_page_texts(raw: bytes) -> tuple[PdfPageText, ...]:
     return tuple(pages)
 
 
+@dataclass(frozen=True, slots=True)
+class _PdfLine:
+    text: str
+    start: int
+    end: int
+
+
+_PDF_PAGE_NUMBER_LINE_PATTERN = re.compile(
+    r"^(?:页次\s*)?(?:第\s*)?(?P<page>\d+)\s*页\s*"
+    r"(?:(?:[/／]\s*)?共\s*(?P<total>\d+)\s*页)?$"
+)
+_PDF_FRACTION_PAGE_LINE_PATTERN = re.compile(
+    r"^[—–-]?\s*(?P<page>\d+)\s*[/／]\s*(?P<total>\d+)\s*[—–-]?$"
+)
+_PDF_CLASSIFICATION_LABEL_PATTERN = re.compile(
+    r"^(?:(?:秘密等级|密级)\s*)?(?:商密|秘密|机密|绝密)"
+    r"\s*(?:[【\[(（][^】\])）]{1,8}[】\])）])?$"
+)
+_TOC_TITLE_PATTERN = re.compile(r"^(?:目\s*录|contents)$", re.IGNORECASE)
+_TOC_LEADER_ENTRY_PATTERN = re.compile(
+    r"(?:[.．…·•]\s*){3,}\s*\d{1,4}\s*$"
+)
+_TOC_NUMBERED_ENTRY_PATTERN = re.compile(
+    r"^(?:第[\u3400-\u4dbf\u4e00-\u9fff]{1,12}章|附表\s*\d+|\d+(?:\.\d+)+)"
+    r"\s+.+\s+\d{1,4}\s*$",
+    re.IGNORECASE,
+)
+_MARGIN_TEMPLATE_KEYWORDS = (
+    "页",
+    "页次",
+    "版本",
+    "文件编码",
+    "秘密等级",
+    "密级",
+    "商密",
+    "confidential",
+)
+
+
+def _pdf_page_lines(page: PdfPageText) -> tuple[_PdfLine, ...]:
+    lines: list[_PdfLine] = []
+    local_offset = 0
+    for raw_line in page.text.splitlines(keepends=True):
+        content = raw_line.rstrip("\r\n")
+        leading = len(content) - len(content.lstrip())
+        trailing = len(content.rstrip())
+        text = content.strip()
+        if text:
+            lines.append(
+                _PdfLine(
+                    text=text,
+                    start=page.source_start + local_offset + leading,
+                    end=page.source_start + local_offset + trailing,
+                )
+            )
+        local_offset += len(raw_line)
+    return tuple(lines)
+
+
+def _is_standalone_page_number(
+    text: str,
+    *,
+    physical_page: int,
+    page_count: int,
+) -> bool:
+    normalized = unicodedata.normalize("NFKC", text).strip()
+    match = _PDF_PAGE_NUMBER_LINE_PATTERN.fullmatch(normalized)
+    if match is None:
+        match = _PDF_FRACTION_PAGE_LINE_PATTERN.fullmatch(normalized)
+    if match is None or int(match.group("page")) != physical_page:
+        return False
+    total = match.groupdict().get("total")
+    return total is None or int(total) == page_count
+
+
+def _margin_signature(text: str) -> str:
+    normalized = unicodedata.normalize("NFKC", text).casefold()
+    normalized = re.sub(r"\s+", " ", normalized).strip()
+    if any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS):
+        normalized = re.sub(r"\d+", "#", normalized)
+    return normalized
+
+
+def _has_margin_metadata_keyword(text: str) -> bool:
+    normalized = unicodedata.normalize("NFKC", text).casefold()
+    return any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS)
+
+
+def _has_meaningful_margin_signature(signature: str) -> bool:
+    return len(re.sub(r"[#\W_]+", "", signature, flags=re.UNICODE)) >= 2
+
+
+def _is_toc_leader_entry(text: str) -> bool:
+    return bool(_TOC_LEADER_ENTRY_PATTERN.search(text))
+
+
+def _is_toc_numbered_entry(text: str) -> bool:
+    return bool(_TOC_NUMBERED_ENTRY_PATTERN.fullmatch(text))
+
+
+def detect_pdf_document_noise(
+    pages: Sequence[PdfPageText],
+) -> tuple[DocumentNoiseSpan, ...]:
+    """识别 PDF 中的独立页码、重复页边内容和高置信目录。
+
+    规则只查看每页顶部 5 行和底部 3 行来推断页眉页脚；目录必须有
+    明显的点引导线密度，避免仅因正文中出现“目录”或章节标题而误删。
+    """
+
+    page_lines = tuple(_pdf_page_lines(page) for page in pages)
+    detected: dict[tuple[int, int], DocumentNoiseSpan] = {}
+
+    def mark(
+        line: _PdfLine,
+        kind: Literal["page_number", "repeated_margin", "table_of_contents"],
+    ) -> None:
+        detected.setdefault(
+            (line.start, line.end),
+            DocumentNoiseSpan(
+                start=line.start,
+                end=line.end,
+                kind=kind,
+            ),
+        )
+
+    for page, lines in zip(pages, page_lines, strict=True):
+        for line in lines:
+            if _is_standalone_page_number(
+                line.text,
+                physical_page=page.page_number,
+                page_count=len(pages),
+            ):
+                mark(line, "page_number")
+        outer_margin_lines = (*lines[:2], *lines[-2:])
+        for line in outer_margin_lines:
+            if _PDF_CLASSIFICATION_LABEL_PATTERN.fullmatch(line.text):
+                mark(line, "repeated_margin")
+
+    # 只在三页及以上文档中推断通用页眉页脚，避免短文档误删。
+    if len(pages) >= 3:
+        signature_pages: dict[str, set[int]] = {}
+        candidate_lines: list[tuple[int, _PdfLine, str]] = []
+        for page_index, lines in enumerate(page_lines):
+            boundary_lines = (
+                *((line, index < 2) for index, line in enumerate(lines[:5])),
+                *((line, index < 2) for index, line in enumerate(reversed(lines[-3:]))),
+            )
+            seen_ranges: set[tuple[int, int]] = set()
+            for line, is_outer_margin in boundary_lines:
+                line_range = (line.start, line.end)
+                if (
+                    line_range in seen_ranges
+                    or line_range in detected
+                    or len(line.text) > 160
+                ):
+                    continue
+                seen_ranges.add(line_range)
+                if not is_outer_margin and not _has_margin_metadata_keyword(line.text):
+                    continue
+                signature = _margin_signature(line.text)
+                if not _has_meaningful_margin_signature(signature):
+                    continue
+                signature_pages.setdefault(signature, set()).add(page_index)
+                candidate_lines.append((page_index, line, signature))
+        minimum_pages = max(3, math.ceil(len(pages) * 0.3))
+        repeated_signatures = {
+            signature
+            for signature, matching_pages in signature_pages.items()
+            if len(matching_pages) >= minimum_pages
+        }
+        for _, line, signature in candidate_lines:
+            if signature in repeated_signatures:
+                mark(line, "repeated_margin")
+
+    # 先依据强证据判定目录页，再补充删除少量不带点引导线的编号目录项。
+    toc_active = False
+    for lines in page_lines:
+        content_lines = [
+            line for line in lines if (line.start, line.end) not in detected
+        ]
+        leader_entries = [line for line in content_lines if _is_toc_leader_entry(line.text)]
+        titles = [line for line in content_lines if _TOC_TITLE_PATTERN.fullmatch(line.text)]
+        starts_toc = bool(titles and len(leader_entries) >= 2)
+        is_toc_dense = bool(
+            len(leader_entries) >= 3
+            and len(leader_entries) / max(1, len(content_lines)) >= 0.5
+        )
+        if not (starts_toc or (toc_active and is_toc_dense)):
+            toc_active = False
+            continue
+        toc_active = True
+        for line in content_lines:
+            if (
+                line in titles
+                or _is_toc_leader_entry(line.text)
+                or _is_toc_numbered_entry(line.text)
+            ):
+                mark(line, "table_of_contents")
+
+    return tuple(sorted(detected.values(), key=lambda span: (span.start, span.end)))
+
+
+def remove_document_noise(
+    text: str,
+    spans: Sequence[DocumentNoiseSpan],
+    *,
+    source_offset: int = 0,
+) -> str:
+    """按原文绝对偏移移除噪声，不改动调用方保留的原文及偏移。"""
+
+    text_end = source_offset + len(text)
+    intersections = sorted(
+        (
+            max(0, span.start - source_offset),
+            min(len(text), span.end - source_offset),
+        )
+        for span in spans
+        if span.start < text_end and span.end > source_offset
+    )
+    if not intersections:
+        return text
+    parts: list[str] = []
+    cursor = 0
+    for start, end in intersections:
+        if end <= cursor:
+            continue
+        if start > cursor:
+            parts.append(text[cursor:start])
+        cursor = end
+    parts.append(text[cursor:])
+    cleaned = normalize_text("".join(parts))
+    return re.sub(r"\n{3,}", "\n\n", cleaned)
+
+
 def _extract_pdf_text(raw: bytes) -> str:
     return "\n\n".join(page.text for page in extract_pdf_page_texts(raw) if page.text)
 
@@ -2737,6 +2980,7 @@ __all__ = [
     "ChunkMethod",
     "DatasetSplit",
     "DocumentHeading",
+    "DocumentNoiseSpan",
     "DocumentStructure",
     "ParsedText",
     "PdfPageText",
@@ -2752,6 +2996,7 @@ __all__ = [
     "desensitize_pii",
     "desensitize_structured_record",
     "detect_document_structure",
+    "detect_pdf_document_noise",
     "detect_text_format",
     "deduplicate_structured_records",
     "estimate_token_count",
@@ -2773,6 +3018,7 @@ __all__ = [
     "preprocess_structured_records",
     "protected_context_ranges",
     "record_fingerprint",
+    "remove_document_noise",
     "score_quality",
     "stable_split",
 ]

backend/tests/test_data_process_algorithms.py

@@ -14,11 +14,13 @@ from pptx.util import Inches
 from pypdf import PdfWriter
 
 from app.modules.data_process.algorithms import (
+    PdfPageText,
     chunk_unstructured,
     content_quality_flags,
     desensitize_pii,
     desensitize_structured_record,
     detect_document_structure,
+    detect_pdf_document_noise,
     detect_text_format,
     estimate_token_count,
     extract_pdf_page_texts,
@@ -30,11 +32,32 @@ from app.modules.data_process.algorithms import (
     parse_text_content,
     preprocess_structured_records,
     record_fingerprint,
+    remove_document_noise,
     score_quality,
     stable_split,
 )
 
 
+def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
+    pages: list[PdfPageText] = []
+    offset = 0
+    for page_number, text in enumerate(texts, start=1):
+        normalized = normalize_text(text)
+        if pages:
+            offset += 2
+        start = offset
+        offset += len(normalized)
+        pages.append(
+            PdfPageText(
+                page_number=page_number,
+                text=normalized,
+                source_start=start,
+                source_end=offset,
+            )
+        )
+    return tuple(pages)
+
+
 def _minimal_pdf(text: str = "Hello PDF") -> bytes:
     stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii")
     objects = [
@@ -206,6 +229,91 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
     assert parsed_pptx.records == ()
 
 
+def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
+    pages = _pdf_page_texts(
+        """
+        远光制度文件 文件编码 2024
+        秘密等级 商密【中】
+        第 1 页 共 5 页
+        正文第一页，关于适用范围的说明。
+        业务提示保留
+        第一页补充说明甲
+        第一页补充说明乙
+        第一页补充说明丙
+        """,
+        """
+        远光制度文件 文件编码 2024
+        秘密等级 商密【中】
+        第 2 页 共 5 页
+        目 录
+        第一章 总则........3
+        第二章 报销申请........4
+        第三章 附则........5
+        """,
+        """
+        远光制度文件 文件编码 2024
+        秘密等级 商密【中】
+        第 3 页 共 5 页
+        1.1 管理要求........6
+        1.2 审批职责 7
+        1.3 费用标准........8
+        1.4 例外处理........9
+        """,
+        """
+        远光制度文件 文件编码 2024
+        秘密等级 商密【中】
+        第 4 页 共 5 页
+        正文中可以说“请参见第 3 页说明”，不应误删。
+        第 99 页 共 100 页
+        系统可用率........99.9%
+        业务提示保留
+        第四页补充说明甲
+        第四页补充说明乙
+        第四页补充说明丙
+        """,
+        """
+        远光制度文件 文件编码 2024
+        秘密等级 商密【中】
+        第 5 页 共 5 页
+        本办法自发布之日起施行。
+        业务提示保留
+        第五页补充说明甲
+        第五页补充说明乙
+        第五页补充说明丙
+        """,
+    )
+    source = "\n\n".join(page.text for page in pages)
+
+    spans = detect_pdf_document_noise(pages)
+    cleaned = remove_document_noise(source, spans)
+
+    assert {span.kind for span in spans} == {
+        "page_number",
+        "repeated_margin",
+        "table_of_contents",
+    }
+    assert "远光制度文件" not in cleaned
+    assert "商密【中】" not in cleaned
+    assert "第 1 页 共 5 页" not in cleaned
+    assert "第一章 总则" not in cleaned
+    assert "1.2 审批职责 7" not in cleaned
+    assert "请参见第 3 页说明" in cleaned
+    assert "第 99 页 共 100 页" in cleaned
+    assert "系统可用率........99.9%" in cleaned
+    assert cleaned.count("业务提示保留") == 3
+
+
+def test_pdf_document_noise_does_not_infer_repeated_margins_for_short_documents() -> None:
+    pages = _pdf_page_texts(
+        "公司内部文件\n正文 A",
+        "公司内部文件\n正文 B",
+    )
+
+    spans = detect_pdf_document_noise(pages)
+
+    assert not any(span.kind == "repeated_margin" for span in spans)
+
+
 def test_xlsx_merged_multilevel_headers_are_flattened_without_losing_columns() -> None:
     workbook = Workbook()
     worksheet = workbook.active

backend/tests/test_data_process_api.py

@@ -12,7 +12,7 @@ from openpyxl import Workbook
 
 from app.api.v1.endpoints import data_process as data_process_endpoint
 from app.api.v1.endpoints.data_process import router
-from app.modules.data_process.algorithms import normalize_text
+from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text
 from app.modules.data_process.storage import (
     DataProcessStorageError,
     LocalDataProcessStorage,
@@ -443,20 +443,49 @@ def _stored_files(storage: LocalDataProcessStorage) -> list[Path]:
     return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()]
 
 
-def _minimal_pdf(text: str = "Hello PDF") -> bytes:
-    stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii")
+def _minimal_pdf_pages(*page_texts: str) -> bytes:
+    if not page_texts:
+        raise ValueError("at least one PDF page is required")
+    font_object_number = 3 + len(page_texts) * 2
+    page_object_numbers = [3 + index * 2 for index in range(len(page_texts))]
     objects = [
         b"<< /Type /Catalog /Pages 2 0 R >>",
-        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
         (
-            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
-            b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>"
+            b"<< /Type /Pages /Kids ["
+            + b" ".join(f"{number} 0 R".encode() for number in page_object_numbers)
+            + b"] /Count "
+            + str(len(page_texts)).encode()
+            + b" >>"
         ),
-        b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n"
-        + stream
-        + b"\nendstream",
-        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
     ]
+    for index, text in enumerate(page_texts):
+        content_object_number = page_object_numbers[index] + 1
+        commands = [b"BT /F1 12 Tf 72 720 Td"]
+        for line_index, line in enumerate(text.splitlines()):
+            escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
+            if line_index:
+                commands.append(b"0 -16 Td")
+            commands.append(f"({escaped}) Tj".encode("ascii"))
+        commands.append(b"ET")
+        stream = b" ".join(commands)
+        objects.extend(
+            [
+                (
+                    b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+                    b"/Resources << /Font << /F1 "
+                    + str(font_object_number).encode()
+                    + b" 0 R >> >> /Contents "
+                    + str(content_object_number).encode()
+                    + b" 0 R >>"
+                ),
+                b"<< /Length "
+                + str(len(stream)).encode()
+                + b" >>\nstream\n"
+                + stream
+                + b"\nendstream",
+            ]
+        )
+    objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
     result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
     offsets = [0]
     for object_number, value in enumerate(objects, start=1):
@@ -478,6 +507,10 @@ def _minimal_pdf(text: str = "Hello PDF") -> bytes:
     return bytes(result)
 
 
+def _minimal_pdf(text: str = "Hello PDF") -> bytes:
+    return _minimal_pdf_pages(text)
+
+
 def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
     client, store, _ = make_client(tmp_path)
     created = client.post(
@@ -1127,6 +1160,59 @@ def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Pat
     assert legacy_pages.status_code == 410
 
 
+def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift(
+    tmp_path: Path,
+) -> None:
+    client, _, _ = make_client(tmp_path)
+    task_id = client.post(
+        "/modelTF/data-process",
+        json={
+            "name": "PDF 文档噪声清理",
+            "process_type": "unstructured",
+            "config": {
+                "chunk_method": "fixed",
+                "chunk_size": 200,
+                "chunk_overlap": 0,
+                "min_chunk_size": 20,
+                "preprocess_options": ["clean_invalid_content"],
+            },
+        },
+    ).json()["data"]["id"]
+    raw = _minimal_pdf_pages(
+        "ACME Internal Manual\nBody page one keeps this guidance and explanation.",
+        "ACME Internal Manual\nContents\n"
+        "Chapter One........3\nChapter Two........4\nAppendix........5",
+        "ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8",
+        "ACME Internal Manual\nBody page four keeps operational details and examples.",
+        "ACME Internal Manual\nBody page five keeps the final effective-date clause.",
+    )
+    uploaded = client.post(
+        f"/modelTF/data-process/{task_id}/source-files",
+        files={"files": ("manual.pdf", raw, "application/pdf")},
+    ).json()["data"]["files"][0]
+    source_content = client.get(
+        f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content"
+    ).json()["data"]["content"]
+
+    built = client.post(f"/modelTF/data-process/{task_id}/preview/build")
+
+    assert built.status_code == 200
+    items = built.json()["data"]["items"]
+    assert items
+    edited = "\n".join(item["edited_content"] for item in items)
+    assert "ACME Internal Manual" not in edited
+    assert "Contents" not in edited
+    assert "Chapter One" not in edited
+    assert "1.2 Approval" not in edited
+    assert "Body page one" in edited
+    assert "Body page five" in edited
+    assert all(
+        item["original_content"]
+        == source_content[item["source_start"] : item["source_end"]]
+        for item in items
+    )
+
+
 def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None:
     client, _, _ = make_client(tmp_path)
     task_id = client.post(
@@ -1450,6 +1536,51 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None
     assert "[EMAIL]" in masked["edited_content"]
 
 
+def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None:
+    source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。")
+    source = {
+        "id": "pdf-source",
+        "name": "manual.pdf",
+        "file_format": "pdf",
+        "content": source_text,
+        "document_noise_spans": (
+            DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"),
+        ),
+    }
+    config = {
+        "chunk_method": "fixed",
+        "chunk_size": 200,
+        "chunk_overlap": 0,
+        "min_chunk_size": 1,
+    }
+
+    cleaned_items = data_process_endpoint._build_preview_items(
+        {
+            "process_type": "unstructured",
+            "config": {**config, "preprocess_options": ["clean_invalid_content"]},
+        },
+        [source],
+    )
+    original_items = data_process_endpoint._build_preview_items(
+        {
+            "process_type": "unstructured",
+            "config": {**config, "preprocess_options": []},
+        },
+        [source],
+    )
+
+    assert len(cleaned_items) == 1
+    cleaned = cleaned_items[0]
+    assert cleaned["original_content"] == source_text[
+        cleaned["source_start"] : cleaned["source_end"]
+    ]
+    assert "重复页眉" not in cleaned["edited_content"]
+    assert "PDF 正文内容" in cleaned["edited_content"]
+    assert cleaned["status"] == "modified"
+    assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"]
+    assert "重复页眉" in original_items[0]["edited_content"]
+
+
 def test_merge_short_content_applies_across_adjacent_structure_sections() -> None:
     content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
     items = _preview_task(