diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 14228f6..7297fef 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -37,6 +37,7 @@ from app.modules.data_process.algorithms import ( desensitize_pii, desensitize_structured_record, detect_document_structure, + detect_pdf_document_noise, estimate_token_count, extract_pdf_page_texts, generate_standard_records, @@ -44,6 +45,7 @@ from app.modules.data_process.algorithms import ( near_duplicate_fingerprint, parse_text_content, preprocess_structured_records, + remove_document_noise, score_quality, ) from app.modules.data_process.generation import generate_model_records @@ -453,13 +455,25 @@ def _build_preview_items( for source in source_files: parsed = _parse_stored_source(source) if process_type == "unstructured": + document_noise_spans = tuple(source.get("document_noise_spans") or ()) chunks = _chunk_source_text(parsed.text, config, preprocess_options) for chunk, heading_path in chunks: + content = ( + remove_document_noise( + chunk.content, + document_noise_spans, + source_offset=chunk.start, + ) + if should_clean_invalid and document_noise_spans + else chunk.content + ) preprocess_flags = content_quality_flags( - chunk.content, + content, min_chars=0, min_tokens=0, ) + if content != chunk.content: + preprocess_flags = (*preprocess_flags, "document_noise_removed") flag_set = set(preprocess_flags) if "clean_invalid_content" in preprocess_options and flag_set & { "empty_content", @@ -475,20 +489,19 @@ def _build_preview_items( }: continue if "deduplicate_content" in preprocess_options: - band_keys = _near_duplicate_band_keys(chunk.content) + band_keys = _near_duplicate_band_keys(content) candidates = { previous for key in band_keys for previous in seen_near_duplicate_bands.get(key, ()) } if any( - _safe_near_duplicate(chunk.content, previous) + _safe_near_duplicate(content, previous) for previous in candidates ): continue for key in band_keys: - seen_near_duplicate_bands.setdefault(key, []).append(chunk.content) - content = chunk.content + seen_near_duplicate_bands.setdefault(key, []).append(content) pii_counts: dict[str, int] = {} if should_desensitize: content, pii_counts = desensitize_pii(content) @@ -512,7 +525,7 @@ def _build_preview_items( "source_end": chunk.end, "source_start_line": chunk.start_line, "source_end_line": chunk.end_line, - "token_count": chunk.token_count, + "token_count": estimate_token_count(content), "status": "modified" if content != chunk.content else "original", "quality_score": quality, } @@ -1234,6 +1247,7 @@ def pull_external_source( def _prepare_preview_items( task_id: str, store: DataProcessStore, + storage: LocalDataProcessStorage, source_file_ids: list[str] | None = None, ) -> list[dict[str, Any]]: task = store.get_task(task_id) @@ -1251,6 +1265,48 @@ def _prepare_preview_items( ] if not sources: raise InvalidStateError("at least one source file is required") + preprocess_options = _preprocess_options(task.get("config") or {}) + if ( + task.get("process_type") == "unstructured" + and preprocess_options & {"clean_invalid", "clean_invalid_content"} + ): + for index, source in enumerate(sources): + if str(source.get("file_format") or "").lower() != "pdf": + continue + storage_object_id = str(source.get("storage_object_id") or "") + actual_size = storage.file_size( + storage_object_id, + expected_task_id=task_id, + expected_source_file_id=str(source["id"]), + ) + if actual_size is None: + logger.info( + "skip PDF document noise detection for unavailable legacy source %s", + source["id"], + ) + continue + expected_size = int(source.get("size_bytes") or 0) + if expected_size and actual_size != expected_size: + raise ValueError("source object size does not match metadata") + raw = b"".join( + storage.iter_bytes( + storage_object_id, + expected_task_id=task_id, + expected_source_file_id=str(source["id"]), + expected_size=actual_size, + ) + ) + pages = extract_pdf_page_texts(raw) + extracted_text = "\n\n".join(page.text for page in pages if page.text) + if extracted_text != str(source.get("content") or ""): + logger.warning( + "skip PDF document noise detection because stored offsets differ for %s", + source["id"], + ) + continue + enriched = dict(source) + enriched["document_noise_spans"] = detect_pdf_document_noise(pages) + sources[index] = enriched items = _build_preview_items(task, sources) if not items and source_file_ids is None: raise InvalidStateError("source files did not produce preview items") @@ -1262,10 +1318,11 @@ def build_preview( task_id: str, payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest), store: DataProcessStore = Depends(get_data_process_store), + storage: LocalDataProcessStorage = Depends(get_data_process_storage), ) -> dict[str, Any]: with api_errors(): selected_ids = payload.source_file_ids - items = _prepare_preview_items(task_id, store, selected_ids) + items = _prepare_preview_items(task_id, store, storage, selected_ids) created = store.replace_preview_items( task_id, items, @@ -1405,9 +1462,10 @@ def start( background_tasks: BackgroundTasks, payload: GenerateRequest = Body(default_factory=GenerateRequest), store: DataProcessStore = Depends(get_data_process_store), + storage: LocalDataProcessStorage = Depends(get_data_process_storage), ) -> dict[str, Any]: with api_errors(): - items = _prepare_preview_items(task_id, store) + items = _prepare_preview_items(task_id, store, storage) store.replace_preview_items(task_id, items) return _start_generation(task_id, payload, background_tasks, store) diff --git a/backend/app/modules/data_process/algorithms.py b/backend/app/modules/data_process/algorithms.py index a293f9e..fd281f3 100644 --- a/backend/app/modules/data_process/algorithms.py +++ b/backend/app/modules/data_process/algorithms.py @@ -175,6 +175,15 @@ class PdfPageText: source_end: int +@dataclass(frozen=True, slots=True) +class DocumentNoiseSpan: + """PDF 中可安全从展示内容移除的文本范围。""" + + start: int + end: int + kind: Literal["page_number", "repeated_margin", "table_of_contents"] + + @dataclass(frozen=True, slots=True) class TextChunk: """带有可追溯来源位置的非结构化文本切片。""" @@ -496,6 +505,240 @@ def extract_pdf_page_texts(raw: bytes) -> tuple[PdfPageText, ...]: return tuple(pages) +@dataclass(frozen=True, slots=True) +class _PdfLine: + text: str + start: int + end: int + + +_PDF_PAGE_NUMBER_LINE_PATTERN = re.compile( + r"^(?:页次\s*)?(?:第\s*)?(?P\d+)\s*页\s*" + r"(?:(?:[/／]\s*)?共\s*(?P\d+)\s*页)?$" +) +_PDF_FRACTION_PAGE_LINE_PATTERN = re.compile( + r"^[—–-]?\s*(?P\d+)\s*[/／]\s*(?P\d+)\s*[—–-]?$" +) +_PDF_CLASSIFICATION_LABEL_PATTERN = re.compile( + r"^(?:(?:秘密等级|密级)\s*)?(?:商密|秘密|机密|绝密)" + r"\s*(?:[【\[(（][^】\])）]{1,8}[】\])）])?$" +) +_TOC_TITLE_PATTERN = re.compile(r"^(?:目\s*录|contents)$", re.IGNORECASE) +_TOC_LEADER_ENTRY_PATTERN = re.compile( + r"(?:[.．…·•]\s*){3,}\s*\d{1,4}\s*$" +) +_TOC_NUMBERED_ENTRY_PATTERN = re.compile( + r"^(?:第[\u3400-\u4dbf\u4e00-\u9fff]{1,12}章|附表\s*\d+|\d+(?:\.\d+)+)" + r"\s+.+\s+\d{1,4}\s*$", + re.IGNORECASE, +) +_MARGIN_TEMPLATE_KEYWORDS = ( + "页", + "页次", + "版本", + "文件编码", + "秘密等级", + "密级", + "商密", + "confidential", +) + + +def _pdf_page_lines(page: PdfPageText) -> tuple[_PdfLine, ...]: + lines: list[_PdfLine] = [] + local_offset = 0 + for raw_line in page.text.splitlines(keepends=True): + content = raw_line.rstrip("\r\n") + leading = len(content) - len(content.lstrip()) + trailing = len(content.rstrip()) + text = content.strip() + if text: + lines.append( + _PdfLine( + text=text, + start=page.source_start + local_offset + leading, + end=page.source_start + local_offset + trailing, + ) + ) + local_offset += len(raw_line) + return tuple(lines) + + +def _is_standalone_page_number( + text: str, + *, + physical_page: int, + page_count: int, +) -> bool: + normalized = unicodedata.normalize("NFKC", text).strip() + match = _PDF_PAGE_NUMBER_LINE_PATTERN.fullmatch(normalized) + if match is None: + match = _PDF_FRACTION_PAGE_LINE_PATTERN.fullmatch(normalized) + if match is None or int(match.group("page")) != physical_page: + return False + total = match.groupdict().get("total") + return total is None or int(total) == page_count + + +def _margin_signature(text: str) -> str: + normalized = unicodedata.normalize("NFKC", text).casefold() + normalized = re.sub(r"\s+", " ", normalized).strip() + if any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS): + normalized = re.sub(r"\d+", "#", normalized) + return normalized + + +def _has_margin_metadata_keyword(text: str) -> bool: + normalized = unicodedata.normalize("NFKC", text).casefold() + return any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS) + + +def _has_meaningful_margin_signature(signature: str) -> bool: + return len(re.sub(r"[#\W_]+", "", signature, flags=re.UNICODE)) >= 2 + + +def _is_toc_leader_entry(text: str) -> bool: + return bool(_TOC_LEADER_ENTRY_PATTERN.search(text)) + + +def _is_toc_numbered_entry(text: str) -> bool: + return bool(_TOC_NUMBERED_ENTRY_PATTERN.fullmatch(text)) + + +def detect_pdf_document_noise( + pages: Sequence[PdfPageText], +) -> tuple[DocumentNoiseSpan, ...]: + """识别 PDF 中的独立页码、重复页边内容和高置信目录。 + + 规则只查看每页顶部 5 行和底部 3 行来推断页眉页脚；目录必须有 + 明显的点引导线密度，避免仅因正文中出现“目录”或章节标题而误删。 + """ + + page_lines = tuple(_pdf_page_lines(page) for page in pages) + detected: dict[tuple[int, int], DocumentNoiseSpan] = {} + + def mark( + line: _PdfLine, + kind: Literal["page_number", "repeated_margin", "table_of_contents"], + ) -> None: + detected.setdefault( + (line.start, line.end), + DocumentNoiseSpan( + start=line.start, + end=line.end, + kind=kind, + ), + ) + + for page, lines in zip(pages, page_lines, strict=True): + for line in lines: + if _is_standalone_page_number( + line.text, + physical_page=page.page_number, + page_count=len(pages), + ): + mark(line, "page_number") + outer_margin_lines = (*lines[:2], *lines[-2:]) + for line in outer_margin_lines: + if _PDF_CLASSIFICATION_LABEL_PATTERN.fullmatch(line.text): + mark(line, "repeated_margin") + + # 只在三页及以上文档中推断通用页眉页脚，避免短文档误删。 + if len(pages) >= 3: + signature_pages: dict[str, set[int]] = {} + candidate_lines: list[tuple[int, _PdfLine, str]] = [] + for page_index, lines in enumerate(page_lines): + boundary_lines = ( + *((line, index < 2) for index, line in enumerate(lines[:5])), + *((line, index < 2) for index, line in enumerate(reversed(lines[-3:]))), + ) + seen_ranges: set[tuple[int, int]] = set() + for line, is_outer_margin in boundary_lines: + line_range = (line.start, line.end) + if ( + line_range in seen_ranges + or line_range in detected + or len(line.text) > 160 + ): + continue + seen_ranges.add(line_range) + if not is_outer_margin and not _has_margin_metadata_keyword(line.text): + continue + signature = _margin_signature(line.text) + if not _has_meaningful_margin_signature(signature): + continue + signature_pages.setdefault(signature, set()).add(page_index) + candidate_lines.append((page_index, line, signature)) + minimum_pages = max(3, math.ceil(len(pages) * 0.3)) + repeated_signatures = { + signature + for signature, matching_pages in signature_pages.items() + if len(matching_pages) >= minimum_pages + } + for _, line, signature in candidate_lines: + if signature in repeated_signatures: + mark(line, "repeated_margin") + + # 先依据强证据判定目录页，再补充删除少量不带点引导线的编号目录项。 + toc_active = False + for lines in page_lines: + content_lines = [ + line for line in lines if (line.start, line.end) not in detected + ] + leader_entries = [line for line in content_lines if _is_toc_leader_entry(line.text)] + titles = [line for line in content_lines if _TOC_TITLE_PATTERN.fullmatch(line.text)] + starts_toc = bool(titles and len(leader_entries) >= 2) + is_toc_dense = bool( + len(leader_entries) >= 3 + and len(leader_entries) / max(1, len(content_lines)) >= 0.5 + ) + if not (starts_toc or (toc_active and is_toc_dense)): + toc_active = False + continue + toc_active = True + for line in content_lines: + if ( + line in titles + or _is_toc_leader_entry(line.text) + or _is_toc_numbered_entry(line.text) + ): + mark(line, "table_of_contents") + + return tuple(sorted(detected.values(), key=lambda span: (span.start, span.end))) + + +def remove_document_noise( + text: str, + spans: Sequence[DocumentNoiseSpan], + *, + source_offset: int = 0, +) -> str: + """按原文绝对偏移移除噪声，不改动调用方保留的原文及偏移。""" + + text_end = source_offset + len(text) + intersections = sorted( + ( + max(0, span.start - source_offset), + min(len(text), span.end - source_offset), + ) + for span in spans + if span.start < text_end and span.end > source_offset + ) + if not intersections: + return text + parts: list[str] = [] + cursor = 0 + for start, end in intersections: + if end <= cursor: + continue + if start > cursor: + parts.append(text[cursor:start]) + cursor = end + parts.append(text[cursor:]) + cleaned = normalize_text("".join(parts)) + return re.sub(r"\n{3,}", "\n\n", cleaned) + + def _extract_pdf_text(raw: bytes) -> str: return "\n\n".join(page.text for page in extract_pdf_page_texts(raw) if page.text) @@ -2737,6 +2980,7 @@ __all__ = [ "ChunkMethod", "DatasetSplit", "DocumentHeading", + "DocumentNoiseSpan", "DocumentStructure", "ParsedText", "PdfPageText", @@ -2752,6 +2996,7 @@ __all__ = [ "desensitize_pii", "desensitize_structured_record", "detect_document_structure", + "detect_pdf_document_noise", "detect_text_format", "deduplicate_structured_records", "estimate_token_count", @@ -2773,6 +3018,7 @@ __all__ = [ "preprocess_structured_records", "protected_context_ranges", "record_fingerprint", + "remove_document_noise", "score_quality", "stable_split", ] diff --git a/backend/tests/test_data_process_algorithms.py b/backend/tests/test_data_process_algorithms.py index 57f8200..a95458a 100644 --- a/backend/tests/test_data_process_algorithms.py +++ b/backend/tests/test_data_process_algorithms.py @@ -14,11 +14,13 @@ from pptx.util import Inches from pypdf import PdfWriter from app.modules.data_process.algorithms import ( + PdfPageText, chunk_unstructured, content_quality_flags, desensitize_pii, desensitize_structured_record, detect_document_structure, + detect_pdf_document_noise, detect_text_format, estimate_token_count, extract_pdf_page_texts, @@ -30,11 +32,32 @@ from app.modules.data_process.algorithms import ( parse_text_content, preprocess_structured_records, record_fingerprint, + remove_document_noise, score_quality, stable_split, ) +def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]: + pages: list[PdfPageText] = [] + offset = 0 + for page_number, text in enumerate(texts, start=1): + normalized = normalize_text(text) + if pages: + offset += 2 + start = offset + offset += len(normalized) + pages.append( + PdfPageText( + page_number=page_number, + text=normalized, + source_start=start, + source_end=offset, + ) + ) + return tuple(pages) + + def _minimal_pdf(text: str = "Hello PDF") -> bytes: stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii") objects = [ @@ -206,6 +229,91 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None: assert parsed_pptx.records == () +def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None: + pages = _pdf_page_texts( + """ + 远光制度文件 文件编码 2024 + 秘密等级 商密【中】 + 第 1 页 共 5 页 + 正文第一页，关于适用范围的说明。 + 业务提示保留 + 第一页补充说明甲 + 第一页补充说明乙 + 第一页补充说明丙 + """, + """ + 远光制度文件 文件编码 2024 + 秘密等级 商密【中】 + 第 2 页 共 5 页 + 目 录 + 第一章 总则........3 + 第二章 报销申请........4 + 第三章 附则........5 + """, + """ + 远光制度文件 文件编码 2024 + 秘密等级 商密【中】 + 第 3 页 共 5 页 + 1.1 管理要求........6 + 1.2 审批职责 7 + 1.3 费用标准........8 + 1.4 例外处理........9 + """, + """ + 远光制度文件 文件编码 2024 + 秘密等级 商密【中】 + 第 4 页 共 5 页 + 正文中可以说“请参见第 3 页说明”，不应误删。 + 第 99 页 共 100 页 + 系统可用率........99.9% + 业务提示保留 + 第四页补充说明甲 + 第四页补充说明乙 + 第四页补充说明丙 + """, + """ + 远光制度文件 文件编码 2024 + 秘密等级 商密【中】 + 第 5 页 共 5 页 + 本办法自发布之日起施行。 + 业务提示保留 + 第五页补充说明甲 + 第五页补充说明乙 + 第五页补充说明丙 + """, + ) + source = "\n\n".join(page.text for page in pages) + + spans = detect_pdf_document_noise(pages) + cleaned = remove_document_noise(source, spans) + + assert {span.kind for span in spans} == { + "page_number", + "repeated_margin", + "table_of_contents", + } + assert "远光制度文件" not in cleaned + assert "商密【中】" not in cleaned + assert "第 1 页 共 5 页" not in cleaned + assert "第一章 总则" not in cleaned + assert "1.2 审批职责 7" not in cleaned + assert "请参见第 3 页说明" in cleaned + assert "第 99 页 共 100 页" in cleaned + assert "系统可用率........99.9%" in cleaned + assert cleaned.count("业务提示保留") == 3 + + +def test_pdf_document_noise_does_not_infer_repeated_margins_for_short_documents() -> None: + pages = _pdf_page_texts( + "公司内部文件\n正文 A", + "公司内部文件\n正文 B", + ) + + spans = detect_pdf_document_noise(pages) + + assert not any(span.kind == "repeated_margin" for span in spans) + + def test_xlsx_merged_multilevel_headers_are_flattened_without_losing_columns() -> None: workbook = Workbook() worksheet = workbook.active diff --git a/backend/tests/test_data_process_api.py b/backend/tests/test_data_process_api.py index c28aa1f..57a8004 100644 --- a/backend/tests/test_data_process_api.py +++ b/backend/tests/test_data_process_api.py @@ -12,7 +12,7 @@ from openpyxl import Workbook from app.api.v1.endpoints import data_process as data_process_endpoint from app.api.v1.endpoints.data_process import router -from app.modules.data_process.algorithms import normalize_text +from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text from app.modules.data_process.storage import ( DataProcessStorageError, LocalDataProcessStorage, @@ -443,20 +443,49 @@ def _stored_files(storage: LocalDataProcessStorage) -> list[Path]: return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()] -def _minimal_pdf(text: str = "Hello PDF") -> bytes: - stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii") +def _minimal_pdf_pages(*page_texts: str) -> bytes: + if not page_texts: + raise ValueError("at least one PDF page is required") + font_object_number = 3 + len(page_texts) * 2 + page_object_numbers = [3 + index * 2 for index in range(len(page_texts))] objects = [ b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", ( - b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " - b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>" + b"<< /Type /Pages /Kids [" + + b" ".join(f"{number} 0 R".encode() for number in page_object_numbers) + + b"] /Count " + + str(len(page_texts)).encode() + + b" >>" ), - b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" - + stream - + b"\nendstream", - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", ] + for index, text in enumerate(page_texts): + content_object_number = page_object_numbers[index] + 1 + commands = [b"BT /F1 12 Tf 72 720 Td"] + for line_index, line in enumerate(text.splitlines()): + escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + if line_index: + commands.append(b"0 -16 Td") + commands.append(f"({escaped}) Tj".encode("ascii")) + commands.append(b"ET") + stream = b" ".join(commands) + objects.extend( + [ + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 " + + str(font_object_number).encode() + + b" 0 R >> >> /Contents " + + str(content_object_number).encode() + + b" 0 R >>" + ), + b"<< /Length " + + str(len(stream)).encode() + + b" >>\nstream\n" + + stream + + b"\nendstream", + ] + ) + objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") offsets = [0] for object_number, value in enumerate(objects, start=1): @@ -478,6 +507,10 @@ def _minimal_pdf(text: str = "Hello PDF") -> bytes: return bytes(result) +def _minimal_pdf(text: str = "Hello PDF") -> bytes: + return _minimal_pdf_pages(text) + + def test_data_process_full_contract_without_database(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) created = client.post( @@ -1127,6 +1160,59 @@ def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Pat assert legacy_pages.status_code == 410 +def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift( + tmp_path: Path, +) -> None: + client, _, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={ + "name": "PDF 文档噪声清理", + "process_type": "unstructured", + "config": { + "chunk_method": "fixed", + "chunk_size": 200, + "chunk_overlap": 0, + "min_chunk_size": 20, + "preprocess_options": ["clean_invalid_content"], + }, + }, + ).json()["data"]["id"] + raw = _minimal_pdf_pages( + "ACME Internal Manual\nBody page one keeps this guidance and explanation.", + "ACME Internal Manual\nContents\n" + "Chapter One........3\nChapter Two........4\nAppendix........5", + "ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8", + "ACME Internal Manual\nBody page four keeps operational details and examples.", + "ACME Internal Manual\nBody page five keeps the final effective-date clause.", + ) + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files={"files": ("manual.pdf", raw, "application/pdf")}, + ).json()["data"]["files"][0] + source_content = client.get( + f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content" + ).json()["data"]["content"] + + built = client.post(f"/modelTF/data-process/{task_id}/preview/build") + + assert built.status_code == 200 + items = built.json()["data"]["items"] + assert items + edited = "\n".join(item["edited_content"] for item in items) + assert "ACME Internal Manual" not in edited + assert "Contents" not in edited + assert "Chapter One" not in edited + assert "1.2 Approval" not in edited + assert "Body page one" in edited + assert "Body page five" in edited + assert all( + item["original_content"] + == source_content[item["source_start"] : item["source_end"]] + for item in items + ) + + def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( @@ -1450,6 +1536,51 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None assert "[EMAIL]" in masked["edited_content"] +def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None: + source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。") + source = { + "id": "pdf-source", + "name": "manual.pdf", + "file_format": "pdf", + "content": source_text, + "document_noise_spans": ( + DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"), + ), + } + config = { + "chunk_method": "fixed", + "chunk_size": 200, + "chunk_overlap": 0, + "min_chunk_size": 1, + } + + cleaned_items = data_process_endpoint._build_preview_items( + { + "process_type": "unstructured", + "config": {**config, "preprocess_options": ["clean_invalid_content"]}, + }, + [source], + ) + original_items = data_process_endpoint._build_preview_items( + { + "process_type": "unstructured", + "config": {**config, "preprocess_options": []}, + }, + [source], + ) + + assert len(cleaned_items) == 1 + cleaned = cleaned_items[0] + assert cleaned["original_content"] == source_text[ + cleaned["source_start"] : cleaned["source_end"] + ] + assert "重复页眉" not in cleaned["edited_content"] + assert "PDF 正文内容" in cleaned["edited_content"] + assert cleaned["status"] == "modified" + assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"] + assert "重复页眉" in original_items[0]["edited_content"] + + def test_merge_short_content_applies_across_adjacent_structure_sections() -> None: content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9)) items = _preview_task(