YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 清理PDF文档级噪声

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-24 15:05:39 +08:00
parent a9b06140d0
commit 3266a6fc09
4 changed files with 561 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
backend/tests/test_data_process_algorithms.py
View File

@@ -14,11 +14,13 @@ from pptx.util import Inches
from pypdf import PdfWriter
from app.modules.data_process.algorithms import (
PdfPageText,
chunk_unstructured,
content_quality_flags,
desensitize_pii,
desensitize_structured_record,
detect_document_structure,
detect_pdf_document_noise,
detect_text_format,
estimate_token_count,
extract_pdf_page_texts,
@@ -30,11 +32,32 @@ from app.modules.data_process.algorithms import (
parse_text_content,
preprocess_structured_records,
record_fingerprint,
remove_document_noise,
score_quality,
stable_split,
)
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
pages: list[PdfPageText] = []
offset = 0
for page_number, text in enumerate(texts, start=1):
normalized = normalize_text(text)
if pages:
offset += 2
start = offset
offset += len(normalized)
pages.append(
PdfPageText(
page_number=page_number,
text=normalized,
source_start=start,
source_end=offset,
)
)
return tuple(pages)
def _minimal_pdf(text: str = "Hello PDF") -> bytes:
stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii")
objects = [
@@ -206,6 +229,91 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
pages = _pdf_page_texts(
"""
远光制度文件 文件编码 2024
秘密等级 商密【中】
第 1 页 共 5 页
正文第一页，关于适用范围的说明。
业务提示保留
第一页补充说明甲
第一页补充说明乙
第一页补充说明丙
""",
"""
远光制度文件 文件编码 2024
秘密等级 商密【中】
第 2 页 共 5 页
目 录
第一章 总则........3
第二章 报销申请........4
第三章 附则........5
""",
"""
远光制度文件 文件编码 2024
秘密等级 商密【中】
第 3 页 共 5 页
1.1 管理要求........6
1.2 审批职责 7
1.3 费用标准........8
1.4 例外处理........9
""",
"""
远光制度文件 文件编码 2024
秘密等级 商密【中】
第 4 页 共 5 页
正文中可以说“请参见第 3 页说明”，不应误删。
第 99 页 共 100 页
系统可用率........99.9%
业务提示保留
第四页补充说明甲
第四页补充说明乙
第四页补充说明丙
""",
"""
远光制度文件 文件编码 2024
秘密等级 商密【中】
第 5 页 共 5 页
本办法自发布之日起施行。
业务提示保留
第五页补充说明甲
第五页补充说明乙
第五页补充说明丙
""",
)
source = "\n\n".join(page.text for page in pages)
spans = detect_pdf_document_noise(pages)
cleaned = remove_document_noise(source, spans)
assert {span.kind for span in spans} == {
"page_number",
"repeated_margin",
"table_of_contents",
}
assert "远光制度文件" not in cleaned
assert "商密【中】" not in cleaned
assert "第 1 页 共 5 页" not in cleaned
assert "第一章 总则" not in cleaned
assert "1.2 审批职责 7" not in cleaned
assert "请参见第 3 页说明" in cleaned
assert "第 99 页 共 100 页" in cleaned
assert "系统可用率........99.9%" in cleaned
assert cleaned.count("业务提示保留") == 3
def test_pdf_document_noise_does_not_infer_repeated_margins_for_short_documents() -> None:
pages = _pdf_page_texts(
"公司内部文件\n正文 A",
"公司内部文件\n正文 B",
)
spans = detect_pdf_document_noise(pages)
assert not any(span.kind == "repeated_margin" for span in spans)
def test_xlsx_merged_multilevel_headers_are_flattened_without_losing_columns() -> None:
workbook = Workbook()
worksheet = workbook.active

151
backend/tests/test_data_process_api.py
View File

@@ -12,7 +12,7 @@ from openpyxl import Workbook
from app.api.v1.endpoints import data_process as data_process_endpoint
from app.api.v1.endpoints.data_process import router
from app.modules.data_process.algorithms import normalize_text
from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text
from app.modules.data_process.storage import (
DataProcessStorageError,
LocalDataProcessStorage,
@@ -443,20 +443,49 @@ def _stored_files(storage: LocalDataProcessStorage) -> list[Path]:
return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()]
def _minimal_pdf(text: str = "Hello PDF") -> bytes:
stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii")
def _minimal_pdf_pages(*page_texts: str) -> bytes:
if not page_texts:
raise ValueError("at least one PDF page is required")
font_object_number = 3 + len(page_texts) * 2
page_object_numbers = [3 + index * 2 for index in range(len(page_texts))]
objects = [
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
(
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>"
b"<< /Type /Pages /Kids ["
+ b" ".join(f"{number} 0 R".encode() for number in page_object_numbers)
+ b"] /Count "
+ str(len(page_texts)).encode()
+ b" >>"
),
b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n"
+ stream
+ b"\nendstream",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
]
for index, text in enumerate(page_texts):
content_object_number = page_object_numbers[index] + 1
commands = [b"BT /F1 12 Tf 72 720 Td"]
for line_index, line in enumerate(text.splitlines()):
escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
if line_index:
commands.append(b"0 -16 Td")
commands.append(f"({escaped}) Tj".encode("ascii"))
commands.append(b"ET")
stream = b" ".join(commands)
objects.extend(
[
(
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
b"/Resources << /Font << /F1 "
+ str(font_object_number).encode()
+ b" 0 R >> >> /Contents "
+ str(content_object_number).encode()
+ b" 0 R >>"
),
b"<< /Length "
+ str(len(stream)).encode()
+ b" >>\nstream\n"
+ stream
+ b"\nendstream",
]
)
objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
offsets = [0]
for object_number, value in enumerate(objects, start=1):
@@ -478,6 +507,10 @@ def _minimal_pdf(text: str = "Hello PDF") -> bytes:
return bytes(result)
def _minimal_pdf(text: str = "Hello PDF") -> bytes:
return _minimal_pdf_pages(text)
def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
created = client.post(
@@ -1127,6 +1160,59 @@ def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Pat
assert legacy_pages.status_code == 410
def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "PDF 文档噪声清理",
"process_type": "unstructured",
"config": {
"chunk_method": "fixed",
"chunk_size": 200,
"chunk_overlap": 0,
"min_chunk_size": 20,
"preprocess_options": ["clean_invalid_content"],
},
},
).json()["data"]["id"]
raw = _minimal_pdf_pages(
"ACME Internal Manual\nBody page one keeps this guidance and explanation.",
"ACME Internal Manual\nContents\n"
"Chapter One........3\nChapter Two........4\nAppendix........5",
"ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8",
"ACME Internal Manual\nBody page four keeps operational details and examples.",
"ACME Internal Manual\nBody page five keeps the final effective-date clause.",
)
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("manual.pdf", raw, "application/pdf")},
).json()["data"]["files"][0]
source_content = client.get(
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content"
).json()["data"]["content"]
built = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert built.status_code == 200
items = built.json()["data"]["items"]
assert items
edited = "\n".join(item["edited_content"] for item in items)
assert "ACME Internal Manual" not in edited
assert "Contents" not in edited
assert "Chapter One" not in edited
assert "1.2 Approval" not in edited
assert "Body page one" in edited
assert "Body page five" in edited
assert all(
item["original_content"]
== source_content[item["source_start"] : item["source_end"]]
for item in items
)
def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
@@ -1450,6 +1536,51 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None
assert "[EMAIL]" in masked["edited_content"]
def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None:
source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。")
source = {
"id": "pdf-source",
"name": "manual.pdf",
"file_format": "pdf",
"content": source_text,
"document_noise_spans": (
DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"),
),
}
config = {
"chunk_method": "fixed",
"chunk_size": 200,
"chunk_overlap": 0,
"min_chunk_size": 1,
}
cleaned_items = data_process_endpoint._build_preview_items(
{
"process_type": "unstructured",
"config": {**config, "preprocess_options": ["clean_invalid_content"]},
},
[source],
)
original_items = data_process_endpoint._build_preview_items(
{
"process_type": "unstructured",
"config": {**config, "preprocess_options": []},
},
[source],
)
assert len(cleaned_items) == 1
cleaned = cleaned_items[0]
assert cleaned["original_content"] == source_text[
cleaned["source_start"] : cleaned["source_end"]
]
assert "重复页眉" not in cleaned["edited_content"]
assert "PDF 正文内容" in cleaned["edited_content"]
assert cleaned["status"] == "modified"
assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"]
assert "重复页眉" in original_items[0]["edited_content"]
def test_merge_short_content_applies_across_adjacent_structure_sections() -> None:
content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
items = _preview_task(