feat(data_process): 模型缓存统一仓库根 .cache 并修复 PDF 页眉页脚清理

- 新增 app/core/cache_paths.py:HF_HOME / tiktoken 缓存统一指向 <repo>/.cache,
  本地与 Docker 路径一致,离线部署打包 .cache 即可
- Dockerfile.backend 的 tiktoken 词表改用官方 SHA 文件名,避免运行时回退重建
- 修复 layout_hybrid 路径不运行 detect_pdf_document_noise 的缺陷:
  needs_pdf_noise 不再与 needs_layout_raw 互斥,PDF 智能预处理在版面切分下也生效
- 新增 layout_noise.py:识别跨页重复的页眉表格标签组并按行剔除,
  解决 docling layout 模型把中文企业 PDF 页眉识别成普通 Table 导致清不掉的问题
- 回收 HybridChunker 丢弃的末尾孤立标题,找回章节标题内容
This commit is contained in:
caoxiaozhu
2026-08-21 10:15:08 +08:00
parent 3b9361c237
commit 03254f8196
14 changed files with 479 additions and 32 deletions

View File

@@ -18,10 +18,12 @@ from pypdf import PdfWriter
from app.modules.data_process.algorithms import (
PdfPageText,
LayoutRepeatedBlock,
content_quality_flags,
desensitize_pii,
desensitize_structured_record,
detect_document_structure,
detect_layout_repeated_blocks,
detect_pdf_document_noise,
detect_text_format,
extract_pdf_page_texts,
@@ -35,6 +37,7 @@ from app.modules.data_process.algorithms import (
preprocess_structured_records_with_lineage,
record_fingerprint,
remove_document_noise,
remove_layout_repeated_blocks,
score_quality,
stable_split,
stable_split_assignments,
@@ -1143,3 +1146,95 @@ def test_generate_standard_records_rejects_out_of_range_count(
) -> None:
with pytest.raises(ValueError, match=r"\[1, 50\]"):
generate_standard_records([], qa_pairs_per_item=qa_pairs_per_item)
def test_layout_repeated_blocks_detects_repeating_header_table() -> None:
"""跨页重复的页眉表格应被识别为重复块(出现 ≥ max(3, ceil(pages*0.3)) 次)。"""
header_table = (
"| 文件编码 | 2024 |\n"
"| - | - |\n"
"| 秘密等级 | 商密【中】 |\n"
"| 现行版本 | 1.0 |\n"
"| 页次 | 第1页 共47页 |\n"
)
body_table = (
"| 支出项目 | 税务票据要求 |\n"
"| - | - |\n"
"| 工资奖金 | 无 |\n"
"| 交通费 | 车票 |\n"
)
doc_items: list[tuple[str, object, str]] = []
for index in range(20):
# 20 个页面里 18 个有页眉表2 个有正文表
text = header_table if index < 18 else body_table
doc_items.append(("table", index, text))
blocks = detect_layout_repeated_blocks(doc_items, page_count=20)
# 仅页眉表对应的标签序列应被识别
assert len(blocks) == 1
assert "文件编码" in blocks[0].labels
assert "秘密等级" in blocks[0].labels
assert blocks[0].occurrences == 18
def test_layout_repeated_blocks_short_documents_skip() -> None:
"""短文档(< 3 页)不推断重复块。"""
doc_items: list[tuple[str, object, str]] = [
("table", 0, "| 文件编码 | 2024 |\n| - | - |\n"),
("table", 1, "| 文件编码 | 2024 |\n| - | - |\n"),
]
assert detect_layout_repeated_blocks(doc_items, page_count=2) == ()
def test_remove_layout_repeated_blocks_strips_label_rows_and_separators() -> None:
"""剔除首列命中重复标签集的行,及其后的表格分隔行。"""
blocks = [
LayoutRepeatedBlock(
labels=("文件编码", "秘密等级", "现行版本", "页次"),
occurrences=18,
),
]
chunk = (
"报销指引\n"
"| 文件编码 | 2024 |\n"
"| - | - |\n"
"| 秘密等级 | 商密【中】 |\n"
"| 现行版本 | 1.0 |\n"
"| 页次 | 第3页 共47页 |\n"
"正文第一段\n"
"| 支出项目 | 税务票据要求 |\n"
"| - | - |\n"
"| 工资奖金 | 无 |\n"
)
cleaned = remove_layout_repeated_blocks(chunk, blocks)
# 重复标签行 + 紧随其后的表格分隔行被剔除;正文与内容表格保留
assert "文件编码" not in cleaned
assert "秘密等级" not in cleaned
assert "现行版本" not in cleaned
assert "页次" not in cleaned
# 第一组表格的 | - | - | 在 文件编码 行之后被一并删除
# (但 cleaned 中可能还有第二个表格的分隔行)
assert cleaned.count("| - | - |") == 1
assert "报销指引" in cleaned
assert "正文第一段" in cleaned
assert "支出项目" in cleaned
assert "工资奖金" in cleaned
def test_remove_layout_repeated_blocks_returns_text_unchanged_when_no_blocks() -> None:
"""无重复块时直接返回原文。"""
chunk = "| 文件编码 | 2024 |\n| 正文 |\n"
assert remove_layout_repeated_blocks(chunk, []) == chunk
assert (
remove_layout_repeated_blocks(
"",
[LayoutRepeatedBlock(labels=("x",), occurrences=5)],
)
== ""
)