feat(data_process): 模型缓存统一仓库根 .cache 并修复 PDF 页眉页脚清理
- 新增 app/core/cache_paths.py:HF_HOME / tiktoken 缓存统一指向 <repo>/.cache, 本地与 Docker 路径一致,离线部署打包 .cache 即可 - Dockerfile.backend 的 tiktoken 词表改用官方 SHA 文件名,避免运行时回退重建 - 修复 layout_hybrid 路径不运行 detect_pdf_document_noise 的缺陷: needs_pdf_noise 不再与 needs_layout_raw 互斥,PDF 智能预处理在版面切分下也生效 - 新增 layout_noise.py:识别跨页重复的页眉表格标签组并按行剔除, 解决 docling layout 模型把中文企业 PDF 页眉识别成普通 Table 导致清不掉的问题 - 回收 HybridChunker 丢弃的末尾孤立标题,找回章节标题内容
This commit is contained in:
@@ -18,10 +18,12 @@ from pypdf import PdfWriter
|
||||
|
||||
from app.modules.data_process.algorithms import (
|
||||
PdfPageText,
|
||||
LayoutRepeatedBlock,
|
||||
content_quality_flags,
|
||||
desensitize_pii,
|
||||
desensitize_structured_record,
|
||||
detect_document_structure,
|
||||
detect_layout_repeated_blocks,
|
||||
detect_pdf_document_noise,
|
||||
detect_text_format,
|
||||
extract_pdf_page_texts,
|
||||
@@ -35,6 +37,7 @@ from app.modules.data_process.algorithms import (
|
||||
preprocess_structured_records_with_lineage,
|
||||
record_fingerprint,
|
||||
remove_document_noise,
|
||||
remove_layout_repeated_blocks,
|
||||
score_quality,
|
||||
stable_split,
|
||||
stable_split_assignments,
|
||||
@@ -1143,3 +1146,95 @@ def test_generate_standard_records_rejects_out_of_range_count(
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match=r"\[1, 50\]"):
|
||||
generate_standard_records([], qa_pairs_per_item=qa_pairs_per_item)
|
||||
|
||||
|
||||
def test_layout_repeated_blocks_detects_repeating_header_table() -> None:
|
||||
"""跨页重复的页眉表格应被识别为重复块(出现 ≥ max(3, ceil(pages*0.3)) 次)。"""
|
||||
|
||||
header_table = (
|
||||
"| 文件编码 | 2024 |\n"
|
||||
"| - | - |\n"
|
||||
"| 秘密等级 | 商密【中】 |\n"
|
||||
"| 现行版本 | 1.0 |\n"
|
||||
"| 页次 | 第1页 共47页 |\n"
|
||||
)
|
||||
body_table = (
|
||||
"| 支出项目 | 税务票据要求 |\n"
|
||||
"| - | - |\n"
|
||||
"| 工资奖金 | 无 |\n"
|
||||
"| 交通费 | 车票 |\n"
|
||||
)
|
||||
doc_items: list[tuple[str, object, str]] = []
|
||||
for index in range(20):
|
||||
# 20 个页面里 18 个有页眉表,2 个有正文表
|
||||
text = header_table if index < 18 else body_table
|
||||
doc_items.append(("table", index, text))
|
||||
|
||||
blocks = detect_layout_repeated_blocks(doc_items, page_count=20)
|
||||
|
||||
# 仅页眉表对应的标签序列应被识别
|
||||
assert len(blocks) == 1
|
||||
assert "文件编码" in blocks[0].labels
|
||||
assert "秘密等级" in blocks[0].labels
|
||||
assert blocks[0].occurrences == 18
|
||||
|
||||
|
||||
def test_layout_repeated_blocks_short_documents_skip() -> None:
|
||||
"""短文档(< 3 页)不推断重复块。"""
|
||||
|
||||
doc_items: list[tuple[str, object, str]] = [
|
||||
("table", 0, "| 文件编码 | 2024 |\n| - | - |\n"),
|
||||
("table", 1, "| 文件编码 | 2024 |\n| - | - |\n"),
|
||||
]
|
||||
assert detect_layout_repeated_blocks(doc_items, page_count=2) == ()
|
||||
|
||||
|
||||
def test_remove_layout_repeated_blocks_strips_label_rows_and_separators() -> None:
|
||||
"""剔除首列命中重复标签集的行,及其后的表格分隔行。"""
|
||||
|
||||
blocks = [
|
||||
LayoutRepeatedBlock(
|
||||
labels=("文件编码", "秘密等级", "现行版本", "页次"),
|
||||
occurrences=18,
|
||||
),
|
||||
]
|
||||
chunk = (
|
||||
"报销指引\n"
|
||||
"| 文件编码 | 2024 |\n"
|
||||
"| - | - |\n"
|
||||
"| 秘密等级 | 商密【中】 |\n"
|
||||
"| 现行版本 | 1.0 |\n"
|
||||
"| 页次 | 第3页 共47页 |\n"
|
||||
"正文第一段\n"
|
||||
"| 支出项目 | 税务票据要求 |\n"
|
||||
"| - | - |\n"
|
||||
"| 工资奖金 | 无 |\n"
|
||||
)
|
||||
cleaned = remove_layout_repeated_blocks(chunk, blocks)
|
||||
|
||||
# 重复标签行 + 紧随其后的表格分隔行被剔除;正文与内容表格保留
|
||||
assert "文件编码" not in cleaned
|
||||
assert "秘密等级" not in cleaned
|
||||
assert "现行版本" not in cleaned
|
||||
assert "页次" not in cleaned
|
||||
# 第一组表格的 | - | - | 在 文件编码 行之后被一并删除
|
||||
# (但 cleaned 中可能还有第二个表格的分隔行)
|
||||
assert cleaned.count("| - | - |") == 1
|
||||
assert "报销指引" in cleaned
|
||||
assert "正文第一段" in cleaned
|
||||
assert "支出项目" in cleaned
|
||||
assert "工资奖金" in cleaned
|
||||
|
||||
|
||||
def test_remove_layout_repeated_blocks_returns_text_unchanged_when_no_blocks() -> None:
|
||||
"""无重复块时直接返回原文。"""
|
||||
|
||||
chunk = "| 文件编码 | 2024 |\n| 正文 |\n"
|
||||
assert remove_layout_repeated_blocks(chunk, []) == chunk
|
||||
assert (
|
||||
remove_layout_repeated_blocks(
|
||||
"",
|
||||
[LayoutRepeatedBlock(labels=("x",), occurrences=5)],
|
||||
)
|
||||
== ""
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user