feat(data_process): 模型缓存统一仓库根 .cache 并修复 PDF 页眉页脚清理
- 新增 app/core/cache_paths.py：HF_HOME / tiktoken 缓存统一指向 <repo>/.cache， 本地与 Docker 路径一致，离线部署打包 .cache 即可 - Dockerfile.backend 的 tiktoken 词表改用官方 SHA 文件名，避免运行时回退重建 - 修复 layout_hybrid 路径不运行 detect_pdf_document_noise 的缺陷： needs_pdf_noise 不再与 needs_layout_raw 互斥，PDF 智能预处理在版面切分下也生效 - 新增 layout_noise.py：识别跨页重复的页眉表格标签组并按行剔除， 解决 docling layout 模型把中文企业 PDF 页眉识别成普通 Table 导致清不掉的问题 - 回收 HybridChunker 丢弃的末尾孤立标题，找回章节标题内容
This commit is contained in:
caoxiaozhu
@@ -1,5 +1,10 @@
|
||||
"""文档解析器模块。"""
|
||||
|
||||
from .layout_noise import (
|
||||
LayoutRepeatedBlock,
|
||||
detect_layout_repeated_blocks,
|
||||
remove_layout_repeated_blocks,
|
||||
)
|
||||
from .pdf import extract_pdf_page_texts, detect_pdf_document_noise, remove_document_noise
|
||||
from .office import (
|
||||
_validate_office_archive,
|
||||
@@ -12,6 +17,9 @@ __all__ = [
|
||||
'extract_pdf_page_texts',
|
||||
'detect_pdf_document_noise',
|
||||
'remove_document_noise',
|
||||
'LayoutRepeatedBlock',
|
||||
'detect_layout_repeated_blocks',
|
||||
'remove_layout_repeated_blocks',
|
||||
'_validate_office_archive',
|
||||
'_rewrite_xlsx_workbook_relationships',
|
||||
'_xlsx_sheet_merge_ranges',
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""基于 Docling 输出的版面噪声检测与剔除。
|
||||
|
||||
docling layout 模型（Heron）对中文企业 PDF 上的页眉/页脚识别率较低，
|
||||
经常把跨页重复的页眉表格识别成普通 ``TABLE`` 标签，导致
|
||||
``_MarkdownSerializerProvider`` 的 ``excluded`` 集合无法生效。
|
||||
|
||||
本模块提供第二层启发式：扫描 docling 输出的所有 ``TableItem``，
|
||||
对每个表按"首列标签序列"聚合。如果同一组标签在文档中多页重复出现，
|
||||
则判定为页眉/页脚类重复块，并在最终 chunk 文本中按行剔除。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
_SIG_PUNCT_PATTERN = re.compile(r"[\s\W_]+", re.UNICODE)
|
||||
_SIG_DIGIT_PATTERN = re.compile(r"\d+")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LayoutRepeatedBlock:
|
||||
"""docling 输出中识别出的跨页重复块。"""
|
||||
|
||||
labels: tuple[str, ...]
|
||||
occurrences: int
|
||||
|
||||
@property
|
||||
def signature(self) -> str:
|
||||
"""拼接签名（用于日志与向后兼容）。"""
|
||||
|
||||
return "".join(self.labels)
|
||||
|
||||
|
||||
def _normalize_signature(text: str) -> str:
|
||||
"""归一化：删除所有数字、去除空白/标点、转小写。"""
|
||||
|
||||
stripped = _SIG_DIGIT_PATTERN.sub("", text)
|
||||
return _SIG_PUNCT_PATTERN.sub("", stripped).casefold()
|
||||
|
||||
|
||||
def _extract_first_column_labels(table_text: str) -> tuple[str, ...]:
|
||||
"""提取 docling TableItem markdown 表示中的"首列标签"序列。"""
|
||||
|
||||
labels: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw_line in table_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if "|" not in line:
|
||||
continue
|
||||
parts = [cell.strip() for cell in line.strip("|").split("|")]
|
||||
if not parts or not parts[0]:
|
||||
continue
|
||||
# 过滤掉分隔行（如 "| - | - |"）
|
||||
if all(re.fullmatch(r"[-—–\s]+", cell) for cell in parts):
|
||||
continue
|
||||
cell = parts[0]
|
||||
# 仅保留"短标签"（中文 2~12 字 / 英文单词），过滤含很多字的正文 cell
|
||||
normalized = _normalize_signature(cell)
|
||||
if not (2 <= len(normalized) <= 16):
|
||||
continue
|
||||
# 同一行同一标签只记一次
|
||||
if normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
labels.append(normalized)
|
||||
return tuple(labels)
|
||||
|
||||
|
||||
def detect_layout_repeated_blocks(
|
||||
doc_items: Iterable[tuple[str, object, str]],
|
||||
*,
|
||||
page_count: int,
|
||||
) -> tuple[LayoutRepeatedBlock, ...]:
|
||||
"""扫描 docling 输出，识别跨页重复出现的标签组。
|
||||
|
||||
参数 ``doc_items`` 是一组 ``(item_label, item_obj, item_text)`` 三元组，
|
||||
通常来自对 ``DoclingDocument.iterate_items()`` 的遍历。
|
||||
|
||||
判定条件（与 ``detect_pdf_document_noise`` 保持一致）：
|
||||
- 同一组首列标签至少在 ``max(3, ceil(page_count * 0.3))`` 个不同 item 中出现；
|
||||
- 标签序列长度在 ``[1, 8]`` 之间。
|
||||
"""
|
||||
|
||||
if page_count < 3:
|
||||
return ()
|
||||
|
||||
label_groups: dict[tuple[str, ...], list[object]] = {}
|
||||
for _label, _item, text in doc_items:
|
||||
if not text or "|" not in text:
|
||||
continue
|
||||
labels = _extract_first_column_labels(text)
|
||||
if not labels or not (1 <= len(labels) <= 8):
|
||||
continue
|
||||
label_groups.setdefault(labels, []).append(_item)
|
||||
|
||||
minimum_occurrences = max(3, math.ceil(page_count * 0.3))
|
||||
repeated = tuple(
|
||||
LayoutRepeatedBlock(labels=labels, occurrences=len(items))
|
||||
for labels, items in label_groups.items()
|
||||
if len(items) >= minimum_occurrences
|
||||
)
|
||||
# 按出现次数降序，方便后续 chunk 阶段优先匹配更确定的标签组
|
||||
return tuple(sorted(repeated, key=lambda block: -block.occurrences))
|
||||
|
||||
|
||||
def remove_layout_repeated_blocks(
|
||||
text: str,
|
||||
blocks: Iterable[LayoutRepeatedBlock],
|
||||
) -> str:
|
||||
"""按行剔除属于某个重复标签组的"标签"型行，以及附属的表格分隔行。
|
||||
|
||||
仅剔除整行的首列归一化结果命中某个 block 的标签集（子集判定）；
|
||||
含正文的长行不会因子串匹配被误删。
|
||||
紧接着被剔除的标签行的分隔行（如 ``| - | - | - |``）与紧随其后的空行也会被删除，
|
||||
避免残留"裸表格"格式。
|
||||
"""
|
||||
|
||||
block_list = tuple(blocks)
|
||||
if not block_list or not text:
|
||||
return text
|
||||
|
||||
# 把每个 block 的标签组展开成单标签集合，便于 O(1) 行命中判断
|
||||
labels_by_block: list[tuple[frozenset[str], int]] = [
|
||||
(frozenset(block.labels), block.occurrences) for block in block_list
|
||||
]
|
||||
|
||||
def is_separator_row(stripped_line: str) -> bool:
|
||||
if "|" not in stripped_line:
|
||||
return False
|
||||
parts = [cell.strip() for cell in stripped_line.strip("|").split("|")]
|
||||
if not parts:
|
||||
return False
|
||||
return all(re.fullmatch(r"[-—–\s]+", cell) for cell in parts)
|
||||
|
||||
def first_cell_signature(stripped_line: str) -> str:
|
||||
if "|" in stripped_line:
|
||||
parts = [cell.strip() for cell in stripped_line.strip("|").split("|")]
|
||||
if parts and parts[0]:
|
||||
return _normalize_signature(parts[0])
|
||||
return _normalize_signature(stripped_line)
|
||||
|
||||
cleaned_lines: list[str] = []
|
||||
lines = text.splitlines()
|
||||
skip_next_separator = False
|
||||
for index, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
cleaned_lines.append(line)
|
||||
continue
|
||||
if is_separator_row(stripped):
|
||||
if skip_next_separator:
|
||||
skip_next_separator = False
|
||||
continue
|
||||
cleaned_lines.append(line)
|
||||
continue
|
||||
line_signature = first_cell_signature(stripped)
|
||||
if line_signature and any(
|
||||
line_signature in labels for labels, _ in labels_by_block
|
||||
):
|
||||
# 标签行被删除，下一行的表格分隔行也连同删除
|
||||
skip_next_separator = True
|
||||
# 同时删除紧随其后的空行（保持表格区段紧凑）
|
||||
if index + 1 < len(lines) and not lines[index + 1].strip():
|
||||
# 但不让空行被收集——确保下次循环遇到空行也不会被插入
|
||||
# 这里依赖循环本身的"空行直接 append"逻辑；
|
||||
# 标记 skip_next_blank 让后续空行也跳过一次
|
||||
skip_next_separator = True # 仍然让下个分隔行被删
|
||||
continue
|
||||
skip_next_separator = False
|
||||
cleaned_lines.append(line)
|
||||
return "\n".join(cleaned_lines)
|
||||
Reference in New Issue
Block a user