feat(data_process): 模型缓存统一仓库根 .cache 并修复 PDF 页眉页脚清理
- 新增 app/core/cache_paths.py：HF_HOME / tiktoken 缓存统一指向 <repo>/.cache， 本地与 Docker 路径一致，离线部署打包 .cache 即可 - Dockerfile.backend 的 tiktoken 词表改用官方 SHA 文件名，避免运行时回退重建 - 修复 layout_hybrid 路径不运行 detect_pdf_document_noise 的缺陷： needs_pdf_noise 不再与 needs_layout_raw 互斥，PDF 智能预处理在版面切分下也生效 - 新增 layout_noise.py：识别跨页重复的页眉表格标签组并按行剔除， 解决 docling layout 模型把中文企业 PDF 页眉识别成普通 Table 导致清不掉的问题 - 回收 HybridChunker 丢弃的末尾孤立标题，找回章节标题内容
This commit is contained in:
caoxiaozhu
@@ -2,9 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
@@ -34,6 +35,8 @@ _LIST_MARKER_PREFIX = re.compile(
|
||||
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
|
||||
_CONVERTER_LOCK = threading.Lock()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentChunk:
|
||||
@@ -65,25 +68,24 @@ def _sentence_chunks(text: str) -> list[str]:
|
||||
@lru_cache(maxsize=1)
|
||||
def _tokenizer() -> tiktoken.Encoding:
|
||||
"""加载 cl100k_base 编码器，优先在线下载，失败时使用本地缓存以支持离线环境。"""
|
||||
import os
|
||||
import base64
|
||||
|
||||
# 先设置缓存目录环境变量
|
||||
offline_cache = os.path.expanduser("~/.cache/tiktoken")
|
||||
os.environ.setdefault("TIKTOKEN_CACHE_DIR", offline_cache)
|
||||
from app.core.cache_paths import tiktoken_cache_dir
|
||||
|
||||
# 缓存目录已在 app.core.cache_paths.setup_local_caches 中统一指向 <repo>/.cache/tiktoken，
|
||||
# 此处直接读取；TIKTOKEN_CACHE_DIR 已在启动阶段写入。
|
||||
offline_cache = tiktoken_cache_dir()
|
||||
|
||||
try:
|
||||
# 尝试标准方式加载
|
||||
# 尝试标准方式加载（环境变量 TIKTOKEN_CACHE_DIR 已被统一设置）
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
except Exception:
|
||||
# 如果失败，尝试手动从本地文件构造
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
local_file = Path(offline_cache) / "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
|
||||
local_file = offline_cache / "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
|
||||
if not local_file.exists():
|
||||
# 尝试另一个可能的文件名
|
||||
local_file = Path(offline_cache) / "cl100k_base.tiktoken"
|
||||
local_file = offline_cache / "cl100k_base.tiktoken"
|
||||
|
||||
if local_file.exists():
|
||||
# 读取 BPE 文件内容
|
||||
@@ -106,7 +108,7 @@ def _tokenizer() -> tiktoken.Encoding:
|
||||
pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
|
||||
mergeable_ranks=mergeable_ranks,
|
||||
special_tokens={
|
||||
"<|endoftext|>": 100257,
|
||||
"": 100257,
|
||||
"<|fim_prefix|>": 100258,
|
||||
"<|fim_middle|>": 100259,
|
||||
"<|fim_suffix|>": 100260,
|
||||
@@ -434,6 +436,12 @@ def chunk_layout_document(
|
||||
from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
|
||||
from docling_core.types.doc import DocItemLabel
|
||||
|
||||
from app.modules.data_process.algorithms import (
|
||||
detect_layout_repeated_blocks,
|
||||
remove_layout_repeated_blocks,
|
||||
)
|
||||
|
||||
convert_started = time.perf_counter()
|
||||
try:
|
||||
with _CONVERTER_LOCK:
|
||||
conversion = _document_converter().convert(
|
||||
@@ -441,6 +449,35 @@ def chunk_layout_document(
|
||||
)
|
||||
except DoclingError as exc:
|
||||
raise ValueError(f"文档版面解析失败: {exc}") from exc
|
||||
logger.info(
|
||||
"layout chunking convert done file=%s elapsed=%.2fs",
|
||||
filename,
|
||||
time.perf_counter() - convert_started,
|
||||
)
|
||||
|
||||
# 第二层启发式：扫描所有 docling item，识别跨页重复出现的短文本块
|
||||
# （docling layout 模型在中文企业 PDF 上把页眉页脚识别成普通 Table，
|
||||
# 因此 _MarkdownSerializerProvider 的标签排除规则收效甚微）。
|
||||
page_count = len(getattr(conversion.document, "pages", {}) or {})
|
||||
layout_items: list[tuple[str, object, str]] = []
|
||||
for item, _level in conversion.document.iterate_items():
|
||||
text = getattr(item, "text", None)
|
||||
if not text and hasattr(item, "export_to_markdown"):
|
||||
try:
|
||||
text = item.export_to_markdown(doc=conversion.document) or ""
|
||||
except TypeError:
|
||||
# 旧版 docling_core 无 doc 参数
|
||||
text = item.export_to_markdown() or ""
|
||||
except Exception:
|
||||
text = ""
|
||||
label = getattr(item, "label", None)
|
||||
label_value = getattr(label, "value", str(label)) if label else ""
|
||||
if text:
|
||||
layout_items.append((label_value, item, text))
|
||||
repeated_blocks = detect_layout_repeated_blocks(
|
||||
layout_items, page_count=page_count
|
||||
)
|
||||
|
||||
chunker = HybridChunker(
|
||||
tokenizer=OpenAITokenizer(tokenizer=_tokenizer(), max_tokens=chunk_size),
|
||||
serializer_provider=_MarkdownSerializerProvider(),
|
||||
@@ -450,6 +487,7 @@ def chunk_layout_document(
|
||||
compact_source, source_offsets = _compact_with_offsets(source_text)
|
||||
compact_start = 0
|
||||
result: list[DocumentChunk] = []
|
||||
covered_refs: set[str] = set()
|
||||
excluded = {
|
||||
DocItemLabel.DOCUMENT_INDEX,
|
||||
DocItemLabel.PAGE_HEADER,
|
||||
@@ -463,6 +501,13 @@ def chunk_layout_document(
|
||||
if not content:
|
||||
continue
|
||||
contextualized = _clean_layout_text(chunker.contextualize(raw_chunk)) or content
|
||||
if repeated_blocks:
|
||||
content = remove_layout_repeated_blocks(content, repeated_blocks)
|
||||
contextualized = remove_layout_repeated_blocks(
|
||||
contextualized, repeated_blocks
|
||||
)
|
||||
if not content:
|
||||
continue
|
||||
start, end, compact_start = _project_layout_span(
|
||||
source_text,
|
||||
content,
|
||||
@@ -476,6 +521,7 @@ def chunk_layout_document(
|
||||
bboxes: list[dict[str, Any]] = []
|
||||
for item in doc_items:
|
||||
refs.append(str(item.self_ref))
|
||||
covered_refs.add(str(item.self_ref))
|
||||
for provenance in item.prov or ():
|
||||
pages.add(int(provenance.page_no))
|
||||
bbox = provenance.bbox
|
||||
@@ -508,6 +554,66 @@ def chunk_layout_document(
|
||||
source_bboxes=tuple(bboxes),
|
||||
)
|
||||
)
|
||||
|
||||
# HybridChunker(merge_peers=True) 会丢弃"末尾无正文的孤立标题"。
|
||||
# OCR 页常只产出一个 heading，内容会被整体吞掉，这里按文档序回收
|
||||
# 未被任何 chunk 覆盖的非排除 item，避免识别出的文字凭空消失。
|
||||
# 注意 heading 会进入 meta.headings 而非 doc_items，其文字已随
|
||||
# contextualize 出现在既有 chunk 里，因此用紧凑文本包含性二次确认，
|
||||
# 防止把正常标题重复回收。
|
||||
chunk_haystack = _compact_with_offsets(
|
||||
"\n".join(chunk.contextualized_content for chunk in result)
|
||||
)[0]
|
||||
uncovered_items = [
|
||||
item
|
||||
for item, _level in conversion.document.iterate_items()
|
||||
if item.label not in excluded
|
||||
and str(item.self_ref) not in covered_refs
|
||||
and (getattr(item, "text", None) or "").strip()
|
||||
and _compact_with_offsets(str(item.text))[0] not in chunk_haystack
|
||||
]
|
||||
for item in uncovered_items:
|
||||
recovered = _clean_layout_text(str(item.text))
|
||||
if not recovered:
|
||||
continue
|
||||
if repeated_blocks:
|
||||
recovered = remove_layout_repeated_blocks(recovered, repeated_blocks)
|
||||
if not recovered:
|
||||
continue
|
||||
pages = {
|
||||
int(provenance.page_no) for provenance in item.prov or ()
|
||||
}
|
||||
bboxes = [
|
||||
{
|
||||
"page": int(provenance.page_no),
|
||||
"left": float(provenance.bbox.l),
|
||||
"top": float(provenance.bbox.t),
|
||||
"right": float(provenance.bbox.r),
|
||||
"bottom": float(provenance.bbox.b),
|
||||
"origin": str(provenance.bbox.coord_origin.value),
|
||||
}
|
||||
for provenance in item.prov or ()
|
||||
]
|
||||
logger.info(
|
||||
"layout chunking recovered uncovered doc item file=%s ref=%s",
|
||||
filename,
|
||||
item.self_ref,
|
||||
)
|
||||
result.append(
|
||||
DocumentChunk(
|
||||
original_content=recovered,
|
||||
contextualized_content=recovered,
|
||||
source_start=None,
|
||||
source_end=None,
|
||||
source_start_line=None,
|
||||
source_end_line=None,
|
||||
token_count=len(_tokenizer().encode(recovered)),
|
||||
heading_path=(),
|
||||
source_pages=tuple(sorted(pages)),
|
||||
doc_item_refs=(str(item.self_ref),),
|
||||
source_bboxes=tuple(bboxes),
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user