feat(data_process): 模型缓存统一仓库根 .cache 并修复 PDF 页眉页脚清理
- 新增 app/core/cache_paths.py:HF_HOME / tiktoken 缓存统一指向 <repo>/.cache, 本地与 Docker 路径一致,离线部署打包 .cache 即可 - Dockerfile.backend 的 tiktoken 词表改用官方 SHA 文件名,避免运行时回退重建 - 修复 layout_hybrid 路径不运行 detect_pdf_document_noise 的缺陷: needs_pdf_noise 不再与 needs_layout_raw 互斥,PDF 智能预处理在版面切分下也生效 - 新增 layout_noise.py:识别跨页重复的页眉表格标签组并按行剔除, 解决 docling layout 模型把中文企业 PDF 页眉识别成普通 Table 导致清不掉的问题 - 回收 HybridChunker 丢弃的末尾孤立标题,找回章节标题内容
This commit is contained in:
@@ -1686,7 +1686,6 @@ def _prepare_preview_items(
|
||||
)
|
||||
needs_pdf_noise = (
|
||||
is_unstructured
|
||||
and not needs_layout_raw
|
||||
and source_format == "pdf"
|
||||
and bool(
|
||||
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||
@@ -1720,17 +1719,18 @@ def _prepare_preview_items(
|
||||
enriched = dict(source)
|
||||
if needs_structured_xlsx or needs_layout_raw:
|
||||
enriched["raw_content"] = raw
|
||||
sources[index] = enriched
|
||||
continue
|
||||
pages = extract_pdf_page_texts(raw)
|
||||
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
||||
if extracted_text != str(source.get("content") or ""):
|
||||
logger.warning(
|
||||
"跳过PDF文档噪声检测(存储偏移量不一致) source_id=%s",
|
||||
source["id"],
|
||||
)
|
||||
continue
|
||||
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
||||
if needs_pdf_noise:
|
||||
# layout_hybrid 路径下也跑 PDF 文本规则噪声检测,
|
||||
# 弥补 docling layout 模型对中文 PDF 页眉/页脚识别率低的问题。
|
||||
pages = extract_pdf_page_texts(raw)
|
||||
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
||||
if extracted_text != str(source.get("content") or ""):
|
||||
logger.warning(
|
||||
"跳过PDF文档噪声检测(存储偏移量不一致) source_id=%s",
|
||||
source["id"],
|
||||
)
|
||||
else:
|
||||
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
||||
sources[index] = enriched
|
||||
items = _build_preview_items(task, sources)
|
||||
if not items and source_file_ids is None and is_unstructured:
|
||||
|
||||
Reference in New Issue
Block a user