feat(data_process): 显式关闭 docling OCR 并拒绝无文本层 PDF

- docling 转换器通过 PdfPipelineOptions 显式设置 do_ocr=False,
  混合 PDF 的图片页不再产出 OCR 文字;不提供重新开启 OCR 的参数。
- 无文本层 PDF 的错误文案改为"扫描版或图片型 PDF 不支持",
  上传阶段整批拒绝,保留混合 PDF 的可处理判定。
- 新增 test_layout_converter_disables_ocr 守护开关状态,
  同步设计文档与 disable-ocr 实施计划/设计说明。
This commit is contained in:
caoxiaozhu
2026-08-18 15:48:30 +08:00
parent 2f48934e66
commit 91b4ae2287
7 changed files with 337 additions and 5 deletions

View File

@@ -1,10 +1,14 @@
from __future__ import annotations
import sys
from types import ModuleType, SimpleNamespace
from llama_index.core.embeddings import MockEmbedding
from app.modules.data_process.document_chunking import (
DocumentChunk,
_compact_with_offsets,
_document_converter,
_project_layout_span,
chunk_fixed_text,
chunk_semantic_text,
@@ -42,6 +46,48 @@ def test_semantic_splitter_uses_llamaindex_and_reapplies_maximum_size() -> None:
assert "".join(chunk.original_content for chunk in chunks) == text
def test_layout_converter_disables_ocr(monkeypatch) -> None:
class FakePipelineOptions:
def __init__(self) -> None:
self.do_ocr = True
class FakePdfFormatOption:
def __init__(self, *, pipeline_options) -> None:
self.pipeline_options = pipeline_options
class FakeDocumentConverter:
def __init__(self, *, format_options) -> None:
self.format_options = format_options
docling_module = ModuleType("docling")
docling_module.__path__ = []
document_converter_module = ModuleType("docling.document_converter")
document_converter_module.DocumentConverter = FakeDocumentConverter
document_converter_module.PdfFormatOption = FakePdfFormatOption
datamodel_module = ModuleType("docling.datamodel")
datamodel_module.__path__ = []
base_models_module = ModuleType("docling.datamodel.base_models")
base_models_module.InputFormat = SimpleNamespace(PDF="pdf")
pipeline_options_module = ModuleType("docling.datamodel.pipeline_options")
pipeline_options_module.PdfPipelineOptions = FakePipelineOptions
for name, module in {
"docling": docling_module,
"docling.document_converter": document_converter_module,
"docling.datamodel": datamodel_module,
"docling.datamodel.base_models": base_models_module,
"docling.datamodel.pipeline_options": pipeline_options_module,
}.items():
monkeypatch.setitem(sys.modules, name, module)
_document_converter.cache_clear()
try:
converter = _document_converter()
options = converter.format_options["pdf"].pipeline_options
assert options.do_ocr is False
finally:
_document_converter.cache_clear()
def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() -> None:
source = "标题\n第一条 这是正文。\n第二条 后续正文。"
compact_source, offsets = _compact_with_offsets(source)