feat(data_process): 显式关闭 docling OCR 并拒绝无文本层 PDF
- docling 转换器通过 PdfPipelineOptions 显式设置 do_ocr=False, 混合 PDF 的图片页不再产出 OCR 文字;不提供重新开启 OCR 的参数。 - 无文本层 PDF 的错误文案改为"扫描版或图片型 PDF 不支持", 上传阶段整批拒绝,保留混合 PDF 的可处理判定。 - 新增 test_layout_converter_disables_ocr 守护开关状态, 同步设计文档与 disable-ocr 实施计划/设计说明。
This commit is contained in:
@@ -654,7 +654,10 @@ def test_office_zip_bomb_and_invalid_pdf_are_rejected_before_parsing() -> None:
|
||||
blank_writer = PdfWriter()
|
||||
blank_writer.add_blank_page(width=612, height=792)
|
||||
blank_writer.write(blank_pdf)
|
||||
with pytest.raises(ValueError, match="scanned PDF requires OCR"):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="scanned or image-only PDF files are not supported",
|
||||
):
|
||||
parse_text_content(blank_pdf.getvalue(), filename="scanned.pdf")
|
||||
|
||||
aes_pdf_without_open_password = parse_text_content(
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
from llama_index.core.embeddings import MockEmbedding
|
||||
|
||||
from app.modules.data_process.document_chunking import (
|
||||
DocumentChunk,
|
||||
_compact_with_offsets,
|
||||
_document_converter,
|
||||
_project_layout_span,
|
||||
chunk_fixed_text,
|
||||
chunk_semantic_text,
|
||||
@@ -42,6 +46,48 @@ def test_semantic_splitter_uses_llamaindex_and_reapplies_maximum_size() -> None:
|
||||
assert "".join(chunk.original_content for chunk in chunks) == text
|
||||
|
||||
|
||||
def test_layout_converter_disables_ocr(monkeypatch) -> None:
|
||||
class FakePipelineOptions:
|
||||
def __init__(self) -> None:
|
||||
self.do_ocr = True
|
||||
|
||||
class FakePdfFormatOption:
|
||||
def __init__(self, *, pipeline_options) -> None:
|
||||
self.pipeline_options = pipeline_options
|
||||
|
||||
class FakeDocumentConverter:
|
||||
def __init__(self, *, format_options) -> None:
|
||||
self.format_options = format_options
|
||||
|
||||
docling_module = ModuleType("docling")
|
||||
docling_module.__path__ = []
|
||||
document_converter_module = ModuleType("docling.document_converter")
|
||||
document_converter_module.DocumentConverter = FakeDocumentConverter
|
||||
document_converter_module.PdfFormatOption = FakePdfFormatOption
|
||||
datamodel_module = ModuleType("docling.datamodel")
|
||||
datamodel_module.__path__ = []
|
||||
base_models_module = ModuleType("docling.datamodel.base_models")
|
||||
base_models_module.InputFormat = SimpleNamespace(PDF="pdf")
|
||||
pipeline_options_module = ModuleType("docling.datamodel.pipeline_options")
|
||||
pipeline_options_module.PdfPipelineOptions = FakePipelineOptions
|
||||
for name, module in {
|
||||
"docling": docling_module,
|
||||
"docling.document_converter": document_converter_module,
|
||||
"docling.datamodel": datamodel_module,
|
||||
"docling.datamodel.base_models": base_models_module,
|
||||
"docling.datamodel.pipeline_options": pipeline_options_module,
|
||||
}.items():
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
|
||||
_document_converter.cache_clear()
|
||||
try:
|
||||
converter = _document_converter()
|
||||
options = converter.format_options["pdf"].pipeline_options
|
||||
assert options.do_ocr is False
|
||||
finally:
|
||||
_document_converter.cache_clear()
|
||||
|
||||
|
||||
def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() -> None:
|
||||
source = "标题\n第一条 这是正文。\n第二条 后续正文。"
|
||||
compact_source, offsets = _compact_with_offsets(source)
|
||||
|
||||
Reference in New Issue
Block a user