feat(ocr): PDF 文本层可用时跳过 worker 调用并补装 poppler-data
- OcrService 提取 PDF 文本层后若有效字符达到阈值,直接构建文档并写入结果缓存,不再触发 OCR worker,仅无文本层时才解析 python_bin/worker_path 调用 worker - _build_text_layer_document 复用 AggregatedOcrDocument 聚合文本层片段,_has_usable_pdf_text_layer 基于 meaningful_char_count 判定 - docker-compose 与 paddleocr bootstrap 脚本补装 poppler-data,保证 PDF 文本层抽取的中文编码正确 - 新增文本层直取与运行时依赖两项 ocr_service 单测
This commit is contained in:
@@ -32,7 +32,7 @@ services:
|
|||||||
- >
|
- >
|
||||||
apt-get update &&
|
apt-get update &&
|
||||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends
|
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends
|
||||||
python3 python3-pip python3-venv fontconfig openssh-server &&
|
python3 python3-pip python3-venv fontconfig openssh-server poppler-data &&
|
||||||
if ! fc-match 'Noto Sans CJK SC' | grep -qi 'Noto'; then if ! timeout "${CJK_FONT_INSTALL_TIMEOUT_SECONDS:-45}" sh -lc 'DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fonts-noto-cjk fonts-noto-cjk-extra'; then printf '%s\n' '[WARN] CJK font installation timed out or failed; continuing startup without blocking the app.'; fi; fi &&
|
if ! fc-match 'Noto Sans CJK SC' | grep -qi 'Noto'; then if ! timeout "${CJK_FONT_INSTALL_TIMEOUT_SECONDS:-45}" sh -lc 'DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fonts-noto-cjk fonts-noto-cjk-extra'; then printf '%s\n' '[WARN] CJK font installation timed out or failed; continuing startup without blocking the app.'; fi; fi &&
|
||||||
printf '%s\n'
|
printf '%s\n'
|
||||||
'<?xml version="1.0"?>'
|
'<?xml version="1.0"?>'
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
apt-get update
|
apt-get update
|
||||||
apt-get install -y --no-install-recommends libgl1 libglib2.0-0 poppler-utils
|
apt-get install -y --no-install-recommends libgl1 libglib2.0-0 poppler-utils poppler-data
|
||||||
|
|
||||||
rm -rf "${OCR_VENV_DIR}"
|
rm -rf "${OCR_VENV_DIR}"
|
||||||
"${PYTHON_BIN}" -m venv "${OCR_VENV_DIR}"
|
"${PYTHON_BIN}" -m venv "${OCR_VENV_DIR}"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
apt-get update
|
apt-get update
|
||||||
apt-get install -y --no-install-recommends libgl1 libglib2.0-0 poppler-utils
|
apt-get install -y --no-install-recommends libgl1 libglib2.0-0 poppler-utils poppler-data
|
||||||
|
|
||||||
"${PYTHON_BIN}" -m venv "${OCR_VENV_DIR}"
|
"${PYTHON_BIN}" -m venv "${OCR_VENV_DIR}"
|
||||||
"${OCR_VENV_DIR}/bin/pip" install --upgrade pip
|
"${OCR_VENV_DIR}/bin/pip" install --upgrade pip
|
||||||
|
|||||||
@@ -77,8 +77,6 @@ class OcrService:
|
|||||||
documents: list[OcrRecognizeDocumentRead] = []
|
documents: list[OcrRecognizeDocumentRead] = []
|
||||||
prepared_inputs: list[PreparedOcrInput] = []
|
prepared_inputs: list[PreparedOcrInput] = []
|
||||||
cleanup_paths: list[Path] = []
|
cleanup_paths: list[Path] = []
|
||||||
python_bin = self._resolve_python_bin()
|
|
||||||
worker_path = self._resolve_worker_path()
|
|
||||||
worker_payload: dict = {}
|
worker_payload: dict = {}
|
||||||
cache_keys_by_source: dict[str, str] = {}
|
cache_keys_by_source: dict[str, str] = {}
|
||||||
|
|
||||||
@@ -144,6 +142,16 @@ class OcrService:
|
|||||||
cleanup_paths=cleanup_paths,
|
cleanup_paths=cleanup_paths,
|
||||||
text_layer=text_layer,
|
text_layer=text_layer,
|
||||||
)
|
)
|
||||||
|
if self._has_usable_pdf_text_layer(text_layer):
|
||||||
|
document = self._build_text_layer_document(
|
||||||
|
filename=normalized_name,
|
||||||
|
media_type=resolved_media_type,
|
||||||
|
text_layer=text_layer,
|
||||||
|
pdf_inputs=pdf_inputs,
|
||||||
|
)
|
||||||
|
documents.append(document)
|
||||||
|
self._write_cached_document(cache_key, document)
|
||||||
|
continue
|
||||||
prepared_inputs.extend(pdf_inputs)
|
prepared_inputs.extend(pdf_inputs)
|
||||||
for item in pdf_inputs:
|
for item in pdf_inputs:
|
||||||
cache_keys_by_source.setdefault(item.source_key, cache_key)
|
cache_keys_by_source.setdefault(item.source_key, cache_key)
|
||||||
@@ -175,6 +183,8 @@ class OcrService:
|
|||||||
cache_keys_by_source[source_key] = cache_key
|
cache_keys_by_source[source_key] = cache_key
|
||||||
|
|
||||||
if prepared_inputs:
|
if prepared_inputs:
|
||||||
|
python_bin = self._resolve_python_bin()
|
||||||
|
worker_path = self._resolve_worker_path()
|
||||||
worker_payload = self._invoke_worker(
|
worker_payload = self._invoke_worker(
|
||||||
python_bin=python_bin,
|
python_bin=python_bin,
|
||||||
worker_path=worker_path,
|
worker_path=worker_path,
|
||||||
@@ -308,6 +318,23 @@ class OcrService:
|
|||||||
while len(cls._result_cache) > OCR_RESULT_CACHE_LIMIT:
|
while len(cls._result_cache) > OCR_RESULT_CACHE_LIMIT:
|
||||||
cls._result_cache.popitem(last=False)
|
cls._result_cache.popitem(last=False)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _write_cached_document(cls, cache_key: str, document: OcrRecognizeDocumentRead) -> None:
|
||||||
|
if not cache_key:
|
||||||
|
return
|
||||||
|
with cls._cache_lock:
|
||||||
|
cls._result_cache[cache_key] = document.model_copy(
|
||||||
|
update={
|
||||||
|
"receipt_id": "",
|
||||||
|
"receipt_status": "",
|
||||||
|
"receipt_preview_url": "",
|
||||||
|
"receipt_source_url": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
cls._result_cache.move_to_end(cache_key)
|
||||||
|
while len(cls._result_cache) > OCR_RESULT_CACHE_LIMIT:
|
||||||
|
cls._result_cache.popitem(last=False)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _resolve_worker_semaphore(cls, limit: int) -> threading.Semaphore:
|
def _resolve_worker_semaphore(cls, limit: int) -> threading.Semaphore:
|
||||||
normalized_limit = max(1, int(limit or 1))
|
normalized_limit = max(1, int(limit or 1))
|
||||||
@@ -568,6 +595,30 @@ class OcrService:
|
|||||||
|
|
||||||
return documents
|
return documents
|
||||||
|
|
||||||
|
def _build_text_layer_document(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
filename: str,
|
||||||
|
media_type: str,
|
||||||
|
text_layer: str,
|
||||||
|
pdf_inputs: list[PreparedOcrInput],
|
||||||
|
) -> OcrRecognizeDocumentRead:
|
||||||
|
first_input = pdf_inputs[0] if pdf_inputs else None
|
||||||
|
aggregated = AggregatedOcrDocument(
|
||||||
|
filename=filename,
|
||||||
|
media_type=media_type,
|
||||||
|
source_key=first_input.source_key if first_input is not None else uuid4().hex,
|
||||||
|
page_count=max(1, len(pdf_inputs)),
|
||||||
|
preview_kind=str(first_input.preview_kind if first_input is not None else ""),
|
||||||
|
preview_data_url=str(first_input.preview_data_url if first_input is not None else ""),
|
||||||
|
)
|
||||||
|
aggregated.text_layer_fragments.append(text_layer)
|
||||||
|
return self._finalize_document(aggregated)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _has_usable_pdf_text_layer(cls, text_layer: str) -> bool:
|
||||||
|
return cls._meaningful_char_count(text_layer) >= 8
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _collect_descriptor_text_layer(descriptors: list[PreparedOcrInput]) -> str:
|
def _collect_descriptor_text_layer(descriptors: list[PreparedOcrInput]) -> str:
|
||||||
for descriptor in descriptors:
|
for descriptor in descriptors:
|
||||||
|
|||||||
@@ -8,6 +8,18 @@ from app.core.config import get_settings
|
|||||||
from app.services.ocr import OcrService
|
from app.services.ocr import OcrService
|
||||||
|
|
||||||
|
|
||||||
|
def test_ocr_runtime_installers_include_poppler_cjk_data() -> None:
|
||||||
|
repo_root = Path(__file__).resolve().parents[2]
|
||||||
|
dependency_sources = [
|
||||||
|
repo_root / "docker-compose.yml",
|
||||||
|
repo_root / "server" / "scripts" / "bootstrap_paddleocr_mobile.sh",
|
||||||
|
repo_root / "server" / "scripts" / "bootstrap_paddleocr_gpu.sh",
|
||||||
|
]
|
||||||
|
|
||||||
|
for path in dependency_sources:
|
||||||
|
assert "poppler-data" in path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
def test_ocr_service_uses_worker_runtime_and_keeps_unsupported_files_as_warnings(
|
def test_ocr_service_uses_worker_runtime_and_keeps_unsupported_files_as_warnings(
|
||||||
monkeypatch,
|
monkeypatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -220,6 +232,59 @@ def test_ocr_service_converts_pdf_to_images_and_returns_image_preview(
|
|||||||
assert recognized.lines[1].page_index == 1
|
assert recognized.lines[1].page_index == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_ocr_service_uses_pdf_text_layer_without_worker_runtime(
|
||||||
|
monkeypatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
def fake_convert_pdf_to_images(self, *, pdf_path: Path, output_dir: Path) -> list[Path]:
|
||||||
|
page = output_dir / "page-1.png"
|
||||||
|
page.write_bytes(b"fake-rendered-page")
|
||||||
|
return [page]
|
||||||
|
|
||||||
|
def fail_resolve_python(self) -> str:
|
||||||
|
raise AssertionError("PDF 文本层可用时不应强制解析 OCR worker。")
|
||||||
|
|
||||||
|
def fail_invoke_worker(self, **kwargs) -> dict:
|
||||||
|
raise AssertionError("PDF 文本层可用时不应调用 OCR worker。")
|
||||||
|
|
||||||
|
monkeypatch.setenv("STORAGE_ROOT_DIR", str(tmp_path / "storage"))
|
||||||
|
monkeypatch.setattr(OcrService, "_resolve_python_bin", fail_resolve_python)
|
||||||
|
monkeypatch.setattr(OcrService, "_resolve_worker_path", lambda self: "worker.py")
|
||||||
|
monkeypatch.setattr(OcrService, "_convert_pdf_to_images", fake_convert_pdf_to_images)
|
||||||
|
monkeypatch.setattr(OcrService, "_invoke_worker", fail_invoke_worker)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
OcrService,
|
||||||
|
"_extract_pdf_text_layer",
|
||||||
|
lambda self, pdf_path: (
|
||||||
|
"电子发票(铁路电子客票)\n"
|
||||||
|
"发票号码:26429165800002785705\n"
|
||||||
|
"武汉站\n"
|
||||||
|
"上海虹桥站\n"
|
||||||
|
"G458\n"
|
||||||
|
"票价:¥354.00\n"
|
||||||
|
"电子客票号:6580061086021391007342026"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
get_settings.cache_clear()
|
||||||
|
try:
|
||||||
|
result = OcrService().recognize_files(
|
||||||
|
[
|
||||||
|
("train-ticket.pdf", b"%PDF-1.7 fake", "application/pdf"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
recognized = result.documents[0]
|
||||||
|
assert result.success_count == 1
|
||||||
|
assert recognized.document_type == "train_ticket"
|
||||||
|
assert "电子发票(铁路电子客票)" in recognized.text
|
||||||
|
assert "电子客票号:6580061086021391007342026" in recognized.text
|
||||||
|
assert any(field.label == "金额" and field.value == "354元" for field in recognized.document_fields)
|
||||||
|
assert recognized.preview_kind == "image"
|
||||||
|
assert recognized.preview_data_url.startswith("data:image/png;base64,")
|
||||||
|
|
||||||
|
|
||||||
def test_ocr_service_reuses_cached_document_for_same_content(
|
def test_ocr_service_reuses_cached_document_for_same_content(
|
||||||
monkeypatch,
|
monkeypatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -351,5 +416,5 @@ def test_ocr_service_prefers_pdf_text_layer_when_rendered_ocr_is_placeholder_hea
|
|||||||
assert "上海虹桥站" in recognized.text
|
assert "上海虹桥站" in recognized.text
|
||||||
assert "□□□□" not in recognized.summary
|
assert "□□□□" not in recognized.summary
|
||||||
assert recognized.document_type == "train_ticket"
|
assert recognized.document_type == "train_ticket"
|
||||||
assert recognized.preview_kind == ""
|
assert recognized.preview_kind == "image"
|
||||||
assert recognized.preview_data_url == ""
|
assert recognized.preview_data_url.startswith("data:image/png;base64,")
|
||||||
|
|||||||
Reference in New Issue
Block a user