YG-Soft/X-Financial
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): 系统缓存清理接口与 OCR 文本层兜底增强

Browse Source 
- 新增 system_cache 模块与 POST /settings/cache/clear，管理员可一键清理 OCR 结果/运行时配置/模型失败冷却/知识库索引/地点语义等进程内缓存
- 各服务暴露 clear_*_cache 方法（ocr/runtime_settings/runtime_chat/knowledge/application_location_semantic），SettingsCacheClearRead 汇总清理项
- OCR 转图片失败时尝试用 PDF 文本层兜底构建识别文档（有效字符≥8），并写结果缓存；OcrService 暴露 clear_result_cache
- receipt_folder 车票过滤补充身份证号关键词，附件文档/操作/展示模块同步适配
- 新增 system_cache_endpoints 测试，更新 openapi_schema/ocr/receipt_folder/attachment_association_jobs 测试
This commit is contained in:
caoxiaozhu
2026-06-24 12:35:51 +08:00
parent 50d2dc579a
commit 9a5ed0e94a
17 changed files with 932 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
server/src/app/api/v1/endpoints/settings.py
View File

@@ -5,18 +5,20 @@ from typing import Annotated
from fastapi import APIRouter, Depends, Header, HTTPException, status
from sqlalchemy.orm import Session
from app.api.deps import get_db
from app.api.deps import CurrentUserContext, get_db, require_admin_user
from app.core.config import get_settings as get_runtime_settings
from app.schemas.common import ErrorResponse
from app.schemas.settings import (
ModelConnectivityTestRead,
ModelConnectivityTestRequest,
RuntimeModelConfigRead,
SettingsCacheClearRead,
SettingsRead,
SettingsWrite,
)
from app.services.model_connectivity import probe_model_connectivity
from app.services.settings import SettingsService
from app.services.system_cache import SystemCacheService
router = APIRouter(prefix="/settings")
DbSession = Annotated[Session, Depends(get_db)]
@@ -93,6 +95,24 @@ def test_model_connectivity(
return probe_model_connectivity(resolved_payload)
@router.post(
"/cache/clear",
response_model=SettingsCacheClearRead,
summary="清理系统缓存",
description="清理 OCR、模型失败冷却、知识库索引和运行时配置等进程内缓存，不删除业务文件或数据库记录。",
responses={
status.HTTP_403_FORBIDDEN: {
"model": ErrorResponse,
"description": "只有管理员可以清理系统缓存。",
}
},
)
def clear_system_cache(
_: Annotated[CurrentUserContext, Depends(require_admin_user)],
) -> SettingsCacheClearRead:
return SystemCacheService().clear_all()
@router.get(
"/runtime-models/{slot}",
response_model=RuntimeModelConfigRead,

6
server/src/app/core/config.py
View File

@@ -169,6 +169,12 @@ def _clear_settings_cache() -> None:
_settings_cache_signature = None
def clear_runtime_settings_cache() -> int:
cleared_count = int(_settings_cache is not None)
_clear_settings_cache()
return cleared_count
def get_settings() -> Settings:
global _settings_cache, _settings_cache_signature

11
server/src/app/schemas/settings.py
View File

@@ -222,6 +222,17 @@ class ModelConnectivityTestRead(BaseModel):
checked_at: datetime
class SettingsCacheClearItemRead(BaseModel):
cacheKey: str
label: str
clearedCount: int = Field(default=0, ge=0)
class SettingsCacheClearRead(BaseModel):
totalCleared: int = Field(default=0, ge=0)
items: list[SettingsCacheClearItemRead] = Field(default_factory=list)
class RuntimeModelConfigRead(BaseModel):
slot: Literal["main", "backup", "embedding", "reranker"]
provider: str

8
server/src/app/services/application_location_semantics.py
View File

@@ -123,6 +123,14 @@ def _load_jieba_posseg() -> Any:
return pseg
def clear_application_location_semantic_caches() -> int:
cleared_count = _load_lac_analyzer.cache_info().currsize
cleared_count += _load_jieba_posseg.cache_info().currsize
_load_lac_analyzer.cache_clear()
_load_jieba_posseg.cache_clear()
return cleared_count
def _iter_jieba_custom_words() -> Iterable[str]:
yield from JIEBA_CUSTOM_WORDS
yield from DIRECT_MUNICIPALITY_DISPLAY

115
server/src/app/services/expense_claim_attachment_document.py
View File

@@ -111,9 +111,20 @@ from app.services.ocr import OcrService
class ExpenseClaimAttachmentDocumentMixin:
def _build_attachment_payload(self, item: ExpenseClaimItem) -> dict[str, Any]:
def _build_attachment_payload(
self,
item: ExpenseClaimItem,
*,
current_user: CurrentUserContext | None = None,
) -> dict[str, Any]:
file_path, media_type, filename = self._resolve_item_attachment_content(item)
metadata = self._attachment_storage.read_meta(file_path)
metadata = self._repair_attachment_metadata_from_source_receipt_if_needed(
file_path=file_path,
metadata=metadata,
item=item,
current_user=current_user,
)
metadata = self._repair_pdf_text_layer_metadata_if_needed(
file_path=file_path,
metadata=metadata,
@@ -164,6 +175,108 @@ class ExpenseClaimAttachmentDocumentMixin:
"requirement_check": requirement_check,
}
def _repair_attachment_metadata_from_source_receipt_if_needed(
self,
*,
file_path: Path,
metadata: dict[str, Any],
item: ExpenseClaimItem,
current_user: CurrentUserContext | None,
) -> dict[str, Any]:
if not metadata or current_user is None:
return metadata
source_receipt_id = str(metadata.get("source_receipt_id") or "").strip()
if not source_receipt_id:
return metadata
if not self._attachment_metadata_needs_source_receipt_repair(metadata):
return metadata
source_document = self._resolve_source_receipt_document(
source_receipt_id=source_receipt_id,
current_user=current_user,
fallback_filename=str(metadata.get("file_name") or file_path.name),
fallback_media_type=str(metadata.get("media_type") or ""),
)
if source_document is None:
return metadata
document_info = self._build_attachment_document_info(source_document)
requirement_check = self._build_attachment_requirement_check(
item=item,
document_info=document_info,
)
preview_meta = self._attachment_presentation.build_preview_meta(
file_path=file_path,
media_type=str(
metadata.get("media_type")
or self._attachment_presentation.resolve_media_type(file_path.name)
),
ocr_document=source_document,
)
metadata.update(
{
"previewable": bool(preview_meta["previewable"]),
"preview_kind": str(preview_meta["preview_kind"]),
"preview_storage_key": str(preview_meta["preview_storage_key"]),
"preview_media_type": str(preview_meta["preview_media_type"]),
"preview_file_name": str(preview_meta["preview_file_name"]),
"preview_rendered_with": str(preview_meta.get("preview_rendered_with") or ""),
"analysis": self._build_attachment_analysis(
document=source_document,
item=item,
claim=getattr(item, "claim", None),
document_info=document_info,
requirement_check=requirement_check,
),
"document_info": document_info,
"requirement_check": requirement_check,
"ocr_status": "recognized",
"ocr_error": "",
"ocr_text": str(getattr(source_document, "text", "") or ""),
"ocr_summary": str(getattr(source_document, "summary", "") or ""),
"ocr_avg_score": float(getattr(source_document, "avg_score", 0.0) or 0.0),
"ocr_line_count": int(getattr(source_document, "line_count", 0) or 0),
"ocr_classification_source": str(
getattr(source_document, "classification_source", "") or ""
),
"ocr_classification_confidence": float(
getattr(source_document, "classification_confidence", 0.0) or 0.0
),
"ocr_classification_evidence": [
str(value)
for value in list(getattr(source_document, "classification_evidence", []) or [])
if str(value).strip()
],
"ocr_warnings": [
str(value)
for value in list(getattr(source_document, "warnings", []) or [])
if str(value).strip()
],
}
)
self._attachment_storage.write_meta(file_path, metadata)
return metadata
@classmethod
def _attachment_metadata_needs_source_receipt_repair(cls, metadata: dict[str, Any]) -> bool:
document_info = metadata.get("document_info")
document_type = ""
fields: list[Any] = []
if isinstance(document_info, dict):
document_type = str(document_info.get("document_type") or "").strip()
fields = list(document_info.get("fields") or [])
return (
str(metadata.get("preview_kind") or "").strip() != "image"
or document_type in {"", "other"}
or not any(
isinstance(field, dict) and str(field.get("value") or "").strip()
for field in fields
)
)
@classmethod
def _attachment_metadata_needs_analysis_refresh(cls, metadata: dict[str, Any]) -> bool:
analysis = metadata.get("analysis")

54
server/src/app/services/expense_claim_attachment_operations.py
View File

@@ -313,8 +313,9 @@ class ExpenseClaimAttachmentOperationsMixin:
if not normalized_receipt_id:
return None
receipt_service = ReceiptFolderService()
try:
receipt = ReceiptFolderService().get_receipt(normalized_receipt_id, current_user)
receipt = receipt_service.get_receipt(normalized_receipt_id, current_user)
except FileNotFoundError:
return None
@@ -325,6 +326,20 @@ class ExpenseClaimAttachmentOperationsMixin:
if not fields:
fields = self._normalize_receipt_document_fields(raw_meta.get("document_fields"))
preview_source_path = None
preview_media_type = ""
preview_file_name = ""
if str(raw_meta.get("preview_kind") or "").strip() == "image":
try:
preview_source_path, preview_media_type, preview_file_name = receipt_service.resolve_preview(
normalized_receipt_id,
current_user,
)
except FileNotFoundError:
preview_source_path = None
preview_media_type = ""
preview_file_name = ""
document = SimpleNamespace(
filename=str(receipt.file_name or fallback_filename or "").strip(),
media_type=str(receipt.media_type or fallback_media_type or "application/octet-stream").strip(),
@@ -359,6 +374,9 @@ class ExpenseClaimAttachmentOperationsMixin:
document_fields=fields,
preview_kind=str(raw_meta.get("preview_kind") or ""),
preview_data_url="",
preview_source_path=str(preview_source_path or ""),
preview_media_type=preview_media_type,
preview_file_name=preview_file_name,
warnings=[
str(value)
for value in list(receipt.warnings or raw_meta.get("ocr_warnings") or [])
@@ -399,8 +417,16 @@ class ExpenseClaimAttachmentOperationsMixin:
source_type = cls._attachment_document_type(source_receipt_document)
upload_type = cls._attachment_document_type(upload_ocr_document)
if source_type in {"", "other"} and upload_type not in {"", "other"}:
return upload_ocr_document
if source_type not in {"", "other"} and upload_type in {"", "other"}:
return source_receipt_document
if (
cls._attachment_has_image_preview(source_receipt_document)
and not cls._attachment_has_image_preview(upload_ocr_document)
and source_score >= upload_score
):
return source_receipt_document
if (
source_type == upload_type
and cls._attachment_document_field_count(source_receipt_document)
@@ -438,6 +464,15 @@ class ExpenseClaimAttachmentOperationsMixin:
return 0
return len(list(getattr(document, "document_fields", []) or []))
@staticmethod
def _attachment_has_image_preview(document: Any | None) -> bool:
if document is None:
return False
return str(getattr(document, "preview_kind", "") or "").strip() == "image" and bool(
str(getattr(document, "preview_data_url", "") or "").strip()
or str(getattr(document, "preview_source_path", "") or "").strip()
)
def get_claim_item_attachment_meta(
self,
*,
@@ -453,7 +488,7 @@ class ExpenseClaimAttachmentOperationsMixin:
if claim is None:
return None
return self._build_attachment_payload(item)
return self._build_attachment_payload(item, current_user=current_user)
def get_claim_item_attachment_content(
self,
@@ -487,7 +522,7 @@ class ExpenseClaimAttachmentOperationsMixin:
if claim is None:
return None
return self._resolve_item_attachment_preview_content(item)
return self._resolve_item_attachment_preview_content(item, current_user=current_user)
def delete_claim_item_attachment(
self,
@@ -740,9 +775,20 @@ class ExpenseClaimAttachmentOperationsMixin:
self._attachment_storage.write_meta(file_path, metadata)
return metadata
def _resolve_item_attachment_preview_content(self, item: ExpenseClaimItem) -> tuple[Path, str, str]:
def _resolve_item_attachment_preview_content(
self,
item: ExpenseClaimItem,
*,
current_user: CurrentUserContext | None = None,
) -> tuple[Path, str, str]:
file_path, media_type, filename = self._resolve_item_attachment_content(item)
metadata = self._attachment_storage.read_meta(file_path)
metadata = self._repair_attachment_metadata_from_source_receipt_if_needed(
file_path=file_path,
metadata=metadata,
item=item,
current_user=current_user,
)
metadata = self._repair_pdf_text_layer_metadata_if_needed(
file_path=file_path,
metadata=metadata,

42
server/src/app/services/expense_claim_attachment_presentation.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import mimetypes
import shutil
from pathlib import Path
from typing import Any
from urllib.parse import quote
@@ -43,6 +44,25 @@ class ExpenseClaimAttachmentPresentation:
"preview_rendered_with": DocumentPreviewAssets.renderer_id_for_source(media_type),
}
preview_source_path = getattr(ocr_document, "preview_source_path", None)
if preview_source_kind == "image" and preview_source_path:
preview_asset = self._copy_preview_asset_from_source(
attachment_dir=file_path.parent,
original_filename=filename,
preview_source_path=Path(preview_source_path),
preview_media_type=str(getattr(ocr_document, "preview_media_type", "") or ""),
)
if preview_asset is not None:
preview_path, preview_media_type, preview_file_name = preview_asset
return {
"previewable": True,
"preview_kind": "image",
"preview_storage_key": self.storage.to_storage_key(preview_path),
"preview_media_type": preview_media_type,
"preview_file_name": preview_file_name,
"preview_rendered_with": DocumentPreviewAssets.renderer_id_for_source(media_type),
}
if preview_kind:
return {
"previewable": True,
@@ -88,6 +108,28 @@ class ExpenseClaimAttachmentPresentation:
preview_data_url=preview_data_url,
)
def _copy_preview_asset_from_source(
self,
*,
attachment_dir: Path,
original_filename: str,
preview_source_path: Path,
preview_media_type: str,
) -> tuple[Path, str, str] | None:
if not preview_source_path.exists() or not preview_source_path.is_file():
return None
suffix = preview_source_path.suffix or DocumentPreviewAssets.PDF_PREVIEW_SUFFIX
preview_name = f"{Path(original_filename).stem}.preview{suffix}"
preview_path = attachment_dir / preview_name
shutil.copyfile(preview_source_path, preview_path)
resolved_media_type = (
preview_media_type
or mimetypes.guess_type(preview_source_path.name)[0]
or DocumentPreviewAssets.PDF_PREVIEW_MEDIA_TYPE
)
return preview_path, resolved_media_type, preview_name
@staticmethod
def build_preview_client_path(claim_id: str, item_id: str) -> str:
return (

7
server/src/app/services/knowledge_rag_local.py
View File

@@ -108,6 +108,13 @@ _index_lock = threading.RLock()
_index_cache: dict[Path, tuple[tuple[int, int], list[dict[str, Any]]]] = {}
def clear_local_knowledge_index_cache() -> int:
with _index_lock:
cleared_count = len(_index_cache)
_index_cache.clear()
return cleared_count
@dataclass(frozen=True, slots=True)
class LocalKnowledgeSearchResult:
hits: list[dict[str, Any]]

59
server/src/app/services/ocr.py
View File

@@ -148,13 +148,23 @@ class OcrService:
for item in pdf_inputs:
cache_keys_by_source.setdefault(item.source_key, cache_key)
except RuntimeError as exc:
documents.append(
OcrRecognizeDocumentRead(
filename=normalized_name,
media_type=resolved_media_type,
warnings=[str(exc)],
)
fallback_document = self._build_pdf_text_layer_fallback_document(
filename=normalized_name,
media_type=resolved_media_type,
text_layer=text_layer,
render_warning=str(exc),
)
if fallback_document is not None:
documents.append(fallback_document)
self._write_cached_document(cache_key, fallback_document)
else:
documents.append(
OcrRecognizeDocumentRead(
filename=normalized_name,
media_type=resolved_media_type,
warnings=[str(exc)],
)
)
continue
source_key = uuid4().hex
@@ -328,6 +338,13 @@ class OcrService:
while len(cls._result_cache) > OCR_RESULT_CACHE_LIMIT:
cls._result_cache.popitem(last=False)
@classmethod
def clear_result_cache(cls) -> int:
with cls._cache_lock:
cleared_count = len(cls._result_cache)
cls._result_cache.clear()
return cleared_count
@classmethod
def _resolve_worker_semaphore(cls, limit: int) -> threading.Semaphore:
normalized_limit = max(1, int(limit or 1))
@@ -425,6 +442,36 @@ class OcrService:
)
return descriptors
def _build_pdf_text_layer_fallback_document(
self,
*,
filename: str,
media_type: str,
text_layer: str,
render_warning: str,
) -> OcrRecognizeDocumentRead | None:
normalized_text = self._normalize_extracted_text(text_layer)
if self._meaningful_char_count(normalized_text) < 8:
return None
aggregated = AggregatedOcrDocument(
filename=filename,
media_type=media_type,
source_key=uuid4().hex,
page_count=1,
warnings=[
str(render_warning or "").strip() or "PDF 转图片失败。",
"PDF 转图片失败，已使用 PDF 文本层继续抽取识别信息。",
],
lines=[
OcrRecognizeLineRead(text=line, page_index=0)
for line in normalized_text.splitlines()
if line.strip()
],
)
aggregated.text_layer_fragments.append(normalized_text)
return self._finalize_document(aggregated)
def _extract_pdf_text_layer(self, pdf_path: Path) -> str:
try:
completed = subprocess.run(

64
server/src/app/services/receipt_folder.py
View File

@@ -889,6 +889,8 @@ class ReceiptFolderTrainTicketMixin:
"无效",
"二维码",
"座席",
"身份",
"身份证号",
"证件",
)
):
@@ -993,6 +995,11 @@ class ReceiptFolderService(ReceiptFolderStorageMixin, ReceiptFolderItemMixin, Re
current_user=current_user,
)
if duplicate_receipt is not None:
duplicate_receipt = self._refresh_duplicate_receipt_from_document_if_stronger(
receipt=duplicate_receipt,
document=document,
current_user=current_user,
)
warning = "已上传过同样的单据，请不要重复上传。"
existing_warnings = [str(item) for item in list(document.warnings or []) if str(item).strip()]
enriched.append(
@@ -1061,6 +1068,7 @@ class ReceiptFolderService(ReceiptFolderStorageMixin, ReceiptFolderItemMixin, Re
if str(value).strip()
],
"document_fields": self._build_ocr_document_fields_from_meta(meta),
"preview_kind": str(meta.get("preview_kind") or document.preview_kind or ""),
}
)
@@ -1073,6 +1081,62 @@ class ReceiptFolderService(ReceiptFolderStorageMixin, ReceiptFolderItemMixin, Re
update["warnings"] = list(dict.fromkeys(warnings))
return document.model_copy(update=update)
def _refresh_duplicate_receipt_from_document_if_stronger(
self,
*,
receipt: ReceiptFolderItemRead,
document: OcrRecognizeDocumentRead,
current_user: CurrentUserContext,
) -> ReceiptFolderItemRead:
try:
meta = self._read_receipt_meta(receipt.id, current_user)
except FileNotFoundError:
return receipt
incoming_meta = self._build_document_meta(document)
if not self._is_incoming_document_meta_stronger(meta, incoming_meta):
return receipt
for key in (
"engine",
"model",
"ocr_text",
"summary",
"ocr_avg_score",
"ocr_line_count",
"page_count",
"document_type",
"document_type_label",
"scene_code",
"scene_label",
"ocr_classification_source",
"ocr_classification_confidence",
"ocr_classification_evidence",
"document_fields",
"ocr_warnings",
):
meta[key] = incoming_meta[key]
meta["updated_at"] = datetime.now(UTC).isoformat()
self._write_meta(self._receipt_dir(self._owner_key(current_user), receipt.id), meta)
return self._build_item(meta)
@staticmethod
def _is_incoming_document_meta_stronger(existing_meta: dict[str, Any], incoming_meta: dict[str, Any]) -> bool:
existing_type = str(existing_meta.get("document_type") or "other").strip() or "other"
incoming_type = str(incoming_meta.get("document_type") or "other").strip() or "other"
existing_fields = [field for field in list(existing_meta.get("document_fields") or []) if isinstance(field, dict)]
incoming_fields = [field for field in list(incoming_meta.get("document_fields") or []) if isinstance(field, dict)]
existing_text = str(existing_meta.get("ocr_text") or "").strip()
incoming_text = str(incoming_meta.get("ocr_text") or "").strip()
if incoming_type != "other" and existing_type == "other":
return True
if incoming_fields and not existing_fields:
return True
if incoming_text and not existing_text:
return True
return False
def _build_ocr_document_fields_from_meta(self, meta: dict[str, Any]) -> list[OcrRecognizeFieldRead]:
return [
OcrRecognizeFieldRead(

6
server/src/app/services/runtime_chat.py
View File

@@ -29,6 +29,12 @@ DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS = 90
_slot_failure_until: dict[str, float] = {}
def clear_runtime_chat_failure_cache() -> int:
cleared_count = len(_slot_failure_until)
_slot_failure_until.clear()
return cleared_count
@dataclass(slots=True)
class RuntimeChatCallTrace:
slot: str

41
server/src/app/services/system_cache.py Normal file
View File

@@ -0,0 +1,41 @@
from __future__ import annotations
from app.core.config import clear_runtime_settings_cache
from app.schemas.settings import SettingsCacheClearItemRead, SettingsCacheClearRead
from app.services.application_location_semantics import clear_application_location_semantic_caches
from app.services.knowledge_rag_local import clear_local_knowledge_index_cache
from app.services.ocr import OcrService
from app.services.runtime_chat import clear_runtime_chat_failure_cache
class SystemCacheService:
def clear_all(self) -> SettingsCacheClearRead:
items = [
SettingsCacheClearItemRead(
cacheKey="ocr_result_cache",
label="OCR 识别结果缓存",
clearedCount=OcrService.clear_result_cache(),
),
SettingsCacheClearItemRead(
cacheKey="runtime_settings_cache",
label="运行时配置缓存",
clearedCount=clear_runtime_settings_cache(),
),
SettingsCacheClearItemRead(
cacheKey="runtime_chat_failure_cache",
label="模型调用失败冷却缓存",
clearedCount=clear_runtime_chat_failure_cache(),
),
SettingsCacheClearItemRead(
cacheKey="knowledge_local_index_cache",
label="知识库本地索引缓存",
clearedCount=clear_local_knowledge_index_cache(),
),
SettingsCacheClearItemRead(
cacheKey="application_location_semantic_cache",
label="地点语义分析缓存",
clearedCount=clear_application_location_semantic_caches(),
),
]
total_cleared = sum(item.clearedCount for item in items)
return SettingsCacheClearRead(totalCleared=total_cleared, items=items)