feat: 重构知识库系统,移除Hermes集成,增强RAG和同步功能
主要变更: - 移除Hermes智能体及相关回调服务 - 新增知识库RAG、同步、调度、规范化和索引任务服务 - 重构orchestrator服务,增强运行时聊天功能 - 更新前端聊天、政策制度、设置等页面样式和逻辑 - 更新expense_claims和document_intelligence服务 - 删除llm_wiki相关服务和测试文件 - 更新docker-compose配置和启动脚本
This commit is contained in:
@@ -8,18 +8,20 @@ version = "0.1.0"
|
||||
description = "Backend service for X-Financial reimbursement and approval platform."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0,<1.0.0",
|
||||
"uvicorn[standard]>=0.30.0,<1.0.0",
|
||||
"sqlalchemy>=2.0.36,<3.0.0",
|
||||
"alembic>=1.14.0,<2.0.0",
|
||||
"psycopg[binary]>=3.2.0,<4.0.0",
|
||||
"PyJWT>=2.9.0,<3.0.0",
|
||||
"pydantic-settings>=2.6.0,<3.0.0",
|
||||
"python-dotenv>=1.0.1,<2.0.0",
|
||||
"email-validator>=2.2.0,<3.0.0",
|
||||
"python-multipart>=0.0.20,<1.0.0",
|
||||
]
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0,<1.0.0",
|
||||
"uvicorn[standard]>=0.30.0,<1.0.0",
|
||||
"sqlalchemy>=2.0.36,<3.0.0",
|
||||
"alembic>=1.14.0,<2.0.0",
|
||||
"psycopg[binary]>=3.2.0,<4.0.0",
|
||||
"PyJWT>=2.9.0,<3.0.0",
|
||||
"pydantic-settings>=2.6.0,<3.0.0",
|
||||
"python-dotenv>=1.0.1,<2.0.0",
|
||||
"email-validator>=2.2.0,<3.0.0",
|
||||
"python-multipart>=0.0.20,<1.0.0",
|
||||
"lightrag-hku>=1.4.16,<1.5.0",
|
||||
"qdrant-client>=1.18.0,<2.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
|
||||
@@ -183,6 +183,10 @@ if [ "${APP_DEBUG:-true}" = "true" ]; then
|
||||
DEFAULT_SERVER_RELOAD="true"
|
||||
fi
|
||||
|
||||
if is_container; then
|
||||
DEFAULT_SERVER_RELOAD="false"
|
||||
fi
|
||||
|
||||
SERVER_RELOAD="${SERVER_RELOAD:-$DEFAULT_SERVER_RELOAD}"
|
||||
|
||||
needs_windows_python() {
|
||||
@@ -236,7 +240,7 @@ run_bootstrap_python() {
|
||||
}
|
||||
|
||||
dependencies_ready() {
|
||||
"$PYTHON_BIN" -c "import alembic, dotenv, email_validator, fastapi, jwt, multipart, psycopg, pydantic_settings, sqlalchemy, uvicorn" >/dev/null 2>&1
|
||||
"$PYTHON_BIN" -c "import alembic, dotenv, email_validator, fastapi, jwt, lightrag, multipart, psycopg, pydantic_settings, qdrant_client, sqlalchemy, uvicorn" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
pip_ready() {
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import get_db
|
||||
from app.api.v1.endpoints.settings import require_hermes_agent_token
|
||||
from app.schemas.common import ErrorResponse
|
||||
from app.schemas.hermes import HermesCallbackRead, HermesCallbackWrite
|
||||
from app.services.hermes_callbacks import HermesCallbackService
|
||||
|
||||
router = APIRouter(prefix="/hermes")
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/callback",
|
||||
response_model=HermesCallbackRead,
|
||||
dependencies=[Depends(require_hermes_agent_token)],
|
||||
summary="接收 Hermes 通用回调",
|
||||
description="所有 Hermes 任务统一通过该入口回传进度或完成结果,服务端依据 type 分发到对应业务处理器。",
|
||||
responses={
|
||||
status.HTTP_400_BAD_REQUEST: {
|
||||
"model": ErrorResponse,
|
||||
"description": "Hermes 回调载荷不合法或任务类型暂不支持。",
|
||||
},
|
||||
status.HTTP_404_NOT_FOUND: {
|
||||
"model": ErrorResponse,
|
||||
"description": "回调引用的 AgentRun 不存在。",
|
||||
},
|
||||
},
|
||||
)
|
||||
def handle_hermes_callback(
|
||||
payload: HermesCallbackWrite,
|
||||
db: DbSession,
|
||||
) -> HermesCallbackRead:
|
||||
try:
|
||||
return HermesCallbackService(db).handle_callback(payload)
|
||||
except LookupError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
File diff suppressed because it is too large
Load Diff
@@ -98,7 +98,7 @@ def test_model_connectivity(
|
||||
response_model=RuntimeModelConfigRead,
|
||||
dependencies=[Depends(require_hermes_agent_token)],
|
||||
summary="读取 Hermes 运行时模型配置",
|
||||
description="供 Hermes 进程读取主模型、备用模型、VLM 或 Embedding 模型的运行时配置。",
|
||||
description="供 Hermes 进程读取主模型、备用模型、Embedding 或 Reranker 模型的运行时配置。",
|
||||
responses={
|
||||
status.HTTP_401_UNAUTHORIZED: {
|
||||
"model": ErrorResponse,
|
||||
|
||||
@@ -7,7 +7,6 @@ from app.api.v1.endpoints.auth import router as auth_router
|
||||
from app.api.v1.endpoints.bootstrap import router as bootstrap_router
|
||||
from app.api.v1.endpoints.employees import router as employees_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
from app.api.v1.endpoints.hermes import router as hermes_router
|
||||
from app.api.v1.endpoints.knowledge import router as knowledge_router
|
||||
from app.api.v1.endpoints.ocr import router as ocr_router
|
||||
from app.api.v1.endpoints.ontology import router as ontology_router
|
||||
@@ -18,7 +17,6 @@ from app.api.v1.endpoints.system_logs import router as system_logs_router
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(health_router, tags=["health"])
|
||||
router.include_router(hermes_router, tags=["hermes"])
|
||||
router.include_router(bootstrap_router, tags=["bootstrap"])
|
||||
router.include_router(auth_router, tags=["auth"])
|
||||
router.include_router(agent_assets_router, tags=["agent-assets"])
|
||||
|
||||
@@ -14,9 +14,11 @@ from app.middleware.logging import AccessLogMiddleware
|
||||
from app.schemas.common import RootStatusRead
|
||||
from app.services.agent_foundation import prepare_agent_foundation
|
||||
from app.services.employee import prepare_employee_directory
|
||||
from app.services.knowledge import prepare_knowledge_library
|
||||
from app.services.llm_wiki_tasks import llm_wiki_task_manager
|
||||
from app.services.hermes_sync import sync_repository_hermes_skills
|
||||
from app.services.knowledge import prepare_knowledge_library
|
||||
from app.services.knowledge_index_tasks import knowledge_index_task_manager
|
||||
from app.services.knowledge_rag import shutdown_knowledge_rag_runtime
|
||||
from app.services.knowledge_scheduler import knowledge_index_scheduler
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -28,6 +30,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
prepare_agent_foundation()
|
||||
prepare_knowledge_library()
|
||||
sync_repository_hermes_skills()
|
||||
knowledge_index_scheduler.start()
|
||||
logger.info(
|
||||
"Server ready - host=%s port=%s prefix=%s",
|
||||
settings.app_host,
|
||||
@@ -35,23 +38,25 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
settings.api_v1_prefix,
|
||||
)
|
||||
yield
|
||||
llm_wiki_task_manager.shutdown()
|
||||
knowledge_index_scheduler.shutdown()
|
||||
knowledge_index_task_manager.shutdown()
|
||||
shutdown_knowledge_rag_runtime()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
|
||||
setup_logging(
|
||||
level=settings.log_level,
|
||||
log_dir=settings.log_dir,
|
||||
enable_file=settings.log_file_enabled,
|
||||
)
|
||||
|
||||
logger = get_logger("app.main")
|
||||
logger.info(
|
||||
"Starting %s (env=%s, debug=%s)", settings.app_name, settings.app_env, settings.app_debug
|
||||
)
|
||||
|
||||
|
||||
setup_logging(
|
||||
level=settings.log_level,
|
||||
log_dir=settings.log_dir,
|
||||
enable_file=settings.log_file_enabled,
|
||||
)
|
||||
|
||||
logger = get_logger("app.main")
|
||||
logger.info(
|
||||
"Starting %s (env=%s, debug=%s)", settings.app_name, settings.app_env, settings.app_debug
|
||||
)
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.app_name,
|
||||
debug=settings.app_debug,
|
||||
@@ -60,31 +65,31 @@ def create_app() -> FastAPI:
|
||||
openapi_tags=OPENAPI_TAGS,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(AccessLogMiddleware)
|
||||
|
||||
if settings.cors_origins:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(api_router, prefix=settings.api_v1_prefix)
|
||||
|
||||
|
||||
app.add_middleware(AccessLogMiddleware)
|
||||
|
||||
if settings.cors_origins:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(api_router, prefix=settings.api_v1_prefix)
|
||||
|
||||
@app.get(
|
||||
"/",
|
||||
tags=["root"],
|
||||
response_model=RootStatusRead,
|
||||
summary="服务根检查",
|
||||
description="用于快速确认后端服务进程已经启动。",
|
||||
description="用于快速确认后端服务进程已启动。",
|
||||
)
|
||||
def root() -> RootStatusRead:
|
||||
return RootStatusRead(message=f"{settings.app_name} is running")
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
@@ -40,6 +40,11 @@ class SystemSetting(Base):
|
||||
embedding_provider: Mapped[str] = mapped_column(String(64), default="GLM")
|
||||
embedding_model: Mapped[str] = mapped_column(String(255), default="Embedding-3")
|
||||
embedding_endpoint: Mapped[str] = mapped_column(String(512), default="https://open.bigmodel.cn/api/paas/v4/")
|
||||
reranker_provider: Mapped[str] = mapped_column(String(64), default="Ali")
|
||||
reranker_model: Mapped[str] = mapped_column(String(255), default="gte-rerank-v2")
|
||||
reranker_endpoint: Mapped[str] = mapped_column(
|
||||
String(512), default="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
|
||||
)
|
||||
onlyoffice_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
onlyoffice_public_url: Mapped[str] = mapped_column(String(512), default="")
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ class SystemSettingSecret(Base):
|
||||
backup_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
vlm_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
embedding_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
reranker_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
onlyoffice_jwt_secret_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
smtp_password_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
HermesCallbackStatus = Literal["running", "succeeded", "failed"]
|
||||
|
||||
|
||||
class HermesCallbackWrite(BaseModel):
|
||||
type: str = Field(min_length=1, max_length=80)
|
||||
run_id: str = Field(min_length=1, max_length=80)
|
||||
status: HermesCallbackStatus
|
||||
summary: str = ""
|
||||
error: str = ""
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class HermesCallbackRead(BaseModel):
|
||||
ok: bool = True
|
||||
accepted: bool = True
|
||||
type: str
|
||||
run_id: str
|
||||
status: HermesCallbackStatus
|
||||
@@ -35,6 +35,7 @@ class KnowledgeDocumentRead(BaseModel):
|
||||
folder: str
|
||||
tag: str
|
||||
time: str
|
||||
ingestTime: str = ""
|
||||
version: str
|
||||
stateCode: int = 1
|
||||
state: str
|
||||
|
||||
@@ -30,7 +30,7 @@ class OcrRecognizeDocumentRead(BaseModel):
|
||||
document_type_label: str = Field(default="其他单据", description="识别出的票据类型名称。")
|
||||
scene_code: str = Field(default="other", description="识别出的票据场景编码。")
|
||||
scene_label: str = Field(default="其他票据", description="识别出的票据场景名称。")
|
||||
classification_source: str = Field(default="rule", description="票据类型判断来源,例如 rule / llm_text / llm_vision。")
|
||||
classification_source: str = Field(default="rule", description="票据类型判断来源,当前固定为 rule。")
|
||||
classification_confidence: float = Field(default=0.0, ge=0.0, le=1.0, description="票据类型判断置信度。")
|
||||
classification_evidence: list[str] = Field(default_factory=list, description="票据类型判断依据摘要。")
|
||||
document_fields: list[OcrRecognizeFieldRead] = Field(
|
||||
|
||||
@@ -58,18 +58,18 @@ class SettingsLlmForm(BaseModel):
|
||||
backupApiKey: str = Field(default="", max_length=1024)
|
||||
backupApiKeyConfigured: bool = False
|
||||
|
||||
vlmProvider: str = Field(min_length=1, max_length=64)
|
||||
vlmModel: str = Field(min_length=1, max_length=255)
|
||||
vlmEndpoint: str = Field(min_length=1, max_length=512)
|
||||
vlmApiKey: str = Field(default="", max_length=1024)
|
||||
vlmApiKeyConfigured: bool = False
|
||||
|
||||
embeddingProvider: str = Field(min_length=1, max_length=64)
|
||||
embeddingModel: str = Field(min_length=1, max_length=255)
|
||||
embeddingEndpoint: str = Field(min_length=1, max_length=512)
|
||||
embeddingApiKey: str = Field(default="", max_length=1024)
|
||||
embeddingApiKeyConfigured: bool = False
|
||||
|
||||
rerankerProvider: str = Field(min_length=1, max_length=64)
|
||||
rerankerModel: str = Field(min_length=1, max_length=255)
|
||||
rerankerEndpoint: str = Field(min_length=1, max_length=512)
|
||||
rerankerApiKey: str = Field(default="", max_length=1024)
|
||||
rerankerApiKeyConfigured: bool = False
|
||||
|
||||
@field_validator(
|
||||
"mainProvider",
|
||||
"mainModel",
|
||||
@@ -79,14 +79,14 @@ class SettingsLlmForm(BaseModel):
|
||||
"backupModel",
|
||||
"backupEndpoint",
|
||||
"backupApiKey",
|
||||
"vlmProvider",
|
||||
"vlmModel",
|
||||
"vlmEndpoint",
|
||||
"vlmApiKey",
|
||||
"embeddingProvider",
|
||||
"embeddingModel",
|
||||
"embeddingEndpoint",
|
||||
"embeddingApiKey",
|
||||
"rerankerProvider",
|
||||
"rerankerModel",
|
||||
"rerankerEndpoint",
|
||||
"rerankerApiKey",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
@@ -185,8 +185,8 @@ class ModelConnectivityTestRequest(BaseModel):
|
||||
endpoint: str = Field(min_length=1, max_length=512)
|
||||
model: str = Field(min_length=1, max_length=255)
|
||||
api_key: str | None = Field(default=None, max_length=1024)
|
||||
capability: Literal["chat", "embedding"] = "chat"
|
||||
slot: Literal["main", "backup", "vlm", "embedding"] | None = None
|
||||
capability: Literal["chat", "embedding", "reranker"] = "chat"
|
||||
slot: Literal["main", "backup", "embedding", "reranker"] | None = None
|
||||
|
||||
@field_validator("provider", "endpoint", "model", "api_key", mode="before")
|
||||
@classmethod
|
||||
@@ -208,9 +208,9 @@ class ModelConnectivityTestRead(BaseModel):
|
||||
|
||||
|
||||
class RuntimeModelConfigRead(BaseModel):
|
||||
slot: Literal["main", "backup", "vlm", "embedding"]
|
||||
slot: Literal["main", "backup", "embedding", "reranker"]
|
||||
provider: str
|
||||
model: str
|
||||
endpoint: str
|
||||
apiKey: str
|
||||
capability: Literal["chat", "embedding"]
|
||||
capability: Literal["chat", "embedding", "reranker"]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,8 +9,6 @@ from typing import Any
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.runtime_chat import RuntimeChatService
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentField:
|
||||
@@ -198,7 +196,7 @@ MERCHANT_PATTERNS = (
|
||||
|
||||
class DocumentIntelligenceService:
|
||||
def __init__(self, db: Session | None = None) -> None:
|
||||
self.runtime_chat_service = RuntimeChatService(db) if db is not None else None
|
||||
self.db = db
|
||||
|
||||
def build_document_insight(
|
||||
self,
|
||||
@@ -254,95 +252,6 @@ class DocumentIntelligenceService:
|
||||
rule_insight: DocumentInsight,
|
||||
fields: tuple[DocumentField, ...],
|
||||
) -> tuple[str, LlmDocumentClassification] | None:
|
||||
if self.runtime_chat_service is None:
|
||||
return None
|
||||
|
||||
trimmed_text = text.strip()
|
||||
if not trimmed_text and not summary.strip():
|
||||
return None
|
||||
|
||||
facts = {
|
||||
"filename": filename,
|
||||
"summary": summary[:300],
|
||||
"ocr_text_excerpt": trimmed_text[:2000],
|
||||
"rule_candidate": {
|
||||
"document_type": rule_insight.document_type,
|
||||
"document_type_label": rule_insight.document_type_label,
|
||||
"scene_code": rule_insight.scene_code,
|
||||
"scene_label": rule_insight.scene_label,
|
||||
"expense_type": rule_insight.expense_type,
|
||||
"confidence": round(rule_insight.classification_confidence, 2),
|
||||
"evidence": list(rule_insight.evidence),
|
||||
},
|
||||
"extracted_fields": [
|
||||
{"key": field.key, "label": field.label, "value": field.value}
|
||||
for field in fields
|
||||
],
|
||||
"allowed_document_types": list(SUPPORTED_DOCUMENT_TYPES),
|
||||
}
|
||||
|
||||
system_prompt = (
|
||||
"你是企业报销票据识别复核器。"
|
||||
"你的任务不是 OCR,而是在已有 OCR 文本和票据预览基础上判断票据类型,并尽量复核关键字段。"
|
||||
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。"
|
||||
"document_type 只能是:"
|
||||
f"{', '.join(SUPPORTED_DOCUMENT_TYPES)}。"
|
||||
"如果证据不足,返回 other。"
|
||||
"严禁编造 OCR 中不存在的商户、酒店、航司、路线或金额。"
|
||||
"如果 OCR 出现冲突碎片,应优先依据票据主体信息,而不是单个噪声词。"
|
||||
"例如滴滴行程单/网约车发票,即使 OCR 混入酒店名称,也不能直接判成酒店票据。"
|
||||
"如果能从 OCR 或图片中明确确认字段,可在 fields 中返回。"
|
||||
"fields 只允许包含 key, label, value,key 只能是 amount, date, merchant_name, invoice_number, "
|
||||
"invoice_code, trip_no, route。无法确认就不要返回该字段。"
|
||||
"输出字段:document_type, scene_code, scene_label, expense_type, confidence, evidence, fields。"
|
||||
)
|
||||
user_prompt = (
|
||||
"请根据以下票据事实给出最终分类 JSON:\n"
|
||||
f"{json.dumps(facts, ensure_ascii=False, indent=2)}\n\n"
|
||||
"示例输出:\n"
|
||||
"{\n"
|
||||
' "document_type": "taxi_receipt",\n'
|
||||
' "scene_code": "transport",\n'
|
||||
' "scene_label": "交通票据",\n'
|
||||
' "expense_type": "transport",\n'
|
||||
' "confidence": 0.86,\n'
|
||||
' "evidence": ["OCR 中出现 滴滴出行、订单号、上车/下车 等交通特征"],\n'
|
||||
' "fields": [{"key": "amount", "label": "金额", "value": "32.5"}]\n'
|
||||
"}"
|
||||
)
|
||||
|
||||
if preview_data_url:
|
||||
response_text = self.runtime_chat_service.complete(
|
||||
[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": user_prompt},
|
||||
{"type": "image_url", "image_url": {"url": preview_data_url}},
|
||||
],
|
||||
},
|
||||
],
|
||||
slot_priority=("vlm",),
|
||||
max_tokens=320,
|
||||
temperature=0.0,
|
||||
)
|
||||
parsed = self._parse_llm_payload(response_text)
|
||||
if parsed is not None:
|
||||
return "llm_vision", parsed
|
||||
|
||||
response_text = self.runtime_chat_service.complete(
|
||||
[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
slot_priority=("main", "backup"),
|
||||
max_tokens=320,
|
||||
temperature=0.0,
|
||||
)
|
||||
parsed = self._parse_llm_payload(response_text)
|
||||
if parsed is not None:
|
||||
return "llm_text", parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -682,11 +682,13 @@ class ExpenseClaimService:
|
||||
raise ValueError("提交前请先补全信息:" + ";".join(missing_fields))
|
||||
|
||||
before_json = self._serialize_claim(claim)
|
||||
review_result = self._run_ai_submission_review(claim)
|
||||
claim.status = str(review_result["status"])
|
||||
claim.approval_stage = str(review_result["approval_stage"])
|
||||
claim.risk_flags_json = list(review_result["risk_flags"])
|
||||
claim.submitted_at = datetime.now(UTC) if claim.status == "submitted" else None
|
||||
# TODO: 后续恢复 AI 验审逻辑
|
||||
# review_result = self._run_ai_submission_review(claim)
|
||||
manager_name = self._resolve_claim_manager_name(claim) or "审批人"
|
||||
claim.status = "submitted"
|
||||
claim.approval_stage = "直属领导审批"
|
||||
claim.risk_flags_json = list(claim.risk_flags_json or [])
|
||||
claim.submitted_at = datetime.now(UTC)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(claim)
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.agent_enums import AgentRunStatus
|
||||
from app.schemas.hermes import HermesCallbackRead, HermesCallbackWrite
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import KNOWLEDGE_INGEST_STATUS_FAILED, KnowledgeService
|
||||
from app.services.llm_wiki import LlmWikiService
|
||||
|
||||
|
||||
class HermesCallbackService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.db = db
|
||||
self.run_service = AgentRunService(db)
|
||||
|
||||
def handle_callback(self, payload: HermesCallbackWrite) -> HermesCallbackRead:
|
||||
run = self.run_service.get_run(payload.run_id)
|
||||
if run is None:
|
||||
raise LookupError("Hermes 回调引用的 AgentRun 不存在。")
|
||||
|
||||
if payload.type == "llm_wiki_sync":
|
||||
self._handle_llm_wiki_sync(payload)
|
||||
else:
|
||||
raise ValueError(f"暂不支持的 Hermes 回调类型:{payload.type}")
|
||||
|
||||
return HermesCallbackRead(
|
||||
type=payload.type,
|
||||
run_id=payload.run_id,
|
||||
status=payload.status,
|
||||
)
|
||||
|
||||
def _handle_llm_wiki_sync(self, payload: HermesCallbackWrite) -> None:
|
||||
run = self.run_service.get_run(payload.run_id)
|
||||
if run is None:
|
||||
raise LookupError("Hermes 回调引用的 AgentRun 不存在。")
|
||||
|
||||
route_json = dict(run.route_json or {})
|
||||
document_ids = [
|
||||
str(item).strip()
|
||||
for item in list(route_json.get("requested_document_ids") or [])
|
||||
if str(item).strip()
|
||||
]
|
||||
if payload.status == "running":
|
||||
self.run_service.merge_route_json(
|
||||
payload.run_id,
|
||||
{
|
||||
"phase": "running",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"callback_status": payload.status,
|
||||
"callback_payload": payload.payload,
|
||||
},
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary=payload.summary or run.result_summary,
|
||||
)
|
||||
return
|
||||
|
||||
if payload.status == "failed":
|
||||
if document_ids:
|
||||
KnowledgeService().set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=payload.run_id,
|
||||
)
|
||||
self.run_service.record_tool_call(
|
||||
run_id=payload.run_id,
|
||||
tool_type="http",
|
||||
tool_name="hermes_callback",
|
||||
request_json=payload.model_dump(mode="json"),
|
||||
response_json={},
|
||||
status="failed",
|
||||
duration_ms=0,
|
||||
error_message=payload.error or payload.summary or "Hermes callback failed",
|
||||
)
|
||||
self.run_service.merge_route_json(
|
||||
payload.run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"callback_status": payload.status,
|
||||
"callback_payload": payload.payload,
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary=payload.summary or payload.error or "Hermes 任务失败。",
|
||||
error_message=payload.error or payload.summary or "Hermes 任务失败。",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
|
||||
result = LlmWikiService(self.db).finalize_agent_batch_callback(
|
||||
agent_run_id=payload.run_id,
|
||||
payload=payload.payload,
|
||||
)
|
||||
self.run_service.record_tool_call(
|
||||
run_id=payload.run_id,
|
||||
tool_type="http",
|
||||
tool_name="hermes_callback",
|
||||
request_json=payload.model_dump(mode="json"),
|
||||
response_json=result.model_dump(mode="json"),
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
self.run_service.merge_route_json(
|
||||
payload.run_id,
|
||||
{
|
||||
"phase": "succeeded",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"callback_status": payload.status,
|
||||
"sync_run_id": result.run_id,
|
||||
"sync_result": result.model_dump(mode="json"),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": result.document_count,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": max(0, len(document_ids) - result.document_count),
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
result_summary=payload.summary or result.summary,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
216
server/src/app/services/knowledge_index_tasks.py
Normal file
216
server/src/app/services/knowledge_index_tasks.py
Normal file
@@ -0,0 +1,216 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentName, AgentRunStatus, AgentToolType
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import (
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
KNOWLEDGE_INGEST_STATUS_INGESTED,
|
||||
KnowledgeService,
|
||||
)
|
||||
from app.services.knowledge_rag import KnowledgeRagService
|
||||
|
||||
logger = get_logger("app.services.knowledge_index_tasks")
|
||||
|
||||
|
||||
class KnowledgeIndexTaskManager:
|
||||
def __init__(self) -> None:
|
||||
self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="knowledge-index")
|
||||
self._futures: dict[str, Future[Any]] = {}
|
||||
|
||||
def submit_sync(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str],
|
||||
force: bool,
|
||||
) -> None:
|
||||
future = self._executor.submit(
|
||||
self._run_sync,
|
||||
agent_run_id,
|
||||
folder,
|
||||
current_user,
|
||||
[str(item).strip() for item in document_ids if str(item).strip()],
|
||||
force,
|
||||
)
|
||||
self._futures[agent_run_id] = future
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
@staticmethod
|
||||
def _run_sync(
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str],
|
||||
force: bool,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
db = session_factory()
|
||||
started = perf_counter()
|
||||
|
||||
try:
|
||||
run_service = AgentRunService(db)
|
||||
knowledge_service = KnowledgeService(db=db)
|
||||
rag_service = KnowledgeRagService(db=db)
|
||||
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "indexing",
|
||||
"folder": folder,
|
||||
"force": force,
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"requested_document_ids": document_ids,
|
||||
"requested_by_username": current_user.username,
|
||||
"requested_by_name": current_user.name,
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 10 if document_ids else 100,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
response = rag_service.index_documents(document_ids=document_ids, force=force)
|
||||
succeeded_document_ids = [
|
||||
str(item).strip()
|
||||
for item in list(response.get("succeeded_document_ids") or [])
|
||||
if str(item).strip()
|
||||
]
|
||||
failed_documents = [
|
||||
item
|
||||
for item in list(response.get("failed_documents") or [])
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
failed_document_ids = [
|
||||
str(item.get("document_id") or "").strip()
|
||||
for item in failed_documents
|
||||
if str(item.get("document_id") or "").strip()
|
||||
]
|
||||
|
||||
if succeeded_document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
succeeded_document_ids,
|
||||
KNOWLEDGE_INGEST_STATUS_INGESTED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
if failed_document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
failed_document_ids,
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
|
||||
duration_ms = int((perf_counter() - started) * 1000)
|
||||
tool_status = "succeeded" if not failed_document_ids else "failed"
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type=AgentToolType.LLM.value,
|
||||
tool_name="lightrag.index_documents",
|
||||
request_json={
|
||||
"agent": AgentName.HERMES.value,
|
||||
"folder": folder,
|
||||
"document_ids": document_ids,
|
||||
"force": force,
|
||||
},
|
||||
response_json=response,
|
||||
status=tool_status,
|
||||
duration_ms=duration_ms,
|
||||
error_message=None if tool_status == "succeeded" else "部分文档索引失败。",
|
||||
)
|
||||
|
||||
completed_documents = len(succeeded_document_ids)
|
||||
failed_count = len(failed_document_ids)
|
||||
total_documents = len(document_ids)
|
||||
summary = (
|
||||
f"LightRAG 已完成 {completed_documents}/{total_documents} 个知识文档索引。"
|
||||
if failed_count == 0
|
||||
else f"LightRAG 已完成 {completed_documents}/{total_documents} 个知识文档索引,失败 {failed_count} 个。"
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "completed",
|
||||
"track_id": str(response.get("track_id") or "").strip(),
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": total_documents,
|
||||
"completed_documents": completed_documents,
|
||||
"failed_documents": failed_count,
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=(
|
||||
AgentRunStatus.SUCCEEDED.value
|
||||
if failed_count == 0
|
||||
else AgentRunStatus.FAILED.value
|
||||
),
|
||||
result_summary=summary,
|
||||
error_message="部分文档索引失败。" if failed_count else None,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
except Exception as exc:
|
||||
try:
|
||||
AgentRunService(db).record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type=AgentToolType.LLM.value,
|
||||
tool_name="lightrag.index_documents",
|
||||
request_json={
|
||||
"agent": AgentName.HERMES.value,
|
||||
"folder": folder,
|
||||
"document_ids": document_ids,
|
||||
"force": force,
|
||||
},
|
||||
response_json={"error": str(exc)},
|
||||
status="failed",
|
||||
duration_ms=int((perf_counter() - started) * 1000),
|
||||
error_message=str(exc),
|
||||
)
|
||||
KnowledgeService(db=db).set_document_ingest_statuses(
|
||||
document_ids,
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
AgentRunService(db).merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": len(document_ids),
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary=str(exc),
|
||||
error_message=str(exc),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Knowledge index task finalization failed run_id=%s", agent_run_id)
|
||||
logger.exception("Knowledge index task failed run_id=%s", agent_run_id)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
knowledge_index_task_manager = KnowledgeIndexTaskManager()
|
||||
414
server/src/app/services/knowledge_normalizer.py
Normal file
414
server/src/app/services/knowledge_normalizer.py
Normal file
@@ -0,0 +1,414 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.services.runtime_chat import RuntimeChatService
|
||||
|
||||
logger = get_logger("app.services.knowledge_normalizer")
|
||||
|
||||
TABLE_MARKER_PATTERN = re.compile(r"表\s*(\d+)")
|
||||
SECTION_HEADING_PATTERN = re.compile(
|
||||
r"^(第[一二三四五六七八九十百零0-9]+[章节]\s*.*|[一二三四五六七八九十]+、.*|([一二三四五六七八九十]+).*|\([一二三四五六七八九十]+\).*)$"
|
||||
)
|
||||
LIST_ITEM_PATTERN = re.compile(r"^[-*•]\s+.+$")
|
||||
NUMBERED_ITEM_PATTERN = re.compile(r"^(?:\d+[.)、]|[①②③④⑤⑥⑦⑧⑨⑩])\s*.+$")
|
||||
ARTICLE_PATTERN = re.compile(r"^(第[一二三四五六七八九十百零0-9]+条)\s*.*$")
|
||||
KEY_VALUE_PATTERN = re.compile(r"^[^::\s][^::]{0,40}[::]\s*.+$")
|
||||
MAX_TABLE_WINDOW_CHARS = 1800
|
||||
MAX_TABLES_PER_DOCUMENT = 8
|
||||
MAX_SECTION_OUTLINE_ITEMS = 12
|
||||
MAX_SECTION_SNIPPETS = 8
|
||||
MAX_SECTION_SNIPPET_CHARS = 220
|
||||
MAX_SECTION_QA_CLUES = 4
|
||||
MAX_TOTAL_QA_CLUES = 24
|
||||
MAX_QA_CLUE_CHARS = 180
|
||||
FACT_KEYWORDS = (
|
||||
"适用",
|
||||
"标准",
|
||||
"条件",
|
||||
"流程",
|
||||
"审批",
|
||||
"提交",
|
||||
"附件",
|
||||
"材料",
|
||||
"票据",
|
||||
"报销",
|
||||
"限额",
|
||||
"金额",
|
||||
"比例",
|
||||
"范围",
|
||||
"对象",
|
||||
"人员",
|
||||
"时限",
|
||||
"工作日",
|
||||
"不得",
|
||||
"可以",
|
||||
"应当",
|
||||
"应",
|
||||
"需",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TableCandidate:
|
||||
title: str
|
||||
excerpt: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SectionCandidate:
|
||||
title: str
|
||||
excerpt: str
|
||||
body_lines: tuple[str, ...]
|
||||
|
||||
|
||||
class KnowledgeNormalizationService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.runtime_chat_service = RuntimeChatService(db)
|
||||
|
||||
def build_enriched_text(self, raw_text: str) -> str:
|
||||
normalized_text = str(raw_text or "").strip()
|
||||
if not normalized_text:
|
||||
return ""
|
||||
|
||||
section_appendix = self._build_section_appendix(normalized_text)
|
||||
answer_clue_appendix = self._build_answer_clue_appendix(normalized_text)
|
||||
normalized_tables: list[str] = []
|
||||
for candidate in self._extract_table_candidates(normalized_text):
|
||||
rendered = self._normalize_table_candidate(candidate)
|
||||
if rendered:
|
||||
normalized_tables.append(f"## {candidate.title}\n\n{rendered}")
|
||||
|
||||
parts: list[str] = []
|
||||
if section_appendix:
|
||||
parts.append(section_appendix)
|
||||
if answer_clue_appendix:
|
||||
parts.append(answer_clue_appendix)
|
||||
if normalized_tables:
|
||||
appendix = "\n\n".join(normalized_tables)
|
||||
parts.append(
|
||||
"# 结构化表格补充\n\n"
|
||||
"以下表格由知识归纳阶段依据原文重新整理,供问答检索时优先理解行列关系。\n\n"
|
||||
f"{appendix}"
|
||||
)
|
||||
|
||||
if not parts:
|
||||
return normalized_text
|
||||
|
||||
parts.append(f"# 原文\n\n{normalized_text}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def _extract_table_candidates(text: str) -> list[TableCandidate]:
|
||||
candidates: list[TableCandidate] = []
|
||||
occupied_ranges: list[tuple[int, int]] = []
|
||||
|
||||
for match in TABLE_MARKER_PATTERN.finditer(text):
|
||||
if len(candidates) >= MAX_TABLES_PER_DOCUMENT:
|
||||
break
|
||||
|
||||
start = text.rfind("\n", 0, match.start())
|
||||
start = 0 if start < 0 else start + 1
|
||||
end = min(len(text), start + MAX_TABLE_WINDOW_CHARS)
|
||||
if any(start < existing_end and end > existing_start for existing_start, existing_end in occupied_ranges):
|
||||
continue
|
||||
|
||||
excerpt = text[start:end].strip()
|
||||
head = excerpt[:360]
|
||||
if "单位:" not in head and "标准" not in head:
|
||||
continue
|
||||
if excerpt.count("\n") < 6 or sum(char.isdigit() for char in excerpt) < 4:
|
||||
continue
|
||||
|
||||
marker = match.group(0).replace(" ", "")
|
||||
first_line = next((line.strip() for line in excerpt.splitlines() if line.strip()), marker)
|
||||
title = first_line if first_line.startswith(marker) else marker
|
||||
candidates.append(TableCandidate(title=title, excerpt=excerpt))
|
||||
occupied_ranges.append((start, end))
|
||||
|
||||
return candidates
|
||||
|
||||
def _normalize_table_candidate(self, candidate: TableCandidate) -> str:
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是制度文档结构化助手。"
|
||||
"只依据用户提供的原文,提炼其中的表格为清晰 Markdown。"
|
||||
"必须严格按照表头从左到右对齐每个数值,不能猜测、不能改列顺序、不能擅自补全。"
|
||||
"只输出一张 Markdown 表格本身,不要输出标题、说明、注释、脚注或正文解释。"
|
||||
"如果原文不足以确认表格关系,只回复“无法确认”。"
|
||||
"不要输出思考过程,不要复述原文,不要添加制度之外的新事实。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"请仅整理下面这段制度表格,标题为《{candidate.title}》。\n\n"
|
||||
f"{candidate.excerpt}"
|
||||
),
|
||||
},
|
||||
]
|
||||
answer = self.runtime_chat_service.complete(
|
||||
messages,
|
||||
max_tokens=900,
|
||||
temperature=0.0,
|
||||
)
|
||||
cleaned = self._sanitize_answer(answer)
|
||||
if not cleaned or cleaned == "无法确认":
|
||||
return ""
|
||||
if cleaned.count("|") < 6:
|
||||
logger.info("Skip non-tabular normalization candidate title=%s", candidate.title)
|
||||
return ""
|
||||
return cleaned
|
||||
|
||||
@staticmethod
|
||||
def _build_section_appendix(text: str) -> str:
|
||||
candidates = KnowledgeNormalizationService._extract_section_candidates(text)
|
||||
if len(candidates) < 2:
|
||||
return ""
|
||||
|
||||
outline = "\n".join(
|
||||
f"- {item.title}"
|
||||
for item in candidates[:MAX_SECTION_OUTLINE_ITEMS]
|
||||
)
|
||||
snippets = "\n\n".join(
|
||||
[
|
||||
f"## {item.title}\n\n{item.excerpt}"
|
||||
for item in candidates[:MAX_SECTION_SNIPPETS]
|
||||
if item.excerpt
|
||||
]
|
||||
)
|
||||
if not snippets:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"# 章节导航\n\n"
|
||||
"以下内容由入库阶段从制度原文中提取,供检索时优先理解制度层级、条目和标准所在章节。\n\n"
|
||||
f"{outline}\n\n"
|
||||
"# 重点章节摘录\n\n"
|
||||
f"{snippets}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_answer_clue_appendix(text: str) -> str:
|
||||
candidates = KnowledgeNormalizationService._extract_section_candidates(text)
|
||||
clue_lines: list[str] = []
|
||||
|
||||
if candidates:
|
||||
for candidate in candidates:
|
||||
clue_lines.extend(
|
||||
KnowledgeNormalizationService._extract_section_clues(candidate)
|
||||
)
|
||||
else:
|
||||
clue_lines.extend(KnowledgeNormalizationService._extract_freeform_clues(text))
|
||||
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in clue_lines:
|
||||
normalized = re.sub(r"\s+", " ", str(item or "")).strip()
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
deduped.append(normalized)
|
||||
if len(deduped) >= MAX_TOTAL_QA_CLUES:
|
||||
break
|
||||
|
||||
if len(deduped) < 2:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"# 问答线索补充\n\n"
|
||||
"以下内容由入库阶段根据章节标题、条款、列表、键值对与相邻正文提炼,"
|
||||
"供问答检索时优先命中更短、更直接的制度依据。\n\n"
|
||||
+ "\n".join(f"- {item}" for item in deduped)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_section_candidates(text: str) -> list[SectionCandidate]:
|
||||
lines = [line.rstrip() for line in str(text or "").splitlines()]
|
||||
sections: list[SectionCandidate] = []
|
||||
current_title = ""
|
||||
current_body: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current_title, current_body
|
||||
if not current_title:
|
||||
current_body = []
|
||||
return
|
||||
excerpt = KnowledgeNormalizationService._build_section_excerpt(current_body)
|
||||
if excerpt:
|
||||
sections.append(
|
||||
SectionCandidate(
|
||||
title=current_title,
|
||||
excerpt=excerpt,
|
||||
body_lines=tuple(current_body),
|
||||
)
|
||||
)
|
||||
current_title = ""
|
||||
current_body = []
|
||||
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
if current_body:
|
||||
current_body.append("")
|
||||
continue
|
||||
|
||||
if SECTION_HEADING_PATTERN.match(line) and len(line) <= 80:
|
||||
flush()
|
||||
current_title = line
|
||||
continue
|
||||
|
||||
if current_title:
|
||||
current_body.append(line)
|
||||
|
||||
flush()
|
||||
return sections
|
||||
|
||||
@staticmethod
|
||||
def _build_section_excerpt(lines: list[str]) -> str:
|
||||
cleaned_lines = [line.strip() for line in lines if line.strip()]
|
||||
if not cleaned_lines:
|
||||
return ""
|
||||
excerpt = ";".join(cleaned_lines[:3]).strip()
|
||||
if len(excerpt) <= MAX_SECTION_SNIPPET_CHARS:
|
||||
return excerpt
|
||||
return f"{excerpt[: MAX_SECTION_SNIPPET_CHARS - 3].rstrip()}..."
|
||||
|
||||
@staticmethod
|
||||
def _extract_section_clues(candidate: SectionCandidate) -> list[str]:
|
||||
clues: list[str] = []
|
||||
fallback: list[str] = []
|
||||
|
||||
for raw_line in candidate.body_lines:
|
||||
normalized_line = KnowledgeNormalizationService._normalize_fact_line(raw_line)
|
||||
if not normalized_line or KnowledgeNormalizationService._is_table_like_line(normalized_line):
|
||||
continue
|
||||
|
||||
fact_units = KnowledgeNormalizationService._split_fact_units(normalized_line)
|
||||
for unit in fact_units:
|
||||
rendered = KnowledgeNormalizationService._render_clue(candidate.title, unit)
|
||||
if not rendered:
|
||||
continue
|
||||
if KnowledgeNormalizationService._looks_like_fact_line(unit):
|
||||
clues.append(rendered)
|
||||
elif len(fallback) < 2:
|
||||
fallback.append(rendered)
|
||||
|
||||
if len(clues) >= MAX_SECTION_QA_CLUES:
|
||||
return clues[:MAX_SECTION_QA_CLUES]
|
||||
|
||||
return clues[:MAX_SECTION_QA_CLUES] or fallback[:2]
|
||||
|
||||
@staticmethod
|
||||
def _extract_freeform_clues(text: str) -> list[str]:
|
||||
clues: list[str] = []
|
||||
for raw_line in str(text or "").splitlines():
|
||||
normalized_line = KnowledgeNormalizationService._normalize_fact_line(raw_line)
|
||||
if (
|
||||
not normalized_line
|
||||
or SECTION_HEADING_PATTERN.match(normalized_line)
|
||||
or KnowledgeNormalizationService._is_table_like_line(normalized_line)
|
||||
or not KnowledgeNormalizationService._looks_like_fact_line(normalized_line)
|
||||
):
|
||||
continue
|
||||
|
||||
for unit in KnowledgeNormalizationService._split_fact_units(normalized_line):
|
||||
rendered = KnowledgeNormalizationService._render_clue("正文", unit)
|
||||
if rendered:
|
||||
clues.append(rendered)
|
||||
if len(clues) >= MAX_TOTAL_QA_CLUES:
|
||||
return clues
|
||||
return clues
|
||||
|
||||
@staticmethod
|
||||
def _split_fact_units(line: str) -> list[str]:
|
||||
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
|
||||
if not normalized:
|
||||
return []
|
||||
if len(normalized) <= MAX_QA_CLUE_CHARS and all(mark not in normalized for mark in (";", ";", "。")):
|
||||
return [normalized]
|
||||
|
||||
units: list[str] = []
|
||||
for part in re.split(r"[;;。]\s*", normalized):
|
||||
cleaned = KnowledgeNormalizationService._normalize_fact_line(part)
|
||||
if not cleaned:
|
||||
continue
|
||||
units.append(cleaned)
|
||||
return units or [KnowledgeNormalizationService._truncate_clue(normalized)]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_fact_line(line: str) -> str:
|
||||
normalized = str(line or "").strip()
|
||||
normalized = re.sub(r"\s+", " ", normalized)
|
||||
return normalized.strip(" -")
|
||||
|
||||
@staticmethod
|
||||
def _is_table_like_line(line: str) -> bool:
|
||||
normalized = str(line or "").strip()
|
||||
if not normalized:
|
||||
return False
|
||||
if normalized.count("|") >= 2:
|
||||
return True
|
||||
if normalized.count("\t") >= 2:
|
||||
return True
|
||||
number_tokens = re.findall(r"\d+(?:[.][0-9]+)?", normalized)
|
||||
if len(number_tokens) >= 3 and len(normalized.split()) >= 4 and not any(
|
||||
punct in normalized for punct in ("。", ";", ";", ":", ":")
|
||||
):
|
||||
return True
|
||||
return "单位:" in normalized and sum(char.isdigit() for char in normalized) >= 3
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_fact_line(line: str) -> bool:
|
||||
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
|
||||
if len(normalized) < 6:
|
||||
return False
|
||||
if TABLE_MARKER_PATTERN.search(normalized) or normalized.startswith(("单位:", "单位:")):
|
||||
return False
|
||||
if (
|
||||
ARTICLE_PATTERN.match(normalized)
|
||||
or LIST_ITEM_PATTERN.match(normalized)
|
||||
or NUMBERED_ITEM_PATTERN.match(normalized)
|
||||
or KEY_VALUE_PATTERN.match(normalized)
|
||||
):
|
||||
return True
|
||||
if any(keyword in normalized for keyword in FACT_KEYWORDS):
|
||||
return True
|
||||
return any(char.isdigit() for char in normalized)
|
||||
|
||||
@staticmethod
|
||||
def _render_clue(section_title: str, line: str) -> str:
|
||||
normalized_line = KnowledgeNormalizationService._truncate_clue(line)
|
||||
if not normalized_line:
|
||||
return ""
|
||||
normalized_title = str(section_title or "").strip()
|
||||
if not normalized_title:
|
||||
return normalized_line
|
||||
return f"{normalized_title}:{normalized_line}"
|
||||
|
||||
@staticmethod
|
||||
def _truncate_clue(line: str) -> str:
|
||||
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
|
||||
if len(normalized) <= MAX_QA_CLUE_CHARS:
|
||||
return normalized
|
||||
return f"{normalized[: MAX_QA_CLUE_CHARS - 3].rstrip()}..."
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_answer(answer: str | None) -> str:
|
||||
cleaned = re.sub(r"<think>.*?</think>", "", str(answer or ""), flags=re.DOTALL | re.IGNORECASE)
|
||||
lines = [line.rstrip() for line in cleaned.strip().splitlines()]
|
||||
table_lines: list[str] = []
|
||||
for line in lines:
|
||||
normalized = line.strip()
|
||||
if "|" not in normalized:
|
||||
if table_lines:
|
||||
break
|
||||
continue
|
||||
table_lines.append(normalized)
|
||||
return "\n".join(table_lines).strip()
|
||||
1261
server/src/app/services/knowledge_rag.py
Normal file
1261
server/src/app/services/knowledge_rag.py
Normal file
File diff suppressed because it is too large
Load Diff
94
server/src/app/services/knowledge_scheduler.py
Normal file
94
server/src/app/services/knowledge_scheduler.py
Normal file
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from datetime import datetime, time, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentRunSource
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.knowledge_sync import KnowledgeSyncDispatchService
|
||||
|
||||
logger = get_logger("app.services.knowledge_scheduler")
|
||||
|
||||
|
||||
class KnowledgeIndexScheduler:
|
||||
def __init__(self) -> None:
|
||||
timezone_name = str(os.environ.get("X_FINANCIAL_SCHEDULER_TZ") or "Asia/Shanghai").strip() or "Asia/Shanghai"
|
||||
self._timezone = ZoneInfo(timezone_name)
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def start(self) -> None:
|
||||
with self._lock:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_loop,
|
||||
name="knowledge-index-scheduler",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
logger.info("Knowledge index scheduler started timezone=%s trigger=00:00", self._timezone.key)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._lock:
|
||||
thread = self._thread
|
||||
self._thread = None
|
||||
self._stop_event.set()
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=3)
|
||||
logger.info("Knowledge index scheduler stopped")
|
||||
|
||||
def _run_loop(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
now = datetime.now(self._timezone)
|
||||
next_run = self._resolve_next_run(now)
|
||||
wait_seconds = max(1.0, (next_run - now).total_seconds())
|
||||
if self._stop_event.wait(wait_seconds):
|
||||
break
|
||||
try:
|
||||
self._run_incremental_sync()
|
||||
except Exception: # pragma: no cover - scheduler best effort logging
|
||||
logger.exception("Scheduled knowledge index sync failed")
|
||||
|
||||
def _run_incremental_sync(self) -> None:
|
||||
db = get_session_factory()()
|
||||
try:
|
||||
current_user = CurrentUserContext(
|
||||
username="hermes",
|
||||
name="Hermes",
|
||||
role_codes=["manager"],
|
||||
is_admin=True,
|
||||
)
|
||||
result = KnowledgeSyncDispatchService(db).queue_sync(
|
||||
current_user=current_user,
|
||||
folder=None,
|
||||
document_ids=None,
|
||||
source=AgentRunSource.SCHEDULE.value,
|
||||
force=False,
|
||||
changed_only=True,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduled knowledge index sync result run_id=%s docs=%s reused=%s summary=%s",
|
||||
result.agent_run_id,
|
||||
len(result.document_ids),
|
||||
result.reused,
|
||||
result.summary,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@staticmethod
|
||||
def _resolve_next_run(now: datetime) -> datetime:
|
||||
today_midnight = datetime.combine(now.date(), time(hour=0, minute=0), tzinfo=now.tzinfo)
|
||||
if now < today_midnight:
|
||||
return today_midnight
|
||||
return today_midnight + timedelta(days=1)
|
||||
|
||||
|
||||
knowledge_index_scheduler = KnowledgeIndexScheduler()
|
||||
244
server/src/app/services/knowledge_sync.py
Normal file
244
server/src/app/services/knowledge_sync.py
Normal file
@@ -0,0 +1,244 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentName, AgentPermissionLevel, AgentRunSource, AgentRunStatus
|
||||
from app.models.agent_asset import AgentAsset
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import (
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
KNOWLEDGE_INGEST_STATUS_SYNCING,
|
||||
KnowledgeService,
|
||||
)
|
||||
from app.services.knowledge_index_tasks import knowledge_index_task_manager
|
||||
|
||||
ALL_KNOWLEDGE_FOLDERS_LABEL = "全部知识库"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class KnowledgeSyncDispatchResult:
|
||||
ok: bool = True
|
||||
agent_run_id: str = ""
|
||||
folder: str = ""
|
||||
document_ids: list[str] = field(default_factory=list)
|
||||
queued_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
status: str = AgentRunStatus.SUCCEEDED.value
|
||||
summary: str = ""
|
||||
reused: bool = False
|
||||
|
||||
|
||||
class KnowledgeSyncDispatchService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.db = db
|
||||
self.run_service = AgentRunService(db)
|
||||
self.knowledge_service = KnowledgeService(db=db)
|
||||
|
||||
def queue_sync(
|
||||
self,
|
||||
*,
|
||||
current_user: CurrentUserContext,
|
||||
folder: str | None = None,
|
||||
document_ids: list[str] | None = None,
|
||||
source: str = AgentRunSource.USER_MESSAGE.value,
|
||||
force: bool = False,
|
||||
changed_only: bool = True,
|
||||
) -> KnowledgeSyncDispatchResult:
|
||||
normalized_folder = str(folder or "").strip() or None
|
||||
folder_label = normalized_folder or ALL_KNOWLEDGE_FOLDERS_LABEL
|
||||
normalized_requested_ids = [
|
||||
str(item).strip()
|
||||
for item in document_ids or []
|
||||
if str(item).strip()
|
||||
]
|
||||
|
||||
all_documents = self.knowledge_service.list_documents_for_ingest(
|
||||
folder=normalized_folder,
|
||||
document_ids=normalized_requested_ids,
|
||||
changed_only=False,
|
||||
)
|
||||
target_documents = self.knowledge_service.list_documents_for_ingest(
|
||||
folder=normalized_folder,
|
||||
document_ids=normalized_requested_ids,
|
||||
changed_only=(False if force else changed_only),
|
||||
)
|
||||
target_document_ids = [
|
||||
str(item.get("id") or "").strip()
|
||||
for item in target_documents
|
||||
if str(item.get("id") or "").strip()
|
||||
]
|
||||
|
||||
if not all_documents:
|
||||
return KnowledgeSyncDispatchResult(
|
||||
folder=folder_label,
|
||||
document_ids=[],
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
summary="当前目录暂无可归纳的知识文档。",
|
||||
)
|
||||
|
||||
if not target_document_ids:
|
||||
return KnowledgeSyncDispatchResult(
|
||||
folder=folder_label,
|
||||
document_ids=[],
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
summary="当前目录没有需要增量归纳的文档。",
|
||||
)
|
||||
|
||||
active_run = self._find_active_run(
|
||||
folder=folder_label,
|
||||
requested_document_ids=target_document_ids,
|
||||
)
|
||||
if active_run is not None:
|
||||
active_document_ids = [
|
||||
str(item).strip()
|
||||
for item in list(active_run.route_json.get("requested_document_ids") or target_document_ids)
|
||||
if str(item).strip()
|
||||
]
|
||||
return KnowledgeSyncDispatchResult(
|
||||
agent_run_id=active_run.run_id,
|
||||
folder=folder_label,
|
||||
document_ids=active_document_ids,
|
||||
queued_at=active_run.started_at,
|
||||
status=active_run.status,
|
||||
summary="已有知识归纳任务正在执行,系统已复用当前任务。",
|
||||
reused=True,
|
||||
)
|
||||
|
||||
task_asset = self.db.scalar(
|
||||
select(AgentAsset).where(AgentAsset.code == "task.hermes.knowledge_index_sync")
|
||||
)
|
||||
run = self.run_service.create_run(
|
||||
agent=AgentName.HERMES.value,
|
||||
source=source,
|
||||
user_id=current_user.username,
|
||||
task_id=task_asset.id if task_asset is not None else None,
|
||||
permission_level=AgentPermissionLevel.READ.value,
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary="知识归纳任务已入队,等待后台执行。",
|
||||
route_json={
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "queued",
|
||||
"folder": folder_label,
|
||||
"force": force,
|
||||
"changed_only": (False if force else changed_only),
|
||||
"requested_document_ids": target_document_ids,
|
||||
"requested_by_username": current_user.username,
|
||||
"requested_by_name": current_user.name,
|
||||
"progress": {
|
||||
"total_documents": len(target_document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
self.knowledge_service.set_document_ingest_statuses(
|
||||
target_document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_SYNCING,
|
||||
agent_run_id=run.run_id,
|
||||
)
|
||||
knowledge_index_task_manager.submit_sync(
|
||||
agent_run_id=run.run_id,
|
||||
folder=folder_label,
|
||||
current_user=current_user,
|
||||
document_ids=target_document_ids,
|
||||
force=force,
|
||||
)
|
||||
return KnowledgeSyncDispatchResult(
|
||||
agent_run_id=run.run_id,
|
||||
folder=folder_label,
|
||||
document_ids=target_document_ids,
|
||||
queued_at=run.started_at,
|
||||
status=run.status,
|
||||
summary="知识归纳任务已进入后台执行,可在日志管理中查看进度。",
|
||||
)
|
||||
except Exception as exc:
|
||||
self.run_service.update_run(
|
||||
run.run_id,
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
error_message=str(exc),
|
||||
result_summary=str(exc),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
self.knowledge_service.set_document_ingest_statuses(
|
||||
target_document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=run.run_id,
|
||||
)
|
||||
raise
|
||||
|
||||
def _find_active_run(
|
||||
self,
|
||||
*,
|
||||
folder: str,
|
||||
requested_document_ids: list[str],
|
||||
):
|
||||
requested_set = {str(item).strip() for item in requested_document_ids if str(item).strip()}
|
||||
|
||||
for item in self.run_service.list_runs(
|
||||
agent=AgentName.HERMES.value,
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
limit=100,
|
||||
):
|
||||
if str(item.route_json.get("job_type") or "").strip() != "knowledge_index_sync":
|
||||
continue
|
||||
|
||||
heartbeat_raw = str(item.route_json.get("heartbeat_at") or "").strip()
|
||||
heartbeat_at = None
|
||||
if heartbeat_raw:
|
||||
try:
|
||||
heartbeat_at = datetime.fromisoformat(heartbeat_raw)
|
||||
except ValueError:
|
||||
heartbeat_at = None
|
||||
|
||||
last_seen_at = heartbeat_at or item.started_at
|
||||
if last_seen_at.tzinfo is None:
|
||||
last_seen_at = last_seen_at.replace(tzinfo=UTC)
|
||||
|
||||
if datetime.now(UTC) - last_seen_at > timedelta(minutes=30):
|
||||
stale_document_ids = [
|
||||
str(document_id).strip()
|
||||
for document_id in list(item.route_json.get("requested_document_ids") or [])
|
||||
if str(document_id).strip()
|
||||
]
|
||||
if stale_document_ids:
|
||||
self.knowledge_service.set_document_ingest_statuses(
|
||||
stale_document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=item.run_id,
|
||||
)
|
||||
self.run_service.merge_route_json(
|
||||
item.run_id,
|
||||
{
|
||||
"phase": "stale_failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary="知识归纳任务长时间无心跳,系统已自动标记失败。",
|
||||
error_message="Knowledge index heartbeat timed out.",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
continue
|
||||
|
||||
active_ids = {
|
||||
str(document_id).strip()
|
||||
for document_id in list(item.route_json.get("requested_document_ids") or [])
|
||||
if str(document_id).strip()
|
||||
}
|
||||
active_folder = str(item.route_json.get("folder") or "").strip()
|
||||
if active_folder == ALL_KNOWLEDGE_FOLDERS_LABEL:
|
||||
if not requested_set or active_ids & requested_set:
|
||||
return item
|
||||
continue
|
||||
if active_folder == folder:
|
||||
if not requested_set or not active_ids or active_ids & requested_set:
|
||||
return item
|
||||
|
||||
return None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,363 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import os
|
||||
import signal
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentRunStatus
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import KNOWLEDGE_INGEST_STATUS_FAILED, KnowledgeService
|
||||
from app.services.llm_wiki import HERMES_AGENT_BATCH_TIMEOUT_SECONDS, LlmWikiService
|
||||
|
||||
logger = get_logger("app.services.llm_wiki_tasks")
|
||||
|
||||
|
||||
class LlmWikiTaskManager:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._threads: dict[str, threading.Thread] = {}
|
||||
|
||||
def submit_sync(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str] | None = None,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
worker = threading.Thread(
|
||||
target=self._run_sync,
|
||||
kwargs={
|
||||
"agent_run_id": agent_run_id,
|
||||
"folder": folder,
|
||||
"current_user": current_user,
|
||||
"document_ids": list(document_ids or []),
|
||||
"force": force,
|
||||
},
|
||||
daemon=True,
|
||||
name=f"llm-wiki-sync-{agent_run_id}",
|
||||
)
|
||||
with self._lock:
|
||||
self._threads[agent_run_id] = worker
|
||||
worker.start()
|
||||
|
||||
def shutdown(self, *, timeout_seconds: float = 1.0) -> None:
|
||||
with self._lock:
|
||||
threads = list(self._threads.items())
|
||||
self._threads.clear()
|
||||
|
||||
for _, worker in threads:
|
||||
if worker.is_alive():
|
||||
worker.join(timeout=timeout_seconds)
|
||||
|
||||
def _run_sync(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str],
|
||||
force: bool,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
db = session_factory()
|
||||
run_service = AgentRunService(db)
|
||||
knowledge_service = KnowledgeService()
|
||||
request_payload = {
|
||||
"folder": folder,
|
||||
"document_ids": list(document_ids),
|
||||
"force": force,
|
||||
}
|
||||
|
||||
try:
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "running",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"job_type": "llm_wiki_sync",
|
||||
"folder": folder,
|
||||
"force": force,
|
||||
"requested_document_ids": list(document_ids),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 0,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary="Hermes 后台归纳任务已启动。",
|
||||
)
|
||||
|
||||
dispatch = LlmWikiService(db).dispatch_agent_batch(
|
||||
folder=folder,
|
||||
document_ids=document_ids,
|
||||
force=force,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
if not dispatch.changed_document_ids:
|
||||
knowledge_service.refresh_document_ingest_statuses(
|
||||
document_ids=document_ids,
|
||||
preserve_syncing=False,
|
||||
)
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_dispatch",
|
||||
request_json=request_payload,
|
||||
response_json={"changed_document_ids": [], "skipped_document_ids": dispatch.skipped_document_ids},
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "succeeded",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": len(dispatch.skipped_document_ids),
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
result_summary="本次所选文档均未变化,未重复派发 Hermes 任务。",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_dispatch",
|
||||
request_json=request_payload,
|
||||
response_json={
|
||||
"changed_document_ids": dispatch.changed_document_ids,
|
||||
"skipped_document_ids": dispatch.skipped_document_ids,
|
||||
"process_id": dispatch.process_id,
|
||||
},
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
current_run = run_service.get_run(agent_run_id)
|
||||
if current_run is not None and current_run.status in {
|
||||
AgentRunStatus.SUCCEEDED.value,
|
||||
AgentRunStatus.FAILED.value,
|
||||
}:
|
||||
return
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "awaiting_callback",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"requested_document_ids": dispatch.changed_document_ids,
|
||||
"skipped_document_ids": dispatch.skipped_document_ids,
|
||||
"hermes_process_id": dispatch.process_id,
|
||||
"hermes_stdout_path": dispatch.stdout_path,
|
||||
"hermes_stderr_path": dispatch.stderr_path,
|
||||
"progress": {
|
||||
"total_documents": len(dispatch.changed_document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": len(dispatch.skipped_document_ids),
|
||||
"percent": 0,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary="Hermes 任务已派发,等待 Agent 主动回调结果。",
|
||||
)
|
||||
self._start_process_monitor(
|
||||
agent_run_id=agent_run_id,
|
||||
document_ids=dispatch.changed_document_ids,
|
||||
process_id=dispatch.process_id,
|
||||
stderr_path=dispatch.stderr_path,
|
||||
timeout_seconds=HERMES_AGENT_BATCH_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Background LLM Wiki sync failed run_id=%s", agent_run_id)
|
||||
if document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_sync",
|
||||
request_json=request_payload,
|
||||
response_json={"error": str(exc)},
|
||||
status="failed",
|
||||
duration_ms=0,
|
||||
error_message=str(exc),
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": len(document_ids),
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary=str(exc),
|
||||
error_message=str(exc),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
with self._lock:
|
||||
self._threads.pop(agent_run_id, None)
|
||||
|
||||
def _start_process_monitor(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
document_ids: list[str],
|
||||
process_id: int,
|
||||
stderr_path: str,
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
worker = threading.Thread(
|
||||
target=self._monitor_process,
|
||||
kwargs={
|
||||
"agent_run_id": agent_run_id,
|
||||
"document_ids": list(document_ids),
|
||||
"process_id": process_id,
|
||||
"stderr_path": stderr_path,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
},
|
||||
daemon=True,
|
||||
name=f"llm-wiki-monitor-{agent_run_id}",
|
||||
)
|
||||
worker.start()
|
||||
|
||||
@staticmethod
|
||||
def _monitor_process(
|
||||
*,
|
||||
agent_run_id: str,
|
||||
document_ids: list[str],
|
||||
process_id: int,
|
||||
stderr_path: str,
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
db = session_factory()
|
||||
run_service = AgentRunService(db)
|
||||
knowledge_service = KnowledgeService()
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3)
|
||||
run = run_service.get_run(agent_run_id)
|
||||
if run is None or run.status != AgentRunStatus.RUNNING.value:
|
||||
return
|
||||
if time.monotonic() - started_at > timeout_seconds:
|
||||
try:
|
||||
os.killpg(process_id, signal.SIGTERM)
|
||||
except OSError:
|
||||
pass
|
||||
error_message = LlmWikiTaskManager._read_process_error(stderr_path)
|
||||
if document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"hermes_process_id": process_id,
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary="Hermes 任务执行超时,已自动终止等待。",
|
||||
error_message=error_message or "Hermes process exceeded callback timeout.",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
if LlmWikiTaskManager._is_process_alive(process_id):
|
||||
continue
|
||||
|
||||
error_message = LlmWikiTaskManager._read_process_error(stderr_path)
|
||||
if document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"hermes_process_id": process_id,
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary="Hermes 进程已退出且未回调结果。",
|
||||
error_message=error_message or "Hermes process exited before callback.",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@staticmethod
|
||||
def _is_process_alive(process_id: int) -> bool:
|
||||
stat_path = Path(f"/proc/{process_id}/stat")
|
||||
if not stat_path.exists():
|
||||
return False
|
||||
try:
|
||||
parts = stat_path.read_text(encoding="utf-8").split()
|
||||
except OSError:
|
||||
return False
|
||||
return len(parts) > 2 and parts[2] != "Z"
|
||||
|
||||
@staticmethod
|
||||
def _read_process_error(stderr_path: str) -> str:
|
||||
path = Path(stderr_path)
|
||||
if not stderr_path or not path.exists():
|
||||
return ""
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
except OSError:
|
||||
return ""
|
||||
return content[-1000:]
|
||||
|
||||
@staticmethod
|
||||
def _write_progress(
|
||||
*,
|
||||
run_service: AgentRunService,
|
||||
agent_run_id: str,
|
||||
payload: dict[str, Any],
|
||||
summary: str,
|
||||
) -> None:
|
||||
patched_payload = dict(payload)
|
||||
patched_payload["heartbeat_at"] = datetime.now(UTC).isoformat()
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
patched_payload,
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary=summary,
|
||||
)
|
||||
|
||||
|
||||
llm_wiki_task_manager = LlmWikiTaskManager()
|
||||
@@ -59,9 +59,18 @@ def _probe_openai_compatible(payload: ModelConnectivityTestRequest) -> int:
|
||||
normalized_endpoint = _normalize_endpoint(payload.endpoint)
|
||||
headers = _build_headers(api_key=payload.api_key, use_bearer=True)
|
||||
|
||||
if payload.capability == "embedding":
|
||||
if payload.capability == "reranker" and payload.provider == "Ali":
|
||||
url, body = _build_ali_reranker_request(payload.model, normalized_endpoint)
|
||||
elif payload.capability == "embedding":
|
||||
url = _ensure_path(normalized_endpoint, "embeddings")
|
||||
body = {"model": payload.model, "input": "connectivity test"}
|
||||
elif payload.capability == "reranker":
|
||||
url = _ensure_path(normalized_endpoint, "rerank")
|
||||
body = {
|
||||
"model": payload.model,
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
}
|
||||
else:
|
||||
url = _ensure_path(normalized_endpoint, "chat/completions")
|
||||
body = {
|
||||
@@ -74,6 +83,35 @@ def _probe_openai_compatible(payload: ModelConnectivityTestRequest) -> int:
|
||||
return status_code
|
||||
|
||||
|
||||
def _build_ali_reranker_request(model: str, endpoint: str) -> tuple[str, dict[str, Any]]:
|
||||
normalized_model = str(model or "").strip()
|
||||
if normalized_model == "qwen3-rerank":
|
||||
return (
|
||||
"https://dashscope.aliyuncs.com/compatible-api/v1/reranks",
|
||||
{
|
||||
"model": normalized_model,
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
"top_n": 1,
|
||||
},
|
||||
)
|
||||
|
||||
return (
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
{
|
||||
"model": normalized_model,
|
||||
"input": {
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
},
|
||||
"parameters": {
|
||||
"return_documents": False,
|
||||
"top_n": 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _probe_ollama(payload: ModelConnectivityTestRequest) -> int:
|
||||
normalized_endpoint = _normalize_endpoint(payload.endpoint)
|
||||
headers = _build_headers(api_key=payload.api_key, use_bearer=False)
|
||||
@@ -81,6 +119,8 @@ def _probe_ollama(payload: ModelConnectivityTestRequest) -> int:
|
||||
if payload.capability == "embedding":
|
||||
url = _ensure_path(normalized_endpoint, "api/embed")
|
||||
body = {"model": payload.model, "input": "connectivity test"}
|
||||
elif payload.capability == "reranker":
|
||||
raise ConnectivityCheckError("Ollama 暂不支持 reranker 连通性探测。", status_code=HTTPStatus.BAD_REQUEST)
|
||||
else:
|
||||
url = _ensure_path(normalized_endpoint, "api/chat")
|
||||
body = {
|
||||
@@ -100,6 +140,12 @@ def _probe_azure_openai(payload: ModelConnectivityTestRequest) -> int:
|
||||
if payload.capability == "embedding":
|
||||
url = f"{deployment_base}/embeddings?api-version={AZURE_API_VERSION}"
|
||||
body = {"input": "connectivity test"}
|
||||
elif payload.capability == "reranker":
|
||||
url = f"{deployment_base}/rerank?api-version={AZURE_API_VERSION}"
|
||||
body = {
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
}
|
||||
else:
|
||||
url = f"{deployment_base}/chat/completions?api-version={AZURE_API_VERSION}"
|
||||
body = {
|
||||
@@ -168,12 +214,13 @@ def _send_json_request(
|
||||
*,
|
||||
headers: dict[str, str],
|
||||
payload: dict[str, Any],
|
||||
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
|
||||
) -> tuple[int, Any]:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
request = Request(url=url, data=data, headers=headers, method=method)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=DEFAULT_TIMEOUT_SECONDS) as response:
|
||||
with urlopen(request, timeout=timeout_seconds) as response:
|
||||
body = response.read().decode("utf-8") if response.length != 0 else ""
|
||||
return response.status, _parse_json_body(body)
|
||||
except HTTPError as exc:
|
||||
|
||||
@@ -243,6 +243,7 @@ STATUS_KEYWORDS = {
|
||||
|
||||
PRIVILEGED_ROLE_CODES = {"manager", "finance", "approver", "executive"}
|
||||
CONTEXTUAL_SCENARIOS = {"expense", "accounts_receivable", "accounts_payable", "knowledge"}
|
||||
KNOWLEDGE_INTENTS = {"query", "explain", "compare"}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -356,7 +357,7 @@ class SemanticOntologyService:
|
||||
rule_scenario = inferred_scenario
|
||||
scenario_score = 0.18
|
||||
|
||||
if self._looks_like_expense_narrative(
|
||||
if session_scenario != "knowledge" and self._looks_like_expense_narrative(
|
||||
compact_query,
|
||||
scenario=rule_scenario,
|
||||
entities=entities,
|
||||
@@ -371,7 +372,7 @@ class SemanticOntologyService:
|
||||
entities=entities,
|
||||
time_range=time_range,
|
||||
)
|
||||
if self._should_inherit_expense_draft(
|
||||
if session_scenario != "knowledge" and self._should_inherit_expense_draft(
|
||||
compact_query,
|
||||
scenario=rule_scenario,
|
||||
entities=entities,
|
||||
@@ -384,17 +385,19 @@ class SemanticOntologyService:
|
||||
intent_score = max(intent_score, 0.18)
|
||||
metrics = self._extract_metrics(compact_query)
|
||||
constraints = self._extract_constraints(compact_query, entities)
|
||||
model_parse = self._parse_with_model(
|
||||
payload=payload,
|
||||
query=query,
|
||||
compact_query=compact_query,
|
||||
fallback_scenario=rule_scenario,
|
||||
fallback_intent=rule_intent,
|
||||
entities=entities,
|
||||
time_range=time_range,
|
||||
metrics=metrics,
|
||||
constraints=constraints,
|
||||
)
|
||||
model_parse = None
|
||||
if session_scenario != "knowledge":
|
||||
model_parse = self._parse_with_model(
|
||||
payload=payload,
|
||||
query=query,
|
||||
compact_query=compact_query,
|
||||
fallback_scenario=rule_scenario,
|
||||
fallback_intent=rule_intent,
|
||||
entities=entities,
|
||||
time_range=time_range,
|
||||
metrics=metrics,
|
||||
constraints=constraints,
|
||||
)
|
||||
scenario = self._resolve_scenario(rule_scenario, model_parse)
|
||||
if session_scenario == "knowledge":
|
||||
scenario = "knowledge"
|
||||
@@ -968,6 +971,12 @@ class SemanticOntologyService:
|
||||
model_parse: LlmOntologyParseResult | None,
|
||||
) -> str:
|
||||
candidate = model_parse.intent if model_parse is not None else fallback_intent
|
||||
if scenario == "knowledge":
|
||||
if candidate in KNOWLEDGE_INTENTS:
|
||||
return candidate
|
||||
if fallback_intent in KNOWLEDGE_INTENTS:
|
||||
return fallback_intent
|
||||
return "query"
|
||||
if candidate == "query" and scenario == "expense":
|
||||
if self._is_generic_expense_prompt(compact_query) or fallback_intent == "draft":
|
||||
return "draft"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from http import HTTPStatus
|
||||
from time import monotonic, sleep
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -18,6 +19,12 @@ from app.services.model_connectivity import (
|
||||
from app.services.settings import SettingsService
|
||||
|
||||
logger = get_logger("app.services.runtime_chat")
|
||||
DEFAULT_RUNTIME_CHAT_TIMEOUT_SECONDS = 45
|
||||
DEFAULT_RUNTIME_CHAT_RETRY_ATTEMPTS = 2
|
||||
DEFAULT_RUNTIME_CHAT_RETRY_DELAY_SECONDS = 0.6
|
||||
DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS = 90
|
||||
|
||||
_slot_failure_until: dict[str, float] = {}
|
||||
|
||||
|
||||
class RuntimeChatService:
|
||||
@@ -32,33 +39,71 @@ class RuntimeChatService:
|
||||
slot_priority: tuple[str, ...] = ("main", "backup"),
|
||||
max_tokens: int = 500,
|
||||
temperature: float = 0.2,
|
||||
timeout_seconds: int | None = None,
|
||||
slot_timeouts: dict[str, int] | None = None,
|
||||
max_attempts: int | None = None,
|
||||
) -> str | None:
|
||||
for slot in slot_priority:
|
||||
config = self._load_chat_slot(slot)
|
||||
if config is None:
|
||||
continue
|
||||
configs = [
|
||||
config
|
||||
for slot in slot_priority
|
||||
if (config := self._load_chat_slot(slot)) is not None
|
||||
]
|
||||
resolved_timeout_seconds = timeout_seconds or DEFAULT_RUNTIME_CHAT_TIMEOUT_SECONDS
|
||||
resolved_slot_timeouts = dict(slot_timeouts or {})
|
||||
resolved_max_attempts = max_attempts or DEFAULT_RUNTIME_CHAT_RETRY_ATTEMPTS
|
||||
|
||||
try:
|
||||
response_text = self._request_chat_completion(
|
||||
config,
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Runtime chat request failed slot=%s provider=%s: %s",
|
||||
slot,
|
||||
config["provider"],
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
if response_text:
|
||||
return response_text.strip()
|
||||
for attempt in range(1, resolved_max_attempts + 1):
|
||||
for config in configs:
|
||||
cache_key = self._build_slot_cache_key(config)
|
||||
if _slot_failure_until.get(cache_key, 0.0) > monotonic():
|
||||
logger.info(
|
||||
"Skip runtime chat slot=%s provider=%s because it is in cooldown",
|
||||
config["slot"],
|
||||
config["provider"],
|
||||
)
|
||||
continue
|
||||
try:
|
||||
response_text = self._request_chat_completion(
|
||||
config,
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=resolved_slot_timeouts.get(
|
||||
config["slot"],
|
||||
resolved_timeout_seconds,
|
||||
),
|
||||
)
|
||||
if response_text:
|
||||
_slot_failure_until.pop(cache_key, None)
|
||||
return response_text.strip()
|
||||
except Exception as exc:
|
||||
_slot_failure_until[cache_key] = (
|
||||
monotonic() + DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS
|
||||
)
|
||||
logger.warning(
|
||||
"Runtime chat request failed slot=%s provider=%s attempt=%s/%s: %s",
|
||||
config["slot"],
|
||||
config["provider"],
|
||||
attempt,
|
||||
resolved_max_attempts,
|
||||
exc,
|
||||
)
|
||||
if attempt < resolved_max_attempts:
|
||||
sleep(DEFAULT_RUNTIME_CHAT_RETRY_DELAY_SECONDS)
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _build_slot_cache_key(config: dict[str, str]) -> str:
|
||||
return "|".join(
|
||||
[
|
||||
str(config.get("slot") or ""),
|
||||
str(config.get("provider") or ""),
|
||||
str(config.get("endpoint") or ""),
|
||||
str(config.get("model") or ""),
|
||||
]
|
||||
)
|
||||
|
||||
def _load_chat_slot(self, slot: str) -> dict[str, str] | None:
|
||||
try:
|
||||
config = self.settings_service.get_runtime_model_config(slot)
|
||||
@@ -95,6 +140,7 @@ class RuntimeChatService:
|
||||
*,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
provider = config["provider"]
|
||||
endpoint = config["endpoint"]
|
||||
@@ -109,6 +155,7 @@ class RuntimeChatService:
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
if provider == "Ollama":
|
||||
@@ -119,38 +166,48 @@ class RuntimeChatService:
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
return self._request_openai_compatible(
|
||||
provider=provider,
|
||||
endpoint=endpoint,
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
def _request_openai_compatible(
|
||||
self,
|
||||
*,
|
||||
provider: str,
|
||||
endpoint: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
url = _ensure_path(_normalize_endpoint(endpoint), "chat/completions")
|
||||
request_payload: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
}
|
||||
if provider == "GLM":
|
||||
request_payload["thinking"] = {"type": "disabled"}
|
||||
|
||||
status_code, payload = _send_json_request(
|
||||
"POST",
|
||||
url,
|
||||
headers=_build_headers(api_key=api_key, use_bearer=True),
|
||||
payload={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
},
|
||||
payload=request_payload,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if status_code >= HTTPStatus.BAD_REQUEST:
|
||||
raise ConnectivityCheckError(
|
||||
@@ -168,6 +225,7 @@ class RuntimeChatService:
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
url = _ensure_path(_normalize_endpoint(endpoint), "api/chat")
|
||||
status_code, payload = _send_json_request(
|
||||
@@ -183,6 +241,7 @@ class RuntimeChatService:
|
||||
"temperature": temperature,
|
||||
},
|
||||
},
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if status_code >= HTTPStatus.BAD_REQUEST:
|
||||
raise ConnectivityCheckError(
|
||||
@@ -200,6 +259,7 @@ class RuntimeChatService:
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
deployment_base = _build_azure_deployment_base(endpoint, model)
|
||||
url = f"{deployment_base}/chat/completions?api-version={AZURE_API_VERSION}"
|
||||
@@ -212,6 +272,7 @@ class RuntimeChatService:
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
},
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if status_code >= HTTPStatus.BAD_REQUEST:
|
||||
raise ConnectivityCheckError(
|
||||
|
||||
@@ -64,29 +64,29 @@ MODEL_SLOT_CONFIGS = {
|
||||
capability="chat",
|
||||
priority=20,
|
||||
),
|
||||
"vlm": ModelSlotConfig(
|
||||
provider_attr="vlm_provider",
|
||||
model_attr="vlm_model",
|
||||
endpoint_attr="vlm_endpoint",
|
||||
legacy_secret_attr="vlm_api_key_encrypted",
|
||||
default_provider="Gemini",
|
||||
default_model="gemini-2.5-flash",
|
||||
default_endpoint="https://generativelanguage.googleapis.com/v1beta/openai/",
|
||||
capability="chat",
|
||||
priority=30,
|
||||
),
|
||||
"embedding": ModelSlotConfig(
|
||||
provider_attr="embedding_provider",
|
||||
model_attr="embedding_model",
|
||||
endpoint_attr="embedding_endpoint",
|
||||
legacy_secret_attr="embedding_api_key_encrypted",
|
||||
default_provider="GLM",
|
||||
default_model="Embedding-3",
|
||||
default_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
capability="embedding",
|
||||
priority=40,
|
||||
),
|
||||
}
|
||||
"embedding": ModelSlotConfig(
|
||||
provider_attr="embedding_provider",
|
||||
model_attr="embedding_model",
|
||||
endpoint_attr="embedding_endpoint",
|
||||
legacy_secret_attr="embedding_api_key_encrypted",
|
||||
default_provider="GLM",
|
||||
default_model="Embedding-3",
|
||||
default_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
capability="embedding",
|
||||
priority=30,
|
||||
),
|
||||
"reranker": ModelSlotConfig(
|
||||
provider_attr="reranker_provider",
|
||||
model_attr="reranker_model",
|
||||
endpoint_attr="reranker_endpoint",
|
||||
legacy_secret_attr="reranker_api_key_encrypted",
|
||||
default_provider="Ali",
|
||||
default_model="gte-rerank-v2",
|
||||
default_endpoint="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
capability="reranker",
|
||||
priority=40,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -138,6 +138,8 @@ class SettingsService:
|
||||
|
||||
if self._sync_onlyoffice_defaults(settings_row, secrets_row):
|
||||
should_commit = True
|
||||
if self._sync_reranker_legacy_defaults(settings_row, secrets_row):
|
||||
should_commit = True
|
||||
|
||||
if should_commit:
|
||||
self.db.commit()
|
||||
@@ -150,13 +152,28 @@ class SettingsService:
|
||||
self,
|
||||
settings_row: SystemSetting,
|
||||
secrets_row: SystemSettingSecret,
|
||||
) -> dict[str, SystemModelSetting]:
|
||||
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
|
||||
should_commit = False
|
||||
|
||||
for slot, config in MODEL_SLOT_CONFIGS.items():
|
||||
if slot in model_rows:
|
||||
continue
|
||||
) -> dict[str, SystemModelSetting]:
|
||||
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
|
||||
should_commit = False
|
||||
|
||||
if "reranker" not in model_rows and "vlm" in model_rows:
|
||||
self.db.execute(
|
||||
text(
|
||||
"UPDATE system_model_settings "
|
||||
"SET slot = 'reranker', capability = :capability, priority = :priority "
|
||||
"WHERE slot = 'vlm'"
|
||||
),
|
||||
{
|
||||
"capability": MODEL_SLOT_CONFIGS["reranker"].capability,
|
||||
"priority": MODEL_SLOT_CONFIGS["reranker"].priority,
|
||||
},
|
||||
)
|
||||
self.db.commit()
|
||||
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
|
||||
|
||||
for slot, config in MODEL_SLOT_CONFIGS.items():
|
||||
if slot in model_rows:
|
||||
continue
|
||||
|
||||
model_row = SystemModelSetting(
|
||||
slot=slot,
|
||||
@@ -224,13 +241,6 @@ class SettingsService:
|
||||
payload.llmForm.backupEndpoint,
|
||||
payload.llmForm.backupApiKey,
|
||||
)
|
||||
self._apply_model_setting(
|
||||
model_rows["vlm"],
|
||||
payload.llmForm.vlmProvider,
|
||||
payload.llmForm.vlmModel,
|
||||
payload.llmForm.vlmEndpoint,
|
||||
payload.llmForm.vlmApiKey,
|
||||
)
|
||||
self._apply_model_setting(
|
||||
model_rows["embedding"],
|
||||
payload.llmForm.embeddingProvider,
|
||||
@@ -238,6 +248,13 @@ class SettingsService:
|
||||
payload.llmForm.embeddingEndpoint,
|
||||
payload.llmForm.embeddingApiKey,
|
||||
)
|
||||
self._apply_model_setting(
|
||||
model_rows["reranker"],
|
||||
payload.llmForm.rerankerProvider,
|
||||
payload.llmForm.rerankerModel,
|
||||
payload.llmForm.rerankerEndpoint,
|
||||
payload.llmForm.rerankerApiKey,
|
||||
)
|
||||
|
||||
if payload.renderForm.enabled and not payload.renderForm.publicUrl:
|
||||
raise ValueError("启用 ONLYOFFICE 时必须配置服务地址。")
|
||||
@@ -252,14 +269,14 @@ class SettingsService:
|
||||
settings_row.main_model = model_rows["main"].model_name
|
||||
settings_row.main_endpoint = model_rows["main"].endpoint
|
||||
settings_row.backup_provider = model_rows["backup"].provider
|
||||
settings_row.backup_model = model_rows["backup"].model_name
|
||||
settings_row.backup_endpoint = model_rows["backup"].endpoint
|
||||
settings_row.vlm_provider = model_rows["vlm"].provider
|
||||
settings_row.vlm_model = model_rows["vlm"].model_name
|
||||
settings_row.vlm_endpoint = model_rows["vlm"].endpoint
|
||||
settings_row.backup_model = model_rows["backup"].model_name
|
||||
settings_row.backup_endpoint = model_rows["backup"].endpoint
|
||||
settings_row.embedding_provider = model_rows["embedding"].provider
|
||||
settings_row.embedding_model = model_rows["embedding"].model_name
|
||||
settings_row.embedding_endpoint = model_rows["embedding"].endpoint
|
||||
settings_row.reranker_provider = model_rows["reranker"].provider
|
||||
settings_row.reranker_model = model_rows["reranker"].model_name
|
||||
settings_row.reranker_endpoint = model_rows["reranker"].endpoint
|
||||
settings_row.onlyoffice_enabled = payload.renderForm.enabled
|
||||
settings_row.onlyoffice_public_url = payload.renderForm.publicUrl
|
||||
|
||||
@@ -428,7 +445,7 @@ class SettingsService:
|
||||
legacy_admin = read_admin_secret() or {}
|
||||
admin_account = str(legacy_admin.get("username", "")).strip() or "superadmin"
|
||||
|
||||
return SystemSetting(
|
||||
return SystemSetting(
|
||||
id=SETTINGS_ROW_ID,
|
||||
company_name=company_name,
|
||||
display_name=company_name,
|
||||
@@ -445,16 +462,16 @@ class SettingsService:
|
||||
login_alert_enabled=True,
|
||||
main_provider="Codex",
|
||||
main_model="codex-mini-latest",
|
||||
main_endpoint="https://api.openai.com/v1",
|
||||
backup_provider="GLM",
|
||||
backup_model="glm-5.1",
|
||||
main_endpoint="https://api.openai.com/v1",
|
||||
backup_provider="GLM",
|
||||
backup_model="glm-5.1",
|
||||
backup_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
vlm_provider="Gemini",
|
||||
vlm_model="gemini-2.5-flash",
|
||||
vlm_endpoint="https://generativelanguage.googleapis.com/v1beta/openai/",
|
||||
embedding_provider="GLM",
|
||||
embedding_model="Embedding-3",
|
||||
embedding_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
reranker_provider="Ali",
|
||||
reranker_model="gte-rerank-v2",
|
||||
reranker_endpoint="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
onlyoffice_enabled=bool(self.runtime_settings.onlyoffice_enabled),
|
||||
onlyoffice_public_url=str(self.runtime_settings.onlyoffice_public_url or "").strip(),
|
||||
log_level="INFO",
|
||||
@@ -542,6 +559,19 @@ class SettingsService:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN onlyoffice_public_url VARCHAR(512) DEFAULT ''"
|
||||
)
|
||||
if "reranker_provider" not in settings_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN reranker_provider VARCHAR(64) DEFAULT 'Ali'"
|
||||
)
|
||||
if "reranker_model" not in settings_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN reranker_model VARCHAR(255) DEFAULT 'gte-rerank-v2'"
|
||||
)
|
||||
if "reranker_endpoint" not in settings_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN reranker_endpoint "
|
||||
"VARCHAR(512) DEFAULT 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank'"
|
||||
)
|
||||
|
||||
if "system_setting_secrets" in table_names:
|
||||
secret_columns = {column["name"] for column in inspector.get_columns("system_setting_secrets")}
|
||||
@@ -549,6 +579,10 @@ class SettingsService:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_setting_secrets ADD COLUMN onlyoffice_jwt_secret_encrypted TEXT DEFAULT ''"
|
||||
)
|
||||
if "reranker_api_key_encrypted" not in secret_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_setting_secrets ADD COLUMN reranker_api_key_encrypted TEXT DEFAULT ''"
|
||||
)
|
||||
|
||||
for statement in migration_statements:
|
||||
self.db.execute(text(statement))
|
||||
@@ -583,16 +617,41 @@ class SettingsService:
|
||||
|
||||
return should_commit
|
||||
|
||||
@staticmethod
|
||||
def _sync_reranker_legacy_defaults(
|
||||
settings_row: SystemSetting,
|
||||
secrets_row: SystemSettingSecret,
|
||||
) -> bool:
|
||||
should_commit = False
|
||||
|
||||
if not str(settings_row.reranker_provider or "").strip() and str(settings_row.vlm_provider or "").strip():
|
||||
settings_row.reranker_provider = settings_row.vlm_provider
|
||||
should_commit = True
|
||||
if not str(settings_row.reranker_model or "").strip() and str(settings_row.vlm_model or "").strip():
|
||||
settings_row.reranker_model = settings_row.vlm_model
|
||||
should_commit = True
|
||||
if not str(settings_row.reranker_endpoint or "").strip() and str(settings_row.vlm_endpoint or "").strip():
|
||||
settings_row.reranker_endpoint = settings_row.vlm_endpoint
|
||||
should_commit = True
|
||||
if (
|
||||
not str(secrets_row.reranker_api_key_encrypted or "").strip()
|
||||
and str(secrets_row.vlm_api_key_encrypted or "").strip()
|
||||
):
|
||||
secrets_row.reranker_api_key_encrypted = secrets_row.vlm_api_key_encrypted
|
||||
should_commit = True
|
||||
|
||||
return should_commit
|
||||
|
||||
@staticmethod
|
||||
def _serialize(
|
||||
settings_row: SystemSetting,
|
||||
secrets_row: SystemSettingSecret,
|
||||
model_rows: dict[str, SystemModelSetting],
|
||||
) -> SettingsRead:
|
||||
main_model = model_rows["main"]
|
||||
backup_model = model_rows["backup"]
|
||||
vlm_model = model_rows["vlm"]
|
||||
embedding_model = model_rows["embedding"]
|
||||
model_rows: dict[str, SystemModelSetting],
|
||||
) -> SettingsRead:
|
||||
main_model = model_rows["main"]
|
||||
backup_model = model_rows["backup"]
|
||||
embedding_model = model_rows["embedding"]
|
||||
reranker_model = model_rows["reranker"]
|
||||
|
||||
return SettingsRead(
|
||||
companyForm={
|
||||
@@ -624,20 +683,20 @@ class SettingsService:
|
||||
"mainApiKey": "",
|
||||
"mainApiKeyConfigured": bool(main_model.api_key_encrypted),
|
||||
"backupProvider": backup_model.provider,
|
||||
"backupModel": backup_model.model_name,
|
||||
"backupEndpoint": backup_model.endpoint,
|
||||
"backupApiKey": "",
|
||||
"backupApiKeyConfigured": bool(backup_model.api_key_encrypted),
|
||||
"vlmProvider": vlm_model.provider,
|
||||
"vlmModel": vlm_model.model_name,
|
||||
"vlmEndpoint": vlm_model.endpoint,
|
||||
"vlmApiKey": "",
|
||||
"vlmApiKeyConfigured": bool(vlm_model.api_key_encrypted),
|
||||
"embeddingProvider": embedding_model.provider,
|
||||
"embeddingModel": embedding_model.model_name,
|
||||
"backupModel": backup_model.model_name,
|
||||
"backupEndpoint": backup_model.endpoint,
|
||||
"backupApiKey": "",
|
||||
"backupApiKeyConfigured": bool(backup_model.api_key_encrypted),
|
||||
"embeddingProvider": embedding_model.provider,
|
||||
"embeddingModel": embedding_model.model_name,
|
||||
"embeddingEndpoint": embedding_model.endpoint,
|
||||
"embeddingApiKey": "",
|
||||
"embeddingApiKeyConfigured": bool(embedding_model.api_key_encrypted),
|
||||
"rerankerProvider": reranker_model.provider,
|
||||
"rerankerModel": reranker_model.model_name,
|
||||
"rerankerEndpoint": reranker_model.endpoint,
|
||||
"rerankerApiKey": "",
|
||||
"rerankerApiKeyConfigured": bool(reranker_model.api_key_encrypted),
|
||||
},
|
||||
renderForm={
|
||||
"enabled": settings_row.onlyoffice_enabled,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,9 @@ Requires-Dist: PyJWT<3.0.0,>=2.9.0
|
||||
Requires-Dist: pydantic-settings<3.0.0,>=2.6.0
|
||||
Requires-Dist: python-dotenv<2.0.0,>=1.0.1
|
||||
Requires-Dist: email-validator<3.0.0,>=2.2.0
|
||||
Requires-Dist: python-multipart<1.0.0,>=0.0.20
|
||||
Requires-Dist: lightrag-hku<1.5.0,>=1.4.16
|
||||
Requires-Dist: qdrant-client<2.0.0,>=1.18.0
|
||||
Provides-Extra: dev
|
||||
Requires-Dist: pytest<9.0.0,>=8.3.0; extra == "dev"
|
||||
Requires-Dist: httpx<1.0.0,>=0.28.0; extra == "dev"
|
||||
|
||||
@@ -16,14 +16,19 @@ src/app/api/v1/endpoints/bootstrap.py
|
||||
src/app/api/v1/endpoints/employees.py
|
||||
src/app/api/v1/endpoints/health.py
|
||||
src/app/api/v1/endpoints/knowledge.py
|
||||
src/app/api/v1/endpoints/ocr.py
|
||||
src/app/api/v1/endpoints/ontology.py
|
||||
src/app/api/v1/endpoints/orchestrator.py
|
||||
src/app/api/v1/endpoints/reimbursements.py
|
||||
src/app/api/v1/endpoints/settings.py
|
||||
src/app/api/v1/endpoints/system_logs.py
|
||||
src/app/core/__init__.py
|
||||
src/app/core/admin_secret.py
|
||||
src/app/core/agent_enums.py
|
||||
src/app/core/bootstrap.py
|
||||
src/app/core/config.py
|
||||
src/app/core/logging.py
|
||||
src/app/core/openapi.py
|
||||
src/app/core/secret_box.py
|
||||
src/app/core/security.py
|
||||
src/app/db/__init__.py
|
||||
@@ -34,6 +39,7 @@ src/app/middleware/__init__.py
|
||||
src/app/middleware/logging.py
|
||||
src/app/models/__init__.py
|
||||
src/app/models/agent_asset.py
|
||||
src/app/models/agent_conversation.py
|
||||
src/app/models/agent_run.py
|
||||
src/app/models/approval.py
|
||||
src/app/models/audit_log.py
|
||||
@@ -59,23 +65,42 @@ src/app/schemas/agent_run.py
|
||||
src/app/schemas/audit_log.py
|
||||
src/app/schemas/auth.py
|
||||
src/app/schemas/bootstrap.py
|
||||
src/app/schemas/common.py
|
||||
src/app/schemas/employee.py
|
||||
src/app/schemas/knowledge.py
|
||||
src/app/schemas/ocr.py
|
||||
src/app/schemas/ontology.py
|
||||
src/app/schemas/orchestrator.py
|
||||
src/app/schemas/reimbursement.py
|
||||
src/app/schemas/settings.py
|
||||
src/app/schemas/system_log.py
|
||||
src/app/schemas/user_agent.py
|
||||
src/app/services/__init__.py
|
||||
src/app/services/agent_assets.py
|
||||
src/app/services/agent_conversations.py
|
||||
src/app/services/agent_foundation.py
|
||||
src/app/services/agent_runs.py
|
||||
src/app/services/audit.py
|
||||
src/app/services/auth.py
|
||||
src/app/services/document_intelligence.py
|
||||
src/app/services/employee.py
|
||||
src/app/services/employee_seed.py
|
||||
src/app/services/expense_claims.py
|
||||
src/app/services/expense_rule_runtime.py
|
||||
src/app/services/hermes_sync.py
|
||||
src/app/services/knowledge.py
|
||||
src/app/services/knowledge_index_tasks.py
|
||||
src/app/services/knowledge_rag.py
|
||||
src/app/services/model_connectivity.py
|
||||
src/app/services/ocr.py
|
||||
src/app/services/ontology.py
|
||||
src/app/services/orchestrator.py
|
||||
src/app/services/reimbursement.py
|
||||
src/app/services/runtime_chat.py
|
||||
src/app/services/settings.py
|
||||
src/app/services/system_hermes.py
|
||||
src/app/services/system_logs.py
|
||||
src/app/services/user_agent.py
|
||||
src/x_financial_server.egg-info/PKG-INFO
|
||||
src/x_financial_server.egg-info/SOURCES.txt
|
||||
src/x_financial_server.egg-info/dependency_links.txt
|
||||
@@ -85,10 +110,19 @@ tests/test_agent_asset_service.py
|
||||
tests/test_agent_foundation_endpoints.py
|
||||
tests/test_auth_service.py
|
||||
tests/test_config_settings_reload.py
|
||||
tests/test_document_intelligence.py
|
||||
tests/test_employee_service.py
|
||||
tests/test_env_file_precedence.py
|
||||
tests/test_expense_claim_service.py
|
||||
tests/test_imports.py
|
||||
tests/test_knowledge_onlyoffice_config.py
|
||||
tests/test_ocr_endpoints.py
|
||||
tests/test_ocr_service.py
|
||||
tests/test_ontology_service.py
|
||||
tests/test_openapi_schema.py
|
||||
tests/test_reimbursement_endpoints.py
|
||||
tests/test_server_start_dependencies.py
|
||||
tests/test_settings_persistence.py
|
||||
tests/test_settings_service.py
|
||||
tests/test_settings_service.py
|
||||
tests/test_system_logs_service.py
|
||||
tests/test_user_agent_service.py
|
||||
@@ -7,6 +7,9 @@ PyJWT<3.0.0,>=2.9.0
|
||||
pydantic-settings<3.0.0,>=2.6.0
|
||||
python-dotenv<2.0.0,>=1.0.1
|
||||
email-validator<3.0.0,>=2.2.0
|
||||
python-multipart<1.0.0,>=0.0.20
|
||||
lightrag-hku<1.5.0,>=1.4.16
|
||||
qdrant-client<2.0.0,>=1.18.0
|
||||
|
||||
[dev]
|
||||
pytest<9.0.0,>=8.3.0
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"file_name": "行程单_2_鄂AX9877.pdf",
|
||||
"storage_key": "0d3102fe-a458-42cf-b30c-4cffeeb74668/c99de539-23cb-4f1a-a21f-a40bce93d54e/行程单_2_鄂AX9877.pdf",
|
||||
"media_type": "application/pdf",
|
||||
"size_bytes": 32459,
|
||||
"uploaded_at": "2026-05-16T08:41:42.540134+00:00",
|
||||
"previewable": true,
|
||||
"preview_kind": "image",
|
||||
"preview_storage_key": "0d3102fe-a458-42cf-b30c-4cffeeb74668/c99de539-23cb-4f1a-a21f-a40bce93d54e/行程单_2_鄂AX9877.preview.png",
|
||||
"preview_media_type": "image/png",
|
||||
"preview_file_name": "行程单_2_鄂AX9877.preview.png",
|
||||
"analysis": {
|
||||
"severity": "pass",
|
||||
"label": "AI提示符合条件",
|
||||
"headline": "AI提示:附件符合基础校验条件",
|
||||
"summary": "已识别到票据类型和关键字段,且符合当前费用场景的附件要求。",
|
||||
"points": [
|
||||
"票据类型:已识别为出租车/网约车票据。",
|
||||
"附件类型要求:当前费用项目为交通费,已识别为出租车/网约车票据,符合当前交通费场景的附件要求。",
|
||||
"金额字段:已识别到与当前明细接近的金额 35.53 元。"
|
||||
],
|
||||
"suggestion": "建议继续核对报销分类、费用说明和业务场景是否一致。"
|
||||
},
|
||||
"document_info": {
|
||||
"document_type": "taxi_receipt",
|
||||
"document_type_label": "出租车/网约车票据",
|
||||
"scene_code": "transport",
|
||||
"scene_label": "交通票据",
|
||||
"fields": [
|
||||
{
|
||||
"key": "amount",
|
||||
"label": "金额",
|
||||
"value": "35.53元"
|
||||
},
|
||||
{
|
||||
"key": "date",
|
||||
"label": "日期",
|
||||
"value": "2026-03-04"
|
||||
},
|
||||
{
|
||||
"key": "merchant_name",
|
||||
"label": "商户",
|
||||
"value": "全季酒店"
|
||||
}
|
||||
]
|
||||
},
|
||||
"requirement_check": {
|
||||
"matches": true,
|
||||
"current_expense_type": "transport",
|
||||
"current_expense_type_label": "交通费",
|
||||
"allowed_scene_labels": [
|
||||
"交通"
|
||||
],
|
||||
"allowed_document_type_labels": [
|
||||
"停车/通行费票据",
|
||||
"一般收据/凭证",
|
||||
"出租车/网约车票据",
|
||||
"增值税发票"
|
||||
],
|
||||
"recognized_scene_code": "transport",
|
||||
"recognized_scene_label": "交通票据",
|
||||
"recognized_document_type": "taxi_receipt",
|
||||
"recognized_document_type_label": "出租车/网约车票据",
|
||||
"mismatch_severity": "high",
|
||||
"rule_code": "rule.expense.scene_submission_standard",
|
||||
"rule_name": "报销场景提交与附件标准",
|
||||
"message": "当前费用项目为交通费,已识别为出租车/网约车票据,符合当前交通费场景的附件要求。"
|
||||
},
|
||||
"ocr_status": "recognized",
|
||||
"ocr_error": "",
|
||||
"ocr_text": "高德地图一打车\n行程单\nAMAP ITINERARY\n1申请时间:2026-03-04\n【行程时间:2026-03-0407:05至2026-03-0407:33\n|行程人手机号:18602700270\n1共计1单行程,合计35.53元\n序号\n服务商\n车型\n上车时间\n城市\n起点\n终点\n金额\n经济型\n2026-03-04\n1\n滴滴出行\n武汉市\n全季酒店武汉工程大学店\n武汉站\n35.53元\n07:05\n页码:1/1",
|
||||
"ocr_summary": "高德地图一打车;行程单;AMAP ITINERARY",
|
||||
"ocr_avg_score": 0.9819406509399414,
|
||||
"ocr_line_count": 25,
|
||||
"ocr_classification_source": "rule",
|
||||
"ocr_classification_confidence": 0.88,
|
||||
"ocr_classification_evidence": [
|
||||
"滴滴出行",
|
||||
"滴滴",
|
||||
"打车",
|
||||
"上车"
|
||||
],
|
||||
"ocr_warnings": []
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"file_name": "行程单_1_鄂A1S987.pdf",
|
||||
"storage_key": "281095c5-d85b-428e-924f-250bdd6e0261/c676b663-8851-4b35-be4e-1ba13d46d35e/行程单_1_鄂A1S987.pdf",
|
||||
"media_type": "application/pdf",
|
||||
"size_bytes": 34880,
|
||||
"uploaded_at": "2026-05-16T08:17:53.656595+00:00",
|
||||
"previewable": true,
|
||||
"preview_kind": "image",
|
||||
"preview_storage_key": "281095c5-d85b-428e-924f-250bdd6e0261/c676b663-8851-4b35-be4e-1ba13d46d35e/行程单_1_鄂A1S987.preview.png",
|
||||
"preview_media_type": "image/png",
|
||||
"preview_file_name": "行程单_1_鄂A1S987.preview.png",
|
||||
"analysis": {
|
||||
"severity": "pass",
|
||||
"label": "AI提示符合条件",
|
||||
"headline": "AI提示:附件符合基础校验条件",
|
||||
"summary": "已识别到票据类型和关键字段,且符合当前费用场景的附件要求。",
|
||||
"points": [
|
||||
"票据类型:已识别为出租车/网约车票据。",
|
||||
"附件类型要求:当前费用项目为交通费,已识别为出租车/网约车票据,符合当前交通费场景的附件要求。",
|
||||
"金额字段:已识别到与当前明细接近的金额 10.30 元。"
|
||||
],
|
||||
"suggestion": "建议继续核对报销分类、费用说明和业务场景是否一致。"
|
||||
},
|
||||
"document_info": {
|
||||
"document_type": "taxi_receipt",
|
||||
"document_type_label": "出租车/网约车票据",
|
||||
"scene_code": "transport",
|
||||
"scene_label": "交通票据",
|
||||
"fields": [
|
||||
{
|
||||
"key": "amount",
|
||||
"label": "金额",
|
||||
"value": "10.3元"
|
||||
},
|
||||
{
|
||||
"key": "date",
|
||||
"label": "日期",
|
||||
"value": "2026-03-01"
|
||||
},
|
||||
{
|
||||
"key": "merchant_name",
|
||||
"label": "商户",
|
||||
"value": "全季酒店"
|
||||
}
|
||||
]
|
||||
},
|
||||
"requirement_check": {
|
||||
"matches": true,
|
||||
"current_expense_type": "transport",
|
||||
"current_expense_type_label": "交通费",
|
||||
"allowed_scene_labels": [
|
||||
"交通"
|
||||
],
|
||||
"allowed_document_type_labels": [
|
||||
"停车/通行费票据",
|
||||
"一般收据/凭证",
|
||||
"出租车/网约车票据",
|
||||
"增值税发票"
|
||||
],
|
||||
"recognized_scene_code": "transport",
|
||||
"recognized_scene_label": "交通票据",
|
||||
"recognized_document_type": "taxi_receipt",
|
||||
"recognized_document_type_label": "出租车/网约车票据",
|
||||
"mismatch_severity": "high",
|
||||
"rule_code": "rule.expense.scene_submission_standard",
|
||||
"rule_name": "报销场景提交与附件标准",
|
||||
"message": "当前费用项目为交通费,已识别为出租车/网约车票据,符合当前交通费场景的附件要求。"
|
||||
},
|
||||
"ocr_status": "recognized",
|
||||
"ocr_error": "",
|
||||
"ocr_text": "高德地图一打车\n行程单\nAMAP ITINERARY\n1申请时间:2026-03-01\n【行程时间:2026-03-0113:23至2026-03-0113:40\n行程人手机号:18602700270\n|共计1单行程,合计10.30元\n序号\n服务商\n车型\n上车时间\n城市\n起点\n终点\n金额\n经济型\n2026-03-01\n1\n滴滴出行\n13:23\n武汉市\n金融港北地铁站\n全季酒店武汉工程大学店\n10.30元\n页码:1/1",
|
||||
"ocr_summary": "高德地图一打车;行程单;AMAP ITINERARY",
|
||||
"ocr_avg_score": 0.9844024634361267,
|
||||
"ocr_line_count": 25,
|
||||
"ocr_classification_source": "rule",
|
||||
"ocr_classification_confidence": 0.88,
|
||||
"ocr_classification_evidence": [
|
||||
"滴滴出行",
|
||||
"滴滴",
|
||||
"打车",
|
||||
"上车"
|
||||
],
|
||||
"ocr_warnings": []
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 63 KiB |
@@ -15,8 +15,12 @@
|
||||
"uploaded_by": "admin",
|
||||
"version_number": 1,
|
||||
"ingest_status": 3,
|
||||
"ingest_status_updated_at": "2026-05-16T01:48:21.849424+00:00",
|
||||
"ingest_agent_run_id": "run_a7b447f69939442f"
|
||||
"ingest_status_updated_at": "2026-05-16T15:37:12.723203+00:00",
|
||||
"ingest_agent_run_id": "run_94562b13f7a54341",
|
||||
"ingest_completed_at": "2026-05-16T15:37:12.723203+00:00",
|
||||
"ingest_document_name": "远光《公司支出管理办法(2024)》.pdf",
|
||||
"ingest_document_updated_at": "2026-05-09T08:39:53.788042+00:00",
|
||||
"ingest_document_sha256": "67a74538bce0dec71ccbb947256cc2c9c0e672d148de49406b967ae1379dbece"
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"bf761bd8eccf402bb676423d64401a56": {
|
||||
"status": "processed",
|
||||
"chunks_count": 11,
|
||||
"chunks_list": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339",
|
||||
"chunk-0e8b903e5d2a7deeadd9ec0ca70d964c",
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26",
|
||||
"chunk-60066b4c758ad553106e2343a99c890e",
|
||||
"chunk-30373ec763ee53fb2c91741699128f30",
|
||||
"chunk-2d84cd4e27b2bcd246988dabe93d2062",
|
||||
"chunk-090b225cc6d57e9bf0cf7e0f34b4760c",
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83",
|
||||
"chunk-cca4d7b1d51b1e831b80471cd168fef0",
|
||||
"chunk-78998358de8a8cc3c018264c9a553b4d",
|
||||
"chunk-37889c882c89c19f96b9b2ca93685014"
|
||||
],
|
||||
"content_summary": "# 章节导航\n\n以下内容由入库阶段从制度原文中提取,供检索时优先理解制度层级、条目和标准所在章节。\n\n- 第一章 总则.............................................................. 4\n- 第二章 职责分工 .......................................................... 4\n- 第三章 支出报销申请与审批 ................................",
|
||||
"content_length": 25627,
|
||||
"created_at": "2026-05-16T15:30:53.520431+00:00",
|
||||
"updated_at": "2026-05-16T15:37:12.723203+00:00",
|
||||
"file_path": "/app/server/storage/knowledge/报销制度/bf761bd8eccf402bb676423d64401a56__远光《公司支出管理办法(2024)》.pdf",
|
||||
"track_id": "insert_20260516_153053_5bdb18b7",
|
||||
"metadata": {
|
||||
"processing_start_time": 1778945453,
|
||||
"processing_end_time": 1778945832
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,225 @@
|
||||
{
|
||||
"bf761bd8eccf402bb676423d64401a56": {
|
||||
"entity_names": [
|
||||
"公对公结算方式",
|
||||
"业务招待",
|
||||
"营销中心",
|
||||
"预算外支出",
|
||||
"第十四条业务招待费",
|
||||
"材料采购",
|
||||
"对外捐赠支出",
|
||||
"第五章附则",
|
||||
"事业部总经理",
|
||||
"第四章重点支出管理规定",
|
||||
"Home Visit Travel Expenses Management Policy",
|
||||
"组织人事部",
|
||||
"三流一致原则",
|
||||
"预算先行原则",
|
||||
"第五条计划财务部主要职责",
|
||||
"业务招待费",
|
||||
"Long-Term Business Accommodation",
|
||||
"无形资产",
|
||||
"各级管理人员",
|
||||
"第二十三条",
|
||||
"第四条归口管理部门主要职责",
|
||||
"附表3",
|
||||
"各委员会主任",
|
||||
"信息管理部",
|
||||
"经办人",
|
||||
"品牌及市场运营中心",
|
||||
"计划财务部",
|
||||
"Department Head",
|
||||
"业务原始凭据",
|
||||
"经办部门",
|
||||
"系统单据",
|
||||
"保证金",
|
||||
"邮递费",
|
||||
"第十一条",
|
||||
"第二十一条",
|
||||
"分级授权原则",
|
||||
"后勤服务部",
|
||||
"Company Business Travel System",
|
||||
"国家电网公司",
|
||||
"Meeting Expenses",
|
||||
"工会委员会",
|
||||
"特殊事项",
|
||||
"前款不清、后款不借",
|
||||
"Advertising and Promotion Expenses",
|
||||
"广告宣传费",
|
||||
"异地挂职锻炼",
|
||||
"Grassroots Manager P4",
|
||||
"Other Employees",
|
||||
"税控系统明细清单",
|
||||
"商旅订票",
|
||||
"分支机构",
|
||||
"低值易耗品",
|
||||
"产品规划设计部",
|
||||
"培训费",
|
||||
"第一条规定义",
|
||||
"经济舱6折及以下",
|
||||
"三个月",
|
||||
"托运费",
|
||||
"Commercial Insurance",
|
||||
"工会经费管理办法",
|
||||
"增值税专用发票",
|
||||
"第七条各级管理人员主要职责",
|
||||
"Compensation and Benefits Expenses",
|
||||
"外聘专家",
|
||||
"第十二条市内交通费",
|
||||
"分类控制原则",
|
||||
"Value Reimbursement System",
|
||||
"第八条支出报销申请",
|
||||
"会议费",
|
||||
"轮船三等舱",
|
||||
"第四条归口管理",
|
||||
"终审岗",
|
||||
"总经理",
|
||||
"中国外汇交易中心",
|
||||
"Middle And Grassroots Manager P4-P6",
|
||||
"办公用品",
|
||||
"办公室(党委办公室)",
|
||||
"控股子公司",
|
||||
"支出审批权限表",
|
||||
"公司领导",
|
||||
"证券与法律事务部",
|
||||
"支出审批流转程序",
|
||||
"第二条适用范围",
|
||||
"出差补贴标准",
|
||||
"附表2",
|
||||
"高铁/动车二等座",
|
||||
"P8",
|
||||
"报销资料规范",
|
||||
"Staff P1-P3",
|
||||
"2024年4月17日",
|
||||
"董事长",
|
||||
"通信费",
|
||||
"财务审核时限",
|
||||
"一万元",
|
||||
"工会支出",
|
||||
"交通工具等级标准",
|
||||
"第二十条",
|
||||
"影像扫描",
|
||||
"异地调动邮寄费",
|
||||
"第二十二条",
|
||||
"基层经理",
|
||||
"第二十四条附件",
|
||||
"公对私结算方式",
|
||||
"第二十三条本办法的归口与实施",
|
||||
"异地挂职锻炼补贴标准",
|
||||
"公司酒店住宿限额标准",
|
||||
"经办部门(个人)",
|
||||
"投标保证金",
|
||||
"远光制度〔2024〕14号",
|
||||
"Business Travel",
|
||||
"Communication Expenses",
|
||||
"交通费",
|
||||
"远光软件股份有限公司",
|
||||
"第三条管理原则",
|
||||
"全资子公司",
|
||||
"第十三条差旅费",
|
||||
"薪酬福利支出",
|
||||
"Relocation Expenses",
|
||||
"住宿费",
|
||||
"公司支出管理办法(2024)",
|
||||
"中层经理",
|
||||
"第六条经办部门主要职责",
|
||||
"Business Trip",
|
||||
"批办分离原则",
|
||||
"备用金借款",
|
||||
"岗位支出业务",
|
||||
"公司支出管理办法",
|
||||
"第二十四条",
|
||||
"报销业务",
|
||||
"第七条管理人员",
|
||||
"外包分包业务",
|
||||
"归口管理部门",
|
||||
"第十条",
|
||||
"财务审核",
|
||||
"备用金",
|
||||
"预付款项",
|
||||
"支出报销申请",
|
||||
"公司团建管理办法",
|
||||
"总工程师",
|
||||
"商旅系统",
|
||||
"Training Expenses",
|
||||
"固定资产",
|
||||
"DAP研发中心",
|
||||
"第五条计划财务部",
|
||||
"第一条目的",
|
||||
"全列软席列车二等座",
|
||||
"涉外业务汇率标准",
|
||||
"客服及商务",
|
||||
"其他支出",
|
||||
"快递费",
|
||||
"需求计划",
|
||||
"党委办公室",
|
||||
"财务信息化系统",
|
||||
"P5及以上",
|
||||
"因公借款",
|
||||
"效益优先原则",
|
||||
"市内交通",
|
||||
"Business Entertainment Expenses",
|
||||
"第二章职责分工",
|
||||
"出差补贴",
|
||||
"人力资源服务部",
|
||||
"P4及以下",
|
||||
"因公用车补贴",
|
||||
"Company Leader P8 And Above",
|
||||
"国网数科公司",
|
||||
"Mailing and Courier Expenses",
|
||||
"季度清理",
|
||||
"附表1",
|
||||
"正式员工",
|
||||
"审批流转程序",
|
||||
"后续审批人",
|
||||
"商密【中】",
|
||||
"High-Level Manager P7",
|
||||
"Home Visit Travel Expenses",
|
||||
"支出成本中心归属",
|
||||
"供应商",
|
||||
"High-Speed Rail And Bullet Train",
|
||||
"资产采购",
|
||||
"附表1:员工支出报销审批权限表",
|
||||
"第九条支出报销审批",
|
||||
"审批权限",
|
||||
"经济舱5折及以下",
|
||||
"产业投资部",
|
||||
"第六条经办部门",
|
||||
"第十条支出成本中心归属",
|
||||
"第九条",
|
||||
"邮件费",
|
||||
"第四章",
|
||||
"第十三条",
|
||||
"Travel Allowance",
|
||||
"第十一条备用金借款",
|
||||
"财务部门",
|
||||
"公司",
|
||||
"中国银行",
|
||||
"Business Travel Ticket Booking",
|
||||
"市内交通费",
|
||||
"发票",
|
||||
"第十二条",
|
||||
"支出报销审批",
|
||||
"经济舱",
|
||||
"第一审批人",
|
||||
"品牌",
|
||||
"火车硬席",
|
||||
"审批时限",
|
||||
"预算内支出",
|
||||
"President",
|
||||
"差旅费",
|
||||
"高层经理",
|
||||
"厉行节约原则",
|
||||
"第三章支出报销申请与审批",
|
||||
"第二条范围",
|
||||
"基建工程",
|
||||
"逐级审批规则",
|
||||
"招标采购规定",
|
||||
"第一章总则"
|
||||
],
|
||||
"count": 215,
|
||||
"create_time": 1778945832,
|
||||
"update_time": 1778945832,
|
||||
"_id": "bf761bd8eccf402bb676423d64401a56"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
{
|
||||
"bf761bd8eccf402bb676423d64401a56": {
|
||||
"relation_pairs": [
|
||||
[
|
||||
"公司支出管理办法",
|
||||
"审批流转程序"
|
||||
],
|
||||
[
|
||||
"供应商",
|
||||
"公司"
|
||||
],
|
||||
[
|
||||
"全资子公司",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"发票",
|
||||
"报销业务"
|
||||
],
|
||||
[
|
||||
"出差补贴",
|
||||
"组织人事部"
|
||||
],
|
||||
[
|
||||
"第十三条差旅费",
|
||||
"第四章重点支出管理规定"
|
||||
],
|
||||
[
|
||||
"公司支出管理办法",
|
||||
"差旅费"
|
||||
],
|
||||
[
|
||||
"第一章总则",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"计划财务部",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"分支机构",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"经办部门",
|
||||
"需求计划"
|
||||
],
|
||||
[
|
||||
"增值税专用发票",
|
||||
"税控系统明细清单"
|
||||
],
|
||||
[
|
||||
"第三章支出报销申请与审批",
|
||||
"财务信息化系统"
|
||||
],
|
||||
[
|
||||
"业务原始凭据",
|
||||
"经办人"
|
||||
],
|
||||
[
|
||||
"第十四条业务招待费",
|
||||
"第四章重点支出管理规定"
|
||||
],
|
||||
[
|
||||
"系统单据",
|
||||
"经办人"
|
||||
],
|
||||
[
|
||||
"第二十三条本办法的归口与实施",
|
||||
"第五章附则"
|
||||
],
|
||||
[
|
||||
"第二十四条",
|
||||
"附表2"
|
||||
],
|
||||
[
|
||||
"第二十三条",
|
||||
"计划财务部"
|
||||
],
|
||||
[
|
||||
"归口管理部门",
|
||||
"报销业务"
|
||||
],
|
||||
[
|
||||
"公司支出管理办法",
|
||||
"投标保证金"
|
||||
],
|
||||
[
|
||||
"对外捐赠支出",
|
||||
"第二十一条"
|
||||
],
|
||||
[
|
||||
"第二十条",
|
||||
"薪酬福利支出"
|
||||
],
|
||||
[
|
||||
"国网数科公司",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"事业部总经理",
|
||||
"逐级审批规则"
|
||||
],
|
||||
[
|
||||
"特殊事项",
|
||||
"终审岗"
|
||||
],
|
||||
[
|
||||
"发票",
|
||||
"经办人"
|
||||
],
|
||||
[
|
||||
"第十一条备用金借款",
|
||||
"第四章重点支出管理规定"
|
||||
],
|
||||
[
|
||||
"归口管理部门",
|
||||
"计划财务部"
|
||||
],
|
||||
[
|
||||
"工会委员会",
|
||||
"工会支出"
|
||||
],
|
||||
[
|
||||
"第二十四条附件",
|
||||
"第五章附则"
|
||||
],
|
||||
[
|
||||
"涉外业务汇率标准",
|
||||
"第二十二条"
|
||||
],
|
||||
[
|
||||
"各级管理人员",
|
||||
"支出报销审批"
|
||||
],
|
||||
[
|
||||
"第十二条市内交通费",
|
||||
"第四章重点支出管理规定"
|
||||
],
|
||||
[
|
||||
"支出审批流转程序",
|
||||
"逐级审批规则"
|
||||
],
|
||||
[
|
||||
"支出审批流转程序",
|
||||
"终审岗"
|
||||
],
|
||||
[
|
||||
"各级管理人员",
|
||||
"报销业务"
|
||||
],
|
||||
[
|
||||
"归口管理部门",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"控股子公司",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"报销业务",
|
||||
"财务部门"
|
||||
],
|
||||
[
|
||||
"报销业务",
|
||||
"经办部门"
|
||||
],
|
||||
[
|
||||
"国家电网公司",
|
||||
"远光软件股份有限公司"
|
||||
],
|
||||
[
|
||||
"第二十四条",
|
||||
"附表1"
|
||||
]
|
||||
],
|
||||
"count": 43,
|
||||
"create_time": 1778945832,
|
||||
"update_time": 1778945832,
|
||||
"_id": "bf761bd8eccf402bb676423d64401a56"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
{
|
||||
"第一章总则<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945805,
|
||||
"update_time": 1778945805,
|
||||
"_id": "第一章总则<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"第三章支出报销申请与审批<SEP>财务信息化系统": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945805,
|
||||
"update_time": 1778945805,
|
||||
"_id": "第三章支出报销申请与审批<SEP>财务信息化系统"
|
||||
},
|
||||
"第十一条备用金借款<SEP>第四章重点支出管理规定": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945805,
|
||||
"update_time": 1778945805,
|
||||
"_id": "第十一条备用金借款<SEP>第四章重点支出管理规定"
|
||||
},
|
||||
"第二十三条本办法的归口与实施<SEP>第五章附则": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945805,
|
||||
"update_time": 1778945805,
|
||||
"_id": "第二十三条本办法的归口与实施<SEP>第五章附则"
|
||||
},
|
||||
"第十二条市内交通费<SEP>第四章重点支出管理规定": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945808,
|
||||
"update_time": 1778945808,
|
||||
"_id": "第十二条市内交通费<SEP>第四章重点支出管理规定"
|
||||
},
|
||||
"归口管理部门<SEP>报销业务": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945808,
|
||||
"update_time": 1778945808,
|
||||
"_id": "归口管理部门<SEP>报销业务"
|
||||
},
|
||||
"第十三条差旅费<SEP>第四章重点支出管理规定": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945808,
|
||||
"update_time": 1778945808,
|
||||
"_id": "第十三条差旅费<SEP>第四章重点支出管理规定"
|
||||
},
|
||||
"第二十四条附件<SEP>第五章附则": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945809,
|
||||
"update_time": 1778945809,
|
||||
"_id": "第二十四条附件<SEP>第五章附则"
|
||||
},
|
||||
"业务原始凭据<SEP>经办人": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945809,
|
||||
"update_time": 1778945809,
|
||||
"_id": "业务原始凭据<SEP>经办人"
|
||||
},
|
||||
"报销业务<SEP>财务部门": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "报销业务<SEP>财务部门"
|
||||
},
|
||||
"增值税专用发票<SEP>税控系统明细清单": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "增值税专用发票<SEP>税控系统明细清单"
|
||||
},
|
||||
"第十四条业务招待费<SEP>第四章重点支出管理规定": {
|
||||
"chunk_ids": [
|
||||
"chunk-a28dc5c0a449bfa3ec07f3ea70720339"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "第十四条业务招待费<SEP>第四章重点支出管理规定"
|
||||
},
|
||||
"经办部门<SEP>需求计划": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "经办部门<SEP>需求计划"
|
||||
},
|
||||
"系统单据<SEP>经办人": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "系统单据<SEP>经办人"
|
||||
},
|
||||
"报销业务<SEP>经办部门": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "报销业务<SEP>经办部门"
|
||||
},
|
||||
"供应商<SEP>公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "供应商<SEP>公司"
|
||||
},
|
||||
"工会委员会<SEP>工会支出": {
|
||||
"chunk_ids": [
|
||||
"chunk-78998358de8a8cc3c018264c9a553b4d"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "工会委员会<SEP>工会支出"
|
||||
},
|
||||
"出差补贴<SEP>组织人事部": {
|
||||
"chunk_ids": [
|
||||
"chunk-78998358de8a8cc3c018264c9a553b4d"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "出差补贴<SEP>组织人事部"
|
||||
},
|
||||
"公司支出管理办法<SEP>差旅费": {
|
||||
"chunk_ids": [
|
||||
"chunk-78998358de8a8cc3c018264c9a553b4d"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945812,
|
||||
"update_time": 1778945812,
|
||||
"_id": "公司支出管理办法<SEP>差旅费"
|
||||
},
|
||||
"各级管理人员<SEP>报销业务": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945813,
|
||||
"update_time": 1778945813,
|
||||
"_id": "各级管理人员<SEP>报销业务"
|
||||
},
|
||||
"公司支出管理办法<SEP>投标保证金": {
|
||||
"chunk_ids": [
|
||||
"chunk-78998358de8a8cc3c018264c9a553b4d"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945815,
|
||||
"update_time": 1778945815,
|
||||
"_id": "公司支出管理办法<SEP>投标保证金"
|
||||
},
|
||||
"第二十条<SEP>薪酬福利支出": {
|
||||
"chunk_ids": [
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945815,
|
||||
"update_time": 1778945815,
|
||||
"_id": "第二十条<SEP>薪酬福利支出"
|
||||
},
|
||||
"对外捐赠支出<SEP>第二十一条": {
|
||||
"chunk_ids": [
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945815,
|
||||
"update_time": 1778945815,
|
||||
"_id": "对外捐赠支出<SEP>第二十一条"
|
||||
},
|
||||
"涉外业务汇率标准<SEP>第二十二条": {
|
||||
"chunk_ids": [
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945827,
|
||||
"update_time": 1778945827,
|
||||
"_id": "涉外业务汇率标准<SEP>第二十二条"
|
||||
},
|
||||
"第二十三条<SEP>计划财务部": {
|
||||
"chunk_ids": [
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945815,
|
||||
"update_time": 1778945815,
|
||||
"_id": "第二十三条<SEP>计划财务部"
|
||||
},
|
||||
"第二十四条<SEP>附表1": {
|
||||
"chunk_ids": [
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945816,
|
||||
"update_time": 1778945816,
|
||||
"_id": "第二十四条<SEP>附表1"
|
||||
},
|
||||
"第二十四条<SEP>附表2": {
|
||||
"chunk_ids": [
|
||||
"chunk-8881e68061e1b668defe35b1cd9d8a83"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945816,
|
||||
"update_time": 1778945816,
|
||||
"_id": "第二十四条<SEP>附表2"
|
||||
},
|
||||
"发票<SEP>报销业务": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945816,
|
||||
"update_time": 1778945816,
|
||||
"_id": "发票<SEP>报销业务"
|
||||
},
|
||||
"支出审批流转程序<SEP>逐级审批规则": {
|
||||
"chunk_ids": [
|
||||
"chunk-37889c882c89c19f96b9b2ca93685014"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945816,
|
||||
"update_time": 1778945816,
|
||||
"_id": "支出审批流转程序<SEP>逐级审批规则"
|
||||
},
|
||||
"公司支出管理办法<SEP>审批流转程序": {
|
||||
"chunk_ids": [
|
||||
"chunk-78998358de8a8cc3c018264c9a553b4d"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945820,
|
||||
"update_time": 1778945820,
|
||||
"_id": "公司支出管理办法<SEP>审批流转程序"
|
||||
},
|
||||
"特殊事项<SEP>终审岗": {
|
||||
"chunk_ids": [
|
||||
"chunk-37889c882c89c19f96b9b2ca93685014"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945820,
|
||||
"update_time": 1778945820,
|
||||
"_id": "特殊事项<SEP>终审岗"
|
||||
},
|
||||
"事业部总经理<SEP>逐级审批规则": {
|
||||
"chunk_ids": [
|
||||
"chunk-37889c882c89c19f96b9b2ca93685014"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945821,
|
||||
"update_time": 1778945821,
|
||||
"_id": "事业部总经理<SEP>逐级审批规则"
|
||||
},
|
||||
"计划财务部<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945821,
|
||||
"update_time": 1778945821,
|
||||
"_id": "计划财务部<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"发票<SEP>经办人": {
|
||||
"chunk_ids": [
|
||||
"chunk-60066b4c758ad553106e2343a99c890e"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945822,
|
||||
"update_time": 1778945822,
|
||||
"_id": "发票<SEP>经办人"
|
||||
},
|
||||
"支出审批流转程序<SEP>终审岗": {
|
||||
"chunk_ids": [
|
||||
"chunk-37889c882c89c19f96b9b2ca93685014"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945823,
|
||||
"update_time": 1778945823,
|
||||
"_id": "支出审批流转程序<SEP>终审岗"
|
||||
},
|
||||
"归口管理部门<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945824,
|
||||
"update_time": 1778945824,
|
||||
"_id": "归口管理部门<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"国家电网公司<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945824,
|
||||
"update_time": 1778945824,
|
||||
"_id": "国家电网公司<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"归口管理部门<SEP>计划财务部": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945824,
|
||||
"update_time": 1778945824,
|
||||
"_id": "归口管理部门<SEP>计划财务部"
|
||||
},
|
||||
"各级管理人员<SEP>支出报销审批": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945824,
|
||||
"update_time": 1778945824,
|
||||
"_id": "各级管理人员<SEP>支出报销审批"
|
||||
},
|
||||
"国网数科公司<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945828,
|
||||
"update_time": 1778945828,
|
||||
"_id": "国网数科公司<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"分支机构<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945831,
|
||||
"update_time": 1778945831,
|
||||
"_id": "分支机构<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"全资子公司<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945832,
|
||||
"update_time": 1778945832,
|
||||
"_id": "全资子公司<SEP>远光软件股份有限公司"
|
||||
},
|
||||
"控股子公司<SEP>远光软件股份有限公司": {
|
||||
"chunk_ids": [
|
||||
"chunk-16edf05e3f89da28ca60c9b8e3101d26"
|
||||
],
|
||||
"count": 1,
|
||||
"create_time": 1778945832,
|
||||
"update_time": 1778945832,
|
||||
"_id": "控股子公司<SEP>远光软件股份有限公司"
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,13 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.services.document_intelligence import DocumentIntelligenceService, build_document_insight
|
||||
from app.services.runtime_chat import RuntimeChatService
|
||||
|
||||
|
||||
def test_build_document_insight_prefers_transport_for_didi_text_with_hotel_noise() -> None:
|
||||
@@ -23,28 +20,7 @@ def test_build_document_insight_prefers_transport_for_didi_text_with_hotel_noise
|
||||
assert any(field.label == "金额" and field.value == "48元" for field in insight.fields)
|
||||
|
||||
|
||||
def test_document_intelligence_service_uses_vlm_result_when_preview_available(monkeypatch) -> None:
|
||||
calls: list[tuple[str, ...]] = []
|
||||
|
||||
def fake_complete(self, messages, *, slot_priority=("main", "backup"), max_tokens=500, temperature=0.2):
|
||||
calls.append(slot_priority)
|
||||
if slot_priority == ("vlm",):
|
||||
assert isinstance(messages[1]["content"], list)
|
||||
return json.dumps(
|
||||
{
|
||||
"document_type": "taxi_receipt",
|
||||
"scene_code": "transport",
|
||||
"scene_label": "交通票据",
|
||||
"expense_type": "transport",
|
||||
"confidence": 0.91,
|
||||
"evidence": ["图片主体为滴滴行程单,OCR 中出现订单号、上车、下车等字段"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(RuntimeChatService, "complete", fake_complete)
|
||||
|
||||
def test_document_intelligence_service_uses_rule_result_when_preview_available() -> None:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
@@ -62,8 +38,7 @@ def test_document_intelligence_service_uses_vlm_result_when_preview_available(mo
|
||||
session.close()
|
||||
|
||||
assert insight.document_type == "taxi_receipt"
|
||||
assert insight.classification_source == "llm_vision"
|
||||
assert calls[0] == ("vlm",)
|
||||
assert insight.classification_source == "rule"
|
||||
|
||||
|
||||
def test_document_intelligence_extracts_larger_decimal_amount_from_multiple_candidates() -> None:
|
||||
@@ -76,28 +51,7 @@ def test_document_intelligence_extracts_larger_decimal_amount_from_multiple_cand
|
||||
assert any(field.label == "金额" and field.value == "13.4元" for field in insight.fields)
|
||||
|
||||
|
||||
def test_document_intelligence_service_uses_vlm_fields_to_correct_amount(monkeypatch) -> None:
|
||||
def fake_complete(self, messages, *, slot_priority=("main", "backup"), max_tokens=500, temperature=0.2):
|
||||
if slot_priority == ("vlm",):
|
||||
return json.dumps(
|
||||
{
|
||||
"document_type": "taxi_receipt",
|
||||
"scene_code": "transport",
|
||||
"scene_label": "交通票据",
|
||||
"expense_type": "transport",
|
||||
"confidence": 0.89,
|
||||
"evidence": ["图片主体为滴滴行程单,金额区域显示 13.4 元"],
|
||||
"fields": [
|
||||
{"key": "amount", "label": "金额", "value": "13.4"},
|
||||
{"key": "merchant_name", "label": "商户", "value": "滴滴出行"},
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(RuntimeChatService, "complete", fake_complete)
|
||||
|
||||
def test_document_intelligence_service_keeps_rule_fields_without_model_correction() -> None:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
@@ -114,5 +68,5 @@ def test_document_intelligence_service_uses_vlm_fields_to_correct_amount(monkeyp
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
assert any(field.label == "金额" and field.value == "13.4元" for field in insight.fields)
|
||||
assert any("大模型复核结果修正" in warning for warning in insight.warnings)
|
||||
assert any(field.label == "金额" and field.value == "1元" for field in insight.fields)
|
||||
assert insight.warnings == ()
|
||||
|
||||
123
server/tests/test_knowledge_normalizer.py
Normal file
123
server/tests/test_knowledge_normalizer.py
Normal file
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.db.base import Base
|
||||
from app.services.knowledge_normalizer import KnowledgeNormalizationService
|
||||
|
||||
|
||||
def build_session_factory() -> sessionmaker[Session]:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
return sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def test_knowledge_normalizer_appends_structured_table(monkeypatch) -> None:
|
||||
session_factory = build_session_factory()
|
||||
raw_text = (
|
||||
"表3 出差补贴标准\n\n"
|
||||
"单位:人民币元/天\n"
|
||||
"补助类型 项目 港澳台 直辖市/特区/西藏 其他地区 国外\n"
|
||||
"餐补 自行解决餐食 75 65 55 140\n"
|
||||
"基本出差补贴 35 35 35 35\n"
|
||||
"合计 110 100 90 175\n"
|
||||
)
|
||||
with session_factory() as db:
|
||||
service = KnowledgeNormalizationService(db)
|
||||
monkeypatch.setattr(
|
||||
service.runtime_chat_service,
|
||||
"complete",
|
||||
lambda *args, **kwargs: (
|
||||
"| 补助类型 | 港澳台 | 直辖市/特区/西藏 | 其他地区 | 国外 |\n"
|
||||
"|---|---:|---:|---:|---:|\n"
|
||||
"| 餐补 | 75 | 65 | 55 | 140 |\n"
|
||||
"| 基本出差补贴 | 35 | 35 | 35 | 35 |\n"
|
||||
"| 合计 | 110 | 100 | 90 | 175 |"
|
||||
),
|
||||
)
|
||||
|
||||
enriched = service.build_enriched_text(raw_text)
|
||||
|
||||
assert enriched.startswith("# 结构化表格补充")
|
||||
assert "| 餐补 | 75 | 65 | 55 | 140 |" in enriched
|
||||
assert enriched.endswith(raw_text.strip())
|
||||
|
||||
|
||||
def test_knowledge_normalizer_keeps_only_markdown_table_body() -> None:
|
||||
cleaned = KnowledgeNormalizationService._sanitize_answer(
|
||||
"## 表3 出差补贴标准\n\n"
|
||||
"| 补助类型 | 港澳台 | 直辖市/特区/西藏 |\n"
|
||||
"|---|---:|---:|\n"
|
||||
"| 餐补 | 75 | 65 |\n\n"
|
||||
"注:主办方统一安排餐食时,不再报销餐补。"
|
||||
)
|
||||
|
||||
assert cleaned == (
|
||||
"| 补助类型 | 港澳台 | 直辖市/特区/西藏 |\n"
|
||||
"|---|---:|---:|\n"
|
||||
"| 餐补 | 75 | 65 |"
|
||||
)
|
||||
|
||||
|
||||
def test_knowledge_normalizer_builds_section_navigation_without_table() -> None:
|
||||
session_factory = build_session_factory()
|
||||
raw_text = (
|
||||
"第一章 总则\n"
|
||||
"本制度适用于员工差旅报销和审批管理。\n\n"
|
||||
"第二章 住宿费标准\n"
|
||||
"住宿费按照出差城市档位和职级标准执行。\n\n"
|
||||
"第三章 交通费标准\n"
|
||||
"交通费应结合出差工具、舱位和审批要求报销。\n"
|
||||
)
|
||||
with session_factory() as db:
|
||||
service = KnowledgeNormalizationService(db)
|
||||
enriched = service.build_enriched_text(raw_text)
|
||||
|
||||
assert enriched.startswith("# 章节导航")
|
||||
assert "- 第一章 总则" in enriched
|
||||
assert "## 第二章 住宿费标准" in enriched
|
||||
assert "# 问答线索补充" in enriched
|
||||
assert "- 第二章 住宿费标准:住宿费按照出差城市档位和职级标准执行" in enriched
|
||||
assert enriched.endswith(raw_text.strip())
|
||||
|
||||
|
||||
def test_knowledge_normalizer_builds_answer_clues_from_lists_and_kv_lines() -> None:
|
||||
session_factory = build_session_factory()
|
||||
raw_text = (
|
||||
"第一章 报销要求\n"
|
||||
"报销时限:费用发生后 30 日内提交申请。\n"
|
||||
"- 超过 30 日需补充审批说明。\n"
|
||||
"第十条 发票遗失的,应先提交遗失说明。\n"
|
||||
)
|
||||
|
||||
with session_factory() as db:
|
||||
service = KnowledgeNormalizationService(db)
|
||||
enriched = service.build_enriched_text(raw_text)
|
||||
|
||||
assert "# 问答线索补充" in enriched
|
||||
assert "- 第一章 报销要求:报销时限:费用发生后 30 日内提交申请" in enriched
|
||||
assert "- 第一章 报销要求:超过 30 日需补充审批说明" in enriched
|
||||
assert "- 第一章 报销要求:第十条 发票遗失的,应先提交遗失说明" in enriched
|
||||
|
||||
|
||||
def test_knowledge_normalizer_builds_answer_clues_without_section_headings() -> None:
|
||||
session_factory = build_session_factory()
|
||||
raw_text = (
|
||||
"报销时限:费用发生后 30 日内提交申请。\n"
|
||||
"超过 30 日需补充审批说明。\n"
|
||||
"审批材料包括发票、行程单和付款凭证。\n"
|
||||
)
|
||||
|
||||
with session_factory() as db:
|
||||
service = KnowledgeNormalizationService(db)
|
||||
enriched = service.build_enriched_text(raw_text)
|
||||
|
||||
assert "# 问答线索补充" in enriched
|
||||
assert "- 正文:报销时限:费用发生后 30 日内提交申请" in enriched
|
||||
assert "- 正文:超过 30 日需补充审批说明" in enriched
|
||||
95
server/tests/test_knowledge_rag_service.py
Normal file
95
server/tests/test_knowledge_rag_service.py
Normal file
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services import knowledge_rag as knowledge_rag_module
|
||||
from app.services.knowledge_rag import KnowledgeRagService
|
||||
|
||||
|
||||
def test_build_hits_prioritizes_structured_table_evidence_for_standard_queries() -> None:
|
||||
hits = KnowledgeRagService._build_hits_from_query_data(
|
||||
query="住宿费标准是多少?",
|
||||
chunks=[
|
||||
{
|
||||
"chunk_id": "plain-1",
|
||||
"file_path": "/tmp/doc-1__差旅制度.md",
|
||||
"content": "住宿费说明文字,提到了出差和报销要求,但没有清晰表格。",
|
||||
},
|
||||
{
|
||||
"chunk_id": "table-1",
|
||||
"file_path": "/tmp/doc-1__差旅制度.md",
|
||||
"content": "# 结构化表格补充\n\n| 城市 | 住宿费标准 |\n| 北京 | 500 |",
|
||||
},
|
||||
],
|
||||
entities=[],
|
||||
limit=2,
|
||||
)
|
||||
|
||||
assert [item["candidate_id"] for item in hits] == ["table-1", "plain-1"]
|
||||
|
||||
|
||||
def test_build_hits_boosts_query_term_matches() -> None:
|
||||
hits = KnowledgeRagService._build_hits_from_query_data(
|
||||
query="招待费报销标准",
|
||||
chunks=[
|
||||
{
|
||||
"chunk_id": "travel-1",
|
||||
"file_path": "/tmp/doc-1__费用制度.md",
|
||||
"content": "差旅费包含交通费、住宿费和餐补标准。",
|
||||
},
|
||||
{
|
||||
"chunk_id": "ent-1",
|
||||
"file_path": "/tmp/doc-1__费用制度.md",
|
||||
"content": "业务招待费报销标准:应结合客户接待场景、人数和审批要求执行。",
|
||||
},
|
||||
],
|
||||
entities=[],
|
||||
limit=2,
|
||||
)
|
||||
|
||||
assert [item["candidate_id"] for item in hits] == ["ent-1", "travel-1"]
|
||||
|
||||
|
||||
def test_build_hits_prioritizes_answer_clue_appendix_for_rule_queries() -> None:
|
||||
hits = KnowledgeRagService._build_hits_from_query_data(
|
||||
query="报销时限是多少?",
|
||||
chunks=[
|
||||
{
|
||||
"chunk_id": "plain-1",
|
||||
"file_path": "/tmp/doc-1__费用制度.md",
|
||||
"content": "本制度用于规范报销流程,员工应遵守公司审批要求。",
|
||||
},
|
||||
{
|
||||
"chunk_id": "clue-1",
|
||||
"file_path": "/tmp/doc-1__费用制度.md",
|
||||
"content": (
|
||||
"# 问答线索补充\n\n"
|
||||
"- 第二章 报销时限:费用发生后 30 日内提交申请。\n"
|
||||
"- 第二章 报销时限:超过 30 日需补充审批说明。"
|
||||
),
|
||||
},
|
||||
],
|
||||
entities=[],
|
||||
limit=2,
|
||||
)
|
||||
|
||||
assert [item["candidate_id"] for item in hits] == ["clue-1", "plain-1"]
|
||||
|
||||
|
||||
def test_resolve_default_qdrant_url_prefers_container_host(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
knowledge_rag_module.socket,
|
||||
"getaddrinfo",
|
||||
lambda hostname, port: [("family", "type", "proto", "canonname", ("172.21.0.2", 0))]
|
||||
if hostname == "qdrant"
|
||||
else [],
|
||||
)
|
||||
|
||||
assert knowledge_rag_module._resolve_default_qdrant_url() == "http://qdrant:6333"
|
||||
|
||||
|
||||
def test_resolve_default_qdrant_url_falls_back_to_loopback(monkeypatch) -> None:
|
||||
def raise_lookup_error(_hostname, _port):
|
||||
raise OSError("lookup failed")
|
||||
|
||||
monkeypatch.setattr(knowledge_rag_module.socket, "getaddrinfo", raise_lookup_error)
|
||||
|
||||
assert knowledge_rag_module._resolve_default_qdrant_url() == "http://127.0.0.1:6333"
|
||||
@@ -1,713 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from subprocess import TimeoutExpired
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.api.deps import CurrentUserContext, get_db
|
||||
from app.core.agent_enums import AgentReviewStatus, AgentRunSource, AgentRunStatus
|
||||
from app.db.base import Base
|
||||
from app.main import create_app
|
||||
from app.schemas.agent_asset import AgentAssetReviewCreate
|
||||
from app.schemas.knowledge import LlmWikiSummaryUpdateWrite
|
||||
from app.services.agent_assets import AgentAssetService
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import (
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
KNOWLEDGE_INGEST_STATUS_INGESTED,
|
||||
KNOWLEDGE_INGEST_STATUS_PUBLISHED,
|
||||
KnowledgeService,
|
||||
)
|
||||
from app.services.llm_wiki import CandidateModelAttempt, LlmWikiService
|
||||
|
||||
|
||||
def build_session() -> Session:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
return session_factory()
|
||||
|
||||
|
||||
def build_client() -> tuple[TestClient, sessionmaker[Session]]:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
app = create_app()
|
||||
|
||||
def override_db() -> Generator[Session, None, None]:
|
||||
db = session_factory()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
app.dependency_overrides[get_db] = override_db
|
||||
return TestClient(app), session_factory
|
||||
|
||||
|
||||
def build_admin_user() -> CurrentUserContext:
|
||||
return CurrentUserContext(
|
||||
username="admin",
|
||||
name="管理员",
|
||||
role_codes=["manager"],
|
||||
is_admin=True,
|
||||
)
|
||||
|
||||
|
||||
def upload_policy_document(storage_root: Path, *, filename: str = "公司差旅报销制度.txt") -> str:
|
||||
service = KnowledgeService(storage_root=storage_root)
|
||||
service.ensure_library_ready()
|
||||
document = service.upload_document(
|
||||
folder="报销制度",
|
||||
filename=filename,
|
||||
content=(
|
||||
"第一章 差旅报销\n"
|
||||
"员工因公出差发生的住宿费应按照公司差旅标准执行。\n"
|
||||
"住宿费超过标准时,必须升级至总经理审批。\n"
|
||||
"报销时必须提供发票、行程单和审批说明。\n"
|
||||
).encode("utf-8"),
|
||||
current_user=build_admin_user(),
|
||||
)
|
||||
return document.id
|
||||
|
||||
|
||||
def upload_multipage_policy_document(storage_root: Path, *, filename: str = "公司支出管理办法.txt") -> str:
|
||||
service = KnowledgeService(storage_root=storage_root)
|
||||
service.ensure_library_ready()
|
||||
document = service.upload_document(
|
||||
folder="报销制度",
|
||||
filename=filename,
|
||||
content=(
|
||||
"商密【中】\n"
|
||||
"关于颁布《公司支出管理办法》的通知\n"
|
||||
"特此通知。\n"
|
||||
"\f"
|
||||
"目录\n"
|
||||
"第一章 总则................................4\n"
|
||||
"第二章 报销审批................................7\n"
|
||||
"\f"
|
||||
"第一条 报销申请\n"
|
||||
"员工提交报销申请时,应附发票、行程单和审批说明。\n"
|
||||
"第二条 报销审批\n"
|
||||
"住宿费超过制度标准时,必须升级至总经理审批。\n"
|
||||
"第三条 附件补充\n"
|
||||
"缺少附件时不得提交报销。\n"
|
||||
"\f"
|
||||
"第四条 财务复核\n"
|
||||
"财务复核时应校验预算、发票真伪和审批链完整性。\n"
|
||||
).encode("utf-8"),
|
||||
current_user=build_admin_user(),
|
||||
)
|
||||
return document.id
|
||||
|
||||
|
||||
def build_candidate_payload(chunk_id: str, *, summary: str = "住宿费超过标准时必须升级审批。") -> dict[str, object]:
|
||||
return {
|
||||
"knowledge_candidates": [
|
||||
{
|
||||
"title": "住宿费升级审批要求",
|
||||
"content": summary,
|
||||
"scenario": "reimbursement_policy",
|
||||
"tags": ["住宿", "审批"],
|
||||
"evidence": [summary],
|
||||
"confidence": 0.91,
|
||||
"source_chunk_ids": [chunk_id],
|
||||
}
|
||||
],
|
||||
"rule_candidates": [
|
||||
{
|
||||
"template_key": "expense_amount_limit_v1",
|
||||
"suggested_rule_name": "住宿费超标审批规则",
|
||||
"summary": "当住宿费超过制度标准时触发升级审批。",
|
||||
"scenario": "travel_standard",
|
||||
"purpose": "识别差旅住宿费是否超出制度标准。",
|
||||
"scope": "适用于员工差旅住宿报销场景。",
|
||||
"inputs": ["expense_type", "amount", "travel_grade"],
|
||||
"judgement_logic": [summary],
|
||||
"outputs": ["approval_required=true", "risk_level=medium"],
|
||||
"admin_note": "上线前需要由财务补充不同职级的金额阈值。",
|
||||
"runtime_rule": {
|
||||
"target": {
|
||||
"expense_types": ["hotel"],
|
||||
"scene_codes": ["travel_standard"],
|
||||
"metric": "item_amount",
|
||||
},
|
||||
"threshold": {
|
||||
"currency": "CNY",
|
||||
"comparator": "gt",
|
||||
"warn_amount": "450.00",
|
||||
"block_amount": "600.00",
|
||||
"source": "document_value",
|
||||
},
|
||||
"exception_policy": {
|
||||
"allow_with_explanation": True,
|
||||
"keywords": ["超标说明", "协议酒店满房"],
|
||||
},
|
||||
"output": {
|
||||
"risk_code": "travel_hotel_limit",
|
||||
"action": "review",
|
||||
"message": "住宿费超过制度标准时需要升级审批。",
|
||||
},
|
||||
},
|
||||
"evidence": [summary],
|
||||
"confidence": 0.93,
|
||||
"source_chunk_ids": [chunk_id],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_invalid_candidate_payload(chunk_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"knowledge_candidates": [],
|
||||
"rule_candidates": [
|
||||
{
|
||||
"template_key": "expense_amount_limit_v1",
|
||||
"suggested_rule_name": "无效金额规则草稿",
|
||||
"summary": "用于验证 schema 强校验。",
|
||||
"scenario": "travel_standard",
|
||||
"purpose": "验证不合规的 runtime_rule 不会落到规则中心。",
|
||||
"scope": "测试场景。",
|
||||
"inputs": ["expense_type", "amount"],
|
||||
"judgement_logic": ["金额超过标准则需审批。"],
|
||||
"outputs": ["approval_required=true"],
|
||||
"admin_note": "此规则故意构造错误阈值。",
|
||||
"runtime_rule": {
|
||||
"target": {
|
||||
"expense_types": ["hotel"],
|
||||
"scene_codes": ["travel_standard"],
|
||||
"metric": "item_amount",
|
||||
},
|
||||
"threshold": {
|
||||
"currency": "CNY",
|
||||
"comparator": "gt",
|
||||
"warn_amount": "600.00",
|
||||
"block_amount": "450.00",
|
||||
"source": "document_value",
|
||||
},
|
||||
"output": {
|
||||
"risk_code": "travel_hotel_limit",
|
||||
"action": "review",
|
||||
"message": "无效阈值。",
|
||||
},
|
||||
},
|
||||
"evidence": ["金额阈值配置不应允许 block 小于 warn。"],
|
||||
"confidence": 0.88,
|
||||
"source_chunk_ids": [chunk_id],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def update_document_timestamp(storage_root: Path, document_id: str, updated_at: str) -> None:
|
||||
index_path = storage_root / "knowledge" / ".index.json"
|
||||
payload = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
for item in payload["documents"]:
|
||||
if item["id"] == document_id:
|
||||
item["updated_at"] = updated_at
|
||||
break
|
||||
index_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def test_llm_wiki_sync_creates_artifacts_and_draft_rule(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
return build_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
result = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
|
||||
assert result.document_count == 1
|
||||
assert result.knowledge_candidate_count == 1
|
||||
assert result.rule_candidate_count == 1
|
||||
assert result.generated_rule_count == 1
|
||||
assert len(result.generated_rule_asset_ids) == 1
|
||||
|
||||
document_dir = tmp_path / "knowledge" / ".llm_wiki" / "documents" / document_id
|
||||
assert (document_dir / "document.json").exists()
|
||||
assert (document_dir / "text.md").exists()
|
||||
assert (document_dir / "chunks.json").exists()
|
||||
assert (document_dir / "knowledge_candidates.json").exists()
|
||||
assert (document_dir / "knowledge_summary.md").exists()
|
||||
assert (document_dir / "rule_candidates.json").exists()
|
||||
|
||||
document_payload = json.loads((document_dir / "document.json").read_text(encoding="utf-8"))
|
||||
assert document_payload["sync_reason"] == "initial_build"
|
||||
assert document_payload["quality_status"] == "formal"
|
||||
assert document_payload["formal_knowledge_candidate_count"] == 1
|
||||
assert document_payload["fallback_knowledge_candidate_count"] == 0
|
||||
|
||||
detail = service.get_document_detail(document_id)
|
||||
assert "公司差旅报销制度.txt 知识总结" in detail.knowledge_summary_markdown
|
||||
assert "住宿费升级审批要求" in detail.knowledge_summary_markdown
|
||||
assert detail.quality_status == "formal"
|
||||
|
||||
asset = AgentAssetService(db).get_asset(result.generated_rule_asset_ids[0])
|
||||
assert asset is not None
|
||||
assert asset.status == "draft"
|
||||
assert asset.config_json["llm_wiki_managed"] is True
|
||||
assert asset.config_json["runtime_rule"]["template_key"] == "expense_amount_limit_v1"
|
||||
assert asset.config_json["runtime_rule"]["threshold"]["block_amount"] == "600.00"
|
||||
assert "```expense-rule" in str(asset.current_version_content)
|
||||
|
||||
|
||||
def test_llm_wiki_document_summary_can_be_updated(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
return build_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
|
||||
updated = service.update_document_summary(
|
||||
document_id,
|
||||
LlmWikiSummaryUpdateWrite(
|
||||
knowledge_summary_markdown="# 人工修订总结\n\n- 住宿费超标必须升级审批。\n- 报销时必须附发票和审批说明。"
|
||||
),
|
||||
)
|
||||
|
||||
assert updated.document_id == document_id
|
||||
assert updated.knowledge_summary_markdown.startswith("# 人工修订总结")
|
||||
|
||||
summary_path = tmp_path / "knowledge" / ".llm_wiki" / "documents" / document_id / "knowledge_summary.md"
|
||||
assert summary_path.read_text(encoding="utf-8").startswith("# 人工修订总结")
|
||||
|
||||
|
||||
def test_llm_wiki_sync_rejects_invalid_runtime_rule_schema(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
return build_invalid_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
result = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
|
||||
assert result.document_count == 1
|
||||
assert result.rule_candidate_count == 1
|
||||
assert result.generated_rule_count == 0
|
||||
|
||||
document_dir = tmp_path / "knowledge" / ".llm_wiki" / "documents" / document_id
|
||||
rule_candidates = json.loads((document_dir / "rule_candidates.json").read_text(encoding="utf-8"))
|
||||
|
||||
assert rule_candidates[0]["validation_status"] == "invalid"
|
||||
assert rule_candidates[0]["status"] == "validation_failed"
|
||||
assert rule_candidates[0]["validation_errors"]
|
||||
assert "block_amount" in " ".join(rule_candidates[0]["validation_errors"])
|
||||
|
||||
|
||||
def test_knowledge_document_state_changes_with_llm_wiki_sync(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
return build_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
knowledge_service = KnowledgeService(storage_root=tmp_path)
|
||||
initial_detail = knowledge_service.get_document_detail(document_id)
|
||||
assert initial_detail.stateCode == KNOWLEDGE_INGEST_STATUS_PUBLISHED
|
||||
assert initial_detail.state == "待归纳"
|
||||
|
||||
with build_session() as db:
|
||||
LlmWikiService(db, storage_root=tmp_path).sync_folder(
|
||||
folder="报销制度",
|
||||
current_user=build_admin_user(),
|
||||
document_ids=[document_id],
|
||||
)
|
||||
|
||||
ingested_detail = knowledge_service.get_document_detail(document_id)
|
||||
assert ingested_detail.stateCode == KNOWLEDGE_INGEST_STATUS_INGESTED
|
||||
assert ingested_detail.state == "已归纳"
|
||||
|
||||
updated_detail = knowledge_service.upload_document(
|
||||
folder="报销制度",
|
||||
filename="公司差旅报销制度.txt",
|
||||
content=(
|
||||
"第一章 差旅报销\n"
|
||||
"员工因公出差发生的住宿费应按照公司差旅标准执行。\n"
|
||||
"新增:超标住宿必须附书面说明。\n"
|
||||
).encode("utf-8"),
|
||||
current_user=build_admin_user(),
|
||||
)
|
||||
assert updated_detail.id == document_id
|
||||
assert updated_detail.stateCode == KNOWLEDGE_INGEST_STATUS_PUBLISHED
|
||||
assert updated_detail.state == "待归纳"
|
||||
|
||||
index_payload = json.loads((tmp_path / "knowledge" / ".index.json").read_text(encoding="utf-8"))
|
||||
stored_entry = next(item for item in index_payload["documents"] if item["id"] == document_id)
|
||||
assert stored_entry["ingest_status"] == KNOWLEDGE_INGEST_STATUS_PUBLISHED
|
||||
|
||||
|
||||
def test_llm_wiki_sync_marks_document_failed_when_ingest_raises(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
raise RuntimeError("simulated llm wiki failure")
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
with pytest.raises(RuntimeError, match="simulated llm wiki failure"):
|
||||
service.sync_folder(
|
||||
folder="报销制度",
|
||||
current_user=build_admin_user(),
|
||||
document_ids=[document_id],
|
||||
)
|
||||
|
||||
detail = KnowledgeService(storage_root=tmp_path).get_document_detail(document_id)
|
||||
assert detail.stateCode == KNOWLEDGE_INGEST_STATUS_FAILED
|
||||
assert detail.state == "归纳失败"
|
||||
|
||||
|
||||
def test_llm_wiki_sync_uses_fallback_candidates_when_system_hermes_times_out(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
|
||||
monkeypatch.setattr(service.system_hermes_service, "is_available", lambda: True)
|
||||
|
||||
def fake_run_query(*args, **kwargs):
|
||||
raise TimeoutExpired(cmd="hermes", timeout=1)
|
||||
|
||||
monkeypatch.setattr(service.system_hermes_service, "run_query", fake_run_query)
|
||||
|
||||
runtime_called = {"count": 0}
|
||||
|
||||
def fail_runtime_complete(*args, **kwargs):
|
||||
runtime_called["count"] += 1
|
||||
raise AssertionError("system hermes timeout should fall back directly to local candidate builder")
|
||||
|
||||
monkeypatch.setattr(service.runtime_chat_service, "complete", fail_runtime_complete)
|
||||
|
||||
result = service.sync_folder(
|
||||
folder="报销制度",
|
||||
current_user=build_admin_user(),
|
||||
document_ids=[document_id],
|
||||
)
|
||||
|
||||
assert result.document_count == 1
|
||||
assert result.knowledge_candidate_count >= 1
|
||||
assert runtime_called["count"] == 0
|
||||
|
||||
knowledge_service = KnowledgeService(storage_root=tmp_path)
|
||||
detail = knowledge_service.get_document_detail(document_id)
|
||||
assert detail.stateCode == KNOWLEDGE_INGEST_STATUS_FAILED
|
||||
assert detail.state == "归纳失败"
|
||||
assert detail.llmWikiAvailable is True
|
||||
assert detail.llmWikiQualityStatus == "fallback_only"
|
||||
|
||||
document_payload = json.loads(
|
||||
(
|
||||
tmp_path
|
||||
/ "knowledge"
|
||||
/ ".llm_wiki"
|
||||
/ "documents"
|
||||
/ document_id
|
||||
/ "document.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert document_payload["quality_status"] == "fallback_only"
|
||||
assert document_payload["formal_knowledge_candidate_count"] == 0
|
||||
assert document_payload["fallback_knowledge_candidate_count"] == 1
|
||||
|
||||
candidates_payload = json.loads(
|
||||
(
|
||||
tmp_path
|
||||
/ "knowledge"
|
||||
/ ".llm_wiki"
|
||||
/ "documents"
|
||||
/ document_id
|
||||
/ "knowledge_candidates.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert candidates_payload[0]["extraction_mode"] == "fallback"
|
||||
assert "fallback_only" in candidates_payload[0]["quality_flags"]
|
||||
|
||||
|
||||
def test_llm_wiki_sync_continues_after_single_group_failure(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_multipage_policy_document(tmp_path, filename="多页支出制度.txt")
|
||||
call_count = {"count": 0}
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
call_count["count"] += 1
|
||||
if call_count["count"] == 1:
|
||||
return CandidateModelAttempt(
|
||||
payload={},
|
||||
source="hermes",
|
||||
ok=False,
|
||||
failure_reason="simulated_timeout",
|
||||
)
|
||||
return build_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
result = service.sync_folder(
|
||||
folder="报销制度",
|
||||
current_user=build_admin_user(),
|
||||
document_ids=[document_id],
|
||||
)
|
||||
detail = service.get_document_detail(document_id)
|
||||
|
||||
assert result.document_count == 1
|
||||
assert call_count["count"] >= 2
|
||||
assert detail.quality_status == "partial_degraded"
|
||||
assert detail.successful_group_count >= 1
|
||||
assert detail.failed_group_count >= 1
|
||||
assert detail.formal_knowledge_candidate_count >= 1
|
||||
|
||||
knowledge_detail = KnowledgeService(storage_root=tmp_path).get_document_detail(document_id)
|
||||
assert knowledge_detail.stateCode == KNOWLEDGE_INGEST_STATUS_INGESTED
|
||||
assert knowledge_detail.llmWikiQualityStatus == "partial_degraded"
|
||||
|
||||
|
||||
def test_llm_wiki_filters_cover_and_catalog_chunks_before_candidate_extraction(tmp_path) -> None:
|
||||
document_id = upload_multipage_policy_document(tmp_path, filename="封面目录过滤测试.txt")
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
text = service.knowledge_service.extract_document_text(document_id)
|
||||
chunks = service._build_chunks(document_id=document_id, text=text)
|
||||
candidate_chunks = service._select_candidate_chunks(chunks)
|
||||
|
||||
assert len(chunks) > len(candidate_chunks)
|
||||
assert candidate_chunks
|
||||
assert all(int(item.get("source_page") or 0) >= 3 for item in candidate_chunks)
|
||||
|
||||
|
||||
def test_llm_wiki_sync_skips_unchanged_and_rebuilds_on_updated_at_change(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
return build_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
|
||||
first = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
second = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
|
||||
assert first.document_count == 1
|
||||
assert second.document_count == 0
|
||||
assert "未变化,跳过" in second.summary
|
||||
|
||||
update_document_timestamp(tmp_path, document_id, "2026-05-15T09:30:00+00:00")
|
||||
third = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
|
||||
assert third.document_count == 1
|
||||
|
||||
document_dir = tmp_path / "knowledge" / ".llm_wiki" / "documents" / document_id
|
||||
document_payload = json.loads((document_dir / "document.json").read_text(encoding="utf-8"))
|
||||
assert document_payload["sync_reason"] == "updated_at_changed"
|
||||
|
||||
|
||||
def test_llm_wiki_sync_does_not_overwrite_active_rule(tmp_path, monkeypatch) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
def fake_call_candidate_model(self, *, entry, chunk_group):
|
||||
return build_candidate_payload(chunk_group[0]["chunk_id"])
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model)
|
||||
|
||||
with build_session() as db:
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
first = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
asset_id = first.generated_rule_asset_ids[0]
|
||||
|
||||
asset_service = AgentAssetService(db)
|
||||
asset_detail = asset_service.get_asset(asset_id)
|
||||
assert asset_detail is not None
|
||||
asset_service.create_review(
|
||||
asset_id,
|
||||
AgentAssetReviewCreate(
|
||||
version=asset_detail.current_version or "v1.0.0",
|
||||
reviewer="管理员",
|
||||
review_status=AgentReviewStatus.APPROVED,
|
||||
review_note="允许上线",
|
||||
),
|
||||
actor="管理员",
|
||||
)
|
||||
activated = asset_service.activate_asset(asset_id, actor="管理员")
|
||||
|
||||
assert activated.status == "active"
|
||||
|
||||
original_version = activated.current_version
|
||||
original_content = activated.current_version_content
|
||||
original_config = activated.config_json
|
||||
|
||||
def fake_call_candidate_model_changed(self, *, entry, chunk_group):
|
||||
return build_candidate_payload(
|
||||
chunk_group[0]["chunk_id"],
|
||||
summary="住宿费超过标准时,必须升级审批并记录超标原因。",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(LlmWikiService, "_call_candidate_model", fake_call_candidate_model_changed)
|
||||
update_document_timestamp(tmp_path, document_id, "2026-05-15T10:00:00+00:00")
|
||||
|
||||
second = service.sync_folder(folder="报销制度", current_user=build_admin_user())
|
||||
refreshed = asset_service.get_asset(asset_id)
|
||||
|
||||
assert second.document_count == 1
|
||||
assert second.generated_rule_count == 0
|
||||
assert refreshed is not None
|
||||
assert refreshed.status == "active"
|
||||
assert refreshed.current_version == original_version
|
||||
assert refreshed.current_version_content == original_content
|
||||
assert refreshed.config_json == original_config
|
||||
|
||||
|
||||
def test_llm_wiki_sync_endpoint_records_agent_run(monkeypatch) -> None:
|
||||
client, session_factory = build_client()
|
||||
|
||||
def fake_submit_sync(*, agent_run_id, folder, current_user, document_ids=None, force=False):
|
||||
with session_factory() as db:
|
||||
service = AgentRunService(db)
|
||||
service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_sync",
|
||||
request_json={
|
||||
"folder": folder,
|
||||
"document_ids": list(document_ids or []),
|
||||
"force": force,
|
||||
},
|
||||
response_json={"run_id": "wiki_test_sync"},
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "succeeded",
|
||||
"sync_run_id": "wiki_test_sync",
|
||||
"progress": {
|
||||
"total_documents": len(document_ids or []),
|
||||
"completed_documents": len(document_ids or []),
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
result_summary="已完成 Hermes LLM Wiki 同步。",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.llm_wiki_tasks.llm_wiki_task_manager.submit_sync",
|
||||
fake_submit_sync,
|
||||
)
|
||||
|
||||
with session_factory() as db:
|
||||
before_count = len(AgentRunService(db).list_runs(limit=100))
|
||||
|
||||
response = client.post(
|
||||
"/api/v1/knowledge/llm-wiki/sync",
|
||||
json={"folder": "报销制度", "force": False},
|
||||
headers={
|
||||
"x-auth-username": "admin",
|
||||
"x-auth-name": "admin",
|
||||
"x-auth-is-admin": "true",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["agent_run_id"].startswith("run_")
|
||||
assert payload["status"] == AgentRunStatus.RUNNING.value
|
||||
|
||||
with session_factory() as db:
|
||||
service = AgentRunService(db)
|
||||
after_runs = service.list_runs(limit=100)
|
||||
assert len(after_runs) == before_count + 1
|
||||
|
||||
latest_run = after_runs[0]
|
||||
assert latest_run.agent == "hermes"
|
||||
assert latest_run.source == AgentRunSource.SCHEDULE.value
|
||||
assert latest_run.status == AgentRunStatus.SUCCEEDED.value
|
||||
assert latest_run.tool_calls
|
||||
assert latest_run.tool_calls[0].tool_name == "system_hermes_llm_wiki_sync"
|
||||
assert latest_run.tool_calls[0].status == "succeeded"
|
||||
assert latest_run.route_json["sync_run_id"] == "wiki_test_sync"
|
||||
|
||||
|
||||
def test_llm_wiki_callback_finalizes_one_whole_document_result(tmp_path) -> None:
|
||||
document_id = upload_policy_document(tmp_path)
|
||||
|
||||
with build_session() as db:
|
||||
run = AgentRunService(db).create_run(
|
||||
agent="hermes",
|
||||
source=AgentRunSource.SCHEDULE.value,
|
||||
user_id="admin",
|
||||
route_json={
|
||||
"job_type": "llm_wiki_sync",
|
||||
"folder": "报销制度",
|
||||
"requested_document_ids": [document_id],
|
||||
"requested_by_username": "admin",
|
||||
"requested_by_name": "管理员",
|
||||
},
|
||||
)
|
||||
service = LlmWikiService(db, storage_root=tmp_path)
|
||||
candidate_payload = build_candidate_payload(f"{document_id}-document")
|
||||
result = service.finalize_agent_batch_callback(
|
||||
agent_run_id=run.run_id,
|
||||
payload={
|
||||
"ok": True,
|
||||
"summary": "Hermes 已完成整文档归纳。",
|
||||
"folder": "报销制度",
|
||||
"documents": [
|
||||
{
|
||||
"document_id": document_id,
|
||||
"knowledge_summary_markdown": "# Hermes 整文档归纳结果",
|
||||
**candidate_payload,
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
detail = service.get_document_detail(document_id)
|
||||
|
||||
assert result.document_count == 1
|
||||
assert result.knowledge_candidate_count == 1
|
||||
assert result.rule_candidate_count == 1
|
||||
assert detail.chunk_count == 1
|
||||
assert len(detail.chunks) == 1
|
||||
assert detail.chunks[0].chunk_id == f"{document_id}-document"
|
||||
assert detail.knowledge_summary_markdown == "# Hermes 整文档归纳结果"
|
||||
assert detail.quality_status == "formal"
|
||||
|
||||
knowledge_detail = KnowledgeService(storage_root=tmp_path).get_document_detail(document_id)
|
||||
assert knowledge_detail.stateCode == KNOWLEDGE_INGEST_STATUS_INGESTED
|
||||
@@ -310,6 +310,44 @@ def test_semantic_ontology_service_keeps_travel_amount_follow_up_in_knowledge_qu
|
||||
assert result.clarification_required is False
|
||||
|
||||
|
||||
def test_semantic_ontology_service_rejects_draft_intent_inside_knowledge_session(monkeypatch) -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = SemanticOntologyService(db)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_parse_with_model",
|
||||
lambda **kwargs: LlmOntologyParseResult(
|
||||
scenario="expense",
|
||||
intent="draft",
|
||||
confidence=0.91,
|
||||
clarification_required=True,
|
||||
clarification_question="请补充招待对象和票据附件。",
|
||||
missing_slots=["participants", "attachments"],
|
||||
ambiguity=[],
|
||||
entity_hints=[],
|
||||
),
|
||||
)
|
||||
|
||||
result = service.parse(
|
||||
OntologyParseRequest(
|
||||
query="我要去北京出差3天,一共可以报销多少钱?",
|
||||
user_id="pytest",
|
||||
context_json={
|
||||
"role_codes": ["employee"],
|
||||
"name": "曹笑竹",
|
||||
"grade": "P3",
|
||||
"session_type": "knowledge",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert result.scenario == "knowledge"
|
||||
assert result.intent == "query"
|
||||
assert result.clarification_required is False
|
||||
assert result.clarification_question is None
|
||||
|
||||
|
||||
def test_semantic_ontology_service_prefers_expense_for_customer_entertainment_narrative() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
184
server/tests/test_runtime_chat_service.py
Normal file
184
server/tests/test_runtime_chat_service.py
Normal file
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.db.base import Base
|
||||
from app.services import runtime_chat as runtime_chat_module
|
||||
from app.services.runtime_chat import RuntimeChatService
|
||||
|
||||
|
||||
def build_session_factory() -> sessionmaker[Session]:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
return sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def _clear_runtime_chat_cooldown() -> None:
|
||||
runtime_chat_module._slot_failure_until.clear()
|
||||
|
||||
|
||||
def test_runtime_chat_fails_over_to_backup_before_retrying_main(monkeypatch) -> None:
|
||||
_clear_runtime_chat_cooldown()
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = RuntimeChatService(db)
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_load_chat_slot(slot: str):
|
||||
return {
|
||||
"slot": slot,
|
||||
"provider": "MiniMax" if slot == "main" else "GLM",
|
||||
"endpoint": "https://example.com/v1",
|
||||
"model": "main-model" if slot == "main" else "backup-model",
|
||||
"apiKey": "secret",
|
||||
}
|
||||
|
||||
def fake_request_chat_completion(config, messages, *, max_tokens, temperature, timeout_seconds):
|
||||
del messages, max_tokens, temperature, timeout_seconds
|
||||
calls.append(config["slot"])
|
||||
if config["slot"] == "main":
|
||||
raise RuntimeError("main unavailable")
|
||||
return "backup answer"
|
||||
|
||||
monkeypatch.setattr(service, "_load_chat_slot", fake_load_chat_slot)
|
||||
monkeypatch.setattr(service, "_request_chat_completion", fake_request_chat_completion)
|
||||
|
||||
answer = service.complete([{"role": "user", "content": "hello"}])
|
||||
|
||||
assert answer == "backup answer"
|
||||
assert calls == ["main", "backup"]
|
||||
|
||||
|
||||
def test_runtime_chat_does_not_rehit_failed_slots_during_cooldown(monkeypatch) -> None:
|
||||
_clear_runtime_chat_cooldown()
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = RuntimeChatService(db)
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_load_chat_slot(slot: str):
|
||||
return {
|
||||
"slot": slot,
|
||||
"provider": slot,
|
||||
"endpoint": "https://example.com/v1",
|
||||
"model": f"{slot}-model",
|
||||
"apiKey": "secret",
|
||||
}
|
||||
|
||||
def fake_request_chat_completion(config, messages, *, max_tokens, temperature, timeout_seconds):
|
||||
del messages, max_tokens, temperature, timeout_seconds
|
||||
calls.append(config["slot"])
|
||||
raise RuntimeError("unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "_load_chat_slot", fake_load_chat_slot)
|
||||
monkeypatch.setattr(service, "_request_chat_completion", fake_request_chat_completion)
|
||||
monkeypatch.setattr("app.services.runtime_chat.sleep", lambda *_args, **_kwargs: None)
|
||||
|
||||
assert service.complete([{"role": "user", "content": "hello"}]) is None
|
||||
assert calls == ["main", "backup"]
|
||||
|
||||
|
||||
def test_runtime_chat_disables_glm_thinking_for_direct_user_answers(monkeypatch) -> None:
|
||||
_clear_runtime_chat_cooldown()
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = RuntimeChatService(db)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_send_json_request(method, url, *, headers, payload, timeout_seconds):
|
||||
captured["method"] = method
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
captured["payload"] = payload
|
||||
captured["timeout_seconds"] = timeout_seconds
|
||||
return 200, {"choices": [{"message": {"content": "ok"}}]}
|
||||
|
||||
monkeypatch.setattr("app.services.runtime_chat._send_json_request", fake_send_json_request)
|
||||
|
||||
answer = service._request_openai_compatible(
|
||||
provider="GLM",
|
||||
endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
model="glm-5.1",
|
||||
api_key="secret",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=32,
|
||||
temperature=0.2,
|
||||
timeout_seconds=17,
|
||||
)
|
||||
|
||||
assert answer == "ok"
|
||||
assert captured["payload"]["thinking"] == {"type": "disabled"}
|
||||
assert captured["timeout_seconds"] == 17
|
||||
|
||||
|
||||
def test_runtime_chat_supports_single_pass_fast_failover(monkeypatch) -> None:
|
||||
_clear_runtime_chat_cooldown()
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = RuntimeChatService(db)
|
||||
calls: list[tuple[str, int]] = []
|
||||
|
||||
def fake_load_chat_slot(slot: str):
|
||||
return {
|
||||
"slot": slot,
|
||||
"provider": slot,
|
||||
"endpoint": "https://example.com/v1",
|
||||
"model": f"{slot}-model",
|
||||
"apiKey": "secret",
|
||||
}
|
||||
|
||||
def fake_request_chat_completion(config, messages, *, max_tokens, temperature, timeout_seconds):
|
||||
del messages, max_tokens, temperature
|
||||
calls.append((config["slot"], timeout_seconds))
|
||||
raise RuntimeError("unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "_load_chat_slot", fake_load_chat_slot)
|
||||
monkeypatch.setattr(service, "_request_chat_completion", fake_request_chat_completion)
|
||||
|
||||
assert (
|
||||
service.complete(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
timeout_seconds=15,
|
||||
slot_timeouts={"main": 8, "backup": 20},
|
||||
max_attempts=1,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert calls == [("main", 8), ("backup", 20)]
|
||||
|
||||
|
||||
def test_runtime_chat_skips_slot_during_cooldown(monkeypatch) -> None:
|
||||
_clear_runtime_chat_cooldown()
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = RuntimeChatService(db)
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_load_chat_slot(slot: str):
|
||||
return {
|
||||
"slot": slot,
|
||||
"provider": slot,
|
||||
"endpoint": "https://example.com/v1",
|
||||
"model": f"{slot}-model",
|
||||
"apiKey": "secret",
|
||||
}
|
||||
|
||||
def fake_request_chat_completion(config, messages, *, max_tokens, temperature, timeout_seconds):
|
||||
del messages, max_tokens, temperature, timeout_seconds
|
||||
calls.append(config["slot"])
|
||||
if config["slot"] == "main":
|
||||
raise RuntimeError("main unavailable")
|
||||
return "backup answer"
|
||||
|
||||
monkeypatch.setattr(service, "_load_chat_slot", fake_load_chat_slot)
|
||||
monkeypatch.setattr(service, "_request_chat_completion", fake_request_chat_completion)
|
||||
|
||||
assert service.complete([{"role": "user", "content": "hello"}], max_attempts=1) == "backup answer"
|
||||
assert service.complete([{"role": "user", "content": "hello again"}], max_attempts=1) == "backup answer"
|
||||
assert calls == ["main", "backup", "backup"]
|
||||
@@ -10,8 +10,9 @@ import yaml
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.core import admin_secret
|
||||
from app.core import secret_box
|
||||
from app.core import admin_secret
|
||||
from app.core import secret_box
|
||||
from app.core.secret_box import encrypt_secret
|
||||
from app.db.base import Base
|
||||
from app.models.system_model_setting import SystemModelSetting
|
||||
from app.models.system_setting import SystemSetting
|
||||
@@ -245,3 +246,55 @@ def test_blank_secret_input_keeps_synced_hermes_api_key(monkeypatch) -> None:
|
||||
hermes_config = yaml.safe_load(get_hermes_config_path().read_text(encoding="utf-8"))
|
||||
assert hermes_config["model"]["default"] == "gpt-5.4-mini"
|
||||
assert hermes_config["model"]["api_key"] == "persisted-main-key"
|
||||
|
||||
|
||||
def test_settings_service_migrates_legacy_vlm_slot_to_reranker(monkeypatch) -> None:
|
||||
temp_dir = build_temp_secret_dir()
|
||||
monkeypatch.setattr(secret_box, "SECRET_KEY_FILE", temp_dir / "settings.key")
|
||||
monkeypatch.setattr(Base.metadata, "create_all", lambda *args, **kwargs: None)
|
||||
monkeypatch.setenv("HERMES_HOME", str(temp_dir / ".hermes"))
|
||||
|
||||
with build_session(temp_dir / "settings.db") as db:
|
||||
service = SettingsService(db)
|
||||
service.get_settings_snapshot()
|
||||
|
||||
settings_row = db.get(SystemSetting, "default")
|
||||
secrets_row = db.get(SystemSettingSecret, "default")
|
||||
reranker_row = db.get(SystemModelSetting, "reranker")
|
||||
|
||||
assert settings_row is not None
|
||||
assert secrets_row is not None
|
||||
assert reranker_row is not None
|
||||
|
||||
db.delete(reranker_row)
|
||||
settings_row.reranker_provider = ""
|
||||
settings_row.reranker_model = ""
|
||||
settings_row.reranker_endpoint = ""
|
||||
settings_row.vlm_provider = "Gemini"
|
||||
settings_row.vlm_model = "legacy-reranker"
|
||||
settings_row.vlm_endpoint = "https://legacy.example.com/v1"
|
||||
secrets_row.reranker_api_key_encrypted = ""
|
||||
secrets_row.vlm_api_key_encrypted = encrypt_secret("legacy-reranker-key")
|
||||
db.add(
|
||||
SystemModelSetting(
|
||||
slot="vlm",
|
||||
provider="Gemini",
|
||||
model_name="legacy-reranker",
|
||||
endpoint="https://legacy.example.com/v1",
|
||||
capability="chat",
|
||||
priority=30,
|
||||
enabled=True,
|
||||
api_key_encrypted=encrypt_secret("legacy-reranker-key"),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
snapshot = service.get_settings_snapshot()
|
||||
|
||||
assert snapshot.llmForm.rerankerProvider == "Gemini"
|
||||
assert snapshot.llmForm.rerankerModel == "legacy-reranker"
|
||||
assert snapshot.llmForm.rerankerEndpoint == "https://legacy.example.com/v1"
|
||||
assert snapshot.llmForm.rerankerApiKeyConfigured is True
|
||||
assert db.get(SystemModelSetting, "vlm") is None
|
||||
assert db.get(SystemModelSetting, "reranker") is not None
|
||||
assert service.load_saved_model_api_key("reranker") == "legacy-reranker-key"
|
||||
|
||||
@@ -64,6 +64,39 @@ def test_probe_azure_embedding_model(monkeypatch) -> None:
|
||||
assert captured["payload"]["input"] == "connectivity test"
|
||||
|
||||
|
||||
def test_probe_openai_compatible_reranker_model(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_send_json_request(method, url, *, headers, payload):
|
||||
captured["method"] = method
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
captured["payload"] = payload
|
||||
return 200, {"results": []}
|
||||
|
||||
monkeypatch.setattr("app.services.model_connectivity._send_json_request", fake_send_json_request)
|
||||
|
||||
result = probe_model_connectivity(
|
||||
ModelConnectivityTestRequest(
|
||||
provider="OpenAI Compatible",
|
||||
endpoint="https://api.example.com/v1",
|
||||
model="reranker-v1",
|
||||
api_key="secret",
|
||||
capability="reranker",
|
||||
)
|
||||
)
|
||||
|
||||
assert result.ok is True
|
||||
assert captured["method"] == "POST"
|
||||
assert captured["url"] == "https://api.example.com/v1/rerank"
|
||||
assert captured["headers"]["Authorization"] == "Bearer secret"
|
||||
assert captured["payload"] == {
|
||||
"model": "reranker-v1",
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
}
|
||||
|
||||
|
||||
def test_probe_ollama_failure_returns_error_payload(monkeypatch) -> None:
|
||||
def fake_send_json_request(method, url, *, headers, payload):
|
||||
raise ConnectivityCheckError("模型不存在或尚未拉取。", status_code=404)
|
||||
|
||||
@@ -83,7 +83,7 @@ def test_user_agent_sanitizes_model_thinking_blocks() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_user_agent_rejects_visible_reasoning_drafts() -> None:
|
||||
def test_user_agent_rejects_visible_reasoning_drafts() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
service = UserAgentService(db)
|
||||
@@ -93,10 +93,43 @@ def test_user_agent_rejects_visible_reasoning_drafts() -> None:
|
||||
"用户问的是:住宿费怎么算?\n让我分析一下:\n1. 实体识别..."
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_user_agent_knowledge_prompt_enforces_knowledge_boundary() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
query="住宿费标准是多少?",
|
||||
user_id="pytest",
|
||||
context_json={"session_type": "knowledge"},
|
||||
)
|
||||
)
|
||||
service = UserAgentService(db)
|
||||
messages = service._build_model_messages(
|
||||
UserAgentRequest(
|
||||
run_id=ontology.run_id,
|
||||
user_id="pytest",
|
||||
message="住宿费标准是多少?",
|
||||
ontology=ontology,
|
||||
tool_payload={"result_type": "knowledge_search", "hits": []},
|
||||
),
|
||||
citations=[],
|
||||
suggested_actions=[],
|
||||
risk_flags=[],
|
||||
draft_payload=None,
|
||||
fallback_answer="",
|
||||
)
|
||||
assert "只能依据 facts.tool_payload.hits、facts.knowledge_answer_evidence" in messages[0]["content"]
|
||||
assert "不能用常识、外部知识或主观推断补齐缺失条件" in messages[0]["content"]
|
||||
assert "不能只依赖排在最前面的片段" in messages[0]["content"]
|
||||
assert "不能把第一列的数值直接套给后面的列名" in messages[0]["content"]
|
||||
assert "knowledge_evidence_blocks" in messages[0]["content"]
|
||||
assert '"knowledge_answer_evidence": []' in messages[1]["content"]
|
||||
|
||||
|
||||
def test_user_agent_knowledge_prompt_enforces_llm_wiki_boundary() -> None:
|
||||
def test_user_agent_knowledge_fallback_is_honest_and_personalized() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
@@ -107,7 +140,48 @@ def test_user_agent_knowledge_prompt_enforces_llm_wiki_boundary() -> None:
|
||||
)
|
||||
)
|
||||
service = UserAgentService(db)
|
||||
messages = service._build_model_messages(
|
||||
answer = service._build_knowledge_search_answer(
|
||||
UserAgentRequest(
|
||||
run_id=ontology.run_id,
|
||||
user_id="pytest",
|
||||
message="住宿费标准是多少?",
|
||||
ontology=ontology,
|
||||
context_json={"name": "张三"},
|
||||
tool_payload={
|
||||
"result_type": "knowledge_search",
|
||||
"hits": [{"title": "差旅费制度", "content": "住宿费标准正文"}],
|
||||
},
|
||||
),
|
||||
citations=[],
|
||||
)
|
||||
|
||||
assert answer.startswith("张三,您好。")
|
||||
assert "答案整理阶段本轮没有及时返回" in answer
|
||||
assert "先给你当前最直接的依据" in answer
|
||||
assert "《差旅费制度》" in answer
|
||||
|
||||
|
||||
def test_user_agent_knowledge_answer_generation_uses_fast_timeouts(monkeypatch) -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
query="住宿费标准是多少?",
|
||||
user_id="pytest",
|
||||
context_json={"session_type": "knowledge"},
|
||||
)
|
||||
)
|
||||
service = UserAgentService(db)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_complete(messages, **kwargs):
|
||||
captured["messages"] = messages
|
||||
captured.update(kwargs)
|
||||
return "测试回答"
|
||||
|
||||
monkeypatch.setattr(service.runtime_chat_service, "complete", fake_complete)
|
||||
|
||||
answer = service._generate_answer_with_model(
|
||||
UserAgentRequest(
|
||||
run_id=ontology.run_id,
|
||||
user_id="pytest",
|
||||
@@ -122,14 +196,177 @@ def test_user_agent_knowledge_prompt_enforces_llm_wiki_boundary() -> None:
|
||||
fallback_answer="",
|
||||
)
|
||||
|
||||
assert "只能依据 tool_payload.hits 中的 LLM Wiki 内容作答" in messages[0]["content"]
|
||||
assert answer == "测试回答"
|
||||
assert captured["timeout_seconds"] == 5
|
||||
assert captured["slot_timeouts"] == {"main": 3, "backup": 5}
|
||||
assert captured["max_attempts"] == 1
|
||||
|
||||
|
||||
def test_user_agent_prefers_structured_knowledge_hit_for_answer_generation() -> None:
|
||||
selected = UserAgentService._select_knowledge_model_hits(
|
||||
{
|
||||
"hits": [
|
||||
{"content": "raw hit 1"},
|
||||
{"content": "raw hit 2"},
|
||||
{"content": "# 问答线索补充\n\n- 第二章 报销时限:费用发生后 30 日内提交申请。"},
|
||||
{"content": "# 结构化表格补充\n\n| 项目 | 金额 |"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert selected[0]["content"].startswith("# 问答线索补充")
|
||||
assert selected[1]["content"].startswith("# 结构化表格补充")
|
||||
assert selected[2]["content"] == "raw hit 1"
|
||||
|
||||
|
||||
def test_user_agent_uses_fast_knowledge_answer_without_model(monkeypatch) -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
query="报销时限是多少?",
|
||||
user_id="pytest",
|
||||
context_json={"session_type": "knowledge"},
|
||||
)
|
||||
)
|
||||
service = UserAgentService(db)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_generate_answer_with_model",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("model should not be called")),
|
||||
)
|
||||
|
||||
response = service.respond(
|
||||
UserAgentRequest(
|
||||
run_id=ontology.run_id,
|
||||
user_id="pytest",
|
||||
message="报销时限是多少?",
|
||||
ontology=ontology,
|
||||
context_json={
|
||||
"name": "张三",
|
||||
"session_type": "knowledge",
|
||||
"user_input_text": "报销时限是多少?",
|
||||
},
|
||||
tool_payload={
|
||||
"result_type": "knowledge_search",
|
||||
"hits": [
|
||||
{
|
||||
"title": "费用报销制度",
|
||||
"content": (
|
||||
"# 问答线索补充\n\n"
|
||||
"- 第二章 报销时限:员工应在费用发生后 30 日内提交报销申请。\n"
|
||||
"- 第二章 报销时限:超过 30 日需补充审批说明。"
|
||||
),
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert response.answer.startswith("张三,您好。")
|
||||
assert "当前能直接确认的是" in response.answer
|
||||
assert "30 日内提交报销申请" in response.answer
|
||||
assert "答案整理阶段本轮没有及时返回" not in response.answer
|
||||
|
||||
|
||||
def test_user_agent_fast_knowledge_answer_renders_relevant_table_preview() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
query="餐补标准是多少?",
|
||||
user_id="pytest",
|
||||
context_json={"session_type": "knowledge"},
|
||||
)
|
||||
)
|
||||
service = UserAgentService(db)
|
||||
|
||||
answer = service._build_fast_knowledge_answer(
|
||||
UserAgentRequest(
|
||||
run_id=ontology.run_id,
|
||||
user_id="pytest",
|
||||
message="餐补标准是多少?",
|
||||
ontology=ontology,
|
||||
context_json={
|
||||
"session_type": "knowledge",
|
||||
"user_input_text": "餐补标准是多少?",
|
||||
},
|
||||
tool_payload={
|
||||
"result_type": "knowledge_search",
|
||||
"hits": [
|
||||
{
|
||||
"title": "费用报销制度",
|
||||
"content": (
|
||||
"# 结构化表格补充\n\n"
|
||||
"## 表3 出差补贴标准\n\n"
|
||||
"| 项目 | 港澳台 | 其他地区 | 国外 |\n"
|
||||
"| --- | --- | --- | --- |\n"
|
||||
"| 餐补 | 75 | 55 | 140 |\n"
|
||||
"| 住宿补贴 | 35 | 35 | 35 |\n"
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
),
|
||||
citations=[],
|
||||
)
|
||||
|
||||
assert answer is not None
|
||||
assert "| 项目 | 港澳台 | 其他地区 | 国外 |" in answer
|
||||
assert "| 餐补 | 75 | 55 | 140 |" in answer
|
||||
|
||||
|
||||
def test_user_agent_fast_knowledge_answer_notes_missing_location_grounding() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
query="前往北京出差的报销标准是什么?",
|
||||
user_id="pytest",
|
||||
context_json={"session_type": "knowledge"},
|
||||
)
|
||||
)
|
||||
service = UserAgentService(db)
|
||||
|
||||
answer = service._build_fast_knowledge_answer(
|
||||
UserAgentRequest(
|
||||
run_id=ontology.run_id,
|
||||
user_id="pytest",
|
||||
message="前往北京出差的报销标准是什么?",
|
||||
ontology=ontology,
|
||||
context_json={
|
||||
"session_type": "knowledge",
|
||||
"user_input_text": "前往北京出差的报销标准是什么?",
|
||||
},
|
||||
tool_payload={
|
||||
"result_type": "knowledge_search",
|
||||
"hits": [
|
||||
{
|
||||
"title": "费用报销制度",
|
||||
"content": (
|
||||
"# 结构化表格补充\n\n"
|
||||
"## 表3 出差补贴标准\n\n"
|
||||
"| 项目 | 港澳台 | 直辖市/特区/西藏 | 其他地区 |\n"
|
||||
"| --- | --- | --- | --- |\n"
|
||||
"| 餐补 | 75 | 65 | 55 |\n"
|
||||
"| 基本补贴 | 35 | 35 | 35 |\n"
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
),
|
||||
citations=[],
|
||||
)
|
||||
|
||||
assert answer is not None
|
||||
assert "没有直接写出“北京”对应的地区档位或映射关系" in answer
|
||||
|
||||
|
||||
def test_user_agent_model_prompt_supports_contextual_personalization() -> None:
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
session_factory = build_session_factory()
|
||||
with session_factory() as db:
|
||||
ontology = SemanticOntologyService(db).parse(
|
||||
OntologyParseRequest(
|
||||
query="我能坐什么舱位?",
|
||||
user_id="pytest",
|
||||
)
|
||||
@@ -159,8 +396,8 @@ def test_user_agent_model_prompt_supports_contextual_personalization() -> None:
|
||||
|
||||
system_prompt = messages[0]["content"]
|
||||
user_prompt = messages[1]["content"]
|
||||
assert "user_grade" in system_prompt
|
||||
assert "conversation_history" in system_prompt
|
||||
assert "context.user_grade" in system_prompt
|
||||
assert "conversation_history" in user_prompt
|
||||
assert '"user_name": "张三"' in user_prompt
|
||||
assert '"user_position": "财务分析师"' in user_prompt
|
||||
assert '"user_grade": "P5"' in user_prompt
|
||||
|
||||
Reference in New Issue
Block a user