feat: 重构知识库系统,移除Hermes集成,增强RAG和同步功能
主要变更: - 移除Hermes智能体及相关回调服务 - 新增知识库RAG、同步、调度、规范化和索引任务服务 - 重构orchestrator服务,增强运行时聊天功能 - 更新前端聊天、政策制度、设置等页面样式和逻辑 - 更新expense_claims和document_intelligence服务 - 删除llm_wiki相关服务和测试文件 - 更新docker-compose配置和启动脚本
This commit is contained in:
@@ -1,44 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import get_db
|
||||
from app.api.v1.endpoints.settings import require_hermes_agent_token
|
||||
from app.schemas.common import ErrorResponse
|
||||
from app.schemas.hermes import HermesCallbackRead, HermesCallbackWrite
|
||||
from app.services.hermes_callbacks import HermesCallbackService
|
||||
|
||||
router = APIRouter(prefix="/hermes")
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/callback",
|
||||
response_model=HermesCallbackRead,
|
||||
dependencies=[Depends(require_hermes_agent_token)],
|
||||
summary="接收 Hermes 通用回调",
|
||||
description="所有 Hermes 任务统一通过该入口回传进度或完成结果,服务端依据 type 分发到对应业务处理器。",
|
||||
responses={
|
||||
status.HTTP_400_BAD_REQUEST: {
|
||||
"model": ErrorResponse,
|
||||
"description": "Hermes 回调载荷不合法或任务类型暂不支持。",
|
||||
},
|
||||
status.HTTP_404_NOT_FOUND: {
|
||||
"model": ErrorResponse,
|
||||
"description": "回调引用的 AgentRun 不存在。",
|
||||
},
|
||||
},
|
||||
)
|
||||
def handle_hermes_callback(
|
||||
payload: HermesCallbackWrite,
|
||||
db: DbSession,
|
||||
) -> HermesCallbackRead:
|
||||
try:
|
||||
return HermesCallbackService(db).handle_callback(payload)
|
||||
except LookupError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
File diff suppressed because it is too large
Load Diff
@@ -98,7 +98,7 @@ def test_model_connectivity(
|
||||
response_model=RuntimeModelConfigRead,
|
||||
dependencies=[Depends(require_hermes_agent_token)],
|
||||
summary="读取 Hermes 运行时模型配置",
|
||||
description="供 Hermes 进程读取主模型、备用模型、VLM 或 Embedding 模型的运行时配置。",
|
||||
description="供 Hermes 进程读取主模型、备用模型、Embedding 或 Reranker 模型的运行时配置。",
|
||||
responses={
|
||||
status.HTTP_401_UNAUTHORIZED: {
|
||||
"model": ErrorResponse,
|
||||
|
||||
@@ -7,7 +7,6 @@ from app.api.v1.endpoints.auth import router as auth_router
|
||||
from app.api.v1.endpoints.bootstrap import router as bootstrap_router
|
||||
from app.api.v1.endpoints.employees import router as employees_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
from app.api.v1.endpoints.hermes import router as hermes_router
|
||||
from app.api.v1.endpoints.knowledge import router as knowledge_router
|
||||
from app.api.v1.endpoints.ocr import router as ocr_router
|
||||
from app.api.v1.endpoints.ontology import router as ontology_router
|
||||
@@ -18,7 +17,6 @@ from app.api.v1.endpoints.system_logs import router as system_logs_router
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(health_router, tags=["health"])
|
||||
router.include_router(hermes_router, tags=["hermes"])
|
||||
router.include_router(bootstrap_router, tags=["bootstrap"])
|
||||
router.include_router(auth_router, tags=["auth"])
|
||||
router.include_router(agent_assets_router, tags=["agent-assets"])
|
||||
|
||||
@@ -14,9 +14,11 @@ from app.middleware.logging import AccessLogMiddleware
|
||||
from app.schemas.common import RootStatusRead
|
||||
from app.services.agent_foundation import prepare_agent_foundation
|
||||
from app.services.employee import prepare_employee_directory
|
||||
from app.services.knowledge import prepare_knowledge_library
|
||||
from app.services.llm_wiki_tasks import llm_wiki_task_manager
|
||||
from app.services.hermes_sync import sync_repository_hermes_skills
|
||||
from app.services.knowledge import prepare_knowledge_library
|
||||
from app.services.knowledge_index_tasks import knowledge_index_task_manager
|
||||
from app.services.knowledge_rag import shutdown_knowledge_rag_runtime
|
||||
from app.services.knowledge_scheduler import knowledge_index_scheduler
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -28,6 +30,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
prepare_agent_foundation()
|
||||
prepare_knowledge_library()
|
||||
sync_repository_hermes_skills()
|
||||
knowledge_index_scheduler.start()
|
||||
logger.info(
|
||||
"Server ready - host=%s port=%s prefix=%s",
|
||||
settings.app_host,
|
||||
@@ -35,23 +38,25 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
settings.api_v1_prefix,
|
||||
)
|
||||
yield
|
||||
llm_wiki_task_manager.shutdown()
|
||||
knowledge_index_scheduler.shutdown()
|
||||
knowledge_index_task_manager.shutdown()
|
||||
shutdown_knowledge_rag_runtime()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
|
||||
setup_logging(
|
||||
level=settings.log_level,
|
||||
log_dir=settings.log_dir,
|
||||
enable_file=settings.log_file_enabled,
|
||||
)
|
||||
|
||||
logger = get_logger("app.main")
|
||||
logger.info(
|
||||
"Starting %s (env=%s, debug=%s)", settings.app_name, settings.app_env, settings.app_debug
|
||||
)
|
||||
|
||||
|
||||
setup_logging(
|
||||
level=settings.log_level,
|
||||
log_dir=settings.log_dir,
|
||||
enable_file=settings.log_file_enabled,
|
||||
)
|
||||
|
||||
logger = get_logger("app.main")
|
||||
logger.info(
|
||||
"Starting %s (env=%s, debug=%s)", settings.app_name, settings.app_env, settings.app_debug
|
||||
)
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.app_name,
|
||||
debug=settings.app_debug,
|
||||
@@ -60,31 +65,31 @@ def create_app() -> FastAPI:
|
||||
openapi_tags=OPENAPI_TAGS,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(AccessLogMiddleware)
|
||||
|
||||
if settings.cors_origins:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(api_router, prefix=settings.api_v1_prefix)
|
||||
|
||||
|
||||
app.add_middleware(AccessLogMiddleware)
|
||||
|
||||
if settings.cors_origins:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(api_router, prefix=settings.api_v1_prefix)
|
||||
|
||||
@app.get(
|
||||
"/",
|
||||
tags=["root"],
|
||||
response_model=RootStatusRead,
|
||||
summary="服务根检查",
|
||||
description="用于快速确认后端服务进程已经启动。",
|
||||
description="用于快速确认后端服务进程已启动。",
|
||||
)
|
||||
def root() -> RootStatusRead:
|
||||
return RootStatusRead(message=f"{settings.app_name} is running")
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
@@ -40,6 +40,11 @@ class SystemSetting(Base):
|
||||
embedding_provider: Mapped[str] = mapped_column(String(64), default="GLM")
|
||||
embedding_model: Mapped[str] = mapped_column(String(255), default="Embedding-3")
|
||||
embedding_endpoint: Mapped[str] = mapped_column(String(512), default="https://open.bigmodel.cn/api/paas/v4/")
|
||||
reranker_provider: Mapped[str] = mapped_column(String(64), default="Ali")
|
||||
reranker_model: Mapped[str] = mapped_column(String(255), default="gte-rerank-v2")
|
||||
reranker_endpoint: Mapped[str] = mapped_column(
|
||||
String(512), default="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
|
||||
)
|
||||
onlyoffice_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
onlyoffice_public_url: Mapped[str] = mapped_column(String(512), default="")
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ class SystemSettingSecret(Base):
|
||||
backup_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
vlm_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
embedding_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
reranker_api_key_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
onlyoffice_jwt_secret_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
smtp_password_encrypted: Mapped[str] = mapped_column(Text, default="")
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
HermesCallbackStatus = Literal["running", "succeeded", "failed"]
|
||||
|
||||
|
||||
class HermesCallbackWrite(BaseModel):
|
||||
type: str = Field(min_length=1, max_length=80)
|
||||
run_id: str = Field(min_length=1, max_length=80)
|
||||
status: HermesCallbackStatus
|
||||
summary: str = ""
|
||||
error: str = ""
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class HermesCallbackRead(BaseModel):
|
||||
ok: bool = True
|
||||
accepted: bool = True
|
||||
type: str
|
||||
run_id: str
|
||||
status: HermesCallbackStatus
|
||||
@@ -35,6 +35,7 @@ class KnowledgeDocumentRead(BaseModel):
|
||||
folder: str
|
||||
tag: str
|
||||
time: str
|
||||
ingestTime: str = ""
|
||||
version: str
|
||||
stateCode: int = 1
|
||||
state: str
|
||||
|
||||
@@ -30,7 +30,7 @@ class OcrRecognizeDocumentRead(BaseModel):
|
||||
document_type_label: str = Field(default="其他单据", description="识别出的票据类型名称。")
|
||||
scene_code: str = Field(default="other", description="识别出的票据场景编码。")
|
||||
scene_label: str = Field(default="其他票据", description="识别出的票据场景名称。")
|
||||
classification_source: str = Field(default="rule", description="票据类型判断来源,例如 rule / llm_text / llm_vision。")
|
||||
classification_source: str = Field(default="rule", description="票据类型判断来源,当前固定为 rule。")
|
||||
classification_confidence: float = Field(default=0.0, ge=0.0, le=1.0, description="票据类型判断置信度。")
|
||||
classification_evidence: list[str] = Field(default_factory=list, description="票据类型判断依据摘要。")
|
||||
document_fields: list[OcrRecognizeFieldRead] = Field(
|
||||
|
||||
@@ -58,18 +58,18 @@ class SettingsLlmForm(BaseModel):
|
||||
backupApiKey: str = Field(default="", max_length=1024)
|
||||
backupApiKeyConfigured: bool = False
|
||||
|
||||
vlmProvider: str = Field(min_length=1, max_length=64)
|
||||
vlmModel: str = Field(min_length=1, max_length=255)
|
||||
vlmEndpoint: str = Field(min_length=1, max_length=512)
|
||||
vlmApiKey: str = Field(default="", max_length=1024)
|
||||
vlmApiKeyConfigured: bool = False
|
||||
|
||||
embeddingProvider: str = Field(min_length=1, max_length=64)
|
||||
embeddingModel: str = Field(min_length=1, max_length=255)
|
||||
embeddingEndpoint: str = Field(min_length=1, max_length=512)
|
||||
embeddingApiKey: str = Field(default="", max_length=1024)
|
||||
embeddingApiKeyConfigured: bool = False
|
||||
|
||||
rerankerProvider: str = Field(min_length=1, max_length=64)
|
||||
rerankerModel: str = Field(min_length=1, max_length=255)
|
||||
rerankerEndpoint: str = Field(min_length=1, max_length=512)
|
||||
rerankerApiKey: str = Field(default="", max_length=1024)
|
||||
rerankerApiKeyConfigured: bool = False
|
||||
|
||||
@field_validator(
|
||||
"mainProvider",
|
||||
"mainModel",
|
||||
@@ -79,14 +79,14 @@ class SettingsLlmForm(BaseModel):
|
||||
"backupModel",
|
||||
"backupEndpoint",
|
||||
"backupApiKey",
|
||||
"vlmProvider",
|
||||
"vlmModel",
|
||||
"vlmEndpoint",
|
||||
"vlmApiKey",
|
||||
"embeddingProvider",
|
||||
"embeddingModel",
|
||||
"embeddingEndpoint",
|
||||
"embeddingApiKey",
|
||||
"rerankerProvider",
|
||||
"rerankerModel",
|
||||
"rerankerEndpoint",
|
||||
"rerankerApiKey",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
@@ -185,8 +185,8 @@ class ModelConnectivityTestRequest(BaseModel):
|
||||
endpoint: str = Field(min_length=1, max_length=512)
|
||||
model: str = Field(min_length=1, max_length=255)
|
||||
api_key: str | None = Field(default=None, max_length=1024)
|
||||
capability: Literal["chat", "embedding"] = "chat"
|
||||
slot: Literal["main", "backup", "vlm", "embedding"] | None = None
|
||||
capability: Literal["chat", "embedding", "reranker"] = "chat"
|
||||
slot: Literal["main", "backup", "embedding", "reranker"] | None = None
|
||||
|
||||
@field_validator("provider", "endpoint", "model", "api_key", mode="before")
|
||||
@classmethod
|
||||
@@ -208,9 +208,9 @@ class ModelConnectivityTestRead(BaseModel):
|
||||
|
||||
|
||||
class RuntimeModelConfigRead(BaseModel):
|
||||
slot: Literal["main", "backup", "vlm", "embedding"]
|
||||
slot: Literal["main", "backup", "embedding", "reranker"]
|
||||
provider: str
|
||||
model: str
|
||||
endpoint: str
|
||||
apiKey: str
|
||||
capability: Literal["chat", "embedding"]
|
||||
capability: Literal["chat", "embedding", "reranker"]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,8 +9,6 @@ from typing import Any
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.runtime_chat import RuntimeChatService
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentField:
|
||||
@@ -198,7 +196,7 @@ MERCHANT_PATTERNS = (
|
||||
|
||||
class DocumentIntelligenceService:
|
||||
def __init__(self, db: Session | None = None) -> None:
|
||||
self.runtime_chat_service = RuntimeChatService(db) if db is not None else None
|
||||
self.db = db
|
||||
|
||||
def build_document_insight(
|
||||
self,
|
||||
@@ -254,95 +252,6 @@ class DocumentIntelligenceService:
|
||||
rule_insight: DocumentInsight,
|
||||
fields: tuple[DocumentField, ...],
|
||||
) -> tuple[str, LlmDocumentClassification] | None:
|
||||
if self.runtime_chat_service is None:
|
||||
return None
|
||||
|
||||
trimmed_text = text.strip()
|
||||
if not trimmed_text and not summary.strip():
|
||||
return None
|
||||
|
||||
facts = {
|
||||
"filename": filename,
|
||||
"summary": summary[:300],
|
||||
"ocr_text_excerpt": trimmed_text[:2000],
|
||||
"rule_candidate": {
|
||||
"document_type": rule_insight.document_type,
|
||||
"document_type_label": rule_insight.document_type_label,
|
||||
"scene_code": rule_insight.scene_code,
|
||||
"scene_label": rule_insight.scene_label,
|
||||
"expense_type": rule_insight.expense_type,
|
||||
"confidence": round(rule_insight.classification_confidence, 2),
|
||||
"evidence": list(rule_insight.evidence),
|
||||
},
|
||||
"extracted_fields": [
|
||||
{"key": field.key, "label": field.label, "value": field.value}
|
||||
for field in fields
|
||||
],
|
||||
"allowed_document_types": list(SUPPORTED_DOCUMENT_TYPES),
|
||||
}
|
||||
|
||||
system_prompt = (
|
||||
"你是企业报销票据识别复核器。"
|
||||
"你的任务不是 OCR,而是在已有 OCR 文本和票据预览基础上判断票据类型,并尽量复核关键字段。"
|
||||
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。"
|
||||
"document_type 只能是:"
|
||||
f"{', '.join(SUPPORTED_DOCUMENT_TYPES)}。"
|
||||
"如果证据不足,返回 other。"
|
||||
"严禁编造 OCR 中不存在的商户、酒店、航司、路线或金额。"
|
||||
"如果 OCR 出现冲突碎片,应优先依据票据主体信息,而不是单个噪声词。"
|
||||
"例如滴滴行程单/网约车发票,即使 OCR 混入酒店名称,也不能直接判成酒店票据。"
|
||||
"如果能从 OCR 或图片中明确确认字段,可在 fields 中返回。"
|
||||
"fields 只允许包含 key, label, value,key 只能是 amount, date, merchant_name, invoice_number, "
|
||||
"invoice_code, trip_no, route。无法确认就不要返回该字段。"
|
||||
"输出字段:document_type, scene_code, scene_label, expense_type, confidence, evidence, fields。"
|
||||
)
|
||||
user_prompt = (
|
||||
"请根据以下票据事实给出最终分类 JSON:\n"
|
||||
f"{json.dumps(facts, ensure_ascii=False, indent=2)}\n\n"
|
||||
"示例输出:\n"
|
||||
"{\n"
|
||||
' "document_type": "taxi_receipt",\n'
|
||||
' "scene_code": "transport",\n'
|
||||
' "scene_label": "交通票据",\n'
|
||||
' "expense_type": "transport",\n'
|
||||
' "confidence": 0.86,\n'
|
||||
' "evidence": ["OCR 中出现 滴滴出行、订单号、上车/下车 等交通特征"],\n'
|
||||
' "fields": [{"key": "amount", "label": "金额", "value": "32.5"}]\n'
|
||||
"}"
|
||||
)
|
||||
|
||||
if preview_data_url:
|
||||
response_text = self.runtime_chat_service.complete(
|
||||
[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": user_prompt},
|
||||
{"type": "image_url", "image_url": {"url": preview_data_url}},
|
||||
],
|
||||
},
|
||||
],
|
||||
slot_priority=("vlm",),
|
||||
max_tokens=320,
|
||||
temperature=0.0,
|
||||
)
|
||||
parsed = self._parse_llm_payload(response_text)
|
||||
if parsed is not None:
|
||||
return "llm_vision", parsed
|
||||
|
||||
response_text = self.runtime_chat_service.complete(
|
||||
[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
slot_priority=("main", "backup"),
|
||||
max_tokens=320,
|
||||
temperature=0.0,
|
||||
)
|
||||
parsed = self._parse_llm_payload(response_text)
|
||||
if parsed is not None:
|
||||
return "llm_text", parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -682,11 +682,13 @@ class ExpenseClaimService:
|
||||
raise ValueError("提交前请先补全信息:" + ";".join(missing_fields))
|
||||
|
||||
before_json = self._serialize_claim(claim)
|
||||
review_result = self._run_ai_submission_review(claim)
|
||||
claim.status = str(review_result["status"])
|
||||
claim.approval_stage = str(review_result["approval_stage"])
|
||||
claim.risk_flags_json = list(review_result["risk_flags"])
|
||||
claim.submitted_at = datetime.now(UTC) if claim.status == "submitted" else None
|
||||
# TODO: 后续恢复 AI 验审逻辑
|
||||
# review_result = self._run_ai_submission_review(claim)
|
||||
manager_name = self._resolve_claim_manager_name(claim) or "审批人"
|
||||
claim.status = "submitted"
|
||||
claim.approval_stage = "直属领导审批"
|
||||
claim.risk_flags_json = list(claim.risk_flags_json or [])
|
||||
claim.submitted_at = datetime.now(UTC)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(claim)
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.agent_enums import AgentRunStatus
|
||||
from app.schemas.hermes import HermesCallbackRead, HermesCallbackWrite
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import KNOWLEDGE_INGEST_STATUS_FAILED, KnowledgeService
|
||||
from app.services.llm_wiki import LlmWikiService
|
||||
|
||||
|
||||
class HermesCallbackService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.db = db
|
||||
self.run_service = AgentRunService(db)
|
||||
|
||||
def handle_callback(self, payload: HermesCallbackWrite) -> HermesCallbackRead:
|
||||
run = self.run_service.get_run(payload.run_id)
|
||||
if run is None:
|
||||
raise LookupError("Hermes 回调引用的 AgentRun 不存在。")
|
||||
|
||||
if payload.type == "llm_wiki_sync":
|
||||
self._handle_llm_wiki_sync(payload)
|
||||
else:
|
||||
raise ValueError(f"暂不支持的 Hermes 回调类型:{payload.type}")
|
||||
|
||||
return HermesCallbackRead(
|
||||
type=payload.type,
|
||||
run_id=payload.run_id,
|
||||
status=payload.status,
|
||||
)
|
||||
|
||||
def _handle_llm_wiki_sync(self, payload: HermesCallbackWrite) -> None:
|
||||
run = self.run_service.get_run(payload.run_id)
|
||||
if run is None:
|
||||
raise LookupError("Hermes 回调引用的 AgentRun 不存在。")
|
||||
|
||||
route_json = dict(run.route_json or {})
|
||||
document_ids = [
|
||||
str(item).strip()
|
||||
for item in list(route_json.get("requested_document_ids") or [])
|
||||
if str(item).strip()
|
||||
]
|
||||
if payload.status == "running":
|
||||
self.run_service.merge_route_json(
|
||||
payload.run_id,
|
||||
{
|
||||
"phase": "running",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"callback_status": payload.status,
|
||||
"callback_payload": payload.payload,
|
||||
},
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary=payload.summary or run.result_summary,
|
||||
)
|
||||
return
|
||||
|
||||
if payload.status == "failed":
|
||||
if document_ids:
|
||||
KnowledgeService().set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=payload.run_id,
|
||||
)
|
||||
self.run_service.record_tool_call(
|
||||
run_id=payload.run_id,
|
||||
tool_type="http",
|
||||
tool_name="hermes_callback",
|
||||
request_json=payload.model_dump(mode="json"),
|
||||
response_json={},
|
||||
status="failed",
|
||||
duration_ms=0,
|
||||
error_message=payload.error or payload.summary or "Hermes callback failed",
|
||||
)
|
||||
self.run_service.merge_route_json(
|
||||
payload.run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"callback_status": payload.status,
|
||||
"callback_payload": payload.payload,
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary=payload.summary or payload.error or "Hermes 任务失败。",
|
||||
error_message=payload.error or payload.summary or "Hermes 任务失败。",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
|
||||
result = LlmWikiService(self.db).finalize_agent_batch_callback(
|
||||
agent_run_id=payload.run_id,
|
||||
payload=payload.payload,
|
||||
)
|
||||
self.run_service.record_tool_call(
|
||||
run_id=payload.run_id,
|
||||
tool_type="http",
|
||||
tool_name="hermes_callback",
|
||||
request_json=payload.model_dump(mode="json"),
|
||||
response_json=result.model_dump(mode="json"),
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
self.run_service.merge_route_json(
|
||||
payload.run_id,
|
||||
{
|
||||
"phase": "succeeded",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"callback_status": payload.status,
|
||||
"sync_run_id": result.run_id,
|
||||
"sync_result": result.model_dump(mode="json"),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": result.document_count,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": max(0, len(document_ids) - result.document_count),
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
result_summary=payload.summary or result.summary,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
216
server/src/app/services/knowledge_index_tasks.py
Normal file
216
server/src/app/services/knowledge_index_tasks.py
Normal file
@@ -0,0 +1,216 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentName, AgentRunStatus, AgentToolType
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import (
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
KNOWLEDGE_INGEST_STATUS_INGESTED,
|
||||
KnowledgeService,
|
||||
)
|
||||
from app.services.knowledge_rag import KnowledgeRagService
|
||||
|
||||
logger = get_logger("app.services.knowledge_index_tasks")
|
||||
|
||||
|
||||
class KnowledgeIndexTaskManager:
|
||||
def __init__(self) -> None:
|
||||
self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="knowledge-index")
|
||||
self._futures: dict[str, Future[Any]] = {}
|
||||
|
||||
def submit_sync(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str],
|
||||
force: bool,
|
||||
) -> None:
|
||||
future = self._executor.submit(
|
||||
self._run_sync,
|
||||
agent_run_id,
|
||||
folder,
|
||||
current_user,
|
||||
[str(item).strip() for item in document_ids if str(item).strip()],
|
||||
force,
|
||||
)
|
||||
self._futures[agent_run_id] = future
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
@staticmethod
|
||||
def _run_sync(
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str],
|
||||
force: bool,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
db = session_factory()
|
||||
started = perf_counter()
|
||||
|
||||
try:
|
||||
run_service = AgentRunService(db)
|
||||
knowledge_service = KnowledgeService(db=db)
|
||||
rag_service = KnowledgeRagService(db=db)
|
||||
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "indexing",
|
||||
"folder": folder,
|
||||
"force": force,
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"requested_document_ids": document_ids,
|
||||
"requested_by_username": current_user.username,
|
||||
"requested_by_name": current_user.name,
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 10 if document_ids else 100,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
response = rag_service.index_documents(document_ids=document_ids, force=force)
|
||||
succeeded_document_ids = [
|
||||
str(item).strip()
|
||||
for item in list(response.get("succeeded_document_ids") or [])
|
||||
if str(item).strip()
|
||||
]
|
||||
failed_documents = [
|
||||
item
|
||||
for item in list(response.get("failed_documents") or [])
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
failed_document_ids = [
|
||||
str(item.get("document_id") or "").strip()
|
||||
for item in failed_documents
|
||||
if str(item.get("document_id") or "").strip()
|
||||
]
|
||||
|
||||
if succeeded_document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
succeeded_document_ids,
|
||||
KNOWLEDGE_INGEST_STATUS_INGESTED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
if failed_document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
failed_document_ids,
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
|
||||
duration_ms = int((perf_counter() - started) * 1000)
|
||||
tool_status = "succeeded" if not failed_document_ids else "failed"
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type=AgentToolType.LLM.value,
|
||||
tool_name="lightrag.index_documents",
|
||||
request_json={
|
||||
"agent": AgentName.HERMES.value,
|
||||
"folder": folder,
|
||||
"document_ids": document_ids,
|
||||
"force": force,
|
||||
},
|
||||
response_json=response,
|
||||
status=tool_status,
|
||||
duration_ms=duration_ms,
|
||||
error_message=None if tool_status == "succeeded" else "部分文档索引失败。",
|
||||
)
|
||||
|
||||
completed_documents = len(succeeded_document_ids)
|
||||
failed_count = len(failed_document_ids)
|
||||
total_documents = len(document_ids)
|
||||
summary = (
|
||||
f"LightRAG 已完成 {completed_documents}/{total_documents} 个知识文档索引。"
|
||||
if failed_count == 0
|
||||
else f"LightRAG 已完成 {completed_documents}/{total_documents} 个知识文档索引,失败 {failed_count} 个。"
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "completed",
|
||||
"track_id": str(response.get("track_id") or "").strip(),
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": total_documents,
|
||||
"completed_documents": completed_documents,
|
||||
"failed_documents": failed_count,
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=(
|
||||
AgentRunStatus.SUCCEEDED.value
|
||||
if failed_count == 0
|
||||
else AgentRunStatus.FAILED.value
|
||||
),
|
||||
result_summary=summary,
|
||||
error_message="部分文档索引失败。" if failed_count else None,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
except Exception as exc:
|
||||
try:
|
||||
AgentRunService(db).record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type=AgentToolType.LLM.value,
|
||||
tool_name="lightrag.index_documents",
|
||||
request_json={
|
||||
"agent": AgentName.HERMES.value,
|
||||
"folder": folder,
|
||||
"document_ids": document_ids,
|
||||
"force": force,
|
||||
},
|
||||
response_json={"error": str(exc)},
|
||||
status="failed",
|
||||
duration_ms=int((perf_counter() - started) * 1000),
|
||||
error_message=str(exc),
|
||||
)
|
||||
KnowledgeService(db=db).set_document_ingest_statuses(
|
||||
document_ids,
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
AgentRunService(db).merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": len(document_ids),
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary=str(exc),
|
||||
error_message=str(exc),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Knowledge index task finalization failed run_id=%s", agent_run_id)
|
||||
logger.exception("Knowledge index task failed run_id=%s", agent_run_id)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
knowledge_index_task_manager = KnowledgeIndexTaskManager()
|
||||
414
server/src/app/services/knowledge_normalizer.py
Normal file
414
server/src/app/services/knowledge_normalizer.py
Normal file
@@ -0,0 +1,414 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.services.runtime_chat import RuntimeChatService
|
||||
|
||||
logger = get_logger("app.services.knowledge_normalizer")
|
||||
|
||||
TABLE_MARKER_PATTERN = re.compile(r"表\s*(\d+)")
|
||||
SECTION_HEADING_PATTERN = re.compile(
|
||||
r"^(第[一二三四五六七八九十百零0-9]+[章节]\s*.*|[一二三四五六七八九十]+、.*|([一二三四五六七八九十]+).*|\([一二三四五六七八九十]+\).*)$"
|
||||
)
|
||||
LIST_ITEM_PATTERN = re.compile(r"^[-*•]\s+.+$")
|
||||
NUMBERED_ITEM_PATTERN = re.compile(r"^(?:\d+[.)、]|[①②③④⑤⑥⑦⑧⑨⑩])\s*.+$")
|
||||
ARTICLE_PATTERN = re.compile(r"^(第[一二三四五六七八九十百零0-9]+条)\s*.*$")
|
||||
KEY_VALUE_PATTERN = re.compile(r"^[^::\s][^::]{0,40}[::]\s*.+$")
|
||||
MAX_TABLE_WINDOW_CHARS = 1800
|
||||
MAX_TABLES_PER_DOCUMENT = 8
|
||||
MAX_SECTION_OUTLINE_ITEMS = 12
|
||||
MAX_SECTION_SNIPPETS = 8
|
||||
MAX_SECTION_SNIPPET_CHARS = 220
|
||||
MAX_SECTION_QA_CLUES = 4
|
||||
MAX_TOTAL_QA_CLUES = 24
|
||||
MAX_QA_CLUE_CHARS = 180
|
||||
FACT_KEYWORDS = (
|
||||
"适用",
|
||||
"标准",
|
||||
"条件",
|
||||
"流程",
|
||||
"审批",
|
||||
"提交",
|
||||
"附件",
|
||||
"材料",
|
||||
"票据",
|
||||
"报销",
|
||||
"限额",
|
||||
"金额",
|
||||
"比例",
|
||||
"范围",
|
||||
"对象",
|
||||
"人员",
|
||||
"时限",
|
||||
"工作日",
|
||||
"不得",
|
||||
"可以",
|
||||
"应当",
|
||||
"应",
|
||||
"需",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TableCandidate:
|
||||
title: str
|
||||
excerpt: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SectionCandidate:
|
||||
title: str
|
||||
excerpt: str
|
||||
body_lines: tuple[str, ...]
|
||||
|
||||
|
||||
class KnowledgeNormalizationService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.runtime_chat_service = RuntimeChatService(db)
|
||||
|
||||
def build_enriched_text(self, raw_text: str) -> str:
|
||||
normalized_text = str(raw_text or "").strip()
|
||||
if not normalized_text:
|
||||
return ""
|
||||
|
||||
section_appendix = self._build_section_appendix(normalized_text)
|
||||
answer_clue_appendix = self._build_answer_clue_appendix(normalized_text)
|
||||
normalized_tables: list[str] = []
|
||||
for candidate in self._extract_table_candidates(normalized_text):
|
||||
rendered = self._normalize_table_candidate(candidate)
|
||||
if rendered:
|
||||
normalized_tables.append(f"## {candidate.title}\n\n{rendered}")
|
||||
|
||||
parts: list[str] = []
|
||||
if section_appendix:
|
||||
parts.append(section_appendix)
|
||||
if answer_clue_appendix:
|
||||
parts.append(answer_clue_appendix)
|
||||
if normalized_tables:
|
||||
appendix = "\n\n".join(normalized_tables)
|
||||
parts.append(
|
||||
"# 结构化表格补充\n\n"
|
||||
"以下表格由知识归纳阶段依据原文重新整理,供问答检索时优先理解行列关系。\n\n"
|
||||
f"{appendix}"
|
||||
)
|
||||
|
||||
if not parts:
|
||||
return normalized_text
|
||||
|
||||
parts.append(f"# 原文\n\n{normalized_text}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def _extract_table_candidates(text: str) -> list[TableCandidate]:
|
||||
candidates: list[TableCandidate] = []
|
||||
occupied_ranges: list[tuple[int, int]] = []
|
||||
|
||||
for match in TABLE_MARKER_PATTERN.finditer(text):
|
||||
if len(candidates) >= MAX_TABLES_PER_DOCUMENT:
|
||||
break
|
||||
|
||||
start = text.rfind("\n", 0, match.start())
|
||||
start = 0 if start < 0 else start + 1
|
||||
end = min(len(text), start + MAX_TABLE_WINDOW_CHARS)
|
||||
if any(start < existing_end and end > existing_start for existing_start, existing_end in occupied_ranges):
|
||||
continue
|
||||
|
||||
excerpt = text[start:end].strip()
|
||||
head = excerpt[:360]
|
||||
if "单位:" not in head and "标准" not in head:
|
||||
continue
|
||||
if excerpt.count("\n") < 6 or sum(char.isdigit() for char in excerpt) < 4:
|
||||
continue
|
||||
|
||||
marker = match.group(0).replace(" ", "")
|
||||
first_line = next((line.strip() for line in excerpt.splitlines() if line.strip()), marker)
|
||||
title = first_line if first_line.startswith(marker) else marker
|
||||
candidates.append(TableCandidate(title=title, excerpt=excerpt))
|
||||
occupied_ranges.append((start, end))
|
||||
|
||||
return candidates
|
||||
|
||||
def _normalize_table_candidate(self, candidate: TableCandidate) -> str:
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是制度文档结构化助手。"
|
||||
"只依据用户提供的原文,提炼其中的表格为清晰 Markdown。"
|
||||
"必须严格按照表头从左到右对齐每个数值,不能猜测、不能改列顺序、不能擅自补全。"
|
||||
"只输出一张 Markdown 表格本身,不要输出标题、说明、注释、脚注或正文解释。"
|
||||
"如果原文不足以确认表格关系,只回复“无法确认”。"
|
||||
"不要输出思考过程,不要复述原文,不要添加制度之外的新事实。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"请仅整理下面这段制度表格,标题为《{candidate.title}》。\n\n"
|
||||
f"{candidate.excerpt}"
|
||||
),
|
||||
},
|
||||
]
|
||||
answer = self.runtime_chat_service.complete(
|
||||
messages,
|
||||
max_tokens=900,
|
||||
temperature=0.0,
|
||||
)
|
||||
cleaned = self._sanitize_answer(answer)
|
||||
if not cleaned or cleaned == "无法确认":
|
||||
return ""
|
||||
if cleaned.count("|") < 6:
|
||||
logger.info("Skip non-tabular normalization candidate title=%s", candidate.title)
|
||||
return ""
|
||||
return cleaned
|
||||
|
||||
@staticmethod
|
||||
def _build_section_appendix(text: str) -> str:
|
||||
candidates = KnowledgeNormalizationService._extract_section_candidates(text)
|
||||
if len(candidates) < 2:
|
||||
return ""
|
||||
|
||||
outline = "\n".join(
|
||||
f"- {item.title}"
|
||||
for item in candidates[:MAX_SECTION_OUTLINE_ITEMS]
|
||||
)
|
||||
snippets = "\n\n".join(
|
||||
[
|
||||
f"## {item.title}\n\n{item.excerpt}"
|
||||
for item in candidates[:MAX_SECTION_SNIPPETS]
|
||||
if item.excerpt
|
||||
]
|
||||
)
|
||||
if not snippets:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"# 章节导航\n\n"
|
||||
"以下内容由入库阶段从制度原文中提取,供检索时优先理解制度层级、条目和标准所在章节。\n\n"
|
||||
f"{outline}\n\n"
|
||||
"# 重点章节摘录\n\n"
|
||||
f"{snippets}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_answer_clue_appendix(text: str) -> str:
|
||||
candidates = KnowledgeNormalizationService._extract_section_candidates(text)
|
||||
clue_lines: list[str] = []
|
||||
|
||||
if candidates:
|
||||
for candidate in candidates:
|
||||
clue_lines.extend(
|
||||
KnowledgeNormalizationService._extract_section_clues(candidate)
|
||||
)
|
||||
else:
|
||||
clue_lines.extend(KnowledgeNormalizationService._extract_freeform_clues(text))
|
||||
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in clue_lines:
|
||||
normalized = re.sub(r"\s+", " ", str(item or "")).strip()
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
deduped.append(normalized)
|
||||
if len(deduped) >= MAX_TOTAL_QA_CLUES:
|
||||
break
|
||||
|
||||
if len(deduped) < 2:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"# 问答线索补充\n\n"
|
||||
"以下内容由入库阶段根据章节标题、条款、列表、键值对与相邻正文提炼,"
|
||||
"供问答检索时优先命中更短、更直接的制度依据。\n\n"
|
||||
+ "\n".join(f"- {item}" for item in deduped)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_section_candidates(text: str) -> list[SectionCandidate]:
|
||||
lines = [line.rstrip() for line in str(text or "").splitlines()]
|
||||
sections: list[SectionCandidate] = []
|
||||
current_title = ""
|
||||
current_body: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current_title, current_body
|
||||
if not current_title:
|
||||
current_body = []
|
||||
return
|
||||
excerpt = KnowledgeNormalizationService._build_section_excerpt(current_body)
|
||||
if excerpt:
|
||||
sections.append(
|
||||
SectionCandidate(
|
||||
title=current_title,
|
||||
excerpt=excerpt,
|
||||
body_lines=tuple(current_body),
|
||||
)
|
||||
)
|
||||
current_title = ""
|
||||
current_body = []
|
||||
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
if current_body:
|
||||
current_body.append("")
|
||||
continue
|
||||
|
||||
if SECTION_HEADING_PATTERN.match(line) and len(line) <= 80:
|
||||
flush()
|
||||
current_title = line
|
||||
continue
|
||||
|
||||
if current_title:
|
||||
current_body.append(line)
|
||||
|
||||
flush()
|
||||
return sections
|
||||
|
||||
@staticmethod
|
||||
def _build_section_excerpt(lines: list[str]) -> str:
|
||||
cleaned_lines = [line.strip() for line in lines if line.strip()]
|
||||
if not cleaned_lines:
|
||||
return ""
|
||||
excerpt = ";".join(cleaned_lines[:3]).strip()
|
||||
if len(excerpt) <= MAX_SECTION_SNIPPET_CHARS:
|
||||
return excerpt
|
||||
return f"{excerpt[: MAX_SECTION_SNIPPET_CHARS - 3].rstrip()}..."
|
||||
|
||||
@staticmethod
|
||||
def _extract_section_clues(candidate: SectionCandidate) -> list[str]:
|
||||
clues: list[str] = []
|
||||
fallback: list[str] = []
|
||||
|
||||
for raw_line in candidate.body_lines:
|
||||
normalized_line = KnowledgeNormalizationService._normalize_fact_line(raw_line)
|
||||
if not normalized_line or KnowledgeNormalizationService._is_table_like_line(normalized_line):
|
||||
continue
|
||||
|
||||
fact_units = KnowledgeNormalizationService._split_fact_units(normalized_line)
|
||||
for unit in fact_units:
|
||||
rendered = KnowledgeNormalizationService._render_clue(candidate.title, unit)
|
||||
if not rendered:
|
||||
continue
|
||||
if KnowledgeNormalizationService._looks_like_fact_line(unit):
|
||||
clues.append(rendered)
|
||||
elif len(fallback) < 2:
|
||||
fallback.append(rendered)
|
||||
|
||||
if len(clues) >= MAX_SECTION_QA_CLUES:
|
||||
return clues[:MAX_SECTION_QA_CLUES]
|
||||
|
||||
return clues[:MAX_SECTION_QA_CLUES] or fallback[:2]
|
||||
|
||||
@staticmethod
|
||||
def _extract_freeform_clues(text: str) -> list[str]:
|
||||
clues: list[str] = []
|
||||
for raw_line in str(text or "").splitlines():
|
||||
normalized_line = KnowledgeNormalizationService._normalize_fact_line(raw_line)
|
||||
if (
|
||||
not normalized_line
|
||||
or SECTION_HEADING_PATTERN.match(normalized_line)
|
||||
or KnowledgeNormalizationService._is_table_like_line(normalized_line)
|
||||
or not KnowledgeNormalizationService._looks_like_fact_line(normalized_line)
|
||||
):
|
||||
continue
|
||||
|
||||
for unit in KnowledgeNormalizationService._split_fact_units(normalized_line):
|
||||
rendered = KnowledgeNormalizationService._render_clue("正文", unit)
|
||||
if rendered:
|
||||
clues.append(rendered)
|
||||
if len(clues) >= MAX_TOTAL_QA_CLUES:
|
||||
return clues
|
||||
return clues
|
||||
|
||||
@staticmethod
|
||||
def _split_fact_units(line: str) -> list[str]:
|
||||
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
|
||||
if not normalized:
|
||||
return []
|
||||
if len(normalized) <= MAX_QA_CLUE_CHARS and all(mark not in normalized for mark in (";", ";", "。")):
|
||||
return [normalized]
|
||||
|
||||
units: list[str] = []
|
||||
for part in re.split(r"[;;。]\s*", normalized):
|
||||
cleaned = KnowledgeNormalizationService._normalize_fact_line(part)
|
||||
if not cleaned:
|
||||
continue
|
||||
units.append(cleaned)
|
||||
return units or [KnowledgeNormalizationService._truncate_clue(normalized)]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_fact_line(line: str) -> str:
|
||||
normalized = str(line or "").strip()
|
||||
normalized = re.sub(r"\s+", " ", normalized)
|
||||
return normalized.strip(" -")
|
||||
|
||||
@staticmethod
|
||||
def _is_table_like_line(line: str) -> bool:
|
||||
normalized = str(line or "").strip()
|
||||
if not normalized:
|
||||
return False
|
||||
if normalized.count("|") >= 2:
|
||||
return True
|
||||
if normalized.count("\t") >= 2:
|
||||
return True
|
||||
number_tokens = re.findall(r"\d+(?:[.][0-9]+)?", normalized)
|
||||
if len(number_tokens) >= 3 and len(normalized.split()) >= 4 and not any(
|
||||
punct in normalized for punct in ("。", ";", ";", ":", ":")
|
||||
):
|
||||
return True
|
||||
return "单位:" in normalized and sum(char.isdigit() for char in normalized) >= 3
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_fact_line(line: str) -> bool:
|
||||
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
|
||||
if len(normalized) < 6:
|
||||
return False
|
||||
if TABLE_MARKER_PATTERN.search(normalized) or normalized.startswith(("单位:", "单位:")):
|
||||
return False
|
||||
if (
|
||||
ARTICLE_PATTERN.match(normalized)
|
||||
or LIST_ITEM_PATTERN.match(normalized)
|
||||
or NUMBERED_ITEM_PATTERN.match(normalized)
|
||||
or KEY_VALUE_PATTERN.match(normalized)
|
||||
):
|
||||
return True
|
||||
if any(keyword in normalized for keyword in FACT_KEYWORDS):
|
||||
return True
|
||||
return any(char.isdigit() for char in normalized)
|
||||
|
||||
@staticmethod
|
||||
def _render_clue(section_title: str, line: str) -> str:
|
||||
normalized_line = KnowledgeNormalizationService._truncate_clue(line)
|
||||
if not normalized_line:
|
||||
return ""
|
||||
normalized_title = str(section_title or "").strip()
|
||||
if not normalized_title:
|
||||
return normalized_line
|
||||
return f"{normalized_title}:{normalized_line}"
|
||||
|
||||
@staticmethod
|
||||
def _truncate_clue(line: str) -> str:
|
||||
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
|
||||
if len(normalized) <= MAX_QA_CLUE_CHARS:
|
||||
return normalized
|
||||
return f"{normalized[: MAX_QA_CLUE_CHARS - 3].rstrip()}..."
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_answer(answer: str | None) -> str:
|
||||
cleaned = re.sub(r"<think>.*?</think>", "", str(answer or ""), flags=re.DOTALL | re.IGNORECASE)
|
||||
lines = [line.rstrip() for line in cleaned.strip().splitlines()]
|
||||
table_lines: list[str] = []
|
||||
for line in lines:
|
||||
normalized = line.strip()
|
||||
if "|" not in normalized:
|
||||
if table_lines:
|
||||
break
|
||||
continue
|
||||
table_lines.append(normalized)
|
||||
return "\n".join(table_lines).strip()
|
||||
1261
server/src/app/services/knowledge_rag.py
Normal file
1261
server/src/app/services/knowledge_rag.py
Normal file
File diff suppressed because it is too large
Load Diff
94
server/src/app/services/knowledge_scheduler.py
Normal file
94
server/src/app/services/knowledge_scheduler.py
Normal file
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from datetime import datetime, time, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentRunSource
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.knowledge_sync import KnowledgeSyncDispatchService
|
||||
|
||||
logger = get_logger("app.services.knowledge_scheduler")
|
||||
|
||||
|
||||
class KnowledgeIndexScheduler:
|
||||
def __init__(self) -> None:
|
||||
timezone_name = str(os.environ.get("X_FINANCIAL_SCHEDULER_TZ") or "Asia/Shanghai").strip() or "Asia/Shanghai"
|
||||
self._timezone = ZoneInfo(timezone_name)
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def start(self) -> None:
|
||||
with self._lock:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_loop,
|
||||
name="knowledge-index-scheduler",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
logger.info("Knowledge index scheduler started timezone=%s trigger=00:00", self._timezone.key)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._lock:
|
||||
thread = self._thread
|
||||
self._thread = None
|
||||
self._stop_event.set()
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=3)
|
||||
logger.info("Knowledge index scheduler stopped")
|
||||
|
||||
def _run_loop(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
now = datetime.now(self._timezone)
|
||||
next_run = self._resolve_next_run(now)
|
||||
wait_seconds = max(1.0, (next_run - now).total_seconds())
|
||||
if self._stop_event.wait(wait_seconds):
|
||||
break
|
||||
try:
|
||||
self._run_incremental_sync()
|
||||
except Exception: # pragma: no cover - scheduler best effort logging
|
||||
logger.exception("Scheduled knowledge index sync failed")
|
||||
|
||||
def _run_incremental_sync(self) -> None:
|
||||
db = get_session_factory()()
|
||||
try:
|
||||
current_user = CurrentUserContext(
|
||||
username="hermes",
|
||||
name="Hermes",
|
||||
role_codes=["manager"],
|
||||
is_admin=True,
|
||||
)
|
||||
result = KnowledgeSyncDispatchService(db).queue_sync(
|
||||
current_user=current_user,
|
||||
folder=None,
|
||||
document_ids=None,
|
||||
source=AgentRunSource.SCHEDULE.value,
|
||||
force=False,
|
||||
changed_only=True,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduled knowledge index sync result run_id=%s docs=%s reused=%s summary=%s",
|
||||
result.agent_run_id,
|
||||
len(result.document_ids),
|
||||
result.reused,
|
||||
result.summary,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@staticmethod
|
||||
def _resolve_next_run(now: datetime) -> datetime:
|
||||
today_midnight = datetime.combine(now.date(), time(hour=0, minute=0), tzinfo=now.tzinfo)
|
||||
if now < today_midnight:
|
||||
return today_midnight
|
||||
return today_midnight + timedelta(days=1)
|
||||
|
||||
|
||||
knowledge_index_scheduler = KnowledgeIndexScheduler()
|
||||
244
server/src/app/services/knowledge_sync.py
Normal file
244
server/src/app/services/knowledge_sync.py
Normal file
@@ -0,0 +1,244 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentName, AgentPermissionLevel, AgentRunSource, AgentRunStatus
|
||||
from app.models.agent_asset import AgentAsset
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import (
|
||||
KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
KNOWLEDGE_INGEST_STATUS_SYNCING,
|
||||
KnowledgeService,
|
||||
)
|
||||
from app.services.knowledge_index_tasks import knowledge_index_task_manager
|
||||
|
||||
ALL_KNOWLEDGE_FOLDERS_LABEL = "全部知识库"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class KnowledgeSyncDispatchResult:
|
||||
ok: bool = True
|
||||
agent_run_id: str = ""
|
||||
folder: str = ""
|
||||
document_ids: list[str] = field(default_factory=list)
|
||||
queued_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
status: str = AgentRunStatus.SUCCEEDED.value
|
||||
summary: str = ""
|
||||
reused: bool = False
|
||||
|
||||
|
||||
class KnowledgeSyncDispatchService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.db = db
|
||||
self.run_service = AgentRunService(db)
|
||||
self.knowledge_service = KnowledgeService(db=db)
|
||||
|
||||
def queue_sync(
|
||||
self,
|
||||
*,
|
||||
current_user: CurrentUserContext,
|
||||
folder: str | None = None,
|
||||
document_ids: list[str] | None = None,
|
||||
source: str = AgentRunSource.USER_MESSAGE.value,
|
||||
force: bool = False,
|
||||
changed_only: bool = True,
|
||||
) -> KnowledgeSyncDispatchResult:
|
||||
normalized_folder = str(folder or "").strip() or None
|
||||
folder_label = normalized_folder or ALL_KNOWLEDGE_FOLDERS_LABEL
|
||||
normalized_requested_ids = [
|
||||
str(item).strip()
|
||||
for item in document_ids or []
|
||||
if str(item).strip()
|
||||
]
|
||||
|
||||
all_documents = self.knowledge_service.list_documents_for_ingest(
|
||||
folder=normalized_folder,
|
||||
document_ids=normalized_requested_ids,
|
||||
changed_only=False,
|
||||
)
|
||||
target_documents = self.knowledge_service.list_documents_for_ingest(
|
||||
folder=normalized_folder,
|
||||
document_ids=normalized_requested_ids,
|
||||
changed_only=(False if force else changed_only),
|
||||
)
|
||||
target_document_ids = [
|
||||
str(item.get("id") or "").strip()
|
||||
for item in target_documents
|
||||
if str(item.get("id") or "").strip()
|
||||
]
|
||||
|
||||
if not all_documents:
|
||||
return KnowledgeSyncDispatchResult(
|
||||
folder=folder_label,
|
||||
document_ids=[],
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
summary="当前目录暂无可归纳的知识文档。",
|
||||
)
|
||||
|
||||
if not target_document_ids:
|
||||
return KnowledgeSyncDispatchResult(
|
||||
folder=folder_label,
|
||||
document_ids=[],
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
summary="当前目录没有需要增量归纳的文档。",
|
||||
)
|
||||
|
||||
active_run = self._find_active_run(
|
||||
folder=folder_label,
|
||||
requested_document_ids=target_document_ids,
|
||||
)
|
||||
if active_run is not None:
|
||||
active_document_ids = [
|
||||
str(item).strip()
|
||||
for item in list(active_run.route_json.get("requested_document_ids") or target_document_ids)
|
||||
if str(item).strip()
|
||||
]
|
||||
return KnowledgeSyncDispatchResult(
|
||||
agent_run_id=active_run.run_id,
|
||||
folder=folder_label,
|
||||
document_ids=active_document_ids,
|
||||
queued_at=active_run.started_at,
|
||||
status=active_run.status,
|
||||
summary="已有知识归纳任务正在执行,系统已复用当前任务。",
|
||||
reused=True,
|
||||
)
|
||||
|
||||
task_asset = self.db.scalar(
|
||||
select(AgentAsset).where(AgentAsset.code == "task.hermes.knowledge_index_sync")
|
||||
)
|
||||
run = self.run_service.create_run(
|
||||
agent=AgentName.HERMES.value,
|
||||
source=source,
|
||||
user_id=current_user.username,
|
||||
task_id=task_asset.id if task_asset is not None else None,
|
||||
permission_level=AgentPermissionLevel.READ.value,
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary="知识归纳任务已入队,等待后台执行。",
|
||||
route_json={
|
||||
"job_type": "knowledge_index_sync",
|
||||
"phase": "queued",
|
||||
"folder": folder_label,
|
||||
"force": force,
|
||||
"changed_only": (False if force else changed_only),
|
||||
"requested_document_ids": target_document_ids,
|
||||
"requested_by_username": current_user.username,
|
||||
"requested_by_name": current_user.name,
|
||||
"progress": {
|
||||
"total_documents": len(target_document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
self.knowledge_service.set_document_ingest_statuses(
|
||||
target_document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_SYNCING,
|
||||
agent_run_id=run.run_id,
|
||||
)
|
||||
knowledge_index_task_manager.submit_sync(
|
||||
agent_run_id=run.run_id,
|
||||
folder=folder_label,
|
||||
current_user=current_user,
|
||||
document_ids=target_document_ids,
|
||||
force=force,
|
||||
)
|
||||
return KnowledgeSyncDispatchResult(
|
||||
agent_run_id=run.run_id,
|
||||
folder=folder_label,
|
||||
document_ids=target_document_ids,
|
||||
queued_at=run.started_at,
|
||||
status=run.status,
|
||||
summary="知识归纳任务已进入后台执行,可在日志管理中查看进度。",
|
||||
)
|
||||
except Exception as exc:
|
||||
self.run_service.update_run(
|
||||
run.run_id,
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
error_message=str(exc),
|
||||
result_summary=str(exc),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
self.knowledge_service.set_document_ingest_statuses(
|
||||
target_document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=run.run_id,
|
||||
)
|
||||
raise
|
||||
|
||||
def _find_active_run(
|
||||
self,
|
||||
*,
|
||||
folder: str,
|
||||
requested_document_ids: list[str],
|
||||
):
|
||||
requested_set = {str(item).strip() for item in requested_document_ids if str(item).strip()}
|
||||
|
||||
for item in self.run_service.list_runs(
|
||||
agent=AgentName.HERMES.value,
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
limit=100,
|
||||
):
|
||||
if str(item.route_json.get("job_type") or "").strip() != "knowledge_index_sync":
|
||||
continue
|
||||
|
||||
heartbeat_raw = str(item.route_json.get("heartbeat_at") or "").strip()
|
||||
heartbeat_at = None
|
||||
if heartbeat_raw:
|
||||
try:
|
||||
heartbeat_at = datetime.fromisoformat(heartbeat_raw)
|
||||
except ValueError:
|
||||
heartbeat_at = None
|
||||
|
||||
last_seen_at = heartbeat_at or item.started_at
|
||||
if last_seen_at.tzinfo is None:
|
||||
last_seen_at = last_seen_at.replace(tzinfo=UTC)
|
||||
|
||||
if datetime.now(UTC) - last_seen_at > timedelta(minutes=30):
|
||||
stale_document_ids = [
|
||||
str(document_id).strip()
|
||||
for document_id in list(item.route_json.get("requested_document_ids") or [])
|
||||
if str(document_id).strip()
|
||||
]
|
||||
if stale_document_ids:
|
||||
self.knowledge_service.set_document_ingest_statuses(
|
||||
stale_document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=item.run_id,
|
||||
)
|
||||
self.run_service.merge_route_json(
|
||||
item.run_id,
|
||||
{
|
||||
"phase": "stale_failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary="知识归纳任务长时间无心跳,系统已自动标记失败。",
|
||||
error_message="Knowledge index heartbeat timed out.",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
continue
|
||||
|
||||
active_ids = {
|
||||
str(document_id).strip()
|
||||
for document_id in list(item.route_json.get("requested_document_ids") or [])
|
||||
if str(document_id).strip()
|
||||
}
|
||||
active_folder = str(item.route_json.get("folder") or "").strip()
|
||||
if active_folder == ALL_KNOWLEDGE_FOLDERS_LABEL:
|
||||
if not requested_set or active_ids & requested_set:
|
||||
return item
|
||||
continue
|
||||
if active_folder == folder:
|
||||
if not requested_set or not active_ids or active_ids & requested_set:
|
||||
return item
|
||||
|
||||
return None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,363 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import os
|
||||
import signal
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.api.deps import CurrentUserContext
|
||||
from app.core.agent_enums import AgentRunStatus
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.agent_runs import AgentRunService
|
||||
from app.services.knowledge import KNOWLEDGE_INGEST_STATUS_FAILED, KnowledgeService
|
||||
from app.services.llm_wiki import HERMES_AGENT_BATCH_TIMEOUT_SECONDS, LlmWikiService
|
||||
|
||||
logger = get_logger("app.services.llm_wiki_tasks")
|
||||
|
||||
|
||||
class LlmWikiTaskManager:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._threads: dict[str, threading.Thread] = {}
|
||||
|
||||
def submit_sync(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str] | None = None,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
worker = threading.Thread(
|
||||
target=self._run_sync,
|
||||
kwargs={
|
||||
"agent_run_id": agent_run_id,
|
||||
"folder": folder,
|
||||
"current_user": current_user,
|
||||
"document_ids": list(document_ids or []),
|
||||
"force": force,
|
||||
},
|
||||
daemon=True,
|
||||
name=f"llm-wiki-sync-{agent_run_id}",
|
||||
)
|
||||
with self._lock:
|
||||
self._threads[agent_run_id] = worker
|
||||
worker.start()
|
||||
|
||||
def shutdown(self, *, timeout_seconds: float = 1.0) -> None:
|
||||
with self._lock:
|
||||
threads = list(self._threads.items())
|
||||
self._threads.clear()
|
||||
|
||||
for _, worker in threads:
|
||||
if worker.is_alive():
|
||||
worker.join(timeout=timeout_seconds)
|
||||
|
||||
def _run_sync(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
folder: str,
|
||||
current_user: CurrentUserContext,
|
||||
document_ids: list[str],
|
||||
force: bool,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
db = session_factory()
|
||||
run_service = AgentRunService(db)
|
||||
knowledge_service = KnowledgeService()
|
||||
request_payload = {
|
||||
"folder": folder,
|
||||
"document_ids": list(document_ids),
|
||||
"force": force,
|
||||
}
|
||||
|
||||
try:
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "running",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"job_type": "llm_wiki_sync",
|
||||
"folder": folder,
|
||||
"force": force,
|
||||
"requested_document_ids": list(document_ids),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": 0,
|
||||
"percent": 0,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary="Hermes 后台归纳任务已启动。",
|
||||
)
|
||||
|
||||
dispatch = LlmWikiService(db).dispatch_agent_batch(
|
||||
folder=folder,
|
||||
document_ids=document_ids,
|
||||
force=force,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
if not dispatch.changed_document_ids:
|
||||
knowledge_service.refresh_document_ingest_statuses(
|
||||
document_ids=document_ids,
|
||||
preserve_syncing=False,
|
||||
)
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_dispatch",
|
||||
request_json=request_payload,
|
||||
response_json={"changed_document_ids": [], "skipped_document_ids": dispatch.skipped_document_ids},
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "succeeded",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": len(dispatch.skipped_document_ids),
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.SUCCEEDED.value,
|
||||
result_summary="本次所选文档均未变化,未重复派发 Hermes 任务。",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_dispatch",
|
||||
request_json=request_payload,
|
||||
response_json={
|
||||
"changed_document_ids": dispatch.changed_document_ids,
|
||||
"skipped_document_ids": dispatch.skipped_document_ids,
|
||||
"process_id": dispatch.process_id,
|
||||
},
|
||||
status="succeeded",
|
||||
duration_ms=0,
|
||||
)
|
||||
current_run = run_service.get_run(agent_run_id)
|
||||
if current_run is not None and current_run.status in {
|
||||
AgentRunStatus.SUCCEEDED.value,
|
||||
AgentRunStatus.FAILED.value,
|
||||
}:
|
||||
return
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "awaiting_callback",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"requested_document_ids": dispatch.changed_document_ids,
|
||||
"skipped_document_ids": dispatch.skipped_document_ids,
|
||||
"hermes_process_id": dispatch.process_id,
|
||||
"hermes_stdout_path": dispatch.stdout_path,
|
||||
"hermes_stderr_path": dispatch.stderr_path,
|
||||
"progress": {
|
||||
"total_documents": len(dispatch.changed_document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": 0,
|
||||
"skipped_documents": len(dispatch.skipped_document_ids),
|
||||
"percent": 0,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary="Hermes 任务已派发,等待 Agent 主动回调结果。",
|
||||
)
|
||||
self._start_process_monitor(
|
||||
agent_run_id=agent_run_id,
|
||||
document_ids=dispatch.changed_document_ids,
|
||||
process_id=dispatch.process_id,
|
||||
stderr_path=dispatch.stderr_path,
|
||||
timeout_seconds=HERMES_AGENT_BATCH_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Background LLM Wiki sync failed run_id=%s", agent_run_id)
|
||||
if document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
run_service.record_tool_call(
|
||||
run_id=agent_run_id,
|
||||
tool_type="llm",
|
||||
tool_name="system_hermes_llm_wiki_sync",
|
||||
request_json=request_payload,
|
||||
response_json={"error": str(exc)},
|
||||
status="failed",
|
||||
duration_ms=0,
|
||||
error_message=str(exc),
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"progress": {
|
||||
"total_documents": len(document_ids),
|
||||
"completed_documents": 0,
|
||||
"failed_documents": len(document_ids),
|
||||
"skipped_documents": 0,
|
||||
"percent": 100,
|
||||
},
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary=str(exc),
|
||||
error_message=str(exc),
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
with self._lock:
|
||||
self._threads.pop(agent_run_id, None)
|
||||
|
||||
def _start_process_monitor(
|
||||
self,
|
||||
*,
|
||||
agent_run_id: str,
|
||||
document_ids: list[str],
|
||||
process_id: int,
|
||||
stderr_path: str,
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
worker = threading.Thread(
|
||||
target=self._monitor_process,
|
||||
kwargs={
|
||||
"agent_run_id": agent_run_id,
|
||||
"document_ids": list(document_ids),
|
||||
"process_id": process_id,
|
||||
"stderr_path": stderr_path,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
},
|
||||
daemon=True,
|
||||
name=f"llm-wiki-monitor-{agent_run_id}",
|
||||
)
|
||||
worker.start()
|
||||
|
||||
@staticmethod
|
||||
def _monitor_process(
|
||||
*,
|
||||
agent_run_id: str,
|
||||
document_ids: list[str],
|
||||
process_id: int,
|
||||
stderr_path: str,
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
db = session_factory()
|
||||
run_service = AgentRunService(db)
|
||||
knowledge_service = KnowledgeService()
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3)
|
||||
run = run_service.get_run(agent_run_id)
|
||||
if run is None or run.status != AgentRunStatus.RUNNING.value:
|
||||
return
|
||||
if time.monotonic() - started_at > timeout_seconds:
|
||||
try:
|
||||
os.killpg(process_id, signal.SIGTERM)
|
||||
except OSError:
|
||||
pass
|
||||
error_message = LlmWikiTaskManager._read_process_error(stderr_path)
|
||||
if document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"hermes_process_id": process_id,
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary="Hermes 任务执行超时,已自动终止等待。",
|
||||
error_message=error_message or "Hermes process exceeded callback timeout.",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
if LlmWikiTaskManager._is_process_alive(process_id):
|
||||
continue
|
||||
|
||||
error_message = LlmWikiTaskManager._read_process_error(stderr_path)
|
||||
if document_ids:
|
||||
knowledge_service.set_document_ingest_statuses(
|
||||
document_ids,
|
||||
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
|
||||
agent_run_id=agent_run_id,
|
||||
)
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
{
|
||||
"phase": "failed",
|
||||
"heartbeat_at": datetime.now(UTC).isoformat(),
|
||||
"hermes_process_id": process_id,
|
||||
},
|
||||
status=AgentRunStatus.FAILED.value,
|
||||
result_summary="Hermes 进程已退出且未回调结果。",
|
||||
error_message=error_message or "Hermes process exited before callback.",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
return
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@staticmethod
|
||||
def _is_process_alive(process_id: int) -> bool:
|
||||
stat_path = Path(f"/proc/{process_id}/stat")
|
||||
if not stat_path.exists():
|
||||
return False
|
||||
try:
|
||||
parts = stat_path.read_text(encoding="utf-8").split()
|
||||
except OSError:
|
||||
return False
|
||||
return len(parts) > 2 and parts[2] != "Z"
|
||||
|
||||
@staticmethod
|
||||
def _read_process_error(stderr_path: str) -> str:
|
||||
path = Path(stderr_path)
|
||||
if not stderr_path or not path.exists():
|
||||
return ""
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
except OSError:
|
||||
return ""
|
||||
return content[-1000:]
|
||||
|
||||
@staticmethod
|
||||
def _write_progress(
|
||||
*,
|
||||
run_service: AgentRunService,
|
||||
agent_run_id: str,
|
||||
payload: dict[str, Any],
|
||||
summary: str,
|
||||
) -> None:
|
||||
patched_payload = dict(payload)
|
||||
patched_payload["heartbeat_at"] = datetime.now(UTC).isoformat()
|
||||
run_service.merge_route_json(
|
||||
agent_run_id,
|
||||
patched_payload,
|
||||
status=AgentRunStatus.RUNNING.value,
|
||||
result_summary=summary,
|
||||
)
|
||||
|
||||
|
||||
llm_wiki_task_manager = LlmWikiTaskManager()
|
||||
@@ -59,9 +59,18 @@ def _probe_openai_compatible(payload: ModelConnectivityTestRequest) -> int:
|
||||
normalized_endpoint = _normalize_endpoint(payload.endpoint)
|
||||
headers = _build_headers(api_key=payload.api_key, use_bearer=True)
|
||||
|
||||
if payload.capability == "embedding":
|
||||
if payload.capability == "reranker" and payload.provider == "Ali":
|
||||
url, body = _build_ali_reranker_request(payload.model, normalized_endpoint)
|
||||
elif payload.capability == "embedding":
|
||||
url = _ensure_path(normalized_endpoint, "embeddings")
|
||||
body = {"model": payload.model, "input": "connectivity test"}
|
||||
elif payload.capability == "reranker":
|
||||
url = _ensure_path(normalized_endpoint, "rerank")
|
||||
body = {
|
||||
"model": payload.model,
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
}
|
||||
else:
|
||||
url = _ensure_path(normalized_endpoint, "chat/completions")
|
||||
body = {
|
||||
@@ -74,6 +83,35 @@ def _probe_openai_compatible(payload: ModelConnectivityTestRequest) -> int:
|
||||
return status_code
|
||||
|
||||
|
||||
def _build_ali_reranker_request(model: str, endpoint: str) -> tuple[str, dict[str, Any]]:
|
||||
normalized_model = str(model or "").strip()
|
||||
if normalized_model == "qwen3-rerank":
|
||||
return (
|
||||
"https://dashscope.aliyuncs.com/compatible-api/v1/reranks",
|
||||
{
|
||||
"model": normalized_model,
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
"top_n": 1,
|
||||
},
|
||||
)
|
||||
|
||||
return (
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
{
|
||||
"model": normalized_model,
|
||||
"input": {
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
},
|
||||
"parameters": {
|
||||
"return_documents": False,
|
||||
"top_n": 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _probe_ollama(payload: ModelConnectivityTestRequest) -> int:
|
||||
normalized_endpoint = _normalize_endpoint(payload.endpoint)
|
||||
headers = _build_headers(api_key=payload.api_key, use_bearer=False)
|
||||
@@ -81,6 +119,8 @@ def _probe_ollama(payload: ModelConnectivityTestRequest) -> int:
|
||||
if payload.capability == "embedding":
|
||||
url = _ensure_path(normalized_endpoint, "api/embed")
|
||||
body = {"model": payload.model, "input": "connectivity test"}
|
||||
elif payload.capability == "reranker":
|
||||
raise ConnectivityCheckError("Ollama 暂不支持 reranker 连通性探测。", status_code=HTTPStatus.BAD_REQUEST)
|
||||
else:
|
||||
url = _ensure_path(normalized_endpoint, "api/chat")
|
||||
body = {
|
||||
@@ -100,6 +140,12 @@ def _probe_azure_openai(payload: ModelConnectivityTestRequest) -> int:
|
||||
if payload.capability == "embedding":
|
||||
url = f"{deployment_base}/embeddings?api-version={AZURE_API_VERSION}"
|
||||
body = {"input": "connectivity test"}
|
||||
elif payload.capability == "reranker":
|
||||
url = f"{deployment_base}/rerank?api-version={AZURE_API_VERSION}"
|
||||
body = {
|
||||
"query": "connectivity test",
|
||||
"documents": ["sample document"],
|
||||
}
|
||||
else:
|
||||
url = f"{deployment_base}/chat/completions?api-version={AZURE_API_VERSION}"
|
||||
body = {
|
||||
@@ -168,12 +214,13 @@ def _send_json_request(
|
||||
*,
|
||||
headers: dict[str, str],
|
||||
payload: dict[str, Any],
|
||||
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
|
||||
) -> tuple[int, Any]:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
request = Request(url=url, data=data, headers=headers, method=method)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=DEFAULT_TIMEOUT_SECONDS) as response:
|
||||
with urlopen(request, timeout=timeout_seconds) as response:
|
||||
body = response.read().decode("utf-8") if response.length != 0 else ""
|
||||
return response.status, _parse_json_body(body)
|
||||
except HTTPError as exc:
|
||||
|
||||
@@ -243,6 +243,7 @@ STATUS_KEYWORDS = {
|
||||
|
||||
PRIVILEGED_ROLE_CODES = {"manager", "finance", "approver", "executive"}
|
||||
CONTEXTUAL_SCENARIOS = {"expense", "accounts_receivable", "accounts_payable", "knowledge"}
|
||||
KNOWLEDGE_INTENTS = {"query", "explain", "compare"}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -356,7 +357,7 @@ class SemanticOntologyService:
|
||||
rule_scenario = inferred_scenario
|
||||
scenario_score = 0.18
|
||||
|
||||
if self._looks_like_expense_narrative(
|
||||
if session_scenario != "knowledge" and self._looks_like_expense_narrative(
|
||||
compact_query,
|
||||
scenario=rule_scenario,
|
||||
entities=entities,
|
||||
@@ -371,7 +372,7 @@ class SemanticOntologyService:
|
||||
entities=entities,
|
||||
time_range=time_range,
|
||||
)
|
||||
if self._should_inherit_expense_draft(
|
||||
if session_scenario != "knowledge" and self._should_inherit_expense_draft(
|
||||
compact_query,
|
||||
scenario=rule_scenario,
|
||||
entities=entities,
|
||||
@@ -384,17 +385,19 @@ class SemanticOntologyService:
|
||||
intent_score = max(intent_score, 0.18)
|
||||
metrics = self._extract_metrics(compact_query)
|
||||
constraints = self._extract_constraints(compact_query, entities)
|
||||
model_parse = self._parse_with_model(
|
||||
payload=payload,
|
||||
query=query,
|
||||
compact_query=compact_query,
|
||||
fallback_scenario=rule_scenario,
|
||||
fallback_intent=rule_intent,
|
||||
entities=entities,
|
||||
time_range=time_range,
|
||||
metrics=metrics,
|
||||
constraints=constraints,
|
||||
)
|
||||
model_parse = None
|
||||
if session_scenario != "knowledge":
|
||||
model_parse = self._parse_with_model(
|
||||
payload=payload,
|
||||
query=query,
|
||||
compact_query=compact_query,
|
||||
fallback_scenario=rule_scenario,
|
||||
fallback_intent=rule_intent,
|
||||
entities=entities,
|
||||
time_range=time_range,
|
||||
metrics=metrics,
|
||||
constraints=constraints,
|
||||
)
|
||||
scenario = self._resolve_scenario(rule_scenario, model_parse)
|
||||
if session_scenario == "knowledge":
|
||||
scenario = "knowledge"
|
||||
@@ -968,6 +971,12 @@ class SemanticOntologyService:
|
||||
model_parse: LlmOntologyParseResult | None,
|
||||
) -> str:
|
||||
candidate = model_parse.intent if model_parse is not None else fallback_intent
|
||||
if scenario == "knowledge":
|
||||
if candidate in KNOWLEDGE_INTENTS:
|
||||
return candidate
|
||||
if fallback_intent in KNOWLEDGE_INTENTS:
|
||||
return fallback_intent
|
||||
return "query"
|
||||
if candidate == "query" and scenario == "expense":
|
||||
if self._is_generic_expense_prompt(compact_query) or fallback_intent == "draft":
|
||||
return "draft"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from http import HTTPStatus
|
||||
from time import monotonic, sleep
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -18,6 +19,12 @@ from app.services.model_connectivity import (
|
||||
from app.services.settings import SettingsService
|
||||
|
||||
logger = get_logger("app.services.runtime_chat")
|
||||
DEFAULT_RUNTIME_CHAT_TIMEOUT_SECONDS = 45
|
||||
DEFAULT_RUNTIME_CHAT_RETRY_ATTEMPTS = 2
|
||||
DEFAULT_RUNTIME_CHAT_RETRY_DELAY_SECONDS = 0.6
|
||||
DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS = 90
|
||||
|
||||
_slot_failure_until: dict[str, float] = {}
|
||||
|
||||
|
||||
class RuntimeChatService:
|
||||
@@ -32,33 +39,71 @@ class RuntimeChatService:
|
||||
slot_priority: tuple[str, ...] = ("main", "backup"),
|
||||
max_tokens: int = 500,
|
||||
temperature: float = 0.2,
|
||||
timeout_seconds: int | None = None,
|
||||
slot_timeouts: dict[str, int] | None = None,
|
||||
max_attempts: int | None = None,
|
||||
) -> str | None:
|
||||
for slot in slot_priority:
|
||||
config = self._load_chat_slot(slot)
|
||||
if config is None:
|
||||
continue
|
||||
configs = [
|
||||
config
|
||||
for slot in slot_priority
|
||||
if (config := self._load_chat_slot(slot)) is not None
|
||||
]
|
||||
resolved_timeout_seconds = timeout_seconds or DEFAULT_RUNTIME_CHAT_TIMEOUT_SECONDS
|
||||
resolved_slot_timeouts = dict(slot_timeouts or {})
|
||||
resolved_max_attempts = max_attempts or DEFAULT_RUNTIME_CHAT_RETRY_ATTEMPTS
|
||||
|
||||
try:
|
||||
response_text = self._request_chat_completion(
|
||||
config,
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Runtime chat request failed slot=%s provider=%s: %s",
|
||||
slot,
|
||||
config["provider"],
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
if response_text:
|
||||
return response_text.strip()
|
||||
for attempt in range(1, resolved_max_attempts + 1):
|
||||
for config in configs:
|
||||
cache_key = self._build_slot_cache_key(config)
|
||||
if _slot_failure_until.get(cache_key, 0.0) > monotonic():
|
||||
logger.info(
|
||||
"Skip runtime chat slot=%s provider=%s because it is in cooldown",
|
||||
config["slot"],
|
||||
config["provider"],
|
||||
)
|
||||
continue
|
||||
try:
|
||||
response_text = self._request_chat_completion(
|
||||
config,
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=resolved_slot_timeouts.get(
|
||||
config["slot"],
|
||||
resolved_timeout_seconds,
|
||||
),
|
||||
)
|
||||
if response_text:
|
||||
_slot_failure_until.pop(cache_key, None)
|
||||
return response_text.strip()
|
||||
except Exception as exc:
|
||||
_slot_failure_until[cache_key] = (
|
||||
monotonic() + DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS
|
||||
)
|
||||
logger.warning(
|
||||
"Runtime chat request failed slot=%s provider=%s attempt=%s/%s: %s",
|
||||
config["slot"],
|
||||
config["provider"],
|
||||
attempt,
|
||||
resolved_max_attempts,
|
||||
exc,
|
||||
)
|
||||
if attempt < resolved_max_attempts:
|
||||
sleep(DEFAULT_RUNTIME_CHAT_RETRY_DELAY_SECONDS)
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _build_slot_cache_key(config: dict[str, str]) -> str:
|
||||
return "|".join(
|
||||
[
|
||||
str(config.get("slot") or ""),
|
||||
str(config.get("provider") or ""),
|
||||
str(config.get("endpoint") or ""),
|
||||
str(config.get("model") or ""),
|
||||
]
|
||||
)
|
||||
|
||||
def _load_chat_slot(self, slot: str) -> dict[str, str] | None:
|
||||
try:
|
||||
config = self.settings_service.get_runtime_model_config(slot)
|
||||
@@ -95,6 +140,7 @@ class RuntimeChatService:
|
||||
*,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
provider = config["provider"]
|
||||
endpoint = config["endpoint"]
|
||||
@@ -109,6 +155,7 @@ class RuntimeChatService:
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
if provider == "Ollama":
|
||||
@@ -119,38 +166,48 @@ class RuntimeChatService:
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
return self._request_openai_compatible(
|
||||
provider=provider,
|
||||
endpoint=endpoint,
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
def _request_openai_compatible(
|
||||
self,
|
||||
*,
|
||||
provider: str,
|
||||
endpoint: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
url = _ensure_path(_normalize_endpoint(endpoint), "chat/completions")
|
||||
request_payload: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
}
|
||||
if provider == "GLM":
|
||||
request_payload["thinking"] = {"type": "disabled"}
|
||||
|
||||
status_code, payload = _send_json_request(
|
||||
"POST",
|
||||
url,
|
||||
headers=_build_headers(api_key=api_key, use_bearer=True),
|
||||
payload={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
},
|
||||
payload=request_payload,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if status_code >= HTTPStatus.BAD_REQUEST:
|
||||
raise ConnectivityCheckError(
|
||||
@@ -168,6 +225,7 @@ class RuntimeChatService:
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
url = _ensure_path(_normalize_endpoint(endpoint), "api/chat")
|
||||
status_code, payload = _send_json_request(
|
||||
@@ -183,6 +241,7 @@ class RuntimeChatService:
|
||||
"temperature": temperature,
|
||||
},
|
||||
},
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if status_code >= HTTPStatus.BAD_REQUEST:
|
||||
raise ConnectivityCheckError(
|
||||
@@ -200,6 +259,7 @@ class RuntimeChatService:
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
deployment_base = _build_azure_deployment_base(endpoint, model)
|
||||
url = f"{deployment_base}/chat/completions?api-version={AZURE_API_VERSION}"
|
||||
@@ -212,6 +272,7 @@ class RuntimeChatService:
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
},
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if status_code >= HTTPStatus.BAD_REQUEST:
|
||||
raise ConnectivityCheckError(
|
||||
|
||||
@@ -64,29 +64,29 @@ MODEL_SLOT_CONFIGS = {
|
||||
capability="chat",
|
||||
priority=20,
|
||||
),
|
||||
"vlm": ModelSlotConfig(
|
||||
provider_attr="vlm_provider",
|
||||
model_attr="vlm_model",
|
||||
endpoint_attr="vlm_endpoint",
|
||||
legacy_secret_attr="vlm_api_key_encrypted",
|
||||
default_provider="Gemini",
|
||||
default_model="gemini-2.5-flash",
|
||||
default_endpoint="https://generativelanguage.googleapis.com/v1beta/openai/",
|
||||
capability="chat",
|
||||
priority=30,
|
||||
),
|
||||
"embedding": ModelSlotConfig(
|
||||
provider_attr="embedding_provider",
|
||||
model_attr="embedding_model",
|
||||
endpoint_attr="embedding_endpoint",
|
||||
legacy_secret_attr="embedding_api_key_encrypted",
|
||||
default_provider="GLM",
|
||||
default_model="Embedding-3",
|
||||
default_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
capability="embedding",
|
||||
priority=40,
|
||||
),
|
||||
}
|
||||
"embedding": ModelSlotConfig(
|
||||
provider_attr="embedding_provider",
|
||||
model_attr="embedding_model",
|
||||
endpoint_attr="embedding_endpoint",
|
||||
legacy_secret_attr="embedding_api_key_encrypted",
|
||||
default_provider="GLM",
|
||||
default_model="Embedding-3",
|
||||
default_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
capability="embedding",
|
||||
priority=30,
|
||||
),
|
||||
"reranker": ModelSlotConfig(
|
||||
provider_attr="reranker_provider",
|
||||
model_attr="reranker_model",
|
||||
endpoint_attr="reranker_endpoint",
|
||||
legacy_secret_attr="reranker_api_key_encrypted",
|
||||
default_provider="Ali",
|
||||
default_model="gte-rerank-v2",
|
||||
default_endpoint="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
capability="reranker",
|
||||
priority=40,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -138,6 +138,8 @@ class SettingsService:
|
||||
|
||||
if self._sync_onlyoffice_defaults(settings_row, secrets_row):
|
||||
should_commit = True
|
||||
if self._sync_reranker_legacy_defaults(settings_row, secrets_row):
|
||||
should_commit = True
|
||||
|
||||
if should_commit:
|
||||
self.db.commit()
|
||||
@@ -150,13 +152,28 @@ class SettingsService:
|
||||
self,
|
||||
settings_row: SystemSetting,
|
||||
secrets_row: SystemSettingSecret,
|
||||
) -> dict[str, SystemModelSetting]:
|
||||
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
|
||||
should_commit = False
|
||||
|
||||
for slot, config in MODEL_SLOT_CONFIGS.items():
|
||||
if slot in model_rows:
|
||||
continue
|
||||
) -> dict[str, SystemModelSetting]:
|
||||
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
|
||||
should_commit = False
|
||||
|
||||
if "reranker" not in model_rows and "vlm" in model_rows:
|
||||
self.db.execute(
|
||||
text(
|
||||
"UPDATE system_model_settings "
|
||||
"SET slot = 'reranker', capability = :capability, priority = :priority "
|
||||
"WHERE slot = 'vlm'"
|
||||
),
|
||||
{
|
||||
"capability": MODEL_SLOT_CONFIGS["reranker"].capability,
|
||||
"priority": MODEL_SLOT_CONFIGS["reranker"].priority,
|
||||
},
|
||||
)
|
||||
self.db.commit()
|
||||
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
|
||||
|
||||
for slot, config in MODEL_SLOT_CONFIGS.items():
|
||||
if slot in model_rows:
|
||||
continue
|
||||
|
||||
model_row = SystemModelSetting(
|
||||
slot=slot,
|
||||
@@ -224,13 +241,6 @@ class SettingsService:
|
||||
payload.llmForm.backupEndpoint,
|
||||
payload.llmForm.backupApiKey,
|
||||
)
|
||||
self._apply_model_setting(
|
||||
model_rows["vlm"],
|
||||
payload.llmForm.vlmProvider,
|
||||
payload.llmForm.vlmModel,
|
||||
payload.llmForm.vlmEndpoint,
|
||||
payload.llmForm.vlmApiKey,
|
||||
)
|
||||
self._apply_model_setting(
|
||||
model_rows["embedding"],
|
||||
payload.llmForm.embeddingProvider,
|
||||
@@ -238,6 +248,13 @@ class SettingsService:
|
||||
payload.llmForm.embeddingEndpoint,
|
||||
payload.llmForm.embeddingApiKey,
|
||||
)
|
||||
self._apply_model_setting(
|
||||
model_rows["reranker"],
|
||||
payload.llmForm.rerankerProvider,
|
||||
payload.llmForm.rerankerModel,
|
||||
payload.llmForm.rerankerEndpoint,
|
||||
payload.llmForm.rerankerApiKey,
|
||||
)
|
||||
|
||||
if payload.renderForm.enabled and not payload.renderForm.publicUrl:
|
||||
raise ValueError("启用 ONLYOFFICE 时必须配置服务地址。")
|
||||
@@ -252,14 +269,14 @@ class SettingsService:
|
||||
settings_row.main_model = model_rows["main"].model_name
|
||||
settings_row.main_endpoint = model_rows["main"].endpoint
|
||||
settings_row.backup_provider = model_rows["backup"].provider
|
||||
settings_row.backup_model = model_rows["backup"].model_name
|
||||
settings_row.backup_endpoint = model_rows["backup"].endpoint
|
||||
settings_row.vlm_provider = model_rows["vlm"].provider
|
||||
settings_row.vlm_model = model_rows["vlm"].model_name
|
||||
settings_row.vlm_endpoint = model_rows["vlm"].endpoint
|
||||
settings_row.backup_model = model_rows["backup"].model_name
|
||||
settings_row.backup_endpoint = model_rows["backup"].endpoint
|
||||
settings_row.embedding_provider = model_rows["embedding"].provider
|
||||
settings_row.embedding_model = model_rows["embedding"].model_name
|
||||
settings_row.embedding_endpoint = model_rows["embedding"].endpoint
|
||||
settings_row.reranker_provider = model_rows["reranker"].provider
|
||||
settings_row.reranker_model = model_rows["reranker"].model_name
|
||||
settings_row.reranker_endpoint = model_rows["reranker"].endpoint
|
||||
settings_row.onlyoffice_enabled = payload.renderForm.enabled
|
||||
settings_row.onlyoffice_public_url = payload.renderForm.publicUrl
|
||||
|
||||
@@ -428,7 +445,7 @@ class SettingsService:
|
||||
legacy_admin = read_admin_secret() or {}
|
||||
admin_account = str(legacy_admin.get("username", "")).strip() or "superadmin"
|
||||
|
||||
return SystemSetting(
|
||||
return SystemSetting(
|
||||
id=SETTINGS_ROW_ID,
|
||||
company_name=company_name,
|
||||
display_name=company_name,
|
||||
@@ -445,16 +462,16 @@ class SettingsService:
|
||||
login_alert_enabled=True,
|
||||
main_provider="Codex",
|
||||
main_model="codex-mini-latest",
|
||||
main_endpoint="https://api.openai.com/v1",
|
||||
backup_provider="GLM",
|
||||
backup_model="glm-5.1",
|
||||
main_endpoint="https://api.openai.com/v1",
|
||||
backup_provider="GLM",
|
||||
backup_model="glm-5.1",
|
||||
backup_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
vlm_provider="Gemini",
|
||||
vlm_model="gemini-2.5-flash",
|
||||
vlm_endpoint="https://generativelanguage.googleapis.com/v1beta/openai/",
|
||||
embedding_provider="GLM",
|
||||
embedding_model="Embedding-3",
|
||||
embedding_endpoint="https://open.bigmodel.cn/api/paas/v4/",
|
||||
reranker_provider="Ali",
|
||||
reranker_model="gte-rerank-v2",
|
||||
reranker_endpoint="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
onlyoffice_enabled=bool(self.runtime_settings.onlyoffice_enabled),
|
||||
onlyoffice_public_url=str(self.runtime_settings.onlyoffice_public_url or "").strip(),
|
||||
log_level="INFO",
|
||||
@@ -542,6 +559,19 @@ class SettingsService:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN onlyoffice_public_url VARCHAR(512) DEFAULT ''"
|
||||
)
|
||||
if "reranker_provider" not in settings_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN reranker_provider VARCHAR(64) DEFAULT 'Ali'"
|
||||
)
|
||||
if "reranker_model" not in settings_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN reranker_model VARCHAR(255) DEFAULT 'gte-rerank-v2'"
|
||||
)
|
||||
if "reranker_endpoint" not in settings_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_settings ADD COLUMN reranker_endpoint "
|
||||
"VARCHAR(512) DEFAULT 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank'"
|
||||
)
|
||||
|
||||
if "system_setting_secrets" in table_names:
|
||||
secret_columns = {column["name"] for column in inspector.get_columns("system_setting_secrets")}
|
||||
@@ -549,6 +579,10 @@ class SettingsService:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_setting_secrets ADD COLUMN onlyoffice_jwt_secret_encrypted TEXT DEFAULT ''"
|
||||
)
|
||||
if "reranker_api_key_encrypted" not in secret_columns:
|
||||
migration_statements.append(
|
||||
"ALTER TABLE system_setting_secrets ADD COLUMN reranker_api_key_encrypted TEXT DEFAULT ''"
|
||||
)
|
||||
|
||||
for statement in migration_statements:
|
||||
self.db.execute(text(statement))
|
||||
@@ -583,16 +617,41 @@ class SettingsService:
|
||||
|
||||
return should_commit
|
||||
|
||||
@staticmethod
|
||||
def _sync_reranker_legacy_defaults(
|
||||
settings_row: SystemSetting,
|
||||
secrets_row: SystemSettingSecret,
|
||||
) -> bool:
|
||||
should_commit = False
|
||||
|
||||
if not str(settings_row.reranker_provider or "").strip() and str(settings_row.vlm_provider or "").strip():
|
||||
settings_row.reranker_provider = settings_row.vlm_provider
|
||||
should_commit = True
|
||||
if not str(settings_row.reranker_model or "").strip() and str(settings_row.vlm_model or "").strip():
|
||||
settings_row.reranker_model = settings_row.vlm_model
|
||||
should_commit = True
|
||||
if not str(settings_row.reranker_endpoint or "").strip() and str(settings_row.vlm_endpoint or "").strip():
|
||||
settings_row.reranker_endpoint = settings_row.vlm_endpoint
|
||||
should_commit = True
|
||||
if (
|
||||
not str(secrets_row.reranker_api_key_encrypted or "").strip()
|
||||
and str(secrets_row.vlm_api_key_encrypted or "").strip()
|
||||
):
|
||||
secrets_row.reranker_api_key_encrypted = secrets_row.vlm_api_key_encrypted
|
||||
should_commit = True
|
||||
|
||||
return should_commit
|
||||
|
||||
@staticmethod
|
||||
def _serialize(
|
||||
settings_row: SystemSetting,
|
||||
secrets_row: SystemSettingSecret,
|
||||
model_rows: dict[str, SystemModelSetting],
|
||||
) -> SettingsRead:
|
||||
main_model = model_rows["main"]
|
||||
backup_model = model_rows["backup"]
|
||||
vlm_model = model_rows["vlm"]
|
||||
embedding_model = model_rows["embedding"]
|
||||
model_rows: dict[str, SystemModelSetting],
|
||||
) -> SettingsRead:
|
||||
main_model = model_rows["main"]
|
||||
backup_model = model_rows["backup"]
|
||||
embedding_model = model_rows["embedding"]
|
||||
reranker_model = model_rows["reranker"]
|
||||
|
||||
return SettingsRead(
|
||||
companyForm={
|
||||
@@ -624,20 +683,20 @@ class SettingsService:
|
||||
"mainApiKey": "",
|
||||
"mainApiKeyConfigured": bool(main_model.api_key_encrypted),
|
||||
"backupProvider": backup_model.provider,
|
||||
"backupModel": backup_model.model_name,
|
||||
"backupEndpoint": backup_model.endpoint,
|
||||
"backupApiKey": "",
|
||||
"backupApiKeyConfigured": bool(backup_model.api_key_encrypted),
|
||||
"vlmProvider": vlm_model.provider,
|
||||
"vlmModel": vlm_model.model_name,
|
||||
"vlmEndpoint": vlm_model.endpoint,
|
||||
"vlmApiKey": "",
|
||||
"vlmApiKeyConfigured": bool(vlm_model.api_key_encrypted),
|
||||
"embeddingProvider": embedding_model.provider,
|
||||
"embeddingModel": embedding_model.model_name,
|
||||
"backupModel": backup_model.model_name,
|
||||
"backupEndpoint": backup_model.endpoint,
|
||||
"backupApiKey": "",
|
||||
"backupApiKeyConfigured": bool(backup_model.api_key_encrypted),
|
||||
"embeddingProvider": embedding_model.provider,
|
||||
"embeddingModel": embedding_model.model_name,
|
||||
"embeddingEndpoint": embedding_model.endpoint,
|
||||
"embeddingApiKey": "",
|
||||
"embeddingApiKeyConfigured": bool(embedding_model.api_key_encrypted),
|
||||
"rerankerProvider": reranker_model.provider,
|
||||
"rerankerModel": reranker_model.model_name,
|
||||
"rerankerEndpoint": reranker_model.endpoint,
|
||||
"rerankerApiKey": "",
|
||||
"rerankerApiKeyConfigured": bool(reranker_model.api_key_encrypted),
|
||||
},
|
||||
renderForm={
|
||||
"enabled": settings_row.onlyoffice_enabled,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,9 @@ Requires-Dist: PyJWT<3.0.0,>=2.9.0
|
||||
Requires-Dist: pydantic-settings<3.0.0,>=2.6.0
|
||||
Requires-Dist: python-dotenv<2.0.0,>=1.0.1
|
||||
Requires-Dist: email-validator<3.0.0,>=2.2.0
|
||||
Requires-Dist: python-multipart<1.0.0,>=0.0.20
|
||||
Requires-Dist: lightrag-hku<1.5.0,>=1.4.16
|
||||
Requires-Dist: qdrant-client<2.0.0,>=1.18.0
|
||||
Provides-Extra: dev
|
||||
Requires-Dist: pytest<9.0.0,>=8.3.0; extra == "dev"
|
||||
Requires-Dist: httpx<1.0.0,>=0.28.0; extra == "dev"
|
||||
|
||||
@@ -16,14 +16,19 @@ src/app/api/v1/endpoints/bootstrap.py
|
||||
src/app/api/v1/endpoints/employees.py
|
||||
src/app/api/v1/endpoints/health.py
|
||||
src/app/api/v1/endpoints/knowledge.py
|
||||
src/app/api/v1/endpoints/ocr.py
|
||||
src/app/api/v1/endpoints/ontology.py
|
||||
src/app/api/v1/endpoints/orchestrator.py
|
||||
src/app/api/v1/endpoints/reimbursements.py
|
||||
src/app/api/v1/endpoints/settings.py
|
||||
src/app/api/v1/endpoints/system_logs.py
|
||||
src/app/core/__init__.py
|
||||
src/app/core/admin_secret.py
|
||||
src/app/core/agent_enums.py
|
||||
src/app/core/bootstrap.py
|
||||
src/app/core/config.py
|
||||
src/app/core/logging.py
|
||||
src/app/core/openapi.py
|
||||
src/app/core/secret_box.py
|
||||
src/app/core/security.py
|
||||
src/app/db/__init__.py
|
||||
@@ -34,6 +39,7 @@ src/app/middleware/__init__.py
|
||||
src/app/middleware/logging.py
|
||||
src/app/models/__init__.py
|
||||
src/app/models/agent_asset.py
|
||||
src/app/models/agent_conversation.py
|
||||
src/app/models/agent_run.py
|
||||
src/app/models/approval.py
|
||||
src/app/models/audit_log.py
|
||||
@@ -59,23 +65,42 @@ src/app/schemas/agent_run.py
|
||||
src/app/schemas/audit_log.py
|
||||
src/app/schemas/auth.py
|
||||
src/app/schemas/bootstrap.py
|
||||
src/app/schemas/common.py
|
||||
src/app/schemas/employee.py
|
||||
src/app/schemas/knowledge.py
|
||||
src/app/schemas/ocr.py
|
||||
src/app/schemas/ontology.py
|
||||
src/app/schemas/orchestrator.py
|
||||
src/app/schemas/reimbursement.py
|
||||
src/app/schemas/settings.py
|
||||
src/app/schemas/system_log.py
|
||||
src/app/schemas/user_agent.py
|
||||
src/app/services/__init__.py
|
||||
src/app/services/agent_assets.py
|
||||
src/app/services/agent_conversations.py
|
||||
src/app/services/agent_foundation.py
|
||||
src/app/services/agent_runs.py
|
||||
src/app/services/audit.py
|
||||
src/app/services/auth.py
|
||||
src/app/services/document_intelligence.py
|
||||
src/app/services/employee.py
|
||||
src/app/services/employee_seed.py
|
||||
src/app/services/expense_claims.py
|
||||
src/app/services/expense_rule_runtime.py
|
||||
src/app/services/hermes_sync.py
|
||||
src/app/services/knowledge.py
|
||||
src/app/services/knowledge_index_tasks.py
|
||||
src/app/services/knowledge_rag.py
|
||||
src/app/services/model_connectivity.py
|
||||
src/app/services/ocr.py
|
||||
src/app/services/ontology.py
|
||||
src/app/services/orchestrator.py
|
||||
src/app/services/reimbursement.py
|
||||
src/app/services/runtime_chat.py
|
||||
src/app/services/settings.py
|
||||
src/app/services/system_hermes.py
|
||||
src/app/services/system_logs.py
|
||||
src/app/services/user_agent.py
|
||||
src/x_financial_server.egg-info/PKG-INFO
|
||||
src/x_financial_server.egg-info/SOURCES.txt
|
||||
src/x_financial_server.egg-info/dependency_links.txt
|
||||
@@ -85,10 +110,19 @@ tests/test_agent_asset_service.py
|
||||
tests/test_agent_foundation_endpoints.py
|
||||
tests/test_auth_service.py
|
||||
tests/test_config_settings_reload.py
|
||||
tests/test_document_intelligence.py
|
||||
tests/test_employee_service.py
|
||||
tests/test_env_file_precedence.py
|
||||
tests/test_expense_claim_service.py
|
||||
tests/test_imports.py
|
||||
tests/test_knowledge_onlyoffice_config.py
|
||||
tests/test_ocr_endpoints.py
|
||||
tests/test_ocr_service.py
|
||||
tests/test_ontology_service.py
|
||||
tests/test_openapi_schema.py
|
||||
tests/test_reimbursement_endpoints.py
|
||||
tests/test_server_start_dependencies.py
|
||||
tests/test_settings_persistence.py
|
||||
tests/test_settings_service.py
|
||||
tests/test_settings_service.py
|
||||
tests/test_system_logs_service.py
|
||||
tests/test_user_agent_service.py
|
||||
@@ -7,6 +7,9 @@ PyJWT<3.0.0,>=2.9.0
|
||||
pydantic-settings<3.0.0,>=2.6.0
|
||||
python-dotenv<2.0.0,>=1.0.1
|
||||
email-validator<3.0.0,>=2.2.0
|
||||
python-multipart<1.0.0,>=0.0.20
|
||||
lightrag-hku<1.5.0,>=1.4.16
|
||||
qdrant-client<2.0.0,>=1.18.0
|
||||
|
||||
[dev]
|
||||
pytest<9.0.0,>=8.3.0
|
||||
|
||||
Reference in New Issue
Block a user