YG-Soft/X-Financial
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 重构知识库系统，移除Hermes集成，增强RAG和同步功能

Browse Source 
主要变更:
- 移除Hermes智能体及相关回调服务
- 新增知识库RAG、同步、调度、规范化和索引任务服务
- 重构orchestrator服务，增强运行时聊天功能
- 更新前端聊天、政策制度、设置等页面样式和逻辑
- 更新expense_claims和document_intelligence服务
- 删除llm_wiki相关服务和测试文件
- 更新docker-compose配置和启动脚本
This commit is contained in:
caoxiaozhu
2026-05-17 08:38:41 +00:00
parent 212c935308
commit 68f663f2f4
308 changed files with 83729 additions and 13588 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2847
server/src/app/services/agent_foundation.py
View File

File diff suppressed because it is too large Load Diff

93
server/src/app/services/document_intelligence.py
View File

@@ -9,8 +9,6 @@ from typing import Any
from pydantic import BaseModel, Field, ValidationError
from sqlalchemy.orm import Session
from app.services.runtime_chat import RuntimeChatService
@dataclass(frozen=True, slots=True)
class DocumentField:
@@ -198,7 +196,7 @@ MERCHANT_PATTERNS = (
class DocumentIntelligenceService:
def __init__(self, db: Session | None = None) -> None:
self.runtime_chat_service = RuntimeChatService(db) if db is not None else None
self.db = db
def build_document_insight(
self,
@@ -254,95 +252,6 @@ class DocumentIntelligenceService:
rule_insight: DocumentInsight,
fields: tuple[DocumentField, ...],
) -> tuple[str, LlmDocumentClassification] | None:
if self.runtime_chat_service is None:
return None
trimmed_text = text.strip()
if not trimmed_text and not summary.strip():
return None
facts = {
"filename": filename,
"summary": summary[:300],
"ocr_text_excerpt": trimmed_text[:2000],
"rule_candidate": {
"document_type": rule_insight.document_type,
"document_type_label": rule_insight.document_type_label,
"scene_code": rule_insight.scene_code,
"scene_label": rule_insight.scene_label,
"expense_type": rule_insight.expense_type,
"confidence": round(rule_insight.classification_confidence, 2),
"evidence": list(rule_insight.evidence),
},
"extracted_fields": [
{"key": field.key, "label": field.label, "value": field.value}
for field in fields
],
"allowed_document_types": list(SUPPORTED_DOCUMENT_TYPES),
}
system_prompt = (
"你是企业报销票据识别复核器。"
"你的任务不是 OCR，而是在已有 OCR 文本和票据预览基础上判断票据类型，并尽量复核关键字段。"
"只输出 JSON 对象，不要输出 Markdown、解释或代码块。"
"document_type 只能是："
f"{', '.join(SUPPORTED_DOCUMENT_TYPES)}。"
"如果证据不足，返回 other。"
"严禁编造 OCR 中不存在的商户、酒店、航司、路线或金额。"
"如果 OCR 出现冲突碎片，应优先依据票据主体信息，而不是单个噪声词。"
"例如滴滴行程单/网约车发票，即使 OCR 混入酒店名称，也不能直接判成酒店票据。"
"如果能从 OCR 或图片中明确确认字段，可在 fields 中返回。"
"fields 只允许包含 key, label, value，key 只能是 amount, date, merchant_name, invoice_number, "
"invoice_code, trip_no, route。无法确认就不要返回该字段。"
"输出字段：document_type, scene_code, scene_label, expense_type, confidence, evidence, fields。"
)
user_prompt = (
"请根据以下票据事实给出最终分类 JSON：\n"
f"{json.dumps(facts, ensure_ascii=False, indent=2)}\n\n"
"示例输出：\n"
"{\n"
' "document_type": "taxi_receipt",\n'
' "scene_code": "transport",\n'
' "scene_label": "交通票据",\n'
' "expense_type": "transport",\n'
' "confidence": 0.86,\n'
' "evidence": ["OCR 中出现 滴滴出行、订单号、上车/下车 等交通特征"],\n'
' "fields": [{"key": "amount", "label": "金额", "value": "32.5"}]\n'
"}"
)
if preview_data_url:
response_text = self.runtime_chat_service.complete(
[
{"role": "system", "content": system_prompt},
{
"role": "user",
"content": [
{"type": "text", "text": user_prompt},
{"type": "image_url", "image_url": {"url": preview_data_url}},
],
},
],
slot_priority=("vlm",),
max_tokens=320,
temperature=0.0,
)
parsed = self._parse_llm_payload(response_text)
if parsed is not None:
return "llm_vision", parsed
response_text = self.runtime_chat_service.complete(
[
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
slot_priority=("main", "backup"),
max_tokens=320,
temperature=0.0,
)
parsed = self._parse_llm_payload(response_text)
if parsed is not None:
return "llm_text", parsed
return None
@staticmethod

12
server/src/app/services/expense_claims.py
View File

@@ -682,11 +682,13 @@ class ExpenseClaimService:
raise ValueError("提交前请先补全信息：" + "；".join(missing_fields))
before_json = self._serialize_claim(claim)
review_result = self._run_ai_submission_review(claim)
claim.status = str(review_result["status"])
claim.approval_stage = str(review_result["approval_stage"])
claim.risk_flags_json = list(review_result["risk_flags"])
claim.submitted_at = datetime.now(UTC) if claim.status == "submitted" else None
# TODO: 后续恢复 AI 验审逻辑
# review_result = self._run_ai_submission_review(claim)
manager_name = self._resolve_claim_manager_name(claim) or "审批人"
claim.status = "submitted"
claim.approval_stage = "直属领导审批"
claim.risk_flags_json = list(claim.risk_flags_json or [])
claim.submitted_at = datetime.now(UTC)
self.db.commit()
self.db.refresh(claim)

124
server/src/app/services/hermes_callbacks.py
View File

@@ -1,124 +0,0 @@
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy.orm import Session
from app.core.agent_enums import AgentRunStatus
from app.schemas.hermes import HermesCallbackRead, HermesCallbackWrite
from app.services.agent_runs import AgentRunService
from app.services.knowledge import KNOWLEDGE_INGEST_STATUS_FAILED, KnowledgeService
from app.services.llm_wiki import LlmWikiService
class HermesCallbackService:
def __init__(self, db: Session) -> None:
self.db = db
self.run_service = AgentRunService(db)
def handle_callback(self, payload: HermesCallbackWrite) -> HermesCallbackRead:
run = self.run_service.get_run(payload.run_id)
if run is None:
raise LookupError("Hermes 回调引用的 AgentRun 不存在。")
if payload.type == "llm_wiki_sync":
self._handle_llm_wiki_sync(payload)
else:
raise ValueError(f"暂不支持的 Hermes 回调类型：{payload.type}")
return HermesCallbackRead(
type=payload.type,
run_id=payload.run_id,
status=payload.status,
)
def _handle_llm_wiki_sync(self, payload: HermesCallbackWrite) -> None:
run = self.run_service.get_run(payload.run_id)
if run is None:
raise LookupError("Hermes 回调引用的 AgentRun 不存在。")
route_json = dict(run.route_json or {})
document_ids = [
str(item).strip()
for item in list(route_json.get("requested_document_ids") or [])
if str(item).strip()
]
if payload.status == "running":
self.run_service.merge_route_json(
payload.run_id,
{
"phase": "running",
"heartbeat_at": datetime.now(UTC).isoformat(),
"callback_status": payload.status,
"callback_payload": payload.payload,
},
status=AgentRunStatus.RUNNING.value,
result_summary=payload.summary or run.result_summary,
)
return
if payload.status == "failed":
if document_ids:
KnowledgeService().set_document_ingest_statuses(
document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=payload.run_id,
)
self.run_service.record_tool_call(
run_id=payload.run_id,
tool_type="http",
tool_name="hermes_callback",
request_json=payload.model_dump(mode="json"),
response_json={},
status="failed",
duration_ms=0,
error_message=payload.error or payload.summary or "Hermes callback failed",
)
self.run_service.merge_route_json(
payload.run_id,
{
"phase": "failed",
"heartbeat_at": datetime.now(UTC).isoformat(),
"callback_status": payload.status,
"callback_payload": payload.payload,
},
status=AgentRunStatus.FAILED.value,
result_summary=payload.summary or payload.error or "Hermes 任务失败。",
error_message=payload.error or payload.summary or "Hermes 任务失败。",
finished_at=datetime.now(UTC),
)
return
result = LlmWikiService(self.db).finalize_agent_batch_callback(
agent_run_id=payload.run_id,
payload=payload.payload,
)
self.run_service.record_tool_call(
run_id=payload.run_id,
tool_type="http",
tool_name="hermes_callback",
request_json=payload.model_dump(mode="json"),
response_json=result.model_dump(mode="json"),
status="succeeded",
duration_ms=0,
)
self.run_service.merge_route_json(
payload.run_id,
{
"phase": "succeeded",
"heartbeat_at": datetime.now(UTC).isoformat(),
"callback_status": payload.status,
"sync_run_id": result.run_id,
"sync_result": result.model_dump(mode="json"),
"progress": {
"total_documents": len(document_ids),
"completed_documents": result.document_count,
"failed_documents": 0,
"skipped_documents": max(0, len(document_ids) - result.document_count),
"percent": 100,
},
},
status=AgentRunStatus.SUCCEEDED.value,
result_summary=payload.summary or result.summary,
finished_at=datetime.now(UTC),
)

1550
server/src/app/services/knowledge.py
View File

File diff suppressed because it is too large Load Diff

216
server/src/app/services/knowledge_index_tasks.py Normal file
View File

@@ -0,0 +1,216 @@
from __future__ import annotations
from concurrent.futures import Future, ThreadPoolExecutor
from datetime import UTC, datetime
from time import perf_counter
from typing import Any
from app.api.deps import CurrentUserContext
from app.core.agent_enums import AgentName, AgentRunStatus, AgentToolType
from app.core.logging import get_logger
from app.db.session import get_session_factory
from app.services.agent_runs import AgentRunService
from app.services.knowledge import (
KNOWLEDGE_INGEST_STATUS_FAILED,
KNOWLEDGE_INGEST_STATUS_INGESTED,
KnowledgeService,
)
from app.services.knowledge_rag import KnowledgeRagService
logger = get_logger("app.services.knowledge_index_tasks")
class KnowledgeIndexTaskManager:
def __init__(self) -> None:
self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="knowledge-index")
self._futures: dict[str, Future[Any]] = {}
def submit_sync(
self,
*,
agent_run_id: str,
folder: str,
current_user: CurrentUserContext,
document_ids: list[str],
force: bool,
) -> None:
future = self._executor.submit(
self._run_sync,
agent_run_id,
folder,
current_user,
[str(item).strip() for item in document_ids if str(item).strip()],
force,
)
self._futures[agent_run_id] = future
def shutdown(self) -> None:
self._executor.shutdown(wait=False, cancel_futures=True)
@staticmethod
def _run_sync(
agent_run_id: str,
folder: str,
current_user: CurrentUserContext,
document_ids: list[str],
force: bool,
) -> None:
session_factory = get_session_factory()
db = session_factory()
started = perf_counter()
try:
run_service = AgentRunService(db)
knowledge_service = KnowledgeService(db=db)
rag_service = KnowledgeRagService(db=db)
run_service.merge_route_json(
agent_run_id,
{
"job_type": "knowledge_index_sync",
"phase": "indexing",
"folder": folder,
"force": force,
"heartbeat_at": datetime.now(UTC).isoformat(),
"requested_document_ids": document_ids,
"requested_by_username": current_user.username,
"requested_by_name": current_user.name,
"progress": {
"total_documents": len(document_ids),
"completed_documents": 0,
"failed_documents": 0,
"skipped_documents": 0,
"percent": 10 if document_ids else 100,
},
},
)
response = rag_service.index_documents(document_ids=document_ids, force=force)
succeeded_document_ids = [
str(item).strip()
for item in list(response.get("succeeded_document_ids") or [])
if str(item).strip()
]
failed_documents = [
item
for item in list(response.get("failed_documents") or [])
if isinstance(item, dict)
]
failed_document_ids = [
str(item.get("document_id") or "").strip()
for item in failed_documents
if str(item.get("document_id") or "").strip()
]
if succeeded_document_ids:
knowledge_service.set_document_ingest_statuses(
succeeded_document_ids,
KNOWLEDGE_INGEST_STATUS_INGESTED,
agent_run_id=agent_run_id,
)
if failed_document_ids:
knowledge_service.set_document_ingest_statuses(
failed_document_ids,
KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=agent_run_id,
)
duration_ms = int((perf_counter() - started) * 1000)
tool_status = "succeeded" if not failed_document_ids else "failed"
run_service.record_tool_call(
run_id=agent_run_id,
tool_type=AgentToolType.LLM.value,
tool_name="lightrag.index_documents",
request_json={
"agent": AgentName.HERMES.value,
"folder": folder,
"document_ids": document_ids,
"force": force,
},
response_json=response,
status=tool_status,
duration_ms=duration_ms,
error_message=None if tool_status == "succeeded" else "部分文档索引失败。",
)
completed_documents = len(succeeded_document_ids)
failed_count = len(failed_document_ids)
total_documents = len(document_ids)
summary = (
f"LightRAG 已完成 {completed_documents}/{total_documents} 个知识文档索引。"
if failed_count == 0
else f"LightRAG 已完成 {completed_documents}/{total_documents} 个知识文档索引，失败 {failed_count} 个。"
)
run_service.merge_route_json(
agent_run_id,
{
"job_type": "knowledge_index_sync",
"phase": "completed",
"track_id": str(response.get("track_id") or "").strip(),
"heartbeat_at": datetime.now(UTC).isoformat(),
"progress": {
"total_documents": total_documents,
"completed_documents": completed_documents,
"failed_documents": failed_count,
"skipped_documents": 0,
"percent": 100,
},
},
status=(
AgentRunStatus.SUCCEEDED.value
if failed_count == 0
else AgentRunStatus.FAILED.value
),
result_summary=summary,
error_message="部分文档索引失败。" if failed_count else None,
finished_at=datetime.now(UTC),
)
except Exception as exc:
try:
AgentRunService(db).record_tool_call(
run_id=agent_run_id,
tool_type=AgentToolType.LLM.value,
tool_name="lightrag.index_documents",
request_json={
"agent": AgentName.HERMES.value,
"folder": folder,
"document_ids": document_ids,
"force": force,
},
response_json={"error": str(exc)},
status="failed",
duration_ms=int((perf_counter() - started) * 1000),
error_message=str(exc),
)
KnowledgeService(db=db).set_document_ingest_statuses(
document_ids,
KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=agent_run_id,
)
AgentRunService(db).merge_route_json(
agent_run_id,
{
"job_type": "knowledge_index_sync",
"phase": "failed",
"heartbeat_at": datetime.now(UTC).isoformat(),
"progress": {
"total_documents": len(document_ids),
"completed_documents": 0,
"failed_documents": len(document_ids),
"skipped_documents": 0,
"percent": 100,
},
},
status=AgentRunStatus.FAILED.value,
result_summary=str(exc),
error_message=str(exc),
finished_at=datetime.now(UTC),
)
except Exception:
logger.exception("Knowledge index task finalization failed run_id=%s", agent_run_id)
logger.exception("Knowledge index task failed run_id=%s", agent_run_id)
finally:
db.close()
knowledge_index_task_manager = KnowledgeIndexTaskManager()

414
server/src/app/services/knowledge_normalizer.py Normal file
View File

@@ -0,0 +1,414 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from sqlalchemy.orm import Session
from app.core.logging import get_logger
from app.services.runtime_chat import RuntimeChatService
logger = get_logger("app.services.knowledge_normalizer")
TABLE_MARKER_PATTERN = re.compile(r"表\s*(\d+)")
SECTION_HEADING_PATTERN = re.compile(
r"^(第[一二三四五六七八九十百零0-9]+[章节]\s*.*|[一二三四五六七八九十]+、.*|（[一二三四五六七八九十]+）.*|\([一二三四五六七八九十]+\).*)$"
)
LIST_ITEM_PATTERN = re.compile(r"^[-*•]\s+.+$")
NUMBERED_ITEM_PATTERN = re.compile(r"^(?:\d+[.)、]|[①②③④⑤⑥⑦⑧⑨⑩])\s*.+$")
ARTICLE_PATTERN = re.compile(r"^(第[一二三四五六七八九十百零0-9]+条)\s*.*$")
KEY_VALUE_PATTERN = re.compile(r"^[^：:\s][^：:]{0,40}[：:]\s*.+$")
MAX_TABLE_WINDOW_CHARS = 1800
MAX_TABLES_PER_DOCUMENT = 8
MAX_SECTION_OUTLINE_ITEMS = 12
MAX_SECTION_SNIPPETS = 8
MAX_SECTION_SNIPPET_CHARS = 220
MAX_SECTION_QA_CLUES = 4
MAX_TOTAL_QA_CLUES = 24
MAX_QA_CLUE_CHARS = 180
FACT_KEYWORDS = (
"适用",
"标准",
"条件",
"流程",
"审批",
"提交",
"附件",
"材料",
"票据",
"报销",
"限额",
"金额",
"比例",
"范围",
"对象",
"人员",
"时限",
"工作日",
"不得",
"可以",
"应当",
"应",
"需",
)
@dataclass(frozen=True, slots=True)
class TableCandidate:
title: str
excerpt: str
@dataclass(frozen=True, slots=True)
class SectionCandidate:
title: str
excerpt: str
body_lines: tuple[str, ...]
class KnowledgeNormalizationService:
def __init__(self, db: Session) -> None:
self.runtime_chat_service = RuntimeChatService(db)
def build_enriched_text(self, raw_text: str) -> str:
normalized_text = str(raw_text or "").strip()
if not normalized_text:
return ""
section_appendix = self._build_section_appendix(normalized_text)
answer_clue_appendix = self._build_answer_clue_appendix(normalized_text)
normalized_tables: list[str] = []
for candidate in self._extract_table_candidates(normalized_text):
rendered = self._normalize_table_candidate(candidate)
if rendered:
normalized_tables.append(f"## {candidate.title}\n\n{rendered}")
parts: list[str] = []
if section_appendix:
parts.append(section_appendix)
if answer_clue_appendix:
parts.append(answer_clue_appendix)
if normalized_tables:
appendix = "\n\n".join(normalized_tables)
parts.append(
"# 结构化表格补充\n\n"
"以下表格由知识归纳阶段依据原文重新整理，供问答检索时优先理解行列关系。\n\n"
f"{appendix}"
)
if not parts:
return normalized_text
parts.append(f"# 原文\n\n{normalized_text}")
return "\n\n".join(parts)
@staticmethod
def _extract_table_candidates(text: str) -> list[TableCandidate]:
candidates: list[TableCandidate] = []
occupied_ranges: list[tuple[int, int]] = []
for match in TABLE_MARKER_PATTERN.finditer(text):
if len(candidates) >= MAX_TABLES_PER_DOCUMENT:
break
start = text.rfind("\n", 0, match.start())
start = 0 if start < 0 else start + 1
end = min(len(text), start + MAX_TABLE_WINDOW_CHARS)
if any(start < existing_end and end > existing_start for existing_start, existing_end in occupied_ranges):
continue
excerpt = text[start:end].strip()
head = excerpt[:360]
if "单位：" not in head and "标准" not in head:
continue
if excerpt.count("\n") < 6 or sum(char.isdigit() for char in excerpt) < 4:
continue
marker = match.group(0).replace(" ", "")
first_line = next((line.strip() for line in excerpt.splitlines() if line.strip()), marker)
title = first_line if first_line.startswith(marker) else marker
candidates.append(TableCandidate(title=title, excerpt=excerpt))
occupied_ranges.append((start, end))
return candidates
def _normalize_table_candidate(self, candidate: TableCandidate) -> str:
messages = [
{
"role": "system",
"content": (
"你是制度文档结构化助手。"
"只依据用户提供的原文，提炼其中的表格为清晰 Markdown。"
"必须严格按照表头从左到右对齐每个数值，不能猜测、不能改列顺序、不能擅自补全。"
"只输出一张 Markdown 表格本身，不要输出标题、说明、注释、脚注或正文解释。"
"如果原文不足以确认表格关系，只回复“无法确认”。"
"不要输出思考过程，不要复述原文，不要添加制度之外的新事实。"
),
},
{
"role": "user",
"content": (
f"请仅整理下面这段制度表格，标题为《{candidate.title}》。\n\n"
f"{candidate.excerpt}"
),
},
]
answer = self.runtime_chat_service.complete(
messages,
max_tokens=900,
temperature=0.0,
)
cleaned = self._sanitize_answer(answer)
if not cleaned or cleaned == "无法确认":
return ""
if cleaned.count("|") < 6:
logger.info("Skip non-tabular normalization candidate title=%s", candidate.title)
return ""
return cleaned
@staticmethod
def _build_section_appendix(text: str) -> str:
candidates = KnowledgeNormalizationService._extract_section_candidates(text)
if len(candidates) < 2:
return ""
outline = "\n".join(
f"- {item.title}"
for item in candidates[:MAX_SECTION_OUTLINE_ITEMS]
)
snippets = "\n\n".join(
[
f"## {item.title}\n\n{item.excerpt}"
for item in candidates[:MAX_SECTION_SNIPPETS]
if item.excerpt
]
)
if not snippets:
return ""
return (
"# 章节导航\n\n"
"以下内容由入库阶段从制度原文中提取，供检索时优先理解制度层级、条目和标准所在章节。\n\n"
f"{outline}\n\n"
"# 重点章节摘录\n\n"
f"{snippets}"
)
@staticmethod
def _build_answer_clue_appendix(text: str) -> str:
candidates = KnowledgeNormalizationService._extract_section_candidates(text)
clue_lines: list[str] = []
if candidates:
for candidate in candidates:
clue_lines.extend(
KnowledgeNormalizationService._extract_section_clues(candidate)
)
else:
clue_lines.extend(KnowledgeNormalizationService._extract_freeform_clues(text))
deduped: list[str] = []
seen: set[str] = set()
for item in clue_lines:
normalized = re.sub(r"\s+", " ", str(item or "")).strip()
if not normalized or normalized in seen:
continue
seen.add(normalized)
deduped.append(normalized)
if len(deduped) >= MAX_TOTAL_QA_CLUES:
break
if len(deduped) < 2:
return ""
return (
"# 问答线索补充\n\n"
"以下内容由入库阶段根据章节标题、条款、列表、键值对与相邻正文提炼，"
"供问答检索时优先命中更短、更直接的制度依据。\n\n"
+ "\n".join(f"- {item}" for item in deduped)
)
@staticmethod
def _extract_section_candidates(text: str) -> list[SectionCandidate]:
lines = [line.rstrip() for line in str(text or "").splitlines()]
sections: list[SectionCandidate] = []
current_title = ""
current_body: list[str] = []
def flush() -> None:
nonlocal current_title, current_body
if not current_title:
current_body = []
return
excerpt = KnowledgeNormalizationService._build_section_excerpt(current_body)
if excerpt:
sections.append(
SectionCandidate(
title=current_title,
excerpt=excerpt,
body_lines=tuple(current_body),
)
)
current_title = ""
current_body = []
for raw_line in lines:
line = raw_line.strip()
if not line:
if current_body:
current_body.append("")
continue
if SECTION_HEADING_PATTERN.match(line) and len(line) <= 80:
flush()
current_title = line
continue
if current_title:
current_body.append(line)
flush()
return sections
@staticmethod
def _build_section_excerpt(lines: list[str]) -> str:
cleaned_lines = [line.strip() for line in lines if line.strip()]
if not cleaned_lines:
return ""
excerpt = "；".join(cleaned_lines[:3]).strip()
if len(excerpt) <= MAX_SECTION_SNIPPET_CHARS:
return excerpt
return f"{excerpt[: MAX_SECTION_SNIPPET_CHARS - 3].rstrip()}..."
@staticmethod
def _extract_section_clues(candidate: SectionCandidate) -> list[str]:
clues: list[str] = []
fallback: list[str] = []
for raw_line in candidate.body_lines:
normalized_line = KnowledgeNormalizationService._normalize_fact_line(raw_line)
if not normalized_line or KnowledgeNormalizationService._is_table_like_line(normalized_line):
continue
fact_units = KnowledgeNormalizationService._split_fact_units(normalized_line)
for unit in fact_units:
rendered = KnowledgeNormalizationService._render_clue(candidate.title, unit)
if not rendered:
continue
if KnowledgeNormalizationService._looks_like_fact_line(unit):
clues.append(rendered)
elif len(fallback) < 2:
fallback.append(rendered)
if len(clues) >= MAX_SECTION_QA_CLUES:
return clues[:MAX_SECTION_QA_CLUES]
return clues[:MAX_SECTION_QA_CLUES] or fallback[:2]
@staticmethod
def _extract_freeform_clues(text: str) -> list[str]:
clues: list[str] = []
for raw_line in str(text or "").splitlines():
normalized_line = KnowledgeNormalizationService._normalize_fact_line(raw_line)
if (
not normalized_line
or SECTION_HEADING_PATTERN.match(normalized_line)
or KnowledgeNormalizationService._is_table_like_line(normalized_line)
or not KnowledgeNormalizationService._looks_like_fact_line(normalized_line)
):
continue
for unit in KnowledgeNormalizationService._split_fact_units(normalized_line):
rendered = KnowledgeNormalizationService._render_clue("正文", unit)
if rendered:
clues.append(rendered)
if len(clues) >= MAX_TOTAL_QA_CLUES:
return clues
return clues
@staticmethod
def _split_fact_units(line: str) -> list[str]:
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
if not normalized:
return []
if len(normalized) <= MAX_QA_CLUE_CHARS and all(mark not in normalized for mark in ("；", ";", "。")):
return [normalized]
units: list[str] = []
for part in re.split(r"[；;。]\s*", normalized):
cleaned = KnowledgeNormalizationService._normalize_fact_line(part)
if not cleaned:
continue
units.append(cleaned)
return units or [KnowledgeNormalizationService._truncate_clue(normalized)]
@staticmethod
def _normalize_fact_line(line: str) -> str:
normalized = str(line or "").strip()
normalized = re.sub(r"\s+", " ", normalized)
return normalized.strip(" -")
@staticmethod
def _is_table_like_line(line: str) -> bool:
normalized = str(line or "").strip()
if not normalized:
return False
if normalized.count("|") >= 2:
return True
if normalized.count("\t") >= 2:
return True
number_tokens = re.findall(r"\d+(?:[.][0-9]+)?", normalized)
if len(number_tokens) >= 3 and len(normalized.split()) >= 4 and not any(
punct in normalized for punct in ("。", "；", ";", "：", ":")
):
return True
return "单位：" in normalized and sum(char.isdigit() for char in normalized) >= 3
@staticmethod
def _looks_like_fact_line(line: str) -> bool:
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
if len(normalized) < 6:
return False
if TABLE_MARKER_PATTERN.search(normalized) or normalized.startswith(("单位：", "单位:")):
return False
if (
ARTICLE_PATTERN.match(normalized)
or LIST_ITEM_PATTERN.match(normalized)
or NUMBERED_ITEM_PATTERN.match(normalized)
or KEY_VALUE_PATTERN.match(normalized)
):
return True
if any(keyword in normalized for keyword in FACT_KEYWORDS):
return True
return any(char.isdigit() for char in normalized)
@staticmethod
def _render_clue(section_title: str, line: str) -> str:
normalized_line = KnowledgeNormalizationService._truncate_clue(line)
if not normalized_line:
return ""
normalized_title = str(section_title or "").strip()
if not normalized_title:
return normalized_line
return f"{normalized_title}：{normalized_line}"
@staticmethod
def _truncate_clue(line: str) -> str:
normalized = KnowledgeNormalizationService._normalize_fact_line(line)
if len(normalized) <= MAX_QA_CLUE_CHARS:
return normalized
return f"{normalized[: MAX_QA_CLUE_CHARS - 3].rstrip()}..."
@staticmethod
def _sanitize_answer(answer: str | None) -> str:
cleaned = re.sub(r"<think>.*?</think>", "", str(answer or ""), flags=re.DOTALL | re.IGNORECASE)
lines = [line.rstrip() for line in cleaned.strip().splitlines()]
table_lines: list[str] = []
for line in lines:
normalized = line.strip()
if "|" not in normalized:
if table_lines:
break
continue
table_lines.append(normalized)
return "\n".join(table_lines).strip()

1261
server/src/app/services/knowledge_rag.py Normal file
View File

File diff suppressed because it is too large Load Diff

94
server/src/app/services/knowledge_scheduler.py Normal file
View File

@@ -0,0 +1,94 @@
from __future__ import annotations
import os
import threading
from datetime import datetime, time, timedelta
from zoneinfo import ZoneInfo
from app.api.deps import CurrentUserContext
from app.core.agent_enums import AgentRunSource
from app.core.logging import get_logger
from app.db.session import get_session_factory
from app.services.knowledge_sync import KnowledgeSyncDispatchService
logger = get_logger("app.services.knowledge_scheduler")
class KnowledgeIndexScheduler:
def __init__(self) -> None:
timezone_name = str(os.environ.get("X_FINANCIAL_SCHEDULER_TZ") or "Asia/Shanghai").strip() or "Asia/Shanghai"
self._timezone = ZoneInfo(timezone_name)
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
self._lock = threading.Lock()
def start(self) -> None:
with self._lock:
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
self._thread = threading.Thread(
target=self._run_loop,
name="knowledge-index-scheduler",
daemon=True,
)
self._thread.start()
logger.info("Knowledge index scheduler started timezone=%s trigger=00:00", self._timezone.key)
def shutdown(self) -> None:
with self._lock:
thread = self._thread
self._thread = None
self._stop_event.set()
if thread is not None and thread.is_alive():
thread.join(timeout=3)
logger.info("Knowledge index scheduler stopped")
def _run_loop(self) -> None:
while not self._stop_event.is_set():
now = datetime.now(self._timezone)
next_run = self._resolve_next_run(now)
wait_seconds = max(1.0, (next_run - now).total_seconds())
if self._stop_event.wait(wait_seconds):
break
try:
self._run_incremental_sync()
except Exception: # pragma: no cover - scheduler best effort logging
logger.exception("Scheduled knowledge index sync failed")
def _run_incremental_sync(self) -> None:
db = get_session_factory()()
try:
current_user = CurrentUserContext(
username="hermes",
name="Hermes",
role_codes=["manager"],
is_admin=True,
)
result = KnowledgeSyncDispatchService(db).queue_sync(
current_user=current_user,
folder=None,
document_ids=None,
source=AgentRunSource.SCHEDULE.value,
force=False,
changed_only=True,
)
logger.info(
"Scheduled knowledge index sync result run_id=%s docs=%s reused=%s summary=%s",
result.agent_run_id,
len(result.document_ids),
result.reused,
result.summary,
)
finally:
db.close()
@staticmethod
def _resolve_next_run(now: datetime) -> datetime:
today_midnight = datetime.combine(now.date(), time(hour=0, minute=0), tzinfo=now.tzinfo)
if now < today_midnight:
return today_midnight
return today_midnight + timedelta(days=1)
knowledge_index_scheduler = KnowledgeIndexScheduler()

244
server/src/app/services/knowledge_sync.py Normal file
View File

@@ -0,0 +1,244 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.api.deps import CurrentUserContext
from app.core.agent_enums import AgentName, AgentPermissionLevel, AgentRunSource, AgentRunStatus
from app.models.agent_asset import AgentAsset
from app.services.agent_runs import AgentRunService
from app.services.knowledge import (
KNOWLEDGE_INGEST_STATUS_FAILED,
KNOWLEDGE_INGEST_STATUS_SYNCING,
KnowledgeService,
)
from app.services.knowledge_index_tasks import knowledge_index_task_manager
ALL_KNOWLEDGE_FOLDERS_LABEL = "全部知识库"
@dataclass(slots=True)
class KnowledgeSyncDispatchResult:
ok: bool = True
agent_run_id: str = ""
folder: str = ""
document_ids: list[str] = field(default_factory=list)
queued_at: datetime = field(default_factory=lambda: datetime.now(UTC))
status: str = AgentRunStatus.SUCCEEDED.value
summary: str = ""
reused: bool = False
class KnowledgeSyncDispatchService:
def __init__(self, db: Session) -> None:
self.db = db
self.run_service = AgentRunService(db)
self.knowledge_service = KnowledgeService(db=db)
def queue_sync(
self,
*,
current_user: CurrentUserContext,
folder: str | None = None,
document_ids: list[str] | None = None,
source: str = AgentRunSource.USER_MESSAGE.value,
force: bool = False,
changed_only: bool = True,
) -> KnowledgeSyncDispatchResult:
normalized_folder = str(folder or "").strip() or None
folder_label = normalized_folder or ALL_KNOWLEDGE_FOLDERS_LABEL
normalized_requested_ids = [
str(item).strip()
for item in document_ids or []
if str(item).strip()
]
all_documents = self.knowledge_service.list_documents_for_ingest(
folder=normalized_folder,
document_ids=normalized_requested_ids,
changed_only=False,
)
target_documents = self.knowledge_service.list_documents_for_ingest(
folder=normalized_folder,
document_ids=normalized_requested_ids,
changed_only=(False if force else changed_only),
)
target_document_ids = [
str(item.get("id") or "").strip()
for item in target_documents
if str(item.get("id") or "").strip()
]
if not all_documents:
return KnowledgeSyncDispatchResult(
folder=folder_label,
document_ids=[],
status=AgentRunStatus.SUCCEEDED.value,
summary="当前目录暂无可归纳的知识文档。",
)
if not target_document_ids:
return KnowledgeSyncDispatchResult(
folder=folder_label,
document_ids=[],
status=AgentRunStatus.SUCCEEDED.value,
summary="当前目录没有需要增量归纳的文档。",
)
active_run = self._find_active_run(
folder=folder_label,
requested_document_ids=target_document_ids,
)
if active_run is not None:
active_document_ids = [
str(item).strip()
for item in list(active_run.route_json.get("requested_document_ids") or target_document_ids)
if str(item).strip()
]
return KnowledgeSyncDispatchResult(
agent_run_id=active_run.run_id,
folder=folder_label,
document_ids=active_document_ids,
queued_at=active_run.started_at,
status=active_run.status,
summary="已有知识归纳任务正在执行，系统已复用当前任务。",
reused=True,
)
task_asset = self.db.scalar(
select(AgentAsset).where(AgentAsset.code == "task.hermes.knowledge_index_sync")
)
run = self.run_service.create_run(
agent=AgentName.HERMES.value,
source=source,
user_id=current_user.username,
task_id=task_asset.id if task_asset is not None else None,
permission_level=AgentPermissionLevel.READ.value,
status=AgentRunStatus.RUNNING.value,
result_summary="知识归纳任务已入队，等待后台执行。",
route_json={
"job_type": "knowledge_index_sync",
"phase": "queued",
"folder": folder_label,
"force": force,
"changed_only": (False if force else changed_only),
"requested_document_ids": target_document_ids,
"requested_by_username": current_user.username,
"requested_by_name": current_user.name,
"progress": {
"total_documents": len(target_document_ids),
"completed_documents": 0,
"failed_documents": 0,
"skipped_documents": 0,
"percent": 0,
},
},
)
try:
self.knowledge_service.set_document_ingest_statuses(
target_document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_SYNCING,
agent_run_id=run.run_id,
)
knowledge_index_task_manager.submit_sync(
agent_run_id=run.run_id,
folder=folder_label,
current_user=current_user,
document_ids=target_document_ids,
force=force,
)
return KnowledgeSyncDispatchResult(
agent_run_id=run.run_id,
folder=folder_label,
document_ids=target_document_ids,
queued_at=run.started_at,
status=run.status,
summary="知识归纳任务已进入后台执行，可在日志管理中查看进度。",
)
except Exception as exc:
self.run_service.update_run(
run.run_id,
status=AgentRunStatus.FAILED.value,
error_message=str(exc),
result_summary=str(exc),
finished_at=datetime.now(UTC),
)
self.knowledge_service.set_document_ingest_statuses(
target_document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=run.run_id,
)
raise
def _find_active_run(
self,
*,
folder: str,
requested_document_ids: list[str],
):
requested_set = {str(item).strip() for item in requested_document_ids if str(item).strip()}
for item in self.run_service.list_runs(
agent=AgentName.HERMES.value,
status=AgentRunStatus.RUNNING.value,
limit=100,
):
if str(item.route_json.get("job_type") or "").strip() != "knowledge_index_sync":
continue
heartbeat_raw = str(item.route_json.get("heartbeat_at") or "").strip()
heartbeat_at = None
if heartbeat_raw:
try:
heartbeat_at = datetime.fromisoformat(heartbeat_raw)
except ValueError:
heartbeat_at = None
last_seen_at = heartbeat_at or item.started_at
if last_seen_at.tzinfo is None:
last_seen_at = last_seen_at.replace(tzinfo=UTC)
if datetime.now(UTC) - last_seen_at > timedelta(minutes=30):
stale_document_ids = [
str(document_id).strip()
for document_id in list(item.route_json.get("requested_document_ids") or [])
if str(document_id).strip()
]
if stale_document_ids:
self.knowledge_service.set_document_ingest_statuses(
stale_document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=item.run_id,
)
self.run_service.merge_route_json(
item.run_id,
{
"phase": "stale_failed",
"heartbeat_at": datetime.now(UTC).isoformat(),
},
status=AgentRunStatus.FAILED.value,
result_summary="知识归纳任务长时间无心跳，系统已自动标记失败。",
error_message="Knowledge index heartbeat timed out.",
finished_at=datetime.now(UTC),
)
continue
active_ids = {
str(document_id).strip()
for document_id in list(item.route_json.get("requested_document_ids") or [])
if str(document_id).strip()
}
active_folder = str(item.route_json.get("folder") or "").strip()
if active_folder == ALL_KNOWLEDGE_FOLDERS_LABEL:
if not requested_set or active_ids & requested_set:
return item
continue
if active_folder == folder:
if not requested_set or not active_ids or active_ids & requested_set:
return item
return None

2900
server/src/app/services/llm_wiki.py
View File

File diff suppressed because it is too large Load Diff

363
server/src/app/services/llm_wiki_tasks.py
View File

@@ -1,363 +0,0 @@
from __future__ import annotations
import threading
import time
import os
import signal
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from app.api.deps import CurrentUserContext
from app.core.agent_enums import AgentRunStatus
from app.core.logging import get_logger
from app.db.session import get_session_factory
from app.services.agent_runs import AgentRunService
from app.services.knowledge import KNOWLEDGE_INGEST_STATUS_FAILED, KnowledgeService
from app.services.llm_wiki import HERMES_AGENT_BATCH_TIMEOUT_SECONDS, LlmWikiService
logger = get_logger("app.services.llm_wiki_tasks")
class LlmWikiTaskManager:
def __init__(self) -> None:
self._lock = threading.RLock()
self._threads: dict[str, threading.Thread] = {}
def submit_sync(
self,
*,
agent_run_id: str,
folder: str,
current_user: CurrentUserContext,
document_ids: list[str] | None = None,
force: bool = False,
) -> None:
worker = threading.Thread(
target=self._run_sync,
kwargs={
"agent_run_id": agent_run_id,
"folder": folder,
"current_user": current_user,
"document_ids": list(document_ids or []),
"force": force,
},
daemon=True,
name=f"llm-wiki-sync-{agent_run_id}",
)
with self._lock:
self._threads[agent_run_id] = worker
worker.start()
def shutdown(self, *, timeout_seconds: float = 1.0) -> None:
with self._lock:
threads = list(self._threads.items())
self._threads.clear()
for _, worker in threads:
if worker.is_alive():
worker.join(timeout=timeout_seconds)
def _run_sync(
self,
*,
agent_run_id: str,
folder: str,
current_user: CurrentUserContext,
document_ids: list[str],
force: bool,
) -> None:
session_factory = get_session_factory()
db = session_factory()
run_service = AgentRunService(db)
knowledge_service = KnowledgeService()
request_payload = {
"folder": folder,
"document_ids": list(document_ids),
"force": force,
}
try:
run_service.merge_route_json(
agent_run_id,
{
"phase": "running",
"heartbeat_at": datetime.now(UTC).isoformat(),
"job_type": "llm_wiki_sync",
"folder": folder,
"force": force,
"requested_document_ids": list(document_ids),
"progress": {
"total_documents": len(document_ids),
"completed_documents": 0,
"failed_documents": 0,
"skipped_documents": 0,
"percent": 0,
},
},
status=AgentRunStatus.RUNNING.value,
result_summary="Hermes 后台归纳任务已启动。",
)
dispatch = LlmWikiService(db).dispatch_agent_batch(
folder=folder,
document_ids=document_ids,
force=force,
agent_run_id=agent_run_id,
)
if not dispatch.changed_document_ids:
knowledge_service.refresh_document_ingest_statuses(
document_ids=document_ids,
preserve_syncing=False,
)
run_service.record_tool_call(
run_id=agent_run_id,
tool_type="llm",
tool_name="system_hermes_llm_wiki_dispatch",
request_json=request_payload,
response_json={"changed_document_ids": [], "skipped_document_ids": dispatch.skipped_document_ids},
status="succeeded",
duration_ms=0,
)
run_service.merge_route_json(
agent_run_id,
{
"phase": "succeeded",
"heartbeat_at": datetime.now(UTC).isoformat(),
"progress": {
"total_documents": len(document_ids),
"completed_documents": 0,
"failed_documents": 0,
"skipped_documents": len(dispatch.skipped_document_ids),
"percent": 100,
},
},
status=AgentRunStatus.SUCCEEDED.value,
result_summary="本次所选文档均未变化，未重复派发 Hermes 任务。",
finished_at=datetime.now(UTC),
)
return
run_service.record_tool_call(
run_id=agent_run_id,
tool_type="llm",
tool_name="system_hermes_llm_wiki_dispatch",
request_json=request_payload,
response_json={
"changed_document_ids": dispatch.changed_document_ids,
"skipped_document_ids": dispatch.skipped_document_ids,
"process_id": dispatch.process_id,
},
status="succeeded",
duration_ms=0,
)
current_run = run_service.get_run(agent_run_id)
if current_run is not None and current_run.status in {
AgentRunStatus.SUCCEEDED.value,
AgentRunStatus.FAILED.value,
}:
return
run_service.merge_route_json(
agent_run_id,
{
"phase": "awaiting_callback",
"heartbeat_at": datetime.now(UTC).isoformat(),
"requested_document_ids": dispatch.changed_document_ids,
"skipped_document_ids": dispatch.skipped_document_ids,
"hermes_process_id": dispatch.process_id,
"hermes_stdout_path": dispatch.stdout_path,
"hermes_stderr_path": dispatch.stderr_path,
"progress": {
"total_documents": len(dispatch.changed_document_ids),
"completed_documents": 0,
"failed_documents": 0,
"skipped_documents": len(dispatch.skipped_document_ids),
"percent": 0,
},
},
status=AgentRunStatus.RUNNING.value,
result_summary="Hermes 任务已派发，等待 Agent 主动回调结果。",
)
self._start_process_monitor(
agent_run_id=agent_run_id,
document_ids=dispatch.changed_document_ids,
process_id=dispatch.process_id,
stderr_path=dispatch.stderr_path,
timeout_seconds=HERMES_AGENT_BATCH_TIMEOUT_SECONDS,
)
except Exception as exc:
logger.exception("Background LLM Wiki sync failed run_id=%s", agent_run_id)
if document_ids:
knowledge_service.set_document_ingest_statuses(
document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=agent_run_id,
)
run_service.record_tool_call(
run_id=agent_run_id,
tool_type="llm",
tool_name="system_hermes_llm_wiki_sync",
request_json=request_payload,
response_json={"error": str(exc)},
status="failed",
duration_ms=0,
error_message=str(exc),
)
run_service.merge_route_json(
agent_run_id,
{
"phase": "failed",
"heartbeat_at": datetime.now(UTC).isoformat(),
"progress": {
"total_documents": len(document_ids),
"completed_documents": 0,
"failed_documents": len(document_ids),
"skipped_documents": 0,
"percent": 100,
},
},
status=AgentRunStatus.FAILED.value,
result_summary=str(exc),
error_message=str(exc),
finished_at=datetime.now(UTC),
)
finally:
db.close()
with self._lock:
self._threads.pop(agent_run_id, None)
def _start_process_monitor(
self,
*,
agent_run_id: str,
document_ids: list[str],
process_id: int,
stderr_path: str,
timeout_seconds: int,
) -> None:
worker = threading.Thread(
target=self._monitor_process,
kwargs={
"agent_run_id": agent_run_id,
"document_ids": list(document_ids),
"process_id": process_id,
"stderr_path": stderr_path,
"timeout_seconds": timeout_seconds,
},
daemon=True,
name=f"llm-wiki-monitor-{agent_run_id}",
)
worker.start()
@staticmethod
def _monitor_process(
*,
agent_run_id: str,
document_ids: list[str],
process_id: int,
stderr_path: str,
timeout_seconds: int,
) -> None:
session_factory = get_session_factory()
db = session_factory()
run_service = AgentRunService(db)
knowledge_service = KnowledgeService()
started_at = time.monotonic()
try:
while True:
time.sleep(3)
run = run_service.get_run(agent_run_id)
if run is None or run.status != AgentRunStatus.RUNNING.value:
return
if time.monotonic() - started_at > timeout_seconds:
try:
os.killpg(process_id, signal.SIGTERM)
except OSError:
pass
error_message = LlmWikiTaskManager._read_process_error(stderr_path)
if document_ids:
knowledge_service.set_document_ingest_statuses(
document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=agent_run_id,
)
run_service.merge_route_json(
agent_run_id,
{
"phase": "failed",
"heartbeat_at": datetime.now(UTC).isoformat(),
"hermes_process_id": process_id,
},
status=AgentRunStatus.FAILED.value,
result_summary="Hermes 任务执行超时，已自动终止等待。",
error_message=error_message or "Hermes process exceeded callback timeout.",
finished_at=datetime.now(UTC),
)
return
if LlmWikiTaskManager._is_process_alive(process_id):
continue
error_message = LlmWikiTaskManager._read_process_error(stderr_path)
if document_ids:
knowledge_service.set_document_ingest_statuses(
document_ids,
status_code=KNOWLEDGE_INGEST_STATUS_FAILED,
agent_run_id=agent_run_id,
)
run_service.merge_route_json(
agent_run_id,
{
"phase": "failed",
"heartbeat_at": datetime.now(UTC).isoformat(),
"hermes_process_id": process_id,
},
status=AgentRunStatus.FAILED.value,
result_summary="Hermes 进程已退出且未回调结果。",
error_message=error_message or "Hermes process exited before callback.",
finished_at=datetime.now(UTC),
)
return
finally:
db.close()
@staticmethod
def _is_process_alive(process_id: int) -> bool:
stat_path = Path(f"/proc/{process_id}/stat")
if not stat_path.exists():
return False
try:
parts = stat_path.read_text(encoding="utf-8").split()
except OSError:
return False
return len(parts) > 2 and parts[2] != "Z"
@staticmethod
def _read_process_error(stderr_path: str) -> str:
path = Path(stderr_path)
if not stderr_path or not path.exists():
return ""
try:
content = path.read_text(encoding="utf-8", errors="replace").strip()
except OSError:
return ""
return content[-1000:]
@staticmethod
def _write_progress(
*,
run_service: AgentRunService,
agent_run_id: str,
payload: dict[str, Any],
summary: str,
) -> None:
patched_payload = dict(payload)
patched_payload["heartbeat_at"] = datetime.now(UTC).isoformat()
run_service.merge_route_json(
agent_run_id,
patched_payload,
status=AgentRunStatus.RUNNING.value,
result_summary=summary,
)
llm_wiki_task_manager = LlmWikiTaskManager()

51
server/src/app/services/model_connectivity.py
View File

@@ -59,9 +59,18 @@ def _probe_openai_compatible(payload: ModelConnectivityTestRequest) -> int:
normalized_endpoint = _normalize_endpoint(payload.endpoint)
headers = _build_headers(api_key=payload.api_key, use_bearer=True)
if payload.capability == "embedding":
if payload.capability == "reranker" and payload.provider == "Ali":
url, body = _build_ali_reranker_request(payload.model, normalized_endpoint)
elif payload.capability == "embedding":
url = _ensure_path(normalized_endpoint, "embeddings")
body = {"model": payload.model, "input": "connectivity test"}
elif payload.capability == "reranker":
url = _ensure_path(normalized_endpoint, "rerank")
body = {
"model": payload.model,
"query": "connectivity test",
"documents": ["sample document"],
}
else:
url = _ensure_path(normalized_endpoint, "chat/completions")
body = {
@@ -74,6 +83,35 @@ def _probe_openai_compatible(payload: ModelConnectivityTestRequest) -> int:
return status_code
def _build_ali_reranker_request(model: str, endpoint: str) -> tuple[str, dict[str, Any]]:
normalized_model = str(model or "").strip()
if normalized_model == "qwen3-rerank":
return (
"https://dashscope.aliyuncs.com/compatible-api/v1/reranks",
{
"model": normalized_model,
"query": "connectivity test",
"documents": ["sample document"],
"top_n": 1,
},
)
return (
"https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
{
"model": normalized_model,
"input": {
"query": "connectivity test",
"documents": ["sample document"],
},
"parameters": {
"return_documents": False,
"top_n": 1,
},
},
)
def _probe_ollama(payload: ModelConnectivityTestRequest) -> int:
normalized_endpoint = _normalize_endpoint(payload.endpoint)
headers = _build_headers(api_key=payload.api_key, use_bearer=False)
@@ -81,6 +119,8 @@ def _probe_ollama(payload: ModelConnectivityTestRequest) -> int:
if payload.capability == "embedding":
url = _ensure_path(normalized_endpoint, "api/embed")
body = {"model": payload.model, "input": "connectivity test"}
elif payload.capability == "reranker":
raise ConnectivityCheckError("Ollama 暂不支持 reranker 连通性探测。", status_code=HTTPStatus.BAD_REQUEST)
else:
url = _ensure_path(normalized_endpoint, "api/chat")
body = {
@@ -100,6 +140,12 @@ def _probe_azure_openai(payload: ModelConnectivityTestRequest) -> int:
if payload.capability == "embedding":
url = f"{deployment_base}/embeddings?api-version={AZURE_API_VERSION}"
body = {"input": "connectivity test"}
elif payload.capability == "reranker":
url = f"{deployment_base}/rerank?api-version={AZURE_API_VERSION}"
body = {
"query": "connectivity test",
"documents": ["sample document"],
}
else:
url = f"{deployment_base}/chat/completions?api-version={AZURE_API_VERSION}"
body = {
@@ -168,12 +214,13 @@ def _send_json_request(
*,
headers: dict[str, str],
payload: dict[str, Any],
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
) -> tuple[int, Any]:
data = json.dumps(payload).encode("utf-8")
request = Request(url=url, data=data, headers=headers, method=method)
try:
with urlopen(request, timeout=DEFAULT_TIMEOUT_SECONDS) as response:
with urlopen(request, timeout=timeout_seconds) as response:
body = response.read().decode("utf-8") if response.length != 0 else ""
return response.status, _parse_json_body(body)
except HTTPError as exc:

35
server/src/app/services/ontology.py
View File

@@ -243,6 +243,7 @@ STATUS_KEYWORDS = {
PRIVILEGED_ROLE_CODES = {"manager", "finance", "approver", "executive"}
CONTEXTUAL_SCENARIOS = {"expense", "accounts_receivable", "accounts_payable", "knowledge"}
KNOWLEDGE_INTENTS = {"query", "explain", "compare"}
@dataclass(slots=True)
@@ -356,7 +357,7 @@ class SemanticOntologyService:
rule_scenario = inferred_scenario
scenario_score = 0.18
if self._looks_like_expense_narrative(
if session_scenario != "knowledge" and self._looks_like_expense_narrative(
compact_query,
scenario=rule_scenario,
entities=entities,
@@ -371,7 +372,7 @@ class SemanticOntologyService:
entities=entities,
time_range=time_range,
)
if self._should_inherit_expense_draft(
if session_scenario != "knowledge" and self._should_inherit_expense_draft(
compact_query,
scenario=rule_scenario,
entities=entities,
@@ -384,17 +385,19 @@ class SemanticOntologyService:
intent_score = max(intent_score, 0.18)
metrics = self._extract_metrics(compact_query)
constraints = self._extract_constraints(compact_query, entities)
model_parse = self._parse_with_model(
payload=payload,
query=query,
compact_query=compact_query,
fallback_scenario=rule_scenario,
fallback_intent=rule_intent,
entities=entities,
time_range=time_range,
metrics=metrics,
constraints=constraints,
)
model_parse = None
if session_scenario != "knowledge":
model_parse = self._parse_with_model(
payload=payload,
query=query,
compact_query=compact_query,
fallback_scenario=rule_scenario,
fallback_intent=rule_intent,
entities=entities,
time_range=time_range,
metrics=metrics,
constraints=constraints,
)
scenario = self._resolve_scenario(rule_scenario, model_parse)
if session_scenario == "knowledge":
scenario = "knowledge"
@@ -968,6 +971,12 @@ class SemanticOntologyService:
model_parse: LlmOntologyParseResult | None,
) -> str:
candidate = model_parse.intent if model_parse is not None else fallback_intent
if scenario == "knowledge":
if candidate in KNOWLEDGE_INTENTS:
return candidate
if fallback_intent in KNOWLEDGE_INTENTS:
return fallback_intent
return "query"
if candidate == "query" and scenario == "expense":
if self._is_generic_expense_prompt(compact_query) or fallback_intent == "draft":
return "draft"

3058
server/src/app/services/orchestrator.py
View File

File diff suppressed because it is too large Load Diff

117
server/src/app/services/runtime_chat.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
from http import HTTPStatus
from time import monotonic, sleep
from typing import Any
from sqlalchemy.orm import Session
@@ -18,6 +19,12 @@ from app.services.model_connectivity import (
from app.services.settings import SettingsService
logger = get_logger("app.services.runtime_chat")
DEFAULT_RUNTIME_CHAT_TIMEOUT_SECONDS = 45
DEFAULT_RUNTIME_CHAT_RETRY_ATTEMPTS = 2
DEFAULT_RUNTIME_CHAT_RETRY_DELAY_SECONDS = 0.6
DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS = 90
_slot_failure_until: dict[str, float] = {}
class RuntimeChatService:
@@ -32,33 +39,71 @@ class RuntimeChatService:
slot_priority: tuple[str, ...] = ("main", "backup"),
max_tokens: int = 500,
temperature: float = 0.2,
timeout_seconds: int | None = None,
slot_timeouts: dict[str, int] | None = None,
max_attempts: int | None = None,
) -> str | None:
for slot in slot_priority:
config = self._load_chat_slot(slot)
if config is None:
continue
configs = [
config
for slot in slot_priority
if (config := self._load_chat_slot(slot)) is not None
]
resolved_timeout_seconds = timeout_seconds or DEFAULT_RUNTIME_CHAT_TIMEOUT_SECONDS
resolved_slot_timeouts = dict(slot_timeouts or {})
resolved_max_attempts = max_attempts or DEFAULT_RUNTIME_CHAT_RETRY_ATTEMPTS
try:
response_text = self._request_chat_completion(
config,
messages,
max_tokens=max_tokens,
temperature=temperature,
)
except Exception as exc:
logger.warning(
"Runtime chat request failed slot=%s provider=%s: %s",
slot,
config["provider"],
exc,
)
continue
if response_text:
return response_text.strip()
for attempt in range(1, resolved_max_attempts + 1):
for config in configs:
cache_key = self._build_slot_cache_key(config)
if _slot_failure_until.get(cache_key, 0.0) > monotonic():
logger.info(
"Skip runtime chat slot=%s provider=%s because it is in cooldown",
config["slot"],
config["provider"],
)
continue
try:
response_text = self._request_chat_completion(
config,
messages,
max_tokens=max_tokens,
temperature=temperature,
timeout_seconds=resolved_slot_timeouts.get(
config["slot"],
resolved_timeout_seconds,
),
)
if response_text:
_slot_failure_until.pop(cache_key, None)
return response_text.strip()
except Exception as exc:
_slot_failure_until[cache_key] = (
monotonic() + DEFAULT_RUNTIME_CHAT_FAILURE_COOLDOWN_SECONDS
)
logger.warning(
"Runtime chat request failed slot=%s provider=%s attempt=%s/%s: %s",
config["slot"],
config["provider"],
attempt,
resolved_max_attempts,
exc,
)
if attempt < resolved_max_attempts:
sleep(DEFAULT_RUNTIME_CHAT_RETRY_DELAY_SECONDS)
return None
@staticmethod
def _build_slot_cache_key(config: dict[str, str]) -> str:
return "|".join(
[
str(config.get("slot") or ""),
str(config.get("provider") or ""),
str(config.get("endpoint") or ""),
str(config.get("model") or ""),
]
)
def _load_chat_slot(self, slot: str) -> dict[str, str] | None:
try:
config = self.settings_service.get_runtime_model_config(slot)
@@ -95,6 +140,7 @@ class RuntimeChatService:
*,
max_tokens: int,
temperature: float,
timeout_seconds: int,
) -> str:
provider = config["provider"]
endpoint = config["endpoint"]
@@ -109,6 +155,7 @@ class RuntimeChatService:
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
timeout_seconds=timeout_seconds,
)
if provider == "Ollama":
@@ -119,38 +166,48 @@ class RuntimeChatService:
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
timeout_seconds=timeout_seconds,
)
return self._request_openai_compatible(
provider=provider,
endpoint=endpoint,
model=model,
api_key=api_key,
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
timeout_seconds=timeout_seconds,
)
def _request_openai_compatible(
self,
*,
provider: str,
endpoint: str,
model: str,
api_key: str,
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
timeout_seconds: int,
) -> str:
url = _ensure_path(_normalize_endpoint(endpoint), "chat/completions")
request_payload: dict[str, Any] = {
"model": model,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
}
if provider == "GLM":
request_payload["thinking"] = {"type": "disabled"}
status_code, payload = _send_json_request(
"POST",
url,
headers=_build_headers(api_key=api_key, use_bearer=True),
payload={
"model": model,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
},
payload=request_payload,
timeout_seconds=timeout_seconds,
)
if status_code >= HTTPStatus.BAD_REQUEST:
raise ConnectivityCheckError(
@@ -168,6 +225,7 @@ class RuntimeChatService:
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
timeout_seconds: int,
) -> str:
url = _ensure_path(_normalize_endpoint(endpoint), "api/chat")
status_code, payload = _send_json_request(
@@ -183,6 +241,7 @@ class RuntimeChatService:
"temperature": temperature,
},
},
timeout_seconds=timeout_seconds,
)
if status_code >= HTTPStatus.BAD_REQUEST:
raise ConnectivityCheckError(
@@ -200,6 +259,7 @@ class RuntimeChatService:
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
timeout_seconds: int,
) -> str:
deployment_base = _build_azure_deployment_base(endpoint, model)
url = f"{deployment_base}/chat/completions?api-version={AZURE_API_VERSION}"
@@ -212,6 +272,7 @@ class RuntimeChatService:
"max_tokens": max_tokens,
"temperature": temperature,
},
timeout_seconds=timeout_seconds,
)
if status_code >= HTTPStatus.BAD_REQUEST:
raise ConnectivityCheckError(

191
server/src/app/services/settings.py
View File

@@ -64,29 +64,29 @@ MODEL_SLOT_CONFIGS = {
capability="chat",
priority=20,
),
"vlm": ModelSlotConfig(
provider_attr="vlm_provider",
model_attr="vlm_model",
endpoint_attr="vlm_endpoint",
legacy_secret_attr="vlm_api_key_encrypted",
default_provider="Gemini",
default_model="gemini-2.5-flash",
default_endpoint="https://generativelanguage.googleapis.com/v1beta/openai/",
capability="chat",
priority=30,
),
"embedding": ModelSlotConfig(
provider_attr="embedding_provider",
model_attr="embedding_model",
endpoint_attr="embedding_endpoint",
legacy_secret_attr="embedding_api_key_encrypted",
default_provider="GLM",
default_model="Embedding-3",
default_endpoint="https://open.bigmodel.cn/api/paas/v4/",
capability="embedding",
priority=40,
),
}
"embedding": ModelSlotConfig(
provider_attr="embedding_provider",
model_attr="embedding_model",
endpoint_attr="embedding_endpoint",
legacy_secret_attr="embedding_api_key_encrypted",
default_provider="GLM",
default_model="Embedding-3",
default_endpoint="https://open.bigmodel.cn/api/paas/v4/",
capability="embedding",
priority=30,
),
"reranker": ModelSlotConfig(
provider_attr="reranker_provider",
model_attr="reranker_model",
endpoint_attr="reranker_endpoint",
legacy_secret_attr="reranker_api_key_encrypted",
default_provider="Ali",
default_model="gte-rerank-v2",
default_endpoint="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
capability="reranker",
priority=40,
),
}
@dataclass(slots=True)
@@ -138,6 +138,8 @@ class SettingsService:
if self._sync_onlyoffice_defaults(settings_row, secrets_row):
should_commit = True
if self._sync_reranker_legacy_defaults(settings_row, secrets_row):
should_commit = True
if should_commit:
self.db.commit()
@@ -150,13 +152,28 @@ class SettingsService:
self,
settings_row: SystemSetting,
secrets_row: SystemSettingSecret,
) -> dict[str, SystemModelSetting]:
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
should_commit = False
for slot, config in MODEL_SLOT_CONFIGS.items():
if slot in model_rows:
continue
) -> dict[str, SystemModelSetting]:
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
should_commit = False
if "reranker" not in model_rows and "vlm" in model_rows:
self.db.execute(
text(
"UPDATE system_model_settings "
"SET slot = 'reranker', capability = :capability, priority = :priority "
"WHERE slot = 'vlm'"
),
{
"capability": MODEL_SLOT_CONFIGS["reranker"].capability,
"priority": MODEL_SLOT_CONFIGS["reranker"].priority,
},
)
self.db.commit()
model_rows = {row.slot: row for row in self.repository.get_model_settings()}
for slot, config in MODEL_SLOT_CONFIGS.items():
if slot in model_rows:
continue
model_row = SystemModelSetting(
slot=slot,
@@ -224,13 +241,6 @@ class SettingsService:
payload.llmForm.backupEndpoint,
payload.llmForm.backupApiKey,
)
self._apply_model_setting(
model_rows["vlm"],
payload.llmForm.vlmProvider,
payload.llmForm.vlmModel,
payload.llmForm.vlmEndpoint,
payload.llmForm.vlmApiKey,
)
self._apply_model_setting(
model_rows["embedding"],
payload.llmForm.embeddingProvider,
@@ -238,6 +248,13 @@ class SettingsService:
payload.llmForm.embeddingEndpoint,
payload.llmForm.embeddingApiKey,
)
self._apply_model_setting(
model_rows["reranker"],
payload.llmForm.rerankerProvider,
payload.llmForm.rerankerModel,
payload.llmForm.rerankerEndpoint,
payload.llmForm.rerankerApiKey,
)
if payload.renderForm.enabled and not payload.renderForm.publicUrl:
raise ValueError("启用 ONLYOFFICE 时必须配置服务地址。")
@@ -252,14 +269,14 @@ class SettingsService:
settings_row.main_model = model_rows["main"].model_name
settings_row.main_endpoint = model_rows["main"].endpoint
settings_row.backup_provider = model_rows["backup"].provider
settings_row.backup_model = model_rows["backup"].model_name
settings_row.backup_endpoint = model_rows["backup"].endpoint
settings_row.vlm_provider = model_rows["vlm"].provider
settings_row.vlm_model = model_rows["vlm"].model_name
settings_row.vlm_endpoint = model_rows["vlm"].endpoint
settings_row.backup_model = model_rows["backup"].model_name
settings_row.backup_endpoint = model_rows["backup"].endpoint
settings_row.embedding_provider = model_rows["embedding"].provider
settings_row.embedding_model = model_rows["embedding"].model_name
settings_row.embedding_endpoint = model_rows["embedding"].endpoint
settings_row.reranker_provider = model_rows["reranker"].provider
settings_row.reranker_model = model_rows["reranker"].model_name
settings_row.reranker_endpoint = model_rows["reranker"].endpoint
settings_row.onlyoffice_enabled = payload.renderForm.enabled
settings_row.onlyoffice_public_url = payload.renderForm.publicUrl
@@ -428,7 +445,7 @@ class SettingsService:
legacy_admin = read_admin_secret() or {}
admin_account = str(legacy_admin.get("username", "")).strip() or "superadmin"
return SystemSetting(
return SystemSetting(
id=SETTINGS_ROW_ID,
company_name=company_name,
display_name=company_name,
@@ -445,16 +462,16 @@ class SettingsService:
login_alert_enabled=True,
main_provider="Codex",
main_model="codex-mini-latest",
main_endpoint="https://api.openai.com/v1",
backup_provider="GLM",
backup_model="glm-5.1",
main_endpoint="https://api.openai.com/v1",
backup_provider="GLM",
backup_model="glm-5.1",
backup_endpoint="https://open.bigmodel.cn/api/paas/v4/",
vlm_provider="Gemini",
vlm_model="gemini-2.5-flash",
vlm_endpoint="https://generativelanguage.googleapis.com/v1beta/openai/",
embedding_provider="GLM",
embedding_model="Embedding-3",
embedding_endpoint="https://open.bigmodel.cn/api/paas/v4/",
reranker_provider="Ali",
reranker_model="gte-rerank-v2",
reranker_endpoint="https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
onlyoffice_enabled=bool(self.runtime_settings.onlyoffice_enabled),
onlyoffice_public_url=str(self.runtime_settings.onlyoffice_public_url or "").strip(),
log_level="INFO",
@@ -542,6 +559,19 @@ class SettingsService:
migration_statements.append(
"ALTER TABLE system_settings ADD COLUMN onlyoffice_public_url VARCHAR(512) DEFAULT ''"
)
if "reranker_provider" not in settings_columns:
migration_statements.append(
"ALTER TABLE system_settings ADD COLUMN reranker_provider VARCHAR(64) DEFAULT 'Ali'"
)
if "reranker_model" not in settings_columns:
migration_statements.append(
"ALTER TABLE system_settings ADD COLUMN reranker_model VARCHAR(255) DEFAULT 'gte-rerank-v2'"
)
if "reranker_endpoint" not in settings_columns:
migration_statements.append(
"ALTER TABLE system_settings ADD COLUMN reranker_endpoint "
"VARCHAR(512) DEFAULT 'https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank'"
)
if "system_setting_secrets" in table_names:
secret_columns = {column["name"] for column in inspector.get_columns("system_setting_secrets")}
@@ -549,6 +579,10 @@ class SettingsService:
migration_statements.append(
"ALTER TABLE system_setting_secrets ADD COLUMN onlyoffice_jwt_secret_encrypted TEXT DEFAULT ''"
)
if "reranker_api_key_encrypted" not in secret_columns:
migration_statements.append(
"ALTER TABLE system_setting_secrets ADD COLUMN reranker_api_key_encrypted TEXT DEFAULT ''"
)
for statement in migration_statements:
self.db.execute(text(statement))
@@ -583,16 +617,41 @@ class SettingsService:
return should_commit
@staticmethod
def _sync_reranker_legacy_defaults(
settings_row: SystemSetting,
secrets_row: SystemSettingSecret,
) -> bool:
should_commit = False
if not str(settings_row.reranker_provider or "").strip() and str(settings_row.vlm_provider or "").strip():
settings_row.reranker_provider = settings_row.vlm_provider
should_commit = True
if not str(settings_row.reranker_model or "").strip() and str(settings_row.vlm_model or "").strip():
settings_row.reranker_model = settings_row.vlm_model
should_commit = True
if not str(settings_row.reranker_endpoint or "").strip() and str(settings_row.vlm_endpoint or "").strip():
settings_row.reranker_endpoint = settings_row.vlm_endpoint
should_commit = True
if (
not str(secrets_row.reranker_api_key_encrypted or "").strip()
and str(secrets_row.vlm_api_key_encrypted or "").strip()
):
secrets_row.reranker_api_key_encrypted = secrets_row.vlm_api_key_encrypted
should_commit = True
return should_commit
@staticmethod
def _serialize(
settings_row: SystemSetting,
secrets_row: SystemSettingSecret,
model_rows: dict[str, SystemModelSetting],
) -> SettingsRead:
main_model = model_rows["main"]
backup_model = model_rows["backup"]
vlm_model = model_rows["vlm"]
embedding_model = model_rows["embedding"]
model_rows: dict[str, SystemModelSetting],
) -> SettingsRead:
main_model = model_rows["main"]
backup_model = model_rows["backup"]
embedding_model = model_rows["embedding"]
reranker_model = model_rows["reranker"]
return SettingsRead(
companyForm={
@@ -624,20 +683,20 @@ class SettingsService:
"mainApiKey": "",
"mainApiKeyConfigured": bool(main_model.api_key_encrypted),
"backupProvider": backup_model.provider,
"backupModel": backup_model.model_name,
"backupEndpoint": backup_model.endpoint,
"backupApiKey": "",
"backupApiKeyConfigured": bool(backup_model.api_key_encrypted),
"vlmProvider": vlm_model.provider,
"vlmModel": vlm_model.model_name,
"vlmEndpoint": vlm_model.endpoint,
"vlmApiKey": "",
"vlmApiKeyConfigured": bool(vlm_model.api_key_encrypted),
"embeddingProvider": embedding_model.provider,
"embeddingModel": embedding_model.model_name,
"backupModel": backup_model.model_name,
"backupEndpoint": backup_model.endpoint,
"backupApiKey": "",
"backupApiKeyConfigured": bool(backup_model.api_key_encrypted),
"embeddingProvider": embedding_model.provider,
"embeddingModel": embedding_model.model_name,
"embeddingEndpoint": embedding_model.endpoint,
"embeddingApiKey": "",
"embeddingApiKeyConfigured": bool(embedding_model.api_key_encrypted),
"rerankerProvider": reranker_model.provider,
"rerankerModel": reranker_model.model_name,
"rerankerEndpoint": reranker_model.endpoint,
"rerankerApiKey": "",
"rerankerApiKeyConfigured": bool(reranker_model.api_key_encrypted),
},
renderForm={
"enabled": settings_row.onlyoffice_enabled,

1021
server/src/app/services/user_agent.py
View File

File diff suppressed because it is too large Load Diff