feat: 新增风险规则生成引擎与知识图谱可视化
后端新增风险规则自动生成和模板执行服务,支持从规则资产 批量生成并持久化风险规则文件;知识库入库日志增强图谱 查询和本地 RAG 回退,前端审计页面增加风险规则模型和流 程图组件,知识入库面板拆分为图谱可视化子组件,报销创 建页面增加引导式流程模型,更新知识库索引数据。
This commit is contained in:
@@ -1,15 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree
|
||||
|
||||
MAX_INGEST_LOG_CHUNKS = 24
|
||||
MAX_INGEST_LOG_ENTITIES = 24
|
||||
MAX_INGEST_LOG_ENTITY_CHUNKS = 48
|
||||
MAX_INGEST_LOG_RELATIONS = 24
|
||||
MAX_INGEST_LOG_SECTIONS = 12
|
||||
MAX_INGEST_LOG_TEXT_PREVIEW = 180
|
||||
MAX_INGEST_LOG_ENTITY_DESCRIPTIONS = 5
|
||||
GRAPHML_NAMESPACE = {"graphml": "http://graphml.graphdrawing.org/xmlns"}
|
||||
GRAPH_PROPERTY_SEPARATOR = "<SEP>"
|
||||
|
||||
INGEST_SECTION_HEADING_PATTERN = re.compile(
|
||||
r"^(?:#{1,4}\s+.+|第[一二三四五六七八九十百零0-9]+[章节条]\s*.*)$"
|
||||
@@ -42,6 +48,7 @@ def build_ingest_document_summary(
|
||||
"entity_count": 0,
|
||||
"relation_count": 0,
|
||||
"entities": [],
|
||||
"entity_chunks": [],
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
@@ -62,6 +69,33 @@ def build_ingest_status_summary(
|
||||
}
|
||||
|
||||
|
||||
def enrich_knowledge_ingest_route_json(
|
||||
route_json: dict[str, Any],
|
||||
*,
|
||||
storage_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
if not isinstance(route_json, dict):
|
||||
return route_json
|
||||
ingest = route_json.get("knowledge_ingest")
|
||||
if not isinstance(ingest, dict):
|
||||
return route_json
|
||||
graph = ingest.get("graph")
|
||||
if not isinstance(graph, dict):
|
||||
return route_json
|
||||
|
||||
workspace = _resolve_lightrag_workspace(route_json)
|
||||
graph_snapshot = _load_lightrag_graph_snapshot(storage_root, workspace=workspace)
|
||||
if not graph_snapshot["entities"] and not graph_snapshot["relations"]:
|
||||
return route_json
|
||||
|
||||
next_route = dict(route_json)
|
||||
next_ingest = dict(ingest)
|
||||
next_graph = _enrich_graph_payload(graph, graph_snapshot)
|
||||
next_ingest["graph"] = next_graph
|
||||
next_route["knowledge_ingest"] = next_ingest
|
||||
return next_route
|
||||
|
||||
|
||||
def build_document_graph_summary(
|
||||
storage_root: Path,
|
||||
*,
|
||||
@@ -74,19 +108,264 @@ def build_document_graph_summary(
|
||||
entities_payload = _load_json_file(workspace_dir / "kv_store_full_entities.json")
|
||||
relations_payload = _load_json_file(workspace_dir / "kv_store_full_relations.json")
|
||||
chunks_payload = _load_json_file(workspace_dir / "kv_store_text_chunks.json")
|
||||
entity_chunks_payload = _load_json_file(workspace_dir / "kv_store_entity_chunks.json")
|
||||
graph_snapshot = _load_lightrag_graph_snapshot(storage_root, workspace=workspace)
|
||||
|
||||
entities = _normalize_document_entities(entities_payload, document_id)
|
||||
relations = _normalize_document_relations(relations_payload, document_id)
|
||||
chunks = _normalize_document_chunks(chunks_payload, document_id)
|
||||
entity_chunks = _normalize_document_entity_chunks(
|
||||
entity_chunks_payload,
|
||||
entities,
|
||||
chunk_ids={str(item.get("id") or "").strip() for item in chunks},
|
||||
)
|
||||
return {
|
||||
"entity_count": len(entities),
|
||||
"relation_count": len(relations),
|
||||
"entities": entities[:MAX_INGEST_LOG_ENTITIES],
|
||||
"relations": relations[:MAX_INGEST_LOG_RELATIONS],
|
||||
"entities": _enrich_entity_list(entities, graph_snapshot)[:MAX_INGEST_LOG_ENTITIES],
|
||||
"relations": _enrich_relation_list(relations, graph_snapshot)[:MAX_INGEST_LOG_RELATIONS],
|
||||
"chunks": chunks[:MAX_INGEST_LOG_CHUNKS],
|
||||
"entity_chunks": entity_chunks[:MAX_INGEST_LOG_ENTITY_CHUNKS],
|
||||
}
|
||||
|
||||
|
||||
def _resolve_lightrag_workspace(route_json: dict[str, Any]) -> str:
|
||||
explicit_workspace = str(
|
||||
route_json.get("lightrag_workspace") or route_json.get("workspace") or ""
|
||||
).strip()
|
||||
if explicit_workspace:
|
||||
return explicit_workspace
|
||||
return os.environ.get("LIGHTRAG_WORKSPACE", "x_financial_knowledge").strip() or "x_financial_knowledge"
|
||||
|
||||
|
||||
def _enrich_graph_payload(
|
||||
graph: dict[str, Any],
|
||||
graph_snapshot: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
next_graph = dict(graph)
|
||||
relation_items = _extract_relation_items(graph.get("relations"))
|
||||
relation_entity_names = [
|
||||
name
|
||||
for relation in relation_items
|
||||
for name in (relation.get("source"), relation.get("target"))
|
||||
]
|
||||
next_graph["entities"] = _enrich_entity_list(
|
||||
_dedupe_text_items(
|
||||
_extract_entity_names(graph.get("entities")) + relation_entity_names
|
||||
),
|
||||
graph_snapshot,
|
||||
)
|
||||
next_graph["relations"] = _enrich_relation_list(relation_items, graph_snapshot)
|
||||
return next_graph
|
||||
|
||||
|
||||
def _enrich_entity_list(
|
||||
entity_names: list[str],
|
||||
graph_snapshot: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
graph_entities = graph_snapshot.get("entities") or {}
|
||||
return [
|
||||
graph_entities.get(entity_name)
|
||||
or {
|
||||
"name": entity_name,
|
||||
"type": "实体",
|
||||
"description": "",
|
||||
"descriptions": [],
|
||||
"properties": {},
|
||||
}
|
||||
for entity_name in entity_names
|
||||
]
|
||||
|
||||
|
||||
def _enrich_relation_list(
|
||||
relations: list[dict[str, Any]],
|
||||
graph_snapshot: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
graph_relations = graph_snapshot.get("relations") or {}
|
||||
enriched_relations: list[dict[str, Any]] = []
|
||||
for relation in relations:
|
||||
source = str(relation.get("source") or "").strip()
|
||||
target = str(relation.get("target") or "").strip()
|
||||
relation_type = str(relation.get("type") or "关联").strip()
|
||||
graph_relation = (
|
||||
graph_relations.get((source, target))
|
||||
or graph_relations.get((target, source))
|
||||
or {}
|
||||
)
|
||||
enriched_relations.append(
|
||||
{
|
||||
**relation,
|
||||
"source": source,
|
||||
"target": target,
|
||||
"type": relation_type,
|
||||
"description": graph_relation.get("description", ""),
|
||||
"keywords": graph_relation.get("keywords", []),
|
||||
"weight": graph_relation.get("weight", relation.get("weight", 1)),
|
||||
"properties": graph_relation.get("properties", {}),
|
||||
}
|
||||
)
|
||||
return enriched_relations
|
||||
|
||||
|
||||
def _load_lightrag_graph_snapshot(storage_root: Path, *, workspace: str) -> dict[str, Any]:
|
||||
graphml_path = (
|
||||
Path(storage_root)
|
||||
/ "knowledge"
|
||||
/ ".lightrag"
|
||||
/ str(workspace).strip()
|
||||
/ "graph_chunk_entity_relation.graphml"
|
||||
)
|
||||
if not graphml_path.exists():
|
||||
return {"entities": {}, "relations": {}}
|
||||
|
||||
try:
|
||||
root = ElementTree.parse(graphml_path).getroot()
|
||||
except (ElementTree.ParseError, OSError):
|
||||
return {"entities": {}, "relations": {}}
|
||||
|
||||
key_names = {
|
||||
str(key.attrib.get("id") or ""): str(key.attrib.get("attr.name") or "")
|
||||
for key in root.findall("graphml:key", GRAPHML_NAMESPACE)
|
||||
}
|
||||
return {
|
||||
"entities": _load_graphml_entities(root, key_names),
|
||||
"relations": _load_graphml_relations(root, key_names),
|
||||
}
|
||||
|
||||
|
||||
def _load_graphml_entities(
|
||||
root: ElementTree.Element,
|
||||
key_names: dict[str, str],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
entities: dict[str, dict[str, Any]] = {}
|
||||
for node in root.findall(".//graphml:node", GRAPHML_NAMESPACE):
|
||||
properties = _read_graphml_data(node, key_names)
|
||||
name = str(properties.get("entity_id") or node.attrib.get("id") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
descriptions = _split_graph_property(properties.get("description"))
|
||||
visible_properties = _filter_graph_properties(properties)
|
||||
entities[name] = {
|
||||
"name": name,
|
||||
"type": str(properties.get("entity_type") or "实体").strip(),
|
||||
"description": descriptions[0] if descriptions else "",
|
||||
"descriptions": descriptions[:MAX_INGEST_LOG_ENTITY_DESCRIPTIONS],
|
||||
"properties": visible_properties,
|
||||
}
|
||||
return entities
|
||||
|
||||
|
||||
def _load_graphml_relations(
|
||||
root: ElementTree.Element,
|
||||
key_names: dict[str, str],
|
||||
) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
relations: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for edge in root.findall(".//graphml:edge", GRAPHML_NAMESPACE):
|
||||
source = str(edge.attrib.get("source") or "").strip()
|
||||
target = str(edge.attrib.get("target") or "").strip()
|
||||
if not source or not target:
|
||||
continue
|
||||
properties = _read_graphml_data(edge, key_names)
|
||||
description_parts = _split_graph_property(properties.get("description"))
|
||||
relations[(source, target)] = {
|
||||
"description": "; ".join(description_parts[:2]),
|
||||
"keywords": _split_graph_keywords(properties.get("keywords"))[:6],
|
||||
"weight": _to_float(properties.get("weight"), default=1.0),
|
||||
"properties": _filter_graph_properties(properties),
|
||||
}
|
||||
return relations
|
||||
|
||||
|
||||
def _read_graphml_data(
|
||||
element: ElementTree.Element,
|
||||
key_names: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
data: dict[str, str] = {}
|
||||
for item in element.findall("graphml:data", GRAPHML_NAMESPACE):
|
||||
key = str(item.attrib.get("key") or "")
|
||||
name = key_names.get(key) or key
|
||||
if not name:
|
||||
continue
|
||||
data[name] = str(item.text or "").strip()
|
||||
return data
|
||||
|
||||
|
||||
def _split_graph_property(value: Any) -> list[str]:
|
||||
return [
|
||||
_truncate_text(part, max_length=MAX_INGEST_LOG_TEXT_PREVIEW)
|
||||
for part in str(value or "").split(GRAPH_PROPERTY_SEPARATOR)
|
||||
if str(part or "").strip()
|
||||
]
|
||||
|
||||
|
||||
def _split_graph_keywords(value: Any) -> list[str]:
|
||||
keywords: list[str] = []
|
||||
for part in str(value or "").split(GRAPH_PROPERTY_SEPARATOR):
|
||||
keywords.extend(part.split(","))
|
||||
return [
|
||||
_truncate_text(keyword, max_length=60)
|
||||
for keyword in keywords
|
||||
if str(keyword or "").strip()
|
||||
]
|
||||
|
||||
|
||||
def _filter_graph_properties(properties: dict[str, Any]) -> dict[str, Any]:
|
||||
hidden_fields = {
|
||||
"source_id",
|
||||
"file_path",
|
||||
"truncate",
|
||||
"description",
|
||||
"keywords",
|
||||
}
|
||||
return {
|
||||
key: value
|
||||
for key, value in properties.items()
|
||||
if key not in hidden_fields and str(value or "").strip()
|
||||
}
|
||||
|
||||
|
||||
def _extract_entity_names(raw_entities: Any) -> list[str]:
|
||||
if not isinstance(raw_entities, list):
|
||||
return []
|
||||
names: list[str] = []
|
||||
for entity in raw_entities:
|
||||
if isinstance(entity, dict):
|
||||
name = str(
|
||||
entity.get("name")
|
||||
or entity.get("entity")
|
||||
or entity.get("entity_id")
|
||||
or entity.get("id")
|
||||
or ""
|
||||
).strip()
|
||||
else:
|
||||
name = str(entity or "").strip()
|
||||
if name:
|
||||
names.append(name)
|
||||
return _dedupe_text_items(names)
|
||||
|
||||
|
||||
def _extract_relation_items(raw_relations: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(raw_relations, list):
|
||||
return []
|
||||
relations: list[dict[str, Any]] = []
|
||||
for relation in raw_relations:
|
||||
if not isinstance(relation, dict):
|
||||
continue
|
||||
source = str(relation.get("source") or relation.get("from") or "").strip()
|
||||
target = str(relation.get("target") or relation.get("to") or "").strip()
|
||||
if not source or not target:
|
||||
continue
|
||||
relations.append(
|
||||
{
|
||||
**relation,
|
||||
"source": source,
|
||||
"target": target,
|
||||
"type": str(relation.get("type") or "关联").strip(),
|
||||
}
|
||||
)
|
||||
return relations
|
||||
|
||||
|
||||
def _extract_ingest_sections(text: str) -> list[dict[str, str]]:
|
||||
sections: list[dict[str, str]] = []
|
||||
lines = [line.strip() for line in str(text or "").splitlines()]
|
||||
@@ -187,11 +466,46 @@ def _normalize_document_chunks(payload: dict[str, Any], document_id: str) -> lis
|
||||
"order": _to_int(raw_chunk.get("chunk_order_index")),
|
||||
"tokens": _to_int(raw_chunk.get("tokens")),
|
||||
"summary": _build_chunk_summary(content),
|
||||
"excerpt": _truncate_text(
|
||||
content,
|
||||
max_length=MAX_INGEST_LOG_TEXT_PREVIEW,
|
||||
),
|
||||
}
|
||||
)
|
||||
return sorted(chunks, key=lambda item: (item["order"], item["id"]))
|
||||
|
||||
|
||||
def _normalize_document_entity_chunks(
|
||||
payload: dict[str, Any],
|
||||
entities: list[str],
|
||||
*,
|
||||
chunk_ids: set[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
if not entities or not chunk_ids:
|
||||
return []
|
||||
|
||||
entity_chunks: list[dict[str, Any]] = []
|
||||
for entity in entities:
|
||||
raw_entry = payload.get(entity) if isinstance(payload, dict) else {}
|
||||
raw_chunk_ids = raw_entry.get("chunk_ids") if isinstance(raw_entry, dict) else []
|
||||
if not isinstance(raw_chunk_ids, list):
|
||||
continue
|
||||
matched_chunk_ids = [
|
||||
str(item or "").strip()
|
||||
for item in raw_chunk_ids
|
||||
if str(item or "").strip() in chunk_ids
|
||||
]
|
||||
if not matched_chunk_ids:
|
||||
continue
|
||||
entity_chunks.append(
|
||||
{
|
||||
"entity": entity,
|
||||
"chunk_ids": _dedupe_text_items(matched_chunk_ids),
|
||||
}
|
||||
)
|
||||
return entity_chunks
|
||||
|
||||
|
||||
def _build_chunk_summary(content: str) -> str:
|
||||
lines = [line.strip() for line in str(content or "").splitlines() if line.strip()]
|
||||
text = next((line for line in lines if len(line) >= 12), lines[0] if lines else "")
|
||||
@@ -217,6 +531,13 @@ def _to_int(value: Any) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _to_float(value: Any, *, default: float = 0.0) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _truncate_text(text: str, *, max_length: int) -> str:
|
||||
normalized = " ".join(str(text or "").split()).strip()
|
||||
if len(normalized) <= max_length:
|
||||
|
||||
Reference in New Issue
Block a user