fix(data_process): 修复 Word 切分行号断档与预览标题缺失
- 正文抽取下钻 SDT 内容控件,目录等内容不再整段丢失 - 切片投影匹配剥离序列化插入的列表自动编号,新增行锚点兜底与游标防回退 - fixed/semantic 切分路径定位失败时保留切片,不再静默丢弃 - Word 预览按段落大纲级别识别标题,未套标题样式的小节正常渲染 - 本地嵌入模型抽为共享单例,供语义分块与质量评分共用
This commit is contained in:
24
backend/app/modules/data_process/algorithms/embedding.py
Normal file
24
backend/app/modules/data_process/algorithms/embedding.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""数据处理算法 - 本地语义嵌入模型共享单例。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def semantic_embedding_model() -> Any:
|
||||
"""加载本地嵌入模型,供语义分块与语义质量评分共用。
|
||||
|
||||
模型可在部署环境覆盖;默认模型体积较小且适合中英文语义判断。
|
||||
返回 LlamaIndex BaseEmbedding,通过 ``get_text_embedding`` 使用。
|
||||
"""
|
||||
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
|
||||
return HuggingFaceEmbedding(
|
||||
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
|
||||
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
|
||||
trust_remote_code=False,
|
||||
)
|
||||
Reference in New Issue
Block a user