fix(data_process): 修复 Word 切分行号断档与预览标题缺失

- 正文抽取下钻 SDT 内容控件，目录等内容不再整段丢失 - 切片投影匹配剥离序列化插入的列表自动编号，新增行锚点兜底与游标防回退 - fixed/semantic 切分路径定位失败时保留切片，不再静默丢弃 - Word 预览按段落大纲级别识别标题，未套标题样式的小节正常渲染 - 本地嵌入模型抽为共享单例，供语义分块与质量评分共用
2026-08-19 14:21:41 +08:00
parent 93449b7f0e
commit f47611020a
6 changed files with 315 additions and 34 deletions
--- a/backend/app/modules/data_process/document_chunking.py
+++ b/backend/app/modules/data_process/document_chunking.py
@@ -18,12 +18,19 @@ from llama_index.core.base.embeddings.base import BaseEmbedding
 from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter

 from app.modules.data_process.algorithms import normalize_text
+from app.modules.data_process.algorithms.embedding import semantic_embedding_model

 ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]

 _PAGE_FURNITURE = re.compile(
    r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$"
 )
+# Docling 的 markdown 序列化会给列表项补上自动编号，而 Word 的编号存放在
+# numbering.xml 中，python-docx 抽取的正文不含这些编号；紧凑匹配前剥掉
+# 行首编号，否则带列表的切片会整体定位失败。
+_LIST_MARKER_PREFIX = re.compile(
+    r"(?m)^[ \t>]*(?:(?:\d{1,3}[.)])+|\([a-zA-Z0-9]{1,3}\)|[a-zA-Z][.)]|[-*+•·])[ \t]+"
+)
 _COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
 _CONVERTER_LOCK = threading.Lock()

@@ -149,18 +156,6 @@ def chunk_fixed_text(
    return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)


-@lru_cache(maxsize=1)
-def _semantic_embedding_model() -> BaseEmbedding:
-    # 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。
-    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
-
-    return HuggingFaceEmbedding(
-        model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
-        device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
-        trust_remote_code=False,
-    )
-
-
 def chunk_semantic_text(
    text: str,
    *,
@@ -175,7 +170,7 @@ def chunk_semantic_text(
    if not normalized:
        return []
    splitter = SemanticSplitterNodeParser.from_defaults(
-        embed_model=embed_model or _semantic_embedding_model(),
+        embed_model=embed_model or semantic_embedding_model(),
        breakpoint_percentile_threshold=breakpoint_percentile_threshold,
        buffer_size=1,
        sentence_splitter=_sentence_chunks,
@@ -193,6 +188,7 @@ def chunk_semantic_text(
        if start is None:
            start = _locate_text(normalized, content, 0)
        if start is None:
+            result.append(_unlocated_chunk(content))
            continue
        if len(_tokenizer().encode(content)) <= chunk_size:
            result.append(_make_text_chunk(normalized, start, start + len(content)))
@@ -203,6 +199,7 @@ def chunk_semantic_text(
                chunk_overlap=chunk_overlap,
            ):
                if child.source_start is None or child.source_end is None:
+                    result.append(_unlocated_chunk(child.original_content))
                    continue
                result.append(
                    _make_text_chunk(
@@ -235,6 +232,7 @@ def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
            if start is None:
                start = _locate_text(source_text, content, 0)
        if start is None:
+            chunks.append(_unlocated_chunk(content))
            continue
        end = start + len(content)
        chunks.append(_make_text_chunk(source_text, start, end))
@@ -247,6 +245,20 @@ def _locate_text(source: str, content: str, start: int) -> int | None:
    return position if position >= 0 else None


+def _unlocated_chunk(content: str) -> DocumentChunk:
+    """正文在源文本中定位失败时保底保留切片，只放弃行号信息。"""
+
+    return DocumentChunk(
+        original_content=content,
+        contextualized_content=content,
+        source_start=None,
+        source_end=None,
+        source_start_line=None,
+        source_end_line=None,
+        token_count=len(_tokenizer().encode(content)),
+    )
+
+
 def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
    content = source[start:end]
    return DocumentChunk(
@@ -316,6 +328,14 @@ def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
    return "".join(compact), offsets


+def _expand_to_line_boundaries(source_text: str, start: int, end: int) -> tuple[int, int]:
+    while start > 0 and source_text[start - 1] not in "\r\n":
+        start -= 1
+    while end < len(source_text) and source_text[end] not in "\r\n":
+        end += 1
+    return start, end
+
+
 def _project_layout_span(
    source_text: str,
    content: str,
@@ -324,21 +344,79 @@ def _project_layout_span(
    source_offsets: list[int],
    compact_start: int,
 ) -> tuple[int | None, int | None, int]:
-    compact_content, _ = _compact_with_offsets(content)
-    if len(compact_content) < 4:
+    for candidate in (content, _LIST_MARKER_PREFIX.sub("", content)):
+        compact_content, _ = _compact_with_offsets(candidate)
+        if len(compact_content) < 4:
+            continue
+        position = compact_source.find(compact_content, compact_start)
+        if position < 0:
+            position = compact_source.find(compact_content)
+        if position < 0:
+            continue
+        start, end = _expand_to_line_boundaries(
+            source_text,
+            source_offsets[position],
+            source_offsets[position + len(compact_content) - 1] + 1,
+        )
+        # 重复内容回退匹配可能命中已消费的更早位置，游标只进不退，
+        # 避免后续切片跟着错位。
+        return start, end, max(compact_start, position + len(compact_content))
+    return _project_layout_span_by_anchors(
+        source_text,
+        content,
+        compact_source=compact_source,
+        source_offsets=source_offsets,
+        compact_start=compact_start,
+    )
+
+
+def _project_layout_span_by_anchors(
+    source_text: str,
+    content: str,
+    *,
+    compact_source: str,
+    source_offsets: list[int],
+    compact_start: int,
+) -> tuple[int | None, int | None, int]:
+    """按行锚点顺序匹配，容忍切片里插入的重复表头等非连续内容。"""
+
+    segments = [
+        compact
+        for compact in (
+            _compact_with_offsets(line)[0]
+            for line in _LIST_MARKER_PREFIX.sub("", content).split("\n")
+        )
+        if len(compact) >= 6
+    ]
+    if not segments:
        return None, None, compact_start
-    position = compact_source.find(compact_content, compact_start)
-    if position < 0:
-        position = compact_source.find(compact_content)
-    if position < 0:
+    total = sum(len(segment) for segment in segments)
+
+    def match_from(cursor: int) -> tuple[list[tuple[int, int]], int]:
+        matched: list[tuple[int, int]] = []
+        position = cursor
+        for segment in segments:
+            found = compact_source.find(segment, position)
+            if found < 0:
+                continue
+            matched.append((found, found + len(segment)))
+            position = found + len(segment)
+        return matched, sum(end - start for start, end in matched)
+
+    matched, covered = match_from(compact_start)
+    if covered * 2 < total:
+        retried, retry_covered = match_from(0)
+        if retry_covered > covered:
+            matched, covered = retried, retry_covered
+    # 覆盖不足一半时宁可不定位，也不能给出错误的行号。
+    if not matched or covered * 2 < total:
        return None, None, compact_start
-    start = source_offsets[position]
-    end = source_offsets[position + len(compact_content) - 1] + 1
-    while start > 0 and source_text[start - 1] not in "\r\n":
-        start -= 1
-    while end < len(source_text) and source_text[end] not in "\r\n":
-        end += 1
-    return start, end, position + len(compact_content)
+    start, end = _expand_to_line_boundaries(
+        source_text,
+        source_offsets[matched[0][0]],
+        source_offsets[matched[-1][1] - 1] + 1,
+    )
+    return start, end, max(compact_start, matched[-1][1])


 def chunk_layout_document(