fix(data_process): 修复 Word 切分行号断档与预览标题缺失

- 正文抽取下钻 SDT 内容控件，目录等内容不再整段丢失
- 切片投影匹配剥离序列化插入的列表自动编号，新增行锚点兜底与游标防回退
- fixed/semantic 切分路径定位失败时保留切片，不再静默丢弃
- Word 预览按段落大纲级别识别标题，未套标题样式的小节正常渲染
- 本地嵌入模型抽为共享单例，供语义分块与质量评分共用

backend/app/modules/data_process/algorithms/embedding.py

@@ -0,0 +1,24 @@
+"""数据处理算法 - 本地语义嵌入模型共享单例。"""
+
+from __future__ import annotations
+
+import os
+from functools import lru_cache
+from typing import Any
+
+
+@lru_cache(maxsize=1)
+def semantic_embedding_model() -> Any:
+    """加载本地嵌入模型，供语义分块与语义质量评分共用。
+
+    模型可在部署环境覆盖；默认模型体积较小且适合中英文语义判断。
+    返回 LlamaIndex BaseEmbedding，通过 ``get_text_embedding`` 使用。
+    """
+
+    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+
+    return HuggingFaceEmbedding(
+        model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
+        device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
+        trust_remote_code=False,
+    )

backend/app/modules/data_process/algorithms/parsers/office.py

@@ -7,12 +7,13 @@ import re
 import unicodedata
 import zipfile
 import xml.etree.ElementTree as ET
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from pathlib import PurePosixPath
 from typing import Any
 from urllib.parse import unquote, urlsplit
 
 from docx import Document
+from docx.oxml.ns import qn
 from docx.oxml.table import CT_Tbl
 from docx.oxml.text.paragraph import CT_P
 from docx.table import Table
@@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
     except zipfile.BadZipFile as exc:
         raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
 
+def iter_document_blocks(parent: Any) -> Iterator[Any]:
+    """按文档顺序产出正文段落与表格，并下钻 SDT 内容控件。
+
+    Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里，只遍历 body
+    直接子级会把这些段落整段丢掉。
+    """
+
+    for child in parent.iterchildren():
+        if isinstance(child, (CT_P, CT_Tbl)):
+            yield child
+        elif child.tag == qn("w:sdt"):
+            content = child.find(qn("w:sdtContent"))
+            if content is not None:
+                yield from iter_document_blocks(content)
+
 def _extract_docx_text(raw: bytes) -> str:
     _validate_office_archive(raw, "docx")
     try:
@@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str:
 
     parts: list[str] = []
     total = 0
-    for child in document.element.body.iterchildren():
+    for child in iter_document_blocks(document.element.body):
         if isinstance(child, CT_P):
             total = _append_bounded_text(parts, Paragraph(child, document).text, total)
             continue

backend/app/modules/data_process/document_chunking.py

@@ -18,12 +18,19 @@ from llama_index.core.base.embeddings.base import BaseEmbedding
 from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
 
 from app.modules.data_process.algorithms import normalize_text
+from app.modules.data_process.algorithms.embedding import semantic_embedding_model
 
 ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
 
 _PAGE_FURNITURE = re.compile(
     r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$"
 )
+# Docling 的 markdown 序列化会给列表项补上自动编号，而 Word 的编号存放在
+# numbering.xml 中，python-docx 抽取的正文不含这些编号；紧凑匹配前剥掉
+# 行首编号，否则带列表的切片会整体定位失败。
+_LIST_MARKER_PREFIX = re.compile(
+    r"(?m)^[ \t>]*(?:(?:\d{1,3}[.)])+|\([a-zA-Z0-9]{1,3}\)|[a-zA-Z][.)]|[-*+•·])[ \t]+"
+)
 _COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
 _CONVERTER_LOCK = threading.Lock()
 
@@ -149,18 +156,6 @@ def chunk_fixed_text(
     return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
 
-@lru_cache(maxsize=1)
-def _semantic_embedding_model() -> BaseEmbedding:
-    # 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。
-    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
-
-    return HuggingFaceEmbedding(
-        model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
-        device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
-        trust_remote_code=False,
-    )
-
-
 def chunk_semantic_text(
     text: str,
     *,
@@ -175,7 +170,7 @@ def chunk_semantic_text(
     if not normalized:
         return []
     splitter = SemanticSplitterNodeParser.from_defaults(
-        embed_model=embed_model or _semantic_embedding_model(),
+        embed_model=embed_model or semantic_embedding_model(),
         breakpoint_percentile_threshold=breakpoint_percentile_threshold,
         buffer_size=1,
         sentence_splitter=_sentence_chunks,
@@ -193,6 +188,7 @@ def chunk_semantic_text(
         if start is None:
             start = _locate_text(normalized, content, 0)
         if start is None:
+            result.append(_unlocated_chunk(content))
             continue
         if len(_tokenizer().encode(content)) <= chunk_size:
             result.append(_make_text_chunk(normalized, start, start + len(content)))
@@ -203,6 +199,7 @@ def chunk_semantic_text(
                 chunk_overlap=chunk_overlap,
             ):
                 if child.source_start is None or child.source_end is None:
+                    result.append(_unlocated_chunk(child.original_content))
                     continue
                 result.append(
                     _make_text_chunk(
@@ -235,6 +232,7 @@ def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
             if start is None:
                 start = _locate_text(source_text, content, 0)
         if start is None:
+            chunks.append(_unlocated_chunk(content))
             continue
         end = start + len(content)
         chunks.append(_make_text_chunk(source_text, start, end))
@@ -247,6 +245,20 @@ def _locate_text(source: str, content: str, start: int) -> int | None:
     return position if position >= 0 else None
 
 
+def _unlocated_chunk(content: str) -> DocumentChunk:
+    """正文在源文本中定位失败时保底保留切片，只放弃行号信息。"""
+
+    return DocumentChunk(
+        original_content=content,
+        contextualized_content=content,
+        source_start=None,
+        source_end=None,
+        source_start_line=None,
+        source_end_line=None,
+        token_count=len(_tokenizer().encode(content)),
+    )
+
+
 def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
     content = source[start:end]
     return DocumentChunk(
@@ -316,6 +328,14 @@ def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
     return "".join(compact), offsets
 
 
+def _expand_to_line_boundaries(source_text: str, start: int, end: int) -> tuple[int, int]:
+    while start > 0 and source_text[start - 1] not in "\r\n":
+        start -= 1
+    while end < len(source_text) and source_text[end] not in "\r\n":
+        end += 1
+    return start, end
+
+
 def _project_layout_span(
     source_text: str,
     content: str,
@@ -324,21 +344,79 @@ def _project_layout_span(
     source_offsets: list[int],
     compact_start: int,
 ) -> tuple[int | None, int | None, int]:
-    compact_content, _ = _compact_with_offsets(content)
-    if len(compact_content) < 4:
+    for candidate in (content, _LIST_MARKER_PREFIX.sub("", content)):
+        compact_content, _ = _compact_with_offsets(candidate)
+        if len(compact_content) < 4:
+            continue
+        position = compact_source.find(compact_content, compact_start)
+        if position < 0:
+            position = compact_source.find(compact_content)
+        if position < 0:
+            continue
+        start, end = _expand_to_line_boundaries(
+            source_text,
+            source_offsets[position],
+            source_offsets[position + len(compact_content) - 1] + 1,
+        )
+        # 重复内容回退匹配可能命中已消费的更早位置，游标只进不退，
+        # 避免后续切片跟着错位。
+        return start, end, max(compact_start, position + len(compact_content))
+    return _project_layout_span_by_anchors(
+        source_text,
+        content,
+        compact_source=compact_source,
+        source_offsets=source_offsets,
+        compact_start=compact_start,
+    )
+
+
+def _project_layout_span_by_anchors(
+    source_text: str,
+    content: str,
+    *,
+    compact_source: str,
+    source_offsets: list[int],
+    compact_start: int,
+) -> tuple[int | None, int | None, int]:
+    """按行锚点顺序匹配，容忍切片里插入的重复表头等非连续内容。"""
+
+    segments = [
+        compact
+        for compact in (
+            _compact_with_offsets(line)[0]
+            for line in _LIST_MARKER_PREFIX.sub("", content).split("\n")
+        )
+        if len(compact) >= 6
+    ]
+    if not segments:
         return None, None, compact_start
-    position = compact_source.find(compact_content, compact_start)
-    if position < 0:
-        position = compact_source.find(compact_content)
-    if position < 0:
+    total = sum(len(segment) for segment in segments)
+
+    def match_from(cursor: int) -> tuple[list[tuple[int, int]], int]:
+        matched: list[tuple[int, int]] = []
+        position = cursor
+        for segment in segments:
+            found = compact_source.find(segment, position)
+            if found < 0:
+                continue
+            matched.append((found, found + len(segment)))
+            position = found + len(segment)
+        return matched, sum(end - start for start, end in matched)
+
+    matched, covered = match_from(compact_start)
+    if covered * 2 < total:
+        retried, retry_covered = match_from(0)
+        if retry_covered > covered:
+            matched, covered = retried, retry_covered
+    # 覆盖不足一半时宁可不定位，也不能给出错误的行号。
+    if not matched or covered * 2 < total:
         return None, None, compact_start
-    start = source_offsets[position]
-    end = source_offsets[position + len(compact_content) - 1] + 1
-    while start > 0 and source_text[start - 1] not in "\r\n":
-        start -= 1
-    while end < len(source_text) and source_text[end] not in "\r\n":
-        end += 1
-    return start, end, position + len(compact_content)
+    start, end = _expand_to_line_boundaries(
+        source_text,
+        source_offsets[matched[0][0]],
+        source_offsets[matched[-1][1] - 1] + 1,
+    )
+    return start, end, max(compact_start, matched[-1][1])
 
 
 def chunk_layout_document(

backend/app/modules/data_process/office_preview.py

@@ -11,6 +11,7 @@ import re
 from typing import Any
 
 from docx import Document
+from docx.oxml.ns import qn
 from docx.oxml.table import CT_Tbl
 from docx.oxml.text.paragraph import CT_P
 from docx.table import Table
@@ -27,6 +28,7 @@ from app.modules.data_process.algorithms import (
     _xlsx_sheet_merge_ranges,
     normalize_text,
 )
+from app.modules.data_process.algorithms.parsers.office import iter_document_blocks
 
 MAX_DOCX_PREVIEW_BLOCKS = 2_000
 MAX_XLSX_PREVIEW_ROWS = 200
@@ -49,12 +51,21 @@ def _docx_alignment(paragraph: Paragraph) -> str:
 
 def _docx_heading_level(paragraph: Paragraph) -> int | None:
     style = paragraph.style
-    if style is None:
-        return None
-    style_name = str(style.name or "")
-    style_id = str(style.style_id or "")
+    style_name = str(style.name or "") if style is not None else ""
+    style_id = str(style.style_id or "") if style is not None else ""
     match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
-    return int(match.group(1)) if match else None
+    if match:
+        return int(match.group(1))
+    # Word 的目录和导航窗格依据大纲级别识别标题；未套标题样式但带
+    # outlineLvl 的段落（如手工排版的编号小节）同样是标题。
+    outline = paragraph._p.find(f"{qn('w:pPr')}/{qn('w:outlineLvl')}")
+    if outline is not None:
+        value = outline.get(qn("w:val"))
+        if value is not None and value.isdigit():
+            level = int(value)
+            if 0 <= level <= 5:
+                return level + 1
+    return None
 
 
 def build_docx_preview(raw: bytes) -> dict[str, Any]:
@@ -84,7 +95,7 @@ def build_docx_preview(raw: bytes) -> dict[str, Any]:
         has_source_content = True
         return text, start, source_cursor
 
-    for child in document.element.body.iterchildren():
+    for child in iter_document_blocks(document.element.body):
         if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
             truncated = True
             break