fix(data_process): 修复 Word 切分行号断档与预览标题缺失

- 正文抽取下钻 SDT 内容控件,目录等内容不再整段丢失
- 切片投影匹配剥离序列化插入的列表自动编号,新增行锚点兜底与游标防回退
- fixed/semantic 切分路径定位失败时保留切片,不再静默丢弃
- Word 预览按段落大纲级别识别标题,未套标题样式的小节正常渲染
- 本地嵌入模型抽为共享单例,供语义分块与质量评分共用
This commit is contained in:
caoxiaozhu
2026-08-19 14:21:41 +08:00
parent 93449b7f0e
commit f47611020a
6 changed files with 315 additions and 34 deletions

View File

@@ -7,12 +7,13 @@ import re
import unicodedata
import zipfile
import xml.etree.ElementTree as ET
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from pathlib import PurePosixPath
from typing import Any
from urllib.parse import unquote, urlsplit
from docx import Document
from docx.oxml.ns import qn
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
@@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
except zipfile.BadZipFile as exc:
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
def iter_document_blocks(parent: Any) -> Iterator[Any]:
"""按文档顺序产出正文段落与表格,并下钻 SDT 内容控件。
Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里,只遍历 body
直接子级会把这些段落整段丢掉。
"""
for child in parent.iterchildren():
if isinstance(child, (CT_P, CT_Tbl)):
yield child
elif child.tag == qn("w:sdt"):
content = child.find(qn("w:sdtContent"))
if content is not None:
yield from iter_document_blocks(content)
def _extract_docx_text(raw: bytes) -> str:
_validate_office_archive(raw, "docx")
try:
@@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str:
parts: list[str] = []
total = 0
for child in document.element.body.iterchildren():
for child in iter_document_blocks(document.element.body):
if isinstance(child, CT_P):
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
continue