fix(data_process): 修复 Word 切分行号断档与预览标题缺失
- 正文抽取下钻 SDT 内容控件,目录等内容不再整段丢失 - 切片投影匹配剥离序列化插入的列表自动编号,新增行锚点兜底与游标防回退 - fixed/semantic 切分路径定位失败时保留切片,不再静默丢弃 - Word 预览按段落大纲级别识别标题,未套标题样式的小节正常渲染 - 本地嵌入模型抽为共享单例,供语义分块与质量评分共用
This commit is contained in:
@@ -7,12 +7,13 @@ import re
|
||||
import unicodedata
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from docx import Document
|
||||
from docx.oxml.ns import qn
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
@@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
|
||||
|
||||
def iter_document_blocks(parent: Any) -> Iterator[Any]:
|
||||
"""按文档顺序产出正文段落与表格,并下钻 SDT 内容控件。
|
||||
|
||||
Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里,只遍历 body
|
||||
直接子级会把这些段落整段丢掉。
|
||||
"""
|
||||
|
||||
for child in parent.iterchildren():
|
||||
if isinstance(child, (CT_P, CT_Tbl)):
|
||||
yield child
|
||||
elif child.tag == qn("w:sdt"):
|
||||
content = child.find(qn("w:sdtContent"))
|
||||
if content is not None:
|
||||
yield from iter_document_blocks(content)
|
||||
|
||||
def _extract_docx_text(raw: bytes) -> str:
|
||||
_validate_office_archive(raw, "docx")
|
||||
try:
|
||||
@@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str:
|
||||
|
||||
parts: list[str] = []
|
||||
total = 0
|
||||
for child in document.element.body.iterchildren():
|
||||
for child in iter_document_blocks(document.element.body):
|
||||
if isinstance(child, CT_P):
|
||||
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user