fix(data_process): 修复 Word 切分行号断档与预览标题缺失

- 正文抽取下钻 SDT 内容控件,目录等内容不再整段丢失
- 切片投影匹配剥离序列化插入的列表自动编号,新增行锚点兜底与游标防回退
- fixed/semantic 切分路径定位失败时保留切片,不再静默丢弃
- Word 预览按段落大纲级别识别标题,未套标题样式的小节正常渲染
- 本地嵌入模型抽为共享单例,供语义分块与质量评分共用
This commit is contained in:
caoxiaozhu
2026-08-19 14:21:41 +08:00
parent 93449b7f0e
commit f47611020a
6 changed files with 315 additions and 34 deletions

View File

@@ -9,6 +9,8 @@ from decimal import Decimal
import pytest
from docx import Document
from docx.oxml import parse_xml
from docx.oxml.ns import nsdecls, qn
from openpyxl import Workbook
from pptx import Presentation
from pptx.util import Inches
@@ -38,6 +40,7 @@ from app.modules.data_process.algorithms import (
stable_split_assignments,
structured_json_dumps,
)
from app.modules.data_process.office_preview import build_docx_preview
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
@@ -318,6 +321,74 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def _docx_with_sdt_bytes() -> bytes:
"""构造带 SDT 目录内容控件的 docx段落顺序为正文、SDT、正文。"""
document = Document()
document.add_paragraph("正文开头。")
sdt = parse_xml(
"<w:sdt %s><w:sdtPr><w:id w:val='1'/></w:sdtPr>"
"<w:sdtContent><w:p><w:r><w:t>目录条目 第一章 概述</w:t></w:r></w:p>"
"</w:sdtContent></w:sdt>" % nsdecls("w")
)
body = document.element.body
sect_pr = body.find(qn("w:sectPr"))
if sect_pr is not None:
sect_pr.addprevious(sdt)
else:
body.append(sdt)
document.add_paragraph("正文结尾。")
output = io.BytesIO()
document.save(output)
return output.getvalue()
def test_docx_extraction_and_preview_include_sdt_content() -> None:
raw = _docx_with_sdt_bytes()
parsed = parse_text_content(raw, filename="toc.docx")
assert "目录条目 第一章 概述" in parsed.text
assert (
parsed.text.index("正文开头。")
< parsed.text.index("目录条目 第一章 概述")
< parsed.text.index("正文结尾。")
)
preview = build_docx_preview(raw)
paragraph_texts = [
block["text"] for block in preview["blocks"] if block["type"] == "paragraph"
]
assert "目录条目 第一章 概述" in paragraph_texts
# 预览偏移必须与正文抽取规则一致,否则前端定位会错位。
sdt_block = next(
block
for block in preview["blocks"]
if block.get("text") == "目录条目 第一章 概述"
)
assert parsed.text[sdt_block["source_start"] : sdt_block["source_end"]] == (
"目录条目 第一章 概述"
)
def test_docx_preview_detects_outline_level_headings() -> None:
"""未套标题样式但设了大纲级别的段落Word 目录按此收录)也按标题渲染。"""
document = Document()
document.add_heading("一级标题", level=1)
plain = document.add_paragraph("4.2.1 数据管理")
p_pr = plain._p.get_or_add_pPr()
p_pr.append(parse_xml("<w:outlineLvl %s w:val='2'/>" % nsdecls("w")))
document.add_paragraph("普通正文段落。")
output = io.BytesIO()
document.save(output)
preview = build_docx_preview(output.getvalue())
blocks = {b["text"]: b for b in preview["blocks"] if b["type"] == "paragraph"}
assert blocks["一级标题"]["heading_level"] == 1
assert blocks["4.2.1 数据管理"]["heading_level"] == 3
assert blocks["普通正文段落。"]["heading_level"] is None
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active