fix(data_process): 修复 Word 切分行号断档与预览标题缺失
- 正文抽取下钻 SDT 内容控件,目录等内容不再整段丢失 - 切片投影匹配剥离序列化插入的列表自动编号,新增行锚点兜底与游标防回退 - fixed/semantic 切分路径定位失败时保留切片,不再静默丢弃 - Word 预览按段落大纲级别识别标题,未套标题样式的小节正常渲染 - 本地嵌入模型抽为共享单例,供语义分块与质量评分共用
This commit is contained in:
@@ -9,6 +9,8 @@ from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
from docx.oxml import parse_xml
|
||||
from docx.oxml.ns import nsdecls, qn
|
||||
from openpyxl import Workbook
|
||||
from pptx import Presentation
|
||||
from pptx.util import Inches
|
||||
@@ -38,6 +40,7 @@ from app.modules.data_process.algorithms import (
|
||||
stable_split_assignments,
|
||||
structured_json_dumps,
|
||||
)
|
||||
from app.modules.data_process.office_preview import build_docx_preview
|
||||
|
||||
|
||||
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
|
||||
@@ -318,6 +321,74 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
assert parsed_pptx.records == ()
|
||||
|
||||
|
||||
def _docx_with_sdt_bytes() -> bytes:
|
||||
"""构造带 SDT 目录内容控件的 docx,段落顺序为正文、SDT、正文。"""
|
||||
|
||||
document = Document()
|
||||
document.add_paragraph("正文开头。")
|
||||
sdt = parse_xml(
|
||||
"<w:sdt %s><w:sdtPr><w:id w:val='1'/></w:sdtPr>"
|
||||
"<w:sdtContent><w:p><w:r><w:t>目录条目 第一章 概述</w:t></w:r></w:p>"
|
||||
"</w:sdtContent></w:sdt>" % nsdecls("w")
|
||||
)
|
||||
body = document.element.body
|
||||
sect_pr = body.find(qn("w:sectPr"))
|
||||
if sect_pr is not None:
|
||||
sect_pr.addprevious(sdt)
|
||||
else:
|
||||
body.append(sdt)
|
||||
document.add_paragraph("正文结尾。")
|
||||
output = io.BytesIO()
|
||||
document.save(output)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def test_docx_extraction_and_preview_include_sdt_content() -> None:
|
||||
raw = _docx_with_sdt_bytes()
|
||||
|
||||
parsed = parse_text_content(raw, filename="toc.docx")
|
||||
assert "目录条目 第一章 概述" in parsed.text
|
||||
assert (
|
||||
parsed.text.index("正文开头。")
|
||||
< parsed.text.index("目录条目 第一章 概述")
|
||||
< parsed.text.index("正文结尾。")
|
||||
)
|
||||
|
||||
preview = build_docx_preview(raw)
|
||||
paragraph_texts = [
|
||||
block["text"] for block in preview["blocks"] if block["type"] == "paragraph"
|
||||
]
|
||||
assert "目录条目 第一章 概述" in paragraph_texts
|
||||
# 预览偏移必须与正文抽取规则一致,否则前端定位会错位。
|
||||
sdt_block = next(
|
||||
block
|
||||
for block in preview["blocks"]
|
||||
if block.get("text") == "目录条目 第一章 概述"
|
||||
)
|
||||
assert parsed.text[sdt_block["source_start"] : sdt_block["source_end"]] == (
|
||||
"目录条目 第一章 概述"
|
||||
)
|
||||
|
||||
|
||||
def test_docx_preview_detects_outline_level_headings() -> None:
|
||||
"""未套标题样式但设了大纲级别的段落(Word 目录按此收录)也按标题渲染。"""
|
||||
|
||||
document = Document()
|
||||
document.add_heading("一级标题", level=1)
|
||||
plain = document.add_paragraph("4.2.1 数据管理")
|
||||
p_pr = plain._p.get_or_add_pPr()
|
||||
p_pr.append(parse_xml("<w:outlineLvl %s w:val='2'/>" % nsdecls("w")))
|
||||
document.add_paragraph("普通正文段落。")
|
||||
output = io.BytesIO()
|
||||
document.save(output)
|
||||
|
||||
preview = build_docx_preview(output.getvalue())
|
||||
blocks = {b["text"]: b for b in preview["blocks"] if b["type"] == "paragraph"}
|
||||
assert blocks["一级标题"]["heading_level"] == 1
|
||||
assert blocks["4.2.1 数据管理"]["heading_level"] == 3
|
||||
assert blocks["普通正文段落。"]["heading_level"] is None
|
||||
|
||||
|
||||
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
||||
workbook = Workbook()
|
||||
first = workbook.active
|
||||
|
||||
Reference in New Issue
Block a user