feat(data-process): 增加 Office 原文件预览接口
This commit is contained in:
caoxiaozhu
@@ -53,6 +53,11 @@ from app.modules.data_process.document_chunking import (
|
||||
merge_short_chunks,
|
||||
)
|
||||
from app.modules.data_process.generation import generate_model_records
|
||||
from app.modules.data_process.office_preview import (
|
||||
MAX_XLSX_PREVIEW_ROWS,
|
||||
build_docx_preview,
|
||||
build_xlsx_preview,
|
||||
)
|
||||
from app.modules.data_process.storage import (
|
||||
LocalDataProcessStorage,
|
||||
StagedSourceObject,
|
||||
@@ -111,6 +116,12 @@ LEGACY_OFFICE_CONVERSIONS = {
|
||||
".xls": ".xlsx",
|
||||
".ppt": ".pptx",
|
||||
}
|
||||
RAW_INLINE_PREVIEW_MEDIA_TYPES = {
|
||||
"pdf": "application/pdf",
|
||||
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
}
|
||||
|
||||
|
||||
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
|
||||
@@ -897,8 +908,13 @@ def source_file_raw(
|
||||
) -> StreamingResponse:
|
||||
with api_errors():
|
||||
source = store.get_source_file(task_id, file_id, include_content=False)
|
||||
if str(source.get("file_format") or "").lower() != "pdf":
|
||||
raise fail(415, "raw inline preview is only available for PDF source files")
|
||||
file_format = str(source.get("file_format") or "").lower()
|
||||
media_type = RAW_INLINE_PREVIEW_MEDIA_TYPES.get(file_format)
|
||||
if media_type is None:
|
||||
raise fail(
|
||||
415,
|
||||
"raw inline preview is only available for PDF and modern Office source files",
|
||||
)
|
||||
storage_object_id = str(source.get("storage_object_id") or "")
|
||||
actual_size = storage.file_size(
|
||||
storage_object_id,
|
||||
@@ -906,14 +922,15 @@ def source_file_raw(
|
||||
expected_source_file_id=file_id,
|
||||
)
|
||||
if actual_size is None:
|
||||
raise fail(410, "the original PDF is unavailable for this legacy source file")
|
||||
raise fail(410, "the original file is unavailable for this legacy source file")
|
||||
expected_size = int(source.get("size_bytes") or 0)
|
||||
if actual_size != expected_size:
|
||||
raise ValueError("source object size does not match metadata")
|
||||
selected_range = _source_byte_range(range_header, actual_size)
|
||||
start, end = selected_range or (0, actual_size - 1)
|
||||
length = end - start + 1
|
||||
name = _safe_file_name(str(source.get("name") or "source.pdf"), "source.pdf")
|
||||
default_name = f"source.{file_format}"
|
||||
name = _safe_file_name(str(source.get("name") or default_name), default_name)
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Cache-Control": "private, no-store",
|
||||
@@ -938,11 +955,61 @@ def source_file_raw(
|
||||
return StreamingResponse(
|
||||
body,
|
||||
status_code=206 if selected_range is not None else 200,
|
||||
media_type="application/pdf",
|
||||
media_type=media_type,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{task_id}/source-files/{file_id}/office-preview")
|
||||
def source_file_office_preview(
|
||||
task_id: str,
|
||||
file_id: str,
|
||||
sheet_index: int = Query(default=0, ge=0),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=100, ge=1, le=MAX_XLSX_PREVIEW_ROWS),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||
) -> dict[str, Any]:
|
||||
"""返回 Word 版式块或 Excel 工作表网格，不把二进制内容下发给组件解析。"""
|
||||
|
||||
with api_errors():
|
||||
source = store.get_source_file(task_id, file_id, include_content=False)
|
||||
file_format = str(source.get("file_format") or "").lower()
|
||||
if file_format not in {"docx", "xlsx"}:
|
||||
raise fail(415, "Office preview is only available for DOCX and XLSX source files")
|
||||
storage_object_id = str(source.get("storage_object_id") or "")
|
||||
actual_size = storage.file_size(
|
||||
storage_object_id,
|
||||
expected_task_id=task_id,
|
||||
expected_source_file_id=file_id,
|
||||
)
|
||||
if actual_size is None:
|
||||
raise fail(410, "the original Office file is unavailable for this legacy source file")
|
||||
expected_size = int(source.get("size_bytes") or 0)
|
||||
if actual_size != expected_size:
|
||||
raise ValueError("source object size does not match metadata")
|
||||
raw = b"".join(
|
||||
storage.iter_bytes(
|
||||
storage_object_id,
|
||||
expected_task_id=task_id,
|
||||
expected_source_file_id=file_id,
|
||||
expected_size=actual_size,
|
||||
)
|
||||
)
|
||||
preview = (
|
||||
build_docx_preview(raw)
|
||||
if file_format == "docx"
|
||||
else build_xlsx_preview(
|
||||
raw,
|
||||
sheet_index=sheet_index,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
preview["file_name"] = str(source.get("name") or f"source.{file_format}")
|
||||
return ok(preview)
|
||||
|
||||
|
||||
@router.get("/{task_id}/source-files/{file_id}/pdf-pages")
|
||||
def source_file_pdf_pages(
|
||||
task_id: str,
|
||||
|
||||
308
backend/app/modules/data_process/office_preview.py
Normal file
308
backend/app/modules/data_process/office_preview.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""Word 与 Excel 原文件的安全、受限预览模型。
|
||||
|
||||
预览只返回浏览器绘制所需的结构化数据，不返回或执行 Office 包中的活动内容。
|
||||
DOCX 的字符偏移与上传时的正文抽取规则保持一致，供前端定位当前切片。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from docx import Document
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from app.modules.data_process.algorithms import (
|
||||
_MAX_WORKBOOK_COLUMNS,
|
||||
_MAX_WORKBOOK_HEADER_SCAN_ROWS,
|
||||
_infer_xlsx_header_region,
|
||||
_normalize_spreadsheet_value,
|
||||
_rewrite_xlsx_workbook_relationships,
|
||||
_validate_office_archive,
|
||||
_xlsx_sheet_merge_ranges,
|
||||
normalize_text,
|
||||
)
|
||||
|
||||
MAX_DOCX_PREVIEW_BLOCKS = 2_000
|
||||
MAX_XLSX_PREVIEW_ROWS = 200
|
||||
|
||||
|
||||
def _docx_alignment(paragraph: Paragraph) -> str:
|
||||
value = paragraph.alignment
|
||||
return {
|
||||
0: "left",
|
||||
1: "center",
|
||||
2: "right",
|
||||
3: "justify",
|
||||
4: "distribute",
|
||||
5: "justify",
|
||||
7: "justify",
|
||||
8: "distribute",
|
||||
9: "distribute",
|
||||
}.get(int(value) if value is not None else -1, "left")
|
||||
|
||||
|
||||
def _docx_heading_level(paragraph: Paragraph) -> int | None:
|
||||
style = paragraph.style
|
||||
if style is None:
|
||||
return None
|
||||
style_name = str(style.name or "")
|
||||
style_id = str(style.style_id or "")
|
||||
match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def build_docx_preview(raw: bytes) -> dict[str, Any]:
|
||||
"""把 DOCX 转为保留标题、段落和表格顺序的浏览器预览模型。"""
|
||||
|
||||
_validate_office_archive(raw, "docx")
|
||||
try:
|
||||
document = Document(io.BytesIO(raw))
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid DOCX file: {exc}") from exc
|
||||
|
||||
blocks: list[dict[str, Any]] = []
|
||||
source_cursor = 0
|
||||
has_source_content = False
|
||||
rendered_blocks = 0
|
||||
truncated = False
|
||||
|
||||
def source_range(value: str) -> tuple[str, int, int] | None:
|
||||
nonlocal source_cursor, has_source_content
|
||||
text = normalize_text(value)
|
||||
if not text:
|
||||
return None
|
||||
if has_source_content:
|
||||
source_cursor += 2
|
||||
start = source_cursor
|
||||
source_cursor += len(text)
|
||||
has_source_content = True
|
||||
return text, start, source_cursor
|
||||
|
||||
for child in document.element.body.iterchildren():
|
||||
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
|
||||
truncated = True
|
||||
break
|
||||
|
||||
if isinstance(child, CT_P):
|
||||
paragraph = Paragraph(child, document)
|
||||
located = source_range(paragraph.text)
|
||||
if located is None:
|
||||
continue
|
||||
text, start, end = located
|
||||
style_name = str(paragraph.style.name or "") if paragraph.style else ""
|
||||
blocks.append(
|
||||
{
|
||||
"type": "paragraph",
|
||||
"text": text,
|
||||
"style": style_name,
|
||||
"heading_level": _docx_heading_level(paragraph),
|
||||
"alignment": _docx_alignment(paragraph),
|
||||
"is_list": "list" in style_name.casefold() or "列表" in style_name,
|
||||
"source_start": start,
|
||||
"source_end": end,
|
||||
}
|
||||
)
|
||||
rendered_blocks += 1
|
||||
continue
|
||||
|
||||
if not isinstance(child, CT_Tbl):
|
||||
continue
|
||||
table = Table(child, document)
|
||||
preview_rows: list[dict[str, Any]] = []
|
||||
for row in table.rows:
|
||||
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
|
||||
truncated = True
|
||||
break
|
||||
cell_values = [normalize_text(cell.text) for cell in row.cells]
|
||||
located = source_range("\t".join(cell_values))
|
||||
if located is None:
|
||||
continue
|
||||
_, start, end = located
|
||||
preview_rows.append(
|
||||
{
|
||||
"cells": cell_values,
|
||||
"source_start": start,
|
||||
"source_end": end,
|
||||
}
|
||||
)
|
||||
rendered_blocks += 1
|
||||
if preview_rows:
|
||||
blocks.append({"type": "table", "rows": preview_rows})
|
||||
if truncated:
|
||||
break
|
||||
|
||||
return {
|
||||
"format": "docx",
|
||||
"blocks": blocks,
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
|
||||
def build_xlsx_preview(
|
||||
raw: bytes,
|
||||
*,
|
||||
sheet_index: int = 0,
|
||||
offset: int = 0,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
"""按工作表分页返回 XLSX 的表头和记录网格。"""
|
||||
|
||||
if sheet_index < 0 or offset < 0:
|
||||
raise ValueError("sheet_index and offset must be non-negative")
|
||||
if limit < 1 or limit > MAX_XLSX_PREVIEW_ROWS:
|
||||
raise ValueError(
|
||||
f"XLSX preview limit must be in [1, {MAX_XLSX_PREVIEW_ROWS}]"
|
||||
)
|
||||
|
||||
_validate_office_archive(raw, "xlsx")
|
||||
merged_by_sheet, normalized_targets = _xlsx_sheet_merge_ranges(raw)
|
||||
workbook_raw = (
|
||||
_rewrite_xlsx_workbook_relationships(raw, normalized_targets)
|
||||
if normalized_targets
|
||||
else raw
|
||||
)
|
||||
try:
|
||||
workbook = load_workbook(
|
||||
io.BytesIO(workbook_raw),
|
||||
read_only=True,
|
||||
data_only=True,
|
||||
keep_links=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid XLSX file: {exc}") from exc
|
||||
|
||||
try:
|
||||
sheets = [
|
||||
{
|
||||
"index": index,
|
||||
"name": worksheet.title,
|
||||
"state": worksheet.sheet_state,
|
||||
}
|
||||
for index, worksheet in enumerate(workbook.worksheets)
|
||||
]
|
||||
if not sheets:
|
||||
raise ValueError("XLSX workbook contains no worksheets")
|
||||
if sheet_index >= len(sheets):
|
||||
raise ValueError("XLSX worksheet index is out of range")
|
||||
|
||||
worksheet = workbook.worksheets[sheet_index]
|
||||
reset_dimensions = getattr(worksheet, "reset_dimensions", None)
|
||||
if callable(reset_dimensions):
|
||||
reset_dimensions()
|
||||
row_iterator = enumerate(worksheet.iter_rows(values_only=True), start=1)
|
||||
buffered_rows: dict[int, tuple[Any, ...]] = {}
|
||||
|
||||
def normalized_values(row: tuple[Any, ...]) -> list[Any]:
|
||||
values = list(row)
|
||||
while values and values[-1] in {None, ""}:
|
||||
values.pop()
|
||||
if len(values) > _MAX_WORKBOOK_COLUMNS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {worksheet.title!r} exceeds "
|
||||
f"{_MAX_WORKBOOK_COLUMNS} columns"
|
||||
)
|
||||
return values
|
||||
|
||||
for row_number, row in row_iterator:
|
||||
values = normalized_values(row)
|
||||
if not values or all(value in {None, ""} for value in values):
|
||||
continue
|
||||
buffered_rows[row_number] = tuple(values)
|
||||
if len(buffered_rows) >= _MAX_WORKBOOK_HEADER_SCAN_ROWS:
|
||||
break
|
||||
|
||||
if not buffered_rows:
|
||||
return {
|
||||
"format": "xlsx",
|
||||
"sheets": sheets,
|
||||
"active_sheet": {
|
||||
"index": sheet_index,
|
||||
"name": worksheet.title,
|
||||
"columns": [],
|
||||
"rows": [],
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"has_more": False,
|
||||
},
|
||||
}
|
||||
|
||||
_, header_end_row, headers = _infer_xlsx_header_region(
|
||||
worksheet.title,
|
||||
buffered_rows,
|
||||
merged_by_sheet.get(worksheet.title, ()),
|
||||
)
|
||||
preview_rows: list[dict[str, Any]] = []
|
||||
record_index = 0
|
||||
has_more = False
|
||||
|
||||
def append_row(row_number: int, values: tuple[Any, ...] | list[Any]) -> bool:
|
||||
nonlocal record_index, has_more
|
||||
row_values = list(values)
|
||||
if len(row_values) > len(headers):
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {worksheet.title!r} has a row wider than its header"
|
||||
)
|
||||
row_values.extend([None] * (len(headers) - len(row_values)))
|
||||
record = {
|
||||
header: _normalize_spreadsheet_value(value)
|
||||
for header, value in zip(headers, row_values, strict=True)
|
||||
}
|
||||
if not any(value not in {"", None} for value in record.values()):
|
||||
return False
|
||||
current_index = record_index
|
||||
record_index += 1
|
||||
if current_index < offset:
|
||||
return False
|
||||
if len(preview_rows) >= limit:
|
||||
has_more = True
|
||||
return True
|
||||
preview_rows.append(
|
||||
{
|
||||
"row_number": row_number,
|
||||
"record_index": current_index,
|
||||
"values": [record[header] for header in headers],
|
||||
"record": record,
|
||||
}
|
||||
)
|
||||
return False
|
||||
|
||||
for row_number, values in buffered_rows.items():
|
||||
if row_number > header_end_row and append_row(row_number, values):
|
||||
break
|
||||
else:
|
||||
for row_number, row in row_iterator:
|
||||
values = normalized_values(row)
|
||||
if not values or all(value in {None, ""} for value in values):
|
||||
continue
|
||||
if append_row(row_number, values):
|
||||
break
|
||||
|
||||
return {
|
||||
"format": "xlsx",
|
||||
"sheets": sheets,
|
||||
"active_sheet": {
|
||||
"index": sheet_index,
|
||||
"name": worksheet.title,
|
||||
"columns": headers,
|
||||
"rows": preview_rows,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"has_more": has_more,
|
||||
},
|
||||
}
|
||||
finally:
|
||||
workbook.close()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_DOCX_PREVIEW_BLOCKS",
|
||||
"MAX_XLSX_PREVIEW_ROWS",
|
||||
"build_docx_preview",
|
||||
"build_xlsx_preview",
|
||||
]
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from docx import Document as WordDocument
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from openpyxl import Workbook
|
||||
@@ -1546,11 +1547,112 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
|
||||
)
|
||||
assert content.status_code == 200
|
||||
assert '"answer":"答案二"' in content.json()["data"]["content"]
|
||||
office_preview_url = (
|
||||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/office-preview"
|
||||
)
|
||||
first_page = client.get(office_preview_url, params={"offset": 0, "limit": 1})
|
||||
assert first_page.status_code == 200
|
||||
preview_data = first_page.json()["data"]
|
||||
assert preview_data["format"] == "xlsx"
|
||||
assert preview_data["sheets"] == [{"index": 0, "name": "Sheet", "state": "visible"}]
|
||||
assert preview_data["active_sheet"]["columns"] == ["question", "answer"]
|
||||
assert preview_data["active_sheet"]["rows"][0]["record"] == {
|
||||
"question": "问题一",
|
||||
"answer": "答案一",
|
||||
}
|
||||
assert preview_data["active_sheet"]["has_more"] is True
|
||||
|
||||
second_page = client.get(office_preview_url, params={"offset": 1, "limit": 1})
|
||||
assert second_page.status_code == 200
|
||||
assert second_page.json()["data"]["active_sheet"]["rows"][0]["record"] == {
|
||||
"question": "问题二",
|
||||
"answer": "答案二",
|
||||
}
|
||||
assert second_page.json()["data"]["active_sheet"]["has_more"] is False
|
||||
|
||||
raw_preview = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/raw"
|
||||
)
|
||||
assert raw_preview.status_code == 200
|
||||
assert raw_preview.content == original_bytes
|
||||
assert raw_preview.headers["content-type"].startswith(
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
)
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["total"] == 2
|
||||
|
||||
|
||||
def test_docx_preview_preserves_document_block_order_and_source_offsets(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "Word 原件预览", "process_type": "unstructured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
document = WordDocument()
|
||||
document.add_heading("费用管理办法", level=1)
|
||||
document.add_paragraph("第一条 本办法用于规范费用报销。")
|
||||
table = document.add_table(rows=2, cols=2)
|
||||
table.cell(0, 0).text = "费用类型"
|
||||
table.cell(0, 1).text = "审批人"
|
||||
table.cell(1, 0).text = "差旅费"
|
||||
table.cell(1, 1).text = "部门负责人"
|
||||
output = BytesIO()
|
||||
document.save(output)
|
||||
original_bytes = output.getvalue()
|
||||
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={
|
||||
"files": (
|
||||
"费用 管理.docx",
|
||||
original_bytes,
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
)
|
||||
},
|
||||
).json()["data"]["files"][0]
|
||||
response = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/office-preview"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()["data"]
|
||||
assert data["format"] == "docx"
|
||||
assert data["file_name"] == "费用 管理.docx"
|
||||
assert data["truncated"] is False
|
||||
assert [block["type"] for block in data["blocks"]] == [
|
||||
"paragraph",
|
||||
"paragraph",
|
||||
"table",
|
||||
]
|
||||
assert data["blocks"][0]["heading_level"] == 1
|
||||
assert data["blocks"][0]["text"] == "费用管理办法"
|
||||
assert data["blocks"][2]["rows"][1]["cells"] == ["差旅费", "部门负责人"]
|
||||
|
||||
source_text = store.get_source_file(task_id, uploaded["id"])["content"]
|
||||
first_paragraph = data["blocks"][0]
|
||||
assert (
|
||||
source_text[first_paragraph["source_start"] : first_paragraph["source_end"]]
|
||||
== first_paragraph["text"]
|
||||
)
|
||||
table_row = data["blocks"][2]["rows"][1]
|
||||
assert (
|
||||
source_text[table_row["source_start"] : table_row["source_end"]]
|
||||
== "\t".join(table_row["cells"])
|
||||
)
|
||||
|
||||
raw_preview = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
|
||||
)
|
||||
assert raw_preview.status_code == 200
|
||||
assert raw_preview.content == original_bytes
|
||||
assert raw_preview.headers["content-type"].startswith(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
|
||||
Reference in New Issue
Block a user