From 9025437a37024afdcd0f075516916a4b7e3bf110 Mon Sep 17 00:00:00 2001 From: caoxiaozhu Date: Mon, 27 Jul 2026 16:12:50 +0800 Subject: [PATCH] =?UTF-8?q?feat(data-process):=20=E5=A2=9E=E5=8A=A0=20Offi?= =?UTF-8?q?ce=20=E5=8E=9F=E6=96=87=E4=BB=B6=E9=A2=84=E8=A7=88=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/endpoints/data_process.py | 77 ++++- .../modules/data_process/office_preview.py | 308 ++++++++++++++++++ backend/tests/test_data_process_api.py | 102 ++++++ 3 files changed, 482 insertions(+), 5 deletions(-) create mode 100644 backend/app/modules/data_process/office_preview.py diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 98a52dc..700e2c2 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -53,6 +53,11 @@ from app.modules.data_process.document_chunking import ( merge_short_chunks, ) from app.modules.data_process.generation import generate_model_records +from app.modules.data_process.office_preview import ( + MAX_XLSX_PREVIEW_ROWS, + build_docx_preview, + build_xlsx_preview, +) from app.modules.data_process.storage import ( LocalDataProcessStorage, StagedSourceObject, @@ -111,6 +116,12 @@ LEGACY_OFFICE_CONVERSIONS = { ".xls": ".xlsx", ".ppt": ".pptx", } +RAW_INLINE_PREVIEW_MEDIA_TYPES = { + "pdf": "application/pdf", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", +} def ok(data: Any = None, message: str = "ok") -> dict[str, Any]: @@ -897,8 +908,13 @@ def source_file_raw( ) -> StreamingResponse: with api_errors(): source = store.get_source_file(task_id, file_id, include_content=False) - if str(source.get("file_format") or "").lower() != "pdf": - raise fail(415, "raw inline preview is only available for PDF source files") + file_format = str(source.get("file_format") or "").lower() + media_type = RAW_INLINE_PREVIEW_MEDIA_TYPES.get(file_format) + if media_type is None: + raise fail( + 415, + "raw inline preview is only available for PDF and modern Office source files", + ) storage_object_id = str(source.get("storage_object_id") or "") actual_size = storage.file_size( storage_object_id, @@ -906,14 +922,15 @@ def source_file_raw( expected_source_file_id=file_id, ) if actual_size is None: - raise fail(410, "the original PDF is unavailable for this legacy source file") + raise fail(410, "the original file is unavailable for this legacy source file") expected_size = int(source.get("size_bytes") or 0) if actual_size != expected_size: raise ValueError("source object size does not match metadata") selected_range = _source_byte_range(range_header, actual_size) start, end = selected_range or (0, actual_size - 1) length = end - start + 1 - name = _safe_file_name(str(source.get("name") or "source.pdf"), "source.pdf") + default_name = f"source.{file_format}" + name = _safe_file_name(str(source.get("name") or default_name), default_name) headers = { "Accept-Ranges": "bytes", "Cache-Control": "private, no-store", @@ -938,11 +955,61 @@ def source_file_raw( return StreamingResponse( body, status_code=206 if selected_range is not None else 200, - media_type="application/pdf", + media_type=media_type, headers=headers, ) +@router.get("/{task_id}/source-files/{file_id}/office-preview") +def source_file_office_preview( + task_id: str, + file_id: str, + sheet_index: int = Query(default=0, ge=0), + offset: int = Query(default=0, ge=0), + limit: int = Query(default=100, ge=1, le=MAX_XLSX_PREVIEW_ROWS), + store: DataProcessStore = Depends(get_data_process_store), + storage: LocalDataProcessStorage = Depends(get_data_process_storage), +) -> dict[str, Any]: + """返回 Word 版式块或 Excel 工作表网格，不把二进制内容下发给组件解析。""" + + with api_errors(): + source = store.get_source_file(task_id, file_id, include_content=False) + file_format = str(source.get("file_format") or "").lower() + if file_format not in {"docx", "xlsx"}: + raise fail(415, "Office preview is only available for DOCX and XLSX source files") + storage_object_id = str(source.get("storage_object_id") or "") + actual_size = storage.file_size( + storage_object_id, + expected_task_id=task_id, + expected_source_file_id=file_id, + ) + if actual_size is None: + raise fail(410, "the original Office file is unavailable for this legacy source file") + expected_size = int(source.get("size_bytes") or 0) + if actual_size != expected_size: + raise ValueError("source object size does not match metadata") + raw = b"".join( + storage.iter_bytes( + storage_object_id, + expected_task_id=task_id, + expected_source_file_id=file_id, + expected_size=actual_size, + ) + ) + preview = ( + build_docx_preview(raw) + if file_format == "docx" + else build_xlsx_preview( + raw, + sheet_index=sheet_index, + offset=offset, + limit=limit, + ) + ) + preview["file_name"] = str(source.get("name") or f"source.{file_format}") + return ok(preview) + + @router.get("/{task_id}/source-files/{file_id}/pdf-pages") def source_file_pdf_pages( task_id: str, diff --git a/backend/app/modules/data_process/office_preview.py b/backend/app/modules/data_process/office_preview.py new file mode 100644 index 0000000..fcfd0ed --- /dev/null +++ b/backend/app/modules/data_process/office_preview.py @@ -0,0 +1,308 @@ +"""Word 与 Excel 原文件的安全、受限预览模型。 + +预览只返回浏览器绘制所需的结构化数据，不返回或执行 Office 包中的活动内容。 +DOCX 的字符偏移与上传时的正文抽取规则保持一致，供前端定位当前切片。 +""" + +from __future__ import annotations + +import io +import re +from typing import Any + +from docx import Document +from docx.oxml.table import CT_Tbl +from docx.oxml.text.paragraph import CT_P +from docx.table import Table +from docx.text.paragraph import Paragraph +from openpyxl import load_workbook + +from app.modules.data_process.algorithms import ( + _MAX_WORKBOOK_COLUMNS, + _MAX_WORKBOOK_HEADER_SCAN_ROWS, + _infer_xlsx_header_region, + _normalize_spreadsheet_value, + _rewrite_xlsx_workbook_relationships, + _validate_office_archive, + _xlsx_sheet_merge_ranges, + normalize_text, +) + +MAX_DOCX_PREVIEW_BLOCKS = 2_000 +MAX_XLSX_PREVIEW_ROWS = 200 + + +def _docx_alignment(paragraph: Paragraph) -> str: + value = paragraph.alignment + return { + 0: "left", + 1: "center", + 2: "right", + 3: "justify", + 4: "distribute", + 5: "justify", + 7: "justify", + 8: "distribute", + 9: "distribute", + }.get(int(value) if value is not None else -1, "left") + + +def _docx_heading_level(paragraph: Paragraph) -> int | None: + style = paragraph.style + if style is None: + return None + style_name = str(style.name or "") + style_id = str(style.style_id or "") + match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE) + return int(match.group(1)) if match else None + + +def build_docx_preview(raw: bytes) -> dict[str, Any]: + """把 DOCX 转为保留标题、段落和表格顺序的浏览器预览模型。""" + + _validate_office_archive(raw, "docx") + try: + document = Document(io.BytesIO(raw)) + except Exception as exc: + raise ValueError(f"invalid DOCX file: {exc}") from exc + + blocks: list[dict[str, Any]] = [] + source_cursor = 0 + has_source_content = False + rendered_blocks = 0 + truncated = False + + def source_range(value: str) -> tuple[str, int, int] | None: + nonlocal source_cursor, has_source_content + text = normalize_text(value) + if not text: + return None + if has_source_content: + source_cursor += 2 + start = source_cursor + source_cursor += len(text) + has_source_content = True + return text, start, source_cursor + + for child in document.element.body.iterchildren(): + if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS: + truncated = True + break + + if isinstance(child, CT_P): + paragraph = Paragraph(child, document) + located = source_range(paragraph.text) + if located is None: + continue + text, start, end = located + style_name = str(paragraph.style.name or "") if paragraph.style else "" + blocks.append( + { + "type": "paragraph", + "text": text, + "style": style_name, + "heading_level": _docx_heading_level(paragraph), + "alignment": _docx_alignment(paragraph), + "is_list": "list" in style_name.casefold() or "列表" in style_name, + "source_start": start, + "source_end": end, + } + ) + rendered_blocks += 1 + continue + + if not isinstance(child, CT_Tbl): + continue + table = Table(child, document) + preview_rows: list[dict[str, Any]] = [] + for row in table.rows: + if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS: + truncated = True + break + cell_values = [normalize_text(cell.text) for cell in row.cells] + located = source_range("\t".join(cell_values)) + if located is None: + continue + _, start, end = located + preview_rows.append( + { + "cells": cell_values, + "source_start": start, + "source_end": end, + } + ) + rendered_blocks += 1 + if preview_rows: + blocks.append({"type": "table", "rows": preview_rows}) + if truncated: + break + + return { + "format": "docx", + "blocks": blocks, + "truncated": truncated, + } + + +def build_xlsx_preview( + raw: bytes, + *, + sheet_index: int = 0, + offset: int = 0, + limit: int = 100, +) -> dict[str, Any]: + """按工作表分页返回 XLSX 的表头和记录网格。""" + + if sheet_index < 0 or offset < 0: + raise ValueError("sheet_index and offset must be non-negative") + if limit < 1 or limit > MAX_XLSX_PREVIEW_ROWS: + raise ValueError( + f"XLSX preview limit must be in [1, {MAX_XLSX_PREVIEW_ROWS}]" + ) + + _validate_office_archive(raw, "xlsx") + merged_by_sheet, normalized_targets = _xlsx_sheet_merge_ranges(raw) + workbook_raw = ( + _rewrite_xlsx_workbook_relationships(raw, normalized_targets) + if normalized_targets + else raw + ) + try: + workbook = load_workbook( + io.BytesIO(workbook_raw), + read_only=True, + data_only=True, + keep_links=False, + ) + except Exception as exc: + raise ValueError(f"invalid XLSX file: {exc}") from exc + + try: + sheets = [ + { + "index": index, + "name": worksheet.title, + "state": worksheet.sheet_state, + } + for index, worksheet in enumerate(workbook.worksheets) + ] + if not sheets: + raise ValueError("XLSX workbook contains no worksheets") + if sheet_index >= len(sheets): + raise ValueError("XLSX worksheet index is out of range") + + worksheet = workbook.worksheets[sheet_index] + reset_dimensions = getattr(worksheet, "reset_dimensions", None) + if callable(reset_dimensions): + reset_dimensions() + row_iterator = enumerate(worksheet.iter_rows(values_only=True), start=1) + buffered_rows: dict[int, tuple[Any, ...]] = {} + + def normalized_values(row: tuple[Any, ...]) -> list[Any]: + values = list(row) + while values and values[-1] in {None, ""}: + values.pop() + if len(values) > _MAX_WORKBOOK_COLUMNS: + raise ValueError( + f"XLSX worksheet {worksheet.title!r} exceeds " + f"{_MAX_WORKBOOK_COLUMNS} columns" + ) + return values + + for row_number, row in row_iterator: + values = normalized_values(row) + if not values or all(value in {None, ""} for value in values): + continue + buffered_rows[row_number] = tuple(values) + if len(buffered_rows) >= _MAX_WORKBOOK_HEADER_SCAN_ROWS: + break + + if not buffered_rows: + return { + "format": "xlsx", + "sheets": sheets, + "active_sheet": { + "index": sheet_index, + "name": worksheet.title, + "columns": [], + "rows": [], + "offset": offset, + "limit": limit, + "has_more": False, + }, + } + + _, header_end_row, headers = _infer_xlsx_header_region( + worksheet.title, + buffered_rows, + merged_by_sheet.get(worksheet.title, ()), + ) + preview_rows: list[dict[str, Any]] = [] + record_index = 0 + has_more = False + + def append_row(row_number: int, values: tuple[Any, ...] | list[Any]) -> bool: + nonlocal record_index, has_more + row_values = list(values) + if len(row_values) > len(headers): + raise ValueError( + f"XLSX worksheet {worksheet.title!r} has a row wider than its header" + ) + row_values.extend([None] * (len(headers) - len(row_values))) + record = { + header: _normalize_spreadsheet_value(value) + for header, value in zip(headers, row_values, strict=True) + } + if not any(value not in {"", None} for value in record.values()): + return False + current_index = record_index + record_index += 1 + if current_index < offset: + return False + if len(preview_rows) >= limit: + has_more = True + return True + preview_rows.append( + { + "row_number": row_number, + "record_index": current_index, + "values": [record[header] for header in headers], + "record": record, + } + ) + return False + + for row_number, values in buffered_rows.items(): + if row_number > header_end_row and append_row(row_number, values): + break + else: + for row_number, row in row_iterator: + values = normalized_values(row) + if not values or all(value in {None, ""} for value in values): + continue + if append_row(row_number, values): + break + + return { + "format": "xlsx", + "sheets": sheets, + "active_sheet": { + "index": sheet_index, + "name": worksheet.title, + "columns": headers, + "rows": preview_rows, + "offset": offset, + "limit": limit, + "has_more": has_more, + }, + } + finally: + workbook.close() + + +__all__ = [ + "MAX_DOCX_PREVIEW_BLOCKS", + "MAX_XLSX_PREVIEW_ROWS", + "build_docx_preview", + "build_xlsx_preview", +] diff --git a/backend/tests/test_data_process_api.py b/backend/tests/test_data_process_api.py index 6f5db51..7f04ef0 100644 --- a/backend/tests/test_data_process_api.py +++ b/backend/tests/test_data_process_api.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any import pytest +from docx import Document as WordDocument from fastapi import FastAPI from fastapi.testclient import TestClient from openpyxl import Workbook @@ -1546,11 +1547,112 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None: ) assert content.status_code == 200 assert '"answer":"答案二"' in content.json()["data"]["content"] + office_preview_url = ( + f"/modelTF/data-process/{task_id}/source-files/{source['id']}/office-preview" + ) + first_page = client.get(office_preview_url, params={"offset": 0, "limit": 1}) + assert first_page.status_code == 200 + preview_data = first_page.json()["data"] + assert preview_data["format"] == "xlsx" + assert preview_data["sheets"] == [{"index": 0, "name": "Sheet", "state": "visible"}] + assert preview_data["active_sheet"]["columns"] == ["question", "answer"] + assert preview_data["active_sheet"]["rows"][0]["record"] == { + "question": "问题一", + "answer": "答案一", + } + assert preview_data["active_sheet"]["has_more"] is True + + second_page = client.get(office_preview_url, params={"offset": 1, "limit": 1}) + assert second_page.status_code == 200 + assert second_page.json()["data"]["active_sheet"]["rows"][0]["record"] == { + "question": "问题二", + "answer": "答案二", + } + assert second_page.json()["data"]["active_sheet"]["has_more"] is False + + raw_preview = client.get( + f"/modelTF/data-process/{task_id}/source-files/{source['id']}/raw" + ) + assert raw_preview.status_code == 200 + assert raw_preview.content == original_bytes + assert raw_preview.headers["content-type"].startswith( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert preview.status_code == 200 assert preview.json()["data"]["total"] == 2 +def test_docx_preview_preserves_document_block_order_and_source_offsets( + tmp_path: Path, +) -> None: + client, store, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={"name": "Word 原件预览", "process_type": "unstructured", "config": {}}, + ).json()["data"]["id"] + document = WordDocument() + document.add_heading("费用管理办法", level=1) + document.add_paragraph("第一条 本办法用于规范费用报销。") + table = document.add_table(rows=2, cols=2) + table.cell(0, 0).text = "费用类型" + table.cell(0, 1).text = "审批人" + table.cell(1, 0).text = "差旅费" + table.cell(1, 1).text = "部门负责人" + output = BytesIO() + document.save(output) + original_bytes = output.getvalue() + + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files={ + "files": ( + "费用 管理.docx", + original_bytes, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + }, + ).json()["data"]["files"][0] + response = client.get( + f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/office-preview" + ) + + assert response.status_code == 200 + data = response.json()["data"] + assert data["format"] == "docx" + assert data["file_name"] == "费用 管理.docx" + assert data["truncated"] is False + assert [block["type"] for block in data["blocks"]] == [ + "paragraph", + "paragraph", + "table", + ] + assert data["blocks"][0]["heading_level"] == 1 + assert data["blocks"][0]["text"] == "费用管理办法" + assert data["blocks"][2]["rows"][1]["cells"] == ["差旅费", "部门负责人"] + + source_text = store.get_source_file(task_id, uploaded["id"])["content"] + first_paragraph = data["blocks"][0] + assert ( + source_text[first_paragraph["source_start"] : first_paragraph["source_end"]] + == first_paragraph["text"] + ) + table_row = data["blocks"][2]["rows"][1] + assert ( + source_text[table_row["source_start"] : table_row["source_end"]] + == "\t".join(table_row["cells"]) + ) + + raw_preview = client.get( + f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw" + ) + assert raw_preview.status_code == 200 + assert raw_preview.content == original_bytes + assert raw_preview.headers["content-type"].startswith( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ) + + def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) task_id = client.post(