refactor: 完整重构 data_process 模块并修复拆分遗留缺陷
将 algorithms.py / store.py 拆分为 algorithms/ 与 store/ 子包，并修复 机械拆分造成的导入与辅助函数缺失： - algorithms/: 补全各子模块依赖与 17 个私有辅助函数、8 个常量；重写 __init__.py 移除坏的 importlib 兜底，分层导入并以局部 import 断开 text_utils<->parsers、quality<->structured_processing 循环依赖。 - store/: 补回 DataProcessStoreError / hashlib / _serialize_value / estimate_token_count 等缺失导入，包入口导出测试与调用方依赖的私有 辅助函数。 - 删除旧单文件 algorithms.py / store.py 及重构残留（_algorithms_old、 backups、refactor 脚本、REFACTORING 文档）。 algorithms 与 store 测试套件 91 项全部通过。
This commit is contained in:
caoxiaozhu
@@ -0,0 +1,19 @@
|
||||
"""文档解析器模块。"""
|
||||
|
||||
from .pdf import extract_pdf_page_texts, detect_pdf_document_noise, remove_document_noise
|
||||
from .office import (
|
||||
_validate_office_archive,
|
||||
_rewrite_xlsx_workbook_relationships,
|
||||
_xlsx_sheet_merge_ranges,
|
||||
_infer_xlsx_header_region,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'extract_pdf_page_texts',
|
||||
'detect_pdf_document_noise',
|
||||
'remove_document_noise',
|
||||
'_validate_office_archive',
|
||||
'_rewrite_xlsx_workbook_relationships',
|
||||
'_xlsx_sheet_merge_ranges',
|
||||
'_infer_xlsx_header_region',
|
||||
]
|
||||
@@ -0,0 +1,89 @@
|
||||
"""数据处理算法 - CSV 解析。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..text_utils import normalize_text
|
||||
|
||||
|
||||
def _source_line_offsets(
|
||||
text: str,
|
||||
start_line: int,
|
||||
end_line: int,
|
||||
) -> tuple[int, int]:
|
||||
"""把 1-based 物理行范围转换为左闭右开的字符范围。"""
|
||||
|
||||
line_starts = [0]
|
||||
line_starts.extend(match.end() for match in re.finditer("\n", text))
|
||||
if start_line < 1 or end_line < start_line or end_line > len(line_starts):
|
||||
raise ValueError("source line range is outside normalized text")
|
||||
source_start = line_starts[start_line - 1]
|
||||
source_end = (
|
||||
line_starts[end_line] - 1
|
||||
if end_line < len(line_starts)
|
||||
else len(text)
|
||||
)
|
||||
return source_start, source_end
|
||||
|
||||
|
||||
def _extract_csv_records_with_locators(
|
||||
text: str,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""从 CSV 文本中提取记录及其行级定位信息。"""
|
||||
|
||||
normalized_text = normalize_text(text)
|
||||
if not normalized_text:
|
||||
return [], []
|
||||
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
|
||||
except csv.Error:
|
||||
dialect = csv.excel
|
||||
reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV header is required")
|
||||
headers = [normalize_text(header or "") for header in reader.fieldnames]
|
||||
if any(not header for header in headers):
|
||||
raise ValueError("CSV header cannot be empty")
|
||||
if len(set(headers)) != len(headers):
|
||||
raise ValueError("CSV headers must be unique")
|
||||
reader.fieldnames = headers
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
locators: list[dict[str, Any]] = []
|
||||
source_lines = normalized_text.splitlines()
|
||||
previous_end_line = reader.line_num
|
||||
for row in reader:
|
||||
end_line = reader.line_num
|
||||
start_line = previous_end_line + 1
|
||||
previous_end_line = end_line
|
||||
while start_line < end_line and not source_lines[start_line - 1].strip():
|
||||
start_line += 1
|
||||
if None in row:
|
||||
raise ValueError("CSV row has more fields than the header")
|
||||
normalized_row = {
|
||||
key: normalize_text(value or "")
|
||||
for key, value in row.items()
|
||||
}
|
||||
if any(value for value in normalized_row.values()):
|
||||
records.append(normalized_row)
|
||||
source_start, source_end = _source_line_offsets(
|
||||
normalized_text,
|
||||
start_line,
|
||||
end_line,
|
||||
)
|
||||
locators.append(
|
||||
{
|
||||
"kind": "csv",
|
||||
"record_index": len(records),
|
||||
"start_line": start_line,
|
||||
"end_line": end_line,
|
||||
"source_start": source_start,
|
||||
"source_end": source_end,
|
||||
}
|
||||
)
|
||||
return records, locators
|
||||
@@ -0,0 +1,395 @@
|
||||
"""数据处理算法 - JSON/JSONL 解析。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from ..text_utils import _normalize_format, _normalize_value, normalize_text
|
||||
from ..types import (
|
||||
_DuplicateJsonKeyError,
|
||||
_JSON_ENVELOPE_KEYS,
|
||||
_JSON_RECORD_ARRAY_KEYS,
|
||||
_JSON_RESPONSE_METADATA_KEYS,
|
||||
_JSON_WRAPPER_METADATA_KEYS,
|
||||
_MAX_JSON_DEPTH,
|
||||
)
|
||||
|
||||
|
||||
def _record_from_value(value: Any, *, normalize: bool = True) -> dict[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return dict(_normalize_value(value)) if normalize else dict(value)
|
||||
return {"value": _normalize_value(value) if normalize else value}
|
||||
|
||||
def _json_pointer_segment(value: Any) -> str:
|
||||
return str(value).replace("~", "~0").replace("/", "~1")
|
||||
|
||||
def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise _DuplicateJsonKeyError(f"duplicate JSON object key: {key!r}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
def _reject_json_constant(value: str) -> Any:
|
||||
raise ValueError(f"non-finite JSON number is not allowed: {value}")
|
||||
|
||||
def _skip_json_whitespace(text: str, offset: int) -> int:
|
||||
while offset < len(text) and text[offset] in " \t\r\n":
|
||||
offset += 1
|
||||
return offset
|
||||
|
||||
def _validate_json_nesting(text: str) -> None:
|
||||
"""在构造 Python 对象前限制容器深度，避免依赖解释器递归阈值。"""
|
||||
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for char in text:
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char in "[{":
|
||||
depth += 1
|
||||
if depth > _MAX_JSON_DEPTH:
|
||||
raise ValueError(
|
||||
f"JSON nesting exceeds the supported depth of {_MAX_JSON_DEPTH}"
|
||||
)
|
||||
elif char in "]}":
|
||||
depth = max(0, depth - 1)
|
||||
|
||||
def _strict_json_loads(text: str) -> tuple[Any, int, int]:
|
||||
"""严格解析单个 JSON 值并返回其左闭右开源码区间。"""
|
||||
|
||||
start = _skip_json_whitespace(text, 0)
|
||||
if start >= len(text):
|
||||
raise ValueError("JSON content is empty")
|
||||
_validate_json_nesting(text)
|
||||
decoder = json.JSONDecoder(
|
||||
object_pairs_hook=_reject_duplicate_json_keys,
|
||||
parse_float=Decimal,
|
||||
parse_int=int,
|
||||
parse_constant=_reject_json_constant,
|
||||
strict=True,
|
||||
)
|
||||
try:
|
||||
payload, end = decoder.raw_decode(text, start)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}"
|
||||
) from exc
|
||||
except RecursionError as exc:
|
||||
raise ValueError("JSON nesting exceeds the supported depth") from exc
|
||||
except _DuplicateJsonKeyError as exc:
|
||||
raise ValueError(str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
# parse_int/parse_float/parse_constant 的异常也必须稳定映射为客户端错误。
|
||||
raise ValueError(f"invalid JSON number: {exc}") from exc
|
||||
trailing = _skip_json_whitespace(text, end)
|
||||
if trailing != len(text):
|
||||
line = text.count("\n", 0, trailing) + 1
|
||||
line_start = text.rfind("\n", 0, trailing) + 1
|
||||
column = trailing - line_start + 1
|
||||
raise ValueError(
|
||||
f"invalid JSON at line {line}, column {column}: extra data"
|
||||
)
|
||||
return payload, start, end
|
||||
|
||||
def _json_value_end(text: str, start: int) -> int:
|
||||
"""在已验证 JSON 中定位一个值的结束偏移，不对数值做二次解析。"""
|
||||
|
||||
if start >= len(text):
|
||||
raise ValueError("invalid JSON source span")
|
||||
first = text[start]
|
||||
if first == '"':
|
||||
escaped = False
|
||||
for offset in range(start + 1, len(text)):
|
||||
char = text[offset]
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
return offset + 1
|
||||
raise ValueError("invalid JSON source span")
|
||||
if first in "[{":
|
||||
stack = [first]
|
||||
in_string = False
|
||||
escaped = False
|
||||
for offset in range(start + 1, len(text)):
|
||||
char = text[offset]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char in "[{":
|
||||
stack.append(char)
|
||||
elif char in "]}":
|
||||
expected = "[" if char == "]" else "{"
|
||||
if not stack or stack[-1] != expected:
|
||||
raise ValueError("invalid JSON source span")
|
||||
stack.pop()
|
||||
if not stack:
|
||||
return offset + 1
|
||||
raise ValueError("invalid JSON source span")
|
||||
end = start
|
||||
while end < len(text) and text[end] not in " \t\r\n,]}":
|
||||
end += 1
|
||||
if end == start:
|
||||
raise ValueError("invalid JSON source span")
|
||||
return end
|
||||
|
||||
def _json_object_value_spans(
|
||||
text: str,
|
||||
start: int,
|
||||
end: int,
|
||||
) -> dict[str, tuple[int, int]]:
|
||||
"""返回已验证 JSON 对象直接子字段的值区间。"""
|
||||
|
||||
if text[start] != "{" or text[end - 1] != "}":
|
||||
raise ValueError("JSON source value is not an object")
|
||||
result: dict[str, tuple[int, int]] = {}
|
||||
offset = _skip_json_whitespace(text, start + 1)
|
||||
key_decoder = json.JSONDecoder()
|
||||
while offset < end - 1:
|
||||
key, key_end = key_decoder.raw_decode(text, offset)
|
||||
if not isinstance(key, str):
|
||||
raise ValueError("invalid JSON object key")
|
||||
offset = _skip_json_whitespace(text, key_end)
|
||||
if offset >= end or text[offset] != ":":
|
||||
raise ValueError("invalid JSON object member")
|
||||
value_start = _skip_json_whitespace(text, offset + 1)
|
||||
value_end = _json_value_end(text, value_start)
|
||||
result[key] = (value_start, value_end)
|
||||
offset = _skip_json_whitespace(text, value_end)
|
||||
if offset >= end - 1:
|
||||
break
|
||||
if text[offset] != ",":
|
||||
raise ValueError("invalid JSON object member")
|
||||
offset = _skip_json_whitespace(text, offset + 1)
|
||||
return result
|
||||
|
||||
def _json_array_item_spans(
|
||||
text: str,
|
||||
start: int,
|
||||
end: int,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""返回已验证 JSON 数组中每个直接元素的源码区间。"""
|
||||
|
||||
if text[start] != "[" or text[end - 1] != "]":
|
||||
raise ValueError("JSON source value is not an array")
|
||||
result: list[tuple[int, int]] = []
|
||||
offset = _skip_json_whitespace(text, start + 1)
|
||||
while offset < end - 1:
|
||||
item_end = _json_value_end(text, offset)
|
||||
result.append((offset, item_end))
|
||||
offset = _skip_json_whitespace(text, item_end)
|
||||
if offset >= end - 1:
|
||||
break
|
||||
if text[offset] != ",":
|
||||
raise ValueError("invalid JSON array item")
|
||||
offset = _skip_json_whitespace(text, offset + 1)
|
||||
return result
|
||||
|
||||
def _json_span_at_path(
|
||||
text: str,
|
||||
root_span: tuple[int, int],
|
||||
path: Sequence[str],
|
||||
) -> tuple[int, int]:
|
||||
span = root_span
|
||||
for key in path:
|
||||
try:
|
||||
span = _json_object_value_spans(text, *span)[key]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"JSON source path cannot be located: {key}") from exc
|
||||
return span
|
||||
|
||||
|
||||
def _pure_json_record_wrapper(
|
||||
payload: Any,
|
||||
) -> tuple[list[Any], tuple[str, ...]] | None:
|
||||
"""识别不会与业务字段冲突的纯记录包装对象。"""
|
||||
|
||||
if not isinstance(payload, Mapping):
|
||||
return None
|
||||
|
||||
def direct_wrapper(
|
||||
value: Mapping[str, Any],
|
||||
metadata_keys: frozenset[str] | set[str] = _JSON_WRAPPER_METADATA_KEYS,
|
||||
) -> tuple[list[Any], tuple[str, ...]] | None:
|
||||
candidates = [
|
||||
key
|
||||
for key in _JSON_RECORD_ARRAY_KEYS
|
||||
if isinstance(value.get(key), list)
|
||||
]
|
||||
if len(candidates) != 1:
|
||||
return None
|
||||
record_key = candidates[0]
|
||||
records = value[record_key]
|
||||
if any(not isinstance(record, Mapping) for record in records):
|
||||
return None
|
||||
if any(
|
||||
key != record_key and key not in metadata_keys
|
||||
for key in value
|
||||
):
|
||||
return None
|
||||
return records, (record_key,)
|
||||
|
||||
direct = direct_wrapper(payload)
|
||||
if direct is not None:
|
||||
return direct
|
||||
|
||||
envelope_keys = [
|
||||
key
|
||||
for key in _JSON_ENVELOPE_KEYS
|
||||
if isinstance(payload.get(key), Mapping)
|
||||
]
|
||||
if len(envelope_keys) != 1:
|
||||
return None
|
||||
envelope_key = envelope_keys[0]
|
||||
if any(
|
||||
key != envelope_key and key not in _JSON_RESPONSE_METADATA_KEYS
|
||||
for key in payload
|
||||
):
|
||||
return None
|
||||
nested = direct_wrapper(
|
||||
payload[envelope_key],
|
||||
_JSON_RESPONSE_METADATA_KEYS,
|
||||
)
|
||||
if nested is None:
|
||||
return None
|
||||
records, nested_path = nested
|
||||
return records, (envelope_key, *nested_path)
|
||||
|
||||
|
||||
def _json_record_locator(
|
||||
text: str,
|
||||
*,
|
||||
record_index: int,
|
||||
json_pointer: str,
|
||||
span: tuple[int, int],
|
||||
) -> dict[str, Any]:
|
||||
source_start, source_end = span
|
||||
start_line = text.count("\n", 0, source_start) + 1
|
||||
last_character = max(source_start, source_end - 1)
|
||||
end_line = text.count("\n", 0, last_character) + 1
|
||||
return {
|
||||
"kind": "json",
|
||||
"record_index": record_index,
|
||||
"json_pointer": json_pointer,
|
||||
"source_start": source_start,
|
||||
"source_end": source_end,
|
||||
"start_line": start_line,
|
||||
"end_line": end_line,
|
||||
}
|
||||
|
||||
|
||||
def _extract_structured_records_with_locators(
|
||||
text: str,
|
||||
file_format: str,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""从 JSON、JSONL 或 CSV 中提取记录。
|
||||
|
||||
JSON 根数组始终表示多条记录；对象仅在满足纯包装契约时展开，其他
|
||||
对象均视为一条业务记录。JSON 字段和值在解析阶段保持原样，只有用户
|
||||
明确选择 ``normalize_format`` 后才会规范化。
|
||||
"""
|
||||
|
||||
normalized_format = _normalize_format(file_format)
|
||||
if normalized_format not in {"json", "jsonl", "csv"}:
|
||||
raise ValueError("structured record extraction only supports JSON, JSONL and CSV")
|
||||
|
||||
if normalized_format == "json":
|
||||
if _skip_json_whitespace(text, 0) == len(text):
|
||||
return [], []
|
||||
payload, root_start, root_end = _strict_json_loads(text)
|
||||
values: Sequence[Any]
|
||||
pointer_path: tuple[str, ...] = ()
|
||||
record_spans: list[tuple[int, int]]
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
record_spans = _json_array_item_spans(text, root_start, root_end)
|
||||
else:
|
||||
wrapper = _pure_json_record_wrapper(payload)
|
||||
if wrapper is None:
|
||||
values = [payload]
|
||||
record_spans = [(root_start, root_end)]
|
||||
else:
|
||||
values, pointer_path = wrapper
|
||||
array_span = _json_span_at_path(
|
||||
text,
|
||||
(root_start, root_end),
|
||||
pointer_path,
|
||||
)
|
||||
record_spans = _json_array_item_spans(text, *array_span)
|
||||
if len(record_spans) != len(values):
|
||||
raise ValueError("JSON record source spans do not match parsed records")
|
||||
records = [_record_from_value(value, normalize=False) for value in values]
|
||||
pointer_prefix = "".join(
|
||||
f"/{_json_pointer_segment(segment)}" for segment in pointer_path
|
||||
)
|
||||
locators = [
|
||||
_json_record_locator(
|
||||
text,
|
||||
record_index=index + 1,
|
||||
json_pointer=(
|
||||
f"{pointer_prefix}/{index}"
|
||||
if pointer_path or isinstance(payload, list)
|
||||
else ""
|
||||
),
|
||||
span=record_spans[index],
|
||||
)
|
||||
for index in range(len(records))
|
||||
]
|
||||
return records, locators
|
||||
|
||||
if normalized_format == "jsonl":
|
||||
records: list[dict[str, Any]] = []
|
||||
locators: list[dict[str, Any]] = []
|
||||
source_offset = 0
|
||||
for line_number, line in enumerate(text.split("\n"), start=1):
|
||||
line_content_end = len(line)
|
||||
if _skip_json_whitespace(line, 0) == line_content_end:
|
||||
source_offset += len(line) + 1
|
||||
continue
|
||||
try:
|
||||
value, value_start, value_end = _strict_json_loads(line)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSONL at line {line_number}: {exc}"
|
||||
) from exc
|
||||
records.append(_record_from_value(value, normalize=False))
|
||||
locators.append(
|
||||
{
|
||||
"kind": "jsonl",
|
||||
"record_index": len(records),
|
||||
"start_line": line_number,
|
||||
"end_line": line_number,
|
||||
"source_start": source_offset + value_start,
|
||||
"source_end": source_offset + value_end,
|
||||
}
|
||||
)
|
||||
source_offset += len(line) + 1
|
||||
return records, locators
|
||||
|
||||
# CSV 分支独立放在 csv_parser 中，避免与 JSON 机制耦合。
|
||||
from .csv_parser import _extract_csv_records_with_locators
|
||||
|
||||
return _extract_csv_records_with_locators(text)
|
||||
700
backend/app/modules/data_process/algorithms/parsers/office.py
Normal file
700
backend/app/modules/data_process/algorithms/parsers/office.py
Normal file
@@ -0,0 +1,700 @@
|
||||
"""数据处理算法 - Office 文档解析。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import unicodedata
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from docx import Document
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils.cell import range_boundaries
|
||||
from pptx import Presentation
|
||||
|
||||
from ..text_utils import _append_bounded_text, _normalize_spreadsheet_value, normalize_text
|
||||
from ..types import (
|
||||
_MAX_ARCHIVE_COMPRESSION_RATIO,
|
||||
_MAX_ARCHIVE_ENTRIES,
|
||||
_MAX_ARCHIVE_ENTRY_BYTES,
|
||||
_MAX_ARCHIVE_UNCOMPRESSED_BYTES,
|
||||
_MAX_PRESENTATION_SLIDES,
|
||||
_MAX_WORKBOOK_CELLS,
|
||||
_MAX_WORKBOOK_COLUMNS,
|
||||
_MAX_WORKBOOK_HEADER_ROWS,
|
||||
_MAX_WORKBOOK_HEADER_SCAN_ROWS,
|
||||
_MAX_WORKBOOK_MERGED_RANGES,
|
||||
_MAX_WORKBOOK_ROWS,
|
||||
_MAX_WORKBOOK_SCANNED_ROWS,
|
||||
_MAX_WORKBOOK_SHEETS,
|
||||
TextFormat,
|
||||
)
|
||||
|
||||
_XLSX_REPORT_METADATA_PATTERN = re.compile(
|
||||
r"^(?:报表|报告|标题|说明|备注|制表|统计|日期|时间|期间|"
|
||||
r"report|title|note|remark|date|time|period)\b",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
|
||||
"""在交给 Office 解析库前限制 ZIP 包规模并拒绝活动 XML。"""
|
||||
|
||||
required_members = {
|
||||
"docx": {"[Content_Types].xml", "word/document.xml"},
|
||||
"xlsx": {"[Content_Types].xml", "xl/workbook.xml"},
|
||||
"pptx": {"[Content_Types].xml", "ppt/presentation.xml"},
|
||||
}
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
|
||||
members = archive.infolist()
|
||||
if len(members) > _MAX_ARCHIVE_ENTRIES:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive contains too many entries "
|
||||
f"(limit {_MAX_ARCHIVE_ENTRIES})"
|
||||
)
|
||||
|
||||
names: set[str] = set()
|
||||
total_size = 0
|
||||
for member in members:
|
||||
path = PurePosixPath(member.filename.replace("\\", "/"))
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive contains an unsafe member path"
|
||||
)
|
||||
if member.filename in names:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive contains duplicate member names"
|
||||
)
|
||||
names.add(member.filename)
|
||||
if member.flag_bits & 0x1:
|
||||
raise ValueError(f"encrypted {file_format.upper()} files are not supported")
|
||||
if member.is_dir():
|
||||
continue
|
||||
if member.file_size > _MAX_ARCHIVE_ENTRY_BYTES:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive entry exceeds "
|
||||
f"{_MAX_ARCHIVE_ENTRY_BYTES} bytes"
|
||||
)
|
||||
total_size += member.file_size
|
||||
if total_size > _MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive expands beyond "
|
||||
f"{_MAX_ARCHIVE_UNCOMPRESSED_BYTES} bytes"
|
||||
)
|
||||
if member.file_size >= 1024 * 1024:
|
||||
if member.compress_size <= 0:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive has an unsafe compression ratio"
|
||||
)
|
||||
ratio = member.file_size / member.compress_size
|
||||
if ratio > _MAX_ARCHIVE_COMPRESSION_RATIO:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive has an unsafe compression ratio"
|
||||
)
|
||||
|
||||
missing = required_members[file_format] - names
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"invalid {file_format.upper()} package: missing "
|
||||
f"{', '.join(sorted(missing))}"
|
||||
)
|
||||
|
||||
for member in members:
|
||||
if member.is_dir() or not member.filename.lower().endswith((".xml", ".rels")):
|
||||
continue
|
||||
with archive.open(member) as stream:
|
||||
prefix = stream.read(min(member.file_size, 1024 * 1024)).upper()
|
||||
if b"<!DOCTYPE" in prefix or b"<!ENTITY" in prefix:
|
||||
raise ValueError(
|
||||
f"{file_format.upper()} archive contains unsupported active XML"
|
||||
)
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
|
||||
|
||||
def _extract_docx_text(raw: bytes) -> str:
|
||||
_validate_office_archive(raw, "docx")
|
||||
try:
|
||||
document = Document(io.BytesIO(raw))
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid DOCX file: {exc}") from exc
|
||||
|
||||
parts: list[str] = []
|
||||
total = 0
|
||||
for child in document.element.body.iterchildren():
|
||||
if isinstance(child, CT_P):
|
||||
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
|
||||
continue
|
||||
if isinstance(child, CT_Tbl):
|
||||
table = Table(child, document)
|
||||
for row in table.rows:
|
||||
cells = [normalize_text(cell.text) for cell in row.cells]
|
||||
total = _append_bounded_text(parts, "\t".join(cells), total)
|
||||
return normalize_text("\n\n".join(parts))
|
||||
|
||||
def _presentation_shape_text(shape: Any) -> list[str]:
|
||||
if getattr(shape, "has_table", False):
|
||||
return [
|
||||
"\t".join(normalize_text(cell.text) for cell in row.cells)
|
||||
for row in shape.table.rows
|
||||
]
|
||||
if getattr(shape, "has_text_frame", False):
|
||||
return [shape.text]
|
||||
child_shapes = getattr(shape, "shapes", None)
|
||||
if child_shapes is not None:
|
||||
values: list[str] = []
|
||||
for child in child_shapes:
|
||||
values.extend(_presentation_shape_text(child))
|
||||
return values
|
||||
return []
|
||||
|
||||
def _extract_pptx_text(raw: bytes) -> str:
|
||||
_validate_office_archive(raw, "pptx")
|
||||
try:
|
||||
presentation = Presentation(io.BytesIO(raw))
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid PPTX file: {exc}") from exc
|
||||
if len(presentation.slides) > _MAX_PRESENTATION_SLIDES:
|
||||
raise ValueError(
|
||||
f"PPTX contains too many slides (limit {_MAX_PRESENTATION_SLIDES})"
|
||||
)
|
||||
|
||||
parts: list[str] = []
|
||||
total = 0
|
||||
for slide in presentation.slides:
|
||||
slide_parts: list[str] = []
|
||||
for shape in slide.shapes:
|
||||
slide_parts.extend(_presentation_shape_text(shape))
|
||||
total = _append_bounded_text(parts, "\n".join(slide_parts), total)
|
||||
return normalize_text("\n\n".join(parts))
|
||||
|
||||
def _xml_local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
def _resolve_xlsx_relationship_target(
|
||||
archive: zipfile.ZipFile,
|
||||
target: str,
|
||||
*,
|
||||
source_part: str = "xl/workbook.xml",
|
||||
) -> str:
|
||||
"""按 OPC URI 规则解析内部关系目标，并保证结果仍位于 ZIP 根内。"""
|
||||
|
||||
raw_target = target.strip()
|
||||
if (
|
||||
raw_target != target
|
||||
or not raw_target
|
||||
or "\\" in raw_target
|
||||
or any(unicodedata.category(char).startswith("C") for char in raw_target)
|
||||
or re.search(r"%(?![0-9A-Fa-f]{2})", raw_target)
|
||||
):
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
try:
|
||||
parsed = urlsplit(raw_target)
|
||||
except ValueError as exc:
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path") from exc
|
||||
if parsed.scheme or parsed.netloc or parsed.query or parsed.fragment:
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
if re.search(r"%(?:2[fF]|5[cC]|0{2})", parsed.path):
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
try:
|
||||
decoded_path = unquote(parsed.path, encoding="utf-8", errors="strict")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path") from exc
|
||||
if not decoded_path or "\\" in decoded_path or "\x00" in decoded_path or "%" in decoded_path:
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
|
||||
parts = [] if decoded_path.startswith("/") else list(PurePosixPath(source_part).parent.parts)
|
||||
for part in decoded_path.lstrip("/").split("/"):
|
||||
if part in {"", "."}:
|
||||
continue
|
||||
if part == "..":
|
||||
if not parts:
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
parts.pop()
|
||||
continue
|
||||
if unicodedata.category(part[0]).startswith("C") or any(
|
||||
unicodedata.category(char).startswith("C") for char in part
|
||||
):
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
parts.append(part)
|
||||
if not parts:
|
||||
raise ValueError("XLSX workbook contains an unsafe worksheet path")
|
||||
|
||||
member_name = "/".join(parts)
|
||||
try:
|
||||
member = archive.getinfo(member_name)
|
||||
except KeyError as exc:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet relationship target does not exist: {member_name}"
|
||||
) from exc
|
||||
if member.is_dir():
|
||||
raise ValueError("XLSX worksheet relationship target must be a file")
|
||||
return member_name
|
||||
|
||||
def _rewrite_xlsx_workbook_relationships(
|
||||
raw: bytes,
|
||||
replacements: Mapping[str, str],
|
||||
) -> bytes:
|
||||
"""把已验证的 worksheet Target 改为解析库稳定支持的包内绝对路径。"""
|
||||
|
||||
relationships_member = "xl/_rels/workbook.xml.rels"
|
||||
output = io.BytesIO()
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as source, zipfile.ZipFile(output, "w") as target:
|
||||
target.comment = source.comment
|
||||
for member in source.infolist():
|
||||
if member.filename == relationships_member:
|
||||
root = ET.fromstring(source.read(member))
|
||||
pending = dict(replacements)
|
||||
for element in root:
|
||||
if _xml_local_name(element.tag) != "Relationship":
|
||||
continue
|
||||
relationship_id = element.attrib.get("Id")
|
||||
if relationship_id in pending:
|
||||
element.set("Target", pending.pop(relationship_id))
|
||||
if pending:
|
||||
raise ValueError(
|
||||
"XLSX workbook relationship changed during normalization"
|
||||
)
|
||||
content = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
||||
target.writestr(member, content)
|
||||
continue
|
||||
if member.is_dir():
|
||||
target.writestr(member, b"")
|
||||
continue
|
||||
with source.open(member) as source_stream, target.open(
|
||||
member,
|
||||
"w",
|
||||
force_zip64=True,
|
||||
) as target_stream:
|
||||
while chunk := source_stream.read(1024 * 1024):
|
||||
target_stream.write(chunk)
|
||||
return output.getvalue()
|
||||
|
||||
def _xlsx_sheet_merge_ranges(
|
||||
raw: bytes,
|
||||
) -> tuple[
|
||||
dict[str, tuple[tuple[int, int, int, int], ...]],
|
||||
dict[str, str],
|
||||
]:
|
||||
"""流式读取 XLSX 合并单元格，不把工作表 XML 整体载入内存。"""
|
||||
|
||||
relationship_namespace = (
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
||||
)
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
|
||||
relationships: dict[str, tuple[str, str, str]] = {}
|
||||
with archive.open("xl/_rels/workbook.xml.rels") as stream:
|
||||
for _, element in ET.iterparse(stream, events=("end",)):
|
||||
if _xml_local_name(element.tag) != "Relationship":
|
||||
element.clear()
|
||||
continue
|
||||
relationship_id = element.attrib.get("Id")
|
||||
target = element.attrib.get("Target")
|
||||
target_mode = element.attrib.get("TargetMode", "Internal")
|
||||
relationship_type = element.attrib.get("Type", "")
|
||||
if relationship_id:
|
||||
if relationship_id in relationships:
|
||||
raise ValueError(
|
||||
"XLSX workbook contains duplicate relationship identifiers"
|
||||
)
|
||||
relationships[relationship_id] = (
|
||||
target or "",
|
||||
target_mode,
|
||||
relationship_type,
|
||||
)
|
||||
element.clear()
|
||||
|
||||
sheet_paths: dict[str, str] = {}
|
||||
normalized_targets: dict[str, str] = {}
|
||||
with archive.open("xl/workbook.xml") as stream:
|
||||
for _, element in ET.iterparse(stream, events=("end",)):
|
||||
if _xml_local_name(element.tag) != "sheet":
|
||||
element.clear()
|
||||
continue
|
||||
title = element.attrib.get("name")
|
||||
relationship_id = element.attrib.get(
|
||||
f"{{{relationship_namespace}}}id"
|
||||
)
|
||||
if title and relationship_id:
|
||||
relationship = relationships.get(relationship_id)
|
||||
if relationship is None:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {title!r} references a missing relationship"
|
||||
)
|
||||
target, target_mode, relationship_type = relationship
|
||||
if target_mode.strip().lower() != "internal":
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {title!r} uses an external relationship"
|
||||
)
|
||||
if not relationship_type.endswith("/worksheet"):
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {title!r} has an invalid relationship type"
|
||||
)
|
||||
member_name = _resolve_xlsx_relationship_target(
|
||||
archive,
|
||||
target,
|
||||
)
|
||||
sheet_paths[title] = member_name
|
||||
canonical_target = f"/{member_name}"
|
||||
if target != canonical_target:
|
||||
normalized_targets[relationship_id] = canonical_target
|
||||
element.clear()
|
||||
|
||||
result: dict[str, tuple[tuple[int, int, int, int], ...]] = {}
|
||||
total_ranges = 0
|
||||
for title, member_name in sheet_paths.items():
|
||||
ranges: list[tuple[int, int, int, int]] = []
|
||||
with archive.open(member_name) as stream:
|
||||
for _, element in ET.iterparse(stream, events=("end",)):
|
||||
if _xml_local_name(element.tag) != "mergeCell":
|
||||
element.clear()
|
||||
continue
|
||||
reference = element.attrib.get("ref")
|
||||
if reference:
|
||||
try:
|
||||
boundaries = range_boundaries(reference)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {title!r} has an invalid merged range"
|
||||
) from exc
|
||||
ranges.append(boundaries)
|
||||
total_ranges += 1
|
||||
if total_ranges > _MAX_WORKBOOK_MERGED_RANGES:
|
||||
raise ValueError(
|
||||
"XLSX workbook contains too many merged ranges "
|
||||
f"(limit {_MAX_WORKBOOK_MERGED_RANGES})"
|
||||
)
|
||||
element.clear()
|
||||
result[title] = tuple(ranges)
|
||||
return result, normalized_targets
|
||||
except (KeyError, ET.ParseError, zipfile.BadZipFile) as exc:
|
||||
raise ValueError(f"invalid XLSX workbook structure: {exc}") from exc
|
||||
|
||||
def _xlsx_header_end_row(
|
||||
first_row: int,
|
||||
rows: Mapping[int, Sequence[Any]],
|
||||
merged_ranges: Sequence[tuple[int, int, int, int]],
|
||||
) -> int:
|
||||
"""在固定深度内闭包表头合并关系，忽略越界或跨空行的可疑级联。"""
|
||||
|
||||
header_end = first_row
|
||||
maximum_end = first_row + _MAX_WORKBOOK_HEADER_ROWS - 1
|
||||
has_header_hierarchy = any(
|
||||
max_column > min_column and min_row == first_row
|
||||
for min_column, min_row, max_column, _ in merged_ranges
|
||||
)
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for min_column, min_row, max_column, max_row in merged_ranges:
|
||||
if min_row < first_row or min_row > header_end or max_row < first_row:
|
||||
continue
|
||||
if max_column == min_column and not has_header_hierarchy:
|
||||
continue
|
||||
candidate = max_row + 1 if max_column > min_column else max_row
|
||||
if candidate <= header_end or candidate > maximum_end:
|
||||
continue
|
||||
if (
|
||||
max_column > min_column
|
||||
and len(_xlsx_nonempty_values(rows.get(min_row, ()))) < 2
|
||||
and len(_xlsx_nonempty_values(rows.get(candidate, ()))) < 2
|
||||
):
|
||||
continue
|
||||
if any(
|
||||
not rows.get(row_number)
|
||||
for row_number in range(header_end + 1, candidate + 1)
|
||||
):
|
||||
continue
|
||||
header_end = candidate
|
||||
changed = True
|
||||
return header_end
|
||||
|
||||
def _xlsx_headers(
|
||||
title: str,
|
||||
rows: Mapping[int, Sequence[Any]],
|
||||
first_row: int,
|
||||
header_end: int,
|
||||
merged_ranges: Sequence[tuple[int, int, int, int]],
|
||||
) -> list[str]:
|
||||
width = max((len(row) for row in rows.values()), default=0)
|
||||
for min_column, min_row, max_column, max_row in merged_ranges:
|
||||
if min_row <= header_end and max_row >= first_row:
|
||||
width = max(width, max_column)
|
||||
if width > _MAX_WORKBOOK_COLUMNS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {title!r} exceeds {_MAX_WORKBOOK_COLUMNS} columns"
|
||||
)
|
||||
|
||||
matrix = [
|
||||
list(rows.get(row_number, ())) + [None] * (width - len(rows.get(row_number, ())))
|
||||
for row_number in range(first_row, header_end + 1)
|
||||
]
|
||||
for min_column, min_row, max_column, max_row in merged_ranges:
|
||||
if min_row > header_end or max_row < first_row:
|
||||
continue
|
||||
source_row = max(first_row, min_row) - first_row
|
||||
source_column = min_column - 1
|
||||
source = matrix[source_row][source_column]
|
||||
for row_number in range(max(first_row, min_row), min(header_end, max_row) + 1):
|
||||
for column_number in range(min_column, max_column + 1):
|
||||
matrix[row_number - first_row][column_number - 1] = source
|
||||
|
||||
headers: list[str] = []
|
||||
for column in range(width):
|
||||
components: list[str] = []
|
||||
for row in matrix:
|
||||
component = normalize_text(str(row[column] or ""))
|
||||
if component and (not components or component != components[-1]):
|
||||
components.append(component)
|
||||
header = ".".join(components)
|
||||
if not header:
|
||||
raise ValueError(f"XLSX worksheet {title!r} contains an empty header")
|
||||
headers.append(header)
|
||||
if len(set(headers)) != len(headers):
|
||||
raise ValueError(f"XLSX worksheet {title!r} contains duplicate headers")
|
||||
return headers
|
||||
|
||||
def _xlsx_nonempty_values(row: Sequence[Any]) -> list[Any]:
|
||||
return [value for value in row if value not in {None, ""}]
|
||||
|
||||
def _infer_xlsx_header_region(
|
||||
title: str,
|
||||
rows: Mapping[int, Sequence[Any]],
|
||||
merged_ranges: Sequence[tuple[int, int, int, int]],
|
||||
) -> tuple[int, int, list[str]]:
|
||||
"""从有界前缀中推断表头，区分报表说明、多级表头和正文合并。"""
|
||||
|
||||
ordered_rows = sorted(rows)
|
||||
if not ordered_rows:
|
||||
return 0, 0, []
|
||||
first_nonempty_row = ordered_rows[0]
|
||||
horizontal_merge_rows = {
|
||||
min_row
|
||||
for min_column, min_row, max_column, _ in merged_ranges
|
||||
if max_column > min_column
|
||||
}
|
||||
candidates: list[tuple[float, int, int, list[str]]] = []
|
||||
for first_row in ordered_rows:
|
||||
raw_values = _xlsx_nonempty_values(rows[first_row])
|
||||
if not raw_values:
|
||||
continue
|
||||
if len(raw_values) < 2 and first_row in horizontal_merge_rows:
|
||||
continue
|
||||
|
||||
header_end = _xlsx_header_end_row(first_row, rows, merged_ranges)
|
||||
header_rows = {
|
||||
row_number: rows[row_number]
|
||||
for row_number in range(first_row, header_end + 1)
|
||||
if row_number in rows
|
||||
}
|
||||
try:
|
||||
headers = _xlsx_headers(
|
||||
title,
|
||||
header_rows,
|
||||
first_row,
|
||||
header_end,
|
||||
merged_ranges,
|
||||
)
|
||||
except ValueError:
|
||||
if header_end == first_row:
|
||||
continue
|
||||
header_end = first_row
|
||||
try:
|
||||
headers = _xlsx_headers(
|
||||
title,
|
||||
{first_row: rows[first_row]},
|
||||
first_row,
|
||||
first_row,
|
||||
merged_ranges,
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
data_rows = [rows[row_number] for row_number in ordered_rows if row_number > header_end]
|
||||
if any(len(row) > len(headers) for row in data_rows):
|
||||
continue
|
||||
|
||||
text_ratio = sum(isinstance(value, str) for value in raw_values) / len(raw_values)
|
||||
score = text_ratio * 6 + min(len(raw_values), 4)
|
||||
if data_rows:
|
||||
first_data_values = _xlsx_nonempty_values(data_rows[0])
|
||||
score += 4 * min(len(first_data_values), len(headers)) / len(headers)
|
||||
if first_data_values:
|
||||
score += (
|
||||
2
|
||||
* sum(
|
||||
not isinstance(value, str)
|
||||
for value in first_data_values
|
||||
)
|
||||
/ len(first_data_values)
|
||||
)
|
||||
first_label = normalize_text(str(raw_values[0]))
|
||||
if len(raw_values) <= 2 and _XLSX_REPORT_METADATA_PATTERN.match(first_label):
|
||||
score -= 8
|
||||
candidates.append((score, first_row, header_end, headers))
|
||||
|
||||
if not candidates:
|
||||
first_row = first_nonempty_row
|
||||
headers = _xlsx_headers(
|
||||
title,
|
||||
{first_row: rows[first_row]},
|
||||
first_row,
|
||||
first_row,
|
||||
(),
|
||||
)
|
||||
return first_row, first_row, headers
|
||||
_, first_row, header_end, headers = max(
|
||||
candidates,
|
||||
key=lambda candidate: (candidate[0], -candidate[1]),
|
||||
)
|
||||
return first_row, header_end, headers
|
||||
|
||||
def _extract_xlsx_records(
|
||||
raw: bytes,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
_validate_office_archive(raw, "xlsx")
|
||||
merged_by_sheet, normalized_targets = _xlsx_sheet_merge_ranges(raw)
|
||||
workbook_raw = (
|
||||
_rewrite_xlsx_workbook_relationships(raw, normalized_targets)
|
||||
if normalized_targets
|
||||
else raw
|
||||
)
|
||||
try:
|
||||
workbook = load_workbook(
|
||||
io.BytesIO(workbook_raw),
|
||||
read_only=True,
|
||||
data_only=True,
|
||||
keep_links=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid XLSX file: {exc}") from exc
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
locators: list[dict[str, Any]] = []
|
||||
total_cells = 0
|
||||
try:
|
||||
if len(workbook.worksheets) > _MAX_WORKBOOK_SHEETS:
|
||||
raise ValueError(
|
||||
f"XLSX contains too many worksheets (limit {_MAX_WORKBOOK_SHEETS})"
|
||||
)
|
||||
for sheet_index, worksheet in enumerate(workbook.worksheets):
|
||||
reset_dimensions = getattr(worksheet, "reset_dimensions", None)
|
||||
if callable(reset_dimensions):
|
||||
reset_dimensions()
|
||||
merged_ranges = merged_by_sheet.get(worksheet.title, ())
|
||||
sheet_rows = 0
|
||||
scanned_rows = 0
|
||||
row_iterator = enumerate(
|
||||
worksheet.iter_rows(values_only=True),
|
||||
start=1,
|
||||
)
|
||||
buffered_rows: dict[int, Sequence[Any]] = {}
|
||||
|
||||
def normalized_row_values(
|
||||
row: Sequence[Any],
|
||||
sheet_title: str = worksheet.title,
|
||||
) -> list[Any]:
|
||||
values = list(row)
|
||||
while values and values[-1] in {None, ""}:
|
||||
values.pop()
|
||||
if len(values) > _MAX_WORKBOOK_COLUMNS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {sheet_title!r} exceeds "
|
||||
f"{_MAX_WORKBOOK_COLUMNS} columns"
|
||||
)
|
||||
return values
|
||||
|
||||
for row_number, row in row_iterator:
|
||||
scanned_rows += 1
|
||||
if scanned_rows > _MAX_WORKBOOK_SCANNED_ROWS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {worksheet.title!r} exceeds "
|
||||
f"{_MAX_WORKBOOK_SCANNED_ROWS} scanned rows"
|
||||
)
|
||||
values = normalized_row_values(row)
|
||||
if not values or all(value in {None, ""} for value in values):
|
||||
continue
|
||||
buffered_rows[row_number] = tuple(values)
|
||||
if len(buffered_rows) >= _MAX_WORKBOOK_HEADER_SCAN_ROWS:
|
||||
break
|
||||
|
||||
if not buffered_rows:
|
||||
continue
|
||||
_, header_end_row, headers = _infer_xlsx_header_region(
|
||||
worksheet.title,
|
||||
buffered_rows,
|
||||
merged_ranges,
|
||||
)
|
||||
|
||||
def append_record(
|
||||
row_number: int,
|
||||
values: Sequence[Any],
|
||||
record_headers: Sequence[str] = tuple(headers),
|
||||
locator_sheet_index: int = sheet_index,
|
||||
sheet_title: str = worksheet.title,
|
||||
) -> None:
|
||||
nonlocal total_cells, sheet_rows
|
||||
row_values = list(values)
|
||||
if len(row_values) > len(record_headers):
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {sheet_title!r} has a row wider than its header"
|
||||
)
|
||||
row_values.extend([None] * (len(record_headers) - len(row_values)))
|
||||
record = {
|
||||
header: _normalize_spreadsheet_value(value)
|
||||
for header, value in zip(record_headers, row_values, strict=True)
|
||||
}
|
||||
if not any(value not in {"", None} for value in record.values()):
|
||||
return
|
||||
sheet_record_index = sheet_rows
|
||||
sheet_rows += 1
|
||||
total_cells += len(record_headers)
|
||||
if sheet_rows > _MAX_WORKBOOK_ROWS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {sheet_title!r} exceeds "
|
||||
f"{_MAX_WORKBOOK_ROWS} data rows"
|
||||
)
|
||||
if total_cells > _MAX_WORKBOOK_CELLS:
|
||||
raise ValueError(
|
||||
f"XLSX workbook exceeds {_MAX_WORKBOOK_CELLS} populated cells"
|
||||
)
|
||||
records.append(record)
|
||||
locators.append(
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": len(records),
|
||||
"sheet_index": locator_sheet_index,
|
||||
"sheet_name": sheet_title,
|
||||
"row_number": row_number,
|
||||
"sheet_record_index": sheet_record_index,
|
||||
}
|
||||
)
|
||||
|
||||
for row_number, values in buffered_rows.items():
|
||||
if row_number > header_end_row:
|
||||
append_record(row_number, values)
|
||||
|
||||
for row_number, row in row_iterator:
|
||||
scanned_rows += 1
|
||||
if scanned_rows > _MAX_WORKBOOK_SCANNED_ROWS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {worksheet.title!r} exceeds "
|
||||
f"{_MAX_WORKBOOK_SCANNED_ROWS} scanned rows"
|
||||
)
|
||||
values = normalized_row_values(row)
|
||||
if not values or all(value in {None, ""} for value in values):
|
||||
continue
|
||||
append_record(row_number, values)
|
||||
finally:
|
||||
workbook.close()
|
||||
return records, locators
|
||||
297
backend/app/modules/data_process/algorithms/parsers/pdf.py
Normal file
297
backend/app/modules/data_process/algorithms/parsers/pdf.py
Normal file
@@ -0,0 +1,297 @@
|
||||
"""数据处理算法 - PDF 文档解析。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Sequence
|
||||
from typing import Literal
|
||||
|
||||
from pypdf import PdfReader
|
||||
|
||||
from ..text_utils import normalize_text
|
||||
from ..types import (
|
||||
DocumentNoiseSpan,
|
||||
PdfPageText,
|
||||
_MAX_EXTRACTED_TEXT_CHARS,
|
||||
_MAX_PDF_PAGES,
|
||||
_PdfLine,
|
||||
)
|
||||
|
||||
_PDF_PAGE_NUMBER_LINE_PATTERN = re.compile(
|
||||
r"^(?:页次\s*)?(?:第\s*)?(?P<page>\d+)\s*页\s*"
|
||||
r"(?:(?:[/／]\s*)?共\s*(?P<total>\d+)\s*页)?$"
|
||||
)
|
||||
_PDF_FRACTION_PAGE_LINE_PATTERN = re.compile(
|
||||
r"^[—–-]?\s*(?P<page>\d+)\s*[/／]\s*(?P<total>\d+)\s*[—–-]?$"
|
||||
)
|
||||
_PDF_CLASSIFICATION_LABEL_PATTERN = re.compile(
|
||||
r"^(?:(?:秘密等级|密级)\s*)?(?:商密|秘密|机密|绝密)"
|
||||
r"\s*(?:[【\[(（][^】\])）]{1,8}[】\])）])?$"
|
||||
)
|
||||
_TOC_TITLE_PATTERN = re.compile(r"^(?:目\s*录|contents)$", re.IGNORECASE)
|
||||
_TOC_LEADER_ENTRY_PATTERN = re.compile(
|
||||
r"(?:[.．…·•]\s*){3,}\s*\d{1,4}\s*$"
|
||||
)
|
||||
_TOC_NUMBERED_ENTRY_PATTERN = re.compile(
|
||||
r"^(?:第[\u3400-\u4dbf\u4e00-\u9fff]{1,12}章|附表\s*\d+|\d+(?:\.\d+)+)"
|
||||
r"\s+.+\s+\d{1,4}\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_MARGIN_TEMPLATE_KEYWORDS = (
|
||||
"页",
|
||||
"页次",
|
||||
"版本",
|
||||
"文件编码",
|
||||
"秘密等级",
|
||||
"密级",
|
||||
"商密",
|
||||
"confidential",
|
||||
)
|
||||
|
||||
|
||||
def extract_pdf_page_texts(raw: bytes) -> tuple[PdfPageText, ...]:
|
||||
"""提取 PDF 各页文本，并保留与切片字符偏移一致的页范围。"""
|
||||
|
||||
if b"%PDF-" not in raw[:1024]:
|
||||
raise ValueError("invalid PDF file: missing PDF header")
|
||||
try:
|
||||
reader = PdfReader(io.BytesIO(raw), strict=True)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid PDF file: {exc}") from exc
|
||||
if reader.is_encrypted and not reader.decrypt(""):
|
||||
raise ValueError("password-protected PDF files are not supported")
|
||||
if len(reader.pages) > _MAX_PDF_PAGES:
|
||||
raise ValueError(f"PDF contains too many pages (limit {_MAX_PDF_PAGES})")
|
||||
|
||||
pages: list[PdfPageText] = []
|
||||
total = 0
|
||||
has_text = False
|
||||
for page_number, page in enumerate(reader.pages, start=1):
|
||||
try:
|
||||
text = normalize_text(page.extract_text() or "")
|
||||
except Exception as exc:
|
||||
raise ValueError(
|
||||
f"failed to extract text from PDF page {page_number}: {exc}"
|
||||
) from exc
|
||||
if not text:
|
||||
pages.append(
|
||||
PdfPageText(
|
||||
page_number=page_number,
|
||||
text="",
|
||||
source_start=total,
|
||||
source_end=total,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if has_text:
|
||||
total += 2
|
||||
start = total
|
||||
total += len(text)
|
||||
if total > _MAX_EXTRACTED_TEXT_CHARS:
|
||||
raise ValueError(
|
||||
f"extracted document text exceeds {_MAX_EXTRACTED_TEXT_CHARS} characters"
|
||||
)
|
||||
pages.append(
|
||||
PdfPageText(
|
||||
page_number=page_number,
|
||||
text=text,
|
||||
source_start=start,
|
||||
source_end=total,
|
||||
)
|
||||
)
|
||||
has_text = True
|
||||
if not has_text:
|
||||
raise ValueError("PDF contains no extractable text; scanned PDF requires OCR")
|
||||
return tuple(pages)
|
||||
|
||||
def _pdf_page_lines(page: PdfPageText) -> tuple[_PdfLine, ...]:
|
||||
lines: list[_PdfLine] = []
|
||||
local_offset = 0
|
||||
for raw_line in page.text.splitlines(keepends=True):
|
||||
content = raw_line.rstrip("\r\n")
|
||||
leading = len(content) - len(content.lstrip())
|
||||
trailing = len(content.rstrip())
|
||||
text = content.strip()
|
||||
if text:
|
||||
lines.append(
|
||||
_PdfLine(
|
||||
text=text,
|
||||
start=page.source_start + local_offset + leading,
|
||||
end=page.source_start + local_offset + trailing,
|
||||
)
|
||||
)
|
||||
local_offset += len(raw_line)
|
||||
return tuple(lines)
|
||||
|
||||
def _is_standalone_page_number(
|
||||
text: str,
|
||||
*,
|
||||
physical_page: int,
|
||||
page_count: int,
|
||||
) -> bool:
|
||||
normalized = unicodedata.normalize("NFKC", text).strip()
|
||||
match = _PDF_PAGE_NUMBER_LINE_PATTERN.fullmatch(normalized)
|
||||
if match is None:
|
||||
match = _PDF_FRACTION_PAGE_LINE_PATTERN.fullmatch(normalized)
|
||||
if match is None or int(match.group("page")) != physical_page:
|
||||
return False
|
||||
total = match.groupdict().get("total")
|
||||
return total is None or int(total) == page_count
|
||||
|
||||
def _margin_signature(text: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKC", text).casefold()
|
||||
normalized = re.sub(r"\s+", " ", normalized).strip()
|
||||
if any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS):
|
||||
normalized = re.sub(r"\d+", "#", normalized)
|
||||
return normalized
|
||||
|
||||
def _has_margin_metadata_keyword(text: str) -> bool:
|
||||
normalized = unicodedata.normalize("NFKC", text).casefold()
|
||||
return any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS)
|
||||
|
||||
def _has_meaningful_margin_signature(signature: str) -> bool:
|
||||
return len(re.sub(r"[#\W_]+", "", signature, flags=re.UNICODE)) >= 2
|
||||
|
||||
def _is_toc_leader_entry(text: str) -> bool:
|
||||
return bool(_TOC_LEADER_ENTRY_PATTERN.search(text))
|
||||
|
||||
def _is_toc_numbered_entry(text: str) -> bool:
|
||||
return bool(_TOC_NUMBERED_ENTRY_PATTERN.fullmatch(text))
|
||||
|
||||
def detect_pdf_document_noise(
|
||||
pages: Sequence[PdfPageText],
|
||||
) -> tuple[DocumentNoiseSpan, ...]:
|
||||
"""识别 PDF 中的独立页码、重复页边内容和高置信目录。
|
||||
|
||||
规则只查看每页顶部 5 行和底部 3 行来推断页眉页脚；目录必须有
|
||||
明显的点引导线密度，避免仅因正文中出现“目录”或章节标题而误删。
|
||||
"""
|
||||
|
||||
page_lines = tuple(_pdf_page_lines(page) for page in pages)
|
||||
detected: dict[tuple[int, int], DocumentNoiseSpan] = {}
|
||||
|
||||
def mark(
|
||||
line: _PdfLine,
|
||||
kind: Literal["page_number", "repeated_margin", "table_of_contents"],
|
||||
) -> None:
|
||||
detected.setdefault(
|
||||
(line.start, line.end),
|
||||
DocumentNoiseSpan(
|
||||
start=line.start,
|
||||
end=line.end,
|
||||
kind=kind,
|
||||
),
|
||||
)
|
||||
|
||||
for page, lines in zip(pages, page_lines, strict=True):
|
||||
for line in lines:
|
||||
if _is_standalone_page_number(
|
||||
line.text,
|
||||
physical_page=page.page_number,
|
||||
page_count=len(pages),
|
||||
):
|
||||
mark(line, "page_number")
|
||||
outer_margin_lines = (*lines[:2], *lines[-2:])
|
||||
for line in outer_margin_lines:
|
||||
if _PDF_CLASSIFICATION_LABEL_PATTERN.fullmatch(line.text):
|
||||
mark(line, "repeated_margin")
|
||||
|
||||
# 只在三页及以上文档中推断通用页眉页脚，避免短文档误删。
|
||||
if len(pages) >= 3:
|
||||
signature_pages: dict[str, set[int]] = {}
|
||||
candidate_lines: list[tuple[int, _PdfLine, str]] = []
|
||||
for page_index, lines in enumerate(page_lines):
|
||||
boundary_lines = (
|
||||
*((line, index < 2) for index, line in enumerate(lines[:5])),
|
||||
*((line, index < 2) for index, line in enumerate(reversed(lines[-3:]))),
|
||||
)
|
||||
seen_ranges: set[tuple[int, int]] = set()
|
||||
for line, is_outer_margin in boundary_lines:
|
||||
line_range = (line.start, line.end)
|
||||
if (
|
||||
line_range in seen_ranges
|
||||
or line_range in detected
|
||||
or len(line.text) > 160
|
||||
):
|
||||
continue
|
||||
seen_ranges.add(line_range)
|
||||
if not is_outer_margin and not _has_margin_metadata_keyword(line.text):
|
||||
continue
|
||||
signature = _margin_signature(line.text)
|
||||
if not _has_meaningful_margin_signature(signature):
|
||||
continue
|
||||
signature_pages.setdefault(signature, set()).add(page_index)
|
||||
candidate_lines.append((page_index, line, signature))
|
||||
minimum_pages = max(3, math.ceil(len(pages) * 0.3))
|
||||
repeated_signatures = {
|
||||
signature
|
||||
for signature, matching_pages in signature_pages.items()
|
||||
if len(matching_pages) >= minimum_pages
|
||||
}
|
||||
for _, line, signature in candidate_lines:
|
||||
if signature in repeated_signatures:
|
||||
mark(line, "repeated_margin")
|
||||
|
||||
# 先依据强证据判定目录页，再补充删除少量不带点引导线的编号目录项。
|
||||
toc_active = False
|
||||
for lines in page_lines:
|
||||
content_lines = [
|
||||
line for line in lines if (line.start, line.end) not in detected
|
||||
]
|
||||
leader_entries = [line for line in content_lines if _is_toc_leader_entry(line.text)]
|
||||
titles = [line for line in content_lines if _TOC_TITLE_PATTERN.fullmatch(line.text)]
|
||||
starts_toc = bool(titles and len(leader_entries) >= 2)
|
||||
is_toc_dense = bool(
|
||||
len(leader_entries) >= 3
|
||||
and len(leader_entries) / max(1, len(content_lines)) >= 0.5
|
||||
)
|
||||
if not (starts_toc or (toc_active and is_toc_dense)):
|
||||
toc_active = False
|
||||
continue
|
||||
toc_active = True
|
||||
for line in content_lines:
|
||||
if (
|
||||
line in titles
|
||||
or _is_toc_leader_entry(line.text)
|
||||
or _is_toc_numbered_entry(line.text)
|
||||
):
|
||||
mark(line, "table_of_contents")
|
||||
|
||||
return tuple(sorted(detected.values(), key=lambda span: (span.start, span.end)))
|
||||
|
||||
def remove_document_noise(
|
||||
text: str,
|
||||
spans: Sequence[DocumentNoiseSpan],
|
||||
*,
|
||||
source_offset: int = 0,
|
||||
) -> str:
|
||||
"""按原文绝对偏移移除噪声，不改动调用方保留的原文及偏移。"""
|
||||
|
||||
text_end = source_offset + len(text)
|
||||
intersections = sorted(
|
||||
(
|
||||
max(0, span.start - source_offset),
|
||||
min(len(text), span.end - source_offset),
|
||||
)
|
||||
for span in spans
|
||||
if span.start < text_end and span.end > source_offset
|
||||
)
|
||||
if not intersections:
|
||||
return text
|
||||
parts: list[str] = []
|
||||
cursor = 0
|
||||
for start, end in intersections:
|
||||
if end <= cursor:
|
||||
continue
|
||||
if start > cursor:
|
||||
parts.append(text[cursor:start])
|
||||
cursor = end
|
||||
parts.append(text[cursor:])
|
||||
cleaned = normalize_text("".join(parts))
|
||||
return re.sub(r"\n{3,}", "\n\n", cleaned)
|
||||
|
||||
def _extract_pdf_text(raw: bytes) -> str:
|
||||
return "\n\n".join(page.text for page in extract_pdf_page_texts(raw) if page.text)
|
||||
Reference in New Issue
Block a user