YG_FT/backend/app/modules/data_process/algorithms.py

"""数据处理模块使用的无副作用算法。

本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
所有偏移量均为 Python 字符串偏移量。
"""

from __future__ import annotations

import csv
import hashlib
import io
import json
import math
import re
import unicodedata
import xml.etree.ElementTree as ET
import zipfile
from collections import Counter
from collections.abc import Iterable, Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass
from datetime import date, datetime, time
from pathlib import Path, PurePosixPath
from typing import Any, Literal
from urllib.parse import unquote, urlsplit

from docx import Document
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
from docx.text.paragraph import Paragraph
from openpyxl import load_workbook
from openpyxl.utils.cell import range_boundaries
from pptx import Presentation
from pypdf import PdfReader

from app.modules.data_process.constants import MAX_QA_PAIRS_PER_ITEM

TextFormat = Literal[
    "json",
    "jsonl",
    "csv",
    "markdown",
    "txt",
    "pdf",
    "docx",
    "xlsx",
    "pptx",
]
DatasetSplit = Literal["train", "validation", "test"]
StructuredPreprocessOption = Literal[
    "clean_invalid",
    "detect_structure",
    "deduplicate",
    "normalize_format",
    "filter_anomaly",
    "desensitize",
]

SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
    "json",
    "jsonl",
    "csv",
    "markdown",
    "txt",
    "pdf",
    "docx",
    "xlsx",
    "pptx",
)

_FORMAT_ALIASES: dict[str, TextFormat] = {
    "json": "json",
    "jsonl": "jsonl",
    "ndjson": "jsonl",
    "csv": "csv",
    "tsv": "csv",
    "md": "markdown",
    "markdown": "markdown",
    "txt": "txt",
    "text": "txt",
    "pdf": "pdf",
    "docx": "docx",
    "xlsx": "xlsx",
    "pptx": "pptx",
}
_LEGACY_OFFICE_FORMATS: dict[str, str] = {
    "doc": "docx",
    "xls": "xlsx",
    "ppt": "pptx",
}
_OFFICE_OPEN_XML_FORMATS = {"docx", "xlsx", "pptx"}
_MAX_ARCHIVE_ENTRIES = 10_000
_MAX_ARCHIVE_UNCOMPRESSED_BYTES = 512 * 1024 * 1024
_MAX_ARCHIVE_ENTRY_BYTES = 128 * 1024 * 1024
_MAX_ARCHIVE_COMPRESSION_RATIO = 200
_MAX_EXTRACTED_TEXT_CHARS = 20_000_000
_MAX_PDF_PAGES = 2_000
_MAX_PRESENTATION_SLIDES = 2_000
_MAX_WORKBOOK_SHEETS = 100
_MAX_WORKBOOK_ROWS = 100_000
_MAX_WORKBOOK_SCANNED_ROWS = 200_000
_MAX_WORKBOOK_COLUMNS = 256
_MAX_WORKBOOK_CELLS = 2_000_000
_MAX_WORKBOOK_HEADER_ROWS = 8
_MAX_WORKBOOK_HEADER_SCAN_ROWS = 64
_MAX_WORKBOOK_MERGED_RANGES = 100_000
_MAX_STRUCTURED_FIELDS = 1_024
_MAX_STRUCTURED_DEPTH = 16
_MAX_ANOMALY_TEXT_CHARS = 1_000_000
_STRUCTURED_OPTIONS = {
    "clean_invalid",
    "detect_structure",
    "deduplicate",
    "normalize_format",
    "filter_anomaly",
    "desensitize",
}
_IDENTITY_FIELD_PATTERN = re.compile(r"(?:^|[._])(?:id|uuid|key|code)$|(?:^|[._]).+_id$")
_MOJIBAKE_MARKERS = ("\ufffd", "锟斤拷", "烫烫烫", "屯屯屯", "Ã", "Â", "â€")
_NAME_FIELD_NAMES = {
    "name",
    "full_name",
    "fullname",
    "real_name",
    "contact_name",
    "customer_name",
    "recipient_name",
    "姓名",
    "真实姓名",
    "联系人",
    "联系人姓名",
    "收件人",
}

_EMAIL_PATTERN = re.compile(
    r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
    r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
    r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
)
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
_CHINESE_NAME_CONTEXT_PATTERN = re.compile(
    r"(?P<label>姓名|真实姓名|联系人(?:姓名)?|收件人)"
    r"(?P<separator>\s*(?:[:：=]|为)\s*|\s+)"
    r"(?P<name>[\u3400-\u4dbf\u4e00-\u9fff·]{2,8})"
)
_ENGLISH_NAME_CONTEXT_PATTERN = re.compile(
    r"(?im)(?P<label>full\s+name|contact\s+name|name)"
    r"(?P<separator>\s*[:=]\s*)"
    r"(?P<name>[A-Za-z][A-Za-z'’-]*(?:[ \t]+[A-Za-z][A-Za-z'’-]*){0,3})"
)
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")


@dataclass(frozen=True, slots=True)
class ParsedText:
    """文本、文档或工作簿的统一解析结果。"""

    format: TextFormat
    text: str
    records: tuple[dict[str, Any], ...]


@dataclass(frozen=True, slots=True)
class PdfPageText:
    """PDF 物理页在统一提取文本中的字符范围。"""

    page_number: int
    text: str
    source_start: int
    source_end: int


@dataclass(frozen=True, slots=True)
class DocumentNoiseSpan:
    """PDF 中可安全从展示内容移除的文本范围。"""

    start: int
    end: int
    kind: Literal["page_number", "repeated_margin", "table_of_contents"]


@dataclass(frozen=True, slots=True)
class QualityScore:
    """标准 instruction/input/output 记录的可解释质量分。"""

    overall: float
    completeness: float
    length: float
    readability: float
    relevance: float
    duplicate: float
    is_valid: bool
    flags: tuple[str, ...]
    fingerprint: str


@dataclass(frozen=True, slots=True)
class DocumentHeading:
    """文档标题的位置和层级。"""

    level: int
    title: str
    line_number: int
    start: int
    end: int


@dataclass(frozen=True, slots=True)
class DocumentStructure:
    """无需外部模型即可复现的文档结构摘要。"""

    line_count: int
    paragraph_count: int
    headings: tuple[DocumentHeading, ...]
    code_block_count: int
    table_block_count: int
    list_block_count: int


def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
    """严格解码 UTF-8 文本，并移除可选 BOM。

    不使用 ``errors='replace'``，避免上传内容损坏后仍被静默接收。
    """

    if isinstance(raw, str):
        return raw.removeprefix("\ufeff")
    if not isinstance(raw, (bytes, bytearray, memoryview)):
        raise TypeError("raw must be bytes-like or str")
    try:
        return bytes(raw).decode("utf-8-sig")
    except UnicodeDecodeError as exc:
        raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc


def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
    """``decode_utf8`` 的语义化别名，供上传服务直接调用。"""

    return decode_utf8(raw)


def normalize_text(text: str) -> str:
    """规范 Unicode、换行和行尾空白，同时保留段落结构。"""

    if not isinstance(text, str):
        raise TypeError("text must be str")
    normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
    normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
    normalized = "".join(
        char
        for char in normalized
        if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
    )
    lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
    return "\n".join(lines).strip()


def _normalize_format(value: str | None) -> TextFormat | None:
    if value is None:
        return None
    normalized = value.strip().lower().removeprefix(".")
    if normalized in _LEGACY_OFFICE_FORMATS:
        replacement = _LEGACY_OFFICE_FORMATS[normalized]
        raise ValueError(
            f"legacy .{normalized} format is not supported; "
            f"convert the file to .{replacement} and upload again"
        )
    try:
        return _FORMAT_ALIASES[normalized]
    except KeyError as exc:
        raise ValueError(f"unsupported text format: {value}") from exc


def detect_text_format(
    *,
    filename: str | None = None,
    text: str = "",
    file_format: str | None = None,
) -> TextFormat:
    """按显式格式、扩展名和内容特征依次识别文本格式。"""

    explicit = _normalize_format(file_format)
    if explicit:
        return explicit

    if filename:
        suffix = Path(filename).suffix.lower().removeprefix(".")
        if suffix in _LEGACY_OFFICE_FORMATS:
            replacement = _LEGACY_OFFICE_FORMATS[suffix]
            raise ValueError(
                f"legacy .{suffix} format is not supported; "
                f"convert the file to .{replacement} and upload again"
            )
        detected = _FORMAT_ALIASES.get(suffix)
        if detected:
            return detected

    stripped = text.strip()
    if stripped:
        if stripped[0] in "[{":
            try:
                json.loads(stripped)
            except json.JSONDecodeError:
                pass
            else:
                return "json"

        nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
        if len(nonempty_lines) > 1:
            try:
                for line in nonempty_lines:
                    json.loads(line)
            except json.JSONDecodeError:
                pass
            else:
                return "jsonl"

        if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
            r"(?m)^\s*\|.+\|\s*$", stripped
        ):
            return "markdown"

        sample = stripped[:8192]
        try:
            dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
            rows = list(csv.reader(io.StringIO(sample), dialect))
            if len(rows) >= 2 and len(rows[0]) >= 2:
                return "csv"
        except csv.Error:
            pass

    return "txt"


def _binary_bytes(
    raw: bytes | bytearray | memoryview | str,
    file_format: TextFormat,
) -> bytes:
    if isinstance(raw, str):
        raise ValueError(f"{file_format.upper()} content must be uploaded as binary data")
    if not isinstance(raw, (bytes, bytearray, memoryview)):
        raise TypeError("raw must be bytes-like or str")
    return bytes(raw)


def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
    """在交给 Office 解析库前限制 ZIP 包规模并拒绝活动 XML。"""

    required_members = {
        "docx": {"[Content_Types].xml", "word/document.xml"},
        "xlsx": {"[Content_Types].xml", "xl/workbook.xml"},
        "pptx": {"[Content_Types].xml", "ppt/presentation.xml"},
    }
    try:
        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
            members = archive.infolist()
            if len(members) > _MAX_ARCHIVE_ENTRIES:
                raise ValueError(
                    f"{file_format.upper()} archive contains too many entries "
                    f"(limit {_MAX_ARCHIVE_ENTRIES})"
                )

            names: set[str] = set()
            total_size = 0
            for member in members:
                path = PurePosixPath(member.filename.replace("\\", "/"))
                if path.is_absolute() or ".." in path.parts:
                    raise ValueError(
                        f"{file_format.upper()} archive contains an unsafe member path"
                    )
                if member.filename in names:
                    raise ValueError(
                        f"{file_format.upper()} archive contains duplicate member names"
                    )
                names.add(member.filename)
                if member.flag_bits & 0x1:
                    raise ValueError(f"encrypted {file_format.upper()} files are not supported")
                if member.is_dir():
                    continue
                if member.file_size > _MAX_ARCHIVE_ENTRY_BYTES:
                    raise ValueError(
                        f"{file_format.upper()} archive entry exceeds "
                        f"{_MAX_ARCHIVE_ENTRY_BYTES} bytes"
                    )
                total_size += member.file_size
                if total_size > _MAX_ARCHIVE_UNCOMPRESSED_BYTES:
                    raise ValueError(
                        f"{file_format.upper()} archive expands beyond "
                        f"{_MAX_ARCHIVE_UNCOMPRESSED_BYTES} bytes"
                    )
                if member.file_size >= 1024 * 1024:
                    if member.compress_size <= 0:
                        raise ValueError(
                            f"{file_format.upper()} archive has an unsafe compression ratio"
                        )
                    ratio = member.file_size / member.compress_size
                    if ratio > _MAX_ARCHIVE_COMPRESSION_RATIO:
                        raise ValueError(
                            f"{file_format.upper()} archive has an unsafe compression ratio"
                        )

            missing = required_members[file_format] - names
            if missing:
                raise ValueError(
                    f"invalid {file_format.upper()} package: missing "
                    f"{', '.join(sorted(missing))}"
                )

            for member in members:
                if member.is_dir() or not member.filename.lower().endswith((".xml", ".rels")):
                    continue
                with archive.open(member) as stream:
                    prefix = stream.read(min(member.file_size, 1024 * 1024)).upper()
                if b"<!DOCTYPE" in prefix or b"<!ENTITY" in prefix:
                    raise ValueError(
                        f"{file_format.upper()} archive contains unsupported active XML"
                    )
    except zipfile.BadZipFile as exc:
        raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc


def _append_bounded_text(parts: list[str], value: Any, total: int) -> int:
    text = normalize_text(str(value or ""))
    if not text:
        return total
    total += len(text)
    if total > _MAX_EXTRACTED_TEXT_CHARS:
        raise ValueError(
            f"extracted document text exceeds {_MAX_EXTRACTED_TEXT_CHARS} characters"
        )
    parts.append(text)
    return total


def extract_pdf_page_texts(raw: bytes) -> tuple[PdfPageText, ...]:
    """提取 PDF 各页文本，并保留与切片字符偏移一致的页范围。"""

    if b"%PDF-" not in raw[:1024]:
        raise ValueError("invalid PDF file: missing PDF header")
    try:
        reader = PdfReader(io.BytesIO(raw), strict=True)
    except Exception as exc:
        raise ValueError(f"invalid PDF file: {exc}") from exc
    if reader.is_encrypted and not reader.decrypt(""):
        raise ValueError("password-protected PDF files are not supported")
    if len(reader.pages) > _MAX_PDF_PAGES:
        raise ValueError(f"PDF contains too many pages (limit {_MAX_PDF_PAGES})")

    pages: list[PdfPageText] = []
    total = 0
    has_text = False
    for page_number, page in enumerate(reader.pages, start=1):
        try:
            text = normalize_text(page.extract_text() or "")
        except Exception as exc:
            raise ValueError(
                f"failed to extract text from PDF page {page_number}: {exc}"
            ) from exc
        if not text:
            pages.append(
                PdfPageText(
                    page_number=page_number,
                    text="",
                    source_start=total,
                    source_end=total,
                )
            )
            continue
        if has_text:
            total += 2
        start = total
        total += len(text)
        if total > _MAX_EXTRACTED_TEXT_CHARS:
            raise ValueError(
                f"extracted document text exceeds {_MAX_EXTRACTED_TEXT_CHARS} characters"
            )
        pages.append(
            PdfPageText(
                page_number=page_number,
                text=text,
                source_start=start,
                source_end=total,
            )
        )
        has_text = True
    if not has_text:
        raise ValueError("PDF contains no extractable text; scanned PDF requires OCR")
    return tuple(pages)


@dataclass(frozen=True, slots=True)
class _PdfLine:
    text: str
    start: int
    end: int


_PDF_PAGE_NUMBER_LINE_PATTERN = re.compile(
    r"^(?:页次\s*)?(?:第\s*)?(?P<page>\d+)\s*页\s*"
    r"(?:(?:[/／]\s*)?共\s*(?P<total>\d+)\s*页)?$"
)
_PDF_FRACTION_PAGE_LINE_PATTERN = re.compile(
    r"^[—–-]?\s*(?P<page>\d+)\s*[/／]\s*(?P<total>\d+)\s*[—–-]?$"
)
_PDF_CLASSIFICATION_LABEL_PATTERN = re.compile(
    r"^(?:(?:秘密等级|密级)\s*)?(?:商密|秘密|机密|绝密)"
    r"\s*(?:[【\[(（][^】\])）]{1,8}[】\])）])?$"
)
_TOC_TITLE_PATTERN = re.compile(r"^(?:目\s*录|contents)$", re.IGNORECASE)
_TOC_LEADER_ENTRY_PATTERN = re.compile(
    r"(?:[.．…·•]\s*){3,}\s*\d{1,4}\s*$"
)
_TOC_NUMBERED_ENTRY_PATTERN = re.compile(
    r"^(?:第[\u3400-\u4dbf\u4e00-\u9fff]{1,12}章|附表\s*\d+|\d+(?:\.\d+)+)"
    r"\s+.+\s+\d{1,4}\s*$",
    re.IGNORECASE,
)
_MARGIN_TEMPLATE_KEYWORDS = (
    "页",
    "页次",
    "版本",
    "文件编码",
    "秘密等级",
    "密级",
    "商密",
    "confidential",
)


def _pdf_page_lines(page: PdfPageText) -> tuple[_PdfLine, ...]:
    lines: list[_PdfLine] = []
    local_offset = 0
    for raw_line in page.text.splitlines(keepends=True):
        content = raw_line.rstrip("\r\n")
        leading = len(content) - len(content.lstrip())
        trailing = len(content.rstrip())
        text = content.strip()
        if text:
            lines.append(
                _PdfLine(
                    text=text,
                    start=page.source_start + local_offset + leading,
                    end=page.source_start + local_offset + trailing,
                )
            )
        local_offset += len(raw_line)
    return tuple(lines)


def _is_standalone_page_number(
    text: str,
    *,
    physical_page: int,
    page_count: int,
) -> bool:
    normalized = unicodedata.normalize("NFKC", text).strip()
    match = _PDF_PAGE_NUMBER_LINE_PATTERN.fullmatch(normalized)
    if match is None:
        match = _PDF_FRACTION_PAGE_LINE_PATTERN.fullmatch(normalized)
    if match is None or int(match.group("page")) != physical_page:
        return False
    total = match.groupdict().get("total")
    return total is None or int(total) == page_count


def _margin_signature(text: str) -> str:
    normalized = unicodedata.normalize("NFKC", text).casefold()
    normalized = re.sub(r"\s+", " ", normalized).strip()
    if any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS):
        normalized = re.sub(r"\d+", "#", normalized)
    return normalized


def _has_margin_metadata_keyword(text: str) -> bool:
    normalized = unicodedata.normalize("NFKC", text).casefold()
    return any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS)


def _has_meaningful_margin_signature(signature: str) -> bool:
    return len(re.sub(r"[#\W_]+", "", signature, flags=re.UNICODE)) >= 2


def _is_toc_leader_entry(text: str) -> bool:
    return bool(_TOC_LEADER_ENTRY_PATTERN.search(text))


def _is_toc_numbered_entry(text: str) -> bool:
    return bool(_TOC_NUMBERED_ENTRY_PATTERN.fullmatch(text))


def detect_pdf_document_noise(
    pages: Sequence[PdfPageText],
) -> tuple[DocumentNoiseSpan, ...]:
    """识别 PDF 中的独立页码、重复页边内容和高置信目录。

    规则只查看每页顶部 5 行和底部 3 行来推断页眉页脚；目录必须有
    明显的点引导线密度，避免仅因正文中出现“目录”或章节标题而误删。
    """

    page_lines = tuple(_pdf_page_lines(page) for page in pages)
    detected: dict[tuple[int, int], DocumentNoiseSpan] = {}

    def mark(
        line: _PdfLine,
        kind: Literal["page_number", "repeated_margin", "table_of_contents"],
    ) -> None:
        detected.setdefault(
            (line.start, line.end),
            DocumentNoiseSpan(
                start=line.start,
                end=line.end,
                kind=kind,
            ),
        )

    for page, lines in zip(pages, page_lines, strict=True):
        for line in lines:
            if _is_standalone_page_number(
                line.text,
                physical_page=page.page_number,
                page_count=len(pages),
            ):
                mark(line, "page_number")
        outer_margin_lines = (*lines[:2], *lines[-2:])
        for line in outer_margin_lines:
            if _PDF_CLASSIFICATION_LABEL_PATTERN.fullmatch(line.text):
                mark(line, "repeated_margin")

    # 只在三页及以上文档中推断通用页眉页脚，避免短文档误删。
    if len(pages) >= 3:
        signature_pages: dict[str, set[int]] = {}
        candidate_lines: list[tuple[int, _PdfLine, str]] = []
        for page_index, lines in enumerate(page_lines):
            boundary_lines = (
                *((line, index < 2) for index, line in enumerate(lines[:5])),
                *((line, index < 2) for index, line in enumerate(reversed(lines[-3:]))),
            )
            seen_ranges: set[tuple[int, int]] = set()
            for line, is_outer_margin in boundary_lines:
                line_range = (line.start, line.end)
                if (
                    line_range in seen_ranges
                    or line_range in detected
                    or len(line.text) > 160
                ):
                    continue
                seen_ranges.add(line_range)
                if not is_outer_margin and not _has_margin_metadata_keyword(line.text):
                    continue
                signature = _margin_signature(line.text)
                if not _has_meaningful_margin_signature(signature):
                    continue
                signature_pages.setdefault(signature, set()).add(page_index)
                candidate_lines.append((page_index, line, signature))
        minimum_pages = max(3, math.ceil(len(pages) * 0.3))
        repeated_signatures = {
            signature
            for signature, matching_pages in signature_pages.items()
            if len(matching_pages) >= minimum_pages
        }
        for _, line, signature in candidate_lines:
            if signature in repeated_signatures:
                mark(line, "repeated_margin")

    # 先依据强证据判定目录页，再补充删除少量不带点引导线的编号目录项。
    toc_active = False
    for lines in page_lines:
        content_lines = [
            line for line in lines if (line.start, line.end) not in detected
        ]
        leader_entries = [line for line in content_lines if _is_toc_leader_entry(line.text)]
        titles = [line for line in content_lines if _TOC_TITLE_PATTERN.fullmatch(line.text)]
        starts_toc = bool(titles and len(leader_entries) >= 2)
        is_toc_dense = bool(
            len(leader_entries) >= 3
            and len(leader_entries) / max(1, len(content_lines)) >= 0.5
        )
        if not (starts_toc or (toc_active and is_toc_dense)):
            toc_active = False
            continue
        toc_active = True
        for line in content_lines:
            if (
                line in titles
                or _is_toc_leader_entry(line.text)
                or _is_toc_numbered_entry(line.text)
            ):
                mark(line, "table_of_contents")

    return tuple(sorted(detected.values(), key=lambda span: (span.start, span.end)))


def remove_document_noise(
    text: str,
    spans: Sequence[DocumentNoiseSpan],
    *,
    source_offset: int = 0,
) -> str:
    """按原文绝对偏移移除噪声，不改动调用方保留的原文及偏移。"""

    text_end = source_offset + len(text)
    intersections = sorted(
        (
            max(0, span.start - source_offset),
            min(len(text), span.end - source_offset),
        )
        for span in spans
        if span.start < text_end and span.end > source_offset
    )
    if not intersections:
        return text
    parts: list[str] = []
    cursor = 0
    for start, end in intersections:
        if end <= cursor:
            continue
        if start > cursor:
            parts.append(text[cursor:start])
        cursor = end
    parts.append(text[cursor:])
    cleaned = normalize_text("".join(parts))
    return re.sub(r"\n{3,}", "\n\n", cleaned)


def _extract_pdf_text(raw: bytes) -> str:
    return "\n\n".join(page.text for page in extract_pdf_page_texts(raw) if page.text)


def _extract_docx_text(raw: bytes) -> str:
    _validate_office_archive(raw, "docx")
    try:
        document = Document(io.BytesIO(raw))
    except Exception as exc:
        raise ValueError(f"invalid DOCX file: {exc}") from exc

    parts: list[str] = []
    total = 0
    for child in document.element.body.iterchildren():
        if isinstance(child, CT_P):
            total = _append_bounded_text(parts, Paragraph(child, document).text, total)
            continue
        if isinstance(child, CT_Tbl):
            table = Table(child, document)
            for row in table.rows:
                cells = [normalize_text(cell.text) for cell in row.cells]
                total = _append_bounded_text(parts, "\t".join(cells), total)
    return normalize_text("\n\n".join(parts))


def _presentation_shape_text(shape: Any) -> list[str]:
    if getattr(shape, "has_table", False):
        return [
            "\t".join(normalize_text(cell.text) for cell in row.cells)
            for row in shape.table.rows
        ]
    if getattr(shape, "has_text_frame", False):
        return [shape.text]
    child_shapes = getattr(shape, "shapes", None)
    if child_shapes is not None:
        values: list[str] = []
        for child in child_shapes:
            values.extend(_presentation_shape_text(child))
        return values
    return []


def _extract_pptx_text(raw: bytes) -> str:
    _validate_office_archive(raw, "pptx")
    try:
        presentation = Presentation(io.BytesIO(raw))
    except Exception as exc:
        raise ValueError(f"invalid PPTX file: {exc}") from exc
    if len(presentation.slides) > _MAX_PRESENTATION_SLIDES:
        raise ValueError(
            f"PPTX contains too many slides (limit {_MAX_PRESENTATION_SLIDES})"
        )

    parts: list[str] = []
    total = 0
    for slide in presentation.slides:
        slide_parts: list[str] = []
        for shape in slide.shapes:
            slide_parts.extend(_presentation_shape_text(shape))
        total = _append_bounded_text(parts, "\n".join(slide_parts), total)
    return normalize_text("\n\n".join(parts))


def _normalize_spreadsheet_value(value: Any) -> Any:
    if value is None:
        return ""
    if isinstance(value, str):
        return normalize_text(value)
    if isinstance(value, (datetime, date, time)):
        return value.isoformat()
    if isinstance(value, (bool, int, float)):
        return value
    return normalize_text(str(value))


def _xml_local_name(tag: str) -> str:
    return tag.rsplit("}", 1)[-1]


def _resolve_xlsx_relationship_target(
    archive: zipfile.ZipFile,
    target: str,
    *,
    source_part: str = "xl/workbook.xml",
) -> str:
    """按 OPC URI 规则解析内部关系目标，并保证结果仍位于 ZIP 根内。"""

    raw_target = target.strip()
    if (
        raw_target != target
        or not raw_target
        or "\\" in raw_target
        or any(unicodedata.category(char).startswith("C") for char in raw_target)
        or re.search(r"%(?![0-9A-Fa-f]{2})", raw_target)
    ):
        raise ValueError("XLSX workbook contains an unsafe worksheet path")
    try:
        parsed = urlsplit(raw_target)
    except ValueError as exc:
        raise ValueError("XLSX workbook contains an unsafe worksheet path") from exc
    if parsed.scheme or parsed.netloc or parsed.query or parsed.fragment:
        raise ValueError("XLSX workbook contains an unsafe worksheet path")
    if re.search(r"%(?:2[fF]|5[cC]|0{2})", parsed.path):
        raise ValueError("XLSX workbook contains an unsafe worksheet path")
    try:
        decoded_path = unquote(parsed.path, encoding="utf-8", errors="strict")
    except UnicodeDecodeError as exc:
        raise ValueError("XLSX workbook contains an unsafe worksheet path") from exc
    if not decoded_path or "\\" in decoded_path or "\x00" in decoded_path or "%" in decoded_path:
        raise ValueError("XLSX workbook contains an unsafe worksheet path")

    parts = [] if decoded_path.startswith("/") else list(PurePosixPath(source_part).parent.parts)
    for part in decoded_path.lstrip("/").split("/"):
        if part in {"", "."}:
            continue
        if part == "..":
            if not parts:
                raise ValueError("XLSX workbook contains an unsafe worksheet path")
            parts.pop()
            continue
        if unicodedata.category(part[0]).startswith("C") or any(
            unicodedata.category(char).startswith("C") for char in part
        ):
            raise ValueError("XLSX workbook contains an unsafe worksheet path")
        parts.append(part)
    if not parts:
        raise ValueError("XLSX workbook contains an unsafe worksheet path")

    member_name = "/".join(parts)
    try:
        member = archive.getinfo(member_name)
    except KeyError as exc:
        raise ValueError(
            f"XLSX worksheet relationship target does not exist: {member_name}"
        ) from exc
    if member.is_dir():
        raise ValueError("XLSX worksheet relationship target must be a file")
    return member_name


def _rewrite_xlsx_workbook_relationships(
    raw: bytes,
    replacements: Mapping[str, str],
) -> bytes:
    """把已验证的 worksheet Target 改为解析库稳定支持的包内绝对路径。"""

    relationships_member = "xl/_rels/workbook.xml.rels"
    output = io.BytesIO()
    with zipfile.ZipFile(io.BytesIO(raw)) as source, zipfile.ZipFile(output, "w") as target:
        target.comment = source.comment
        for member in source.infolist():
            if member.filename == relationships_member:
                root = ET.fromstring(source.read(member))
                pending = dict(replacements)
                for element in root:
                    if _xml_local_name(element.tag) != "Relationship":
                        continue
                    relationship_id = element.attrib.get("Id")
                    if relationship_id in pending:
                        element.set("Target", pending.pop(relationship_id))
                if pending:
                    raise ValueError(
                        "XLSX workbook relationship changed during normalization"
                    )
                content = ET.tostring(root, encoding="utf-8", xml_declaration=True)
                target.writestr(member, content)
                continue
            if member.is_dir():
                target.writestr(member, b"")
                continue
            with source.open(member) as source_stream, target.open(
                member,
                "w",
                force_zip64=True,
            ) as target_stream:
                while chunk := source_stream.read(1024 * 1024):
                    target_stream.write(chunk)
    return output.getvalue()


def _xlsx_sheet_merge_ranges(
    raw: bytes,
) -> tuple[
    dict[str, tuple[tuple[int, int, int, int], ...]],
    dict[str, str],
]:
    """流式读取 XLSX 合并单元格，不把工作表 XML 整体载入内存。"""

    relationship_namespace = (
        "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
    )
    try:
        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
            relationships: dict[str, tuple[str, str, str]] = {}
            with archive.open("xl/_rels/workbook.xml.rels") as stream:
                for _, element in ET.iterparse(stream, events=("end",)):
                    if _xml_local_name(element.tag) != "Relationship":
                        element.clear()
                        continue
                    relationship_id = element.attrib.get("Id")
                    target = element.attrib.get("Target")
                    target_mode = element.attrib.get("TargetMode", "Internal")
                    relationship_type = element.attrib.get("Type", "")
                    if relationship_id:
                        if relationship_id in relationships:
                            raise ValueError(
                                "XLSX workbook contains duplicate relationship identifiers"
                            )
                        relationships[relationship_id] = (
                            target or "",
                            target_mode,
                            relationship_type,
                        )
                    element.clear()

            sheet_paths: dict[str, str] = {}
            normalized_targets: dict[str, str] = {}
            with archive.open("xl/workbook.xml") as stream:
                for _, element in ET.iterparse(stream, events=("end",)):
                    if _xml_local_name(element.tag) != "sheet":
                        element.clear()
                        continue
                    title = element.attrib.get("name")
                    relationship_id = element.attrib.get(
                        f"{{{relationship_namespace}}}id"
                    )
                    if title and relationship_id:
                        relationship = relationships.get(relationship_id)
                        if relationship is None:
                            raise ValueError(
                                f"XLSX worksheet {title!r} references a missing relationship"
                            )
                        target, target_mode, relationship_type = relationship
                        if target_mode.strip().lower() != "internal":
                            raise ValueError(
                                f"XLSX worksheet {title!r} uses an external relationship"
                            )
                        if not relationship_type.endswith("/worksheet"):
                            raise ValueError(
                                f"XLSX worksheet {title!r} has an invalid relationship type"
                            )
                        member_name = _resolve_xlsx_relationship_target(
                            archive,
                            target,
                        )
                        sheet_paths[title] = member_name
                        canonical_target = f"/{member_name}"
                        if target != canonical_target:
                            normalized_targets[relationship_id] = canonical_target
                    element.clear()

            result: dict[str, tuple[tuple[int, int, int, int], ...]] = {}
            total_ranges = 0
            for title, member_name in sheet_paths.items():
                ranges: list[tuple[int, int, int, int]] = []
                with archive.open(member_name) as stream:
                    for _, element in ET.iterparse(stream, events=("end",)):
                        if _xml_local_name(element.tag) != "mergeCell":
                            element.clear()
                            continue
                        reference = element.attrib.get("ref")
                        if reference:
                            try:
                                boundaries = range_boundaries(reference)
                            except ValueError as exc:
                                raise ValueError(
                                    f"XLSX worksheet {title!r} has an invalid merged range"
                                ) from exc
                            ranges.append(boundaries)
                            total_ranges += 1
                            if total_ranges > _MAX_WORKBOOK_MERGED_RANGES:
                                raise ValueError(
                                    "XLSX workbook contains too many merged ranges "
                                    f"(limit {_MAX_WORKBOOK_MERGED_RANGES})"
                                )
                        element.clear()
                result[title] = tuple(ranges)
            return result, normalized_targets
    except (KeyError, ET.ParseError, zipfile.BadZipFile) as exc:
        raise ValueError(f"invalid XLSX workbook structure: {exc}") from exc


def _xlsx_header_end_row(
    first_row: int,
    rows: Mapping[int, Sequence[Any]],
    merged_ranges: Sequence[tuple[int, int, int, int]],
) -> int:
    """在固定深度内闭包表头合并关系，忽略越界或跨空行的可疑级联。"""

    header_end = first_row
    maximum_end = first_row + _MAX_WORKBOOK_HEADER_ROWS - 1
    has_header_hierarchy = any(
        max_column > min_column and min_row == first_row
        for min_column, min_row, max_column, _ in merged_ranges
    )
    changed = True
    while changed:
        changed = False
        for min_column, min_row, max_column, max_row in merged_ranges:
            if min_row < first_row or min_row > header_end or max_row < first_row:
                continue
            if max_column == min_column and not has_header_hierarchy:
                continue
            candidate = max_row + 1 if max_column > min_column else max_row
            if candidate <= header_end or candidate > maximum_end:
                continue
            if (
                max_column > min_column
                and len(_xlsx_nonempty_values(rows.get(min_row, ()))) < 2
                and len(_xlsx_nonempty_values(rows.get(candidate, ()))) < 2
            ):
                continue
            if any(
                not rows.get(row_number)
                for row_number in range(header_end + 1, candidate + 1)
            ):
                continue
            header_end = candidate
            changed = True
    return header_end


def _xlsx_headers(
    title: str,
    rows: Mapping[int, Sequence[Any]],
    first_row: int,
    header_end: int,
    merged_ranges: Sequence[tuple[int, int, int, int]],
) -> list[str]:
    width = max((len(row) for row in rows.values()), default=0)
    for min_column, min_row, max_column, max_row in merged_ranges:
        if min_row <= header_end and max_row >= first_row:
            width = max(width, max_column)
    if width > _MAX_WORKBOOK_COLUMNS:
        raise ValueError(
            f"XLSX worksheet {title!r} exceeds {_MAX_WORKBOOK_COLUMNS} columns"
        )

    matrix = [
        list(rows.get(row_number, ())) + [None] * (width - len(rows.get(row_number, ())))
        for row_number in range(first_row, header_end + 1)
    ]
    for min_column, min_row, max_column, max_row in merged_ranges:
        if min_row > header_end or max_row < first_row:
            continue
        source_row = max(first_row, min_row) - first_row
        source_column = min_column - 1
        source = matrix[source_row][source_column]
        for row_number in range(max(first_row, min_row), min(header_end, max_row) + 1):
            for column_number in range(min_column, max_column + 1):
                matrix[row_number - first_row][column_number - 1] = source

    headers: list[str] = []
    for column in range(width):
        components: list[str] = []
        for row in matrix:
            component = normalize_text(str(row[column] or ""))
            if component and (not components or component != components[-1]):
                components.append(component)
        header = ".".join(components)
        if not header:
            raise ValueError(f"XLSX worksheet {title!r} contains an empty header")
        headers.append(header)
    if len(set(headers)) != len(headers):
        raise ValueError(f"XLSX worksheet {title!r} contains duplicate headers")
    return headers


_XLSX_REPORT_METADATA_PATTERN = re.compile(
    r"^(?:报表|报告|标题|说明|备注|制表|统计|日期|时间|期间|"
    r"report|title|note|remark|date|time|period)\b",
    flags=re.IGNORECASE,
)


def _xlsx_nonempty_values(row: Sequence[Any]) -> list[Any]:
    return [value for value in row if value not in {None, ""}]


def _infer_xlsx_header_region(
    title: str,
    rows: Mapping[int, Sequence[Any]],
    merged_ranges: Sequence[tuple[int, int, int, int]],
) -> tuple[int, int, list[str]]:
    """从有界前缀中推断表头，区分报表说明、多级表头和正文合并。"""

    ordered_rows = sorted(rows)
    if not ordered_rows:
        return 0, 0, []
    first_nonempty_row = ordered_rows[0]
    horizontal_merge_rows = {
        min_row
        for min_column, min_row, max_column, _ in merged_ranges
        if max_column > min_column
    }
    candidates: list[tuple[float, int, int, list[str]]] = []
    for first_row in ordered_rows:
        raw_values = _xlsx_nonempty_values(rows[first_row])
        if not raw_values:
            continue
        if len(raw_values) < 2 and first_row in horizontal_merge_rows:
            continue

        header_end = _xlsx_header_end_row(first_row, rows, merged_ranges)
        header_rows = {
            row_number: rows[row_number]
            for row_number in range(first_row, header_end + 1)
            if row_number in rows
        }
        try:
            headers = _xlsx_headers(
                title,
                header_rows,
                first_row,
                header_end,
                merged_ranges,
            )
        except ValueError:
            if header_end == first_row:
                continue
            header_end = first_row
            try:
                headers = _xlsx_headers(
                    title,
                    {first_row: rows[first_row]},
                    first_row,
                    first_row,
                    merged_ranges,
                )
            except ValueError:
                continue

        data_rows = [rows[row_number] for row_number in ordered_rows if row_number > header_end]
        if any(len(row) > len(headers) for row in data_rows):
            continue

        text_ratio = sum(isinstance(value, str) for value in raw_values) / len(raw_values)
        score = text_ratio * 6 + min(len(raw_values), 4)
        if data_rows:
            first_data_values = _xlsx_nonempty_values(data_rows[0])
            score += 4 * min(len(first_data_values), len(headers)) / len(headers)
            if first_data_values:
                score += (
                    2
                    * sum(
                        not isinstance(value, str)
                        for value in first_data_values
                    )
                    / len(first_data_values)
                )
        first_label = normalize_text(str(raw_values[0]))
        if len(raw_values) <= 2 and _XLSX_REPORT_METADATA_PATTERN.match(first_label):
            score -= 8
        candidates.append((score, first_row, header_end, headers))

    if not candidates:
        first_row = first_nonempty_row
        headers = _xlsx_headers(
            title,
            {first_row: rows[first_row]},
            first_row,
            first_row,
            (),
        )
        return first_row, first_row, headers
    _, first_row, header_end, headers = max(
        candidates,
        key=lambda candidate: (candidate[0], -candidate[1]),
    )
    return first_row, header_end, headers


def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]:
    _validate_office_archive(raw, "xlsx")
    merged_by_sheet, normalized_targets = _xlsx_sheet_merge_ranges(raw)
    workbook_raw = (
        _rewrite_xlsx_workbook_relationships(raw, normalized_targets)
        if normalized_targets
        else raw
    )
    try:
        workbook = load_workbook(
            io.BytesIO(workbook_raw),
            read_only=True,
            data_only=True,
            keep_links=False,
        )
    except Exception as exc:
        raise ValueError(f"invalid XLSX file: {exc}") from exc

    records: list[dict[str, Any]] = []
    total_cells = 0
    try:
        if len(workbook.worksheets) > _MAX_WORKBOOK_SHEETS:
            raise ValueError(
                f"XLSX contains too many worksheets (limit {_MAX_WORKBOOK_SHEETS})"
            )
        for worksheet in workbook.worksheets:
            reset_dimensions = getattr(worksheet, "reset_dimensions", None)
            if callable(reset_dimensions):
                reset_dimensions()
            merged_ranges = merged_by_sheet.get(worksheet.title, ())
            sheet_rows = 0
            scanned_rows = 0
            row_iterator = enumerate(
                worksheet.iter_rows(values_only=True),
                start=1,
            )
            buffered_rows: dict[int, Sequence[Any]] = {}

            def normalized_row_values(row: Sequence[Any]) -> list[Any]:
                values = list(row)
                while values and values[-1] in {None, ""}:
                    values.pop()
                if len(values) > _MAX_WORKBOOK_COLUMNS:
                    raise ValueError(
                        f"XLSX worksheet {worksheet.title!r} exceeds "
                        f"{_MAX_WORKBOOK_COLUMNS} columns"
                    )
                return values

            for row_number, row in row_iterator:
                scanned_rows += 1
                if scanned_rows > _MAX_WORKBOOK_SCANNED_ROWS:
                    raise ValueError(
                        f"XLSX worksheet {worksheet.title!r} exceeds "
                        f"{_MAX_WORKBOOK_SCANNED_ROWS} scanned rows"
                    )
                values = normalized_row_values(row)
                if not values or all(value in {None, ""} for value in values):
                    continue
                buffered_rows[row_number] = tuple(values)
                if len(buffered_rows) >= _MAX_WORKBOOK_HEADER_SCAN_ROWS:
                    break

            if not buffered_rows:
                continue
            _, header_end_row, headers = _infer_xlsx_header_region(
                worksheet.title,
                buffered_rows,
                merged_ranges,
            )

            def append_record(values: Sequence[Any]) -> None:
                nonlocal total_cells, sheet_rows
                row_values = list(values)
                if len(row_values) > len(headers):
                    raise ValueError(
                        f"XLSX worksheet {worksheet.title!r} has a row wider than its header"
                    )
                row_values.extend([None] * (len(headers) - len(row_values)))
                record = {
                    header: _normalize_spreadsheet_value(value)
                    for header, value in zip(headers, row_values, strict=True)
                }
                if not any(value not in {"", None} for value in record.values()):
                    return
                sheet_rows += 1
                total_cells += len(headers)
                if sheet_rows > _MAX_WORKBOOK_ROWS:
                    raise ValueError(
                        f"XLSX worksheet {worksheet.title!r} exceeds "
                        f"{_MAX_WORKBOOK_ROWS} data rows"
                    )
                if total_cells > _MAX_WORKBOOK_CELLS:
                    raise ValueError(
                        f"XLSX workbook exceeds {_MAX_WORKBOOK_CELLS} populated cells"
                    )
                records.append(record)

            for row_number, values in buffered_rows.items():
                if row_number > header_end_row:
                    append_record(values)

            for _, row in row_iterator:
                scanned_rows += 1
                if scanned_rows > _MAX_WORKBOOK_SCANNED_ROWS:
                    raise ValueError(
                        f"XLSX worksheet {worksheet.title!r} exceeds "
                        f"{_MAX_WORKBOOK_SCANNED_ROWS} scanned rows"
                    )
                values = normalized_row_values(row)
                if not values or all(value in {None, ""} for value in values):
                    continue
                append_record(values)
    finally:
        workbook.close()
    return records


def _normalize_value(value: Any) -> Any:
    if isinstance(value, str):
        return normalize_text(value)
    if isinstance(value, Mapping):
        return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
    if isinstance(value, list):
        return [_normalize_value(item) for item in value]
    return value


def _record_from_value(value: Any) -> dict[str, Any]:
    if isinstance(value, Mapping):
        return dict(_normalize_value(value))
    return {"value": _normalize_value(value)}


def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
    """从 JSON、JSONL 或 CSV 中提取规范化记录。

    JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组；
    其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
    """

    normalized_format = _normalize_format(file_format)
    if normalized_format not in {"json", "jsonl", "csv"}:
        raise ValueError("structured record extraction only supports JSON, JSONL and CSV")

    normalized_text = normalize_text(text)
    if not normalized_text:
        return []

    if normalized_format == "json":
        try:
            payload = json.loads(normalized_text)
        except json.JSONDecodeError as exc:
            raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
        values: Sequence[Any]
        if isinstance(payload, list):
            values = payload
        elif isinstance(payload, Mapping):
            nested = next(
                (
                    payload[key]
                    for key in ("records", "data", "items", "rows")
                    if isinstance(payload.get(key), list)
                ),
                None,
            )
            values = nested if isinstance(nested, list) else [payload]
        else:
            values = [payload]
        return [_record_from_value(value) for value in values]

    if normalized_format == "jsonl":
        records: list[dict[str, Any]] = []
        for line_number, line in enumerate(normalized_text.splitlines(), start=1):
            if not line.strip():
                continue
            try:
                value = json.loads(line)
            except json.JSONDecodeError as exc:
                raise ValueError(
                    f"invalid JSONL at line {line_number}, "
                    f"column {exc.colno}: {exc.msg}"
                ) from exc
            records.append(_record_from_value(value))
        return records

    try:
        dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
    except csv.Error:
        dialect = csv.excel
    reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
    if not reader.fieldnames:
        raise ValueError("CSV header is required")
    headers = [normalize_text(header or "") for header in reader.fieldnames]
    if any(not header for header in headers):
        raise ValueError("CSV header cannot be empty")
    if len(set(headers)) != len(headers):
        raise ValueError("CSV headers must be unique")
    reader.fieldnames = headers

    records = []
    for row in reader:
        if None in row:
            raise ValueError("CSV row has more fields than the header")
        normalized_row = {
            key: normalize_text(value or "")
            for key, value in row.items()
        }
        if any(value for value in normalized_row.values()):
            records.append(normalized_row)
    return records


def parse_text_content(
    raw: bytes | bytearray | memoryview | str,
    *,
    filename: str | None = None,
    file_format: str | None = None,
) -> ParsedText:
    """安全解析 UTF-8 文本、文本型 PDF 和现代 Office 文件。"""

    detected_format = detect_text_format(
        filename=filename,
        text="",
        file_format=file_format,
    )
    if detected_format in _OFFICE_OPEN_XML_FORMATS or detected_format == "pdf":
        binary = _binary_bytes(raw, detected_format)
        if detected_format == "pdf":
            text = _extract_pdf_text(binary)
            return ParsedText(format=detected_format, text=text, records=())
        if detected_format == "docx":
            text = _extract_docx_text(binary)
            return ParsedText(format=detected_format, text=text, records=())
        if detected_format == "pptx":
            text = _extract_pptx_text(binary)
            return ParsedText(format=detected_format, text=text, records=())

        records = _extract_xlsx_records(binary)
        text = "\n".join(
            json.dumps(record, ensure_ascii=False, separators=(",", ":"))
            for record in records
        )
        return ParsedText(
            format=detected_format,
            text=normalize_text(text),
            records=tuple(records),
        )

    text = normalize_text(decode_utf8(raw))
    detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
    records: list[dict[str, Any]] = []
    if detected_format in {"json", "jsonl", "csv"}:
        records = extract_structured_records(text, detected_format)
    return ParsedText(format=detected_format, text=text, records=tuple(records))


def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
    """掩码邮箱、手机号、身份证号及有明确上下文的姓名。"""

    if not isinstance(text, str):
        raise TypeError("text must be str")
    counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}

    def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
        def replacer(_: re.Match[str]) -> str:
            counts[kind] += 1
            return replacement

        return pattern.sub(replacer, value)

    masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
    masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
    masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)

    def replace_context_name(match: re.Match[str]) -> str:
        counts["name"] = counts.get("name", 0) + 1
        return f"{match.group('label')}{match.group('separator')}[NAME]"

    masked = _CHINESE_NAME_CONTEXT_PATTERN.sub(replace_context_name, masked)
    masked = _ENGLISH_NAME_CONTEXT_PATTERN.sub(replace_context_name, masked)
    counts["total"] = sum(counts.values())
    return masked, counts


def _is_empty_value(value: Any) -> bool:
    if value is None:
        return True
    if isinstance(value, str):
        return not normalize_text(value)
    if isinstance(value, Mapping):
        return not value or all(_is_empty_value(item) for item in value.values())
    if isinstance(value, (list, tuple, set, frozenset)):
        return not value or all(_is_empty_value(item) for item in value)
    return False


def _canonical_value(value: Any) -> Any:
    if value is None or isinstance(value, (bool, int)):
        return value
    if isinstance(value, float):
        if math.isfinite(value):
            return value
        return str(value).lower()
    if isinstance(value, str):
        return normalize_text(value)
    if isinstance(value, (datetime, date, time)):
        return value.isoformat()
    if isinstance(value, Mapping):
        normalized: dict[str, Any] = {}
        for key, item in sorted(value.items(), key=lambda pair: str(pair[0])):
            normalized_key = normalize_text(str(key))
            if not normalized_key:
                raise ValueError("structured record contains an empty field name")
            if normalized_key in normalized:
                raise ValueError(
                    f"structured record fields collide after normalization: {normalized_key}"
                )
            normalized[normalized_key] = _canonical_value(item)
        return normalized
    if isinstance(value, (list, tuple)):
        return [_canonical_value(item) for item in value]
    if isinstance(value, (set, frozenset)):
        items = [_canonical_value(item) for item in value]
        return sorted(
            items,
            key=lambda item: json.dumps(
                item,
                ensure_ascii=False,
                sort_keys=True,
                separators=(",", ":"),
            ),
        )
    if isinstance(value, (bytes, bytearray, memoryview)):
        return bytes(value).hex()
    return normalize_text(str(value))


def canonical_record_json(record: Mapping[str, Any]) -> str:
    """生成与字段顺序无关、可用于比较和落库的 canonical JSON。"""

    if not isinstance(record, Mapping):
        raise TypeError("record must be a mapping")
    return json.dumps(
        _canonical_value(record),
        ensure_ascii=False,
        sort_keys=True,
        separators=(",", ":"),
        allow_nan=False,
    )


def _normalize_field_name(value: Any, style: str) -> str:
    name = normalize_text(str(value))
    if style == "preserve":
        return name
    if style == "lower":
        return name.lower()
    if style != "snake_case":
        raise ValueError("field_name_style must be snake_case, lower or preserve")
    name = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", name)
    name = re.sub(r"[^\w\u3400-\u4dbf\u4e00-\u9fff]+", "_", name, flags=re.UNICODE)
    return re.sub(r"_+", "_", name).strip("_").lower()


def normalize_structured_record(
    record: Mapping[str, Any],
    *,
    field_name_style: str = "snake_case",
) -> dict[str, Any]:
    """规范字段名、Unicode/空白、容器类型和不可 JSON 化的标量。"""

    if not isinstance(record, Mapping):
        raise TypeError("record must be a mapping")
    normalized: dict[str, Any] = {}
    for key, value in record.items():
        normalized_key = _normalize_field_name(key, field_name_style)
        if not normalized_key:
            raise ValueError("structured record contains an empty field name")
        if normalized_key in normalized:
            raise ValueError(
                f"structured record fields collide after normalization: {normalized_key}"
            )
        normalized[normalized_key] = _canonical_value(value)
    return dict(sorted(normalized.items()))


def _embedded_structure(value: Any) -> Any:
    if not isinstance(value, str):
        return value
    candidate = value.strip()
    if not candidate or candidate[0] not in "[{":
        return value
    try:
        parsed = json.loads(candidate)
    except json.JSONDecodeError:
        return value
    return parsed if isinstance(parsed, (Mapping, list)) else value


def flatten_structured_record(
    record: Mapping[str, Any],
    *,
    separator: str = ".",
) -> dict[str, Any]:
    """把嵌套对象展平；数组保留为 canonical JSON 兼容值。"""

    if not isinstance(record, Mapping):
        raise TypeError("record must be a mapping")
    if not separator:
        raise ValueError("separator cannot be empty")
    flattened: dict[str, Any] = {}

    def visit(value: Any, path: tuple[str, ...], depth: int) -> None:
        if depth > _MAX_STRUCTURED_DEPTH:
            raise ValueError(
                f"structured record nesting exceeds {_MAX_STRUCTURED_DEPTH} levels"
            )
        value = _embedded_structure(value)
        if isinstance(value, Mapping):
            if not value and path:
                key = separator.join(path)
                flattened[key] = {}
                return
            for child_key, child_value in value.items():
                normalized_key = normalize_text(str(child_key))
                if not normalized_key:
                    raise ValueError("structured record contains an empty field name")
                visit(child_value, (*path, normalized_key), depth + 1)
            return
        key = separator.join(path)
        if key in flattened:
            raise ValueError(f"structured record fields collide while flattening: {key}")
        flattened[key] = _canonical_value(value)
        if len(flattened) > _MAX_STRUCTURED_FIELDS:
            raise ValueError(
                f"structured record exceeds {_MAX_STRUCTURED_FIELDS} flattened fields"
            )

    for field, value in record.items():
        field_name = normalize_text(str(field))
        if not field_name:
            raise ValueError("structured record contains an empty field name")
        visit(value, (field_name,), 1)
    return flattened


def _clean_invalid_structured_records(
    records: Sequence[Mapping[str, Any]],
) -> list[dict[str, Any]]:
    if not records:
        return []
    fields: list[str] = []
    for record in records:
        for field in record:
            if field not in fields:
                fields.append(field)
    active_fields = [
        field
        for field in fields
        if any(not _is_empty_value(record.get(field)) for record in records)
    ]
    if not active_fields:
        return []
    identity_fields = [
        field
        for field in active_fields
        if _IDENTITY_FIELD_PATTERN.search(_normalize_field_name(field, "snake_case"))
    ]
    cleaned: list[dict[str, Any]] = []
    for record in records:
        values = {field: deepcopy(record.get(field)) for field in active_fields}
        # 空记录一定无效；存在身份字段时只把身份字段缺失视作“残缺行”，
        # 避免因为备注等可选列为空而误删有效业务数据。
        if all(_is_empty_value(value) for value in values.values()):
            continue
        if identity_fields and any(_is_empty_value(values[field]) for field in identity_fields):
            continue
        cleaned.append(values)
    return cleaned


def _percentile(values: Sequence[float], fraction: float) -> float:
    if not values:
        raise ValueError("cannot calculate a percentile of an empty sequence")
    ordered = sorted(values)
    position = (len(ordered) - 1) * fraction
    lower = math.floor(position)
    upper = math.ceil(position)
    if lower == upper:
        return ordered[lower]
    weight = position - lower
    return ordered[lower] * (1 - weight) + ordered[upper] * weight


def content_quality_flags(
    text: str,
    *,
    min_chars: int = 20,
    min_tokens: int = 5,
    max_chars: int = _MAX_ANOMALY_TEXT_CHARS,
) -> tuple[str, ...]:
    """返回非结构化内容的确定性低质量原因。"""

    if min_chars < 0 or min_tokens < 0 or max_chars <= 0:
        raise ValueError("content quality limits must be non-negative")
    normalized = normalize_text(text)
    if not normalized:
        return ("empty_content",)
    flags: list[str] = []
    if len(normalized) < min_chars or estimate_token_count(normalized) < min_tokens:
        flags.append("content_too_short")
    if len(normalized) > max_chars:
        flags.append("content_too_long")
    if any(marker in normalized for marker in _MOJIBAKE_MARKERS):
        flags.append("mojibake")
    nonspace = [char for char in normalized if not char.isspace()]
    if nonspace:
        readable_ratio = sum(
            char.isprintable()
            and unicodedata.category(char) not in {"Co", "Cs", "Cn"}
            for char in nonspace
        ) / len(nonspace)
        if readable_ratio < 0.85:
            flags.append("low_printable_ratio")
        if len(nonspace) >= 100:
            most_common = Counter(nonspace).most_common(1)[0][1]
            if most_common / len(nonspace) > 0.9:
                flags.append("repetitive_content")
    return tuple(dict.fromkeys(flags))


def is_low_quality_content(
    text: str,
    *,
    min_chars: int = 20,
    min_tokens: int = 5,
    max_chars: int = _MAX_ANOMALY_TEXT_CHARS,
) -> bool:
    """判断内容是否命中任一低质量规则。"""

    return bool(
        content_quality_flags(
            text,
            min_chars=min_chars,
            min_tokens=min_tokens,
            max_chars=max_chars,
        )
    )


def filter_anomalous_structured_records(
    records: Sequence[Mapping[str, Any]],
    *,
    iqr_multiplier: float = 1.5,
) -> list[dict[str, Any]]:
    """按字段级数值 IQR、乱码和极端文本长度过滤异常记录。"""

    if iqr_multiplier <= 0:
        raise ValueError("iqr_multiplier must be greater than 0")
    numeric_values: dict[str, list[float]] = {}
    text_lengths: dict[str, list[float]] = {}
    for record in records:
        for field, value in record.items():
            if (
                isinstance(value, (int, float))
                and not isinstance(value, bool)
                and not _IDENTITY_FIELD_PATTERN.search(
                    _normalize_field_name(field, "snake_case")
                )
            ):
                number = float(value)
                if math.isfinite(number):
                    numeric_values.setdefault(field, []).append(number)
            elif isinstance(value, str) and value:
                text_lengths.setdefault(field, []).append(float(len(value)))

    numeric_bounds: dict[str, tuple[float, float]] = {}
    for field, values in numeric_values.items():
        # 小样本不做统计异常判断，避免把合法长尾值误删。
        if len(values) < 8:
            continue
        first_quartile = _percentile(values, 0.25)
        third_quartile = _percentile(values, 0.75)
        spread = third_quartile - first_quartile
        numeric_bounds[field] = (
            first_quartile - iqr_multiplier * spread,
            third_quartile + iqr_multiplier * spread,
        )

    text_upper_bounds: dict[str, float] = {}
    for field, lengths in text_lengths.items():
        if len(lengths) < 8:
            continue
        first_quartile = _percentile(lengths, 0.25)
        third_quartile = _percentile(lengths, 0.75)
        spread = third_quartile - first_quartile
        text_upper_bounds[field] = max(512.0, third_quartile + 3 * spread)

    accepted: list[dict[str, Any]] = []
    for record in records:
        anomalous = False
        for field, value in record.items():
            if (
                isinstance(value, (int, float))
                and not isinstance(value, bool)
                and not _IDENTITY_FIELD_PATTERN.search(
                    _normalize_field_name(field, "snake_case")
                )
            ):
                number = float(value)
                if not math.isfinite(number):
                    anomalous = True
                    break
                bounds = numeric_bounds.get(field)
                if bounds and not bounds[0] <= number <= bounds[1]:
                    anomalous = True
                    break
            if isinstance(value, str):
                flags = content_quality_flags(value, min_chars=0, min_tokens=0)
                if {"content_too_long", "mojibake", "low_printable_ratio"} & set(flags):
                    anomalous = True
                    break
                upper_bound = text_upper_bounds.get(field)
                if upper_bound is not None and len(value) > upper_bound:
                    anomalous = True
                    break
        if not anomalous:
            accepted.append(deepcopy(dict(record)))
    return accepted


def _identity_tokens(record: Mapping[str, Any]) -> set[tuple[str, str]]:
    tokens: set[tuple[str, str]] = set()
    for field, value in record.items():
        normalized_field = _normalize_field_name(field, "snake_case")
        if not _IDENTITY_FIELD_PATTERN.search(normalized_field) or _is_empty_value(value):
            continue
        tokens.add(
            (
                normalized_field,
                json.dumps(
                    _canonical_value(value),
                    ensure_ascii=False,
                    sort_keys=True,
                    separators=(",", ":"),
                ),
            )
        )
    return tokens


def deduplicate_structured_records(
    records: Sequence[Mapping[str, Any]],
) -> list[dict[str, Any]]:
    """按整条 canonical JSON 和 id/uuid/key/code/*_id 字段稳定去重。"""

    exact_seen: set[str] = set()
    identity_seen: set[tuple[str, str]] = set()
    unique: list[dict[str, Any]] = []
    for record in records:
        fingerprint = hashlib.sha256(canonical_record_json(record).encode("utf-8")).hexdigest()
        identities = _identity_tokens(record)
        if fingerprint in exact_seen or identities & identity_seen:
            continue
        exact_seen.add(fingerprint)
        identity_seen.update(identities)
        unique.append(deepcopy(dict(record)))
    return unique


def _is_name_field(field: Any) -> bool:
    normalized = _normalize_field_name(field, "snake_case")
    if normalized in _NAME_FIELD_NAMES:
        return True
    suffixes = ("_name", "_full_name", "_姓名", "_真实姓名", "_联系人", "_联系人姓名")
    return normalized.endswith(suffixes)


def desensitize_structured_record(
    record: Mapping[str, Any],
) -> tuple[dict[str, Any], dict[str, int]]:
    """递归脱敏结构化姓名字段及任意文本中的手机号、邮箱、身份证号。"""

    if not isinstance(record, Mapping):
        raise TypeError("record must be a mapping")
    counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}

    def add_counts(values: Mapping[str, int]) -> None:
        for kind, count in values.items():
            if kind != "total" and count:
                counts[kind] = counts.get(kind, 0) + count

    def visit(value: Any, field: Any = "") -> Any:
        if isinstance(value, Mapping):
            return {key: visit(item, key) for key, item in value.items()}
        if isinstance(value, list):
            return [visit(item, field) for item in value]
        if isinstance(value, tuple):
            return [visit(item, field) for item in value]
        if _is_name_field(field) and not _is_empty_value(value):
            counts["name"] = counts.get("name", 0) + 1
            return "[NAME]"
        if isinstance(value, str):
            masked, found = desensitize_pii(value)
            add_counts(found)
            return masked
        return deepcopy(value)

    masked = {key: visit(value, key) for key, value in record.items()}
    counts["total"] = sum(counts.values())
    return masked, counts


def _structured_options(options: Iterable[str] | Mapping[str, Any]) -> set[str]:
    if isinstance(options, str):
        raise TypeError("options must be an iterable or mapping of option names")
    if isinstance(options, Mapping):
        enabled = {str(key) for key, value in options.items() if bool(value)}
    else:
        enabled = {str(option) for option in options}
    unknown = enabled - _STRUCTURED_OPTIONS
    if unknown:
        raise ValueError(f"unsupported structured preprocess options: {', '.join(sorted(unknown))}")
    return enabled


def preprocess_structured_records(
    records: Iterable[Mapping[str, Any]],
    options: Iterable[str] | Mapping[str, Any],
) -> list[dict[str, Any]]:
    """按界面选项执行确定性、无副作用的结构化数据预处理。"""

    enabled = _structured_options(options)
    current: list[dict[str, Any]] = []
    for record in records:
        if not isinstance(record, Mapping):
            if "clean_invalid" in enabled:
                continue
            raise TypeError("structured records must contain mappings")
        value = deepcopy(dict(record))
        if "detect_structure" in enabled:
            value = flatten_structured_record(value)
        if "normalize_format" in enabled:
            value = normalize_structured_record(value)
        current.append(value)

    if "clean_invalid" in enabled:
        current = _clean_invalid_structured_records(current)
    if "filter_anomaly" in enabled:
        current = filter_anomalous_structured_records(current)
    if "deduplicate" in enabled:
        current = deduplicate_structured_records(current)
    if "desensitize" in enabled:
        current = [desensitize_structured_record(record)[0] for record in current]
    return current


def estimate_token_count(text: str) -> int:
    """无分词器依赖的确定性 token 估算，用于预览与保护性限流。"""

    return len(_TOKEN_PATTERN.findall(text))


def _protected_markdown_ranges(
    text: str,
    *,
    preserve_code_blocks: bool,
    preserve_tables: bool,
    preserve_lists: bool,
) -> list[tuple[int, int]]:
    """找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""

    lines: list[tuple[int, int, str]] = []
    cursor = 0
    for raw_line in text.splitlines(keepends=True):
        end = cursor + len(raw_line)
        lines.append((cursor, end, raw_line.rstrip("\r\n")))
        cursor = end
    if cursor < len(text) or not lines:
        lines.append((cursor, len(text), text[cursor:]))

    ranges: list[tuple[int, int]] = []
    code_line_indexes: set[int] = set()
    if preserve_code_blocks:
        open_block: tuple[int, str, int] | None = None
        for index, (start, end, content) in enumerate(lines):
            fence = re.match(r"^\s*(`{3,}|~{3,})", content)
            if not fence:
                continue
            marker = fence.group(1)[0]
            length = len(fence.group(1))
            if open_block is None:
                open_block = (index, marker, length)
                continue
            first_index, open_marker, open_length = open_block
            if marker == open_marker and length >= open_length:
                ranges.append((lines[first_index][0], end))
                code_line_indexes.update(range(first_index, index + 1))
                open_block = None
        if open_block is not None:
            first_index = open_block[0]
            ranges.append((lines[first_index][0], len(text)))
            code_line_indexes.update(range(first_index, len(lines)))

    if preserve_tables:
        index = 0
        while index + 1 < len(lines):
            if index in code_line_indexes:
                index += 1
                continue
            header = lines[index][2].strip()
            separator = lines[index + 1][2].strip().strip("|")
            cells = [cell.strip() for cell in separator.split("|")]
            if (
                "|" not in header
                or len(cells) < 2
                or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
            ):
                index += 1
                continue
            end_index = index + 1
            while (
                end_index + 1 < len(lines)
                and end_index + 1 not in code_line_indexes
                and lines[end_index + 1][2].strip()
                and "|" in lines[end_index + 1][2]
            ):
                end_index += 1
            ranges.append((lines[index][0], lines[end_index][1]))
            index = end_index + 1

    if preserve_lists:
        list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
        continuation_pattern = re.compile(r"^\s{2,}\S")
        index = 0
        while index < len(lines):
            if index in code_line_indexes or not list_pattern.match(lines[index][2]):
                index += 1
                continue
            end_index = index
            item_count = 1
            while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
                next_line = lines[end_index + 1][2]
                if list_pattern.match(next_line):
                    item_count += 1
                    end_index += 1
                elif continuation_pattern.match(next_line):
                    end_index += 1
                else:
                    break
            if item_count >= 2:
                ranges.append((lines[index][0], lines[end_index][1]))
            index = end_index + 1

    merged: list[tuple[int, int]] = []
    for start, end in sorted(ranges):
        if merged and start < merged[-1][1]:
            merged[-1] = (merged[-1][0], max(merged[-1][1], end))
        else:
            merged.append((start, end))
    return merged


def protected_context_ranges(
    text: str,
    *,
    preserve_code_blocks: bool = True,
    preserve_tables: bool = True,
    preserve_lists: bool = True,
) -> tuple[tuple[int, int], ...]:
    """返回代码块、表格和列表的不可拆分区间。

    返回的偏移量基于规范化后的文本；调用方应在同一份 ``normalize_text``
    结果上使用这些区间。
    """

    normalized = normalize_text(text)
    return tuple(
        _protected_markdown_ranges(
            normalized,
            preserve_code_blocks=preserve_code_blocks,
            preserve_tables=preserve_tables,
            preserve_lists=preserve_lists,
        )
    )


def expand_to_context_boundaries(
    text: str,
    start: int,
    end: int,
    *,
    preserve_paragraph: bool = True,
    preserve_code_blocks: bool = True,
    preserve_tables: bool = True,
    preserve_lists: bool = True,
) -> tuple[int, int]:
    """将一个文本区间扩展到段落及受保护 Markdown 结构边界。"""

    normalized = normalize_text(text)
    if isinstance(start, bool) or isinstance(end, bool):
        raise TypeError("start and end must be integers")
    if not isinstance(start, int) or not isinstance(end, int):
        raise TypeError("start and end must be integers")
    if not 0 <= start <= end <= len(normalized):
        raise ValueError("start and end must define a valid normalized text range")

    expanded_start = start
    expanded_end = end
    if preserve_paragraph and normalized:
        paragraph_start = normalized.rfind("\n\n", 0, start)
        expanded_start = 0 if paragraph_start < 0 else paragraph_start + 2
        paragraph_end = normalized.find("\n\n", end)
        expanded_end = len(normalized) if paragraph_end < 0 else paragraph_end

    ranges = protected_context_ranges(
        normalized,
        preserve_code_blocks=preserve_code_blocks,
        preserve_tables=preserve_tables,
        preserve_lists=preserve_lists,
    )
    changed = True
    while changed:
        changed = False
        for range_start, range_end in ranges:
            overlaps = range_start < expanded_end and range_end > expanded_start
            contains_boundary = (
                range_start <= expanded_start < range_end
                or range_start < expanded_end <= range_end
            )
            if not overlaps and not contains_boundary:
                continue
            next_start = min(expanded_start, range_start)
            next_end = max(expanded_end, range_end)
            if (next_start, next_end) != (expanded_start, expanded_end):
                expanded_start, expanded_end = next_start, next_end
                changed = True
    return expanded_start, expanded_end


def detect_document_structure(text: str) -> DocumentStructure:
    """识别 Markdown、中文章节和数字编号标题及不可拆分块。"""

    normalized = normalize_text(text)
    if not normalized:
        return DocumentStructure(
            line_count=0,
            paragraph_count=0,
            headings=(),
            code_block_count=0,
            table_block_count=0,
            list_block_count=0,
        )

    code_ranges = protected_context_ranges(
        normalized,
        preserve_code_blocks=True,
        preserve_tables=False,
        preserve_lists=False,
    )
    table_candidates = protected_context_ranges(
        normalized,
        preserve_code_blocks=False,
        preserve_tables=True,
        preserve_lists=False,
    )
    list_candidates = protected_context_ranges(
        normalized,
        preserve_code_blocks=False,
        preserve_tables=False,
        preserve_lists=True,
    )
    table_ranges = tuple(
        item
        for item in table_candidates
        if not any(
            item[0] < code_end and item[1] > code_start
            for code_start, code_end in code_ranges
        )
    )
    list_ranges = tuple(
        item
        for item in list_candidates
        if not any(
            item[0] < code_end and item[1] > code_start
            for code_start, code_end in code_ranges
        )
    )
    markdown_heading = re.compile(r"^\s*(?P<marks>#{1,6})\s+(?P<title>.+?)\s*#*\s*$")
    chinese_heading = re.compile(
        r"^\s*(?P<title>第[一二三四五六七八九十百千万0-9]+[章节篇部分].*)$"
    )
    numbered_heading = re.compile(
        r"^\s*(?P<number>\d+(?:\.\d+)*)(?:[、.]|\s+)\s*(?P<title>\S.*)$"
    )

    headings: list[DocumentHeading] = []
    cursor = 0
    for line_number, raw_line in enumerate(normalized.splitlines(keepends=True), start=1):
        line = raw_line.rstrip("\n")
        line_end = cursor + len(line)
        if not any(range_start <= cursor < range_end for range_start, range_end in code_ranges):
            match = markdown_heading.match(line)
            if match:
                headings.append(
                    DocumentHeading(
                        level=len(match.group("marks")),
                        title=normalize_text(match.group("title")),
                        line_number=line_number,
                        start=cursor,
                        end=line_end,
                    )
                )
            else:
                match = chinese_heading.match(line)
                if match:
                    headings.append(
                        DocumentHeading(
                            level=1,
                            title=normalize_text(match.group("title")),
                            line_number=line_number,
                            start=cursor,
                            end=line_end,
                        )
                    )
                else:
                    match = numbered_heading.match(line)
                    if match:
                        headings.append(
                            DocumentHeading(
                                level=min(6, match.group("number").count(".") + 1),
                                title=normalize_text(match.group("title")),
                                line_number=line_number,
                                start=cursor,
                                end=line_end,
                            )
                        )
        cursor += len(raw_line)

    paragraphs = [part for part in re.split(r"\n\s*\n", normalized) if part.strip()]
    return DocumentStructure(
        line_count=len(normalized.splitlines()),
        paragraph_count=len(paragraphs),
        headings=tuple(headings),
        code_block_count=len(code_ranges),
        table_block_count=len(table_ranges),
        list_block_count=len(list_ranges),
    )


def merge_short_blocks(
    blocks: Iterable[str],
    *,
    min_token_count: int = 100,
    separator: str = "\n\n",
) -> list[str]:
    """按原顺序合并短内容块，并把末尾残块归入前一块。"""

    if isinstance(min_token_count, bool) or not isinstance(min_token_count, int):
        raise TypeError("min_token_count must be an integer")
    if min_token_count <= 0:
        raise ValueError("min_token_count must be greater than 0")
    if not isinstance(separator, str):
        raise TypeError("separator must be str")

    merged: list[str] = []
    pending: list[str] = []
    pending_tokens = 0
    for block in blocks:
        if not isinstance(block, str):
            raise TypeError("blocks must contain strings")
        normalized = normalize_text(block)
        if not normalized:
            continue
        token_count = estimate_token_count(normalized)
        if not pending and token_count >= min_token_count:
            merged.append(normalized)
            continue
        pending.append(normalized)
        pending_tokens += token_count
        if pending_tokens >= min_token_count:
            merged.append(separator.join(pending))
            pending = []
            pending_tokens = 0

    if pending:
        tail = separator.join(pending)
        if merged:
            merged[-1] = separator.join((merged[-1], tail))
        else:
            merged.append(tail)
    return merged


def _near_duplicate_features(text: str, shingle_size: int) -> tuple[str, ...]:
    if isinstance(shingle_size, bool) or not isinstance(shingle_size, int):
        raise TypeError("shingle_size must be an integer")
    if shingle_size <= 0:
        raise ValueError("shingle_size must be greater than 0")
    tokens = re.findall(
        r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+",
        normalize_text(text).casefold(),
    )
    if not tokens:
        return ()
    if len(tokens) < shingle_size:
        return ("\x1f".join(tokens),)
    return tuple(
        "\x1f".join(tokens[index : index + shingle_size])
        for index in range(len(tokens) - shingle_size + 1)
    )


def near_duplicate_fingerprint(text: str, *, shingle_size: int = 3) -> str:
    """生成 64 位 SimHash 指纹，用于低成本近重复候选筛选。"""

    if isinstance(shingle_size, bool) or not isinstance(shingle_size, int):
        raise TypeError("shingle_size must be an integer")
    if shingle_size <= 0:
        raise ValueError("shingle_size must be greater than 0")
    features = Counter(_near_duplicate_features(text, shingle_size))
    if not features:
        return "0" * 16
    vector = [0] * 64
    for feature, weight in features.items():
        digest = int.from_bytes(hashlib.sha256(feature.encode("utf-8")).digest()[:8], "big")
        for bit in range(64):
            vector[bit] += weight if digest & (1 << bit) else -weight
    fingerprint = sum(1 << bit for bit, value in enumerate(vector) if value >= 0)
    return f"{fingerprint:016x}"


def fingerprints_are_near_duplicate(
    left: str,
    right: str,
    *,
    max_hamming_distance: int = 3,
) -> bool:
    """比较两个 64 位十六进制 SimHash 指纹。"""

    if isinstance(max_hamming_distance, bool) or not isinstance(max_hamming_distance, int):
        raise TypeError("max_hamming_distance must be an integer")
    if not 0 <= max_hamming_distance <= 64:
        raise ValueError("max_hamming_distance must be in [0, 64]")
    if not re.fullmatch(r"[0-9a-fA-F]{16}", left) or not re.fullmatch(
        r"[0-9a-fA-F]{16}", right
    ):
        raise ValueError("fingerprints must be 16-character hexadecimal strings")
    distance = (int(left, 16) ^ int(right, 16)).bit_count()
    return distance <= max_hamming_distance


def is_near_duplicate(
    left: str,
    right: str,
    *,
    shingle_size: int = 3,
    similarity_threshold: float = 0.9,
    max_hamming_distance: int = 3,
) -> bool:
    """结合词片 Jaccard 和 SimHash 判断两段内容是否近重复。"""

    if isinstance(similarity_threshold, bool) or not isinstance(
        similarity_threshold, (int, float)
    ):
        raise TypeError("similarity_threshold must be a number")
    if not 0 <= similarity_threshold <= 1:
        raise ValueError("similarity_threshold must be in [0, 1]")
    if isinstance(max_hamming_distance, bool) or not isinstance(max_hamming_distance, int):
        raise TypeError("max_hamming_distance must be an integer")
    if not 0 <= max_hamming_distance <= 64:
        raise ValueError("max_hamming_distance must be in [0, 64]")
    left_normalized = normalize_text(left)
    right_normalized = normalize_text(right)
    if not left_normalized or not right_normalized:
        return left_normalized == right_normalized
    if left_normalized.casefold() == right_normalized.casefold():
        return True
    left_features = set(_near_duplicate_features(left_normalized, shingle_size))
    right_features = set(_near_duplicate_features(right_normalized, shingle_size))
    union = left_features | right_features
    similarity = len(left_features & right_features) / len(union) if union else 1.0
    if similarity >= similarity_threshold:
        return True
    return fingerprints_are_near_duplicate(
        near_duplicate_fingerprint(left_normalized, shingle_size=shingle_size),
        near_duplicate_fingerprint(right_normalized, shingle_size=shingle_size),
        max_hamming_distance=max_hamming_distance,
    )


def record_fingerprint(record: Mapping[str, Any]) -> str:
    """计算与字典键顺序无关的稳定记录指纹。"""

    canonical = {
        "instruction": normalize_text(str(record.get("instruction") or "")),
        "input": normalize_text(str(record.get("input") or "")),
        "output": normalize_text(str(record.get("output") or "")),
    }
    raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
    return hashlib.sha256(raw.encode("utf-8")).hexdigest()


def _readability_score(text: str) -> float:
    if not text:
        return 0.0
    nonspace = [char for char in text if not char.isspace()]
    if not nonspace:
        return 0.0
    printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
    useful_ratio = sum(
        char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
        for char in nonspace
    ) / len(nonspace)
    return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)


def _internal_duplicate_score(text: str) -> float:
    units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
    if len(units) <= 1:
        return 100.0
    return round(100 * len(set(units)) / len(units), 2)


def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
    """估算结果与来源文本的词元覆盖率。

    这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
    时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
    """

    source = normalize_text(source_content)
    if not source:
        return 100.0
    candidate = normalize_text(
        "\n".join(
            str(record.get(field) or "") for field in ("instruction", "input", "output")
        )
    )

    def semantic_tokens(text: str) -> set[str]:
        return {
            token.lower()
            for token in _TOKEN_PATTERN.findall(text)
            if token.isalnum() or "\u3400" <= token <= "\u9fff"
        }

    source_tokens = semantic_tokens(source)
    candidate_tokens = semantic_tokens(candidate)
    if not candidate_tokens:
        return 0.0
    if not source_tokens:
        return 0.0
    return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)


def score_quality(
    record: Mapping[str, Any],
    *,
    min_output_length: int = 20,
    source_content: str = "",
    known_fingerprints: Iterable[str] = (),
    threshold: float = 60.0,
) -> QualityScore:
    """按完整性、长度、可读性、来源相关性和重复度计算质量分。"""

    if min_output_length <= 0:
        raise ValueError("min_output_length must be greater than 0")
    if not 0 <= threshold <= 100:
        raise ValueError("threshold must be in [0, 100]")

    instruction = normalize_text(str(record.get("instruction") or ""))
    input_text = normalize_text(str(record.get("input") or ""))
    output = normalize_text(str(record.get("output") or ""))
    flags: list[str] = []

    completeness = 100.0
    if not instruction:
        completeness -= 50
        flags.append("missing_instruction")
    if not output:
        completeness -= 50
        flags.append("missing_output")

    output_length = len(output)
    length_score = round(min(100.0, output_length / min_output_length * 100), 2)
    if output_length < min_output_length:
        flags.append("output_too_short")

    readability = _readability_score("\n".join((instruction, input_text, output)))
    if readability < 70:
        flags.append("low_readability")

    relevance = _source_relevance_score(record, source_content)
    if source_content and relevance < 30:
        flags.append("low_source_relevance")

    fingerprint = record_fingerprint(record)
    known = set(known_fingerprints)
    duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
    if duplicate == 0:
        flags.append("duplicate_record")
    elif duplicate < 70:
        flags.append("repetitive_output")

    overall = round(
        completeness * 0.35
        + length_score * 0.20
        + readability * 0.20
        + relevance * 0.15
        + duplicate * 0.10,
        2,
    )
    hard_valid = bool(instruction and output)
    return QualityScore(
        overall=overall,
        completeness=completeness,
        length=length_score,
        readability=readability,
        relevance=relevance,
        duplicate=duplicate,
        is_valid=hard_valid and overall >= threshold,
        flags=tuple(flags),
        fingerprint=fingerprint,
    )


def stable_split(
    value: str | int,
    split: Mapping[str, int] | None = None,
    *,
    seed: str = "",
) -> DatasetSplit:
    """按稳定哈希将记录划分到 train/validation/test。"""

    ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
    required = {"train", "validation", "test"}
    if set(ratios) != required:
        raise ValueError("split must contain exactly train, validation and test")
    if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
        raise ValueError("split ratios must be non-negative integers")
    if sum(ratios.values()) != 100:
        raise ValueError("split ratios must sum to 100")

    digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
    bucket = int.from_bytes(digest[:8], "big") % 10_000
    train_boundary = ratios["train"] * 100
    validation_boundary = train_boundary + ratios["validation"] * 100
    if bucket < train_boundary:
        return "train"
    if bucket < validation_boundary:
        return "validation"
    return "test"


def stable_split_assignments(
    values: Sequence[str | int],
    split: Mapping[str, int] | None = None,
    *,
    seed: str = "",
) -> list[DatasetSplit]:
    """按稳定顺序和精确配额批量划分数据集。

    单条哈希分桶只能在大样本下近似比例。这里先按哈希稳定排序，再用
    最大余数法计算各切分配额，确保小数据集也严格遵循配置比例。
    """

    ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
    # 复用单条划分的参数校验，避免两套规则逐渐漂移。
    stable_split("validation", ratios, seed=seed)
    if not values:
        return []

    split_order: tuple[DatasetSplit, ...] = ("train", "validation", "test")
    exact = {name: len(values) * ratios[name] / 100 for name in split_order}
    quotas = {name: math.floor(exact[name]) for name in split_order}
    remaining = len(values) - sum(quotas.values())
    remainder_order = sorted(
        split_order,
        key=lambda name: (-(exact[name] - quotas[name]), split_order.index(name)),
    )
    for name in remainder_order[:remaining]:
        quotas[name] += 1

    ranked_indices = sorted(
        range(len(values)),
        key=lambda index: (
            hashlib.sha256(f"{seed}:{values[index]}".encode("utf-8")).digest(),
            index,
        ),
    )
    assignments: list[DatasetSplit] = ["train"] * len(values)
    cursor = 0
    for name in split_order:
        for index in ranked_indices[cursor : cursor + quotas[name]]:
            assignments[index] = name
        cursor += quotas[name]
    return assignments


def _preview_content(item: Mapping[str, Any]) -> str:
    for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
        value = item.get(field)
        if value is not None:
            return normalize_text(str(value))
    return ""


def _standard_fields(content: str) -> tuple[str, str, str]:
    if not content:
        return "", "", ""

    try:
        payload = json.loads(content)
    except json.JSONDecodeError:
        payload = None
    if isinstance(payload, Mapping):
        instruction = next(
            (
                str(payload[key])
                for key in ("instruction", "question", "prompt")
                if payload.get(key) is not None
            ),
            "",
        )
        input_text = next(
            (str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
            "",
        )
        output = next(
            (str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
            "",
        )
        if instruction or output:
            return normalize_text(instruction), normalize_text(input_text), normalize_text(output)

    question_answer = re.match(
        r"^\s*(?:问|question)\s*[:：]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[:：]\s*(.+)\s*$",
        content,
        flags=re.IGNORECASE | re.DOTALL,
    )
    if question_answer:
        return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))

    lines = [line.strip() for line in content.splitlines() if line.strip()]
    first_line = re.sub(r"^(?:问|question)\s*[:：]\s*", "", lines[0], flags=re.IGNORECASE)
    output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
    return normalize_text(first_line), "", output


def generate_standard_records(
    preview_items: Iterable[Mapping[str, Any]],
    *,
    qa_pairs_per_item: int = 1,
    semantic_enrichment: bool = False,
    split: Mapping[str, int] | None = None,
    split_seed: str = "",
) -> list[dict[str, Any]]:
    """把预览内容确定性转换为标准 instruction/input/output 记录。

    该函数只负责本地标准化，不冒充 LLM；服务层可将其作为无模型模式或
    LLM 响应解析后的统一落库步骤。
    """

    if not 1 <= qa_pairs_per_item <= MAX_QA_PAIRS_PER_ITEM:
        raise ValueError(
            f"qa_pairs_per_item must be in [1, {MAX_QA_PAIRS_PER_ITEM}]"
        )
    prefixes = (
        "请结合实际情况说明：",
        "请用通俗易懂的方式说明：",
        "请从实际应用角度说明：",
        "请简洁自然地说明：",
        "请详细解答：",
    )
    results: list[dict[str, Any]] = []
    for item_index, item in enumerate(preview_items):
        content = _preview_content(item)
        instruction, input_text, output = _standard_fields(content)
        preview_id = str(item.get("id") or f"preview-{item_index + 1}")
        for variant_index in range(qa_pairs_per_item):
            variant_instruction = instruction
            if variant_index:
                if semantic_enrichment:
                    prefix = prefixes[variant_index % len(prefixes)]
                    if variant_index >= len(prefixes):
                        prefix = (
                            f"{prefix.removesuffix('：')}"
                            f"（问法 {variant_index + 1}）："
                        )
                    variant_instruction = f"{prefix}{instruction}"
                else:
                    variant_instruction = f"{instruction}(问法 {variant_index + 1})"
            raw_id = f"{preview_id}:{variant_index + 1}"
            result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
            status = "valid" if variant_instruction and output else "invalid"
            results.append(
                {
                    "id": result_id,
                    "preview_item_id": preview_id,
                    "instruction": variant_instruction,
                    "input": input_text,
                    "output": output,
                    "original_instruction": variant_instruction,
                    "original_input": input_text,
                    "original_output": output,
                    "status": status,
                    "split": "train",
                }
            )
    assignments = stable_split_assignments(
        [str(result["id"]) for result in results],
        split,
        seed=split_seed,
    )
    for result, assignment in zip(results, assignments, strict=True):
        result["split"] = assignment
    return results


__all__ = [
    "DatasetSplit",
    "DocumentHeading",
    "DocumentNoiseSpan",
    "DocumentStructure",
    "ParsedText",
    "PdfPageText",
    "QualityScore",
    "SUPPORTED_TEXT_FORMATS",
    "StructuredPreprocessOption",
    "TextFormat",
    "canonical_record_json",
    "content_quality_flags",
    "decode_utf8",
    "desensitize_pii",
    "desensitize_structured_record",
    "detect_document_structure",
    "detect_pdf_document_noise",
    "detect_text_format",
    "deduplicate_structured_records",
    "estimate_token_count",
    "expand_to_context_boundaries",
    "extract_pdf_page_texts",
    "extract_structured_records",
    "filter_anomalous_structured_records",
    "fingerprints_are_near_duplicate",
    "flatten_structured_record",
    "generate_standard_records",
    "is_low_quality_content",
    "is_near_duplicate",
    "merge_short_blocks",
    "near_duplicate_fingerprint",
    "normalize_structured_record",
    "normalize_text",
    "parse_text_content",
    "parse_utf8_text",
    "preprocess_structured_records",
    "protected_context_ranges",
    "record_fingerprint",
    "remove_document_noise",
    "score_quality",
    "stable_split",
    "stable_split_assignments",
]