"""数据处理模块使用的无副作用算法。

本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
``source[chunk.start:chunk.end]``。
"""

from __future__ import annotations

import csv
import hashlib
import io
import json
import re
import unicodedata
from bisect import bisect_left
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal


TextFormat = Literal["json", "jsonl", "csv", "markdown", "txt"]
ChunkMethod = Literal["semantic", "heading", "fixed", "custom"]
DatasetSplit = Literal["train", "validation", "test"]

SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
    "json",
    "jsonl",
    "csv",
    "markdown",
    "txt",
)

_FORMAT_ALIASES: dict[str, TextFormat] = {
    "json": "json",
    "jsonl": "jsonl",
    "ndjson": "jsonl",
    "csv": "csv",
    "tsv": "csv",
    "md": "markdown",
    "markdown": "markdown",
    "txt": "txt",
    "text": "txt",
}

_EMAIL_PATTERN = re.compile(
    r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
    r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
    r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
)
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
_HEADING_PATTERN = re.compile(
    r"(?m)^(?:#{1,6}\s+|第[一二三四五六七八九十百千万0-9]+[章节篇部分]\s*|"
    r"\d+(?:\.\d+)*[、.\s]+)"
)
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")


@dataclass(frozen=True, slots=True)
class ParsedText:
    """UTF-8 文本的解析结果。"""

    format: TextFormat
    text: str
    records: tuple[dict[str, Any], ...]


@dataclass(frozen=True, slots=True)
class TextChunk:
    """带有可追溯来源位置的非结构化文本切片。"""

    content: str
    start: int
    end: int
    start_line: int
    end_line: int
    token_count: int


@dataclass(frozen=True, slots=True)
class QualityScore:
    """标准 instruction/input/output 记录的可解释质量分。"""

    overall: float
    completeness: float
    length: float
    readability: float
    relevance: float
    duplicate: float
    is_valid: bool
    flags: tuple[str, ...]
    fingerprint: str


def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
    """严格解码 UTF-8 文本，并移除可选 BOM。

    不使用 ``errors='replace'``，避免上传内容损坏后仍被静默接收。
    """

    if isinstance(raw, str):
        return raw.removeprefix("\ufeff")
    if not isinstance(raw, (bytes, bytearray, memoryview)):
        raise TypeError("raw must be bytes-like or str")
    try:
        return bytes(raw).decode("utf-8-sig")
    except UnicodeDecodeError as exc:
        raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc


def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
    """``decode_utf8`` 的语义化别名，供上传服务直接调用。"""

    return decode_utf8(raw)


def normalize_text(text: str) -> str:
    """规范 Unicode、换行和行尾空白，同时保留段落结构。"""

    if not isinstance(text, str):
        raise TypeError("text must be str")
    normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
    normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
    normalized = "".join(
        char
        for char in normalized
        if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
    )
    lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
    return "\n".join(lines).strip()


def _normalize_format(value: str | None) -> TextFormat | None:
    if value is None:
        return None
    normalized = value.strip().lower().removeprefix(".")
    try:
        return _FORMAT_ALIASES[normalized]
    except KeyError as exc:
        raise ValueError(f"unsupported text format: {value}") from exc


def detect_text_format(
    *,
    filename: str | None = None,
    text: str = "",
    file_format: str | None = None,
) -> TextFormat:
    """按显式格式、扩展名和内容特征依次识别文本格式。"""

    explicit = _normalize_format(file_format)
    if explicit:
        return explicit

    if filename:
        suffix = Path(filename).suffix.lower().removeprefix(".")
        detected = _FORMAT_ALIASES.get(suffix)
        if detected:
            return detected

    stripped = text.strip()
    if stripped:
        if stripped[0] in "[{":
            try:
                json.loads(stripped)
            except json.JSONDecodeError:
                pass
            else:
                return "json"

        nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
        if len(nonempty_lines) > 1:
            try:
                for line in nonempty_lines:
                    json.loads(line)
            except json.JSONDecodeError:
                pass
            else:
                return "jsonl"

        if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
            r"(?m)^\s*\|.+\|\s*$", stripped
        ):
            return "markdown"

        sample = stripped[:8192]
        try:
            dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
            rows = list(csv.reader(io.StringIO(sample), dialect))
            if len(rows) >= 2 and len(rows[0]) >= 2:
                return "csv"
        except csv.Error:
            pass

    return "txt"


def _normalize_value(value: Any) -> Any:
    if isinstance(value, str):
        return normalize_text(value)
    if isinstance(value, Mapping):
        return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
    if isinstance(value, list):
        return [_normalize_value(item) for item in value]
    return value


def _record_from_value(value: Any) -> dict[str, Any]:
    if isinstance(value, Mapping):
        return dict(_normalize_value(value))
    return {"value": _normalize_value(value)}


def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
    """从 JSON、JSONL 或 CSV 中提取规范化记录。

    JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组；
    其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
    """

    normalized_format = _normalize_format(file_format)
    if normalized_format not in {"json", "jsonl", "csv"}:
        raise ValueError("structured record extraction only supports JSON, JSONL and CSV")

    normalized_text = normalize_text(text)
    if not normalized_text:
        return []

    if normalized_format == "json":
        try:
            payload = json.loads(normalized_text)
        except json.JSONDecodeError as exc:
            raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
        values: Sequence[Any]
        if isinstance(payload, list):
            values = payload
        elif isinstance(payload, Mapping):
            nested = next(
                (
                    payload[key]
                    for key in ("records", "data", "items", "rows")
                    if isinstance(payload.get(key), list)
                ),
                None,
            )
            values = nested if isinstance(nested, list) else [payload]
        else:
            values = [payload]
        return [_record_from_value(value) for value in values]

    if normalized_format == "jsonl":
        records: list[dict[str, Any]] = []
        for line_number, line in enumerate(normalized_text.splitlines(), start=1):
            if not line.strip():
                continue
            try:
                value = json.loads(line)
            except json.JSONDecodeError as exc:
                raise ValueError(
                    f"invalid JSONL at line {line_number}, "
                    f"column {exc.colno}: {exc.msg}"
                ) from exc
            records.append(_record_from_value(value))
        return records

    try:
        dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
    except csv.Error:
        dialect = csv.excel
    reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
    if not reader.fieldnames:
        raise ValueError("CSV header is required")
    headers = [normalize_text(header or "") for header in reader.fieldnames]
    if any(not header for header in headers):
        raise ValueError("CSV header cannot be empty")
    if len(set(headers)) != len(headers):
        raise ValueError("CSV headers must be unique")
    reader.fieldnames = headers

    records = []
    for row in reader:
        if None in row:
            raise ValueError("CSV row has more fields than the header")
        normalized_row = {
            key: normalize_text(value or "")
            for key, value in row.items()
        }
        if any(value for value in normalized_row.values()):
            records.append(normalized_row)
    return records


def parse_text_content(
    raw: bytes | bytearray | memoryview | str,
    *,
    filename: str | None = None,
    file_format: str | None = None,
) -> ParsedText:
    """严格解码并解析支持的 UTF-8 文本格式。"""

    text = normalize_text(decode_utf8(raw))
    detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
    records: list[dict[str, Any]] = []
    if detected_format in {"json", "jsonl", "csv"}:
        records = extract_structured_records(text, detected_format)
    return ParsedText(format=detected_format, text=text, records=tuple(records))


def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
    """掩码邮箱、中国大陆手机号和 15/18 位身份证号，并返回命中统计。"""

    if not isinstance(text, str):
        raise TypeError("text must be str")
    counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}

    def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
        def replacer(_: re.Match[str]) -> str:
            counts[kind] += 1
            return replacement

        return pattern.sub(replacer, value)

    masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
    masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
    masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)
    counts["total"] = sum(counts.values())
    return masked, counts


def estimate_token_count(text: str) -> int:
    """无分词器依赖的确定性 token 估算，用于预览与保护性限流。"""

    return len(_TOKEN_PATTERN.findall(text))


def _token_spans(text: str) -> list[tuple[int, int]]:
    return [match.span() for match in _TOKEN_PATTERN.finditer(text)]


def _line_number(newline_offsets: list[int], offset: int) -> int:
    # 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
    return bisect_left(newline_offsets, offset) + 1


def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
    starts = [span[0] for span in spans]
    return bisect_left(starts, offset)


def _protected_markdown_ranges(
    text: str,
    *,
    preserve_code_blocks: bool,
    preserve_tables: bool,
    preserve_lists: bool,
) -> list[tuple[int, int]]:
    """找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""

    lines: list[tuple[int, int, str]] = []
    cursor = 0
    for raw_line in text.splitlines(keepends=True):
        end = cursor + len(raw_line)
        lines.append((cursor, end, raw_line.rstrip("\r\n")))
        cursor = end
    if cursor < len(text) or not lines:
        lines.append((cursor, len(text), text[cursor:]))

    ranges: list[tuple[int, int]] = []
    code_line_indexes: set[int] = set()
    if preserve_code_blocks:
        open_block: tuple[int, str, int] | None = None
        for index, (start, end, content) in enumerate(lines):
            fence = re.match(r"^\s*(`{3,}|~{3,})", content)
            if not fence:
                continue
            marker = fence.group(1)[0]
            length = len(fence.group(1))
            if open_block is None:
                open_block = (index, marker, length)
                continue
            first_index, open_marker, open_length = open_block
            if marker == open_marker and length >= open_length:
                ranges.append((lines[first_index][0], end))
                code_line_indexes.update(range(first_index, index + 1))
                open_block = None
        if open_block is not None:
            first_index = open_block[0]
            ranges.append((lines[first_index][0], len(text)))
            code_line_indexes.update(range(first_index, len(lines)))

    if preserve_tables:
        index = 0
        while index + 1 < len(lines):
            if index in code_line_indexes:
                index += 1
                continue
            header = lines[index][2].strip()
            separator = lines[index + 1][2].strip().strip("|")
            cells = [cell.strip() for cell in separator.split("|")]
            if (
                "|" not in header
                or len(cells) < 2
                or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
            ):
                index += 1
                continue
            end_index = index + 1
            while (
                end_index + 1 < len(lines)
                and end_index + 1 not in code_line_indexes
                and lines[end_index + 1][2].strip()
                and "|" in lines[end_index + 1][2]
            ):
                end_index += 1
            ranges.append((lines[index][0], lines[end_index][1]))
            index = end_index + 1

    if preserve_lists:
        list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
        continuation_pattern = re.compile(r"^\s{2,}\S")
        index = 0
        while index < len(lines):
            if index in code_line_indexes or not list_pattern.match(lines[index][2]):
                index += 1
                continue
            end_index = index
            item_count = 1
            while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
                next_line = lines[end_index + 1][2]
                if list_pattern.match(next_line):
                    item_count += 1
                    end_index += 1
                elif continuation_pattern.match(next_line):
                    end_index += 1
                else:
                    break
            if item_count >= 2:
                ranges.append((lines[index][0], lines[end_index][1]))
            index = end_index + 1

    merged: list[tuple[int, int]] = []
    for start, end in sorted(ranges):
        if merged and start < merged[-1][1]:
            merged[-1] = (merged[-1][0], max(merged[-1][1], end))
        else:
            merged.append((start, end))
    return merged


def _range_containing(
    ranges: Sequence[tuple[int, int]], offset: int
) -> tuple[int, int] | None:
    return next((item for item in ranges if item[0] < offset < item[1]), None)


def _boundary_for_method(
    text: str,
    spans: list[tuple[int, int]],
    start_index: int,
    ideal_end_index: int,
    minimum_end_index: int,
    method: ChunkMethod,
    custom_delimiter: str,
) -> tuple[int, int | None]:
    if method == "fixed":
        return ideal_end_index, None

    start_offset = spans[start_index][0]
    ideal_end_offset = spans[ideal_end_index - 1][1]
    minimum_end_offset = spans[minimum_end_index - 1][1]
    search_text = text[start_offset:ideal_end_offset]

    if method == "custom":
        delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
        if not delimiter:
            raise ValueError("custom_delimiter is required for custom chunking")
        relative_minimum = max(0, minimum_end_offset - start_offset)
        delimiter_start = search_text.rfind(delimiter, relative_minimum)
        if delimiter_start >= 0:
            boundary_offset = start_offset + delimiter_start + len(delimiter)
            boundary_index = _token_index_at_or_after(spans, boundary_offset)
            if boundary_index > start_index:
                return min(boundary_index, ideal_end_index), boundary_offset
        return ideal_end_index, None

    if method == "heading":
        heading_offsets = [
            start_offset + match.start()
            for match in _HEADING_PATTERN.finditer(search_text)
            if start_offset + match.start() >= minimum_end_offset
        ]
        if heading_offsets:
            boundary_offset = heading_offsets[-1]
            boundary_index = _token_index_at_or_after(spans, boundary_offset)
            if start_index < boundary_index <= ideal_end_index:
                return boundary_index, boundary_offset

    semantic_boundaries = [
        start_offset + match.end()
        for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(search_text)
        if start_offset + match.end() >= minimum_end_offset
    ]
    if semantic_boundaries:
        boundary_offset = semantic_boundaries[-1]
        boundary_index = _token_index_at_or_after(spans, boundary_offset)
        if boundary_index > start_index:
            return min(boundary_index, ideal_end_index), boundary_offset
    return ideal_end_index, None


def chunk_unstructured(
    text: str,
    *,
    method: ChunkMethod = "semantic",
    chunk_size: int = 800,
    chunk_overlap: int = 100,
    min_chunk_size: int = 100,
    custom_delimiter: str = "",
    preserve_code_blocks: bool = False,
    preserve_tables: bool = False,
    preserve_lists: bool = False,
) -> list[TextChunk]:
    """按估算 token 切分非结构化文本。

    overlap 足够时精确保留配置数量；短边界下会自动收缩，并且每轮至少推进
    一个 token，避免异常配置或分隔符造成死循环。
    """

    if method not in {"semantic", "heading", "fixed", "custom"}:
        raise ValueError(f"unsupported chunk method: {method}")
    if chunk_size <= 0:
        raise ValueError("chunk_size must be greater than 0")
    if chunk_overlap < 0 or chunk_overlap >= chunk_size:
        raise ValueError("chunk_overlap must be in [0, chunk_size)")
    if min_chunk_size <= 0 or min_chunk_size > chunk_size:
        raise ValueError("min_chunk_size must be in [1, chunk_size]")
    if chunk_overlap + min_chunk_size > chunk_size:
        raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
    if method == "custom" and not custom_delimiter:
        raise ValueError("custom_delimiter is required for custom chunking")

    normalized = normalize_text(text)
    if not normalized:
        return []
    spans = _token_spans(normalized)
    if not spans:
        return []

    newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
    protected_ranges = _protected_markdown_ranges(
        normalized,
        preserve_code_blocks=preserve_code_blocks,
        preserve_tables=preserve_tables,
        preserve_lists=preserve_lists,
    )
    chunks: list[TextChunk] = []
    start_index = 0

    while start_index < len(spans):
        ideal_end_index = min(len(spans), start_index + chunk_size)
        if ideal_end_index == len(spans):
            end_index, end_override = ideal_end_index, len(normalized)
        else:
            minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
            end_index, end_override = _boundary_for_method(
                normalized,
                spans,
                start_index,
                ideal_end_index,
                minimum_end_index,
                method,
                custom_delimiter,
            )
        if end_index <= start_index:
            end_index = min(len(spans), start_index + chunk_size)
            end_override = None

        start_offset = spans[start_index][0]
        end_offset = end_override if end_override is not None else spans[end_index - 1][1]
        end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
        split_range = _range_containing(protected_ranges, end_offset)
        if split_range:
            before_index = _token_index_at_or_after(spans, split_range[0])
            if before_index - start_index >= min_chunk_size:
                end_index = before_index
                end_offset = split_range[0]
            else:
                end_index = min(
                    len(spans),
                    max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
                )
                end_offset = split_range[1]
        content = normalized[start_offset:end_offset]
        chunks.append(
            TextChunk(
                content=content,
                start=start_offset,
                end=end_offset,
                start_line=_line_number(newline_offsets, start_offset),
                end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
                token_count=end_index - start_index,
            )
        )

        if end_index >= len(spans):
            break
        next_start = max(start_index + 1, end_index - chunk_overlap)
        overlap_range = _range_containing(protected_ranges, spans[next_start][0])
        if overlap_range:
            candidate = _token_index_at_or_after(spans, overlap_range[0])
            if candidate <= start_index:
                candidate = _token_index_at_or_after(spans, overlap_range[1])
            next_start = min(len(spans), max(start_index + 1, candidate))
        start_index = next_start

    return chunks


def record_fingerprint(record: Mapping[str, Any]) -> str:
    """计算与字典键顺序无关的稳定记录指纹。"""

    canonical = {
        "instruction": normalize_text(str(record.get("instruction") or "")),
        "input": normalize_text(str(record.get("input") or "")),
        "output": normalize_text(str(record.get("output") or "")),
    }
    raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
    return hashlib.sha256(raw.encode("utf-8")).hexdigest()


def _readability_score(text: str) -> float:
    if not text:
        return 0.0
    nonspace = [char for char in text if not char.isspace()]
    if not nonspace:
        return 0.0
    printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
    useful_ratio = sum(
        char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
        for char in nonspace
    ) / len(nonspace)
    return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)


def _internal_duplicate_score(text: str) -> float:
    units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
    if len(units) <= 1:
        return 100.0
    return round(100 * len(set(units)) / len(units), 2)


def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
    """估算结果与来源文本的词元覆盖率。

    这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
    时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
    """

    source = normalize_text(source_content)
    if not source:
        return 100.0
    candidate = normalize_text(
        "\n".join(
            str(record.get(field) or "") for field in ("instruction", "input", "output")
        )
    )

    def semantic_tokens(text: str) -> set[str]:
        return {
            token.lower()
            for token in _TOKEN_PATTERN.findall(text)
            if token.isalnum() or "\u3400" <= token <= "\u9fff"
        }

    source_tokens = semantic_tokens(source)
    candidate_tokens = semantic_tokens(candidate)
    if not candidate_tokens:
        return 0.0
    if not source_tokens:
        return 0.0
    return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)


def score_quality(
    record: Mapping[str, Any],
    *,
    min_output_length: int = 20,
    source_content: str = "",
    known_fingerprints: Iterable[str] = (),
    threshold: float = 60.0,
) -> QualityScore:
    """按完整性、长度、可读性、来源相关性和重复度计算质量分。"""

    if min_output_length <= 0:
        raise ValueError("min_output_length must be greater than 0")
    if not 0 <= threshold <= 100:
        raise ValueError("threshold must be in [0, 100]")

    instruction = normalize_text(str(record.get("instruction") or ""))
    input_text = normalize_text(str(record.get("input") or ""))
    output = normalize_text(str(record.get("output") or ""))
    flags: list[str] = []

    completeness = 100.0
    if not instruction:
        completeness -= 50
        flags.append("missing_instruction")
    if not output:
        completeness -= 50
        flags.append("missing_output")

    output_length = len(output)
    length_score = round(min(100.0, output_length / min_output_length * 100), 2)
    if output_length < min_output_length:
        flags.append("output_too_short")

    readability = _readability_score("\n".join((instruction, input_text, output)))
    if readability < 70:
        flags.append("low_readability")

    relevance = _source_relevance_score(record, source_content)
    if source_content and relevance < 30:
        flags.append("low_source_relevance")

    fingerprint = record_fingerprint(record)
    known = set(known_fingerprints)
    duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
    if duplicate == 0:
        flags.append("duplicate_record")
    elif duplicate < 70:
        flags.append("repetitive_output")

    overall = round(
        completeness * 0.35
        + length_score * 0.20
        + readability * 0.20
        + relevance * 0.15
        + duplicate * 0.10,
        2,
    )
    hard_valid = bool(instruction and output)
    return QualityScore(
        overall=overall,
        completeness=completeness,
        length=length_score,
        readability=readability,
        relevance=relevance,
        duplicate=duplicate,
        is_valid=hard_valid and overall >= threshold,
        flags=tuple(flags),
        fingerprint=fingerprint,
    )


def stable_split(
    value: str | int,
    split: Mapping[str, int] | None = None,
    *,
    seed: str = "",
) -> DatasetSplit:
    """按稳定哈希将记录划分到 train/validation/test。"""

    ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
    required = {"train", "validation", "test"}
    if set(ratios) != required:
        raise ValueError("split must contain exactly train, validation and test")
    if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
        raise ValueError("split ratios must be non-negative integers")
    if sum(ratios.values()) != 100:
        raise ValueError("split ratios must sum to 100")

    digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
    bucket = int.from_bytes(digest[:8], "big") % 10_000
    train_boundary = ratios["train"] * 100
    validation_boundary = train_boundary + ratios["validation"] * 100
    if bucket < train_boundary:
        return "train"
    if bucket < validation_boundary:
        return "validation"
    return "test"


def _preview_content(item: Mapping[str, Any]) -> str:
    for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
        value = item.get(field)
        if value is not None:
            return normalize_text(str(value))
    return ""


def _standard_fields(content: str) -> tuple[str, str, str]:
    if not content:
        return "", "", ""

    try:
        payload = json.loads(content)
    except json.JSONDecodeError:
        payload = None
    if isinstance(payload, Mapping):
        instruction = next(
            (
                str(payload[key])
                for key in ("instruction", "question", "prompt")
                if payload.get(key) is not None
            ),
            "",
        )
        input_text = next(
            (str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
            "",
        )
        output = next(
            (str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
            "",
        )
        if instruction or output:
            return normalize_text(instruction), normalize_text(input_text), normalize_text(output)

    question_answer = re.match(
        r"^\s*(?:问|question)\s*[:：]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[:：]\s*(.+)\s*$",
        content,
        flags=re.IGNORECASE | re.DOTALL,
    )
    if question_answer:
        return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))

    lines = [line.strip() for line in content.splitlines() if line.strip()]
    first_line = re.sub(r"^(?:问|question)\s*[:：]\s*", "", lines[0], flags=re.IGNORECASE)
    output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
    return normalize_text(first_line), "", output


def generate_standard_records(
    preview_items: Iterable[Mapping[str, Any]],
    *,
    qa_pairs_per_item: int = 1,
    semantic_enrichment: bool = False,
    split: Mapping[str, int] | None = None,
    split_seed: str = "",
) -> list[dict[str, Any]]:
    """把预览内容确定性转换为标准 instruction/input/output 记录。

    该函数只负责本地标准化，不冒充 LLM；服务层可将其作为无模型模式或
    LLM 响应解析后的统一落库步骤。
    """

    if not 1 <= qa_pairs_per_item <= 5:
        raise ValueError("qa_pairs_per_item must be in [1, 5]")
    prefixes = (
        "请结合实际情况说明：",
        "请用通俗易懂的方式说明：",
        "请从实际应用角度说明：",
        "请简洁自然地说明：",
        "请详细解答：",
    )
    results: list[dict[str, Any]] = []
    for item_index, item in enumerate(preview_items):
        content = _preview_content(item)
        instruction, input_text, output = _standard_fields(content)
        preview_id = str(item.get("id") or f"preview-{item_index + 1}")
        for variant_index in range(qa_pairs_per_item):
            variant_instruction = instruction
            if variant_index:
                if semantic_enrichment:
                    variant_instruction = f"{prefixes[variant_index]}{instruction}"
                else:
                    variant_instruction = f"{instruction}(问法 {variant_index + 1})"
            raw_id = f"{preview_id}:{variant_index + 1}"
            result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
            status = "valid" if variant_instruction and output else "invalid"
            results.append(
                {
                    "id": result_id,
                    "preview_item_id": preview_id,
                    "instruction": variant_instruction,
                    "input": input_text,
                    "output": output,
                    "original_instruction": variant_instruction,
                    "original_input": input_text,
                    "original_output": output,
                    "status": status,
                    "split": stable_split(result_id, split, seed=split_seed),
                }
            )
    return results


__all__ = [
    "ChunkMethod",
    "DatasetSplit",
    "ParsedText",
    "QualityScore",
    "SUPPORTED_TEXT_FORMATS",
    "TextChunk",
    "TextFormat",
    "chunk_unstructured",
    "decode_utf8",
    "desensitize_pii",
    "detect_text_format",
    "estimate_token_count",
    "extract_structured_records",
    "generate_standard_records",
    "normalize_text",
    "parse_text_content",
    "parse_utf8_text",
    "record_fingerprint",
    "score_quality",
    "stable_split",
]