913 lines
32 KiB
Python
913 lines
32 KiB
Python
|
"""数据处理模块使用的无副作用算法。
|
|||
|
|
|
|||
|
|
本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
|
|||
|
|
所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
|
|||
|
|
``source[chunk.start:chunk.end]``。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import csv
|
|||
|
|
import hashlib
|
|||
|
|
import io
|
|||
|
|
import json
|
|||
|
|
import re
|
|||
|
|
import unicodedata
|
|||
|
|
from bisect import bisect_left
|
|||
|
|
from collections.abc import Iterable, Mapping, Sequence
|
|||
|
|
from dataclasses import dataclass
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Any, Literal
|
|||
|
|
|
|||
|
|
|
|||
|
|
TextFormat = Literal["json", "jsonl", "csv", "markdown", "txt"]
|
|||
|
|
ChunkMethod = Literal["semantic", "heading", "fixed", "custom"]
|
|||
|
|
DatasetSplit = Literal["train", "validation", "test"]
|
|||
|
|
|
|||
|
|
SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
|
|||
|
|
"json",
|
|||
|
|
"jsonl",
|
|||
|
|
"csv",
|
|||
|
|
"markdown",
|
|||
|
|
"txt",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
_FORMAT_ALIASES: dict[str, TextFormat] = {
|
|||
|
|
"json": "json",
|
|||
|
|
"jsonl": "jsonl",
|
|||
|
|
"ndjson": "jsonl",
|
|||
|
|
"csv": "csv",
|
|||
|
|
"tsv": "csv",
|
|||
|
|
"md": "markdown",
|
|||
|
|
"markdown": "markdown",
|
|||
|
|
"txt": "txt",
|
|||
|
|
"text": "txt",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_EMAIL_PATTERN = re.compile(
|
|||
|
|
r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
|
|||
|
|
r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
|
|||
|
|
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
|
|||
|
|
)
|
|||
|
|
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
|
|||
|
|
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
|
|||
|
|
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
|
|||
|
|
_HEADING_PATTERN = re.compile(
|
|||
|
|
r"(?m)^(?:#{1,6}\s+|第[一二三四五六七八九十百千万0-9]+[章节篇部分]\s*|"
|
|||
|
|
r"\d+(?:\.\d+)*[、.\s]+)"
|
|||
|
|
)
|
|||
|
|
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True, slots=True)
|
|||
|
|
class ParsedText:
|
|||
|
|
"""UTF-8 文本的解析结果。"""
|
|||
|
|
|
|||
|
|
format: TextFormat
|
|||
|
|
text: str
|
|||
|
|
records: tuple[dict[str, Any], ...]
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True, slots=True)
|
|||
|
|
class TextChunk:
|
|||
|
|
"""带有可追溯来源位置的非结构化文本切片。"""
|
|||
|
|
|
|||
|
|
content: str
|
|||
|
|
start: int
|
|||
|
|
end: int
|
|||
|
|
start_line: int
|
|||
|
|
end_line: int
|
|||
|
|
token_count: int
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True, slots=True)
|
|||
|
|
class QualityScore:
|
|||
|
|
"""标准 instruction/input/output 记录的可解释质量分。"""
|
|||
|
|
|
|||
|
|
overall: float
|
|||
|
|
completeness: float
|
|||
|
|
length: float
|
|||
|
|
readability: float
|
|||
|
|
relevance: float
|
|||
|
|
duplicate: float
|
|||
|
|
is_valid: bool
|
|||
|
|
flags: tuple[str, ...]
|
|||
|
|
fingerprint: str
|
|||
|
|
|
|||
|
|
|
|||
|
|
def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
|
|||
|
|
"""严格解码 UTF-8 文本，并移除可选 BOM。
|
|||
|
|
|
|||
|
|
不使用 ``errors='replace'``，避免上传内容损坏后仍被静默接收。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
if isinstance(raw, str):
|
|||
|
|
return raw.removeprefix("\ufeff")
|
|||
|
|
if not isinstance(raw, (bytes, bytearray, memoryview)):
|
|||
|
|
raise TypeError("raw must be bytes-like or str")
|
|||
|
|
try:
|
|||
|
|
return bytes(raw).decode("utf-8-sig")
|
|||
|
|
except UnicodeDecodeError as exc:
|
|||
|
|
raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
|
|||
|
|
"""``decode_utf8`` 的语义化别名，供上传服务直接调用。"""
|
|||
|
|
|
|||
|
|
return decode_utf8(raw)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_text(text: str) -> str:
|
|||
|
|
"""规范 Unicode、换行和行尾空白，同时保留段落结构。"""
|
|||
|
|
|
|||
|
|
if not isinstance(text, str):
|
|||
|
|
raise TypeError("text must be str")
|
|||
|
|
normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
|
|||
|
|
normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
|
|||
|
|
normalized = "".join(
|
|||
|
|
char
|
|||
|
|
for char in normalized
|
|||
|
|
if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
|
|||
|
|
)
|
|||
|
|
lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
|
|||
|
|
return "\n".join(lines).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _normalize_format(value: str | None) -> TextFormat | None:
|
|||
|
|
if value is None:
|
|||
|
|
return None
|
|||
|
|
normalized = value.strip().lower().removeprefix(".")
|
|||
|
|
try:
|
|||
|
|
return _FORMAT_ALIASES[normalized]
|
|||
|
|
except KeyError as exc:
|
|||
|
|
raise ValueError(f"unsupported text format: {value}") from exc
|
|||
|
|
|
|||
|
|
|
|||
|
|
def detect_text_format(
|
|||
|
|
*,
|
|||
|
|
filename: str | None = None,
|
|||
|
|
text: str = "",
|
|||
|
|
file_format: str | None = None,
|
|||
|
|
) -> TextFormat:
|
|||
|
|
"""按显式格式、扩展名和内容特征依次识别文本格式。"""
|
|||
|
|
|
|||
|
|
explicit = _normalize_format(file_format)
|
|||
|
|
if explicit:
|
|||
|
|
return explicit
|
|||
|
|
|
|||
|
|
if filename:
|
|||
|
|
suffix = Path(filename).suffix.lower().removeprefix(".")
|
|||
|
|
detected = _FORMAT_ALIASES.get(suffix)
|
|||
|
|
if detected:
|
|||
|
|
return detected
|
|||
|
|
|
|||
|
|
stripped = text.strip()
|
|||
|
|
if stripped:
|
|||
|
|
if stripped[0] in "[{":
|
|||
|
|
try:
|
|||
|
|
json.loads(stripped)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
pass
|
|||
|
|
else:
|
|||
|
|
return "json"
|
|||
|
|
|
|||
|
|
nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
|
|||
|
|
if len(nonempty_lines) > 1:
|
|||
|
|
try:
|
|||
|
|
for line in nonempty_lines:
|
|||
|
|
json.loads(line)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
pass
|
|||
|
|
else:
|
|||
|
|
return "jsonl"
|
|||
|
|
|
|||
|
|
if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
|
|||
|
|
r"(?m)^\s*\|.+\|\s*$", stripped
|
|||
|
|
):
|
|||
|
|
return "markdown"
|
|||
|
|
|
|||
|
|
sample = stripped[:8192]
|
|||
|
|
try:
|
|||
|
|
dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
|
|||
|
|
rows = list(csv.reader(io.StringIO(sample), dialect))
|
|||
|
|
if len(rows) >= 2 and len(rows[0]) >= 2:
|
|||
|
|
return "csv"
|
|||
|
|
except csv.Error:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
return "txt"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _normalize_value(value: Any) -> Any:
|
|||
|
|
if isinstance(value, str):
|
|||
|
|
return normalize_text(value)
|
|||
|
|
if isinstance(value, Mapping):
|
|||
|
|
return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
|
|||
|
|
if isinstance(value, list):
|
|||
|
|
return [_normalize_value(item) for item in value]
|
|||
|
|
return value
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _record_from_value(value: Any) -> dict[str, Any]:
|
|||
|
|
if isinstance(value, Mapping):
|
|||
|
|
return dict(_normalize_value(value))
|
|||
|
|
return {"value": _normalize_value(value)}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
|
|||
|
|
"""从 JSON、JSONL 或 CSV 中提取规范化记录。
|
|||
|
|
|
|||
|
|
JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组；
|
|||
|
|
其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
normalized_format = _normalize_format(file_format)
|
|||
|
|
if normalized_format not in {"json", "jsonl", "csv"}:
|
|||
|
|
raise ValueError("structured record extraction only supports JSON, JSONL and CSV")
|
|||
|
|
|
|||
|
|
normalized_text = normalize_text(text)
|
|||
|
|
if not normalized_text:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
if normalized_format == "json":
|
|||
|
|
try:
|
|||
|
|
payload = json.loads(normalized_text)
|
|||
|
|
except json.JSONDecodeError as exc:
|
|||
|
|
raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
|
|||
|
|
values: Sequence[Any]
|
|||
|
|
if isinstance(payload, list):
|
|||
|
|
values = payload
|
|||
|
|
elif isinstance(payload, Mapping):
|
|||
|
|
nested = next(
|
|||
|
|
(
|
|||
|
|
payload[key]
|
|||
|
|
for key in ("records", "data", "items", "rows")
|
|||
|
|
if isinstance(payload.get(key), list)
|
|||
|
|
),
|
|||
|
|
None,
|
|||
|
|
)
|
|||
|
|
values = nested if isinstance(nested, list) else [payload]
|
|||
|
|
else:
|
|||
|
|
values = [payload]
|
|||
|
|
return [_record_from_value(value) for value in values]
|
|||
|
|
|
|||
|
|
if normalized_format == "jsonl":
|
|||
|
|
records: list[dict[str, Any]] = []
|
|||
|
|
for line_number, line in enumerate(normalized_text.splitlines(), start=1):
|
|||
|
|
if not line.strip():
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
value = json.loads(line)
|
|||
|
|
except json.JSONDecodeError as exc:
|
|||
|
|
raise ValueError(
|
|||
|
|
f"invalid JSONL at line {line_number}, "
|
|||
|
|
f"column {exc.colno}: {exc.msg}"
|
|||
|
|
) from exc
|
|||
|
|
records.append(_record_from_value(value))
|
|||
|
|
return records
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
|
|||
|
|
except csv.Error:
|
|||
|
|
dialect = csv.excel
|
|||
|
|
reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
|
|||
|
|
if not reader.fieldnames:
|
|||
|
|
raise ValueError("CSV header is required")
|
|||
|
|
headers = [normalize_text(header or "") for header in reader.fieldnames]
|
|||
|
|
if any(not header for header in headers):
|
|||
|
|
raise ValueError("CSV header cannot be empty")
|
|||
|
|
if len(set(headers)) != len(headers):
|
|||
|
|
raise ValueError("CSV headers must be unique")
|
|||
|
|
reader.fieldnames = headers
|
|||
|
|
|
|||
|
|
records = []
|
|||
|
|
for row in reader:
|
|||
|
|
if None in row:
|
|||
|
|
raise ValueError("CSV row has more fields than the header")
|
|||
|
|
normalized_row = {
|
|||
|
|
key: normalize_text(value or "")
|
|||
|
|
for key, value in row.items()
|
|||
|
|
}
|
|||
|
|
if any(value for value in normalized_row.values()):
|
|||
|
|
records.append(normalized_row)
|
|||
|
|
return records
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_text_content(
|
|||
|
|
raw: bytes | bytearray | memoryview | str,
|
|||
|
|
*,
|
|||
|
|
filename: str | None = None,
|
|||
|
|
file_format: str | None = None,
|
|||
|
|
) -> ParsedText:
|
|||
|
|
"""严格解码并解析支持的 UTF-8 文本格式。"""
|
|||
|
|
|
|||
|
|
text = normalize_text(decode_utf8(raw))
|
|||
|
|
detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
|
|||
|
|
records: list[dict[str, Any]] = []
|
|||
|
|
if detected_format in {"json", "jsonl", "csv"}:
|
|||
|
|
records = extract_structured_records(text, detected_format)
|
|||
|
|
return ParsedText(format=detected_format, text=text, records=tuple(records))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
|
|||
|
|
"""掩码邮箱、中国大陆手机号和 15/18 位身份证号，并返回命中统计。"""
|
|||
|
|
|
|||
|
|
if not isinstance(text, str):
|
|||
|
|
raise TypeError("text must be str")
|
|||
|
|
counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}
|
|||
|
|
|
|||
|
|
def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
|
|||
|
|
def replacer(_: re.Match[str]) -> str:
|
|||
|
|
counts[kind] += 1
|
|||
|
|
return replacement
|
|||
|
|
|
|||
|
|
return pattern.sub(replacer, value)
|
|||
|
|
|
|||
|
|
masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
|
|||
|
|
masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
|
|||
|
|
masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)
|
|||
|
|
counts["total"] = sum(counts.values())
|
|||
|
|
return masked, counts
|
|||
|
|
|
|||
|
|
|
|||
|
|
def estimate_token_count(text: str) -> int:
|
|||
|
|
"""无分词器依赖的确定性 token 估算，用于预览与保护性限流。"""
|
|||
|
|
|
|||
|
|
return len(_TOKEN_PATTERN.findall(text))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _token_spans(text: str) -> list[tuple[int, int]]:
|
|||
|
|
return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _line_number(newline_offsets: list[int], offset: int) -> int:
|
|||
|
|
# 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
|
|||
|
|
return bisect_left(newline_offsets, offset) + 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
|
|||
|
|
starts = [span[0] for span in spans]
|
|||
|
|
return bisect_left(starts, offset)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _protected_markdown_ranges(
|
|||
|
|
text: str,
|
|||
|
|
*,
|
|||
|
|
preserve_code_blocks: bool,
|
|||
|
|
preserve_tables: bool,
|
|||
|
|
preserve_lists: bool,
|
|||
|
|
) -> list[tuple[int, int]]:
|
|||
|
|
"""找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""
|
|||
|
|
|
|||
|
|
lines: list[tuple[int, int, str]] = []
|
|||
|
|
cursor = 0
|
|||
|
|
for raw_line in text.splitlines(keepends=True):
|
|||
|
|
end = cursor + len(raw_line)
|
|||
|
|
lines.append((cursor, end, raw_line.rstrip("\r\n")))
|
|||
|
|
cursor = end
|
|||
|
|
if cursor < len(text) or not lines:
|
|||
|
|
lines.append((cursor, len(text), text[cursor:]))
|
|||
|
|
|
|||
|
|
ranges: list[tuple[int, int]] = []
|
|||
|
|
code_line_indexes: set[int] = set()
|
|||
|
|
if preserve_code_blocks:
|
|||
|
|
open_block: tuple[int, str, int] | None = None
|
|||
|
|
for index, (start, end, content) in enumerate(lines):
|
|||
|
|
fence = re.match(r"^\s*(`{3,}|~{3,})", content)
|
|||
|
|
if not fence:
|
|||
|
|
continue
|
|||
|
|
marker = fence.group(1)[0]
|
|||
|
|
length = len(fence.group(1))
|
|||
|
|
if open_block is None:
|
|||
|
|
open_block = (index, marker, length)
|
|||
|
|
continue
|
|||
|
|
first_index, open_marker, open_length = open_block
|
|||
|
|
if marker == open_marker and length >= open_length:
|
|||
|
|
ranges.append((lines[first_index][0], end))
|
|||
|
|
code_line_indexes.update(range(first_index, index + 1))
|
|||
|
|
open_block = None
|
|||
|
|
if open_block is not None:
|
|||
|
|
first_index = open_block[0]
|
|||
|
|
ranges.append((lines[first_index][0], len(text)))
|
|||
|
|
code_line_indexes.update(range(first_index, len(lines)))
|
|||
|
|
|
|||
|
|
if preserve_tables:
|
|||
|
|
index = 0
|
|||
|
|
while index + 1 < len(lines):
|
|||
|
|
if index in code_line_indexes:
|
|||
|
|
index += 1
|
|||
|
|
continue
|
|||
|
|
header = lines[index][2].strip()
|
|||
|
|
separator = lines[index + 1][2].strip().strip("|")
|
|||
|
|
cells = [cell.strip() for cell in separator.split("|")]
|
|||
|
|
if (
|
|||
|
|
"|" not in header
|
|||
|
|
or len(cells) < 2
|
|||
|
|
or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
|
|||
|
|
):
|
|||
|
|
index += 1
|
|||
|
|
continue
|
|||
|
|
end_index = index + 1
|
|||
|
|
while (
|
|||
|
|
end_index + 1 < len(lines)
|
|||
|
|
and end_index + 1 not in code_line_indexes
|
|||
|
|
and lines[end_index + 1][2].strip()
|
|||
|
|
and "|" in lines[end_index + 1][2]
|
|||
|
|
):
|
|||
|
|
end_index += 1
|
|||
|
|
ranges.append((lines[index][0], lines[end_index][1]))
|
|||
|
|
index = end_index + 1
|
|||
|
|
|
|||
|
|
if preserve_lists:
|
|||
|
|
list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
|
|||
|
|
continuation_pattern = re.compile(r"^\s{2,}\S")
|
|||
|
|
index = 0
|
|||
|
|
while index < len(lines):
|
|||
|
|
if index in code_line_indexes or not list_pattern.match(lines[index][2]):
|
|||
|
|
index += 1
|
|||
|
|
continue
|
|||
|
|
end_index = index
|
|||
|
|
item_count = 1
|
|||
|
|
while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
|
|||
|
|
next_line = lines[end_index + 1][2]
|
|||
|
|
if list_pattern.match(next_line):
|
|||
|
|
item_count += 1
|
|||
|
|
end_index += 1
|
|||
|
|
elif continuation_pattern.match(next_line):
|
|||
|
|
end_index += 1
|
|||
|
|
else:
|
|||
|
|
break
|
|||
|
|
if item_count >= 2:
|
|||
|
|
ranges.append((lines[index][0], lines[end_index][1]))
|
|||
|
|
index = end_index + 1
|
|||
|
|
|
|||
|
|
merged: list[tuple[int, int]] = []
|
|||
|
|
for start, end in sorted(ranges):
|
|||
|
|
if merged and start < merged[-1][1]:
|
|||
|
|
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
|||
|
|
else:
|
|||
|
|
merged.append((start, end))
|
|||
|
|
return merged
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _range_containing(
|
|||
|
|
ranges: Sequence[tuple[int, int]], offset: int
|
|||
|
|
) -> tuple[int, int] | None:
|
|||
|
|
return next((item for item in ranges if item[0] < offset < item[1]), None)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _boundary_for_method(
|
|||
|
|
text: str,
|
|||
|
|
spans: list[tuple[int, int]],
|
|||
|
|
start_index: int,
|
|||
|
|
ideal_end_index: int,
|
|||
|
|
minimum_end_index: int,
|
|||
|
|
method: ChunkMethod,
|
|||
|
|
custom_delimiter: str,
|
|||
|
|
) -> tuple[int, int | None]:
|
|||
|
|
if method == "fixed":
|
|||
|
|
return ideal_end_index, None
|
|||
|
|
|
|||
|
|
start_offset = spans[start_index][0]
|
|||
|
|
ideal_end_offset = spans[ideal_end_index - 1][1]
|
|||
|
|
minimum_end_offset = spans[minimum_end_index - 1][1]
|
|||
|
|
search_text = text[start_offset:ideal_end_offset]
|
|||
|
|
|
|||
|
|
if method == "custom":
|
|||
|
|
delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
|
|||
|
|
if not delimiter:
|
|||
|
|
raise ValueError("custom_delimiter is required for custom chunking")
|
|||
|
|
relative_minimum = max(0, minimum_end_offset - start_offset)
|
|||
|
|
delimiter_start = search_text.rfind(delimiter, relative_minimum)
|
|||
|
|
if delimiter_start >= 0:
|
|||
|
|
boundary_offset = start_offset + delimiter_start + len(delimiter)
|
|||
|
|
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
|||
|
|
if boundary_index > start_index:
|
|||
|
|
return min(boundary_index, ideal_end_index), boundary_offset
|
|||
|
|
return ideal_end_index, None
|
|||
|
|
|
|||
|
|
if method == "heading":
|
|||
|
|
heading_offsets = [
|
|||
|
|
start_offset + match.start()
|
|||
|
|
for match in _HEADING_PATTERN.finditer(search_text)
|
|||
|
|
if start_offset + match.start() >= minimum_end_offset
|
|||
|
|
]
|
|||
|
|
if heading_offsets:
|
|||
|
|
boundary_offset = heading_offsets[-1]
|
|||
|
|
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
|||
|
|
if start_index < boundary_index <= ideal_end_index:
|
|||
|
|
return boundary_index, boundary_offset
|
|||
|
|
|
|||
|
|
semantic_boundaries = [
|
|||
|
|
start_offset + match.end()
|
|||
|
|
for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(search_text)
|
|||
|
|
if start_offset + match.end() >= minimum_end_offset
|
|||
|
|
]
|
|||
|
|
if semantic_boundaries:
|
|||
|
|
boundary_offset = semantic_boundaries[-1]
|
|||
|
|
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
|||
|
|
if boundary_index > start_index:
|
|||
|
|
return min(boundary_index, ideal_end_index), boundary_offset
|
|||
|
|
return ideal_end_index, None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def chunk_unstructured(
|
|||
|
|
text: str,
|
|||
|
|
*,
|
|||
|
|
method: ChunkMethod = "semantic",
|
|||
|
|
chunk_size: int = 800,
|
|||
|
|
chunk_overlap: int = 100,
|
|||
|
|
min_chunk_size: int = 100,
|
|||
|
|
custom_delimiter: str = "",
|
|||
|
|
preserve_code_blocks: bool = False,
|
|||
|
|
preserve_tables: bool = False,
|
|||
|
|
preserve_lists: bool = False,
|
|||
|
|
) -> list[TextChunk]:
|
|||
|
|
"""按估算 token 切分非结构化文本。
|
|||
|
|
|
|||
|
|
overlap 足够时精确保留配置数量；短边界下会自动收缩，并且每轮至少推进
|
|||
|
|
一个 token，避免异常配置或分隔符造成死循环。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
if method not in {"semantic", "heading", "fixed", "custom"}:
|
|||
|
|
raise ValueError(f"unsupported chunk method: {method}")
|
|||
|
|
if chunk_size <= 0:
|
|||
|
|
raise ValueError("chunk_size must be greater than 0")
|
|||
|
|
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
|||
|
|
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
|||
|
|
if min_chunk_size <= 0 or min_chunk_size > chunk_size:
|
|||
|
|
raise ValueError("min_chunk_size must be in [1, chunk_size]")
|
|||
|
|
if chunk_overlap + min_chunk_size > chunk_size:
|
|||
|
|
raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
|
|||
|
|
if method == "custom" and not custom_delimiter:
|
|||
|
|
raise ValueError("custom_delimiter is required for custom chunking")
|
|||
|
|
|
|||
|
|
normalized = normalize_text(text)
|
|||
|
|
if not normalized:
|
|||
|
|
return []
|
|||
|
|
spans = _token_spans(normalized)
|
|||
|
|
if not spans:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
|||
|
|
protected_ranges = _protected_markdown_ranges(
|
|||
|
|
normalized,
|
|||
|
|
preserve_code_blocks=preserve_code_blocks,
|
|||
|
|
preserve_tables=preserve_tables,
|
|||
|
|
preserve_lists=preserve_lists,
|
|||
|
|
)
|
|||
|
|
chunks: list[TextChunk] = []
|
|||
|
|
start_index = 0
|
|||
|
|
|
|||
|
|
while start_index < len(spans):
|
|||
|
|
ideal_end_index = min(len(spans), start_index + chunk_size)
|
|||
|
|
if ideal_end_index == len(spans):
|
|||
|
|
end_index, end_override = ideal_end_index, len(normalized)
|
|||
|
|
else:
|
|||
|
|
minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
|
|||
|
|
end_index, end_override = _boundary_for_method(
|
|||
|
|
normalized,
|
|||
|
|
spans,
|
|||
|
|
start_index,
|
|||
|
|
ideal_end_index,
|
|||
|
|
minimum_end_index,
|
|||
|
|
method,
|
|||
|
|
custom_delimiter,
|
|||
|
|
)
|
|||
|
|
if end_index <= start_index:
|
|||
|
|
end_index = min(len(spans), start_index + chunk_size)
|
|||
|
|
end_override = None
|
|||
|
|
|
|||
|
|
start_offset = spans[start_index][0]
|
|||
|
|
end_offset = end_override if end_override is not None else spans[end_index - 1][1]
|
|||
|
|
end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
|
|||
|
|
split_range = _range_containing(protected_ranges, end_offset)
|
|||
|
|
if split_range:
|
|||
|
|
before_index = _token_index_at_or_after(spans, split_range[0])
|
|||
|
|
if before_index - start_index >= min_chunk_size:
|
|||
|
|
end_index = before_index
|
|||
|
|
end_offset = split_range[0]
|
|||
|
|
else:
|
|||
|
|
end_index = min(
|
|||
|
|
len(spans),
|
|||
|
|
max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
|
|||
|
|
)
|
|||
|
|
end_offset = split_range[1]
|
|||
|
|
content = normalized[start_offset:end_offset]
|
|||
|
|
chunks.append(
|
|||
|
|
TextChunk(
|
|||
|
|
content=content,
|
|||
|
|
start=start_offset,
|
|||
|
|
end=end_offset,
|
|||
|
|
start_line=_line_number(newline_offsets, start_offset),
|
|||
|
|
end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
|
|||
|
|
token_count=end_index - start_index,
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if end_index >= len(spans):
|
|||
|
|
break
|
|||
|
|
next_start = max(start_index + 1, end_index - chunk_overlap)
|
|||
|
|
overlap_range = _range_containing(protected_ranges, spans[next_start][0])
|
|||
|
|
if overlap_range:
|
|||
|
|
candidate = _token_index_at_or_after(spans, overlap_range[0])
|
|||
|
|
if candidate <= start_index:
|
|||
|
|
candidate = _token_index_at_or_after(spans, overlap_range[1])
|
|||
|
|
next_start = min(len(spans), max(start_index + 1, candidate))
|
|||
|
|
start_index = next_start
|
|||
|
|
|
|||
|
|
return chunks
|
|||
|
|
|
|||
|
|
|
|||
|
|
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
|||
|
|
"""计算与字典键顺序无关的稳定记录指纹。"""
|
|||
|
|
|
|||
|
|
canonical = {
|
|||
|
|
"instruction": normalize_text(str(record.get("instruction") or "")),
|
|||
|
|
"input": normalize_text(str(record.get("input") or "")),
|
|||
|
|
"output": normalize_text(str(record.get("output") or "")),
|
|||
|
|
}
|
|||
|
|
raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|||
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _readability_score(text: str) -> float:
|
|||
|
|
if not text:
|
|||
|
|
return 0.0
|
|||
|
|
nonspace = [char for char in text if not char.isspace()]
|
|||
|
|
if not nonspace:
|
|||
|
|
return 0.0
|
|||
|
|
printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
|
|||
|
|
useful_ratio = sum(
|
|||
|
|
char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
|
|||
|
|
for char in nonspace
|
|||
|
|
) / len(nonspace)
|
|||
|
|
return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _internal_duplicate_score(text: str) -> float:
|
|||
|
|
units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
|
|||
|
|
if len(units) <= 1:
|
|||
|
|
return 100.0
|
|||
|
|
return round(100 * len(set(units)) / len(units), 2)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
|
|||
|
|
"""估算结果与来源文本的词元覆盖率。
|
|||
|
|
|
|||
|
|
这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
|
|||
|
|
时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
source = normalize_text(source_content)
|
|||
|
|
if not source:
|
|||
|
|
return 100.0
|
|||
|
|
candidate = normalize_text(
|
|||
|
|
"\n".join(
|
|||
|
|
str(record.get(field) or "") for field in ("instruction", "input", "output")
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def semantic_tokens(text: str) -> set[str]:
|
|||
|
|
return {
|
|||
|
|
token.lower()
|
|||
|
|
for token in _TOKEN_PATTERN.findall(text)
|
|||
|
|
if token.isalnum() or "\u3400" <= token <= "\u9fff"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
source_tokens = semantic_tokens(source)
|
|||
|
|
candidate_tokens = semantic_tokens(candidate)
|
|||
|
|
if not candidate_tokens:
|
|||
|
|
return 0.0
|
|||
|
|
if not source_tokens:
|
|||
|
|
return 0.0
|
|||
|
|
return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def score_quality(
|
|||
|
|
record: Mapping[str, Any],
|
|||
|
|
*,
|
|||
|
|
min_output_length: int = 20,
|
|||
|
|
source_content: str = "",
|
|||
|
|
known_fingerprints: Iterable[str] = (),
|
|||
|
|
threshold: float = 60.0,
|
|||
|
|
) -> QualityScore:
|
|||
|
|
"""按完整性、长度、可读性、来源相关性和重复度计算质量分。"""
|
|||
|
|
|
|||
|
|
if min_output_length <= 0:
|
|||
|
|
raise ValueError("min_output_length must be greater than 0")
|
|||
|
|
if not 0 <= threshold <= 100:
|
|||
|
|
raise ValueError("threshold must be in [0, 100]")
|
|||
|
|
|
|||
|
|
instruction = normalize_text(str(record.get("instruction") or ""))
|
|||
|
|
input_text = normalize_text(str(record.get("input") or ""))
|
|||
|
|
output = normalize_text(str(record.get("output") or ""))
|
|||
|
|
flags: list[str] = []
|
|||
|
|
|
|||
|
|
completeness = 100.0
|
|||
|
|
if not instruction:
|
|||
|
|
completeness -= 50
|
|||
|
|
flags.append("missing_instruction")
|
|||
|
|
if not output:
|
|||
|
|
completeness -= 50
|
|||
|
|
flags.append("missing_output")
|
|||
|
|
|
|||
|
|
output_length = len(output)
|
|||
|
|
length_score = round(min(100.0, output_length / min_output_length * 100), 2)
|
|||
|
|
if output_length < min_output_length:
|
|||
|
|
flags.append("output_too_short")
|
|||
|
|
|
|||
|
|
readability = _readability_score("\n".join((instruction, input_text, output)))
|
|||
|
|
if readability < 70:
|
|||
|
|
flags.append("low_readability")
|
|||
|
|
|
|||
|
|
relevance = _source_relevance_score(record, source_content)
|
|||
|
|
if source_content and relevance < 30:
|
|||
|
|
flags.append("low_source_relevance")
|
|||
|
|
|
|||
|
|
fingerprint = record_fingerprint(record)
|
|||
|
|
known = set(known_fingerprints)
|
|||
|
|
duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
|
|||
|
|
if duplicate == 0:
|
|||
|
|
flags.append("duplicate_record")
|
|||
|
|
elif duplicate < 70:
|
|||
|
|
flags.append("repetitive_output")
|
|||
|
|
|
|||
|
|
overall = round(
|
|||
|
|
completeness * 0.35
|
|||
|
|
+ length_score * 0.20
|
|||
|
|
+ readability * 0.20
|
|||
|
|
+ relevance * 0.15
|
|||
|
|
+ duplicate * 0.10,
|
|||
|
|
2,
|
|||
|
|
)
|
|||
|
|
hard_valid = bool(instruction and output)
|
|||
|
|
return QualityScore(
|
|||
|
|
overall=overall,
|
|||
|
|
completeness=completeness,
|
|||
|
|
length=length_score,
|
|||
|
|
readability=readability,
|
|||
|
|
relevance=relevance,
|
|||
|
|
duplicate=duplicate,
|
|||
|
|
is_valid=hard_valid and overall >= threshold,
|
|||
|
|
flags=tuple(flags),
|
|||
|
|
fingerprint=fingerprint,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def stable_split(
|
|||
|
|
value: str | int,
|
|||
|
|
split: Mapping[str, int] | None = None,
|
|||
|
|
*,
|
|||
|
|
seed: str = "",
|
|||
|
|
) -> DatasetSplit:
|
|||
|
|
"""按稳定哈希将记录划分到 train/validation/test。"""
|
|||
|
|
|
|||
|
|
ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
|
|||
|
|
required = {"train", "validation", "test"}
|
|||
|
|
if set(ratios) != required:
|
|||
|
|
raise ValueError("split must contain exactly train, validation and test")
|
|||
|
|
if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
|
|||
|
|
raise ValueError("split ratios must be non-negative integers")
|
|||
|
|
if sum(ratios.values()) != 100:
|
|||
|
|
raise ValueError("split ratios must sum to 100")
|
|||
|
|
|
|||
|
|
digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
|
|||
|
|
bucket = int.from_bytes(digest[:8], "big") % 10_000
|
|||
|
|
train_boundary = ratios["train"] * 100
|
|||
|
|
validation_boundary = train_boundary + ratios["validation"] * 100
|
|||
|
|
if bucket < train_boundary:
|
|||
|
|
return "train"
|
|||
|
|
if bucket < validation_boundary:
|
|||
|
|
return "validation"
|
|||
|
|
return "test"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _preview_content(item: Mapping[str, Any]) -> str:
|
|||
|
|
for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
|
|||
|
|
value = item.get(field)
|
|||
|
|
if value is not None:
|
|||
|
|
return normalize_text(str(value))
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _standard_fields(content: str) -> tuple[str, str, str]:
|
|||
|
|
if not content:
|
|||
|
|
return "", "", ""
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
payload = json.loads(content)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
payload = None
|
|||
|
|
if isinstance(payload, Mapping):
|
|||
|
|
instruction = next(
|
|||
|
|
(
|
|||
|
|
str(payload[key])
|
|||
|
|
for key in ("instruction", "question", "prompt")
|
|||
|
|
if payload.get(key) is not None
|
|||
|
|
),
|
|||
|
|
"",
|
|||
|
|
)
|
|||
|
|
input_text = next(
|
|||
|
|
(str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
|
|||
|
|
"",
|
|||
|
|
)
|
|||
|
|
output = next(
|
|||
|
|
(str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
|
|||
|
|
"",
|
|||
|
|
)
|
|||
|
|
if instruction or output:
|
|||
|
|
return normalize_text(instruction), normalize_text(input_text), normalize_text(output)
|
|||
|
|
|
|||
|
|
question_answer = re.match(
|
|||
|
|
r"^\s*(?:问|question)\s*[:：]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[:：]\s*(.+)\s*$",
|
|||
|
|
content,
|
|||
|
|
flags=re.IGNORECASE | re.DOTALL,
|
|||
|
|
)
|
|||
|
|
if question_answer:
|
|||
|
|
return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))
|
|||
|
|
|
|||
|
|
lines = [line.strip() for line in content.splitlines() if line.strip()]
|
|||
|
|
first_line = re.sub(r"^(?:问|question)\s*[:：]\s*", "", lines[0], flags=re.IGNORECASE)
|
|||
|
|
output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
|
|||
|
|
return normalize_text(first_line), "", output
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_standard_records(
|
|||
|
|
preview_items: Iterable[Mapping[str, Any]],
|
|||
|
|
*,
|
|||
|
|
qa_pairs_per_item: int = 1,
|
|||
|
|
semantic_enrichment: bool = False,
|
|||
|
|
split: Mapping[str, int] | None = None,
|
|||
|
|
split_seed: str = "",
|
|||
|
|
) -> list[dict[str, Any]]:
|
|||
|
|
"""把预览内容确定性转换为标准 instruction/input/output 记录。
|
|||
|
|
|
|||
|
|
该函数只负责本地标准化，不冒充 LLM；服务层可将其作为无模型模式或
|
|||
|
|
LLM 响应解析后的统一落库步骤。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
if not 1 <= qa_pairs_per_item <= 5:
|
|||
|
|
raise ValueError("qa_pairs_per_item must be in [1, 5]")
|
|||
|
|
prefixes = (
|
|||
|
|
"请结合实际情况说明：",
|
|||
|
|
"请用通俗易懂的方式说明：",
|
|||
|
|
"请从实际应用角度说明：",
|
|||
|
|
"请简洁自然地说明：",
|
|||
|
|
"请详细解答：",
|
|||
|
|
)
|
|||
|
|
results: list[dict[str, Any]] = []
|
|||
|
|
for item_index, item in enumerate(preview_items):
|
|||
|
|
content = _preview_content(item)
|
|||
|
|
instruction, input_text, output = _standard_fields(content)
|
|||
|
|
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
|||
|
|
for variant_index in range(qa_pairs_per_item):
|
|||
|
|
variant_instruction = instruction
|
|||
|
|
if variant_index:
|
|||
|
|
if semantic_enrichment:
|
|||
|
|
variant_instruction = f"{prefixes[variant_index]}{instruction}"
|
|||
|
|
else:
|
|||
|
|
variant_instruction = f"{instruction}(问法 {variant_index + 1})"
|
|||
|
|
raw_id = f"{preview_id}:{variant_index + 1}"
|
|||
|
|
result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
|
|||
|
|
status = "valid" if variant_instruction and output else "invalid"
|
|||
|
|
results.append(
|
|||
|
|
{
|
|||
|
|
"id": result_id,
|
|||
|
|
"preview_item_id": preview_id,
|
|||
|
|
"instruction": variant_instruction,
|
|||
|
|
"input": input_text,
|
|||
|
|
"output": output,
|
|||
|
|
"original_instruction": variant_instruction,
|
|||
|
|
"original_input": input_text,
|
|||
|
|
"original_output": output,
|
|||
|
|
"status": status,
|
|||
|
|
"split": stable_split(result_id, split, seed=split_seed),
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
__all__ = [
|
|||
|
|
"ChunkMethod",
|
|||
|
|
"DatasetSplit",
|
|||
|
|
"ParsedText",
|
|||
|
|
"QualityScore",
|
|||
|
|
"SUPPORTED_TEXT_FORMATS",
|
|||
|
|
"TextChunk",
|
|||
|
|
"TextFormat",
|
|||
|
|
"chunk_unstructured",
|
|||
|
|
"decode_utf8",
|
|||
|
|
"desensitize_pii",
|
|||
|
|
"detect_text_format",
|
|||
|
|
"estimate_token_count",
|
|||
|
|
"extract_structured_records",
|
|||
|
|
"generate_standard_records",
|
|||
|
|
"normalize_text",
|
|||
|
|
"parse_text_content",
|
|||
|
|
"parse_utf8_text",
|
|||
|
|
"record_fingerprint",
|
|||
|
|
"score_quality",
|
|||
|
|
"stable_split",
|
|||
|
|
]
|