feat: 完成数据处理接口与前端接入
This commit is contained in:
caoxiaozhu
912
backend/app/modules/data_process/algorithms.py
Normal file
912
backend/app/modules/data_process/algorithms.py
Normal file
@@ -0,0 +1,912 @@
|
||||
"""数据处理模块使用的无副作用算法。
|
||||
|
||||
本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
|
||||
所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
|
||||
``source[chunk.start:chunk.end]``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from bisect import bisect_left
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
TextFormat = Literal["json", "jsonl", "csv", "markdown", "txt"]
|
||||
ChunkMethod = Literal["semantic", "heading", "fixed", "custom"]
|
||||
DatasetSplit = Literal["train", "validation", "test"]
|
||||
|
||||
SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
|
||||
"json",
|
||||
"jsonl",
|
||||
"csv",
|
||||
"markdown",
|
||||
"txt",
|
||||
)
|
||||
|
||||
_FORMAT_ALIASES: dict[str, TextFormat] = {
|
||||
"json": "json",
|
||||
"jsonl": "jsonl",
|
||||
"ndjson": "jsonl",
|
||||
"csv": "csv",
|
||||
"tsv": "csv",
|
||||
"md": "markdown",
|
||||
"markdown": "markdown",
|
||||
"txt": "txt",
|
||||
"text": "txt",
|
||||
}
|
||||
|
||||
_EMAIL_PATTERN = re.compile(
|
||||
r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
|
||||
r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
|
||||
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
|
||||
)
|
||||
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
|
||||
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
|
||||
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
|
||||
_HEADING_PATTERN = re.compile(
|
||||
r"(?m)^(?:#{1,6}\s+|第[一二三四五六七八九十百千万0-9]+[章节篇部分]\s*|"
|
||||
r"\d+(?:\.\d+)*[、.\s]+)"
|
||||
)
|
||||
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedText:
|
||||
"""UTF-8 文本的解析结果。"""
|
||||
|
||||
format: TextFormat
|
||||
text: str
|
||||
records: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TextChunk:
|
||||
"""带有可追溯来源位置的非结构化文本切片。"""
|
||||
|
||||
content: str
|
||||
start: int
|
||||
end: int
|
||||
start_line: int
|
||||
end_line: int
|
||||
token_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QualityScore:
|
||||
"""标准 instruction/input/output 记录的可解释质量分。"""
|
||||
|
||||
overall: float
|
||||
completeness: float
|
||||
length: float
|
||||
readability: float
|
||||
relevance: float
|
||||
duplicate: float
|
||||
is_valid: bool
|
||||
flags: tuple[str, ...]
|
||||
fingerprint: str
|
||||
|
||||
|
||||
def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""严格解码 UTF-8 文本，并移除可选 BOM。
|
||||
|
||||
不使用 ``errors='replace'``，避免上传内容损坏后仍被静默接收。
|
||||
"""
|
||||
|
||||
if isinstance(raw, str):
|
||||
return raw.removeprefix("\ufeff")
|
||||
if not isinstance(raw, (bytes, bytearray, memoryview)):
|
||||
raise TypeError("raw must be bytes-like or str")
|
||||
try:
|
||||
return bytes(raw).decode("utf-8-sig")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc
|
||||
|
||||
|
||||
def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""``decode_utf8`` 的语义化别名，供上传服务直接调用。"""
|
||||
|
||||
return decode_utf8(raw)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""规范 Unicode、换行和行尾空白，同时保留段落结构。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
|
||||
normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
|
||||
normalized = "".join(
|
||||
char
|
||||
for char in normalized
|
||||
if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
|
||||
)
|
||||
lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _normalize_format(value: str | None) -> TextFormat | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip().lower().removeprefix(".")
|
||||
try:
|
||||
return _FORMAT_ALIASES[normalized]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unsupported text format: {value}") from exc
|
||||
|
||||
|
||||
def detect_text_format(
|
||||
*,
|
||||
filename: str | None = None,
|
||||
text: str = "",
|
||||
file_format: str | None = None,
|
||||
) -> TextFormat:
|
||||
"""按显式格式、扩展名和内容特征依次识别文本格式。"""
|
||||
|
||||
explicit = _normalize_format(file_format)
|
||||
if explicit:
|
||||
return explicit
|
||||
|
||||
if filename:
|
||||
suffix = Path(filename).suffix.lower().removeprefix(".")
|
||||
detected = _FORMAT_ALIASES.get(suffix)
|
||||
if detected:
|
||||
return detected
|
||||
|
||||
stripped = text.strip()
|
||||
if stripped:
|
||||
if stripped[0] in "[{":
|
||||
try:
|
||||
json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "json"
|
||||
|
||||
nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
|
||||
if len(nonempty_lines) > 1:
|
||||
try:
|
||||
for line in nonempty_lines:
|
||||
json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "jsonl"
|
||||
|
||||
if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
|
||||
r"(?m)^\s*\|.+\|\s*$", stripped
|
||||
):
|
||||
return "markdown"
|
||||
|
||||
sample = stripped[:8192]
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
|
||||
rows = list(csv.reader(io.StringIO(sample), dialect))
|
||||
if len(rows) >= 2 and len(rows[0]) >= 2:
|
||||
return "csv"
|
||||
except csv.Error:
|
||||
pass
|
||||
|
||||
return "txt"
|
||||
|
||||
|
||||
def _normalize_value(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return normalize_text(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_normalize_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _record_from_value(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return dict(_normalize_value(value))
|
||||
return {"value": _normalize_value(value)}
|
||||
|
||||
|
||||
def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
|
||||
"""从 JSON、JSONL 或 CSV 中提取规范化记录。
|
||||
|
||||
JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组；
|
||||
其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
|
||||
"""
|
||||
|
||||
normalized_format = _normalize_format(file_format)
|
||||
if normalized_format not in {"json", "jsonl", "csv"}:
|
||||
raise ValueError("structured record extraction only supports JSON, JSONL and CSV")
|
||||
|
||||
normalized_text = normalize_text(text)
|
||||
if not normalized_text:
|
||||
return []
|
||||
|
||||
if normalized_format == "json":
|
||||
try:
|
||||
payload = json.loads(normalized_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
|
||||
values: Sequence[Any]
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("records", "data", "items", "rows")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
values = [payload]
|
||||
return [_record_from_value(value) for value in values]
|
||||
|
||||
if normalized_format == "jsonl":
|
||||
records: list[dict[str, Any]] = []
|
||||
for line_number, line in enumerate(normalized_text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSONL at line {line_number}, "
|
||||
f"column {exc.colno}: {exc.msg}"
|
||||
) from exc
|
||||
records.append(_record_from_value(value))
|
||||
return records
|
||||
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
|
||||
except csv.Error:
|
||||
dialect = csv.excel
|
||||
reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV header is required")
|
||||
headers = [normalize_text(header or "") for header in reader.fieldnames]
|
||||
if any(not header for header in headers):
|
||||
raise ValueError("CSV header cannot be empty")
|
||||
if len(set(headers)) != len(headers):
|
||||
raise ValueError("CSV headers must be unique")
|
||||
reader.fieldnames = headers
|
||||
|
||||
records = []
|
||||
for row in reader:
|
||||
if None in row:
|
||||
raise ValueError("CSV row has more fields than the header")
|
||||
normalized_row = {
|
||||
key: normalize_text(value or "")
|
||||
for key, value in row.items()
|
||||
}
|
||||
if any(value for value in normalized_row.values()):
|
||||
records.append(normalized_row)
|
||||
return records
|
||||
|
||||
|
||||
def parse_text_content(
|
||||
raw: bytes | bytearray | memoryview | str,
|
||||
*,
|
||||
filename: str | None = None,
|
||||
file_format: str | None = None,
|
||||
) -> ParsedText:
|
||||
"""严格解码并解析支持的 UTF-8 文本格式。"""
|
||||
|
||||
text = normalize_text(decode_utf8(raw))
|
||||
detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
|
||||
records: list[dict[str, Any]] = []
|
||||
if detected_format in {"json", "jsonl", "csv"}:
|
||||
records = extract_structured_records(text, detected_format)
|
||||
return ParsedText(format=detected_format, text=text, records=tuple(records))
|
||||
|
||||
|
||||
def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
|
||||
"""掩码邮箱、中国大陆手机号和 15/18 位身份证号，并返回命中统计。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}
|
||||
|
||||
def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
|
||||
def replacer(_: re.Match[str]) -> str:
|
||||
counts[kind] += 1
|
||||
return replacement
|
||||
|
||||
return pattern.sub(replacer, value)
|
||||
|
||||
masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
|
||||
masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
|
||||
masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)
|
||||
counts["total"] = sum(counts.values())
|
||||
return masked, counts
|
||||
|
||||
|
||||
def estimate_token_count(text: str) -> int:
|
||||
"""无分词器依赖的确定性 token 估算，用于预览与保护性限流。"""
|
||||
|
||||
return len(_TOKEN_PATTERN.findall(text))
|
||||
|
||||
|
||||
def _token_spans(text: str) -> list[tuple[int, int]]:
|
||||
return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
|
||||
|
||||
|
||||
def _line_number(newline_offsets: list[int], offset: int) -> int:
|
||||
# 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
|
||||
return bisect_left(newline_offsets, offset) + 1
|
||||
|
||||
|
||||
def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
|
||||
starts = [span[0] for span in spans]
|
||||
return bisect_left(starts, offset)
|
||||
|
||||
|
||||
def _protected_markdown_ranges(
|
||||
text: str,
|
||||
*,
|
||||
preserve_code_blocks: bool,
|
||||
preserve_tables: bool,
|
||||
preserve_lists: bool,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""
|
||||
|
||||
lines: list[tuple[int, int, str]] = []
|
||||
cursor = 0
|
||||
for raw_line in text.splitlines(keepends=True):
|
||||
end = cursor + len(raw_line)
|
||||
lines.append((cursor, end, raw_line.rstrip("\r\n")))
|
||||
cursor = end
|
||||
if cursor < len(text) or not lines:
|
||||
lines.append((cursor, len(text), text[cursor:]))
|
||||
|
||||
ranges: list[tuple[int, int]] = []
|
||||
code_line_indexes: set[int] = set()
|
||||
if preserve_code_blocks:
|
||||
open_block: tuple[int, str, int] | None = None
|
||||
for index, (start, end, content) in enumerate(lines):
|
||||
fence = re.match(r"^\s*(`{3,}|~{3,})", content)
|
||||
if not fence:
|
||||
continue
|
||||
marker = fence.group(1)[0]
|
||||
length = len(fence.group(1))
|
||||
if open_block is None:
|
||||
open_block = (index, marker, length)
|
||||
continue
|
||||
first_index, open_marker, open_length = open_block
|
||||
if marker == open_marker and length >= open_length:
|
||||
ranges.append((lines[first_index][0], end))
|
||||
code_line_indexes.update(range(first_index, index + 1))
|
||||
open_block = None
|
||||
if open_block is not None:
|
||||
first_index = open_block[0]
|
||||
ranges.append((lines[first_index][0], len(text)))
|
||||
code_line_indexes.update(range(first_index, len(lines)))
|
||||
|
||||
if preserve_tables:
|
||||
index = 0
|
||||
while index + 1 < len(lines):
|
||||
if index in code_line_indexes:
|
||||
index += 1
|
||||
continue
|
||||
header = lines[index][2].strip()
|
||||
separator = lines[index + 1][2].strip().strip("|")
|
||||
cells = [cell.strip() for cell in separator.split("|")]
|
||||
if (
|
||||
"|" not in header
|
||||
or len(cells) < 2
|
||||
or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index + 1
|
||||
while (
|
||||
end_index + 1 < len(lines)
|
||||
and end_index + 1 not in code_line_indexes
|
||||
and lines[end_index + 1][2].strip()
|
||||
and "|" in lines[end_index + 1][2]
|
||||
):
|
||||
end_index += 1
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
if preserve_lists:
|
||||
list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
|
||||
continuation_pattern = re.compile(r"^\s{2,}\S")
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
if index in code_line_indexes or not list_pattern.match(lines[index][2]):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index
|
||||
item_count = 1
|
||||
while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
|
||||
next_line = lines[end_index + 1][2]
|
||||
if list_pattern.match(next_line):
|
||||
item_count += 1
|
||||
end_index += 1
|
||||
elif continuation_pattern.match(next_line):
|
||||
end_index += 1
|
||||
else:
|
||||
break
|
||||
if item_count >= 2:
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
merged: list[tuple[int, int]] = []
|
||||
for start, end in sorted(ranges):
|
||||
if merged and start < merged[-1][1]:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
return merged
|
||||
|
||||
|
||||
def _range_containing(
|
||||
ranges: Sequence[tuple[int, int]], offset: int
|
||||
) -> tuple[int, int] | None:
|
||||
return next((item for item in ranges if item[0] < offset < item[1]), None)
|
||||
|
||||
|
||||
def _boundary_for_method(
|
||||
text: str,
|
||||
spans: list[tuple[int, int]],
|
||||
start_index: int,
|
||||
ideal_end_index: int,
|
||||
minimum_end_index: int,
|
||||
method: ChunkMethod,
|
||||
custom_delimiter: str,
|
||||
) -> tuple[int, int | None]:
|
||||
if method == "fixed":
|
||||
return ideal_end_index, None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
ideal_end_offset = spans[ideal_end_index - 1][1]
|
||||
minimum_end_offset = spans[minimum_end_index - 1][1]
|
||||
search_text = text[start_offset:ideal_end_offset]
|
||||
|
||||
if method == "custom":
|
||||
delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
|
||||
if not delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
relative_minimum = max(0, minimum_end_offset - start_offset)
|
||||
delimiter_start = search_text.rfind(delimiter, relative_minimum)
|
||||
if delimiter_start >= 0:
|
||||
boundary_offset = start_offset + delimiter_start + len(delimiter)
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
if method == "heading":
|
||||
heading_offsets = [
|
||||
start_offset + match.start()
|
||||
for match in _HEADING_PATTERN.finditer(search_text)
|
||||
if start_offset + match.start() >= minimum_end_offset
|
||||
]
|
||||
if heading_offsets:
|
||||
boundary_offset = heading_offsets[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if start_index < boundary_index <= ideal_end_index:
|
||||
return boundary_index, boundary_offset
|
||||
|
||||
semantic_boundaries = [
|
||||
start_offset + match.end()
|
||||
for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(search_text)
|
||||
if start_offset + match.end() >= minimum_end_offset
|
||||
]
|
||||
if semantic_boundaries:
|
||||
boundary_offset = semantic_boundaries[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
|
||||
def chunk_unstructured(
|
||||
text: str,
|
||||
*,
|
||||
method: ChunkMethod = "semantic",
|
||||
chunk_size: int = 800,
|
||||
chunk_overlap: int = 100,
|
||||
min_chunk_size: int = 100,
|
||||
custom_delimiter: str = "",
|
||||
preserve_code_blocks: bool = False,
|
||||
preserve_tables: bool = False,
|
||||
preserve_lists: bool = False,
|
||||
) -> list[TextChunk]:
|
||||
"""按估算 token 切分非结构化文本。
|
||||
|
||||
overlap 足够时精确保留配置数量；短边界下会自动收缩，并且每轮至少推进
|
||||
一个 token，避免异常配置或分隔符造成死循环。
|
||||
"""
|
||||
|
||||
if method not in {"semantic", "heading", "fixed", "custom"}:
|
||||
raise ValueError(f"unsupported chunk method: {method}")
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be greater than 0")
|
||||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if min_chunk_size <= 0 or min_chunk_size > chunk_size:
|
||||
raise ValueError("min_chunk_size must be in [1, chunk_size]")
|
||||
if chunk_overlap + min_chunk_size > chunk_size:
|
||||
raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
|
||||
if method == "custom" and not custom_delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
spans = _token_spans(normalized)
|
||||
if not spans:
|
||||
return []
|
||||
|
||||
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
||||
protected_ranges = _protected_markdown_ranges(
|
||||
normalized,
|
||||
preserve_code_blocks=preserve_code_blocks,
|
||||
preserve_tables=preserve_tables,
|
||||
preserve_lists=preserve_lists,
|
||||
)
|
||||
chunks: list[TextChunk] = []
|
||||
start_index = 0
|
||||
|
||||
while start_index < len(spans):
|
||||
ideal_end_index = min(len(spans), start_index + chunk_size)
|
||||
if ideal_end_index == len(spans):
|
||||
end_index, end_override = ideal_end_index, len(normalized)
|
||||
else:
|
||||
minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
|
||||
end_index, end_override = _boundary_for_method(
|
||||
normalized,
|
||||
spans,
|
||||
start_index,
|
||||
ideal_end_index,
|
||||
minimum_end_index,
|
||||
method,
|
||||
custom_delimiter,
|
||||
)
|
||||
if end_index <= start_index:
|
||||
end_index = min(len(spans), start_index + chunk_size)
|
||||
end_override = None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
end_offset = end_override if end_override is not None else spans[end_index - 1][1]
|
||||
end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
|
||||
split_range = _range_containing(protected_ranges, end_offset)
|
||||
if split_range:
|
||||
before_index = _token_index_at_or_after(spans, split_range[0])
|
||||
if before_index - start_index >= min_chunk_size:
|
||||
end_index = before_index
|
||||
end_offset = split_range[0]
|
||||
else:
|
||||
end_index = min(
|
||||
len(spans),
|
||||
max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
|
||||
)
|
||||
end_offset = split_range[1]
|
||||
content = normalized[start_offset:end_offset]
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
content=content,
|
||||
start=start_offset,
|
||||
end=end_offset,
|
||||
start_line=_line_number(newline_offsets, start_offset),
|
||||
end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
|
||||
token_count=end_index - start_index,
|
||||
)
|
||||
)
|
||||
|
||||
if end_index >= len(spans):
|
||||
break
|
||||
next_start = max(start_index + 1, end_index - chunk_overlap)
|
||||
overlap_range = _range_containing(protected_ranges, spans[next_start][0])
|
||||
if overlap_range:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[0])
|
||||
if candidate <= start_index:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[1])
|
||||
next_start = min(len(spans), max(start_index + 1, candidate))
|
||||
start_index = next_start
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
||||
"""计算与字典键顺序无关的稳定记录指纹。"""
|
||||
|
||||
canonical = {
|
||||
"instruction": normalize_text(str(record.get("instruction") or "")),
|
||||
"input": normalize_text(str(record.get("input") or "")),
|
||||
"output": normalize_text(str(record.get("output") or "")),
|
||||
}
|
||||
raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _readability_score(text: str) -> float:
|
||||
if not text:
|
||||
return 0.0
|
||||
nonspace = [char for char in text if not char.isspace()]
|
||||
if not nonspace:
|
||||
return 0.0
|
||||
printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
|
||||
useful_ratio = sum(
|
||||
char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
|
||||
for char in nonspace
|
||||
) / len(nonspace)
|
||||
return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)
|
||||
|
||||
|
||||
def _internal_duplicate_score(text: str) -> float:
|
||||
units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
|
||||
if len(units) <= 1:
|
||||
return 100.0
|
||||
return round(100 * len(set(units)) / len(units), 2)
|
||||
|
||||
|
||||
def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
|
||||
"""估算结果与来源文本的词元覆盖率。
|
||||
|
||||
这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
|
||||
时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
|
||||
"""
|
||||
|
||||
source = normalize_text(source_content)
|
||||
if not source:
|
||||
return 100.0
|
||||
candidate = normalize_text(
|
||||
"\n".join(
|
||||
str(record.get(field) or "") for field in ("instruction", "input", "output")
|
||||
)
|
||||
)
|
||||
|
||||
def semantic_tokens(text: str) -> set[str]:
|
||||
return {
|
||||
token.lower()
|
||||
for token in _TOKEN_PATTERN.findall(text)
|
||||
if token.isalnum() or "\u3400" <= token <= "\u9fff"
|
||||
}
|
||||
|
||||
source_tokens = semantic_tokens(source)
|
||||
candidate_tokens = semantic_tokens(candidate)
|
||||
if not candidate_tokens:
|
||||
return 0.0
|
||||
if not source_tokens:
|
||||
return 0.0
|
||||
return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)
|
||||
|
||||
|
||||
def score_quality(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
min_output_length: int = 20,
|
||||
source_content: str = "",
|
||||
known_fingerprints: Iterable[str] = (),
|
||||
threshold: float = 60.0,
|
||||
) -> QualityScore:
|
||||
"""按完整性、长度、可读性、来源相关性和重复度计算质量分。"""
|
||||
|
||||
if min_output_length <= 0:
|
||||
raise ValueError("min_output_length must be greater than 0")
|
||||
if not 0 <= threshold <= 100:
|
||||
raise ValueError("threshold must be in [0, 100]")
|
||||
|
||||
instruction = normalize_text(str(record.get("instruction") or ""))
|
||||
input_text = normalize_text(str(record.get("input") or ""))
|
||||
output = normalize_text(str(record.get("output") or ""))
|
||||
flags: list[str] = []
|
||||
|
||||
completeness = 100.0
|
||||
if not instruction:
|
||||
completeness -= 50
|
||||
flags.append("missing_instruction")
|
||||
if not output:
|
||||
completeness -= 50
|
||||
flags.append("missing_output")
|
||||
|
||||
output_length = len(output)
|
||||
length_score = round(min(100.0, output_length / min_output_length * 100), 2)
|
||||
if output_length < min_output_length:
|
||||
flags.append("output_too_short")
|
||||
|
||||
readability = _readability_score("\n".join((instruction, input_text, output)))
|
||||
if readability < 70:
|
||||
flags.append("low_readability")
|
||||
|
||||
relevance = _source_relevance_score(record, source_content)
|
||||
if source_content and relevance < 30:
|
||||
flags.append("low_source_relevance")
|
||||
|
||||
fingerprint = record_fingerprint(record)
|
||||
known = set(known_fingerprints)
|
||||
duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
|
||||
if duplicate == 0:
|
||||
flags.append("duplicate_record")
|
||||
elif duplicate < 70:
|
||||
flags.append("repetitive_output")
|
||||
|
||||
overall = round(
|
||||
completeness * 0.35
|
||||
+ length_score * 0.20
|
||||
+ readability * 0.20
|
||||
+ relevance * 0.15
|
||||
+ duplicate * 0.10,
|
||||
2,
|
||||
)
|
||||
hard_valid = bool(instruction and output)
|
||||
return QualityScore(
|
||||
overall=overall,
|
||||
completeness=completeness,
|
||||
length=length_score,
|
||||
readability=readability,
|
||||
relevance=relevance,
|
||||
duplicate=duplicate,
|
||||
is_valid=hard_valid and overall >= threshold,
|
||||
flags=tuple(flags),
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def stable_split(
|
||||
value: str | int,
|
||||
split: Mapping[str, int] | None = None,
|
||||
*,
|
||||
seed: str = "",
|
||||
) -> DatasetSplit:
|
||||
"""按稳定哈希将记录划分到 train/validation/test。"""
|
||||
|
||||
ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
|
||||
required = {"train", "validation", "test"}
|
||||
if set(ratios) != required:
|
||||
raise ValueError("split must contain exactly train, validation and test")
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
|
||||
raise ValueError("split ratios must be non-negative integers")
|
||||
if sum(ratios.values()) != 100:
|
||||
raise ValueError("split ratios must sum to 100")
|
||||
|
||||
digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
|
||||
bucket = int.from_bytes(digest[:8], "big") % 10_000
|
||||
train_boundary = ratios["train"] * 100
|
||||
validation_boundary = train_boundary + ratios["validation"] * 100
|
||||
if bucket < train_boundary:
|
||||
return "train"
|
||||
if bucket < validation_boundary:
|
||||
return "validation"
|
||||
return "test"
|
||||
|
||||
|
||||
def _preview_content(item: Mapping[str, Any]) -> str:
|
||||
for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
|
||||
value = item.get(field)
|
||||
if value is not None:
|
||||
return normalize_text(str(value))
|
||||
return ""
|
||||
|
||||
|
||||
def _standard_fields(content: str) -> tuple[str, str, str]:
|
||||
if not content:
|
||||
return "", "", ""
|
||||
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, Mapping):
|
||||
instruction = next(
|
||||
(
|
||||
str(payload[key])
|
||||
for key in ("instruction", "question", "prompt")
|
||||
if payload.get(key) is not None
|
||||
),
|
||||
"",
|
||||
)
|
||||
input_text = next(
|
||||
(str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
output = next(
|
||||
(str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
if instruction or output:
|
||||
return normalize_text(instruction), normalize_text(input_text), normalize_text(output)
|
||||
|
||||
question_answer = re.match(
|
||||
r"^\s*(?:问|question)\s*[:：]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[:：]\s*(.+)\s*$",
|
||||
content,
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if question_answer:
|
||||
return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))
|
||||
|
||||
lines = [line.strip() for line in content.splitlines() if line.strip()]
|
||||
first_line = re.sub(r"^(?:问|question)\s*[:：]\s*", "", lines[0], flags=re.IGNORECASE)
|
||||
output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
|
||||
return normalize_text(first_line), "", output
|
||||
|
||||
|
||||
def generate_standard_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
qa_pairs_per_item: int = 1,
|
||||
semantic_enrichment: bool = False,
|
||||
split: Mapping[str, int] | None = None,
|
||||
split_seed: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""把预览内容确定性转换为标准 instruction/input/output 记录。
|
||||
|
||||
该函数只负责本地标准化，不冒充 LLM；服务层可将其作为无模型模式或
|
||||
LLM 响应解析后的统一落库步骤。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ValueError("qa_pairs_per_item must be in [1, 5]")
|
||||
prefixes = (
|
||||
"请结合实际情况说明：",
|
||||
"请用通俗易懂的方式说明：",
|
||||
"请从实际应用角度说明：",
|
||||
"请简洁自然地说明：",
|
||||
"请详细解答：",
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for item_index, item in enumerate(preview_items):
|
||||
content = _preview_content(item)
|
||||
instruction, input_text, output = _standard_fields(content)
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
for variant_index in range(qa_pairs_per_item):
|
||||
variant_instruction = instruction
|
||||
if variant_index:
|
||||
if semantic_enrichment:
|
||||
variant_instruction = f"{prefixes[variant_index]}{instruction}"
|
||||
else:
|
||||
variant_instruction = f"{instruction}(问法 {variant_index + 1})"
|
||||
raw_id = f"{preview_id}:{variant_index + 1}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
|
||||
status = "valid" if variant_instruction and output else "invalid"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": variant_instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": variant_instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": status,
|
||||
"split": stable_split(result_id, split, seed=split_seed),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChunkMethod",
|
||||
"DatasetSplit",
|
||||
"ParsedText",
|
||||
"QualityScore",
|
||||
"SUPPORTED_TEXT_FORMATS",
|
||||
"TextChunk",
|
||||
"TextFormat",
|
||||
"chunk_unstructured",
|
||||
"decode_utf8",
|
||||
"desensitize_pii",
|
||||
"detect_text_format",
|
||||
"estimate_token_count",
|
||||
"extract_structured_records",
|
||||
"generate_standard_records",
|
||||
"normalize_text",
|
||||
"parse_text_content",
|
||||
"parse_utf8_text",
|
||||
"record_fingerprint",
|
||||
"score_quality",
|
||||
"stable_split",
|
||||
]
|
||||
Reference in New Issue
Block a user