ca012893f7
将 algorithms.py / store.py 拆分为 algorithms/ 与 store/ 子包，并修复 机械拆分造成的导入与辅助函数缺失： - algorithms/: 补全各子模块依赖与 17 个私有辅助函数、8 个常量；重写 __init__.py 移除坏的 importlib 兜底，分层导入并以局部 import 断开 text_utils<->parsers、quality<->structured_processing 循环依赖。 - store/: 补回 DataProcessStoreError / hashlib / _serialize_value / estimate_token_count 等缺失导入，包入口导出测试与调用方依赖的私有 辅助函数。 - 删除旧单文件 algorithms.py / store.py 及重构残留（_algorithms_old、 backups、refactor 脚本、REFACTORING 文档）。 algorithms 与 store 测试套件 91 项全部通过。
344 lines
12 KiB
Python
344 lines
12 KiB
Python
"""数据处理算法 - 质量评分和去重。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
from collections import Counter
|
||
from collections.abc import Iterable, Mapping, Sequence
|
||
from copy import deepcopy
|
||
from typing import Any
|
||
|
||
from .text_utils import normalize_text
|
||
from .types import (
|
||
_MAX_ANOMALY_TEXT_CHARS,
|
||
_MOJIBAKE_MARKERS,
|
||
_TOKEN_PATTERN,
|
||
ProcessedStructuredRecord,
|
||
QualityScore,
|
||
)
|
||
|
||
|
||
def estimate_token_count(text: str) -> int:
|
||
"""粗略估计文本的 token 数量。"""
|
||
return len(_TOKEN_PATTERN.findall(text))
|
||
|
||
|
||
def content_quality_flags(
|
||
text: str,
|
||
*,
|
||
min_chars: int = 20,
|
||
min_tokens: int = 5,
|
||
max_chars: int = _MAX_ANOMALY_TEXT_CHARS,
|
||
) -> tuple[str, ...]:
|
||
"""返回非结构化内容的确定性低质量原因。"""
|
||
|
||
if min_chars < 0 or min_tokens < 0 or max_chars <= 0:
|
||
raise ValueError("content quality limits must be non-negative")
|
||
normalized = normalize_text(text)
|
||
if not normalized:
|
||
return ("empty_content",)
|
||
flags: list[str] = []
|
||
if len(normalized) < min_chars or estimate_token_count(normalized) < min_tokens:
|
||
flags.append("content_too_short")
|
||
if len(normalized) > max_chars:
|
||
flags.append("content_too_long")
|
||
if any(marker in normalized for marker in _MOJIBAKE_MARKERS):
|
||
flags.append("mojibake")
|
||
nonspace = [char for char in normalized if not char.isspace()]
|
||
if nonspace:
|
||
readable_ratio = sum(
|
||
char.isprintable()
|
||
and unicodedata.category(char) not in {"Co", "Cs", "Cn"}
|
||
for char in nonspace
|
||
) / len(nonspace)
|
||
if readable_ratio < 0.85:
|
||
flags.append("low_printable_ratio")
|
||
if len(nonspace) >= 100:
|
||
most_common = Counter(nonspace).most_common(1)[0][1]
|
||
if most_common / len(nonspace) > 0.9:
|
||
flags.append("repetitive_content")
|
||
return tuple(dict.fromkeys(flags))
|
||
|
||
def is_low_quality_content(
|
||
text: str,
|
||
*,
|
||
min_chars: int = 20,
|
||
min_tokens: int = 5,
|
||
max_chars: int = _MAX_ANOMALY_TEXT_CHARS,
|
||
) -> bool:
|
||
"""判断内容是否命中任一低质量规则。"""
|
||
|
||
return bool(
|
||
content_quality_flags(
|
||
text,
|
||
min_chars=min_chars,
|
||
min_tokens=min_tokens,
|
||
max_chars=max_chars,
|
||
)
|
||
)
|
||
|
||
def _deduplicate_structured_entries(
|
||
entries: Sequence[ProcessedStructuredRecord],
|
||
) -> list[ProcessedStructuredRecord]:
|
||
"""仅按整条 canonical JSON 稳定去重，避免误删同 ID 的更新记录。"""
|
||
|
||
# canonical_record_json 位于 structured_processing，延迟导入以断开循环依赖。
|
||
from .structured_processing import canonical_record_json
|
||
|
||
exact_seen: set[str] = set()
|
||
unique: list[ProcessedStructuredRecord] = []
|
||
for entry in entries:
|
||
record = entry.record
|
||
fingerprint = hashlib.sha256(canonical_record_json(record).encode("utf-8")).hexdigest()
|
||
if fingerprint in exact_seen:
|
||
continue
|
||
exact_seen.add(fingerprint)
|
||
unique.append(
|
||
ProcessedStructuredRecord(
|
||
entry.source_index,
|
||
deepcopy(dict(record)),
|
||
)
|
||
)
|
||
return unique
|
||
|
||
def deduplicate_structured_records(
|
||
records: Sequence[Mapping[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
"""仅按整条 canonical JSON 稳定去重。"""
|
||
|
||
entries = [
|
||
ProcessedStructuredRecord(index, deepcopy(dict(record)))
|
||
for index, record in enumerate(records)
|
||
]
|
||
return [entry.record for entry in _deduplicate_structured_entries(entries)]
|
||
|
||
def _near_duplicate_features(text: str, shingle_size: int) -> tuple[str, ...]:
|
||
if isinstance(shingle_size, bool) or not isinstance(shingle_size, int):
|
||
raise TypeError("shingle_size must be an integer")
|
||
if shingle_size <= 0:
|
||
raise ValueError("shingle_size must be greater than 0")
|
||
tokens = re.findall(
|
||
r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+",
|
||
normalize_text(text).casefold(),
|
||
)
|
||
if not tokens:
|
||
return ()
|
||
if len(tokens) < shingle_size:
|
||
return ("\x1f".join(tokens),)
|
||
return tuple(
|
||
"\x1f".join(tokens[index : index + shingle_size])
|
||
for index in range(len(tokens) - shingle_size + 1)
|
||
)
|
||
|
||
def near_duplicate_fingerprint(text: str, *, shingle_size: int = 3) -> str:
|
||
"""生成 64 位 SimHash 指纹，用于低成本近重复候选筛选。"""
|
||
|
||
if isinstance(shingle_size, bool) or not isinstance(shingle_size, int):
|
||
raise TypeError("shingle_size must be an integer")
|
||
if shingle_size <= 0:
|
||
raise ValueError("shingle_size must be greater than 0")
|
||
features = Counter(_near_duplicate_features(text, shingle_size))
|
||
if not features:
|
||
return "0" * 16
|
||
vector = [0] * 64
|
||
for feature, weight in features.items():
|
||
digest = int.from_bytes(hashlib.sha256(feature.encode("utf-8")).digest()[:8], "big")
|
||
for bit in range(64):
|
||
vector[bit] += weight if digest & (1 << bit) else -weight
|
||
fingerprint = sum(1 << bit for bit, value in enumerate(vector) if value >= 0)
|
||
return f"{fingerprint:016x}"
|
||
|
||
def fingerprints_are_near_duplicate(
|
||
left: str,
|
||
right: str,
|
||
*,
|
||
max_hamming_distance: int = 3,
|
||
) -> bool:
|
||
"""比较两个 64 位十六进制 SimHash 指纹。"""
|
||
|
||
if isinstance(max_hamming_distance, bool) or not isinstance(max_hamming_distance, int):
|
||
raise TypeError("max_hamming_distance must be an integer")
|
||
if not 0 <= max_hamming_distance <= 64:
|
||
raise ValueError("max_hamming_distance must be in [0, 64]")
|
||
if not re.fullmatch(r"[0-9a-fA-F]{16}", left) or not re.fullmatch(
|
||
r"[0-9a-fA-F]{16}", right
|
||
):
|
||
raise ValueError("fingerprints must be 16-character hexadecimal strings")
|
||
distance = (int(left, 16) ^ int(right, 16)).bit_count()
|
||
return distance <= max_hamming_distance
|
||
|
||
def is_near_duplicate(
|
||
left: str,
|
||
right: str,
|
||
*,
|
||
shingle_size: int = 3,
|
||
similarity_threshold: float = 0.9,
|
||
max_hamming_distance: int = 3,
|
||
) -> bool:
|
||
"""结合词片 Jaccard 和 SimHash 判断两段内容是否近重复。"""
|
||
|
||
if isinstance(similarity_threshold, bool) or not isinstance(
|
||
similarity_threshold, (int, float)
|
||
):
|
||
raise TypeError("similarity_threshold must be a number")
|
||
if not 0 <= similarity_threshold <= 1:
|
||
raise ValueError("similarity_threshold must be in [0, 1]")
|
||
if isinstance(max_hamming_distance, bool) or not isinstance(max_hamming_distance, int):
|
||
raise TypeError("max_hamming_distance must be an integer")
|
||
if not 0 <= max_hamming_distance <= 64:
|
||
raise ValueError("max_hamming_distance must be in [0, 64]")
|
||
left_normalized = normalize_text(left)
|
||
right_normalized = normalize_text(right)
|
||
if not left_normalized or not right_normalized:
|
||
return left_normalized == right_normalized
|
||
if left_normalized.casefold() == right_normalized.casefold():
|
||
return True
|
||
left_features = set(_near_duplicate_features(left_normalized, shingle_size))
|
||
right_features = set(_near_duplicate_features(right_normalized, shingle_size))
|
||
union = left_features | right_features
|
||
similarity = len(left_features & right_features) / len(union) if union else 1.0
|
||
if similarity >= similarity_threshold:
|
||
return True
|
||
return fingerprints_are_near_duplicate(
|
||
near_duplicate_fingerprint(left_normalized, shingle_size=shingle_size),
|
||
near_duplicate_fingerprint(right_normalized, shingle_size=shingle_size),
|
||
max_hamming_distance=max_hamming_distance,
|
||
)
|
||
|
||
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
||
"""计算与字典键顺序无关的稳定记录指纹。"""
|
||
|
||
canonical = {
|
||
"instruction": normalize_text(str(record.get("instruction") or "")),
|
||
"input": normalize_text(str(record.get("input") or "")),
|
||
"output": normalize_text(str(record.get("output") or "")),
|
||
}
|
||
raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||
|
||
def _readability_score(text: str) -> float:
|
||
if not text:
|
||
return 0.0
|
||
nonspace = [char for char in text if not char.isspace()]
|
||
if not nonspace:
|
||
return 0.0
|
||
printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
|
||
useful_ratio = sum(
|
||
char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
|
||
for char in nonspace
|
||
) / len(nonspace)
|
||
return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)
|
||
|
||
|
||
def _internal_duplicate_score(text: str) -> float:
|
||
units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
|
||
if len(units) <= 1:
|
||
return 100.0
|
||
return round(100 * len(set(units)) / len(units), 2)
|
||
|
||
|
||
def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
|
||
"""估算结果与来源文本的词元覆盖率。
|
||
|
||
这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
|
||
时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
|
||
"""
|
||
|
||
source = normalize_text(source_content)
|
||
if not source:
|
||
return 100.0
|
||
candidate = normalize_text(
|
||
"\n".join(
|
||
str(record.get(field) or "") for field in ("instruction", "input", "output")
|
||
)
|
||
)
|
||
|
||
def semantic_tokens(text: str) -> set[str]:
|
||
return {
|
||
token.lower()
|
||
for token in _TOKEN_PATTERN.findall(text)
|
||
if token.isalnum() or "\u3400" <= token <= "\u9fff"
|
||
}
|
||
|
||
source_tokens = semantic_tokens(source)
|
||
candidate_tokens = semantic_tokens(candidate)
|
||
if not candidate_tokens:
|
||
return 0.0
|
||
if not source_tokens:
|
||
return 0.0
|
||
return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)
|
||
|
||
|
||
def score_quality(
|
||
record: Mapping[str, Any],
|
||
*,
|
||
min_output_length: int = 20,
|
||
source_content: str = "",
|
||
known_fingerprints: Iterable[str] = (),
|
||
threshold: float = 60.0,
|
||
) -> QualityScore:
|
||
"""按完整性、长度、可读性、来源相关性和重复度计算质量分。"""
|
||
|
||
if min_output_length <= 0:
|
||
raise ValueError("min_output_length must be greater than 0")
|
||
if not 0 <= threshold <= 100:
|
||
raise ValueError("threshold must be in [0, 100]")
|
||
|
||
instruction = normalize_text(str(record.get("instruction") or ""))
|
||
input_text = normalize_text(str(record.get("input") or ""))
|
||
output = normalize_text(str(record.get("output") or ""))
|
||
flags: list[str] = []
|
||
|
||
completeness = 100.0
|
||
if not instruction:
|
||
completeness -= 50
|
||
flags.append("missing_instruction")
|
||
if not output:
|
||
completeness -= 50
|
||
flags.append("missing_output")
|
||
|
||
output_length = len(output)
|
||
length_score = round(min(100.0, output_length / min_output_length * 100), 2)
|
||
if output_length < min_output_length:
|
||
flags.append("output_too_short")
|
||
|
||
readability = _readability_score("\n".join((instruction, input_text, output)))
|
||
if readability < 70:
|
||
flags.append("low_readability")
|
||
|
||
relevance = _source_relevance_score(record, source_content)
|
||
if source_content and relevance < 30:
|
||
flags.append("low_source_relevance")
|
||
|
||
fingerprint = record_fingerprint(record)
|
||
known = set(known_fingerprints)
|
||
duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
|
||
if duplicate == 0:
|
||
flags.append("duplicate_record")
|
||
elif duplicate < 70:
|
||
flags.append("repetitive_output")
|
||
|
||
overall = round(
|
||
completeness * 0.35
|
||
+ length_score * 0.20
|
||
+ readability * 0.20
|
||
+ relevance * 0.15
|
||
+ duplicate * 0.10,
|
||
2,
|
||
)
|
||
hard_valid = bool(instruction and output)
|
||
return QualityScore(
|
||
overall=overall,
|
||
completeness=completeness,
|
||
length=length_score,
|
||
readability=readability,
|
||
relevance=relevance,
|
||
duplicate=duplicate,
|
||
is_valid=hard_valid and overall >= threshold,
|
||
flags=tuple(flags),
|
||
fingerprint=fingerprint,
|
||
)
|