feat(data_process): 问答对数据评测体系与质量分雷达图

- 三层评测:规则层沿用原五维规则分,语义层用本地 BGE 向量算问答/来源
  相关性,评审层复用生成模型按 rubric 打分(忠实度/正确性/清晰度等,
  区分 standard/reasoning/dpo 输出类型),任一层失败自动降级
- 组合分 = 规则 35% + 语义 20% + 评审 45%,缺层自动重归一
- 新增 results/evaluate-batch 批量评测接口,镜像批量重生成的并发、
  乐观锁与部分成功语义;生成阶段不再展示质量分
- 详情页与结果编辑页新增"数据评测"按钮和批量进度;质量分列悬停弹出
  雷达图浮窗(评审 5 维 + 语义 2 维、三层分项、评审理由)
- 手动编辑/恢复后重算规则与语义层并丢弃过期评审分,雷达图不再展示
  失效数据
This commit is contained in:
caoxiaozhu
2026-08-19 14:21:54 +08:00
parent f47611020a
commit 81c2f85c3a
16 changed files with 1917 additions and 33 deletions

View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
import math
import re
import unicodedata
from collections import Counter
@@ -341,3 +342,87 @@ def score_quality(
flags=tuple(flags),
fingerprint=fingerprint,
)
def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
if not left or not right or len(left) != len(right):
return 0.0
dot = math.fsum(a * b for a, b in zip(left, right))
norm_left = math.sqrt(math.fsum(a * a for a in left))
norm_right = math.sqrt(math.fsum(b * b for b in right))
if not norm_left or not norm_right:
return 0.0
return dot / (norm_left * norm_right)
def semantic_quality_scores(
record: Mapping[str, Any],
*,
source_content: str = "",
embed_model: Any = None,
) -> dict[str, Any] | None:
"""用本地嵌入向量计算语义相关性0-100
返回 ``question_answer``(问题↔答案)、``answer_source``(答案↔来源,
无来源时缺省)与 ``overall``;嵌入模型不可用时返回 None 降级,不阻断流程。
"""
try:
if embed_model is None:
from .embedding import semantic_embedding_model
embed_model = semantic_embedding_model()
if embed_model is None:
return None
question = normalize_text(
" ".join(
str(record.get(field) or "")
for field in ("instruction", "input")
)
)
answer = normalize_text(
str(record.get("output") or "") or str(record.get("chosen") or "")
)
source = normalize_text(source_content)
texts = [text for text in {question, answer, source} if text]
if not texts:
return None
vectors = {text: embed_model.get_text_embedding(text) for text in texts}
except Exception:
return None
scores: dict[str, Any] = {}
if question and answer:
scores["question_answer"] = round(
100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2
)
if answer and source:
scores["answer_source"] = round(
100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2
)
if not scores:
return None
scores["overall"] = round(sum(scores.values()) / len(scores), 2)
return scores
def composite_overall(
*,
rule: float | None,
semantic: float | None = None,
judge: float | None = None,
) -> float:
"""三层加权组合:规则 35% + 语义 20% + 评审 45%,缺失层自动重归一。"""
if rule is None:
rule = 0.0
if judge is not None and semantic is not None:
overall = rule * 0.35 + semantic * 0.20 + judge * 0.45
elif semantic is not None:
overall = rule * 0.60 + semantic * 0.40
elif judge is not None:
overall = rule * 0.55 + judge * 0.45
else:
overall = rule
return round(max(0.0, min(100.0, overall)), 2)