feat(data_process): 问答对数据评测体系与质量分雷达图
- 三层评测:规则层沿用原五维规则分,语义层用本地 BGE 向量算问答/来源 相关性,评审层复用生成模型按 rubric 打分(忠实度/正确性/清晰度等, 区分 standard/reasoning/dpo 输出类型),任一层失败自动降级 - 组合分 = 规则 35% + 语义 20% + 评审 45%,缺层自动重归一 - 新增 results/evaluate-batch 批量评测接口,镜像批量重生成的并发、 乐观锁与部分成功语义;生成阶段不再展示质量分 - 详情页与结果编辑页新增"数据评测"按钮和批量进度;质量分列悬停弹出 雷达图浮窗(评审 5 维 + 语义 2 维、三层分项、评审理由) - 手动编辑/恢复后重算规则与语义层并丢弃过期评审分,雷达图不再展示 失效数据
This commit is contained in:
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
@@ -341,3 +342,87 @@ def score_quality(
|
||||
flags=tuple(flags),
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
|
||||
if not left or not right or len(left) != len(right):
|
||||
return 0.0
|
||||
dot = math.fsum(a * b for a, b in zip(left, right))
|
||||
norm_left = math.sqrt(math.fsum(a * a for a in left))
|
||||
norm_right = math.sqrt(math.fsum(b * b for b in right))
|
||||
if not norm_left or not norm_right:
|
||||
return 0.0
|
||||
return dot / (norm_left * norm_right)
|
||||
|
||||
|
||||
def semantic_quality_scores(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
source_content: str = "",
|
||||
embed_model: Any = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""用本地嵌入向量计算语义相关性(0-100)。
|
||||
|
||||
返回 ``question_answer``(问题↔答案)、``answer_source``(答案↔来源,
|
||||
无来源时缺省)与 ``overall``;嵌入模型不可用时返回 None 降级,不阻断流程。
|
||||
"""
|
||||
|
||||
try:
|
||||
if embed_model is None:
|
||||
from .embedding import semantic_embedding_model
|
||||
|
||||
embed_model = semantic_embedding_model()
|
||||
if embed_model is None:
|
||||
return None
|
||||
|
||||
question = normalize_text(
|
||||
" ".join(
|
||||
str(record.get(field) or "")
|
||||
for field in ("instruction", "input")
|
||||
)
|
||||
)
|
||||
answer = normalize_text(
|
||||
str(record.get("output") or "") or str(record.get("chosen") or "")
|
||||
)
|
||||
source = normalize_text(source_content)
|
||||
texts = [text for text in {question, answer, source} if text]
|
||||
if not texts:
|
||||
return None
|
||||
vectors = {text: embed_model.get_text_embedding(text) for text in texts}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
scores: dict[str, Any] = {}
|
||||
if question and answer:
|
||||
scores["question_answer"] = round(
|
||||
100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2
|
||||
)
|
||||
if answer and source:
|
||||
scores["answer_source"] = round(
|
||||
100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2
|
||||
)
|
||||
if not scores:
|
||||
return None
|
||||
scores["overall"] = round(sum(scores.values()) / len(scores), 2)
|
||||
return scores
|
||||
|
||||
|
||||
def composite_overall(
|
||||
*,
|
||||
rule: float | None,
|
||||
semantic: float | None = None,
|
||||
judge: float | None = None,
|
||||
) -> float:
|
||||
"""三层加权组合:规则 35% + 语义 20% + 评审 45%,缺失层自动重归一。"""
|
||||
|
||||
if rule is None:
|
||||
rule = 0.0
|
||||
if judge is not None and semantic is not None:
|
||||
overall = rule * 0.35 + semantic * 0.20 + judge * 0.45
|
||||
elif semantic is not None:
|
||||
overall = rule * 0.60 + semantic * 0.40
|
||||
elif judge is not None:
|
||||
overall = rule * 0.55 + judge * 0.45
|
||||
else:
|
||||
overall = rule
|
||||
return round(max(0.0, min(100.0, overall)), 2)
|
||||
|
||||
Reference in New Issue
Block a user