feat(data_process): 问答对数据评测体系与质量分雷达图
- 三层评测：规则层沿用原五维规则分，语义层用本地 BGE 向量算问答/来源 相关性，评审层复用生成模型按 rubric 打分（忠实度/正确性/清晰度等， 区分 standard/reasoning/dpo 输出类型），任一层失败自动降级 - 组合分 = 规则 35% + 语义 20% + 评审 45%，缺层自动重归一 - 新增 results/evaluate-batch 批量评测接口，镜像批量重生成的并发、 乐观锁与部分成功语义；生成阶段不再展示质量分 - 详情页与结果编辑页新增"数据评测"按钮和批量进度；质量分列悬停弹出 雷达图浮窗（评审 5 维 + 语义 2 维、三层分项、评审理由） - 手动编辑/恢复后重算规则与语义层并丢弃过期评审分，雷达图不再展示 失效数据
This commit is contained in:
caoxiaozhu
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
@@ -341,3 +342,87 @@ def score_quality(
|
||||
flags=tuple(flags),
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
|
||||
if not left or not right or len(left) != len(right):
|
||||
return 0.0
|
||||
dot = math.fsum(a * b for a, b in zip(left, right))
|
||||
norm_left = math.sqrt(math.fsum(a * a for a in left))
|
||||
norm_right = math.sqrt(math.fsum(b * b for b in right))
|
||||
if not norm_left or not norm_right:
|
||||
return 0.0
|
||||
return dot / (norm_left * norm_right)
|
||||
|
||||
|
||||
def semantic_quality_scores(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
source_content: str = "",
|
||||
embed_model: Any = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""用本地嵌入向量计算语义相关性（0-100）。
|
||||
|
||||
返回 ``question_answer``（问题↔答案）、``answer_source``（答案↔来源，
|
||||
无来源时缺省）与 ``overall``；嵌入模型不可用时返回 None 降级，不阻断流程。
|
||||
"""
|
||||
|
||||
try:
|
||||
if embed_model is None:
|
||||
from .embedding import semantic_embedding_model
|
||||
|
||||
embed_model = semantic_embedding_model()
|
||||
if embed_model is None:
|
||||
return None
|
||||
|
||||
question = normalize_text(
|
||||
" ".join(
|
||||
str(record.get(field) or "")
|
||||
for field in ("instruction", "input")
|
||||
)
|
||||
)
|
||||
answer = normalize_text(
|
||||
str(record.get("output") or "") or str(record.get("chosen") or "")
|
||||
)
|
||||
source = normalize_text(source_content)
|
||||
texts = [text for text in {question, answer, source} if text]
|
||||
if not texts:
|
||||
return None
|
||||
vectors = {text: embed_model.get_text_embedding(text) for text in texts}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
scores: dict[str, Any] = {}
|
||||
if question and answer:
|
||||
scores["question_answer"] = round(
|
||||
100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2
|
||||
)
|
||||
if answer and source:
|
||||
scores["answer_source"] = round(
|
||||
100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2
|
||||
)
|
||||
if not scores:
|
||||
return None
|
||||
scores["overall"] = round(sum(scores.values()) / len(scores), 2)
|
||||
return scores
|
||||
|
||||
|
||||
def composite_overall(
|
||||
*,
|
||||
rule: float | None,
|
||||
semantic: float | None = None,
|
||||
judge: float | None = None,
|
||||
) -> float:
|
||||
"""三层加权组合：规则 35% + 语义 20% + 评审 45%，缺失层自动重归一。"""
|
||||
|
||||
if rule is None:
|
||||
rule = 0.0
|
||||
if judge is not None and semantic is not None:
|
||||
overall = rule * 0.35 + semantic * 0.20 + judge * 0.45
|
||||
elif semantic is not None:
|
||||
overall = rule * 0.60 + semantic * 0.40
|
||||
elif judge is not None:
|
||||
overall = rule * 0.55 + judge * 0.45
|
||||
else:
|
||||
overall = rule
|
||||
return round(max(0.0, min(100.0, overall)), 2)
|
||||
|
||||
313
backend/app/modules/data_process/evaluation.py
Normal file
313
backend/app/modules/data_process/evaluation.py
Normal file
@@ -0,0 +1,313 @@
|
||||
"""数据处理 - 生成结果的多层质量评测。
|
||||
|
||||
三层体系：规则层（确定性规则分）+ 语义层（本地嵌入向量）+ 评审层
|
||||
（复用生成模型按 rubric 打分的 LLM-as-judge）。任一层失败自动降级，
|
||||
评测永远返回可用结果，不阻断调用方流程。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .algorithms import normalize_text, score_quality
|
||||
from .algorithms.quality import composite_overall, semantic_quality_scores
|
||||
from .generation import (
|
||||
ModelGenerationError,
|
||||
_is_retryable_generation_error,
|
||||
_json_payload,
|
||||
_message_content,
|
||||
chat_completions_url,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 送入评审提示词的来源正文上限，避免超长切片挤占评分输出空间。
|
||||
_MAX_JUDGE_SOURCE_CHARS = 6000
|
||||
|
||||
_JUDGE_DIMENSIONS: dict[str, tuple[str, ...]] = {
|
||||
"standard": (
|
||||
"faithfulness",
|
||||
"correctness",
|
||||
"clarity",
|
||||
"completeness",
|
||||
"alignment",
|
||||
),
|
||||
"reasoning": (
|
||||
"faithfulness",
|
||||
"correctness",
|
||||
"clarity",
|
||||
"completeness",
|
||||
"alignment",
|
||||
"reasoning_validity",
|
||||
),
|
||||
"dpo": (
|
||||
"clarity",
|
||||
"chosen_quality",
|
||||
"rejected_quality",
|
||||
"preference_reasonableness",
|
||||
"faithfulness",
|
||||
),
|
||||
}
|
||||
|
||||
_DIMENSION_LABELS: dict[str, str] = {
|
||||
"faithfulness": "忠实度",
|
||||
"correctness": "正确性",
|
||||
"clarity": "问题清晰度",
|
||||
"completeness": "回答完整性",
|
||||
"alignment": "指令对齐",
|
||||
"reasoning_validity": "推理有效性",
|
||||
"chosen_quality": "chosen 回答质量",
|
||||
"rejected_quality": "rejected 回答质量",
|
||||
"preference_reasonableness": "偏好区分合理性",
|
||||
}
|
||||
|
||||
_DIMENSION_RULES: dict[str, str] = {
|
||||
"faithfulness": "忠实度：答案的全部陈述是否被参考资料支持，没有编造、没有引入资料之外的信息；未提供参考资料时按答案内部自洽性评估",
|
||||
"correctness": "正确性：答案中的事实、概念与计算是否正确",
|
||||
"clarity": "问题清晰度：问题是否清晰、自包含、无歧义，脱离上下文也能理解",
|
||||
"completeness": "回答完整性：答案是否充分、直接地回应了问题的全部要点",
|
||||
"alignment": "指令对齐：答案的形式与范围是否符合问题的要求（如格式、语言、范围限定）",
|
||||
"reasoning_validity": "推理有效性：思维链步骤是否逻辑连贯、无跳步或循环论证，结论是否由推理过程自然得出",
|
||||
"chosen_quality": "chosen 回答质量：更优回答的正确性、完整性与表述质量",
|
||||
"rejected_quality": "rejected 回答质量：较差回答是否仍具备基本可读性，使对比训练有意义",
|
||||
"preference_reasonableness": "偏好区分合理性：chosen 是否明显优于 rejected，且优劣差异与问题直接相关",
|
||||
}
|
||||
|
||||
|
||||
def _judge_system_prompt(output_type: str) -> str:
|
||||
dimensions = _JUDGE_DIMENSIONS[output_type]
|
||||
rules = "\n".join(f"- {_DIMENSION_RULES[name]}" for name in dimensions)
|
||||
scores_schema = ", ".join(f'"{name}": 1-5' for name in dimensions)
|
||||
return (
|
||||
"你是大模型训练数据质量评审员。严格依据用户消息中的【参考资料】评审这条训练数据，逐维度按 1-5 分打分：\n"
|
||||
f"{rules}\n"
|
||||
"评分锚点：5 分=完全符合维度描述；3 分=基本符合但有明显不足；1 分=严重不符合。\n"
|
||||
"忠实度只依据参考资料与公认常识判断，无法得到支持的陈述必须扣分；不要因为答案冗长而加分。\n"
|
||||
"只输出一个 JSON 对象，不要输出 JSON 之外的任何文字。\n"
|
||||
'输出格式：{"scores": {' + scores_schema + '}, "reason": "一句话总评", "issues": ["具体问题，没有则为空数组"]}'
|
||||
)
|
||||
|
||||
|
||||
def _judge_user_prompt(record: Mapping[str, Any], source_content: str) -> str:
|
||||
source = normalize_text(source_content)[:_MAX_JUDGE_SOURCE_CHARS] or "（无参考资料）"
|
||||
instruction = normalize_text(str(record.get("instruction") or "")) or "（空）"
|
||||
input_text = normalize_text(str(record.get("input") or ""))
|
||||
sections = [f"【参考资料】\n{source}", f"【问题】\n{instruction}"]
|
||||
if input_text:
|
||||
sections.append(f"【输入】\n{input_text}")
|
||||
if record.get("chosen") or record.get("rejected"):
|
||||
sections.append(f"【更优回答 chosen】\n{normalize_text(str(record.get('chosen') or '')) or '（空）'}")
|
||||
sections.append(f"【较差回答 rejected】\n{normalize_text(str(record.get('rejected') or '')) or '（空）'}")
|
||||
else:
|
||||
output = normalize_text(str(record.get("output") or ""))
|
||||
sections.append(f"【回答】\n{output or '（空）'}")
|
||||
return "\n\n".join(sections)
|
||||
|
||||
|
||||
def _validated_judge_payload(payload: Any, output_type: str) -> dict[str, Any]:
|
||||
if not isinstance(payload, Mapping):
|
||||
raise ModelGenerationError("评审响应不是 JSON 对象")
|
||||
raw_scores = payload.get("scores")
|
||||
if not isinstance(raw_scores, Mapping):
|
||||
raise ModelGenerationError("评审响应缺少 scores 对象")
|
||||
expected = _JUDGE_DIMENSIONS[output_type]
|
||||
scores: dict[str, float] = {}
|
||||
for name in expected:
|
||||
value = raw_scores.get(name)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ModelGenerationError(f"评审响应缺少维度 {name} 的有效分数")
|
||||
scores[name] = round(max(1.0, min(5.0, float(value))), 1)
|
||||
issues = payload.get("issues")
|
||||
if not isinstance(issues, list):
|
||||
issues = []
|
||||
issues = [str(item)[:200] for item in issues if str(item).strip()][:8]
|
||||
reason = normalize_text(str(payload.get("reason") or ""))[:300]
|
||||
return {
|
||||
"scores": scores,
|
||||
"overall": round(sum(scores.values()) / len(scores) * 20, 2),
|
||||
"reason": reason,
|
||||
"issues": issues,
|
||||
}
|
||||
|
||||
|
||||
def _judge_record(
|
||||
record: Mapping[str, Any],
|
||||
source_content: str,
|
||||
*,
|
||||
model: Mapping[str, Any],
|
||||
config: Mapping[str, Any],
|
||||
client: httpx.Client | None,
|
||||
) -> dict[str, Any] | None:
|
||||
output_type = str(config.get("output_type") or "standard").strip().lower()
|
||||
if output_type not in _JUDGE_DIMENSIONS:
|
||||
output_type = "standard"
|
||||
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
||||
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||
if not model_name:
|
||||
raise ModelGenerationError("generation model name is required")
|
||||
temperature = 0.1
|
||||
max_tokens = max(256, min(2048, int(config.get("max_tokens", 1024) or 1024)))
|
||||
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60) or 60)))
|
||||
retries = max(0, min(5, int(config.get("generation_retries", 2) or 2)))
|
||||
headers = {"Content-Type": "application/json"}
|
||||
api_key = str(model.get("api_key") or "").strip()
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
request_payload: dict[str, Any] = {
|
||||
"model": model_name,
|
||||
"messages": [
|
||||
{"role": "system", "content": _judge_system_prompt(output_type)},
|
||||
{"role": "user", "content": _judge_user_prompt(record, source_content)},
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if bool(config.get("json_mode", False)):
|
||||
request_payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
owns_client = client is None
|
||||
http_client = client or httpx.Client(timeout=timeout)
|
||||
try:
|
||||
last_error: Exception | None = None
|
||||
for _ in range(retries + 1):
|
||||
try:
|
||||
response = http_client.post(endpoint, headers=headers, json=request_payload)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if not isinstance(body, Mapping):
|
||||
raise ModelGenerationError("model response body must be a JSON object")
|
||||
judged = _validated_judge_payload(
|
||||
_json_payload(_message_content(body)),
|
||||
output_type,
|
||||
)
|
||||
judged["model"] = model_name
|
||||
judged["output_type"] = output_type
|
||||
return judged
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if not _is_retryable_generation_error(exc):
|
||||
break
|
||||
raise ModelGenerationError(f"质量评审调用失败: {last_error}")
|
||||
finally:
|
||||
if owns_client:
|
||||
http_client.close()
|
||||
|
||||
|
||||
def evaluate_result_record(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
source_content: str = "",
|
||||
model: Mapping[str, Any] | None = None,
|
||||
config: Mapping[str, Any] | None = None,
|
||||
client: httpx.Client | None = None,
|
||||
embed_model: Any = None,
|
||||
min_output_length: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
"""对一条生成结果执行三层评测，返回可直接落库的 quality_score 字典。
|
||||
|
||||
规则层字段保持原样平铺（向后兼容既有读取方）；新增 ``semantic``、
|
||||
``judge``、``layers``、``evaluated`` 与组合 ``overall``。
|
||||
"""
|
||||
|
||||
config_dict = dict(config or {})
|
||||
rule = score_quality(
|
||||
record,
|
||||
min_output_length=min_output_length,
|
||||
source_content=source_content,
|
||||
)
|
||||
quality: dict[str, Any] = asdict(rule)
|
||||
|
||||
semantic = semantic_quality_scores(
|
||||
record,
|
||||
source_content=source_content,
|
||||
embed_model=embed_model,
|
||||
)
|
||||
judge: dict[str, Any] | None = None
|
||||
if model is not None:
|
||||
try:
|
||||
judge = _judge_record(
|
||||
record,
|
||||
source_content,
|
||||
model=model,
|
||||
config=config_dict,
|
||||
client=client,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"data process judge evaluation degraded: %s",
|
||||
exc,
|
||||
)
|
||||
|
||||
layers = {
|
||||
"rule": rule.overall,
|
||||
"semantic": semantic.get("overall") if semantic else None,
|
||||
"judge": judge.get("overall") if judge else None,
|
||||
}
|
||||
quality.update(
|
||||
semantic=semantic,
|
||||
judge=judge,
|
||||
layers=layers,
|
||||
evaluated=True,
|
||||
evaluated_at=datetime.now(UTC).isoformat(),
|
||||
overall=composite_overall(
|
||||
rule=layers["rule"],
|
||||
semantic=layers["semantic"],
|
||||
judge=layers["judge"],
|
||||
),
|
||||
)
|
||||
return quality
|
||||
|
||||
|
||||
def reevaluate_edited_record(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
source_content: str = "",
|
||||
previous_quality: Mapping[str, Any] | None = None,
|
||||
embed_model: Any = None,
|
||||
min_output_length: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
"""手动编辑/恢复后重算规则与语义层，丢弃已过期的评审层。
|
||||
|
||||
编辑会改变内容，旧的评审分不再可信；规则与语义层本地重算零成本。
|
||||
``evaluated`` 标记沿用原值，保证已评测过的结果编辑后仍有可用分数。
|
||||
"""
|
||||
|
||||
rule = score_quality(
|
||||
record,
|
||||
min_output_length=min_output_length,
|
||||
source_content=source_content,
|
||||
)
|
||||
quality: dict[str, Any] = asdict(rule)
|
||||
semantic = semantic_quality_scores(
|
||||
record,
|
||||
source_content=source_content,
|
||||
embed_model=embed_model,
|
||||
)
|
||||
previous = dict(previous_quality or {})
|
||||
evaluated = bool(previous.get("evaluated"))
|
||||
layers = {
|
||||
"rule": rule.overall,
|
||||
"semantic": semantic.get("overall") if semantic else None,
|
||||
"judge": None,
|
||||
}
|
||||
quality.update(
|
||||
semantic=semantic,
|
||||
judge=None,
|
||||
layers=layers,
|
||||
evaluated=evaluated,
|
||||
evaluated_at=(
|
||||
datetime.now(UTC).isoformat() if evaluated else None
|
||||
),
|
||||
overall=composite_overall(
|
||||
rule=layers["rule"],
|
||||
semantic=layers["semantic"],
|
||||
),
|
||||
)
|
||||
return quality
|
||||
Reference in New Issue
Block a user