feat(data_process): 问答对数据评测体系与质量分雷达图
- 三层评测:规则层沿用原五维规则分,语义层用本地 BGE 向量算问答/来源 相关性,评审层复用生成模型按 rubric 打分(忠实度/正确性/清晰度等, 区分 standard/reasoning/dpo 输出类型),任一层失败自动降级 - 组合分 = 规则 35% + 语义 20% + 评审 45%,缺层自动重归一 - 新增 results/evaluate-batch 批量评测接口,镜像批量重生成的并发、 乐观锁与部分成功语义;生成阶段不再展示质量分 - 详情页与结果编辑页新增"数据评测"按钮和批量进度;质量分列悬停弹出 雷达图浮窗(评审 5 维 + 语义 2 维、三层分项、评审理由) - 手动编辑/恢复后重算规则与语义层并丢弃过期评审分,雷达图不再展示 失效数据
This commit is contained in:
@@ -60,6 +60,10 @@ from app.modules.data_process.document_chunking import (
|
||||
chunk_semantic_text,
|
||||
merge_short_chunks,
|
||||
)
|
||||
from app.modules.data_process.evaluation import (
|
||||
evaluate_result_record,
|
||||
reevaluate_edited_record,
|
||||
)
|
||||
from app.modules.data_process.generation import generate_model_records
|
||||
from app.modules.data_process.office_preview import (
|
||||
MAX_XLSX_PREVIEW_ROWS,
|
||||
@@ -96,6 +100,7 @@ from app.schemas.data_process import (
|
||||
PreviewItemUpdate,
|
||||
ProcessType,
|
||||
PublishRequest,
|
||||
ResultBatchEvaluateRequest,
|
||||
ResultBatchRegenerateRequest,
|
||||
ResultRegenerateRequest,
|
||||
ResultUpdate,
|
||||
@@ -2039,12 +2044,13 @@ def update_result(
|
||||
or 20
|
||||
),
|
||||
)
|
||||
quality = score_quality(
|
||||
# 编辑后内容已变化:重算规则与语义层,旧的评审分不再可信直接丢弃。
|
||||
update["quality_score"] = reevaluate_edited_record(
|
||||
merged,
|
||||
min_output_length=minimum,
|
||||
source_content=source_content,
|
||||
previous_quality=current.get("quality_score"),
|
||||
min_output_length=minimum,
|
||||
)
|
||||
update["quality_score"] = asdict(quality)
|
||||
result = store.update_result(
|
||||
task_id,
|
||||
result_id,
|
||||
@@ -2089,11 +2095,6 @@ def restore_result(
|
||||
or 20
|
||||
),
|
||||
)
|
||||
quality = score_quality(
|
||||
restored,
|
||||
min_output_length=minimum,
|
||||
source_content=source_content,
|
||||
)
|
||||
restored = store.update_result(
|
||||
task_id,
|
||||
result_id,
|
||||
@@ -2103,7 +2104,12 @@ def restore_result(
|
||||
"output": restored["output"],
|
||||
"chosen": restored["chosen"],
|
||||
"rejected": restored["rejected"],
|
||||
"quality_score": asdict(quality),
|
||||
"quality_score": reevaluate_edited_record(
|
||||
restored,
|
||||
source_content=source_content,
|
||||
previous_quality=current.get("quality_score"),
|
||||
min_output_length=minimum,
|
||||
),
|
||||
"expected_updated_at": current.get("updated_at"),
|
||||
},
|
||||
)
|
||||
@@ -2266,6 +2272,46 @@ def _safe_regeneration_error(exc: Exception) -> str:
|
||||
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
|
||||
|
||||
|
||||
def _evaluate_result_in_place(
|
||||
task_id: str,
|
||||
current: dict[str, Any],
|
||||
source_content: str,
|
||||
config: dict[str, Any],
|
||||
evaluation_model: dict[str, Any] | None,
|
||||
store: DataProcessStore,
|
||||
*,
|
||||
expected_updated_at: str,
|
||||
model_client: httpx.Client | None = None,
|
||||
minimum: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
"""评测单条结果并落库;复用逐结果互斥锁避免与重生成并发写冲突。"""
|
||||
|
||||
result_id = str(current["id"])
|
||||
with _claim_result_regeneration(task_id, result_id):
|
||||
quality = evaluate_result_record(
|
||||
{
|
||||
"instruction": current.get("instruction"),
|
||||
"input": current.get("input"),
|
||||
"output": current.get("output"),
|
||||
"chosen": current.get("chosen"),
|
||||
"rejected": current.get("rejected"),
|
||||
},
|
||||
source_content=source_content,
|
||||
model=evaluation_model,
|
||||
config=config,
|
||||
client=model_client,
|
||||
min_output_length=minimum,
|
||||
)
|
||||
return store.update_result(
|
||||
task_id,
|
||||
result_id,
|
||||
{
|
||||
"quality_score": quality,
|
||||
"expected_updated_at": expected_updated_at,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/results/regenerate-batch")
|
||||
def regenerate_results_batch(
|
||||
task_id: str,
|
||||
@@ -2439,6 +2485,186 @@ def regenerate_results_batch(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/results/evaluate-batch")
|
||||
def evaluate_results_batch(
|
||||
task_id: str,
|
||||
payload: ResultBatchEvaluateRequest,
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
) -> dict[str, Any]:
|
||||
"""对一批结果执行三层质量评测(规则+语义+评审),允许部分成功。"""
|
||||
|
||||
started_at = time.perf_counter()
|
||||
batch_id = new_id("dpeb")
|
||||
with api_errors():
|
||||
task = store.get_task(task_id)
|
||||
if task.get("status") == "running":
|
||||
raise ConflictError("data process task is running")
|
||||
if task.get("output_dataset_id"):
|
||||
raise InvalidStateError("published results cannot be evaluated")
|
||||
config = task.get("config") or {}
|
||||
evaluation_model: dict[str, Any] | None = None
|
||||
model_id = _value(config, "generation_model_id", "generationModelId", None)
|
||||
if model_id:
|
||||
try:
|
||||
evaluation_model = store.get_generation_model(str(model_id))
|
||||
except NotFoundError:
|
||||
logger.warning(
|
||||
"data process evaluation model unavailable, judge layer "
|
||||
"skipped task_id=%s model_id=%s",
|
||||
task_id,
|
||||
model_id,
|
||||
)
|
||||
evaluation_config = {
|
||||
**config,
|
||||
"output_type": str(
|
||||
_value(config, "output_type", "outputType", "standard")
|
||||
).strip().lower(),
|
||||
}
|
||||
minimum = max(
|
||||
1,
|
||||
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
|
||||
)
|
||||
|
||||
prepared: list[tuple[int, dict[str, Any], str, str]] = []
|
||||
failures: list[tuple[int, dict[str, str]]] = []
|
||||
for index, requested in enumerate(payload.items):
|
||||
try:
|
||||
current = store.get_result(task_id, requested.result_id)
|
||||
if requested.expected_updated_at != str(current.get("updated_at") or ""):
|
||||
raise ConflictError("data process result was modified by another request")
|
||||
source_content = ""
|
||||
preview_id = current.get("preview_item_id")
|
||||
if preview_id:
|
||||
preview = store.get_preview_item(task_id, str(preview_id))
|
||||
source_content = str(
|
||||
preview.get("edited_content")
|
||||
or preview.get("original_content")
|
||||
or ""
|
||||
)
|
||||
prepared.append(
|
||||
(index, current, source_content, requested.expected_updated_at)
|
||||
)
|
||||
except ConflictError as exc:
|
||||
failures.append((index, {
|
||||
"result_id": requested.result_id,
|
||||
"code": "conflict",
|
||||
"message": _safe_regeneration_error(exc),
|
||||
}))
|
||||
except (NotFoundError, InvalidStateError) as exc:
|
||||
failures.append((index, {
|
||||
"result_id": requested.result_id,
|
||||
"code": "skipped",
|
||||
"message": _safe_regeneration_error(exc),
|
||||
}))
|
||||
|
||||
logger.info(
|
||||
"data process result batch evaluation started batch_id=%s task_id=%s "
|
||||
"requested=%s prepared=%s judge_enabled=%s",
|
||||
batch_id,
|
||||
task_id,
|
||||
len(payload.items),
|
||||
len(prepared),
|
||||
evaluation_model is not None,
|
||||
)
|
||||
successes: list[tuple[int, dict[str, Any]]] = []
|
||||
if prepared:
|
||||
try:
|
||||
from app.modules.data_process.algorithms.embedding import (
|
||||
semantic_embedding_model,
|
||||
)
|
||||
|
||||
semantic_embedding_model()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"data process semantic embedding unavailable, semantic layer "
|
||||
"will be skipped batch_id=%s",
|
||||
batch_id,
|
||||
)
|
||||
request_timeout = _result_regeneration_timeout(config)
|
||||
model_timeout = httpx.Timeout(
|
||||
request_timeout,
|
||||
connect=min(10.0, request_timeout),
|
||||
)
|
||||
model_limits = httpx.Limits(
|
||||
max_connections=RESULT_REGENERATION_CONCURRENCY,
|
||||
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
|
||||
)
|
||||
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
|
||||
ThreadPoolExecutor(
|
||||
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
||||
thread_name_prefix="data-result-evaluation",
|
||||
) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
_evaluate_result_in_place,
|
||||
task_id,
|
||||
current,
|
||||
source_content,
|
||||
evaluation_config,
|
||||
evaluation_model,
|
||||
store,
|
||||
expected_updated_at=expected_updated_at,
|
||||
model_client=model_client if evaluation_model else None,
|
||||
minimum=minimum,
|
||||
): (index, str(current["id"]), time.perf_counter())
|
||||
for index, current, source_content, expected_updated_at in prepared
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
index, result_id, item_started_at = futures[future]
|
||||
try:
|
||||
evaluated = future.result()
|
||||
successes.append((index, evaluated))
|
||||
outcome = "succeeded"
|
||||
except ConflictError as exc:
|
||||
outcome = "conflict"
|
||||
failures.append((index, {
|
||||
"result_id": result_id,
|
||||
"code": outcome,
|
||||
"message": _safe_regeneration_error(exc),
|
||||
}))
|
||||
except Exception as exc:
|
||||
outcome = "evaluation_failed"
|
||||
failures.append((index, {
|
||||
"result_id": result_id,
|
||||
"code": outcome,
|
||||
"message": _safe_regeneration_error(exc),
|
||||
}))
|
||||
logger.info(
|
||||
"data process result batch evaluation item finished "
|
||||
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
|
||||
batch_id,
|
||||
task_id,
|
||||
result_id,
|
||||
outcome,
|
||||
(time.perf_counter() - item_started_at) * 1000,
|
||||
)
|
||||
|
||||
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
|
||||
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
|
||||
duration_ms = (time.perf_counter() - started_at) * 1000
|
||||
logger.info(
|
||||
"data process result batch evaluation completed batch_id=%s task_id=%s "
|
||||
"succeeded=%s failed=%s duration_ms=%.2f",
|
||||
batch_id,
|
||||
task_id,
|
||||
len(success_items),
|
||||
len(failure_items),
|
||||
duration_ms,
|
||||
)
|
||||
return ok(
|
||||
{
|
||||
"batch_id": batch_id,
|
||||
"total": len(payload.items),
|
||||
"succeeded": len(success_items),
|
||||
"failed": len(failure_items),
|
||||
"duration_ms": round(duration_ms, 2),
|
||||
"items": success_items,
|
||||
"failures": failure_items,
|
||||
},
|
||||
"data process results evaluated",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/results/{result_id}/regenerate")
|
||||
def regenerate_result(
|
||||
task_id: str,
|
||||
|
||||
Reference in New Issue
Block a user