YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 完善后台生成与失败重试

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-28 10:56:05 +08:00
parent 8a6a6574bb
commit 01d2e6c76a
178 changed files with 4472 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files

626
backend/app/api/v1/endpoints/data_process.py
View File

@@ -7,12 +7,16 @@ import logging
import os
import re
import socket
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from dataclasses import asdict
from pathlib import Path
from threading import BoundedSemaphore, Lock
from typing import Any, Iterator, Literal
from urllib.parse import quote, urlsplit
import httpx
import psycopg
from fastapi import (
APIRouter,
@@ -77,6 +81,7 @@ from app.schemas.data_process import (
DataProcessStatus,
DataProcessTaskCreate,
DataProcessTaskUpdate,
DataProcessWorkflowStepUpdate,
ExternalPullRequest,
ExternalSourceRequest,
GenerateRequest,
@@ -85,6 +90,8 @@ from app.schemas.data_process import (
PreviewItemUpdate,
ProcessType,
PublishRequest,
ResultBatchRegenerateRequest,
ResultRegenerateRequest,
ResultUpdate,
)
@@ -122,6 +129,12 @@ RAW_INLINE_PREVIEW_MEDIA_TYPES = {
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
}
RESULT_REGENERATION_CONCURRENCY = 4
RESULT_REGENERATION_RETRIES = 0
RESULT_REGENERATION_TIMEOUT_SECONDS = 60.0
_result_regeneration_slots = BoundedSemaphore(RESULT_REGENERATION_CONCURRENCY)
_result_regeneration_claims_lock = Lock()
_active_result_regenerations: set[tuple[str, str]] = set()
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
@@ -147,8 +160,11 @@ def api_errors() -> Iterator[None]:
raise fail(409, str(exc)) from exc
except (DataProcessStoreError, ValueError) as exc:
raise fail(400, str(exc)) from exc
except psycopg.errors.UndefinedTable as exc:
raise fail(503, "data process schema is not installed; run schema_cli --check") from exc
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn) as exc:
raise fail(
503,
"data process schema is missing or out of date; run schema_cli --check",
) from exc
except psycopg.OperationalError as exc:
raise fail(503, "data process database is unavailable") from exc
@@ -205,7 +221,10 @@ def _commit_source_batch(
storage.delete(item.reference)
except Exception:
# 文件系统回滚失败不能覆盖数据库抛出的根因，并继续清理其余对象。
pass
logger.exception(
"failed to roll back data process source object task_id=%s",
task_id,
)
raise
@@ -533,9 +552,21 @@ def _all_preview_items(store: DataProcessStore, task_id: str) -> list[dict[str,
def _run_generation(
store: DataProcessStore, task_id: str, generation_run_id: str
) -> None:
started_at = time.perf_counter()
logger.info(
"data process generation worker started task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
try:
task = store.get_task(task_id)
if not store.generation_is_running(task_id, generation_run_id):
logger.info(
"data process generation worker skipped inactive run task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
return
all_preview_items = _all_preview_items(store, task_id)
preview_items = [
@@ -619,6 +650,12 @@ def _run_generation(
len(preview_items),
len(preview_items),
):
logger.info(
"data process generation stopped before completion task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
return
known_fingerprints: set[str] = set()
@@ -678,7 +715,7 @@ def _run_generation(
# stop 请求可能在纯函数计算期间到达，最终写入前再次检查状态。
if store.generation_is_running(task_id, generation_run_id):
store.complete_generation(
completed = store.complete_generation(
task_id,
accepted,
generation_run_id=generation_run_id,
@@ -686,7 +723,32 @@ def _run_generation(
duplicate_count=duplicate_count,
error_count=error_count,
)
except Exception as exc: # noqa: BLE001 - background failures must be persisted
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
"duration_ms=%.2f",
task_id,
generation_run_id,
completed.get("output_count", len(accepted)),
completed.get("filtered_count", filtered_count),
completed.get("duplicate_count", duplicate_count),
completed.get("error_count", error_count),
(time.perf_counter() - started_at) * 1000,
)
else:
logger.info(
"data process generation stopped before result persistence task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
except Exception as exc:
logger.exception(
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
task_id,
generation_run_id,
(time.perf_counter() - started_at) * 1000,
)
try:
if store.generation_is_running(task_id, generation_run_id):
store.mark_failed(
@@ -695,7 +757,12 @@ def _run_generation(
generation_run_id=generation_run_id,
)
except Exception:
return
logger.exception(
"failed to persist data process generation failure task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
@router.get("")
@@ -757,6 +824,19 @@ def update_task(
)
@router.put("/{task_id}/workflow-step")
def update_workflow_step(
task_id: str,
payload: DataProcessWorkflowStepUpdate,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(
store.update_workflow_step(task_id, payload.workflow_step.value),
"data process workflow step updated",
)
@router.post("/{task_id}/regenerate")
def prepare_regeneration(
task_id: str,
@@ -1338,6 +1418,150 @@ def _prepare_preview_items(
return items
def _run_preview(
store: DataProcessStore,
storage: LocalDataProcessStorage,
task_id: str,
preview_run_id: str,
source_file_ids: list[str],
) -> None:
"""后台逐文件切分；所有写入均由 preview_run_id 保护。"""
started_at = time.perf_counter()
logger.info(
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
task_id,
preview_run_id,
len(source_file_ids),
)
try:
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
return
total_files = len(source_file_ids)
total_items = 0
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
if not store.preview_is_running(task_id, preview_run_id):
logger.info(
"data process preview cancelled task_id=%s preview_run_id=%s "
"completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files - 1,
total_files,
)
return
items = _prepare_preview_items(
task_id,
store,
storage,
[source_file_id],
)
if not items:
raise InvalidStateError(
f"source file did not produce preview items: {source_file_id}"
)
created = store.replace_preview_items(
task_id,
items,
source_file_ids=[source_file_id],
preview_run_id=preview_run_id,
)
total_items += len(created)
if not store.update_preview_progress(
task_id,
preview_run_id,
completed_files,
total_files,
):
logger.info(
"data process preview stopped before progress update task_id=%s "
"preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files,
total_files,
)
return
if store.complete_preview(task_id, preview_run_id):
logger.info(
"data process preview completed task_id=%s preview_run_id=%s "
"total_files=%s total_items=%s duration_ms=%.2f",
task_id,
preview_run_id,
total_files,
total_items,
(time.perf_counter() - started_at) * 1000,
)
else:
logger.info(
"data process preview completion ignored for inactive run task_id=%s "
"preview_run_id=%s",
task_id,
preview_run_id,
)
except Exception as exc:
logger.exception(
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
task_id,
preview_run_id,
(time.perf_counter() - started_at) * 1000,
)
try:
if store.preview_is_running(task_id, preview_run_id):
store.mark_preview_failed(
task_id,
str(exc),
preview_run_id=preview_run_id,
)
except Exception:
logger.exception(
"failed to persist data process preview failure task_id=%s "
"preview_run_id=%s",
task_id,
preview_run_id,
)
@router.post("/{task_id}/preview/start", status_code=202)
def start_preview(
task_id: str,
background_tasks: BackgroundTasks,
payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest),
store: DataProcessStore = Depends(get_data_process_store),
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
) -> dict[str, Any]:
with api_errors():
task, selected_ids = store.start_preview(
task_id,
source_file_ids=payload.source_file_ids,
)
preview_run_id = str(task["preview_run_id"])
progress = store.preview_progress(task_id)
background_tasks.add_task(
_run_preview,
store,
storage,
task_id,
preview_run_id,
selected_ids,
)
return ok(progress, "data process preview started")
@router.get("/{task_id}/preview/progress")
def preview_progress(
task_id: str,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.preview_progress(task_id))
@router.post("/{task_id}/preview/build")
def build_preview(
task_id: str,
@@ -1537,6 +1761,15 @@ def results(
)
@router.post("/{task_id}/confirm-results")
def confirm_results(
task_id: str,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.confirm_results(task_id), "data process results confirmed")
@router.put("/{task_id}/results/{result_id}")
def update_result(
task_id: str,
@@ -1635,6 +1868,387 @@ def restore_result(
return ok(restored, "data process result restored")
class _ResultRegenerationFailed(InvalidStateError):
"""模型返回或质量校验失败，原失败结果必须保持不变。"""
def _assert_result_regeneration_allowed(task: dict[str, Any]) -> None:
if task.get("status") != "completed" or task.get("workflow_step") != "results":
raise InvalidStateError("task is not editing generation results")
if task.get("results_confirmed"):
raise InvalidStateError("confirmed results cannot be regenerated")
if task.get("output_dataset_id"):
raise InvalidStateError("published results cannot be regenerated")
def _result_regeneration_model(
task: dict[str, Any],
store: DataProcessStore,
) -> tuple[dict[str, Any], dict[str, Any]]:
config = task.get("config") or {}
model_id = _value(config, "generation_model_id", "generationModelId", None)
if not model_id:
raise InvalidStateError("task does not have a generation model")
return config, store.get_generation_model(str(model_id))
def _result_regeneration_timeout(config: dict[str, Any]) -> float:
configured = float(
_value(config, "request_timeout_seconds", "requestTimeoutSeconds", 60)
)
return max(1.0, min(RESULT_REGENERATION_TIMEOUT_SECONDS, configured))
@contextmanager
def _claim_result_regeneration(task_id: str, result_id: str) -> Iterator[None]:
key = (task_id, result_id)
with _result_regeneration_claims_lock:
if key in _active_result_regenerations:
raise ConflictError("data process result regeneration is already running")
_active_result_regenerations.add(key)
try:
yield
finally:
with _result_regeneration_claims_lock:
_active_result_regenerations.discard(key)
def _generate_result_replacement(
task_id: str,
current: dict[str, Any],
preview: dict[str, Any],
config: dict[str, Any],
generation_model: dict[str, Any],
model_client: httpx.Client | None = None,
) -> dict[str, Any]:
source_content = str(
preview.get("edited_content") or preview.get("original_content") or ""
).strip()
if not source_content or preview.get("status") == "invalid":
raise InvalidStateError("result source preview item is invalid or empty")
output_type = str(
_value(config, "output_type", "outputType", "standard")
).strip().lower()
previous_instruction = str(current.get("instruction") or "")[:1000]
previous_output = str(current.get("output") or "")[:1000]
base_prompt = str(
_value(config, "generation_prompt", "generationPrompt", "") or ""
)
regeneration_instruction = (
"这是一次失败结果的重新生成。请使用新的提问角度和表达，"
"不要复述旧结果。旧问题："
f"{previous_instruction or '无'}；旧答案：{previous_output or '无'}。"
)
runtime_config = {
**config,
"generation_prompt": f"{base_prompt}\n{regeneration_instruction}".strip(),
"output_type": output_type,
"reasoning_detail": _value(
config, "reasoning_detail", "reasoningDetail", "normal"
),
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
"json_mode": _value(config, "json_mode", "jsonMode", False),
# 交互式重新生成只做一次新尝试，避免继承整任务的重试配置后长时间等待。
"generation_retries": RESULT_REGENERATION_RETRIES,
"request_timeout_seconds": _result_regeneration_timeout(config),
}
with _result_regeneration_slots:
generated = generate_model_records(
[preview],
model=generation_model,
config=runtime_config,
task_id=task_id,
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=model_client,
)
if not generated or generated[0].get("status") == "invalid":
reason = str(
(generated[0] if generated else {}).get("error")
or "model generation failed"
)
raise _ResultRegenerationFailed(f"重新生成结果仍无效：{reason}")
minimum = max(
1,
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
)
replacement = generated[0]
quality = score_quality(
replacement,
min_output_length=minimum,
source_content=source_content,
)
if not quality.is_valid:
reason = ", ".join(quality.flags) or "quality validation failed"
raise _ResultRegenerationFailed(f"重新生成结果未通过质量校验：{reason}")
replacement["quality_score"] = asdict(quality)
replacement["status"] = "valid"
replacement["error"] = None
return replacement
def _regenerate_result_in_place(
task_id: str,
current: dict[str, Any],
preview: dict[str, Any],
config: dict[str, Any],
generation_model: dict[str, Any],
store: DataProcessStore,
*,
expected_updated_at: str,
model_client: httpx.Client | None = None,
) -> dict[str, Any]:
result_id = str(current["id"])
with _claim_result_regeneration(task_id, result_id):
replacement = _generate_result_replacement(
task_id,
current,
preview,
config,
generation_model,
model_client,
)
return store.replace_generated_result(
task_id,
result_id,
replacement,
expected_updated_at=expected_updated_at,
)
def _safe_regeneration_error(exc: Exception) -> str:
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
@router.post("/{task_id}/results/regenerate-batch")
def regenerate_results_batch(
task_id: str,
payload: ResultBatchRegenerateRequest,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
"""并发重新生成一批失败结果；每条独立提交并允许部分成功。"""
started_at = time.perf_counter()
batch_id = new_id("dprb")
with api_errors():
task = store.get_task(task_id)
_assert_result_regeneration_allowed(task)
config, generation_model = _result_regeneration_model(task, store)
prepared: list[tuple[int, dict[str, Any], dict[str, Any], str]] = []
failures: list[tuple[int, dict[str, str]]] = []
for index, requested in enumerate(payload.items):
try:
current = store.get_result(task_id, requested.result_id)
if current.get("status") != "invalid":
raise InvalidStateError("only an invalid result can be regenerated")
if requested.expected_updated_at != str(current.get("updated_at") or ""):
raise ConflictError("data process result was modified by another request")
preview_id = current.get("preview_item_id")
if not preview_id:
raise InvalidStateError(
"result is not associated with a source preview item"
)
preview = store.get_preview_item(task_id, str(preview_id))
prepared.append(
(index, current, preview, requested.expected_updated_at)
)
except ConflictError as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "conflict",
"message": _safe_regeneration_error(exc),
}))
except (NotFoundError, InvalidStateError) as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "skipped",
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch regeneration started batch_id=%s task_id=%s "
"requested=%s prepared=%s concurrency=%s",
batch_id,
task_id,
len(payload.items),
len(prepared),
min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
)
successes: list[tuple[int, dict[str, Any]]] = []
if prepared:
request_timeout = _result_regeneration_timeout(config)
model_timeout = httpx.Timeout(
request_timeout,
connect=min(10.0, request_timeout),
)
model_limits = httpx.Limits(
max_connections=RESULT_REGENERATION_CONCURRENCY,
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
# httpx.Client 支持跨线程复用，批次内共享连接池可减少重复建连开销。
with (
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor,
):
futures = {
executor.submit(
_regenerate_result_in_place,
task_id,
current,
preview,
config,
generation_model,
store,
expected_updated_at=expected_updated_at,
model_client=model_client,
): (index, str(current["id"]), time.perf_counter())
for index, current, preview, expected_updated_at in prepared
}
for future in as_completed(futures):
index, result_id, item_started_at = futures[future]
try:
regenerated = future.result()
successes.append((index, regenerated))
outcome = "succeeded"
except ConflictError as exc:
outcome = "conflict"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except _ResultRegenerationFailed as exc:
outcome = "generation_failed"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except (NotFoundError, InvalidStateError) as exc:
outcome = "skipped"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except Exception as exc: # pragma: no cover - defensive boundary
outcome = "internal_error"
logger.exception(
"data process result batch regeneration crashed "
"batch_id=%s task_id=%s result_id=%s",
batch_id,
task_id,
result_id,
)
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch item finished batch_id=%s task_id=%s "
"result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
outcome,
(time.perf_counter() - item_started_at) * 1000,
)
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
remaining_invalid_count = int(
store.list_results(
task_id,
page=1,
page_size=1,
status="invalid",
)["total"]
)
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch regeneration completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
len(failure_items),
remaining_invalid_count,
duration_ms,
)
return ok(
{
"batch_id": batch_id,
"total": len(payload.items),
"succeeded": len(success_items),
"failed": len(failure_items),
"remaining_invalid_count": remaining_invalid_count,
"duration_ms": round(duration_ms, 2),
"items": success_items,
"failures": failure_items,
},
"data process results regenerated",
)
@router.post("/{task_id}/results/{result_id}/regenerate")
def regenerate_result(
task_id: str,
result_id: str,
payload: ResultRegenerateRequest,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
"""只重新生成一个失败结果，成功后原位替换且不影响其他结果。"""
started_at = time.perf_counter()
with api_errors():
task = store.get_task(task_id)
_assert_result_regeneration_allowed(task)
current = store.get_result(task_id, result_id)
if current.get("status") != "invalid":
raise InvalidStateError("only an invalid result can be regenerated")
if payload.expected_updated_at != str(current.get("updated_at") or ""):
raise ConflictError("data process result was modified by another request")
preview_id = current.get("preview_item_id")
if not preview_id:
raise InvalidStateError("result is not associated with a source preview item")
preview = store.get_preview_item(task_id, str(preview_id))
config, generation_model = _result_regeneration_model(task, store)
try:
result = _regenerate_result_in_place(
task_id,
current,
preview,
config,
generation_model,
store,
expected_updated_at=payload.expected_updated_at,
)
except _ResultRegenerationFailed as exc:
logger.warning(
"data process result regeneration failed task_id=%s result_id=%s "
"duration_ms=%.2f reason=%s",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
_safe_regeneration_error(exc),
)
raise
logger.info(
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
)
return ok(result, "data process result regenerated")
@router.post("/{task_id}/publish")
def publish(
task_id: str,