YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 完善后台生成与失败重试

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-28 10:56:05 +08:00
parent 8a6a6574bb
commit 01d2e6c76a
178 changed files with 4472 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files

626
backend/app/api/v1/endpoints/data_process.py
View File

@@ -7,12 +7,16 @@ import logging
import os
import re
import socket
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from dataclasses import asdict
from pathlib import Path
from threading import BoundedSemaphore, Lock
from typing import Any, Iterator, Literal
from urllib.parse import quote, urlsplit
import httpx
import psycopg
from fastapi import (
APIRouter,
@@ -77,6 +81,7 @@ from app.schemas.data_process import (
DataProcessStatus,
DataProcessTaskCreate,
DataProcessTaskUpdate,
DataProcessWorkflowStepUpdate,
ExternalPullRequest,
ExternalSourceRequest,
GenerateRequest,
@@ -85,6 +90,8 @@ from app.schemas.data_process import (
PreviewItemUpdate,
ProcessType,
PublishRequest,
ResultBatchRegenerateRequest,
ResultRegenerateRequest,
ResultUpdate,
)
@@ -122,6 +129,12 @@ RAW_INLINE_PREVIEW_MEDIA_TYPES = {
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
}
RESULT_REGENERATION_CONCURRENCY = 4
RESULT_REGENERATION_RETRIES = 0
RESULT_REGENERATION_TIMEOUT_SECONDS = 60.0
_result_regeneration_slots = BoundedSemaphore(RESULT_REGENERATION_CONCURRENCY)
_result_regeneration_claims_lock = Lock()
_active_result_regenerations: set[tuple[str, str]] = set()
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
@@ -147,8 +160,11 @@ def api_errors() -> Iterator[None]:
raise fail(409, str(exc)) from exc
except (DataProcessStoreError, ValueError) as exc:
raise fail(400, str(exc)) from exc
except psycopg.errors.UndefinedTable as exc:
raise fail(503, "data process schema is not installed; run schema_cli --check") from exc
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn) as exc:
raise fail(
503,
"data process schema is missing or out of date; run schema_cli --check",
) from exc
except psycopg.OperationalError as exc:
raise fail(503, "data process database is unavailable") from exc
@@ -205,7 +221,10 @@ def _commit_source_batch(
storage.delete(item.reference)
except Exception:
# 文件系统回滚失败不能覆盖数据库抛出的根因，并继续清理其余对象。
pass
logger.exception(
"failed to roll back data process source object task_id=%s",
task_id,
)
raise
@@ -533,9 +552,21 @@ def _all_preview_items(store: DataProcessStore, task_id: str) -> list[dict[str,
def _run_generation(
store: DataProcessStore, task_id: str, generation_run_id: str
) -> None:
started_at = time.perf_counter()
logger.info(
"data process generation worker started task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
try:
task = store.get_task(task_id)
if not store.generation_is_running(task_id, generation_run_id):
logger.info(
"data process generation worker skipped inactive run task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
return
all_preview_items = _all_preview_items(store, task_id)
preview_items = [
@@ -619,6 +650,12 @@ def _run_generation(
len(preview_items),
len(preview_items),
):
logger.info(
"data process generation stopped before completion task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
return
known_fingerprints: set[str] = set()
@@ -678,7 +715,7 @@ def _run_generation(
# stop 请求可能在纯函数计算期间到达，最终写入前再次检查状态。
if store.generation_is_running(task_id, generation_run_id):
store.complete_generation(
completed = store.complete_generation(
task_id,
accepted,
generation_run_id=generation_run_id,
@@ -686,7 +723,32 @@ def _run_generation(
duplicate_count=duplicate_count,
error_count=error_count,
)
except Exception as exc: # noqa: BLE001 - background failures must be persisted
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
"duration_ms=%.2f",
task_id,
generation_run_id,
completed.get("output_count", len(accepted)),
completed.get("filtered_count", filtered_count),
completed.get("duplicate_count", duplicate_count),
completed.get("error_count", error_count),
(time.perf_counter() - started_at) * 1000,
)
else:
logger.info(
"data process generation stopped before result persistence task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
except Exception as exc:
logger.exception(
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
task_id,
generation_run_id,
(time.perf_counter() - started_at) * 1000,
)
try:
if store.generation_is_running(task_id, generation_run_id):
store.mark_failed(
@@ -695,7 +757,12 @@ def _run_generation(
generation_run_id=generation_run_id,
)
except Exception:
return
logger.exception(
"failed to persist data process generation failure task_id=%s "
"generation_run_id=%s",
task_id,
generation_run_id,
)
@router.get("")
@@ -757,6 +824,19 @@ def update_task(
)
@router.put("/{task_id}/workflow-step")
def update_workflow_step(
task_id: str,
payload: DataProcessWorkflowStepUpdate,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(
store.update_workflow_step(task_id, payload.workflow_step.value),
"data process workflow step updated",
)
@router.post("/{task_id}/regenerate")
def prepare_regeneration(
task_id: str,
@@ -1338,6 +1418,150 @@ def _prepare_preview_items(
return items
def _run_preview(
store: DataProcessStore,
storage: LocalDataProcessStorage,
task_id: str,
preview_run_id: str,
source_file_ids: list[str],
) -> None:
"""后台逐文件切分；所有写入均由 preview_run_id 保护。"""
started_at = time.perf_counter()
logger.info(
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
task_id,
preview_run_id,
len(source_file_ids),
)
try:
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
return
total_files = len(source_file_ids)
total_items = 0
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
if not store.preview_is_running(task_id, preview_run_id):
logger.info(
"data process preview cancelled task_id=%s preview_run_id=%s "
"completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files - 1,
total_files,
)
return
items = _prepare_preview_items(
task_id,
store,
storage,
[source_file_id],
)
if not items:
raise InvalidStateError(
f"source file did not produce preview items: {source_file_id}"
)
created = store.replace_preview_items(
task_id,
items,
source_file_ids=[source_file_id],
preview_run_id=preview_run_id,
)
total_items += len(created)
if not store.update_preview_progress(
task_id,
preview_run_id,
completed_files,
total_files,
):
logger.info(
"data process preview stopped before progress update task_id=%s "
"preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files,
total_files,
)
return
if store.complete_preview(task_id, preview_run_id):
logger.info(
"data process preview completed task_id=%s preview_run_id=%s "
"total_files=%s total_items=%s duration_ms=%.2f",
task_id,
preview_run_id,
total_files,
total_items,
(time.perf_counter() - started_at) * 1000,
)
else:
logger.info(
"data process preview completion ignored for inactive run task_id=%s "
"preview_run_id=%s",
task_id,
preview_run_id,
)
except Exception as exc:
logger.exception(
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
task_id,
preview_run_id,
(time.perf_counter() - started_at) * 1000,
)
try:
if store.preview_is_running(task_id, preview_run_id):
store.mark_preview_failed(
task_id,
str(exc),
preview_run_id=preview_run_id,
)
except Exception:
logger.exception(
"failed to persist data process preview failure task_id=%s "
"preview_run_id=%s",
task_id,
preview_run_id,
)
@router.post("/{task_id}/preview/start", status_code=202)
def start_preview(
task_id: str,
background_tasks: BackgroundTasks,
payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest),
store: DataProcessStore = Depends(get_data_process_store),
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
) -> dict[str, Any]:
with api_errors():
task, selected_ids = store.start_preview(
task_id,
source_file_ids=payload.source_file_ids,
)
preview_run_id = str(task["preview_run_id"])
progress = store.preview_progress(task_id)
background_tasks.add_task(
_run_preview,
store,
storage,
task_id,
preview_run_id,
selected_ids,
)
return ok(progress, "data process preview started")
@router.get("/{task_id}/preview/progress")
def preview_progress(
task_id: str,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.preview_progress(task_id))
@router.post("/{task_id}/preview/build")
def build_preview(
task_id: str,
@@ -1537,6 +1761,15 @@ def results(
)
@router.post("/{task_id}/confirm-results")
def confirm_results(
task_id: str,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.confirm_results(task_id), "data process results confirmed")
@router.put("/{task_id}/results/{result_id}")
def update_result(
task_id: str,
@@ -1635,6 +1868,387 @@ def restore_result(
return ok(restored, "data process result restored")
class _ResultRegenerationFailed(InvalidStateError):
"""模型返回或质量校验失败，原失败结果必须保持不变。"""
def _assert_result_regeneration_allowed(task: dict[str, Any]) -> None:
if task.get("status") != "completed" or task.get("workflow_step") != "results":
raise InvalidStateError("task is not editing generation results")
if task.get("results_confirmed"):
raise InvalidStateError("confirmed results cannot be regenerated")
if task.get("output_dataset_id"):
raise InvalidStateError("published results cannot be regenerated")
def _result_regeneration_model(
task: dict[str, Any],
store: DataProcessStore,
) -> tuple[dict[str, Any], dict[str, Any]]:
config = task.get("config") or {}
model_id = _value(config, "generation_model_id", "generationModelId", None)
if not model_id:
raise InvalidStateError("task does not have a generation model")
return config, store.get_generation_model(str(model_id))
def _result_regeneration_timeout(config: dict[str, Any]) -> float:
configured = float(
_value(config, "request_timeout_seconds", "requestTimeoutSeconds", 60)
)
return max(1.0, min(RESULT_REGENERATION_TIMEOUT_SECONDS, configured))
@contextmanager
def _claim_result_regeneration(task_id: str, result_id: str) -> Iterator[None]:
key = (task_id, result_id)
with _result_regeneration_claims_lock:
if key in _active_result_regenerations:
raise ConflictError("data process result regeneration is already running")
_active_result_regenerations.add(key)
try:
yield
finally:
with _result_regeneration_claims_lock:
_active_result_regenerations.discard(key)
def _generate_result_replacement(
task_id: str,
current: dict[str, Any],
preview: dict[str, Any],
config: dict[str, Any],
generation_model: dict[str, Any],
model_client: httpx.Client | None = None,
) -> dict[str, Any]:
source_content = str(
preview.get("edited_content") or preview.get("original_content") or ""
).strip()
if not source_content or preview.get("status") == "invalid":
raise InvalidStateError("result source preview item is invalid or empty")
output_type = str(
_value(config, "output_type", "outputType", "standard")
).strip().lower()
previous_instruction = str(current.get("instruction") or "")[:1000]
previous_output = str(current.get("output") or "")[:1000]
base_prompt = str(
_value(config, "generation_prompt", "generationPrompt", "") or ""
)
regeneration_instruction = (
"这是一次失败结果的重新生成。请使用新的提问角度和表达，"
"不要复述旧结果。旧问题："
f"{previous_instruction or '无'}；旧答案：{previous_output or '无'}。"
)
runtime_config = {
**config,
"generation_prompt": f"{base_prompt}\n{regeneration_instruction}".strip(),
"output_type": output_type,
"reasoning_detail": _value(
config, "reasoning_detail", "reasoningDetail", "normal"
),
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
"json_mode": _value(config, "json_mode", "jsonMode", False),
# 交互式重新生成只做一次新尝试，避免继承整任务的重试配置后长时间等待。
"generation_retries": RESULT_REGENERATION_RETRIES,
"request_timeout_seconds": _result_regeneration_timeout(config),
}
with _result_regeneration_slots:
generated = generate_model_records(
[preview],
model=generation_model,
config=runtime_config,
task_id=task_id,
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=model_client,
)
if not generated or generated[0].get("status") == "invalid":
reason = str(
(generated[0] if generated else {}).get("error")
or "model generation failed"
)
raise _ResultRegenerationFailed(f"重新生成结果仍无效：{reason}")
minimum = max(
1,
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
)
replacement = generated[0]
quality = score_quality(
replacement,
min_output_length=minimum,
source_content=source_content,
)
if not quality.is_valid:
reason = ", ".join(quality.flags) or "quality validation failed"
raise _ResultRegenerationFailed(f"重新生成结果未通过质量校验：{reason}")
replacement["quality_score"] = asdict(quality)
replacement["status"] = "valid"
replacement["error"] = None
return replacement
def _regenerate_result_in_place(
task_id: str,
current: dict[str, Any],
preview: dict[str, Any],
config: dict[str, Any],
generation_model: dict[str, Any],
store: DataProcessStore,
*,
expected_updated_at: str,
model_client: httpx.Client | None = None,
) -> dict[str, Any]:
result_id = str(current["id"])
with _claim_result_regeneration(task_id, result_id):
replacement = _generate_result_replacement(
task_id,
current,
preview,
config,
generation_model,
model_client,
)
return store.replace_generated_result(
task_id,
result_id,
replacement,
expected_updated_at=expected_updated_at,
)
def _safe_regeneration_error(exc: Exception) -> str:
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
@router.post("/{task_id}/results/regenerate-batch")
def regenerate_results_batch(
task_id: str,
payload: ResultBatchRegenerateRequest,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
"""并发重新生成一批失败结果；每条独立提交并允许部分成功。"""
started_at = time.perf_counter()
batch_id = new_id("dprb")
with api_errors():
task = store.get_task(task_id)
_assert_result_regeneration_allowed(task)
config, generation_model = _result_regeneration_model(task, store)
prepared: list[tuple[int, dict[str, Any], dict[str, Any], str]] = []
failures: list[tuple[int, dict[str, str]]] = []
for index, requested in enumerate(payload.items):
try:
current = store.get_result(task_id, requested.result_id)
if current.get("status") != "invalid":
raise InvalidStateError("only an invalid result can be regenerated")
if requested.expected_updated_at != str(current.get("updated_at") or ""):
raise ConflictError("data process result was modified by another request")
preview_id = current.get("preview_item_id")
if not preview_id:
raise InvalidStateError(
"result is not associated with a source preview item"
)
preview = store.get_preview_item(task_id, str(preview_id))
prepared.append(
(index, current, preview, requested.expected_updated_at)
)
except ConflictError as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "conflict",
"message": _safe_regeneration_error(exc),
}))
except (NotFoundError, InvalidStateError) as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "skipped",
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch regeneration started batch_id=%s task_id=%s "
"requested=%s prepared=%s concurrency=%s",
batch_id,
task_id,
len(payload.items),
len(prepared),
min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
)
successes: list[tuple[int, dict[str, Any]]] = []
if prepared:
request_timeout = _result_regeneration_timeout(config)
model_timeout = httpx.Timeout(
request_timeout,
connect=min(10.0, request_timeout),
)
model_limits = httpx.Limits(
max_connections=RESULT_REGENERATION_CONCURRENCY,
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
# httpx.Client 支持跨线程复用，批次内共享连接池可减少重复建连开销。
with (
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor,
):
futures = {
executor.submit(
_regenerate_result_in_place,
task_id,
current,
preview,
config,
generation_model,
store,
expected_updated_at=expected_updated_at,
model_client=model_client,
): (index, str(current["id"]), time.perf_counter())
for index, current, preview, expected_updated_at in prepared
}
for future in as_completed(futures):
index, result_id, item_started_at = futures[future]
try:
regenerated = future.result()
successes.append((index, regenerated))
outcome = "succeeded"
except ConflictError as exc:
outcome = "conflict"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except _ResultRegenerationFailed as exc:
outcome = "generation_failed"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except (NotFoundError, InvalidStateError) as exc:
outcome = "skipped"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except Exception as exc: # pragma: no cover - defensive boundary
outcome = "internal_error"
logger.exception(
"data process result batch regeneration crashed "
"batch_id=%s task_id=%s result_id=%s",
batch_id,
task_id,
result_id,
)
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch item finished batch_id=%s task_id=%s "
"result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
outcome,
(time.perf_counter() - item_started_at) * 1000,
)
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
remaining_invalid_count = int(
store.list_results(
task_id,
page=1,
page_size=1,
status="invalid",
)["total"]
)
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch regeneration completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
len(failure_items),
remaining_invalid_count,
duration_ms,
)
return ok(
{
"batch_id": batch_id,
"total": len(payload.items),
"succeeded": len(success_items),
"failed": len(failure_items),
"remaining_invalid_count": remaining_invalid_count,
"duration_ms": round(duration_ms, 2),
"items": success_items,
"failures": failure_items,
},
"data process results regenerated",
)
@router.post("/{task_id}/results/{result_id}/regenerate")
def regenerate_result(
task_id: str,
result_id: str,
payload: ResultRegenerateRequest,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
"""只重新生成一个失败结果，成功后原位替换且不影响其他结果。"""
started_at = time.perf_counter()
with api_errors():
task = store.get_task(task_id)
_assert_result_regeneration_allowed(task)
current = store.get_result(task_id, result_id)
if current.get("status") != "invalid":
raise InvalidStateError("only an invalid result can be regenerated")
if payload.expected_updated_at != str(current.get("updated_at") or ""):
raise ConflictError("data process result was modified by another request")
preview_id = current.get("preview_item_id")
if not preview_id:
raise InvalidStateError("result is not associated with a source preview item")
preview = store.get_preview_item(task_id, str(preview_id))
config, generation_model = _result_regeneration_model(task, store)
try:
result = _regenerate_result_in_place(
task_id,
current,
preview,
config,
generation_model,
store,
expected_updated_at=payload.expected_updated_at,
)
except _ResultRegenerationFailed as exc:
logger.warning(
"data process result regeneration failed task_id=%s result_id=%s "
"duration_ms=%.2f reason=%s",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
_safe_regeneration_error(exc),
)
raise
logger.info(
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
)
return ok(result, "data process result regenerated")
@router.post("/{task_id}/publish")
def publish(
task_id: str,

108
backend/app/db/sql/002_data_process.sql
View File

@@ -80,6 +80,17 @@ CREATE TABLE IF NOT EXISTS data_process_tasks (
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
failure_reason TEXT,
generation_run_id TEXT,
results_confirmed BOOLEAN NOT NULL DEFAULT TRUE,
workflow_step VARCHAR(20) NOT NULL DEFAULT 'create'
CHECK (workflow_step IN ('create', 'model', 'upload', 'preview', 'generate', 'results')),
preview_status VARCHAR(20) NOT NULL DEFAULT 'idle'
CHECK (preview_status IN ('idle', 'queued', 'running', 'completed', 'failed', 'cancelled')),
preview_progress NUMERIC(5,2) NOT NULL DEFAULT 0
CHECK (preview_progress >= 0 AND preview_progress <= 100),
preview_run_id TEXT,
preview_failure_reason TEXT,
preview_total_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_total_files >= 0),
preview_completed_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_completed_files >= 0),
tenant_id TEXT,
project_id TEXT,
owner_id TEXT,
@@ -95,6 +106,87 @@ CREATE TABLE IF NOT EXISTS data_process_tasks (
);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
-- 历史任务在引入六步确认流程前已经完成审核，默认保留为已确认；
-- 新任务由创建接口显式写入 FALSE，并在第六步确认后转为 TRUE。
ALTER TABLE data_process_tasks
ADD COLUMN IF NOT EXISTS results_confirmed BOOLEAN NOT NULL DEFAULT TRUE;
UPDATE data_process_tasks
SET results_confirmed=FALSE
WHERE status <> 'completed' AND results_confirmed=TRUE;
-- 先以可空列接入旧库，才能只回填历史行；随后再收紧默认值与约束。
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS workflow_step VARCHAR(20);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_status VARCHAR(20);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_progress NUMERIC(5,2);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_run_id TEXT;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_failure_reason TEXT;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_total_files INTEGER;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_completed_files INTEGER;
CREATE TEMP TABLE data_process_workflow_backfill_ids ON COMMIT DROP AS
SELECT id FROM data_process_tasks WHERE workflow_step IS NULL;
UPDATE data_process_tasks task
SET workflow_step = CASE
WHEN task.status IN ('running', 'failed', 'stopped') THEN 'generate'
WHEN task.status = 'completed' AND task.results_confirmed=FALSE THEN 'generate'
WHEN task.status = 'completed' THEN 'results'
ELSE 'create'
END
WHERE task.workflow_step IS NULL;
UPDATE data_process_tasks
SET preview_status='idle', preview_progress=0,
preview_total_files=0, preview_completed_files=0
WHERE preview_status IS NULL OR preview_progress IS NULL
OR preview_total_files IS NULL OR preview_completed_files IS NULL;
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET DEFAULT 'create';
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET DEFAULT 'idle';
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET DEFAULT 0;
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET DEFAULT 0;
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET DEFAULT 0;
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET NOT NULL;
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid='data_process_tasks'::regclass
AND conname='ck_data_process_tasks_workflow_step'
) THEN
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_workflow_step
CHECK (workflow_step IN ('create', 'model', 'upload', 'preview', 'generate', 'results'));
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid='data_process_tasks'::regclass
AND conname='ck_data_process_tasks_preview_status'
) THEN
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_preview_status
CHECK (preview_status IN ('idle', 'queued', 'running', 'completed', 'failed', 'cancelled'));
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid='data_process_tasks'::regclass
AND conname='ck_data_process_tasks_preview_progress'
) THEN
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_preview_progress
CHECK (preview_progress >= 0 AND preview_progress <= 100);
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid='data_process_tasks'::regclass
AND conname='ck_data_process_tasks_preview_file_counts'
) THEN
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_preview_file_counts
CHECK (preview_total_files >= 0 AND preview_completed_files >= 0
AND preview_completed_files <= preview_total_files);
END IF;
END $$;
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
ON data_process_tasks(name) WHERE deleted_at IS NULL;
@@ -153,6 +245,22 @@ CREATE TABLE IF NOT EXISTS data_process_preview_items (
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
ON data_process_preview_items(task_id, source_file_id, created_at);
-- 子表在新库中到这里才存在；只修复本次新增 workflow_step 前的历史任务。
UPDATE data_process_tasks task
SET workflow_step = CASE
WHEN EXISTS (
SELECT 1 FROM data_process_preview_items preview
WHERE preview.task_id=task.id
) THEN 'preview'
WHEN EXISTS (
SELECT 1 FROM data_process_source_files source_file
WHERE source_file.task_id=task.id AND source_file.deleted_at IS NULL
) THEN 'upload'
ELSE task.workflow_step
END
WHERE task.id IN (SELECT id FROM data_process_workflow_backfill_ids)
AND task.status='pending';
CREATE TABLE IF NOT EXISTS data_process_results (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,

202
backend/app/modules/data_process/generation.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
import logging
import re
from collections.abc import Callable, Iterable, Mapping
from typing import Any
@@ -22,6 +23,10 @@ class ModelGenerationError(ValueError):
"""模型配置、响应或调用失败。"""
class _TerminalModelGenerationError(ModelGenerationError):
"""使用相同参数重试也无法恢复的模型响应错误。"""
OUTPUT_TYPE_STANDARD = "standard"
OUTPUT_TYPE_REASONING = "reasoning"
SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
@@ -31,6 +36,25 @@ SUPPORTED_REASONING_DETAILS = {
REASONING_DETAIL_NORMAL,
REASONING_DETAIL_DETAILED,
}
MINIMAX_M3_API_HOSTS = {"api.minimax.io", "api.minimaxi.com"}
MINIMAX_M3_MIN_COMPLETION_TOKENS = 4096
logger = logging.getLogger(__name__)
def _is_retryable_generation_error(exc: Exception) -> bool:
if isinstance(exc, _TerminalModelGenerationError):
return False
if isinstance(exc, httpx.HTTPStatusError):
status_code = exc.response.status_code
return status_code in {408, 425, 429} or status_code >= 500
if isinstance(exc, httpx.RequestError):
return True
return isinstance(exc, (json.JSONDecodeError, ModelGenerationError))
def _is_official_minimax_m3(endpoint: str, model_name: str) -> bool:
host = (urlsplit(endpoint).hostname or "").casefold()
return host in MINIMAX_M3_API_HOSTS and model_name.casefold() == "minimax-m3"
def chat_completions_url(value: str) -> str:
@@ -59,32 +83,139 @@ def chat_completions_url(value: str) -> str:
return urlunsplit((parsed.scheme, parsed.netloc, target_path, "", ""))
def _message_content(payload: Mapping[str, Any]) -> str:
def _response_choice(payload: Mapping[str, Any]) -> Mapping[str, Any]:
try:
content = payload["choices"][0]["message"]["content"]
choice = payload["choices"][0]
except (KeyError, IndexError, TypeError) as exc:
raise ModelGenerationError(
"model response does not contain choices[0].message.content"
) from exc
raise ModelGenerationError("模型响应缺少 choices[0]") from exc
if not isinstance(choice, Mapping):
raise ModelGenerationError("模型响应 choices[0] 不是对象")
return choice
def _response_finish_reason(payload: Mapping[str, Any]) -> str:
try:
return str(_response_choice(payload).get("finish_reason") or "").strip().lower()
except ModelGenerationError:
return ""
def _response_content_length(payload: Mapping[str, Any]) -> int:
try:
message = _response_choice(payload).get("message")
if not isinstance(message, Mapping):
return 0
content = message.get("content")
if isinstance(content, str):
return len(content)
if isinstance(content, list):
return sum(
len(str(item.get("text") or ""))
for item in content
if isinstance(item, Mapping)
)
except ModelGenerationError:
pass
return 0
def _raise_for_terminal_response(payload: Mapping[str, Any]) -> Mapping[str, Any]:
choice = _response_choice(payload)
base_response = payload.get("base_resp")
status_code: Any = None
status_message = ""
if isinstance(base_response, Mapping):
status_code = base_response.get("status_code")
status_message = re.sub(
r"\s+", " ", str(base_response.get("status_msg") or "")
).strip()[:200]
if bool(payload.get("input_sensitive")) or status_code in {1026, "1026"}:
raise _TerminalModelGenerationError(
f"模型输入触发内容安全拦截（code={status_code or 1026}）"
)
if bool(payload.get("output_sensitive")) or status_code in {1027, "1027"}:
raise _TerminalModelGenerationError(
f"模型输出触发内容安全拦截（code={status_code or 1027}）"
)
finish_reason = str(choice.get("finish_reason") or "").strip().lower()
if finish_reason == "length":
raise _TerminalModelGenerationError(
"模型输出因达到 Token 上限被截断（finish_reason=length），"
"请提高最大输出长度后重试"
)
if finish_reason == "content_filter":
raise _TerminalModelGenerationError(
"模型输出被内容安全策略拦截（finish_reason=content_filter）"
)
if finish_reason in {"tool_calls", "function_call"}:
raise _TerminalModelGenerationError(
f"模型返回了当前生成任务不支持的工具调用（finish_reason={finish_reason}）"
)
if status_code not in {None, "", 0, "0"}:
detail = f"：{status_message}" if status_message else ""
raise _TerminalModelGenerationError(
f"模型服务返回业务错误（code={status_code}）{detail}"
)
return choice
def _message_content(payload: Mapping[str, Any]) -> str:
choice = _raise_for_terminal_response(payload)
message = choice.get("message")
if not isinstance(message, Mapping):
raise ModelGenerationError("模型响应缺少 choices[0].message")
content = message.get("content")
if isinstance(content, str):
return content
if isinstance(content, list):
result = content
elif isinstance(content, list):
parts = [
str(item.get("text") or "")
for item in content
if isinstance(item, Mapping) and item.get("type") in {None, "text", "output_text"}
]
if parts:
return "".join(parts)
raise ModelGenerationError("model response content must be text")
result = "".join(parts)
elif content is None:
result = ""
else:
raise ModelGenerationError("模型响应 content 必须是文本")
if not result.strip():
raise ModelGenerationError("模型返回的最终内容为空，未生成可解析的 JSON")
return result
def _json_documents(content: str) -> list[Any]:
decoder = json.JSONDecoder()
documents: list[Any] = []
cursor = 0
while cursor < len(content):
match = re.search(r"[\[{]", content[cursor:])
if not match:
break
start = cursor + match.start()
try:
value, end = decoder.raw_decode(content[start:])
except json.JSONDecodeError:
cursor = start + 1
continue
if isinstance(value, (Mapping, list)):
documents.append(value)
cursor = start + max(end, 1)
return documents
def _json_payload(content: str) -> Any:
# 只移除模型在 JSON 之前自行输出的思考过程，不能破坏 JSON 字段中的训练内容。
cleaned = content.strip()
if re.match(r"^\s*<think>", cleaned, flags=re.IGNORECASE) and not re.match(
r"^\s*<think>[\s\S]*?</think>", cleaned, flags=re.IGNORECASE
):
raise ModelGenerationError("模型思考内容未闭合，响应可能已被截断")
cleaned = re.sub(
r"^\s*<think>[\s\S]*?</think>\s*",
r"^\s*(?:<think>[\s\S]*?</think>\s*)+",
"",
content,
cleaned,
count=1,
flags=re.IGNORECASE,
).strip()
@@ -93,10 +224,16 @@ def _json_payload(content: str) -> Any:
cleaned = fenced.group(1).strip()
try:
return json.loads(cleaned)
except json.JSONDecodeError as exc:
except json.JSONDecodeError as direct_error:
documents = _json_documents(cleaned)
if len(documents) == 1:
return documents[0]
if len(documents) > 1:
raise ModelGenerationError("模型响应包含多个 JSON 对象，无法确定应使用哪一个")
raise ModelGenerationError(
f"model response is not valid JSON at line {exc.lineno}, column {exc.colno}"
) from exc
"模型响应中没有找到唯一且完整的 JSON 对象"
f"（第 {direct_error.lineno} 行，第 {direct_error.colno} 列）"
) from direct_error
def _result_items(payload: Any) -> list[Mapping[str, Any]]:
@@ -206,6 +343,7 @@ def generate_model_records(
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
if not model_name:
raise ModelGenerationError("generation model name is required")
is_minimax_m3 = _is_official_minimax_m3(endpoint, model_name)
temperature = float(config.get("temperature", 0.7))
max_tokens = int(config.get("max_tokens", 1024))
@@ -246,9 +384,18 @@ def generate_model_records(
reasoning_detail=reasoning_detail,
),
"temperature": temperature,
"max_tokens": max_tokens,
}
if bool(config.get("json_mode", False)):
if is_minimax_m3:
request_payload.update(
reasoning_split=True,
max_completion_tokens=max(
max_tokens,
MINIMAX_M3_MIN_COMPLETION_TOKENS,
),
)
else:
request_payload["max_tokens"] = max_tokens
if bool(config.get("json_mode", False)) and not is_minimax_m3:
request_payload["response_format"] = {"type": "json_object"}
last_error: Exception | None = None
@@ -264,7 +411,24 @@ def generate_model_records(
body = response.json()
if not isinstance(body, Mapping):
raise ModelGenerationError("model response body must be a JSON object")
candidate_items = _result_items(_json_payload(_message_content(body)))
try:
candidate_items = _result_items(
_json_payload(_message_content(body))
)
except ModelGenerationError as exc:
logger.warning(
"data process model response rejected task_id=%s model=%s "
"finish_reason=%s response_chars=%s input_sensitive=%s "
"output_sensitive=%s reason=%s",
task_id,
model_name,
_response_finish_reason(body) or "missing",
_response_content_length(body),
bool(body.get("input_sensitive")),
bool(body.get("output_sensitive")),
str(exc),
)
raise
if len(candidate_items) < batch_count:
raise ModelGenerationError(
"model response contains fewer result objects than requested: "
@@ -278,6 +442,8 @@ def generate_model_records(
ModelGenerationError,
) as exc:
last_error = exc
if not _is_retryable_generation_error(exc):
break
if generated_items is None:
error_message = str(last_error or "model generation failed")[:2000]

27
backend/app/modules/data_process/schema_cli.py
View File

@@ -7,6 +7,18 @@ from urllib.parse import urlsplit
from app.modules.data_process.store import DataProcessStore
REQUIRED_TASK_COLUMNS = (
"generation_run_id",
"results_confirmed",
"workflow_step",
"preview_status",
"preview_progress",
"preview_run_id",
"preview_failure_reason",
"preview_total_files",
"preview_completed_files",
)
def _target_label(database_url: str) -> str:
parsed = urlsplit(database_url)
@@ -18,14 +30,13 @@ def _schema_ready(store: DataProcessStore) -> bool:
with store.connect() as conn:
row = conn.execute(
"""
SELECT EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema=current_schema()
AND table_name='data_process_tasks'
AND column_name='generation_run_id'
) AS ready
"""
SELECT COUNT(*) = %s AS ready
FROM information_schema.columns
WHERE table_schema=current_schema()
AND table_name='data_process_tasks'
AND column_name = ANY(%s)
""",
(len(REQUIRED_TASK_COLUMNS), list(REQUIRED_TASK_COLUMNS)),
).fetchone()
return bool(row and row["ready"])

482
backend/app/modules/data_process/store.py
View File

@@ -20,6 +20,8 @@ from app.modules.data_process.algorithms import estimate_token_count, stable_spl
TASK_STATUSES = {"pending", "running", "completed", "failed", "stopped"}
EDITABLE_STATUSES = {"pending", "failed", "stopped", "completed"}
ACTIVE_PREVIEW_STATUSES = {"queued", "running"}
WORKFLOW_STEPS = {"create", "model", "upload", "preview", "generate", "results"}
_PREVIEW_CONFIG_ALIASES = {
"preprocess_options": "preprocessOptions",
@@ -339,9 +341,10 @@ class DataProcessStore:
"""
INSERT INTO data_process_tasks
(id, name, description, status, process_type, source_dataset_id, config,
progress, tenant_id, project_id, owner_id, created_by, updated_by,
progress, results_confirmed, tenant_id, project_id, owner_id, created_by, updated_by,
created_at, updated_at)
VALUES (%s, %s, %s, 'pending', %s, %s, %s, 0, %s, %s, %s, %s, %s, %s, %s)
VALUES (%s, %s, %s, 'pending', %s, %s, %s, 0, FALSE,
%s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
@@ -442,9 +445,30 @@ class DataProcessStore:
def _ensure_editable(task: dict[str, Any]) -> None:
if task["status"] not in EDITABLE_STATUSES:
raise InvalidStateError(f"task cannot be edited while status is {task['status']}")
if task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
raise InvalidStateError("task cannot be edited while preview is running")
if task.get("output_dataset_id") and not _is_regeneration_prepared(task):
raise InvalidStateError("published task cannot be edited")
def update_workflow_step(self, task_id: str, workflow_step: str) -> dict[str, Any]:
"""独立保存向导位置，不触发配置或结果失效逻辑。"""
if workflow_step not in WORKFLOW_STEPS:
raise ValueError("invalid data process workflow step")
with self.connect() as conn:
row = conn.execute(
"""
UPDATE data_process_tasks
SET workflow_step=%s, updated_at=%s
WHERE id=%s AND deleted_at IS NULL
RETURNING *
""",
(workflow_step, utcnow(), task_id),
).fetchone()
if not row:
raise NotFoundError("data process task not found")
return _public_task(_decode_row(row)) or {}
def recover_legacy_aborted_regeneration(self, task_id: str) -> dict[str, Any]:
"""恢复旧版在真正开始生成前误删的上一轮结果。
@@ -593,7 +617,8 @@ class DataProcessStore:
output_count=%s, filtered_count=0, duplicate_count=0,
error_count=(SELECT COUNT(*) FROM data_process_results
WHERE task_id=%s AND status='invalid'),
failure_reason=NULL, updated_at=%s
failure_reason=NULL, results_confirmed=TRUE,
workflow_step='results', updated_at=%s
WHERE id=%s
""",
(
@@ -660,6 +685,13 @@ class DataProcessStore:
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"results_confirmed": False,
"preview_status": "idle",
"preview_progress": 0,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": 0,
"preview_completed_files": 0,
"started_at": None,
"completed_at": None,
}
@@ -709,6 +741,8 @@ class DataProcessStore:
task = self._task_in_connection(conn, task_id, for_update=True)
if task["status"] == "running":
raise ConflictError("running task cannot be prepared for regeneration")
if task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
raise ConflictError("running preview cannot be prepared for regeneration")
current_updated_at = _serialize_value(task.get("updated_at"))
if payload["expected_updated_at"] != current_updated_at:
@@ -785,14 +819,19 @@ class DataProcessStore:
def delete_task(self, task_id: str, *, deleted_by: str | None = None) -> None:
with self.connect() as conn:
task = self._task_in_connection(conn, task_id, for_update=True)
if task["status"] == "running":
raise InvalidStateError("running task must be stopped before deletion")
self._task_in_connection(conn, task_id, for_update=True)
now = utcnow()
conn.execute(
"""
UPDATE data_process_tasks
SET deleted_at=%s, deleted_by=%s, updated_at=%s
SET status=CASE WHEN status='running' THEN 'stopped' ELSE status END,
generation_run_id=NULL,
preview_status=CASE
WHEN preview_status IN ('queued', 'running') THEN 'cancelled'
ELSE preview_status
END,
preview_run_id=NULL,
deleted_at=%s, deleted_by=%s, updated_at=%s
WHERE id=%s
""",
(now, deleted_by, now, task_id),
@@ -918,7 +957,10 @@ class DataProcessStore:
SELECT COALESCE(SUM(record_count), 0)
FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL
), updated_at=%s
), workflow_step='upload', preview_status='idle',
preview_progress=0, preview_run_id=NULL,
preview_failure_reason=NULL, preview_total_files=0,
preview_completed_files=0, updated_at=%s
WHERE id=%s
""",
(task_id, now, task_id),
@@ -934,7 +976,12 @@ class DataProcessStore:
UPDATE data_process_tasks
SET status='pending', progress=%s, output_count=0, filtered_count=0,
duplicate_count=0, error_count=0, failure_reason=NULL,
generation_run_id=NULL, started_at=NULL, completed_at=NULL,
generation_run_id=NULL, results_confirmed=FALSE,
workflow_step='upload', preview_status='idle',
preview_progress=0, preview_run_id=NULL,
preview_failure_reason=NULL, preview_total_files=0,
preview_completed_files=0,
started_at=NULL, completed_at=NULL,
input_count=(
SELECT COALESCE(SUM(record_count), 0)
FROM data_process_source_files
@@ -1035,7 +1082,10 @@ class DataProcessStore:
SET input_count=(SELECT COALESCE(SUM(record_count), 0)
FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL),
updated_at=%s
workflow_step='upload', preview_status='idle',
preview_progress=0, preview_run_id=NULL,
preview_failure_reason=NULL, preview_total_files=0,
preview_completed_files=0, updated_at=%s
WHERE id=%s
""",
(task_id, now, task_id),
@@ -1049,6 +1099,11 @@ class DataProcessStore:
UPDATE data_process_tasks
SET status='pending', progress=0, output_count=0, filtered_count=0,
duplicate_count=0, error_count=0, failure_reason=NULL,
results_confirmed=FALSE,
workflow_step='upload', preview_status='idle',
preview_progress=0, preview_run_id=NULL,
preview_failure_reason=NULL, preview_total_files=0,
preview_completed_files=0,
input_count=(SELECT COALESCE(SUM(record_count), 0)
FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL),
@@ -1064,6 +1119,7 @@ class DataProcessStore:
items: Sequence[dict[str, Any]],
*,
source_file_ids: Sequence[str] | None = None,
preview_run_id: str | None = None,
) -> list[dict[str, Any]]:
selected_ids = (
list(dict.fromkeys(str(file_id) for file_id in source_file_ids))
@@ -1081,11 +1137,21 @@ class DataProcessStore:
}
if unexpected:
raise ValueError("preview items contain an unselected source file")
preview_file_count = len(selected_ids) if selected_ids is not None else len(
{str(item.get("source_file_id") or "") for item in items}
)
is_direct_build = preview_run_id is None
now = utcnow()
with self.connect() as conn:
task = self._task_in_connection(conn, task_id, for_update=True)
self._ensure_editable(task)
if is_direct_build:
self._ensure_editable(task)
elif (
task.get("preview_run_id") != preview_run_id
or task.get("preview_status") != "running"
):
raise InvalidStateError("preview run is no longer active")
regeneration_prepared = _is_regeneration_prepared(task)
if not regeneration_prepared:
conn.execute("DELETE FROM data_process_results WHERE task_id=%s", (task_id,))
@@ -1144,22 +1210,249 @@ class DataProcessStore:
).fetchone()
created.append(_decode_row(row) or {})
if regeneration_prepared:
conn.execute(
"UPDATE data_process_tasks SET updated_at=%s WHERE id=%s",
(now, task_id),
)
if is_direct_build:
conn.execute(
"""
UPDATE data_process_tasks
SET workflow_step='preview', preview_status='completed',
preview_progress=100, preview_run_id=NULL,
preview_failure_reason=NULL, preview_total_files=%s,
preview_completed_files=%s, updated_at=%s
WHERE id=%s
""",
(preview_file_count, preview_file_count, now, task_id),
)
else:
conn.execute(
"UPDATE data_process_tasks SET updated_at=%s WHERE id=%s",
(now, task_id),
)
else:
conn.execute(
"""
UPDATE data_process_tasks
SET status='pending', progress=20, output_count=0, filtered_count=0,
duplicate_count=0, error_count=0, failure_reason=NULL, updated_at=%s
duplicate_count=0, error_count=0, failure_reason=NULL,
results_confirmed=FALSE,
workflow_step=CASE WHEN %s THEN 'preview' ELSE workflow_step END,
preview_status=CASE WHEN %s THEN 'completed' ELSE preview_status END,
preview_progress=CASE WHEN %s THEN 100 ELSE preview_progress END,
preview_run_id=CASE WHEN %s THEN NULL ELSE preview_run_id END,
preview_failure_reason=CASE WHEN %s THEN NULL ELSE preview_failure_reason END,
preview_total_files=CASE WHEN %s THEN %s ELSE preview_total_files END,
preview_completed_files=CASE WHEN %s THEN %s ELSE preview_completed_files END,
updated_at=%s
WHERE id=%s
""",
(now, task_id),
(
is_direct_build,
is_direct_build,
is_direct_build,
is_direct_build,
is_direct_build,
is_direct_build,
preview_file_count,
is_direct_build,
preview_file_count,
now,
task_id,
),
)
return created
def start_preview(
self,
task_id: str,
*,
source_file_ids: Sequence[str] | None = None,
) -> tuple[dict[str, Any], list[str]]:
"""创建一轮持久化切分任务，并返回本轮固定的源文件集合。"""
requested_ids = (
list(dict.fromkeys(str(file_id) for file_id in source_file_ids))
if source_file_ids is not None
else None
)
if requested_ids is not None and (
not requested_ids or any(not file_id for file_id in requested_ids)
):
raise ValueError("source_file_ids must contain non-empty ids")
with self.connect() as conn:
task = self._task_in_connection(conn, task_id, for_update=True)
self._ensure_editable(task)
if requested_ids is None:
rows = conn.execute(
"""
SELECT id FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL
ORDER BY created_at, id
""",
(task_id,),
).fetchall()
else:
rows = conn.execute(
"""
SELECT id FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL AND id=ANY(%s)
ORDER BY created_at, id
""",
(task_id, requested_ids),
).fetchall()
selected_ids = [str(row["id"]) for row in rows]
if not selected_ids:
raise InvalidStateError("at least one source file is required")
if requested_ids is not None:
missing = set(requested_ids) - set(selected_ids)
if missing:
raise NotFoundError(
f"source files not found: {', '.join(sorted(missing))}"
)
regeneration_prepared = _is_regeneration_prepared(task)
if not regeneration_prepared:
conn.execute("DELETE FROM data_process_results WHERE task_id=%s", (task_id,))
preview_run_id = new_id("dpprun")
now = utcnow()
row = conn.execute(
"""
UPDATE data_process_tasks
SET status=CASE WHEN %s THEN status ELSE 'pending' END,
progress=CASE WHEN %s THEN progress ELSE 0 END,
output_count=CASE WHEN %s THEN output_count ELSE 0 END,
filtered_count=CASE WHEN %s THEN filtered_count ELSE 0 END,
duplicate_count=CASE WHEN %s THEN duplicate_count ELSE 0 END,
error_count=CASE WHEN %s THEN error_count ELSE 0 END,
failure_reason=CASE WHEN %s THEN failure_reason ELSE NULL END,
results_confirmed=CASE WHEN %s THEN results_confirmed ELSE FALSE END,
workflow_step='upload', preview_status='queued', preview_progress=0,
preview_run_id=%s, preview_failure_reason=NULL,
preview_total_files=%s, preview_completed_files=0,
updated_at=%s
WHERE id=%s AND deleted_at IS NULL
RETURNING *
""",
(
regeneration_prepared,
regeneration_prepared,
regeneration_prepared,
regeneration_prepared,
regeneration_prepared,
regeneration_prepared,
regeneration_prepared,
regeneration_prepared,
preview_run_id,
len(selected_ids),
now,
task_id,
),
).fetchone()
return _public_task(_decode_row(row)) or {}, selected_ids
def mark_preview_running(self, task_id: str, preview_run_id: str) -> bool:
with self.connect() as conn:
row = conn.execute(
"""
UPDATE data_process_tasks
SET preview_status='running', updated_at=%s
WHERE id=%s AND deleted_at IS NULL
AND preview_status='queued' AND preview_run_id=%s
RETURNING id
""",
(utcnow(), task_id, preview_run_id),
).fetchone()
return row is not None
def preview_is_running(self, task_id: str, preview_run_id: str) -> bool:
with self.connect() as conn:
row = conn.execute(
"""
SELECT preview_status, preview_run_id
FROM data_process_tasks
WHERE id=%s AND deleted_at IS NULL
""",
(task_id,),
).fetchone()
return bool(
row
and row.get("preview_status") in ACTIVE_PREVIEW_STATUSES
and row.get("preview_run_id") == preview_run_id
)
def update_preview_progress(
self,
task_id: str,
preview_run_id: str,
completed_files: int,
total_files: int,
) -> bool:
total = max(1, total_files)
completed = min(max(0, completed_files), total)
progress = completed / total * 100
with self.connect() as conn:
row = conn.execute(
"""
UPDATE data_process_tasks
SET preview_progress=%s, preview_completed_files=%s, updated_at=%s
WHERE id=%s AND deleted_at IS NULL
AND preview_status='running' AND preview_run_id=%s
RETURNING id
""",
(progress, completed, utcnow(), task_id, preview_run_id),
).fetchone()
return row is not None
def complete_preview(self, task_id: str, preview_run_id: str) -> bool:
with self.connect() as conn:
row = conn.execute(
"""
UPDATE data_process_tasks
SET workflow_step='preview', preview_status='completed',
preview_progress=100, preview_run_id=NULL,
preview_failure_reason=NULL,
preview_completed_files=preview_total_files, updated_at=%s
WHERE id=%s AND deleted_at IS NULL
AND preview_status='running' AND preview_run_id=%s
RETURNING id
""",
(utcnow(), task_id, preview_run_id),
).fetchone()
return row is not None
def mark_preview_failed(
self,
task_id: str,
reason: str,
*,
preview_run_id: str,
) -> bool:
with self.connect() as conn:
row = conn.execute(
"""
UPDATE data_process_tasks
SET preview_status='failed', preview_run_id=NULL,
preview_failure_reason=%s, updated_at=%s
WHERE id=%s AND deleted_at IS NULL
AND preview_status IN ('queued', 'running') AND preview_run_id=%s
RETURNING id
""",
(reason[:4000], utcnow(), task_id, preview_run_id),
).fetchone()
return row is not None
def preview_progress(self, task_id: str) -> dict[str, Any]:
task = self.get_task(task_id)
return {
"task_id": task["id"],
"workflow_step": task.get("workflow_step") or "create",
"preview_status": task.get("preview_status") or "idle",
"preview_progress": float(task.get("preview_progress") or 0),
"preview_run_id": task.get("preview_run_id"),
"preview_failure_reason": task.get("preview_failure_reason"),
"preview_total_files": int(task.get("preview_total_files") or 0),
"preview_completed_files": int(task.get("preview_completed_files") or 0),
}
def list_preview_items(
self,
task_id: str,
@@ -1332,7 +1625,7 @@ class DataProcessStore:
UPDATE data_process_tasks
SET status='pending', progress=20, output_count=0, filtered_count=0,
duplicate_count=0, error_count=0, failure_reason=NULL,
generation_run_id=NULL, updated_at=%s
generation_run_id=NULL, results_confirmed=FALSE, updated_at=%s
WHERE id=%s
""",
(now, task_id),
@@ -1348,6 +1641,8 @@ class DataProcessStore:
raise InvalidStateError("published task cannot be regenerated")
if task["status"] == "running":
raise ConflictError("data process task is already running")
if task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
raise ConflictError("preview is still running")
preview_count = conn.execute(
"SELECT COUNT(*) AS count FROM data_process_preview_items WHERE task_id=%s",
(task_id,),
@@ -1365,7 +1660,8 @@ class DataProcessStore:
SET config=%s, status='running', progress=30, failure_reason=NULL,
started_at=%s, completed_at=NULL, output_dataset_id=NULL,
output_count=0, filtered_count=0, duplicate_count=0, error_count=0,
generation_run_id=%s, updated_at=%s
generation_run_id=%s, results_confirmed=FALSE,
workflow_step='generate', updated_at=%s
WHERE id=%s
RETURNING *
""",
@@ -1391,10 +1687,19 @@ class DataProcessStore:
return _decode_row(row) or {}
def generation_is_running(self, task_id: str, generation_run_id: str) -> bool:
task = self.get_task(task_id)
return (
task["status"] == "running"
and task.get("generation_run_id") == generation_run_id
with self.connect() as conn:
row = conn.execute(
"""
SELECT status, generation_run_id
FROM data_process_tasks
WHERE id=%s AND deleted_at IS NULL
""",
(task_id,),
).fetchone()
return bool(
row
and row.get("status") == "running"
and row.get("generation_run_id") == generation_run_id
)
def update_generation_progress(
@@ -1469,7 +1774,8 @@ class DataProcessStore:
UPDATE data_process_tasks
SET status='completed', progress=100, output_count=%s, filtered_count=%s,
duplicate_count=%s, error_count=%s, failure_reason=NULL,
completed_at=%s, generation_run_id=NULL, updated_at=%s
completed_at=%s, generation_run_id=NULL, results_confirmed=FALSE,
updated_at=%s
WHERE id=%s AND generation_run_id=%s RETURNING *
""",
(
@@ -1519,10 +1825,59 @@ class DataProcessStore:
"duplicate_count": int(task.get("duplicate_count") or 0),
"error_count": int(task.get("error_count") or 0),
"failure_reason": task.get("failure_reason"),
"results_confirmed": bool(task.get("results_confirmed")),
"started_at": task.get("started_at"),
"completed_at": task.get("completed_at"),
}
def confirm_results(self, task_id: str) -> dict[str, Any]:
"""确认第六步结果，确认前再次校验所有生成记录。"""
with self.connect() as conn:
task = self._task_in_connection(conn, task_id, for_update=True)
if task["status"] != "completed":
raise InvalidStateError("only a completed task can confirm results")
if task.get("workflow_step") != "results":
raise InvalidStateError("workflow must be on results before confirmation")
if task.get("results_confirmed"):
return task
rows = conn.execute(
"""
SELECT status, instruction, output
FROM data_process_results
WHERE task_id=%s
""",
(task_id,),
).fetchall()
if not rows:
raise InvalidStateError("task has no results to confirm")
invalid_count = sum(
1
for row in rows
if row["status"] == "invalid"
or not str(row.get("instruction") or "").strip()
or not str(row.get("output") or "").strip()
or (
_task_output_type(task) == "reasoning"
and not _reasoning_output_is_valid(row.get("output"))
)
)
if invalid_count:
raise InvalidStateError(
f"task contains {invalid_count} invalid results"
)
row = conn.execute(
"""
UPDATE data_process_tasks
SET results_confirmed=TRUE, updated_at=%s
WHERE id=%s RETURNING *
""",
(utcnow(), task_id),
).fetchone()
return _decode_row(row) or {}
def list_results(
self,
task_id: str,
@@ -1651,6 +2006,83 @@ class DataProcessStore:
)
return _decode_row(row) or {}
def replace_generated_result(
self,
task_id: str,
result_id: str,
replacement: dict[str, Any],
*,
expected_updated_at: str,
) -> dict[str, Any]:
"""用新模型结果原位替换失败项，并将新内容设为恢复基线。"""
with self.connect() as conn:
task = self._task_in_connection(conn, task_id, for_update=True)
if task["status"] != "completed" or task.get("workflow_step") != "results":
raise InvalidStateError("task is not editing generation results")
if task.get("results_confirmed"):
raise InvalidStateError("confirmed results cannot be regenerated")
if task.get("output_dataset_id"):
raise InvalidStateError("published results cannot be regenerated")
current = conn.execute(
"""SELECT * FROM data_process_results
WHERE id=%s AND task_id=%s FOR UPDATE""",
(result_id, task_id),
).fetchone()
if not current:
raise NotFoundError("data process result not found")
if current.get("status") != "invalid":
raise InvalidStateError("only an invalid result can be regenerated")
current_updated_at = _serialize_value(current.get("updated_at"))
if expected_updated_at != current_updated_at:
raise ConflictError("data process result was modified by another request")
instruction = str(replacement.get("instruction") or "").strip()
input_text = str(replacement.get("input") or "").strip()
output = str(replacement.get("output") or "").strip()
quality_score = replacement.get("quality_score") or {}
if not instruction or not output or not bool(quality_score.get("is_valid")):
raise InvalidStateError("regenerated result did not pass quality validation")
if _task_output_type(task) == "reasoning" and not _reasoning_output_is_valid(output):
raise InvalidStateError("regenerated reasoning result has an invalid output format")
now = utcnow()
row = conn.execute(
"""
UPDATE data_process_results
SET instruction=%s, input=%s, output=%s,
original_instruction=%s, original_input=%s, original_output=%s,
status='valid', error=NULL, quality_score=%s, updated_at=%s
WHERE id=%s AND task_id=%s
RETURNING *
""",
(
instruction,
input_text,
output,
instruction,
input_text,
output,
json_dumps(quality_score),
now,
result_id,
task_id,
),
).fetchone()
conn.execute(
"""
UPDATE data_process_tasks
SET error_count=(
SELECT COUNT(*) FROM data_process_results
WHERE task_id=%s AND status='invalid'
), updated_at=%s
WHERE id=%s
""",
(task_id, now, task_id),
)
return _decode_row(row) or {}
def get_generation_model(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute(
@@ -1704,6 +2136,8 @@ class DataProcessStore:
)
if task["status"] != "completed":
raise InvalidStateError("only a completed task can be published")
if not task.get("results_confirmed"):
raise InvalidStateError("results must be confirmed before publishing")
rows = conn.execute(
"""
SELECT * FROM data_process_results

52
backend/app/schemas/data_process.py
View File

@@ -109,6 +109,24 @@ class DataProcessStatus(StrEnum):
stopped = "stopped"
class DataProcessWorkflowStep(StrEnum):
create = "create"
model = "model"
upload = "upload"
preview = "preview"
generate = "generate"
results = "results"
class DataProcessPreviewStatus(StrEnum):
idle = "idle"
queued = "queued"
running = "running"
completed = "completed"
failed = "failed"
cancelled = "cancelled"
class ProcessType(StrEnum):
structured = "structured"
unstructured = "unstructured"
@@ -164,6 +182,14 @@ class DataProcessTaskUpdate(BaseModel):
return self
class DataProcessWorkflowStepUpdate(BaseModel):
"""仅保存创建向导位置，不修改配置或使下游产物失效。"""
model_config = ConfigDict(extra="forbid")
workflow_step: DataProcessWorkflowStep
class DataProcessRegenerateRequest(BaseModel):
"""以一份完整配置准备任务重新生成。
@@ -288,6 +314,32 @@ class ResultUpdate(BaseModel):
expected_updated_at: str | None = None
class ResultRegenerateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
expected_updated_at: str = Field(min_length=1, max_length=100)
class ResultBatchRegenerateItem(BaseModel):
model_config = ConfigDict(extra="forbid")
result_id: str = Field(min_length=1, max_length=100)
expected_updated_at: str = Field(min_length=1, max_length=100)
class ResultBatchRegenerateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
items: list[ResultBatchRegenerateItem] = Field(min_length=1, max_length=100)
@model_validator(mode="after")
def validate_unique_results(self) -> "ResultBatchRegenerateRequest":
result_ids = [item.result_id for item in self.items]
if len(result_ids) != len(set(result_ids)):
raise ValueError("result_id values must be unique")
return self
class DatasetSplit(BaseModel):
model_config = ConfigDict(extra="forbid")

794
backend/tests/test_data_process_api.py
View File

@@ -3,11 +3,13 @@ from __future__ import annotations
from copy import deepcopy
from io import BytesIO
from pathlib import Path
from threading import Barrier, Lock
from typing import Any
import psycopg
import pytest
from docx import Document as WordDocument
from fastapi import FastAPI
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from openpyxl import Workbook
@@ -31,6 +33,7 @@ class FakeDataProcessStore:
self.previews: dict[str, list[dict[str, Any]]] = {}
self.results: dict[str, list[dict[str, Any]]] = {}
self.datasets: dict[str, dict[str, Any]] = {}
self.models: dict[str, dict[str, Any]] = {}
self.regeneration_prepared: set[str] = set()
self.sequence = 0
@@ -73,6 +76,14 @@ class FakeDataProcessStore:
"error_count": 0,
"failure_reason": None,
"output_dataset_id": None,
"results_confirmed": False,
"workflow_step": "create",
"preview_status": "idle",
"preview_progress": 0,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": 0,
"preview_completed_files": 0,
}
self.tasks[task_id] = task
self.sources[task_id] = []
@@ -105,6 +116,11 @@ class FakeDataProcessStore:
self.tasks[task_id].update(deepcopy(payload))
return self.get_task(task_id)
def update_workflow_step(self, task_id: str, workflow_step: str) -> dict[str, Any]:
self.get_task(task_id)
self.tasks[task_id]["workflow_step"] = workflow_step
return self.get_task(task_id)
def prepare_regeneration(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
task = self.tasks.get(task_id)
if task is None:
@@ -136,8 +152,6 @@ class FakeDataProcessStore:
def delete_task(self, task_id: str, **_: Any) -> None:
self.get_task(task_id)
if self.tasks[task_id]["status"] == "running":
raise InvalidStateError("running task must be stopped before deletion")
del self.tasks[task_id]
def list_source_files(self, task_id: str) -> list[dict[str, Any]]:
@@ -197,6 +211,13 @@ class FakeDataProcessStore:
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"workflow_step": "upload",
"preview_status": "idle",
"preview_progress": 0,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": 0,
"preview_completed_files": 0,
"started_at": None,
"completed_at": None,
}
@@ -254,6 +275,15 @@ class FakeDataProcessStore:
item for item in self.previews[task_id] if item["source_file_id"] != file_id
]
self.results[task_id] = []
self.tasks[task_id].update(
workflow_step="upload",
preview_status="idle",
preview_progress=0,
preview_run_id=None,
preview_failure_reason=None,
preview_total_files=0,
preview_completed_files=0,
)
def replace_preview_items(
self,
@@ -261,7 +291,10 @@ class FakeDataProcessStore:
items: list[dict[str, Any]],
*,
source_file_ids: list[str] | None = None,
preview_run_id: str | None = None,
) -> list[dict[str, Any]]:
if preview_run_id is not None and not self.preview_is_running(task_id, preview_run_id):
raise InvalidStateError("preview run is no longer active")
created = [
{"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items
]
@@ -276,8 +309,121 @@ class FakeDataProcessStore:
] + created
self.results[task_id] = []
self.tasks[task_id]["progress"] = 20
if preview_run_id is None:
file_count = len(source_file_ids or {item["source_file_id"] for item in created})
self.tasks[task_id].update(
workflow_step="preview",
preview_status="completed",
preview_progress=100,
preview_run_id=None,
preview_failure_reason=None,
preview_total_files=file_count,
preview_completed_files=file_count,
)
return deepcopy(created)
def start_preview(
self,
task_id: str,
*,
source_file_ids: list[str] | None = None,
) -> tuple[dict[str, Any], list[str]]:
task = self.tasks[task_id]
if task.get("preview_status") in {"queued", "running"}:
raise InvalidStateError("task cannot be edited while preview is running")
selected_ids = source_file_ids or [str(item["id"]) for item in self.sources[task_id]]
found = {str(item["id"]) for item in self.sources[task_id]}
missing = set(selected_ids) - found
if missing:
raise NotFoundError(f"source files not found: {', '.join(sorted(missing))}")
if not selected_ids:
raise InvalidStateError("at least one source file is required")
run_id = self._id("dpprun")
self.results[task_id] = []
task.update(
status="pending",
workflow_step="upload",
preview_status="queued",
preview_progress=0,
preview_run_id=run_id,
preview_failure_reason=None,
preview_total_files=len(selected_ids),
preview_completed_files=0,
results_confirmed=False,
)
return self.get_task(task_id), list(selected_ids)
def mark_preview_running(self, task_id: str, preview_run_id: str) -> bool:
task = self.tasks.get(task_id)
if not task or task.get("preview_status") != "queued" or task.get("preview_run_id") != preview_run_id:
return False
task["preview_status"] = "running"
return True
def preview_is_running(self, task_id: str, preview_run_id: str) -> bool:
task = self.tasks.get(task_id)
return bool(
task
and task.get("preview_status") in {"queued", "running"}
and task.get("preview_run_id") == preview_run_id
)
def update_preview_progress(
self,
task_id: str,
preview_run_id: str,
completed_files: int,
total_files: int,
) -> bool:
if not self.preview_is_running(task_id, preview_run_id):
return False
self.tasks[task_id]["preview_completed_files"] = completed_files
self.tasks[task_id]["preview_progress"] = completed_files / max(1, total_files) * 100
return True
def complete_preview(self, task_id: str, preview_run_id: str) -> bool:
if not self.preview_is_running(task_id, preview_run_id):
return False
task = self.tasks[task_id]
task.update(
workflow_step="preview",
preview_status="completed",
preview_progress=100,
preview_run_id=None,
preview_failure_reason=None,
preview_completed_files=task["preview_total_files"],
)
return True
def mark_preview_failed(
self,
task_id: str,
reason: str,
*,
preview_run_id: str,
) -> bool:
if not self.preview_is_running(task_id, preview_run_id):
return False
self.tasks[task_id].update(
preview_status="failed",
preview_run_id=None,
preview_failure_reason=reason,
)
return True
def preview_progress(self, task_id: str) -> dict[str, Any]:
task = self.get_task(task_id)
return {
"task_id": task_id,
"workflow_step": task["workflow_step"],
"preview_status": task["preview_status"],
"preview_progress": task["preview_progress"],
"preview_run_id": task["preview_run_id"],
"preview_failure_reason": task["preview_failure_reason"],
"preview_total_files": task["preview_total_files"],
"preview_completed_files": task["preview_completed_files"],
}
def list_preview_items(
self,
task_id: str,
@@ -348,15 +494,19 @@ class FakeDataProcessStore:
progress=30,
output_dataset_id=None,
output_count=0,
results_confirmed=False,
workflow_step="generate",
generation_run_id=self._id("dprun"),
)
self.regeneration_prepared.discard(task_id)
return self.get_task(task_id)
def generation_is_running(self, task_id: str, generation_run_id: str) -> bool:
task = self.tasks.get(task_id)
return (
self.tasks[task_id]["status"] == "running"
and self.tasks[task_id].get("generation_run_id") == generation_run_id
bool(task)
and task["status"] == "running"
and task.get("generation_run_id") == generation_run_id
)
def update_generation_progress(
@@ -389,6 +539,7 @@ class FakeDataProcessStore:
status="completed",
progress=100,
output_count=len(results),
results_confirmed=False,
generation_run_id=None,
**counts,
)
@@ -416,6 +567,7 @@ class FakeDataProcessStore:
result = {key: task.get(key) for key in (
"status", "progress", "input_count", "output_count",
"filtered_count", "duplicate_count", "error_count", "failure_reason",
"results_confirmed",
)}
result["task_id"] = task["id"]
return result
@@ -488,6 +640,66 @@ class FakeDataProcessStore:
item["status"] = "valid"
return deepcopy(item)
def get_generation_model(self, model_id: str) -> dict[str, Any]:
model = self.models.get(model_id)
if not model:
raise NotFoundError("generation model not found")
return deepcopy(model)
def replace_generated_result(
self,
task_id: str,
result_id: str,
replacement: dict[str, Any],
*,
expected_updated_at: str,
) -> dict[str, Any]:
task = self.tasks[task_id]
if task["status"] != "completed" or task.get("workflow_step") != "results":
raise InvalidStateError("task is not editing generation results")
if task.get("results_confirmed") or task.get("output_dataset_id"):
raise InvalidStateError("confirmed or published results cannot be regenerated")
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
if not item:
raise NotFoundError("data process result not found")
if item["status"] != "invalid":
raise InvalidStateError("only an invalid result can be regenerated")
if expected_updated_at != item.get("updated_at"):
raise InvalidStateError("data process result was modified by another request")
for field in ("instruction", "input", "output"):
item[field] = replacement[field]
item[f"original_{field}"] = replacement[field]
item.update(
status=replacement["status"],
error=replacement.get("error"),
quality_score=deepcopy(replacement["quality_score"]),
updated_at="2026-07-27T22:00:00Z",
)
task["error_count"] = sum(
result["status"] == "invalid" for result in self.results[task_id]
)
return deepcopy(item)
def confirm_results(self, task_id: str) -> dict[str, Any]:
task = self.tasks[task_id]
if task["status"] != "completed":
raise InvalidStateError("only a completed task can confirm results")
if task.get("workflow_step") != "results":
raise InvalidStateError("workflow must be on results before confirmation")
invalid_count = sum(
1
for item in self.results[task_id]
if item["status"] == "invalid"
or not item["instruction"].strip()
or not item["output"].strip()
)
if invalid_count:
raise InvalidStateError(f"task contains {invalid_count} invalid results")
if not self.results[task_id]:
raise InvalidStateError("task has no results to confirm")
task["results_confirmed"] = True
return self.get_task(task_id)
def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
task = self.tasks[task_id]
if task_id in self.regeneration_prepared:
@@ -508,6 +720,8 @@ class FakeDataProcessStore:
}
if task["status"] != "completed":
raise InvalidStateError("only a completed task can be published")
if not task.get("results_confirmed"):
raise InvalidStateError("results must be confirmed before publishing")
split_specs = (
("train", "训练集"),
("val", "验证集"),
@@ -625,6 +839,14 @@ def _minimal_pdf(text: str = "Hello PDF") -> bytes:
return _minimal_pdf_pages(text)
def test_outdated_data_process_schema_returns_actionable_503() -> None:
with pytest.raises(HTTPException) as captured, data_process_endpoint.api_errors():
raise psycopg.errors.UndefinedColumn("missing runtime column")
assert captured.value.status_code == 503
assert "schema is missing or out of date" in captured.value.detail["message"]
def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
created = client.post(
@@ -636,6 +858,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
},
)
assert created.status_code == 200
assert created.json()["data"]["results_confirmed"] is False
task_id = created.json()["data"]["id"]
source_content = (
@@ -687,8 +910,10 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
assert generated.status_code == 200
assert generated.json()["data"]["results_confirmed"] is False
progress = client.get(f"/modelTF/data-process/{task_id}/progress")
assert progress.json()["data"]["status"] == "completed"
assert progress.json()["data"]["results_confirmed"] is False
result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
assert result_page["total"] == 2
keyword_page = client.get(
@@ -719,6 +944,17 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
assert restored.json()["data"]["status"] == "valid"
assert store.tasks[task_id]["error_count"] == 0
workflow = client.put(
f"/modelTF/data-process/{task_id}/workflow-step",
json={"workflow_step": "results"},
)
assert workflow.status_code == 200
confirmed = client.post(
f"/modelTF/data-process/{task_id}/confirm-results"
)
assert confirmed.status_code == 200
assert confirmed.json()["data"]["results_confirmed"] is True
publish_payload = {"dataset_name": "客服问答清洗集"}
first_publish = client.post(
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
@@ -761,6 +997,120 @@ def test_task_list_exposes_document_and_generation_counts(
assert item["output_dataset_id"] is None
def test_workflow_step_update_is_independent_and_validated(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "步骤持久化", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.previews[task_id] = [{"id": "preview-1"}]
store.results[task_id] = [{"id": "result-1"}]
store.tasks[task_id].update(status="running", generation_run_id="run-1")
updated = client.put(
f"/modelTF/data-process/{task_id}/workflow-step",
json={"workflow_step": "generate"},
)
assert updated.status_code == 200
assert updated.json()["data"]["workflow_step"] == "generate"
assert store.previews[task_id] == [{"id": "preview-1"}]
assert store.results[task_id] == [{"id": "result-1"}]
assert store.tasks[task_id]["status"] == "running"
assert client.put(
f"/modelTF/data-process/{task_id}/workflow-step",
json={"workflow_step": "unknown"},
).status_code == 422
def test_background_preview_persists_progress_and_items(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "后台切分", "process_type": "structured", "config": {}},
).json()["data"]["id"]
source = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={
"files": (
"one.jsonl",
b'{"question":"What is one?","answer":"One."}\n',
"application/jsonl",
)
},
).json()["data"]["files"][0]
started = client.post(
f"/modelTF/data-process/{task_id}/preview/start",
json={"source_file_ids": [source["id"]]},
)
assert started.status_code == 202
assert started.json()["data"]["preview_status"] == "queued"
assert started.json()["data"]["preview_run_id"]
progress = client.get(
f"/modelTF/data-process/{task_id}/preview/progress"
).json()["data"]
assert progress == {
"task_id": task_id,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100.0,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": 1,
"preview_completed_files": 1,
}
assert client.get(
f"/modelTF/data-process/{task_id}/preview"
).json()["data"]["total"] == 1
def test_stale_preview_worker_cannot_replace_new_run(tmp_path: Path) -> None:
_, store, storage = make_client(tmp_path)
task = store.create_task(
{"name": "切分代次", "process_type": "structured", "config": {}}
)
task_id = str(task["id"])
store.sources[task_id] = [{"id": "source-1"}]
first, source_ids = store.start_preview(task_id, source_file_ids=["source-1"])
first_run_id = str(first["preview_run_id"])
store.tasks[task_id].update(preview_status="cancelled", preview_run_id=None)
second, _ = store.start_preview(task_id, source_file_ids=["source-1"])
data_process_endpoint._run_preview(
store,
storage,
task_id,
first_run_id,
source_ids,
)
assert store.tasks[task_id]["preview_status"] == "queued"
assert store.tasks[task_id]["preview_run_id"] == second["preview_run_id"]
assert store.previews[task_id] == []
def test_delete_invalidates_active_generation_and_preview(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "删除运行任务", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
status="running",
generation_run_id="generation-1",
preview_status="running",
preview_run_id="preview-1",
)
deleted = client.delete(f"/modelTF/data-process/{task_id}")
assert deleted.status_code == 200
assert store.generation_is_running(task_id, "generation-1") is False
assert store.preview_is_running(task_id, "preview-1") is False
def test_task_detail_uses_returned_source_files_as_document_count(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
@@ -811,6 +1161,404 @@ def test_generation_start_response_clears_previous_output_count(
assert store.tasks[task_id]["output_count"] == 0
def test_results_must_be_generated_and_valid_before_confirmation(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "结果确认门禁", "process_type": "structured", "config": {}},
).json()["data"]["id"]
pending = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
assert pending.status_code == 409
store.tasks[task_id].update(status="completed", progress=100, workflow_step="generate")
store.results[task_id] = [
{
"id": "result_valid",
"status": "valid",
"instruction": "问题",
"input": "",
"output": "答案",
}
]
wrong_step = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
assert wrong_step.status_code == 409
store.tasks[task_id]["workflow_step"] = "results"
store.results[task_id] = [
{
"id": "result_invalid",
"status": "invalid",
"instruction": "问题",
"input": "",
"output": "",
}
]
invalid = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
assert invalid.status_code == 409
store.results[task_id][0].update(status="valid", output="答案")
confirmed = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
assert confirmed.status_code == 200
assert confirmed.json()["data"]["results_confirmed"] is True
assert client.post(
f"/modelTF/data-process/{task_id}/confirm-results"
).status_code == 200
def test_invalid_result_can_be_regenerated_in_place(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "单条重生成",
"process_type": "structured",
"config": {
"generation_model_id": "model-1",
"output_type": "standard",
"min_output_length": 5,
"generation_retries": 5,
"request_timeout_seconds": 120,
},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
progress=100,
workflow_step="results",
results_confirmed=False,
error_count=1,
)
store.models["model-1"] = {
"id": "model-1",
"name": "测试模型",
"online_model_name": "test-model",
"api_url": "https://model.example/v1",
"api_key": "secret",
}
store.previews[task_id] = [{
"id": "preview-1",
"status": "original",
"original_content": "申请编号字段用于标识报销申请。",
"edited_content": "申请编号字段用于标识报销申请。",
}]
store.results[task_id] = [{
"id": "result-1",
"preview_item_id": "preview-1",
"instruction": "模型生成失败，请人工补充",
"input": "申请编号字段用于标识报销申请。",
"output": "",
"original_instruction": "模型生成失败，请人工补充",
"original_input": "申请编号字段用于标识报销申请。",
"original_output": "",
"status": "invalid",
"error": "model response is not valid JSON",
"split": "train",
"quality_score": {},
"updated_at": "2026-07-27T21:00:00Z",
}]
captured: dict[str, Any] = {}
def fake_generate(preview_items: Any, **kwargs: Any) -> list[dict[str, Any]]:
captured["preview_items"] = list(preview_items)
captured["qa_pairs_per_item"] = kwargs["qa_pairs_per_item"]
captured["config"] = kwargs["config"]
return [{
"id": "temporary-result",
"preview_item_id": "preview-1",
"instruction": "申请编号字段有什么作用？",
"input": "申请编号字段用于标识报销申请。",
"output": "申请编号字段用于唯一标识一笔报销申请。",
"original_instruction": "申请编号字段有什么作用？",
"original_input": "申请编号字段用于标识报销申请。",
"original_output": "申请编号字段用于唯一标识一笔报销申请。",
"status": "valid",
"error": None,
"split": "train",
}]
monkeypatch.setattr(data_process_endpoint, "generate_model_records", fake_generate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/result-1/regenerate",
json={"expected_updated_at": "2026-07-27T21:00:00Z"},
)
assert response.status_code == 200
regenerated = response.json()["data"]
assert regenerated["id"] == "result-1"
assert regenerated["status"] == "valid"
assert regenerated["instruction"] == regenerated["original_instruction"]
assert regenerated["output"] == regenerated["original_output"]
assert store.tasks[task_id]["error_count"] == 0
assert captured["qa_pairs_per_item"] == 1
assert [item["id"] for item in captured["preview_items"]] == ["preview-1"]
assert captured["config"]["generation_retries"] == 0
assert captured["config"]["request_timeout_seconds"] == 60
def test_failed_result_regeneration_keeps_the_original_error(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "失败结果保留",
"process_type": "structured",
"config": {
"generation_model_id": "model-1",
"output_type": "standard",
},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
progress=100,
workflow_step="results",
results_confirmed=False,
error_count=1,
)
store.models["model-1"] = {
"id": "model-1",
"online_model_name": "test-model",
"api_url": "https://model.example/v1",
"api_key": "secret",
}
store.previews[task_id] = [{
"id": "preview-1",
"status": "original",
"original_content": "原始内容",
"edited_content": "原始内容",
}]
original_result = {
"id": "result-1",
"preview_item_id": "preview-1",
"instruction": "模型生成失败，请人工补充",
"input": "原始内容",
"output": "",
"status": "invalid",
"error": "first failure",
"updated_at": "2026-07-27T21:30:00Z",
}
store.results[task_id] = [original_result.copy()]
monkeypatch.setattr(
data_process_endpoint,
"generate_model_records",
lambda *args, **kwargs: [{
"status": "invalid",
"error": "second failure",
}],
)
response = client.post(
f"/modelTF/data-process/{task_id}/results/result-1/regenerate",
json={"expected_updated_at": "2026-07-27T21:30:00Z"},
)
assert response.status_code == 409
assert store.results[task_id] == [original_result]
assert store.tasks[task_id]["error_count"] == 1
def test_failed_results_can_be_regenerated_in_parallel_with_partial_success(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "批量重生成",
"process_type": "structured",
"config": {
"generation_model_id": "model-1",
"output_type": "standard",
"min_output_length": 5,
},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
progress=100,
workflow_step="results",
results_confirmed=False,
error_count=2,
)
store.models["model-1"] = {
"id": "model-1",
"online_model_name": "test-model",
"api_url": "https://model.example/v1",
"api_key": "secret",
}
store.previews[task_id] = [
{
"id": "preview-1",
"status": "original",
"original_content": "申请编号用于唯一标识一笔报销申请。",
"edited_content": "申请编号用于唯一标识一笔报销申请。",
},
{
"id": "preview-2",
"status": "original",
"original_content": "联系电话用于联系申请人。",
"edited_content": "联系电话用于联系申请人。",
},
]
original_results = [
{
"id": "result-1",
"preview_item_id": "preview-1",
"instruction": "模型生成失败，请人工补充",
"input": "申请编号用于唯一标识一笔报销申请。",
"output": "",
"original_instruction": "模型生成失败，请人工补充",
"original_input": "申请编号用于唯一标识一笔报销申请。",
"original_output": "",
"status": "invalid",
"error": "first failure",
"split": "train",
"quality_score": {},
"updated_at": "2026-07-28T09:00:00Z",
},
{
"id": "result-2",
"preview_item_id": "preview-2",
"instruction": "模型生成失败，请人工补充",
"input": "联系电话用于联系申请人。",
"output": "",
"original_instruction": "模型生成失败，请人工补充",
"original_input": "联系电话用于联系申请人。",
"original_output": "",
"status": "invalid",
"error": "first failure",
"split": "train",
"quality_score": {},
"updated_at": "2026-07-28T09:00:01Z",
},
]
store.results[task_id] = deepcopy(original_results)
barrier = Barrier(2, timeout=2)
activity_lock = Lock()
active_calls = 0
max_active_calls = 0
model_clients: list[Any] = []
def fake_generate(preview_items: Any, **kwargs: Any) -> list[dict[str, Any]]:
nonlocal active_calls, max_active_calls
preview = next(iter(preview_items))
with activity_lock:
active_calls += 1
max_active_calls = max(max_active_calls, active_calls)
model_clients.append(kwargs.get("client"))
try:
barrier.wait()
if preview["id"] == "preview-2":
return [{"status": "invalid", "error": "second failure"}]
return [{
"id": "temporary-result",
"preview_item_id": preview["id"],
"instruction": "申请编号有什么作用？",
"input": preview["edited_content"],
"output": "申请编号用于唯一标识一笔报销申请。",
"status": "valid",
"error": None,
"split": "train",
}]
finally:
with activity_lock:
active_calls -= 1
monkeypatch.setattr(data_process_endpoint, "generate_model_records", fake_generate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/regenerate-batch",
json={
"items": [
{
"result_id": "result-1",
"expected_updated_at": "2026-07-28T09:00:00Z",
},
{
"result_id": "result-2",
"expected_updated_at": "2026-07-28T09:00:01Z",
},
],
},
)
assert response.status_code == 200
data = response.json()["data"]
assert data["total"] == 2
assert data["succeeded"] == 1
assert data["failed"] == 1
assert data["remaining_invalid_count"] == 1
assert [item["id"] for item in data["items"]] == ["result-1"]
assert data["failures"][0]["result_id"] == "result-2"
assert store.results[task_id][0]["status"] == "valid"
assert store.results[task_id][0]["id"] == "result-1"
assert store.results[task_id][1] == original_results[1]
assert store.tasks[task_id]["error_count"] == 1
assert max_active_calls == 2
assert all(client is not None for client in model_clients)
assert len({id(client) for client in model_clients}) == 1
@pytest.mark.parametrize(
"locked_state",
[
{"results_confirmed": True},
{"output_dataset_id": "dataset-published"},
],
ids=["confirmed", "published"],
)
def test_batch_result_regeneration_rejects_locked_tasks_before_model_call(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
locked_state: dict[str, Any],
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "批量重生成门禁",
"process_type": "structured",
"config": {"generation_model_id": "model-1"},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
workflow_step="results",
results_confirmed=False,
)
store.tasks[task_id].update(locked_state)
model_calls = 0
def fake_generate(*args: Any, **kwargs: Any) -> list[dict[str, Any]]:
nonlocal model_calls
model_calls += 1
return []
monkeypatch.setattr(data_process_endpoint, "generate_model_records", fake_generate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/regenerate-batch",
json={
"items": [{
"result_id": "result-1",
"expected_updated_at": "2026-07-28T09:00:00Z",
}],
},
)
assert response.status_code == 409
assert model_calls == 0
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
tmp_path: Path,
) -> None:
@@ -1026,6 +1774,7 @@ def test_published_split_datasets_remain_in_detail_after_regeneration(
"status": "completed",
"updated_at": "2026-07-27T09:00:00Z",
"output_count": 1,
"results_confirmed": True,
}
)
store.results[task_id] = [
@@ -1440,6 +2189,41 @@ def test_reasoning_output_requires_generation_model() -> None:
assert failed["failure_reason"] == "思维链输出必须配置可用的数据生成模型"
def test_generation_failure_is_written_to_structured_log(caplog: pytest.LogCaptureFixture) -> None:
store = FakeDataProcessStore()
task = store.create_task(
{
"name": "生成失败日志",
"process_type": "structured",
"config": {"output_type": "reasoning"},
}
)
task_id = task["id"]
store.replace_preview_items(
task_id,
[
{
"source_file_id": None,
"original_content": "需要推理的来源内容",
"edited_content": "需要推理的来源内容",
"status": "manual",
}
],
)
started = store.start_generation(task_id, replace_existing=True)
with caplog.at_level("INFO", logger=data_process_endpoint.__name__):
data_process_endpoint._run_generation(
store,
task_id,
started["generation_run_id"],
)
messages = [record.getMessage() for record in caplog.records]
assert any("generation worker started" in message for message in messages)
assert any("generation failed" in message for message in messages)
def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(

444
backend/tests/test_data_process_generation.py
View File

@@ -88,6 +88,344 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
assert progress_updates == [(1, 1)]
def test_minimax_m3_uses_split_reasoning_and_completion_token_budget() -> None:
requests: list[dict[str, object]] = []
def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
requests.append(payload)
return httpx.Response(
200,
json={
"choices": [{
"finish_reason": "stop",
"message": {
"reasoning_content": "模型内部思考不应混入业务 JSON",
"content": json.dumps({
"items": [{
"instruction": "申请编号有什么作用？",
"reasoning": "来源说明它用于标识报销申请。",
"answer": "它用于唯一标识一笔报销申请。",
}],
}, ensure_ascii=False),
},
}],
"output_sensitive": False,
"base_resp": {"status_code": 0, "status_msg": ""},
},
)
records = generate_model_records(
[{"id": "preview-minimax", "edited_content": "申请编号用于标识报销申请。"}],
model={
"name": "MiniMax",
"online_model_name": "MiniMax-M3",
"api_url": "https://api.minimaxi.com/v1",
},
config={
"output_type": "reasoning",
"json_mode": True,
"max_tokens": 1024,
"generation_retries": 0,
},
task_id="task-minimax",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert records[0]["status"] == "valid"
assert len(requests) == 1
assert requests[0]["reasoning_split"] is True
assert requests[0]["max_completion_tokens"] >= 4096
assert "max_tokens" not in requests[0]
assert "response_format" not in requests[0]
def test_minimax_m3_keeps_larger_configured_completion_budget() -> None:
requests: list[dict[str, object]] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(json.loads(request.content))
return httpx.Response(
200,
json={
"choices": [{
"message": {
"content": json.dumps({
"items": [{
"instruction": "问题",
"output": "这是满足测试要求的完整答案。",
}],
}, ensure_ascii=False),
},
}],
},
)
generate_model_records(
[{"id": "preview-minimax-budget", "edited_content": "来源正文"}],
model={
"online_model_name": "MiniMax-M3",
"api_url": "https://api.minimax.io/v1",
},
config={"max_tokens": 8192, "generation_retries": 0},
task_id="task-minimax-budget",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert requests[0]["max_completion_tokens"] == 8192
def test_minimax_m3_name_on_custom_proxy_keeps_generic_openai_parameters() -> None:
requests: list[dict[str, object]] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(json.loads(request.content))
return httpx.Response(
200,
json={
"choices": [{
"message": {
"content": json.dumps({
"items": [{
"instruction": "问题",
"output": "这是代理服务返回的完整答案。",
}],
}, ensure_ascii=False),
},
}],
},
)
generate_model_records(
[{"id": "preview-minimax-proxy", "edited_content": "来源正文"}],
model={
"online_model_name": "MiniMax-M3",
"api_url": "https://model-proxy.example/v1",
},
config={"max_tokens": 1024, "json_mode": True, "generation_retries": 0},
task_id="task-minimax-proxy",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert requests[0]["max_tokens"] == 1024
assert requests[0]["response_format"] == {"type": "json_object"}
assert "reasoning_split" not in requests[0]
assert "max_completion_tokens" not in requests[0]
def test_generate_model_records_extracts_json_surrounded_by_model_explanation() -> None:
content = "模型结果如下：\n```json\n" + json.dumps(
{
"items": [{
"instruction": "字段有什么作用？",
"output": "该字段用于唯一标识记录。",
}],
},
ensure_ascii=False,
) + "\n```\n生成完毕。"
client = httpx.Client(
transport=httpx.MockTransport(
lambda _: httpx.Response(
200,
json={"choices": [{"message": {"content": content}}]},
)
)
)
records = generate_model_records(
[{"id": "preview-explanation", "edited_content": "字段用于唯一标识记录。"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 0},
task_id="task-explanation",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=client,
)
assert records[0]["status"] == "valid"
assert records[0]["output"] == "该字段用于唯一标识记录。"
def test_generate_model_records_reports_token_truncation_instead_of_json_error() -> None:
client = httpx.Client(
transport=httpx.MockTransport(
lambda _: httpx.Response(
200,
json={
"choices": [{
"finish_reason": "length",
"message": {"content": ""},
}],
"output_sensitive": False,
},
)
)
)
records = generate_model_records(
[{"id": "preview-truncated", "edited_content": "来源正文"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 0},
task_id="task-truncated",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=client,
)
assert records[0]["status"] == "invalid"
assert "Token" in records[0]["error"]
assert "截断" in records[0]["error"]
def test_token_truncation_is_not_retried_even_when_json_looks_complete() -> None:
request_count = 0
content = json.dumps({
"items": [{
"instruction": "问题",
"output": "表面完整但服务端已声明截断。",
}],
}, ensure_ascii=False)
def handler(_: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
return httpx.Response(
200,
json={
"choices": [{
"finish_reason": "length",
"message": {"content": content},
}],
},
)
records = generate_model_records(
[{"id": "preview-length", "edited_content": "来源正文"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 5},
task_id="task-length",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert request_count == 1
assert records[0]["status"] == "invalid"
assert "finish_reason=length" in records[0]["error"]
def test_sensitive_model_response_is_not_retried_or_saved() -> None:
request_count = 0
def handler(_: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
return httpx.Response(
200,
json={
"choices": [{
"finish_reason": "stop",
"message": {"content": "{}"},
}],
"output_sensitive": True,
"base_resp": {"status_code": 1027, "status_msg": "output sensitive"},
},
)
records = generate_model_records(
[{"id": "preview-sensitive", "edited_content": "来源正文"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 5},
task_id="task-sensitive",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert request_count == 1
assert records[0]["status"] == "invalid"
assert "安全拦截" in records[0]["error"]
assert "1027" in records[0]["error"]
def test_empty_model_content_can_retry_then_succeed() -> None:
request_count = 0
def handler(_: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
if request_count == 1:
return httpx.Response(
200,
json={"choices": [{"finish_reason": "stop", "message": {"content": ""}}]},
)
return httpx.Response(
200,
json={
"choices": [{
"finish_reason": "stop",
"message": {
"content": json.dumps({
"items": [{
"instruction": "问题",
"output": "第二次请求返回了完整答案。",
}],
}, ensure_ascii=False),
},
}],
},
)
records = generate_model_records(
[{"id": "preview-empty-retry", "edited_content": "来源正文"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 1},
task_id="task-empty-retry",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert request_count == 2
assert records[0]["status"] == "valid"
def test_multiple_top_level_json_documents_are_rejected_as_ambiguous() -> None:
first = json.dumps({
"items": [{"instruction": "问题一", "output": "答案一"}],
}, ensure_ascii=False)
second = json.dumps({
"items": [{"instruction": "问题二", "output": "答案二"}],
}, ensure_ascii=False)
client = httpx.Client(
transport=httpx.MockTransport(
lambda _: httpx.Response(
200,
json={"choices": [{"message": {"content": f"{first}\n{second}"}}]},
)
)
)
records = generate_model_records(
[{"id": "preview-ambiguous", "edited_content": "来源正文"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 0},
task_id="task-ambiguous",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=client,
)
assert records[0]["status"] == "invalid"
assert "多个 JSON" in records[0]["error"]
def test_generate_model_records_builds_reasoning_output_with_think_tags() -> None:
def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
@@ -414,6 +752,112 @@ def test_generate_model_records_retries_short_batch_then_marks_it_invalid() -> N
assert "expected 10, got 1" in records[0]["error"]
def test_generate_model_records_does_not_retry_non_retryable_http_errors() -> None:
request_count = 0
def handler(_: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
return httpx.Response(401, json={"error": {"message": "unauthorized"}})
records = generate_model_records(
[{"id": "preview-auth", "edited_content": "来源内容"}],
model={
"api_url": "https://model.example/v1",
"online_model_name": "test-model",
"api_key": "invalid",
},
config={"generation_retries": 5},
task_id="task-auth",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert request_count == 1
assert records[0]["status"] == "invalid"
assert "401" in records[0]["error"]
@pytest.mark.parametrize("status_code", [408, 425, 429, 500])
def test_generate_model_records_retries_retryable_http_statuses(
status_code: int,
) -> None:
request_count = 0
def handler(_: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
if request_count == 1:
return httpx.Response(status_code)
return httpx.Response(
200,
json={
"choices": [{
"message": {
"content": json.dumps({
"items": [{
"instruction": "来源内容是什么？",
"output": "这是用于验证可重试错误的来源内容。",
}],
}, ensure_ascii=False),
},
}],
},
)
records = generate_model_records(
[{"id": "preview-retryable", "edited_content": "来源内容"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 1},
task_id="task-retryable",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert request_count == 2
assert records[0]["status"] == "valid"
def test_generate_model_records_retries_transient_network_errors() -> None:
request_count = 0
def handler(request: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
if request_count == 1:
raise httpx.ConnectError("temporary connection failure", request=request)
return httpx.Response(
200,
json={
"choices": [{
"message": {
"content": json.dumps({
"items": [{
"instruction": "网络恢复了吗？",
"output": "临时连接错误后，第二次模型请求已经成功。",
}],
}, ensure_ascii=False),
},
}],
},
)
records = generate_model_records(
[{"id": "preview-network", "edited_content": "网络重试来源"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"generation_retries": 1},
task_id="task-network",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert request_count == 2
assert records[0]["status"] == "valid"
@pytest.mark.parametrize("qa_pairs_per_item", [0, 51])
def test_generate_model_records_rejects_out_of_range_count(
qa_pairs_per_item: int,

34
backend/tests/test_data_process_migration.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
from pathlib import Path
from app.modules.data_process.schema_cli import _target_label
from app.modules.data_process.schema_cli import REQUIRED_TASK_COLUMNS, _target_label
def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
@@ -18,6 +18,24 @@ def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
assert "requires 001_platform_runtime.sql first" in sql
assert "supports only the current TEXT runtime schema" in sql
assert "generation_run_id" in sql
assert "results_confirmed BOOLEAN NOT NULL DEFAULT TRUE" in sql
assert "WHERE status <> 'completed' AND results_confirmed=TRUE" in sql
assert "ADD COLUMN IF NOT EXISTS workflow_step VARCHAR(20)" in sql
assert "ADD COLUMN IF NOT EXISTS preview_status VARCHAR(20)" in sql
assert "ADD COLUMN IF NOT EXISTS preview_progress NUMERIC(5,2)" in sql
assert "ADD COLUMN IF NOT EXISTS preview_run_id TEXT" in sql
assert "ADD COLUMN IF NOT EXISTS preview_failure_reason TEXT" in sql
assert "ADD COLUMN IF NOT EXISTS preview_total_files INTEGER" in sql
assert "ADD COLUMN IF NOT EXISTS preview_completed_files INTEGER" in sql
assert "data_process_workflow_backfill_ids" in sql
assert "ck_data_process_tasks_workflow_step" in sql
assert "ck_data_process_tasks_preview_status" in sql
assert "ck_data_process_tasks_preview_progress" in sql
assert "ck_data_process_tasks_preview_file_counts" in sql
for value in ("create", "model", "upload", "preview", "generate", "results"):
assert f"'{value}'" in sql
for value in ("idle", "queued", "running", "completed", "failed", "cancelled"):
assert f"'{value}'" in sql
assert "CREATE TABLE IF NOT EXISTS data_process_results" in sql
assert sql.count("BEGIN;") == 1
assert sql.rstrip().endswith("COMMIT;")
@@ -27,3 +45,17 @@ def test_schema_cli_target_label_never_contains_credentials() -> None:
label = _target_label("postgresql://secret-user:secret-password@db.example:5433/yg_ft")
assert label == "db.example:5433/yg_ft"
assert "secret" not in label
def test_schema_check_requires_current_runtime_columns() -> None:
assert REQUIRED_TASK_COLUMNS == (
"generation_run_id",
"results_confirmed",
"workflow_step",
"preview_status",
"preview_progress",
"preview_run_id",
"preview_failure_reason",
"preview_total_files",
"preview_completed_files",
)

111
backend/tests/test_data_process_store.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import inspect
import json
from collections.abc import Iterator
from contextlib import contextmanager
@@ -20,6 +21,14 @@ from app.modules.data_process.store import (
)
def test_preview_replace_sql_never_uses_untyped_null_placeholders() -> None:
source = inspect.getsource(DataProcessStore.replace_preview_items)
assert "%s IS NULL" not in source
assert "is_direct_build = preview_run_id is None" in source
assert "workflow_step=CASE WHEN %s THEN 'preview'" in source
class _Result:
def __init__(self, *, row: dict[str, Any] | None = None, rows: list[dict[str, Any]] | None = None):
self.row = row
@@ -146,6 +155,7 @@ class _PublishStore(DataProcessStore):
return {
"id": task_id,
"status": "completed",
"results_confirmed": True,
"description": "",
"config": self._task_config,
"output_dataset_id": train_dataset and train_dataset["id"],
@@ -454,12 +464,14 @@ class _LegacyRecoveryConnection:
record["preview_item_id"] = params[1]
return _Result()
if normalized.startswith("UPDATE data_process_tasks SET status='completed'"):
assert "workflow_step='results'" in normalized
self.task.update(
{
"status": "completed",
"progress": 100,
"output_dataset_id": params[0],
"output_count": params[1],
"workflow_step": "results",
}
)
return _Result()
@@ -496,6 +508,7 @@ class _StartGenerationConnection:
config=config,
output_dataset_id="dataset_train" if published_prepared else None,
output_count=28,
results_confirmed=published_prepared,
),
"generation_run_id": None,
}
@@ -512,6 +525,7 @@ class _StartGenerationConnection:
assert normalized.startswith("UPDATE data_process_tasks SET config=%s, status='running'")
assert "output_dataset_id=NULL" in normalized
assert "output_count=0" in normalized
assert "results_confirmed=FALSE" in normalized
self.task.update(
{
"config": params[0],
@@ -526,6 +540,7 @@ class _StartGenerationConnection:
"duplicate_count": 0,
"error_count": 0,
"generation_run_id": params[2],
"results_confirmed": False,
"updated_at": params[3],
}
)
@@ -713,6 +728,7 @@ def test_start_generation_clears_previous_output_count() -> None:
assert task["status"] == "running"
assert task["output_count"] == 0
assert task["results_confirmed"] is False
assert conn.results == []
@@ -737,6 +753,7 @@ def test_prepared_published_task_survives_generation_preflight_failure() -> None
assert conn.task["status"] == "completed"
assert conn.task["output_dataset_id"] == "dataset_train"
assert conn.task["output_count"] == 28
assert conn.task["results_confirmed"] is True
assert "_regeneration_prepared" in conn.task["config"]
assert conn.results == [{"id": "old-result"}]
@@ -752,6 +769,7 @@ def test_legacy_aborted_regeneration_recovers_results_and_published_state() -> N
assert conn.task["progress"] == 100
assert conn.task["output_dataset_id"] == "dataset_train"
assert conn.task["output_count"] == 2
assert conn.task["workflow_step"] == "results"
assert conn.task["started_at"] is None
assert conn.task["completed_at"] is None
assert [item["id"] for item in conn.results] == ["result_train", "result_test"]
@@ -1185,3 +1203,96 @@ def test_source_storage_descriptor_rejects_unowned_or_unsupported_references(
"dpt_task",
"dpsf_source",
)
class _LifecycleConnection:
def __init__(self) -> None:
self.task: dict[str, Any] = {
"id": "task-lifecycle",
"status": "running",
"generation_run_id": "generation-active",
"workflow_step": "generate",
"preview_status": "running",
"preview_progress": Decimal("40.00"),
"preview_run_id": "preview-active",
"preview_failure_reason": None,
"preview_total_files": 5,
"preview_completed_files": 2,
"deleted_at": None,
"deleted_by": None,
}
self.last_update_sql = ""
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if params is not None:
assert normalized.count("%s") == len(params)
if normalized.startswith("SELECT * FROM data_process_tasks"):
row = None if self.task["deleted_at"] is not None else dict(self.task)
return _Result(row=row)
if normalized.startswith("UPDATE data_process_tasks SET workflow_step="):
self.last_update_sql = normalized
workflow_step, updated_at, task_id = params
assert task_id == self.task["id"]
self.task.update(workflow_step=workflow_step, updated_at=updated_at)
return _Result(row=dict(self.task))
if normalized.startswith("UPDATE data_process_tasks SET status=CASE"):
self.last_update_sql = normalized
deleted_at, deleted_by, updated_at, task_id = params
assert task_id == self.task["id"]
self.task.update(
status="stopped",
generation_run_id=None,
preview_status="cancelled",
preview_run_id=None,
deleted_at=deleted_at,
deleted_by=deleted_by,
updated_at=updated_at,
)
return _Result()
if normalized.startswith("SELECT status, generation_run_id"):
row = None if self.task["deleted_at"] is not None else dict(self.task)
return _Result(row=row)
if normalized.startswith("SELECT preview_status, preview_run_id"):
row = None if self.task["deleted_at"] is not None else dict(self.task)
return _Result(row=row)
raise AssertionError(f"unexpected SQL: {normalized}")
class _LifecycleStore(DataProcessStore):
def __init__(self, conn: _LifecycleConnection) -> None:
self._conn = conn
@contextmanager
def connect(self) -> Iterator[_LifecycleConnection]:
yield self._conn
def test_workflow_step_update_does_not_invalidate_active_runs() -> None:
conn = _LifecycleConnection()
task = _LifecycleStore(conn).update_workflow_step("task-lifecycle", "results")
assert task["workflow_step"] == "results"
assert task["status"] == "running"
assert task["generation_run_id"] == "generation-active"
assert task["preview_status"] == "running"
assert task["preview_run_id"] == "preview-active"
assert "generation_run_id" not in conn.last_update_sql
assert "preview_run_id" not in conn.last_update_sql
def test_delete_atomically_invalidates_generation_and_preview_runs() -> None:
conn = _LifecycleConnection()
store = _LifecycleStore(conn)
store.delete_task("task-lifecycle", deleted_by="user-1")
assert conn.task["status"] == "stopped"
assert conn.task["generation_run_id"] is None
assert conn.task["preview_status"] == "cancelled"
assert conn.task["preview_run_id"] is None
assert conn.task["deleted_by"] == "user-1"
assert conn.task["deleted_at"] is not None
assert store.generation_is_running("task-lifecycle", "generation-active") is False
assert store.preview_is_running("task-lifecycle", "preview-active") is False