3492 lines
126 KiB
Python
3492 lines
126 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
from copy import deepcopy
|
||
from io import BytesIO
|
||
from pathlib import Path
|
||
from threading import Barrier, Lock
|
||
from typing import Any
|
||
|
||
import psycopg
|
||
import pytest
|
||
from docx import Document as WordDocument
|
||
from fastapi import FastAPI, HTTPException
|
||
from fastapi.testclient import TestClient
|
||
from openpyxl import Workbook
|
||
|
||
from app.api.v1.endpoints import data_process as data_process_endpoint
|
||
from app.api.v1.endpoints.data_process import router
|
||
from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text
|
||
from app.modules.data_process.storage import (
|
||
DataProcessStorageError,
|
||
LocalDataProcessStorage,
|
||
get_data_process_storage,
|
||
)
|
||
from app.modules.data_process.store import (
|
||
InvalidStateError,
|
||
NotFoundError,
|
||
get_data_process_store,
|
||
repeat_task_id,
|
||
)
|
||
|
||
|
||
class FakeDataProcessStore:
|
||
"""接口测试专用内存实现，确保测试不会连接或迁移真实数据库。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.tasks: dict[str, dict[str, Any]] = {}
|
||
self.sources: dict[str, list[dict[str, Any]]] = {}
|
||
self.previews: dict[str, list[dict[str, Any]]] = {}
|
||
self.results: dict[str, list[dict[str, Any]]] = {}
|
||
self.datasets: dict[str, dict[str, Any]] = {}
|
||
self.models: dict[str, dict[str, Any]] = {}
|
||
self.regeneration_prepared: set[str] = set()
|
||
self.repeat_requests: dict[tuple[str, str], str] = {}
|
||
self.sequence = 0
|
||
|
||
def _id(self, prefix: str) -> str:
|
||
self.sequence += 1
|
||
return f"{prefix}_{self.sequence}"
|
||
|
||
def list_tasks(self, *, page: int, page_size: int, **filters: Any) -> dict[str, Any]:
|
||
items = [
|
||
{
|
||
**item,
|
||
"source_file_count": len(self.sources.get(str(item["id"]), [])),
|
||
}
|
||
for item in self.tasks.values()
|
||
]
|
||
for field in ("status", "process_type", "tenant_id", "project_id"):
|
||
if filters.get(field):
|
||
items = [item for item in items if item.get(field) == filters[field]]
|
||
keyword = filters.get("keyword")
|
||
if keyword:
|
||
items = [item for item in items if keyword in item["name"]]
|
||
return {
|
||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||
"total": len(items),
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
def create_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
task_id = self._id("dpt")
|
||
task = {
|
||
"id": task_id,
|
||
**deepcopy(payload),
|
||
"status": "pending",
|
||
"progress": 0,
|
||
"input_count": 0,
|
||
"output_count": 0,
|
||
"filtered_count": 0,
|
||
"duplicate_count": 0,
|
||
"error_count": 0,
|
||
"failure_reason": None,
|
||
"output_dataset_id": None,
|
||
"results_confirmed": False,
|
||
"workflow_step": "create",
|
||
"preview_status": "idle",
|
||
"preview_progress": 0,
|
||
"preview_run_id": None,
|
||
"preview_failure_reason": None,
|
||
"preview_total_files": 0,
|
||
"preview_completed_files": 0,
|
||
}
|
||
self.tasks[task_id] = task
|
||
self.sources[task_id] = []
|
||
self.previews[task_id] = []
|
||
self.results[task_id] = []
|
||
return deepcopy(task)
|
||
|
||
def get_task(self, task_id: str) -> dict[str, Any]:
|
||
if task_id not in self.tasks:
|
||
raise NotFoundError("data process task not found")
|
||
task = deepcopy(self.tasks[task_id])
|
||
split_order = {"train": 0, "val": 1, "test": 2}
|
||
task["output_datasets"] = sorted(
|
||
(
|
||
deepcopy(dataset)
|
||
for dataset in self.datasets.values()
|
||
if dataset.get("source_task_id") == task_id
|
||
and dataset.get("deleted_at") is None
|
||
),
|
||
key=lambda dataset: split_order.get(str(dataset.get("type")), 3),
|
||
)
|
||
return task
|
||
|
||
def recover_legacy_aborted_regeneration(self, task_id: str) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
return {"recovered": False, "result_count": 0}
|
||
|
||
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
self.tasks[task_id].update(deepcopy(payload))
|
||
return self.get_task(task_id)
|
||
|
||
def update_workflow_step(self, task_id: str, workflow_step: str) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
self.tasks[task_id]["workflow_step"] = workflow_step
|
||
return self.get_task(task_id)
|
||
|
||
def prepare_regeneration(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
task = self.tasks.get(task_id)
|
||
if task is None:
|
||
raise NotFoundError("data process task not found")
|
||
if task["status"] == "running":
|
||
raise InvalidStateError("running task cannot be prepared for regeneration")
|
||
if payload["expected_updated_at"] != task.get("updated_at"):
|
||
raise InvalidStateError("data process task was modified by another request")
|
||
if payload["process_type"] != task["process_type"]:
|
||
raise InvalidStateError("process_type cannot be changed during regeneration")
|
||
published_outputs_preserved = bool(task.get("output_dataset_id")) or any(
|
||
dataset.get("source_task_id") == task_id
|
||
for dataset in self.datasets.values()
|
||
)
|
||
task.update(
|
||
{
|
||
"name": payload["name"],
|
||
"description": payload["description"],
|
||
"config": deepcopy(payload["config"]),
|
||
"updated_at": "2026-07-25T20:00:00Z",
|
||
}
|
||
)
|
||
self.regeneration_prepared.add(task_id)
|
||
return {
|
||
"task": deepcopy(task),
|
||
"preview_invalidated": False,
|
||
"published_outputs_preserved": published_outputs_preserved,
|
||
}
|
||
|
||
def _repeat_response(
|
||
self,
|
||
source_task_id: str,
|
||
repeated_task_id: str,
|
||
*,
|
||
created: bool,
|
||
) -> dict[str, Any]:
|
||
task = self.get_task(repeated_task_id)
|
||
task["source_file_count"] = len(self.sources[repeated_task_id])
|
||
task["preview_count"] = len(self.previews[repeated_task_id])
|
||
return {
|
||
"task": task,
|
||
"source_task_id": source_task_id,
|
||
"created": created,
|
||
"copied_source_file_count": len(self.sources[repeated_task_id]),
|
||
"copied_preview_count": len(self.previews[repeated_task_id]),
|
||
}
|
||
|
||
def find_repeated_task(
|
||
self,
|
||
source_task_id: str,
|
||
request_id: str,
|
||
) -> dict[str, Any] | None:
|
||
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
|
||
if repeated_task_id is None:
|
||
return None
|
||
return self._repeat_response(
|
||
source_task_id,
|
||
repeated_task_id,
|
||
created=False,
|
||
)
|
||
|
||
def repeat_task(
|
||
self,
|
||
source_task_id: str,
|
||
*,
|
||
expected_updated_at: str,
|
||
request_id: str,
|
||
file_copies: dict[str, dict[str, str]],
|
||
) -> dict[str, Any]:
|
||
existing = self.find_repeated_task(source_task_id, request_id)
|
||
if existing is not None:
|
||
return existing
|
||
source_task = self.get_task(source_task_id)
|
||
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
|
||
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
|
||
if source_task.get("updated_at") != expected_updated_at:
|
||
raise InvalidStateError("源任务已被其他操作修改，请刷新后重试")
|
||
source_files = self.sources[source_task_id]
|
||
if set(file_copies) != {str(item["id"]) for item in source_files}:
|
||
raise InvalidStateError("源文件快照已变化，请刷新后重试")
|
||
if not self.previews[source_task_id]:
|
||
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
|
||
|
||
repeated_task_id = repeat_task_id(source_task_id, request_id)
|
||
suffix = f"（再次生成-{repeated_task_id[-6:]}）"
|
||
task = {
|
||
**deepcopy(source_task),
|
||
"id": repeated_task_id,
|
||
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
|
||
"status": "pending",
|
||
"progress": 20,
|
||
"output_dataset_id": None,
|
||
"output_datasets": [],
|
||
"output_count": 0,
|
||
"filtered_count": 0,
|
||
"duplicate_count": 0,
|
||
"error_count": 0,
|
||
"failure_reason": None,
|
||
"generation_run_id": None,
|
||
"results_confirmed": False,
|
||
"workflow_step": "preview",
|
||
"preview_status": "completed",
|
||
"preview_progress": 100,
|
||
"preview_run_id": None,
|
||
"preview_failure_reason": None,
|
||
"preview_total_files": len(source_files),
|
||
"preview_completed_files": len(source_files),
|
||
"started_at": None,
|
||
"completed_at": None,
|
||
}
|
||
self.tasks[repeated_task_id] = task
|
||
self.sources[repeated_task_id] = []
|
||
file_id_map: dict[str, str] = {}
|
||
for source in source_files:
|
||
old_file_id = str(source["id"])
|
||
copy = file_copies[old_file_id]
|
||
file_id_map[old_file_id] = copy["id"]
|
||
self.sources[repeated_task_id].append(
|
||
{
|
||
**deepcopy(source),
|
||
"id": copy["id"],
|
||
"task_id": repeated_task_id,
|
||
"storage_object_id": copy["storage_object_id"],
|
||
}
|
||
)
|
||
self.previews[repeated_task_id] = [
|
||
{
|
||
**deepcopy(item),
|
||
"id": self._id("dpp"),
|
||
"task_id": repeated_task_id,
|
||
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
|
||
if item.get("source_file_id")
|
||
else None,
|
||
}
|
||
for item in self.previews[source_task_id]
|
||
]
|
||
self.results[repeated_task_id] = []
|
||
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
|
||
return self._repeat_response(
|
||
source_task_id,
|
||
repeated_task_id,
|
||
created=True,
|
||
)
|
||
|
||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||
self.get_task(task_id)
|
||
del self.tasks[task_id]
|
||
|
||
def list_source_files(self, task_id: str) -> list[dict[str, Any]]:
|
||
self.get_task(task_id)
|
||
return [
|
||
{key: value for key, value in item.items() if key != "content"}
|
||
for item in self.sources[task_id]
|
||
]
|
||
|
||
def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
values = deepcopy(payload)
|
||
source_id = str(values.pop("id", None) or self._id("dpsf"))
|
||
storage_object_id = str(
|
||
values.pop("storage_object_id", None)
|
||
or f"db://data-process/{task_id}/{source_id}/v1"
|
||
)
|
||
raw_size = int(values.pop("raw_size"))
|
||
metadata = deepcopy(values.pop("metadata", {}))
|
||
metadata.setdefault(
|
||
"storage_backend",
|
||
"local" if storage_object_id.startswith("local://data-process/") else "database",
|
||
)
|
||
source = {
|
||
"id": source_id,
|
||
"task_id": task_id,
|
||
"version_no": 1,
|
||
"storage_object_id": storage_object_id,
|
||
"size_bytes": raw_size,
|
||
"metadata": metadata,
|
||
**values,
|
||
}
|
||
self.sources[task_id].append(source)
|
||
self.tasks[task_id]["input_count"] += payload["record_count"]
|
||
return {key: value for key, value in deepcopy(source).items() if key != "content"}
|
||
|
||
def add_source_files(
|
||
self, task_id: str, files: list[dict[str, Any]]
|
||
) -> list[dict[str, Any]]:
|
||
# 先验证整个批次，模拟数据库事务的 all-or-nothing 语义。
|
||
checksums = {item["checksum_sha256"] for item in self.sources.get(task_id, [])}
|
||
incoming: set[str] = set()
|
||
for payload in files:
|
||
checksum = payload["checksum_sha256"]
|
||
if checksum in checksums or checksum in incoming:
|
||
raise ValueError("the same source file content is already attached to this task")
|
||
incoming.add(checksum)
|
||
created = [self.add_source_file(task_id, **payload) for payload in files]
|
||
self.results[task_id] = []
|
||
self.tasks[task_id].update(
|
||
{
|
||
"status": "pending",
|
||
"progress": 20 if self.previews[task_id] else 0,
|
||
"output_count": 0,
|
||
"filtered_count": 0,
|
||
"duplicate_count": 0,
|
||
"error_count": 0,
|
||
"failure_reason": None,
|
||
"generation_run_id": None,
|
||
"workflow_step": "upload",
|
||
"preview_status": "idle",
|
||
"preview_progress": 0,
|
||
"preview_run_id": None,
|
||
"preview_failure_reason": None,
|
||
"preview_total_files": 0,
|
||
"preview_completed_files": 0,
|
||
"started_at": None,
|
||
"completed_at": None,
|
||
}
|
||
)
|
||
return created
|
||
|
||
def get_source_file(
|
||
self, task_id: str, file_id: str, *, include_content: bool = True
|
||
) -> dict[str, Any]:
|
||
source = next(
|
||
(item for item in self.sources.get(task_id, []) if item["id"] == file_id),
|
||
None,
|
||
)
|
||
if not source:
|
||
raise NotFoundError("source file not found")
|
||
result = deepcopy(source)
|
||
if not include_content:
|
||
result.pop("content", None)
|
||
return result
|
||
|
||
def source_content_window(
|
||
self, task_id: str, file_id: str, offset: int, limit: int
|
||
) -> dict[str, Any]:
|
||
source = self.get_source_file(task_id, file_id)
|
||
content = source.pop("content")
|
||
return {
|
||
"file": source,
|
||
"content": content[offset : offset + limit],
|
||
"offset": offset,
|
||
"limit": limit,
|
||
"total_chars": len(content),
|
||
"has_more": offset + limit < len(content),
|
||
}
|
||
|
||
def source_content_lines(
|
||
self, task_id: str, file_id: str, start_line: int, line_count: int
|
||
) -> dict[str, Any]:
|
||
source = self.get_source_file(task_id, file_id)
|
||
lines = source.pop("content").splitlines(keepends=True)
|
||
selected = lines[start_line - 1 : start_line - 1 + line_count]
|
||
return {
|
||
"file": source,
|
||
"content": "".join(selected),
|
||
"start_line": start_line,
|
||
"end_line": start_line - 1 + len(selected),
|
||
"line_count": len(selected),
|
||
"total_lines": len(lines),
|
||
"has_more": start_line - 1 + len(selected) < len(lines),
|
||
}
|
||
|
||
def delete_source_file(self, task_id: str, file_id: str) -> None:
|
||
self.get_source_file(task_id, file_id)
|
||
self.sources[task_id] = [item for item in self.sources[task_id] if item["id"] != file_id]
|
||
self.previews[task_id] = [
|
||
item for item in self.previews[task_id] if item["source_file_id"] != file_id
|
||
]
|
||
self.results[task_id] = []
|
||
self.tasks[task_id].update(
|
||
workflow_step="upload",
|
||
preview_status="idle",
|
||
preview_progress=0,
|
||
preview_run_id=None,
|
||
preview_failure_reason=None,
|
||
preview_total_files=0,
|
||
preview_completed_files=0,
|
||
)
|
||
|
||
def replace_preview_items(
|
||
self,
|
||
task_id: str,
|
||
items: list[dict[str, Any]],
|
||
*,
|
||
source_file_ids: list[str] | None = None,
|
||
preview_run_id: str | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
if preview_run_id is not None and not self.preview_is_running(task_id, preview_run_id):
|
||
raise InvalidStateError("preview run is no longer active")
|
||
created = [
|
||
{"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items
|
||
]
|
||
if source_file_ids is None:
|
||
self.previews[task_id] = created
|
||
else:
|
||
selected = set(source_file_ids)
|
||
self.previews[task_id] = [
|
||
item
|
||
for item in self.previews[task_id]
|
||
if item["source_file_id"] not in selected
|
||
] + created
|
||
self.results[task_id] = []
|
||
self.tasks[task_id]["progress"] = 20
|
||
if preview_run_id is None:
|
||
file_count = len(source_file_ids or {item["source_file_id"] for item in created})
|
||
self.tasks[task_id].update(
|
||
workflow_step="preview",
|
||
preview_status="completed",
|
||
preview_progress=100,
|
||
preview_run_id=None,
|
||
preview_failure_reason=None,
|
||
preview_total_files=file_count,
|
||
preview_completed_files=file_count,
|
||
)
|
||
return deepcopy(created)
|
||
|
||
def start_preview(
|
||
self,
|
||
task_id: str,
|
||
*,
|
||
source_file_ids: list[str] | None = None,
|
||
) -> tuple[dict[str, Any], list[str]]:
|
||
task = self.tasks[task_id]
|
||
if task.get("preview_status") in {"queued", "running"}:
|
||
raise InvalidStateError("task cannot be edited while preview is running")
|
||
selected_ids = source_file_ids or [str(item["id"]) for item in self.sources[task_id]]
|
||
found = {str(item["id"]) for item in self.sources[task_id]}
|
||
missing = set(selected_ids) - found
|
||
if missing:
|
||
raise NotFoundError(f"source files not found: {', '.join(sorted(missing))}")
|
||
if not selected_ids:
|
||
raise InvalidStateError("at least one source file is required")
|
||
run_id = self._id("dpprun")
|
||
self.results[task_id] = []
|
||
task.update(
|
||
status="pending",
|
||
workflow_step="upload",
|
||
preview_status="queued",
|
||
preview_progress=0,
|
||
preview_run_id=run_id,
|
||
preview_failure_reason=None,
|
||
preview_total_files=len(selected_ids),
|
||
preview_completed_files=0,
|
||
results_confirmed=False,
|
||
)
|
||
return self.get_task(task_id), list(selected_ids)
|
||
|
||
def mark_preview_running(self, task_id: str, preview_run_id: str) -> bool:
|
||
task = self.tasks.get(task_id)
|
||
if not task or task.get("preview_status") != "queued" or task.get("preview_run_id") != preview_run_id:
|
||
return False
|
||
task["preview_status"] = "running"
|
||
return True
|
||
|
||
def preview_is_running(self, task_id: str, preview_run_id: str) -> bool:
|
||
task = self.tasks.get(task_id)
|
||
return bool(
|
||
task
|
||
and task.get("preview_status") in {"queued", "running"}
|
||
and task.get("preview_run_id") == preview_run_id
|
||
)
|
||
|
||
def update_preview_progress(
|
||
self,
|
||
task_id: str,
|
||
preview_run_id: str,
|
||
completed_files: int,
|
||
total_files: int,
|
||
) -> bool:
|
||
if not self.preview_is_running(task_id, preview_run_id):
|
||
return False
|
||
self.tasks[task_id]["preview_completed_files"] = completed_files
|
||
self.tasks[task_id]["preview_progress"] = completed_files / max(1, total_files) * 100
|
||
return True
|
||
|
||
def complete_preview(self, task_id: str, preview_run_id: str) -> bool:
|
||
if not self.preview_is_running(task_id, preview_run_id):
|
||
return False
|
||
task = self.tasks[task_id]
|
||
task.update(
|
||
workflow_step="preview",
|
||
preview_status="completed",
|
||
preview_progress=100,
|
||
preview_run_id=None,
|
||
preview_failure_reason=None,
|
||
preview_completed_files=task["preview_total_files"],
|
||
)
|
||
return True
|
||
|
||
def mark_preview_failed(
|
||
self,
|
||
task_id: str,
|
||
reason: str,
|
||
*,
|
||
preview_run_id: str,
|
||
) -> bool:
|
||
if not self.preview_is_running(task_id, preview_run_id):
|
||
return False
|
||
self.tasks[task_id].update(
|
||
preview_status="failed",
|
||
preview_run_id=None,
|
||
preview_failure_reason=reason,
|
||
)
|
||
return True
|
||
|
||
def preview_progress(self, task_id: str) -> dict[str, Any]:
|
||
task = self.get_task(task_id)
|
||
return {
|
||
"task_id": task_id,
|
||
"workflow_step": task["workflow_step"],
|
||
"preview_status": task["preview_status"],
|
||
"preview_progress": task["preview_progress"],
|
||
"preview_run_id": task["preview_run_id"],
|
||
"preview_failure_reason": task["preview_failure_reason"],
|
||
"preview_total_files": task["preview_total_files"],
|
||
"preview_completed_files": task["preview_completed_files"],
|
||
}
|
||
|
||
def list_preview_items(
|
||
self,
|
||
task_id: str,
|
||
*,
|
||
page: int,
|
||
page_size: int,
|
||
source_file_id: str | None = None,
|
||
keyword: str | None = None,
|
||
) -> dict[str, Any]:
|
||
items = self.previews[task_id]
|
||
if source_file_id:
|
||
items = [item for item in items if item["source_file_id"] == source_file_id]
|
||
if keyword:
|
||
items = [item for item in items if keyword in item["edited_content"]]
|
||
return {
|
||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||
"total": len(items),
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]:
|
||
item = next(
|
||
(item for item in self.previews.get(task_id, []) if item["id"] == preview_id),
|
||
None,
|
||
)
|
||
if not item:
|
||
raise NotFoundError("preview item not found")
|
||
return deepcopy(item)
|
||
|
||
def create_preview_item(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
item = {"id": self._id("dpp"), "task_id": task_id, **deepcopy(payload)}
|
||
self.previews[task_id].append(item)
|
||
self.results[task_id] = []
|
||
return deepcopy(item)
|
||
|
||
def update_preview_item(
|
||
self, task_id: str, preview_id: str, payload: dict[str, Any]
|
||
) -> dict[str, Any]:
|
||
item = next(
|
||
(item for item in self.previews[task_id] if item["id"] == preview_id),
|
||
None,
|
||
)
|
||
if not item:
|
||
raise NotFoundError("preview item not found")
|
||
item.update(deepcopy(payload))
|
||
self.results[task_id] = []
|
||
return deepcopy(item)
|
||
|
||
def delete_preview_item(self, task_id: str, preview_id: str) -> None:
|
||
before = len(self.previews[task_id])
|
||
self.previews[task_id] = [
|
||
item for item in self.previews[task_id] if item["id"] != preview_id
|
||
]
|
||
if len(self.previews[task_id]) == before:
|
||
raise NotFoundError("preview item not found")
|
||
|
||
def start_generation(self, task_id: str, *, replace_existing: bool) -> dict[str, Any]:
|
||
if not self.previews[task_id]:
|
||
raise InvalidStateError("preview must be built before generation")
|
||
task = self.tasks[task_id]
|
||
if task.get("output_dataset_id") and task_id not in self.regeneration_prepared:
|
||
raise InvalidStateError("published task cannot be regenerated")
|
||
if replace_existing:
|
||
self.results[task_id] = []
|
||
task.update(
|
||
status="running",
|
||
progress=30,
|
||
output_dataset_id=None,
|
||
output_count=0,
|
||
results_confirmed=False,
|
||
workflow_step="generate",
|
||
generation_run_id=self._id("dprun"),
|
||
)
|
||
self.regeneration_prepared.discard(task_id)
|
||
return self.get_task(task_id)
|
||
|
||
def generation_is_running(self, task_id: str, generation_run_id: str) -> bool:
|
||
task = self.tasks.get(task_id)
|
||
return (
|
||
bool(task)
|
||
and task["status"] == "running"
|
||
and task.get("generation_run_id") == generation_run_id
|
||
)
|
||
|
||
def update_generation_progress(
|
||
self,
|
||
task_id: str,
|
||
generation_run_id: str,
|
||
processed_count: int,
|
||
total_count: int,
|
||
) -> bool:
|
||
if not self.generation_is_running(task_id, generation_run_id):
|
||
return False
|
||
self.tasks[task_id]["progress"] = min(
|
||
95,
|
||
30 + processed_count / max(1, total_count) * 65,
|
||
)
|
||
return True
|
||
|
||
def complete_generation(
|
||
self,
|
||
task_id: str,
|
||
results: list[dict[str, Any]],
|
||
*,
|
||
generation_run_id: str,
|
||
**counts: Any,
|
||
) -> dict[str, Any]:
|
||
if not self.generation_is_running(task_id, generation_run_id):
|
||
return self.get_task(task_id)
|
||
self.results[task_id] = deepcopy(results)
|
||
self.tasks[task_id].update(
|
||
status="completed",
|
||
progress=100,
|
||
output_count=len(results),
|
||
results_confirmed=False,
|
||
generation_run_id=None,
|
||
**counts,
|
||
)
|
||
return self.get_task(task_id)
|
||
|
||
def mark_failed(
|
||
self, task_id: str, reason: str, *, generation_run_id: str
|
||
) -> dict[str, Any]:
|
||
if self.generation_is_running(task_id, generation_run_id):
|
||
self.tasks[task_id].update(
|
||
status="failed",
|
||
failure_reason=reason,
|
||
generation_run_id=None,
|
||
)
|
||
return self.get_task(task_id)
|
||
|
||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||
if self.tasks[task_id]["status"] != "running":
|
||
raise InvalidStateError("only a running task can be stopped")
|
||
self.tasks[task_id].update(status="stopped", generation_run_id=None)
|
||
return self.get_task(task_id)
|
||
|
||
def progress(self, task_id: str) -> dict[str, Any]:
|
||
task = self.get_task(task_id)
|
||
result = {key: task.get(key) for key in (
|
||
"status", "progress", "input_count", "output_count",
|
||
"filtered_count", "duplicate_count", "error_count", "failure_reason",
|
||
"results_confirmed",
|
||
)}
|
||
result["task_id"] = task["id"]
|
||
return result
|
||
|
||
def list_results(
|
||
self,
|
||
task_id: str,
|
||
*,
|
||
page: int,
|
||
page_size: int,
|
||
status: str | None = None,
|
||
split: str | None = None,
|
||
keyword: str | None = None,
|
||
) -> dict[str, Any]:
|
||
items = self.results[task_id]
|
||
if status:
|
||
items = [item for item in items if item["status"] == status]
|
||
if split:
|
||
items = [item for item in items if item["split"] == split]
|
||
if keyword:
|
||
items = [
|
||
item
|
||
for item in items
|
||
if any(keyword in item[field] for field in ("instruction", "input", "output"))
|
||
]
|
||
return {
|
||
"items": deepcopy(items),
|
||
"total": len(items),
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
def update_result(
|
||
self, task_id: str, result_id: str, payload: dict[str, Any]
|
||
) -> dict[str, Any]:
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
for field in ("instruction", "input", "output", "quality_score"):
|
||
if field in payload:
|
||
item[field] = deepcopy(payload[field])
|
||
hard_valid = bool(item["instruction"].strip() and item["output"].strip())
|
||
quality_valid = bool((item.get("quality_score") or {}).get("is_valid", hard_valid))
|
||
changed = any(
|
||
item[field] != item[f"original_{field}"]
|
||
for field in ("instruction", "input", "output")
|
||
)
|
||
item["status"] = (
|
||
"invalid"
|
||
if not hard_valid or not quality_valid
|
||
else "modified" if changed else "valid"
|
||
)
|
||
self.tasks[task_id]["error_count"] = sum(
|
||
result["status"] == "invalid" for result in self.results[task_id]
|
||
)
|
||
return deepcopy(item)
|
||
|
||
def get_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
return deepcopy(item)
|
||
|
||
def restore_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
for field in ("instruction", "input", "output"):
|
||
item[field] = item[f"original_{field}"]
|
||
item["status"] = "valid"
|
||
return deepcopy(item)
|
||
|
||
def get_generation_model(self, model_id: str) -> dict[str, Any]:
|
||
model = self.models.get(model_id)
|
||
if not model:
|
||
raise NotFoundError("generation model not found")
|
||
return deepcopy(model)
|
||
|
||
def replace_generated_result(
|
||
self,
|
||
task_id: str,
|
||
result_id: str,
|
||
replacement: dict[str, Any],
|
||
*,
|
||
expected_updated_at: str,
|
||
) -> dict[str, Any]:
|
||
task = self.tasks[task_id]
|
||
if task["status"] != "completed" or task.get("workflow_step") != "results":
|
||
raise InvalidStateError("task is not editing generation results")
|
||
if task.get("results_confirmed") or task.get("output_dataset_id"):
|
||
raise InvalidStateError("confirmed or published results cannot be regenerated")
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
if item["status"] != "invalid":
|
||
raise InvalidStateError("only an invalid result can be regenerated")
|
||
if expected_updated_at != item.get("updated_at"):
|
||
raise InvalidStateError("data process result was modified by another request")
|
||
for field in ("instruction", "input", "output"):
|
||
item[field] = replacement[field]
|
||
item[f"original_{field}"] = replacement[field]
|
||
item.update(
|
||
status=replacement["status"],
|
||
error=replacement.get("error"),
|
||
quality_score=deepcopy(replacement["quality_score"]),
|
||
updated_at="2026-07-27T22:00:00Z",
|
||
)
|
||
task["error_count"] = sum(
|
||
result["status"] == "invalid" for result in self.results[task_id]
|
||
)
|
||
return deepcopy(item)
|
||
|
||
def confirm_results(self, task_id: str) -> dict[str, Any]:
|
||
task = self.tasks[task_id]
|
||
if task["status"] != "completed":
|
||
raise InvalidStateError("only a completed task can confirm results")
|
||
if task.get("workflow_step") != "results":
|
||
raise InvalidStateError("workflow must be on results before confirmation")
|
||
invalid_count = sum(
|
||
1
|
||
for item in self.results[task_id]
|
||
if item["status"] == "invalid"
|
||
or not item["instruction"].strip()
|
||
or not item["output"].strip()
|
||
)
|
||
if invalid_count:
|
||
raise InvalidStateError(f"task contains {invalid_count} invalid results")
|
||
if not self.results[task_id]:
|
||
raise InvalidStateError("task has no results to confirm")
|
||
task["results_confirmed"] = True
|
||
return self.get_task(task_id)
|
||
|
||
def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
task = self.tasks[task_id]
|
||
if task_id in self.regeneration_prepared:
|
||
raise InvalidStateError("regeneration must start and complete before publishing")
|
||
published = [
|
||
dataset
|
||
for dataset in self.datasets.values()
|
||
if dataset.get("source_task_id") == task_id
|
||
and dataset.get("deleted_at") is None
|
||
]
|
||
if task.get("output_dataset_id"):
|
||
train_dataset = self.datasets[task["output_dataset_id"]]
|
||
return {
|
||
"dataset": deepcopy(train_dataset),
|
||
"datasets": deepcopy(published),
|
||
"output_datasets": deepcopy(published),
|
||
"created": False,
|
||
}
|
||
if task["status"] != "completed":
|
||
raise InvalidStateError("only a completed task can be published")
|
||
if not task.get("results_confirmed"):
|
||
raise InvalidStateError("results must be confirmed before publishing")
|
||
split_specs = (
|
||
("train", "训练集"),
|
||
("val", "验证集"),
|
||
("test", "测试集"),
|
||
)
|
||
for dataset_type, label in split_specs:
|
||
dataset_id = self._id("dataset")
|
||
self.datasets[dataset_id] = {
|
||
"id": dataset_id,
|
||
"name": f"{payload['dataset_name']}-{label}",
|
||
"type": dataset_type,
|
||
"count": len(self.results[task_id]),
|
||
"source": "task",
|
||
"task_id": task_id,
|
||
"source_task_id": task_id,
|
||
"deleted_at": None,
|
||
}
|
||
published = [
|
||
dataset
|
||
for dataset in self.datasets.values()
|
||
if dataset.get("source_task_id") == task_id
|
||
and dataset.get("deleted_at") is None
|
||
]
|
||
train_dataset = next(dataset for dataset in published if dataset["type"] == "train")
|
||
task["output_dataset_id"] = train_dataset["id"]
|
||
return {
|
||
"dataset": deepcopy(train_dataset),
|
||
"datasets": deepcopy(published),
|
||
"output_datasets": deepcopy(published),
|
||
"created": True,
|
||
}
|
||
|
||
|
||
def make_client(
|
||
tmp_path: Path,
|
||
) -> tuple[TestClient, FakeDataProcessStore, LocalDataProcessStorage]:
|
||
store = FakeDataProcessStore()
|
||
storage = LocalDataProcessStorage(tmp_path / "data-process")
|
||
app = FastAPI()
|
||
app.include_router(router, prefix="/modelTF")
|
||
app.dependency_overrides[get_data_process_store] = lambda: store
|
||
app.dependency_overrides[get_data_process_storage] = lambda: storage
|
||
return TestClient(app), store, storage
|
||
|
||
|
||
def _stored_files(storage: LocalDataProcessStorage) -> list[Path]:
|
||
return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()]
|
||
|
||
|
||
def _minimal_pdf_pages(*page_texts: str) -> bytes:
|
||
if not page_texts:
|
||
raise ValueError("at least one PDF page is required")
|
||
font_object_number = 3 + len(page_texts) * 2
|
||
page_object_numbers = [3 + index * 2 for index in range(len(page_texts))]
|
||
objects = [
|
||
b"<< /Type /Catalog /Pages 2 0 R >>",
|
||
(
|
||
b"<< /Type /Pages /Kids ["
|
||
+ b" ".join(f"{number} 0 R".encode() for number in page_object_numbers)
|
||
+ b"] /Count "
|
||
+ str(len(page_texts)).encode()
|
||
+ b" >>"
|
||
),
|
||
]
|
||
for index, text in enumerate(page_texts):
|
||
content_object_number = page_object_numbers[index] + 1
|
||
commands = [b"BT /F1 12 Tf 72 720 Td"]
|
||
for line_index, line in enumerate(text.splitlines()):
|
||
escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
||
if line_index:
|
||
commands.append(b"0 -16 Td")
|
||
commands.append(f"({escaped}) Tj".encode("ascii"))
|
||
commands.append(b"ET")
|
||
stream = b" ".join(commands)
|
||
objects.extend(
|
||
[
|
||
(
|
||
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
|
||
b"/Resources << /Font << /F1 "
|
||
+ str(font_object_number).encode()
|
||
+ b" 0 R >> >> /Contents "
|
||
+ str(content_object_number).encode()
|
||
+ b" 0 R >>"
|
||
),
|
||
b"<< /Length "
|
||
+ str(len(stream)).encode()
|
||
+ b" >>\nstream\n"
|
||
+ stream
|
||
+ b"\nendstream",
|
||
]
|
||
)
|
||
objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
|
||
result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
|
||
offsets = [0]
|
||
for object_number, value in enumerate(objects, start=1):
|
||
offsets.append(len(result))
|
||
result.extend(f"{object_number} 0 obj\n".encode())
|
||
result.extend(value)
|
||
result.extend(b"\nendobj\n")
|
||
xref_offset = len(result)
|
||
result.extend(f"xref\n0 {len(objects) + 1}\n".encode())
|
||
result.extend(b"0000000000 65535 f \n")
|
||
for offset in offsets[1:]:
|
||
result.extend(f"{offset:010d} 00000 n \n".encode())
|
||
result.extend(
|
||
(
|
||
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
|
||
f"startxref\n{xref_offset}\n%%EOF\n"
|
||
).encode()
|
||
)
|
||
return bytes(result)
|
||
|
||
|
||
def _minimal_pdf(text: str = "Hello PDF") -> bytes:
|
||
return _minimal_pdf_pages(text)
|
||
|
||
|
||
def test_outdated_data_process_schema_returns_actionable_503() -> None:
|
||
with pytest.raises(HTTPException) as captured, data_process_endpoint.api_errors():
|
||
raise psycopg.errors.UndefinedColumn("missing runtime column")
|
||
|
||
assert captured.value.status_code == 503
|
||
assert "schema is missing or out of date" in captured.value.detail["message"]
|
||
|
||
|
||
def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
created = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "客服问答处理",
|
||
"process_type": "structured",
|
||
"config": {"dataset_split": {"train": 80, "validation": 10, "test": 10}},
|
||
},
|
||
)
|
||
assert created.status_code == 200
|
||
assert created.json()["data"]["results_confirmed"] is False
|
||
task_id = created.json()["data"]["id"]
|
||
|
||
source_content = (
|
||
'{"question":"如何修改地址？",'
|
||
'"answer":"订单发货前可在订单详情申请修改收货地址。"}\n'
|
||
'{"question":"如何申请退款？",'
|
||
'"answer":"请在订单详情提交退款申请并等待审核处理。"}\n'
|
||
)
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("customer.jsonl", source_content.encode(), "application/jsonl")},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
source = uploaded.json()["data"]["files"][0]
|
||
assert len(source["checksum_sha256"]) == 64
|
||
assert source["version_no"] == 1
|
||
|
||
window = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||
params={"offset": 0, "limit": 20},
|
||
)
|
||
assert window.status_code == 200
|
||
assert window.json()["data"]["has_more"] is True
|
||
line_window = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||
params={"start_line": 2, "line_count": 1},
|
||
)
|
||
assert line_window.json()["data"]["start_line"] == 2
|
||
assert line_window.json()["data"]["end_line"] == 2
|
||
assert line_window.json()["data"]["total_lines"] == 2
|
||
|
||
preview = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_ids": [source["id"]]},
|
||
)
|
||
assert preview.status_code == 200
|
||
assert preview.json()["data"]["total"] == 2
|
||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||
assert listed_preview.json()["data"]["total"] == 2
|
||
preview_item = listed_preview.json()["data"]["items"][0]
|
||
source_locator = preview_item["quality_score"]["source_locator"]
|
||
assert source_locator == {
|
||
"kind": "jsonl",
|
||
"record_index": 1,
|
||
"start_line": 1,
|
||
"end_line": 1,
|
||
"source_start": 0,
|
||
"source_end": len(preview_item["original_content"]),
|
||
}
|
||
updated_preview = client.put(
|
||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||
json={
|
||
"edited_content": preview_item["edited_content"],
|
||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||
},
|
||
)
|
||
assert "quality_score" in updated_preview.json()["data"]
|
||
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
|
||
|
||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
assert generated.status_code == 200
|
||
assert generated.json()["data"]["results_confirmed"] is False
|
||
progress = client.get(f"/modelTF/data-process/{task_id}/progress")
|
||
assert progress.json()["data"]["status"] == "completed"
|
||
assert progress.json()["data"]["results_confirmed"] is False
|
||
result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
|
||
assert result_page["total"] == 2
|
||
keyword_page = client.get(
|
||
f"/modelTF/data-process/{task_id}/results", params={"keyword": "地址"}
|
||
).json()["data"]
|
||
assert keyword_page["total"] == 1
|
||
|
||
result = result_page["items"][0]
|
||
edited = client.put(
|
||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||
json={
|
||
"output": "人工修改后的完整答案。",
|
||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||
},
|
||
)
|
||
assert edited.json()["data"]["status"] == "modified"
|
||
assert "quality_score" in edited.json()["data"]
|
||
invalid_edit = client.put(
|
||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||
json={"output": ""},
|
||
)
|
||
assert invalid_edit.json()["data"]["status"] == "invalid"
|
||
assert store.tasks[task_id]["error_count"] == 1
|
||
restored = client.post(
|
||
f"/modelTF/data-process/{task_id}/results/{result['id']}/restore"
|
||
)
|
||
assert restored.json()["data"]["output"] == result["original_output"]
|
||
assert restored.json()["data"]["status"] == "valid"
|
||
assert store.tasks[task_id]["error_count"] == 0
|
||
|
||
workflow = client.put(
|
||
f"/modelTF/data-process/{task_id}/workflow-step",
|
||
json={"workflow_step": "results"},
|
||
)
|
||
assert workflow.status_code == 200
|
||
confirmed = client.post(
|
||
f"/modelTF/data-process/{task_id}/confirm-results"
|
||
)
|
||
assert confirmed.status_code == 200
|
||
assert confirmed.json()["data"]["results_confirmed"] is True
|
||
|
||
publish_payload = {"dataset_name": "客服问答清洗集"}
|
||
first_publish = client.post(
|
||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||
)
|
||
second_publish = client.post(
|
||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||
)
|
||
assert first_publish.json()["data"]["created"] is True
|
||
assert second_publish.json()["data"]["created"] is False
|
||
assert (
|
||
first_publish.json()["data"]["dataset"]["id"]
|
||
== second_publish.json()["data"]["dataset"]["id"]
|
||
)
|
||
|
||
|
||
def test_task_list_exposes_document_and_generation_counts(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "列表契约", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.sources[task_id] = [{"id": "source-1"}, {"id": "source-2"}]
|
||
store.tasks[task_id].update(
|
||
{
|
||
"status": "pending",
|
||
"output_count": 17,
|
||
"output_dataset_id": None,
|
||
}
|
||
)
|
||
|
||
response = client.get("/modelTF/data-process")
|
||
|
||
assert response.status_code == 200
|
||
item = response.json()["data"]["items"][0]
|
||
assert item["status"] == "pending"
|
||
assert item["source_file_count"] == 2
|
||
assert item["output_count"] == 17
|
||
assert item["output_dataset_id"] is None
|
||
|
||
|
||
def test_workflow_step_update_is_independent_and_validated(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "步骤持久化", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.previews[task_id] = [{"id": "preview-1"}]
|
||
store.results[task_id] = [{"id": "result-1"}]
|
||
store.tasks[task_id].update(status="running", generation_run_id="run-1")
|
||
|
||
updated = client.put(
|
||
f"/modelTF/data-process/{task_id}/workflow-step",
|
||
json={"workflow_step": "generate"},
|
||
)
|
||
|
||
assert updated.status_code == 200
|
||
assert updated.json()["data"]["workflow_step"] == "generate"
|
||
assert store.previews[task_id] == [{"id": "preview-1"}]
|
||
assert store.results[task_id] == [{"id": "result-1"}]
|
||
assert store.tasks[task_id]["status"] == "running"
|
||
assert client.put(
|
||
f"/modelTF/data-process/{task_id}/workflow-step",
|
||
json={"workflow_step": "unknown"},
|
||
).status_code == 422
|
||
|
||
|
||
def test_background_preview_persists_progress_and_items(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "后台切分", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
source = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"one.jsonl",
|
||
b'{"question":"What is one?","answer":"One."}\n',
|
||
"application/jsonl",
|
||
)
|
||
},
|
||
).json()["data"]["files"][0]
|
||
|
||
started = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/start",
|
||
json={"source_file_ids": [source["id"]]},
|
||
)
|
||
|
||
assert started.status_code == 202
|
||
assert started.json()["data"]["preview_status"] == "queued"
|
||
assert started.json()["data"]["preview_run_id"]
|
||
progress = client.get(
|
||
f"/modelTF/data-process/{task_id}/preview/progress"
|
||
).json()["data"]
|
||
assert progress == {
|
||
"task_id": task_id,
|
||
"workflow_step": "preview",
|
||
"preview_status": "completed",
|
||
"preview_progress": 100.0,
|
||
"preview_run_id": None,
|
||
"preview_failure_reason": None,
|
||
"preview_total_files": 1,
|
||
"preview_completed_files": 1,
|
||
}
|
||
assert client.get(
|
||
f"/modelTF/data-process/{task_id}/preview"
|
||
).json()["data"]["total"] == 1
|
||
|
||
|
||
def test_stale_preview_worker_cannot_replace_new_run(tmp_path: Path) -> None:
|
||
_, store, storage = make_client(tmp_path)
|
||
task = store.create_task(
|
||
{"name": "切分代次", "process_type": "structured", "config": {}}
|
||
)
|
||
task_id = str(task["id"])
|
||
store.sources[task_id] = [{"id": "source-1"}]
|
||
first, source_ids = store.start_preview(task_id, source_file_ids=["source-1"])
|
||
first_run_id = str(first["preview_run_id"])
|
||
store.tasks[task_id].update(preview_status="cancelled", preview_run_id=None)
|
||
second, _ = store.start_preview(task_id, source_file_ids=["source-1"])
|
||
|
||
data_process_endpoint._run_preview(
|
||
store,
|
||
storage,
|
||
task_id,
|
||
first_run_id,
|
||
source_ids,
|
||
)
|
||
|
||
assert store.tasks[task_id]["preview_status"] == "queued"
|
||
assert store.tasks[task_id]["preview_run_id"] == second["preview_run_id"]
|
||
assert store.previews[task_id] == []
|
||
|
||
|
||
def test_delete_invalidates_active_generation_and_preview(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "删除运行任务", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
status="running",
|
||
generation_run_id="generation-1",
|
||
preview_status="running",
|
||
preview_run_id="preview-1",
|
||
)
|
||
|
||
deleted = client.delete(f"/modelTF/data-process/{task_id}")
|
||
|
||
assert deleted.status_code == 200
|
||
assert store.generation_is_running(task_id, "generation-1") is False
|
||
assert store.preview_is_running(task_id, "preview-1") is False
|
||
|
||
|
||
def test_task_detail_uses_returned_source_files_as_document_count(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "详情文档数", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["source_file_count"] = 99
|
||
store.sources[task_id] = [
|
||
{"id": "source-1", "name": "一.pdf", "content": "正文一"},
|
||
{"id": "source-2", "name": "二.pdf", "content": "正文二"},
|
||
]
|
||
|
||
response = client.get(f"/modelTF/data-process/{task_id}")
|
||
|
||
assert response.status_code == 200
|
||
detail = response.json()["data"]
|
||
assert len(detail["source_files"]) == 2
|
||
assert detail["source_file_count"] == 2
|
||
|
||
|
||
def test_generation_start_response_clears_previous_output_count(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "新一轮生成", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["output_count"] = 28
|
||
store.previews[task_id] = [
|
||
{
|
||
"id": "preview-1",
|
||
"source_file_id": None,
|
||
"original_content": '{"question":"新问题","answer":"新答案"}',
|
||
"edited_content": '{"question":"新问题","answer":"新答案"}',
|
||
"status": "original",
|
||
}
|
||
]
|
||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *args: None)
|
||
|
||
response = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
|
||
assert response.status_code == 200
|
||
progress = response.json()["data"]
|
||
assert progress["status"] == "running"
|
||
assert progress["output_count"] == 0
|
||
assert store.tasks[task_id]["output_count"] == 0
|
||
|
||
|
||
def test_results_must_be_generated_and_valid_before_confirmation(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "结果确认门禁", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
|
||
pending = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
|
||
assert pending.status_code == 409
|
||
|
||
store.tasks[task_id].update(status="completed", progress=100, workflow_step="generate")
|
||
store.results[task_id] = [
|
||
{
|
||
"id": "result_valid",
|
||
"status": "valid",
|
||
"instruction": "问题",
|
||
"input": "",
|
||
"output": "答案",
|
||
}
|
||
]
|
||
wrong_step = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
|
||
assert wrong_step.status_code == 409
|
||
|
||
store.tasks[task_id]["workflow_step"] = "results"
|
||
store.results[task_id] = [
|
||
{
|
||
"id": "result_invalid",
|
||
"status": "invalid",
|
||
"instruction": "问题",
|
||
"input": "",
|
||
"output": "",
|
||
}
|
||
]
|
||
invalid = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
|
||
assert invalid.status_code == 409
|
||
|
||
store.results[task_id][0].update(status="valid", output="答案")
|
||
confirmed = client.post(f"/modelTF/data-process/{task_id}/confirm-results")
|
||
assert confirmed.status_code == 200
|
||
assert confirmed.json()["data"]["results_confirmed"] is True
|
||
assert client.post(
|
||
f"/modelTF/data-process/{task_id}/confirm-results"
|
||
).status_code == 200
|
||
|
||
|
||
def test_invalid_result_can_be_regenerated_in_place(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "单条重生成",
|
||
"process_type": "structured",
|
||
"config": {
|
||
"generation_model_id": "model-1",
|
||
"output_type": "standard",
|
||
"min_output_length": 5,
|
||
"generation_retries": 5,
|
||
"request_timeout_seconds": 120,
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
status="completed",
|
||
progress=100,
|
||
workflow_step="results",
|
||
results_confirmed=False,
|
||
error_count=1,
|
||
)
|
||
store.models["model-1"] = {
|
||
"id": "model-1",
|
||
"name": "测试模型",
|
||
"online_model_name": "test-model",
|
||
"api_url": "https://model.example/v1",
|
||
"api_key": "secret",
|
||
}
|
||
store.previews[task_id] = [{
|
||
"id": "preview-1",
|
||
"status": "original",
|
||
"original_content": "申请编号字段用于标识报销申请。",
|
||
"edited_content": "申请编号字段用于标识报销申请。",
|
||
}]
|
||
store.results[task_id] = [{
|
||
"id": "result-1",
|
||
"preview_item_id": "preview-1",
|
||
"instruction": "模型生成失败，请人工补充",
|
||
"input": "申请编号字段用于标识报销申请。",
|
||
"output": "",
|
||
"original_instruction": "模型生成失败，请人工补充",
|
||
"original_input": "申请编号字段用于标识报销申请。",
|
||
"original_output": "",
|
||
"status": "invalid",
|
||
"error": "model response is not valid JSON",
|
||
"split": "train",
|
||
"quality_score": {},
|
||
"updated_at": "2026-07-27T21:00:00Z",
|
||
}]
|
||
|
||
captured: dict[str, Any] = {}
|
||
|
||
def fake_generate(preview_items: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
||
captured["preview_items"] = list(preview_items)
|
||
captured["qa_pairs_per_item"] = kwargs["qa_pairs_per_item"]
|
||
captured["config"] = kwargs["config"]
|
||
return [{
|
||
"id": "temporary-result",
|
||
"preview_item_id": "preview-1",
|
||
"instruction": "申请编号字段有什么作用？",
|
||
"input": "申请编号字段用于标识报销申请。",
|
||
"output": "申请编号字段用于唯一标识一笔报销申请。",
|
||
"original_instruction": "申请编号字段有什么作用？",
|
||
"original_input": "申请编号字段用于标识报销申请。",
|
||
"original_output": "申请编号字段用于唯一标识一笔报销申请。",
|
||
"status": "valid",
|
||
"error": None,
|
||
"split": "train",
|
||
}]
|
||
|
||
monkeypatch.setattr(data_process_endpoint, "generate_model_records", fake_generate)
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/results/result-1/regenerate",
|
||
json={"expected_updated_at": "2026-07-27T21:00:00Z"},
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
regenerated = response.json()["data"]
|
||
assert regenerated["id"] == "result-1"
|
||
assert regenerated["status"] == "valid"
|
||
assert regenerated["instruction"] == regenerated["original_instruction"]
|
||
assert regenerated["output"] == regenerated["original_output"]
|
||
assert store.tasks[task_id]["error_count"] == 0
|
||
assert captured["qa_pairs_per_item"] == 1
|
||
assert [item["id"] for item in captured["preview_items"]] == ["preview-1"]
|
||
assert captured["config"]["generation_retries"] == 0
|
||
assert captured["config"]["request_timeout_seconds"] == 60
|
||
|
||
|
||
def test_failed_result_regeneration_keeps_the_original_error(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "失败结果保留",
|
||
"process_type": "structured",
|
||
"config": {
|
||
"generation_model_id": "model-1",
|
||
"output_type": "standard",
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
status="completed",
|
||
progress=100,
|
||
workflow_step="results",
|
||
results_confirmed=False,
|
||
error_count=1,
|
||
)
|
||
store.models["model-1"] = {
|
||
"id": "model-1",
|
||
"online_model_name": "test-model",
|
||
"api_url": "https://model.example/v1",
|
||
"api_key": "secret",
|
||
}
|
||
store.previews[task_id] = [{
|
||
"id": "preview-1",
|
||
"status": "original",
|
||
"original_content": "原始内容",
|
||
"edited_content": "原始内容",
|
||
}]
|
||
original_result = {
|
||
"id": "result-1",
|
||
"preview_item_id": "preview-1",
|
||
"instruction": "模型生成失败，请人工补充",
|
||
"input": "原始内容",
|
||
"output": "",
|
||
"status": "invalid",
|
||
"error": "first failure",
|
||
"updated_at": "2026-07-27T21:30:00Z",
|
||
}
|
||
store.results[task_id] = [original_result.copy()]
|
||
|
||
monkeypatch.setattr(
|
||
data_process_endpoint,
|
||
"generate_model_records",
|
||
lambda *args, **kwargs: [{
|
||
"status": "invalid",
|
||
"error": "second failure",
|
||
}],
|
||
)
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/results/result-1/regenerate",
|
||
json={"expected_updated_at": "2026-07-27T21:30:00Z"},
|
||
)
|
||
|
||
assert response.status_code == 409
|
||
assert store.results[task_id] == [original_result]
|
||
assert store.tasks[task_id]["error_count"] == 1
|
||
|
||
|
||
def test_failed_results_can_be_regenerated_in_parallel_with_partial_success(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "批量重生成",
|
||
"process_type": "structured",
|
||
"config": {
|
||
"generation_model_id": "model-1",
|
||
"output_type": "standard",
|
||
"min_output_length": 5,
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
status="completed",
|
||
progress=100,
|
||
workflow_step="results",
|
||
results_confirmed=False,
|
||
error_count=2,
|
||
)
|
||
store.models["model-1"] = {
|
||
"id": "model-1",
|
||
"online_model_name": "test-model",
|
||
"api_url": "https://model.example/v1",
|
||
"api_key": "secret",
|
||
}
|
||
store.previews[task_id] = [
|
||
{
|
||
"id": "preview-1",
|
||
"status": "original",
|
||
"original_content": "申请编号用于唯一标识一笔报销申请。",
|
||
"edited_content": "申请编号用于唯一标识一笔报销申请。",
|
||
},
|
||
{
|
||
"id": "preview-2",
|
||
"status": "original",
|
||
"original_content": "联系电话用于联系申请人。",
|
||
"edited_content": "联系电话用于联系申请人。",
|
||
},
|
||
]
|
||
original_results = [
|
||
{
|
||
"id": "result-1",
|
||
"preview_item_id": "preview-1",
|
||
"instruction": "模型生成失败，请人工补充",
|
||
"input": "申请编号用于唯一标识一笔报销申请。",
|
||
"output": "",
|
||
"original_instruction": "模型生成失败，请人工补充",
|
||
"original_input": "申请编号用于唯一标识一笔报销申请。",
|
||
"original_output": "",
|
||
"status": "invalid",
|
||
"error": "first failure",
|
||
"split": "train",
|
||
"quality_score": {},
|
||
"updated_at": "2026-07-28T09:00:00Z",
|
||
},
|
||
{
|
||
"id": "result-2",
|
||
"preview_item_id": "preview-2",
|
||
"instruction": "模型生成失败，请人工补充",
|
||
"input": "联系电话用于联系申请人。",
|
||
"output": "",
|
||
"original_instruction": "模型生成失败，请人工补充",
|
||
"original_input": "联系电话用于联系申请人。",
|
||
"original_output": "",
|
||
"status": "invalid",
|
||
"error": "first failure",
|
||
"split": "train",
|
||
"quality_score": {},
|
||
"updated_at": "2026-07-28T09:00:01Z",
|
||
},
|
||
]
|
||
store.results[task_id] = deepcopy(original_results)
|
||
barrier = Barrier(2, timeout=2)
|
||
activity_lock = Lock()
|
||
active_calls = 0
|
||
max_active_calls = 0
|
||
model_clients: list[Any] = []
|
||
|
||
def fake_generate(preview_items: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
||
nonlocal active_calls, max_active_calls
|
||
preview = next(iter(preview_items))
|
||
with activity_lock:
|
||
active_calls += 1
|
||
max_active_calls = max(max_active_calls, active_calls)
|
||
model_clients.append(kwargs.get("client"))
|
||
try:
|
||
barrier.wait()
|
||
if preview["id"] == "preview-2":
|
||
return [{"status": "invalid", "error": "second failure"}]
|
||
return [{
|
||
"id": "temporary-result",
|
||
"preview_item_id": preview["id"],
|
||
"instruction": "申请编号有什么作用？",
|
||
"input": preview["edited_content"],
|
||
"output": "申请编号用于唯一标识一笔报销申请。",
|
||
"status": "valid",
|
||
"error": None,
|
||
"split": "train",
|
||
}]
|
||
finally:
|
||
with activity_lock:
|
||
active_calls -= 1
|
||
|
||
monkeypatch.setattr(data_process_endpoint, "generate_model_records", fake_generate)
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/results/regenerate-batch",
|
||
json={
|
||
"items": [
|
||
{
|
||
"result_id": "result-1",
|
||
"expected_updated_at": "2026-07-28T09:00:00Z",
|
||
},
|
||
{
|
||
"result_id": "result-2",
|
||
"expected_updated_at": "2026-07-28T09:00:01Z",
|
||
},
|
||
],
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
data = response.json()["data"]
|
||
assert data["total"] == 2
|
||
assert data["succeeded"] == 1
|
||
assert data["failed"] == 1
|
||
assert data["remaining_invalid_count"] == 1
|
||
assert [item["id"] for item in data["items"]] == ["result-1"]
|
||
assert data["failures"][0]["result_id"] == "result-2"
|
||
assert store.results[task_id][0]["status"] == "valid"
|
||
assert store.results[task_id][0]["id"] == "result-1"
|
||
assert store.results[task_id][1] == original_results[1]
|
||
assert store.tasks[task_id]["error_count"] == 1
|
||
assert max_active_calls == 2
|
||
assert all(client is not None for client in model_clients)
|
||
assert len({id(client) for client in model_clients}) == 1
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"locked_state",
|
||
[
|
||
{"results_confirmed": True},
|
||
{"output_dataset_id": "dataset-published"},
|
||
],
|
||
ids=["confirmed", "published"],
|
||
)
|
||
def test_batch_result_regeneration_rejects_locked_tasks_before_model_call(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
locked_state: dict[str, Any],
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "批量重生成门禁",
|
||
"process_type": "structured",
|
||
"config": {"generation_model_id": "model-1"},
|
||
},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
status="completed",
|
||
workflow_step="results",
|
||
results_confirmed=False,
|
||
)
|
||
store.tasks[task_id].update(locked_state)
|
||
model_calls = 0
|
||
|
||
def fake_generate(*args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
||
nonlocal model_calls
|
||
model_calls += 1
|
||
return []
|
||
|
||
monkeypatch.setattr(data_process_endpoint, "generate_model_records", fake_generate)
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/results/regenerate-batch",
|
||
json={
|
||
"items": [{
|
||
"result_id": "result-1",
|
||
"expected_updated_at": "2026-07-28T09:00:00Z",
|
||
}],
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 409
|
||
assert model_calls == 0
|
||
|
||
|
||
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "逐文件预览", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("first.jsonl", b'{"id":1}\n', "application/jsonl")),
|
||
(
|
||
"files",
|
||
("second.jsonl", b'{"id":2}\n{"id":3}\n', "application/jsonl"),
|
||
),
|
||
],
|
||
)
|
||
assert uploaded.status_code == 200
|
||
first_source, second_source = uploaded.json()["data"]["files"]
|
||
|
||
first_build = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_ids": [first_source["id"]]},
|
||
)
|
||
assert first_build.status_code == 200
|
||
first_data = first_build.json()["data"]
|
||
assert first_data["file_counts"] == {first_source["id"]: 1}
|
||
assert first_data["files"] == [
|
||
{
|
||
"source_file_id": first_source["id"],
|
||
"preview_count": 1,
|
||
"status": "completed",
|
||
}
|
||
]
|
||
first_item = first_data["items"][0]
|
||
edited = client.put(
|
||
f"/modelTF/data-process/{task_id}/preview/{first_item['id']}",
|
||
json={"edited_content": "人工确认后的第一文件预览"},
|
||
)
|
||
assert edited.status_code == 200
|
||
|
||
second_build = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_id": second_source["id"]},
|
||
)
|
||
assert second_build.status_code == 200
|
||
second_data = second_build.json()["data"]
|
||
assert second_data["file_counts"] == {second_source["id"]: 2}
|
||
assert second_data["files"] == [
|
||
{
|
||
"source_file_id": second_source["id"],
|
||
"preview_count": 2,
|
||
"status": "completed",
|
||
}
|
||
]
|
||
assert {item["source_file_id"] for item in store.previews[task_id]} == {
|
||
first_source["id"],
|
||
second_source["id"],
|
||
}
|
||
preserved_first = next(
|
||
item
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == first_source["id"]
|
||
)
|
||
assert preserved_first["id"] == first_item["id"]
|
||
assert preserved_first["edited_content"] == "人工确认后的第一文件预览"
|
||
|
||
previous_second_ids = {
|
||
item["id"]
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == second_source["id"]
|
||
}
|
||
next(
|
||
source
|
||
for source in store.sources[task_id]
|
||
if source["id"] == second_source["id"]
|
||
)["content"] = '{"id":4}\n'
|
||
rebuilt = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_ids": [second_source["id"]]},
|
||
)
|
||
assert rebuilt.status_code == 200
|
||
assert rebuilt.json()["data"]["file_counts"] == {second_source["id"]: 1}
|
||
current_second_ids = {
|
||
item["id"]
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == second_source["id"]
|
||
}
|
||
assert current_second_ids.isdisjoint(previous_second_ids)
|
||
assert len(current_second_ids) == 1
|
||
assert next(
|
||
item
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == first_source["id"]
|
||
)["id"] == first_item["id"]
|
||
|
||
|
||
def test_preview_build_rejects_unknown_and_cross_task_source_file_ids(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
first_task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "归属任务一", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
second_task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "归属任务二", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
foreign_source = client.post(
|
||
f"/modelTF/data-process/{second_task_id}/source-files",
|
||
files={"files": ("foreign.jsonl", b'{"id":2}\n', "application/jsonl")},
|
||
).json()["data"]["files"][0]
|
||
|
||
unknown = client.post(
|
||
f"/modelTF/data-process/{first_task_id}/preview/build",
|
||
json={"source_file_ids": ["dpsf_not_found"]},
|
||
)
|
||
assert unknown.status_code == 404
|
||
foreign = client.post(
|
||
f"/modelTF/data-process/{first_task_id}/preview/build",
|
||
json={"source_file_id": foreign_source["id"]},
|
||
)
|
||
assert foreign.status_code == 404
|
||
ambiguous = client.post(
|
||
f"/modelTF/data-process/{first_task_id}/preview/build",
|
||
json={
|
||
"source_file_id": foreign_source["id"],
|
||
"source_file_ids": [foreign_source["id"]],
|
||
},
|
||
)
|
||
assert ambiguous.status_code == 422
|
||
|
||
|
||
def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "外部数据", "process_type": "external", "config": {}},
|
||
).json()["data"]["id"]
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/external/test",
|
||
json={"type": "mysql", "url": "mysql://db.example/test"},
|
||
)
|
||
assert response.status_code == 501
|
||
assert response.json()["detail"]["code"] == 501
|
||
|
||
|
||
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "待重新生成", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
{
|
||
"status": "completed",
|
||
"updated_at": "2026-07-25T19:00:00Z",
|
||
"output_dataset_id": "dataset_train",
|
||
"output_count": 2,
|
||
}
|
||
)
|
||
store.datasets["dataset_train"] = {
|
||
"id": "dataset_train",
|
||
"name": "原训练集",
|
||
"type": "train",
|
||
"source_task_id": task_id,
|
||
"deleted_at": None,
|
||
}
|
||
store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}]
|
||
store.results[task_id] = [{"id": "result_1"}]
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/regenerate",
|
||
json={
|
||
"name": "重新生成后名称",
|
||
"description": "更换生成模型",
|
||
"process_type": "structured",
|
||
"config": {"generation_model_id": "model_2"},
|
||
"expected_updated_at": "2026-07-25T19:00:00Z",
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
data = response.json()["data"]
|
||
assert data["task"]["status"] == "completed"
|
||
assert data["task"]["output_dataset_id"] == "dataset_train"
|
||
assert data["task"]["output_count"] == 2
|
||
assert data["preview_invalidated"] is False
|
||
assert data["published_outputs_preserved"] is True
|
||
assert store.results[task_id] == [{"id": "result_1"}]
|
||
assert store.previews[task_id][0]["id"] == "preview_1"
|
||
|
||
detail = client.get(f"/modelTF/data-process/{task_id}").json()["data"]
|
||
assert detail["status"] == "completed"
|
||
assert detail["output_dataset_id"] == "dataset_train"
|
||
assert detail["output_count"] == 2
|
||
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
|
||
|
||
|
||
def test_completed_task_can_repeat_into_an_independent_background_task(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "原始生成任务",
|
||
"process_type": "structured",
|
||
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
|
||
},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
source_id = uploaded.json()["data"]["files"][0]["id"]
|
||
built = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"replace_existing": True},
|
||
)
|
||
assert built.status_code == 200
|
||
store.tasks[task_id].update(
|
||
status="completed",
|
||
progress=100,
|
||
results_confirmed=True,
|
||
workflow_step="results",
|
||
output_count=1,
|
||
output_dataset_id="dataset-original",
|
||
updated_at="2026-07-28T12:00:00Z",
|
||
)
|
||
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
|
||
store.datasets["dataset-original"] = {
|
||
"id": "dataset-original",
|
||
"name": "原数据集",
|
||
"type": "train",
|
||
"source_task_id": task_id,
|
||
"deleted_at": None,
|
||
}
|
||
original_task = deepcopy(store.tasks[task_id])
|
||
original_sources = deepcopy(store.sources[task_id])
|
||
original_previews = deepcopy(store.previews[task_id])
|
||
original_results = deepcopy(store.results[task_id])
|
||
original_datasets = deepcopy(store.datasets)
|
||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
|
||
|
||
payload = {
|
||
"expected_updated_at": "2026-07-28T12:00:00Z",
|
||
"request_id": "repeat-request-0001",
|
||
}
|
||
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||
|
||
assert response.status_code == 202
|
||
repeated = response.json()["data"]
|
||
repeated_task_id = repeated["task"]["id"]
|
||
assert repeated["created"] is True
|
||
assert repeated_task_id != task_id
|
||
assert repeated["task"]["status"] == "running"
|
||
assert repeated["task"]["workflow_step"] == "generate"
|
||
assert repeated["copied_source_file_count"] == 1
|
||
assert repeated["copied_preview_count"] == len(original_previews)
|
||
assert store.tasks[task_id] == original_task
|
||
assert store.sources[task_id] == original_sources
|
||
assert store.previews[task_id] == original_previews
|
||
assert store.results[task_id] == original_results
|
||
assert store.datasets == original_datasets
|
||
|
||
repeated_source = store.sources[repeated_task_id][0]
|
||
repeated_preview = store.previews[repeated_task_id][0]
|
||
assert repeated_source["id"] != source_id
|
||
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
|
||
assert repeated_preview["id"] != original_previews[0]["id"]
|
||
assert repeated_preview["source_file_id"] == repeated_source["id"]
|
||
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
|
||
|
||
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||
assert replay.status_code == 202
|
||
assert replay.json()["data"]["created"] is False
|
||
assert replay.json()["data"]["task"]["id"] == repeated_task_id
|
||
assert len(store.tasks) == 2
|
||
assert len(store.sources[repeated_task_id]) == 1
|
||
|
||
|
||
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "源任务", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
status="completed",
|
||
results_confirmed=True,
|
||
updated_at="2026-07-28T12:00:00Z",
|
||
)
|
||
before = deepcopy(store.tasks)
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/repeat",
|
||
json={
|
||
"expected_updated_at": "2026-07-28T11:59:59Z",
|
||
"request_id": "repeat-request-stale",
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 409
|
||
assert store.tasks == before
|
||
|
||
|
||
def test_published_split_datasets_remain_in_detail_after_regeneration(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "保留旧发布数据", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
{
|
||
"status": "completed",
|
||
"updated_at": "2026-07-27T09:00:00Z",
|
||
"output_count": 1,
|
||
"results_confirmed": True,
|
||
}
|
||
)
|
||
store.results[task_id] = [
|
||
{
|
||
"id": "result_1",
|
||
"status": "valid",
|
||
"instruction": "问题",
|
||
"input": "",
|
||
"output": "答案",
|
||
}
|
||
]
|
||
store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}]
|
||
|
||
published = client.post(
|
||
f"/modelTF/data-process/{task_id}/publish",
|
||
json={"dataset_name": "保留旧发布数据集"},
|
||
)
|
||
assert published.status_code == 200
|
||
published_datasets = published.json()["data"]["datasets"]
|
||
assert len(published_datasets) == 3
|
||
published_ids = {item["id"] for item in published_datasets}
|
||
assert store.tasks[task_id]["output_dataset_id"] in published_ids
|
||
|
||
regenerated = client.post(
|
||
f"/modelTF/data-process/{task_id}/regenerate",
|
||
json={
|
||
"name": "保留旧发布数据",
|
||
"description": "更换生成配置后退出",
|
||
"process_type": "structured",
|
||
"config": {"generation_model_id": "model_2"},
|
||
"expected_updated_at": "2026-07-27T09:00:00Z",
|
||
},
|
||
)
|
||
assert regenerated.status_code == 200
|
||
original_output_dataset_id = store.tasks[task_id]["output_dataset_id"]
|
||
prepared_task = regenerated.json()["data"]["task"]
|
||
assert prepared_task["status"] == "completed"
|
||
assert prepared_task["output_dataset_id"] == original_output_dataset_id
|
||
assert prepared_task["output_count"] == 1
|
||
assert store.results[task_id][0]["id"] == "result_1"
|
||
|
||
detail = client.get(f"/modelTF/data-process/{task_id}")
|
||
assert detail.status_code == 200
|
||
detail_data = detail.json()["data"]
|
||
assert detail_data["status"] == "completed"
|
||
assert detail_data["output_dataset_id"] == original_output_dataset_id
|
||
assert detail_data["output_count"] == 1
|
||
assert len(detail_data["output_datasets"]) == 3
|
||
assert {item["id"] for item in detail_data["output_datasets"]} == published_ids
|
||
assert set(store.datasets) == published_ids
|
||
retained_results = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
|
||
assert retained_results["total"] == 1
|
||
assert retained_results["items"][0]["id"] == "result_1"
|
||
|
||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *args: None)
|
||
started = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
assert started.status_code == 200
|
||
running = started.json()["data"]
|
||
assert running["status"] == "running"
|
||
assert running["output_count"] == 0
|
||
assert store.results[task_id] == []
|
||
assert set(store.datasets) == published_ids
|
||
|
||
running_detail = client.get(f"/modelTF/data-process/{task_id}").json()["data"]
|
||
assert running_detail["status"] == "running"
|
||
assert running_detail["output_dataset_id"] is None
|
||
assert running_detail["output_count"] == 0
|
||
assert {item["id"] for item in running_detail["output_datasets"]} == published_ids
|
||
|
||
|
||
def test_regenerate_endpoint_validates_snapshot_and_locked_process_type(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "并发校验", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["updated_at"] = "2026-07-25T19:00:00Z"
|
||
payload = {
|
||
"name": "并发校验",
|
||
"description": "",
|
||
"process_type": "structured",
|
||
"config": {},
|
||
"expected_updated_at": "stale",
|
||
}
|
||
|
||
stale = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
|
||
assert stale.status_code == 409
|
||
|
||
payload.update(
|
||
{
|
||
"process_type": "unstructured",
|
||
"expected_updated_at": "2026-07-25T19:00:00Z",
|
||
}
|
||
)
|
||
locked_type = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
|
||
assert locked_type.status_code == 409
|
||
|
||
for missing_field in ("expected_updated_at", "description", "config"):
|
||
missing_required_field = client.post(
|
||
f"/modelTF/data-process/{task_id}/regenerate",
|
||
json={key: value for key, value in payload.items() if key != missing_field},
|
||
)
|
||
assert missing_required_field.status_code == 422
|
||
|
||
|
||
def test_config_validation_and_stop_state(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
invalid = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "错误切片配置",
|
||
"process_type": "unstructured",
|
||
"config": {
|
||
"dataset_split": {"train": 80, "validation": 30, "test": 0},
|
||
"chunk_size": 100,
|
||
"chunk_overlap": 90,
|
||
"min_chunk_size": 20,
|
||
},
|
||
},
|
||
)
|
||
assert invalid.status_code == 422
|
||
|
||
semantic = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "语义切分策略",
|
||
"process_type": "unstructured",
|
||
"config": {"chunk_method": "semantic"},
|
||
},
|
||
)
|
||
assert semantic.status_code == 200
|
||
|
||
removed_custom_method = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "已移除的自定义分隔符",
|
||
"process_type": "unstructured",
|
||
"config": {"chunk_method": "custom"},
|
||
},
|
||
)
|
||
assert removed_custom_method.status_code == 422
|
||
assert "chunk_method" in removed_custom_method.text
|
||
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "可停止任务", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["status"] = "running"
|
||
stopped = client.post(f"/modelTF/data-process/{task_id}/stop")
|
||
assert stopped.status_code == 200
|
||
assert stopped.json()["data"]["status"] == "stopped"
|
||
|
||
|
||
@pytest.mark.parametrize("config_key", ["qa_pairs_per_row", "qa_pairs_per_chunk"])
|
||
@pytest.mark.parametrize("count", [1, 50])
|
||
def test_qa_pair_config_accepts_supported_boundaries(
|
||
tmp_path: Path,
|
||
config_key: str,
|
||
count: int,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
response = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "问答数量边界",
|
||
"process_type": "unstructured",
|
||
"config": {config_key: count},
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
|
||
|
||
@pytest.mark.parametrize("config_key", ["qa_pairs_per_row", "qa_pairs_per_chunk"])
|
||
@pytest.mark.parametrize("count", [0, 51])
|
||
def test_qa_pair_config_rejects_out_of_range_boundaries(
|
||
tmp_path: Path,
|
||
config_key: str,
|
||
count: int,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
response = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "问答数量越界",
|
||
"process_type": "unstructured",
|
||
"config": {config_key: count},
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 422
|
||
assert "[1, 50]" in response.text
|
||
|
||
|
||
def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "批量上传", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
|
||
duplicate_batch = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("first.txt", b"same content", "text/plain")),
|
||
("files", ("second.txt", b"same content", "text/plain")),
|
||
],
|
||
)
|
||
assert duplicate_batch.status_code == 400
|
||
assert store.sources[task_id] == []
|
||
assert _stored_files(storage) == []
|
||
|
||
parse_failure = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("valid.txt", "先暂存的内容".encode(), "text/plain")),
|
||
("files", ("broken.txt", b"\xff", "text/plain")),
|
||
],
|
||
)
|
||
assert parse_failure.status_code == 400
|
||
assert store.sources[task_id] == []
|
||
assert _stored_files(storage) == []
|
||
|
||
empty = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("empty.txt", b"", "text/plain")},
|
||
)
|
||
assert empty.status_code == 400
|
||
assert store.sources[task_id] == []
|
||
assert _stored_files(storage) == []
|
||
|
||
|
||
def test_incremental_upload_keeps_existing_file_previews(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "增量上传预览", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
first_batch = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("first.txt", "第一个文件内容".encode(), "text/plain")),
|
||
("files", ("second.txt", "第二个文件内容".encode(), "text/plain")),
|
||
],
|
||
)
|
||
assert first_batch.status_code == 200
|
||
first_source_ids = {item["id"] for item in first_batch.json()["data"]["files"]}
|
||
built = client.post(f"/modelTF/data-process/{task_id}/preview/build", json={})
|
||
assert built.status_code == 200
|
||
assert {item["source_file_id"] for item in built.json()["data"]["items"]} == first_source_ids
|
||
store.results[task_id] = [{"id": "old_result"}]
|
||
|
||
third = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("third.txt", "第三个文件内容".encode(), "text/plain")},
|
||
)
|
||
|
||
assert third.status_code == 200
|
||
previews = client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]
|
||
assert previews["total"] == 2
|
||
assert {item["source_file_id"] for item in previews["items"]} == first_source_ids
|
||
assert store.results[task_id] == []
|
||
assert store.tasks[task_id]["progress"] == 20
|
||
|
||
|
||
def test_upload_preserves_store_error_when_storage_rollback_fails(
|
||
tmp_path: Path,
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "回滚异常", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
cleanup_attempts: list[str] = []
|
||
|
||
def fail_store(*_: Any, **__: Any) -> list[dict[str, Any]]:
|
||
raise ValueError("simulated database transaction failure")
|
||
|
||
def fail_cleanup(reference: str, **_: Any) -> bool:
|
||
cleanup_attempts.append(reference)
|
||
raise OSError("simulated storage cleanup failure")
|
||
|
||
monkeypatch.setattr(store, "add_source_files", fail_store)
|
||
monkeypatch.setattr(storage, "delete", fail_cleanup)
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("rollback.txt", b"rollback payload", "text/plain")},
|
||
)
|
||
|
||
assert response.status_code == 400
|
||
assert response.json()["detail"]["message"] == "simulated database transaction failure"
|
||
assert len(cleanup_attempts) == 1
|
||
assert store.sources[task_id] == []
|
||
|
||
|
||
def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "去重与质量筛选",
|
||
"process_type": "structured",
|
||
"config": {
|
||
"preprocess_options": ["clean_invalid", "deduplicate"],
|
||
"quality_filter_enabled": True,
|
||
"filter_low_quality": False,
|
||
"filter_short_content": True,
|
||
"min_output_length": 100,
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
content = (
|
||
'{"question":"问题","answer":"短答案"}\n'
|
||
'{"question":"问题","answer":"短答案"}\n'
|
||
).encode()
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("duplicates.jsonl", content, "application/jsonl")},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
assert preview.json()["data"]["total"] == 1
|
||
|
||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
assert generated.status_code == 200
|
||
progress = client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]
|
||
assert progress["status"] == "completed"
|
||
assert progress["filtered_count"] == 1
|
||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||
|
||
|
||
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "先去重再脱敏",
|
||
"process_type": "structured",
|
||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||
},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"names.jsonl",
|
||
(
|
||
'{"name":"张三","role":"开发"}\n'
|
||
'{"name":"李四","role":"开发"}\n'
|
||
),
|
||
"application/jsonl",
|
||
)
|
||
},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
|
||
assert preview.status_code == 200
|
||
items = preview.json()["data"]["items"]
|
||
assert len(items) == 2
|
||
assert len({item["original_content"] for item in items}) == 2
|
||
assert {item["edited_content"] for item in items} == {
|
||
'{"name":"[NAME]","role":"开发"}'
|
||
}
|
||
|
||
|
||
def test_structured_deduplication_removes_identical_rows_across_sources(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "跨源原文去重",
|
||
"process_type": "structured",
|
||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||
},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
(
|
||
"files",
|
||
(
|
||
"first.jsonl",
|
||
'{"name":"张三","role":"开发"}\n',
|
||
"application/jsonl",
|
||
),
|
||
),
|
||
(
|
||
"files",
|
||
(
|
||
"second.jsonl",
|
||
'\n{"name":"张三","role":"开发"}\n',
|
||
"application/jsonl",
|
||
),
|
||
),
|
||
],
|
||
)
|
||
assert uploaded.status_code == 200
|
||
first_source, second_source = uploaded.json()["data"]["files"]
|
||
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
|
||
assert preview.status_code == 200
|
||
data = preview.json()["data"]
|
||
assert data["total"] == 1
|
||
assert data["file_counts"] == {
|
||
first_source["id"]: 1,
|
||
second_source["id"]: 0,
|
||
}
|
||
|
||
|
||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||
store = FakeDataProcessStore()
|
||
task = store.create_task(
|
||
{"name": "并发代次", "process_type": "structured", "config": {}}
|
||
)
|
||
task_id = task["id"]
|
||
store.replace_preview_items(
|
||
task_id,
|
||
[
|
||
{
|
||
"source_file_id": None,
|
||
"original_content": "来源内容",
|
||
"edited_content": "来源内容",
|
||
"status": "manual",
|
||
}
|
||
],
|
||
)
|
||
first = store.start_generation(task_id, replace_existing=True)
|
||
first_run_id = first["generation_run_id"]
|
||
second_run_id = ""
|
||
|
||
def restart_while_old_worker_runs(*_: Any, **__: Any) -> list[dict[str, Any]]:
|
||
nonlocal second_run_id
|
||
store.stop_task(task_id)
|
||
second = store.start_generation(task_id, replace_existing=True)
|
||
second_run_id = second["generation_run_id"]
|
||
return []
|
||
|
||
monkeypatch.setattr(
|
||
data_process_endpoint,
|
||
"generate_standard_records",
|
||
restart_while_old_worker_runs,
|
||
)
|
||
data_process_endpoint._run_generation(store, task_id, first_run_id)
|
||
|
||
assert second_run_id and second_run_id != first_run_id
|
||
assert store.tasks[task_id]["status"] == "running"
|
||
assert store.tasks[task_id]["generation_run_id"] == second_run_id
|
||
assert store.results[task_id] == []
|
||
store.mark_failed(task_id, "old failure", generation_run_id=first_run_id)
|
||
assert store.tasks[task_id]["status"] == "running"
|
||
|
||
|
||
def test_reasoning_output_requires_generation_model() -> None:
|
||
store = FakeDataProcessStore()
|
||
task = store.create_task(
|
||
{
|
||
"name": "思维链模型校验",
|
||
"process_type": "structured",
|
||
"config": {"output_type": "reasoning"},
|
||
}
|
||
)
|
||
task_id = task["id"]
|
||
store.replace_preview_items(
|
||
task_id,
|
||
[
|
||
{
|
||
"source_file_id": None,
|
||
"original_content": "需要推理的来源内容",
|
||
"edited_content": "需要推理的来源内容",
|
||
"status": "manual",
|
||
}
|
||
],
|
||
)
|
||
started = store.start_generation(task_id, replace_existing=True)
|
||
|
||
data_process_endpoint._run_generation(
|
||
store,
|
||
task_id,
|
||
started["generation_run_id"],
|
||
)
|
||
|
||
failed = store.get_task(task_id)
|
||
assert failed["status"] == "failed"
|
||
assert failed["failure_reason"] == "思维链输出必须配置可用的数据生成模型"
|
||
|
||
|
||
def test_generation_failure_is_written_to_structured_log(caplog: pytest.LogCaptureFixture) -> None:
|
||
store = FakeDataProcessStore()
|
||
task = store.create_task(
|
||
{
|
||
"name": "生成失败日志",
|
||
"process_type": "structured",
|
||
"config": {"output_type": "reasoning"},
|
||
}
|
||
)
|
||
task_id = task["id"]
|
||
store.replace_preview_items(
|
||
task_id,
|
||
[
|
||
{
|
||
"source_file_id": None,
|
||
"original_content": "需要推理的来源内容",
|
||
"edited_content": "需要推理的来源内容",
|
||
"status": "manual",
|
||
}
|
||
],
|
||
)
|
||
started = store.start_generation(task_id, replace_existing=True)
|
||
|
||
with caplog.at_level("INFO", logger=data_process_endpoint.__name__):
|
||
data_process_endpoint._run_generation(
|
||
store,
|
||
task_id,
|
||
started["generation_run_id"],
|
||
)
|
||
|
||
messages = [record.getMessage() for record in caplog.records]
|
||
assert any("generation worker started" in message for message in messages)
|
||
assert any("generation failed" in message for message in messages)
|
||
|
||
|
||
def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "状态保护", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
response = client.put(
|
||
f"/modelTF/data-process/{task_id}/results/not-created",
|
||
json={"instruction": "", "output": "", "status": "valid"},
|
||
)
|
||
assert response.status_code == 422
|
||
|
||
|
||
def test_start_rebuilds_preview_and_generates_in_one_request(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "一键处理", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"one.jsonl",
|
||
b'{"question":"What is one?","answer":"One."}\n',
|
||
"application/jsonl",
|
||
)
|
||
},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
|
||
started = client.post(f"/modelTF/data-process/{task_id}/start")
|
||
assert started.status_code == 200
|
||
assert started.json()["data"]["task_id"] == task_id
|
||
assert started.json()["data"]["status"] == "running"
|
||
assert client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]["status"] == "completed"
|
||
assert client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]["total"] == 1
|
||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1
|
||
|
||
|
||
def test_unsupported_upload_format_returns_415(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "格式限制", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("payload.exe", b"not supported", "application/octet-stream")},
|
||
)
|
||
assert response.status_code == 415
|
||
|
||
legacy = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("document.doc", b"legacy", "application/msword")},
|
||
)
|
||
assert legacy.status_code == 415
|
||
assert "convert the file to .docx" in legacy.json()["detail"]["message"]
|
||
|
||
|
||
def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "XLSX 上传", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
workbook = Workbook()
|
||
worksheet = workbook.active
|
||
worksheet.append(["question", "answer"])
|
||
worksheet.append(["问题一", "答案一"])
|
||
worksheet.append(["问题二", "答案二"])
|
||
output = BytesIO()
|
||
workbook.save(output)
|
||
workbook.close()
|
||
original_bytes = output.getvalue()
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"records.xlsx",
|
||
original_bytes,
|
||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||
)
|
||
},
|
||
)
|
||
assert response.status_code == 200
|
||
source = response.json()["data"]["files"][0]
|
||
assert source["file_format"] == "xlsx"
|
||
assert source["record_count"] == 2
|
||
assert source["size_bytes"] == len(original_bytes)
|
||
assert source["storage_object_id"].startswith("local://data-process/")
|
||
assert str(storage.root) not in response.text
|
||
assert storage.read(source["storage_object_id"]) == original_bytes
|
||
|
||
stored_source = store.get_source_file(task_id, source["id"])
|
||
assert stored_source["id"] == source["id"]
|
||
assert stored_source["storage_object_id"] == source["storage_object_id"]
|
||
assert stored_source["metadata"]["storage_backend"] == "local"
|
||
assert stored_source["metadata"]["original_size_bytes"] == len(original_bytes)
|
||
assert '"question":"问题一"' in stored_source["content"]
|
||
|
||
content = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content"
|
||
)
|
||
assert content.status_code == 200
|
||
assert '"answer":"答案二"' in content.json()["data"]["content"]
|
||
office_preview_url = (
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/office-preview"
|
||
)
|
||
first_page = client.get(office_preview_url, params={"offset": 0, "limit": 1})
|
||
assert first_page.status_code == 200
|
||
preview_data = first_page.json()["data"]
|
||
assert preview_data["format"] == "xlsx"
|
||
assert preview_data["sheets"] == [{"index": 0, "name": "Sheet", "state": "visible"}]
|
||
assert preview_data["active_sheet"]["columns"] == ["question", "answer"]
|
||
assert preview_data["active_sheet"]["rows"][0]["record"] == {
|
||
"question": "问题一",
|
||
"answer": "答案一",
|
||
}
|
||
assert preview_data["active_sheet"]["has_more"] is True
|
||
|
||
second_page = client.get(office_preview_url, params={"offset": 1, "limit": 1})
|
||
assert second_page.status_code == 200
|
||
assert second_page.json()["data"]["active_sheet"]["rows"][0]["record"] == {
|
||
"question": "问题二",
|
||
"answer": "答案二",
|
||
}
|
||
assert second_page.json()["data"]["active_sheet"]["has_more"] is False
|
||
|
||
raw_preview = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/raw"
|
||
)
|
||
assert raw_preview.status_code == 200
|
||
assert raw_preview.content == original_bytes
|
||
assert raw_preview.headers["content-type"].startswith(
|
||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||
)
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
assert preview.status_code == 200
|
||
preview_items = preview.json()["data"]["items"]
|
||
assert len(preview_items) == 2
|
||
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
|
||
{
|
||
"kind": "xlsx",
|
||
"record_index": 1,
|
||
"sheet_index": 0,
|
||
"sheet_name": "Sheet",
|
||
"row_number": 2,
|
||
"sheet_record_index": 0,
|
||
},
|
||
{
|
||
"kind": "xlsx",
|
||
"record_index": 2,
|
||
"sheet_index": 0,
|
||
"sheet_name": "Sheet",
|
||
"row_number": 3,
|
||
"sheet_record_index": 1,
|
||
},
|
||
]
|
||
|
||
|
||
def test_docx_preview_preserves_document_block_order_and_source_offsets(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "Word 原件预览", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
document = WordDocument()
|
||
document.add_heading("费用管理办法", level=1)
|
||
document.add_paragraph("第一条 本办法用于规范费用报销。")
|
||
table = document.add_table(rows=2, cols=2)
|
||
table.cell(0, 0).text = "费用类型"
|
||
table.cell(0, 1).text = "审批人"
|
||
table.cell(1, 0).text = "差旅费"
|
||
table.cell(1, 1).text = "部门负责人"
|
||
output = BytesIO()
|
||
document.save(output)
|
||
original_bytes = output.getvalue()
|
||
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"费用 管理.docx",
|
||
original_bytes,
|
||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||
)
|
||
},
|
||
).json()["data"]["files"][0]
|
||
response = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/office-preview"
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
data = response.json()["data"]
|
||
assert data["format"] == "docx"
|
||
assert data["file_name"] == "费用 管理.docx"
|
||
assert data["truncated"] is False
|
||
assert [block["type"] for block in data["blocks"]] == [
|
||
"paragraph",
|
||
"paragraph",
|
||
"table",
|
||
]
|
||
assert data["blocks"][0]["heading_level"] == 1
|
||
assert data["blocks"][0]["text"] == "费用管理办法"
|
||
assert data["blocks"][2]["rows"][1]["cells"] == ["差旅费", "部门负责人"]
|
||
|
||
source_text = store.get_source_file(task_id, uploaded["id"])["content"]
|
||
first_paragraph = data["blocks"][0]
|
||
assert (
|
||
source_text[first_paragraph["source_start"] : first_paragraph["source_end"]]
|
||
== first_paragraph["text"]
|
||
)
|
||
table_row = data["blocks"][2]["rows"][1]
|
||
assert (
|
||
source_text[table_row["source_start"] : table_row["source_end"]]
|
||
== "\t".join(table_row["cells"])
|
||
)
|
||
|
||
raw_preview = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
|
||
)
|
||
assert raw_preview.status_code == 200
|
||
assert raw_preview.content == original_bytes
|
||
assert raw_preview.headers["content-type"].startswith(
|
||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||
)
|
||
|
||
|
||
def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "PDF 原件预览", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
original_pdf = _minimal_pdf()
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("说明 文档.pdf", original_pdf, "application/pdf")},
|
||
).json()["data"]["files"][0]
|
||
raw_url = f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
|
||
|
||
full = client.get(raw_url)
|
||
assert full.status_code == 200
|
||
assert full.content == original_pdf
|
||
assert full.headers["content-type"] == "application/pdf"
|
||
assert full.headers["accept-ranges"] == "bytes"
|
||
assert full.headers["cache-control"] == "private, no-store"
|
||
assert full.headers["content-length"] == str(len(original_pdf))
|
||
assert full.headers["content-disposition"].startswith("inline;")
|
||
assert "%E8%AF%B4%E6%98%8E%20%E6%96%87%E6%A1%A3.pdf" in full.headers[
|
||
"content-disposition"
|
||
]
|
||
assert full.headers["etag"] == f'"{uploaded["checksum_sha256"]}"'
|
||
|
||
partial = client.get(raw_url, headers={"Range": "bytes=5-14"})
|
||
assert partial.status_code == 206
|
||
assert partial.content == original_pdf[5:15]
|
||
assert partial.headers["content-range"] == f"bytes 5-14/{len(original_pdf)}"
|
||
assert partial.headers["content-length"] == "10"
|
||
|
||
suffix = client.get(raw_url, headers={"Range": "bytes=-8"})
|
||
assert suffix.status_code == 206
|
||
assert suffix.content == original_pdf[-8:]
|
||
|
||
invalid = client.get(raw_url, headers={"Range": "bytes=0-1,4-5"})
|
||
assert invalid.status_code == 416
|
||
assert invalid.headers["content-range"] == f"bytes */{len(original_pdf)}"
|
||
|
||
pages_url = (
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages"
|
||
)
|
||
pages = client.get(pages_url)
|
||
assert pages.status_code == 200
|
||
assert pages.json()["data"] == {
|
||
"page_count": 1,
|
||
"pages": [
|
||
{
|
||
"page_number": 1,
|
||
"source_start": 0,
|
||
"source_end": len("Hello PDF"),
|
||
}
|
||
],
|
||
}
|
||
|
||
legacy_id = "dpsf_legacy_pdf"
|
||
store.add_source_file(
|
||
task_id,
|
||
id=legacy_id,
|
||
storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1",
|
||
name="legacy.pdf",
|
||
content="legacy extracted PDF text",
|
||
raw_size=len(original_pdf),
|
||
checksum_sha256="a" * 64,
|
||
file_format="pdf",
|
||
record_count=1,
|
||
metadata={"legacy": True},
|
||
)
|
||
legacy = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/raw"
|
||
)
|
||
assert legacy.status_code == 410
|
||
legacy_pages = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/pdf-pages"
|
||
)
|
||
assert legacy_pages.status_code == 410
|
||
|
||
|
||
def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "PDF 文档噪声清理",
|
||
"process_type": "unstructured",
|
||
"config": {
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 20,
|
||
"preprocess_options": ["clean_invalid_content"],
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
raw = _minimal_pdf_pages(
|
||
"ACME Internal Manual\nBody page one keeps this guidance and explanation.",
|
||
"ACME Internal Manual\nContents\n"
|
||
"Chapter One........3\nChapter Two........4\nAppendix........5",
|
||
"ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8",
|
||
"ACME Internal Manual\nBody page four keeps operational details and examples.",
|
||
"ACME Internal Manual\nBody page five keeps the final effective-date clause.",
|
||
)
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("manual.pdf", raw, "application/pdf")},
|
||
).json()["data"]["files"][0]
|
||
source_content = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content"
|
||
).json()["data"]["content"]
|
||
|
||
built = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
|
||
assert built.status_code == 200
|
||
items = built.json()["data"]["items"]
|
||
assert items
|
||
edited = "\n".join(item["edited_content"] for item in items)
|
||
assert "ACME Internal Manual" not in edited
|
||
assert "Contents" not in edited
|
||
assert "Chapter One" not in edited
|
||
assert "1.2 Approval" not in edited
|
||
assert "Body page one" in edited
|
||
assert "Body page five" in edited
|
||
assert all(
|
||
item["original_content"]
|
||
== source_content[item["source_start"] : item["source_end"]]
|
||
for item in items
|
||
)
|
||
|
||
|
||
def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "非 PDF 原件", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("notes.txt", b"plain source text", "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
|
||
response = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
|
||
)
|
||
assert response.status_code == 415
|
||
pages_response = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages"
|
||
)
|
||
assert pages_response.status_code == 415
|
||
|
||
|
||
def test_delete_source_removes_owned_local_object_and_accepts_legacy_db_reference(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "删除原件", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("原件.txt", "本地原始内容".encode(), "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
reference = uploaded["storage_object_id"]
|
||
assert storage.read(reference) == "本地原始内容".encode()
|
||
|
||
deleted = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}"
|
||
)
|
||
assert deleted.status_code == 200
|
||
assert deleted.json()["data"]["storage_cleanup_pending"] is False
|
||
with pytest.raises(DataProcessStorageError, match="does not exist"):
|
||
storage.read(reference)
|
||
|
||
legacy_id = "dpsf_legacy"
|
||
store.add_source_file(
|
||
task_id,
|
||
id=legacy_id,
|
||
storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1",
|
||
name="legacy.txt",
|
||
content="旧记录正文",
|
||
raw_size=len("旧记录正文".encode()),
|
||
checksum_sha256="a" * 64,
|
||
file_format="txt",
|
||
record_count=1,
|
||
metadata={"legacy": True},
|
||
)
|
||
legacy_deleted = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}"
|
||
)
|
||
assert legacy_deleted.status_code == 200
|
||
assert legacy_deleted.json()["data"]["storage_cleanup_pending"] is False
|
||
|
||
|
||
def test_delete_reports_pending_cleanup_after_database_soft_delete(
|
||
tmp_path: Path,
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "待清理原件", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("pending.txt", b"pending cleanup", "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
|
||
def fail_cleanup(*_: Any, **__: Any) -> bool:
|
||
raise OSError("simulated storage failure")
|
||
|
||
monkeypatch.setattr(storage, "delete", fail_cleanup)
|
||
response = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}"
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
assert response.json()["data"]["storage_cleanup_pending"] is True
|
||
with pytest.raises(NotFoundError):
|
||
store.get_source_file(task_id, uploaded["id"])
|
||
assert storage.read(uploaded["storage_object_id"]) == b"pending cleanup"
|
||
|
||
|
||
def test_delete_rejects_polluted_reference_owned_by_another_source(tmp_path: Path) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "归属校验", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("safe.txt", b"owned content", "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
target_reference = uploaded["storage_object_id"]
|
||
|
||
polluted_id = "dpsf_polluted"
|
||
store.add_source_file(
|
||
task_id,
|
||
id=polluted_id,
|
||
storage_object_id=target_reference,
|
||
name="polluted.txt",
|
||
content="polluted",
|
||
raw_size=8,
|
||
checksum_sha256="b" * 64,
|
||
file_format="txt",
|
||
record_count=1,
|
||
metadata={},
|
||
)
|
||
rejected = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{polluted_id}"
|
||
)
|
||
assert rejected.status_code == 400
|
||
assert storage.read(target_reference) == b"owned content"
|
||
assert store.get_source_file(task_id, polluted_id)["id"] == polluted_id
|
||
|
||
|
||
def test_upload_format_must_match_process_type(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
structured_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "结构化格式约束", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
structured_pdf = client.post(
|
||
f"/modelTF/data-process/{structured_id}/source-files",
|
||
files={"files": ("manual.pdf", b"not parsed", "application/pdf")},
|
||
)
|
||
assert structured_pdf.status_code == 415
|
||
|
||
unstructured_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "非结构化格式约束", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
unstructured_xlsx = client.post(
|
||
f"/modelTF/data-process/{unstructured_id}/source-files",
|
||
files={"files": ("records.xlsx", b"not parsed", "application/octet-stream")},
|
||
)
|
||
assert unstructured_xlsx.status_code == 415
|
||
|
||
external_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "外部数据格式约束", "process_type": "external", "config": {}},
|
||
).json()["data"]["id"]
|
||
external_upload = client.post(
|
||
f"/modelTF/data-process/{external_id}/source-files",
|
||
files={"files": ("records.jsonl", b'{"id":1}', "application/jsonl")},
|
||
)
|
||
assert external_upload.status_code == 409
|
||
|
||
|
||
def _preview_task(
|
||
content: str,
|
||
*,
|
||
options: list[str],
|
||
config: dict[str, Any] | None = None,
|
||
source_id: str = "source-1",
|
||
file_format: str = "txt",
|
||
) -> list[dict[str, Any]]:
|
||
task_config = {
|
||
"preprocess_options": options,
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 20,
|
||
"min_chunk_size": 20,
|
||
**(config or {}),
|
||
}
|
||
return data_process_endpoint._build_preview_items(
|
||
{"process_type": "unstructured", "config": task_config},
|
||
[
|
||
{
|
||
"id": source_id,
|
||
"name": f"{source_id}.{file_format}",
|
||
"file_format": file_format,
|
||
"content": content,
|
||
}
|
||
],
|
||
)
|
||
|
||
|
||
def _structured_preview_task(
|
||
content: str,
|
||
*,
|
||
file_format: str,
|
||
options: list[str] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
return data_process_endpoint._build_preview_items(
|
||
{
|
||
"process_type": "structured",
|
||
"config": {"preprocess_options": options or []},
|
||
},
|
||
[
|
||
{
|
||
"id": "structured-source",
|
||
"name": f"records.{file_format}",
|
||
"file_format": file_format,
|
||
"content": content,
|
||
}
|
||
],
|
||
)
|
||
|
||
|
||
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
|
||
json_source = '{"records":[{"id":1},{"id":2}]}'
|
||
json_items = _structured_preview_task(
|
||
json_source,
|
||
file_format="json",
|
||
)
|
||
assert [
|
||
item["quality_score"]["source_locator"]["json_pointer"]
|
||
for item in json_items
|
||
] == ["/records/0", "/records/1"]
|
||
assert [
|
||
json_source[item["source_start"] : item["source_end"]]
|
||
for item in json_items
|
||
] == ['{"id":1}', '{"id":2}']
|
||
assert [item["source_start_line"] for item in json_items] == [1, 1]
|
||
|
||
jsonl_source = '{"id":1}\n\n{"id":2}'
|
||
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
|
||
assert [
|
||
item["quality_score"]["source_locator"]["record_index"]
|
||
for item in jsonl_items
|
||
] == [1, 2]
|
||
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
|
||
assert [
|
||
jsonl_source[item["source_start"] : item["source_end"]]
|
||
for item in jsonl_items
|
||
] == ['{"id":1}', '{"id":2}']
|
||
|
||
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
|
||
csv_items = _structured_preview_task(csv_source, file_format="csv")
|
||
assert [
|
||
(item["source_start_line"], item["source_end_line"])
|
||
for item in csv_items
|
||
] == [(2, 3), (5, 5)]
|
||
assert [
|
||
csv_source[item["source_start"] : item["source_end"]]
|
||
for item in csv_items
|
||
] == ['1,"hello\nworld"', "2,plain"]
|
||
|
||
|
||
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "空 JSON", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("empty-array.json", "[]", "application/json")),
|
||
(
|
||
"files",
|
||
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
|
||
),
|
||
],
|
||
)
|
||
|
||
assert uploaded.status_code == 200
|
||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
assert preview.status_code == 200
|
||
assert preview.json()["data"]["items"] == []
|
||
assert preview.json()["data"]["total"] == 0
|
||
assert set(preview.json()["data"]["file_counts"].values()) == {0}
|
||
|
||
|
||
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
invalid_sources = (
|
||
("duplicate.json", '{"id":1,"id":2}'),
|
||
("duplicate.jsonl", '{"id":1,"id":2}\n'),
|
||
("nan.json", '{"value":NaN}'),
|
||
("infinity.json", '{"value":Infinity}'),
|
||
("control.json", '{"value":"bad\x00control"}'),
|
||
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
|
||
)
|
||
|
||
for filename, content in invalid_sources:
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": (filename, content, "application/json")},
|
||
)
|
||
assert response.status_code == 400, (filename, response.text)
|
||
|
||
|
||
def test_structured_json_preview_preserves_precision_and_business_data_field(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
precise = '{"value":0.123456789012345678901234567890}'
|
||
business = '{"id":7,"data":[{"id":8}]}'
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("precise.json", precise, "application/json")),
|
||
("files", ("business.json", business, "application/json")),
|
||
],
|
||
)
|
||
assert uploaded.status_code == 200
|
||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
|
||
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
assert preview.status_code == 200
|
||
items = preview.json()["data"]["items"]
|
||
assert [item["original_content"] for item in items] == [precise, business]
|
||
assert [
|
||
item["quality_score"]["source_locator"]["json_pointer"] for item in items
|
||
] == ["", ""]
|
||
assert [item["source_start"] for item in items] == [0, 0]
|
||
|
||
|
||
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
|
||
source_records = [
|
||
{"id": "A", "amount": 10, "empty": ""},
|
||
{"id": "A", "amount": 10, "empty": ""},
|
||
{"id": "", "amount": 11, "empty": ""},
|
||
{"id": "B", "amount": 11, "empty": ""},
|
||
{"id": "C", "amount": 12, "empty": ""},
|
||
{"id": "D", "amount": 12, "empty": ""},
|
||
{"id": "E", "amount": 13, "empty": ""},
|
||
{"id": "F", "amount": 13, "empty": ""},
|
||
{"id": "G", "amount": 14, "empty": ""},
|
||
{"id": "H", "amount": 1000, "empty": ""},
|
||
]
|
||
source = "\n".join(
|
||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||
for record in source_records
|
||
)
|
||
items = _structured_preview_task(
|
||
source,
|
||
file_format="jsonl",
|
||
options=["clean_invalid", "deduplicate", "filter_anomaly"],
|
||
)
|
||
|
||
assert [
|
||
item["quality_score"]["source_locator"]["record_index"]
|
||
for item in items
|
||
] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||
assert [json.loads(item["original_content"])["id"] for item in items] == [
|
||
"A",
|
||
"",
|
||
"B",
|
||
"C",
|
||
"D",
|
||
"E",
|
||
"F",
|
||
"G",
|
||
]
|
||
|
||
|
||
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
|
||
source_records = [
|
||
{"customer_id": "C-1", "status": "old"},
|
||
{"customer_id": "C-1", "status": "new"},
|
||
{"status": "old", "customer_id": "C-1"},
|
||
]
|
||
source = "\n".join(
|
||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||
for record in source_records
|
||
)
|
||
|
||
items = _structured_preview_task(
|
||
source,
|
||
file_format="jsonl",
|
||
options=["clean_invalid", "deduplicate"],
|
||
)
|
||
|
||
assert [
|
||
item["quality_score"]["source_locator"]["record_index"]
|
||
for item in items
|
||
] == [1, 2]
|
||
assert [item["source_start_line"] for item in items] == [1, 2]
|
||
assert [json.loads(item["original_content"])["status"] for item in items] == [
|
||
"old",
|
||
"new",
|
||
]
|
||
|
||
|
||
def test_fixed_preview_preserves_source_offsets() -> None:
|
||
content = (
|
||
"# 第一章\n"
|
||
+ " ".join(f"alpha{index}" for index in range(18))
|
||
+ "\n# 第二章\n"
|
||
+ " ".join(f"beta{index}" for index in range(18))
|
||
)
|
||
normalized = normalize_text(content)
|
||
second_chapter_start = normalized.index("# 第二章")
|
||
common_config = {"chunk_size": 10, "chunk_overlap": 3, "min_chunk_size": 4}
|
||
|
||
default_items = _preview_task(
|
||
content,
|
||
options=["preserve_context"],
|
||
config=common_config,
|
||
)
|
||
fixed_items = _preview_task(
|
||
content,
|
||
options=["preserve_context"],
|
||
config={**common_config, "chunk_method": "fixed"},
|
||
)
|
||
|
||
def snapshot(items: list[dict[str, Any]]) -> list[tuple[Any, ...]]:
|
||
return [
|
||
(
|
||
item["original_content"],
|
||
item["source_start"],
|
||
item["source_end"],
|
||
item["source_start_line"],
|
||
item["source_end_line"],
|
||
)
|
||
for item in items
|
||
]
|
||
|
||
assert snapshot(default_items) == snapshot(fixed_items)
|
||
assert all(
|
||
item["original_content"]
|
||
== normalized[item["source_start"] : item["source_end"]]
|
||
for item in fixed_items
|
||
)
|
||
second_chapter_items = [
|
||
item for item in fixed_items if item["source_start"] >= second_chapter_start
|
||
]
|
||
assert second_chapter_items
|
||
|
||
|
||
def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None:
|
||
repeated = "@" * 120
|
||
assert len(_preview_task(repeated, options=[])) == 1
|
||
assert _preview_task(repeated, options=["clean_invalid_content"]) == []
|
||
|
||
mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。"
|
||
assert len(_preview_task(mojibake, options=[])) == 1
|
||
assert _preview_task(mojibake, options=["filter_low_quality"]) == []
|
||
|
||
first = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"
|
||
second = "alpha beta gamma, delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"
|
||
sources = [
|
||
{"id": "near-1", "name": "one.txt", "file_format": "txt", "content": first},
|
||
{"id": "near-2", "name": "two.txt", "file_format": "txt", "content": second},
|
||
]
|
||
base_task = {
|
||
"process_type": "unstructured",
|
||
"config": {
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 1,
|
||
"preprocess_options": [],
|
||
},
|
||
}
|
||
assert len(data_process_endpoint._build_preview_items(base_task, sources)) == 2
|
||
deduplicated_task = deepcopy(base_task)
|
||
deduplicated_task["config"]["preprocess_options"] = ["deduplicate_content"]
|
||
assert len(data_process_endpoint._build_preview_items(deduplicated_task, sources)) == 1
|
||
|
||
context_text = " ".join(f"token{index}" for index in range(45))
|
||
no_context = _preview_task(
|
||
context_text,
|
||
options=[],
|
||
config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5},
|
||
)
|
||
with_context = _preview_task(
|
||
context_text,
|
||
options=["preserve_context"],
|
||
config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5},
|
||
)
|
||
assert no_context[1]["source_start"] >= no_context[0]["source_end"]
|
||
assert with_context[1]["source_start"] < with_context[0]["source_end"]
|
||
|
||
sensitive = "联系人：张三，手机 13800138000，邮箱 user@example.com。"
|
||
plain = _preview_task(sensitive, options=[])[0]
|
||
masked = _preview_task(sensitive, options=["desensitize"])[0]
|
||
assert "张三" in plain["edited_content"]
|
||
assert "联系人:[NAME]" in masked["edited_content"]
|
||
assert "[PHONE]" in masked["edited_content"]
|
||
assert "[EMAIL]" in masked["edited_content"]
|
||
|
||
|
||
def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None:
|
||
source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。")
|
||
source = {
|
||
"id": "pdf-source",
|
||
"name": "manual.pdf",
|
||
"file_format": "pdf",
|
||
"content": source_text,
|
||
"document_noise_spans": (
|
||
DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"),
|
||
),
|
||
}
|
||
config = {
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 1,
|
||
}
|
||
|
||
cleaned_items = data_process_endpoint._build_preview_items(
|
||
{
|
||
"process_type": "unstructured",
|
||
"config": {**config, "preprocess_options": ["clean_invalid_content"]},
|
||
},
|
||
[source],
|
||
)
|
||
original_items = data_process_endpoint._build_preview_items(
|
||
{
|
||
"process_type": "unstructured",
|
||
"config": {**config, "preprocess_options": []},
|
||
},
|
||
[source],
|
||
)
|
||
|
||
assert len(cleaned_items) == 1
|
||
cleaned = cleaned_items[0]
|
||
assert cleaned["original_content"] == source_text[
|
||
cleaned["source_start"] : cleaned["source_end"]
|
||
]
|
||
assert "重复页眉" not in cleaned["edited_content"]
|
||
assert "PDF 正文内容" in cleaned["edited_content"]
|
||
assert cleaned["status"] == "modified"
|
||
assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"]
|
||
assert "重复页眉" in original_items[0]["edited_content"]
|
||
|
||
|
||
def test_merge_short_content_applies_across_adjacent_fixed_chunks() -> None:
|
||
content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
|
||
items = _preview_task(
|
||
content,
|
||
options=["merge_short_content"],
|
||
config={
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 40,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 20,
|
||
},
|
||
)
|
||
|
||
assert len(items) == 3
|
||
assert all(item["token_count"] <= 40 for item in items)
|
||
assert items[0]["source_start_line"] == 1
|
||
assert items[-1]["source_end_line"] == 16
|
||
|
||
|
||
def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None:
|
||
for file_format in ("pdf", "docx", "pptx"):
|
||
items = _preview_task(
|
||
f"{file_format.upper()} 已抽取正文，可直接进入切片处理。",
|
||
options=[],
|
||
file_format=file_format,
|
||
)
|
||
assert len(items) == 1
|
||
assert "已抽取正文" in items[0]["edited_content"]
|