feat(data-process): 支持任务重新生成

This commit is contained in:
caoxiaozhu
2026-07-25 22:40:55 +08:00
parent 64d7414b04
commit 396d3f6f47
5 changed files with 667 additions and 4 deletions

View File

@@ -82,6 +82,43 @@ class FakeDataProcessStore:
self.tasks[task_id].update(deepcopy(payload))
return self.get_task(task_id)
def prepare_regeneration(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
task = self.tasks.get(task_id)
if task is None:
raise NotFoundError("data process task not found")
if task["status"] == "running":
raise InvalidStateError("running task cannot be prepared for regeneration")
if payload["expected_updated_at"] != task.get("updated_at"):
raise InvalidStateError("data process task was modified by another request")
if payload["process_type"] != task["process_type"]:
raise InvalidStateError("process_type cannot be changed during regeneration")
published_outputs_preserved = bool(task.get("output_dataset_id"))
self.results[task_id] = []
task.update(
{
"name": payload["name"],
"description": payload["description"],
"config": deepcopy(payload["config"]),
"status": "pending",
"progress": 20,
"output_dataset_id": None,
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"started_at": None,
"completed_at": None,
"updated_at": "2026-07-25T20:00:00Z",
}
)
return {
"task": deepcopy(task),
"preview_invalidated": False,
"published_outputs_preserved": published_outputs_preserved,
}
def delete_task(self, task_id: str, **_: Any) -> None:
self.get_task(task_id)
if self.tasks[task_id]["status"] == "running":
@@ -133,7 +170,23 @@ class FakeDataProcessStore:
if checksum in checksums or checksum in incoming:
raise ValueError("the same source file content is already attached to this task")
incoming.add(checksum)
return [self.add_source_file(task_id, **payload) for payload in files]
created = [self.add_source_file(task_id, **payload) for payload in files]
self.results[task_id] = []
self.tasks[task_id].update(
{
"status": "pending",
"progress": 20 if self.previews[task_id] else 0,
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"started_at": None,
"completed_at": None,
}
)
return created
def get_source_file(
self, task_id: str, file_id: str, *, include_content: bool = True
@@ -769,6 +822,81 @@ def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
assert response.json()["detail"]["code"] == 501
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "待重新生成", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
{
"status": "completed",
"updated_at": "2026-07-25T19:00:00Z",
"output_dataset_id": "dataset_train",
"output_count": 2,
}
)
store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}]
store.results[task_id] = [{"id": "result_1"}]
response = client.post(
f"/modelTF/data-process/{task_id}/regenerate",
json={
"name": "重新生成后名称",
"description": "更换生成模型",
"process_type": "structured",
"config": {"generation_model_id": "model_2"},
"expected_updated_at": "2026-07-25T19:00:00Z",
},
)
assert response.status_code == 200
data = response.json()["data"]
assert data["task"]["status"] == "pending"
assert data["task"]["output_dataset_id"] is None
assert data["preview_invalidated"] is False
assert data["published_outputs_preserved"] is True
assert store.results[task_id] == []
assert store.previews[task_id][0]["id"] == "preview_1"
def test_regenerate_endpoint_validates_snapshot_and_locked_process_type(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "并发校验", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id]["updated_at"] = "2026-07-25T19:00:00Z"
payload = {
"name": "并发校验",
"description": "",
"process_type": "structured",
"config": {},
"expected_updated_at": "stale",
}
stale = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
assert stale.status_code == 409
payload.update(
{
"process_type": "unstructured",
"expected_updated_at": "2026-07-25T19:00:00Z",
}
)
locked_type = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
assert locked_type.status_code == 409
for missing_field in ("expected_updated_at", "description", "config"):
missing_required_field = client.post(
f"/modelTF/data-process/{task_id}/regenerate",
json={key: value for key, value in payload.items() if key != missing_field},
)
assert missing_required_field.status_code == 422
def test_config_validation_and_stop_state(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
invalid = client.post(
@@ -855,6 +983,39 @@ def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) ->
assert _stored_files(storage) == []
def test_incremental_upload_keeps_existing_file_previews(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "增量上传预览", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
first_batch = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("first.txt", "第一个文件内容".encode(), "text/plain")),
("files", ("second.txt", "第二个文件内容".encode(), "text/plain")),
],
)
assert first_batch.status_code == 200
first_source_ids = {item["id"] for item in first_batch.json()["data"]["files"]}
built = client.post(f"/modelTF/data-process/{task_id}/preview/build", json={})
assert built.status_code == 200
assert {item["source_file_id"] for item in built.json()["data"]["items"]} == first_source_ids
store.results[task_id] = [{"id": "old_result"}]
third = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("third.txt", "第三个文件内容".encode(), "text/plain")},
)
assert third.status_code == 200
previews = client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]
assert previews["total"] == 2
assert {item["source_file_id"] for item in previews["items"]} == first_source_ids
assert store.results[task_id] == []
assert store.tasks[task_id]["progress"] == 20
def test_upload_preserves_store_error_when_storage_rollback_fails(
tmp_path: Path,
monkeypatch: Any,