feat(data-process): 支持任务重新生成
This commit is contained in:
@@ -82,6 +82,43 @@ class FakeDataProcessStore:
|
||||
self.tasks[task_id].update(deepcopy(payload))
|
||||
return self.get_task(task_id)
|
||||
|
||||
def prepare_regeneration(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task = self.tasks.get(task_id)
|
||||
if task is None:
|
||||
raise NotFoundError("data process task not found")
|
||||
if task["status"] == "running":
|
||||
raise InvalidStateError("running task cannot be prepared for regeneration")
|
||||
if payload["expected_updated_at"] != task.get("updated_at"):
|
||||
raise InvalidStateError("data process task was modified by another request")
|
||||
if payload["process_type"] != task["process_type"]:
|
||||
raise InvalidStateError("process_type cannot be changed during regeneration")
|
||||
published_outputs_preserved = bool(task.get("output_dataset_id"))
|
||||
self.results[task_id] = []
|
||||
task.update(
|
||||
{
|
||||
"name": payload["name"],
|
||||
"description": payload["description"],
|
||||
"config": deepcopy(payload["config"]),
|
||||
"status": "pending",
|
||||
"progress": 20,
|
||||
"output_dataset_id": None,
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"generation_run_id": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
"updated_at": "2026-07-25T20:00:00Z",
|
||||
}
|
||||
)
|
||||
return {
|
||||
"task": deepcopy(task),
|
||||
"preview_invalidated": False,
|
||||
"published_outputs_preserved": published_outputs_preserved,
|
||||
}
|
||||
|
||||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||||
self.get_task(task_id)
|
||||
if self.tasks[task_id]["status"] == "running":
|
||||
@@ -133,7 +170,23 @@ class FakeDataProcessStore:
|
||||
if checksum in checksums or checksum in incoming:
|
||||
raise ValueError("the same source file content is already attached to this task")
|
||||
incoming.add(checksum)
|
||||
return [self.add_source_file(task_id, **payload) for payload in files]
|
||||
created = [self.add_source_file(task_id, **payload) for payload in files]
|
||||
self.results[task_id] = []
|
||||
self.tasks[task_id].update(
|
||||
{
|
||||
"status": "pending",
|
||||
"progress": 20 if self.previews[task_id] else 0,
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"generation_run_id": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
}
|
||||
)
|
||||
return created
|
||||
|
||||
def get_source_file(
|
||||
self, task_id: str, file_id: str, *, include_content: bool = True
|
||||
@@ -769,6 +822,81 @@ def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
|
||||
assert response.json()["detail"]["code"] == 501
|
||||
|
||||
|
||||
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "待重新生成", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id].update(
|
||||
{
|
||||
"status": "completed",
|
||||
"updated_at": "2026-07-25T19:00:00Z",
|
||||
"output_dataset_id": "dataset_train",
|
||||
"output_count": 2,
|
||||
}
|
||||
)
|
||||
store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}]
|
||||
store.results[task_id] = [{"id": "result_1"}]
|
||||
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/regenerate",
|
||||
json={
|
||||
"name": "重新生成后名称",
|
||||
"description": "更换生成模型",
|
||||
"process_type": "structured",
|
||||
"config": {"generation_model_id": "model_2"},
|
||||
"expected_updated_at": "2026-07-25T19:00:00Z",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()["data"]
|
||||
assert data["task"]["status"] == "pending"
|
||||
assert data["task"]["output_dataset_id"] is None
|
||||
assert data["preview_invalidated"] is False
|
||||
assert data["published_outputs_preserved"] is True
|
||||
assert store.results[task_id] == []
|
||||
assert store.previews[task_id][0]["id"] == "preview_1"
|
||||
|
||||
|
||||
def test_regenerate_endpoint_validates_snapshot_and_locked_process_type(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "并发校验", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id]["updated_at"] = "2026-07-25T19:00:00Z"
|
||||
payload = {
|
||||
"name": "并发校验",
|
||||
"description": "",
|
||||
"process_type": "structured",
|
||||
"config": {},
|
||||
"expected_updated_at": "stale",
|
||||
}
|
||||
|
||||
stale = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
|
||||
assert stale.status_code == 409
|
||||
|
||||
payload.update(
|
||||
{
|
||||
"process_type": "unstructured",
|
||||
"expected_updated_at": "2026-07-25T19:00:00Z",
|
||||
}
|
||||
)
|
||||
locked_type = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
|
||||
assert locked_type.status_code == 409
|
||||
|
||||
for missing_field in ("expected_updated_at", "description", "config"):
|
||||
missing_required_field = client.post(
|
||||
f"/modelTF/data-process/{task_id}/regenerate",
|
||||
json={key: value for key, value in payload.items() if key != missing_field},
|
||||
)
|
||||
assert missing_required_field.status_code == 422
|
||||
|
||||
|
||||
def test_config_validation_and_stop_state(tmp_path: Path) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
invalid = client.post(
|
||||
@@ -855,6 +983,39 @@ def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) ->
|
||||
assert _stored_files(storage) == []
|
||||
|
||||
|
||||
def test_incremental_upload_keeps_existing_file_previews(tmp_path: Path) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "增量上传预览", "process_type": "unstructured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
first_batch = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("first.txt", "第一个文件内容".encode(), "text/plain")),
|
||||
("files", ("second.txt", "第二个文件内容".encode(), "text/plain")),
|
||||
],
|
||||
)
|
||||
assert first_batch.status_code == 200
|
||||
first_source_ids = {item["id"] for item in first_batch.json()["data"]["files"]}
|
||||
built = client.post(f"/modelTF/data-process/{task_id}/preview/build", json={})
|
||||
assert built.status_code == 200
|
||||
assert {item["source_file_id"] for item in built.json()["data"]["items"]} == first_source_ids
|
||||
store.results[task_id] = [{"id": "old_result"}]
|
||||
|
||||
third = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("third.txt", "第三个文件内容".encode(), "text/plain")},
|
||||
)
|
||||
|
||||
assert third.status_code == 200
|
||||
previews = client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]
|
||||
assert previews["total"] == 2
|
||||
assert {item["source_file_id"] for item in previews["items"]} == first_source_ids
|
||||
assert store.results[task_id] == []
|
||||
assert store.tasks[task_id]["progress"] == 20
|
||||
|
||||
|
||||
def test_upload_preserves_store_error_when_storage_rollback_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch: Any,
|
||||
|
||||
Reference in New Issue
Block a user