fix(data-process): 延迟重新生成破坏性变更

This commit is contained in:
caoxiaozhu
2026-07-27 10:43:42 +08:00
parent 03bd0b6d03
commit 53014bb381
3 changed files with 299 additions and 143 deletions

View File

@@ -225,18 +225,7 @@ class _RegenerationConnection:
"name": params[0],
"description": params[1],
"config": params[2],
"status": "pending",
"progress": params[3],
"output_dataset_id": None,
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"started_at": None,
"completed_at": None,
"updated_at": params[4],
"updated_at": params[3],
}
)
return _Result(row=dict(self.task))
@@ -369,39 +358,50 @@ class _TaskListStore(DataProcessStore):
class _StartGenerationConnection:
def __init__(self) -> None:
def __init__(self, *, published_prepared: bool = False, preview_count: int = 1) -> None:
config = {"chunk_method": "fixed", "temperature": 0.7}
if published_prepared:
config["_regeneration_prepared"] = {
"prepared": True,
"preview_invalidated": False,
}
self.task = {
**_regeneration_task(
status="pending",
output_dataset_id=None,
status="completed" if published_prepared else "pending",
config=config,
output_dataset_id="dataset_train" if published_prepared else None,
output_count=28,
),
"generation_run_id": None,
}
self.preview_count = preview_count
self.results = [{"id": "old-result"}]
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if normalized.startswith("SELECT COUNT(*) AS count FROM data_process_preview_items"):
return _Result(row={"count": 1})
return _Result(row={"count": self.preview_count})
if normalized.startswith("DELETE FROM data_process_results"):
self.results.clear()
return _Result()
assert normalized.startswith("UPDATE data_process_tasks SET status='running'")
assert normalized.startswith("UPDATE data_process_tasks SET config=%s, status='running'")
assert "output_dataset_id=NULL" in normalized
assert "output_count=0" in normalized
self.task.update(
{
"config": params[0],
"status": "running",
"progress": 30,
"output_count": 0,
"output_dataset_id": None,
"failure_reason": None,
"started_at": params[0],
"started_at": params[1],
"completed_at": None,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"generation_run_id": params[1],
"updated_at": params[2],
"generation_run_id": params[2],
"updated_at": params[3],
}
)
return _Result(row=dict(self.task))
@@ -591,6 +591,31 @@ def test_start_generation_clears_previous_output_count() -> None:
assert conn.results == []
def test_prepared_published_task_is_only_cleared_when_generation_starts() -> None:
conn = _StartGenerationConnection(published_prepared=True)
task = _StartGenerationStore(conn).start_generation("task-1")
assert task["status"] == "running"
assert task["output_dataset_id"] is None
assert task["output_count"] == 0
assert "_regeneration_prepared" not in task["config"]
assert conn.results == []
def test_prepared_published_task_survives_generation_preflight_failure() -> None:
conn = _StartGenerationConnection(published_prepared=True, preview_count=0)
with pytest.raises(InvalidStateError, match="preview must be built"):
_StartGenerationStore(conn).start_generation("task-1")
assert conn.task["status"] == "completed"
assert conn.task["output_dataset_id"] == "dataset_train"
assert conn.task["output_count"] == 28
assert "_regeneration_prepared" in conn.task["config"]
assert conn.results == [{"id": "old-result"}]
def _legacy_published_datasets(task_id: str = "task-1") -> list[dict[str, Any]]:
specs = (
("dataset_train", "制度问答-训练集", "train", "train", 22),
@@ -640,13 +665,16 @@ def test_prepare_regeneration_preserves_outputs_sources_and_generation_only_prev
assert result["preview_invalidated"] is False
assert result["published_outputs_preserved"] is True
assert result["task"]["output_dataset_id"] is None
assert result["task"]["status"] == "pending"
assert result["task"]["progress"] == 20
assert result["task"]["output_count"] == 0
assert result["task"]["started_at"] is None
assert result["task"]["completed_at"] is None
assert conn.results == []
assert result["task"]["output_dataset_id"] == "dataset_train"
assert result["task"]["status"] == "completed"
assert result["task"]["progress"] == 100
assert result["task"]["output_count"] == 28
assert result["task"]["started_at"] == "2026-07-25T18:00:00Z"
assert result["task"]["completed_at"] == "2026-07-25T18:05:00Z"
assert "_regeneration_prepared" not in result["task"]["config"]
stored_config = json.loads(conn.task["config"])
assert stored_config["_regeneration_prepared"]["prepared"] is True
assert conn.results == [{"id": "result_1"}]
assert conn.previews == [{"id": "preview_1"}]
assert conn.datasets == original_datasets
assert conn.sources == original_sources
@@ -694,7 +722,7 @@ def test_prepare_regeneration_backfills_and_keeps_legacy_task_datasets_visible()
)
assert result["published_outputs_preserved"] is True
assert result["task"]["output_dataset_id"] is None
assert result["task"]["output_dataset_id"] == "dataset_train"
assert len(conn.datasets) == 5
assert all(
item["source_task_id"] == "task-1" for item in conn.datasets[:3]
@@ -712,7 +740,7 @@ def test_prepare_regeneration_backfills_and_keeps_legacy_task_datasets_visible()
]
def test_prepare_regeneration_deletes_preview_when_chunk_configuration_changes() -> None:
def test_prepare_regeneration_defers_preview_deletion_when_chunk_configuration_changes() -> None:
conn = _RegenerationConnection(_regeneration_task(output_dataset_id=None))
result = _RegenerationStore(conn).prepare_regeneration(
@@ -728,9 +756,11 @@ def test_prepare_regeneration_deletes_preview_when_chunk_configuration_changes()
assert result["preview_invalidated"] is True
assert result["published_outputs_preserved"] is True
assert result["task"]["progress"] == 0
assert conn.previews == []
assert conn.results == []
assert result["task"]["status"] == "completed"
assert result["task"]["progress"] == 100
assert result["task"]["output_count"] == 28
assert conn.previews == [{"id": "preview_1"}]
assert conn.results == [{"id": "result_1"}]
assert len(conn.datasets) == 3