fix(data-process): 延迟重新生成破坏性变更
This commit is contained in:
@@ -225,18 +225,7 @@ class _RegenerationConnection:
|
||||
"name": params[0],
|
||||
"description": params[1],
|
||||
"config": params[2],
|
||||
"status": "pending",
|
||||
"progress": params[3],
|
||||
"output_dataset_id": None,
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"generation_run_id": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
"updated_at": params[4],
|
||||
"updated_at": params[3],
|
||||
}
|
||||
)
|
||||
return _Result(row=dict(self.task))
|
||||
@@ -369,39 +358,50 @@ class _TaskListStore(DataProcessStore):
|
||||
|
||||
|
||||
class _StartGenerationConnection:
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, *, published_prepared: bool = False, preview_count: int = 1) -> None:
|
||||
config = {"chunk_method": "fixed", "temperature": 0.7}
|
||||
if published_prepared:
|
||||
config["_regeneration_prepared"] = {
|
||||
"prepared": True,
|
||||
"preview_invalidated": False,
|
||||
}
|
||||
self.task = {
|
||||
**_regeneration_task(
|
||||
status="pending",
|
||||
output_dataset_id=None,
|
||||
status="completed" if published_prepared else "pending",
|
||||
config=config,
|
||||
output_dataset_id="dataset_train" if published_prepared else None,
|
||||
output_count=28,
|
||||
),
|
||||
"generation_run_id": None,
|
||||
}
|
||||
self.preview_count = preview_count
|
||||
self.results = [{"id": "old-result"}]
|
||||
|
||||
def execute(self, sql: str, params: Any = None) -> _Result:
|
||||
normalized = " ".join(sql.split())
|
||||
if normalized.startswith("SELECT COUNT(*) AS count FROM data_process_preview_items"):
|
||||
return _Result(row={"count": 1})
|
||||
return _Result(row={"count": self.preview_count})
|
||||
if normalized.startswith("DELETE FROM data_process_results"):
|
||||
self.results.clear()
|
||||
return _Result()
|
||||
assert normalized.startswith("UPDATE data_process_tasks SET status='running'")
|
||||
assert normalized.startswith("UPDATE data_process_tasks SET config=%s, status='running'")
|
||||
assert "output_dataset_id=NULL" in normalized
|
||||
assert "output_count=0" in normalized
|
||||
self.task.update(
|
||||
{
|
||||
"config": params[0],
|
||||
"status": "running",
|
||||
"progress": 30,
|
||||
"output_count": 0,
|
||||
"output_dataset_id": None,
|
||||
"failure_reason": None,
|
||||
"started_at": params[0],
|
||||
"started_at": params[1],
|
||||
"completed_at": None,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"generation_run_id": params[1],
|
||||
"updated_at": params[2],
|
||||
"generation_run_id": params[2],
|
||||
"updated_at": params[3],
|
||||
}
|
||||
)
|
||||
return _Result(row=dict(self.task))
|
||||
@@ -591,6 +591,31 @@ def test_start_generation_clears_previous_output_count() -> None:
|
||||
assert conn.results == []
|
||||
|
||||
|
||||
def test_prepared_published_task_is_only_cleared_when_generation_starts() -> None:
|
||||
conn = _StartGenerationConnection(published_prepared=True)
|
||||
|
||||
task = _StartGenerationStore(conn).start_generation("task-1")
|
||||
|
||||
assert task["status"] == "running"
|
||||
assert task["output_dataset_id"] is None
|
||||
assert task["output_count"] == 0
|
||||
assert "_regeneration_prepared" not in task["config"]
|
||||
assert conn.results == []
|
||||
|
||||
|
||||
def test_prepared_published_task_survives_generation_preflight_failure() -> None:
|
||||
conn = _StartGenerationConnection(published_prepared=True, preview_count=0)
|
||||
|
||||
with pytest.raises(InvalidStateError, match="preview must be built"):
|
||||
_StartGenerationStore(conn).start_generation("task-1")
|
||||
|
||||
assert conn.task["status"] == "completed"
|
||||
assert conn.task["output_dataset_id"] == "dataset_train"
|
||||
assert conn.task["output_count"] == 28
|
||||
assert "_regeneration_prepared" in conn.task["config"]
|
||||
assert conn.results == [{"id": "old-result"}]
|
||||
|
||||
|
||||
def _legacy_published_datasets(task_id: str = "task-1") -> list[dict[str, Any]]:
|
||||
specs = (
|
||||
("dataset_train", "制度问答-训练集", "train", "train", 22),
|
||||
@@ -640,13 +665,16 @@ def test_prepare_regeneration_preserves_outputs_sources_and_generation_only_prev
|
||||
|
||||
assert result["preview_invalidated"] is False
|
||||
assert result["published_outputs_preserved"] is True
|
||||
assert result["task"]["output_dataset_id"] is None
|
||||
assert result["task"]["status"] == "pending"
|
||||
assert result["task"]["progress"] == 20
|
||||
assert result["task"]["output_count"] == 0
|
||||
assert result["task"]["started_at"] is None
|
||||
assert result["task"]["completed_at"] is None
|
||||
assert conn.results == []
|
||||
assert result["task"]["output_dataset_id"] == "dataset_train"
|
||||
assert result["task"]["status"] == "completed"
|
||||
assert result["task"]["progress"] == 100
|
||||
assert result["task"]["output_count"] == 28
|
||||
assert result["task"]["started_at"] == "2026-07-25T18:00:00Z"
|
||||
assert result["task"]["completed_at"] == "2026-07-25T18:05:00Z"
|
||||
assert "_regeneration_prepared" not in result["task"]["config"]
|
||||
stored_config = json.loads(conn.task["config"])
|
||||
assert stored_config["_regeneration_prepared"]["prepared"] is True
|
||||
assert conn.results == [{"id": "result_1"}]
|
||||
assert conn.previews == [{"id": "preview_1"}]
|
||||
assert conn.datasets == original_datasets
|
||||
assert conn.sources == original_sources
|
||||
@@ -694,7 +722,7 @@ def test_prepare_regeneration_backfills_and_keeps_legacy_task_datasets_visible()
|
||||
)
|
||||
|
||||
assert result["published_outputs_preserved"] is True
|
||||
assert result["task"]["output_dataset_id"] is None
|
||||
assert result["task"]["output_dataset_id"] == "dataset_train"
|
||||
assert len(conn.datasets) == 5
|
||||
assert all(
|
||||
item["source_task_id"] == "task-1" for item in conn.datasets[:3]
|
||||
@@ -712,7 +740,7 @@ def test_prepare_regeneration_backfills_and_keeps_legacy_task_datasets_visible()
|
||||
]
|
||||
|
||||
|
||||
def test_prepare_regeneration_deletes_preview_when_chunk_configuration_changes() -> None:
|
||||
def test_prepare_regeneration_defers_preview_deletion_when_chunk_configuration_changes() -> None:
|
||||
conn = _RegenerationConnection(_regeneration_task(output_dataset_id=None))
|
||||
|
||||
result = _RegenerationStore(conn).prepare_regeneration(
|
||||
@@ -728,9 +756,11 @@ def test_prepare_regeneration_deletes_preview_when_chunk_configuration_changes()
|
||||
|
||||
assert result["preview_invalidated"] is True
|
||||
assert result["published_outputs_preserved"] is True
|
||||
assert result["task"]["progress"] == 0
|
||||
assert conn.previews == []
|
||||
assert conn.results == []
|
||||
assert result["task"]["status"] == "completed"
|
||||
assert result["task"]["progress"] == 100
|
||||
assert result["task"]["output_count"] == 28
|
||||
assert conn.previews == [{"id": "preview_1"}]
|
||||
assert conn.results == [{"id": "result_1"}]
|
||||
assert len(conn.datasets) == 3
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user