fix(data-process): 保留重新生成前的已发布数据集
This commit is contained in:
@@ -174,6 +174,7 @@ def _decode_row(row: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
"metadata": {},
|
||||
"quality_score": {},
|
||||
"versions": [],
|
||||
"output_datasets": [],
|
||||
}.items():
|
||||
if key in item:
|
||||
item[key] = _json_value(item[key], default)
|
||||
@@ -318,7 +319,12 @@ class DataProcessStore:
|
||||
) ORDER BY CASE dataset.type
|
||||
WHEN 'train' THEN 1 WHEN 'val' THEN 2 WHEN 'test' THEN 3 ELSE 4 END), '[]'::json)
|
||||
FROM datasets dataset
|
||||
WHERE dataset.source_task_id=task.id AND dataset.source='task')
|
||||
WHERE dataset.source='task'
|
||||
AND dataset.deleted_at IS NULL
|
||||
AND (
|
||||
dataset.source_task_id=task.id
|
||||
OR (dataset.source_task_id IS NULL AND dataset.task_id=task.id)
|
||||
))
|
||||
AS output_datasets,
|
||||
CASE
|
||||
WHEN task.started_at IS NOT NULL AND task.completed_at IS NOT NULL
|
||||
@@ -459,14 +465,30 @@ class DataProcessStore:
|
||||
current_config,
|
||||
next_config,
|
||||
)
|
||||
now = utcnow()
|
||||
# 002 迁移前发布的数据集只有 task_id。先补齐新关联字段,保证
|
||||
# 解除任务输出指针后,详情和后续重新发布仍能定位原来的三份数据集。
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE datasets
|
||||
SET source_task_id=%s, updated_at=%s
|
||||
WHERE source='task' AND source_task_id IS NULL AND task_id=%s
|
||||
AND deleted_at IS NULL
|
||||
""",
|
||||
(task_id, now, task_id),
|
||||
)
|
||||
published_row = conn.execute(
|
||||
"""
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM datasets
|
||||
WHERE source_task_id=%s AND source='task'
|
||||
WHERE source='task' AND deleted_at IS NULL
|
||||
AND (
|
||||
source_task_id=%s
|
||||
OR (source_task_id IS NULL AND task_id=%s)
|
||||
)
|
||||
) AS exists
|
||||
""",
|
||||
(task_id,),
|
||||
(task_id, task_id),
|
||||
).fetchone()
|
||||
published_outputs_preserved = bool(task.get("output_dataset_id")) or bool(
|
||||
published_row and published_row.get("exists")
|
||||
@@ -493,7 +515,6 @@ class DataProcessStore:
|
||||
).fetchone()
|
||||
preview_count = int((preview_row or {}).get("count") or 0)
|
||||
|
||||
now = utcnow()
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
@@ -1460,10 +1481,14 @@ class DataProcessStore:
|
||||
existing_datasets = conn.execute(
|
||||
"""
|
||||
SELECT * FROM datasets
|
||||
WHERE source_task_id=%s AND source='task'
|
||||
WHERE source='task' AND deleted_at IS NULL
|
||||
AND (
|
||||
source_task_id=%s
|
||||
OR (source_task_id IS NULL AND task_id=%s)
|
||||
)
|
||||
ORDER BY created_at, id
|
||||
""",
|
||||
(task_id,),
|
||||
(task_id, task_id),
|
||||
).fetchall()
|
||||
existing_by_split: dict[str, dict[str, Any]] = {}
|
||||
primary_existing = None
|
||||
|
||||
Reference in New Issue
Block a user