fix(data-process): 保留重新生成前的已发布数据集

This commit is contained in:
caoxiaozhu
2026-07-27 09:39:19 +08:00
parent 915f994c45
commit e4ea1f168c
2 changed files with 308 additions and 16 deletions

View File

@@ -174,6 +174,7 @@ def _decode_row(row: dict[str, Any] | None) -> dict[str, Any] | None:
"metadata": {},
"quality_score": {},
"versions": [],
"output_datasets": [],
}.items():
if key in item:
item[key] = _json_value(item[key], default)
@@ -318,7 +319,12 @@ class DataProcessStore:
) ORDER BY CASE dataset.type
WHEN 'train' THEN 1 WHEN 'val' THEN 2 WHEN 'test' THEN 3 ELSE 4 END), '[]'::json)
FROM datasets dataset
WHERE dataset.source_task_id=task.id AND dataset.source='task')
WHERE dataset.source='task'
AND dataset.deleted_at IS NULL
AND (
dataset.source_task_id=task.id
OR (dataset.source_task_id IS NULL AND dataset.task_id=task.id)
))
AS output_datasets,
CASE
WHEN task.started_at IS NOT NULL AND task.completed_at IS NOT NULL
@@ -459,14 +465,30 @@ class DataProcessStore:
current_config,
next_config,
)
now = utcnow()
# 002 迁移前发布的数据集只有 task_id。先补齐新关联字段保证
# 解除任务输出指针后,详情和后续重新发布仍能定位原来的三份数据集。
conn.execute(
"""
UPDATE datasets
SET source_task_id=%s, updated_at=%s
WHERE source='task' AND source_task_id IS NULL AND task_id=%s
AND deleted_at IS NULL
""",
(task_id, now, task_id),
)
published_row = conn.execute(
"""
SELECT EXISTS(
SELECT 1 FROM datasets
WHERE source_task_id=%s AND source='task'
WHERE source='task' AND deleted_at IS NULL
AND (
source_task_id=%s
OR (source_task_id IS NULL AND task_id=%s)
)
) AS exists
""",
(task_id,),
(task_id, task_id),
).fetchone()
published_outputs_preserved = bool(task.get("output_dataset_id")) or bool(
published_row and published_row.get("exists")
@@ -493,7 +515,6 @@ class DataProcessStore:
).fetchone()
preview_count = int((preview_row or {}).get("count") or 0)
now = utcnow()
row = conn.execute(
"""
UPDATE data_process_tasks
@@ -1460,10 +1481,14 @@ class DataProcessStore:
existing_datasets = conn.execute(
"""
SELECT * FROM datasets
WHERE source_task_id=%s AND source='task'
WHERE source='task' AND deleted_at IS NULL
AND (
source_task_id=%s
OR (source_task_id IS NULL AND task_id=%s)
)
ORDER BY created_at, id
""",
(task_id,),
(task_id, task_id),
).fetchall()
existing_by_split: dict[str, dict[str, Any]] = {}
primary_existing = None