fix: 完善数据预处理与 JSON 上传链路

backend/app/modules/data_process/storage.py

@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
         self._issued_staged_objects[temporary_path] = staged
         return staged
 
+    def stage_copy(
+        self,
+        *,
+        batch_id: str,
+        source_reference: str,
+        expected_source_task_id: str,
+        expected_source_file_id: str,
+        task_id: str,
+        source_file_id: str,
+        version: int,
+        name: str,
+    ) -> StagedSourceObject:
+        """为不可变源对象创建独立目录项，不把大文件重新读入内存。"""
+
+        batch_id = _safe_component(batch_id, "batch id")
+        task_id = _safe_component(task_id, "task id")
+        source_file_id = _safe_component(source_file_id, "source file id")
+        if isinstance(version, bool) or not isinstance(version, int) or version < 1:
+            raise DataProcessStorageError("invalid source file version")
+        basename = _safe_basename(name)
+        source_relative = self._relative_from_reference(source_reference)
+        if source_relative is None:
+            raise DataProcessStorageError("original source object is not available")
+        self._assert_expected_owner(
+            source_relative,
+            expected_task_id=expected_source_task_id,
+            expected_source_file_id=expected_source_file_id,
+        )
+        descriptor, source_info = self._open_read_descriptor(source_relative)
+        os.close(descriptor)
+
+        batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
+        temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
+        source_path = self._path_for_relative(source_relative)
+        try:
+            os.link(source_path, temporary_path, follow_symlinks=False)
+            copy_info = temporary_path.lstat()
+            if (
+                not stat.S_ISREG(copy_info.st_mode)
+                or source_info.st_dev != copy_info.st_dev
+                or source_info.st_ino != copy_info.st_ino
+            ):
+                raise DataProcessStorageError("source storage object changed while copying")
+        except Exception:
+            temporary_path.unlink(missing_ok=True)
+            raise
+
+        relative_path = PurePosixPath(
+            task_id,
+            source_file_id,
+            f"v{version}",
+            basename,
+        )
+        reference = (
+            "local://data-process/"
+            f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
+        )
+        staged = StagedSourceObject(reference, temporary_path, relative_path)
+        self._issued_staged_objects[temporary_path] = staged
+        return staged
+
     def publish(self, objects: Iterable[StagedSourceObject]) -> None:
         staged = list(objects)
         published: list[StagedSourceObject] = []

backend/app/modules/data_process/store.py

@@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = {
     "preserve_lists": True,
 }
 _REGENERATION_MARKER_KEY = "_regeneration_prepared"
+_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id"
+_REPEAT_REQUEST_KEY = "_repeat_request_id"
+_INTERNAL_CONFIG_KEYS = {
+    _REGENERATION_MARKER_KEY,
+    _REPEAT_SOURCE_TASK_KEY,
+    _REPEAT_REQUEST_KEY,
+}
 
 
 class DataProcessStoreError(RuntimeError):
@@ -71,6 +78,13 @@ def new_id(prefix: str) -> str:
     return f"{prefix}_{uuid.uuid4().hex[:20]}"
 
 
+def repeat_task_id(source_task_id: str, request_id: str) -> str:
+    """按源任务和请求幂等键生成稳定的新任务 ID。"""
+
+    digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest()
+    return f"dpt_{digest[:20]}"
+
+
 def json_dumps(value: Any) -> str:
     return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
 
@@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool:
     return _regeneration_marker(task) is not None
 
 
+def _business_config(config: dict[str, Any] | None) -> dict[str, Any]:
+    """过滤只供服务端维护的工作流标记。"""
+
+    return {
+        key: value
+        for key, value in (config or {}).items()
+        if key not in _INTERNAL_CONFIG_KEYS
+    }
+
+
 def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None:
-    """从 API 任务快照中移除服务端内部重新生成标记。"""
+    """从 API 任务快照中移除服务端内部工作流标记。"""
 
     if item is None:
         return None
     public = dict(item)
     config = public.get("config")
-    if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config:
-        public["config"] = {
-            key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY
-        }
+    if isinstance(config, dict):
+        public["config"] = _business_config(config)
     return public
 
 
@@ -353,13 +375,7 @@ class DataProcessStore:
                         payload.get("description") or "",
                         payload["process_type"],
                         payload.get("source_dataset_id"),
-                        json_dumps(
-                            {
-                                key: value
-                                for key, value in (payload.get("config") or {}).items()
-                                if key != _REGENERATION_MARKER_KEY
-                            }
-                        ),
+                        json_dumps(_business_config(payload.get("config"))),
                         payload.get("tenant_id"),
                         payload.get("project_id"),
                         payload.get("owner_id"),
@@ -373,6 +389,268 @@ class DataProcessStore:
             raise ConflictError("data process task name already exists") from exc
         return _public_task(_decode_row(row)) or {}
 
+    @staticmethod
+    def _repeat_response(
+        conn: psycopg.Connection[dict[str, Any]],
+        row: dict[str, Any],
+        *,
+        source_task_id: str,
+        created: bool,
+    ) -> dict[str, Any]:
+        task_id = str(row["id"])
+        counts = conn.execute(
+            """
+            SELECT
+              (SELECT COUNT(*) FROM data_process_source_files
+               WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count,
+              (SELECT COUNT(*) FROM data_process_preview_items
+               WHERE task_id=%s) AS preview_count
+            """,
+            (task_id, task_id),
+        ).fetchone() or {}
+        task = _public_task(_decode_row(row)) or {}
+        task["source_file_count"] = int(counts.get("source_file_count") or 0)
+        task["preview_count"] = int(counts.get("preview_count") or 0)
+        return {
+            "task": task,
+            "source_task_id": source_task_id,
+            "created": created,
+            "copied_source_file_count": task["source_file_count"],
+            "copied_preview_count": task["preview_count"],
+        }
+
+    def find_repeated_task(
+        self,
+        source_task_id: str,
+        request_id: str,
+    ) -> dict[str, Any] | None:
+        """查找同一幂等请求已创建的新任务。"""
+
+        task_id = repeat_task_id(source_task_id, request_id)
+        with self.connect() as conn:
+            row = conn.execute(
+                "SELECT * FROM data_process_tasks WHERE id=%s",
+                (task_id,),
+            ).fetchone()
+            if row is None:
+                return None
+            decoded = _decode_row(row) or {}
+            config = decoded.get("config") or {}
+            if (
+                config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
+                or config.get(_REPEAT_REQUEST_KEY) != request_id
+            ):
+                raise ConflictError("再次生成请求与现有任务冲突")
+            if decoded.get("deleted_at"):
+                raise ConflictError("此次再次生成创建的任务已被删除，请重新发起")
+            return self._repeat_response(
+                conn,
+                row,
+                source_task_id=source_task_id,
+                created=False,
+            )
+
+    def repeat_task(
+        self,
+        source_task_id: str,
+        *,
+        expected_updated_at: str,
+        request_id: str,
+        file_copies: dict[str, dict[str, str]],
+    ) -> dict[str, Any]:
+        """复制已确认任务的配置、源文件和预览，结果与发布数据保持独立。"""
+
+        task_id = repeat_task_id(source_task_id, request_id)
+        now = utcnow()
+        try:
+            with self.connect() as conn:
+                existing = conn.execute(
+                    "SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE",
+                    (task_id,),
+                ).fetchone()
+                if existing is not None:
+                    decoded = _decode_row(existing) or {}
+                    config = decoded.get("config") or {}
+                    if (
+                        config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
+                        or config.get(_REPEAT_REQUEST_KEY) != request_id
+                    ):
+                        raise ConflictError("再次生成请求与现有任务冲突")
+                    if decoded.get("deleted_at"):
+                        raise ConflictError("此次再次生成创建的任务已被删除，请重新发起")
+                    return self._repeat_response(
+                        conn,
+                        existing,
+                        source_task_id=source_task_id,
+                        created=False,
+                    )
+
+                source_task = self._task_in_connection(
+                    conn,
+                    source_task_id,
+                    for_update=True,
+                )
+                if (
+                    source_task.get("status") != "completed"
+                    or source_task.get("results_confirmed") is False
+                ):
+                    raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
+                if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
+                    raise ConflictError("源任务仍在处理切分，暂时不能再次生成")
+                if expected_updated_at != _serialize_value(source_task.get("updated_at")):
+                    raise ConflictError("源任务已被其他操作修改，请刷新后重试")
+
+                source_files = conn.execute(
+                    """
+                    SELECT * FROM data_process_source_files
+                    WHERE task_id=%s AND deleted_at IS NULL
+                    ORDER BY created_at, id
+                    """,
+                    (source_task_id,),
+                ).fetchall()
+                source_file_ids = {str(row["id"]) for row in source_files}
+                if source_file_ids != set(file_copies):
+                    raise ConflictError("源文件快照已变化，请刷新后重试")
+                previews = conn.execute(
+                    """
+                    SELECT * FROM data_process_preview_items
+                    WHERE task_id=%s
+                    ORDER BY source_file_id NULLS LAST, source_start NULLS LAST,
+                             created_at, id
+                    """,
+                    (source_task_id,),
+                ).fetchall()
+                if not previews:
+                    raise InvalidStateError("源任务没有可用于再次生成的切分结果")
+
+                suffix = f"（再次生成-{task_id[-6:]}）"
+                base_name = str(source_task.get("name") or "数据处理任务")
+                repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}"
+                repeated_config = _business_config(source_task.get("config") or {})
+                repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id
+                repeated_config[_REPEAT_REQUEST_KEY] = request_id
+                input_count = sum(int(row.get("record_count") or 0) for row in source_files)
+                task_row = conn.execute(
+                    """
+                    INSERT INTO data_process_tasks
+                    (id, name, description, status, process_type, source_dataset_id,
+                     output_dataset_id, config, progress, input_count, output_count,
+                     filtered_count, duplicate_count, error_count, failure_reason,
+                     generation_run_id, results_confirmed, workflow_step,
+                     preview_status, preview_progress, preview_run_id,
+                     preview_failure_reason, preview_total_files,
+                     preview_completed_files, tenant_id, project_id, owner_id,
+                     approval_status, created_by, updated_by, created_at, updated_at)
+                    VALUES
+                    (%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0,
+                     0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100,
+                     NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                    RETURNING *
+                    """,
+                    (
+                        task_id,
+                        repeated_name,
+                        source_task.get("description") or "",
+                        source_task["process_type"],
+                        source_task.get("source_dataset_id"),
+                        json_dumps(repeated_config),
+                        input_count,
+                        len(source_files),
+                        len(source_files),
+                        source_task.get("tenant_id"),
+                        source_task.get("project_id"),
+                        source_task.get("owner_id"),
+                        source_task.get("approval_status") or "not_required",
+                        source_task.get("created_by"),
+                        source_task.get("created_by"),
+                        now,
+                        now,
+                    ),
+                ).fetchone()
+
+                file_id_map: dict[str, str] = {}
+                for source in source_files:
+                    old_file_id = str(source["id"])
+                    copy = file_copies[old_file_id]
+                    new_file_id = str(copy["id"])
+                    storage_object_id, metadata = _source_storage_descriptor(
+                        {
+                            "storage_object_id": copy["storage_object_id"],
+                            "metadata": _json_value(source.get("metadata"), {}),
+                        },
+                        task_id,
+                        new_file_id,
+                    )
+                    file_id_map[old_file_id] = new_file_id
+                    conn.execute(
+                        """
+                        INSERT INTO data_process_source_files
+                        (id, task_id, storage_object_id, name, size_bytes, record_count,
+                         file_format, checksum_sha256, version_no, content,
+                         content_preview, metadata, tenant_id, project_id, created_by,
+                         created_at, updated_at)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s,
+                                %s, %s, %s, %s, %s)
+                        """,
+                        (
+                            new_file_id,
+                            task_id,
+                            storage_object_id,
+                            source["name"],
+                            source.get("size_bytes") or 0,
+                            source.get("record_count") or 0,
+                            source.get("file_format"),
+                            source["checksum_sha256"],
+                            source.get("content") or "",
+                            source.get("content_preview"),
+                            json_dumps(metadata),
+                            source_task.get("tenant_id"),
+                            source_task.get("project_id"),
+                            source.get("created_by") or source_task.get("created_by"),
+                            now,
+                            now,
+                        ),
+                    )
+
+                for preview in previews:
+                    old_source_file_id = preview.get("source_file_id")
+                    conn.execute(
+                        """
+                        INSERT INTO data_process_preview_items
+                        (id, task_id, source_file_id, original_content, edited_content,
+                         source_start, source_end, source_start_line, source_end_line,
+                         token_count, status, quality_score, created_at, updated_at)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
+                                %s, %s)
+                        """,
+                        (
+                            new_id("dpp"),
+                            task_id,
+                            file_id_map.get(str(old_source_file_id))
+                            if old_source_file_id
+                            else None,
+                            preview.get("original_content") or "",
+                            preview.get("edited_content") or "",
+                            preview.get("source_start"),
+                            preview.get("source_end"),
+                            preview.get("source_start_line"),
+                            preview.get("source_end_line"),
+                            max(0, int(preview.get("token_count") or 0)),
+                            preview.get("status") or "original",
+                            json_dumps(_json_value(preview.get("quality_score"), {})),
+                            now,
+                            now,
+                        ),
+                    )
+                return self._repeat_response(
+                    conn,
+                    task_row or {},
+                    source_task_id=source_task_id,
+                    created=True,
+                )
+        except psycopg.errors.UniqueViolation as exc:
+            raise ConflictError("再次生成任务名称或请求发生冲突，请重试") from exc
+
     def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]:
         lock = " FOR UPDATE" if for_update else ""
         with self.connect() as conn:
@@ -654,14 +932,11 @@ class DataProcessStore:
                         "process type and source dataset cannot change during regeneration"
                     )
                 if payload.get("config") is not None:
-                    next_config = {
-                        key: value
-                        for key, value in payload["config"].items()
-                        if key != _REGENERATION_MARKER_KEY
-                    }
-                    current_marker = _regeneration_marker(task)
-                    if current_marker:
-                        next_config[_REGENERATION_MARKER_KEY] = current_marker
+                    next_config = _business_config(payload["config"])
+                    current_config = dict(task.get("config") or {})
+                    for key in _INTERNAL_CONFIG_KEYS:
+                        if key in current_config:
+                            next_config[key] = current_config[key]
                     values["config"] = json_dumps(next_config)
                 invalidates_results = (
                     ("config" in payload and payload.get("config") != task.get("config"))
@@ -753,8 +1028,10 @@ class DataProcessStore:
                     raise InvalidStateError("process_type cannot be changed during regeneration")
 
                 current_config = dict(task.get("config") or {})
-                next_config = dict(payload.get("config") or {})
-                next_config.pop(_REGENERATION_MARKER_KEY, None)
+                next_config = _business_config(payload.get("config"))
+                for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY):
+                    if key in current_config:
+                        next_config[key] = current_config[key]
                 preview_invalidated = _preview_config_changed(
                     process_type,
                     current_config,