"""数据处理存储层 - 预览管理。""" from __future__ import annotations from typing import Any from collections.abc import Sequence import psycopg from .base import ( StoreBase, utcnow, new_id, repeat_task_id, json_dumps, _json_value, _decode_row, _public_task, _business_config, _preview_config_value, _preview_config_changed, _preview_config_projection, _normalized_preprocess_options, _regeneration_marker, _is_regeneration_prepared, _task_output_type, _task_reasoning_detail, _reasoning_output_is_valid, _dpo_fields_are_valid, _source_storage_descriptor, _serialize_value, NotFoundError, ConflictError, InvalidStateError, EDITABLE_STATUSES, ACTIVE_PREVIEW_STATUSES, WORKFLOW_STEPS, _REGENERATION_MARKER_KEY, _REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY, ) from ..algorithms import estimate_token_count # noqa: E402 class PreviewMixin: """预览管理 Mixin。""" def replace_preview_items( self, task_id: str, items: Sequence[dict[str, Any]], *, source_file_ids: Sequence[str] | None = None, preview_run_id: str | None = None, ) -> list[dict[str, Any]]: selected_ids = ( list(dict.fromkeys(str(file_id) for file_id in source_file_ids)) if source_file_ids is not None else None ) if selected_ids is not None: if not selected_ids or any(not file_id for file_id in selected_ids): raise ValueError("source_file_ids must contain non-empty ids") selected_set = set(selected_ids) unexpected = { str(item.get("source_file_id") or "") for item in items if str(item.get("source_file_id") or "") not in selected_set } if unexpected: raise ValueError("preview items contain an unselected source file") preview_file_count = len(selected_ids) if selected_ids is not None else len( {str(item.get("source_file_id") or "") for item in items} ) is_direct_build = preview_run_id is None now = utcnow() with self.connect() as conn: task = self._task_in_connection(conn, task_id, for_update=True) if is_direct_build: self._ensure_editable(task) elif ( task.get("preview_run_id") != preview_run_id or task.get("preview_status") != "running" ): raise InvalidStateError("preview run is no longer active") regeneration_prepared = _is_regeneration_prepared(task) if not regeneration_prepared: conn.execute("DELETE FROM data_process_results WHERE task_id=%s", (task_id,)) if selected_ids is None: conn.execute( "DELETE FROM data_process_preview_items WHERE task_id=%s", (task_id,) ) else: rows = conn.execute( """ SELECT id FROM data_process_source_files WHERE task_id=%s AND deleted_at IS NULL AND id=ANY(%s) """, (task_id, selected_ids), ).fetchall() found = {str(row["id"]) for row in rows} missing = set(selected_ids) - found if missing: raise NotFoundError( f"source files not found: {', '.join(sorted(missing))}" ) conn.execute( """ DELETE FROM data_process_preview_items WHERE task_id=%s AND source_file_id=ANY(%s) """, (task_id, selected_ids), ) created: list[dict[str, Any]] = [] for item in items: row = conn.execute( """ INSERT INTO data_process_preview_items (id, task_id, source_file_id, original_content, edited_content, source_start, source_end, source_start_line, source_end_line, token_count, status, quality_score, created_at, updated_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING * """, ( item.get("id") or new_id("dpp"), task_id, item.get("source_file_id"), item.get("original_content") or "", item.get("edited_content", item.get("original_content") or ""), item.get("source_start"), item.get("source_end"), item.get("source_start_line"), item.get("source_end_line"), max(0, int(item.get("token_count") or 0)), item.get("status") or "original", json_dumps(item.get("quality_score") or {}), now, now, ), ).fetchone() created.append(_decode_row(row) or {}) if regeneration_prepared: if is_direct_build: conn.execute( """ UPDATE data_process_tasks SET workflow_step='preview', preview_status='completed', preview_progress=100, preview_run_id=NULL, preview_failure_reason=NULL, preview_total_files=%s, preview_completed_files=%s, updated_at=%s WHERE id=%s """, (preview_file_count, preview_file_count, now, task_id), ) else: conn.execute( "UPDATE data_process_tasks SET updated_at=%s WHERE id=%s", (now, task_id), ) else: conn.execute( """ UPDATE data_process_tasks SET status='pending', progress=20, output_count=0, filtered_count=0, duplicate_count=0, error_count=0, failure_reason=NULL, results_confirmed=FALSE, workflow_step=CASE WHEN %s THEN 'preview' ELSE workflow_step END, preview_status=CASE WHEN %s THEN 'completed' ELSE preview_status END, preview_progress=CASE WHEN %s THEN 100 ELSE preview_progress END, preview_run_id=CASE WHEN %s THEN NULL ELSE preview_run_id END, preview_failure_reason=CASE WHEN %s THEN NULL ELSE preview_failure_reason END, preview_total_files=CASE WHEN %s THEN %s ELSE preview_total_files END, preview_completed_files=CASE WHEN %s THEN %s ELSE preview_completed_files END, updated_at=%s WHERE id=%s """, ( is_direct_build, is_direct_build, is_direct_build, is_direct_build, is_direct_build, is_direct_build, preview_file_count, is_direct_build, preview_file_count, now, task_id, ), ) return created def start_preview( self, task_id: str, *, source_file_ids: Sequence[str] | None = None, ) -> tuple[dict[str, Any], list[str]]: """创建一轮持久化切分任务,并返回本轮固定的源文件集合。""" requested_ids = ( list(dict.fromkeys(str(file_id) for file_id in source_file_ids)) if source_file_ids is not None else None ) if requested_ids is not None and ( not requested_ids or any(not file_id for file_id in requested_ids) ): raise ValueError("source_file_ids must contain non-empty ids") with self.connect() as conn: task = self._task_in_connection(conn, task_id, for_update=True) self._ensure_editable(task) if requested_ids is None: rows = conn.execute( """ SELECT id FROM data_process_source_files WHERE task_id=%s AND deleted_at IS NULL ORDER BY created_at, id """, (task_id,), ).fetchall() else: rows = conn.execute( """ SELECT id FROM data_process_source_files WHERE task_id=%s AND deleted_at IS NULL AND id=ANY(%s) ORDER BY created_at, id """, (task_id, requested_ids), ).fetchall() selected_ids = [str(row["id"]) for row in rows] if not selected_ids: raise InvalidStateError("at least one source file is required") if requested_ids is not None: missing = set(requested_ids) - set(selected_ids) if missing: raise NotFoundError( f"source files not found: {', '.join(sorted(missing))}" ) regeneration_prepared = _is_regeneration_prepared(task) if not regeneration_prepared: conn.execute("DELETE FROM data_process_results WHERE task_id=%s", (task_id,)) preview_run_id = new_id("dpprun") now = utcnow() row = conn.execute( """ UPDATE data_process_tasks SET status=CASE WHEN %s THEN status ELSE 'pending' END, progress=CASE WHEN %s THEN progress ELSE 0 END, output_count=CASE WHEN %s THEN output_count ELSE 0 END, filtered_count=CASE WHEN %s THEN filtered_count ELSE 0 END, duplicate_count=CASE WHEN %s THEN duplicate_count ELSE 0 END, error_count=CASE WHEN %s THEN error_count ELSE 0 END, failure_reason=CASE WHEN %s THEN failure_reason ELSE NULL END, results_confirmed=CASE WHEN %s THEN results_confirmed ELSE FALSE END, workflow_step='upload', preview_status='queued', preview_progress=0, preview_run_id=%s, preview_failure_reason=NULL, preview_total_files=%s, preview_completed_files=0, updated_at=%s WHERE id=%s AND deleted_at IS NULL RETURNING * """, ( regeneration_prepared, regeneration_prepared, regeneration_prepared, regeneration_prepared, regeneration_prepared, regeneration_prepared, regeneration_prepared, regeneration_prepared, preview_run_id, len(selected_ids), now, task_id, ), ).fetchone() return _public_task(_decode_row(row)) or {}, selected_ids def mark_preview_running(self, task_id: str, preview_run_id: str) -> bool: with self.connect() as conn: row = conn.execute( """ UPDATE data_process_tasks SET preview_status='running', updated_at=%s WHERE id=%s AND deleted_at IS NULL AND preview_status='queued' AND preview_run_id=%s RETURNING id """, (utcnow(), task_id, preview_run_id), ).fetchone() return row is not None def preview_is_running(self, task_id: str, preview_run_id: str) -> bool: with self.connect() as conn: row = conn.execute( """ SELECT preview_status, preview_run_id FROM data_process_tasks WHERE id=%s AND deleted_at IS NULL """, (task_id,), ).fetchone() return bool( row and row.get("preview_status") in ACTIVE_PREVIEW_STATUSES and row.get("preview_run_id") == preview_run_id ) def update_preview_progress( self, task_id: str, preview_run_id: str, completed_files: int, total_files: int, ) -> bool: total = max(1, total_files) completed = min(max(0, completed_files), total) progress = completed / total * 100 with self.connect() as conn: row = conn.execute( """ UPDATE data_process_tasks SET preview_progress=%s, preview_completed_files=%s, updated_at=%s WHERE id=%s AND deleted_at IS NULL AND preview_status='running' AND preview_run_id=%s RETURNING id """, (progress, completed, utcnow(), task_id, preview_run_id), ).fetchone() return row is not None def complete_preview(self, task_id: str, preview_run_id: str) -> bool: with self.connect() as conn: row = conn.execute( """ UPDATE data_process_tasks SET workflow_step='preview', preview_status='completed', preview_progress=100, preview_run_id=NULL, preview_failure_reason=NULL, preview_completed_files=preview_total_files, updated_at=%s WHERE id=%s AND deleted_at IS NULL AND preview_status='running' AND preview_run_id=%s RETURNING id """, (utcnow(), task_id, preview_run_id), ).fetchone() return row is not None def mark_preview_failed( self, task_id: str, reason: str, *, preview_run_id: str, ) -> bool: with self.connect() as conn: row = conn.execute( """ UPDATE data_process_tasks SET preview_status='failed', preview_run_id=NULL, preview_failure_reason=%s, updated_at=%s WHERE id=%s AND deleted_at IS NULL AND preview_status IN ('queued', 'running') AND preview_run_id=%s RETURNING id """, (reason[:4000], utcnow(), task_id, preview_run_id), ).fetchone() return row is not None def preview_progress(self, task_id: str) -> dict[str, Any]: task = self.get_task(task_id) return { "task_id": task["id"], "workflow_step": task.get("workflow_step") or "create", "preview_status": task.get("preview_status") or "idle", "preview_progress": float(task.get("preview_progress") or 0), "preview_run_id": task.get("preview_run_id"), "preview_failure_reason": task.get("preview_failure_reason"), "preview_total_files": int(task.get("preview_total_files") or 0), "preview_completed_files": int(task.get("preview_completed_files") or 0), } def list_preview_items( self, task_id: str, *, source_file_id: str | None = None, page: int = 1, page_size: int = 200, keyword: str | None = None, ) -> dict[str, Any]: self.get_task(task_id) clauses = ["task_id=%s"] params: list[Any] = [task_id] if source_file_id: clauses.append("source_file_id=%s") params.append(source_file_id) if keyword: clauses.append("(original_content ILIKE %s OR edited_content ILIKE %s)") pattern = f"%{keyword.strip()}%" params.extend([pattern, pattern]) where = " AND ".join(clauses) with self.connect() as conn: total = conn.execute( f"SELECT COUNT(*) AS count FROM data_process_preview_items WHERE {where}", params ).fetchone()["count"] rows = conn.execute( f""" SELECT * FROM data_process_preview_items WHERE {where} ORDER BY source_file_id NULLS LAST, source_start NULLS LAST, created_at, id LIMIT %s OFFSET %s """, [*params, page_size, (page - 1) * page_size], ).fetchall() return { "items": [_decode_row(row) for row in rows], "total": int(total), "page": page, "page_size": page_size, } def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]: with self.connect() as conn: row = conn.execute( "SELECT * FROM data_process_preview_items WHERE id=%s AND task_id=%s", (preview_id, task_id), ).fetchone() if not row: raise NotFoundError("preview item not found") return _decode_row(row) or {} def create_preview_item(self, task_id: str, item: dict[str, Any]) -> dict[str, Any]: now = utcnow() with self.connect() as conn: task = self._task_in_connection(conn, task_id, for_update=True) self._ensure_editable(task) if item.get("source_file_id"): source = conn.execute( """ SELECT id FROM data_process_source_files WHERE id=%s AND task_id=%s AND deleted_at IS NULL """, (item["source_file_id"], task_id), ).fetchone() if not source: raise NotFoundError("source file not found") row = conn.execute( """ INSERT INTO data_process_preview_items (id, task_id, source_file_id, original_content, edited_content, source_start, source_end, source_start_line, source_end_line, token_count, status, quality_score, created_at, updated_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING * """, ( new_id("dpp"), task_id, item.get("source_file_id"), item.get("original_content") or "", item.get("edited_content") or "", item.get("source_start"), item.get("source_end"), item.get("source_start_line"), item.get("source_end_line"), max(0, int(item.get("token_count") or 0)), item.get("status") or "manual", json_dumps(item.get("quality_score") or {}), now, now, ), ).fetchone() self._invalidate_results(conn, task, task_id, now) return _decode_row(row) or {} def update_preview_item( self, task_id: str, preview_id: str, payload: dict[str, Any] ) -> dict[str, Any]: with self.connect() as conn: task = self._task_in_connection(conn, task_id, for_update=True) self._ensure_editable(task) existing = conn.execute( "SELECT * FROM data_process_preview_items WHERE id=%s AND task_id=%s", (preview_id, task_id), ).fetchone() if not existing: raise NotFoundError("preview item not found") expected_updated_at = payload.get("expected_updated_at") current_updated_at = _serialize_value(existing.get("updated_at")) if expected_updated_at and expected_updated_at != current_updated_at: raise ConflictError("preview item was modified by another request") edited = payload["edited_content"] status = payload.get("status") if not status: if not edited.strip(): status = "invalid" elif edited == existing["original_content"]: status = "original" else: status = "modified" now = utcnow() row = conn.execute( """ UPDATE data_process_preview_items SET edited_content=%s, token_count=%s, status=%s, quality_score=%s, updated_at=%s WHERE id=%s AND task_id=%s RETURNING * """, ( edited, estimate_token_count(edited), status, json_dumps(payload.get("quality_score") or {}), now, preview_id, task_id, ), ).fetchone() self._invalidate_results(conn, task, task_id, now) return _decode_row(row) or {} def delete_preview_item(self, task_id: str, preview_id: str) -> None: with self.connect() as conn: task = self._task_in_connection(conn, task_id, for_update=True) self._ensure_editable(task) row = conn.execute( "DELETE FROM data_process_preview_items WHERE id=%s AND task_id=%s RETURNING id", (preview_id, task_id), ).fetchone() if not row: raise NotFoundError("preview item not found") self._invalidate_results(conn, task, task_id, utcnow())