refactor: 完整重构 data_process 模块并修复拆分遗留缺陷
将 algorithms.py / store.py 拆分为 algorithms/ 与 store/ 子包，并修复 机械拆分造成的导入与辅助函数缺失： - algorithms/: 补全各子模块依赖与 17 个私有辅助函数、8 个常量；重写 __init__.py 移除坏的 importlib 兜底，分层导入并以局部 import 断开 text_utils<->parsers、quality<->structured_processing 循环依赖。 - store/: 补回 DataProcessStoreError / hashlib / _serialize_value / estimate_token_count 等缺失导入，包入口导出测试与调用方依赖的私有 辅助函数。 - 删除旧单文件 algorithms.py / store.py 及重构残留（_algorithms_old、 backups、refactor 脚本、REFACTORING 文档）。 algorithms 与 store 测试套件 91 项全部通过。
This commit is contained in:
caoxiaozhu
539
backend/app/modules/data_process/store/preview.py
Normal file
539
backend/app/modules/data_process/store/preview.py
Normal file
@@ -0,0 +1,539 @@
|
||||
"""数据处理存储层 - 预览管理。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from collections.abc import Sequence
|
||||
|
||||
import psycopg
|
||||
|
||||
from .base import (
|
||||
StoreBase,
|
||||
utcnow,
|
||||
new_id,
|
||||
repeat_task_id,
|
||||
json_dumps,
|
||||
_json_value,
|
||||
_decode_row,
|
||||
_public_task,
|
||||
_business_config,
|
||||
_preview_config_value,
|
||||
_preview_config_changed,
|
||||
_preview_config_projection,
|
||||
_normalized_preprocess_options,
|
||||
_regeneration_marker,
|
||||
_is_regeneration_prepared,
|
||||
_task_output_type,
|
||||
_task_reasoning_detail,
|
||||
_reasoning_output_is_valid,
|
||||
_dpo_fields_are_valid,
|
||||
_source_storage_descriptor,
|
||||
_serialize_value,
|
||||
NotFoundError,
|
||||
ConflictError,
|
||||
InvalidStateError,
|
||||
EDITABLE_STATUSES,
|
||||
ACTIVE_PREVIEW_STATUSES,
|
||||
WORKFLOW_STEPS,
|
||||
_REGENERATION_MARKER_KEY,
|
||||
_REPEAT_SOURCE_TASK_KEY,
|
||||
_REPEAT_REQUEST_KEY,
|
||||
)
|
||||
from ..algorithms import estimate_token_count # noqa: E402
|
||||
|
||||
|
||||
class PreviewMixin:
|
||||
"""预览管理 Mixin。"""
|
||||
|
||||
def replace_preview_items(
|
||||
self,
|
||||
task_id: str,
|
||||
items: Sequence[dict[str, Any]],
|
||||
*,
|
||||
source_file_ids: Sequence[str] | None = None,
|
||||
preview_run_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
selected_ids = (
|
||||
list(dict.fromkeys(str(file_id) for file_id in source_file_ids))
|
||||
if source_file_ids is not None
|
||||
else None
|
||||
)
|
||||
if selected_ids is not None:
|
||||
if not selected_ids or any(not file_id for file_id in selected_ids):
|
||||
raise ValueError("source_file_ids must contain non-empty ids")
|
||||
selected_set = set(selected_ids)
|
||||
unexpected = {
|
||||
str(item.get("source_file_id") or "")
|
||||
for item in items
|
||||
if str(item.get("source_file_id") or "") not in selected_set
|
||||
}
|
||||
if unexpected:
|
||||
raise ValueError("preview items contain an unselected source file")
|
||||
preview_file_count = len(selected_ids) if selected_ids is not None else len(
|
||||
{str(item.get("source_file_id") or "") for item in items}
|
||||
)
|
||||
is_direct_build = preview_run_id is None
|
||||
|
||||
now = utcnow()
|
||||
with self.connect() as conn:
|
||||
task = self._task_in_connection(conn, task_id, for_update=True)
|
||||
if is_direct_build:
|
||||
self._ensure_editable(task)
|
||||
elif (
|
||||
task.get("preview_run_id") != preview_run_id
|
||||
or task.get("preview_status") != "running"
|
||||
):
|
||||
raise InvalidStateError("preview run is no longer active")
|
||||
regeneration_prepared = _is_regeneration_prepared(task)
|
||||
if not regeneration_prepared:
|
||||
conn.execute("DELETE FROM data_process_results WHERE task_id=%s", (task_id,))
|
||||
if selected_ids is None:
|
||||
conn.execute(
|
||||
"DELETE FROM data_process_preview_items WHERE task_id=%s", (task_id,)
|
||||
)
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL AND id=ANY(%s)
|
||||
""",
|
||||
(task_id, selected_ids),
|
||||
).fetchall()
|
||||
found = {str(row["id"]) for row in rows}
|
||||
missing = set(selected_ids) - found
|
||||
if missing:
|
||||
raise NotFoundError(
|
||||
f"source files not found: {', '.join(sorted(missing))}"
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
DELETE FROM data_process_preview_items
|
||||
WHERE task_id=%s AND source_file_id=ANY(%s)
|
||||
""",
|
||||
(task_id, selected_ids),
|
||||
)
|
||||
created: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
row = conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_preview_items
|
||||
(id, task_id, source_file_id, original_content, edited_content,
|
||||
source_start, source_end, source_start_line, source_end_line,
|
||||
token_count, status, quality_score, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
item.get("id") or new_id("dpp"),
|
||||
task_id,
|
||||
item.get("source_file_id"),
|
||||
item.get("original_content") or "",
|
||||
item.get("edited_content", item.get("original_content") or ""),
|
||||
item.get("source_start"),
|
||||
item.get("source_end"),
|
||||
item.get("source_start_line"),
|
||||
item.get("source_end_line"),
|
||||
max(0, int(item.get("token_count") or 0)),
|
||||
item.get("status") or "original",
|
||||
json_dumps(item.get("quality_score") or {}),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
).fetchone()
|
||||
created.append(_decode_row(row) or {})
|
||||
if regeneration_prepared:
|
||||
if is_direct_build:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET workflow_step='preview', preview_status='completed',
|
||||
preview_progress=100, preview_run_id=NULL,
|
||||
preview_failure_reason=NULL, preview_total_files=%s,
|
||||
preview_completed_files=%s, updated_at=%s
|
||||
WHERE id=%s
|
||||
""",
|
||||
(preview_file_count, preview_file_count, now, task_id),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE data_process_tasks SET updated_at=%s WHERE id=%s",
|
||||
(now, task_id),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET status='pending', progress=20, output_count=0, filtered_count=0,
|
||||
duplicate_count=0, error_count=0, failure_reason=NULL,
|
||||
results_confirmed=FALSE,
|
||||
workflow_step=CASE WHEN %s THEN 'preview' ELSE workflow_step END,
|
||||
preview_status=CASE WHEN %s THEN 'completed' ELSE preview_status END,
|
||||
preview_progress=CASE WHEN %s THEN 100 ELSE preview_progress END,
|
||||
preview_run_id=CASE WHEN %s THEN NULL ELSE preview_run_id END,
|
||||
preview_failure_reason=CASE WHEN %s THEN NULL ELSE preview_failure_reason END,
|
||||
preview_total_files=CASE WHEN %s THEN %s ELSE preview_total_files END,
|
||||
preview_completed_files=CASE WHEN %s THEN %s ELSE preview_completed_files END,
|
||||
updated_at=%s
|
||||
WHERE id=%s
|
||||
""",
|
||||
(
|
||||
is_direct_build,
|
||||
is_direct_build,
|
||||
is_direct_build,
|
||||
is_direct_build,
|
||||
is_direct_build,
|
||||
is_direct_build,
|
||||
preview_file_count,
|
||||
is_direct_build,
|
||||
preview_file_count,
|
||||
now,
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
return created
|
||||
|
||||
def start_preview(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source_file_ids: Sequence[str] | None = None,
|
||||
) -> tuple[dict[str, Any], list[str]]:
|
||||
"""创建一轮持久化切分任务，并返回本轮固定的源文件集合。"""
|
||||
|
||||
requested_ids = (
|
||||
list(dict.fromkeys(str(file_id) for file_id in source_file_ids))
|
||||
if source_file_ids is not None
|
||||
else None
|
||||
)
|
||||
if requested_ids is not None and (
|
||||
not requested_ids or any(not file_id for file_id in requested_ids)
|
||||
):
|
||||
raise ValueError("source_file_ids must contain non-empty ids")
|
||||
|
||||
with self.connect() as conn:
|
||||
task = self._task_in_connection(conn, task_id, for_update=True)
|
||||
self._ensure_editable(task)
|
||||
if requested_ids is None:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL
|
||||
ORDER BY created_at, id
|
||||
""",
|
||||
(task_id,),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL AND id=ANY(%s)
|
||||
ORDER BY created_at, id
|
||||
""",
|
||||
(task_id, requested_ids),
|
||||
).fetchall()
|
||||
selected_ids = [str(row["id"]) for row in rows]
|
||||
if not selected_ids:
|
||||
raise InvalidStateError("at least one source file is required")
|
||||
if requested_ids is not None:
|
||||
missing = set(requested_ids) - set(selected_ids)
|
||||
if missing:
|
||||
raise NotFoundError(
|
||||
f"source files not found: {', '.join(sorted(missing))}"
|
||||
)
|
||||
|
||||
regeneration_prepared = _is_regeneration_prepared(task)
|
||||
if not regeneration_prepared:
|
||||
conn.execute("DELETE FROM data_process_results WHERE task_id=%s", (task_id,))
|
||||
preview_run_id = new_id("dpprun")
|
||||
now = utcnow()
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET status=CASE WHEN %s THEN status ELSE 'pending' END,
|
||||
progress=CASE WHEN %s THEN progress ELSE 0 END,
|
||||
output_count=CASE WHEN %s THEN output_count ELSE 0 END,
|
||||
filtered_count=CASE WHEN %s THEN filtered_count ELSE 0 END,
|
||||
duplicate_count=CASE WHEN %s THEN duplicate_count ELSE 0 END,
|
||||
error_count=CASE WHEN %s THEN error_count ELSE 0 END,
|
||||
failure_reason=CASE WHEN %s THEN failure_reason ELSE NULL END,
|
||||
results_confirmed=CASE WHEN %s THEN results_confirmed ELSE FALSE END,
|
||||
workflow_step='upload', preview_status='queued', preview_progress=0,
|
||||
preview_run_id=%s, preview_failure_reason=NULL,
|
||||
preview_total_files=%s, preview_completed_files=0,
|
||||
updated_at=%s
|
||||
WHERE id=%s AND deleted_at IS NULL
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
regeneration_prepared,
|
||||
preview_run_id,
|
||||
len(selected_ids),
|
||||
now,
|
||||
task_id,
|
||||
),
|
||||
).fetchone()
|
||||
return _public_task(_decode_row(row)) or {}, selected_ids
|
||||
|
||||
def mark_preview_running(self, task_id: str, preview_run_id: str) -> bool:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET preview_status='running', updated_at=%s
|
||||
WHERE id=%s AND deleted_at IS NULL
|
||||
AND preview_status='queued' AND preview_run_id=%s
|
||||
RETURNING id
|
||||
""",
|
||||
(utcnow(), task_id, preview_run_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def preview_is_running(self, task_id: str, preview_run_id: str) -> bool:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT preview_status, preview_run_id
|
||||
FROM data_process_tasks
|
||||
WHERE id=%s AND deleted_at IS NULL
|
||||
""",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
return bool(
|
||||
row
|
||||
and row.get("preview_status") in ACTIVE_PREVIEW_STATUSES
|
||||
and row.get("preview_run_id") == preview_run_id
|
||||
)
|
||||
|
||||
def update_preview_progress(
|
||||
self,
|
||||
task_id: str,
|
||||
preview_run_id: str,
|
||||
completed_files: int,
|
||||
total_files: int,
|
||||
) -> bool:
|
||||
total = max(1, total_files)
|
||||
completed = min(max(0, completed_files), total)
|
||||
progress = completed / total * 100
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET preview_progress=%s, preview_completed_files=%s, updated_at=%s
|
||||
WHERE id=%s AND deleted_at IS NULL
|
||||
AND preview_status='running' AND preview_run_id=%s
|
||||
RETURNING id
|
||||
""",
|
||||
(progress, completed, utcnow(), task_id, preview_run_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def complete_preview(self, task_id: str, preview_run_id: str) -> bool:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET workflow_step='preview', preview_status='completed',
|
||||
preview_progress=100, preview_run_id=NULL,
|
||||
preview_failure_reason=NULL,
|
||||
preview_completed_files=preview_total_files, updated_at=%s
|
||||
WHERE id=%s AND deleted_at IS NULL
|
||||
AND preview_status='running' AND preview_run_id=%s
|
||||
RETURNING id
|
||||
""",
|
||||
(utcnow(), task_id, preview_run_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def mark_preview_failed(
|
||||
self,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
*,
|
||||
preview_run_id: str,
|
||||
) -> bool:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_tasks
|
||||
SET preview_status='failed', preview_run_id=NULL,
|
||||
preview_failure_reason=%s, updated_at=%s
|
||||
WHERE id=%s AND deleted_at IS NULL
|
||||
AND preview_status IN ('queued', 'running') AND preview_run_id=%s
|
||||
RETURNING id
|
||||
""",
|
||||
(reason[:4000], utcnow(), task_id, preview_run_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def preview_progress(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.get_task(task_id)
|
||||
return {
|
||||
"task_id": task["id"],
|
||||
"workflow_step": task.get("workflow_step") or "create",
|
||||
"preview_status": task.get("preview_status") or "idle",
|
||||
"preview_progress": float(task.get("preview_progress") or 0),
|
||||
"preview_run_id": task.get("preview_run_id"),
|
||||
"preview_failure_reason": task.get("preview_failure_reason"),
|
||||
"preview_total_files": int(task.get("preview_total_files") or 0),
|
||||
"preview_completed_files": int(task.get("preview_completed_files") or 0),
|
||||
}
|
||||
|
||||
def list_preview_items(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source_file_id: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 200,
|
||||
keyword: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.get_task(task_id)
|
||||
clauses = ["task_id=%s"]
|
||||
params: list[Any] = [task_id]
|
||||
if source_file_id:
|
||||
clauses.append("source_file_id=%s")
|
||||
params.append(source_file_id)
|
||||
if keyword:
|
||||
clauses.append("(original_content ILIKE %s OR edited_content ILIKE %s)")
|
||||
pattern = f"%{keyword.strip()}%"
|
||||
params.extend([pattern, pattern])
|
||||
where = " AND ".join(clauses)
|
||||
with self.connect() as conn:
|
||||
total = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM data_process_preview_items WHERE {where}", params
|
||||
).fetchone()["count"]
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT * FROM data_process_preview_items
|
||||
WHERE {where}
|
||||
ORDER BY source_file_id NULLS LAST, source_start NULLS LAST, created_at, id
|
||||
LIMIT %s OFFSET %s
|
||||
""",
|
||||
[*params, page_size, (page - 1) * page_size],
|
||||
).fetchall()
|
||||
return {
|
||||
"items": [_decode_row(row) for row in rows],
|
||||
"total": int(total),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM data_process_preview_items WHERE id=%s AND task_id=%s",
|
||||
(preview_id, task_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise NotFoundError("preview item not found")
|
||||
return _decode_row(row) or {}
|
||||
|
||||
def create_preview_item(self, task_id: str, item: dict[str, Any]) -> dict[str, Any]:
|
||||
now = utcnow()
|
||||
with self.connect() as conn:
|
||||
task = self._task_in_connection(conn, task_id, for_update=True)
|
||||
self._ensure_editable(task)
|
||||
if item.get("source_file_id"):
|
||||
source = conn.execute(
|
||||
"""
|
||||
SELECT id FROM data_process_source_files
|
||||
WHERE id=%s AND task_id=%s AND deleted_at IS NULL
|
||||
""",
|
||||
(item["source_file_id"], task_id),
|
||||
).fetchone()
|
||||
if not source:
|
||||
raise NotFoundError("source file not found")
|
||||
row = conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_preview_items
|
||||
(id, task_id, source_file_id, original_content, edited_content,
|
||||
source_start, source_end, source_start_line, source_end_line,
|
||||
token_count, status, quality_score, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
new_id("dpp"),
|
||||
task_id,
|
||||
item.get("source_file_id"),
|
||||
item.get("original_content") or "",
|
||||
item.get("edited_content") or "",
|
||||
item.get("source_start"),
|
||||
item.get("source_end"),
|
||||
item.get("source_start_line"),
|
||||
item.get("source_end_line"),
|
||||
max(0, int(item.get("token_count") or 0)),
|
||||
item.get("status") or "manual",
|
||||
json_dumps(item.get("quality_score") or {}),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
).fetchone()
|
||||
self._invalidate_results(conn, task, task_id, now)
|
||||
return _decode_row(row) or {}
|
||||
|
||||
def update_preview_item(
|
||||
self, task_id: str, preview_id: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
task = self._task_in_connection(conn, task_id, for_update=True)
|
||||
self._ensure_editable(task)
|
||||
existing = conn.execute(
|
||||
"SELECT * FROM data_process_preview_items WHERE id=%s AND task_id=%s",
|
||||
(preview_id, task_id),
|
||||
).fetchone()
|
||||
if not existing:
|
||||
raise NotFoundError("preview item not found")
|
||||
expected_updated_at = payload.get("expected_updated_at")
|
||||
current_updated_at = _serialize_value(existing.get("updated_at"))
|
||||
if expected_updated_at and expected_updated_at != current_updated_at:
|
||||
raise ConflictError("preview item was modified by another request")
|
||||
edited = payload["edited_content"]
|
||||
status = payload.get("status")
|
||||
if not status:
|
||||
if not edited.strip():
|
||||
status = "invalid"
|
||||
elif edited == existing["original_content"]:
|
||||
status = "original"
|
||||
else:
|
||||
status = "modified"
|
||||
now = utcnow()
|
||||
row = conn.execute(
|
||||
"""
|
||||
UPDATE data_process_preview_items
|
||||
SET edited_content=%s, token_count=%s, status=%s, quality_score=%s,
|
||||
updated_at=%s
|
||||
WHERE id=%s AND task_id=%s
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
edited,
|
||||
estimate_token_count(edited),
|
||||
status,
|
||||
json_dumps(payload.get("quality_score") or {}),
|
||||
now,
|
||||
preview_id,
|
||||
task_id,
|
||||
),
|
||||
).fetchone()
|
||||
self._invalidate_results(conn, task, task_id, now)
|
||||
return _decode_row(row) or {}
|
||||
|
||||
def delete_preview_item(self, task_id: str, preview_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
task = self._task_in_connection(conn, task_id, for_update=True)
|
||||
self._ensure_editable(task)
|
||||
row = conn.execute(
|
||||
"DELETE FROM data_process_preview_items WHERE id=%s AND task_id=%s RETURNING id",
|
||||
(preview_id, task_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise NotFoundError("preview item not found")
|
||||
self._invalidate_results(conn, task, task_id, utcnow())
|
||||
Reference in New Issue
Block a user