fix: 完善数据预处理与 JSON 上传链路

This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions

View File

@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
_preview_config_changed,
_reasoning_output_is_valid,
_source_storage_descriptor,
repeat_task_id,
)
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
yield self._conn
class _RepeatConnection:
def __init__(self) -> None:
self.source_files = [
{
"id": "source-old",
"name": "source.jsonl",
"size_bytes": 12,
"record_count": 1,
"file_format": "jsonl",
"checksum_sha256": "a" * 64,
"content": '{"id":1}\n',
"content_preview": '{"id":1}',
"metadata": {"storage_backend": "local"},
"created_by": "user-1",
}
]
self.source_previews = [
{
"id": "preview-old",
"source_file_id": "source-old",
"original_content": '{"id":1}',
"edited_content": '{"id":1,"checked":true}',
"source_start": 0,
"source_end": 8,
"source_start_line": 1,
"source_end_line": 1,
"token_count": 5,
"status": "modified",
"quality_score": {"overall": 90},
}
]
self.created_task: dict[str, Any] | None = None
self.created_files: list[dict[str, Any]] = []
self.created_previews: list[dict[str, Any]] = []
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if params is not None:
assert normalized.count("%s") == len(params)
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
return _Result(row=None)
if normalized.startswith("SELECT * FROM data_process_source_files"):
return _Result(rows=[dict(item) for item in self.source_files])
if normalized.startswith("SELECT * FROM data_process_preview_items"):
return _Result(rows=[dict(item) for item in self.source_previews])
if normalized.startswith("INSERT INTO data_process_tasks"):
self.created_task = {
"id": params[0],
"name": params[1],
"description": params[2],
"status": "pending",
"process_type": params[3],
"source_dataset_id": params[4],
"config": params[5],
"progress": 20,
"input_count": params[6],
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_total_files": params[7],
"preview_completed_files": params[8],
"created_at": params[15],
"updated_at": params[16],
}
return _Result(row=dict(self.created_task))
if normalized.startswith("INSERT INTO data_process_source_files"):
self.created_files.append(
{
"id": params[0],
"task_id": params[1],
"storage_object_id": params[2],
"content": params[8],
}
)
return _Result()
if normalized.startswith("INSERT INTO data_process_preview_items"):
self.created_previews.append(
{
"id": params[0],
"task_id": params[1],
"source_file_id": params[2],
"edited_content": params[4],
}
)
return _Result()
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
return _Result(
row={
"source_file_count": len(self.created_files),
"preview_count": len(self.created_previews),
}
)
raise AssertionError(f"unexpected SQL: {normalized}")
class _RepeatStore(DataProcessStore):
def __init__(self, conn: _RepeatConnection) -> None:
self._conn = conn
@contextmanager
def connect(self) -> Iterator[_RepeatConnection]:
yield self._conn
def _task_in_connection(
self,
conn: Any,
task_id: str,
*,
for_update: bool = False,
) -> dict[str, Any]:
assert task_id == "task-source"
assert for_update is True
return {
"id": task_id,
"name": "原任务",
"description": "原描述",
"status": "completed",
"process_type": "structured",
"source_dataset_id": None,
"config": {
"temperature": 0.3,
"_regeneration_prepared": {"prepared": True},
},
"results_confirmed": True,
"preview_status": "completed",
"tenant_id": "tenant-1",
"project_id": "project-1",
"owner_id": "owner-1",
"created_by": "user-1",
"updated_at": "2026-07-28T12:00:00Z",
}
class _TaskListConnection:
def __init__(self) -> None:
self.task = {
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
conn = _RepeatConnection()
store = _RepeatStore(conn)
request_id = "repeat-request-0001"
target_task_id = repeat_task_id("task-source", request_id)
repeated = store.repeat_task(
"task-source",
expected_updated_at="2026-07-28T12:00:00Z",
request_id=request_id,
file_copies={
"source-old": {
"id": "source-new",
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
}
},
)
assert repeated["created"] is True
assert repeated["task"]["id"] == target_task_id
assert repeated["task"]["config"] == {"temperature": 0.3}
assert repeated["task"]["results_confirmed"] is False
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == 1
assert conn.created_files == [
{
"id": "source-new",
"task_id": target_task_id,
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
"content": '{"id":1}\n',
}
]
assert conn.created_previews[0]["task_id"] == target_task_id
assert conn.created_previews[0]["source_file_id"] == "source-new"
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
decoded = _decode_row(
{