fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
|
||||
_preview_config_changed,
|
||||
_reasoning_output_is_valid,
|
||||
_source_storage_descriptor,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
|
||||
yield self._conn
|
||||
|
||||
|
||||
class _RepeatConnection:
|
||||
def __init__(self) -> None:
|
||||
self.source_files = [
|
||||
{
|
||||
"id": "source-old",
|
||||
"name": "source.jsonl",
|
||||
"size_bytes": 12,
|
||||
"record_count": 1,
|
||||
"file_format": "jsonl",
|
||||
"checksum_sha256": "a" * 64,
|
||||
"content": '{"id":1}\n',
|
||||
"content_preview": '{"id":1}',
|
||||
"metadata": {"storage_backend": "local"},
|
||||
"created_by": "user-1",
|
||||
}
|
||||
]
|
||||
self.source_previews = [
|
||||
{
|
||||
"id": "preview-old",
|
||||
"source_file_id": "source-old",
|
||||
"original_content": '{"id":1}',
|
||||
"edited_content": '{"id":1,"checked":true}',
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"source_start_line": 1,
|
||||
"source_end_line": 1,
|
||||
"token_count": 5,
|
||||
"status": "modified",
|
||||
"quality_score": {"overall": 90},
|
||||
}
|
||||
]
|
||||
self.created_task: dict[str, Any] | None = None
|
||||
self.created_files: list[dict[str, Any]] = []
|
||||
self.created_previews: list[dict[str, Any]] = []
|
||||
|
||||
def execute(self, sql: str, params: Any = None) -> _Result:
|
||||
normalized = " ".join(sql.split())
|
||||
if params is not None:
|
||||
assert normalized.count("%s") == len(params)
|
||||
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
|
||||
return _Result(row=None)
|
||||
if normalized.startswith("SELECT * FROM data_process_source_files"):
|
||||
return _Result(rows=[dict(item) for item in self.source_files])
|
||||
if normalized.startswith("SELECT * FROM data_process_preview_items"):
|
||||
return _Result(rows=[dict(item) for item in self.source_previews])
|
||||
if normalized.startswith("INSERT INTO data_process_tasks"):
|
||||
self.created_task = {
|
||||
"id": params[0],
|
||||
"name": params[1],
|
||||
"description": params[2],
|
||||
"status": "pending",
|
||||
"process_type": params[3],
|
||||
"source_dataset_id": params[4],
|
||||
"config": params[5],
|
||||
"progress": 20,
|
||||
"input_count": params[6],
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_total_files": params[7],
|
||||
"preview_completed_files": params[8],
|
||||
"created_at": params[15],
|
||||
"updated_at": params[16],
|
||||
}
|
||||
return _Result(row=dict(self.created_task))
|
||||
if normalized.startswith("INSERT INTO data_process_source_files"):
|
||||
self.created_files.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"storage_object_id": params[2],
|
||||
"content": params[8],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("INSERT INTO data_process_preview_items"):
|
||||
self.created_previews.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"source_file_id": params[2],
|
||||
"edited_content": params[4],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
|
||||
return _Result(
|
||||
row={
|
||||
"source_file_count": len(self.created_files),
|
||||
"preview_count": len(self.created_previews),
|
||||
}
|
||||
)
|
||||
raise AssertionError(f"unexpected SQL: {normalized}")
|
||||
|
||||
|
||||
class _RepeatStore(DataProcessStore):
|
||||
def __init__(self, conn: _RepeatConnection) -> None:
|
||||
self._conn = conn
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[_RepeatConnection]:
|
||||
yield self._conn
|
||||
|
||||
def _task_in_connection(
|
||||
self,
|
||||
conn: Any,
|
||||
task_id: str,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
assert task_id == "task-source"
|
||||
assert for_update is True
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": "原任务",
|
||||
"description": "原描述",
|
||||
"status": "completed",
|
||||
"process_type": "structured",
|
||||
"source_dataset_id": None,
|
||||
"config": {
|
||||
"temperature": 0.3,
|
||||
"_regeneration_prepared": {"prepared": True},
|
||||
},
|
||||
"results_confirmed": True,
|
||||
"preview_status": "completed",
|
||||
"tenant_id": "tenant-1",
|
||||
"project_id": "project-1",
|
||||
"owner_id": "owner-1",
|
||||
"created_by": "user-1",
|
||||
"updated_at": "2026-07-28T12:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
class _TaskListConnection:
|
||||
def __init__(self) -> None:
|
||||
self.task = {
|
||||
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
|
||||
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
|
||||
|
||||
|
||||
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
|
||||
conn = _RepeatConnection()
|
||||
store = _RepeatStore(conn)
|
||||
request_id = "repeat-request-0001"
|
||||
target_task_id = repeat_task_id("task-source", request_id)
|
||||
|
||||
repeated = store.repeat_task(
|
||||
"task-source",
|
||||
expected_updated_at="2026-07-28T12:00:00Z",
|
||||
request_id=request_id,
|
||||
file_copies={
|
||||
"source-old": {
|
||||
"id": "source-new",
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert repeated["created"] is True
|
||||
assert repeated["task"]["id"] == target_task_id
|
||||
assert repeated["task"]["config"] == {"temperature": 0.3}
|
||||
assert repeated["task"]["results_confirmed"] is False
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == 1
|
||||
assert conn.created_files == [
|
||||
{
|
||||
"id": "source-new",
|
||||
"task_id": target_task_id,
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
"content": '{"id":1}\n',
|
||||
}
|
||||
]
|
||||
assert conn.created_previews[0]["task_id"] == target_task_id
|
||||
assert conn.created_previews[0]["source_file_id"] == "source-new"
|
||||
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
|
||||
|
||||
|
||||
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
|
||||
decoded = _decode_row(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user