YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(data-process): 修正任务详情数据契约

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-27 10:03:46 +08:00
parent 42c0e4f5c2
commit b14b2ecf22
4 changed files with 247 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
backend/tests/test_data_process_api.py
View File

@@ -81,7 +81,18 @@ class FakeDataProcessStore:
def get_task(self, task_id: str) -> dict[str, Any]:
if task_id not in self.tasks:
raise NotFoundError("data process task not found")
return deepcopy(self.tasks[task_id])
task = deepcopy(self.tasks[task_id])
split_order = {"train": 0, "val": 1, "test": 2}
task["output_datasets"] = sorted(
(
deepcopy(dataset)
for dataset in self.datasets.values()
if dataset.get("source_task_id") == task_id
and dataset.get("deleted_at") is None
),
key=lambda dataset: split_order.get(str(dataset.get("type")), 3),
)
return task
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
self.get_task(task_id)
@@ -334,6 +345,7 @@ class FakeDataProcessStore:
self.tasks[task_id].update(
status="running",
progress=30,
output_count=0,
generation_run_id=self._id("dprun"),
)
return self.get_task(task_id)
@@ -475,15 +487,53 @@ class FakeDataProcessStore:
def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
task = self.tasks[task_id]
published = [
dataset
for dataset in self.datasets.values()
if dataset.get("source_task_id") == task_id
and dataset.get("deleted_at") is None
]
if task.get("output_dataset_id"):
return {"dataset": deepcopy(self.datasets[task["output_dataset_id"]]), "created": False}
train_dataset = self.datasets[task["output_dataset_id"]]
return {
"dataset": deepcopy(train_dataset),
"datasets": deepcopy(published),
"output_datasets": deepcopy(published),
"created": False,
}
if task["status"] != "completed":
raise InvalidStateError("only a completed task can be published")
dataset_id = self._id("dataset")
dataset = {"id": dataset_id, "name": payload["dataset_name"], "source_task_id": task_id}
self.datasets[dataset_id] = dataset
task["output_dataset_id"] = dataset_id
return {"dataset": deepcopy(dataset), "created": True}
split_specs = (
("train", "训练集"),
("val", "验证集"),
("test", "测试集"),
)
for dataset_type, label in split_specs:
dataset_id = self._id("dataset")
self.datasets[dataset_id] = {
"id": dataset_id,
"name": f"{payload['dataset_name']}-{label}",
"type": dataset_type,
"count": len(self.results[task_id]),
"source": "task",
"task_id": task_id,
"source_task_id": task_id,
"deleted_at": None,
}
published = [
dataset
for dataset in self.datasets.values()
if dataset.get("source_task_id") == task_id
and dataset.get("deleted_at") is None
]
train_dataset = next(dataset for dataset in published if dataset["type"] == "train")
task["output_dataset_id"] = train_dataset["id"]
return {
"dataset": deepcopy(train_dataset),
"datasets": deepcopy(published),
"output_datasets": deepcopy(published),
"created": True,
}
def make_client(
@@ -706,6 +756,56 @@ def test_task_list_exposes_document_and_generation_counts(
assert item["output_dataset_id"] is None
def test_task_detail_uses_returned_source_files_as_document_count(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "详情文档数", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id]["source_file_count"] = 99
store.sources[task_id] = [
{"id": "source-1", "name": "一.pdf", "content": "正文一"},
{"id": "source-2", "name": "二.pdf", "content": "正文二"},
]
response = client.get(f"/modelTF/data-process/{task_id}")
assert response.status_code == 200
detail = response.json()["data"]
assert len(detail["source_files"]) == 2
assert detail["source_file_count"] == 2
def test_generation_start_response_clears_previous_output_count(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "新一轮生成", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id]["output_count"] = 28
store.previews[task_id] = [
{
"id": "preview-1",
"source_file_id": None,
"original_content": '{"question":"新问题","answer":"新答案"}',
"edited_content": '{"question":"新问题","answer":"新答案"}',
"status": "original",
}
]
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *args: None)
response = client.post(f"/modelTF/data-process/{task_id}/generate")
assert response.status_code == 200
progress = response.json()["data"]
assert progress["status"] == "running"
assert progress["output_count"] == 0
assert store.tasks[task_id]["output_count"] == 0
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
tmp_path: Path,
) -> None:
@@ -893,6 +993,64 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
assert store.previews[task_id][0]["id"] == "preview_1"
def test_published_split_datasets_remain_in_detail_after_regeneration(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "保留旧发布数据", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
{
"status": "completed",
"updated_at": "2026-07-27T09:00:00Z",
"output_count": 1,
}
)
store.results[task_id] = [
{
"id": "result_1",
"status": "valid",
"instruction": "问题",
"input": "",
"output": "答案",
}
]
published = client.post(
f"/modelTF/data-process/{task_id}/publish",
json={"dataset_name": "保留旧发布数据集"},
)
assert published.status_code == 200
published_datasets = published.json()["data"]["datasets"]
assert len(published_datasets) == 3
published_ids = {item["id"] for item in published_datasets}
assert store.tasks[task_id]["output_dataset_id"] in published_ids
regenerated = client.post(
f"/modelTF/data-process/{task_id}/regenerate",
json={
"name": "保留旧发布数据",
"description": "更换生成配置后退出",
"process_type": "structured",
"config": {"generation_model_id": "model_2"},
"expected_updated_at": "2026-07-27T09:00:00Z",
},
)
assert regenerated.status_code == 200
assert regenerated.json()["data"]["task"]["output_dataset_id"] is None
detail = client.get(f"/modelTF/data-process/{task_id}")
assert detail.status_code == 200
detail_data = detail.json()["data"]
assert detail_data["status"] == "pending"
assert detail_data["output_dataset_id"] is None
assert len(detail_data["output_datasets"]) == 3
assert {item["id"] for item in detail_data["output_datasets"]} == published_ids
assert set(store.datasets) == published_ids
def test_regenerate_endpoint_validates_snapshot_and_locked_process_type(
tmp_path: Path,
) -> None: