fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
caoxiaozhu
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
|
||||
LocalDataProcessStorage,
|
||||
get_data_process_storage,
|
||||
)
|
||||
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
|
||||
from app.modules.data_process.store import (
|
||||
InvalidStateError,
|
||||
NotFoundError,
|
||||
get_data_process_store,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
class FakeDataProcessStore:
|
||||
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
|
||||
self.datasets: dict[str, dict[str, Any]] = {}
|
||||
self.models: dict[str, dict[str, Any]] = {}
|
||||
self.regeneration_prepared: set[str] = set()
|
||||
self.repeat_requests: dict[tuple[str, str], str] = {}
|
||||
self.sequence = 0
|
||||
|
||||
def _id(self, prefix: str) -> str:
|
||||
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
|
||||
"published_outputs_preserved": published_outputs_preserved,
|
||||
}
|
||||
|
||||
def _repeat_response(
|
||||
self,
|
||||
source_task_id: str,
|
||||
repeated_task_id: str,
|
||||
*,
|
||||
created: bool,
|
||||
) -> dict[str, Any]:
|
||||
task = self.get_task(repeated_task_id)
|
||||
task["source_file_count"] = len(self.sources[repeated_task_id])
|
||||
task["preview_count"] = len(self.previews[repeated_task_id])
|
||||
return {
|
||||
"task": task,
|
||||
"source_task_id": source_task_id,
|
||||
"created": created,
|
||||
"copied_source_file_count": len(self.sources[repeated_task_id]),
|
||||
"copied_preview_count": len(self.previews[repeated_task_id]),
|
||||
}
|
||||
|
||||
def find_repeated_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
|
||||
if repeated_task_id is None:
|
||||
return None
|
||||
return self._repeat_response(
|
||||
source_task_id,
|
||||
repeated_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
def repeat_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
*,
|
||||
expected_updated_at: str,
|
||||
request_id: str,
|
||||
file_copies: dict[str, dict[str, str]],
|
||||
) -> dict[str, Any]:
|
||||
existing = self.find_repeated_task(source_task_id, request_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
source_task = self.get_task(source_task_id)
|
||||
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
|
||||
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
|
||||
if source_task.get("updated_at") != expected_updated_at:
|
||||
raise InvalidStateError("源任务已被其他操作修改，请刷新后重试")
|
||||
source_files = self.sources[source_task_id]
|
||||
if set(file_copies) != {str(item["id"]) for item in source_files}:
|
||||
raise InvalidStateError("源文件快照已变化，请刷新后重试")
|
||||
if not self.previews[source_task_id]:
|
||||
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
|
||||
|
||||
repeated_task_id = repeat_task_id(source_task_id, request_id)
|
||||
suffix = f"（再次生成-{repeated_task_id[-6:]}）"
|
||||
task = {
|
||||
**deepcopy(source_task),
|
||||
"id": repeated_task_id,
|
||||
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
|
||||
"status": "pending",
|
||||
"progress": 20,
|
||||
"output_dataset_id": None,
|
||||
"output_datasets": [],
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"generation_run_id": None,
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_run_id": None,
|
||||
"preview_failure_reason": None,
|
||||
"preview_total_files": len(source_files),
|
||||
"preview_completed_files": len(source_files),
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
}
|
||||
self.tasks[repeated_task_id] = task
|
||||
self.sources[repeated_task_id] = []
|
||||
file_id_map: dict[str, str] = {}
|
||||
for source in source_files:
|
||||
old_file_id = str(source["id"])
|
||||
copy = file_copies[old_file_id]
|
||||
file_id_map[old_file_id] = copy["id"]
|
||||
self.sources[repeated_task_id].append(
|
||||
{
|
||||
**deepcopy(source),
|
||||
"id": copy["id"],
|
||||
"task_id": repeated_task_id,
|
||||
"storage_object_id": copy["storage_object_id"],
|
||||
}
|
||||
)
|
||||
self.previews[repeated_task_id] = [
|
||||
{
|
||||
**deepcopy(item),
|
||||
"id": self._id("dpp"),
|
||||
"task_id": repeated_task_id,
|
||||
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
|
||||
if item.get("source_file_id")
|
||||
else None,
|
||||
}
|
||||
for item in self.previews[source_task_id]
|
||||
]
|
||||
self.results[repeated_task_id] = []
|
||||
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
|
||||
return self._repeat_response(
|
||||
source_task_id,
|
||||
repeated_task_id,
|
||||
created=True,
|
||||
)
|
||||
|
||||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||||
self.get_task(task_id)
|
||||
del self.tasks[task_id]
|
||||
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||||
assert listed_preview.json()["data"]["total"] == 2
|
||||
preview_item = listed_preview.json()["data"]["items"][0]
|
||||
source_locator = preview_item["quality_score"]["source_locator"]
|
||||
assert source_locator == {
|
||||
"kind": "jsonl",
|
||||
"record_index": 1,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
"source_start": 0,
|
||||
"source_end": len(preview_item["original_content"]),
|
||||
}
|
||||
updated_preview = client.put(
|
||||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||||
json={
|
||||
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||||
},
|
||||
)
|
||||
assert "quality_score" in updated_preview.json()["data"]
|
||||
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
|
||||
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
|
||||
|
||||
|
||||
def test_completed_task_can_repeat_into_an_independent_background_task(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
client, store, storage = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "原始生成任务",
|
||||
"process_type": "structured",
|
||||
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
source_id = uploaded.json()["data"]["files"][0]["id"]
|
||||
built = client.post(
|
||||
f"/modelTF/data-process/{task_id}/preview/build",
|
||||
json={"replace_existing": True},
|
||||
)
|
||||
assert built.status_code == 200
|
||||
store.tasks[task_id].update(
|
||||
status="completed",
|
||||
progress=100,
|
||||
results_confirmed=True,
|
||||
workflow_step="results",
|
||||
output_count=1,
|
||||
output_dataset_id="dataset-original",
|
||||
updated_at="2026-07-28T12:00:00Z",
|
||||
)
|
||||
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
|
||||
store.datasets["dataset-original"] = {
|
||||
"id": "dataset-original",
|
||||
"name": "原数据集",
|
||||
"type": "train",
|
||||
"source_task_id": task_id,
|
||||
"deleted_at": None,
|
||||
}
|
||||
original_task = deepcopy(store.tasks[task_id])
|
||||
original_sources = deepcopy(store.sources[task_id])
|
||||
original_previews = deepcopy(store.previews[task_id])
|
||||
original_results = deepcopy(store.results[task_id])
|
||||
original_datasets = deepcopy(store.datasets)
|
||||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
|
||||
|
||||
payload = {
|
||||
"expected_updated_at": "2026-07-28T12:00:00Z",
|
||||
"request_id": "repeat-request-0001",
|
||||
}
|
||||
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||||
|
||||
assert response.status_code == 202
|
||||
repeated = response.json()["data"]
|
||||
repeated_task_id = repeated["task"]["id"]
|
||||
assert repeated["created"] is True
|
||||
assert repeated_task_id != task_id
|
||||
assert repeated["task"]["status"] == "running"
|
||||
assert repeated["task"]["workflow_step"] == "generate"
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == len(original_previews)
|
||||
assert store.tasks[task_id] == original_task
|
||||
assert store.sources[task_id] == original_sources
|
||||
assert store.previews[task_id] == original_previews
|
||||
assert store.results[task_id] == original_results
|
||||
assert store.datasets == original_datasets
|
||||
|
||||
repeated_source = store.sources[repeated_task_id][0]
|
||||
repeated_preview = store.previews[repeated_task_id][0]
|
||||
assert repeated_source["id"] != source_id
|
||||
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
|
||||
assert repeated_preview["id"] != original_previews[0]["id"]
|
||||
assert repeated_preview["source_file_id"] == repeated_source["id"]
|
||||
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
|
||||
|
||||
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||||
assert replay.status_code == 202
|
||||
assert replay.json()["data"]["created"] is False
|
||||
assert replay.json()["data"]["task"]["id"] == repeated_task_id
|
||||
assert len(store.tasks) == 2
|
||||
assert len(store.sources[repeated_task_id]) == 1
|
||||
|
||||
|
||||
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "源任务", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id].update(
|
||||
status="completed",
|
||||
results_confirmed=True,
|
||||
updated_at="2026-07-28T12:00:00Z",
|
||||
)
|
||||
before = deepcopy(store.tasks)
|
||||
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/repeat",
|
||||
json={
|
||||
"expected_updated_at": "2026-07-28T11:59:59Z",
|
||||
"request_id": "repeat-request-stale",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 409
|
||||
assert store.tasks == before
|
||||
|
||||
|
||||
def test_published_split_datasets_remain_in_detail_after_regeneration(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||||
|
||||
|
||||
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "先去重再脱敏",
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={
|
||||
"files": (
|
||||
"names.jsonl",
|
||||
(
|
||||
'{"name":"张三","role":"开发"}\n'
|
||||
'{"name":"李四","role":"开发"}\n'
|
||||
),
|
||||
"application/jsonl",
|
||||
)
|
||||
},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
|
||||
assert preview.status_code == 200
|
||||
items = preview.json()["data"]["items"]
|
||||
assert len(items) == 2
|
||||
assert len({item["original_content"] for item in items}) == 2
|
||||
assert {item["edited_content"] for item in items} == {
|
||||
'{"name":"[NAME]","role":"开发"}'
|
||||
}
|
||||
|
||||
|
||||
def test_structured_deduplication_removes_identical_rows_across_sources(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "跨源原文去重",
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
(
|
||||
"files",
|
||||
(
|
||||
"first.jsonl",
|
||||
'{"name":"张三","role":"开发"}\n',
|
||||
"application/jsonl",
|
||||
),
|
||||
),
|
||||
(
|
||||
"files",
|
||||
(
|
||||
"second.jsonl",
|
||||
'\n{"name":"张三","role":"开发"}\n',
|
||||
"application/jsonl",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
first_source, second_source = uploaded.json()["data"]["files"]
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
|
||||
assert preview.status_code == 200
|
||||
data = preview.json()["data"]
|
||||
assert data["total"] == 1
|
||||
assert data["file_counts"] == {
|
||||
first_source["id"]: 1,
|
||||
second_source["id"]: 0,
|
||||
}
|
||||
|
||||
|
||||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||||
store = FakeDataProcessStore()
|
||||
task = store.create_task(
|
||||
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
|
||||
)
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["total"] == 2
|
||||
preview_items = preview.json()["data"]["items"]
|
||||
assert len(preview_items) == 2
|
||||
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "Sheet",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "Sheet",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_docx_preview_preserves_document_block_order_and_source_offsets(
|
||||
@@ -2757,6 +3105,218 @@ def _preview_task(
|
||||
)
|
||||
|
||||
|
||||
def _structured_preview_task(
|
||||
content: str,
|
||||
*,
|
||||
file_format: str,
|
||||
options: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
return data_process_endpoint._build_preview_items(
|
||||
{
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": options or []},
|
||||
},
|
||||
[
|
||||
{
|
||||
"id": "structured-source",
|
||||
"name": f"records.{file_format}",
|
||||
"file_format": file_format,
|
||||
"content": content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
|
||||
json_source = '{"records":[{"id":1},{"id":2}]}'
|
||||
json_items = _structured_preview_task(
|
||||
json_source,
|
||||
file_format="json",
|
||||
)
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["json_pointer"]
|
||||
for item in json_items
|
||||
] == ["/records/0", "/records/1"]
|
||||
assert [
|
||||
json_source[item["source_start"] : item["source_end"]]
|
||||
for item in json_items
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
assert [item["source_start_line"] for item in json_items] == [1, 1]
|
||||
|
||||
jsonl_source = '{"id":1}\n\n{"id":2}'
|
||||
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in jsonl_items
|
||||
] == [1, 2]
|
||||
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
|
||||
assert [
|
||||
jsonl_source[item["source_start"] : item["source_end"]]
|
||||
for item in jsonl_items
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
|
||||
csv_items = _structured_preview_task(csv_source, file_format="csv")
|
||||
assert [
|
||||
(item["source_start_line"], item["source_end_line"])
|
||||
for item in csv_items
|
||||
] == [(2, 3), (5, 5)]
|
||||
assert [
|
||||
csv_source[item["source_start"] : item["source_end"]]
|
||||
for item in csv_items
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "空 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("empty-array.json", "[]", "application/json")),
|
||||
(
|
||||
"files",
|
||||
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
assert uploaded.status_code == 200
|
||||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["items"] == []
|
||||
assert preview.json()["data"]["total"] == 0
|
||||
assert set(preview.json()["data"]["file_counts"].values()) == {0}
|
||||
|
||||
|
||||
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
invalid_sources = (
|
||||
("duplicate.json", '{"id":1,"id":2}'),
|
||||
("duplicate.jsonl", '{"id":1,"id":2}\n'),
|
||||
("nan.json", '{"value":NaN}'),
|
||||
("infinity.json", '{"value":Infinity}'),
|
||||
("control.json", '{"value":"bad\x00control"}'),
|
||||
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
|
||||
)
|
||||
|
||||
for filename, content in invalid_sources:
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": (filename, content, "application/json")},
|
||||
)
|
||||
assert response.status_code == 400, (filename, response.text)
|
||||
|
||||
|
||||
def test_structured_json_preview_preserves_precision_and_business_data_field(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
precise = '{"value":0.123456789012345678901234567890}'
|
||||
business = '{"id":7,"data":[{"id":8}]}'
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("precise.json", precise, "application/json")),
|
||||
("files", ("business.json", business, "application/json")),
|
||||
],
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
items = preview.json()["data"]["items"]
|
||||
assert [item["original_content"] for item in items] == [precise, business]
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["json_pointer"] for item in items
|
||||
] == ["", ""]
|
||||
assert [item["source_start"] for item in items] == [0, 0]
|
||||
|
||||
|
||||
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
|
||||
source_records = [
|
||||
{"id": "A", "amount": 10, "empty": ""},
|
||||
{"id": "A", "amount": 10, "empty": ""},
|
||||
{"id": "", "amount": 11, "empty": ""},
|
||||
{"id": "B", "amount": 11, "empty": ""},
|
||||
{"id": "C", "amount": 12, "empty": ""},
|
||||
{"id": "D", "amount": 12, "empty": ""},
|
||||
{"id": "E", "amount": 13, "empty": ""},
|
||||
{"id": "F", "amount": 13, "empty": ""},
|
||||
{"id": "G", "amount": 14, "empty": ""},
|
||||
{"id": "H", "amount": 1000, "empty": ""},
|
||||
]
|
||||
source = "\n".join(
|
||||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
for record in source_records
|
||||
)
|
||||
items = _structured_preview_task(
|
||||
source,
|
||||
file_format="jsonl",
|
||||
options=["clean_invalid", "deduplicate", "filter_anomaly"],
|
||||
)
|
||||
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in items
|
||||
] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [json.loads(item["original_content"])["id"] for item in items] == [
|
||||
"A",
|
||||
"",
|
||||
"B",
|
||||
"C",
|
||||
"D",
|
||||
"E",
|
||||
"F",
|
||||
"G",
|
||||
]
|
||||
|
||||
|
||||
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
|
||||
source_records = [
|
||||
{"customer_id": "C-1", "status": "old"},
|
||||
{"customer_id": "C-1", "status": "new"},
|
||||
{"status": "old", "customer_id": "C-1"},
|
||||
]
|
||||
source = "\n".join(
|
||||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
for record in source_records
|
||||
)
|
||||
|
||||
items = _structured_preview_task(
|
||||
source,
|
||||
file_format="jsonl",
|
||||
options=["clean_invalid", "deduplicate"],
|
||||
)
|
||||
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in items
|
||||
] == [1, 2]
|
||||
assert [item["source_start_line"] for item in items] == [1, 2]
|
||||
assert [json.loads(item["original_content"])["status"] for item in items] == [
|
||||
"old",
|
||||
"new",
|
||||
]
|
||||
|
||||
|
||||
def test_fixed_preview_preserves_source_offsets() -> None:
|
||||
content = (
|
||||
"# 第一章\n"
|
||||
|
||||
Reference in New Issue
Block a user