YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 完善数据预处理与 JSON 上传链路

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files

303
backend/tests/test_data_process_algorithms.py
View File

@@ -5,6 +5,7 @@ import json
import xml.etree.ElementTree as ET
import zipfile
from datetime import datetime
from decimal import Decimal
import pytest
from docx import Document
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
normalize_text,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
record_fingerprint,
remove_document_noise,
score_quality,
stable_split,
stable_split_assignments,
structured_json_dumps,
)
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
assert parsed_txt.text == "普通文本"
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
root_json = parse_text_content('{"id":1}', filename="root.json")
assert root_json.record_locators == (
{
"kind": "json",
"record_index": 1,
"json_pointer": "",
"source_start": 0,
"source_end": 8,
"start_line": 1,
"end_line": 1,
},
)
wrapped_json = parse_text_content(
'{"records":[{"id":1},{"id":1}]}',
filename="wrapped.json",
)
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
"/records/0",
"/records/1",
]
parsed_jsonl = parse_text_content(
'{"id":1}\r\n\r\n{"id":1}',
filename="records.jsonl",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_jsonl.record_locators
] == [(1, 1, 1), (2, 3, 3)]
assert [
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_jsonl.record_locators
] == ['{"id":1}', '{"id":1}']
parsed_csv = parse_text_content(
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
filename="records.csv",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_csv.record_locators
] == [(1, 2, 3), (2, 5, 5)]
assert [
parsed_csv.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_csv.record_locators
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
processed = preprocess_structured_records_with_lineage(
[
{"id": "A", "value": "first", "empty": ""},
{"id": "", "value": "invalid", "empty": ""},
{"id": "A", "value": "duplicate identity", "empty": ""},
{"id": "B", "value": "second", "empty": ""},
],
["clean_invalid", "deduplicate"],
)
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
assert [entry.record for entry in processed] == [
{"id": "A", "value": "first"},
{"id": "", "value": "invalid"},
{"id": "A", "value": "duplicate identity"},
{"id": "B", "value": "second"},
]
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
assert parsed_pdf.format == "pdf"
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
)
assert parsed_xlsx.record_locators == (
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 3,
"sheet_record_index": 1,
},
)
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active
first.title = "甲表"
first.append(["说明"])
first.append([])
first.append(["id", "value"])
first.append([1, "same"])
first.append([1, "same"])
second = workbook.create_sheet("乙表")
second.append(["id", "value"])
second.append([1, "same"])
output = io.BytesIO()
workbook.save(output)
workbook.close()
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
assert parsed.records == (
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
)
assert [
(
locator["record_index"],
locator["sheet_index"],
locator["sheet_name"],
locator["row_number"],
locator["sheet_record_index"],
)
for locator in parsed.record_locators
] == [
(1, 0, "甲表", 4, 0),
(2, 0, "甲表", 5, 1),
(3, 1, "乙表", 2, 0),
]
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
pages = _pdf_page_texts(
"""
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
"json",
)
assert result == [{"text": "内容"}]
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
assert extract_structured_records(
'{"items":[{"text":" 内容 "}],"total":1}',
"json",
) == [{"text": " 内容 "}]
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
source = '{"code":"００１","text":" 内容 ","quote":"＂"}'
parsed = parse_text_content(source, filename="records.json")
assert parsed.text == source
assert parsed.records == (
{"code": "００１", "text": " 内容 ", "quote": "＂"},
)
invalid_values = (
'{"id":1,"id":2}',
'{"nested":{"id":1,"id":2}}',
'{"value":NaN}',
'{"value":Infinity}',
'{"value":-Infinity}',
'{"value":"bad\x00control"}',
)
for invalid in invalid_values:
with pytest.raises(ValueError):
parse_text_content(invalid, filename="invalid.json")
with pytest.raises(ValueError):
parse_text_content("｛\"id\":1｝", filename="invalid.json")
with pytest.raises(ValueError, match="nesting exceeds"):
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
source = (
' {"code":"００１","text":" 内容 ",'
'"value":0.123456789012345678901234567890}\r\n\r\n'
'{"id":2}\r\n'
)
parsed = parse_text_content(source, filename="records.jsonl")
assert parsed.text == source
assert parsed.records[0] == {
"code": "００１",
"text": " 内容 ",
"value": Decimal("0.123456789012345678901234567890"),
}
assert [
source[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == [
(
'{"code":"００１","text":" 内容 ",'
'"value":0.123456789012345678901234567890}'
),
'{"id":2}',
]
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
parse_text_content(invalid, filename="invalid.jsonl")
def test_json_record_contract_avoids_business_field_collisions() -> None:
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
{"id": 1},
{"id": 2},
]
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
{"id": 1, "data": [{"id": 2}]}
]
assert extract_structured_records(
'{"records":[{"id":1}],"data":[{"id":2}]}',
"json",
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
assert extract_structured_records(
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
"json",
) == [{"id": 1}]
assert extract_structured_records(
'{"payload":{"data":[{"id":2}],"total":1}}',
"json",
) == [{"id": 2}]
assert extract_structured_records('{"records":[],"total":0}', "json") == []
# 包装数组中的非对象不是记录集合，整体按一条业务对象保留。
assert extract_structured_records('{"data":[1,2]}', "json") == [
{"data": [1, 2]}
]
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
pretty = (
'{\n "records": [\n {"id": 1},\n'
' {\n "id": 2\n }\n ],\n "total": 2\n}'
)
parsed = parse_text_content(pretty, filename="pretty.json")
assert [
pretty[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id": 1}', '{\n "id": 2\n }']
assert [
(locator["start_line"], locator["end_line"])
for locator in parsed.record_locators
] == [(3, 3), (4, 6)]
minified = '[{"id":1},{"id":2}]'
parsed = parse_text_content(minified, filename="minified.json")
assert [
minified[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id":1}', '{"id":2}']
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
parsed = parse_text_content(source, filename="precise.json")
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
assert parsed.records[1]["value"] == Decimal("1e400")
assert structured_json_dumps(parsed.records[0]) == (
'{"value":0.123456789012345678901234567890}'
)
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
assert isinstance(parsed.records[0]["value"], Decimal)
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
assert preprocess_structured_records(clean_source, []) == clean_source
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
{"id": "1", "name": "有效"},
{"id": "", "name": "缺少关键字段"},
{"id": "2", "name": "有效"},
]
hierarchy = [
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
{"id": "", "parent_id": "", "name": "", "empty": ""},
]
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
{"id": "1", "parent_id": None, "name": "根节点"},
{"id": "2", "parent_id": "1", "name": "子节点"},
]
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
assert "profile" in preprocess_structured_records(nested, [])[0]
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
duplicates = [
{"customer_id": "C-1", "value": "first"},
{"customer_id": "C-1", "value": "updated"},
{"customer_id": "C-1", "value": "first"},
{"customer_id": "", "value": "blank-one"},
{"customer_id": "", "value": "blank-two"},
]
assert len(preprocess_structured_records(duplicates, [])) == 4
assert len(preprocess_structured_records(duplicates, [])) == 5
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
assert [record["value"] for record in deduplicated] == [
"first",
"updated",
"blank-one",
"blank-two",
]
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
)
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
masked, counts = desensitize_structured_record(
{
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "张三",
"contact_name": "李四",
"姓名": "王五",
"profile.name": "赵六",
}
)
assert masked == {
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "[NAME]",
"contact_name": "[NAME]",
"姓名": "[NAME]",
"profile.name": "[NAME]",
}
assert counts == {
"email": 0,
"phone": 0,
"id_card": 0,
"name": 4,
"total": 4,
}
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
valid = {
"instruction": "如何修改收货地址？",

564
backend/tests/test_data_process_api.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
from copy import deepcopy
from io import BytesIO
from pathlib import Path
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
LocalDataProcessStorage,
get_data_process_storage,
)
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
from app.modules.data_process.store import (
InvalidStateError,
NotFoundError,
get_data_process_store,
repeat_task_id,
)
class FakeDataProcessStore:
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
self.datasets: dict[str, dict[str, Any]] = {}
self.models: dict[str, dict[str, Any]] = {}
self.regeneration_prepared: set[str] = set()
self.repeat_requests: dict[tuple[str, str], str] = {}
self.sequence = 0
def _id(self, prefix: str) -> str:
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
"published_outputs_preserved": published_outputs_preserved,
}
def _repeat_response(
self,
source_task_id: str,
repeated_task_id: str,
*,
created: bool,
) -> dict[str, Any]:
task = self.get_task(repeated_task_id)
task["source_file_count"] = len(self.sources[repeated_task_id])
task["preview_count"] = len(self.previews[repeated_task_id])
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": len(self.sources[repeated_task_id]),
"copied_preview_count": len(self.previews[repeated_task_id]),
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
if repeated_task_id is None:
return None
return self._repeat_response(
source_task_id,
repeated_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
existing = self.find_repeated_task(source_task_id, request_id)
if existing is not None:
return existing
source_task = self.get_task(source_task_id)
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("updated_at") != expected_updated_at:
raise InvalidStateError("源任务已被其他操作修改，请刷新后重试")
source_files = self.sources[source_task_id]
if set(file_copies) != {str(item["id"]) for item in source_files}:
raise InvalidStateError("源文件快照已变化，请刷新后重试")
if not self.previews[source_task_id]:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
repeated_task_id = repeat_task_id(source_task_id, request_id)
suffix = f"（再次生成-{repeated_task_id[-6:]}）"
task = {
**deepcopy(source_task),
"id": repeated_task_id,
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
"status": "pending",
"progress": 20,
"output_dataset_id": None,
"output_datasets": [],
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": len(source_files),
"preview_completed_files": len(source_files),
"started_at": None,
"completed_at": None,
}
self.tasks[repeated_task_id] = task
self.sources[repeated_task_id] = []
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
file_id_map[old_file_id] = copy["id"]
self.sources[repeated_task_id].append(
{
**deepcopy(source),
"id": copy["id"],
"task_id": repeated_task_id,
"storage_object_id": copy["storage_object_id"],
}
)
self.previews[repeated_task_id] = [
{
**deepcopy(item),
"id": self._id("dpp"),
"task_id": repeated_task_id,
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
if item.get("source_file_id")
else None,
}
for item in self.previews[source_task_id]
]
self.results[repeated_task_id] = []
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
return self._repeat_response(
source_task_id,
repeated_task_id,
created=True,
)
def delete_task(self, task_id: str, **_: Any) -> None:
self.get_task(task_id)
del self.tasks[task_id]
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
assert listed_preview.json()["data"]["total"] == 2
preview_item = listed_preview.json()["data"]["items"][0]
source_locator = preview_item["quality_score"]["source_locator"]
assert source_locator == {
"kind": "jsonl",
"record_index": 1,
"start_line": 1,
"end_line": 1,
"source_start": 0,
"source_end": len(preview_item["original_content"]),
}
updated_preview = client.put(
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
json={
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
},
)
assert "quality_score" in updated_preview.json()["data"]
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
assert generated.status_code == 200
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
def test_completed_task_can_repeat_into_an_independent_background_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "原始生成任务",
"process_type": "structured",
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
)
assert uploaded.status_code == 200
source_id = uploaded.json()["data"]["files"][0]["id"]
built = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"replace_existing": True},
)
assert built.status_code == 200
store.tasks[task_id].update(
status="completed",
progress=100,
results_confirmed=True,
workflow_step="results",
output_count=1,
output_dataset_id="dataset-original",
updated_at="2026-07-28T12:00:00Z",
)
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
store.datasets["dataset-original"] = {
"id": "dataset-original",
"name": "原数据集",
"type": "train",
"source_task_id": task_id,
"deleted_at": None,
}
original_task = deepcopy(store.tasks[task_id])
original_sources = deepcopy(store.sources[task_id])
original_previews = deepcopy(store.previews[task_id])
original_results = deepcopy(store.results[task_id])
original_datasets = deepcopy(store.datasets)
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
payload = {
"expected_updated_at": "2026-07-28T12:00:00Z",
"request_id": "repeat-request-0001",
}
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert response.status_code == 202
repeated = response.json()["data"]
repeated_task_id = repeated["task"]["id"]
assert repeated["created"] is True
assert repeated_task_id != task_id
assert repeated["task"]["status"] == "running"
assert repeated["task"]["workflow_step"] == "generate"
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == len(original_previews)
assert store.tasks[task_id] == original_task
assert store.sources[task_id] == original_sources
assert store.previews[task_id] == original_previews
assert store.results[task_id] == original_results
assert store.datasets == original_datasets
repeated_source = store.sources[repeated_task_id][0]
repeated_preview = store.previews[repeated_task_id][0]
assert repeated_source["id"] != source_id
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
assert repeated_preview["id"] != original_previews[0]["id"]
assert repeated_preview["source_file_id"] == repeated_source["id"]
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert replay.status_code == 202
assert replay.json()["data"]["created"] is False
assert replay.json()["data"]["task"]["id"] == repeated_task_id
assert len(store.tasks) == 2
assert len(store.sources[repeated_task_id]) == 1
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "源任务", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
results_confirmed=True,
updated_at="2026-07-28T12:00:00Z",
)
before = deepcopy(store.tasks)
response = client.post(
f"/modelTF/data-process/{task_id}/repeat",
json={
"expected_updated_at": "2026-07-28T11:59:59Z",
"request_id": "repeat-request-stale",
},
)
assert response.status_code == 409
assert store.tasks == before
def test_published_split_datasets_remain_in_detail_after_regeneration(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "先去重再脱敏",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={
"files": (
"names.jsonl",
(
'{"name":"张三","role":"开发"}\n'
'{"name":"李四","role":"开发"}\n'
),
"application/jsonl",
)
},
)
assert uploaded.status_code == 200
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert len(items) == 2
assert len({item["original_content"] for item in items}) == 2
assert {item["edited_content"] for item in items} == {
'{"name":"[NAME]","role":"开发"}'
}
def test_structured_deduplication_removes_identical_rows_across_sources(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "跨源原文去重",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
(
"files",
(
"first.jsonl",
'{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
(
"files",
(
"second.jsonl",
'\n{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
],
)
assert uploaded.status_code == 200
first_source, second_source = uploaded.json()["data"]["files"]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
data = preview.json()["data"]
assert data["total"] == 1
assert data["file_counts"] == {
first_source["id"]: 1,
second_source["id"]: 0,
}
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
store = FakeDataProcessStore()
task = store.create_task(
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
)
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["total"] == 2
preview_items = preview.json()["data"]["items"]
assert len(preview_items) == 2
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 3,
"sheet_record_index": 1,
},
]
def test_docx_preview_preserves_document_block_order_and_source_offsets(
@@ -2757,6 +3105,218 @@ def _preview_task(
)
def _structured_preview_task(
content: str,
*,
file_format: str,
options: list[str] | None = None,
) -> list[dict[str, Any]]:
return data_process_endpoint._build_preview_items(
{
"process_type": "structured",
"config": {"preprocess_options": options or []},
},
[
{
"id": "structured-source",
"name": f"records.{file_format}",
"file_format": file_format,
"content": content,
}
],
)
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
json_source = '{"records":[{"id":1},{"id":2}]}'
json_items = _structured_preview_task(
json_source,
file_format="json",
)
assert [
item["quality_score"]["source_locator"]["json_pointer"]
for item in json_items
] == ["/records/0", "/records/1"]
assert [
json_source[item["source_start"] : item["source_end"]]
for item in json_items
] == ['{"id":1}', '{"id":2}']
assert [item["source_start_line"] for item in json_items] == [1, 1]
jsonl_source = '{"id":1}\n\n{"id":2}'
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in jsonl_items
] == [1, 2]
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
assert [
jsonl_source[item["source_start"] : item["source_end"]]
for item in jsonl_items
] == ['{"id":1}', '{"id":2}']
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
csv_items = _structured_preview_task(csv_source, file_format="csv")
assert [
(item["source_start_line"], item["source_end_line"])
for item in csv_items
] == [(2, 3), (5, 5)]
assert [
csv_source[item["source_start"] : item["source_end"]]
for item in csv_items
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "空 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("empty-array.json", "[]", "application/json")),
(
"files",
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["items"] == []
assert preview.json()["data"]["total"] == 0
assert set(preview.json()["data"]["file_counts"].values()) == {0}
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
invalid_sources = (
("duplicate.json", '{"id":1,"id":2}'),
("duplicate.jsonl", '{"id":1,"id":2}\n'),
("nan.json", '{"value":NaN}'),
("infinity.json", '{"value":Infinity}'),
("control.json", '{"value":"bad\x00control"}'),
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
)
for filename, content in invalid_sources:
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": (filename, content, "application/json")},
)
assert response.status_code == 400, (filename, response.text)
def test_structured_json_preview_preserves_precision_and_business_data_field(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
precise = '{"value":0.123456789012345678901234567890}'
business = '{"id":7,"data":[{"id":8}]}'
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("precise.json", precise, "application/json")),
("files", ("business.json", business, "application/json")),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert [item["original_content"] for item in items] == [precise, business]
assert [
item["quality_score"]["source_locator"]["json_pointer"] for item in items
] == ["", ""]
assert [item["source_start"] for item in items] == [0, 0]
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
source_records = [
{"id": "A", "amount": 10, "empty": ""},
{"id": "A", "amount": 10, "empty": ""},
{"id": "", "amount": 11, "empty": ""},
{"id": "B", "amount": 11, "empty": ""},
{"id": "C", "amount": 12, "empty": ""},
{"id": "D", "amount": 12, "empty": ""},
{"id": "E", "amount": 13, "empty": ""},
{"id": "F", "amount": 13, "empty": ""},
{"id": "G", "amount": 14, "empty": ""},
{"id": "H", "amount": 1000, "empty": ""},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate", "filter_anomaly"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [json.loads(item["original_content"])["id"] for item in items] == [
"A",
"",
"B",
"C",
"D",
"E",
"F",
"G",
]
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
source_records = [
{"customer_id": "C-1", "status": "old"},
{"customer_id": "C-1", "status": "new"},
{"status": "old", "customer_id": "C-1"},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 2]
assert [item["source_start_line"] for item in items] == [1, 2]
assert [json.loads(item["original_content"])["status"] for item in items] == [
"old",
"new",
]
def test_fixed_preview_preserves_source_offsets() -> None:
content = (
"# 第一章\n"

34
backend/tests/test_data_process_storage.py
View File

@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
_assert_staging_empty(storage)
def test_stage_copy_creates_an_independently_deletable_source_object(
tmp_path: Path,
) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")
original = _stage(storage, content=b"immutable source")
storage.publish([original])
copied = storage.stage_copy(
batch_id="batch-copy",
source_reference=original.reference,
expected_source_task_id="task-1",
expected_source_file_id="source-1",
task_id="task-2",
source_file_id="source-2",
version=1,
name="source.txt",
)
storage.publish([copied])
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
original.reference,
expected_task_id="task-1",
expected_source_file_id="source-1",
) is True
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
copied.reference,
expected_task_id="task-2",
expected_source_file_id="source-2",
) is True
_assert_staging_empty(storage)
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")

176
backend/tests/test_data_process_store.py
View File

@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
_preview_config_changed,
_reasoning_output_is_valid,
_source_storage_descriptor,
repeat_task_id,
)
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
yield self._conn
class _RepeatConnection:
def __init__(self) -> None:
self.source_files = [
{
"id": "source-old",
"name": "source.jsonl",
"size_bytes": 12,
"record_count": 1,
"file_format": "jsonl",
"checksum_sha256": "a" * 64,
"content": '{"id":1}\n',
"content_preview": '{"id":1}',
"metadata": {"storage_backend": "local"},
"created_by": "user-1",
}
]
self.source_previews = [
{
"id": "preview-old",
"source_file_id": "source-old",
"original_content": '{"id":1}',
"edited_content": '{"id":1,"checked":true}',
"source_start": 0,
"source_end": 8,
"source_start_line": 1,
"source_end_line": 1,
"token_count": 5,
"status": "modified",
"quality_score": {"overall": 90},
}
]
self.created_task: dict[str, Any] | None = None
self.created_files: list[dict[str, Any]] = []
self.created_previews: list[dict[str, Any]] = []
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if params is not None:
assert normalized.count("%s") == len(params)
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
return _Result(row=None)
if normalized.startswith("SELECT * FROM data_process_source_files"):
return _Result(rows=[dict(item) for item in self.source_files])
if normalized.startswith("SELECT * FROM data_process_preview_items"):
return _Result(rows=[dict(item) for item in self.source_previews])
if normalized.startswith("INSERT INTO data_process_tasks"):
self.created_task = {
"id": params[0],
"name": params[1],
"description": params[2],
"status": "pending",
"process_type": params[3],
"source_dataset_id": params[4],
"config": params[5],
"progress": 20,
"input_count": params[6],
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_total_files": params[7],
"preview_completed_files": params[8],
"created_at": params[15],
"updated_at": params[16],
}
return _Result(row=dict(self.created_task))
if normalized.startswith("INSERT INTO data_process_source_files"):
self.created_files.append(
{
"id": params[0],
"task_id": params[1],
"storage_object_id": params[2],
"content": params[8],
}
)
return _Result()
if normalized.startswith("INSERT INTO data_process_preview_items"):
self.created_previews.append(
{
"id": params[0],
"task_id": params[1],
"source_file_id": params[2],
"edited_content": params[4],
}
)
return _Result()
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
return _Result(
row={
"source_file_count": len(self.created_files),
"preview_count": len(self.created_previews),
}
)
raise AssertionError(f"unexpected SQL: {normalized}")
class _RepeatStore(DataProcessStore):
def __init__(self, conn: _RepeatConnection) -> None:
self._conn = conn
@contextmanager
def connect(self) -> Iterator[_RepeatConnection]:
yield self._conn
def _task_in_connection(
self,
conn: Any,
task_id: str,
*,
for_update: bool = False,
) -> dict[str, Any]:
assert task_id == "task-source"
assert for_update is True
return {
"id": task_id,
"name": "原任务",
"description": "原描述",
"status": "completed",
"process_type": "structured",
"source_dataset_id": None,
"config": {
"temperature": 0.3,
"_regeneration_prepared": {"prepared": True},
},
"results_confirmed": True,
"preview_status": "completed",
"tenant_id": "tenant-1",
"project_id": "project-1",
"owner_id": "owner-1",
"created_by": "user-1",
"updated_at": "2026-07-28T12:00:00Z",
}
class _TaskListConnection:
def __init__(self) -> None:
self.task = {
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
conn = _RepeatConnection()
store = _RepeatStore(conn)
request_id = "repeat-request-0001"
target_task_id = repeat_task_id("task-source", request_id)
repeated = store.repeat_task(
"task-source",
expected_updated_at="2026-07-28T12:00:00Z",
request_id=request_id,
file_copies={
"source-old": {
"id": "source-new",
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
}
},
)
assert repeated["created"] is True
assert repeated["task"]["id"] == target_task_id
assert repeated["task"]["config"] == {"temperature": 0.3}
assert repeated["task"]["results_confirmed"] is False
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == 1
assert conn.created_files == [
{
"id": "source-new",
"task_id": target_task_id,
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
"content": '{"id":1}\n',
}
]
assert conn.created_previews[0]["task_id"] == target_task_id
assert conn.created_previews[0]["source_file_id"] == "source-new"
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
decoded = _decode_row(
{