fix: 完善数据预处理与 JSON 上传链路

This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions

View File

@@ -8,12 +8,14 @@ import os
import re
import socket
import time
from collections.abc import Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from copy import deepcopy
from dataclasses import asdict
from pathlib import Path
from threading import BoundedSemaphore, Lock
from typing import Any, Iterator, Literal
from typing import Any, Literal
from urllib.parse import quote, urlsplit
import httpx
@@ -45,9 +47,10 @@ from app.modules.data_process.algorithms import (
is_near_duplicate,
near_duplicate_fingerprint,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
remove_document_noise,
score_quality,
structured_json_dumps,
)
from app.modules.data_process.document_chunking import (
DocumentChunk,
@@ -75,9 +78,11 @@ from app.modules.data_process.store import (
NotFoundError,
get_data_process_store,
new_id,
repeat_task_id,
)
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessRepeatRequest,
DataProcessStatus,
DataProcessTaskCreate,
DataProcessTaskUpdate,
@@ -261,7 +266,14 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
content = str(source.get("content") or "")
file_format = str(source.get("file_format") or "").lower()
if file_format == "xlsx":
# XLSX 上传阶段已安全解析为 JSONL 后入库。
raw_content = source.get("raw_content")
if isinstance(raw_content, bytes):
return parse_text_content(
raw_content,
filename=str(source.get("name") or "source.xlsx"),
file_format="xlsx",
)
# 兼容原始对象已缺失的历史文件:退化为上传阶段生成的 JSONL。
return parse_text_content(content, file_format="jsonl")
if file_format in {"pdf", "docx", "pptx"}:
# 文档上传阶段已抽取文本,预览阶段只需要对正文切片。
@@ -381,11 +393,12 @@ def _build_preview_items(
seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {}
items: list[dict[str, Any]] = []
def append_item(item: dict[str, Any]) -> None:
def append_item(item: dict[str, Any], *, dedup_content: str) -> None:
content = str(item.get("edited_content") or "").strip()
if should_clean_invalid and not content:
return
content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
# 去重必须基于脱敏前内容,否则不同原文可能在替换 PII 后被错误合并。
content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest()
if should_deduplicate and content_hash in seen_content_hashes:
return
seen_content_hashes.add(content_hash)
@@ -447,6 +460,7 @@ def _build_preview_items(
continue
for key in band_keys:
seen_near_duplicate_bands.setdefault(key, []).append(content)
dedup_content = content
pii_counts: dict[str, int] = {}
if should_desensitize:
content, pii_counts = desensitize_pii(content)
@@ -476,7 +490,8 @@ def _build_preview_items(
else "original"
),
"quality_score": quality,
}
},
dedup_content=dedup_content,
)
continue
@@ -488,48 +503,76 @@ def _build_preview_items(
"filter_anomaly",
}
source_records = list(parsed.records)
processed_records = preprocess_structured_records(
processed_records = preprocess_structured_records_with_lineage(
source_records,
structured_options,
)
if not processed_records and parsed.text and not source_records:
processed_records = [{"value": parsed.text}]
same_cardinality = len(processed_records) == len(source_records)
for index, record in enumerate(processed_records):
original_record = source_records[index] if same_cardinality else record
original_content = json.dumps(
original_record,
ensure_ascii=False,
separators=(",", ":"),
for processed in processed_records:
source_index = processed.source_index
record = processed.record
original_record = (
source_records[source_index]
if source_index < len(source_records)
else record
)
source_locator = (
deepcopy(parsed.record_locators[source_index])
if source_index < len(parsed.record_locators)
else None
)
original_content = structured_json_dumps(original_record)
pii_counts: dict[str, int] = {}
edited_record = record
dedup_content = (
canonical_record_json(record)
if "normalize_format" in preprocess_options
else structured_json_dumps(record)
)
if should_desensitize:
edited_record, pii_counts = desensitize_structured_record(record)
content = (
canonical_record_json(edited_record)
if "normalize_format" in preprocess_options
else json.dumps(
edited_record,
ensure_ascii=False,
separators=(",", ":"),
)
else structured_json_dumps(edited_record)
)
quality = _preview_quality(content, config)
quality["pii_replacements"] = pii_counts
if source_locator is not None:
quality["source_locator"] = source_locator
source_start = (
source_locator.get("source_start")
if source_locator is not None
else None
)
source_end = (
source_locator.get("source_end")
if source_locator is not None
else None
)
source_start_line = (
source_locator.get("start_line")
if source_locator is not None
else None
)
source_end_line = (
source_locator.get("end_line")
if source_locator is not None
else None
)
append_item(
{
"source_file_id": source["id"],
"original_content": original_content,
"edited_content": content,
"source_start": None,
"source_end": None,
"source_start_line": None,
"source_end_line": None,
"source_start": source_start,
"source_end": source_end,
"source_start_line": source_start_line,
"source_end_line": source_end_line,
"token_count": estimate_token_count(content),
"status": "modified" if content != original_content else "original",
"quality_score": quality,
}
},
dedup_content=dedup_content,
)
return items
@@ -850,6 +893,135 @@ def prepare_regeneration(
)
def _repeat_file_copies(
store: DataProcessStore,
storage: LocalDataProcessStorage,
source_task_id: str,
request_id: str,
) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]:
"""为新任务创建独立的源文件引用,避免删除任一任务时互相影响。"""
target_task_id = repeat_task_id(source_task_id, request_id)
copies: dict[str, dict[str, str]] = {}
staged: list[StagedSourceObject] = []
batch_id = storage.new_batch_id()
for summary in store.list_source_files(source_task_id):
old_file_id = str(summary["id"])
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
new_file_id = new_id("dpsf")
old_reference = str(source.get("storage_object_id") or "")
if old_reference.startswith("local://data-process/"):
staged_object = storage.stage_copy(
batch_id=batch_id,
source_reference=old_reference,
expected_source_task_id=source_task_id,
expected_source_file_id=old_file_id,
task_id=target_task_id,
source_file_id=new_file_id,
version=1,
name=str(source["name"]),
)
staged.append(staged_object)
new_reference = staged_object.reference
elif old_reference.startswith("db://data-process/") or not old_reference:
new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1"
else:
raise ValueError("源任务包含不受支持的文件存储引用")
copies[old_file_id] = {
"id": new_file_id,
"storage_object_id": new_reference,
}
return copies, staged
def _remove_repeated_storage_objects(
storage: LocalDataProcessStorage,
task_id: str,
staged: list[StagedSourceObject],
copies: dict[str, dict[str, str]],
) -> None:
source_file_ids = {
str(copy["storage_object_id"]): str(copy["id"])
for copy in copies.values()
}
for item in staged:
try:
storage.delete(
item.reference,
expected_task_id=task_id,
expected_source_file_id=source_file_ids[item.reference],
)
except Exception:
logger.exception(
"failed to roll back repeated data process source object task_id=%s",
task_id,
)
@router.post("/{task_id}/repeat", status_code=202)
def repeat_generation(
task_id: str,
payload: DataProcessRepeatRequest,
background_tasks: BackgroundTasks,
store: DataProcessStore = Depends(get_data_process_store),
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
) -> dict[str, Any]:
"""按原任务快照创建独立任务,并立即在后台开始新一批生成。"""
with api_errors():
repeated = store.find_repeated_task(task_id, payload.request_id)
staged: list[StagedSourceObject] = []
target_task_id = repeat_task_id(task_id, payload.request_id)
if repeated is None:
copies, staged = _repeat_file_copies(
store,
storage,
task_id,
payload.request_id,
)
storage.publish(staged)
try:
repeated = store.repeat_task(
task_id,
expected_updated_at=payload.expected_updated_at,
request_id=payload.request_id,
file_copies=copies,
)
except Exception:
_remove_repeated_storage_objects(
storage,
target_task_id,
staged,
copies,
)
raise
if not repeated["created"]:
_remove_repeated_storage_objects(
storage,
target_task_id,
staged,
copies,
)
repeated_task = repeated["task"]
if repeated_task.get("status") == "pending":
try:
started = store.start_generation(target_task_id, replace_existing=True)
background_tasks.add_task(
_run_generation,
store,
target_task_id,
str(started["generation_run_id"]),
)
except ConflictError:
latest = store.get_task(target_task_id)
if latest.get("status") != "running":
raise
repeated["task"] = store.get_task(target_task_id)
repeated["progress"] = store.progress(target_task_id)
return ok(repeated, "已按原配置创建新任务并开始后台生成")
@router.delete("/{task_id}")
def delete_task(
task_id: str,
@@ -919,7 +1091,7 @@ async def upload_source_files(
f"{suffix} is not supported for {process_type} data processing",
)
parsed = parse_text_content(raw, filename=name)
if not parsed.text:
if not parsed.text.strip():
raise fail(400, f"source file is empty: {name}")
batch_size += len(raw)
if batch_size > MAX_SOURCE_BATCH_BYTES:
@@ -934,7 +1106,11 @@ async def upload_source_files(
content=raw,
)
staged.append(staged_object)
record_count = len(parsed.records) or (1 if parsed.text else 0)
record_count = (
len(parsed.records)
if process_type == "structured"
else (1 if parsed.text else 0)
)
prepared.append(
{
"id": source_file_id,
@@ -1362,15 +1538,28 @@ def _prepare_preview_items(
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
)
is_unstructured = task.get("process_type") == "unstructured"
if is_unstructured and (
needs_unstructured_raw = is_unstructured and (
chunk_method == "layout_hybrid"
or preprocess_options & {"clean_invalid", "clean_invalid_content"}
):
)
has_structured_xlsx = not is_unstructured and any(
str(source.get("file_format") or "").lower() == "xlsx"
for source in sources
)
if needs_unstructured_raw or has_structured_xlsx:
for index, source in enumerate(sources):
if (
chunk_method != "layout_hybrid"
and str(source.get("file_format") or "").lower() != "pdf"
):
source_format = str(source.get("file_format") or "").lower()
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
needs_pdf_noise = (
is_unstructured
and not needs_layout_raw
and source_format == "pdf"
and bool(
preprocess_options & {"clean_invalid", "clean_invalid_content"}
)
)
if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise):
continue
storage_object_id = str(source.get("storage_object_id") or "")
actual_size = storage.file_size(
@@ -1379,7 +1568,7 @@ def _prepare_preview_items(
expected_source_file_id=str(source["id"]),
)
if actual_size is None:
if chunk_method == "layout_hybrid":
if needs_layout_raw:
raise InvalidStateError(
"版面结构混合切分无法读取原始文件,请重新上传后再处理"
)
@@ -1396,12 +1585,10 @@ def _prepare_preview_items(
)
)
enriched = dict(source)
if chunk_method == "layout_hybrid":
if needs_structured_xlsx or needs_layout_raw:
enriched["raw_content"] = raw
sources[index] = enriched
continue
if str(source.get("file_format") or "").lower() != "pdf":
continue
pages = extract_pdf_page_texts(raw)
extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""):
@@ -1413,7 +1600,7 @@ def _prepare_preview_items(
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
sources[index] = enriched
items = _build_preview_items(task, sources)
if not items and source_file_ids is None:
if not items and source_file_ids is None and is_unstructured:
raise InvalidStateError("source files did not produce preview items")
return items
@@ -1435,6 +1622,7 @@ def _run_preview(
len(source_file_ids),
)
try:
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
@@ -1461,7 +1649,7 @@ def _run_preview(
storage,
[source_file_id],
)
if not items:
if not items and is_unstructured:
raise InvalidStateError(
f"source file did not produce preview items: {source_file_id}"
)
@@ -1657,8 +1845,13 @@ def update_preview_item(
) -> dict[str, Any]:
with api_errors():
task = store.get_task(task_id)
existing = store.get_preview_item(task_id, preview_id)
update = payload.model_dump(exclude_unset=True, mode="json")
update["quality_score"] = _preview_quality(payload.edited_content, task.get("config") or {})
quality = _preview_quality(payload.edited_content, task.get("config") or {})
source_locator = (existing.get("quality_score") or {}).get("source_locator")
if isinstance(source_locator, Mapping):
quality["source_locator"] = deepcopy(dict(source_locator))
update["quality_score"] = quality
item = store.update_preview_item(
task_id,
preview_id,

File diff suppressed because it is too large Load Diff

View File

@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
self._issued_staged_objects[temporary_path] = staged
return staged
def stage_copy(
self,
*,
batch_id: str,
source_reference: str,
expected_source_task_id: str,
expected_source_file_id: str,
task_id: str,
source_file_id: str,
version: int,
name: str,
) -> StagedSourceObject:
"""为不可变源对象创建独立目录项,不把大文件重新读入内存。"""
batch_id = _safe_component(batch_id, "batch id")
task_id = _safe_component(task_id, "task id")
source_file_id = _safe_component(source_file_id, "source file id")
if isinstance(version, bool) or not isinstance(version, int) or version < 1:
raise DataProcessStorageError("invalid source file version")
basename = _safe_basename(name)
source_relative = self._relative_from_reference(source_reference)
if source_relative is None:
raise DataProcessStorageError("original source object is not available")
self._assert_expected_owner(
source_relative,
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
source_path = self._path_for_relative(source_relative)
try:
os.link(source_path, temporary_path, follow_symlinks=False)
copy_info = temporary_path.lstat()
if (
not stat.S_ISREG(copy_info.st_mode)
or source_info.st_dev != copy_info.st_dev
or source_info.st_ino != copy_info.st_ino
):
raise DataProcessStorageError("source storage object changed while copying")
except Exception:
temporary_path.unlink(missing_ok=True)
raise
relative_path = PurePosixPath(
task_id,
source_file_id,
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
def publish(self, objects: Iterable[StagedSourceObject]) -> None:
staged = list(objects)
published: list[StagedSourceObject] = []

View File

@@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = {
"preserve_lists": True,
}
_REGENERATION_MARKER_KEY = "_regeneration_prepared"
_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id"
_REPEAT_REQUEST_KEY = "_repeat_request_id"
_INTERNAL_CONFIG_KEYS = {
_REGENERATION_MARKER_KEY,
_REPEAT_SOURCE_TASK_KEY,
_REPEAT_REQUEST_KEY,
}
class DataProcessStoreError(RuntimeError):
@@ -71,6 +78,13 @@ def new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:20]}"
def repeat_task_id(source_task_id: str, request_id: str) -> str:
"""按源任务和请求幂等键生成稳定的新任务 ID。"""
digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest()
return f"dpt_{digest[:20]}"
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
@@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool:
return _regeneration_marker(task) is not None
def _business_config(config: dict[str, Any] | None) -> dict[str, Any]:
"""过滤只供服务端维护的工作流标记。"""
return {
key: value
for key, value in (config or {}).items()
if key not in _INTERNAL_CONFIG_KEYS
}
def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None:
"""从 API 任务快照中移除服务端内部重新生成标记。"""
"""从 API 任务快照中移除服务端内部工作流标记。"""
if item is None:
return None
public = dict(item)
config = public.get("config")
if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config:
public["config"] = {
key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY
}
if isinstance(config, dict):
public["config"] = _business_config(config)
return public
@@ -353,13 +375,7 @@ class DataProcessStore:
payload.get("description") or "",
payload["process_type"],
payload.get("source_dataset_id"),
json_dumps(
{
key: value
for key, value in (payload.get("config") or {}).items()
if key != _REGENERATION_MARKER_KEY
}
),
json_dumps(_business_config(payload.get("config"))),
payload.get("tenant_id"),
payload.get("project_id"),
payload.get("owner_id"),
@@ -373,6 +389,268 @@ class DataProcessStore:
raise ConflictError("data process task name already exists") from exc
return _public_task(_decode_row(row)) or {}
@staticmethod
def _repeat_response(
conn: psycopg.Connection[dict[str, Any]],
row: dict[str, Any],
*,
source_task_id: str,
created: bool,
) -> dict[str, Any]:
task_id = str(row["id"])
counts = conn.execute(
"""
SELECT
(SELECT COUNT(*) FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count,
(SELECT COUNT(*) FROM data_process_preview_items
WHERE task_id=%s) AS preview_count
""",
(task_id, task_id),
).fetchone() or {}
task = _public_task(_decode_row(row)) or {}
task["source_file_count"] = int(counts.get("source_file_count") or 0)
task["preview_count"] = int(counts.get("preview_count") or 0)
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": task["source_file_count"],
"copied_preview_count": task["preview_count"],
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
"""查找同一幂等请求已创建的新任务。"""
task_id = repeat_task_id(source_task_id, request_id)
with self.connect() as conn:
row = conn.execute(
"SELECT * FROM data_process_tasks WHERE id=%s",
(task_id,),
).fetchone()
if row is None:
return None
decoded = _decode_row(row) or {}
config = decoded.get("config") or {}
if (
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
or config.get(_REPEAT_REQUEST_KEY) != request_id
):
raise ConflictError("再次生成请求与现有任务冲突")
if decoded.get("deleted_at"):
raise ConflictError("此次再次生成创建的任务已被删除,请重新发起")
return self._repeat_response(
conn,
row,
source_task_id=source_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
"""复制已确认任务的配置、源文件和预览,结果与发布数据保持独立。"""
task_id = repeat_task_id(source_task_id, request_id)
now = utcnow()
try:
with self.connect() as conn:
existing = conn.execute(
"SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE",
(task_id,),
).fetchone()
if existing is not None:
decoded = _decode_row(existing) or {}
config = decoded.get("config") or {}
if (
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
or config.get(_REPEAT_REQUEST_KEY) != request_id
):
raise ConflictError("再次生成请求与现有任务冲突")
if decoded.get("deleted_at"):
raise ConflictError("此次再次生成创建的任务已被删除,请重新发起")
return self._repeat_response(
conn,
existing,
source_task_id=source_task_id,
created=False,
)
source_task = self._task_in_connection(
conn,
source_task_id,
for_update=True,
)
if (
source_task.get("status") != "completed"
or source_task.get("results_confirmed") is False
):
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
raise ConflictError("源任务仍在处理切分,暂时不能再次生成")
if expected_updated_at != _serialize_value(source_task.get("updated_at")):
raise ConflictError("源任务已被其他操作修改,请刷新后重试")
source_files = conn.execute(
"""
SELECT * FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL
ORDER BY created_at, id
""",
(source_task_id,),
).fetchall()
source_file_ids = {str(row["id"]) for row in source_files}
if source_file_ids != set(file_copies):
raise ConflictError("源文件快照已变化,请刷新后重试")
previews = conn.execute(
"""
SELECT * FROM data_process_preview_items
WHERE task_id=%s
ORDER BY source_file_id NULLS LAST, source_start NULLS LAST,
created_at, id
""",
(source_task_id,),
).fetchall()
if not previews:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
suffix = f"(再次生成-{task_id[-6:]}"
base_name = str(source_task.get("name") or "数据处理任务")
repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}"
repeated_config = _business_config(source_task.get("config") or {})
repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id
repeated_config[_REPEAT_REQUEST_KEY] = request_id
input_count = sum(int(row.get("record_count") or 0) for row in source_files)
task_row = conn.execute(
"""
INSERT INTO data_process_tasks
(id, name, description, status, process_type, source_dataset_id,
output_dataset_id, config, progress, input_count, output_count,
filtered_count, duplicate_count, error_count, failure_reason,
generation_run_id, results_confirmed, workflow_step,
preview_status, preview_progress, preview_run_id,
preview_failure_reason, preview_total_files,
preview_completed_files, tenant_id, project_id, owner_id,
approval_status, created_by, updated_by, created_at, updated_at)
VALUES
(%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0,
0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100,
NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
task_id,
repeated_name,
source_task.get("description") or "",
source_task["process_type"],
source_task.get("source_dataset_id"),
json_dumps(repeated_config),
input_count,
len(source_files),
len(source_files),
source_task.get("tenant_id"),
source_task.get("project_id"),
source_task.get("owner_id"),
source_task.get("approval_status") or "not_required",
source_task.get("created_by"),
source_task.get("created_by"),
now,
now,
),
).fetchone()
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
new_file_id = str(copy["id"])
storage_object_id, metadata = _source_storage_descriptor(
{
"storage_object_id": copy["storage_object_id"],
"metadata": _json_value(source.get("metadata"), {}),
},
task_id,
new_file_id,
)
file_id_map[old_file_id] = new_file_id
conn.execute(
"""
INSERT INTO data_process_source_files
(id, task_id, storage_object_id, name, size_bytes, record_count,
file_format, checksum_sha256, version_no, content,
content_preview, metadata, tenant_id, project_id, created_by,
created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s,
%s, %s, %s, %s, %s)
""",
(
new_file_id,
task_id,
storage_object_id,
source["name"],
source.get("size_bytes") or 0,
source.get("record_count") or 0,
source.get("file_format"),
source["checksum_sha256"],
source.get("content") or "",
source.get("content_preview"),
json_dumps(metadata),
source_task.get("tenant_id"),
source_task.get("project_id"),
source.get("created_by") or source_task.get("created_by"),
now,
now,
),
)
for preview in previews:
old_source_file_id = preview.get("source_file_id")
conn.execute(
"""
INSERT INTO data_process_preview_items
(id, task_id, source_file_id, original_content, edited_content,
source_start, source_end, source_start_line, source_end_line,
token_count, status, quality_score, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s)
""",
(
new_id("dpp"),
task_id,
file_id_map.get(str(old_source_file_id))
if old_source_file_id
else None,
preview.get("original_content") or "",
preview.get("edited_content") or "",
preview.get("source_start"),
preview.get("source_end"),
preview.get("source_start_line"),
preview.get("source_end_line"),
max(0, int(preview.get("token_count") or 0)),
preview.get("status") or "original",
json_dumps(_json_value(preview.get("quality_score"), {})),
now,
now,
),
)
return self._repeat_response(
conn,
task_row or {},
source_task_id=source_task_id,
created=True,
)
except psycopg.errors.UniqueViolation as exc:
raise ConflictError("再次生成任务名称或请求发生冲突,请重试") from exc
def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]:
lock = " FOR UPDATE" if for_update else ""
with self.connect() as conn:
@@ -654,14 +932,11 @@ class DataProcessStore:
"process type and source dataset cannot change during regeneration"
)
if payload.get("config") is not None:
next_config = {
key: value
for key, value in payload["config"].items()
if key != _REGENERATION_MARKER_KEY
}
current_marker = _regeneration_marker(task)
if current_marker:
next_config[_REGENERATION_MARKER_KEY] = current_marker
next_config = _business_config(payload["config"])
current_config = dict(task.get("config") or {})
for key in _INTERNAL_CONFIG_KEYS:
if key in current_config:
next_config[key] = current_config[key]
values["config"] = json_dumps(next_config)
invalidates_results = (
("config" in payload and payload.get("config") != task.get("config"))
@@ -753,8 +1028,10 @@ class DataProcessStore:
raise InvalidStateError("process_type cannot be changed during regeneration")
current_config = dict(task.get("config") or {})
next_config = dict(payload.get("config") or {})
next_config.pop(_REGENERATION_MARKER_KEY, None)
next_config = _business_config(payload.get("config"))
for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY):
if key in current_config:
next_config[key] = current_config[key]
preview_invalidated = _preview_config_changed(
process_type,
current_config,

View File

@@ -220,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel):
return self
class DataProcessRepeatRequest(BaseModel):
"""按已确认任务的完整快照创建一批独立的新生成结果。"""
model_config = ConfigDict(extra="forbid")
expected_updated_at: str = Field(min_length=1)
request_id: str = Field(
min_length=8,
max_length=80,
pattern=r"^[A-Za-z0-9_-]+$",
)
class PreviewBuildRequest(BaseModel):
model_config = ConfigDict(extra="forbid")

View File

@@ -5,6 +5,7 @@ import json
import xml.etree.ElementTree as ET
import zipfile
from datetime import datetime
from decimal import Decimal
import pytest
from docx import Document
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
normalize_text,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
record_fingerprint,
remove_document_noise,
score_quality,
stable_split,
stable_split_assignments,
structured_json_dumps,
)
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
assert parsed_txt.text == "普通文本"
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
root_json = parse_text_content('{"id":1}', filename="root.json")
assert root_json.record_locators == (
{
"kind": "json",
"record_index": 1,
"json_pointer": "",
"source_start": 0,
"source_end": 8,
"start_line": 1,
"end_line": 1,
},
)
wrapped_json = parse_text_content(
'{"records":[{"id":1},{"id":1}]}',
filename="wrapped.json",
)
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
"/records/0",
"/records/1",
]
parsed_jsonl = parse_text_content(
'{"id":1}\r\n\r\n{"id":1}',
filename="records.jsonl",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_jsonl.record_locators
] == [(1, 1, 1), (2, 3, 3)]
assert [
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_jsonl.record_locators
] == ['{"id":1}', '{"id":1}']
parsed_csv = parse_text_content(
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
filename="records.csv",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_csv.record_locators
] == [(1, 2, 3), (2, 5, 5)]
assert [
parsed_csv.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_csv.record_locators
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
processed = preprocess_structured_records_with_lineage(
[
{"id": "A", "value": "first", "empty": ""},
{"id": "", "value": "invalid", "empty": ""},
{"id": "A", "value": "duplicate identity", "empty": ""},
{"id": "B", "value": "second", "empty": ""},
],
["clean_invalid", "deduplicate"],
)
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
assert [entry.record for entry in processed] == [
{"id": "A", "value": "first"},
{"id": "", "value": "invalid"},
{"id": "A", "value": "duplicate identity"},
{"id": "B", "value": "second"},
]
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
assert parsed_pdf.format == "pdf"
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
)
assert parsed_xlsx.record_locators == (
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 3,
"sheet_record_index": 1,
},
)
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active
first.title = "甲表"
first.append(["说明"])
first.append([])
first.append(["id", "value"])
first.append([1, "same"])
first.append([1, "same"])
second = workbook.create_sheet("乙表")
second.append(["id", "value"])
second.append([1, "same"])
output = io.BytesIO()
workbook.save(output)
workbook.close()
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
assert parsed.records == (
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
)
assert [
(
locator["record_index"],
locator["sheet_index"],
locator["sheet_name"],
locator["row_number"],
locator["sheet_record_index"],
)
for locator in parsed.record_locators
] == [
(1, 0, "甲表", 4, 0),
(2, 0, "甲表", 5, 1),
(3, 1, "乙表", 2, 0),
]
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
pages = _pdf_page_texts(
"""
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
"json",
)
assert result == [{"text": "内容"}]
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
assert extract_structured_records(
'{"items":[{"text":" 内容 "}],"total":1}',
"json",
) == [{"text": " 内容 "}]
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
source = '{"code":"","text":" 内容 ","quote":""}'
parsed = parse_text_content(source, filename="records.json")
assert parsed.text == source
assert parsed.records == (
{"code": "", "text": " 内容 ", "quote": ""},
)
invalid_values = (
'{"id":1,"id":2}',
'{"nested":{"id":1,"id":2}}',
'{"value":NaN}',
'{"value":Infinity}',
'{"value":-Infinity}',
'{"value":"bad\x00control"}',
)
for invalid in invalid_values:
with pytest.raises(ValueError):
parse_text_content(invalid, filename="invalid.json")
with pytest.raises(ValueError):
parse_text_content("\"id\":1", filename="invalid.json")
with pytest.raises(ValueError, match="nesting exceeds"):
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
source = (
' {"code":"","text":" 内容 ",'
'"value":0.123456789012345678901234567890}\r\n\r\n'
'{"id":2}\r\n'
)
parsed = parse_text_content(source, filename="records.jsonl")
assert parsed.text == source
assert parsed.records[0] == {
"code": "",
"text": " 内容 ",
"value": Decimal("0.123456789012345678901234567890"),
}
assert [
source[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == [
(
'{"code":"","text":" 内容 ",'
'"value":0.123456789012345678901234567890}'
),
'{"id":2}',
]
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
parse_text_content(invalid, filename="invalid.jsonl")
def test_json_record_contract_avoids_business_field_collisions() -> None:
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
{"id": 1},
{"id": 2},
]
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
{"id": 1, "data": [{"id": 2}]}
]
assert extract_structured_records(
'{"records":[{"id":1}],"data":[{"id":2}]}',
"json",
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
assert extract_structured_records(
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
"json",
) == [{"id": 1}]
assert extract_structured_records(
'{"payload":{"data":[{"id":2}],"total":1}}',
"json",
) == [{"id": 2}]
assert extract_structured_records('{"records":[],"total":0}', "json") == []
# 包装数组中的非对象不是记录集合,整体按一条业务对象保留。
assert extract_structured_records('{"data":[1,2]}', "json") == [
{"data": [1, 2]}
]
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
pretty = (
'{\n "records": [\n {"id": 1},\n'
' {\n "id": 2\n }\n ],\n "total": 2\n}'
)
parsed = parse_text_content(pretty, filename="pretty.json")
assert [
pretty[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id": 1}', '{\n "id": 2\n }']
assert [
(locator["start_line"], locator["end_line"])
for locator in parsed.record_locators
] == [(3, 3), (4, 6)]
minified = '[{"id":1},{"id":2}]'
parsed = parse_text_content(minified, filename="minified.json")
assert [
minified[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id":1}', '{"id":2}']
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
parsed = parse_text_content(source, filename="precise.json")
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
assert parsed.records[1]["value"] == Decimal("1e400")
assert structured_json_dumps(parsed.records[0]) == (
'{"value":0.123456789012345678901234567890}'
)
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
assert isinstance(parsed.records[0]["value"], Decimal)
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
assert preprocess_structured_records(clean_source, []) == clean_source
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
{"id": "1", "name": "有效"},
{"id": "", "name": "缺少关键字段"},
{"id": "2", "name": "有效"},
]
hierarchy = [
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
{"id": "", "parent_id": "", "name": "", "empty": ""},
]
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
{"id": "1", "parent_id": None, "name": "根节点"},
{"id": "2", "parent_id": "1", "name": "子节点"},
]
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
assert "profile" in preprocess_structured_records(nested, [])[0]
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
duplicates = [
{"customer_id": "C-1", "value": "first"},
{"customer_id": "C-1", "value": "updated"},
{"customer_id": "C-1", "value": "first"},
{"customer_id": "", "value": "blank-one"},
{"customer_id": "", "value": "blank-two"},
]
assert len(preprocess_structured_records(duplicates, [])) == 4
assert len(preprocess_structured_records(duplicates, [])) == 5
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
assert [record["value"] for record in deduplicated] == [
"first",
"updated",
"blank-one",
"blank-two",
]
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
)
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
masked, counts = desensitize_structured_record(
{
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "张三",
"contact_name": "李四",
"姓名": "王五",
"profile.name": "赵六",
}
)
assert masked == {
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "[NAME]",
"contact_name": "[NAME]",
"姓名": "[NAME]",
"profile.name": "[NAME]",
}
assert counts == {
"email": 0,
"phone": 0,
"id_card": 0,
"name": 4,
"total": 4,
}
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
valid = {
"instruction": "如何修改收货地址?",

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
from copy import deepcopy
from io import BytesIO
from pathlib import Path
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
LocalDataProcessStorage,
get_data_process_storage,
)
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
from app.modules.data_process.store import (
InvalidStateError,
NotFoundError,
get_data_process_store,
repeat_task_id,
)
class FakeDataProcessStore:
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
self.datasets: dict[str, dict[str, Any]] = {}
self.models: dict[str, dict[str, Any]] = {}
self.regeneration_prepared: set[str] = set()
self.repeat_requests: dict[tuple[str, str], str] = {}
self.sequence = 0
def _id(self, prefix: str) -> str:
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
"published_outputs_preserved": published_outputs_preserved,
}
def _repeat_response(
self,
source_task_id: str,
repeated_task_id: str,
*,
created: bool,
) -> dict[str, Any]:
task = self.get_task(repeated_task_id)
task["source_file_count"] = len(self.sources[repeated_task_id])
task["preview_count"] = len(self.previews[repeated_task_id])
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": len(self.sources[repeated_task_id]),
"copied_preview_count": len(self.previews[repeated_task_id]),
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
if repeated_task_id is None:
return None
return self._repeat_response(
source_task_id,
repeated_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
existing = self.find_repeated_task(source_task_id, request_id)
if existing is not None:
return existing
source_task = self.get_task(source_task_id)
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("updated_at") != expected_updated_at:
raise InvalidStateError("源任务已被其他操作修改,请刷新后重试")
source_files = self.sources[source_task_id]
if set(file_copies) != {str(item["id"]) for item in source_files}:
raise InvalidStateError("源文件快照已变化,请刷新后重试")
if not self.previews[source_task_id]:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
repeated_task_id = repeat_task_id(source_task_id, request_id)
suffix = f"(再次生成-{repeated_task_id[-6:]}"
task = {
**deepcopy(source_task),
"id": repeated_task_id,
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
"status": "pending",
"progress": 20,
"output_dataset_id": None,
"output_datasets": [],
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": len(source_files),
"preview_completed_files": len(source_files),
"started_at": None,
"completed_at": None,
}
self.tasks[repeated_task_id] = task
self.sources[repeated_task_id] = []
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
file_id_map[old_file_id] = copy["id"]
self.sources[repeated_task_id].append(
{
**deepcopy(source),
"id": copy["id"],
"task_id": repeated_task_id,
"storage_object_id": copy["storage_object_id"],
}
)
self.previews[repeated_task_id] = [
{
**deepcopy(item),
"id": self._id("dpp"),
"task_id": repeated_task_id,
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
if item.get("source_file_id")
else None,
}
for item in self.previews[source_task_id]
]
self.results[repeated_task_id] = []
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
return self._repeat_response(
source_task_id,
repeated_task_id,
created=True,
)
def delete_task(self, task_id: str, **_: Any) -> None:
self.get_task(task_id)
del self.tasks[task_id]
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
assert listed_preview.json()["data"]["total"] == 2
preview_item = listed_preview.json()["data"]["items"][0]
source_locator = preview_item["quality_score"]["source_locator"]
assert source_locator == {
"kind": "jsonl",
"record_index": 1,
"start_line": 1,
"end_line": 1,
"source_start": 0,
"source_end": len(preview_item["original_content"]),
}
updated_preview = client.put(
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
json={
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
},
)
assert "quality_score" in updated_preview.json()["data"]
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
assert generated.status_code == 200
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
def test_completed_task_can_repeat_into_an_independent_background_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "原始生成任务",
"process_type": "structured",
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
)
assert uploaded.status_code == 200
source_id = uploaded.json()["data"]["files"][0]["id"]
built = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"replace_existing": True},
)
assert built.status_code == 200
store.tasks[task_id].update(
status="completed",
progress=100,
results_confirmed=True,
workflow_step="results",
output_count=1,
output_dataset_id="dataset-original",
updated_at="2026-07-28T12:00:00Z",
)
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
store.datasets["dataset-original"] = {
"id": "dataset-original",
"name": "原数据集",
"type": "train",
"source_task_id": task_id,
"deleted_at": None,
}
original_task = deepcopy(store.tasks[task_id])
original_sources = deepcopy(store.sources[task_id])
original_previews = deepcopy(store.previews[task_id])
original_results = deepcopy(store.results[task_id])
original_datasets = deepcopy(store.datasets)
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
payload = {
"expected_updated_at": "2026-07-28T12:00:00Z",
"request_id": "repeat-request-0001",
}
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert response.status_code == 202
repeated = response.json()["data"]
repeated_task_id = repeated["task"]["id"]
assert repeated["created"] is True
assert repeated_task_id != task_id
assert repeated["task"]["status"] == "running"
assert repeated["task"]["workflow_step"] == "generate"
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == len(original_previews)
assert store.tasks[task_id] == original_task
assert store.sources[task_id] == original_sources
assert store.previews[task_id] == original_previews
assert store.results[task_id] == original_results
assert store.datasets == original_datasets
repeated_source = store.sources[repeated_task_id][0]
repeated_preview = store.previews[repeated_task_id][0]
assert repeated_source["id"] != source_id
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
assert repeated_preview["id"] != original_previews[0]["id"]
assert repeated_preview["source_file_id"] == repeated_source["id"]
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert replay.status_code == 202
assert replay.json()["data"]["created"] is False
assert replay.json()["data"]["task"]["id"] == repeated_task_id
assert len(store.tasks) == 2
assert len(store.sources[repeated_task_id]) == 1
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "源任务", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
results_confirmed=True,
updated_at="2026-07-28T12:00:00Z",
)
before = deepcopy(store.tasks)
response = client.post(
f"/modelTF/data-process/{task_id}/repeat",
json={
"expected_updated_at": "2026-07-28T11:59:59Z",
"request_id": "repeat-request-stale",
},
)
assert response.status_code == 409
assert store.tasks == before
def test_published_split_datasets_remain_in_detail_after_regeneration(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "先去重再脱敏",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={
"files": (
"names.jsonl",
(
'{"name":"张三","role":"开发"}\n'
'{"name":"李四","role":"开发"}\n'
),
"application/jsonl",
)
},
)
assert uploaded.status_code == 200
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert len(items) == 2
assert len({item["original_content"] for item in items}) == 2
assert {item["edited_content"] for item in items} == {
'{"name":"[NAME]","role":"开发"}'
}
def test_structured_deduplication_removes_identical_rows_across_sources(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "跨源原文去重",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
(
"files",
(
"first.jsonl",
'{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
(
"files",
(
"second.jsonl",
'\n{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
],
)
assert uploaded.status_code == 200
first_source, second_source = uploaded.json()["data"]["files"]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
data = preview.json()["data"]
assert data["total"] == 1
assert data["file_counts"] == {
first_source["id"]: 1,
second_source["id"]: 0,
}
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
store = FakeDataProcessStore()
task = store.create_task(
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
)
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["total"] == 2
preview_items = preview.json()["data"]["items"]
assert len(preview_items) == 2
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 3,
"sheet_record_index": 1,
},
]
def test_docx_preview_preserves_document_block_order_and_source_offsets(
@@ -2757,6 +3105,218 @@ def _preview_task(
)
def _structured_preview_task(
content: str,
*,
file_format: str,
options: list[str] | None = None,
) -> list[dict[str, Any]]:
return data_process_endpoint._build_preview_items(
{
"process_type": "structured",
"config": {"preprocess_options": options or []},
},
[
{
"id": "structured-source",
"name": f"records.{file_format}",
"file_format": file_format,
"content": content,
}
],
)
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
json_source = '{"records":[{"id":1},{"id":2}]}'
json_items = _structured_preview_task(
json_source,
file_format="json",
)
assert [
item["quality_score"]["source_locator"]["json_pointer"]
for item in json_items
] == ["/records/0", "/records/1"]
assert [
json_source[item["source_start"] : item["source_end"]]
for item in json_items
] == ['{"id":1}', '{"id":2}']
assert [item["source_start_line"] for item in json_items] == [1, 1]
jsonl_source = '{"id":1}\n\n{"id":2}'
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in jsonl_items
] == [1, 2]
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
assert [
jsonl_source[item["source_start"] : item["source_end"]]
for item in jsonl_items
] == ['{"id":1}', '{"id":2}']
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
csv_items = _structured_preview_task(csv_source, file_format="csv")
assert [
(item["source_start_line"], item["source_end_line"])
for item in csv_items
] == [(2, 3), (5, 5)]
assert [
csv_source[item["source_start"] : item["source_end"]]
for item in csv_items
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "空 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("empty-array.json", "[]", "application/json")),
(
"files",
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["items"] == []
assert preview.json()["data"]["total"] == 0
assert set(preview.json()["data"]["file_counts"].values()) == {0}
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
invalid_sources = (
("duplicate.json", '{"id":1,"id":2}'),
("duplicate.jsonl", '{"id":1,"id":2}\n'),
("nan.json", '{"value":NaN}'),
("infinity.json", '{"value":Infinity}'),
("control.json", '{"value":"bad\x00control"}'),
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
)
for filename, content in invalid_sources:
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": (filename, content, "application/json")},
)
assert response.status_code == 400, (filename, response.text)
def test_structured_json_preview_preserves_precision_and_business_data_field(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
precise = '{"value":0.123456789012345678901234567890}'
business = '{"id":7,"data":[{"id":8}]}'
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("precise.json", precise, "application/json")),
("files", ("business.json", business, "application/json")),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert [item["original_content"] for item in items] == [precise, business]
assert [
item["quality_score"]["source_locator"]["json_pointer"] for item in items
] == ["", ""]
assert [item["source_start"] for item in items] == [0, 0]
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
source_records = [
{"id": "A", "amount": 10, "empty": ""},
{"id": "A", "amount": 10, "empty": ""},
{"id": "", "amount": 11, "empty": ""},
{"id": "B", "amount": 11, "empty": ""},
{"id": "C", "amount": 12, "empty": ""},
{"id": "D", "amount": 12, "empty": ""},
{"id": "E", "amount": 13, "empty": ""},
{"id": "F", "amount": 13, "empty": ""},
{"id": "G", "amount": 14, "empty": ""},
{"id": "H", "amount": 1000, "empty": ""},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate", "filter_anomaly"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [json.loads(item["original_content"])["id"] for item in items] == [
"A",
"",
"B",
"C",
"D",
"E",
"F",
"G",
]
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
source_records = [
{"customer_id": "C-1", "status": "old"},
{"customer_id": "C-1", "status": "new"},
{"status": "old", "customer_id": "C-1"},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 2]
assert [item["source_start_line"] for item in items] == [1, 2]
assert [json.loads(item["original_content"])["status"] for item in items] == [
"old",
"new",
]
def test_fixed_preview_preserves_source_offsets() -> None:
content = (
"# 第一章\n"

View File

@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
_assert_staging_empty(storage)
def test_stage_copy_creates_an_independently_deletable_source_object(
tmp_path: Path,
) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")
original = _stage(storage, content=b"immutable source")
storage.publish([original])
copied = storage.stage_copy(
batch_id="batch-copy",
source_reference=original.reference,
expected_source_task_id="task-1",
expected_source_file_id="source-1",
task_id="task-2",
source_file_id="source-2",
version=1,
name="source.txt",
)
storage.publish([copied])
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
original.reference,
expected_task_id="task-1",
expected_source_file_id="source-1",
) is True
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
copied.reference,
expected_task_id="task-2",
expected_source_file_id="source-2",
) is True
_assert_staging_empty(storage)
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")

View File

@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
_preview_config_changed,
_reasoning_output_is_valid,
_source_storage_descriptor,
repeat_task_id,
)
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
yield self._conn
class _RepeatConnection:
def __init__(self) -> None:
self.source_files = [
{
"id": "source-old",
"name": "source.jsonl",
"size_bytes": 12,
"record_count": 1,
"file_format": "jsonl",
"checksum_sha256": "a" * 64,
"content": '{"id":1}\n',
"content_preview": '{"id":1}',
"metadata": {"storage_backend": "local"},
"created_by": "user-1",
}
]
self.source_previews = [
{
"id": "preview-old",
"source_file_id": "source-old",
"original_content": '{"id":1}',
"edited_content": '{"id":1,"checked":true}',
"source_start": 0,
"source_end": 8,
"source_start_line": 1,
"source_end_line": 1,
"token_count": 5,
"status": "modified",
"quality_score": {"overall": 90},
}
]
self.created_task: dict[str, Any] | None = None
self.created_files: list[dict[str, Any]] = []
self.created_previews: list[dict[str, Any]] = []
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if params is not None:
assert normalized.count("%s") == len(params)
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
return _Result(row=None)
if normalized.startswith("SELECT * FROM data_process_source_files"):
return _Result(rows=[dict(item) for item in self.source_files])
if normalized.startswith("SELECT * FROM data_process_preview_items"):
return _Result(rows=[dict(item) for item in self.source_previews])
if normalized.startswith("INSERT INTO data_process_tasks"):
self.created_task = {
"id": params[0],
"name": params[1],
"description": params[2],
"status": "pending",
"process_type": params[3],
"source_dataset_id": params[4],
"config": params[5],
"progress": 20,
"input_count": params[6],
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_total_files": params[7],
"preview_completed_files": params[8],
"created_at": params[15],
"updated_at": params[16],
}
return _Result(row=dict(self.created_task))
if normalized.startswith("INSERT INTO data_process_source_files"):
self.created_files.append(
{
"id": params[0],
"task_id": params[1],
"storage_object_id": params[2],
"content": params[8],
}
)
return _Result()
if normalized.startswith("INSERT INTO data_process_preview_items"):
self.created_previews.append(
{
"id": params[0],
"task_id": params[1],
"source_file_id": params[2],
"edited_content": params[4],
}
)
return _Result()
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
return _Result(
row={
"source_file_count": len(self.created_files),
"preview_count": len(self.created_previews),
}
)
raise AssertionError(f"unexpected SQL: {normalized}")
class _RepeatStore(DataProcessStore):
def __init__(self, conn: _RepeatConnection) -> None:
self._conn = conn
@contextmanager
def connect(self) -> Iterator[_RepeatConnection]:
yield self._conn
def _task_in_connection(
self,
conn: Any,
task_id: str,
*,
for_update: bool = False,
) -> dict[str, Any]:
assert task_id == "task-source"
assert for_update is True
return {
"id": task_id,
"name": "原任务",
"description": "原描述",
"status": "completed",
"process_type": "structured",
"source_dataset_id": None,
"config": {
"temperature": 0.3,
"_regeneration_prepared": {"prepared": True},
},
"results_confirmed": True,
"preview_status": "completed",
"tenant_id": "tenant-1",
"project_id": "project-1",
"owner_id": "owner-1",
"created_by": "user-1",
"updated_at": "2026-07-28T12:00:00Z",
}
class _TaskListConnection:
def __init__(self) -> None:
self.task = {
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
conn = _RepeatConnection()
store = _RepeatStore(conn)
request_id = "repeat-request-0001"
target_task_id = repeat_task_id("task-source", request_id)
repeated = store.repeat_task(
"task-source",
expected_updated_at="2026-07-28T12:00:00Z",
request_id=request_id,
file_copies={
"source-old": {
"id": "source-new",
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
}
},
)
assert repeated["created"] is True
assert repeated["task"]["id"] == target_task_id
assert repeated["task"]["config"] == {"temperature": 0.3}
assert repeated["task"]["results_confirmed"] is False
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == 1
assert conn.created_files == [
{
"id": "source-new",
"task_id": target_task_id,
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
"content": '{"id":1}\n',
}
]
assert conn.created_previews[0]["task_id"] == target_task_id
assert conn.created_previews[0]["source_file_id"] == "source-new"
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
decoded = _decode_row(
{