fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
@@ -8,12 +8,14 @@ import os
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from threading import BoundedSemaphore, Lock
|
||||
from typing import Any, Iterator, Literal
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
import httpx
|
||||
@@ -45,9 +47,10 @@ from app.modules.data_process.algorithms import (
|
||||
is_near_duplicate,
|
||||
near_duplicate_fingerprint,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
preprocess_structured_records_with_lineage,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
structured_json_dumps,
|
||||
)
|
||||
from app.modules.data_process.document_chunking import (
|
||||
DocumentChunk,
|
||||
@@ -75,9 +78,11 @@ from app.modules.data_process.store import (
|
||||
NotFoundError,
|
||||
get_data_process_store,
|
||||
new_id,
|
||||
repeat_task_id,
|
||||
)
|
||||
from app.schemas.data_process import (
|
||||
DataProcessRegenerateRequest,
|
||||
DataProcessRepeatRequest,
|
||||
DataProcessStatus,
|
||||
DataProcessTaskCreate,
|
||||
DataProcessTaskUpdate,
|
||||
@@ -261,7 +266,14 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
|
||||
content = str(source.get("content") or "")
|
||||
file_format = str(source.get("file_format") or "").lower()
|
||||
if file_format == "xlsx":
|
||||
# XLSX 上传阶段已安全解析为 JSONL 后入库。
|
||||
raw_content = source.get("raw_content")
|
||||
if isinstance(raw_content, bytes):
|
||||
return parse_text_content(
|
||||
raw_content,
|
||||
filename=str(source.get("name") or "source.xlsx"),
|
||||
file_format="xlsx",
|
||||
)
|
||||
# 兼容原始对象已缺失的历史文件:退化为上传阶段生成的 JSONL。
|
||||
return parse_text_content(content, file_format="jsonl")
|
||||
if file_format in {"pdf", "docx", "pptx"}:
|
||||
# 文档上传阶段已抽取文本,预览阶段只需要对正文切片。
|
||||
@@ -381,11 +393,12 @@ def _build_preview_items(
|
||||
seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {}
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
def append_item(item: dict[str, Any]) -> None:
|
||||
def append_item(item: dict[str, Any], *, dedup_content: str) -> None:
|
||||
content = str(item.get("edited_content") or "").strip()
|
||||
if should_clean_invalid and not content:
|
||||
return
|
||||
content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
# 去重必须基于脱敏前内容,否则不同原文可能在替换 PII 后被错误合并。
|
||||
content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest()
|
||||
if should_deduplicate and content_hash in seen_content_hashes:
|
||||
return
|
||||
seen_content_hashes.add(content_hash)
|
||||
@@ -447,6 +460,7 @@ def _build_preview_items(
|
||||
continue
|
||||
for key in band_keys:
|
||||
seen_near_duplicate_bands.setdefault(key, []).append(content)
|
||||
dedup_content = content
|
||||
pii_counts: dict[str, int] = {}
|
||||
if should_desensitize:
|
||||
content, pii_counts = desensitize_pii(content)
|
||||
@@ -476,7 +490,8 @@ def _build_preview_items(
|
||||
else "original"
|
||||
),
|
||||
"quality_score": quality,
|
||||
}
|
||||
},
|
||||
dedup_content=dedup_content,
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -488,48 +503,76 @@ def _build_preview_items(
|
||||
"filter_anomaly",
|
||||
}
|
||||
source_records = list(parsed.records)
|
||||
processed_records = preprocess_structured_records(
|
||||
processed_records = preprocess_structured_records_with_lineage(
|
||||
source_records,
|
||||
structured_options,
|
||||
)
|
||||
if not processed_records and parsed.text and not source_records:
|
||||
processed_records = [{"value": parsed.text}]
|
||||
same_cardinality = len(processed_records) == len(source_records)
|
||||
for index, record in enumerate(processed_records):
|
||||
original_record = source_records[index] if same_cardinality else record
|
||||
original_content = json.dumps(
|
||||
original_record,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
for processed in processed_records:
|
||||
source_index = processed.source_index
|
||||
record = processed.record
|
||||
original_record = (
|
||||
source_records[source_index]
|
||||
if source_index < len(source_records)
|
||||
else record
|
||||
)
|
||||
source_locator = (
|
||||
deepcopy(parsed.record_locators[source_index])
|
||||
if source_index < len(parsed.record_locators)
|
||||
else None
|
||||
)
|
||||
original_content = structured_json_dumps(original_record)
|
||||
pii_counts: dict[str, int] = {}
|
||||
edited_record = record
|
||||
dedup_content = (
|
||||
canonical_record_json(record)
|
||||
if "normalize_format" in preprocess_options
|
||||
else structured_json_dumps(record)
|
||||
)
|
||||
if should_desensitize:
|
||||
edited_record, pii_counts = desensitize_structured_record(record)
|
||||
content = (
|
||||
canonical_record_json(edited_record)
|
||||
if "normalize_format" in preprocess_options
|
||||
else json.dumps(
|
||||
edited_record,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
else structured_json_dumps(edited_record)
|
||||
)
|
||||
quality = _preview_quality(content, config)
|
||||
quality["pii_replacements"] = pii_counts
|
||||
if source_locator is not None:
|
||||
quality["source_locator"] = source_locator
|
||||
source_start = (
|
||||
source_locator.get("source_start")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
source_end = (
|
||||
source_locator.get("source_end")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
source_start_line = (
|
||||
source_locator.get("start_line")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
source_end_line = (
|
||||
source_locator.get("end_line")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
append_item(
|
||||
{
|
||||
"source_file_id": source["id"],
|
||||
"original_content": original_content,
|
||||
"edited_content": content,
|
||||
"source_start": None,
|
||||
"source_end": None,
|
||||
"source_start_line": None,
|
||||
"source_end_line": None,
|
||||
"source_start": source_start,
|
||||
"source_end": source_end,
|
||||
"source_start_line": source_start_line,
|
||||
"source_end_line": source_end_line,
|
||||
"token_count": estimate_token_count(content),
|
||||
"status": "modified" if content != original_content else "original",
|
||||
"quality_score": quality,
|
||||
}
|
||||
},
|
||||
dedup_content=dedup_content,
|
||||
)
|
||||
return items
|
||||
|
||||
@@ -850,6 +893,135 @@ def prepare_regeneration(
|
||||
)
|
||||
|
||||
|
||||
def _repeat_file_copies(
|
||||
store: DataProcessStore,
|
||||
storage: LocalDataProcessStorage,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]:
|
||||
"""为新任务创建独立的源文件引用,避免删除任一任务时互相影响。"""
|
||||
|
||||
target_task_id = repeat_task_id(source_task_id, request_id)
|
||||
copies: dict[str, dict[str, str]] = {}
|
||||
staged: list[StagedSourceObject] = []
|
||||
batch_id = storage.new_batch_id()
|
||||
for summary in store.list_source_files(source_task_id):
|
||||
old_file_id = str(summary["id"])
|
||||
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
|
||||
new_file_id = new_id("dpsf")
|
||||
old_reference = str(source.get("storage_object_id") or "")
|
||||
if old_reference.startswith("local://data-process/"):
|
||||
staged_object = storage.stage_copy(
|
||||
batch_id=batch_id,
|
||||
source_reference=old_reference,
|
||||
expected_source_task_id=source_task_id,
|
||||
expected_source_file_id=old_file_id,
|
||||
task_id=target_task_id,
|
||||
source_file_id=new_file_id,
|
||||
version=1,
|
||||
name=str(source["name"]),
|
||||
)
|
||||
staged.append(staged_object)
|
||||
new_reference = staged_object.reference
|
||||
elif old_reference.startswith("db://data-process/") or not old_reference:
|
||||
new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1"
|
||||
else:
|
||||
raise ValueError("源任务包含不受支持的文件存储引用")
|
||||
copies[old_file_id] = {
|
||||
"id": new_file_id,
|
||||
"storage_object_id": new_reference,
|
||||
}
|
||||
return copies, staged
|
||||
|
||||
|
||||
def _remove_repeated_storage_objects(
|
||||
storage: LocalDataProcessStorage,
|
||||
task_id: str,
|
||||
staged: list[StagedSourceObject],
|
||||
copies: dict[str, dict[str, str]],
|
||||
) -> None:
|
||||
source_file_ids = {
|
||||
str(copy["storage_object_id"]): str(copy["id"])
|
||||
for copy in copies.values()
|
||||
}
|
||||
for item in staged:
|
||||
try:
|
||||
storage.delete(
|
||||
item.reference,
|
||||
expected_task_id=task_id,
|
||||
expected_source_file_id=source_file_ids[item.reference],
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"failed to roll back repeated data process source object task_id=%s",
|
||||
task_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/repeat", status_code=202)
|
||||
def repeat_generation(
|
||||
task_id: str,
|
||||
payload: DataProcessRepeatRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||
) -> dict[str, Any]:
|
||||
"""按原任务快照创建独立任务,并立即在后台开始新一批生成。"""
|
||||
|
||||
with api_errors():
|
||||
repeated = store.find_repeated_task(task_id, payload.request_id)
|
||||
staged: list[StagedSourceObject] = []
|
||||
target_task_id = repeat_task_id(task_id, payload.request_id)
|
||||
if repeated is None:
|
||||
copies, staged = _repeat_file_copies(
|
||||
store,
|
||||
storage,
|
||||
task_id,
|
||||
payload.request_id,
|
||||
)
|
||||
storage.publish(staged)
|
||||
try:
|
||||
repeated = store.repeat_task(
|
||||
task_id,
|
||||
expected_updated_at=payload.expected_updated_at,
|
||||
request_id=payload.request_id,
|
||||
file_copies=copies,
|
||||
)
|
||||
except Exception:
|
||||
_remove_repeated_storage_objects(
|
||||
storage,
|
||||
target_task_id,
|
||||
staged,
|
||||
copies,
|
||||
)
|
||||
raise
|
||||
if not repeated["created"]:
|
||||
_remove_repeated_storage_objects(
|
||||
storage,
|
||||
target_task_id,
|
||||
staged,
|
||||
copies,
|
||||
)
|
||||
|
||||
repeated_task = repeated["task"]
|
||||
if repeated_task.get("status") == "pending":
|
||||
try:
|
||||
started = store.start_generation(target_task_id, replace_existing=True)
|
||||
background_tasks.add_task(
|
||||
_run_generation,
|
||||
store,
|
||||
target_task_id,
|
||||
str(started["generation_run_id"]),
|
||||
)
|
||||
except ConflictError:
|
||||
latest = store.get_task(target_task_id)
|
||||
if latest.get("status") != "running":
|
||||
raise
|
||||
repeated["task"] = store.get_task(target_task_id)
|
||||
repeated["progress"] = store.progress(target_task_id)
|
||||
return ok(repeated, "已按原配置创建新任务并开始后台生成")
|
||||
|
||||
|
||||
@router.delete("/{task_id}")
|
||||
def delete_task(
|
||||
task_id: str,
|
||||
@@ -919,7 +1091,7 @@ async def upload_source_files(
|
||||
f"{suffix} is not supported for {process_type} data processing",
|
||||
)
|
||||
parsed = parse_text_content(raw, filename=name)
|
||||
if not parsed.text:
|
||||
if not parsed.text.strip():
|
||||
raise fail(400, f"source file is empty: {name}")
|
||||
batch_size += len(raw)
|
||||
if batch_size > MAX_SOURCE_BATCH_BYTES:
|
||||
@@ -934,7 +1106,11 @@ async def upload_source_files(
|
||||
content=raw,
|
||||
)
|
||||
staged.append(staged_object)
|
||||
record_count = len(parsed.records) or (1 if parsed.text else 0)
|
||||
record_count = (
|
||||
len(parsed.records)
|
||||
if process_type == "structured"
|
||||
else (1 if parsed.text else 0)
|
||||
)
|
||||
prepared.append(
|
||||
{
|
||||
"id": source_file_id,
|
||||
@@ -1362,15 +1538,28 @@ def _prepare_preview_items(
|
||||
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
|
||||
)
|
||||
is_unstructured = task.get("process_type") == "unstructured"
|
||||
if is_unstructured and (
|
||||
needs_unstructured_raw = is_unstructured and (
|
||||
chunk_method == "layout_hybrid"
|
||||
or preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||
):
|
||||
)
|
||||
has_structured_xlsx = not is_unstructured and any(
|
||||
str(source.get("file_format") or "").lower() == "xlsx"
|
||||
for source in sources
|
||||
)
|
||||
if needs_unstructured_raw or has_structured_xlsx:
|
||||
for index, source in enumerate(sources):
|
||||
if (
|
||||
chunk_method != "layout_hybrid"
|
||||
and str(source.get("file_format") or "").lower() != "pdf"
|
||||
):
|
||||
source_format = str(source.get("file_format") or "").lower()
|
||||
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
|
||||
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
|
||||
needs_pdf_noise = (
|
||||
is_unstructured
|
||||
and not needs_layout_raw
|
||||
and source_format == "pdf"
|
||||
and bool(
|
||||
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||
)
|
||||
)
|
||||
if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise):
|
||||
continue
|
||||
storage_object_id = str(source.get("storage_object_id") or "")
|
||||
actual_size = storage.file_size(
|
||||
@@ -1379,7 +1568,7 @@ def _prepare_preview_items(
|
||||
expected_source_file_id=str(source["id"]),
|
||||
)
|
||||
if actual_size is None:
|
||||
if chunk_method == "layout_hybrid":
|
||||
if needs_layout_raw:
|
||||
raise InvalidStateError(
|
||||
"版面结构混合切分无法读取原始文件,请重新上传后再处理"
|
||||
)
|
||||
@@ -1396,12 +1585,10 @@ def _prepare_preview_items(
|
||||
)
|
||||
)
|
||||
enriched = dict(source)
|
||||
if chunk_method == "layout_hybrid":
|
||||
if needs_structured_xlsx or needs_layout_raw:
|
||||
enriched["raw_content"] = raw
|
||||
sources[index] = enriched
|
||||
continue
|
||||
if str(source.get("file_format") or "").lower() != "pdf":
|
||||
continue
|
||||
pages = extract_pdf_page_texts(raw)
|
||||
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
||||
if extracted_text != str(source.get("content") or ""):
|
||||
@@ -1413,7 +1600,7 @@ def _prepare_preview_items(
|
||||
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
||||
sources[index] = enriched
|
||||
items = _build_preview_items(task, sources)
|
||||
if not items and source_file_ids is None:
|
||||
if not items and source_file_ids is None and is_unstructured:
|
||||
raise InvalidStateError("source files did not produce preview items")
|
||||
return items
|
||||
|
||||
@@ -1435,6 +1622,7 @@ def _run_preview(
|
||||
len(source_file_ids),
|
||||
)
|
||||
try:
|
||||
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
|
||||
if not store.mark_preview_running(task_id, preview_run_id):
|
||||
logger.info(
|
||||
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
|
||||
@@ -1461,7 +1649,7 @@ def _run_preview(
|
||||
storage,
|
||||
[source_file_id],
|
||||
)
|
||||
if not items:
|
||||
if not items and is_unstructured:
|
||||
raise InvalidStateError(
|
||||
f"source file did not produce preview items: {source_file_id}"
|
||||
)
|
||||
@@ -1657,8 +1845,13 @@ def update_preview_item(
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
task = store.get_task(task_id)
|
||||
existing = store.get_preview_item(task_id, preview_id)
|
||||
update = payload.model_dump(exclude_unset=True, mode="json")
|
||||
update["quality_score"] = _preview_quality(payload.edited_content, task.get("config") or {})
|
||||
quality = _preview_quality(payload.edited_content, task.get("config") or {})
|
||||
source_locator = (existing.get("quality_score") or {}).get("source_locator")
|
||||
if isinstance(source_locator, Mapping):
|
||||
quality["source_locator"] = deepcopy(dict(source_locator))
|
||||
update["quality_score"] = quality
|
||||
item = store.update_preview_item(
|
||||
task_id,
|
||||
preview_id,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
|
||||
self._issued_staged_objects[temporary_path] = staged
|
||||
return staged
|
||||
|
||||
def stage_copy(
|
||||
self,
|
||||
*,
|
||||
batch_id: str,
|
||||
source_reference: str,
|
||||
expected_source_task_id: str,
|
||||
expected_source_file_id: str,
|
||||
task_id: str,
|
||||
source_file_id: str,
|
||||
version: int,
|
||||
name: str,
|
||||
) -> StagedSourceObject:
|
||||
"""为不可变源对象创建独立目录项,不把大文件重新读入内存。"""
|
||||
|
||||
batch_id = _safe_component(batch_id, "batch id")
|
||||
task_id = _safe_component(task_id, "task id")
|
||||
source_file_id = _safe_component(source_file_id, "source file id")
|
||||
if isinstance(version, bool) or not isinstance(version, int) or version < 1:
|
||||
raise DataProcessStorageError("invalid source file version")
|
||||
basename = _safe_basename(name)
|
||||
source_relative = self._relative_from_reference(source_reference)
|
||||
if source_relative is None:
|
||||
raise DataProcessStorageError("original source object is not available")
|
||||
self._assert_expected_owner(
|
||||
source_relative,
|
||||
expected_task_id=expected_source_task_id,
|
||||
expected_source_file_id=expected_source_file_id,
|
||||
)
|
||||
descriptor, source_info = self._open_read_descriptor(source_relative)
|
||||
os.close(descriptor)
|
||||
|
||||
batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
|
||||
temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
|
||||
source_path = self._path_for_relative(source_relative)
|
||||
try:
|
||||
os.link(source_path, temporary_path, follow_symlinks=False)
|
||||
copy_info = temporary_path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(copy_info.st_mode)
|
||||
or source_info.st_dev != copy_info.st_dev
|
||||
or source_info.st_ino != copy_info.st_ino
|
||||
):
|
||||
raise DataProcessStorageError("source storage object changed while copying")
|
||||
except Exception:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
relative_path = PurePosixPath(
|
||||
task_id,
|
||||
source_file_id,
|
||||
f"v{version}",
|
||||
basename,
|
||||
)
|
||||
reference = (
|
||||
"local://data-process/"
|
||||
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
|
||||
)
|
||||
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
||||
self._issued_staged_objects[temporary_path] = staged
|
||||
return staged
|
||||
|
||||
def publish(self, objects: Iterable[StagedSourceObject]) -> None:
|
||||
staged = list(objects)
|
||||
published: list[StagedSourceObject] = []
|
||||
|
||||
@@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = {
|
||||
"preserve_lists": True,
|
||||
}
|
||||
_REGENERATION_MARKER_KEY = "_regeneration_prepared"
|
||||
_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id"
|
||||
_REPEAT_REQUEST_KEY = "_repeat_request_id"
|
||||
_INTERNAL_CONFIG_KEYS = {
|
||||
_REGENERATION_MARKER_KEY,
|
||||
_REPEAT_SOURCE_TASK_KEY,
|
||||
_REPEAT_REQUEST_KEY,
|
||||
}
|
||||
|
||||
|
||||
class DataProcessStoreError(RuntimeError):
|
||||
@@ -71,6 +78,13 @@ def new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:20]}"
|
||||
|
||||
|
||||
def repeat_task_id(source_task_id: str, request_id: str) -> str:
|
||||
"""按源任务和请求幂等键生成稳定的新任务 ID。"""
|
||||
|
||||
digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest()
|
||||
return f"dpt_{digest[:20]}"
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
@@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool:
|
||||
return _regeneration_marker(task) is not None
|
||||
|
||||
|
||||
def _business_config(config: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""过滤只供服务端维护的工作流标记。"""
|
||||
|
||||
return {
|
||||
key: value
|
||||
for key, value in (config or {}).items()
|
||||
if key not in _INTERNAL_CONFIG_KEYS
|
||||
}
|
||||
|
||||
|
||||
def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
"""从 API 任务快照中移除服务端内部重新生成标记。"""
|
||||
"""从 API 任务快照中移除服务端内部工作流标记。"""
|
||||
|
||||
if item is None:
|
||||
return None
|
||||
public = dict(item)
|
||||
config = public.get("config")
|
||||
if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config:
|
||||
public["config"] = {
|
||||
key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY
|
||||
}
|
||||
if isinstance(config, dict):
|
||||
public["config"] = _business_config(config)
|
||||
return public
|
||||
|
||||
|
||||
@@ -353,13 +375,7 @@ class DataProcessStore:
|
||||
payload.get("description") or "",
|
||||
payload["process_type"],
|
||||
payload.get("source_dataset_id"),
|
||||
json_dumps(
|
||||
{
|
||||
key: value
|
||||
for key, value in (payload.get("config") or {}).items()
|
||||
if key != _REGENERATION_MARKER_KEY
|
||||
}
|
||||
),
|
||||
json_dumps(_business_config(payload.get("config"))),
|
||||
payload.get("tenant_id"),
|
||||
payload.get("project_id"),
|
||||
payload.get("owner_id"),
|
||||
@@ -373,6 +389,268 @@ class DataProcessStore:
|
||||
raise ConflictError("data process task name already exists") from exc
|
||||
return _public_task(_decode_row(row)) or {}
|
||||
|
||||
@staticmethod
|
||||
def _repeat_response(
|
||||
conn: psycopg.Connection[dict[str, Any]],
|
||||
row: dict[str, Any],
|
||||
*,
|
||||
source_task_id: str,
|
||||
created: bool,
|
||||
) -> dict[str, Any]:
|
||||
task_id = str(row["id"])
|
||||
counts = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count,
|
||||
(SELECT COUNT(*) FROM data_process_preview_items
|
||||
WHERE task_id=%s) AS preview_count
|
||||
""",
|
||||
(task_id, task_id),
|
||||
).fetchone() or {}
|
||||
task = _public_task(_decode_row(row)) or {}
|
||||
task["source_file_count"] = int(counts.get("source_file_count") or 0)
|
||||
task["preview_count"] = int(counts.get("preview_count") or 0)
|
||||
return {
|
||||
"task": task,
|
||||
"source_task_id": source_task_id,
|
||||
"created": created,
|
||||
"copied_source_file_count": task["source_file_count"],
|
||||
"copied_preview_count": task["preview_count"],
|
||||
}
|
||||
|
||||
def find_repeated_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""查找同一幂等请求已创建的新任务。"""
|
||||
|
||||
task_id = repeat_task_id(source_task_id, request_id)
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM data_process_tasks WHERE id=%s",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
decoded = _decode_row(row) or {}
|
||||
config = decoded.get("config") or {}
|
||||
if (
|
||||
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
|
||||
or config.get(_REPEAT_REQUEST_KEY) != request_id
|
||||
):
|
||||
raise ConflictError("再次生成请求与现有任务冲突")
|
||||
if decoded.get("deleted_at"):
|
||||
raise ConflictError("此次再次生成创建的任务已被删除,请重新发起")
|
||||
return self._repeat_response(
|
||||
conn,
|
||||
row,
|
||||
source_task_id=source_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
def repeat_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
*,
|
||||
expected_updated_at: str,
|
||||
request_id: str,
|
||||
file_copies: dict[str, dict[str, str]],
|
||||
) -> dict[str, Any]:
|
||||
"""复制已确认任务的配置、源文件和预览,结果与发布数据保持独立。"""
|
||||
|
||||
task_id = repeat_task_id(source_task_id, request_id)
|
||||
now = utcnow()
|
||||
try:
|
||||
with self.connect() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
decoded = _decode_row(existing) or {}
|
||||
config = decoded.get("config") or {}
|
||||
if (
|
||||
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
|
||||
or config.get(_REPEAT_REQUEST_KEY) != request_id
|
||||
):
|
||||
raise ConflictError("再次生成请求与现有任务冲突")
|
||||
if decoded.get("deleted_at"):
|
||||
raise ConflictError("此次再次生成创建的任务已被删除,请重新发起")
|
||||
return self._repeat_response(
|
||||
conn,
|
||||
existing,
|
||||
source_task_id=source_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
source_task = self._task_in_connection(
|
||||
conn,
|
||||
source_task_id,
|
||||
for_update=True,
|
||||
)
|
||||
if (
|
||||
source_task.get("status") != "completed"
|
||||
or source_task.get("results_confirmed") is False
|
||||
):
|
||||
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
|
||||
if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
|
||||
raise ConflictError("源任务仍在处理切分,暂时不能再次生成")
|
||||
if expected_updated_at != _serialize_value(source_task.get("updated_at")):
|
||||
raise ConflictError("源任务已被其他操作修改,请刷新后重试")
|
||||
|
||||
source_files = conn.execute(
|
||||
"""
|
||||
SELECT * FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL
|
||||
ORDER BY created_at, id
|
||||
""",
|
||||
(source_task_id,),
|
||||
).fetchall()
|
||||
source_file_ids = {str(row["id"]) for row in source_files}
|
||||
if source_file_ids != set(file_copies):
|
||||
raise ConflictError("源文件快照已变化,请刷新后重试")
|
||||
previews = conn.execute(
|
||||
"""
|
||||
SELECT * FROM data_process_preview_items
|
||||
WHERE task_id=%s
|
||||
ORDER BY source_file_id NULLS LAST, source_start NULLS LAST,
|
||||
created_at, id
|
||||
""",
|
||||
(source_task_id,),
|
||||
).fetchall()
|
||||
if not previews:
|
||||
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
|
||||
|
||||
suffix = f"(再次生成-{task_id[-6:]})"
|
||||
base_name = str(source_task.get("name") or "数据处理任务")
|
||||
repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}"
|
||||
repeated_config = _business_config(source_task.get("config") or {})
|
||||
repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id
|
||||
repeated_config[_REPEAT_REQUEST_KEY] = request_id
|
||||
input_count = sum(int(row.get("record_count") or 0) for row in source_files)
|
||||
task_row = conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_tasks
|
||||
(id, name, description, status, process_type, source_dataset_id,
|
||||
output_dataset_id, config, progress, input_count, output_count,
|
||||
filtered_count, duplicate_count, error_count, failure_reason,
|
||||
generation_run_id, results_confirmed, workflow_step,
|
||||
preview_status, preview_progress, preview_run_id,
|
||||
preview_failure_reason, preview_total_files,
|
||||
preview_completed_files, tenant_id, project_id, owner_id,
|
||||
approval_status, created_by, updated_by, created_at, updated_at)
|
||||
VALUES
|
||||
(%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0,
|
||||
0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100,
|
||||
NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
task_id,
|
||||
repeated_name,
|
||||
source_task.get("description") or "",
|
||||
source_task["process_type"],
|
||||
source_task.get("source_dataset_id"),
|
||||
json_dumps(repeated_config),
|
||||
input_count,
|
||||
len(source_files),
|
||||
len(source_files),
|
||||
source_task.get("tenant_id"),
|
||||
source_task.get("project_id"),
|
||||
source_task.get("owner_id"),
|
||||
source_task.get("approval_status") or "not_required",
|
||||
source_task.get("created_by"),
|
||||
source_task.get("created_by"),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
file_id_map: dict[str, str] = {}
|
||||
for source in source_files:
|
||||
old_file_id = str(source["id"])
|
||||
copy = file_copies[old_file_id]
|
||||
new_file_id = str(copy["id"])
|
||||
storage_object_id, metadata = _source_storage_descriptor(
|
||||
{
|
||||
"storage_object_id": copy["storage_object_id"],
|
||||
"metadata": _json_value(source.get("metadata"), {}),
|
||||
},
|
||||
task_id,
|
||||
new_file_id,
|
||||
)
|
||||
file_id_map[old_file_id] = new_file_id
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_source_files
|
||||
(id, task_id, storage_object_id, name, size_bytes, record_count,
|
||||
file_format, checksum_sha256, version_no, content,
|
||||
content_preview, metadata, tenant_id, project_id, created_by,
|
||||
created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
new_file_id,
|
||||
task_id,
|
||||
storage_object_id,
|
||||
source["name"],
|
||||
source.get("size_bytes") or 0,
|
||||
source.get("record_count") or 0,
|
||||
source.get("file_format"),
|
||||
source["checksum_sha256"],
|
||||
source.get("content") or "",
|
||||
source.get("content_preview"),
|
||||
json_dumps(metadata),
|
||||
source_task.get("tenant_id"),
|
||||
source_task.get("project_id"),
|
||||
source.get("created_by") or source_task.get("created_by"),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
|
||||
for preview in previews:
|
||||
old_source_file_id = preview.get("source_file_id")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_preview_items
|
||||
(id, task_id, source_file_id, original_content, edited_content,
|
||||
source_start, source_end, source_start_line, source_end_line,
|
||||
token_count, status, quality_score, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s)
|
||||
""",
|
||||
(
|
||||
new_id("dpp"),
|
||||
task_id,
|
||||
file_id_map.get(str(old_source_file_id))
|
||||
if old_source_file_id
|
||||
else None,
|
||||
preview.get("original_content") or "",
|
||||
preview.get("edited_content") or "",
|
||||
preview.get("source_start"),
|
||||
preview.get("source_end"),
|
||||
preview.get("source_start_line"),
|
||||
preview.get("source_end_line"),
|
||||
max(0, int(preview.get("token_count") or 0)),
|
||||
preview.get("status") or "original",
|
||||
json_dumps(_json_value(preview.get("quality_score"), {})),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
return self._repeat_response(
|
||||
conn,
|
||||
task_row or {},
|
||||
source_task_id=source_task_id,
|
||||
created=True,
|
||||
)
|
||||
except psycopg.errors.UniqueViolation as exc:
|
||||
raise ConflictError("再次生成任务名称或请求发生冲突,请重试") from exc
|
||||
|
||||
def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]:
|
||||
lock = " FOR UPDATE" if for_update else ""
|
||||
with self.connect() as conn:
|
||||
@@ -654,14 +932,11 @@ class DataProcessStore:
|
||||
"process type and source dataset cannot change during regeneration"
|
||||
)
|
||||
if payload.get("config") is not None:
|
||||
next_config = {
|
||||
key: value
|
||||
for key, value in payload["config"].items()
|
||||
if key != _REGENERATION_MARKER_KEY
|
||||
}
|
||||
current_marker = _regeneration_marker(task)
|
||||
if current_marker:
|
||||
next_config[_REGENERATION_MARKER_KEY] = current_marker
|
||||
next_config = _business_config(payload["config"])
|
||||
current_config = dict(task.get("config") or {})
|
||||
for key in _INTERNAL_CONFIG_KEYS:
|
||||
if key in current_config:
|
||||
next_config[key] = current_config[key]
|
||||
values["config"] = json_dumps(next_config)
|
||||
invalidates_results = (
|
||||
("config" in payload and payload.get("config") != task.get("config"))
|
||||
@@ -753,8 +1028,10 @@ class DataProcessStore:
|
||||
raise InvalidStateError("process_type cannot be changed during regeneration")
|
||||
|
||||
current_config = dict(task.get("config") or {})
|
||||
next_config = dict(payload.get("config") or {})
|
||||
next_config.pop(_REGENERATION_MARKER_KEY, None)
|
||||
next_config = _business_config(payload.get("config"))
|
||||
for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY):
|
||||
if key in current_config:
|
||||
next_config[key] = current_config[key]
|
||||
preview_invalidated = _preview_config_changed(
|
||||
process_type,
|
||||
current_config,
|
||||
|
||||
@@ -220,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessRepeatRequest(BaseModel):
|
||||
"""按已确认任务的完整快照创建一批独立的新生成结果。"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
expected_updated_at: str = Field(min_length=1)
|
||||
request_id: str = Field(
|
||||
min_length=8,
|
||||
max_length=80,
|
||||
pattern=r"^[A-Za-z0-9_-]+$",
|
||||
)
|
||||
|
||||
|
||||
class PreviewBuildRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
|
||||
normalize_text,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
preprocess_structured_records_with_lineage,
|
||||
record_fingerprint,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
stable_split,
|
||||
stable_split_assignments,
|
||||
structured_json_dumps,
|
||||
)
|
||||
|
||||
|
||||
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
|
||||
assert parsed_txt.text == "普通文本"
|
||||
|
||||
|
||||
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
|
||||
root_json = parse_text_content('{"id":1}', filename="root.json")
|
||||
assert root_json.record_locators == (
|
||||
{
|
||||
"kind": "json",
|
||||
"record_index": 1,
|
||||
"json_pointer": "",
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
},
|
||||
)
|
||||
|
||||
wrapped_json = parse_text_content(
|
||||
'{"records":[{"id":1},{"id":1}]}',
|
||||
filename="wrapped.json",
|
||||
)
|
||||
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
|
||||
"/records/0",
|
||||
"/records/1",
|
||||
]
|
||||
|
||||
parsed_jsonl = parse_text_content(
|
||||
'{"id":1}\r\n\r\n{"id":1}',
|
||||
filename="records.jsonl",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == [(1, 1, 1), (2, 3, 3)]
|
||||
assert [
|
||||
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == ['{"id":1}', '{"id":1}']
|
||||
|
||||
parsed_csv = parse_text_content(
|
||||
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
|
||||
filename="records.csv",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_csv.record_locators
|
||||
] == [(1, 2, 3), (2, 5, 5)]
|
||||
assert [
|
||||
parsed_csv.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_csv.record_locators
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
|
||||
processed = preprocess_structured_records_with_lineage(
|
||||
[
|
||||
{"id": "A", "value": "first", "empty": ""},
|
||||
{"id": "", "value": "invalid", "empty": ""},
|
||||
{"id": "A", "value": "duplicate identity", "empty": ""},
|
||||
{"id": "B", "value": "second", "empty": ""},
|
||||
],
|
||||
["clean_invalid", "deduplicate"],
|
||||
)
|
||||
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
|
||||
assert [entry.record for entry in processed] == [
|
||||
{"id": "A", "value": "first"},
|
||||
{"id": "", "value": "invalid"},
|
||||
{"id": "A", "value": "duplicate identity"},
|
||||
{"id": "B", "value": "second"},
|
||||
]
|
||||
|
||||
|
||||
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
|
||||
assert parsed_pdf.format == "pdf"
|
||||
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
|
||||
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
|
||||
)
|
||||
assert parsed_xlsx.record_locators == (
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
)
|
||||
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
|
||||
|
||||
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
|
||||
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
assert parsed_pptx.records == ()
|
||||
|
||||
|
||||
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
||||
workbook = Workbook()
|
||||
first = workbook.active
|
||||
first.title = "甲表"
|
||||
first.append(["说明"])
|
||||
first.append([])
|
||||
first.append(["id", "value"])
|
||||
first.append([1, "same"])
|
||||
first.append([1, "same"])
|
||||
second = workbook.create_sheet("乙表")
|
||||
second.append(["id", "value"])
|
||||
second.append([1, "same"])
|
||||
output = io.BytesIO()
|
||||
workbook.save(output)
|
||||
workbook.close()
|
||||
|
||||
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
|
||||
assert parsed.records == (
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
)
|
||||
assert [
|
||||
(
|
||||
locator["record_index"],
|
||||
locator["sheet_index"],
|
||||
locator["sheet_name"],
|
||||
locator["row_number"],
|
||||
locator["sheet_record_index"],
|
||||
)
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(1, 0, "甲表", 4, 0),
|
||||
(2, 0, "甲表", 5, 1),
|
||||
(3, 1, "乙表", 2, 0),
|
||||
]
|
||||
|
||||
|
||||
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
|
||||
pages = _pdf_page_texts(
|
||||
"""
|
||||
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
|
||||
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
|
||||
"json",
|
||||
)
|
||||
assert result == [{"text": "内容"}]
|
||||
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
|
||||
|
||||
assert extract_structured_records(
|
||||
'{"items":[{"text":" 内容 "}],"total":1}',
|
||||
"json",
|
||||
) == [{"text": " 内容 "}]
|
||||
|
||||
|
||||
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
|
||||
source = '{"code":"001","text":" 内容 ","quote":"""}'
|
||||
parsed = parse_text_content(source, filename="records.json")
|
||||
assert parsed.text == source
|
||||
assert parsed.records == (
|
||||
{"code": "001", "text": " 内容 ", "quote": """},
|
||||
)
|
||||
|
||||
invalid_values = (
|
||||
'{"id":1,"id":2}',
|
||||
'{"nested":{"id":1,"id":2}}',
|
||||
'{"value":NaN}',
|
||||
'{"value":Infinity}',
|
||||
'{"value":-Infinity}',
|
||||
'{"value":"bad\x00control"}',
|
||||
)
|
||||
for invalid in invalid_values:
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content(invalid, filename="invalid.json")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content("{\"id\":1}", filename="invalid.json")
|
||||
with pytest.raises(ValueError, match="nesting exceeds"):
|
||||
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
|
||||
|
||||
|
||||
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
|
||||
source = (
|
||||
' {"code":"001","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}\r\n\r\n'
|
||||
'{"id":2}\r\n'
|
||||
)
|
||||
parsed = parse_text_content(source, filename="records.jsonl")
|
||||
assert parsed.text == source
|
||||
assert parsed.records[0] == {
|
||||
"code": "001",
|
||||
"text": " 内容 ",
|
||||
"value": Decimal("0.123456789012345678901234567890"),
|
||||
}
|
||||
assert [
|
||||
source[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(
|
||||
'{"code":"001","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}'
|
||||
),
|
||||
'{"id":2}',
|
||||
]
|
||||
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
|
||||
|
||||
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
|
||||
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
|
||||
parse_text_content(invalid, filename="invalid.jsonl")
|
||||
|
||||
|
||||
def test_json_record_contract_avoids_business_field_collisions() -> None:
|
||||
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
|
||||
{"id": 1},
|
||||
{"id": 2},
|
||||
]
|
||||
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
|
||||
{"id": 1, "data": [{"id": 2}]}
|
||||
]
|
||||
assert extract_structured_records(
|
||||
'{"records":[{"id":1}],"data":[{"id":2}]}',
|
||||
"json",
|
||||
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
|
||||
assert extract_structured_records(
|
||||
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
|
||||
"json",
|
||||
) == [{"id": 1}]
|
||||
assert extract_structured_records(
|
||||
'{"payload":{"data":[{"id":2}],"total":1}}',
|
||||
"json",
|
||||
) == [{"id": 2}]
|
||||
assert extract_structured_records('{"records":[],"total":0}', "json") == []
|
||||
# 包装数组中的非对象不是记录集合,整体按一条业务对象保留。
|
||||
assert extract_structured_records('{"data":[1,2]}', "json") == [
|
||||
{"data": [1, 2]}
|
||||
]
|
||||
|
||||
|
||||
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
|
||||
pretty = (
|
||||
'{\n "records": [\n {"id": 1},\n'
|
||||
' {\n "id": 2\n }\n ],\n "total": 2\n}'
|
||||
)
|
||||
parsed = parse_text_content(pretty, filename="pretty.json")
|
||||
assert [
|
||||
pretty[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id": 1}', '{\n "id": 2\n }']
|
||||
assert [
|
||||
(locator["start_line"], locator["end_line"])
|
||||
for locator in parsed.record_locators
|
||||
] == [(3, 3), (4, 6)]
|
||||
|
||||
minified = '[{"id":1},{"id":2}]'
|
||||
parsed = parse_text_content(minified, filename="minified.json")
|
||||
assert [
|
||||
minified[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
|
||||
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
|
||||
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
|
||||
parsed = parse_text_content(source, filename="precise.json")
|
||||
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
|
||||
assert parsed.records[1]["value"] == Decimal("1e400")
|
||||
assert structured_json_dumps(parsed.records[0]) == (
|
||||
'{"value":0.123456789012345678901234567890}'
|
||||
)
|
||||
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
|
||||
assert isinstance(parsed.records[0]["value"], Decimal)
|
||||
|
||||
|
||||
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
|
||||
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
assert preprocess_structured_records(clean_source, []) == clean_source
|
||||
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
|
||||
{"id": "1", "name": "有效"},
|
||||
{"id": "", "name": "缺少关键字段"},
|
||||
{"id": "2", "name": "有效"},
|
||||
]
|
||||
|
||||
hierarchy = [
|
||||
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
|
||||
{"id": "", "parent_id": "", "name": "", "empty": ""},
|
||||
]
|
||||
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
|
||||
{"id": "1", "parent_id": None, "name": "根节点"},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点"},
|
||||
]
|
||||
|
||||
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
|
||||
assert "profile" in preprocess_structured_records(nested, [])[0]
|
||||
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
|
||||
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
duplicates = [
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "C-1", "value": "updated"},
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "", "value": "blank-one"},
|
||||
{"customer_id": "", "value": "blank-two"},
|
||||
]
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 4
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 5
|
||||
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
|
||||
assert [record["value"] for record in deduplicated] == [
|
||||
"first",
|
||||
"updated",
|
||||
"blank-one",
|
||||
"blank-two",
|
||||
]
|
||||
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
|
||||
masked, counts = desensitize_structured_record(
|
||||
{
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "张三",
|
||||
"contact_name": "李四",
|
||||
"姓名": "王五",
|
||||
"profile.name": "赵六",
|
||||
}
|
||||
)
|
||||
|
||||
assert masked == {
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "[NAME]",
|
||||
"contact_name": "[NAME]",
|
||||
"姓名": "[NAME]",
|
||||
"profile.name": "[NAME]",
|
||||
}
|
||||
assert counts == {
|
||||
"email": 0,
|
||||
"phone": 0,
|
||||
"id_card": 0,
|
||||
"name": 4,
|
||||
"total": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
|
||||
valid = {
|
||||
"instruction": "如何修改收货地址?",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
|
||||
LocalDataProcessStorage,
|
||||
get_data_process_storage,
|
||||
)
|
||||
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
|
||||
from app.modules.data_process.store import (
|
||||
InvalidStateError,
|
||||
NotFoundError,
|
||||
get_data_process_store,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
class FakeDataProcessStore:
|
||||
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
|
||||
self.datasets: dict[str, dict[str, Any]] = {}
|
||||
self.models: dict[str, dict[str, Any]] = {}
|
||||
self.regeneration_prepared: set[str] = set()
|
||||
self.repeat_requests: dict[tuple[str, str], str] = {}
|
||||
self.sequence = 0
|
||||
|
||||
def _id(self, prefix: str) -> str:
|
||||
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
|
||||
"published_outputs_preserved": published_outputs_preserved,
|
||||
}
|
||||
|
||||
def _repeat_response(
|
||||
self,
|
||||
source_task_id: str,
|
||||
repeated_task_id: str,
|
||||
*,
|
||||
created: bool,
|
||||
) -> dict[str, Any]:
|
||||
task = self.get_task(repeated_task_id)
|
||||
task["source_file_count"] = len(self.sources[repeated_task_id])
|
||||
task["preview_count"] = len(self.previews[repeated_task_id])
|
||||
return {
|
||||
"task": task,
|
||||
"source_task_id": source_task_id,
|
||||
"created": created,
|
||||
"copied_source_file_count": len(self.sources[repeated_task_id]),
|
||||
"copied_preview_count": len(self.previews[repeated_task_id]),
|
||||
}
|
||||
|
||||
def find_repeated_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
|
||||
if repeated_task_id is None:
|
||||
return None
|
||||
return self._repeat_response(
|
||||
source_task_id,
|
||||
repeated_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
def repeat_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
*,
|
||||
expected_updated_at: str,
|
||||
request_id: str,
|
||||
file_copies: dict[str, dict[str, str]],
|
||||
) -> dict[str, Any]:
|
||||
existing = self.find_repeated_task(source_task_id, request_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
source_task = self.get_task(source_task_id)
|
||||
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
|
||||
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
|
||||
if source_task.get("updated_at") != expected_updated_at:
|
||||
raise InvalidStateError("源任务已被其他操作修改,请刷新后重试")
|
||||
source_files = self.sources[source_task_id]
|
||||
if set(file_copies) != {str(item["id"]) for item in source_files}:
|
||||
raise InvalidStateError("源文件快照已变化,请刷新后重试")
|
||||
if not self.previews[source_task_id]:
|
||||
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
|
||||
|
||||
repeated_task_id = repeat_task_id(source_task_id, request_id)
|
||||
suffix = f"(再次生成-{repeated_task_id[-6:]})"
|
||||
task = {
|
||||
**deepcopy(source_task),
|
||||
"id": repeated_task_id,
|
||||
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
|
||||
"status": "pending",
|
||||
"progress": 20,
|
||||
"output_dataset_id": None,
|
||||
"output_datasets": [],
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"generation_run_id": None,
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_run_id": None,
|
||||
"preview_failure_reason": None,
|
||||
"preview_total_files": len(source_files),
|
||||
"preview_completed_files": len(source_files),
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
}
|
||||
self.tasks[repeated_task_id] = task
|
||||
self.sources[repeated_task_id] = []
|
||||
file_id_map: dict[str, str] = {}
|
||||
for source in source_files:
|
||||
old_file_id = str(source["id"])
|
||||
copy = file_copies[old_file_id]
|
||||
file_id_map[old_file_id] = copy["id"]
|
||||
self.sources[repeated_task_id].append(
|
||||
{
|
||||
**deepcopy(source),
|
||||
"id": copy["id"],
|
||||
"task_id": repeated_task_id,
|
||||
"storage_object_id": copy["storage_object_id"],
|
||||
}
|
||||
)
|
||||
self.previews[repeated_task_id] = [
|
||||
{
|
||||
**deepcopy(item),
|
||||
"id": self._id("dpp"),
|
||||
"task_id": repeated_task_id,
|
||||
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
|
||||
if item.get("source_file_id")
|
||||
else None,
|
||||
}
|
||||
for item in self.previews[source_task_id]
|
||||
]
|
||||
self.results[repeated_task_id] = []
|
||||
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
|
||||
return self._repeat_response(
|
||||
source_task_id,
|
||||
repeated_task_id,
|
||||
created=True,
|
||||
)
|
||||
|
||||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||||
self.get_task(task_id)
|
||||
del self.tasks[task_id]
|
||||
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||||
assert listed_preview.json()["data"]["total"] == 2
|
||||
preview_item = listed_preview.json()["data"]["items"][0]
|
||||
source_locator = preview_item["quality_score"]["source_locator"]
|
||||
assert source_locator == {
|
||||
"kind": "jsonl",
|
||||
"record_index": 1,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
"source_start": 0,
|
||||
"source_end": len(preview_item["original_content"]),
|
||||
}
|
||||
updated_preview = client.put(
|
||||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||||
json={
|
||||
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||||
},
|
||||
)
|
||||
assert "quality_score" in updated_preview.json()["data"]
|
||||
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
|
||||
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
|
||||
|
||||
|
||||
def test_completed_task_can_repeat_into_an_independent_background_task(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
client, store, storage = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "原始生成任务",
|
||||
"process_type": "structured",
|
||||
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
source_id = uploaded.json()["data"]["files"][0]["id"]
|
||||
built = client.post(
|
||||
f"/modelTF/data-process/{task_id}/preview/build",
|
||||
json={"replace_existing": True},
|
||||
)
|
||||
assert built.status_code == 200
|
||||
store.tasks[task_id].update(
|
||||
status="completed",
|
||||
progress=100,
|
||||
results_confirmed=True,
|
||||
workflow_step="results",
|
||||
output_count=1,
|
||||
output_dataset_id="dataset-original",
|
||||
updated_at="2026-07-28T12:00:00Z",
|
||||
)
|
||||
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
|
||||
store.datasets["dataset-original"] = {
|
||||
"id": "dataset-original",
|
||||
"name": "原数据集",
|
||||
"type": "train",
|
||||
"source_task_id": task_id,
|
||||
"deleted_at": None,
|
||||
}
|
||||
original_task = deepcopy(store.tasks[task_id])
|
||||
original_sources = deepcopy(store.sources[task_id])
|
||||
original_previews = deepcopy(store.previews[task_id])
|
||||
original_results = deepcopy(store.results[task_id])
|
||||
original_datasets = deepcopy(store.datasets)
|
||||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
|
||||
|
||||
payload = {
|
||||
"expected_updated_at": "2026-07-28T12:00:00Z",
|
||||
"request_id": "repeat-request-0001",
|
||||
}
|
||||
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||||
|
||||
assert response.status_code == 202
|
||||
repeated = response.json()["data"]
|
||||
repeated_task_id = repeated["task"]["id"]
|
||||
assert repeated["created"] is True
|
||||
assert repeated_task_id != task_id
|
||||
assert repeated["task"]["status"] == "running"
|
||||
assert repeated["task"]["workflow_step"] == "generate"
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == len(original_previews)
|
||||
assert store.tasks[task_id] == original_task
|
||||
assert store.sources[task_id] == original_sources
|
||||
assert store.previews[task_id] == original_previews
|
||||
assert store.results[task_id] == original_results
|
||||
assert store.datasets == original_datasets
|
||||
|
||||
repeated_source = store.sources[repeated_task_id][0]
|
||||
repeated_preview = store.previews[repeated_task_id][0]
|
||||
assert repeated_source["id"] != source_id
|
||||
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
|
||||
assert repeated_preview["id"] != original_previews[0]["id"]
|
||||
assert repeated_preview["source_file_id"] == repeated_source["id"]
|
||||
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
|
||||
|
||||
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||||
assert replay.status_code == 202
|
||||
assert replay.json()["data"]["created"] is False
|
||||
assert replay.json()["data"]["task"]["id"] == repeated_task_id
|
||||
assert len(store.tasks) == 2
|
||||
assert len(store.sources[repeated_task_id]) == 1
|
||||
|
||||
|
||||
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "源任务", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id].update(
|
||||
status="completed",
|
||||
results_confirmed=True,
|
||||
updated_at="2026-07-28T12:00:00Z",
|
||||
)
|
||||
before = deepcopy(store.tasks)
|
||||
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/repeat",
|
||||
json={
|
||||
"expected_updated_at": "2026-07-28T11:59:59Z",
|
||||
"request_id": "repeat-request-stale",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 409
|
||||
assert store.tasks == before
|
||||
|
||||
|
||||
def test_published_split_datasets_remain_in_detail_after_regeneration(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||||
|
||||
|
||||
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "先去重再脱敏",
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={
|
||||
"files": (
|
||||
"names.jsonl",
|
||||
(
|
||||
'{"name":"张三","role":"开发"}\n'
|
||||
'{"name":"李四","role":"开发"}\n'
|
||||
),
|
||||
"application/jsonl",
|
||||
)
|
||||
},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
|
||||
assert preview.status_code == 200
|
||||
items = preview.json()["data"]["items"]
|
||||
assert len(items) == 2
|
||||
assert len({item["original_content"] for item in items}) == 2
|
||||
assert {item["edited_content"] for item in items} == {
|
||||
'{"name":"[NAME]","role":"开发"}'
|
||||
}
|
||||
|
||||
|
||||
def test_structured_deduplication_removes_identical_rows_across_sources(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "跨源原文去重",
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
(
|
||||
"files",
|
||||
(
|
||||
"first.jsonl",
|
||||
'{"name":"张三","role":"开发"}\n',
|
||||
"application/jsonl",
|
||||
),
|
||||
),
|
||||
(
|
||||
"files",
|
||||
(
|
||||
"second.jsonl",
|
||||
'\n{"name":"张三","role":"开发"}\n',
|
||||
"application/jsonl",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
first_source, second_source = uploaded.json()["data"]["files"]
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
|
||||
assert preview.status_code == 200
|
||||
data = preview.json()["data"]
|
||||
assert data["total"] == 1
|
||||
assert data["file_counts"] == {
|
||||
first_source["id"]: 1,
|
||||
second_source["id"]: 0,
|
||||
}
|
||||
|
||||
|
||||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||||
store = FakeDataProcessStore()
|
||||
task = store.create_task(
|
||||
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
|
||||
)
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["total"] == 2
|
||||
preview_items = preview.json()["data"]["items"]
|
||||
assert len(preview_items) == 2
|
||||
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "Sheet",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "Sheet",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_docx_preview_preserves_document_block_order_and_source_offsets(
|
||||
@@ -2757,6 +3105,218 @@ def _preview_task(
|
||||
)
|
||||
|
||||
|
||||
def _structured_preview_task(
|
||||
content: str,
|
||||
*,
|
||||
file_format: str,
|
||||
options: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
return data_process_endpoint._build_preview_items(
|
||||
{
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": options or []},
|
||||
},
|
||||
[
|
||||
{
|
||||
"id": "structured-source",
|
||||
"name": f"records.{file_format}",
|
||||
"file_format": file_format,
|
||||
"content": content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
|
||||
json_source = '{"records":[{"id":1},{"id":2}]}'
|
||||
json_items = _structured_preview_task(
|
||||
json_source,
|
||||
file_format="json",
|
||||
)
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["json_pointer"]
|
||||
for item in json_items
|
||||
] == ["/records/0", "/records/1"]
|
||||
assert [
|
||||
json_source[item["source_start"] : item["source_end"]]
|
||||
for item in json_items
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
assert [item["source_start_line"] for item in json_items] == [1, 1]
|
||||
|
||||
jsonl_source = '{"id":1}\n\n{"id":2}'
|
||||
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in jsonl_items
|
||||
] == [1, 2]
|
||||
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
|
||||
assert [
|
||||
jsonl_source[item["source_start"] : item["source_end"]]
|
||||
for item in jsonl_items
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
|
||||
csv_items = _structured_preview_task(csv_source, file_format="csv")
|
||||
assert [
|
||||
(item["source_start_line"], item["source_end_line"])
|
||||
for item in csv_items
|
||||
] == [(2, 3), (5, 5)]
|
||||
assert [
|
||||
csv_source[item["source_start"] : item["source_end"]]
|
||||
for item in csv_items
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "空 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("empty-array.json", "[]", "application/json")),
|
||||
(
|
||||
"files",
|
||||
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
assert uploaded.status_code == 200
|
||||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["items"] == []
|
||||
assert preview.json()["data"]["total"] == 0
|
||||
assert set(preview.json()["data"]["file_counts"].values()) == {0}
|
||||
|
||||
|
||||
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
invalid_sources = (
|
||||
("duplicate.json", '{"id":1,"id":2}'),
|
||||
("duplicate.jsonl", '{"id":1,"id":2}\n'),
|
||||
("nan.json", '{"value":NaN}'),
|
||||
("infinity.json", '{"value":Infinity}'),
|
||||
("control.json", '{"value":"bad\x00control"}'),
|
||||
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
|
||||
)
|
||||
|
||||
for filename, content in invalid_sources:
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": (filename, content, "application/json")},
|
||||
)
|
||||
assert response.status_code == 400, (filename, response.text)
|
||||
|
||||
|
||||
def test_structured_json_preview_preserves_precision_and_business_data_field(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
precise = '{"value":0.123456789012345678901234567890}'
|
||||
business = '{"id":7,"data":[{"id":8}]}'
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("precise.json", precise, "application/json")),
|
||||
("files", ("business.json", business, "application/json")),
|
||||
],
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
items = preview.json()["data"]["items"]
|
||||
assert [item["original_content"] for item in items] == [precise, business]
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["json_pointer"] for item in items
|
||||
] == ["", ""]
|
||||
assert [item["source_start"] for item in items] == [0, 0]
|
||||
|
||||
|
||||
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
|
||||
source_records = [
|
||||
{"id": "A", "amount": 10, "empty": ""},
|
||||
{"id": "A", "amount": 10, "empty": ""},
|
||||
{"id": "", "amount": 11, "empty": ""},
|
||||
{"id": "B", "amount": 11, "empty": ""},
|
||||
{"id": "C", "amount": 12, "empty": ""},
|
||||
{"id": "D", "amount": 12, "empty": ""},
|
||||
{"id": "E", "amount": 13, "empty": ""},
|
||||
{"id": "F", "amount": 13, "empty": ""},
|
||||
{"id": "G", "amount": 14, "empty": ""},
|
||||
{"id": "H", "amount": 1000, "empty": ""},
|
||||
]
|
||||
source = "\n".join(
|
||||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
for record in source_records
|
||||
)
|
||||
items = _structured_preview_task(
|
||||
source,
|
||||
file_format="jsonl",
|
||||
options=["clean_invalid", "deduplicate", "filter_anomaly"],
|
||||
)
|
||||
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in items
|
||||
] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [json.loads(item["original_content"])["id"] for item in items] == [
|
||||
"A",
|
||||
"",
|
||||
"B",
|
||||
"C",
|
||||
"D",
|
||||
"E",
|
||||
"F",
|
||||
"G",
|
||||
]
|
||||
|
||||
|
||||
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
|
||||
source_records = [
|
||||
{"customer_id": "C-1", "status": "old"},
|
||||
{"customer_id": "C-1", "status": "new"},
|
||||
{"status": "old", "customer_id": "C-1"},
|
||||
]
|
||||
source = "\n".join(
|
||||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
for record in source_records
|
||||
)
|
||||
|
||||
items = _structured_preview_task(
|
||||
source,
|
||||
file_format="jsonl",
|
||||
options=["clean_invalid", "deduplicate"],
|
||||
)
|
||||
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in items
|
||||
] == [1, 2]
|
||||
assert [item["source_start_line"] for item in items] == [1, 2]
|
||||
assert [json.loads(item["original_content"])["status"] for item in items] == [
|
||||
"old",
|
||||
"new",
|
||||
]
|
||||
|
||||
|
||||
def test_fixed_preview_preserves_source_offsets() -> None:
|
||||
content = (
|
||||
"# 第一章\n"
|
||||
|
||||
@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
|
||||
_assert_staging_empty(storage)
|
||||
|
||||
|
||||
def test_stage_copy_creates_an_independently_deletable_source_object(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
storage = LocalDataProcessStorage(tmp_path / "storage")
|
||||
original = _stage(storage, content=b"immutable source")
|
||||
storage.publish([original])
|
||||
|
||||
copied = storage.stage_copy(
|
||||
batch_id="batch-copy",
|
||||
source_reference=original.reference,
|
||||
expected_source_task_id="task-1",
|
||||
expected_source_file_id="source-1",
|
||||
task_id="task-2",
|
||||
source_file_id="source-2",
|
||||
version=1,
|
||||
name="source.txt",
|
||||
)
|
||||
storage.publish([copied])
|
||||
|
||||
assert storage.read(copied.reference) == b"immutable source"
|
||||
assert storage.delete(
|
||||
original.reference,
|
||||
expected_task_id="task-1",
|
||||
expected_source_file_id="source-1",
|
||||
) is True
|
||||
assert storage.read(copied.reference) == b"immutable source"
|
||||
assert storage.delete(
|
||||
copied.reference,
|
||||
expected_task_id="task-2",
|
||||
expected_source_file_id="source-2",
|
||||
) is True
|
||||
_assert_staging_empty(storage)
|
||||
|
||||
|
||||
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
|
||||
storage = LocalDataProcessStorage(tmp_path / "storage")
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
|
||||
_preview_config_changed,
|
||||
_reasoning_output_is_valid,
|
||||
_source_storage_descriptor,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
|
||||
yield self._conn
|
||||
|
||||
|
||||
class _RepeatConnection:
|
||||
def __init__(self) -> None:
|
||||
self.source_files = [
|
||||
{
|
||||
"id": "source-old",
|
||||
"name": "source.jsonl",
|
||||
"size_bytes": 12,
|
||||
"record_count": 1,
|
||||
"file_format": "jsonl",
|
||||
"checksum_sha256": "a" * 64,
|
||||
"content": '{"id":1}\n',
|
||||
"content_preview": '{"id":1}',
|
||||
"metadata": {"storage_backend": "local"},
|
||||
"created_by": "user-1",
|
||||
}
|
||||
]
|
||||
self.source_previews = [
|
||||
{
|
||||
"id": "preview-old",
|
||||
"source_file_id": "source-old",
|
||||
"original_content": '{"id":1}',
|
||||
"edited_content": '{"id":1,"checked":true}',
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"source_start_line": 1,
|
||||
"source_end_line": 1,
|
||||
"token_count": 5,
|
||||
"status": "modified",
|
||||
"quality_score": {"overall": 90},
|
||||
}
|
||||
]
|
||||
self.created_task: dict[str, Any] | None = None
|
||||
self.created_files: list[dict[str, Any]] = []
|
||||
self.created_previews: list[dict[str, Any]] = []
|
||||
|
||||
def execute(self, sql: str, params: Any = None) -> _Result:
|
||||
normalized = " ".join(sql.split())
|
||||
if params is not None:
|
||||
assert normalized.count("%s") == len(params)
|
||||
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
|
||||
return _Result(row=None)
|
||||
if normalized.startswith("SELECT * FROM data_process_source_files"):
|
||||
return _Result(rows=[dict(item) for item in self.source_files])
|
||||
if normalized.startswith("SELECT * FROM data_process_preview_items"):
|
||||
return _Result(rows=[dict(item) for item in self.source_previews])
|
||||
if normalized.startswith("INSERT INTO data_process_tasks"):
|
||||
self.created_task = {
|
||||
"id": params[0],
|
||||
"name": params[1],
|
||||
"description": params[2],
|
||||
"status": "pending",
|
||||
"process_type": params[3],
|
||||
"source_dataset_id": params[4],
|
||||
"config": params[5],
|
||||
"progress": 20,
|
||||
"input_count": params[6],
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_total_files": params[7],
|
||||
"preview_completed_files": params[8],
|
||||
"created_at": params[15],
|
||||
"updated_at": params[16],
|
||||
}
|
||||
return _Result(row=dict(self.created_task))
|
||||
if normalized.startswith("INSERT INTO data_process_source_files"):
|
||||
self.created_files.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"storage_object_id": params[2],
|
||||
"content": params[8],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("INSERT INTO data_process_preview_items"):
|
||||
self.created_previews.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"source_file_id": params[2],
|
||||
"edited_content": params[4],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
|
||||
return _Result(
|
||||
row={
|
||||
"source_file_count": len(self.created_files),
|
||||
"preview_count": len(self.created_previews),
|
||||
}
|
||||
)
|
||||
raise AssertionError(f"unexpected SQL: {normalized}")
|
||||
|
||||
|
||||
class _RepeatStore(DataProcessStore):
|
||||
def __init__(self, conn: _RepeatConnection) -> None:
|
||||
self._conn = conn
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[_RepeatConnection]:
|
||||
yield self._conn
|
||||
|
||||
def _task_in_connection(
|
||||
self,
|
||||
conn: Any,
|
||||
task_id: str,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
assert task_id == "task-source"
|
||||
assert for_update is True
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": "原任务",
|
||||
"description": "原描述",
|
||||
"status": "completed",
|
||||
"process_type": "structured",
|
||||
"source_dataset_id": None,
|
||||
"config": {
|
||||
"temperature": 0.3,
|
||||
"_regeneration_prepared": {"prepared": True},
|
||||
},
|
||||
"results_confirmed": True,
|
||||
"preview_status": "completed",
|
||||
"tenant_id": "tenant-1",
|
||||
"project_id": "project-1",
|
||||
"owner_id": "owner-1",
|
||||
"created_by": "user-1",
|
||||
"updated_at": "2026-07-28T12:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
class _TaskListConnection:
|
||||
def __init__(self) -> None:
|
||||
self.task = {
|
||||
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
|
||||
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
|
||||
|
||||
|
||||
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
|
||||
conn = _RepeatConnection()
|
||||
store = _RepeatStore(conn)
|
||||
request_id = "repeat-request-0001"
|
||||
target_task_id = repeat_task_id("task-source", request_id)
|
||||
|
||||
repeated = store.repeat_task(
|
||||
"task-source",
|
||||
expected_updated_at="2026-07-28T12:00:00Z",
|
||||
request_id=request_id,
|
||||
file_copies={
|
||||
"source-old": {
|
||||
"id": "source-new",
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert repeated["created"] is True
|
||||
assert repeated["task"]["id"] == target_task_id
|
||||
assert repeated["task"]["config"] == {"temperature": 0.3}
|
||||
assert repeated["task"]["results_confirmed"] is False
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == 1
|
||||
assert conn.created_files == [
|
||||
{
|
||||
"id": "source-new",
|
||||
"task_id": target_task_id,
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
"content": '{"id":1}\n',
|
||||
}
|
||||
]
|
||||
assert conn.created_previews[0]["task_id"] == target_task_id
|
||||
assert conn.created_previews[0]["source_file_id"] == "source-new"
|
||||
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
|
||||
|
||||
|
||||
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
|
||||
decoded = _decode_row(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user