YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 完善数据预处理与 JSON 上传链路

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files

275
backend/app/api/v1/endpoints/data_process.py
View File

@@ -8,12 +8,14 @@ import os
import re
import socket
import time
from collections.abc import Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from copy import deepcopy
from dataclasses import asdict
from pathlib import Path
from threading import BoundedSemaphore, Lock
from typing import Any, Iterator, Literal
from typing import Any, Literal
from urllib.parse import quote, urlsplit
import httpx
@@ -45,9 +47,10 @@ from app.modules.data_process.algorithms import (
is_near_duplicate,
near_duplicate_fingerprint,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
remove_document_noise,
score_quality,
structured_json_dumps,
)
from app.modules.data_process.document_chunking import (
DocumentChunk,
@@ -75,9 +78,11 @@ from app.modules.data_process.store import (
NotFoundError,
get_data_process_store,
new_id,
repeat_task_id,
)
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessRepeatRequest,
DataProcessStatus,
DataProcessTaskCreate,
DataProcessTaskUpdate,
@@ -261,7 +266,14 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
content = str(source.get("content") or "")
file_format = str(source.get("file_format") or "").lower()
if file_format == "xlsx":
# XLSX 上传阶段已安全解析为 JSONL 后入库。
raw_content = source.get("raw_content")
if isinstance(raw_content, bytes):
return parse_text_content(
raw_content,
filename=str(source.get("name") or "source.xlsx"),
file_format="xlsx",
)
# 兼容原始对象已缺失的历史文件：退化为上传阶段生成的 JSONL。
return parse_text_content(content, file_format="jsonl")
if file_format in {"pdf", "docx", "pptx"}:
# 文档上传阶段已抽取文本，预览阶段只需要对正文切片。
@@ -381,11 +393,12 @@ def _build_preview_items(
seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {}
items: list[dict[str, Any]] = []
def append_item(item: dict[str, Any]) -> None:
def append_item(item: dict[str, Any], *, dedup_content: str) -> None:
content = str(item.get("edited_content") or "").strip()
if should_clean_invalid and not content:
return
content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
# 去重必须基于脱敏前内容，否则不同原文可能在替换 PII 后被错误合并。
content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest()
if should_deduplicate and content_hash in seen_content_hashes:
return
seen_content_hashes.add(content_hash)
@@ -447,6 +460,7 @@ def _build_preview_items(
continue
for key in band_keys:
seen_near_duplicate_bands.setdefault(key, []).append(content)
dedup_content = content
pii_counts: dict[str, int] = {}
if should_desensitize:
content, pii_counts = desensitize_pii(content)
@@ -476,7 +490,8 @@ def _build_preview_items(
else "original"
),
"quality_score": quality,
}
},
dedup_content=dedup_content,
)
continue
@@ -488,48 +503,76 @@ def _build_preview_items(
"filter_anomaly",
}
source_records = list(parsed.records)
processed_records = preprocess_structured_records(
processed_records = preprocess_structured_records_with_lineage(
source_records,
structured_options,
)
if not processed_records and parsed.text and not source_records:
processed_records = [{"value": parsed.text}]
same_cardinality = len(processed_records) == len(source_records)
for index, record in enumerate(processed_records):
original_record = source_records[index] if same_cardinality else record
original_content = json.dumps(
original_record,
ensure_ascii=False,
separators=(",", ":"),
for processed in processed_records:
source_index = processed.source_index
record = processed.record
original_record = (
source_records[source_index]
if source_index < len(source_records)
else record
)
source_locator = (
deepcopy(parsed.record_locators[source_index])
if source_index < len(parsed.record_locators)
else None
)
original_content = structured_json_dumps(original_record)
pii_counts: dict[str, int] = {}
edited_record = record
dedup_content = (
canonical_record_json(record)
if "normalize_format" in preprocess_options
else structured_json_dumps(record)
)
if should_desensitize:
edited_record, pii_counts = desensitize_structured_record(record)
content = (
canonical_record_json(edited_record)
if "normalize_format" in preprocess_options
else json.dumps(
edited_record,
ensure_ascii=False,
separators=(",", ":"),
)
else structured_json_dumps(edited_record)
)
quality = _preview_quality(content, config)
quality["pii_replacements"] = pii_counts
if source_locator is not None:
quality["source_locator"] = source_locator
source_start = (
source_locator.get("source_start")
if source_locator is not None
else None
)
source_end = (
source_locator.get("source_end")
if source_locator is not None
else None
)
source_start_line = (
source_locator.get("start_line")
if source_locator is not None
else None
)
source_end_line = (
source_locator.get("end_line")
if source_locator is not None
else None
)
append_item(
{
"source_file_id": source["id"],
"original_content": original_content,
"edited_content": content,
"source_start": None,
"source_end": None,
"source_start_line": None,
"source_end_line": None,
"source_start": source_start,
"source_end": source_end,
"source_start_line": source_start_line,
"source_end_line": source_end_line,
"token_count": estimate_token_count(content),
"status": "modified" if content != original_content else "original",
"quality_score": quality,
}
},
dedup_content=dedup_content,
)
return items
@@ -850,6 +893,135 @@ def prepare_regeneration(
)
def _repeat_file_copies(
store: DataProcessStore,
storage: LocalDataProcessStorage,
source_task_id: str,
request_id: str,
) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]:
"""为新任务创建独立的源文件引用，避免删除任一任务时互相影响。"""
target_task_id = repeat_task_id(source_task_id, request_id)
copies: dict[str, dict[str, str]] = {}
staged: list[StagedSourceObject] = []
batch_id = storage.new_batch_id()
for summary in store.list_source_files(source_task_id):
old_file_id = str(summary["id"])
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
new_file_id = new_id("dpsf")
old_reference = str(source.get("storage_object_id") or "")
if old_reference.startswith("local://data-process/"):
staged_object = storage.stage_copy(
batch_id=batch_id,
source_reference=old_reference,
expected_source_task_id=source_task_id,
expected_source_file_id=old_file_id,
task_id=target_task_id,
source_file_id=new_file_id,
version=1,
name=str(source["name"]),
)
staged.append(staged_object)
new_reference = staged_object.reference
elif old_reference.startswith("db://data-process/") or not old_reference:
new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1"
else:
raise ValueError("源任务包含不受支持的文件存储引用")
copies[old_file_id] = {
"id": new_file_id,
"storage_object_id": new_reference,
}
return copies, staged
def _remove_repeated_storage_objects(
storage: LocalDataProcessStorage,
task_id: str,
staged: list[StagedSourceObject],
copies: dict[str, dict[str, str]],
) -> None:
source_file_ids = {
str(copy["storage_object_id"]): str(copy["id"])
for copy in copies.values()
}
for item in staged:
try:
storage.delete(
item.reference,
expected_task_id=task_id,
expected_source_file_id=source_file_ids[item.reference],
)
except Exception:
logger.exception(
"failed to roll back repeated data process source object task_id=%s",
task_id,
)
@router.post("/{task_id}/repeat", status_code=202)
def repeat_generation(
task_id: str,
payload: DataProcessRepeatRequest,
background_tasks: BackgroundTasks,
store: DataProcessStore = Depends(get_data_process_store),
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
) -> dict[str, Any]:
"""按原任务快照创建独立任务，并立即在后台开始新一批生成。"""
with api_errors():
repeated = store.find_repeated_task(task_id, payload.request_id)
staged: list[StagedSourceObject] = []
target_task_id = repeat_task_id(task_id, payload.request_id)
if repeated is None:
copies, staged = _repeat_file_copies(
store,
storage,
task_id,
payload.request_id,
)
storage.publish(staged)
try:
repeated = store.repeat_task(
task_id,
expected_updated_at=payload.expected_updated_at,
request_id=payload.request_id,
file_copies=copies,
)
except Exception:
_remove_repeated_storage_objects(
storage,
target_task_id,
staged,
copies,
)
raise
if not repeated["created"]:
_remove_repeated_storage_objects(
storage,
target_task_id,
staged,
copies,
)
repeated_task = repeated["task"]
if repeated_task.get("status") == "pending":
try:
started = store.start_generation(target_task_id, replace_existing=True)
background_tasks.add_task(
_run_generation,
store,
target_task_id,
str(started["generation_run_id"]),
)
except ConflictError:
latest = store.get_task(target_task_id)
if latest.get("status") != "running":
raise
repeated["task"] = store.get_task(target_task_id)
repeated["progress"] = store.progress(target_task_id)
return ok(repeated, "已按原配置创建新任务并开始后台生成")
@router.delete("/{task_id}")
def delete_task(
task_id: str,
@@ -919,7 +1091,7 @@ async def upload_source_files(
f"{suffix} is not supported for {process_type} data processing",
)
parsed = parse_text_content(raw, filename=name)
if not parsed.text:
if not parsed.text.strip():
raise fail(400, f"source file is empty: {name}")
batch_size += len(raw)
if batch_size > MAX_SOURCE_BATCH_BYTES:
@@ -934,7 +1106,11 @@ async def upload_source_files(
content=raw,
)
staged.append(staged_object)
record_count = len(parsed.records) or (1 if parsed.text else 0)
record_count = (
len(parsed.records)
if process_type == "structured"
else (1 if parsed.text else 0)
)
prepared.append(
{
"id": source_file_id,
@@ -1362,15 +1538,28 @@ def _prepare_preview_items(
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
)
is_unstructured = task.get("process_type") == "unstructured"
if is_unstructured and (
needs_unstructured_raw = is_unstructured and (
chunk_method == "layout_hybrid"
or preprocess_options & {"clean_invalid", "clean_invalid_content"}
):
)
has_structured_xlsx = not is_unstructured and any(
str(source.get("file_format") or "").lower() == "xlsx"
for source in sources
)
if needs_unstructured_raw or has_structured_xlsx:
for index, source in enumerate(sources):
if (
chunk_method != "layout_hybrid"
and str(source.get("file_format") or "").lower() != "pdf"
):
source_format = str(source.get("file_format") or "").lower()
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
needs_pdf_noise = (
is_unstructured
and not needs_layout_raw
and source_format == "pdf"
and bool(
preprocess_options & {"clean_invalid", "clean_invalid_content"}
)
)
if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise):
continue
storage_object_id = str(source.get("storage_object_id") or "")
actual_size = storage.file_size(
@@ -1379,7 +1568,7 @@ def _prepare_preview_items(
expected_source_file_id=str(source["id"]),
)
if actual_size is None:
if chunk_method == "layout_hybrid":
if needs_layout_raw:
raise InvalidStateError(
"版面结构混合切分无法读取原始文件，请重新上传后再处理"
)
@@ -1396,12 +1585,10 @@ def _prepare_preview_items(
)
)
enriched = dict(source)
if chunk_method == "layout_hybrid":
if needs_structured_xlsx or needs_layout_raw:
enriched["raw_content"] = raw
sources[index] = enriched
continue
if str(source.get("file_format") or "").lower() != "pdf":
continue
pages = extract_pdf_page_texts(raw)
extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""):
@@ -1413,7 +1600,7 @@ def _prepare_preview_items(
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
sources[index] = enriched
items = _build_preview_items(task, sources)
if not items and source_file_ids is None:
if not items and source_file_ids is None and is_unstructured:
raise InvalidStateError("source files did not produce preview items")
return items
@@ -1435,6 +1622,7 @@ def _run_preview(
len(source_file_ids),
)
try:
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
@@ -1461,7 +1649,7 @@ def _run_preview(
storage,
[source_file_id],
)
if not items:
if not items and is_unstructured:
raise InvalidStateError(
f"source file did not produce preview items: {source_file_id}"
)
@@ -1657,8 +1845,13 @@ def update_preview_item(
) -> dict[str, Any]:
with api_errors():
task = store.get_task(task_id)
existing = store.get_preview_item(task_id, preview_id)
update = payload.model_dump(exclude_unset=True, mode="json")
update["quality_score"] = _preview_quality(payload.edited_content, task.get("config") or {})
quality = _preview_quality(payload.edited_content, task.get("config") or {})
source_locator = (existing.get("quality_score") or {}).get("source_locator")
if isinstance(source_locator, Mapping):
quality["source_locator"] = deepcopy(dict(source_locator))
update["quality_score"] = quality
item = store.update_preview_item(
task_id,
preview_id,

845
backend/app/modules/data_process/algorithms.py
View File

File diff suppressed because it is too large Load Diff

61
backend/app/modules/data_process/storage.py
View File

@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
self._issued_staged_objects[temporary_path] = staged
return staged
def stage_copy(
self,
*,
batch_id: str,
source_reference: str,
expected_source_task_id: str,
expected_source_file_id: str,
task_id: str,
source_file_id: str,
version: int,
name: str,
) -> StagedSourceObject:
"""为不可变源对象创建独立目录项，不把大文件重新读入内存。"""
batch_id = _safe_component(batch_id, "batch id")
task_id = _safe_component(task_id, "task id")
source_file_id = _safe_component(source_file_id, "source file id")
if isinstance(version, bool) or not isinstance(version, int) or version < 1:
raise DataProcessStorageError("invalid source file version")
basename = _safe_basename(name)
source_relative = self._relative_from_reference(source_reference)
if source_relative is None:
raise DataProcessStorageError("original source object is not available")
self._assert_expected_owner(
source_relative,
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
source_path = self._path_for_relative(source_relative)
try:
os.link(source_path, temporary_path, follow_symlinks=False)
copy_info = temporary_path.lstat()
if (
not stat.S_ISREG(copy_info.st_mode)
or source_info.st_dev != copy_info.st_dev
or source_info.st_ino != copy_info.st_ino
):
raise DataProcessStorageError("source storage object changed while copying")
except Exception:
temporary_path.unlink(missing_ok=True)
raise
relative_path = PurePosixPath(
task_id,
source_file_id,
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
def publish(self, objects: Iterable[StagedSourceObject]) -> None:
staged = list(objects)
published: list[StagedSourceObject] = []

321
backend/app/modules/data_process/store.py
View File

@@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = {
"preserve_lists": True,
}
_REGENERATION_MARKER_KEY = "_regeneration_prepared"
_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id"
_REPEAT_REQUEST_KEY = "_repeat_request_id"
_INTERNAL_CONFIG_KEYS = {
_REGENERATION_MARKER_KEY,
_REPEAT_SOURCE_TASK_KEY,
_REPEAT_REQUEST_KEY,
}
class DataProcessStoreError(RuntimeError):
@@ -71,6 +78,13 @@ def new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:20]}"
def repeat_task_id(source_task_id: str, request_id: str) -> str:
"""按源任务和请求幂等键生成稳定的新任务 ID。"""
digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest()
return f"dpt_{digest[:20]}"
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
@@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool:
return _regeneration_marker(task) is not None
def _business_config(config: dict[str, Any] | None) -> dict[str, Any]:
"""过滤只供服务端维护的工作流标记。"""
return {
key: value
for key, value in (config or {}).items()
if key not in _INTERNAL_CONFIG_KEYS
}
def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None:
"""从 API 任务快照中移除服务端内部重新生成标记。"""
"""从 API 任务快照中移除服务端内部工作流标记。"""
if item is None:
return None
public = dict(item)
config = public.get("config")
if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config:
public["config"] = {
key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY
}
if isinstance(config, dict):
public["config"] = _business_config(config)
return public
@@ -353,13 +375,7 @@ class DataProcessStore:
payload.get("description") or "",
payload["process_type"],
payload.get("source_dataset_id"),
json_dumps(
{
key: value
for key, value in (payload.get("config") or {}).items()
if key != _REGENERATION_MARKER_KEY
}
),
json_dumps(_business_config(payload.get("config"))),
payload.get("tenant_id"),
payload.get("project_id"),
payload.get("owner_id"),
@@ -373,6 +389,268 @@ class DataProcessStore:
raise ConflictError("data process task name already exists") from exc
return _public_task(_decode_row(row)) or {}
@staticmethod
def _repeat_response(
conn: psycopg.Connection[dict[str, Any]],
row: dict[str, Any],
*,
source_task_id: str,
created: bool,
) -> dict[str, Any]:
task_id = str(row["id"])
counts = conn.execute(
"""
SELECT
(SELECT COUNT(*) FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count,
(SELECT COUNT(*) FROM data_process_preview_items
WHERE task_id=%s) AS preview_count
""",
(task_id, task_id),
).fetchone() or {}
task = _public_task(_decode_row(row)) or {}
task["source_file_count"] = int(counts.get("source_file_count") or 0)
task["preview_count"] = int(counts.get("preview_count") or 0)
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": task["source_file_count"],
"copied_preview_count": task["preview_count"],
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
"""查找同一幂等请求已创建的新任务。"""
task_id = repeat_task_id(source_task_id, request_id)
with self.connect() as conn:
row = conn.execute(
"SELECT * FROM data_process_tasks WHERE id=%s",
(task_id,),
).fetchone()
if row is None:
return None
decoded = _decode_row(row) or {}
config = decoded.get("config") or {}
if (
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
or config.get(_REPEAT_REQUEST_KEY) != request_id
):
raise ConflictError("再次生成请求与现有任务冲突")
if decoded.get("deleted_at"):
raise ConflictError("此次再次生成创建的任务已被删除，请重新发起")
return self._repeat_response(
conn,
row,
source_task_id=source_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
"""复制已确认任务的配置、源文件和预览，结果与发布数据保持独立。"""
task_id = repeat_task_id(source_task_id, request_id)
now = utcnow()
try:
with self.connect() as conn:
existing = conn.execute(
"SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE",
(task_id,),
).fetchone()
if existing is not None:
decoded = _decode_row(existing) or {}
config = decoded.get("config") or {}
if (
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
or config.get(_REPEAT_REQUEST_KEY) != request_id
):
raise ConflictError("再次生成请求与现有任务冲突")
if decoded.get("deleted_at"):
raise ConflictError("此次再次生成创建的任务已被删除，请重新发起")
return self._repeat_response(
conn,
existing,
source_task_id=source_task_id,
created=False,
)
source_task = self._task_in_connection(
conn,
source_task_id,
for_update=True,
)
if (
source_task.get("status") != "completed"
or source_task.get("results_confirmed") is False
):
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
raise ConflictError("源任务仍在处理切分，暂时不能再次生成")
if expected_updated_at != _serialize_value(source_task.get("updated_at")):
raise ConflictError("源任务已被其他操作修改，请刷新后重试")
source_files = conn.execute(
"""
SELECT * FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL
ORDER BY created_at, id
""",
(source_task_id,),
).fetchall()
source_file_ids = {str(row["id"]) for row in source_files}
if source_file_ids != set(file_copies):
raise ConflictError("源文件快照已变化，请刷新后重试")
previews = conn.execute(
"""
SELECT * FROM data_process_preview_items
WHERE task_id=%s
ORDER BY source_file_id NULLS LAST, source_start NULLS LAST,
created_at, id
""",
(source_task_id,),
).fetchall()
if not previews:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
suffix = f"（再次生成-{task_id[-6:]}）"
base_name = str(source_task.get("name") or "数据处理任务")
repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}"
repeated_config = _business_config(source_task.get("config") or {})
repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id
repeated_config[_REPEAT_REQUEST_KEY] = request_id
input_count = sum(int(row.get("record_count") or 0) for row in source_files)
task_row = conn.execute(
"""
INSERT INTO data_process_tasks
(id, name, description, status, process_type, source_dataset_id,
output_dataset_id, config, progress, input_count, output_count,
filtered_count, duplicate_count, error_count, failure_reason,
generation_run_id, results_confirmed, workflow_step,
preview_status, preview_progress, preview_run_id,
preview_failure_reason, preview_total_files,
preview_completed_files, tenant_id, project_id, owner_id,
approval_status, created_by, updated_by, created_at, updated_at)
VALUES
(%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0,
0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100,
NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
task_id,
repeated_name,
source_task.get("description") or "",
source_task["process_type"],
source_task.get("source_dataset_id"),
json_dumps(repeated_config),
input_count,
len(source_files),
len(source_files),
source_task.get("tenant_id"),
source_task.get("project_id"),
source_task.get("owner_id"),
source_task.get("approval_status") or "not_required",
source_task.get("created_by"),
source_task.get("created_by"),
now,
now,
),
).fetchone()
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
new_file_id = str(copy["id"])
storage_object_id, metadata = _source_storage_descriptor(
{
"storage_object_id": copy["storage_object_id"],
"metadata": _json_value(source.get("metadata"), {}),
},
task_id,
new_file_id,
)
file_id_map[old_file_id] = new_file_id
conn.execute(
"""
INSERT INTO data_process_source_files
(id, task_id, storage_object_id, name, size_bytes, record_count,
file_format, checksum_sha256, version_no, content,
content_preview, metadata, tenant_id, project_id, created_by,
created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s,
%s, %s, %s, %s, %s)
""",
(
new_file_id,
task_id,
storage_object_id,
source["name"],
source.get("size_bytes") or 0,
source.get("record_count") or 0,
source.get("file_format"),
source["checksum_sha256"],
source.get("content") or "",
source.get("content_preview"),
json_dumps(metadata),
source_task.get("tenant_id"),
source_task.get("project_id"),
source.get("created_by") or source_task.get("created_by"),
now,
now,
),
)
for preview in previews:
old_source_file_id = preview.get("source_file_id")
conn.execute(
"""
INSERT INTO data_process_preview_items
(id, task_id, source_file_id, original_content, edited_content,
source_start, source_end, source_start_line, source_end_line,
token_count, status, quality_score, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s)
""",
(
new_id("dpp"),
task_id,
file_id_map.get(str(old_source_file_id))
if old_source_file_id
else None,
preview.get("original_content") or "",
preview.get("edited_content") or "",
preview.get("source_start"),
preview.get("source_end"),
preview.get("source_start_line"),
preview.get("source_end_line"),
max(0, int(preview.get("token_count") or 0)),
preview.get("status") or "original",
json_dumps(_json_value(preview.get("quality_score"), {})),
now,
now,
),
)
return self._repeat_response(
conn,
task_row or {},
source_task_id=source_task_id,
created=True,
)
except psycopg.errors.UniqueViolation as exc:
raise ConflictError("再次生成任务名称或请求发生冲突，请重试") from exc
def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]:
lock = " FOR UPDATE" if for_update else ""
with self.connect() as conn:
@@ -654,14 +932,11 @@ class DataProcessStore:
"process type and source dataset cannot change during regeneration"
)
if payload.get("config") is not None:
next_config = {
key: value
for key, value in payload["config"].items()
if key != _REGENERATION_MARKER_KEY
}
current_marker = _regeneration_marker(task)
if current_marker:
next_config[_REGENERATION_MARKER_KEY] = current_marker
next_config = _business_config(payload["config"])
current_config = dict(task.get("config") or {})
for key in _INTERNAL_CONFIG_KEYS:
if key in current_config:
next_config[key] = current_config[key]
values["config"] = json_dumps(next_config)
invalidates_results = (
("config" in payload and payload.get("config") != task.get("config"))
@@ -753,8 +1028,10 @@ class DataProcessStore:
raise InvalidStateError("process_type cannot be changed during regeneration")
current_config = dict(task.get("config") or {})
next_config = dict(payload.get("config") or {})
next_config.pop(_REGENERATION_MARKER_KEY, None)
next_config = _business_config(payload.get("config"))
for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY):
if key in current_config:
next_config[key] = current_config[key]
preview_invalidated = _preview_config_changed(
process_type,
current_config,

13
backend/app/schemas/data_process.py
View File

@@ -220,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel):
return self
class DataProcessRepeatRequest(BaseModel):
"""按已确认任务的完整快照创建一批独立的新生成结果。"""
model_config = ConfigDict(extra="forbid")
expected_updated_at: str = Field(min_length=1)
request_id: str = Field(
min_length=8,
max_length=80,
pattern=r"^[A-Za-z0-9_-]+$",
)
class PreviewBuildRequest(BaseModel):
model_config = ConfigDict(extra="forbid")

303
backend/tests/test_data_process_algorithms.py
View File

@@ -5,6 +5,7 @@ import json
import xml.etree.ElementTree as ET
import zipfile
from datetime import datetime
from decimal import Decimal
import pytest
from docx import Document
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
normalize_text,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
record_fingerprint,
remove_document_noise,
score_quality,
stable_split,
stable_split_assignments,
structured_json_dumps,
)
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
assert parsed_txt.text == "普通文本"
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
root_json = parse_text_content('{"id":1}', filename="root.json")
assert root_json.record_locators == (
{
"kind": "json",
"record_index": 1,
"json_pointer": "",
"source_start": 0,
"source_end": 8,
"start_line": 1,
"end_line": 1,
},
)
wrapped_json = parse_text_content(
'{"records":[{"id":1},{"id":1}]}',
filename="wrapped.json",
)
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
"/records/0",
"/records/1",
]
parsed_jsonl = parse_text_content(
'{"id":1}\r\n\r\n{"id":1}',
filename="records.jsonl",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_jsonl.record_locators
] == [(1, 1, 1), (2, 3, 3)]
assert [
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_jsonl.record_locators
] == ['{"id":1}', '{"id":1}']
parsed_csv = parse_text_content(
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
filename="records.csv",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_csv.record_locators
] == [(1, 2, 3), (2, 5, 5)]
assert [
parsed_csv.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_csv.record_locators
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
processed = preprocess_structured_records_with_lineage(
[
{"id": "A", "value": "first", "empty": ""},
{"id": "", "value": "invalid", "empty": ""},
{"id": "A", "value": "duplicate identity", "empty": ""},
{"id": "B", "value": "second", "empty": ""},
],
["clean_invalid", "deduplicate"],
)
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
assert [entry.record for entry in processed] == [
{"id": "A", "value": "first"},
{"id": "", "value": "invalid"},
{"id": "A", "value": "duplicate identity"},
{"id": "B", "value": "second"},
]
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
assert parsed_pdf.format == "pdf"
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
)
assert parsed_xlsx.record_locators == (
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 3,
"sheet_record_index": 1,
},
)
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active
first.title = "甲表"
first.append(["说明"])
first.append([])
first.append(["id", "value"])
first.append([1, "same"])
first.append([1, "same"])
second = workbook.create_sheet("乙表")
second.append(["id", "value"])
second.append([1, "same"])
output = io.BytesIO()
workbook.save(output)
workbook.close()
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
assert parsed.records == (
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
)
assert [
(
locator["record_index"],
locator["sheet_index"],
locator["sheet_name"],
locator["row_number"],
locator["sheet_record_index"],
)
for locator in parsed.record_locators
] == [
(1, 0, "甲表", 4, 0),
(2, 0, "甲表", 5, 1),
(3, 1, "乙表", 2, 0),
]
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
pages = _pdf_page_texts(
"""
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
"json",
)
assert result == [{"text": "内容"}]
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
assert extract_structured_records(
'{"items":[{"text":" 内容 "}],"total":1}',
"json",
) == [{"text": " 内容 "}]
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
source = '{"code":"００１","text":" 内容 ","quote":"＂"}'
parsed = parse_text_content(source, filename="records.json")
assert parsed.text == source
assert parsed.records == (
{"code": "００１", "text": " 内容 ", "quote": "＂"},
)
invalid_values = (
'{"id":1,"id":2}',
'{"nested":{"id":1,"id":2}}',
'{"value":NaN}',
'{"value":Infinity}',
'{"value":-Infinity}',
'{"value":"bad\x00control"}',
)
for invalid in invalid_values:
with pytest.raises(ValueError):
parse_text_content(invalid, filename="invalid.json")
with pytest.raises(ValueError):
parse_text_content("｛\"id\":1｝", filename="invalid.json")
with pytest.raises(ValueError, match="nesting exceeds"):
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
source = (
' {"code":"００１","text":" 内容 ",'
'"value":0.123456789012345678901234567890}\r\n\r\n'
'{"id":2}\r\n'
)
parsed = parse_text_content(source, filename="records.jsonl")
assert parsed.text == source
assert parsed.records[0] == {
"code": "００１",
"text": " 内容 ",
"value": Decimal("0.123456789012345678901234567890"),
}
assert [
source[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == [
(
'{"code":"００１","text":" 内容 ",'
'"value":0.123456789012345678901234567890}'
),
'{"id":2}',
]
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
parse_text_content(invalid, filename="invalid.jsonl")
def test_json_record_contract_avoids_business_field_collisions() -> None:
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
{"id": 1},
{"id": 2},
]
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
{"id": 1, "data": [{"id": 2}]}
]
assert extract_structured_records(
'{"records":[{"id":1}],"data":[{"id":2}]}',
"json",
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
assert extract_structured_records(
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
"json",
) == [{"id": 1}]
assert extract_structured_records(
'{"payload":{"data":[{"id":2}],"total":1}}',
"json",
) == [{"id": 2}]
assert extract_structured_records('{"records":[],"total":0}', "json") == []
# 包装数组中的非对象不是记录集合，整体按一条业务对象保留。
assert extract_structured_records('{"data":[1,2]}', "json") == [
{"data": [1, 2]}
]
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
pretty = (
'{\n "records": [\n {"id": 1},\n'
' {\n "id": 2\n }\n ],\n "total": 2\n}'
)
parsed = parse_text_content(pretty, filename="pretty.json")
assert [
pretty[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id": 1}', '{\n "id": 2\n }']
assert [
(locator["start_line"], locator["end_line"])
for locator in parsed.record_locators
] == [(3, 3), (4, 6)]
minified = '[{"id":1},{"id":2}]'
parsed = parse_text_content(minified, filename="minified.json")
assert [
minified[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id":1}', '{"id":2}']
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
parsed = parse_text_content(source, filename="precise.json")
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
assert parsed.records[1]["value"] == Decimal("1e400")
assert structured_json_dumps(parsed.records[0]) == (
'{"value":0.123456789012345678901234567890}'
)
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
assert isinstance(parsed.records[0]["value"], Decimal)
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
assert preprocess_structured_records(clean_source, []) == clean_source
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
{"id": "1", "name": "有效"},
{"id": "", "name": "缺少关键字段"},
{"id": "2", "name": "有效"},
]
hierarchy = [
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
{"id": "", "parent_id": "", "name": "", "empty": ""},
]
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
{"id": "1", "parent_id": None, "name": "根节点"},
{"id": "2", "parent_id": "1", "name": "子节点"},
]
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
assert "profile" in preprocess_structured_records(nested, [])[0]
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
duplicates = [
{"customer_id": "C-1", "value": "first"},
{"customer_id": "C-1", "value": "updated"},
{"customer_id": "C-1", "value": "first"},
{"customer_id": "", "value": "blank-one"},
{"customer_id": "", "value": "blank-two"},
]
assert len(preprocess_structured_records(duplicates, [])) == 4
assert len(preprocess_structured_records(duplicates, [])) == 5
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
assert [record["value"] for record in deduplicated] == [
"first",
"updated",
"blank-one",
"blank-two",
]
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
)
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
masked, counts = desensitize_structured_record(
{
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "张三",
"contact_name": "李四",
"姓名": "王五",
"profile.name": "赵六",
}
)
assert masked == {
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "[NAME]",
"contact_name": "[NAME]",
"姓名": "[NAME]",
"profile.name": "[NAME]",
}
assert counts == {
"email": 0,
"phone": 0,
"id_card": 0,
"name": 4,
"total": 4,
}
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
valid = {
"instruction": "如何修改收货地址？",

564
backend/tests/test_data_process_api.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
from copy import deepcopy
from io import BytesIO
from pathlib import Path
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
LocalDataProcessStorage,
get_data_process_storage,
)
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
from app.modules.data_process.store import (
InvalidStateError,
NotFoundError,
get_data_process_store,
repeat_task_id,
)
class FakeDataProcessStore:
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
self.datasets: dict[str, dict[str, Any]] = {}
self.models: dict[str, dict[str, Any]] = {}
self.regeneration_prepared: set[str] = set()
self.repeat_requests: dict[tuple[str, str], str] = {}
self.sequence = 0
def _id(self, prefix: str) -> str:
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
"published_outputs_preserved": published_outputs_preserved,
}
def _repeat_response(
self,
source_task_id: str,
repeated_task_id: str,
*,
created: bool,
) -> dict[str, Any]:
task = self.get_task(repeated_task_id)
task["source_file_count"] = len(self.sources[repeated_task_id])
task["preview_count"] = len(self.previews[repeated_task_id])
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": len(self.sources[repeated_task_id]),
"copied_preview_count": len(self.previews[repeated_task_id]),
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
if repeated_task_id is None:
return None
return self._repeat_response(
source_task_id,
repeated_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
existing = self.find_repeated_task(source_task_id, request_id)
if existing is not None:
return existing
source_task = self.get_task(source_task_id)
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("updated_at") != expected_updated_at:
raise InvalidStateError("源任务已被其他操作修改，请刷新后重试")
source_files = self.sources[source_task_id]
if set(file_copies) != {str(item["id"]) for item in source_files}:
raise InvalidStateError("源文件快照已变化，请刷新后重试")
if not self.previews[source_task_id]:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
repeated_task_id = repeat_task_id(source_task_id, request_id)
suffix = f"（再次生成-{repeated_task_id[-6:]}）"
task = {
**deepcopy(source_task),
"id": repeated_task_id,
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
"status": "pending",
"progress": 20,
"output_dataset_id": None,
"output_datasets": [],
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": len(source_files),
"preview_completed_files": len(source_files),
"started_at": None,
"completed_at": None,
}
self.tasks[repeated_task_id] = task
self.sources[repeated_task_id] = []
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
file_id_map[old_file_id] = copy["id"]
self.sources[repeated_task_id].append(
{
**deepcopy(source),
"id": copy["id"],
"task_id": repeated_task_id,
"storage_object_id": copy["storage_object_id"],
}
)
self.previews[repeated_task_id] = [
{
**deepcopy(item),
"id": self._id("dpp"),
"task_id": repeated_task_id,
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
if item.get("source_file_id")
else None,
}
for item in self.previews[source_task_id]
]
self.results[repeated_task_id] = []
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
return self._repeat_response(
source_task_id,
repeated_task_id,
created=True,
)
def delete_task(self, task_id: str, **_: Any) -> None:
self.get_task(task_id)
del self.tasks[task_id]
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
assert listed_preview.json()["data"]["total"] == 2
preview_item = listed_preview.json()["data"]["items"][0]
source_locator = preview_item["quality_score"]["source_locator"]
assert source_locator == {
"kind": "jsonl",
"record_index": 1,
"start_line": 1,
"end_line": 1,
"source_start": 0,
"source_end": len(preview_item["original_content"]),
}
updated_preview = client.put(
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
json={
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
},
)
assert "quality_score" in updated_preview.json()["data"]
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
assert generated.status_code == 200
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
def test_completed_task_can_repeat_into_an_independent_background_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "原始生成任务",
"process_type": "structured",
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
)
assert uploaded.status_code == 200
source_id = uploaded.json()["data"]["files"][0]["id"]
built = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"replace_existing": True},
)
assert built.status_code == 200
store.tasks[task_id].update(
status="completed",
progress=100,
results_confirmed=True,
workflow_step="results",
output_count=1,
output_dataset_id="dataset-original",
updated_at="2026-07-28T12:00:00Z",
)
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
store.datasets["dataset-original"] = {
"id": "dataset-original",
"name": "原数据集",
"type": "train",
"source_task_id": task_id,
"deleted_at": None,
}
original_task = deepcopy(store.tasks[task_id])
original_sources = deepcopy(store.sources[task_id])
original_previews = deepcopy(store.previews[task_id])
original_results = deepcopy(store.results[task_id])
original_datasets = deepcopy(store.datasets)
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
payload = {
"expected_updated_at": "2026-07-28T12:00:00Z",
"request_id": "repeat-request-0001",
}
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert response.status_code == 202
repeated = response.json()["data"]
repeated_task_id = repeated["task"]["id"]
assert repeated["created"] is True
assert repeated_task_id != task_id
assert repeated["task"]["status"] == "running"
assert repeated["task"]["workflow_step"] == "generate"
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == len(original_previews)
assert store.tasks[task_id] == original_task
assert store.sources[task_id] == original_sources
assert store.previews[task_id] == original_previews
assert store.results[task_id] == original_results
assert store.datasets == original_datasets
repeated_source = store.sources[repeated_task_id][0]
repeated_preview = store.previews[repeated_task_id][0]
assert repeated_source["id"] != source_id
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
assert repeated_preview["id"] != original_previews[0]["id"]
assert repeated_preview["source_file_id"] == repeated_source["id"]
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert replay.status_code == 202
assert replay.json()["data"]["created"] is False
assert replay.json()["data"]["task"]["id"] == repeated_task_id
assert len(store.tasks) == 2
assert len(store.sources[repeated_task_id]) == 1
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "源任务", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
results_confirmed=True,
updated_at="2026-07-28T12:00:00Z",
)
before = deepcopy(store.tasks)
response = client.post(
f"/modelTF/data-process/{task_id}/repeat",
json={
"expected_updated_at": "2026-07-28T11:59:59Z",
"request_id": "repeat-request-stale",
},
)
assert response.status_code == 409
assert store.tasks == before
def test_published_split_datasets_remain_in_detail_after_regeneration(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "先去重再脱敏",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={
"files": (
"names.jsonl",
(
'{"name":"张三","role":"开发"}\n'
'{"name":"李四","role":"开发"}\n'
),
"application/jsonl",
)
},
)
assert uploaded.status_code == 200
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert len(items) == 2
assert len({item["original_content"] for item in items}) == 2
assert {item["edited_content"] for item in items} == {
'{"name":"[NAME]","role":"开发"}'
}
def test_structured_deduplication_removes_identical_rows_across_sources(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "跨源原文去重",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
(
"files",
(
"first.jsonl",
'{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
(
"files",
(
"second.jsonl",
'\n{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
],
)
assert uploaded.status_code == 200
first_source, second_source = uploaded.json()["data"]["files"]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
data = preview.json()["data"]
assert data["total"] == 1
assert data["file_counts"] == {
first_source["id"]: 1,
second_source["id"]: 0,
}
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
store = FakeDataProcessStore()
task = store.create_task(
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
)
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["total"] == 2
preview_items = preview.json()["data"]["items"]
assert len(preview_items) == 2
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 3,
"sheet_record_index": 1,
},
]
def test_docx_preview_preserves_document_block_order_and_source_offsets(
@@ -2757,6 +3105,218 @@ def _preview_task(
)
def _structured_preview_task(
content: str,
*,
file_format: str,
options: list[str] | None = None,
) -> list[dict[str, Any]]:
return data_process_endpoint._build_preview_items(
{
"process_type": "structured",
"config": {"preprocess_options": options or []},
},
[
{
"id": "structured-source",
"name": f"records.{file_format}",
"file_format": file_format,
"content": content,
}
],
)
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
json_source = '{"records":[{"id":1},{"id":2}]}'
json_items = _structured_preview_task(
json_source,
file_format="json",
)
assert [
item["quality_score"]["source_locator"]["json_pointer"]
for item in json_items
] == ["/records/0", "/records/1"]
assert [
json_source[item["source_start"] : item["source_end"]]
for item in json_items
] == ['{"id":1}', '{"id":2}']
assert [item["source_start_line"] for item in json_items] == [1, 1]
jsonl_source = '{"id":1}\n\n{"id":2}'
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in jsonl_items
] == [1, 2]
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
assert [
jsonl_source[item["source_start"] : item["source_end"]]
for item in jsonl_items
] == ['{"id":1}', '{"id":2}']
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
csv_items = _structured_preview_task(csv_source, file_format="csv")
assert [
(item["source_start_line"], item["source_end_line"])
for item in csv_items
] == [(2, 3), (5, 5)]
assert [
csv_source[item["source_start"] : item["source_end"]]
for item in csv_items
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "空 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("empty-array.json", "[]", "application/json")),
(
"files",
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["items"] == []
assert preview.json()["data"]["total"] == 0
assert set(preview.json()["data"]["file_counts"].values()) == {0}
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
invalid_sources = (
("duplicate.json", '{"id":1,"id":2}'),
("duplicate.jsonl", '{"id":1,"id":2}\n'),
("nan.json", '{"value":NaN}'),
("infinity.json", '{"value":Infinity}'),
("control.json", '{"value":"bad\x00control"}'),
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
)
for filename, content in invalid_sources:
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": (filename, content, "application/json")},
)
assert response.status_code == 400, (filename, response.text)
def test_structured_json_preview_preserves_precision_and_business_data_field(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
precise = '{"value":0.123456789012345678901234567890}'
business = '{"id":7,"data":[{"id":8}]}'
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("precise.json", precise, "application/json")),
("files", ("business.json", business, "application/json")),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert [item["original_content"] for item in items] == [precise, business]
assert [
item["quality_score"]["source_locator"]["json_pointer"] for item in items
] == ["", ""]
assert [item["source_start"] for item in items] == [0, 0]
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
source_records = [
{"id": "A", "amount": 10, "empty": ""},
{"id": "A", "amount": 10, "empty": ""},
{"id": "", "amount": 11, "empty": ""},
{"id": "B", "amount": 11, "empty": ""},
{"id": "C", "amount": 12, "empty": ""},
{"id": "D", "amount": 12, "empty": ""},
{"id": "E", "amount": 13, "empty": ""},
{"id": "F", "amount": 13, "empty": ""},
{"id": "G", "amount": 14, "empty": ""},
{"id": "H", "amount": 1000, "empty": ""},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate", "filter_anomaly"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [json.loads(item["original_content"])["id"] for item in items] == [
"A",
"",
"B",
"C",
"D",
"E",
"F",
"G",
]
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
source_records = [
{"customer_id": "C-1", "status": "old"},
{"customer_id": "C-1", "status": "new"},
{"status": "old", "customer_id": "C-1"},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 2]
assert [item["source_start_line"] for item in items] == [1, 2]
assert [json.loads(item["original_content"])["status"] for item in items] == [
"old",
"new",
]
def test_fixed_preview_preserves_source_offsets() -> None:
content = (
"# 第一章\n"

34
backend/tests/test_data_process_storage.py
View File

@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
_assert_staging_empty(storage)
def test_stage_copy_creates_an_independently_deletable_source_object(
tmp_path: Path,
) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")
original = _stage(storage, content=b"immutable source")
storage.publish([original])
copied = storage.stage_copy(
batch_id="batch-copy",
source_reference=original.reference,
expected_source_task_id="task-1",
expected_source_file_id="source-1",
task_id="task-2",
source_file_id="source-2",
version=1,
name="source.txt",
)
storage.publish([copied])
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
original.reference,
expected_task_id="task-1",
expected_source_file_id="source-1",
) is True
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
copied.reference,
expected_task_id="task-2",
expected_source_file_id="source-2",
) is True
_assert_staging_empty(storage)
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")

176
backend/tests/test_data_process_store.py
View File

@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
_preview_config_changed,
_reasoning_output_is_valid,
_source_storage_descriptor,
repeat_task_id,
)
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
yield self._conn
class _RepeatConnection:
def __init__(self) -> None:
self.source_files = [
{
"id": "source-old",
"name": "source.jsonl",
"size_bytes": 12,
"record_count": 1,
"file_format": "jsonl",
"checksum_sha256": "a" * 64,
"content": '{"id":1}\n',
"content_preview": '{"id":1}',
"metadata": {"storage_backend": "local"},
"created_by": "user-1",
}
]
self.source_previews = [
{
"id": "preview-old",
"source_file_id": "source-old",
"original_content": '{"id":1}',
"edited_content": '{"id":1,"checked":true}',
"source_start": 0,
"source_end": 8,
"source_start_line": 1,
"source_end_line": 1,
"token_count": 5,
"status": "modified",
"quality_score": {"overall": 90},
}
]
self.created_task: dict[str, Any] | None = None
self.created_files: list[dict[str, Any]] = []
self.created_previews: list[dict[str, Any]] = []
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if params is not None:
assert normalized.count("%s") == len(params)
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
return _Result(row=None)
if normalized.startswith("SELECT * FROM data_process_source_files"):
return _Result(rows=[dict(item) for item in self.source_files])
if normalized.startswith("SELECT * FROM data_process_preview_items"):
return _Result(rows=[dict(item) for item in self.source_previews])
if normalized.startswith("INSERT INTO data_process_tasks"):
self.created_task = {
"id": params[0],
"name": params[1],
"description": params[2],
"status": "pending",
"process_type": params[3],
"source_dataset_id": params[4],
"config": params[5],
"progress": 20,
"input_count": params[6],
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_total_files": params[7],
"preview_completed_files": params[8],
"created_at": params[15],
"updated_at": params[16],
}
return _Result(row=dict(self.created_task))
if normalized.startswith("INSERT INTO data_process_source_files"):
self.created_files.append(
{
"id": params[0],
"task_id": params[1],
"storage_object_id": params[2],
"content": params[8],
}
)
return _Result()
if normalized.startswith("INSERT INTO data_process_preview_items"):
self.created_previews.append(
{
"id": params[0],
"task_id": params[1],
"source_file_id": params[2],
"edited_content": params[4],
}
)
return _Result()
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
return _Result(
row={
"source_file_count": len(self.created_files),
"preview_count": len(self.created_previews),
}
)
raise AssertionError(f"unexpected SQL: {normalized}")
class _RepeatStore(DataProcessStore):
def __init__(self, conn: _RepeatConnection) -> None:
self._conn = conn
@contextmanager
def connect(self) -> Iterator[_RepeatConnection]:
yield self._conn
def _task_in_connection(
self,
conn: Any,
task_id: str,
*,
for_update: bool = False,
) -> dict[str, Any]:
assert task_id == "task-source"
assert for_update is True
return {
"id": task_id,
"name": "原任务",
"description": "原描述",
"status": "completed",
"process_type": "structured",
"source_dataset_id": None,
"config": {
"temperature": 0.3,
"_regeneration_prepared": {"prepared": True},
},
"results_confirmed": True,
"preview_status": "completed",
"tenant_id": "tenant-1",
"project_id": "project-1",
"owner_id": "owner-1",
"created_by": "user-1",
"updated_at": "2026-07-28T12:00:00Z",
}
class _TaskListConnection:
def __init__(self) -> None:
self.task = {
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
conn = _RepeatConnection()
store = _RepeatStore(conn)
request_id = "repeat-request-0001"
target_task_id = repeat_task_id("task-source", request_id)
repeated = store.repeat_task(
"task-source",
expected_updated_at="2026-07-28T12:00:00Z",
request_id=request_id,
file_copies={
"source-old": {
"id": "source-new",
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
}
},
)
assert repeated["created"] is True
assert repeated["task"]["id"] == target_task_id
assert repeated["task"]["config"] == {"temperature": 0.3}
assert repeated["task"]["results_confirmed"] is False
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == 1
assert conn.created_files == [
{
"id": "source-new",
"task_id": target_task_id,
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
"content": '{"id":1}\n',
}
]
assert conn.created_previews[0]["task_id"] == target_task_id
assert conn.created_previews[0]["source_file_id"] == "source-new"
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
decoded = _decode_row(
{

14
frontend/scripts/regression-data-process-detail.mjs
View File

@@ -83,6 +83,11 @@ assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?d
assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐')
assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集')
assert.match(detailSource, /function startRegeneration\(\)[\s\S]*?name: 'data-process-regenerate'[\s\S]*?params: \{ id: taskId\.value \}/, '重新生成按钮没有携带原任务 ID 进入命名路由')
assert.match(detailSource, /const canRepeatGeneration = computed[\s\S]*?status === 'completed'[\s\S]*?results_confirmed !== false[\s\S]*?previewCount\.value > 0/, '已完成任务缺少再次生成资格判断')
assert.match(detailSource, /repeatDataProcessTask\(taskId\.value,[\s\S]*?expected_updated_at: detail\.value\.updated_at[\s\S]*?request_id: repeatRequestId\.value/, '再次生成没有携带源任务版本和幂等请求 ID')
assert.match(detailSource, /name: 'data-process-workflow'[\s\S]*?params: \{ id: repeated\.task\.id \}/, '再次生成成功后没有进入新任务工作流')
assert.match(detailSource, /原任务和原结果不会被修改/, '再次生成确认提示没有说明原任务保持不变')
assert.match(detailSource, /v-if="canRepeatGeneration"[\s\S]*?@click="repeatGeneration"[\s\S]*?按原配置再生成一批/, '已完成任务详情缺少再次生成新批次入口')
assert.match(detailSource, /const canRegenerate = computed\(\(\) => \{[\s\S]*?status === 'pending'[\s\S]*?status === 'failed'[\s\S]*?status === 'stopped'[\s\S]*?status === 'completed'[\s\S]*?outputDatasetId\.value[\s\S]*?hasPublishedOutputs\.value/, '详情页没有覆盖指针已清空但旧发布数据集仍存在的重新生成任务')
assert.match(detailSource, /v-if="canRegenerate"[\s\S]*?@click="startRegeneration"[\s\S]*?重新生成/, '可恢复任务没有收敛为单一重新生成入口')
assert.match(detailSource, /v-if="detail\.status === 'completed' && !hasCurrentPublishedDataset"[\s\S]*?@click="openPublishDialog"[\s\S]*?发布为三个数据集/, '未发布或发布指针失效的完成任务没有保留发布入口')
@@ -115,6 +120,11 @@ assert.match(detailSource, /inputMetricCount\.toLocaleString\(\) \}\} \{\{ input
assert.match(detailSource, /sourceFileCount\.toLocaleString\(\) \}\} 个/, '源文件数量缺少个数单位')
assert.match(detailSource, /<span>生成结果<\/span><strong>\{\{ numeric\(detail\.output_count\)\.toLocaleString\(\) \}\} 条<\/strong>/, '生成结果数量缺少条数单位或仍误称成功输出')
assert.match(detailSource, /const configExpanded = ref\(false\)/, '处理配置没有默认收起')
assert.match(detailSource, /appendGroup\(\['clean_invalid', 'deduplicate'\], '数据清洗'\)/, '详情页没有将完整清洗配置合并为数据清洗')
assert.match(detailSource, /appendGroup\(\['detect_structure', 'normalize_format'\], '结构标准化'\)/, '详情页没有将完整结构配置合并为结构标准化')
assert.match(detailSource, /历史部分配置/, '详情页没有标识旧任务的半组选项')
assert.match(detailSource, /异常数据过滤（历史规则）/, '详情页没有标识已停用的历史异常过滤规则')
assert.match(detailSource, /new Set\(value\.map/, '详情页没有去除历史预处理配置中的重复值')
assert.match(detailSource, /:aria-expanded="configExpanded"/, '处理配置折叠按钮缺少无障碍状态')
assert.match(detailSource, /<el-collapse-transition>[\s\S]*?v-show="configExpanded"/, '处理配置没有折叠过渡或内容状态')
assert.doesNotMatch(detailSource, /const (?:detailMap|completedResults)\b|TODO: 接入真实接口/, '详情页仍包含本地 Mock 数据')
@@ -126,6 +136,7 @@ for (const apiName of [
'updateDataProcessResult',
'restoreDataProcessResult',
'publishDataProcess',
'repeatDataProcessTask',
]) {
assert.match(
apiSource,
@@ -136,5 +147,8 @@ for (const apiName of [
assert.match(apiSource, /keyword\?: string; status\?: string; split\?: string/, '结果列表 API 缺少服务端筛选参数')
assert.match(apiSource, /\/results\/\$\{encodeURIComponent\(resultId\)\}/, '结果资源路径没有安全编码结果 ID')
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/publish`/, '发布 API 路径不正确')
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/repeat`/, '再次生成 API 路径不正确')
assert.match(typesSource, /interface DataProcessRepeatPayload[\s\S]*?expected_updated_at: string[\s\S]*?request_id: string/, '再次生成请求契约不完整')
assert.match(typesSource, /interface DataProcessRepeatResult[\s\S]*?task: DataProcessTask[\s\S]*?source_task_id: string[\s\S]*?created: boolean/, '再次生成响应契约不完整')
console.log('数据处理任务详情真实 API 回归检查通过')

187
frontend/scripts/regression-data-process-wizard.mjs
View File

@@ -183,8 +183,36 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte
assert.ok(typesSource.includes(field), `PreviewItem 缺少字段：${field}`)
}
assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识')
assert.match(typesSource, /sourceLocator\?: PreviewSourceLocator/, 'PreviewItem 缺少结构化来源定位契约')
assert.match(typesSource, /headingPath\?: string\[\]/, 'PreviewItem 缺少非结构化标题路径')
assert.match(typesSource, /PreviewSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, '前端来源定位 kind 未使用明确联合类型')
assert.match(contractTypesSource, /DataProcessSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, 'API 来源定位 kind 未使用明确联合类型')
for (const field of ['kind', 'record_index', 'start_line', 'end_line', 'source_start', 'source_end', 'json_pointer', 'sheet_index', 'sheet_name', 'row_number', 'sheet_record_index']) {
assert.ok(typesSource.includes(field), `PreviewSourceLocator 缺少字段：${field}`)
assert.ok(contractTypesSource.includes(field), `后端来源定位契约缺少字段：${field}`)
}
assert.match(contractTypesSource, /source_locator\?: DataProcessSourceLocator/, '质量信息缺少来源定位契约')
assert.match(contractTypesSource, /heading_path\?: string\[\]/, '质量信息缺少标题路径契约')
assert.match(viewSource, /const sourceLocator = item\.quality_score\?\.source_locator/, '预览映射丢失来源定位')
assert.match(viewSource, /sourceStart:\s*item\.source_start\s*\?\?\s*sourceLocator\?\.source_start/, 'JSON locator 的字符起点没有映射到预览项')
assert.match(viewSource, /sourceEnd:\s*item\.source_end\s*\?\?\s*sourceLocator\?\.source_end/, 'JSON locator 的字符终点没有映射到预览项')
assert.match(viewSource, /sourceStartLine:\s*item\.source_start_line\s*\?\?\s*sourceLocator\?\.start_line/, 'JSON locator 的起始行没有映射到预览项')
assert.match(viewSource, /sourceEndLine:\s*item\.source_end_line\s*\?\?\s*sourceLocator\?\.end_line/, 'JSON locator 的结束行没有映射到预览项')
assert.match(viewSource, /headingPath:[\s\S]*?item\.quality_score\?\.heading_path/, '预览映射丢失标题路径')
assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤')
assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数')
assert.match(modelSource, /export function sourceLineWindow/, '缺少有界源文件行窗口函数')
assert.match(modelSource, /maxLines:\s*number/, '源文件行窗口缺少最大渲染行数参数')
assert.doesNotMatch(modelSource, /\.split\(\s*['"]\\n['"]\s*\)/, '源文件行窗口仍会先对全文 split')
assert.match(modelSource, /lines\.length < limit/, '源文件行扫描没有受最大行数约束')
assert.match(modelSource, /unicodeCodePointLength/, '源文件字符偏移未与后端 Unicode code point 计数保持一致')
assert.match(modelSource, /export function sourceLineNumberAtOffset/, '字符偏移缺少无数组的行号解析函数')
const manualPreviewHelperStart = modelSource.indexOf('export function isManualPreviewItem(')
const manualPreviewHelperEnd = modelSource.indexOf('\n}', manualPreviewHelperStart)
assert.ok(manualPreviewHelperStart >= 0, '缺少统一的手动预览项判定函数')
const manualPreviewHelperSource = modelSource.slice(manualPreviewHelperStart, manualPreviewHelperEnd + 2)
for (const field of ['status', 'originalContent', 'sourceStart', 'sourceEnd', 'sourceStartLine', 'sourceEndLine', 'sourcePages', 'sourceLocator']) {
assert.ok(manualPreviewHelperSource.includes(field), `手动预览项判定缺少来源字段：${field}`)
}
assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法')
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
const previewBuildBindingStart = viewSource.indexOf('useDataProcessPreviewBuild()')
@@ -210,6 +238,30 @@ for (const marker of [
}
assert.match(previewSource, /sourceStart/, '第四步未使用来源起始偏移')
assert.match(previewSource, /sourceEnd/, '第四步未使用来源结束偏移')
const lineRangeStart = previewSource.indexOf('function lineRange(item: PreviewItem)')
const lineRangeEnd = previewSource.indexOf('\n}', lineRangeStart)
const lineRangeSource = previewSource.slice(lineRangeStart, lineRangeEnd + 2)
assert.match(lineRangeSource, /isManualPreviewItem\(item\)[\s\S]*?手动新增，无源文件定位/, '来源标签仍会把缺少行偏移的正常记录误判为手动新增')
assert.match(lineRangeSource, /props\.processType === 'unstructured'[\s\S]*?来源：源文件记录/, '结构化来源记录缺少无行偏移时的准确标签')
assert.doesNotMatch(lineRangeSource, /sourceStartLine == null[^\n]*手动新增/, '来源标签仍直接以缺少行号判定手动新增')
assert.match(lineRangeSource, /sheet_name[\s\S]*?row_number[\s\S]*?来源：\$\{sheet\} · 第 \$\{locator\.row_number\} 行/, 'XLSX 来源标签没有展示工作表和物理行号')
assert.match(lineRangeSource, /json_pointer[\s\S]*?JSON 路径/, 'JSON 来源标签没有展示 JSON 路径')
assert.match(lineRangeSource, /locator\?\.kind === 'json'[\s\S]*?JSON 根对象/, 'JSON 根对象来源标签被空 JSON Pointer 错误降级')
assert.match(lineRangeSource, /locatedLines[\s\S]*?第 \$\{locatedLines\.start\}[\s\S]*?locatedLines\.end/, 'JSONL/CSV 来源标签没有展示行范围')
assert.match(lineRangeSource, /headingPath[\s\S]*?章节：/, '非结构化来源标签没有合并标题路径')
assert.match(previewSource, /sourceLocator\?\.start_line[\s\S]*?sourceLocator\?\.end_line/, '文本预览没有优先使用后端行号定位')
assert.match(previewSource, /sourceLocator\?\.source_start\s*\?\?\s*item\.sourceStart/, '文本预览没有优先使用 locator 字符起点')
assert.match(previewSource, /sourceLocator\?\.source_end\s*\?\?\s*item\.sourceEnd/, '文本预览没有优先使用 locator 字符终点')
assert.match(previewSource, /data-line-number="line\.number"/, '文本预览行缺少稳定行号定位标识')
assert.match(previewSource, /isLineHighlighted\(line\.number, line\.start, line\.end\)/, '文本预览没有按物理行号高亮')
assert.match(previewSource, /querySelector<HTMLElement>\(`\[data-line-number=/, '选中记录后没有按物理行号滚动定位')
assert.match(previewSource, /const SOURCE_LINE_RENDER_LIMIT = 240/, '源文件查看器缺少安全渲染上限')
assert.match(previewSource, /const SOURCE_LINE_CHARACTER_LIMIT = 4_000/, '源文件查看器缺少单行字符渲染上限')
assert.match(previewSource, /sourceLineWindow\([\s\S]*?SOURCE_LINE_RENDER_LIMIT/, '源文件查看器没有使用有界行窗口')
assert.match(previewSource, /SOURCE_LINE_RENDER_LIMIT,[\s\S]*?SOURCE_LINE_CHARACTER_LIMIT,[\s\S]*?selectedSourceLine\.value,[\s\S]*?selectedSourceOffset\.value/, '单行超大 JSON 没有围绕选中来源构建字符窗口')
assert.match(previewSource, /sourceWindowStartLine/, '源文件查看器缺少窗口起始行状态')
assert.match(previewSource, /showPreviousSourceWindow[\s\S]*?showNextSourceWindow/, '源文件查看器缺少前后窗口导航')
assert.match(previewSource, /sourceLineNumberAtOffset\(props\.sourceText/, '仅有字符偏移时没有解析目标物理行')
assert.match(previewSource, /filterable/, '文件选择器必须可搜索')
assert.match(previewSource, /当前文件/, '预览缺少当前文件切换器')
assert.doesNotMatch(previewSource, /located-badge|sync-label|已定位到/, '源文件栏不应显示冗余定位提示')
@@ -269,6 +321,18 @@ for (const marker of [
]) {
assert.ok(officeViewerSource.includes(marker), `Word/XLSX 预览缺少结构或行为：${marker}`)
}
assert.match(officeViewerSource, /const selectedXlsxLocator = computed/, 'XLSX 查看器没有读取精确来源定位')
assert.match(officeViewerSource, /row\.row_number === locator\.row_number/, 'XLSX 查看器没有按物理行号精确高亮')
assert.match(officeViewerSource, /row\.record_index === locator\.sheet_record_index/, 'XLSX 查看器没有按工作表记录序号精确高亮')
assert.match(officeViewerSource, /Math\.floor\(locator\.sheet_record_index \/ XLSX_PAGE_SIZE\) \* XLSX_PAGE_SIZE/, 'XLSX 查看器没有按记录序号自动计算分页')
assert.match(officeViewerSource, /activeSheetIndex\.value = targetSheet[\s\S]*?pageOffset\.value = targetOffset[\s\S]*?loadPreview\(\)/, '切换记录时 XLSX 查看器没有自动切工作表和分页')
const xlsxHighlightStart = officeViewerSource.indexOf('function xlsxRowHighlighted(')
const xlsxHighlightEnd = officeViewerSource.indexOf('\n}', xlsxHighlightStart)
const xlsxHighlightSource = officeViewerSource.slice(xlsxHighlightStart, xlsxHighlightEnd + 2)
assert.ok(
xlsxHighlightSource.indexOf('locator.row_number') < xlsxHighlightSource.indexOf('selectedRecordKey.value'),
'XLSX 查看器没有把精确定位放在原内容比对 fallback 之前',
)
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
@@ -454,7 +518,7 @@ assert.match(
)
assert.match(
workflowInitializationSource,
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?goToStep\(resumeStep\)[\s\S]*?resumeGeneration/,
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)/,
'生成运行中时没有强制回到第五步并接管后台进度',
)
const startGenerationHandler = viewSource.slice(
@@ -463,7 +527,35 @@ const startGenerationHandler = viewSource.slice(
)
assert.match(startGenerationHandler, /await persistWorkflowStep\('generate'\)[\s\S]*?await startGeneration\(\)[\s\S]*?dirty\.value = false/, '开始生成没有持久化第五步或启动真实后台任务')
assert.doesNotMatch(startGenerationHandler, /router\.(?:push|replace)|allowLeave\s*=\s*true/, '开始生成后应停留在第五步，不得自动跳回列表')
assert.match(viewSource, /:disabled="currentStepId === 'generate' \|\| previewBuilding \|\| sourceUploading"/, '第五步底部返回按钮没有固定禁用')
assert.match(
generationSource,
/const canReturnFromGeneration = computed\(\(\) => \([\s\S]*?generation\.status === 'idle'[\s\S]*?!generationStarting\.value[\s\S]*?!generationRestoring\.value/,
'第五步返回权限没有区分未启动、启动中和恢复中状态',
)
assert.match(
viewSource,
/:disabled="\(currentStepId === 'generate' && !canReturnFromGeneration\) \|\| previewBuilding \|\| sourceUploading"/,
'第五步尚未启动生成时返回按钮仍被禁用',
)
const handleBackStart = viewSource.indexOf('async function handleBack()')
const handleBackEnd = viewSource.indexOf('\n}', handleBackStart)
const handleBackSource = viewSource.slice(handleBackStart, handleBackEnd + 2)
assert.match(
handleBackSource,
/currentStepId\.value === 'generate' && !canReturnFromGeneration\.value/,
'第五步处理函数仍无条件拦截返回',
)
assert.match(
generationSource,
/async function resumeGeneration\(\)[\s\S]*?generationRestoring\.value = true[\s\S]*?await getDataProcessProgress\(taskId\)[\s\S]*?generationRestoring\.value = false/,
'恢复已启动任务时存在短暂可返回的 idle 窗口',
)
assert.match(viewSource, /const resume = resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)[\s\S]*?await resume/, '第五步展示时未先启动恢复锁')
assert.match(
generationSource,
/const generationStarting = ref\(false\)[\s\S]*?generationStarting\.value = true[\s\S]*?generationStarting\.value = false/,
'点击开始生成后到请求启动前没有锁定返回状态',
)
assert.match(
viewSource,
/generation\.status === 'success'[\s\S]*?persistWorkflowStep\('results'\)/,
@@ -506,32 +598,34 @@ assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTy
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
const expectedStructuredOptions = [
['clean_invalid', '清理无效数据', '清理全空列，并剔除关键字段残缺的数据行'],
const expectedStructuredGroups = [
[
'detect_structure',
'嵌套结构展平',
'展平嵌套对象和可解析的 JSON 字段；Excel 表头与合并单元格在上传时自动解析',
"values: ['clean_invalid', 'deduplicate']",
'数据清洗',
'清理全空列和空记录，并删除内容完全相同的记录；不会猜测可空字段是否必填',
],
[
'deduplicate',
'重复记录去重',
'按整行内容或 id、uuid、key、code、*_id 等身份字段去重，暂不支持自定义组合字段',
"values: ['detect_structure', 'normalize_format']",
'结构标准化',
'展平嵌套对象和可解析的 JSON 字段，并统一编码、空白、字段名和 JSON 序列化格式',
],
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值，并过滤乱码等异常记录'],
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
["values: ['desensitize']", '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
]
for (const [value, label, description] of expectedStructuredOptions) {
assert.ok(structuredOptionsSource.includes(`value: '${value}'`), `结构化预处理缺少值：${value}`)
for (const [values, label, description] of expectedStructuredGroups) {
assert.ok(structuredOptionsSource.includes(values), `结构化预处理组合值不准确：${label}`)
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签：${label}`)
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确：${value}`)
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确：${label}`)
}
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)]
.map((match) => match[1])
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重')
assert.equal(expectedStructuredGroups.length, 3, '结构化预处理应收敛为 3 项')
const preprocessGroupsSource = structuredOptionsSource.slice(
structuredOptionsSource.indexOf('const PREPROCESS_GROUPS'),
structuredOptionsSource.indexOf('const legacyAnomalyFilterEnabled'),
)
assert.doesNotMatch(preprocessGroupsSource, /异常数据过滤|filter_anomaly|IQR/, '结构化新任务仍暴露异常数据过滤')
assert.match(structuredOptionsSource, /:indeterminate="groupIndeterminate\(group\.values\)"/, '历史部分选中的组合项没有半选回显')
assert.match(structuredOptionsSource, /function updatePreprocessGroup\([\s\S]*?new Set\(props\.options\.preprocessOptions\)[\s\S]*?next\.add\(value\)[\s\S]*?next\.delete\(value\)[\s\S]*?\[\.\.\.next\]/, '结构化预处理组合开关没有原子化更新或去重内部选项')
assert.match(typesSource, /仅用于恢复历史任务[\s\S]*?\| 'filter_anomaly'/, '异常数据过滤缺少历史兼容类型')
assert.match(structuredOptionsSource, /legacyAnomalyFilterEnabled[\s\S]*?历史任务[\s\S]*?结果可复现/, '历史异常过滤配置没有透明提示')
assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
for (const splitName of ['训练集', '验证集', '测试集']) {
assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`)
@@ -585,8 +679,20 @@ for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json',
}
assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别')
assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示')
assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验')
assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本')
const sourceValidationStart = sourceUploadWorkerSource.indexOf('export function validateSourceFileSelection(')
const sourceValidationEnd = sourceUploadWorkerSource.indexOf('\n}\n\nfunction unicodeCodePointLength', sourceValidationStart)
assert.ok(sourceValidationStart >= 0 && sourceValidationEnd > sourceValidationStart, '无法定位源文件选择校验函数')
const sourceValidationSource = sourceUploadWorkerSource.slice(sourceValidationStart, sourceValidationEnd + 2)
assert.doesNotMatch(sourceValidationSource, /file\.name === raw\.name[\s\S]{0,160}file\.size === raw\.size|同名且同大小/, '不同内容但同名同大小的文件仍会被前端误拒绝')
assert.match(sourceValidationSource, /selectedFiles\.length >= MAX_SOURCE_FILE_COUNT/, '移除伪重复校验时误删了文件数量限制')
assert.match(sourceValidationSource, /selectedBytes \+ raw\.size > MAX_SOURCE_BATCH_BYTES/, '移除伪重复校验时误删了批次大小限制')
assert.doesNotMatch(sourceUploadWorkerSource, /job\.file\.arrayBuffer\(|new TextDecoder/, '上传前仍把整个文本文件读入浏览器内存')
assert.match(sourceUploadWorkerSource, /export async function loadCanonicalSourceContent[\s\S]*?offset,[\s\S]*?limit: SOURCE_CONTENT_PAGE_CHARS/, '服务端 canonical content 没有按有界字符窗口读取')
assert.match(sourceUploadWorkerSource, /pending\.content = await loadCanonicalSourceContent\(currentTaskId, source\.id\)/, '上传成功后没有统一使用服务端 canonical content')
assert.doesNotMatch(sourceUploadWorkerSource, /\brawFile:\s*job\.file\b/, '上传成功状态仍长期保留原始 File')
assert.doesNotMatch(typesSource, /\brawFile\??:\s*File\b/, '上传状态类型仍长期持有原始 File')
assert.doesNotMatch(viewSource, /\brawFile:\s*raw\b/, '待上传列表仍复制保存原始 File')
assert.match(apiSource, /params:\s*\{[\s\S]*?offset\?: number[\s\S]*?limit\?: number[\s\S]*?\}/, '正文 API 前端契约缺少字符窗口参数')
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%')
assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时')
@@ -609,7 +715,7 @@ assert.match(
/export interface DataProcessPreviewProgress[\s\S]*?workflow_step: DataProcessWorkflowStep[\s\S]*?preview_status: DataProcessPreviewStatus[\s\S]*?preview_progress: number[\s\S]*?preview_run_id/,
'后台切分进度契约缺少步骤、状态、进度或任务代次',
)
for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
for (const field of ['status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段：${field}`)
}
assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整')
@@ -853,13 +959,23 @@ const defaultStructuredPreprocess = defaultPreprocessValues(
)
assert.deepEqual(
defaultStructuredPreprocess,
['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
'结构化默认预处理配置不准确',
[],
'结构化新任务不应默认勾选预处理',
)
assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复')
const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions')
assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整')
assert.deepEqual(defaultUnstructuredPreprocess, [], '非结构化新任务不应默认勾选预处理')
assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复')
for (const field of ['preserveTables', 'preserveCodeBlocks', 'preserveLists']) {
assert.match(
stateSource,
new RegExp(`${field}:\\s*false`),
`非结构化预处理选项 ${field} 不应默认开启`,
)
}
assert.match(structuredOptionsSource, /默认不执行预处理，请按数据情况自行选择/, '结构化预处理缺少默认不勾选说明')
assert.match(unstructuredOptionsSource, /默认不执行预处理，请按文档情况自行选择/, '非结构化预处理缺少默认不勾选说明')
assert.doesNotMatch(unstructuredOptionsSource, /默认启用结构感知/, '非结构化预处理仍保留默认启用的误导文案')
const backendConfigStart = viewSource.indexOf('function toBackendConfig()')
const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart)
@@ -934,7 +1050,7 @@ for (const [field, fallback] of [
)
}
assert.match(regenerationSource, /getDataProcessTask\(sourceTaskId\.value\)/, '重新生成没有加载原任务')
assert.match(regenerationSource, /while \(true\)[\s\S]*?getDataProcessSourceContent[\s\S]*?has_more/, '重新生成没有分页加载完整源正文')
assert.match(regenerationSource, /loadCanonicalSourceContent\(taskId, file\.id\)/, '重新生成没有复用分页 canonical 正文加载器')
assert.match(regenerationSource, /getDataProcessPreview\(taskId, \{ page: 1, page_size: 500 \}\)[\s\S]*?for \(let page = 2; page <= pages;/, '重新生成没有分页加载全部现有切片')
assert.match(viewSource, /if \(hydrating\.value\) return/, '任务水合期间仍可能触发重置副作用')
assert.match(regenerationSource, /currentSignature !== originalPreviewConfigSignature\.value[\s\S]*?currentSignature === confirmedPreviewConfigSignature\.value/, '切分变更确认没有按原签名和已确认签名去重')
@@ -948,7 +1064,7 @@ assert.match(regenerationSource, /if \(regenerationPrepared\.value\) \{[\s\S]*?g
assert.match(regenerationSource, /regenerationPrepared\.value = true/, '重新生成提交成功后没有记录服务端已变更状态')
assert.match(regenerationSource, /hydrateWorkspace\(regeneratedTask, !regenerated\.preview_invalidated\)/, '重新生成没有按 preview_invalidated 决定保留或清空切片')
assert.match(regenerationSource, /重新生成配置已保存，但工作区恢复失败/, '重新生成配置已保存但水合失败时缺少可恢复错误状态')
assert.match(regenerationSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
assert.match(sourceUploadWorkerSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
assert.doesNotMatch(regenerationSource, /binaryDocument[\s\S]*?mapDataProcessSourceFile\(file, ''\)/, '二进制源正文加载失败时不能静默降级为空内容')
assert.match(nextFromModelSource, /if \(isRegeneration\.value\) \{[\s\S]*?prepareRegeneration\(taskPayload\(\)\)/, '重新生成每次从模型步骤继续时没有调用专用接口')
assert.doesNotMatch(nextFromModelSource, /isRegeneration\.value && !taskId\.value/, '重新生成提交一次后可能错误转为普通任务更新')
@@ -1017,6 +1133,17 @@ for (const mutationFunction of [
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
}
const updatePreviewContentStart = viewSource.indexOf('function updatePreviewContent(')
const updatePreviewContentEnd = viewSource.indexOf('\n}', updatePreviewContentStart)
const updatePreviewContentSource = viewSource.slice(updatePreviewContentStart, updatePreviewContentEnd + 2)
assert.match(updatePreviewContentSource, /isManualPreviewItem\(item\)/, '编辑预览内容仍未按稳定来源信息区分手动项')
assert.doesNotMatch(updatePreviewContentSource, /sourceStart == null/, '结构化来源记录编辑后仍会被误标为手动项')
const restorePreviewItemStart = viewSource.indexOf('function restorePreviewItem(')
const restorePreviewItemEnd = viewSource.indexOf('\n}', restorePreviewItemStart)
const restorePreviewItemSource = viewSource.slice(restorePreviewItemStart, restorePreviewItemEnd + 2)
assert.match(restorePreviewItemSource, /isManualPreviewItem\(item\)/, '恢复预览内容没有使用统一的手动项判定')
assert.doesNotMatch(restorePreviewItemSource, /sourceStart == null/, '结构化来源记录仍因缺少字符偏移而无法恢复')
assert.match(previewSource, /v-if="!isManualPreviewItem\(editingItem\)"/, '结构化来源记录的恢复原文按钮仍被错误隐藏')
assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责')
function findNextStyleBlockStart(source, startIndex) {

20
frontend/src/api/modules/dataProcess.ts
View File

@@ -14,6 +14,8 @@ import type {
DataProcessProgress,
DataProcessRegeneratePayload,
DataProcessRegenerateResult,
DataProcessRepeatPayload,
DataProcessRepeatResult,
DataProcessPublishPayload,
DataProcessPublishResult,
DataProcessQualityScore,
@@ -56,6 +58,8 @@ export type {
DataProcessProgress,
DataProcessRegeneratePayload,
DataProcessRegenerateResult,
DataProcessRepeatPayload,
DataProcessRepeatResult,
DataProcessPublishPayload,
DataProcessPublishResult,
DataProcessQualityScore,
@@ -117,6 +121,15 @@ export const regenerateDataProcessTask = (
payload,
)
export const repeatDataProcessTask = (
taskId: string | number,
payload: DataProcessRepeatPayload,
) => post<DataProcessRepeatResult>(
`/data-process/${encodeURIComponent(taskId)}/repeat`,
payload,
{ timeout: 5 * 60 * 1000 },
)
export const deleteDataProcessTask = (taskId: string | number) =>
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
@@ -150,7 +163,12 @@ export const deleteDataProcessSourceFile = (taskId: string | number, fileId: str
export const getDataProcessSourceContent = (
taskId: string | number,
fileId: string | number,
params: { start_line?: number; line_count?: number } = {},
params: {
start_line?: number
line_count?: number
offset?: number
limit?: number
} = {},
) => get<DataProcessSourceContent>(
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/content`,
params,

33
frontend/src/types/dataProcess.ts
View File

@@ -99,6 +99,20 @@ export interface DataProcessRegenerateResult {
published_outputs_preserved: boolean
}
export interface DataProcessRepeatPayload {
expected_updated_at: string
request_id: string
}
export interface DataProcessRepeatResult {
task: DataProcessTask
source_task_id: string
created: boolean
copied_source_file_count: number
copied_preview_count: number
progress: DataProcessProgress
}
export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload>
export interface DataProcessSourceFile {
@@ -232,6 +246,22 @@ export interface DataProcessPreviewItem {
updated_at?: string
}
export type DataProcessSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
export interface DataProcessSourceLocator {
kind: DataProcessSourceLocatorKind
record_index?: number | null
start_line?: number | null
end_line?: number | null
source_start?: number | null
source_end?: number | null
json_pointer?: string | null
sheet_index?: number | null
sheet_name?: string | null
row_number?: number | null
sheet_record_index?: number | null
}
export interface DataProcessPreviewBuildPayload {
replace_existing?: true
source_file_ids?: Array<string | number>
@@ -369,6 +399,9 @@ export interface DataProcessQualityScore {
is_valid?: boolean
flags?: string[]
fingerprint?: string
source_pages?: number[]
heading_path?: string[]
source_locator?: DataProcessSourceLocator
[key: string]: unknown
}

54
frontend/src/views/data-process/DataProcessCreateView.vue
View File

@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel'
import {
createDefaultStructuredOptions,
createDefaultUnstructuredOptions,
@@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
useDataProcessSourceUpload,
validateSourceFileSelection,
@@ -33,7 +34,6 @@ import {
deleteDataProcessPreview,
deleteDataProcessSourceFile,
getDataProcessPreview,
getDataProcessSourceContent,
pullDataProcessExternalSource,
testDataProcessExternalSource,
updateDataProcessPreview,
@@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false)
let allowLeave = false
const {
bulkRegeneration,
canReturnFromGeneration,
generation,
generationStarting,
regeneratingResultId,
resultRegenerationBusy,
results,
@@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => {
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
return 'fa-arrow-right'
})
const previousStepLabel = computed(() => currentStep.value > 0
? WIZARD_STEPS[currentStep.value - 1].title
: '')
@@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload {
}
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
const sourceLocator = item.quality_score?.source_locator
return {
id: String(item.id),
sourceFileId: String(item.source_file_id),
originalContent: item.original_content,
editedContent: item.edited_content,
savedEditedContent: item.edited_content,
sourceStart: item.source_start,
sourceEnd: item.source_end,
sourceStartLine: item.source_start_line,
sourceEndLine: item.source_end_line,
sourceStart: item.source_start ?? sourceLocator?.source_start ?? null,
sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null,
sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null,
sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null,
tokenCount: item.token_count,
status: item.status,
sourcePages: Array.isArray(item.quality_score?.source_pages)
? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
: [],
sourceLocator,
headingPath: Array.isArray(item.quality_score?.heading_path)
? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string')
: [],
updatedAt: item.updated_at,
}
}
@@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) {
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
uploadedFiles.value.push({
uid: localUid,
rawFile: raw,
name: raw.name,
size: raw.size,
count: 0,
@@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) {
previewProgress: 0,
})
dirty.value = true
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
enqueueSourceUpload({ uid: localUid, file: raw })
}
async function useSampleFile() {
@@ -488,11 +493,8 @@ async function handlePullData() {
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
const newFiles: UploadedDataFile[] = []
for (const file of response.files) {
const source = await getDataProcessSourceContent(taskId.value, file.id, {
start_line: 1,
line_count: 5000,
})
newFiles.push(mapDataProcessSourceFile(file, source.content))
const content = await loadCanonicalSourceContent(taskId.value, file.id)
newFiles.push(mapDataProcessSourceFile(file, content))
}
uploadedFiles.value.push(...newFiles)
externalConnected.value = true
@@ -734,9 +736,14 @@ function selectPreviewItem(id: string) {
function updatePreviewContent(id: string, value: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item) return
const isManual = isManualPreviewItem(item)
item.editedContent = value
item.tokenCount = estimateTokenCount(value)
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
item.status = !value.trim()
? 'invalid'
: value === item.originalContent
? 'original'
: isManual ? 'manual' : 'modified'
resetDownstream()
dirty.value = true
}
@@ -758,7 +765,7 @@ async function syncPreviewChanges() {
function restorePreviewItem(id: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item || item.sourceStart == null) return
if (!item || isManualPreviewItem(item)) return
item.editedContent = item.originalContent
item.tokenCount = estimateTokenCount(item.originalContent)
item.status = 'original'
@@ -897,7 +904,7 @@ async function handleBack() {
ElMessage.warning('请等待当前文件切分完成')
return
}
if (currentStepId.value === 'generate') return
if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return
if (currentStep.value > 0) {
const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id
if (!targetStep) return
@@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() {
if (sourceTask.status === 'running') resumeStep = 'generate'
if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload'
if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate'
if (resumeStep === 'generate' || resumeStep === 'results') {
const resume = resumeGeneration()
goToStep(resumeStep)
await resume
return
}
goToStep(resumeStep)
if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration()
}
onBeforeUnmount(() => {
@@ -1154,7 +1166,7 @@ onMounted(() => {
<div class="footer-left">
<el-button
v-if="currentStep > 0"
:disabled="currentStepId === 'generate' || previewBuilding || sourceUploading"
:disabled="(currentStepId === 'generate' && !canReturnFromGeneration) || previewBuilding || sourceUploading"
@click="handleBack"
>
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回：{{ previousStepLabel }}
@@ -1167,8 +1179,8 @@ onMounted(() => {
<el-button
class="wizard-primary-action"
type="primary"
:loading="modelSubmitLoading || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

161
frontend/src/views/data-process/DataProcessDetailView.vue
View File

@@ -9,6 +9,7 @@ import {
getDataProcessResults,
getDataProcessTask,
publishDataProcess,
repeatDataProcessTask,
restoreDataProcessResult,
updateDataProcessResult,
} from '@/api/modules/dataProcess'
@@ -42,6 +43,8 @@ const savingResult = ref(false)
const restoringResultId = ref<string | number | null>(null)
const publishDialogVisible = ref(false)
const publishing = ref(false)
const repeatGenerating = ref(false)
const repeatRequestId = ref('')
const configExpanded = ref(false)
const resultCellTooltipOptions = {
popperClass: 'data-process-result-tooltip',
@@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record<string, string> = {
preserve_context: '保留上下文',
}
const structuredPreprocessOptionKeys = new Set([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
function formatStructuredPreprocessOptions(value: unknown[]) {
const options = [...new Set(value.map((item) => String(item)))]
const selected = new Set(options)
const consumed = new Set<string>()
const labels: string[] = []
function appendGroup(values: string[], groupLabel: string) {
const selectedValues = values.filter((item) => selected.has(item))
selectedValues.forEach((item) => consumed.add(item))
if (selectedValues.length === values.length) {
labels.push(groupLabel)
return
}
selectedValues.forEach((item) => {
labels.push(`${preprocessOptionLabelMap[item] || item}（历史部分配置）`)
})
}
appendGroup(['clean_invalid', 'deduplicate'], '数据清洗')
appendGroup(['detect_structure', 'normalize_format'], '结构标准化')
if (selected.has('desensitize')) {
consumed.add('desensitize')
labels.push('敏感信息脱敏')
}
if (selected.has('filter_anomaly')) {
consumed.add('filter_anomaly')
labels.push('异常数据过滤（历史规则）')
}
options.forEach((item) => {
if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item)
})
return labels.length ? labels.join('、') : '-'
}
function numeric(value: unknown) {
const parsed = typeof value === 'number' ? value : Number(value)
return Number.isFinite(parsed) ? parsed : 0
@@ -199,6 +247,11 @@ const canRegenerate = computed(() => {
|| status === 'stopped'
|| (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value))
})
const canRepeatGeneration = computed(() => (
detail.value?.status === 'completed'
&& detail.value.results_confirmed !== false
&& previewCount.value > 0
))
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
@@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) {
}
if (Array.isArray(value)) {
if (key === 'preprocess_options') {
return value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
const containsStructuredOption = value.some((item) => (
structuredPreprocessOptionKeys.has(String(item))
))
return containsStructuredOption
? formatStructuredPreprocessOptions(value)
: value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
}
return value.length ? value.join('、') : '-'
}
@@ -503,6 +561,48 @@ function startRegeneration() {
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
}
function createRepeatRequestId() {
if (typeof globalThis.crypto?.randomUUID === 'function') {
return globalThis.crypto.randomUUID()
}
return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
}
async function repeatGeneration() {
if (!detail.value?.updated_at || repeatGenerating.value) return
try {
await ElMessageBox.confirm(
'系统会复制当前配置、源文件和切分结果，创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。',
'按原配置再生成一批？',
{
confirmButtonText: '创建并开始生成',
cancelButtonText: '取消',
type: 'info',
},
)
} catch {
return
}
repeatGenerating.value = true
repeatRequestId.value ||= createRepeatRequestId()
try {
const repeated = await repeatDataProcessTask(taskId.value, {
expected_updated_at: detail.value.updated_at,
request_id: repeatRequestId.value,
})
ElMessage.success(repeated.created ? '已创建新任务，正在后台生成' : '已恢复此前创建的新任务')
await router.push({
name: 'data-process-workflow',
params: { id: repeated.task.id },
})
} catch {
// 保留幂等请求 ID；网络超时后再次点击不会重复创建任务。
} finally {
repeatGenerating.value = false
}
}
watch([currentPage, pageSize], () => void loadResults())
onMounted(loadPage)
@@ -520,22 +620,32 @@ onBeforeUnmount(() => {
<el-tag :type="displayStatus.type" size="small" effect="light">
{{ displayStatus.label }}
</el-tag>
<el-button
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
class="publish-button"
type="primary"
@click="openPublishDialog"
>
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
</el-button>
<el-button
v-if="canRegenerate"
class="publish-button"
type="primary"
@click="startRegeneration"
>
<i class="fa fa-refresh" style="margin-right: 4px;" />重新生成
</el-button>
<div class="heading-actions">
<el-button
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
type="primary"
@click="openPublishDialog"
>
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
</el-button>
<el-button
v-if="canRepeatGeneration"
type="primary"
:loading="repeatGenerating"
:disabled="repeatGenerating"
@click="repeatGeneration"
>
<i class="fa fa-clone" style="margin-right: 4px;" />按原配置再生成一批
</el-button>
<el-button
v-if="canRegenerate"
type="warning"
plain
@click="startRegeneration"
>
<i class="fa fa-refresh" style="margin-right: 4px;" />覆盖当前任务重新生成
</el-button>
</div>
</div>
<p>{{ detail.description || '暂无任务描述' }}</p>
<dl class="heading-meta">
@@ -804,7 +914,15 @@ onBeforeUnmount(() => {
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
}
.publish-button { margin-left: auto; }
.heading-actions {
margin-left: auto;
display: flex;
flex-wrap: wrap;
justify-content: flex-end;
gap: 8px;
:deep(.el-button + .el-button) { margin-left: 0; }
}
.load-state-actions { display: flex; gap: 10px; }
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
@@ -968,7 +1086,8 @@ onBeforeUnmount(() => {
@media (max-width: 720px) {
.metric-grid, .config-grid { grid-template-columns: 1fr; }
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
.publish-button { width: 100%; margin-left: 0; }
.heading-actions { width: 100%; margin-left: 0; }
.heading-actions :deep(.el-button) { width: 100%; }
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
.result-toolbar { align-items: stretch; flex-direction: column; }
.result-filters { padding: 0 16px 16px; flex-direction: column; }

56
frontend/src/views/data-process/create/OfficeSourceViewer.vue
View File

@@ -46,6 +46,13 @@ const sourceUrl = computed(() => (
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
: ''
))
const selectedXlsxLocator = computed(() => {
const locator = props.selectedItem?.sourceLocator
if (!locator) return null
const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name)
const hasRow = locator.row_number != null || locator.sheet_record_index != null
return hasSheet && hasRow ? locator : null
})
const visibleRowRange = computed(() => {
const sheet = xlsxPreview.value?.active_sheet
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
@@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => {
})
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
const locator = selectedXlsxLocator.value
const sheet = xlsxPreview.value?.active_sheet
if (locator && sheet) {
const sheetMatches = locator.sheet_index != null
? sheet.index === locator.sheet_index
: sheet.name === locator.sheet_name
if (!sheetMatches) return false
if (locator.row_number != null) return row.row_number === locator.row_number
return row.record_index === locator.sheet_record_index
}
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
}
@@ -119,8 +136,11 @@ async function locateSelectedItem() {
async function loadPreview(options: { reset?: boolean } = {}) {
const sequence = ++loadSequence
if (options.reset) {
activeSheetIndex.value = 0
pageOffset.value = 0
const locator = selectedXlsxLocator.value
activeSheetIndex.value = locator?.sheet_index ?? 0
pageOffset.value = locator?.sheet_record_index == null
? 0
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
preview.value = null
}
errorMessage.value = ''
@@ -178,8 +198,34 @@ watch(
)
watch(
() => props.selectedItem?.id,
() => void locateSelectedItem(),
() => [
props.selectedItem?.id,
props.selectedItem?.sourceLocator?.sheet_index,
props.selectedItem?.sourceLocator?.sheet_record_index,
props.selectedItem?.sourceLocator?.row_number,
],
() => {
const locator = selectedXlsxLocator.value
if (!locator || isDocx.value) {
void locateSelectedItem()
return
}
const targetSheet = locator.sheet_index ?? activeSheetIndex.value
const targetOffset = locator.sheet_record_index == null
? pageOffset.value
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
const activeSheet = xlsxPreview.value?.active_sheet
if (
activeSheet?.index === targetSheet
&& activeSheet.offset === targetOffset
) {
void locateSelectedItem()
return
}
activeSheetIndex.value = targetSheet
pageOffset.value = targetOffset
void loadPreview()
},
)
</script>
@@ -301,6 +347,8 @@ watch(
:key="row.row_number"
class="xlsx-row"
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
:data-row-number="row.row_number"
:data-record-index="row.record_index"
>
<th class="row-number-cell">{{ row.row_number }}</th>
<td

176
frontend/src/views/data-process/create/PreviewCompareStep.vue
View File

@@ -2,7 +2,11 @@
import { computed, nextTick, ref, watch } from 'vue'
import OfficeSourceViewer from './OfficeSourceViewer.vue'
import PdfSourceViewer from './PdfSourceViewer.vue'
import { sourceLines } from './previewModel'
import {
isManualPreviewItem,
sourceLineNumberAtOffset,
sourceLineWindow,
} from './previewModel'
import type { PreviewItem, ProcessType } from './types'
const props = defineProps<{
@@ -31,9 +35,11 @@ const sourceViewerRef = ref<HTMLElement | null>(null)
const search = ref('')
const currentPage = ref(1)
const PREVIEW_PAGE_SIZE = 10
const SOURCE_LINE_RENDER_LIMIT = 240
const SOURCE_LINE_CHARACTER_LIMIT = 4_000
const sourceWindowStartLine = ref(1)
const editingItemId = ref<string | null>(null)
const editorDraft = ref('')
const lines = computed(() => sourceLines(props.sourceText))
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
const normalizedFileFormat = computed(() => (
@@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => (
))
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
const selectedSourceOffset = computed(() => {
const item = selectedItem.value
return item ? sourceOffsetRange(item)?.start ?? null : null
})
const selectedSourceLine = computed(() => {
const item = selectedItem.value
if (!item) return null
return sourceLineRange(item)?.start
?? (selectedSourceOffset.value == null
? null
: sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value))
})
const visibleSourceWindow = computed(() => sourceLineWindow(
props.sourceText,
sourceWindowStartLine.value,
SOURCE_LINE_RENDER_LIMIT,
SOURCE_LINE_CHARACTER_LIMIT,
selectedSourceLine.value,
selectedSourceOffset.value,
))
const lines = computed(() => visibleSourceWindow.value.lines)
const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !search.value.trim()
@@ -58,10 +85,27 @@ const pagedItems = computed(() => {
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
function isLineHighlighted(lineStart: number, lineEnd: number) {
function sourceLineRange(item: PreviewItem) {
const start = item.sourceLocator?.start_line ?? item.sourceStartLine
const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start
return start == null ? null : { start, end: end ?? start }
}
function sourceOffsetRange(item: PreviewItem) {
const start = item.sourceLocator?.source_start ?? item.sourceStart
const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start
return start == null ? null : { start, end: Math.max(start, end ?? start) }
}
function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) {
const item = selectedItem.value
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd
if (!item) return false
const lineRange = sourceLineRange(item)
if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end
const offsetRange = sourceOffsetRange(item)
if (!offsetRange) return false
const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end)
return lineEnd >= offsetRange.start && lineStart < effectiveEnd
}
function selectItem(id: string) {
@@ -107,34 +151,88 @@ watch(search, () => {
watch(() => props.selectedFileId, closeEditor)
watch(selectedItem, async (item) => {
watch([selectedItem, () => props.sourceText], async ([item]) => {
if (!item) return
const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id)
if (visibleIndex >= 0) {
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
}
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
if (isPdfSource.value || isOfficeSource.value) return
const itemLineRange = sourceLineRange(item)
const itemOffsetRange = sourceOffsetRange(item)
if (!itemLineRange && !itemOffsetRange) {
sourceWindowStartLine.value = 1
return
}
const targetLine = selectedSourceLine.value
?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0)
sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3))
await nextTick()
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
const exactTarget = sourceViewerRef.value
?.querySelector<HTMLElement>(`[data-line-number="${targetLine}"]`)
const target = exactTarget
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
target?.scrollIntoView({ block: 'center', behavior: 'smooth' })
}, { immediate: true })
async function showPreviousSourceWindow() {
sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT)
await nextTick()
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
}
async function showNextSourceWindow() {
if (!visibleSourceWindow.value.hasMore) return
sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1
await nextTick()
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
}
function itemNumber(item: PreviewItem) {
return props.items.findIndex((entry) => entry.id === item.id) + 1
}
function lineRange(item: PreviewItem) {
if (item.sourcePages?.length) {
const first = item.sourcePages[0]
const last = item.sourcePages[item.sourcePages.length - 1]
return first === last ? `来源：第 ${first} 页` : `来源：第 ${first}–${last} 页`
if (isManualPreviewItem(item)) return '手动新增，无源文件定位'
const locator = item.sourceLocator
const locatedLines = sourceLineRange(item)
if (props.processType === 'unstructured') {
const parts: string[] = []
if (item.sourcePages?.length) {
const first = item.sourcePages[0]
const last = item.sourcePages[item.sourcePages.length - 1]
parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`)
}
if (locatedLines) {
parts.push(
locatedLines.start === locatedLines.end
? `第 ${locatedLines.start} 行`
: `第 ${locatedLines.start}–${locatedLines.end} 行`,
)
}
if (item.headingPath?.length) parts.push(`章节：${item.headingPath.join(' / ')}`)
return parts.length ? `来源：${parts.join(' · ')}` : '来源：源文件内容（无精确定位）'
}
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增，无源文件定位'
return item.sourceStartLine === item.sourceEndLine
? `来源：第 ${item.sourceStartLine} 行`
: `来源：第 ${item.sourceStartLine}–${item.sourceEndLine} 行`
if (locator?.kind === 'xlsx') {
const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}`
return locator.row_number != null
? `来源：${sheet} · 第 ${locator.row_number} 行`
: `来源：${sheet}`
}
if (locator?.kind === 'json') {
return locator.json_pointer
? `来源：JSON 路径 ${locator.json_pointer}`
: '来源：JSON 根对象'
}
if (locatedLines) {
return locatedLines.start === locatedLines.end
? `来源：第 ${locatedLines.start} 行`
: `来源：第 ${locatedLines.start}–${locatedLines.end} 行`
}
return '来源：源文件记录'
}
</script>
@@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
<div>
<strong>源文件 · {{ fileName }}</strong>
</div>
<div
v-if="!isPdfSource && !isOfficeSource && lines.length"
class="source-window-controls"
aria-label="源文件行窗口"
>
<span>第 {{ visibleSourceWindow.startLine }}–{{ visibleSourceWindow.endLine }} 行</span>
<el-button
link
size="small"
aria-label="查看上一段源文件"
:disabled="!visibleSourceWindow.hasPrevious"
@click="showPreviousSourceWindow"
>
上一段
</el-button>
<el-button
link
size="small"
aria-label="查看下一段源文件"
:disabled="!visibleSourceWindow.hasMore"
@click="showNextSourceWindow"
>
下一段
</el-button>
</div>
</div>
<PdfSourceViewer
@@ -197,8 +320,9 @@ function lineRange(item: PreviewItem) {
v-for="line in lines"
:key="line.number"
class="source-line"
:class="{ 'is-highlighted': isLineHighlighted(line.start, line.end) }"
:class="{ 'is-highlighted': isLineHighlighted(line.number, line.start, line.end) }"
:data-source-start="line.start"
:data-line-number="line.number"
>
<span class="line-number">{{ line.number }}</span>
<span class="line-content">{{ line.content || ' ' }}</span>
@@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) {
/>
<div class="editor-actions">
<el-button
v-if="editingItem.sourceStart != null"
v-if="!isManualPreviewItem(editingItem)"
link
@click="restoreItem"
>
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) {
}
}
.source-window-controls {
flex: none;
gap: 2px !important;
> span {
margin-right: 4px;
color: #8a93a3;
font-size: 11px;
white-space: nowrap;
}
:deep(.el-button) {
margin-left: 0;
}
}
.source-viewer {
flex: 1;
height: 538px;

94
frontend/src/views/data-process/create/StructuredOptionsPanel.vue
View File

@@ -1,4 +1,5 @@
<script setup lang="ts">
import { computed } from 'vue'
import type {
GenerationControlOptions,
PreprocessOption,
@@ -21,27 +22,32 @@ const emit = defineEmits<{
'update:options': [value: StructuredProcessOptions]
}>()
const PREPROCESS_OPTIONS: Array<{
value: PreprocessOption
const PREPROCESS_GROUPS: Array<{
values: PreprocessOption[]
label: string
description: string
}> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列，并剔除关键字段残缺的数据行' },
{
value: 'detect_structure',
label: '嵌套结构展平',
description: '展平嵌套对象和可解析的 JSON 字段；Excel 表头与合并单元格在上传时自动解析',
values: ['clean_invalid', 'deduplicate'],
label: '数据清洗',
description: '清理全空列和空记录，并删除内容完全相同的记录；不会猜测可空字段是否必填',
},
{
value: 'deduplicate',
label: '重复记录去重',
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重，暂不支持自定义组合字段',
values: ['detect_structure', 'normalize_format'],
label: '结构标准化',
description: '展平嵌套对象和可解析的 JSON 字段，并统一编码、空白、字段名和 JSON 序列化格式',
},
{
values: ['desensitize'],
label: '敏感信息脱敏',
description: '识别并脱敏姓名、手机号、邮箱和身份证号',
},
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值，并过滤乱码等异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
]
const legacyAnomalyFilterEnabled = computed(() => (
props.options.preprocessOptions.includes('filter_anomaly')
))
function updateField<K extends keyof StructuredProcessOptions>(
field: K,
value: StructuredProcessOptions[K],
@@ -57,14 +63,26 @@ function updateQaPairsPerRow(value: number | undefined) {
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = Array.from(new Set(value.filter(
(option): option is PreprocessOption => (
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
),
)))
updateField('preprocessOptions', preprocessOptions)
function selectedCount(values: PreprocessOption[]) {
return values.filter((value) => props.options.preprocessOptions.includes(value)).length
}
function groupSelected(values: PreprocessOption[]) {
return selectedCount(values) === values.length
}
function groupIndeterminate(values: PreprocessOption[]) {
const count = selectedCount(values)
return count > 0 && count < values.length
}
function updatePreprocessGroup(values: PreprocessOption[], checked: string | number | boolean) {
const next = new Set(props.options.preprocessOptions)
values.forEach((value) => {
if (Boolean(checked)) next.add(value)
else next.delete(value)
})
updateField('preprocessOptions', [...next])
}
</script>
@@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>选择在生成问答对之前需要执行的数据处理方式</p>
<p>默认不执行预处理，请按数据情况自行选择</p>
</div>
</div>
<el-checkbox-group
:model-value="options.preprocessOptions"
class="preprocess-option-grid"
@update:model-value="updatePreprocessOptions"
>
<el-checkbox
v-for="option in PREPROCESS_OPTIONS"
:key="option.value"
:value="option.value"
<div class="preprocess-option-grid">
<label
v-for="group in PREPROCESS_GROUPS"
:key="group.label"
class="preprocess-option"
:class="{ 'is-checked': groupSelected(group.values) }"
>
<el-checkbox
:model-value="groupSelected(group.values)"
:indeterminate="groupIndeterminate(group.values)"
@update:model-value="updatePreprocessGroup(group.values, $event)"
/>
<span class="preprocess-option-copy">
<strong>{{ option.label }}</strong>
<small>{{ option.description }}</small>
<strong>{{ group.label }}</strong>
<small>{{ group.description }}</small>
</span>
</el-checkbox>
</el-checkbox-group>
</label>
</div>
<el-alert
v-if="legacyAnomalyFilterEnabled"
class="legacy-preprocess-alert"
type="warning"
:closable="false"
title="该历史任务仍启用了已停用的“异常数据过滤”；为保证结果可复现，本次继续保留"
/>
</div>
<div class="form-section generation-options-section">

2
frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue
View File

@@ -119,7 +119,7 @@ defineExpose({ revealValidation })
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>默认启用结构感知的推荐策略，只需决定是否需要脱敏</p>
<p>默认不执行预处理，请按文档情况自行选择</p>
</div>
</div>
<div class="preprocess-option-grid">

29
frontend/src/views/data-process/create/dataProcessCreateState.ts
View File

@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
export function createDefaultStructuredOptions(): StructuredProcessOptions {
return {
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
preprocessOptions: [],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
return {
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
preprocessOptions: [],
chunkMethod: 'layout_hybrid',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
semanticBreakpointPercentile: 95,
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
const defaults = createDefaultStructuredOptions()
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
const supportedPreprocessOptions = new Set<PreprocessOption>([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
return {
...defaults,
...generationOptionsFromConfig(config, defaults),
preprocessOptions: Array.isArray(preprocessOptions)
? preprocessOptions.map(String) as PreprocessOption[]
? Array.from(new Set(preprocessOptions.map(String).filter(
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
)))
: defaults.preprocessOptions,
semanticEnrichment: Boolean(configValue(
config,

148
frontend/src/views/data-process/create/previewModel.ts
View File

@@ -1,4 +1,4 @@
import type { SourceLine } from './types'
import type { PreviewItem, SourceLine } from './types'
/** 仅用于“使用示例”上传；正式预览和切片全部由后端生成。 */
export const DEFAULT_SOURCE_TEXT = [
@@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [
'答：复利是将上一期利息加入本金，再计算下一期利息。',
].join('\n')
/**
* 把后端返回的字符偏移映射为源文件行，仅负责界面高亮，不参与切片。
*/
export function sourceLines(sourceText: string): SourceLine[] {
const rawLines = sourceText.split('\n')
let cursor = 0
export interface SourceLineWindow {
lines: SourceLine[]
startLine: number
endLine: number
hasPrevious: boolean
hasMore: boolean
}
return rawLines.map((content, index) => {
const start = cursor
const end = start + content.length
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
return { number: index + 1, content, start, end }
})
function unicodeCodePointLength(value: string, start = 0, end = value.length) {
let length = 0
let index = start
while (index < end) {
const codePoint = value.codePointAt(index)
index += codePoint != null && codePoint > 0xffff ? 2 : 1
length += 1
}
return length
}
function advanceCodePoints(value: string, start: number, end: number, count: number) {
let index = start
let remaining = Math.max(0, count)
while (index < end && remaining > 0) {
const codePoint = value.codePointAt(index)
index += codePoint != null && codePoint > 0xffff ? 2 : 1
remaining -= 1
}
return index
}
/**
* 只扫描并返回当前可见行窗口，不对全文 split，避免大文件生成巨量字符串数组。
* 字符定位场景可开启 code point 偏移，以与后端 Python 的字符计数保持一致。
*/
export function sourceLineWindow(
sourceText: string,
requestedStartLine: number,
maxLines: number,
maxCharactersPerLine: number,
focusLine: number | null = null,
focusOffset: number | null = null,
): SourceLineWindow {
const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1)
const limit = Math.max(1, Math.trunc(maxLines) || 1)
const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1)
const trackUnicodeOffsets = focusOffset != null
const lines: SourceLine[] = []
let lineNumber = 1
let jsCursor = 0
let sourceCursor = 0
while (jsCursor <= sourceText.length && lineNumber < startLine) {
const newlineIndex = sourceText.indexOf('\n', jsCursor)
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
sourceCursor = trackUnicodeOffsets
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0)
: (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1)
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
lineNumber += 1
}
while (jsCursor <= sourceText.length && lines.length < limit) {
const newlineIndex = sourceText.indexOf('\n', jsCursor)
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
const fullSourceEnd = trackUnicodeOffsets
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd)
: jsEnd
const focusedStart = focusLine === lineNumber && focusOffset != null
? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3))
: sourceCursor
const segmentSourceStart = Math.min(
focusedStart,
Math.max(sourceCursor, fullSourceEnd - characterLimit),
)
const relativeSegmentStart = trackUnicodeOffsets
? segmentSourceStart - sourceCursor
: Math.max(0, segmentSourceStart - jsCursor)
const segmentJsStart = advanceCodePoints(
sourceText,
jsCursor,
jsEnd,
relativeSegmentStart,
)
const segmentJsEnd = advanceCodePoints(
sourceText,
segmentJsStart,
jsEnd,
characterLimit,
)
const segmentLength = trackUnicodeOffsets
? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd)
: segmentJsEnd - segmentJsStart
const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart
const end = start + segmentLength
const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}`
lines.push({ number: lineNumber, content, start, end })
sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0)
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
lineNumber += 1
}
return {
lines,
startLine: lines[0]?.number ?? startLine,
endLine: lines[lines.length - 1]?.number ?? startLine,
hasPrevious: startLine > 1,
hasMore: jsCursor <= sourceText.length,
}
}
/** 根据后端 code point 偏移查找物理行号，不构建全文行数组。 */
export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) {
const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0)
let offset = 0
let lineNumber = 1
for (const character of sourceText) {
if (offset >= normalizedOffset) break
if (character === '\n') lineNumber += 1
offset += 1
}
return lineNumber
}
/**
* 手动新增项可能先以空内容保存为 invalid，编辑后又由后端标记为 modified，
* 因此不能只依赖可变的 status；空原文且完全没有来源定位才是稳定兜底。
*/
export function isManualPreviewItem(item: PreviewItem): boolean {
const hasSourceLocation = item.sourceStart != null
|| item.sourceEnd != null
|| item.sourceStartLine != null
|| item.sourceEndLine != null
|| Boolean(item.sourcePages?.length)
|| Boolean(item.sourceLocator)
return item.status === 'manual' || (!item.originalContent && !hasSourceLocation)
}
/** 与后端预览 token 估算规则一致，仅用于编辑中的即时计数。 */

20
frontend/src/views/data-process/create/types.ts
View File

@@ -24,6 +24,7 @@ export type PreprocessOption =
| 'detect_structure'
| 'deduplicate'
| 'normalize_format'
/** 仅用于恢复历史任务，新任务界面不再提供。 */
| 'filter_anomaly'
| 'desensitize'
@@ -94,7 +95,6 @@ export interface ExternalDataSource {
export interface UploadedDataFile {
uid: string | number
sourceFileId?: string
rawFile?: File
name: string
size: number
count: number
@@ -118,6 +118,22 @@ export interface SourceLine {
end: number
}
export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
export interface PreviewSourceLocator {
kind: PreviewSourceLocatorKind
record_index?: number | null
start_line?: number | null
end_line?: number | null
source_start?: number | null
source_end?: number | null
json_pointer?: string | null
sheet_index?: number | null
sheet_name?: string | null
row_number?: number | null
sheet_record_index?: number | null
}
export interface PreviewItem {
id: string
sourceFileId: string
@@ -129,6 +145,8 @@ export interface PreviewItem {
sourceStartLine: number | null
sourceEndLine: number | null
sourcePages?: number[]
sourceLocator?: PreviewSourceLocator
headingPath?: string[]
tokenCount: number
status: 'original' | 'modified' | 'manual' | 'invalid'
qualityScore?: number

23
frontend/src/views/data-process/create/useDataProcessGeneration.ts
View File

@@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
let generationTimer: ReturnType<typeof setTimeout> | null = null
let generationRun = 0
let pollFailureCount = 0
let generationStarting = false
const generationStarting = ref(false)
const generationRestoring = ref(false)
const canReturnFromGeneration = computed(() => (
generation.status === 'idle' && !generationStarting.value && !generationRestoring.value
))
function stopGenerationTimer() {
generationRun += 1
@@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
}
async function startGeneration() {
if (generationStarting || generation.status === 'running') return false
if (generationStarting.value || generation.status === 'running') return false
const taskId = bindings.taskId.value
if (!taskId) {
ElMessage.error('任务尚未创建，请返回上一步重试')
return false
}
generationStarting = true
generationStarting.value = true
let runId: number | null = null
try {
const canStart = await bindings.beforeGenerate?.()
@@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
generation.message = error instanceof Error ? error.message : '启动数据处理失败，请重试。'
return false
} finally {
generationStarting = false
generationStarting.value = false
}
}
async function resumeGeneration() {
const taskId = bindings.taskId.value
if (!taskId) return
stopGenerationTimer()
const activeRunId = generationRun
pollFailureCount = 0
generationRestoring.value = true
try {
stopGenerationTimer()
const activeRunId = generationRun
pollFailureCount = 0
const progress = await getDataProcessProgress(taskId)
if (activeRunId !== generationRun) return
if (progress.status === 'running') {
@@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
} catch (error) {
generation.status = 'failed'
generation.message = error instanceof Error ? error.message : '查询任务进度失败，请重试。'
} finally {
generationRestoring.value = false
}
}
@@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
return {
bulkRegeneration,
canReturnFromGeneration,
generation,
generationStarting,
regeneratingResultId,
resultRegenerationBusy,
results,

26
frontend/src/views/data-process/create/useDataProcessRegeneration.ts
View File

@@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
import { useRoute } from 'vue-router'
import {
getDataProcessPreview,
getDataProcessSourceContent,
getDataProcessTask,
regenerateDataProcessTask,
} from '@/api/modules/dataProcess'
@@ -15,7 +14,10 @@ import {
createStructuredOptionsFromConfig,
createUnstructuredOptionsFromConfig,
} from './dataProcessCreateState'
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
} from './useDataProcessSourceUpload'
import type {
PreviewItem,
ProcessType,
@@ -50,24 +52,6 @@ interface RegenerationBindings {
resetDownstream: () => void
}
async function loadSourceContent(taskId: string, fileId: string | number) {
const chunks: string[] = []
let startLine = 1
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
start_line: startLine,
line_count: 10_000,
})
chunks.push(source.content || '')
if (!source.has_more) break
const nextLine = Number(source.end_line || startLine) + 1
if (nextLine <= startLine) break
startLine = nextLine
}
// source_content_lines 已保留原始换行；分页之间直接拼接，避免凭空增加空行并破坏偏移。
return chunks.join('')
}
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
const items = [...first.items]
@@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) {
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
const taskId = String(task.id)
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id))
)))
bindings.previewItems.value = preservePreviews
? await loadAllPreviews(taskId, bindings.mapPreviewItem)

61
frontend/src/views/data-process/create/useDataProcessSourceUpload.ts
View File

@@ -6,7 +6,6 @@ import {
} from '@/api/modules/dataProcess'
import type { ProcessType, UploadedDataFile } from './types'
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
const MAX_SOURCE_FILE_COUNT = 20
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
interface SourceUploadJob {
uid: string
file: File
extension: string
}
interface SourceUploadOptions {
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
}
}
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
}
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
}
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
return { valid: true, extension }
}
function unicodeCodePointLength(value: string) {
let length = 0
for (const _character of value) length += 1
return length
}
/** 分页读取服务端保存的规范化正文，避免重新使用浏览器本地解码结果。 */
export async function loadCanonicalSourceContent(
taskId: string | number,
fileId: string | number,
) {
const chunks: string[] = []
let offset = 0
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
offset,
limit: SOURCE_CONTENT_PAGE_CHARS,
})
const content = source.content || ''
chunks.push(content)
if (!source.has_more) break
const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常，请删除文件后重试')
offset = nextOffset
}
return chunks.join('')
}
export function mapDataProcessSourceFile(
file: DataProcessSourceFile,
content = '',
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
pending.uploadError = undefined
try {
let content = ''
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
} catch {
throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试')
}
if (!content.trim()) throw new Error('不能上传空文件')
}
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
pending.uploadProgress = progress
})
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
// 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
Object.assign(pending, mapDataProcessSourceFile(source), {
rawFile: job.file,
status: 'uploading',
uploadProgress: 99,
})
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
start_line: 1,
line_count: 10_000,
})
pending.content = parsed.content
} catch {
// 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。
}
} else {
pending.content = content
try {
pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
} catch {
throw new Error('文件已上传，但服务端规范化内容读取失败，请删除文件后重试')
}
pending.status = 'ready'