From b975de02da956365bdf53e8bada37b801d8632eb Mon Sep 17 00:00:00 2001 From: caoxiaozhu Date: Thu, 30 Jul 2026 16:53:54 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=AE=8C=E5=96=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=A2=84=E5=A4=84=E7=90=86=E4=B8=8E=20JSON=20=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/endpoints/data_process.py | 275 +++++- .../app/modules/data_process/algorithms.py | 845 +++++++++++++++--- backend/app/modules/data_process/storage.py | 61 ++ backend/app/modules/data_process/store.py | 321 ++++++- backend/app/schemas/data_process.py | 13 + backend/tests/test_data_process_algorithms.py | 303 ++++++- backend/tests/test_data_process_api.py | 564 +++++++++++- backend/tests/test_data_process_storage.py | 34 + backend/tests/test_data_process_store.py | 176 ++++ .../regression-data-process-detail.mjs | 14 + .../regression-data-process-wizard.mjs | 187 +++- frontend/src/api/modules/dataProcess.ts | 20 +- frontend/src/types/dataProcess.ts | 33 + .../data-process/DataProcessCreateView.vue | 54 +- .../data-process/DataProcessDetailView.vue | 161 +++- .../create/OfficeSourceViewer.vue | 56 +- .../create/PreviewCompareStep.vue | 176 +++- .../create/StructuredOptionsPanel.vue | 94 +- .../create/UnstructuredOptionsPanel.vue | 2 +- .../create/dataProcessCreateState.ts | 29 +- .../views/data-process/create/previewModel.ts | 148 ++- .../src/views/data-process/create/types.ts | 20 +- .../create/useDataProcessGeneration.ts | 23 +- .../create/useDataProcessRegeneration.ts | 26 +- .../create/useDataProcessSourceUpload.ts | 61 +- 25 files changed, 3277 insertions(+), 419 deletions(-) diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 1500a64..0b52c1d 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -8,12 +8,14 @@ import os import re import socket import time +from collections.abc import Iterator, Mapping from concurrent.futures import ThreadPoolExecutor, as_completed from contextlib import contextmanager +from copy import deepcopy from dataclasses import asdict from pathlib import Path from threading import BoundedSemaphore, Lock -from typing import Any, Iterator, Literal +from typing import Any, Literal from urllib.parse import quote, urlsplit import httpx @@ -45,9 +47,10 @@ from app.modules.data_process.algorithms import ( is_near_duplicate, near_duplicate_fingerprint, parse_text_content, - preprocess_structured_records, + preprocess_structured_records_with_lineage, remove_document_noise, score_quality, + structured_json_dumps, ) from app.modules.data_process.document_chunking import ( DocumentChunk, @@ -75,9 +78,11 @@ from app.modules.data_process.store import ( NotFoundError, get_data_process_store, new_id, + repeat_task_id, ) from app.schemas.data_process import ( DataProcessRegenerateRequest, + DataProcessRepeatRequest, DataProcessStatus, DataProcessTaskCreate, DataProcessTaskUpdate, @@ -261,7 +266,14 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText: content = str(source.get("content") or "") file_format = str(source.get("file_format") or "").lower() if file_format == "xlsx": - # XLSX 上传阶段已安全解析为 JSONL 后入库。 + raw_content = source.get("raw_content") + if isinstance(raw_content, bytes): + return parse_text_content( + raw_content, + filename=str(source.get("name") or "source.xlsx"), + file_format="xlsx", + ) + # 兼容原始对象已缺失的历史文件：退化为上传阶段生成的 JSONL。 return parse_text_content(content, file_format="jsonl") if file_format in {"pdf", "docx", "pptx"}: # 文档上传阶段已抽取文本，预览阶段只需要对正文切片。 @@ -381,11 +393,12 @@ def _build_preview_items( seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {} items: list[dict[str, Any]] = [] - def append_item(item: dict[str, Any]) -> None: + def append_item(item: dict[str, Any], *, dedup_content: str) -> None: content = str(item.get("edited_content") or "").strip() if should_clean_invalid and not content: return - content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + # 去重必须基于脱敏前内容，否则不同原文可能在替换 PII 后被错误合并。 + content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest() if should_deduplicate and content_hash in seen_content_hashes: return seen_content_hashes.add(content_hash) @@ -447,6 +460,7 @@ def _build_preview_items( continue for key in band_keys: seen_near_duplicate_bands.setdefault(key, []).append(content) + dedup_content = content pii_counts: dict[str, int] = {} if should_desensitize: content, pii_counts = desensitize_pii(content) @@ -476,7 +490,8 @@ def _build_preview_items( else "original" ), "quality_score": quality, - } + }, + dedup_content=dedup_content, ) continue @@ -488,48 +503,76 @@ def _build_preview_items( "filter_anomaly", } source_records = list(parsed.records) - processed_records = preprocess_structured_records( + processed_records = preprocess_structured_records_with_lineage( source_records, structured_options, ) - if not processed_records and parsed.text and not source_records: - processed_records = [{"value": parsed.text}] - same_cardinality = len(processed_records) == len(source_records) - for index, record in enumerate(processed_records): - original_record = source_records[index] if same_cardinality else record - original_content = json.dumps( - original_record, - ensure_ascii=False, - separators=(",", ":"), + for processed in processed_records: + source_index = processed.source_index + record = processed.record + original_record = ( + source_records[source_index] + if source_index < len(source_records) + else record ) + source_locator = ( + deepcopy(parsed.record_locators[source_index]) + if source_index < len(parsed.record_locators) + else None + ) + original_content = structured_json_dumps(original_record) pii_counts: dict[str, int] = {} edited_record = record + dedup_content = ( + canonical_record_json(record) + if "normalize_format" in preprocess_options + else structured_json_dumps(record) + ) if should_desensitize: edited_record, pii_counts = desensitize_structured_record(record) content = ( canonical_record_json(edited_record) if "normalize_format" in preprocess_options - else json.dumps( - edited_record, - ensure_ascii=False, - separators=(",", ":"), - ) + else structured_json_dumps(edited_record) ) quality = _preview_quality(content, config) quality["pii_replacements"] = pii_counts + if source_locator is not None: + quality["source_locator"] = source_locator + source_start = ( + source_locator.get("source_start") + if source_locator is not None + else None + ) + source_end = ( + source_locator.get("source_end") + if source_locator is not None + else None + ) + source_start_line = ( + source_locator.get("start_line") + if source_locator is not None + else None + ) + source_end_line = ( + source_locator.get("end_line") + if source_locator is not None + else None + ) append_item( { "source_file_id": source["id"], "original_content": original_content, "edited_content": content, - "source_start": None, - "source_end": None, - "source_start_line": None, - "source_end_line": None, + "source_start": source_start, + "source_end": source_end, + "source_start_line": source_start_line, + "source_end_line": source_end_line, "token_count": estimate_token_count(content), "status": "modified" if content != original_content else "original", "quality_score": quality, - } + }, + dedup_content=dedup_content, ) return items @@ -850,6 +893,135 @@ def prepare_regeneration( ) +def _repeat_file_copies( + store: DataProcessStore, + storage: LocalDataProcessStorage, + source_task_id: str, + request_id: str, +) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]: + """为新任务创建独立的源文件引用，避免删除任一任务时互相影响。""" + + target_task_id = repeat_task_id(source_task_id, request_id) + copies: dict[str, dict[str, str]] = {} + staged: list[StagedSourceObject] = [] + batch_id = storage.new_batch_id() + for summary in store.list_source_files(source_task_id): + old_file_id = str(summary["id"]) + source = store.get_source_file(source_task_id, old_file_id, include_content=True) + new_file_id = new_id("dpsf") + old_reference = str(source.get("storage_object_id") or "") + if old_reference.startswith("local://data-process/"): + staged_object = storage.stage_copy( + batch_id=batch_id, + source_reference=old_reference, + expected_source_task_id=source_task_id, + expected_source_file_id=old_file_id, + task_id=target_task_id, + source_file_id=new_file_id, + version=1, + name=str(source["name"]), + ) + staged.append(staged_object) + new_reference = staged_object.reference + elif old_reference.startswith("db://data-process/") or not old_reference: + new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1" + else: + raise ValueError("源任务包含不受支持的文件存储引用") + copies[old_file_id] = { + "id": new_file_id, + "storage_object_id": new_reference, + } + return copies, staged + + +def _remove_repeated_storage_objects( + storage: LocalDataProcessStorage, + task_id: str, + staged: list[StagedSourceObject], + copies: dict[str, dict[str, str]], +) -> None: + source_file_ids = { + str(copy["storage_object_id"]): str(copy["id"]) + for copy in copies.values() + } + for item in staged: + try: + storage.delete( + item.reference, + expected_task_id=task_id, + expected_source_file_id=source_file_ids[item.reference], + ) + except Exception: + logger.exception( + "failed to roll back repeated data process source object task_id=%s", + task_id, + ) + + +@router.post("/{task_id}/repeat", status_code=202) +def repeat_generation( + task_id: str, + payload: DataProcessRepeatRequest, + background_tasks: BackgroundTasks, + store: DataProcessStore = Depends(get_data_process_store), + storage: LocalDataProcessStorage = Depends(get_data_process_storage), +) -> dict[str, Any]: + """按原任务快照创建独立任务，并立即在后台开始新一批生成。""" + + with api_errors(): + repeated = store.find_repeated_task(task_id, payload.request_id) + staged: list[StagedSourceObject] = [] + target_task_id = repeat_task_id(task_id, payload.request_id) + if repeated is None: + copies, staged = _repeat_file_copies( + store, + storage, + task_id, + payload.request_id, + ) + storage.publish(staged) + try: + repeated = store.repeat_task( + task_id, + expected_updated_at=payload.expected_updated_at, + request_id=payload.request_id, + file_copies=copies, + ) + except Exception: + _remove_repeated_storage_objects( + storage, + target_task_id, + staged, + copies, + ) + raise + if not repeated["created"]: + _remove_repeated_storage_objects( + storage, + target_task_id, + staged, + copies, + ) + + repeated_task = repeated["task"] + if repeated_task.get("status") == "pending": + try: + started = store.start_generation(target_task_id, replace_existing=True) + background_tasks.add_task( + _run_generation, + store, + target_task_id, + str(started["generation_run_id"]), + ) + except ConflictError: + latest = store.get_task(target_task_id) + if latest.get("status") != "running": + raise + repeated["task"] = store.get_task(target_task_id) + repeated["progress"] = store.progress(target_task_id) + return ok(repeated, "已按原配置创建新任务并开始后台生成") + + @router.delete("/{task_id}") def delete_task( task_id: str, @@ -919,7 +1091,7 @@ async def upload_source_files( f"{suffix} is not supported for {process_type} data processing", ) parsed = parse_text_content(raw, filename=name) - if not parsed.text: + if not parsed.text.strip(): raise fail(400, f"source file is empty: {name}") batch_size += len(raw) if batch_size > MAX_SOURCE_BATCH_BYTES: @@ -934,7 +1106,11 @@ async def upload_source_files( content=raw, ) staged.append(staged_object) - record_count = len(parsed.records) or (1 if parsed.text else 0) + record_count = ( + len(parsed.records) + if process_type == "structured" + else (1 if parsed.text else 0) + ) prepared.append( { "id": source_file_id, @@ -1362,15 +1538,28 @@ def _prepare_preview_items( _value(config, "chunk_method", "chunkMethod", "layout_hybrid") ) is_unstructured = task.get("process_type") == "unstructured" - if is_unstructured and ( + needs_unstructured_raw = is_unstructured and ( chunk_method == "layout_hybrid" or preprocess_options & {"clean_invalid", "clean_invalid_content"} - ): + ) + has_structured_xlsx = not is_unstructured and any( + str(source.get("file_format") or "").lower() == "xlsx" + for source in sources + ) + if needs_unstructured_raw or has_structured_xlsx: for index, source in enumerate(sources): - if ( - chunk_method != "layout_hybrid" - and str(source.get("file_format") or "").lower() != "pdf" - ): + source_format = str(source.get("file_format") or "").lower() + needs_structured_xlsx = not is_unstructured and source_format == "xlsx" + needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid" + needs_pdf_noise = ( + is_unstructured + and not needs_layout_raw + and source_format == "pdf" + and bool( + preprocess_options & {"clean_invalid", "clean_invalid_content"} + ) + ) + if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise): continue storage_object_id = str(source.get("storage_object_id") or "") actual_size = storage.file_size( @@ -1379,7 +1568,7 @@ def _prepare_preview_items( expected_source_file_id=str(source["id"]), ) if actual_size is None: - if chunk_method == "layout_hybrid": + if needs_layout_raw: raise InvalidStateError( "版面结构混合切分无法读取原始文件，请重新上传后再处理" ) @@ -1396,12 +1585,10 @@ def _prepare_preview_items( ) ) enriched = dict(source) - if chunk_method == "layout_hybrid": + if needs_structured_xlsx or needs_layout_raw: enriched["raw_content"] = raw sources[index] = enriched continue - if str(source.get("file_format") or "").lower() != "pdf": - continue pages = extract_pdf_page_texts(raw) extracted_text = "\n\n".join(page.text for page in pages if page.text) if extracted_text != str(source.get("content") or ""): @@ -1413,7 +1600,7 @@ def _prepare_preview_items( enriched["document_noise_spans"] = detect_pdf_document_noise(pages) sources[index] = enriched items = _build_preview_items(task, sources) - if not items and source_file_ids is None: + if not items and source_file_ids is None and is_unstructured: raise InvalidStateError("source files did not produce preview items") return items @@ -1435,6 +1622,7 @@ def _run_preview( len(source_file_ids), ) try: + is_unstructured = store.get_task(task_id).get("process_type") == "unstructured" if not store.mark_preview_running(task_id, preview_run_id): logger.info( "data process preview skipped inactive run task_id=%s preview_run_id=%s", @@ -1461,7 +1649,7 @@ def _run_preview( storage, [source_file_id], ) - if not items: + if not items and is_unstructured: raise InvalidStateError( f"source file did not produce preview items: {source_file_id}" ) @@ -1657,8 +1845,13 @@ def update_preview_item( ) -> dict[str, Any]: with api_errors(): task = store.get_task(task_id) + existing = store.get_preview_item(task_id, preview_id) update = payload.model_dump(exclude_unset=True, mode="json") - update["quality_score"] = _preview_quality(payload.edited_content, task.get("config") or {}) + quality = _preview_quality(payload.edited_content, task.get("config") or {}) + source_locator = (existing.get("quality_score") or {}).get("source_locator") + if isinstance(source_locator, Mapping): + quality["source_locator"] = deepcopy(dict(source_locator)) + update["quality_score"] = quality item = store.update_preview_item( task_id, preview_id, diff --git a/backend/app/modules/data_process/algorithms.py b/backend/app/modules/data_process/algorithms.py index 07d4b38..a0204dd 100644 --- a/backend/app/modules/data_process/algorithms.py +++ b/backend/app/modules/data_process/algorithms.py @@ -20,6 +20,7 @@ from collections.abc import Iterable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass from datetime import date, datetime, time +from decimal import Decimal from pathlib import Path, PurePosixPath from typing import Any, Literal from urllib.parse import unquote, urlsplit @@ -107,6 +108,7 @@ _MAX_WORKBOOK_HEADER_SCAN_ROWS = 64 _MAX_WORKBOOK_MERGED_RANGES = 100_000 _MAX_STRUCTURED_FIELDS = 1_024 _MAX_STRUCTURED_DEPTH = 16 +_MAX_JSON_DEPTH = 64 _MAX_ANOMALY_TEXT_CHARS = 1_000_000 _STRUCTURED_OPTIONS = { "clean_invalid", @@ -118,6 +120,35 @@ _STRUCTURED_OPTIONS = { } _IDENTITY_FIELD_PATTERN = re.compile(r"(?:^|[._])(?:id|uuid|key|code)$|(?:^|[._]).+_id$") _MOJIBAKE_MARKERS = ("\ufffd", "锟斤拷", "烫烫烫", "屯屯屯", "Ã", "Â", "â€") +_JSON_RECORD_ARRAY_KEYS = ("records", "data", "items", "rows") +_JSON_ENVELOPE_KEYS = ("response", "payload") +_JSON_WRAPPER_METADATA_KEYS = frozenset( + { + "page", + "page_size", + "pageSize", + "per_page", + "perPage", + "total", + "total_count", + "totalCount", + "count", + "offset", + "limit", + "cursor", + "next_cursor", + "nextCursor", + "has_more", + "hasMore", + } +) +_JSON_RESPONSE_METADATA_KEYS = _JSON_WRAPPER_METADATA_KEYS | { + "success", + "status", + "code", + "message", + "error", +} _NAME_FIELD_NAMES = { "name", "full_name", @@ -127,10 +158,13 @@ _NAME_FIELD_NAMES = { "customer_name", "recipient_name", "姓名", + "中文姓名", "真实姓名", "联系人", "联系人姓名", + "客户姓名", "收件人", + "收件人姓名", } _EMAIL_PATTERN = re.compile( @@ -160,6 +194,15 @@ class ParsedText: format: TextFormat text: str records: tuple[dict[str, Any], ...] + record_locators: tuple[dict[str, Any], ...] = () + + +@dataclass(frozen=True, slots=True) +class ProcessedStructuredRecord: + """保留原始记录索引的结构化预处理结果。""" + + source_index: int + record: dict[str, Any] @dataclass(frozen=True, slots=True) @@ -1196,7 +1239,9 @@ def _infer_xlsx_header_region( return first_row, header_end, headers -def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]: +def _extract_xlsx_records( + raw: bytes, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: _validate_office_archive(raw, "xlsx") merged_by_sheet, normalized_targets = _xlsx_sheet_merge_ranges(raw) workbook_raw = ( @@ -1215,13 +1260,14 @@ def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]: raise ValueError(f"invalid XLSX file: {exc}") from exc records: list[dict[str, Any]] = [] + locators: list[dict[str, Any]] = [] total_cells = 0 try: if len(workbook.worksheets) > _MAX_WORKBOOK_SHEETS: raise ValueError( f"XLSX contains too many worksheets (limit {_MAX_WORKBOOK_SHEETS})" ) - for worksheet in workbook.worksheets: + for sheet_index, worksheet in enumerate(workbook.worksheets): reset_dimensions = getattr(worksheet, "reset_dimensions", None) if callable(reset_dimensions): reset_dimensions() @@ -1234,13 +1280,16 @@ def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]: ) buffered_rows: dict[int, Sequence[Any]] = {} - def normalized_row_values(row: Sequence[Any]) -> list[Any]: + def normalized_row_values( + row: Sequence[Any], + sheet_title: str = worksheet.title, + ) -> list[Any]: values = list(row) while values and values[-1] in {None, ""}: values.pop() if len(values) > _MAX_WORKBOOK_COLUMNS: raise ValueError( - f"XLSX worksheet {worksheet.title!r} exceeds " + f"XLSX worksheet {sheet_title!r} exceeds " f"{_MAX_WORKBOOK_COLUMNS} columns" ) return values @@ -1267,25 +1316,32 @@ def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]: merged_ranges, ) - def append_record(values: Sequence[Any]) -> None: + def append_record( + row_number: int, + values: Sequence[Any], + record_headers: Sequence[str] = tuple(headers), + locator_sheet_index: int = sheet_index, + sheet_title: str = worksheet.title, + ) -> None: nonlocal total_cells, sheet_rows row_values = list(values) - if len(row_values) > len(headers): + if len(row_values) > len(record_headers): raise ValueError( - f"XLSX worksheet {worksheet.title!r} has a row wider than its header" + f"XLSX worksheet {sheet_title!r} has a row wider than its header" ) - row_values.extend([None] * (len(headers) - len(row_values))) + row_values.extend([None] * (len(record_headers) - len(row_values))) record = { header: _normalize_spreadsheet_value(value) - for header, value in zip(headers, row_values, strict=True) + for header, value in zip(record_headers, row_values, strict=True) } if not any(value not in {"", None} for value in record.values()): return + sheet_record_index = sheet_rows sheet_rows += 1 - total_cells += len(headers) + total_cells += len(record_headers) if sheet_rows > _MAX_WORKBOOK_ROWS: raise ValueError( - f"XLSX worksheet {worksheet.title!r} exceeds " + f"XLSX worksheet {sheet_title!r} exceeds " f"{_MAX_WORKBOOK_ROWS} data rows" ) if total_cells > _MAX_WORKBOOK_CELLS: @@ -1293,12 +1349,22 @@ def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]: f"XLSX workbook exceeds {_MAX_WORKBOOK_CELLS} populated cells" ) records.append(record) + locators.append( + { + "kind": "xlsx", + "record_index": len(records), + "sheet_index": locator_sheet_index, + "sheet_name": sheet_title, + "row_number": row_number, + "sheet_record_index": sheet_record_index, + } + ) for row_number, values in buffered_rows.items(): if row_number > header_end_row: - append_record(values) + append_record(row_number, values) - for _, row in row_iterator: + for row_number, row in row_iterator: scanned_rows += 1 if scanned_rows > _MAX_WORKBOOK_SCANNED_ROWS: raise ValueError( @@ -1308,10 +1374,10 @@ def _extract_xlsx_records(raw: bytes) -> list[dict[str, Any]]: values = normalized_row_values(row) if not values or all(value in {None, ""} for value in values): continue - append_record(values) + append_record(row_number, values) finally: workbook.close() - return records + return records, locators def _normalize_value(value: Any) -> Any: @@ -1324,63 +1390,413 @@ def _normalize_value(value: Any) -> Any: return value -def _record_from_value(value: Any) -> dict[str, Any]: +def _record_from_value(value: Any, *, normalize: bool = True) -> dict[str, Any]: if isinstance(value, Mapping): - return dict(_normalize_value(value)) - return {"value": _normalize_value(value)} + return dict(_normalize_value(value)) if normalize else dict(value) + return {"value": _normalize_value(value) if normalize else value} -def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]: - """从 JSON、JSONL 或 CSV 中提取规范化记录。 +def _json_pointer_segment(value: Any) -> str: + return str(value).replace("~", "~0").replace("/", "~1") - JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组； - 其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。 + +class _DuplicateJsonKeyError(ValueError): + """严格 JSON 解析时发现同一对象内的重复键。""" + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _DuplicateJsonKeyError(f"duplicate JSON object key: {key!r}") + result[key] = value + return result + + +def _reject_json_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON number is not allowed: {value}") + + +def _skip_json_whitespace(text: str, offset: int) -> int: + while offset < len(text) and text[offset] in " \t\r\n": + offset += 1 + return offset + + +def _validate_json_nesting(text: str) -> None: + """在构造 Python 对象前限制容器深度，避免依赖解释器递归阈值。""" + + depth = 0 + in_string = False + escaped = False + for char in text: + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char in "[{": + depth += 1 + if depth > _MAX_JSON_DEPTH: + raise ValueError( + f"JSON nesting exceeds the supported depth of {_MAX_JSON_DEPTH}" + ) + elif char in "]}": + depth = max(0, depth - 1) + + +def _strict_json_loads(text: str) -> tuple[Any, int, int]: + """严格解析单个 JSON 值并返回其左闭右开源码区间。""" + + start = _skip_json_whitespace(text, 0) + if start >= len(text): + raise ValueError("JSON content is empty") + _validate_json_nesting(text) + decoder = json.JSONDecoder( + object_pairs_hook=_reject_duplicate_json_keys, + parse_float=Decimal, + parse_int=int, + parse_constant=_reject_json_constant, + strict=True, + ) + try: + payload, end = decoder.raw_decode(text, start) + except json.JSONDecodeError as exc: + raise ValueError( + f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}" + ) from exc + except RecursionError as exc: + raise ValueError("JSON nesting exceeds the supported depth") from exc + except _DuplicateJsonKeyError as exc: + raise ValueError(str(exc)) from exc + except ValueError as exc: + # parse_int/parse_float/parse_constant 的异常也必须稳定映射为客户端错误。 + raise ValueError(f"invalid JSON number: {exc}") from exc + trailing = _skip_json_whitespace(text, end) + if trailing != len(text): + line = text.count("\n", 0, trailing) + 1 + line_start = text.rfind("\n", 0, trailing) + 1 + column = trailing - line_start + 1 + raise ValueError( + f"invalid JSON at line {line}, column {column}: extra data" + ) + return payload, start, end + + +def _json_value_end(text: str, start: int) -> int: + """在已验证 JSON 中定位一个值的结束偏移，不对数值做二次解析。""" + + if start >= len(text): + raise ValueError("invalid JSON source span") + first = text[start] + if first == '"': + escaped = False + for offset in range(start + 1, len(text)): + char = text[offset] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + return offset + 1 + raise ValueError("invalid JSON source span") + if first in "[{": + stack = [first] + in_string = False + escaped = False + for offset in range(start + 1, len(text)): + char = text[offset] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char in "[{": + stack.append(char) + elif char in "]}": + expected = "[" if char == "]" else "{" + if not stack or stack[-1] != expected: + raise ValueError("invalid JSON source span") + stack.pop() + if not stack: + return offset + 1 + raise ValueError("invalid JSON source span") + end = start + while end < len(text) and text[end] not in " \t\r\n,]}": + end += 1 + if end == start: + raise ValueError("invalid JSON source span") + return end + + +def _json_object_value_spans( + text: str, + start: int, + end: int, +) -> dict[str, tuple[int, int]]: + """返回已验证 JSON 对象直接子字段的值区间。""" + + if text[start] != "{" or text[end - 1] != "}": + raise ValueError("JSON source value is not an object") + result: dict[str, tuple[int, int]] = {} + offset = _skip_json_whitespace(text, start + 1) + key_decoder = json.JSONDecoder() + while offset < end - 1: + key, key_end = key_decoder.raw_decode(text, offset) + if not isinstance(key, str): + raise ValueError("invalid JSON object key") + offset = _skip_json_whitespace(text, key_end) + if offset >= end or text[offset] != ":": + raise ValueError("invalid JSON object member") + value_start = _skip_json_whitespace(text, offset + 1) + value_end = _json_value_end(text, value_start) + result[key] = (value_start, value_end) + offset = _skip_json_whitespace(text, value_end) + if offset >= end - 1: + break + if text[offset] != ",": + raise ValueError("invalid JSON object member") + offset = _skip_json_whitespace(text, offset + 1) + return result + + +def _json_array_item_spans( + text: str, + start: int, + end: int, +) -> list[tuple[int, int]]: + """返回已验证 JSON 数组中每个直接元素的源码区间。""" + + if text[start] != "[" or text[end - 1] != "]": + raise ValueError("JSON source value is not an array") + result: list[tuple[int, int]] = [] + offset = _skip_json_whitespace(text, start + 1) + while offset < end - 1: + item_end = _json_value_end(text, offset) + result.append((offset, item_end)) + offset = _skip_json_whitespace(text, item_end) + if offset >= end - 1: + break + if text[offset] != ",": + raise ValueError("invalid JSON array item") + offset = _skip_json_whitespace(text, offset + 1) + return result + + +def _json_span_at_path( + text: str, + root_span: tuple[int, int], + path: Sequence[str], +) -> tuple[int, int]: + span = root_span + for key in path: + try: + span = _json_object_value_spans(text, *span)[key] + except KeyError as exc: + raise ValueError(f"JSON source path cannot be located: {key}") from exc + return span + + +def _pure_json_record_wrapper( + payload: Any, +) -> tuple[list[Any], tuple[str, ...]] | None: + """识别不会与业务字段冲突的纯记录包装对象。""" + + if not isinstance(payload, Mapping): + return None + + def direct_wrapper( + value: Mapping[str, Any], + metadata_keys: frozenset[str] | set[str] = _JSON_WRAPPER_METADATA_KEYS, + ) -> tuple[list[Any], tuple[str, ...]] | None: + candidates = [ + key + for key in _JSON_RECORD_ARRAY_KEYS + if isinstance(value.get(key), list) + ] + if len(candidates) != 1: + return None + record_key = candidates[0] + records = value[record_key] + if any(not isinstance(record, Mapping) for record in records): + return None + if any( + key != record_key and key not in metadata_keys + for key in value + ): + return None + return records, (record_key,) + + direct = direct_wrapper(payload) + if direct is not None: + return direct + + envelope_keys = [ + key + for key in _JSON_ENVELOPE_KEYS + if isinstance(payload.get(key), Mapping) + ] + if len(envelope_keys) != 1: + return None + envelope_key = envelope_keys[0] + if any( + key != envelope_key and key not in _JSON_RESPONSE_METADATA_KEYS + for key in payload + ): + return None + nested = direct_wrapper( + payload[envelope_key], + _JSON_RESPONSE_METADATA_KEYS, + ) + if nested is None: + return None + records, nested_path = nested + return records, (envelope_key, *nested_path) + + +def _json_record_locator( + text: str, + *, + record_index: int, + json_pointer: str, + span: tuple[int, int], +) -> dict[str, Any]: + source_start, source_end = span + start_line = text.count("\n", 0, source_start) + 1 + last_character = max(source_start, source_end - 1) + end_line = text.count("\n", 0, last_character) + 1 + return { + "kind": "json", + "record_index": record_index, + "json_pointer": json_pointer, + "source_start": source_start, + "source_end": source_end, + "start_line": start_line, + "end_line": end_line, + } + + +def _source_line_offsets( + text: str, + start_line: int, + end_line: int, +) -> tuple[int, int]: + """把 1-based 物理行范围转换为左闭右开的字符范围。""" + + line_starts = [0] + line_starts.extend(match.end() for match in re.finditer("\n", text)) + if start_line < 1 or end_line < start_line or end_line > len(line_starts): + raise ValueError("source line range is outside normalized text") + source_start = line_starts[start_line - 1] + source_end = ( + line_starts[end_line] - 1 + if end_line < len(line_starts) + else len(text) + ) + return source_start, source_end + + +def _extract_structured_records_with_locators( + text: str, + file_format: str, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """从 JSON、JSONL 或 CSV 中提取记录。 + + JSON 根数组始终表示多条记录；对象仅在满足纯包装契约时展开，其他 + 对象均视为一条业务记录。JSON 字段和值在解析阶段保持原样，只有用户 + 明确选择 ``normalize_format`` 后才会规范化。 """ normalized_format = _normalize_format(file_format) if normalized_format not in {"json", "jsonl", "csv"}: raise ValueError("structured record extraction only supports JSON, JSONL and CSV") - normalized_text = normalize_text(text) - if not normalized_text: - return [] - if normalized_format == "json": - try: - payload = json.loads(normalized_text) - except json.JSONDecodeError as exc: - raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc + if _skip_json_whitespace(text, 0) == len(text): + return [], [] + payload, root_start, root_end = _strict_json_loads(text) values: Sequence[Any] + pointer_path: tuple[str, ...] = () + record_spans: list[tuple[int, int]] if isinstance(payload, list): values = payload - elif isinstance(payload, Mapping): - nested = next( - ( - payload[key] - for key in ("records", "data", "items", "rows") - if isinstance(payload.get(key), list) - ), - None, - ) - values = nested if isinstance(nested, list) else [payload] + record_spans = _json_array_item_spans(text, root_start, root_end) else: - values = [payload] - return [_record_from_value(value) for value in values] + wrapper = _pure_json_record_wrapper(payload) + if wrapper is None: + values = [payload] + record_spans = [(root_start, root_end)] + else: + values, pointer_path = wrapper + array_span = _json_span_at_path( + text, + (root_start, root_end), + pointer_path, + ) + record_spans = _json_array_item_spans(text, *array_span) + if len(record_spans) != len(values): + raise ValueError("JSON record source spans do not match parsed records") + records = [_record_from_value(value, normalize=False) for value in values] + pointer_prefix = "".join( + f"/{_json_pointer_segment(segment)}" for segment in pointer_path + ) + locators = [ + _json_record_locator( + text, + record_index=index + 1, + json_pointer=( + f"{pointer_prefix}/{index}" + if pointer_path or isinstance(payload, list) + else "" + ), + span=record_spans[index], + ) + for index in range(len(records)) + ] + return records, locators if normalized_format == "jsonl": records: list[dict[str, Any]] = [] - for line_number, line in enumerate(normalized_text.splitlines(), start=1): - if not line.strip(): + locators: list[dict[str, Any]] = [] + source_offset = 0 + for line_number, line in enumerate(text.split("\n"), start=1): + line_content_end = len(line) + if _skip_json_whitespace(line, 0) == line_content_end: + source_offset += len(line) + 1 continue try: - value = json.loads(line) - except json.JSONDecodeError as exc: + value, value_start, value_end = _strict_json_loads(line) + except ValueError as exc: raise ValueError( - f"invalid JSONL at line {line_number}, " - f"column {exc.colno}: {exc.msg}" + f"invalid JSONL at line {line_number}: {exc}" ) from exc - records.append(_record_from_value(value)) - return records + records.append(_record_from_value(value, normalize=False)) + locators.append( + { + "kind": "jsonl", + "record_index": len(records), + "start_line": line_number, + "end_line": line_number, + "source_start": source_offset + value_start, + "source_end": source_offset + value_end, + } + ) + source_offset += len(line) + 1 + return records, locators + + normalized_text = normalize_text(text) + if not normalized_text: + return [], [] try: dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;") @@ -1396,8 +1812,16 @@ def extract_structured_records(text: str, file_format: str) -> list[dict[str, An raise ValueError("CSV headers must be unique") reader.fieldnames = headers - records = [] + records: list[dict[str, Any]] = [] + locators = [] + source_lines = normalized_text.splitlines() + previous_end_line = reader.line_num for row in reader: + end_line = reader.line_num + start_line = previous_end_line + 1 + previous_end_line = end_line + while start_line < end_line and not source_lines[start_line - 1].strip(): + start_line += 1 if None in row: raise ValueError("CSV row has more fields than the header") normalized_row = { @@ -1406,6 +1830,28 @@ def extract_structured_records(text: str, file_format: str) -> list[dict[str, An } if any(value for value in normalized_row.values()): records.append(normalized_row) + source_start, source_end = _source_line_offsets( + normalized_text, + start_line, + end_line, + ) + locators.append( + { + "kind": "csv", + "record_index": len(records), + "start_line": start_line, + "end_line": end_line, + "source_start": source_start, + "source_end": source_end, + } + ) + return records, locators + + +def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]: + """从 JSON、JSONL 或 CSV 中提取规范化记录。""" + + records, _ = _extract_structured_records_with_locators(text, file_format) return records @@ -1434,7 +1880,7 @@ def parse_text_content( text = _extract_pptx_text(binary) return ParsedText(format=detected_format, text=text, records=()) - records = _extract_xlsx_records(binary) + records, record_locators = _extract_xlsx_records(binary) text = "\n".join( json.dumps(record, ensure_ascii=False, separators=(",", ":")) for record in records @@ -1443,14 +1889,35 @@ def parse_text_content( format=detected_format, text=normalize_text(text), records=tuple(records), + record_locators=tuple(record_locators), ) - text = normalize_text(decode_utf8(raw)) - detected_format = detect_text_format(filename=filename, text=text, file_format=file_format) + decoded_text = decode_utf8(raw) + detected_format = detect_text_format( + filename=filename, + text=decoded_text, + file_format=file_format, + ) + # JSON/JSONL 是有损规范化的禁区：NFKC、控制字符删除或 trim 都可能改变字段值、 + # 掩盖非法输入，甚至把原本合法的字符串变成语法错误。其他格式保持历史行为。 + text = ( + decoded_text + if detected_format in {"json", "jsonl"} + else normalize_text(decoded_text) + ) records: list[dict[str, Any]] = [] + record_locators: list[dict[str, Any]] = [] if detected_format in {"json", "jsonl", "csv"}: - records = extract_structured_records(text, detected_format) - return ParsedText(format=detected_format, text=text, records=tuple(records)) + records, record_locators = _extract_structured_records_with_locators( + text, + detected_format, + ) + return ParsedText( + format=detected_format, + text=text, + records=tuple(records), + record_locators=tuple(record_locators), + ) def desensitize_pii(text: str) -> tuple[str, dict[str, int]]: @@ -1496,10 +1963,14 @@ def _is_empty_value(value: Any) -> bool: def _canonical_value(value: Any) -> Any: if value is None or isinstance(value, (bool, int)): return value + if isinstance(value, Decimal): + if not value.is_finite(): + raise ValueError("non-finite JSON number is not allowed") + return value if isinstance(value, float): - if math.isfinite(value): - return value - return str(value).lower() + if not math.isfinite(value): + raise ValueError("non-finite JSON number is not allowed") + return value if isinstance(value, str): return normalize_text(value) if isinstance(value, (datetime, date, time)): @@ -1522,30 +1993,68 @@ def _canonical_value(value: Any) -> Any: items = [_canonical_value(item) for item in value] return sorted( items, - key=lambda item: json.dumps( - item, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ), + key=lambda item: structured_json_dumps(item, sort_keys=True), ) if isinstance(value, (bytes, bytearray, memoryview)): return bytes(value).hex() return normalize_text(str(value)) +def structured_json_dumps(value: Any, *, sort_keys: bool = False) -> str: + """序列化紧凑 JSON，并把 ``Decimal`` 保持为原值对应的 JSON 数字。 + + 标准库会要求先把 ``Decimal`` 转成 float 或字符串；前者可能静默舍入， + 后者会改变 JSON 类型。这里直接输出有限 Decimal 的十进制表示。 + """ + + def serialize(item: Any) -> str: + if item is None: + return "null" + if item is True: + return "true" + if item is False: + return "false" + if isinstance(item, int): + return str(item) + if isinstance(item, Decimal): + if not item.is_finite(): + raise ValueError("non-finite JSON number is not allowed") + return str(item) + if isinstance(item, float): + if not math.isfinite(item): + raise ValueError("non-finite JSON number is not allowed") + return json.dumps(item, allow_nan=False) + if isinstance(item, str): + return json.dumps(item, ensure_ascii=False) + if isinstance(item, Mapping): + pairs: list[tuple[str, Any]] = [] + seen_keys: set[str] = set() + for key, child in item.items(): + if not isinstance(key, str): + raise TypeError("JSON object keys must be strings") + if key in seen_keys: + raise ValueError(f"duplicate JSON object key: {key!r}") + seen_keys.add(key) + pairs.append((key, child)) + if sort_keys: + pairs.sort(key=lambda pair: pair[0]) + return "{" + ",".join( + f"{json.dumps(key, ensure_ascii=False)}:{serialize(child)}" + for key, child in pairs + ) + "}" + if isinstance(item, (list, tuple)): + return "[" + ",".join(serialize(child) for child in item) + "]" + raise TypeError(f"value of type {type(item).__name__} is not JSON serializable") + + return serialize(value) + + def canonical_record_json(record: Mapping[str, Any]) -> str: """生成与字段顺序无关、可用于比较和落库的 canonical JSON。""" if not isinstance(record, Mapping): raise TypeError("record must be a mapping") - return json.dumps( - _canonical_value(record), - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) + return structured_json_dumps(_canonical_value(record), sort_keys=True) def _normalize_field_name(value: Any, style: str) -> str: @@ -1643,41 +2152,46 @@ def flatten_structured_record( return flattened -def _clean_invalid_structured_records( - records: Sequence[Mapping[str, Any]], -) -> list[dict[str, Any]]: - if not records: +def _clean_invalid_structured_entries( + entries: Sequence[ProcessedStructuredRecord], +) -> list[ProcessedStructuredRecord]: + if not entries: return [] fields: list[str] = [] - for record in records: + for entry in entries: + record = entry.record for field in record: if field not in fields: fields.append(field) active_fields = [ field for field in fields - if any(not _is_empty_value(record.get(field)) for record in records) + if any(not _is_empty_value(entry.record.get(field)) for entry in entries) ] if not active_fields: return [] - identity_fields = [ - field - for field in active_fields - if _IDENTITY_FIELD_PATTERN.search(_normalize_field_name(field, "snake_case")) - ] - cleaned: list[dict[str, Any]] = [] - for record in records: + cleaned: list[ProcessedStructuredRecord] = [] + for entry in entries: + record = entry.record values = {field: deepcopy(record.get(field)) for field in active_fields} - # 空记录一定无效；存在身份字段时只把身份字段缺失视作“残缺行”， - # 避免因为备注等可选列为空而误删有效业务数据。 + # 清洗只依据整行是否为空。外键、父级 ID 等字段天然允许为空，不能 + # 因为字段名以 *_id 结尾就把它们全部提升为联合必填项。 if all(_is_empty_value(value) for value in values.values()): continue - if identity_fields and any(_is_empty_value(values[field]) for field in identity_fields): - continue - cleaned.append(values) + cleaned.append(ProcessedStructuredRecord(entry.source_index, values)) return cleaned +def _clean_invalid_structured_records( + records: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + entries = [ + ProcessedStructuredRecord(index, deepcopy(dict(record))) + for index, record in enumerate(records) + ] + return [entry.record for entry in _clean_invalid_structured_entries(entries)] + + def _percentile(values: Sequence[float], fraction: float) -> float: if not values: raise ValueError("cannot calculate a percentile of an empty sequence") @@ -1747,18 +2261,19 @@ def is_low_quality_content( ) -def filter_anomalous_structured_records( - records: Sequence[Mapping[str, Any]], +def _filter_anomalous_structured_entries( + entries: Sequence[ProcessedStructuredRecord], *, iqr_multiplier: float = 1.5, -) -> list[dict[str, Any]]: +) -> list[ProcessedStructuredRecord]: """按字段级数值 IQR、乱码和极端文本长度过滤异常记录。""" if iqr_multiplier <= 0: raise ValueError("iqr_multiplier must be greater than 0") numeric_values: dict[str, list[float]] = {} text_lengths: dict[str, list[float]] = {} - for record in records: + for entry in entries: + record = entry.record for field, value in record.items(): if ( isinstance(value, (int, float)) @@ -1795,8 +2310,9 @@ def filter_anomalous_structured_records( spread = third_quartile - first_quartile text_upper_bounds[field] = max(512.0, third_quartile + 3 * spread) - accepted: list[dict[str, Any]] = [] - for record in records: + accepted: list[ProcessedStructuredRecord] = [] + for entry in entries: + record = entry.record anomalous = False for field, value in record.items(): if ( @@ -1824,55 +2340,85 @@ def filter_anomalous_structured_records( anomalous = True break if not anomalous: - accepted.append(deepcopy(dict(record))) + accepted.append( + ProcessedStructuredRecord( + entry.source_index, + deepcopy(dict(record)), + ) + ) return accepted -def _identity_tokens(record: Mapping[str, Any]) -> set[tuple[str, str]]: - tokens: set[tuple[str, str]] = set() - for field, value in record.items(): - normalized_field = _normalize_field_name(field, "snake_case") - if not _IDENTITY_FIELD_PATTERN.search(normalized_field) or _is_empty_value(value): +def filter_anomalous_structured_records( + records: Sequence[Mapping[str, Any]], + *, + iqr_multiplier: float = 1.5, +) -> list[dict[str, Any]]: + """按字段级数值 IQR、乱码和极端文本长度过滤异常记录。""" + + entries = [ + ProcessedStructuredRecord(index, deepcopy(dict(record))) + for index, record in enumerate(records) + ] + return [ + entry.record + for entry in _filter_anomalous_structured_entries( + entries, + iqr_multiplier=iqr_multiplier, + ) + ] + + +def _deduplicate_structured_entries( + entries: Sequence[ProcessedStructuredRecord], +) -> list[ProcessedStructuredRecord]: + """仅按整条 canonical JSON 稳定去重，避免误删同 ID 的更新记录。""" + + exact_seen: set[str] = set() + unique: list[ProcessedStructuredRecord] = [] + for entry in entries: + record = entry.record + fingerprint = hashlib.sha256(canonical_record_json(record).encode("utf-8")).hexdigest() + if fingerprint in exact_seen: continue - tokens.add( - ( - normalized_field, - json.dumps( - _canonical_value(value), - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ), + exact_seen.add(fingerprint) + unique.append( + ProcessedStructuredRecord( + entry.source_index, + deepcopy(dict(record)), ) ) - return tokens + return unique def deduplicate_structured_records( records: Sequence[Mapping[str, Any]], ) -> list[dict[str, Any]]: - """按整条 canonical JSON 和 id/uuid/key/code/*_id 字段稳定去重。""" + """仅按整条 canonical JSON 稳定去重。""" - exact_seen: set[str] = set() - identity_seen: set[tuple[str, str]] = set() - unique: list[dict[str, Any]] = [] - for record in records: - fingerprint = hashlib.sha256(canonical_record_json(record).encode("utf-8")).hexdigest() - identities = _identity_tokens(record) - if fingerprint in exact_seen or identities & identity_seen: - continue - exact_seen.add(fingerprint) - identity_seen.update(identities) - unique.append(deepcopy(dict(record))) - return unique + entries = [ + ProcessedStructuredRecord(index, deepcopy(dict(record))) + for index, record in enumerate(records) + ] + return [entry.record for entry in _deduplicate_structured_entries(entries)] def _is_name_field(field: Any) -> bool: - normalized = _normalize_field_name(field, "snake_case") - if normalized in _NAME_FIELD_NAMES: + raw_field = normalize_text(str(field)) + if not raw_field: + return False + + # 只匹配明确表示自然人姓名的字段，避免将 table_name、product_name、 + # chinese_name 等业务名称或元数据字段误判为个人敏感信息。 + if _normalize_field_name(raw_field, "snake_case") in _NAME_FIELD_NAMES: return True - suffixes = ("_name", "_full_name", "_姓名", "_真实姓名", "_联系人", "_联系人姓名") - return normalized.endswith(suffixes) + + # detect_structure 会使用点号生成扁平化路径（例如 profile.name）；此时仅 + # 判断最后一个路径段，不能退回到宽泛的 ``*_name`` 后缀匹配。 + if "." not in raw_field: + return False + leaf_field = raw_field.rsplit(".", 1)[-1] + return _normalize_field_name(leaf_field, "snake_case") in _NAME_FIELD_NAMES def desensitize_structured_record( @@ -1923,15 +2469,15 @@ def _structured_options(options: Iterable[str] | Mapping[str, Any]) -> set[str]: return enabled -def preprocess_structured_records( +def preprocess_structured_records_with_lineage( records: Iterable[Mapping[str, Any]], options: Iterable[str] | Mapping[str, Any], -) -> list[dict[str, Any]]: - """按界面选项执行确定性、无副作用的结构化数据预处理。""" +) -> list[ProcessedStructuredRecord]: + """执行结构化预处理，并保留每条结果在原始输入中的稳定索引。""" enabled = _structured_options(options) - current: list[dict[str, Any]] = [] - for record in records: + current: list[ProcessedStructuredRecord] = [] + for source_index, record in enumerate(records): if not isinstance(record, Mapping): if "clean_invalid" in enabled: continue @@ -1941,19 +2487,37 @@ def preprocess_structured_records( value = flatten_structured_record(value) if "normalize_format" in enabled: value = normalize_structured_record(value) - current.append(value) + current.append(ProcessedStructuredRecord(source_index, value)) if "clean_invalid" in enabled: - current = _clean_invalid_structured_records(current) + current = _clean_invalid_structured_entries(current) if "filter_anomaly" in enabled: - current = filter_anomalous_structured_records(current) + current = _filter_anomalous_structured_entries(current) if "deduplicate" in enabled: - current = deduplicate_structured_records(current) + current = _deduplicate_structured_entries(current) if "desensitize" in enabled: - current = [desensitize_structured_record(record)[0] for record in current] + current = [ + ProcessedStructuredRecord( + entry.source_index, + desensitize_structured_record(entry.record)[0], + ) + for entry in current + ] return current +def preprocess_structured_records( + records: Iterable[Mapping[str, Any]], + options: Iterable[str] | Mapping[str, Any], +) -> list[dict[str, Any]]: + """按界面选项执行确定性、无副作用的结构化数据预处理。""" + + return [ + entry.record + for entry in preprocess_structured_records_with_lineage(records, options) + ] + + def estimate_token_count(text: str) -> int: """无分词器依赖的确定性 token 估算，用于预览与保护性限流。""" @@ -2724,6 +3288,7 @@ __all__ = [ "DocumentStructure", "ParsedText", "PdfPageText", + "ProcessedStructuredRecord", "QualityScore", "SUPPORTED_TEXT_FORMATS", "StructuredPreprocessOption", @@ -2754,10 +3319,12 @@ __all__ = [ "parse_text_content", "parse_utf8_text", "preprocess_structured_records", + "preprocess_structured_records_with_lineage", "protected_context_ranges", "record_fingerprint", "remove_document_noise", "score_quality", "stable_split", "stable_split_assignments", + "structured_json_dumps", ] diff --git a/backend/app/modules/data_process/storage.py b/backend/app/modules/data_process/storage.py index b6051e3..23b1666 100644 --- a/backend/app/modules/data_process/storage.py +++ b/backend/app/modules/data_process/storage.py @@ -137,6 +137,67 @@ class LocalDataProcessStorage: self._issued_staged_objects[temporary_path] = staged return staged + def stage_copy( + self, + *, + batch_id: str, + source_reference: str, + expected_source_task_id: str, + expected_source_file_id: str, + task_id: str, + source_file_id: str, + version: int, + name: str, + ) -> StagedSourceObject: + """为不可变源对象创建独立目录项，不把大文件重新读入内存。""" + + batch_id = _safe_component(batch_id, "batch id") + task_id = _safe_component(task_id, "task id") + source_file_id = _safe_component(source_file_id, "source file id") + if isinstance(version, bool) or not isinstance(version, int) or version < 1: + raise DataProcessStorageError("invalid source file version") + basename = _safe_basename(name) + source_relative = self._relative_from_reference(source_reference) + if source_relative is None: + raise DataProcessStorageError("original source object is not available") + self._assert_expected_owner( + source_relative, + expected_task_id=expected_source_task_id, + expected_source_file_id=expected_source_file_id, + ) + descriptor, source_info = self._open_read_descriptor(source_relative) + os.close(descriptor) + + batch_directory = self._ensure_directory(self._root / ".staging" / batch_id) + temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp" + source_path = self._path_for_relative(source_relative) + try: + os.link(source_path, temporary_path, follow_symlinks=False) + copy_info = temporary_path.lstat() + if ( + not stat.S_ISREG(copy_info.st_mode) + or source_info.st_dev != copy_info.st_dev + or source_info.st_ino != copy_info.st_ino + ): + raise DataProcessStorageError("source storage object changed while copying") + except Exception: + temporary_path.unlink(missing_ok=True) + raise + + relative_path = PurePosixPath( + task_id, + source_file_id, + f"v{version}", + basename, + ) + reference = ( + "local://data-process/" + f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}" + ) + staged = StagedSourceObject(reference, temporary_path, relative_path) + self._issued_staged_objects[temporary_path] = staged + return staged + def publish(self, objects: Iterable[StagedSourceObject]) -> None: staged = list(objects) published: list[StagedSourceObject] = [] diff --git a/backend/app/modules/data_process/store.py b/backend/app/modules/data_process/store.py index f99dd75..06925b6 100644 --- a/backend/app/modules/data_process/store.py +++ b/backend/app/modules/data_process/store.py @@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = { "preserve_lists": True, } _REGENERATION_MARKER_KEY = "_regeneration_prepared" +_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id" +_REPEAT_REQUEST_KEY = "_repeat_request_id" +_INTERNAL_CONFIG_KEYS = { + _REGENERATION_MARKER_KEY, + _REPEAT_SOURCE_TASK_KEY, + _REPEAT_REQUEST_KEY, +} class DataProcessStoreError(RuntimeError): @@ -71,6 +78,13 @@ def new_id(prefix: str) -> str: return f"{prefix}_{uuid.uuid4().hex[:20]}" +def repeat_task_id(source_task_id: str, request_id: str) -> str: + """按源任务和请求幂等键生成稳定的新任务 ID。""" + + digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest() + return f"dpt_{digest[:20]}" + + def json_dumps(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) @@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool: return _regeneration_marker(task) is not None +def _business_config(config: dict[str, Any] | None) -> dict[str, Any]: + """过滤只供服务端维护的工作流标记。""" + + return { + key: value + for key, value in (config or {}).items() + if key not in _INTERNAL_CONFIG_KEYS + } + + def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None: - """从 API 任务快照中移除服务端内部重新生成标记。""" + """从 API 任务快照中移除服务端内部工作流标记。""" if item is None: return None public = dict(item) config = public.get("config") - if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config: - public["config"] = { - key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY - } + if isinstance(config, dict): + public["config"] = _business_config(config) return public @@ -353,13 +375,7 @@ class DataProcessStore: payload.get("description") or "", payload["process_type"], payload.get("source_dataset_id"), - json_dumps( - { - key: value - for key, value in (payload.get("config") or {}).items() - if key != _REGENERATION_MARKER_KEY - } - ), + json_dumps(_business_config(payload.get("config"))), payload.get("tenant_id"), payload.get("project_id"), payload.get("owner_id"), @@ -373,6 +389,268 @@ class DataProcessStore: raise ConflictError("data process task name already exists") from exc return _public_task(_decode_row(row)) or {} + @staticmethod + def _repeat_response( + conn: psycopg.Connection[dict[str, Any]], + row: dict[str, Any], + *, + source_task_id: str, + created: bool, + ) -> dict[str, Any]: + task_id = str(row["id"]) + counts = conn.execute( + """ + SELECT + (SELECT COUNT(*) FROM data_process_source_files + WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count, + (SELECT COUNT(*) FROM data_process_preview_items + WHERE task_id=%s) AS preview_count + """, + (task_id, task_id), + ).fetchone() or {} + task = _public_task(_decode_row(row)) or {} + task["source_file_count"] = int(counts.get("source_file_count") or 0) + task["preview_count"] = int(counts.get("preview_count") or 0) + return { + "task": task, + "source_task_id": source_task_id, + "created": created, + "copied_source_file_count": task["source_file_count"], + "copied_preview_count": task["preview_count"], + } + + def find_repeated_task( + self, + source_task_id: str, + request_id: str, + ) -> dict[str, Any] | None: + """查找同一幂等请求已创建的新任务。""" + + task_id = repeat_task_id(source_task_id, request_id) + with self.connect() as conn: + row = conn.execute( + "SELECT * FROM data_process_tasks WHERE id=%s", + (task_id,), + ).fetchone() + if row is None: + return None + decoded = _decode_row(row) or {} + config = decoded.get("config") or {} + if ( + config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id + or config.get(_REPEAT_REQUEST_KEY) != request_id + ): + raise ConflictError("再次生成请求与现有任务冲突") + if decoded.get("deleted_at"): + raise ConflictError("此次再次生成创建的任务已被删除，请重新发起") + return self._repeat_response( + conn, + row, + source_task_id=source_task_id, + created=False, + ) + + def repeat_task( + self, + source_task_id: str, + *, + expected_updated_at: str, + request_id: str, + file_copies: dict[str, dict[str, str]], + ) -> dict[str, Any]: + """复制已确认任务的配置、源文件和预览，结果与发布数据保持独立。""" + + task_id = repeat_task_id(source_task_id, request_id) + now = utcnow() + try: + with self.connect() as conn: + existing = conn.execute( + "SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE", + (task_id,), + ).fetchone() + if existing is not None: + decoded = _decode_row(existing) or {} + config = decoded.get("config") or {} + if ( + config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id + or config.get(_REPEAT_REQUEST_KEY) != request_id + ): + raise ConflictError("再次生成请求与现有任务冲突") + if decoded.get("deleted_at"): + raise ConflictError("此次再次生成创建的任务已被删除，请重新发起") + return self._repeat_response( + conn, + existing, + source_task_id=source_task_id, + created=False, + ) + + source_task = self._task_in_connection( + conn, + source_task_id, + for_update=True, + ) + if ( + source_task.get("status") != "completed" + or source_task.get("results_confirmed") is False + ): + raise InvalidStateError("只有已完成并确认结果的任务可以再次生成") + if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES: + raise ConflictError("源任务仍在处理切分，暂时不能再次生成") + if expected_updated_at != _serialize_value(source_task.get("updated_at")): + raise ConflictError("源任务已被其他操作修改，请刷新后重试") + + source_files = conn.execute( + """ + SELECT * FROM data_process_source_files + WHERE task_id=%s AND deleted_at IS NULL + ORDER BY created_at, id + """, + (source_task_id,), + ).fetchall() + source_file_ids = {str(row["id"]) for row in source_files} + if source_file_ids != set(file_copies): + raise ConflictError("源文件快照已变化，请刷新后重试") + previews = conn.execute( + """ + SELECT * FROM data_process_preview_items + WHERE task_id=%s + ORDER BY source_file_id NULLS LAST, source_start NULLS LAST, + created_at, id + """, + (source_task_id,), + ).fetchall() + if not previews: + raise InvalidStateError("源任务没有可用于再次生成的切分结果") + + suffix = f"（再次生成-{task_id[-6:]}）" + base_name = str(source_task.get("name") or "数据处理任务") + repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}" + repeated_config = _business_config(source_task.get("config") or {}) + repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id + repeated_config[_REPEAT_REQUEST_KEY] = request_id + input_count = sum(int(row.get("record_count") or 0) for row in source_files) + task_row = conn.execute( + """ + INSERT INTO data_process_tasks + (id, name, description, status, process_type, source_dataset_id, + output_dataset_id, config, progress, input_count, output_count, + filtered_count, duplicate_count, error_count, failure_reason, + generation_run_id, results_confirmed, workflow_step, + preview_status, preview_progress, preview_run_id, + preview_failure_reason, preview_total_files, + preview_completed_files, tenant_id, project_id, owner_id, + approval_status, created_by, updated_by, created_at, updated_at) + VALUES + (%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0, + 0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100, + NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING * + """, + ( + task_id, + repeated_name, + source_task.get("description") or "", + source_task["process_type"], + source_task.get("source_dataset_id"), + json_dumps(repeated_config), + input_count, + len(source_files), + len(source_files), + source_task.get("tenant_id"), + source_task.get("project_id"), + source_task.get("owner_id"), + source_task.get("approval_status") or "not_required", + source_task.get("created_by"), + source_task.get("created_by"), + now, + now, + ), + ).fetchone() + + file_id_map: dict[str, str] = {} + for source in source_files: + old_file_id = str(source["id"]) + copy = file_copies[old_file_id] + new_file_id = str(copy["id"]) + storage_object_id, metadata = _source_storage_descriptor( + { + "storage_object_id": copy["storage_object_id"], + "metadata": _json_value(source.get("metadata"), {}), + }, + task_id, + new_file_id, + ) + file_id_map[old_file_id] = new_file_id + conn.execute( + """ + INSERT INTO data_process_source_files + (id, task_id, storage_object_id, name, size_bytes, record_count, + file_format, checksum_sha256, version_no, content, + content_preview, metadata, tenant_id, project_id, created_by, + created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s, + %s, %s, %s, %s, %s) + """, + ( + new_file_id, + task_id, + storage_object_id, + source["name"], + source.get("size_bytes") or 0, + source.get("record_count") or 0, + source.get("file_format"), + source["checksum_sha256"], + source.get("content") or "", + source.get("content_preview"), + json_dumps(metadata), + source_task.get("tenant_id"), + source_task.get("project_id"), + source.get("created_by") or source_task.get("created_by"), + now, + now, + ), + ) + + for preview in previews: + old_source_file_id = preview.get("source_file_id") + conn.execute( + """ + INSERT INTO data_process_preview_items + (id, task_id, source_file_id, original_content, edited_content, + source_start, source_end, source_start_line, source_end_line, + token_count, status, quality_score, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s) + """, + ( + new_id("dpp"), + task_id, + file_id_map.get(str(old_source_file_id)) + if old_source_file_id + else None, + preview.get("original_content") or "", + preview.get("edited_content") or "", + preview.get("source_start"), + preview.get("source_end"), + preview.get("source_start_line"), + preview.get("source_end_line"), + max(0, int(preview.get("token_count") or 0)), + preview.get("status") or "original", + json_dumps(_json_value(preview.get("quality_score"), {})), + now, + now, + ), + ) + return self._repeat_response( + conn, + task_row or {}, + source_task_id=source_task_id, + created=True, + ) + except psycopg.errors.UniqueViolation as exc: + raise ConflictError("再次生成任务名称或请求发生冲突，请重试") from exc + def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]: lock = " FOR UPDATE" if for_update else "" with self.connect() as conn: @@ -654,14 +932,11 @@ class DataProcessStore: "process type and source dataset cannot change during regeneration" ) if payload.get("config") is not None: - next_config = { - key: value - for key, value in payload["config"].items() - if key != _REGENERATION_MARKER_KEY - } - current_marker = _regeneration_marker(task) - if current_marker: - next_config[_REGENERATION_MARKER_KEY] = current_marker + next_config = _business_config(payload["config"]) + current_config = dict(task.get("config") or {}) + for key in _INTERNAL_CONFIG_KEYS: + if key in current_config: + next_config[key] = current_config[key] values["config"] = json_dumps(next_config) invalidates_results = ( ("config" in payload and payload.get("config") != task.get("config")) @@ -753,8 +1028,10 @@ class DataProcessStore: raise InvalidStateError("process_type cannot be changed during regeneration") current_config = dict(task.get("config") or {}) - next_config = dict(payload.get("config") or {}) - next_config.pop(_REGENERATION_MARKER_KEY, None) + next_config = _business_config(payload.get("config")) + for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY): + if key in current_config: + next_config[key] = current_config[key] preview_invalidated = _preview_config_changed( process_type, current_config, diff --git a/backend/app/schemas/data_process.py b/backend/app/schemas/data_process.py index 744c505..1798121 100644 --- a/backend/app/schemas/data_process.py +++ b/backend/app/schemas/data_process.py @@ -220,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel): return self +class DataProcessRepeatRequest(BaseModel): + """按已确认任务的完整快照创建一批独立的新生成结果。""" + + model_config = ConfigDict(extra="forbid") + + expected_updated_at: str = Field(min_length=1) + request_id: str = Field( + min_length=8, + max_length=80, + pattern=r"^[A-Za-z0-9_-]+$", + ) + + class PreviewBuildRequest(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/backend/tests/test_data_process_algorithms.py b/backend/tests/test_data_process_algorithms.py index 1fa8520..5647d35 100644 --- a/backend/tests/test_data_process_algorithms.py +++ b/backend/tests/test_data_process_algorithms.py @@ -5,6 +5,7 @@ import json import xml.etree.ElementTree as ET import zipfile from datetime import datetime +from decimal import Decimal import pytest from docx import Document @@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import ( normalize_text, parse_text_content, preprocess_structured_records, + preprocess_structured_records_with_lineage, record_fingerprint, remove_document_noise, score_quality, stable_split, stable_split_assignments, + structured_json_dumps, ) @@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None: assert parsed_txt.text == "普通文本" +def test_structured_text_record_locators_preserve_logical_source_positions() -> None: + root_json = parse_text_content('{"id":1}', filename="root.json") + assert root_json.record_locators == ( + { + "kind": "json", + "record_index": 1, + "json_pointer": "", + "source_start": 0, + "source_end": 8, + "start_line": 1, + "end_line": 1, + }, + ) + + wrapped_json = parse_text_content( + '{"records":[{"id":1},{"id":1}]}', + filename="wrapped.json", + ) + assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [ + "/records/0", + "/records/1", + ] + + parsed_jsonl = parse_text_content( + '{"id":1}\r\n\r\n{"id":1}', + filename="records.jsonl", + ) + assert [ + (locator["record_index"], locator["start_line"], locator["end_line"]) + for locator in parsed_jsonl.record_locators + ] == [(1, 1, 1), (2, 3, 3)] + assert [ + parsed_jsonl.text[locator["source_start"] : locator["source_end"]] + for locator in parsed_jsonl.record_locators + ] == ['{"id":1}', '{"id":1}'] + + parsed_csv = parse_text_content( + 'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain', + filename="records.csv", + ) + assert [ + (locator["record_index"], locator["start_line"], locator["end_line"]) + for locator in parsed_csv.record_locators + ] == [(1, 2, 3), (2, 5, 5)] + assert [ + parsed_csv.text[locator["source_start"] : locator["source_end"]] + for locator in parsed_csv.record_locators + ] == ['1,"hello\nworld"', "2,plain"] + + +def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None: + processed = preprocess_structured_records_with_lineage( + [ + {"id": "A", "value": "first", "empty": ""}, + {"id": "", "value": "invalid", "empty": ""}, + {"id": "A", "value": "duplicate identity", "empty": ""}, + {"id": "B", "value": "second", "empty": ""}, + ], + ["clean_invalid", "deduplicate"], + ) + assert [entry.source_index for entry in processed] == [0, 1, 2, 3] + assert [entry.record for entry in processed] == [ + {"id": "A", "value": "first"}, + {"id": "", "value": "invalid"}, + {"id": "A", "value": "duplicate identity"}, + {"id": "B", "value": "second"}, + ] + + def test_parse_pdf_docx_xlsx_and_pptx() -> None: parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf") assert parsed_pdf.format == "pdf" @@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None: {"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"}, {"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"}, ) + assert parsed_xlsx.record_locators == ( + { + "kind": "xlsx", + "record_index": 1, + "sheet_index": 0, + "sheet_name": "数据", + "row_number": 2, + "sheet_record_index": 0, + }, + { + "kind": "xlsx", + "record_index": 2, + "sheet_index": 0, + "sheet_name": "数据", + "row_number": 3, + "sheet_record_index": 1, + }, + ) assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0] parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx") @@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None: assert parsed_pptx.records == () +def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None: + workbook = Workbook() + first = workbook.active + first.title = "甲表" + first.append(["说明"]) + first.append([]) + first.append(["id", "value"]) + first.append([1, "same"]) + first.append([1, "same"]) + second = workbook.create_sheet("乙表") + second.append(["id", "value"]) + second.append([1, "same"]) + output = io.BytesIO() + workbook.save(output) + workbook.close() + + parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx") + assert parsed.records == ( + {"id": 1, "value": "same"}, + {"id": 1, "value": "same"}, + {"id": 1, "value": "same"}, + ) + assert [ + ( + locator["record_index"], + locator["sheet_index"], + locator["sheet_name"], + locator["row_number"], + locator["sheet_record_index"], + ) + for locator in parsed.record_locators + ] == [ + (1, 0, "甲表", 4, 0), + (2, 0, "甲表", 5, 1), + (3, 1, "乙表", 2, 0), + ] + + def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None: pages = _pdf_page_texts( """ @@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None: json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False), "json", ) - assert result == [{"text": "内容"}] + assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}] + + assert extract_structured_records( + '{"items":[{"text":" 内容 "}],"total":1}', + "json", + ) == [{"text": " 内容 "}] + + +def test_json_parsing_is_strict_and_preserves_field_values() -> None: + source = '{"code":"００１","text":" 内容 ","quote":"＂"}' + parsed = parse_text_content(source, filename="records.json") + assert parsed.text == source + assert parsed.records == ( + {"code": "００１", "text": " 内容 ", "quote": "＂"}, + ) + + invalid_values = ( + '{"id":1,"id":2}', + '{"nested":{"id":1,"id":2}}', + '{"value":NaN}', + '{"value":Infinity}', + '{"value":-Infinity}', + '{"value":"bad\x00control"}', + ) + for invalid in invalid_values: + with pytest.raises(ValueError): + parse_text_content(invalid, filename="invalid.json") + + with pytest.raises(ValueError): + parse_text_content("｛\"id\":1｝", filename="invalid.json") + with pytest.raises(ValueError, match="nesting exceeds"): + parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json") + + +def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None: + source = ( + ' {"code":"００１","text":" 内容 ",' + '"value":0.123456789012345678901234567890}\r\n\r\n' + '{"id":2}\r\n' + ) + parsed = parse_text_content(source, filename="records.jsonl") + assert parsed.text == source + assert parsed.records[0] == { + "code": "００１", + "text": " 内容 ", + "value": Decimal("0.123456789012345678901234567890"), + } + assert [ + source[locator["source_start"] : locator["source_end"]] + for locator in parsed.record_locators + ] == [ + ( + '{"code":"００１","text":" 内容 ",' + '"value":0.123456789012345678901234567890}' + ), + '{"id":2}', + ] + assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3] + + for invalid in ('{"id":1,"id":2}', '{"value":NaN}'): + with pytest.raises(ValueError, match="invalid JSONL at line 1"): + parse_text_content(invalid, filename="invalid.jsonl") + + +def test_json_record_contract_avoids_business_field_collisions() -> None: + assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [ + {"id": 1}, + {"id": 2}, + ] + assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [ + {"id": 1, "data": [{"id": 2}]} + ] + assert extract_structured_records( + '{"records":[{"id":1}],"data":[{"id":2}]}', + "json", + ) == [{"records": [{"id": 1}], "data": [{"id": 2}]}] + assert extract_structured_records( + '{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}', + "json", + ) == [{"id": 1}] + assert extract_structured_records( + '{"payload":{"data":[{"id":2}],"total":1}}', + "json", + ) == [{"id": 2}] + assert extract_structured_records('{"records":[],"total":0}', "json") == [] + # 包装数组中的非对象不是记录集合，整体按一条业务对象保留。 + assert extract_structured_records('{"data":[1,2]}', "json") == [ + {"data": [1, 2]} + ] + + +def test_json_record_locators_cover_pretty_and_minified_sources() -> None: + pretty = ( + '{\n "records": [\n {"id": 1},\n' + ' {\n "id": 2\n }\n ],\n "total": 2\n}' + ) + parsed = parse_text_content(pretty, filename="pretty.json") + assert [ + pretty[locator["source_start"] : locator["source_end"]] + for locator in parsed.record_locators + ] == ['{"id": 1}', '{\n "id": 2\n }'] + assert [ + (locator["start_line"], locator["end_line"]) + for locator in parsed.record_locators + ] == [(3, 3), (4, 6)] + + minified = '[{"id":1},{"id":2}]' + parsed = parse_text_content(minified, filename="minified.json") + assert [ + minified[locator["source_start"] : locator["source_end"]] + for locator in parsed.record_locators + ] == ['{"id":1}', '{"id":2}'] + + +def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None: + source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]' + parsed = parse_text_content(source, filename="precise.json") + assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890") + assert parsed.records[1]["value"] == Decimal("1e400") + assert structured_json_dumps(parsed.records[0]) == ( + '{"value":0.123456789012345678901234567890}' + ) + assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}' + assert isinstance(parsed.records[0]["value"], Decimal) def test_desensitize_pii_returns_masked_text_and_counts() -> None: @@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None: assert preprocess_structured_records(clean_source, []) == clean_source assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [ {"id": "1", "name": "有效"}, + {"id": "", "name": "缺少关键字段"}, {"id": "2", "name": "有效"}, ] + hierarchy = [ + {"id": "1", "parent_id": None, "name": "根节点", "empty": ""}, + {"id": "2", "parent_id": "1", "name": "子节点", "empty": ""}, + {"id": "", "parent_id": "", "name": "", "empty": ""}, + ] + assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [ + {"id": "1", "parent_id": None, "name": "根节点"}, + {"id": "2", "parent_id": "1", "name": "子节点"}, + ] + nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}] assert "profile" in preprocess_structured_records(nested, [])[0] assert preprocess_structured_records(nested, ["detect_structure"])[0] == { @@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None: duplicates = [ {"customer_id": "C-1", "value": "first"}, {"customer_id": "C-1", "value": "updated"}, + {"customer_id": "C-1", "value": "first"}, {"customer_id": "", "value": "blank-one"}, {"customer_id": "", "value": "blank-two"}, ] - assert len(preprocess_structured_records(duplicates, [])) == 4 + assert len(preprocess_structured_records(duplicates, [])) == 5 deduplicated = preprocess_structured_records(duplicates, ["deduplicate"]) assert [record["value"] for record in deduplicated] == [ "first", + "updated", "blank-one", "blank-two", ] @@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None: ) +def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None: + masked, counts = desensitize_structured_record( + { + "table_name": "customer_profile", + "chinese_name": "zh_CN", + "english_name": "en_US", + "product_name": "智能助手", + "metadata.table_name": "customer_archive", + "name": "张三", + "contact_name": "李四", + "姓名": "王五", + "profile.name": "赵六", + } + ) + + assert masked == { + "table_name": "customer_profile", + "chinese_name": "zh_CN", + "english_name": "en_US", + "product_name": "智能助手", + "metadata.table_name": "customer_archive", + "name": "[NAME]", + "contact_name": "[NAME]", + "姓名": "[NAME]", + "profile.name": "[NAME]", + } + assert counts == { + "email": 0, + "phone": 0, + "id_card": 0, + "name": 4, + "total": 4, + } + + def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None: valid = { "instruction": "如何修改收货地址？", diff --git a/backend/tests/test_data_process_api.py b/backend/tests/test_data_process_api.py index 8a7d192..c93c4dc 100644 --- a/backend/tests/test_data_process_api.py +++ b/backend/tests/test_data_process_api.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from copy import deepcopy from io import BytesIO from pathlib import Path @@ -21,7 +22,12 @@ from app.modules.data_process.storage import ( LocalDataProcessStorage, get_data_process_storage, ) -from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store +from app.modules.data_process.store import ( + InvalidStateError, + NotFoundError, + get_data_process_store, + repeat_task_id, +) class FakeDataProcessStore: @@ -35,6 +41,7 @@ class FakeDataProcessStore: self.datasets: dict[str, dict[str, Any]] = {} self.models: dict[str, dict[str, Any]] = {} self.regeneration_prepared: set[str] = set() + self.repeat_requests: dict[tuple[str, str], str] = {} self.sequence = 0 def _id(self, prefix: str) -> str: @@ -150,6 +157,121 @@ class FakeDataProcessStore: "published_outputs_preserved": published_outputs_preserved, } + def _repeat_response( + self, + source_task_id: str, + repeated_task_id: str, + *, + created: bool, + ) -> dict[str, Any]: + task = self.get_task(repeated_task_id) + task["source_file_count"] = len(self.sources[repeated_task_id]) + task["preview_count"] = len(self.previews[repeated_task_id]) + return { + "task": task, + "source_task_id": source_task_id, + "created": created, + "copied_source_file_count": len(self.sources[repeated_task_id]), + "copied_preview_count": len(self.previews[repeated_task_id]), + } + + def find_repeated_task( + self, + source_task_id: str, + request_id: str, + ) -> dict[str, Any] | None: + repeated_task_id = self.repeat_requests.get((source_task_id, request_id)) + if repeated_task_id is None: + return None + return self._repeat_response( + source_task_id, + repeated_task_id, + created=False, + ) + + def repeat_task( + self, + source_task_id: str, + *, + expected_updated_at: str, + request_id: str, + file_copies: dict[str, dict[str, str]], + ) -> dict[str, Any]: + existing = self.find_repeated_task(source_task_id, request_id) + if existing is not None: + return existing + source_task = self.get_task(source_task_id) + if source_task["status"] != "completed" or source_task.get("results_confirmed") is False: + raise InvalidStateError("只有已完成并确认结果的任务可以再次生成") + if source_task.get("updated_at") != expected_updated_at: + raise InvalidStateError("源任务已被其他操作修改，请刷新后重试") + source_files = self.sources[source_task_id] + if set(file_copies) != {str(item["id"]) for item in source_files}: + raise InvalidStateError("源文件快照已变化，请刷新后重试") + if not self.previews[source_task_id]: + raise InvalidStateError("源任务没有可用于再次生成的切分结果") + + repeated_task_id = repeat_task_id(source_task_id, request_id) + suffix = f"（再次生成-{repeated_task_id[-6:]}）" + task = { + **deepcopy(source_task), + "id": repeated_task_id, + "name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}", + "status": "pending", + "progress": 20, + "output_dataset_id": None, + "output_datasets": [], + "output_count": 0, + "filtered_count": 0, + "duplicate_count": 0, + "error_count": 0, + "failure_reason": None, + "generation_run_id": None, + "results_confirmed": False, + "workflow_step": "preview", + "preview_status": "completed", + "preview_progress": 100, + "preview_run_id": None, + "preview_failure_reason": None, + "preview_total_files": len(source_files), + "preview_completed_files": len(source_files), + "started_at": None, + "completed_at": None, + } + self.tasks[repeated_task_id] = task + self.sources[repeated_task_id] = [] + file_id_map: dict[str, str] = {} + for source in source_files: + old_file_id = str(source["id"]) + copy = file_copies[old_file_id] + file_id_map[old_file_id] = copy["id"] + self.sources[repeated_task_id].append( + { + **deepcopy(source), + "id": copy["id"], + "task_id": repeated_task_id, + "storage_object_id": copy["storage_object_id"], + } + ) + self.previews[repeated_task_id] = [ + { + **deepcopy(item), + "id": self._id("dpp"), + "task_id": repeated_task_id, + "source_file_id": file_id_map.get(str(item.get("source_file_id"))) + if item.get("source_file_id") + else None, + } + for item in self.previews[source_task_id] + ] + self.results[repeated_task_id] = [] + self.repeat_requests[(source_task_id, request_id)] = repeated_task_id + return self._repeat_response( + source_task_id, + repeated_task_id, + created=True, + ) + def delete_task(self, task_id: str, **_: Any) -> None: self.get_task(task_id) del self.tasks[task_id] @@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None: listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview") assert listed_preview.json()["data"]["total"] == 2 preview_item = listed_preview.json()["data"]["items"][0] + source_locator = preview_item["quality_score"]["source_locator"] + assert source_locator == { + "kind": "jsonl", + "record_index": 1, + "start_line": 1, + "end_line": 1, + "source_start": 0, + "source_end": len(preview_item["original_content"]), + } updated_preview = client.put( f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}", json={ @@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None: }, ) assert "quality_score" in updated_preview.json()["data"] + assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator generated = client.post(f"/modelTF/data-process/{task_id}/generate") assert generated.status_code == 200 @@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"] +def test_completed_task_can_repeat_into_an_independent_background_task( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, store, storage = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={ + "name": "原始生成任务", + "process_type": "structured", + "config": {"qa_pairs_per_row": 1, "temperature": 0.3}, + }, + ).json()["data"]["id"] + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")}, + ) + assert uploaded.status_code == 200 + source_id = uploaded.json()["data"]["files"][0]["id"] + built = client.post( + f"/modelTF/data-process/{task_id}/preview/build", + json={"replace_existing": True}, + ) + assert built.status_code == 200 + store.tasks[task_id].update( + status="completed", + progress=100, + results_confirmed=True, + workflow_step="results", + output_count=1, + output_dataset_id="dataset-original", + updated_at="2026-07-28T12:00:00Z", + ) + store.results[task_id] = [{"id": "result-original", "output": "原结果"}] + store.datasets["dataset-original"] = { + "id": "dataset-original", + "name": "原数据集", + "type": "train", + "source_task_id": task_id, + "deleted_at": None, + } + original_task = deepcopy(store.tasks[task_id]) + original_sources = deepcopy(store.sources[task_id]) + original_previews = deepcopy(store.previews[task_id]) + original_results = deepcopy(store.results[task_id]) + original_datasets = deepcopy(store.datasets) + monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None) + + payload = { + "expected_updated_at": "2026-07-28T12:00:00Z", + "request_id": "repeat-request-0001", + } + response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload) + + assert response.status_code == 202 + repeated = response.json()["data"] + repeated_task_id = repeated["task"]["id"] + assert repeated["created"] is True + assert repeated_task_id != task_id + assert repeated["task"]["status"] == "running" + assert repeated["task"]["workflow_step"] == "generate" + assert repeated["copied_source_file_count"] == 1 + assert repeated["copied_preview_count"] == len(original_previews) + assert store.tasks[task_id] == original_task + assert store.sources[task_id] == original_sources + assert store.previews[task_id] == original_previews + assert store.results[task_id] == original_results + assert store.datasets == original_datasets + + repeated_source = store.sources[repeated_task_id][0] + repeated_preview = store.previews[repeated_task_id][0] + assert repeated_source["id"] != source_id + assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"] + assert repeated_preview["id"] != original_previews[0]["id"] + assert repeated_preview["source_file_id"] == repeated_source["id"] + assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n' + + replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload) + assert replay.status_code == 202 + assert replay.json()["data"]["created"] is False + assert replay.json()["data"]["task"]["id"] == repeated_task_id + assert len(store.tasks) == 2 + assert len(store.sources[repeated_task_id]) == 1 + + +def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task( + tmp_path: Path, +) -> None: + client, store, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={"name": "源任务", "process_type": "structured", "config": {}}, + ).json()["data"]["id"] + store.tasks[task_id].update( + status="completed", + results_confirmed=True, + updated_at="2026-07-28T12:00:00Z", + ) + before = deepcopy(store.tasks) + + response = client.post( + f"/modelTF/data-process/{task_id}/repeat", + json={ + "expected_updated_at": "2026-07-28T11:59:59Z", + "request_id": "repeat-request-stale", + }, + ) + + assert response.status_code == 409 + assert store.tasks == before + + def test_published_split_datasets_remain_in_detail_after_regeneration( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results( assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0 +def test_structured_deduplication_preserves_distinct_rows_after_desensitization( + tmp_path: Path, +) -> None: + client, _, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={ + "name": "先去重再脱敏", + "process_type": "structured", + "config": {"preprocess_options": ["deduplicate", "desensitize"]}, + }, + ).json()["data"]["id"] + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files={ + "files": ( + "names.jsonl", + ( + '{"name":"张三","role":"开发"}\n' + '{"name":"李四","role":"开发"}\n' + ), + "application/jsonl", + ) + }, + ) + assert uploaded.status_code == 200 + + preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") + + assert preview.status_code == 200 + items = preview.json()["data"]["items"] + assert len(items) == 2 + assert len({item["original_content"] for item in items}) == 2 + assert {item["edited_content"] for item in items} == { + '{"name":"[NAME]","role":"开发"}' + } + + +def test_structured_deduplication_removes_identical_rows_across_sources( + tmp_path: Path, +) -> None: + client, _, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={ + "name": "跨源原文去重", + "process_type": "structured", + "config": {"preprocess_options": ["deduplicate", "desensitize"]}, + }, + ).json()["data"]["id"] + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files=[ + ( + "files", + ( + "first.jsonl", + '{"name":"张三","role":"开发"}\n', + "application/jsonl", + ), + ), + ( + "files", + ( + "second.jsonl", + '\n{"name":"张三","role":"开发"}\n', + "application/jsonl", + ), + ), + ], + ) + assert uploaded.status_code == 200 + first_source, second_source = uploaded.json()["data"]["files"] + + preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") + + assert preview.status_code == 200 + data = preview.json()["data"] + assert data["total"] == 1 + assert data["file_counts"] == { + first_source["id"]: 1, + second_source["id"]: 0, + } + + def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None: store = FakeDataProcessStore() task = store.create_task( @@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None: ) preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert preview.status_code == 200 - assert preview.json()["data"]["total"] == 2 + preview_items = preview.json()["data"]["items"] + assert len(preview_items) == 2 + assert [item["quality_score"]["source_locator"] for item in preview_items] == [ + { + "kind": "xlsx", + "record_index": 1, + "sheet_index": 0, + "sheet_name": "Sheet", + "row_number": 2, + "sheet_record_index": 0, + }, + { + "kind": "xlsx", + "record_index": 2, + "sheet_index": 0, + "sheet_name": "Sheet", + "row_number": 3, + "sheet_record_index": 1, + }, + ] def test_docx_preview_preserves_document_block_order_and_source_offsets( @@ -2757,6 +3105,218 @@ def _preview_task( ) +def _structured_preview_task( + content: str, + *, + file_format: str, + options: list[str] | None = None, +) -> list[dict[str, Any]]: + return data_process_endpoint._build_preview_items( + { + "process_type": "structured", + "config": {"preprocess_options": options or []}, + }, + [ + { + "id": "structured-source", + "name": f"records.{file_format}", + "file_format": file_format, + "content": content, + } + ], + ) + + +def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None: + json_source = '{"records":[{"id":1},{"id":2}]}' + json_items = _structured_preview_task( + json_source, + file_format="json", + ) + assert [ + item["quality_score"]["source_locator"]["json_pointer"] + for item in json_items + ] == ["/records/0", "/records/1"] + assert [ + json_source[item["source_start"] : item["source_end"]] + for item in json_items + ] == ['{"id":1}', '{"id":2}'] + assert [item["source_start_line"] for item in json_items] == [1, 1] + + jsonl_source = '{"id":1}\n\n{"id":2}' + jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl") + assert [ + item["quality_score"]["source_locator"]["record_index"] + for item in jsonl_items + ] == [1, 2] + assert [item["source_start_line"] for item in jsonl_items] == [1, 3] + assert [ + jsonl_source[item["source_start"] : item["source_end"]] + for item in jsonl_items + ] == ['{"id":1}', '{"id":2}'] + + csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain' + csv_items = _structured_preview_task(csv_source, file_format="csv") + assert [ + (item["source_start_line"], item["source_end_line"]) + for item in csv_items + ] == [(2, 3), (5, 5)] + assert [ + csv_source[item["source_start"] : item["source_end"]] + for item in csv_items + ] == ['1,"hello\nworld"', "2,plain"] + + +def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None: + client, _, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={"name": "空 JSON", "process_type": "structured", "config": {}}, + ).json()["data"]["id"] + + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files=[ + ("files", ("empty-array.json", "[]", "application/json")), + ( + "files", + ("empty-wrapper.json", '{"records":[],"total":0}', "application/json"), + ), + ], + ) + + assert uploaded.status_code == 200 + assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0] + preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") + assert preview.status_code == 200 + assert preview.json()["data"]["items"] == [] + assert preview.json()["data"]["total"] == 0 + assert set(preview.json()["data"]["file_counts"].values()) == {0} + + +def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers( + tmp_path: Path, +) -> None: + client, _, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={"name": "严格 JSON", "process_type": "structured", "config": {}}, + ).json()["data"]["id"] + invalid_sources = ( + ("duplicate.json", '{"id":1,"id":2}'), + ("duplicate.jsonl", '{"id":1,"id":2}\n'), + ("nan.json", '{"value":NaN}'), + ("infinity.json", '{"value":Infinity}'), + ("control.json", '{"value":"bad\x00control"}'), + ("deep.json", "[" * 10_000 + "0" + "]" * 10_000), + ) + + for filename, content in invalid_sources: + response = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files={"files": (filename, content, "application/json")}, + ) + assert response.status_code == 400, (filename, response.text) + + +def test_structured_json_preview_preserves_precision_and_business_data_field( + tmp_path: Path, +) -> None: + client, _, _ = make_client(tmp_path) + task_id = client.post( + "/modelTF/data-process", + json={"name": "无损 JSON", "process_type": "structured", "config": {}}, + ).json()["data"]["id"] + precise = '{"value":0.123456789012345678901234567890}' + business = '{"id":7,"data":[{"id":8}]}' + uploaded = client.post( + f"/modelTF/data-process/{task_id}/source-files", + files=[ + ("files", ("precise.json", precise, "application/json")), + ("files", ("business.json", business, "application/json")), + ], + ) + assert uploaded.status_code == 200 + assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1] + + preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") + assert preview.status_code == 200 + items = preview.json()["data"]["items"] + assert [item["original_content"] for item in items] == [precise, business] + assert [ + item["quality_score"]["source_locator"]["json_pointer"] for item in items + ] == ["", ""] + assert [item["source_start"] for item in items] == [0, 0] + + +def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None: + source_records = [ + {"id": "A", "amount": 10, "empty": ""}, + {"id": "A", "amount": 10, "empty": ""}, + {"id": "", "amount": 11, "empty": ""}, + {"id": "B", "amount": 11, "empty": ""}, + {"id": "C", "amount": 12, "empty": ""}, + {"id": "D", "amount": 12, "empty": ""}, + {"id": "E", "amount": 13, "empty": ""}, + {"id": "F", "amount": 13, "empty": ""}, + {"id": "G", "amount": 14, "empty": ""}, + {"id": "H", "amount": 1000, "empty": ""}, + ] + source = "\n".join( + json.dumps(record, ensure_ascii=False, separators=(",", ":")) + for record in source_records + ) + items = _structured_preview_task( + source, + file_format="jsonl", + options=["clean_invalid", "deduplicate", "filter_anomaly"], + ) + + assert [ + item["quality_score"]["source_locator"]["record_index"] + for item in items + ] == [1, 3, 4, 5, 6, 7, 8, 9] + assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9] + assert [json.loads(item["original_content"])["id"] for item in items] == [ + "A", + "", + "B", + "C", + "D", + "E", + "F", + "G", + ] + + +def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None: + source_records = [ + {"customer_id": "C-1", "status": "old"}, + {"customer_id": "C-1", "status": "new"}, + {"status": "old", "customer_id": "C-1"}, + ] + source = "\n".join( + json.dumps(record, ensure_ascii=False, separators=(",", ":")) + for record in source_records + ) + + items = _structured_preview_task( + source, + file_format="jsonl", + options=["clean_invalid", "deduplicate"], + ) + + assert [ + item["quality_score"]["source_locator"]["record_index"] + for item in items + ] == [1, 2] + assert [item["source_start_line"] for item in items] == [1, 2] + assert [json.loads(item["original_content"])["status"] for item in items] == [ + "old", + "new", + ] + + def test_fixed_preview_preserves_source_offsets() -> None: content = ( "# 第一章\n" diff --git a/backend/tests/test_data_process_storage.py b/backend/tests/test_data_process_storage.py index b31f6c2..cd14e3f 100644 --- a/backend/tests/test_data_process_storage.py +++ b/backend/tests/test_data_process_storage.py @@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat _assert_staging_empty(storage) +def test_stage_copy_creates_an_independently_deletable_source_object( + tmp_path: Path, +) -> None: + storage = LocalDataProcessStorage(tmp_path / "storage") + original = _stage(storage, content=b"immutable source") + storage.publish([original]) + + copied = storage.stage_copy( + batch_id="batch-copy", + source_reference=original.reference, + expected_source_task_id="task-1", + expected_source_file_id="source-1", + task_id="task-2", + source_file_id="source-2", + version=1, + name="source.txt", + ) + storage.publish([copied]) + + assert storage.read(copied.reference) == b"immutable source" + assert storage.delete( + original.reference, + expected_task_id="task-1", + expected_source_file_id="source-1", + ) is True + assert storage.read(copied.reference) == b"immutable source" + assert storage.delete( + copied.reference, + expected_task_id="task-2", + expected_source_file_id="source-2", + ) is True + _assert_staging_empty(storage) + + def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None: storage = LocalDataProcessStorage(tmp_path / "storage") diff --git a/backend/tests/test_data_process_store.py b/backend/tests/test_data_process_store.py index 15982e0..f397bfc 100644 --- a/backend/tests/test_data_process_store.py +++ b/backend/tests/test_data_process_store.py @@ -18,6 +18,7 @@ from app.modules.data_process.store import ( _preview_config_changed, _reasoning_output_is_valid, _source_storage_descriptor, + repeat_task_id, ) @@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore): yield self._conn +class _RepeatConnection: + def __init__(self) -> None: + self.source_files = [ + { + "id": "source-old", + "name": "source.jsonl", + "size_bytes": 12, + "record_count": 1, + "file_format": "jsonl", + "checksum_sha256": "a" * 64, + "content": '{"id":1}\n', + "content_preview": '{"id":1}', + "metadata": {"storage_backend": "local"}, + "created_by": "user-1", + } + ] + self.source_previews = [ + { + "id": "preview-old", + "source_file_id": "source-old", + "original_content": '{"id":1}', + "edited_content": '{"id":1,"checked":true}', + "source_start": 0, + "source_end": 8, + "source_start_line": 1, + "source_end_line": 1, + "token_count": 5, + "status": "modified", + "quality_score": {"overall": 90}, + } + ] + self.created_task: dict[str, Any] | None = None + self.created_files: list[dict[str, Any]] = [] + self.created_previews: list[dict[str, Any]] = [] + + def execute(self, sql: str, params: Any = None) -> _Result: + normalized = " ".join(sql.split()) + if params is not None: + assert normalized.count("%s") == len(params) + if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="): + return _Result(row=None) + if normalized.startswith("SELECT * FROM data_process_source_files"): + return _Result(rows=[dict(item) for item in self.source_files]) + if normalized.startswith("SELECT * FROM data_process_preview_items"): + return _Result(rows=[dict(item) for item in self.source_previews]) + if normalized.startswith("INSERT INTO data_process_tasks"): + self.created_task = { + "id": params[0], + "name": params[1], + "description": params[2], + "status": "pending", + "process_type": params[3], + "source_dataset_id": params[4], + "config": params[5], + "progress": 20, + "input_count": params[6], + "results_confirmed": False, + "workflow_step": "preview", + "preview_status": "completed", + "preview_progress": 100, + "preview_total_files": params[7], + "preview_completed_files": params[8], + "created_at": params[15], + "updated_at": params[16], + } + return _Result(row=dict(self.created_task)) + if normalized.startswith("INSERT INTO data_process_source_files"): + self.created_files.append( + { + "id": params[0], + "task_id": params[1], + "storage_object_id": params[2], + "content": params[8], + } + ) + return _Result() + if normalized.startswith("INSERT INTO data_process_preview_items"): + self.created_previews.append( + { + "id": params[0], + "task_id": params[1], + "source_file_id": params[2], + "edited_content": params[4], + } + ) + return _Result() + if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"): + return _Result( + row={ + "source_file_count": len(self.created_files), + "preview_count": len(self.created_previews), + } + ) + raise AssertionError(f"unexpected SQL: {normalized}") + + +class _RepeatStore(DataProcessStore): + def __init__(self, conn: _RepeatConnection) -> None: + self._conn = conn + + @contextmanager + def connect(self) -> Iterator[_RepeatConnection]: + yield self._conn + + def _task_in_connection( + self, + conn: Any, + task_id: str, + *, + for_update: bool = False, + ) -> dict[str, Any]: + assert task_id == "task-source" + assert for_update is True + return { + "id": task_id, + "name": "原任务", + "description": "原描述", + "status": "completed", + "process_type": "structured", + "source_dataset_id": None, + "config": { + "temperature": 0.3, + "_regeneration_prepared": {"prepared": True}, + }, + "results_confirmed": True, + "preview_status": "completed", + "tenant_id": "tenant-1", + "project_id": "project-1", + "owner_id": "owner-1", + "created_by": "user-1", + "updated_at": "2026-07-28T12:00:00Z", + } + + class _TaskListConnection: def __init__(self) -> None: self.task = { @@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None assert decoded == {"progress": 100.0, "duration_seconds": 389.0} +def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None: + conn = _RepeatConnection() + store = _RepeatStore(conn) + request_id = "repeat-request-0001" + target_task_id = repeat_task_id("task-source", request_id) + + repeated = store.repeat_task( + "task-source", + expected_updated_at="2026-07-28T12:00:00Z", + request_id=request_id, + file_copies={ + "source-old": { + "id": "source-new", + "storage_object_id": ( + f"local://data-process/{target_task_id}/source-new/v1/source.jsonl" + ), + } + }, + ) + + assert repeated["created"] is True + assert repeated["task"]["id"] == target_task_id + assert repeated["task"]["config"] == {"temperature": 0.3} + assert repeated["task"]["results_confirmed"] is False + assert repeated["copied_source_file_count"] == 1 + assert repeated["copied_preview_count"] == 1 + assert conn.created_files == [ + { + "id": "source-new", + "task_id": target_task_id, + "storage_object_id": ( + f"local://data-process/{target_task_id}/source-new/v1/source.jsonl" + ), + "content": '{"id":1}\n', + } + ] + assert conn.created_previews[0]["task_id"] == target_task_id + assert conn.created_previews[0]["source_file_id"] == "source-new" + assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}' + + def test_decode_row_decodes_aggregated_output_datasets_json() -> None: decoded = _decode_row( { diff --git a/frontend/scripts/regression-data-process-detail.mjs b/frontend/scripts/regression-data-process-detail.mjs index 83ae5c2..865194b 100644 --- a/frontend/scripts/regression-data-process-detail.mjs +++ b/frontend/scripts/regression-data-process-detail.mjs @@ -83,6 +83,11 @@ assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?d assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐') assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集') assert.match(detailSource, /function startRegeneration\(\)[\s\S]*?name: 'data-process-regenerate'[\s\S]*?params: \{ id: taskId\.value \}/, '重新生成按钮没有携带原任务 ID 进入命名路由') +assert.match(detailSource, /const canRepeatGeneration = computed[\s\S]*?status === 'completed'[\s\S]*?results_confirmed !== false[\s\S]*?previewCount\.value > 0/, '已完成任务缺少再次生成资格判断') +assert.match(detailSource, /repeatDataProcessTask\(taskId\.value,[\s\S]*?expected_updated_at: detail\.value\.updated_at[\s\S]*?request_id: repeatRequestId\.value/, '再次生成没有携带源任务版本和幂等请求 ID') +assert.match(detailSource, /name: 'data-process-workflow'[\s\S]*?params: \{ id: repeated\.task\.id \}/, '再次生成成功后没有进入新任务工作流') +assert.match(detailSource, /原任务和原结果不会被修改/, '再次生成确认提示没有说明原任务保持不变') +assert.match(detailSource, /v-if="canRepeatGeneration"[\s\S]*?@click="repeatGeneration"[\s\S]*?按原配置再生成一批/, '已完成任务详情缺少再次生成新批次入口') assert.match(detailSource, /const canRegenerate = computed\(\(\) => \{[\s\S]*?status === 'pending'[\s\S]*?status === 'failed'[\s\S]*?status === 'stopped'[\s\S]*?status === 'completed'[\s\S]*?outputDatasetId\.value[\s\S]*?hasPublishedOutputs\.value/, '详情页没有覆盖指针已清空但旧发布数据集仍存在的重新生成任务') assert.match(detailSource, /v-if="canRegenerate"[\s\S]*?@click="startRegeneration"[\s\S]*?重新生成/, '可恢复任务没有收敛为单一重新生成入口') assert.match(detailSource, /v-if="detail\.status === 'completed' && !hasCurrentPublishedDataset"[\s\S]*?@click="openPublishDialog"[\s\S]*?发布为三个数据集/, '未发布或发布指针失效的完成任务没有保留发布入口') @@ -115,6 +120,11 @@ assert.match(detailSource, /inputMetricCount\.toLocaleString\(\) \}\} \{\{ input assert.match(detailSource, /sourceFileCount\.toLocaleString\(\) \}\} 个/, '源文件数量缺少个数单位') assert.match(detailSource, /生成结果<\/span>\{\{ numeric\(detail\.output_count\)\.toLocaleString\(\) \}\} 条<\/strong>/, '生成结果数量缺少条数单位或仍误称成功输出') assert.match(detailSource, /const configExpanded = ref\(false\)/, '处理配置没有默认收起') +assert.match(detailSource, /appendGroup\(\['clean_invalid', 'deduplicate'\], '数据清洗'\)/, '详情页没有将完整清洗配置合并为数据清洗') +assert.match(detailSource, /appendGroup\(\['detect_structure', 'normalize_format'\], '结构标准化'\)/, '详情页没有将完整结构配置合并为结构标准化') +assert.match(detailSource, /历史部分配置/, '详情页没有标识旧任务的半组选项') +assert.match(detailSource, /异常数据过滤（历史规则）/, '详情页没有标识已停用的历史异常过滤规则') +assert.match(detailSource, /new Set\(value\.map/, '详情页没有去除历史预处理配置中的重复值') assert.match(detailSource, /:aria-expanded="configExpanded"/, '处理配置折叠按钮缺少无障碍状态') assert.match(detailSource, /[\s\S]*?v-show="configExpanded"/, '处理配置没有折叠过渡或内容状态') assert.doesNotMatch(detailSource, /const (?:detailMap|completedResults)\b|TODO: 接入真实接口/, '详情页仍包含本地 Mock 数据') @@ -126,6 +136,7 @@ for (const apiName of [ 'updateDataProcessResult', 'restoreDataProcessResult', 'publishDataProcess', + 'repeatDataProcessTask', ]) { assert.match( apiSource, @@ -136,5 +147,8 @@ for (const apiName of [ assert.match(apiSource, /keyword\?: string; status\?: string; split\?: string/, '结果列表 API 缺少服务端筛选参数') assert.match(apiSource, /\/results\/\$\{encodeURIComponent\(resultId\)\}/, '结果资源路径没有安全编码结果 ID') assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/publish`/, '发布 API 路径不正确') +assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/repeat`/, '再次生成 API 路径不正确') +assert.match(typesSource, /interface DataProcessRepeatPayload[\s\S]*?expected_updated_at: string[\s\S]*?request_id: string/, '再次生成请求契约不完整') +assert.match(typesSource, /interface DataProcessRepeatResult[\s\S]*?task: DataProcessTask[\s\S]*?source_task_id: string[\s\S]*?created: boolean/, '再次生成响应契约不完整') console.log('数据处理任务详情真实 API 回归检查通过') diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index e44e71c..0172d63 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -183,8 +183,36 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte assert.ok(typesSource.includes(field), `PreviewItem 缺少字段：${field}`) } assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识') +assert.match(typesSource, /sourceLocator\?: PreviewSourceLocator/, 'PreviewItem 缺少结构化来源定位契约') +assert.match(typesSource, /headingPath\?: string\[\]/, 'PreviewItem 缺少非结构化标题路径') +assert.match(typesSource, /PreviewSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, '前端来源定位 kind 未使用明确联合类型') +assert.match(contractTypesSource, /DataProcessSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, 'API 来源定位 kind 未使用明确联合类型') +for (const field of ['kind', 'record_index', 'start_line', 'end_line', 'source_start', 'source_end', 'json_pointer', 'sheet_index', 'sheet_name', 'row_number', 'sheet_record_index']) { + assert.ok(typesSource.includes(field), `PreviewSourceLocator 缺少字段：${field}`) + assert.ok(contractTypesSource.includes(field), `后端来源定位契约缺少字段：${field}`) +} +assert.match(contractTypesSource, /source_locator\?: DataProcessSourceLocator/, '质量信息缺少来源定位契约') +assert.match(contractTypesSource, /heading_path\?: string\[\]/, '质量信息缺少标题路径契约') +assert.match(viewSource, /const sourceLocator = item\.quality_score\?\.source_locator/, '预览映射丢失来源定位') +assert.match(viewSource, /sourceStart:\s*item\.source_start\s*\?\?\s*sourceLocator\?\.source_start/, 'JSON locator 的字符起点没有映射到预览项') +assert.match(viewSource, /sourceEnd:\s*item\.source_end\s*\?\?\s*sourceLocator\?\.source_end/, 'JSON locator 的字符终点没有映射到预览项') +assert.match(viewSource, /sourceStartLine:\s*item\.source_start_line\s*\?\?\s*sourceLocator\?\.start_line/, 'JSON locator 的起始行没有映射到预览项') +assert.match(viewSource, /sourceEndLine:\s*item\.source_end_line\s*\?\?\s*sourceLocator\?\.end_line/, 'JSON locator 的结束行没有映射到预览项') +assert.match(viewSource, /headingPath:[\s\S]*?item\.quality_score\?\.heading_path/, '预览映射丢失标题路径') assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤') -assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数') +assert.match(modelSource, /export function sourceLineWindow/, '缺少有界源文件行窗口函数') +assert.match(modelSource, /maxLines:\s*number/, '源文件行窗口缺少最大渲染行数参数') +assert.doesNotMatch(modelSource, /\.split\(\s*['"]\\n['"]\s*\)/, '源文件行窗口仍会先对全文 split') +assert.match(modelSource, /lines\.length < limit/, '源文件行扫描没有受最大行数约束') +assert.match(modelSource, /unicodeCodePointLength/, '源文件字符偏移未与后端 Unicode code point 计数保持一致') +assert.match(modelSource, /export function sourceLineNumberAtOffset/, '字符偏移缺少无数组的行号解析函数') +const manualPreviewHelperStart = modelSource.indexOf('export function isManualPreviewItem(') +const manualPreviewHelperEnd = modelSource.indexOf('\n}', manualPreviewHelperStart) +assert.ok(manualPreviewHelperStart >= 0, '缺少统一的手动预览项判定函数') +const manualPreviewHelperSource = modelSource.slice(manualPreviewHelperStart, manualPreviewHelperEnd + 2) +for (const field of ['status', 'originalContent', 'sourceStart', 'sourceEnd', 'sourceStartLine', 'sourceEndLine', 'sourcePages', 'sourceLocator']) { + assert.ok(manualPreviewHelperSource.includes(field), `手动预览项判定缺少来源字段：${field}`) +} assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法') assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态') const previewBuildBindingStart = viewSource.indexOf('useDataProcessPreviewBuild()') @@ -210,6 +238,30 @@ for (const marker of [ } assert.match(previewSource, /sourceStart/, '第四步未使用来源起始偏移') assert.match(previewSource, /sourceEnd/, '第四步未使用来源结束偏移') +const lineRangeStart = previewSource.indexOf('function lineRange(item: PreviewItem)') +const lineRangeEnd = previewSource.indexOf('\n}', lineRangeStart) +const lineRangeSource = previewSource.slice(lineRangeStart, lineRangeEnd + 2) +assert.match(lineRangeSource, /isManualPreviewItem\(item\)[\s\S]*?手动新增，无源文件定位/, '来源标签仍会把缺少行偏移的正常记录误判为手动新增') +assert.match(lineRangeSource, /props\.processType === 'unstructured'[\s\S]*?来源：源文件记录/, '结构化来源记录缺少无行偏移时的准确标签') +assert.doesNotMatch(lineRangeSource, /sourceStartLine == null[^\n]*手动新增/, '来源标签仍直接以缺少行号判定手动新增') +assert.match(lineRangeSource, /sheet_name[\s\S]*?row_number[\s\S]*?来源：\$\{sheet\} · 第 \$\{locator\.row_number\} 行/, 'XLSX 来源标签没有展示工作表和物理行号') +assert.match(lineRangeSource, /json_pointer[\s\S]*?JSON 路径/, 'JSON 来源标签没有展示 JSON 路径') +assert.match(lineRangeSource, /locator\?\.kind === 'json'[\s\S]*?JSON 根对象/, 'JSON 根对象来源标签被空 JSON Pointer 错误降级') +assert.match(lineRangeSource, /locatedLines[\s\S]*?第 \$\{locatedLines\.start\}[\s\S]*?locatedLines\.end/, 'JSONL/CSV 来源标签没有展示行范围') +assert.match(lineRangeSource, /headingPath[\s\S]*?章节：/, '非结构化来源标签没有合并标题路径') +assert.match(previewSource, /sourceLocator\?\.start_line[\s\S]*?sourceLocator\?\.end_line/, '文本预览没有优先使用后端行号定位') +assert.match(previewSource, /sourceLocator\?\.source_start\s*\?\?\s*item\.sourceStart/, '文本预览没有优先使用 locator 字符起点') +assert.match(previewSource, /sourceLocator\?\.source_end\s*\?\?\s*item\.sourceEnd/, '文本预览没有优先使用 locator 字符终点') +assert.match(previewSource, /data-line-number="line\.number"/, '文本预览行缺少稳定行号定位标识') +assert.match(previewSource, /isLineHighlighted\(line\.number, line\.start, line\.end\)/, '文本预览没有按物理行号高亮') +assert.match(previewSource, /querySelector\(`\[data-line-number=/, '选中记录后没有按物理行号滚动定位') +assert.match(previewSource, /const SOURCE_LINE_RENDER_LIMIT = 240/, '源文件查看器缺少安全渲染上限') +assert.match(previewSource, /const SOURCE_LINE_CHARACTER_LIMIT = 4_000/, '源文件查看器缺少单行字符渲染上限') +assert.match(previewSource, /sourceLineWindow\([\s\S]*?SOURCE_LINE_RENDER_LIMIT/, '源文件查看器没有使用有界行窗口') +assert.match(previewSource, /SOURCE_LINE_RENDER_LIMIT,[\s\S]*?SOURCE_LINE_CHARACTER_LIMIT,[\s\S]*?selectedSourceLine\.value,[\s\S]*?selectedSourceOffset\.value/, '单行超大 JSON 没有围绕选中来源构建字符窗口') +assert.match(previewSource, /sourceWindowStartLine/, '源文件查看器缺少窗口起始行状态') +assert.match(previewSource, /showPreviousSourceWindow[\s\S]*?showNextSourceWindow/, '源文件查看器缺少前后窗口导航') +assert.match(previewSource, /sourceLineNumberAtOffset\(props\.sourceText/, '仅有字符偏移时没有解析目标物理行') assert.match(previewSource, /filterable/, '文件选择器必须可搜索') assert.match(previewSource, /当前文件/, '预览缺少当前文件切换器') assert.doesNotMatch(previewSource, /located-badge|sync-label|已定位到/, '源文件栏不应显示冗余定位提示') @@ -269,6 +321,18 @@ for (const marker of [ ]) { assert.ok(officeViewerSource.includes(marker), `Word/XLSX 预览缺少结构或行为：${marker}`) } +assert.match(officeViewerSource, /const selectedXlsxLocator = computed/, 'XLSX 查看器没有读取精确来源定位') +assert.match(officeViewerSource, /row\.row_number === locator\.row_number/, 'XLSX 查看器没有按物理行号精确高亮') +assert.match(officeViewerSource, /row\.record_index === locator\.sheet_record_index/, 'XLSX 查看器没有按工作表记录序号精确高亮') +assert.match(officeViewerSource, /Math\.floor\(locator\.sheet_record_index \/ XLSX_PAGE_SIZE\) \* XLSX_PAGE_SIZE/, 'XLSX 查看器没有按记录序号自动计算分页') +assert.match(officeViewerSource, /activeSheetIndex\.value = targetSheet[\s\S]*?pageOffset\.value = targetOffset[\s\S]*?loadPreview\(\)/, '切换记录时 XLSX 查看器没有自动切工作表和分页') +const xlsxHighlightStart = officeViewerSource.indexOf('function xlsxRowHighlighted(') +const xlsxHighlightEnd = officeViewerSource.indexOf('\n}', xlsxHighlightStart) +const xlsxHighlightSource = officeViewerSource.slice(xlsxHighlightStart, xlsxHighlightEnd + 2) +assert.ok( + xlsxHighlightSource.indexOf('locator.row_number') < xlsxHighlightSource.indexOf('selectedRecordKey.value'), + 'XLSX 查看器没有把精确定位放在原内容比对 fallback 之前', +) const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue') const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue') @@ -454,7 +518,7 @@ assert.match( ) assert.match( workflowInitializationSource, - /sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?goToStep\(resumeStep\)[\s\S]*?resumeGeneration/, + /sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)/, '生成运行中时没有强制回到第五步并接管后台进度', ) const startGenerationHandler = viewSource.slice( @@ -463,7 +527,35 @@ const startGenerationHandler = viewSource.slice( ) assert.match(startGenerationHandler, /await persistWorkflowStep\('generate'\)[\s\S]*?await startGeneration\(\)[\s\S]*?dirty\.value = false/, '开始生成没有持久化第五步或启动真实后台任务') assert.doesNotMatch(startGenerationHandler, /router\.(?:push|replace)|allowLeave\s*=\s*true/, '开始生成后应停留在第五步，不得自动跳回列表') -assert.match(viewSource, /:disabled="currentStepId === 'generate' \|\| previewBuilding \|\| sourceUploading"/, '第五步底部返回按钮没有固定禁用') +assert.match( + generationSource, + /const canReturnFromGeneration = computed\(\(\) => \([\s\S]*?generation\.status === 'idle'[\s\S]*?!generationStarting\.value[\s\S]*?!generationRestoring\.value/, + '第五步返回权限没有区分未启动、启动中和恢复中状态', +) +assert.match( + viewSource, + /:disabled="\(currentStepId === 'generate' && !canReturnFromGeneration\) \|\| previewBuilding \|\| sourceUploading"/, + '第五步尚未启动生成时返回按钮仍被禁用', +) +const handleBackStart = viewSource.indexOf('async function handleBack()') +const handleBackEnd = viewSource.indexOf('\n}', handleBackStart) +const handleBackSource = viewSource.slice(handleBackStart, handleBackEnd + 2) +assert.match( + handleBackSource, + /currentStepId\.value === 'generate' && !canReturnFromGeneration\.value/, + '第五步处理函数仍无条件拦截返回', +) +assert.match( + generationSource, + /async function resumeGeneration\(\)[\s\S]*?generationRestoring\.value = true[\s\S]*?await getDataProcessProgress\(taskId\)[\s\S]*?generationRestoring\.value = false/, + '恢复已启动任务时存在短暂可返回的 idle 窗口', +) +assert.match(viewSource, /const resume = resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)[\s\S]*?await resume/, '第五步展示时未先启动恢复锁') +assert.match( + generationSource, + /const generationStarting = ref\(false\)[\s\S]*?generationStarting\.value = true[\s\S]*?generationStarting\.value = false/, + '点击开始生成后到请求启动前没有锁定返回状态', +) assert.match( viewSource, /generation\.status === 'success'[\s\S]*?persistWorkflowStep\('results'\)/, @@ -506,32 +598,34 @@ assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTy assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择') assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示') -const expectedStructuredOptions = [ - ['clean_invalid', '清理无效数据', '清理全空列，并剔除关键字段残缺的数据行'], +const expectedStructuredGroups = [ [ - 'detect_structure', - '嵌套结构展平', - '展平嵌套对象和可解析的 JSON 字段；Excel 表头与合并单元格在上传时自动解析', + "values: ['clean_invalid', 'deduplicate']", + '数据清洗', + '清理全空列和空记录，并删除内容完全相同的记录；不会猜测可空字段是否必填', ], [ - 'deduplicate', - '重复记录去重', - '按整行内容或 id、uuid、key、code、*_id 等身份字段去重，暂不支持自定义组合字段', + "values: ['detect_structure', 'normalize_format']", + '结构标准化', + '展平嵌套对象和可解析的 JSON 字段，并统一编码、空白、字段名和 JSON 序列化格式', ], - ['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'], - ['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值，并过滤乱码等异常记录'], - ['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'], + ["values: ['desensitize']", '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'], ] -for (const [value, label, description] of expectedStructuredOptions) { - assert.ok(structuredOptionsSource.includes(`value: '${value}'`), `结构化预处理缺少值：${value}`) +for (const [values, label, description] of expectedStructuredGroups) { + assert.ok(structuredOptionsSource.includes(values), `结构化预处理组合值不准确：${label}`) assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签：${label}`) - assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确：${value}`) + assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确：${label}`) } -const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)] - .map((match) => match[1]) -assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确') -assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一') -assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重') +assert.equal(expectedStructuredGroups.length, 3, '结构化预处理应收敛为 3 项') +const preprocessGroupsSource = structuredOptionsSource.slice( + structuredOptionsSource.indexOf('const PREPROCESS_GROUPS'), + structuredOptionsSource.indexOf('const legacyAnomalyFilterEnabled'), +) +assert.doesNotMatch(preprocessGroupsSource, /异常数据过滤|filter_anomaly|IQR/, '结构化新任务仍暴露异常数据过滤') +assert.match(structuredOptionsSource, /:indeterminate="groupIndeterminate\(group\.values\)"/, '历史部分选中的组合项没有半选回显') +assert.match(structuredOptionsSource, /function updatePreprocessGroup\([\s\S]*?new Set\(props\.options\.preprocessOptions\)[\s\S]*?next\.add\(value\)[\s\S]*?next\.delete\(value\)[\s\S]*?\[\.\.\.next\]/, '结构化预处理组合开关没有原子化更新或去重内部选项') +assert.match(typesSource, /仅用于恢复历史任务[\s\S]*?\| 'filter_anomaly'/, '异常数据过滤缺少历史兼容类型') +assert.match(structuredOptionsSource, /legacyAnomalyFilterEnabled[\s\S]*?历史任务[\s\S]*?结果可复现/, '历史异常过滤配置没有透明提示') assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类') for (const splitName of ['训练集', '验证集', '测试集']) { assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`) @@ -585,8 +679,20 @@ for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', } assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别') assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示') -assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验') -assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本') +const sourceValidationStart = sourceUploadWorkerSource.indexOf('export function validateSourceFileSelection(') +const sourceValidationEnd = sourceUploadWorkerSource.indexOf('\n}\n\nfunction unicodeCodePointLength', sourceValidationStart) +assert.ok(sourceValidationStart >= 0 && sourceValidationEnd > sourceValidationStart, '无法定位源文件选择校验函数') +const sourceValidationSource = sourceUploadWorkerSource.slice(sourceValidationStart, sourceValidationEnd + 2) +assert.doesNotMatch(sourceValidationSource, /file\.name === raw\.name[\s\S]{0,160}file\.size === raw\.size|同名且同大小/, '不同内容但同名同大小的文件仍会被前端误拒绝') +assert.match(sourceValidationSource, /selectedFiles\.length >= MAX_SOURCE_FILE_COUNT/, '移除伪重复校验时误删了文件数量限制') +assert.match(sourceValidationSource, /selectedBytes \+ raw\.size > MAX_SOURCE_BATCH_BYTES/, '移除伪重复校验时误删了批次大小限制') +assert.doesNotMatch(sourceUploadWorkerSource, /job\.file\.arrayBuffer\(|new TextDecoder/, '上传前仍把整个文本文件读入浏览器内存') +assert.match(sourceUploadWorkerSource, /export async function loadCanonicalSourceContent[\s\S]*?offset,[\s\S]*?limit: SOURCE_CONTENT_PAGE_CHARS/, '服务端 canonical content 没有按有界字符窗口读取') +assert.match(sourceUploadWorkerSource, /pending\.content = await loadCanonicalSourceContent\(currentTaskId, source\.id\)/, '上传成功后没有统一使用服务端 canonical content') +assert.doesNotMatch(sourceUploadWorkerSource, /\brawFile:\s*job\.file\b/, '上传成功状态仍长期保留原始 File') +assert.doesNotMatch(typesSource, /\brawFile\??:\s*File\b/, '上传状态类型仍长期持有原始 File') +assert.doesNotMatch(viewSource, /\brawFile:\s*raw\b/, '待上传列表仍复制保存原始 File') +assert.match(apiSource, /params:\s*\{[\s\S]*?offset\?: number[\s\S]*?limit\?: number[\s\S]*?\}/, '正文 API 前端契约缺少字符窗口参数') assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段') assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%') assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时') @@ -609,7 +715,7 @@ assert.match( /export interface DataProcessPreviewProgress[\s\S]*?workflow_step: DataProcessWorkflowStep[\s\S]*?preview_status: DataProcessPreviewStatus[\s\S]*?preview_progress: number[\s\S]*?preview_run_id/, '后台切分进度契约缺少步骤、状态、进度或任务代次', ) -for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) { +for (const field of ['status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) { assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段：${field}`) } assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整') @@ -853,13 +959,23 @@ const defaultStructuredPreprocess = defaultPreprocessValues( ) assert.deepEqual( defaultStructuredPreprocess, - ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], - '结构化默认预处理配置不准确', + [], + '结构化新任务不应默认勾选预处理', ) assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复') const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions') -assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整') +assert.deepEqual(defaultUnstructuredPreprocess, [], '非结构化新任务不应默认勾选预处理') assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复') +for (const field of ['preserveTables', 'preserveCodeBlocks', 'preserveLists']) { + assert.match( + stateSource, + new RegExp(`${field}:\\s*false`), + `非结构化预处理选项 ${field} 不应默认开启`, + ) +} +assert.match(structuredOptionsSource, /默认不执行预处理，请按数据情况自行选择/, '结构化预处理缺少默认不勾选说明') +assert.match(unstructuredOptionsSource, /默认不执行预处理，请按文档情况自行选择/, '非结构化预处理缺少默认不勾选说明') +assert.doesNotMatch(unstructuredOptionsSource, /默认启用结构感知/, '非结构化预处理仍保留默认启用的误导文案') const backendConfigStart = viewSource.indexOf('function toBackendConfig()') const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart) @@ -934,7 +1050,7 @@ for (const [field, fallback] of [ ) } assert.match(regenerationSource, /getDataProcessTask\(sourceTaskId\.value\)/, '重新生成没有加载原任务') -assert.match(regenerationSource, /while \(true\)[\s\S]*?getDataProcessSourceContent[\s\S]*?has_more/, '重新生成没有分页加载完整源正文') +assert.match(regenerationSource, /loadCanonicalSourceContent\(taskId, file\.id\)/, '重新生成没有复用分页 canonical 正文加载器') assert.match(regenerationSource, /getDataProcessPreview\(taskId, \{ page: 1, page_size: 500 \}\)[\s\S]*?for \(let page = 2; page <= pages;/, '重新生成没有分页加载全部现有切片') assert.match(viewSource, /if \(hydrating\.value\) return/, '任务水合期间仍可能触发重置副作用') assert.match(regenerationSource, /currentSignature !== originalPreviewConfigSignature\.value[\s\S]*?currentSignature === confirmedPreviewConfigSignature\.value/, '切分变更确认没有按原签名和已确认签名去重') @@ -948,7 +1064,7 @@ assert.match(regenerationSource, /if \(regenerationPrepared\.value\) \{[\s\S]*?g assert.match(regenerationSource, /regenerationPrepared\.value = true/, '重新生成提交成功后没有记录服务端已变更状态') assert.match(regenerationSource, /hydrateWorkspace\(regeneratedTask, !regenerated\.preview_invalidated\)/, '重新生成没有按 preview_invalidated 决定保留或清空切片') assert.match(regenerationSource, /重新生成配置已保存，但工作区恢复失败/, '重新生成配置已保存但水合失败时缺少可恢复错误状态') -assert.match(regenerationSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行') +assert.match(sourceUploadWorkerSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行') assert.doesNotMatch(regenerationSource, /binaryDocument[\s\S]*?mapDataProcessSourceFile\(file, ''\)/, '二进制源正文加载失败时不能静默降级为空内容') assert.match(nextFromModelSource, /if \(isRegeneration\.value\) \{[\s\S]*?prepareRegeneration\(taskPayload\(\)\)/, '重新生成每次从模型步骤继续时没有调用专用接口') assert.doesNotMatch(nextFromModelSource, /isRegeneration\.value && !taskId\.value/, '重新生成提交一次后可能错误转为普通任务更新') @@ -1017,6 +1133,17 @@ for (const mutationFunction of [ const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd) assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`) } +const updatePreviewContentStart = viewSource.indexOf('function updatePreviewContent(') +const updatePreviewContentEnd = viewSource.indexOf('\n}', updatePreviewContentStart) +const updatePreviewContentSource = viewSource.slice(updatePreviewContentStart, updatePreviewContentEnd + 2) +assert.match(updatePreviewContentSource, /isManualPreviewItem\(item\)/, '编辑预览内容仍未按稳定来源信息区分手动项') +assert.doesNotMatch(updatePreviewContentSource, /sourceStart == null/, '结构化来源记录编辑后仍会被误标为手动项') +const restorePreviewItemStart = viewSource.indexOf('function restorePreviewItem(') +const restorePreviewItemEnd = viewSource.indexOf('\n}', restorePreviewItemStart) +const restorePreviewItemSource = viewSource.slice(restorePreviewItemStart, restorePreviewItemEnd + 2) +assert.match(restorePreviewItemSource, /isManualPreviewItem\(item\)/, '恢复预览内容没有使用统一的手动项判定') +assert.doesNotMatch(restorePreviewItemSource, /sourceStart == null/, '结构化来源记录仍因缺少字符偏移而无法恢复') +assert.match(previewSource, /v-if="!isManualPreviewItem\(editingItem\)"/, '结构化来源记录的恢复原文按钮仍被错误隐藏') assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责') function findNextStyleBlockStart(source, startIndex) { diff --git a/frontend/src/api/modules/dataProcess.ts b/frontend/src/api/modules/dataProcess.ts index 0fc2bf3..a9c35e0 100644 --- a/frontend/src/api/modules/dataProcess.ts +++ b/frontend/src/api/modules/dataProcess.ts @@ -14,6 +14,8 @@ import type { DataProcessProgress, DataProcessRegeneratePayload, DataProcessRegenerateResult, + DataProcessRepeatPayload, + DataProcessRepeatResult, DataProcessPublishPayload, DataProcessPublishResult, DataProcessQualityScore, @@ -56,6 +58,8 @@ export type { DataProcessProgress, DataProcessRegeneratePayload, DataProcessRegenerateResult, + DataProcessRepeatPayload, + DataProcessRepeatResult, DataProcessPublishPayload, DataProcessPublishResult, DataProcessQualityScore, @@ -117,6 +121,15 @@ export const regenerateDataProcessTask = ( payload, ) +export const repeatDataProcessTask = ( + taskId: string | number, + payload: DataProcessRepeatPayload, +) => post( + `/data-process/${encodeURIComponent(taskId)}/repeat`, + payload, + { timeout: 5 * 60 * 1000 }, +) + export const deleteDataProcessTask = (taskId: string | number) => del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`) @@ -150,7 +163,12 @@ export const deleteDataProcessSourceFile = (taskId: string | number, fileId: str export const getDataProcessSourceContent = ( taskId: string | number, fileId: string | number, - params: { start_line?: number; line_count?: number } = {}, + params: { + start_line?: number + line_count?: number + offset?: number + limit?: number + } = {}, ) => get( `/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/content`, params, diff --git a/frontend/src/types/dataProcess.ts b/frontend/src/types/dataProcess.ts index 75c882a..3581c06 100644 --- a/frontend/src/types/dataProcess.ts +++ b/frontend/src/types/dataProcess.ts @@ -99,6 +99,20 @@ export interface DataProcessRegenerateResult { published_outputs_preserved: boolean } +export interface DataProcessRepeatPayload { + expected_updated_at: string + request_id: string +} + +export interface DataProcessRepeatResult { + task: DataProcessTask + source_task_id: string + created: boolean + copied_source_file_count: number + copied_preview_count: number + progress: DataProcessProgress +} + export type DataProcessTaskUpdatePayload = Partial export interface DataProcessSourceFile { @@ -232,6 +246,22 @@ export interface DataProcessPreviewItem { updated_at?: string } +export type DataProcessSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx' + +export interface DataProcessSourceLocator { + kind: DataProcessSourceLocatorKind + record_index?: number | null + start_line?: number | null + end_line?: number | null + source_start?: number | null + source_end?: number | null + json_pointer?: string | null + sheet_index?: number | null + sheet_name?: string | null + row_number?: number | null + sheet_record_index?: number | null +} + export interface DataProcessPreviewBuildPayload { replace_existing?: true source_file_ids?: Array @@ -369,6 +399,9 @@ export interface DataProcessQualityScore { is_valid?: boolean flags?: string[] fingerprint?: string + source_pages?: number[] + heading_path?: string[] + source_locator?: DataProcessSourceLocator [key: string]: unknown } diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index 268eaec..79cf870 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue' import PreviewCompareStep from './create/PreviewCompareStep.vue' import GenerationStep from './create/GenerationStep.vue' import ResultEditorStep from './create/ResultEditorStep.vue' -import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel' +import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel' import { createDefaultStructuredOptions, createDefaultUnstructuredOptions, @@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration' import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild' import { useDataProcessRegeneration } from './create/useDataProcessRegeneration' import { + loadCanonicalSourceContent, mapDataProcessSourceFile, useDataProcessSourceUpload, validateSourceFileSelection, @@ -33,7 +34,6 @@ import { deleteDataProcessPreview, deleteDataProcessSourceFile, getDataProcessPreview, - getDataProcessSourceContent, pullDataProcessExternalSource, testDataProcessExternalSource, updateDataProcessPreview, @@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false) let allowLeave = false const { bulkRegeneration, + canReturnFromGeneration, generation, + generationStarting, regeneratingResultId, resultRegenerationBusy, results, @@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => { if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play' return 'fa-arrow-right' }) - const previousStepLabel = computed(() => currentStep.value > 0 ? WIZARD_STEPS[currentStep.value - 1].title : '') @@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload { } function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem { + const sourceLocator = item.quality_score?.source_locator return { id: String(item.id), sourceFileId: String(item.source_file_id), originalContent: item.original_content, editedContent: item.edited_content, savedEditedContent: item.edited_content, - sourceStart: item.source_start, - sourceEnd: item.source_end, - sourceStartLine: item.source_start_line, - sourceEndLine: item.source_end_line, + sourceStart: item.source_start ?? sourceLocator?.source_start ?? null, + sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null, + sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null, + sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null, tokenCount: item.token_count, status: item.status, sourcePages: Array.isArray(item.quality_score?.source_pages) ? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number') : [], + sourceLocator, + headingPath: Array.isArray(item.quality_score?.heading_path) + ? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string') + : [], updatedAt: item.updated_at, } } @@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) { const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}` uploadedFiles.value.push({ uid: localUid, - rawFile: raw, name: raw.name, size: raw.size, count: 0, @@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) { previewProgress: 0, }) dirty.value = true - enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension }) + enqueueSourceUpload({ uid: localUid, file: raw }) } async function useSampleFile() { @@ -488,11 +493,8 @@ async function handlePullData() { const response = await pullDataProcessExternalSource(taskId.value, externalPayload()) const newFiles: UploadedDataFile[] = [] for (const file of response.files) { - const source = await getDataProcessSourceContent(taskId.value, file.id, { - start_line: 1, - line_count: 5000, - }) - newFiles.push(mapDataProcessSourceFile(file, source.content)) + const content = await loadCanonicalSourceContent(taskId.value, file.id) + newFiles.push(mapDataProcessSourceFile(file, content)) } uploadedFiles.value.push(...newFiles) externalConnected.value = true @@ -734,9 +736,14 @@ function selectPreviewItem(id: string) { function updatePreviewContent(id: string, value: string) { const item = previewItems.value.find((entry) => entry.id === id) if (!item) return + const isManual = isManualPreviewItem(item) item.editedContent = value item.tokenCount = estimateTokenCount(value) - item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified' + item.status = !value.trim() + ? 'invalid' + : value === item.originalContent + ? 'original' + : isManual ? 'manual' : 'modified' resetDownstream() dirty.value = true } @@ -758,7 +765,7 @@ async function syncPreviewChanges() { function restorePreviewItem(id: string) { const item = previewItems.value.find((entry) => entry.id === id) - if (!item || item.sourceStart == null) return + if (!item || isManualPreviewItem(item)) return item.editedContent = item.originalContent item.tokenCount = estimateTokenCount(item.originalContent) item.status = 'original' @@ -897,7 +904,7 @@ async function handleBack() { ElMessage.warning('请等待当前文件切分完成') return } - if (currentStepId.value === 'generate') return + if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return if (currentStep.value > 0) { const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id if (!targetStep) return @@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() { if (sourceTask.status === 'running') resumeStep = 'generate' if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload' if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate' + if (resumeStep === 'generate' || resumeStep === 'results') { + const resume = resumeGeneration() + goToStep(resumeStep) + await resume + return + } goToStep(resumeStep) - if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration() } onBeforeUnmount(() => { @@ -1154,7 +1166,7 @@ onMounted(() => {
返回：{{ previousStepLabel }} @@ -1167,8 +1179,8 @@ onMounted(() => { {{ primaryActionLabel }} diff --git a/frontend/src/views/data-process/DataProcessDetailView.vue b/frontend/src/views/data-process/DataProcessDetailView.vue index 000a645..2a0ce81 100644 --- a/frontend/src/views/data-process/DataProcessDetailView.vue +++ b/frontend/src/views/data-process/DataProcessDetailView.vue @@ -9,6 +9,7 @@ import { getDataProcessResults, getDataProcessTask, publishDataProcess, + repeatDataProcessTask, restoreDataProcessResult, updateDataProcessResult, } from '@/api/modules/dataProcess' @@ -42,6 +43,8 @@ const savingResult = ref(false) const restoringResultId = ref(null) const publishDialogVisible = ref(false) const publishing = ref(false) +const repeatGenerating = ref(false) +const repeatRequestId = ref('') const configExpanded = ref(false) const resultCellTooltipOptions = { popperClass: 'data-process-result-tooltip', @@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record = { preserve_context: '保留上下文', } +const structuredPreprocessOptionKeys = new Set([ + 'clean_invalid', + 'deduplicate', + 'detect_structure', + 'normalize_format', + 'desensitize', + 'filter_anomaly', +]) + +function formatStructuredPreprocessOptions(value: unknown[]) { + const options = [...new Set(value.map((item) => String(item)))] + const selected = new Set(options) + const consumed = new Set() + const labels: string[] = [] + + function appendGroup(values: string[], groupLabel: string) { + const selectedValues = values.filter((item) => selected.has(item)) + selectedValues.forEach((item) => consumed.add(item)) + if (selectedValues.length === values.length) { + labels.push(groupLabel) + return + } + selectedValues.forEach((item) => { + labels.push(`${preprocessOptionLabelMap[item] || item}（历史部分配置）`) + }) + } + + appendGroup(['clean_invalid', 'deduplicate'], '数据清洗') + appendGroup(['detect_structure', 'normalize_format'], '结构标准化') + + if (selected.has('desensitize')) { + consumed.add('desensitize') + labels.push('敏感信息脱敏') + } + if (selected.has('filter_anomaly')) { + consumed.add('filter_anomaly') + labels.push('异常数据过滤（历史规则）') + } + + options.forEach((item) => { + if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item) + }) + return labels.length ? labels.join('、') : '-' +} + function numeric(value: unknown) { const parsed = typeof value === 'number' ? value : Number(value) return Number.isFinite(parsed) ? parsed : 0 @@ -199,6 +247,11 @@ const canRegenerate = computed(() => { || status === 'stopped' || (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value)) }) +const canRepeatGeneration = computed(() => ( + detail.value?.status === 'completed' + && detail.value.results_confirmed !== false + && previewCount.value > 0 +)) const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-') const createTime = computed(() => detail.value?.create_time || detail.value?.created_at) const startTime = computed(() => detail.value?.start_time || detail.value?.started_at) @@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) { } if (Array.isArray(value)) { if (key === 'preprocess_options') { - return value.length - ? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、') - : '-' + const containsStructuredOption = value.some((item) => ( + structuredPreprocessOptionKeys.has(String(item)) + )) + return containsStructuredOption + ? formatStructuredPreprocessOptions(value) + : value.length + ? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、') + : '-' } return value.length ? value.join('、') : '-' } @@ -503,6 +561,48 @@ function startRegeneration() { void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } }) } +function createRepeatRequestId() { + if (typeof globalThis.crypto?.randomUUID === 'function') { + return globalThis.crypto.randomUUID() + } + return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}` +} + +async function repeatGeneration() { + if (!detail.value?.updated_at || repeatGenerating.value) return + try { + await ElMessageBox.confirm( + '系统会复制当前配置、源文件和切分结果，创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。', + '按原配置再生成一批？', + { + confirmButtonText: '创建并开始生成', + cancelButtonText: '取消', + type: 'info', + }, + ) + } catch { + return + } + + repeatGenerating.value = true + repeatRequestId.value ||= createRepeatRequestId() + try { + const repeated = await repeatDataProcessTask(taskId.value, { + expected_updated_at: detail.value.updated_at, + request_id: repeatRequestId.value, + }) + ElMessage.success(repeated.created ? '已创建新任务，正在后台生成' : '已恢复此前创建的新任务') + await router.push({ + name: 'data-process-workflow', + params: { id: repeated.task.id }, + }) + } catch { + // 保留幂等请求 ID；网络超时后再次点击不会重复创建任务。 + } finally { + repeatGenerating.value = false + } +} + watch([currentPage, pageSize], () => void loadResults()) onMounted(loadPage) @@ -520,22 +620,32 @@ onBeforeUnmount(() => { {{ displayStatus.label }} - - 发布为三个数据集 - - - 重新生成 - +
+ + 发布为三个数据集 + + + 按原配置再生成一批 + + + 覆盖当前任务重新生成 + +

{{ detail.description || '暂无任务描述' }}

@@ -804,7 +914,15 @@ onBeforeUnmount(() => { > p { margin: 8px 0 0; color: #64748b; font-size: 13px; } } -.publish-button { margin-left: auto; } +.heading-actions { + margin-left: auto; + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 8px; + + :deep(.el-button + .el-button) { margin-left: 0; } +} .load-state-actions { display: flex; gap: 10px; } .compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; } .publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; } @@ -968,7 +1086,8 @@ onBeforeUnmount(() => { @media (max-width: 720px) { .metric-grid, .config-grid { grid-template-columns: 1fr; } .detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; } - .publish-button { width: 100%; margin-left: 0; } + .heading-actions { width: 100%; margin-left: 0; } + .heading-actions :deep(.el-button) { width: 100%; } .publish-form-grid { grid-template-columns: 1fr; gap: 0; } .result-toolbar { align-items: stretch; flex-direction: column; } .result-filters { padding: 0 16px 16px; flex-direction: column; } diff --git a/frontend/src/views/data-process/create/OfficeSourceViewer.vue b/frontend/src/views/data-process/create/OfficeSourceViewer.vue index 1dfe363..563e668 100644 --- a/frontend/src/views/data-process/create/OfficeSourceViewer.vue +++ b/frontend/src/views/data-process/create/OfficeSourceViewer.vue @@ -46,6 +46,13 @@ const sourceUrl = computed(() => ( ? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId) : '' )) +const selectedXlsxLocator = computed(() => { + const locator = props.selectedItem?.sourceLocator + if (!locator) return null + const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name) + const hasRow = locator.row_number != null || locator.sheet_record_index != null + return hasSheet && hasRow ? locator : null +}) const visibleRowRange = computed(() => { const sheet = xlsxPreview.value?.active_sheet if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录' @@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => { }) function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) { + const locator = selectedXlsxLocator.value + const sheet = xlsxPreview.value?.active_sheet + if (locator && sheet) { + const sheetMatches = locator.sheet_index != null + ? sheet.index === locator.sheet_index + : sheet.name === locator.sheet_name + if (!sheetMatches) return false + if (locator.row_number != null) return row.row_number === locator.row_number + return row.record_index === locator.sheet_record_index + } return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value) } @@ -119,8 +136,11 @@ async function locateSelectedItem() { async function loadPreview(options: { reset?: boolean } = {}) { const sequence = ++loadSequence if (options.reset) { - activeSheetIndex.value = 0 - pageOffset.value = 0 + const locator = selectedXlsxLocator.value + activeSheetIndex.value = locator?.sheet_index ?? 0 + pageOffset.value = locator?.sheet_record_index == null + ? 0 + : Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE preview.value = null } errorMessage.value = '' @@ -178,8 +198,34 @@ watch( ) watch( - () => props.selectedItem?.id, - () => void locateSelectedItem(), + () => [ + props.selectedItem?.id, + props.selectedItem?.sourceLocator?.sheet_index, + props.selectedItem?.sourceLocator?.sheet_record_index, + props.selectedItem?.sourceLocator?.row_number, + ], + () => { + const locator = selectedXlsxLocator.value + if (!locator || isDocx.value) { + void locateSelectedItem() + return + } + const targetSheet = locator.sheet_index ?? activeSheetIndex.value + const targetOffset = locator.sheet_record_index == null + ? pageOffset.value + : Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE + const activeSheet = xlsxPreview.value?.active_sheet + if ( + activeSheet?.index === targetSheet + && activeSheet.offset === targetOffset + ) { + void locateSelectedItem() + return + } + activeSheetIndex.value = targetSheet + pageOffset.value = targetOffset + void loadPreview() + }, ) @@ -301,6 +347,8 @@ watch( :key="row.row_number" class="xlsx-row" :class="{ 'is-highlighted': xlsxRowHighlighted(row) }" + :data-row-number="row.row_number" + :data-record-index="row.record_index" > {{ row.row_number }} (null) const search = ref('') const currentPage = ref(1) const PREVIEW_PAGE_SIZE = 10 +const SOURCE_LINE_RENDER_LIMIT = 240 +const SOURCE_LINE_CHARACTER_LIMIT = 4_000 +const sourceWindowStartLine = ref(1) const editingItemId = ref(null) const editorDraft = ref('') -const lines = computed(() => sourceLines(props.sourceText)) const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0]) const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value)) const normalizedFileFormat = computed(() => ( @@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => ( )) const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf') const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value)) +const selectedSourceOffset = computed(() => { + const item = selectedItem.value + return item ? sourceOffsetRange(item)?.start ?? null : null +}) +const selectedSourceLine = computed(() => { + const item = selectedItem.value + if (!item) return null + return sourceLineRange(item)?.start + ?? (selectedSourceOffset.value == null + ? null + : sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value)) +}) +const visibleSourceWindow = computed(() => sourceLineWindow( + props.sourceText, + sourceWindowStartLine.value, + SOURCE_LINE_RENDER_LIMIT, + SOURCE_LINE_CHARACTER_LIMIT, + selectedSourceLine.value, + selectedSourceOffset.value, +)) +const lines = computed(() => visibleSourceWindow.value.lines) const filteredItems = computed(() => props.items.filter((item, index) => { const matchesSearch = !search.value.trim() @@ -58,10 +85,27 @@ const pagedItems = computed(() => { const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id)) -function isLineHighlighted(lineStart: number, lineEnd: number) { +function sourceLineRange(item: PreviewItem) { + const start = item.sourceLocator?.start_line ?? item.sourceStartLine + const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start + return start == null ? null : { start, end: end ?? start } +} + +function sourceOffsetRange(item: PreviewItem) { + const start = item.sourceLocator?.source_start ?? item.sourceStart + const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start + return start == null ? null : { start, end: Math.max(start, end ?? start) } +} + +function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) { const item = selectedItem.value - if (!item || item.sourceStart == null || item.sourceEnd == null) return false - return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd + if (!item) return false + const lineRange = sourceLineRange(item) + if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end + const offsetRange = sourceOffsetRange(item) + if (!offsetRange) return false + const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end) + return lineEnd >= offsetRange.start && lineStart < effectiveEnd } function selectItem(id: string) { @@ -107,34 +151,88 @@ watch(search, () => { watch(() => props.selectedFileId, closeEditor) -watch(selectedItem, async (item) => { +watch([selectedItem, () => props.sourceText], async ([item]) => { if (!item) return const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id) if (visibleIndex >= 0) { currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1 } - if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return + if (isPdfSource.value || isOfficeSource.value) return + const itemLineRange = sourceLineRange(item) + const itemOffsetRange = sourceOffsetRange(item) + if (!itemLineRange && !itemOffsetRange) { + sourceWindowStartLine.value = 1 + return + } + const targetLine = selectedSourceLine.value + ?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0) + sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3)) await nextTick() - const target = sourceViewerRef.value?.querySelector(`[data-source-start="${item.sourceStart}"]`) + const exactTarget = sourceViewerRef.value + ?.querySelector(`[data-line-number="${targetLine}"]`) + const target = exactTarget ?? sourceViewerRef.value?.querySelector('.source-line.is-highlighted') target?.scrollIntoView({ block: 'center', behavior: 'smooth' }) }, { immediate: true }) +async function showPreviousSourceWindow() { + sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT) + await nextTick() + if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0 +} + +async function showNextSourceWindow() { + if (!visibleSourceWindow.value.hasMore) return + sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1 + await nextTick() + if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0 +} + function itemNumber(item: PreviewItem) { return props.items.findIndex((entry) => entry.id === item.id) + 1 } function lineRange(item: PreviewItem) { - if (item.sourcePages?.length) { - const first = item.sourcePages[0] - const last = item.sourcePages[item.sourcePages.length - 1] - return first === last ? `来源：第 ${first} 页` : `来源：第 ${first}–${last} 页` + if (isManualPreviewItem(item)) return '手动新增，无源文件定位' + + const locator = item.sourceLocator + const locatedLines = sourceLineRange(item) + if (props.processType === 'unstructured') { + const parts: string[] = [] + if (item.sourcePages?.length) { + const first = item.sourcePages[0] + const last = item.sourcePages[item.sourcePages.length - 1] + parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`) + } + if (locatedLines) { + parts.push( + locatedLines.start === locatedLines.end + ? `第 ${locatedLines.start} 行` + : `第 ${locatedLines.start}–${locatedLines.end} 行`, + ) + } + if (item.headingPath?.length) parts.push(`章节：${item.headingPath.join(' / ')}`) + return parts.length ? `来源：${parts.join(' · ')}` : '来源：源文件内容（无精确定位）' } - if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增，无源文件定位' - return item.sourceStartLine === item.sourceEndLine - ? `来源：第 ${item.sourceStartLine} 行` - : `来源：第 ${item.sourceStartLine}–${item.sourceEndLine} 行` + + if (locator?.kind === 'xlsx') { + const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}` + return locator.row_number != null + ? `来源：${sheet} · 第 ${locator.row_number} 行` + : `来源：${sheet}` + } + if (locator?.kind === 'json') { + return locator.json_pointer + ? `来源：JSON 路径 ${locator.json_pointer}` + : '来源：JSON 根对象' + } + if (locatedLines) { + return locatedLines.start === locatedLines.end + ? `来源：第 ${locatedLines.start} 行` + : `来源：第 ${locatedLines.start}–${locatedLines.end} 行` + } + return '来源：源文件记录' } @@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
源文件 · {{ fileName }}
+
+ 第 {{ visibleSourceWindow.startLine }}–{{ visibleSourceWindow.endLine }} 行 + + 上一段 + + + 下一段 + +
{{ line.number }} {{ line.content || ' ' }} @@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) { />
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) { } } +.source-window-controls { + flex: none; + gap: 2px !important; + + > span { + margin-right: 4px; + color: #8a93a3; + font-size: 11px; + white-space: nowrap; + } + + :deep(.el-button) { + margin-left: 0; + } +} + .source-viewer { flex: 1; height: 538px; diff --git a/frontend/src/views/data-process/create/StructuredOptionsPanel.vue b/frontend/src/views/data-process/create/StructuredOptionsPanel.vue index f7930ae..0b5b7f0 100644 --- a/frontend/src/views/data-process/create/StructuredOptionsPanel.vue +++ b/frontend/src/views/data-process/create/StructuredOptionsPanel.vue @@ -1,4 +1,5 @@ @@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array) {

预处理选项

-

选择在生成问答对之前需要执行的数据处理方式

+

默认不执行预处理，请按数据情况自行选择

- - + + +
+
diff --git a/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue b/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue index b1cde5c..66438ca 100644 --- a/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue +++ b/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue @@ -119,7 +119,7 @@ defineExpose({ revealValidation })

预处理选项

-

默认启用结构感知的推荐策略，只需决定是否需要脱敏

+

默认不执行预处理，请按文档情况自行选择

diff --git a/frontend/src/views/data-process/create/dataProcessCreateState.ts b/frontend/src/views/data-process/create/dataProcessCreateState.ts index b7453ea..40b403f 100644 --- a/frontend/src/views/data-process/create/dataProcessCreateState.ts +++ b/frontend/src/views/data-process/create/dataProcessCreateState.ts @@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) { export function createDefaultStructuredOptions(): StructuredProcessOptions { return { - preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], + preprocessOptions: [], semanticEnrichment: false, qaPairsPerRow: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, @@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions { export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions { return { - preprocessOptions: [ - 'clean_invalid_content', - 'detect_document_structure', - 'merge_short_content', - 'filter_low_quality', - 'deduplicate_content', - 'preserve_context', - ], + preprocessOptions: [], chunkMethod: 'layout_hybrid', chunkSize: 800, chunkOverlap: 100, minChunkSize: 100, semanticBreakpointPercentile: 95, - preserveTables: true, - preserveCodeBlocks: true, - preserveLists: true, + preserveTables: false, + preserveCodeBlocks: false, + preserveLists: false, semanticEnrichment: false, qaPairsPerChunk: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, @@ -218,11 +211,21 @@ function generationOptionsFromConfig( export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions { const defaults = createDefaultStructuredOptions() const preprocessOptions = configValue(config, 'preprocess_options', []) + const supportedPreprocessOptions = new Set([ + 'clean_invalid', + 'deduplicate', + 'detect_structure', + 'normalize_format', + 'desensitize', + 'filter_anomaly', + ]) return { ...defaults, ...generationOptionsFromConfig(config, defaults), preprocessOptions: Array.isArray(preprocessOptions) - ? preprocessOptions.map(String) as PreprocessOption[] + ? Array.from(new Set(preprocessOptions.map(String).filter( + (option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption), + ))) : defaults.preprocessOptions, semanticEnrichment: Boolean(configValue( config, diff --git a/frontend/src/views/data-process/create/previewModel.ts b/frontend/src/views/data-process/create/previewModel.ts index 64c5d73..143e109 100644 --- a/frontend/src/views/data-process/create/previewModel.ts +++ b/frontend/src/views/data-process/create/previewModel.ts @@ -1,4 +1,4 @@ -import type { SourceLine } from './types' +import type { PreviewItem, SourceLine } from './types' /** 仅用于“使用示例”上传；正式预览和切片全部由后端生成。 */ export const DEFAULT_SOURCE_TEXT = [ @@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [ '答：复利是将上一期利息加入本金，再计算下一期利息。', ].join('\n') -/** - * 把后端返回的字符偏移映射为源文件行，仅负责界面高亮，不参与切片。 - */ -export function sourceLines(sourceText: string): SourceLine[] { - const rawLines = sourceText.split('\n') - let cursor = 0 +export interface SourceLineWindow { + lines: SourceLine[] + startLine: number + endLine: number + hasPrevious: boolean + hasMore: boolean +} - return rawLines.map((content, index) => { - const start = cursor - const end = start + content.length - cursor = end + (index < rawLines.length - 1 ? 1 : 0) - return { number: index + 1, content, start, end } - }) +function unicodeCodePointLength(value: string, start = 0, end = value.length) { + let length = 0 + let index = start + while (index < end) { + const codePoint = value.codePointAt(index) + index += codePoint != null && codePoint > 0xffff ? 2 : 1 + length += 1 + } + return length +} + +function advanceCodePoints(value: string, start: number, end: number, count: number) { + let index = start + let remaining = Math.max(0, count) + while (index < end && remaining > 0) { + const codePoint = value.codePointAt(index) + index += codePoint != null && codePoint > 0xffff ? 2 : 1 + remaining -= 1 + } + return index +} + +/** + * 只扫描并返回当前可见行窗口，不对全文 split，避免大文件生成巨量字符串数组。 + * 字符定位场景可开启 code point 偏移，以与后端 Python 的字符计数保持一致。 + */ +export function sourceLineWindow( + sourceText: string, + requestedStartLine: number, + maxLines: number, + maxCharactersPerLine: number, + focusLine: number | null = null, + focusOffset: number | null = null, +): SourceLineWindow { + const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1) + const limit = Math.max(1, Math.trunc(maxLines) || 1) + const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1) + const trackUnicodeOffsets = focusOffset != null + const lines: SourceLine[] = [] + let lineNumber = 1 + let jsCursor = 0 + let sourceCursor = 0 + + while (jsCursor <= sourceText.length && lineNumber < startLine) { + const newlineIndex = sourceText.indexOf('\n', jsCursor) + const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length + sourceCursor = trackUnicodeOffsets + ? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0) + : (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1) + jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1 + lineNumber += 1 + } + + while (jsCursor <= sourceText.length && lines.length < limit) { + const newlineIndex = sourceText.indexOf('\n', jsCursor) + const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length + const fullSourceEnd = trackUnicodeOffsets + ? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + : jsEnd + const focusedStart = focusLine === lineNumber && focusOffset != null + ? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3)) + : sourceCursor + const segmentSourceStart = Math.min( + focusedStart, + Math.max(sourceCursor, fullSourceEnd - characterLimit), + ) + const relativeSegmentStart = trackUnicodeOffsets + ? segmentSourceStart - sourceCursor + : Math.max(0, segmentSourceStart - jsCursor) + const segmentJsStart = advanceCodePoints( + sourceText, + jsCursor, + jsEnd, + relativeSegmentStart, + ) + const segmentJsEnd = advanceCodePoints( + sourceText, + segmentJsStart, + jsEnd, + characterLimit, + ) + const segmentLength = trackUnicodeOffsets + ? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd) + : segmentJsEnd - segmentJsStart + const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart + const end = start + segmentLength + const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}` + lines.push({ number: lineNumber, content, start, end }) + sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0) + jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1 + lineNumber += 1 + } + + return { + lines, + startLine: lines[0]?.number ?? startLine, + endLine: lines[lines.length - 1]?.number ?? startLine, + hasPrevious: startLine > 1, + hasMore: jsCursor <= sourceText.length, + } +} + +/** 根据后端 code point 偏移查找物理行号，不构建全文行数组。 */ +export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) { + const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0) + let offset = 0 + let lineNumber = 1 + for (const character of sourceText) { + if (offset >= normalizedOffset) break + if (character === '\n') lineNumber += 1 + offset += 1 + } + return lineNumber +} + +/** + * 手动新增项可能先以空内容保存为 invalid，编辑后又由后端标记为 modified， + * 因此不能只依赖可变的 status；空原文且完全没有来源定位才是稳定兜底。 + */ +export function isManualPreviewItem(item: PreviewItem): boolean { + const hasSourceLocation = item.sourceStart != null + || item.sourceEnd != null + || item.sourceStartLine != null + || item.sourceEndLine != null + || Boolean(item.sourcePages?.length) + || Boolean(item.sourceLocator) + return item.status === 'manual' || (!item.originalContent && !hasSourceLocation) } /** 与后端预览 token 估算规则一致，仅用于编辑中的即时计数。 */ diff --git a/frontend/src/views/data-process/create/types.ts b/frontend/src/views/data-process/create/types.ts index 017c018..a6d1e9d 100644 --- a/frontend/src/views/data-process/create/types.ts +++ b/frontend/src/views/data-process/create/types.ts @@ -24,6 +24,7 @@ export type PreprocessOption = | 'detect_structure' | 'deduplicate' | 'normalize_format' + /** 仅用于恢复历史任务，新任务界面不再提供。 */ | 'filter_anomaly' | 'desensitize' @@ -94,7 +95,6 @@ export interface ExternalDataSource { export interface UploadedDataFile { uid: string | number sourceFileId?: string - rawFile?: File name: string size: number count: number @@ -118,6 +118,22 @@ export interface SourceLine { end: number } +export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx' + +export interface PreviewSourceLocator { + kind: PreviewSourceLocatorKind + record_index?: number | null + start_line?: number | null + end_line?: number | null + source_start?: number | null + source_end?: number | null + json_pointer?: string | null + sheet_index?: number | null + sheet_name?: string | null + row_number?: number | null + sheet_record_index?: number | null +} + export interface PreviewItem { id: string sourceFileId: string @@ -129,6 +145,8 @@ export interface PreviewItem { sourceStartLine: number | null sourceEndLine: number | null sourcePages?: number[] + sourceLocator?: PreviewSourceLocator + headingPath?: string[] tokenCount: number status: 'original' | 'modified' | 'manual' | 'invalid' qualityScore?: number diff --git a/frontend/src/views/data-process/create/useDataProcessGeneration.ts b/frontend/src/views/data-process/create/useDataProcessGeneration.ts index 7adbbd2..73ea4d2 100644 --- a/frontend/src/views/data-process/create/useDataProcessGeneration.ts +++ b/frontend/src/views/data-process/create/useDataProcessGeneration.ts @@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) { let generationTimer: ReturnType | null = null let generationRun = 0 let pollFailureCount = 0 - let generationStarting = false + const generationStarting = ref(false) + const generationRestoring = ref(false) + const canReturnFromGeneration = computed(() => ( + generation.status === 'idle' && !generationStarting.value && !generationRestoring.value + )) function stopGenerationTimer() { generationRun += 1 @@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) { } async function startGeneration() { - if (generationStarting || generation.status === 'running') return false + if (generationStarting.value || generation.status === 'running') return false const taskId = bindings.taskId.value if (!taskId) { ElMessage.error('任务尚未创建，请返回上一步重试') return false } - generationStarting = true + generationStarting.value = true let runId: number | null = null try { const canStart = await bindings.beforeGenerate?.() @@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) { generation.message = error instanceof Error ? error.message : '启动数据处理失败，请重试。' return false } finally { - generationStarting = false + generationStarting.value = false } } async function resumeGeneration() { const taskId = bindings.taskId.value if (!taskId) return - stopGenerationTimer() - const activeRunId = generationRun - pollFailureCount = 0 + generationRestoring.value = true try { + stopGenerationTimer() + const activeRunId = generationRun + pollFailureCount = 0 const progress = await getDataProcessProgress(taskId) if (activeRunId !== generationRun) return if (progress.status === 'running') { @@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) { } catch (error) { generation.status = 'failed' generation.message = error instanceof Error ? error.message : '查询任务进度失败，请重试。' + } finally { + generationRestoring.value = false } } @@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) { return { bulkRegeneration, + canReturnFromGeneration, generation, + generationStarting, regeneratingResultId, resultRegenerationBusy, results, diff --git a/frontend/src/views/data-process/create/useDataProcessRegeneration.ts b/frontend/src/views/data-process/create/useDataProcessRegeneration.ts index 77906df..41d6b8f 100644 --- a/frontend/src/views/data-process/create/useDataProcessRegeneration.ts +++ b/frontend/src/views/data-process/create/useDataProcessRegeneration.ts @@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue' import { useRoute } from 'vue-router' import { getDataProcessPreview, - getDataProcessSourceContent, getDataProcessTask, regenerateDataProcessTask, } from '@/api/modules/dataProcess' @@ -15,7 +14,10 @@ import { createStructuredOptionsFromConfig, createUnstructuredOptionsFromConfig, } from './dataProcessCreateState' -import { mapDataProcessSourceFile } from './useDataProcessSourceUpload' +import { + loadCanonicalSourceContent, + mapDataProcessSourceFile, +} from './useDataProcessSourceUpload' import type { PreviewItem, ProcessType, @@ -50,24 +52,6 @@ interface RegenerationBindings { resetDownstream: () => void } -async function loadSourceContent(taskId: string, fileId: string | number) { - const chunks: string[] = [] - let startLine = 1 - while (true) { - const source = await getDataProcessSourceContent(taskId, fileId, { - start_line: startLine, - line_count: 10_000, - }) - chunks.push(source.content || '') - if (!source.has_more) break - const nextLine = Number(source.end_line || startLine) + 1 - if (nextLine <= startLine) break - startLine = nextLine - } - // source_content_lines 已保留原始换行；分页之间直接拼接，避免凭空增加空行并破坏偏移。 - return chunks.join('') -} - async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) { const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 }) const items = [...first.items] @@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) { async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) { const taskId = String(task.id) bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => ( - mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id)) + mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id)) ))) bindings.previewItems.value = preservePreviews ? await loadAllPreviews(taskId, bindings.mapPreviewItem) diff --git a/frontend/src/views/data-process/create/useDataProcessSourceUpload.ts b/frontend/src/views/data-process/create/useDataProcessSourceUpload.ts index ffad87c..b1858a1 100644 --- a/frontend/src/views/data-process/create/useDataProcessSourceUpload.ts +++ b/frontend/src/views/data-process/create/useDataProcessSourceUpload.ts @@ -6,7 +6,6 @@ import { } from '@/api/modules/dataProcess' import type { ProcessType, UploadedDataFile } from './types' -const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx']) const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx']) const UNSTRUCTURED_FILE_EXTENSIONS = new Set([ 'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson', @@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt']) const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024 const MAX_SOURCE_FILE_COUNT = 20 const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024 +const SOURCE_CONTENT_PAGE_CHARS = 1_000_000 interface SourceUploadJob { uid: string file: File - extension: string } interface SourceUploadOptions { @@ -60,9 +59,6 @@ export function validateSourceFileSelection( : '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX', } } - if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) { - return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' } - } if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) { return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` } } @@ -73,6 +69,34 @@ export function validateSourceFileSelection( return { valid: true, extension } } +function unicodeCodePointLength(value: string) { + let length = 0 + for (const _character of value) length += 1 + return length +} + +/** 分页读取服务端保存的规范化正文，避免重新使用浏览器本地解码结果。 */ +export async function loadCanonicalSourceContent( + taskId: string | number, + fileId: string | number, +) { + const chunks: string[] = [] + let offset = 0 + while (true) { + const source = await getDataProcessSourceContent(taskId, fileId, { + offset, + limit: SOURCE_CONTENT_PAGE_CHARS, + }) + const content = source.content || '' + chunks.push(content) + if (!source.has_more) break + const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content) + if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常，请删除文件后重试') + offset = nextOffset + } + return chunks.join('') +} + export function mapDataProcessSourceFile( file: DataProcessSourceFile, content = '', @@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) { pending.uploadError = undefined try { - let content = '' - if (!BINARY_FILE_EXTENSIONS.has(job.extension)) { - try { - content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer()) - } catch { - throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试') - } - if (!content.trim()) throw new Error('不能上传空文件') - } - const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => { pending.uploadProgress = progress }) @@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) { // 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。 Object.assign(pending, mapDataProcessSourceFile(source), { - rawFile: job.file, status: 'uploading', uploadProgress: 99, }) - if (BINARY_FILE_EXTENSIONS.has(job.extension)) { - try { - const parsed = await getDataProcessSourceContent(currentTaskId, source.id, { - start_line: 1, - line_count: 10_000, - }) - pending.content = parsed.content - } catch { - // 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。 - } - } else { - pending.content = content + try { + pending.content = await loadCanonicalSourceContent(currentTaskId, source.id) + } catch { + throw new Error('文件已上传，但服务端规范化内容读取失败，请删除文件后重试') } pending.status = 'ready'