from __future__ import annotations from copy import deepcopy from io import BytesIO from pathlib import Path from typing import Any import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from openpyxl import Workbook from app.api.v1.endpoints import data_process as data_process_endpoint from app.api.v1.endpoints.data_process import router from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text from app.modules.data_process.storage import ( DataProcessStorageError, LocalDataProcessStorage, get_data_process_storage, ) from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store class FakeDataProcessStore: """接口测试专用内存实现，确保测试不会连接或迁移真实数据库。""" def __init__(self) -> None: self.tasks: dict[str, dict[str, Any]] = {} self.sources: dict[str, list[dict[str, Any]]] = {} self.previews: dict[str, list[dict[str, Any]]] = {} self.results: dict[str, list[dict[str, Any]]] = {} self.datasets: dict[str, dict[str, Any]] = {} self.sequence = 0 def _id(self, prefix: str) -> str: self.sequence += 1 return f"{prefix}_{self.sequence}" def list_tasks(self, *, page: int, page_size: int, **filters: Any) -> dict[str, Any]: items = [ { **item, "source_file_count": len(self.sources.get(str(item["id"]), [])), } for item in self.tasks.values() ] for field in ("status", "process_type", "tenant_id", "project_id"): if filters.get(field): items = [item for item in items if item.get(field) == filters[field]] keyword = filters.get("keyword") if keyword: items = [item for item in items if keyword in item["name"]] return { "items": deepcopy(items[(page - 1) * page_size : page * page_size]), "total": len(items), "page": page, "page_size": page_size, } def create_task(self, payload: dict[str, Any]) -> dict[str, Any]: task_id = self._id("dpt") task = { "id": task_id, **deepcopy(payload), "status": "pending", "progress": 0, "input_count": 0, "output_count": 0, "filtered_count": 0, "duplicate_count": 0, "error_count": 0, "failure_reason": None, "output_dataset_id": None, } self.tasks[task_id] = task self.sources[task_id] = [] self.previews[task_id] = [] self.results[task_id] = [] return deepcopy(task) def get_task(self, task_id: str) -> dict[str, Any]: if task_id not in self.tasks: raise NotFoundError("data process task not found") task = deepcopy(self.tasks[task_id]) split_order = {"train": 0, "val": 1, "test": 2} task["output_datasets"] = sorted( ( deepcopy(dataset) for dataset in self.datasets.values() if dataset.get("source_task_id") == task_id and dataset.get("deleted_at") is None ), key=lambda dataset: split_order.get(str(dataset.get("type")), 3), ) return task def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: self.get_task(task_id) self.tasks[task_id].update(deepcopy(payload)) return self.get_task(task_id) def prepare_regeneration(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: task = self.tasks.get(task_id) if task is None: raise NotFoundError("data process task not found") if task["status"] == "running": raise InvalidStateError("running task cannot be prepared for regeneration") if payload["expected_updated_at"] != task.get("updated_at"): raise InvalidStateError("data process task was modified by another request") if payload["process_type"] != task["process_type"]: raise InvalidStateError("process_type cannot be changed during regeneration") published_outputs_preserved = bool(task.get("output_dataset_id")) self.results[task_id] = [] task.update( { "name": payload["name"], "description": payload["description"], "config": deepcopy(payload["config"]), "status": "pending", "progress": 20, "output_dataset_id": None, "output_count": 0, "filtered_count": 0, "duplicate_count": 0, "error_count": 0, "failure_reason": None, "generation_run_id": None, "started_at": None, "completed_at": None, "updated_at": "2026-07-25T20:00:00Z", } ) return { "task": deepcopy(task), "preview_invalidated": False, "published_outputs_preserved": published_outputs_preserved, } def delete_task(self, task_id: str, **_: Any) -> None: self.get_task(task_id) if self.tasks[task_id]["status"] == "running": raise InvalidStateError("running task must be stopped before deletion") del self.tasks[task_id] def list_source_files(self, task_id: str) -> list[dict[str, Any]]: self.get_task(task_id) return [ {key: value for key, value in item.items() if key != "content"} for item in self.sources[task_id] ] def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]: self.get_task(task_id) values = deepcopy(payload) source_id = str(values.pop("id", None) or self._id("dpsf")) storage_object_id = str( values.pop("storage_object_id", None) or f"db://data-process/{task_id}/{source_id}/v1" ) raw_size = int(values.pop("raw_size")) metadata = deepcopy(values.pop("metadata", {})) metadata.setdefault( "storage_backend", "local" if storage_object_id.startswith("local://data-process/") else "database", ) source = { "id": source_id, "task_id": task_id, "version_no": 1, "storage_object_id": storage_object_id, "size_bytes": raw_size, "metadata": metadata, **values, } self.sources[task_id].append(source) self.tasks[task_id]["input_count"] += payload["record_count"] return {key: value for key, value in deepcopy(source).items() if key != "content"} def add_source_files( self, task_id: str, files: list[dict[str, Any]] ) -> list[dict[str, Any]]: # 先验证整个批次，模拟数据库事务的 all-or-nothing 语义。 checksums = {item["checksum_sha256"] for item in self.sources.get(task_id, [])} incoming: set[str] = set() for payload in files: checksum = payload["checksum_sha256"] if checksum in checksums or checksum in incoming: raise ValueError("the same source file content is already attached to this task") incoming.add(checksum) created = [self.add_source_file(task_id, **payload) for payload in files] self.results[task_id] = [] self.tasks[task_id].update( { "status": "pending", "progress": 20 if self.previews[task_id] else 0, "output_count": 0, "filtered_count": 0, "duplicate_count": 0, "error_count": 0, "failure_reason": None, "generation_run_id": None, "started_at": None, "completed_at": None, } ) return created def get_source_file( self, task_id: str, file_id: str, *, include_content: bool = True ) -> dict[str, Any]: source = next( (item for item in self.sources.get(task_id, []) if item["id"] == file_id), None, ) if not source: raise NotFoundError("source file not found") result = deepcopy(source) if not include_content: result.pop("content", None) return result def source_content_window( self, task_id: str, file_id: str, offset: int, limit: int ) -> dict[str, Any]: source = self.get_source_file(task_id, file_id) content = source.pop("content") return { "file": source, "content": content[offset : offset + limit], "offset": offset, "limit": limit, "total_chars": len(content), "has_more": offset + limit < len(content), } def source_content_lines( self, task_id: str, file_id: str, start_line: int, line_count: int ) -> dict[str, Any]: source = self.get_source_file(task_id, file_id) lines = source.pop("content").splitlines(keepends=True) selected = lines[start_line - 1 : start_line - 1 + line_count] return { "file": source, "content": "".join(selected), "start_line": start_line, "end_line": start_line - 1 + len(selected), "line_count": len(selected), "total_lines": len(lines), "has_more": start_line - 1 + len(selected) < len(lines), } def delete_source_file(self, task_id: str, file_id: str) -> None: self.get_source_file(task_id, file_id) self.sources[task_id] = [item for item in self.sources[task_id] if item["id"] != file_id] self.previews[task_id] = [ item for item in self.previews[task_id] if item["source_file_id"] != file_id ] self.results[task_id] = [] def replace_preview_items( self, task_id: str, items: list[dict[str, Any]], *, source_file_ids: list[str] | None = None, ) -> list[dict[str, Any]]: created = [ {"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items ] if source_file_ids is None: self.previews[task_id] = created else: selected = set(source_file_ids) self.previews[task_id] = [ item for item in self.previews[task_id] if item["source_file_id"] not in selected ] + created self.results[task_id] = [] self.tasks[task_id]["progress"] = 20 return deepcopy(created) def list_preview_items( self, task_id: str, *, page: int, page_size: int, source_file_id: str | None = None, keyword: str | None = None, ) -> dict[str, Any]: items = self.previews[task_id] if source_file_id: items = [item for item in items if item["source_file_id"] == source_file_id] if keyword: items = [item for item in items if keyword in item["edited_content"]] return { "items": deepcopy(items[(page - 1) * page_size : page * page_size]), "total": len(items), "page": page, "page_size": page_size, } def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]: item = next( (item for item in self.previews.get(task_id, []) if item["id"] == preview_id), None, ) if not item: raise NotFoundError("preview item not found") return deepcopy(item) def create_preview_item(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: item = {"id": self._id("dpp"), "task_id": task_id, **deepcopy(payload)} self.previews[task_id].append(item) self.results[task_id] = [] return deepcopy(item) def update_preview_item( self, task_id: str, preview_id: str, payload: dict[str, Any] ) -> dict[str, Any]: item = next( (item for item in self.previews[task_id] if item["id"] == preview_id), None, ) if not item: raise NotFoundError("preview item not found") item.update(deepcopy(payload)) self.results[task_id] = [] return deepcopy(item) def delete_preview_item(self, task_id: str, preview_id: str) -> None: before = len(self.previews[task_id]) self.previews[task_id] = [ item for item in self.previews[task_id] if item["id"] != preview_id ] if len(self.previews[task_id]) == before: raise NotFoundError("preview item not found") def start_generation(self, task_id: str, *, replace_existing: bool) -> dict[str, Any]: if not self.previews[task_id]: raise InvalidStateError("preview must be built before generation") if replace_existing: self.results[task_id] = [] self.tasks[task_id].update( status="running", progress=30, output_count=0, generation_run_id=self._id("dprun"), ) return self.get_task(task_id) def generation_is_running(self, task_id: str, generation_run_id: str) -> bool: return ( self.tasks[task_id]["status"] == "running" and self.tasks[task_id].get("generation_run_id") == generation_run_id ) def update_generation_progress( self, task_id: str, generation_run_id: str, processed_count: int, total_count: int, ) -> bool: if not self.generation_is_running(task_id, generation_run_id): return False self.tasks[task_id]["progress"] = min( 95, 30 + processed_count / max(1, total_count) * 65, ) return True def complete_generation( self, task_id: str, results: list[dict[str, Any]], *, generation_run_id: str, **counts: Any, ) -> dict[str, Any]: if not self.generation_is_running(task_id, generation_run_id): return self.get_task(task_id) self.results[task_id] = deepcopy(results) self.tasks[task_id].update( status="completed", progress=100, output_count=len(results), generation_run_id=None, **counts, ) return self.get_task(task_id) def mark_failed( self, task_id: str, reason: str, *, generation_run_id: str ) -> dict[str, Any]: if self.generation_is_running(task_id, generation_run_id): self.tasks[task_id].update( status="failed", failure_reason=reason, generation_run_id=None, ) return self.get_task(task_id) def stop_task(self, task_id: str) -> dict[str, Any]: if self.tasks[task_id]["status"] != "running": raise InvalidStateError("only a running task can be stopped") self.tasks[task_id].update(status="stopped", generation_run_id=None) return self.get_task(task_id) def progress(self, task_id: str) -> dict[str, Any]: task = self.get_task(task_id) result = {key: task.get(key) for key in ( "status", "progress", "input_count", "output_count", "filtered_count", "duplicate_count", "error_count", "failure_reason", )} result["task_id"] = task["id"] return result def list_results( self, task_id: str, *, page: int, page_size: int, status: str | None = None, split: str | None = None, keyword: str | None = None, ) -> dict[str, Any]: items = self.results[task_id] if status: items = [item for item in items if item["status"] == status] if split: items = [item for item in items if item["split"] == split] if keyword: items = [ item for item in items if any(keyword in item[field] for field in ("instruction", "input", "output")) ] return { "items": deepcopy(items), "total": len(items), "page": page, "page_size": page_size, } def update_result( self, task_id: str, result_id: str, payload: dict[str, Any] ) -> dict[str, Any]: item = next((item for item in self.results[task_id] if item["id"] == result_id), None) if not item: raise NotFoundError("data process result not found") for field in ("instruction", "input", "output", "quality_score"): if field in payload: item[field] = deepcopy(payload[field]) hard_valid = bool(item["instruction"].strip() and item["output"].strip()) quality_valid = bool((item.get("quality_score") or {}).get("is_valid", hard_valid)) changed = any( item[field] != item[f"original_{field}"] for field in ("instruction", "input", "output") ) item["status"] = ( "invalid" if not hard_valid or not quality_valid else "modified" if changed else "valid" ) self.tasks[task_id]["error_count"] = sum( result["status"] == "invalid" for result in self.results[task_id] ) return deepcopy(item) def get_result(self, task_id: str, result_id: str) -> dict[str, Any]: item = next((item for item in self.results[task_id] if item["id"] == result_id), None) if not item: raise NotFoundError("data process result not found") return deepcopy(item) def restore_result(self, task_id: str, result_id: str) -> dict[str, Any]: item = next((item for item in self.results[task_id] if item["id"] == result_id), None) if not item: raise NotFoundError("data process result not found") for field in ("instruction", "input", "output"): item[field] = item[f"original_{field}"] item["status"] = "valid" return deepcopy(item) def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: task = self.tasks[task_id] published = [ dataset for dataset in self.datasets.values() if dataset.get("source_task_id") == task_id and dataset.get("deleted_at") is None ] if task.get("output_dataset_id"): train_dataset = self.datasets[task["output_dataset_id"]] return { "dataset": deepcopy(train_dataset), "datasets": deepcopy(published), "output_datasets": deepcopy(published), "created": False, } if task["status"] != "completed": raise InvalidStateError("only a completed task can be published") split_specs = ( ("train", "训练集"), ("val", "验证集"), ("test", "测试集"), ) for dataset_type, label in split_specs: dataset_id = self._id("dataset") self.datasets[dataset_id] = { "id": dataset_id, "name": f"{payload['dataset_name']}-{label}", "type": dataset_type, "count": len(self.results[task_id]), "source": "task", "task_id": task_id, "source_task_id": task_id, "deleted_at": None, } published = [ dataset for dataset in self.datasets.values() if dataset.get("source_task_id") == task_id and dataset.get("deleted_at") is None ] train_dataset = next(dataset for dataset in published if dataset["type"] == "train") task["output_dataset_id"] = train_dataset["id"] return { "dataset": deepcopy(train_dataset), "datasets": deepcopy(published), "output_datasets": deepcopy(published), "created": True, } def make_client( tmp_path: Path, ) -> tuple[TestClient, FakeDataProcessStore, LocalDataProcessStorage]: store = FakeDataProcessStore() storage = LocalDataProcessStorage(tmp_path / "data-process") app = FastAPI() app.include_router(router, prefix="/modelTF") app.dependency_overrides[get_data_process_store] = lambda: store app.dependency_overrides[get_data_process_storage] = lambda: storage return TestClient(app), store, storage def _stored_files(storage: LocalDataProcessStorage) -> list[Path]: return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()] def _minimal_pdf_pages(*page_texts: str) -> bytes: if not page_texts: raise ValueError("at least one PDF page is required") font_object_number = 3 + len(page_texts) * 2 page_object_numbers = [3 + index * 2 for index in range(len(page_texts))] objects = [ b"<< /Type /Catalog /Pages 2 0 R >>", ( b"<< /Type /Pages /Kids [" + b" ".join(f"{number} 0 R".encode() for number in page_object_numbers) + b"] /Count " + str(len(page_texts)).encode() + b" >>" ), ] for index, text in enumerate(page_texts): content_object_number = page_object_numbers[index] + 1 commands = [b"BT /F1 12 Tf 72 720 Td"] for line_index, line in enumerate(text.splitlines()): escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") if line_index: commands.append(b"0 -16 Td") commands.append(f"({escaped}) Tj".encode("ascii")) commands.append(b"ET") stream = b" ".join(commands) objects.extend( [ ( b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " b"/Resources << /Font << /F1 " + str(font_object_number).encode() + b" 0 R >> >> /Contents " + str(content_object_number).encode() + b" 0 R >>" ), b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream", ] ) objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") offsets = [0] for object_number, value in enumerate(objects, start=1): offsets.append(len(result)) result.extend(f"{object_number} 0 obj\n".encode()) result.extend(value) result.extend(b"\nendobj\n") xref_offset = len(result) result.extend(f"xref\n0 {len(objects) + 1}\n".encode()) result.extend(b"0000000000 65535 f \n") for offset in offsets[1:]: result.extend(f"{offset:010d} 00000 n \n".encode()) result.extend( ( f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" f"startxref\n{xref_offset}\n%%EOF\n" ).encode() ) return bytes(result) def _minimal_pdf(text: str = "Hello PDF") -> bytes: return _minimal_pdf_pages(text) def test_data_process_full_contract_without_database(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) created = client.post( "/modelTF/data-process", json={ "name": "客服问答处理", "process_type": "structured", "config": {"dataset_split": {"train": 80, "validation": 10, "test": 10}}, }, ) assert created.status_code == 200 task_id = created.json()["data"]["id"] source_content = ( '{"question":"如何修改地址？",' '"answer":"订单发货前可在订单详情申请修改收货地址。"}\n' '{"question":"如何申请退款？",' '"answer":"请在订单详情提交退款申请并等待审核处理。"}\n' ) uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("customer.jsonl", source_content.encode(), "application/jsonl")}, ) assert uploaded.status_code == 200 source = uploaded.json()["data"]["files"][0] assert len(source["checksum_sha256"]) == 64 assert source["version_no"] == 1 window = client.get( f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content", params={"offset": 0, "limit": 20}, ) assert window.status_code == 200 assert window.json()["data"]["has_more"] is True line_window = client.get( f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content", params={"start_line": 2, "line_count": 1}, ) assert line_window.json()["data"]["start_line"] == 2 assert line_window.json()["data"]["end_line"] == 2 assert line_window.json()["data"]["total_lines"] == 2 preview = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_ids": [source["id"]]}, ) assert preview.status_code == 200 assert preview.json()["data"]["total"] == 2 listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview") assert listed_preview.json()["data"]["total"] == 2 preview_item = listed_preview.json()["data"]["items"][0] updated_preview = client.put( f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}", json={ "edited_content": preview_item["edited_content"], "expected_updated_at": "2026-07-23T00:00:00Z", }, ) assert "quality_score" in updated_preview.json()["data"] generated = client.post(f"/modelTF/data-process/{task_id}/generate") assert generated.status_code == 200 progress = client.get(f"/modelTF/data-process/{task_id}/progress") assert progress.json()["data"]["status"] == "completed" result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"] assert result_page["total"] == 2 keyword_page = client.get( f"/modelTF/data-process/{task_id}/results", params={"keyword": "地址"} ).json()["data"] assert keyword_page["total"] == 1 result = result_page["items"][0] edited = client.put( f"/modelTF/data-process/{task_id}/results/{result['id']}", json={ "output": "人工修改后的完整答案。", "expected_updated_at": "2026-07-23T00:00:00Z", }, ) assert edited.json()["data"]["status"] == "modified" assert "quality_score" in edited.json()["data"] invalid_edit = client.put( f"/modelTF/data-process/{task_id}/results/{result['id']}", json={"output": ""}, ) assert invalid_edit.json()["data"]["status"] == "invalid" assert store.tasks[task_id]["error_count"] == 1 restored = client.post( f"/modelTF/data-process/{task_id}/results/{result['id']}/restore" ) assert restored.json()["data"]["output"] == result["original_output"] assert restored.json()["data"]["status"] == "valid" assert store.tasks[task_id]["error_count"] == 0 publish_payload = {"dataset_name": "客服问答清洗集"} first_publish = client.post( f"/modelTF/data-process/{task_id}/publish", json=publish_payload ) second_publish = client.post( f"/modelTF/data-process/{task_id}/publish", json=publish_payload ) assert first_publish.json()["data"]["created"] is True assert second_publish.json()["data"]["created"] is False assert ( first_publish.json()["data"]["dataset"]["id"] == second_publish.json()["data"]["dataset"]["id"] ) def test_task_list_exposes_document_and_generation_counts( tmp_path: Path, ) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "列表契约", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] store.sources[task_id] = [{"id": "source-1"}, {"id": "source-2"}] store.tasks[task_id].update( { "status": "pending", "output_count": 17, "output_dataset_id": None, } ) response = client.get("/modelTF/data-process") assert response.status_code == 200 item = response.json()["data"]["items"][0] assert item["status"] == "pending" assert item["source_file_count"] == 2 assert item["output_count"] == 17 assert item["output_dataset_id"] is None def test_task_detail_uses_returned_source_files_as_document_count(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "详情文档数", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id]["source_file_count"] = 99 store.sources[task_id] = [ {"id": "source-1", "name": "一.pdf", "content": "正文一"}, {"id": "source-2", "name": "二.pdf", "content": "正文二"}, ] response = client.get(f"/modelTF/data-process/{task_id}") assert response.status_code == 200 detail = response.json()["data"] assert len(detail["source_files"]) == 2 assert detail["source_file_count"] == 2 def test_generation_start_response_clears_previous_output_count( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "新一轮生成", "process_type": "structured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id]["output_count"] = 28 store.previews[task_id] = [ { "id": "preview-1", "source_file_id": None, "original_content": '{"question":"新问题","answer":"新答案"}', "edited_content": '{"question":"新问题","answer":"新答案"}', "status": "original", } ] monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *args: None) response = client.post(f"/modelTF/data-process/{task_id}/generate") assert response.status_code == 200 progress = response.json()["data"] assert progress["status"] == "running" assert progress["output_count"] == 0 assert store.tasks[task_id]["output_count"] == 0 def test_preview_build_replaces_only_selected_files_and_reports_file_counts( tmp_path: Path, ) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "逐文件预览", "process_type": "structured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("first.jsonl", b'{"id":1}\n', "application/jsonl")), ( "files", ("second.jsonl", b'{"id":2}\n{"id":3}\n', "application/jsonl"), ), ], ) assert uploaded.status_code == 200 first_source, second_source = uploaded.json()["data"]["files"] first_build = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_ids": [first_source["id"]]}, ) assert first_build.status_code == 200 first_data = first_build.json()["data"] assert first_data["file_counts"] == {first_source["id"]: 1} assert first_data["files"] == [ { "source_file_id": first_source["id"], "preview_count": 1, "status": "completed", } ] first_item = first_data["items"][0] edited = client.put( f"/modelTF/data-process/{task_id}/preview/{first_item['id']}", json={"edited_content": "人工确认后的第一文件预览"}, ) assert edited.status_code == 200 second_build = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_id": second_source["id"]}, ) assert second_build.status_code == 200 second_data = second_build.json()["data"] assert second_data["file_counts"] == {second_source["id"]: 2} assert second_data["files"] == [ { "source_file_id": second_source["id"], "preview_count": 2, "status": "completed", } ] assert {item["source_file_id"] for item in store.previews[task_id]} == { first_source["id"], second_source["id"], } preserved_first = next( item for item in store.previews[task_id] if item["source_file_id"] == first_source["id"] ) assert preserved_first["id"] == first_item["id"] assert preserved_first["edited_content"] == "人工确认后的第一文件预览" previous_second_ids = { item["id"] for item in store.previews[task_id] if item["source_file_id"] == second_source["id"] } next( source for source in store.sources[task_id] if source["id"] == second_source["id"] )["content"] = '{"id":4}\n' rebuilt = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_ids": [second_source["id"]]}, ) assert rebuilt.status_code == 200 assert rebuilt.json()["data"]["file_counts"] == {second_source["id"]: 1} current_second_ids = { item["id"] for item in store.previews[task_id] if item["source_file_id"] == second_source["id"] } assert current_second_ids.isdisjoint(previous_second_ids) assert len(current_second_ids) == 1 assert next( item for item in store.previews[task_id] if item["source_file_id"] == first_source["id"] )["id"] == first_item["id"] def test_preview_build_rejects_unknown_and_cross_task_source_file_ids( tmp_path: Path, ) -> None: client, _, _ = make_client(tmp_path) first_task_id = client.post( "/modelTF/data-process", json={"name": "归属任务一", "process_type": "structured", "config": {}}, ).json()["data"]["id"] second_task_id = client.post( "/modelTF/data-process", json={"name": "归属任务二", "process_type": "structured", "config": {}}, ).json()["data"]["id"] foreign_source = client.post( f"/modelTF/data-process/{second_task_id}/source-files", files={"files": ("foreign.jsonl", b'{"id":2}\n', "application/jsonl")}, ).json()["data"]["files"][0] unknown = client.post( f"/modelTF/data-process/{first_task_id}/preview/build", json={"source_file_ids": ["dpsf_not_found"]}, ) assert unknown.status_code == 404 foreign = client.post( f"/modelTF/data-process/{first_task_id}/preview/build", json={"source_file_id": foreign_source["id"]}, ) assert foreign.status_code == 404 ambiguous = client.post( f"/modelTF/data-process/{first_task_id}/preview/build", json={ "source_file_id": foreign_source["id"], "source_file_ids": [foreign_source["id"]], }, ) assert ambiguous.status_code == 422 def test_external_source_never_returns_fake_success(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "外部数据", "process_type": "external", "config": {}}, ).json()["data"]["id"] response = client.post( f"/modelTF/data-process/{task_id}/external/test", json={"type": "mysql", "url": "mysql://db.example/test"}, ) assert response.status_code == 501 assert response.json()["detail"]["code"] == 501 def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "待重新生成", "process_type": "structured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id].update( { "status": "completed", "updated_at": "2026-07-25T19:00:00Z", "output_dataset_id": "dataset_train", "output_count": 2, } ) store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}] store.results[task_id] = [{"id": "result_1"}] response = client.post( f"/modelTF/data-process/{task_id}/regenerate", json={ "name": "重新生成后名称", "description": "更换生成模型", "process_type": "structured", "config": {"generation_model_id": "model_2"}, "expected_updated_at": "2026-07-25T19:00:00Z", }, ) assert response.status_code == 200 data = response.json()["data"] assert data["task"]["status"] == "pending" assert data["task"]["output_dataset_id"] is None assert data["preview_invalidated"] is False assert data["published_outputs_preserved"] is True assert store.results[task_id] == [] assert store.previews[task_id][0]["id"] == "preview_1" def test_published_split_datasets_remain_in_detail_after_regeneration( tmp_path: Path, ) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "保留旧发布数据", "process_type": "structured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id].update( { "status": "completed", "updated_at": "2026-07-27T09:00:00Z", "output_count": 1, } ) store.results[task_id] = [ { "id": "result_1", "status": "valid", "instruction": "问题", "input": "", "output": "答案", } ] published = client.post( f"/modelTF/data-process/{task_id}/publish", json={"dataset_name": "保留旧发布数据集"}, ) assert published.status_code == 200 published_datasets = published.json()["data"]["datasets"] assert len(published_datasets) == 3 published_ids = {item["id"] for item in published_datasets} assert store.tasks[task_id]["output_dataset_id"] in published_ids regenerated = client.post( f"/modelTF/data-process/{task_id}/regenerate", json={ "name": "保留旧发布数据", "description": "更换生成配置后退出", "process_type": "structured", "config": {"generation_model_id": "model_2"}, "expected_updated_at": "2026-07-27T09:00:00Z", }, ) assert regenerated.status_code == 200 assert regenerated.json()["data"]["task"]["output_dataset_id"] is None detail = client.get(f"/modelTF/data-process/{task_id}") assert detail.status_code == 200 detail_data = detail.json()["data"] assert detail_data["status"] == "pending" assert detail_data["output_dataset_id"] is None assert len(detail_data["output_datasets"]) == 3 assert {item["id"] for item in detail_data["output_datasets"]} == published_ids assert set(store.datasets) == published_ids def test_regenerate_endpoint_validates_snapshot_and_locked_process_type( tmp_path: Path, ) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "并发校验", "process_type": "structured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id]["updated_at"] = "2026-07-25T19:00:00Z" payload = { "name": "并发校验", "description": "", "process_type": "structured", "config": {}, "expected_updated_at": "stale", } stale = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload) assert stale.status_code == 409 payload.update( { "process_type": "unstructured", "expected_updated_at": "2026-07-25T19:00:00Z", } ) locked_type = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload) assert locked_type.status_code == 409 for missing_field in ("expected_updated_at", "description", "config"): missing_required_field = client.post( f"/modelTF/data-process/{task_id}/regenerate", json={key: value for key, value in payload.items() if key != missing_field}, ) assert missing_required_field.status_code == 422 def test_config_validation_and_stop_state(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) invalid = client.post( "/modelTF/data-process", json={ "name": "错误切片配置", "process_type": "unstructured", "config": { "dataset_split": {"train": 80, "validation": 30, "test": 0}, "chunk_size": 100, "chunk_overlap": 90, "min_chunk_size": 20, }, }, ) assert invalid.status_code == 422 semantic = client.post( "/modelTF/data-process", json={ "name": "语义切分策略", "process_type": "unstructured", "config": {"chunk_method": "semantic"}, }, ) assert semantic.status_code == 200 removed_custom_method = client.post( "/modelTF/data-process", json={ "name": "已移除的自定义分隔符", "process_type": "unstructured", "config": {"chunk_method": "custom"}, }, ) assert removed_custom_method.status_code == 422 assert "chunk_method" in removed_custom_method.text task_id = client.post( "/modelTF/data-process", json={"name": "可停止任务", "process_type": "structured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id]["status"] = "running" stopped = client.post(f"/modelTF/data-process/{task_id}/stop") assert stopped.status_code == 200 assert stopped.json()["data"]["status"] == "stopped" @pytest.mark.parametrize("config_key", ["qa_pairs_per_row", "qa_pairs_per_chunk"]) @pytest.mark.parametrize("count", [1, 50]) def test_qa_pair_config_accepts_supported_boundaries( tmp_path: Path, config_key: str, count: int, ) -> None: client, _, _ = make_client(tmp_path) response = client.post( "/modelTF/data-process", json={ "name": "问答数量边界", "process_type": "unstructured", "config": {config_key: count}, }, ) assert response.status_code == 200 @pytest.mark.parametrize("config_key", ["qa_pairs_per_row", "qa_pairs_per_chunk"]) @pytest.mark.parametrize("count", [0, 51]) def test_qa_pair_config_rejects_out_of_range_boundaries( tmp_path: Path, config_key: str, count: int, ) -> None: client, _, _ = make_client(tmp_path) response = client.post( "/modelTF/data-process", json={ "name": "问答数量越界", "process_type": "unstructured", "config": {config_key: count}, }, ) assert response.status_code == 422 assert "[1, 50]" in response.text def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "批量上传", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] duplicate_batch = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("first.txt", b"same content", "text/plain")), ("files", ("second.txt", b"same content", "text/plain")), ], ) assert duplicate_batch.status_code == 400 assert store.sources[task_id] == [] assert _stored_files(storage) == [] parse_failure = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("valid.txt", "先暂存的内容".encode(), "text/plain")), ("files", ("broken.txt", b"\xff", "text/plain")), ], ) assert parse_failure.status_code == 400 assert store.sources[task_id] == [] assert _stored_files(storage) == [] empty = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("empty.txt", b"", "text/plain")}, ) assert empty.status_code == 400 assert store.sources[task_id] == [] assert _stored_files(storage) == [] def test_incremental_upload_keeps_existing_file_previews(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "增量上传预览", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] first_batch = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("first.txt", "第一个文件内容".encode(), "text/plain")), ("files", ("second.txt", "第二个文件内容".encode(), "text/plain")), ], ) assert first_batch.status_code == 200 first_source_ids = {item["id"] for item in first_batch.json()["data"]["files"]} built = client.post(f"/modelTF/data-process/{task_id}/preview/build", json={}) assert built.status_code == 200 assert {item["source_file_id"] for item in built.json()["data"]["items"]} == first_source_ids store.results[task_id] = [{"id": "old_result"}] third = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("third.txt", "第三个文件内容".encode(), "text/plain")}, ) assert third.status_code == 200 previews = client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"] assert previews["total"] == 2 assert {item["source_file_id"] for item in previews["items"]} == first_source_ids assert store.results[task_id] == [] assert store.tasks[task_id]["progress"] == 20 def test_upload_preserves_store_error_when_storage_rollback_fails( tmp_path: Path, monkeypatch: Any, ) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "回滚异常", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] cleanup_attempts: list[str] = [] def fail_store(*_: Any, **__: Any) -> list[dict[str, Any]]: raise ValueError("simulated database transaction failure") def fail_cleanup(reference: str, **_: Any) -> bool: cleanup_attempts.append(reference) raise OSError("simulated storage cleanup failure") monkeypatch.setattr(store, "add_source_files", fail_store) monkeypatch.setattr(storage, "delete", fail_cleanup) response = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("rollback.txt", b"rollback payload", "text/plain")}, ) assert response.status_code == 400 assert response.json()["detail"]["message"] == "simulated database transaction failure" assert len(cleanup_attempts) == 1 assert store.sources[task_id] == [] def test_preprocess_deduplicates_and_quality_filter_removes_short_results( tmp_path: Path, ) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={ "name": "去重与质量筛选", "process_type": "structured", "config": { "preprocess_options": ["clean_invalid", "deduplicate"], "quality_filter_enabled": True, "filter_low_quality": False, "filter_short_content": True, "min_output_length": 100, }, }, ).json()["data"]["id"] content = ( '{"question":"问题","answer":"短答案"}\n' '{"question":"问题","answer":"短答案"}\n' ).encode() uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("duplicates.jsonl", content, "application/jsonl")}, ) assert uploaded.status_code == 200 preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert preview.json()["data"]["total"] == 1 generated = client.post(f"/modelTF/data-process/{task_id}/generate") assert generated.status_code == 200 progress = client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"] assert progress["status"] == "completed" assert progress["filtered_count"] == 1 assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0 def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None: store = FakeDataProcessStore() task = store.create_task( {"name": "并发代次", "process_type": "structured", "config": {}} ) task_id = task["id"] store.replace_preview_items( task_id, [ { "source_file_id": None, "original_content": "来源内容", "edited_content": "来源内容", "status": "manual", } ], ) first = store.start_generation(task_id, replace_existing=True) first_run_id = first["generation_run_id"] second_run_id = "" def restart_while_old_worker_runs(*_: Any, **__: Any) -> list[dict[str, Any]]: nonlocal second_run_id store.stop_task(task_id) second = store.start_generation(task_id, replace_existing=True) second_run_id = second["generation_run_id"] return [] monkeypatch.setattr( data_process_endpoint, "generate_standard_records", restart_while_old_worker_runs, ) data_process_endpoint._run_generation(store, task_id, first_run_id) assert second_run_id and second_run_id != first_run_id assert store.tasks[task_id]["status"] == "running" assert store.tasks[task_id]["generation_run_id"] == second_run_id assert store.results[task_id] == [] store.mark_failed(task_id, "old failure", generation_run_id=first_run_id) assert store.tasks[task_id]["status"] == "running" def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "状态保护", "process_type": "structured", "config": {}}, ).json()["data"]["id"] response = client.put( f"/modelTF/data-process/{task_id}/results/not-created", json={"instruction": "", "output": "", "status": "valid"}, ) assert response.status_code == 422 def test_start_rebuilds_preview_and_generates_in_one_request(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "一键处理", "process_type": "structured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={ "files": ( "one.jsonl", b'{"question":"What is one?","answer":"One."}\n', "application/jsonl", ) }, ) assert uploaded.status_code == 200 started = client.post(f"/modelTF/data-process/{task_id}/start") assert started.status_code == 200 assert started.json()["data"]["task_id"] == task_id assert started.json()["data"]["status"] == "running" assert client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]["status"] == "completed" assert client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]["total"] == 1 assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1 def test_unsupported_upload_format_returns_415(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "格式限制", "process_type": "structured", "config": {}}, ).json()["data"]["id"] response = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("payload.exe", b"not supported", "application/octet-stream")}, ) assert response.status_code == 415 legacy = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("document.doc", b"legacy", "application/msword")}, ) assert legacy.status_code == 415 assert "convert the file to .docx" in legacy.json()["detail"]["message"] def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "XLSX 上传", "process_type": "structured", "config": {}}, ).json()["data"]["id"] workbook = Workbook() worksheet = workbook.active worksheet.append(["question", "answer"]) worksheet.append(["问题一", "答案一"]) worksheet.append(["问题二", "答案二"]) output = BytesIO() workbook.save(output) workbook.close() original_bytes = output.getvalue() response = client.post( f"/modelTF/data-process/{task_id}/source-files", files={ "files": ( "records.xlsx", original_bytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) }, ) assert response.status_code == 200 source = response.json()["data"]["files"][0] assert source["file_format"] == "xlsx" assert source["record_count"] == 2 assert source["size_bytes"] == len(original_bytes) assert source["storage_object_id"].startswith("local://data-process/") assert str(storage.root) not in response.text assert storage.read(source["storage_object_id"]) == original_bytes stored_source = store.get_source_file(task_id, source["id"]) assert stored_source["id"] == source["id"] assert stored_source["storage_object_id"] == source["storage_object_id"] assert stored_source["metadata"]["storage_backend"] == "local" assert stored_source["metadata"]["original_size_bytes"] == len(original_bytes) assert '"question":"问题一"' in stored_source["content"] content = client.get( f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content" ) assert content.status_code == 200 assert '"answer":"答案二"' in content.json()["data"]["content"] preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert preview.status_code == 200 assert preview.json()["data"]["total"] == 2 def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "PDF 原件预览", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] original_pdf = _minimal_pdf() uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("说明 文档.pdf", original_pdf, "application/pdf")}, ).json()["data"]["files"][0] raw_url = f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw" full = client.get(raw_url) assert full.status_code == 200 assert full.content == original_pdf assert full.headers["content-type"] == "application/pdf" assert full.headers["accept-ranges"] == "bytes" assert full.headers["cache-control"] == "private, no-store" assert full.headers["content-length"] == str(len(original_pdf)) assert full.headers["content-disposition"].startswith("inline;") assert "%E8%AF%B4%E6%98%8E%20%E6%96%87%E6%A1%A3.pdf" in full.headers[ "content-disposition" ] assert full.headers["etag"] == f'"{uploaded["checksum_sha256"]}"' partial = client.get(raw_url, headers={"Range": "bytes=5-14"}) assert partial.status_code == 206 assert partial.content == original_pdf[5:15] assert partial.headers["content-range"] == f"bytes 5-14/{len(original_pdf)}" assert partial.headers["content-length"] == "10" suffix = client.get(raw_url, headers={"Range": "bytes=-8"}) assert suffix.status_code == 206 assert suffix.content == original_pdf[-8:] invalid = client.get(raw_url, headers={"Range": "bytes=0-1,4-5"}) assert invalid.status_code == 416 assert invalid.headers["content-range"] == f"bytes */{len(original_pdf)}" pages_url = ( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages" ) pages = client.get(pages_url) assert pages.status_code == 200 assert pages.json()["data"] == { "page_count": 1, "pages": [ { "page_number": 1, "source_start": 0, "source_end": len("Hello PDF"), } ], } legacy_id = "dpsf_legacy_pdf" store.add_source_file( task_id, id=legacy_id, storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1", name="legacy.pdf", content="legacy extracted PDF text", raw_size=len(original_pdf), checksum_sha256="a" * 64, file_format="pdf", record_count=1, metadata={"legacy": True}, ) legacy = client.get( f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/raw" ) assert legacy.status_code == 410 legacy_pages = client.get( f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/pdf-pages" ) assert legacy_pages.status_code == 410 def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift( tmp_path: Path, ) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={ "name": "PDF 文档噪声清理", "process_type": "unstructured", "config": { "chunk_method": "fixed", "chunk_size": 200, "chunk_overlap": 0, "min_chunk_size": 20, "preprocess_options": ["clean_invalid_content"], }, }, ).json()["data"]["id"] raw = _minimal_pdf_pages( "ACME Internal Manual\nBody page one keeps this guidance and explanation.", "ACME Internal Manual\nContents\n" "Chapter One........3\nChapter Two........4\nAppendix........5", "ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8", "ACME Internal Manual\nBody page four keeps operational details and examples.", "ACME Internal Manual\nBody page five keeps the final effective-date clause.", ) uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("manual.pdf", raw, "application/pdf")}, ).json()["data"]["files"][0] source_content = client.get( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content" ).json()["data"]["content"] built = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert built.status_code == 200 items = built.json()["data"]["items"] assert items edited = "\n".join(item["edited_content"] for item in items) assert "ACME Internal Manual" not in edited assert "Contents" not in edited assert "Chapter One" not in edited assert "1.2 Approval" not in edited assert "Body page one" in edited assert "Body page five" in edited assert all( item["original_content"] == source_content[item["source_start"] : item["source_end"]] for item in items ) def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "非 PDF 原件", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("notes.txt", b"plain source text", "text/plain")}, ).json()["data"]["files"][0] response = client.get( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw" ) assert response.status_code == 415 pages_response = client.get( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages" ) assert pages_response.status_code == 415 def test_delete_source_removes_owned_local_object_and_accepts_legacy_db_reference( tmp_path: Path, ) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "删除原件", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("原件.txt", "本地原始内容".encode(), "text/plain")}, ).json()["data"]["files"][0] reference = uploaded["storage_object_id"] assert storage.read(reference) == "本地原始内容".encode() deleted = client.delete( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}" ) assert deleted.status_code == 200 assert deleted.json()["data"]["storage_cleanup_pending"] is False with pytest.raises(DataProcessStorageError, match="does not exist"): storage.read(reference) legacy_id = "dpsf_legacy" store.add_source_file( task_id, id=legacy_id, storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1", name="legacy.txt", content="旧记录正文", raw_size=len("旧记录正文".encode()), checksum_sha256="a" * 64, file_format="txt", record_count=1, metadata={"legacy": True}, ) legacy_deleted = client.delete( f"/modelTF/data-process/{task_id}/source-files/{legacy_id}" ) assert legacy_deleted.status_code == 200 assert legacy_deleted.json()["data"]["storage_cleanup_pending"] is False def test_delete_reports_pending_cleanup_after_database_soft_delete( tmp_path: Path, monkeypatch: Any, ) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "待清理原件", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("pending.txt", b"pending cleanup", "text/plain")}, ).json()["data"]["files"][0] def fail_cleanup(*_: Any, **__: Any) -> bool: raise OSError("simulated storage failure") monkeypatch.setattr(storage, "delete", fail_cleanup) response = client.delete( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}" ) assert response.status_code == 200 assert response.json()["data"]["storage_cleanup_pending"] is True with pytest.raises(NotFoundError): store.get_source_file(task_id, uploaded["id"]) assert storage.read(uploaded["storage_object_id"]) == b"pending cleanup" def test_delete_rejects_polluted_reference_owned_by_another_source(tmp_path: Path) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "归属校验", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("safe.txt", b"owned content", "text/plain")}, ).json()["data"]["files"][0] target_reference = uploaded["storage_object_id"] polluted_id = "dpsf_polluted" store.add_source_file( task_id, id=polluted_id, storage_object_id=target_reference, name="polluted.txt", content="polluted", raw_size=8, checksum_sha256="b" * 64, file_format="txt", record_count=1, metadata={}, ) rejected = client.delete( f"/modelTF/data-process/{task_id}/source-files/{polluted_id}" ) assert rejected.status_code == 400 assert storage.read(target_reference) == b"owned content" assert store.get_source_file(task_id, polluted_id)["id"] == polluted_id def test_upload_format_must_match_process_type(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) structured_id = client.post( "/modelTF/data-process", json={"name": "结构化格式约束", "process_type": "structured", "config": {}}, ).json()["data"]["id"] structured_pdf = client.post( f"/modelTF/data-process/{structured_id}/source-files", files={"files": ("manual.pdf", b"not parsed", "application/pdf")}, ) assert structured_pdf.status_code == 415 unstructured_id = client.post( "/modelTF/data-process", json={"name": "非结构化格式约束", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] unstructured_xlsx = client.post( f"/modelTF/data-process/{unstructured_id}/source-files", files={"files": ("records.xlsx", b"not parsed", "application/octet-stream")}, ) assert unstructured_xlsx.status_code == 415 external_id = client.post( "/modelTF/data-process", json={"name": "外部数据格式约束", "process_type": "external", "config": {}}, ).json()["data"]["id"] external_upload = client.post( f"/modelTF/data-process/{external_id}/source-files", files={"files": ("records.jsonl", b'{"id":1}', "application/jsonl")}, ) assert external_upload.status_code == 409 def _preview_task( content: str, *, options: list[str], config: dict[str, Any] | None = None, source_id: str = "source-1", file_format: str = "txt", ) -> list[dict[str, Any]]: task_config = { "preprocess_options": options, "chunk_method": "fixed", "chunk_size": 200, "chunk_overlap": 20, "min_chunk_size": 20, **(config or {}), } return data_process_endpoint._build_preview_items( {"process_type": "unstructured", "config": task_config}, [ { "id": source_id, "name": f"{source_id}.{file_format}", "file_format": file_format, "content": content, } ], ) def test_fixed_preview_preserves_source_offsets() -> None: content = ( "# 第一章\n" + " ".join(f"alpha{index}" for index in range(18)) + "\n# 第二章\n" + " ".join(f"beta{index}" for index in range(18)) ) normalized = normalize_text(content) second_chapter_start = normalized.index("# 第二章") common_config = {"chunk_size": 10, "chunk_overlap": 3, "min_chunk_size": 4} default_items = _preview_task( content, options=["preserve_context"], config=common_config, ) fixed_items = _preview_task( content, options=["preserve_context"], config={**common_config, "chunk_method": "fixed"}, ) def snapshot(items: list[dict[str, Any]]) -> list[tuple[Any, ...]]: return [ ( item["original_content"], item["source_start"], item["source_end"], item["source_start_line"], item["source_end_line"], ) for item in items ] assert snapshot(default_items) == snapshot(fixed_items) assert all( item["original_content"] == normalized[item["source_start"] : item["source_end"]] for item in fixed_items ) second_chapter_items = [ item for item in fixed_items if item["source_start"] >= second_chapter_start ] assert second_chapter_items def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None: repeated = "@" * 120 assert len(_preview_task(repeated, options=[])) == 1 assert _preview_task(repeated, options=["clean_invalid_content"]) == [] mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。" assert len(_preview_task(mojibake, options=[])) == 1 assert _preview_task(mojibake, options=["filter_low_quality"]) == [] first = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega" second = "alpha beta gamma, delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega" sources = [ {"id": "near-1", "name": "one.txt", "file_format": "txt", "content": first}, {"id": "near-2", "name": "two.txt", "file_format": "txt", "content": second}, ] base_task = { "process_type": "unstructured", "config": { "chunk_method": "fixed", "chunk_size": 200, "chunk_overlap": 0, "min_chunk_size": 1, "preprocess_options": [], }, } assert len(data_process_endpoint._build_preview_items(base_task, sources)) == 2 deduplicated_task = deepcopy(base_task) deduplicated_task["config"]["preprocess_options"] = ["deduplicate_content"] assert len(data_process_endpoint._build_preview_items(deduplicated_task, sources)) == 1 context_text = " ".join(f"token{index}" for index in range(45)) no_context = _preview_task( context_text, options=[], config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5}, ) with_context = _preview_task( context_text, options=["preserve_context"], config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5}, ) assert no_context[1]["source_start"] >= no_context[0]["source_end"] assert with_context[1]["source_start"] < with_context[0]["source_end"] sensitive = "联系人：张三，手机 13800138000，邮箱 user@example.com。" plain = _preview_task(sensitive, options=[])[0] masked = _preview_task(sensitive, options=["desensitize"])[0] assert "张三" in plain["edited_content"] assert "联系人:[NAME]" in masked["edited_content"] assert "[PHONE]" in masked["edited_content"] assert "[EMAIL]" in masked["edited_content"] def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None: source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。") source = { "id": "pdf-source", "name": "manual.pdf", "file_format": "pdf", "content": source_text, "document_noise_spans": ( DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"), ), } config = { "chunk_method": "fixed", "chunk_size": 200, "chunk_overlap": 0, "min_chunk_size": 1, } cleaned_items = data_process_endpoint._build_preview_items( { "process_type": "unstructured", "config": {**config, "preprocess_options": ["clean_invalid_content"]}, }, [source], ) original_items = data_process_endpoint._build_preview_items( { "process_type": "unstructured", "config": {**config, "preprocess_options": []}, }, [source], ) assert len(cleaned_items) == 1 cleaned = cleaned_items[0] assert cleaned["original_content"] == source_text[ cleaned["source_start"] : cleaned["source_end"] ] assert "重复页眉" not in cleaned["edited_content"] assert "PDF 正文内容" in cleaned["edited_content"] assert cleaned["status"] == "modified" assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"] assert "重复页眉" in original_items[0]["edited_content"] def test_merge_short_content_applies_across_adjacent_fixed_chunks() -> None: content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9)) items = _preview_task( content, options=["merge_short_content"], config={ "chunk_method": "fixed", "chunk_size": 40, "chunk_overlap": 0, "min_chunk_size": 20, }, ) assert len(items) == 3 assert all(item["token_count"] <= 40 for item in items) assert items[0]["source_start_line"] == 1 assert items[-1]["source_end_line"] == 16 def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None: for file_format in ("pdf", "docx", "pptx"): items = _preview_task( f"{file_format.upper()} 已抽取正文，可直接进入切片处理。", options=[], file_format=file_format, ) assert len(items) == 1 assert "已抽取正文" in items[0]["edited_content"]