from __future__ import annotations from copy import deepcopy from io import BytesIO from pathlib import Path from typing import Any import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from openpyxl import Workbook from app.api.v1.endpoints import data_process as data_process_endpoint from app.api.v1.endpoints.data_process import router from app.modules.data_process.algorithms import normalize_text from app.modules.data_process.storage import ( DataProcessStorageError, LocalDataProcessStorage, get_data_process_storage, ) from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store class FakeDataProcessStore: """接口测试专用内存实现，确保测试不会连接或迁移真实数据库。""" def __init__(self) -> None: self.tasks: dict[str, dict[str, Any]] = {} self.sources: dict[str, list[dict[str, Any]]] = {} self.previews: dict[str, list[dict[str, Any]]] = {} self.results: dict[str, list[dict[str, Any]]] = {} self.datasets: dict[str, dict[str, Any]] = {} self.sequence = 0 def _id(self, prefix: str) -> str: self.sequence += 1 return f"{prefix}_{self.sequence}" def list_tasks(self, *, page: int, page_size: int, **filters: Any) -> dict[str, Any]: items = list(self.tasks.values()) for field in ("status", "process_type", "tenant_id", "project_id"): if filters.get(field): items = [item for item in items if item.get(field) == filters[field]] keyword = filters.get("keyword") if keyword: items = [item for item in items if keyword in item["name"]] return { "items": deepcopy(items[(page - 1) * page_size : page * page_size]), "total": len(items), "page": page, "page_size": page_size, } def create_task(self, payload: dict[str, Any]) -> dict[str, Any]: task_id = self._id("dpt") task = { "id": task_id, **deepcopy(payload), "status": "pending", "progress": 0, "input_count": 0, "output_count": 0, "filtered_count": 0, "duplicate_count": 0, "error_count": 0, "failure_reason": None, "output_dataset_id": None, } self.tasks[task_id] = task self.sources[task_id] = [] self.previews[task_id] = [] self.results[task_id] = [] return deepcopy(task) def get_task(self, task_id: str) -> dict[str, Any]: if task_id not in self.tasks: raise NotFoundError("data process task not found") return deepcopy(self.tasks[task_id]) def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: self.get_task(task_id) self.tasks[task_id].update(deepcopy(payload)) return self.get_task(task_id) def delete_task(self, task_id: str, **_: Any) -> None: self.get_task(task_id) if self.tasks[task_id]["status"] == "running": raise InvalidStateError("running task must be stopped before deletion") del self.tasks[task_id] def list_source_files(self, task_id: str) -> list[dict[str, Any]]: self.get_task(task_id) return [ {key: value for key, value in item.items() if key != "content"} for item in self.sources[task_id] ] def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]: self.get_task(task_id) values = deepcopy(payload) source_id = str(values.pop("id", None) or self._id("dpsf")) storage_object_id = str( values.pop("storage_object_id", None) or f"db://data-process/{task_id}/{source_id}/v1" ) raw_size = int(values.pop("raw_size")) metadata = deepcopy(values.pop("metadata", {})) metadata.setdefault( "storage_backend", "local" if storage_object_id.startswith("local://data-process/") else "database", ) source = { "id": source_id, "task_id": task_id, "version_no": 1, "storage_object_id": storage_object_id, "size_bytes": raw_size, "metadata": metadata, **values, } self.sources[task_id].append(source) self.tasks[task_id]["input_count"] += payload["record_count"] return {key: value for key, value in deepcopy(source).items() if key != "content"} def add_source_files( self, task_id: str, files: list[dict[str, Any]] ) -> list[dict[str, Any]]: # 先验证整个批次，模拟数据库事务的 all-or-nothing 语义。 checksums = {item["checksum_sha256"] for item in self.sources.get(task_id, [])} incoming: set[str] = set() for payload in files: checksum = payload["checksum_sha256"] if checksum in checksums or checksum in incoming: raise ValueError("the same source file content is already attached to this task") incoming.add(checksum) return [self.add_source_file(task_id, **payload) for payload in files] def get_source_file( self, task_id: str, file_id: str, *, include_content: bool = True ) -> dict[str, Any]: source = next( (item for item in self.sources.get(task_id, []) if item["id"] == file_id), None, ) if not source: raise NotFoundError("source file not found") result = deepcopy(source) if not include_content: result.pop("content", None) return result def source_content_window( self, task_id: str, file_id: str, offset: int, limit: int ) -> dict[str, Any]: source = self.get_source_file(task_id, file_id) content = source.pop("content") return { "file": source, "content": content[offset : offset + limit], "offset": offset, "limit": limit, "total_chars": len(content), "has_more": offset + limit < len(content), } def source_content_lines( self, task_id: str, file_id: str, start_line: int, line_count: int ) -> dict[str, Any]: source = self.get_source_file(task_id, file_id) lines = source.pop("content").splitlines(keepends=True) selected = lines[start_line - 1 : start_line - 1 + line_count] return { "file": source, "content": "".join(selected), "start_line": start_line, "end_line": start_line - 1 + len(selected), "line_count": len(selected), "total_lines": len(lines), "has_more": start_line - 1 + len(selected) < len(lines), } def delete_source_file(self, task_id: str, file_id: str) -> None: self.get_source_file(task_id, file_id) self.sources[task_id] = [item for item in self.sources[task_id] if item["id"] != file_id] self.previews[task_id] = [ item for item in self.previews[task_id] if item["source_file_id"] != file_id ] self.results[task_id] = [] def replace_preview_items( self, task_id: str, items: list[dict[str, Any]], *, source_file_ids: list[str] | None = None, ) -> list[dict[str, Any]]: created = [ {"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items ] if source_file_ids is None: self.previews[task_id] = created else: selected = set(source_file_ids) self.previews[task_id] = [ item for item in self.previews[task_id] if item["source_file_id"] not in selected ] + created self.results[task_id] = [] self.tasks[task_id]["progress"] = 20 return deepcopy(created) def list_preview_items( self, task_id: str, *, page: int, page_size: int, source_file_id: str | None = None, keyword: str | None = None, ) -> dict[str, Any]: items = self.previews[task_id] if source_file_id: items = [item for item in items if item["source_file_id"] == source_file_id] if keyword: items = [item for item in items if keyword in item["edited_content"]] return { "items": deepcopy(items[(page - 1) * page_size : page * page_size]), "total": len(items), "page": page, "page_size": page_size, } def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]: item = next( (item for item in self.previews.get(task_id, []) if item["id"] == preview_id), None, ) if not item: raise NotFoundError("preview item not found") return deepcopy(item) def create_preview_item(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: item = {"id": self._id("dpp"), "task_id": task_id, **deepcopy(payload)} self.previews[task_id].append(item) self.results[task_id] = [] return deepcopy(item) def update_preview_item( self, task_id: str, preview_id: str, payload: dict[str, Any] ) -> dict[str, Any]: item = next( (item for item in self.previews[task_id] if item["id"] == preview_id), None, ) if not item: raise NotFoundError("preview item not found") item.update(deepcopy(payload)) self.results[task_id] = [] return deepcopy(item) def delete_preview_item(self, task_id: str, preview_id: str) -> None: before = len(self.previews[task_id]) self.previews[task_id] = [ item for item in self.previews[task_id] if item["id"] != preview_id ] if len(self.previews[task_id]) == before: raise NotFoundError("preview item not found") def start_generation(self, task_id: str, *, replace_existing: bool) -> dict[str, Any]: if not self.previews[task_id]: raise InvalidStateError("preview must be built before generation") if replace_existing: self.results[task_id] = [] self.tasks[task_id].update( status="running", progress=30, generation_run_id=self._id("dprun"), ) return self.get_task(task_id) def generation_is_running(self, task_id: str, generation_run_id: str) -> bool: return ( self.tasks[task_id]["status"] == "running" and self.tasks[task_id].get("generation_run_id") == generation_run_id ) def update_generation_progress( self, task_id: str, generation_run_id: str, processed_count: int, total_count: int, ) -> bool: if not self.generation_is_running(task_id, generation_run_id): return False self.tasks[task_id]["progress"] = min( 95, 30 + processed_count / max(1, total_count) * 65, ) return True def complete_generation( self, task_id: str, results: list[dict[str, Any]], *, generation_run_id: str, **counts: Any, ) -> dict[str, Any]: if not self.generation_is_running(task_id, generation_run_id): return self.get_task(task_id) self.results[task_id] = deepcopy(results) self.tasks[task_id].update( status="completed", progress=100, output_count=len(results), generation_run_id=None, **counts, ) return self.get_task(task_id) def mark_failed( self, task_id: str, reason: str, *, generation_run_id: str ) -> dict[str, Any]: if self.generation_is_running(task_id, generation_run_id): self.tasks[task_id].update( status="failed", failure_reason=reason, generation_run_id=None, ) return self.get_task(task_id) def stop_task(self, task_id: str) -> dict[str, Any]: if self.tasks[task_id]["status"] != "running": raise InvalidStateError("only a running task can be stopped") self.tasks[task_id].update(status="stopped", generation_run_id=None) return self.get_task(task_id) def progress(self, task_id: str) -> dict[str, Any]: task = self.get_task(task_id) result = {key: task.get(key) for key in ( "status", "progress", "input_count", "output_count", "filtered_count", "duplicate_count", "error_count", "failure_reason", )} result["task_id"] = task["id"] return result def list_results( self, task_id: str, *, page: int, page_size: int, status: str | None = None, split: str | None = None, keyword: str | None = None, ) -> dict[str, Any]: items = self.results[task_id] if status: items = [item for item in items if item["status"] == status] if split: items = [item for item in items if item["split"] == split] if keyword: items = [ item for item in items if any(keyword in item[field] for field in ("instruction", "input", "output")) ] return { "items": deepcopy(items), "total": len(items), "page": page, "page_size": page_size, } def update_result( self, task_id: str, result_id: str, payload: dict[str, Any] ) -> dict[str, Any]: item = next((item for item in self.results[task_id] if item["id"] == result_id), None) if not item: raise NotFoundError("data process result not found") for field in ("instruction", "input", "output", "quality_score"): if field in payload: item[field] = deepcopy(payload[field]) hard_valid = bool(item["instruction"].strip() and item["output"].strip()) quality_valid = bool((item.get("quality_score") or {}).get("is_valid", hard_valid)) changed = any( item[field] != item[f"original_{field}"] for field in ("instruction", "input", "output") ) item["status"] = ( "invalid" if not hard_valid or not quality_valid else "modified" if changed else "valid" ) self.tasks[task_id]["error_count"] = sum( result["status"] == "invalid" for result in self.results[task_id] ) return deepcopy(item) def get_result(self, task_id: str, result_id: str) -> dict[str, Any]: item = next((item for item in self.results[task_id] if item["id"] == result_id), None) if not item: raise NotFoundError("data process result not found") return deepcopy(item) def restore_result(self, task_id: str, result_id: str) -> dict[str, Any]: item = next((item for item in self.results[task_id] if item["id"] == result_id), None) if not item: raise NotFoundError("data process result not found") for field in ("instruction", "input", "output"): item[field] = item[f"original_{field}"] item["status"] = "valid" return deepcopy(item) def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: task = self.tasks[task_id] if task.get("output_dataset_id"): return {"dataset": deepcopy(self.datasets[task["output_dataset_id"]]), "created": False} if task["status"] != "completed": raise InvalidStateError("only a completed task can be published") dataset_id = self._id("dataset") dataset = {"id": dataset_id, "name": payload["dataset_name"], "source_task_id": task_id} self.datasets[dataset_id] = dataset task["output_dataset_id"] = dataset_id return {"dataset": deepcopy(dataset), "created": True} def make_client( tmp_path: Path, ) -> tuple[TestClient, FakeDataProcessStore, LocalDataProcessStorage]: store = FakeDataProcessStore() storage = LocalDataProcessStorage(tmp_path / "data-process") app = FastAPI() app.include_router(router, prefix="/modelTF") app.dependency_overrides[get_data_process_store] = lambda: store app.dependency_overrides[get_data_process_storage] = lambda: storage return TestClient(app), store, storage def _stored_files(storage: LocalDataProcessStorage) -> list[Path]: return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()] def _minimal_pdf(text: str = "Hello PDF") -> bytes: stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii") objects = [ b"<< /Type /Catalog /Pages 2 0 R >>", b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", ( b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>" ), b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream", b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", ] result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") offsets = [0] for object_number, value in enumerate(objects, start=1): offsets.append(len(result)) result.extend(f"{object_number} 0 obj\n".encode()) result.extend(value) result.extend(b"\nendobj\n") xref_offset = len(result) result.extend(f"xref\n0 {len(objects) + 1}\n".encode()) result.extend(b"0000000000 65535 f \n") for offset in offsets[1:]: result.extend(f"{offset:010d} 00000 n \n".encode()) result.extend( ( f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" f"startxref\n{xref_offset}\n%%EOF\n" ).encode() ) return bytes(result) def test_data_process_full_contract_without_database(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) created = client.post( "/modelTF/data-process", json={ "name": "客服问答处理", "process_type": "structured", "config": {"dataset_split": {"train": 80, "validation": 10, "test": 10}}, }, ) assert created.status_code == 200 task_id = created.json()["data"]["id"] source_content = ( '{"question":"如何修改地址？",' '"answer":"订单发货前可在订单详情申请修改收货地址。"}\n' '{"question":"如何申请退款？",' '"answer":"请在订单详情提交退款申请并等待审核处理。"}\n' ) uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("customer.jsonl", source_content.encode(), "application/jsonl")}, ) assert uploaded.status_code == 200 source = uploaded.json()["data"]["files"][0] assert len(source["checksum_sha256"]) == 64 assert source["version_no"] == 1 window = client.get( f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content", params={"offset": 0, "limit": 20}, ) assert window.status_code == 200 assert window.json()["data"]["has_more"] is True line_window = client.get( f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content", params={"start_line": 2, "line_count": 1}, ) assert line_window.json()["data"]["start_line"] == 2 assert line_window.json()["data"]["end_line"] == 2 assert line_window.json()["data"]["total_lines"] == 2 preview = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_ids": [source["id"]]}, ) assert preview.status_code == 200 assert preview.json()["data"]["total"] == 2 listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview") assert listed_preview.json()["data"]["total"] == 2 preview_item = listed_preview.json()["data"]["items"][0] updated_preview = client.put( f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}", json={ "edited_content": preview_item["edited_content"], "expected_updated_at": "2026-07-23T00:00:00Z", }, ) assert "quality_score" in updated_preview.json()["data"] generated = client.post(f"/modelTF/data-process/{task_id}/generate") assert generated.status_code == 200 progress = client.get(f"/modelTF/data-process/{task_id}/progress") assert progress.json()["data"]["status"] == "completed" result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"] assert result_page["total"] == 2 keyword_page = client.get( f"/modelTF/data-process/{task_id}/results", params={"keyword": "地址"} ).json()["data"] assert keyword_page["total"] == 1 result = result_page["items"][0] edited = client.put( f"/modelTF/data-process/{task_id}/results/{result['id']}", json={ "output": "人工修改后的完整答案。", "expected_updated_at": "2026-07-23T00:00:00Z", }, ) assert edited.json()["data"]["status"] == "modified" assert "quality_score" in edited.json()["data"] invalid_edit = client.put( f"/modelTF/data-process/{task_id}/results/{result['id']}", json={"output": ""}, ) assert invalid_edit.json()["data"]["status"] == "invalid" assert store.tasks[task_id]["error_count"] == 1 restored = client.post( f"/modelTF/data-process/{task_id}/results/{result['id']}/restore" ) assert restored.json()["data"]["output"] == result["original_output"] assert restored.json()["data"]["status"] == "valid" assert store.tasks[task_id]["error_count"] == 0 publish_payload = {"dataset_name": "客服问答清洗集"} first_publish = client.post( f"/modelTF/data-process/{task_id}/publish", json=publish_payload ) second_publish = client.post( f"/modelTF/data-process/{task_id}/publish", json=publish_payload ) assert first_publish.json()["data"]["created"] is True assert second_publish.json()["data"]["created"] is False assert ( first_publish.json()["data"]["dataset"]["id"] == second_publish.json()["data"]["dataset"]["id"] ) def test_preview_build_replaces_only_selected_files_and_reports_file_counts( tmp_path: Path, ) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "逐文件预览", "process_type": "structured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("first.jsonl", b'{"id":1}\n', "application/jsonl")), ( "files", ("second.jsonl", b'{"id":2}\n{"id":3}\n', "application/jsonl"), ), ], ) assert uploaded.status_code == 200 first_source, second_source = uploaded.json()["data"]["files"] first_build = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_ids": [first_source["id"]]}, ) assert first_build.status_code == 200 first_data = first_build.json()["data"] assert first_data["file_counts"] == {first_source["id"]: 1} assert first_data["files"] == [ { "source_file_id": first_source["id"], "preview_count": 1, "status": "completed", } ] first_item = first_data["items"][0] edited = client.put( f"/modelTF/data-process/{task_id}/preview/{first_item['id']}", json={"edited_content": "人工确认后的第一文件预览"}, ) assert edited.status_code == 200 second_build = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_id": second_source["id"]}, ) assert second_build.status_code == 200 second_data = second_build.json()["data"] assert second_data["file_counts"] == {second_source["id"]: 2} assert second_data["files"] == [ { "source_file_id": second_source["id"], "preview_count": 2, "status": "completed", } ] assert {item["source_file_id"] for item in store.previews[task_id]} == { first_source["id"], second_source["id"], } preserved_first = next( item for item in store.previews[task_id] if item["source_file_id"] == first_source["id"] ) assert preserved_first["id"] == first_item["id"] assert preserved_first["edited_content"] == "人工确认后的第一文件预览" previous_second_ids = { item["id"] for item in store.previews[task_id] if item["source_file_id"] == second_source["id"] } next( source for source in store.sources[task_id] if source["id"] == second_source["id"] )["content"] = '{"id":4}\n' rebuilt = client.post( f"/modelTF/data-process/{task_id}/preview/build", json={"source_file_ids": [second_source["id"]]}, ) assert rebuilt.status_code == 200 assert rebuilt.json()["data"]["file_counts"] == {second_source["id"]: 1} current_second_ids = { item["id"] for item in store.previews[task_id] if item["source_file_id"] == second_source["id"] } assert current_second_ids.isdisjoint(previous_second_ids) assert len(current_second_ids) == 1 assert next( item for item in store.previews[task_id] if item["source_file_id"] == first_source["id"] )["id"] == first_item["id"] def test_preview_build_rejects_unknown_and_cross_task_source_file_ids( tmp_path: Path, ) -> None: client, _, _ = make_client(tmp_path) first_task_id = client.post( "/modelTF/data-process", json={"name": "归属任务一", "process_type": "structured", "config": {}}, ).json()["data"]["id"] second_task_id = client.post( "/modelTF/data-process", json={"name": "归属任务二", "process_type": "structured", "config": {}}, ).json()["data"]["id"] foreign_source = client.post( f"/modelTF/data-process/{second_task_id}/source-files", files={"files": ("foreign.jsonl", b'{"id":2}\n', "application/jsonl")}, ).json()["data"]["files"][0] unknown = client.post( f"/modelTF/data-process/{first_task_id}/preview/build", json={"source_file_ids": ["dpsf_not_found"]}, ) assert unknown.status_code == 404 foreign = client.post( f"/modelTF/data-process/{first_task_id}/preview/build", json={"source_file_id": foreign_source["id"]}, ) assert foreign.status_code == 404 ambiguous = client.post( f"/modelTF/data-process/{first_task_id}/preview/build", json={ "source_file_id": foreign_source["id"], "source_file_ids": [foreign_source["id"]], }, ) assert ambiguous.status_code == 422 def test_external_source_never_returns_fake_success(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "外部数据", "process_type": "external", "config": {}}, ).json()["data"]["id"] response = client.post( f"/modelTF/data-process/{task_id}/external/test", json={"type": "mysql", "url": "mysql://db.example/test"}, ) assert response.status_code == 501 assert response.json()["detail"]["code"] == 501 def test_config_validation_and_stop_state(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) invalid = client.post( "/modelTF/data-process", json={ "name": "错误切片配置", "process_type": "unstructured", "config": { "dataset_split": {"train": 80, "validation": 30, "test": 0}, "chunk_size": 100, "chunk_overlap": 90, "min_chunk_size": 20, }, }, ) assert invalid.status_code == 422 legacy_semantic = client.post( "/modelTF/data-process", json={ "name": "旧切分策略", "process_type": "unstructured", "config": {"chunk_method": "semantic"}, }, ) assert legacy_semantic.status_code == 422 assert "chunk_method" in legacy_semantic.text missing_custom_delimiter = client.post( "/modelTF/data-process", json={ "name": "缺少自定义分隔符", "process_type": "unstructured", "config": {"chunk_method": "custom"}, }, ) assert missing_custom_delimiter.status_code == 422 assert "custom_delimiter" in missing_custom_delimiter.text task_id = client.post( "/modelTF/data-process", json={"name": "可停止任务", "process_type": "structured", "config": {}}, ).json()["data"]["id"] store.tasks[task_id]["status"] = "running" stopped = client.post(f"/modelTF/data-process/{task_id}/stop") assert stopped.status_code == 200 assert stopped.json()["data"]["status"] == "stopped" def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "批量上传", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] duplicate_batch = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("first.txt", b"same content", "text/plain")), ("files", ("second.txt", b"same content", "text/plain")), ], ) assert duplicate_batch.status_code == 400 assert store.sources[task_id] == [] assert _stored_files(storage) == [] parse_failure = client.post( f"/modelTF/data-process/{task_id}/source-files", files=[ ("files", ("valid.txt", "先暂存的内容".encode(), "text/plain")), ("files", ("broken.txt", b"\xff", "text/plain")), ], ) assert parse_failure.status_code == 400 assert store.sources[task_id] == [] assert _stored_files(storage) == [] empty = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("empty.txt", b"", "text/plain")}, ) assert empty.status_code == 400 assert store.sources[task_id] == [] assert _stored_files(storage) == [] def test_upload_preserves_store_error_when_storage_rollback_fails( tmp_path: Path, monkeypatch: Any, ) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "回滚异常", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] cleanup_attempts: list[str] = [] def fail_store(*_: Any, **__: Any) -> list[dict[str, Any]]: raise ValueError("simulated database transaction failure") def fail_cleanup(reference: str, **_: Any) -> bool: cleanup_attempts.append(reference) raise OSError("simulated storage cleanup failure") monkeypatch.setattr(store, "add_source_files", fail_store) monkeypatch.setattr(storage, "delete", fail_cleanup) response = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("rollback.txt", b"rollback payload", "text/plain")}, ) assert response.status_code == 400 assert response.json()["detail"]["message"] == "simulated database transaction failure" assert len(cleanup_attempts) == 1 assert store.sources[task_id] == [] def test_preprocess_deduplicates_and_quality_filter_removes_short_results( tmp_path: Path, ) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={ "name": "去重与质量筛选", "process_type": "structured", "config": { "preprocess_options": ["clean_invalid", "deduplicate"], "quality_filter_enabled": True, "filter_low_quality": False, "filter_short_content": True, "min_output_length": 100, }, }, ).json()["data"]["id"] content = ( '{"question":"问题","answer":"短答案"}\n' '{"question":"问题","answer":"短答案"}\n' ).encode() uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("duplicates.jsonl", content, "application/jsonl")}, ) assert uploaded.status_code == 200 preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert preview.json()["data"]["total"] == 1 generated = client.post(f"/modelTF/data-process/{task_id}/generate") assert generated.status_code == 200 progress = client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"] assert progress["status"] == "completed" assert progress["filtered_count"] == 1 assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0 def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None: store = FakeDataProcessStore() task = store.create_task( {"name": "并发代次", "process_type": "structured", "config": {}} ) task_id = task["id"] store.replace_preview_items( task_id, [ { "source_file_id": None, "original_content": "来源内容", "edited_content": "来源内容", "status": "manual", } ], ) first = store.start_generation(task_id, replace_existing=True) first_run_id = first["generation_run_id"] second_run_id = "" def restart_while_old_worker_runs(*_: Any, **__: Any) -> list[dict[str, Any]]: nonlocal second_run_id store.stop_task(task_id) second = store.start_generation(task_id, replace_existing=True) second_run_id = second["generation_run_id"] return [] monkeypatch.setattr( data_process_endpoint, "generate_standard_records", restart_while_old_worker_runs, ) data_process_endpoint._run_generation(store, task_id, first_run_id) assert second_run_id and second_run_id != first_run_id assert store.tasks[task_id]["status"] == "running" assert store.tasks[task_id]["generation_run_id"] == second_run_id assert store.results[task_id] == [] store.mark_failed(task_id, "old failure", generation_run_id=first_run_id) assert store.tasks[task_id]["status"] == "running" def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "状态保护", "process_type": "structured", "config": {}}, ).json()["data"]["id"] response = client.put( f"/modelTF/data-process/{task_id}/results/not-created", json={"instruction": "", "output": "", "status": "valid"}, ) assert response.status_code == 422 def test_start_rebuilds_preview_and_generates_in_one_request(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "一键处理", "process_type": "structured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={ "files": ( "one.jsonl", b'{"question":"What is one?","answer":"One."}\n', "application/jsonl", ) }, ) assert uploaded.status_code == 200 started = client.post(f"/modelTF/data-process/{task_id}/start") assert started.status_code == 200 assert started.json()["data"]["task_id"] == task_id assert started.json()["data"]["status"] == "running" assert client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]["status"] == "completed" assert client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]["total"] == 1 assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1 def test_unsupported_upload_format_returns_415(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "格式限制", "process_type": "structured", "config": {}}, ).json()["data"]["id"] response = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("payload.exe", b"not supported", "application/octet-stream")}, ) assert response.status_code == 415 legacy = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("document.doc", b"legacy", "application/msword")}, ) assert legacy.status_code == 415 assert "convert the file to .docx" in legacy.json()["detail"]["message"] def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "XLSX 上传", "process_type": "structured", "config": {}}, ).json()["data"]["id"] workbook = Workbook() worksheet = workbook.active worksheet.append(["question", "answer"]) worksheet.append(["问题一", "答案一"]) worksheet.append(["问题二", "答案二"]) output = BytesIO() workbook.save(output) workbook.close() original_bytes = output.getvalue() response = client.post( f"/modelTF/data-process/{task_id}/source-files", files={ "files": ( "records.xlsx", original_bytes, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) }, ) assert response.status_code == 200 source = response.json()["data"]["files"][0] assert source["file_format"] == "xlsx" assert source["record_count"] == 2 assert source["size_bytes"] == len(original_bytes) assert source["storage_object_id"].startswith("local://data-process/") assert str(storage.root) not in response.text assert storage.read(source["storage_object_id"]) == original_bytes stored_source = store.get_source_file(task_id, source["id"]) assert stored_source["id"] == source["id"] assert stored_source["storage_object_id"] == source["storage_object_id"] assert stored_source["metadata"]["storage_backend"] == "local" assert stored_source["metadata"]["original_size_bytes"] == len(original_bytes) assert '"question":"问题一"' in stored_source["content"] content = client.get( f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content" ) assert content.status_code == 200 assert '"answer":"答案二"' in content.json()["data"]["content"] preview = client.post(f"/modelTF/data-process/{task_id}/preview/build") assert preview.status_code == 200 assert preview.json()["data"]["total"] == 2 def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None: client, store, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "PDF 原件预览", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] original_pdf = _minimal_pdf() uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("说明 文档.pdf", original_pdf, "application/pdf")}, ).json()["data"]["files"][0] raw_url = f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw" full = client.get(raw_url) assert full.status_code == 200 assert full.content == original_pdf assert full.headers["content-type"] == "application/pdf" assert full.headers["accept-ranges"] == "bytes" assert full.headers["cache-control"] == "private, no-store" assert full.headers["content-length"] == str(len(original_pdf)) assert full.headers["content-disposition"].startswith("inline;") assert "%E8%AF%B4%E6%98%8E%20%E6%96%87%E6%A1%A3.pdf" in full.headers[ "content-disposition" ] assert full.headers["etag"] == f'"{uploaded["checksum_sha256"]}"' partial = client.get(raw_url, headers={"Range": "bytes=5-14"}) assert partial.status_code == 206 assert partial.content == original_pdf[5:15] assert partial.headers["content-range"] == f"bytes 5-14/{len(original_pdf)}" assert partial.headers["content-length"] == "10" suffix = client.get(raw_url, headers={"Range": "bytes=-8"}) assert suffix.status_code == 206 assert suffix.content == original_pdf[-8:] invalid = client.get(raw_url, headers={"Range": "bytes=0-1,4-5"}) assert invalid.status_code == 416 assert invalid.headers["content-range"] == f"bytes */{len(original_pdf)}" pages_url = ( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages" ) pages = client.get(pages_url) assert pages.status_code == 200 assert pages.json()["data"] == { "page_count": 1, "pages": [ { "page_number": 1, "source_start": 0, "source_end": len("Hello PDF"), } ], } legacy_id = "dpsf_legacy_pdf" store.add_source_file( task_id, id=legacy_id, storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1", name="legacy.pdf", content="legacy extracted PDF text", raw_size=len(original_pdf), checksum_sha256="a" * 64, file_format="pdf", record_count=1, metadata={"legacy": True}, ) legacy = client.get( f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/raw" ) assert legacy.status_code == 410 legacy_pages = client.get( f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/pdf-pages" ) assert legacy_pages.status_code == 410 def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "非 PDF 原件", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("notes.txt", b"plain source text", "text/plain")}, ).json()["data"]["files"][0] response = client.get( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw" ) assert response.status_code == 415 pages_response = client.get( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages" ) assert pages_response.status_code == 415 def test_delete_source_removes_owned_local_object_and_accepts_legacy_db_reference( tmp_path: Path, ) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "删除原件", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("原件.txt", "本地原始内容".encode(), "text/plain")}, ).json()["data"]["files"][0] reference = uploaded["storage_object_id"] assert storage.read(reference) == "本地原始内容".encode() deleted = client.delete( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}" ) assert deleted.status_code == 200 assert deleted.json()["data"]["storage_cleanup_pending"] is False with pytest.raises(DataProcessStorageError, match="does not exist"): storage.read(reference) legacy_id = "dpsf_legacy" store.add_source_file( task_id, id=legacy_id, storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1", name="legacy.txt", content="旧记录正文", raw_size=len("旧记录正文".encode()), checksum_sha256="a" * 64, file_format="txt", record_count=1, metadata={"legacy": True}, ) legacy_deleted = client.delete( f"/modelTF/data-process/{task_id}/source-files/{legacy_id}" ) assert legacy_deleted.status_code == 200 assert legacy_deleted.json()["data"]["storage_cleanup_pending"] is False def test_delete_reports_pending_cleanup_after_database_soft_delete( tmp_path: Path, monkeypatch: Any, ) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "待清理原件", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("pending.txt", b"pending cleanup", "text/plain")}, ).json()["data"]["files"][0] def fail_cleanup(*_: Any, **__: Any) -> bool: raise OSError("simulated storage failure") monkeypatch.setattr(storage, "delete", fail_cleanup) response = client.delete( f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}" ) assert response.status_code == 200 assert response.json()["data"]["storage_cleanup_pending"] is True with pytest.raises(NotFoundError): store.get_source_file(task_id, uploaded["id"]) assert storage.read(uploaded["storage_object_id"]) == b"pending cleanup" def test_delete_rejects_polluted_reference_owned_by_another_source(tmp_path: Path) -> None: client, store, storage = make_client(tmp_path) task_id = client.post( "/modelTF/data-process", json={"name": "归属校验", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] uploaded = client.post( f"/modelTF/data-process/{task_id}/source-files", files={"files": ("safe.txt", b"owned content", "text/plain")}, ).json()["data"]["files"][0] target_reference = uploaded["storage_object_id"] polluted_id = "dpsf_polluted" store.add_source_file( task_id, id=polluted_id, storage_object_id=target_reference, name="polluted.txt", content="polluted", raw_size=8, checksum_sha256="b" * 64, file_format="txt", record_count=1, metadata={}, ) rejected = client.delete( f"/modelTF/data-process/{task_id}/source-files/{polluted_id}" ) assert rejected.status_code == 400 assert storage.read(target_reference) == b"owned content" assert store.get_source_file(task_id, polluted_id)["id"] == polluted_id def test_upload_format_must_match_process_type(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) structured_id = client.post( "/modelTF/data-process", json={"name": "结构化格式约束", "process_type": "structured", "config": {}}, ).json()["data"]["id"] structured_pdf = client.post( f"/modelTF/data-process/{structured_id}/source-files", files={"files": ("manual.pdf", b"not parsed", "application/pdf")}, ) assert structured_pdf.status_code == 415 unstructured_id = client.post( "/modelTF/data-process", json={"name": "非结构化格式约束", "process_type": "unstructured", "config": {}}, ).json()["data"]["id"] unstructured_xlsx = client.post( f"/modelTF/data-process/{unstructured_id}/source-files", files={"files": ("records.xlsx", b"not parsed", "application/octet-stream")}, ) assert unstructured_xlsx.status_code == 415 external_id = client.post( "/modelTF/data-process", json={"name": "外部数据格式约束", "process_type": "external", "config": {}}, ).json()["data"]["id"] external_upload = client.post( f"/modelTF/data-process/{external_id}/source-files", files={"files": ("records.jsonl", b'{"id":1}', "application/jsonl")}, ) assert external_upload.status_code == 409 def _preview_task( content: str, *, options: list[str], config: dict[str, Any] | None = None, source_id: str = "source-1", file_format: str = "txt", ) -> list[dict[str, Any]]: task_config = { "preprocess_options": options, "chunk_size": 200, "chunk_overlap": 20, "min_chunk_size": 20, **(config or {}), } return data_process_endpoint._build_preview_items( {"process_type": "unstructured", "config": task_config}, [ { "id": source_id, "name": f"{source_id}.{file_format}", "file_format": file_format, "content": content, } ], ) def test_default_and_structure_preview_split_headings_without_cross_section_overlap() -> None: content = ( "# 第一章\n" + " ".join(f"alpha{index}" for index in range(18)) + "\n# 第二章\n" + " ".join(f"beta{index}" for index in range(18)) ) normalized = normalize_text(content) second_chapter_start = normalized.index("# 第二章") common_config = {"chunk_size": 10, "chunk_overlap": 3, "min_chunk_size": 4} default_items = _preview_task( content, options=["preserve_context"], config=common_config, ) structure_items = _preview_task( content, options=["preserve_context"], config={**common_config, "chunk_method": "structure"}, ) def snapshot(items: list[dict[str, Any]]) -> list[tuple[Any, ...]]: return [ ( item["original_content"], item["source_start"], item["source_end"], item["source_start_line"], item["source_end_line"], ) for item in items ] assert snapshot(default_items) == snapshot(structure_items) assert all( item["original_content"] == normalized[item["source_start"] : item["source_end"]] for item in structure_items ) assert all( not (item["source_start"] < second_chapter_start < item["source_end"]) for item in structure_items ) second_chapter_items = [ item for item in structure_items if item["source_start"] >= second_chapter_start ] assert second_chapter_items[0]["source_start"] == second_chapter_start assert second_chapter_items[0]["original_content"].startswith("# 第二章") def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None: repeated = "@" * 120 assert len(_preview_task(repeated, options=[])) == 1 assert _preview_task(repeated, options=["clean_invalid_content"]) == [] structured_text = "# 第一章\n" + "甲。" * 30 + "\n# 第二章\n" + "乙。" * 30 detected = _preview_task( structured_text, options=["detect_document_structure"], config={"chunk_method": "fixed", "chunk_size": 20, "min_chunk_size": 5}, ) undetected = _preview_task( structured_text, options=[], config={"chunk_method": "fixed", "chunk_size": 20, "min_chunk_size": 5}, ) assert all("heading_path" in item["quality_score"] for item in detected) assert {tuple(item["quality_score"]["heading_path"]) for item in detected} == { ("第一章",), ("第二章",), } assert all("heading_path" not in item["quality_score"] for item in undetected) assert all(not ("第一章" in item["edited_content"] and "第二章" in item["edited_content"]) for item in detected) short_lead = "a b. c d e f g h i j k l m n o p q r s t u v w x y z" without_merge = _preview_task( short_lead, options=[], config={"chunk_method": "structure", "chunk_size": 12, "min_chunk_size": 5}, ) with_merge = _preview_task( short_lead, options=["merge_short_content"], config={"chunk_method": "structure", "chunk_size": 12, "min_chunk_size": 5}, ) assert without_merge[0]["token_count"] < 5 assert with_merge[0]["token_count"] >= 5 mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。" assert len(_preview_task(mojibake, options=[])) == 1 assert _preview_task(mojibake, options=["filter_low_quality"]) == [] first = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega" second = "alpha beta gamma, delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega" sources = [ {"id": "near-1", "name": "one.txt", "file_format": "txt", "content": first}, {"id": "near-2", "name": "two.txt", "file_format": "txt", "content": second}, ] base_task = { "process_type": "unstructured", "config": { "chunk_method": "fixed", "chunk_size": 200, "chunk_overlap": 0, "min_chunk_size": 1, "preprocess_options": [], }, } assert len(data_process_endpoint._build_preview_items(base_task, sources)) == 2 deduplicated_task = deepcopy(base_task) deduplicated_task["config"]["preprocess_options"] = ["deduplicate_content"] assert len(data_process_endpoint._build_preview_items(deduplicated_task, sources)) == 1 context_text = " ".join(f"token{index}" for index in range(45)) no_context = _preview_task( context_text, options=[], config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5}, ) with_context = _preview_task( context_text, options=["preserve_context"], config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5}, ) assert no_context[1]["source_start"] >= no_context[0]["source_end"] assert with_context[1]["source_start"] < with_context[0]["source_end"] sensitive = "联系人：张三，手机 13800138000，邮箱 user@example.com。" plain = _preview_task(sensitive, options=[])[0] masked = _preview_task(sensitive, options=["desensitize"])[0] assert "张三" in plain["edited_content"] assert "联系人:[NAME]" in masked["edited_content"] assert "[PHONE]" in masked["edited_content"] assert "[EMAIL]" in masked["edited_content"] def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None: for file_format in ("pdf", "docx", "pptx"): items = _preview_task( f"{file_format.upper()} 已抽取正文，可直接进入切片处理。", options=[], file_format=file_format, ) assert len(items) == 1 assert "已抽取正文" in items[0]["edited_content"]