2012 lines
74 KiB
Python
2012 lines
74 KiB
Python
from __future__ import annotations
|
||
|
||
from copy import deepcopy
|
||
from io import BytesIO
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
from openpyxl import Workbook
|
||
|
||
from app.api.v1.endpoints import data_process as data_process_endpoint
|
||
from app.api.v1.endpoints.data_process import router
|
||
from app.modules.data_process.algorithms import DocumentNoiseSpan, normalize_text
|
||
from app.modules.data_process.storage import (
|
||
DataProcessStorageError,
|
||
LocalDataProcessStorage,
|
||
get_data_process_storage,
|
||
)
|
||
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
|
||
|
||
|
||
class FakeDataProcessStore:
|
||
"""接口测试专用内存实现，确保测试不会连接或迁移真实数据库。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.tasks: dict[str, dict[str, Any]] = {}
|
||
self.sources: dict[str, list[dict[str, Any]]] = {}
|
||
self.previews: dict[str, list[dict[str, Any]]] = {}
|
||
self.results: dict[str, list[dict[str, Any]]] = {}
|
||
self.datasets: dict[str, dict[str, Any]] = {}
|
||
self.regeneration_prepared: set[str] = set()
|
||
self.sequence = 0
|
||
|
||
def _id(self, prefix: str) -> str:
|
||
self.sequence += 1
|
||
return f"{prefix}_{self.sequence}"
|
||
|
||
def list_tasks(self, *, page: int, page_size: int, **filters: Any) -> dict[str, Any]:
|
||
items = [
|
||
{
|
||
**item,
|
||
"source_file_count": len(self.sources.get(str(item["id"]), [])),
|
||
}
|
||
for item in self.tasks.values()
|
||
]
|
||
for field in ("status", "process_type", "tenant_id", "project_id"):
|
||
if filters.get(field):
|
||
items = [item for item in items if item.get(field) == filters[field]]
|
||
keyword = filters.get("keyword")
|
||
if keyword:
|
||
items = [item for item in items if keyword in item["name"]]
|
||
return {
|
||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||
"total": len(items),
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
def create_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
task_id = self._id("dpt")
|
||
task = {
|
||
"id": task_id,
|
||
**deepcopy(payload),
|
||
"status": "pending",
|
||
"progress": 0,
|
||
"input_count": 0,
|
||
"output_count": 0,
|
||
"filtered_count": 0,
|
||
"duplicate_count": 0,
|
||
"error_count": 0,
|
||
"failure_reason": None,
|
||
"output_dataset_id": None,
|
||
}
|
||
self.tasks[task_id] = task
|
||
self.sources[task_id] = []
|
||
self.previews[task_id] = []
|
||
self.results[task_id] = []
|
||
return deepcopy(task)
|
||
|
||
def get_task(self, task_id: str) -> dict[str, Any]:
|
||
if task_id not in self.tasks:
|
||
raise NotFoundError("data process task not found")
|
||
task = deepcopy(self.tasks[task_id])
|
||
split_order = {"train": 0, "val": 1, "test": 2}
|
||
task["output_datasets"] = sorted(
|
||
(
|
||
deepcopy(dataset)
|
||
for dataset in self.datasets.values()
|
||
if dataset.get("source_task_id") == task_id
|
||
and dataset.get("deleted_at") is None
|
||
),
|
||
key=lambda dataset: split_order.get(str(dataset.get("type")), 3),
|
||
)
|
||
return task
|
||
|
||
def recover_legacy_aborted_regeneration(self, task_id: str) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
return {"recovered": False, "result_count": 0}
|
||
|
||
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
self.tasks[task_id].update(deepcopy(payload))
|
||
return self.get_task(task_id)
|
||
|
||
def prepare_regeneration(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
task = self.tasks.get(task_id)
|
||
if task is None:
|
||
raise NotFoundError("data process task not found")
|
||
if task["status"] == "running":
|
||
raise InvalidStateError("running task cannot be prepared for regeneration")
|
||
if payload["expected_updated_at"] != task.get("updated_at"):
|
||
raise InvalidStateError("data process task was modified by another request")
|
||
if payload["process_type"] != task["process_type"]:
|
||
raise InvalidStateError("process_type cannot be changed during regeneration")
|
||
published_outputs_preserved = bool(task.get("output_dataset_id")) or any(
|
||
dataset.get("source_task_id") == task_id
|
||
for dataset in self.datasets.values()
|
||
)
|
||
task.update(
|
||
{
|
||
"name": payload["name"],
|
||
"description": payload["description"],
|
||
"config": deepcopy(payload["config"]),
|
||
"updated_at": "2026-07-25T20:00:00Z",
|
||
}
|
||
)
|
||
self.regeneration_prepared.add(task_id)
|
||
return {
|
||
"task": deepcopy(task),
|
||
"preview_invalidated": False,
|
||
"published_outputs_preserved": published_outputs_preserved,
|
||
}
|
||
|
||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||
self.get_task(task_id)
|
||
if self.tasks[task_id]["status"] == "running":
|
||
raise InvalidStateError("running task must be stopped before deletion")
|
||
del self.tasks[task_id]
|
||
|
||
def list_source_files(self, task_id: str) -> list[dict[str, Any]]:
|
||
self.get_task(task_id)
|
||
return [
|
||
{key: value for key, value in item.items() if key != "content"}
|
||
for item in self.sources[task_id]
|
||
]
|
||
|
||
def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]:
|
||
self.get_task(task_id)
|
||
values = deepcopy(payload)
|
||
source_id = str(values.pop("id", None) or self._id("dpsf"))
|
||
storage_object_id = str(
|
||
values.pop("storage_object_id", None)
|
||
or f"db://data-process/{task_id}/{source_id}/v1"
|
||
)
|
||
raw_size = int(values.pop("raw_size"))
|
||
metadata = deepcopy(values.pop("metadata", {}))
|
||
metadata.setdefault(
|
||
"storage_backend",
|
||
"local" if storage_object_id.startswith("local://data-process/") else "database",
|
||
)
|
||
source = {
|
||
"id": source_id,
|
||
"task_id": task_id,
|
||
"version_no": 1,
|
||
"storage_object_id": storage_object_id,
|
||
"size_bytes": raw_size,
|
||
"metadata": metadata,
|
||
**values,
|
||
}
|
||
self.sources[task_id].append(source)
|
||
self.tasks[task_id]["input_count"] += payload["record_count"]
|
||
return {key: value for key, value in deepcopy(source).items() if key != "content"}
|
||
|
||
def add_source_files(
|
||
self, task_id: str, files: list[dict[str, Any]]
|
||
) -> list[dict[str, Any]]:
|
||
# 先验证整个批次，模拟数据库事务的 all-or-nothing 语义。
|
||
checksums = {item["checksum_sha256"] for item in self.sources.get(task_id, [])}
|
||
incoming: set[str] = set()
|
||
for payload in files:
|
||
checksum = payload["checksum_sha256"]
|
||
if checksum in checksums or checksum in incoming:
|
||
raise ValueError("the same source file content is already attached to this task")
|
||
incoming.add(checksum)
|
||
created = [self.add_source_file(task_id, **payload) for payload in files]
|
||
self.results[task_id] = []
|
||
self.tasks[task_id].update(
|
||
{
|
||
"status": "pending",
|
||
"progress": 20 if self.previews[task_id] else 0,
|
||
"output_count": 0,
|
||
"filtered_count": 0,
|
||
"duplicate_count": 0,
|
||
"error_count": 0,
|
||
"failure_reason": None,
|
||
"generation_run_id": None,
|
||
"started_at": None,
|
||
"completed_at": None,
|
||
}
|
||
)
|
||
return created
|
||
|
||
def get_source_file(
|
||
self, task_id: str, file_id: str, *, include_content: bool = True
|
||
) -> dict[str, Any]:
|
||
source = next(
|
||
(item for item in self.sources.get(task_id, []) if item["id"] == file_id),
|
||
None,
|
||
)
|
||
if not source:
|
||
raise NotFoundError("source file not found")
|
||
result = deepcopy(source)
|
||
if not include_content:
|
||
result.pop("content", None)
|
||
return result
|
||
|
||
def source_content_window(
|
||
self, task_id: str, file_id: str, offset: int, limit: int
|
||
) -> dict[str, Any]:
|
||
source = self.get_source_file(task_id, file_id)
|
||
content = source.pop("content")
|
||
return {
|
||
"file": source,
|
||
"content": content[offset : offset + limit],
|
||
"offset": offset,
|
||
"limit": limit,
|
||
"total_chars": len(content),
|
||
"has_more": offset + limit < len(content),
|
||
}
|
||
|
||
def source_content_lines(
|
||
self, task_id: str, file_id: str, start_line: int, line_count: int
|
||
) -> dict[str, Any]:
|
||
source = self.get_source_file(task_id, file_id)
|
||
lines = source.pop("content").splitlines(keepends=True)
|
||
selected = lines[start_line - 1 : start_line - 1 + line_count]
|
||
return {
|
||
"file": source,
|
||
"content": "".join(selected),
|
||
"start_line": start_line,
|
||
"end_line": start_line - 1 + len(selected),
|
||
"line_count": len(selected),
|
||
"total_lines": len(lines),
|
||
"has_more": start_line - 1 + len(selected) < len(lines),
|
||
}
|
||
|
||
def delete_source_file(self, task_id: str, file_id: str) -> None:
|
||
self.get_source_file(task_id, file_id)
|
||
self.sources[task_id] = [item for item in self.sources[task_id] if item["id"] != file_id]
|
||
self.previews[task_id] = [
|
||
item for item in self.previews[task_id] if item["source_file_id"] != file_id
|
||
]
|
||
self.results[task_id] = []
|
||
|
||
def replace_preview_items(
|
||
self,
|
||
task_id: str,
|
||
items: list[dict[str, Any]],
|
||
*,
|
||
source_file_ids: list[str] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
created = [
|
||
{"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items
|
||
]
|
||
if source_file_ids is None:
|
||
self.previews[task_id] = created
|
||
else:
|
||
selected = set(source_file_ids)
|
||
self.previews[task_id] = [
|
||
item
|
||
for item in self.previews[task_id]
|
||
if item["source_file_id"] not in selected
|
||
] + created
|
||
self.results[task_id] = []
|
||
self.tasks[task_id]["progress"] = 20
|
||
return deepcopy(created)
|
||
|
||
def list_preview_items(
|
||
self,
|
||
task_id: str,
|
||
*,
|
||
page: int,
|
||
page_size: int,
|
||
source_file_id: str | None = None,
|
||
keyword: str | None = None,
|
||
) -> dict[str, Any]:
|
||
items = self.previews[task_id]
|
||
if source_file_id:
|
||
items = [item for item in items if item["source_file_id"] == source_file_id]
|
||
if keyword:
|
||
items = [item for item in items if keyword in item["edited_content"]]
|
||
return {
|
||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||
"total": len(items),
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]:
|
||
item = next(
|
||
(item for item in self.previews.get(task_id, []) if item["id"] == preview_id),
|
||
None,
|
||
)
|
||
if not item:
|
||
raise NotFoundError("preview item not found")
|
||
return deepcopy(item)
|
||
|
||
def create_preview_item(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
item = {"id": self._id("dpp"), "task_id": task_id, **deepcopy(payload)}
|
||
self.previews[task_id].append(item)
|
||
self.results[task_id] = []
|
||
return deepcopy(item)
|
||
|
||
def update_preview_item(
|
||
self, task_id: str, preview_id: str, payload: dict[str, Any]
|
||
) -> dict[str, Any]:
|
||
item = next(
|
||
(item for item in self.previews[task_id] if item["id"] == preview_id),
|
||
None,
|
||
)
|
||
if not item:
|
||
raise NotFoundError("preview item not found")
|
||
item.update(deepcopy(payload))
|
||
self.results[task_id] = []
|
||
return deepcopy(item)
|
||
|
||
def delete_preview_item(self, task_id: str, preview_id: str) -> None:
|
||
before = len(self.previews[task_id])
|
||
self.previews[task_id] = [
|
||
item for item in self.previews[task_id] if item["id"] != preview_id
|
||
]
|
||
if len(self.previews[task_id]) == before:
|
||
raise NotFoundError("preview item not found")
|
||
|
||
def start_generation(self, task_id: str, *, replace_existing: bool) -> dict[str, Any]:
|
||
if not self.previews[task_id]:
|
||
raise InvalidStateError("preview must be built before generation")
|
||
task = self.tasks[task_id]
|
||
if task.get("output_dataset_id") and task_id not in self.regeneration_prepared:
|
||
raise InvalidStateError("published task cannot be regenerated")
|
||
if replace_existing:
|
||
self.results[task_id] = []
|
||
task.update(
|
||
status="running",
|
||
progress=30,
|
||
output_dataset_id=None,
|
||
output_count=0,
|
||
generation_run_id=self._id("dprun"),
|
||
)
|
||
self.regeneration_prepared.discard(task_id)
|
||
return self.get_task(task_id)
|
||
|
||
def generation_is_running(self, task_id: str, generation_run_id: str) -> bool:
|
||
return (
|
||
self.tasks[task_id]["status"] == "running"
|
||
and self.tasks[task_id].get("generation_run_id") == generation_run_id
|
||
)
|
||
|
||
def update_generation_progress(
|
||
self,
|
||
task_id: str,
|
||
generation_run_id: str,
|
||
processed_count: int,
|
||
total_count: int,
|
||
) -> bool:
|
||
if not self.generation_is_running(task_id, generation_run_id):
|
||
return False
|
||
self.tasks[task_id]["progress"] = min(
|
||
95,
|
||
30 + processed_count / max(1, total_count) * 65,
|
||
)
|
||
return True
|
||
|
||
def complete_generation(
|
||
self,
|
||
task_id: str,
|
||
results: list[dict[str, Any]],
|
||
*,
|
||
generation_run_id: str,
|
||
**counts: Any,
|
||
) -> dict[str, Any]:
|
||
if not self.generation_is_running(task_id, generation_run_id):
|
||
return self.get_task(task_id)
|
||
self.results[task_id] = deepcopy(results)
|
||
self.tasks[task_id].update(
|
||
status="completed",
|
||
progress=100,
|
||
output_count=len(results),
|
||
generation_run_id=None,
|
||
**counts,
|
||
)
|
||
return self.get_task(task_id)
|
||
|
||
def mark_failed(
|
||
self, task_id: str, reason: str, *, generation_run_id: str
|
||
) -> dict[str, Any]:
|
||
if self.generation_is_running(task_id, generation_run_id):
|
||
self.tasks[task_id].update(
|
||
status="failed",
|
||
failure_reason=reason,
|
||
generation_run_id=None,
|
||
)
|
||
return self.get_task(task_id)
|
||
|
||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||
if self.tasks[task_id]["status"] != "running":
|
||
raise InvalidStateError("only a running task can be stopped")
|
||
self.tasks[task_id].update(status="stopped", generation_run_id=None)
|
||
return self.get_task(task_id)
|
||
|
||
def progress(self, task_id: str) -> dict[str, Any]:
|
||
task = self.get_task(task_id)
|
||
result = {key: task.get(key) for key in (
|
||
"status", "progress", "input_count", "output_count",
|
||
"filtered_count", "duplicate_count", "error_count", "failure_reason",
|
||
)}
|
||
result["task_id"] = task["id"]
|
||
return result
|
||
|
||
def list_results(
|
||
self,
|
||
task_id: str,
|
||
*,
|
||
page: int,
|
||
page_size: int,
|
||
status: str | None = None,
|
||
split: str | None = None,
|
||
keyword: str | None = None,
|
||
) -> dict[str, Any]:
|
||
items = self.results[task_id]
|
||
if status:
|
||
items = [item for item in items if item["status"] == status]
|
||
if split:
|
||
items = [item for item in items if item["split"] == split]
|
||
if keyword:
|
||
items = [
|
||
item
|
||
for item in items
|
||
if any(keyword in item[field] for field in ("instruction", "input", "output"))
|
||
]
|
||
return {
|
||
"items": deepcopy(items),
|
||
"total": len(items),
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
def update_result(
|
||
self, task_id: str, result_id: str, payload: dict[str, Any]
|
||
) -> dict[str, Any]:
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
for field in ("instruction", "input", "output", "quality_score"):
|
||
if field in payload:
|
||
item[field] = deepcopy(payload[field])
|
||
hard_valid = bool(item["instruction"].strip() and item["output"].strip())
|
||
quality_valid = bool((item.get("quality_score") or {}).get("is_valid", hard_valid))
|
||
changed = any(
|
||
item[field] != item[f"original_{field}"]
|
||
for field in ("instruction", "input", "output")
|
||
)
|
||
item["status"] = (
|
||
"invalid"
|
||
if not hard_valid or not quality_valid
|
||
else "modified" if changed else "valid"
|
||
)
|
||
self.tasks[task_id]["error_count"] = sum(
|
||
result["status"] == "invalid" for result in self.results[task_id]
|
||
)
|
||
return deepcopy(item)
|
||
|
||
def get_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
return deepcopy(item)
|
||
|
||
def restore_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||
if not item:
|
||
raise NotFoundError("data process result not found")
|
||
for field in ("instruction", "input", "output"):
|
||
item[field] = item[f"original_{field}"]
|
||
item["status"] = "valid"
|
||
return deepcopy(item)
|
||
|
||
def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||
task = self.tasks[task_id]
|
||
if task_id in self.regeneration_prepared:
|
||
raise InvalidStateError("regeneration must start and complete before publishing")
|
||
published = [
|
||
dataset
|
||
for dataset in self.datasets.values()
|
||
if dataset.get("source_task_id") == task_id
|
||
and dataset.get("deleted_at") is None
|
||
]
|
||
if task.get("output_dataset_id"):
|
||
train_dataset = self.datasets[task["output_dataset_id"]]
|
||
return {
|
||
"dataset": deepcopy(train_dataset),
|
||
"datasets": deepcopy(published),
|
||
"output_datasets": deepcopy(published),
|
||
"created": False,
|
||
}
|
||
if task["status"] != "completed":
|
||
raise InvalidStateError("only a completed task can be published")
|
||
split_specs = (
|
||
("train", "训练集"),
|
||
("val", "验证集"),
|
||
("test", "测试集"),
|
||
)
|
||
for dataset_type, label in split_specs:
|
||
dataset_id = self._id("dataset")
|
||
self.datasets[dataset_id] = {
|
||
"id": dataset_id,
|
||
"name": f"{payload['dataset_name']}-{label}",
|
||
"type": dataset_type,
|
||
"count": len(self.results[task_id]),
|
||
"source": "task",
|
||
"task_id": task_id,
|
||
"source_task_id": task_id,
|
||
"deleted_at": None,
|
||
}
|
||
published = [
|
||
dataset
|
||
for dataset in self.datasets.values()
|
||
if dataset.get("source_task_id") == task_id
|
||
and dataset.get("deleted_at") is None
|
||
]
|
||
train_dataset = next(dataset for dataset in published if dataset["type"] == "train")
|
||
task["output_dataset_id"] = train_dataset["id"]
|
||
return {
|
||
"dataset": deepcopy(train_dataset),
|
||
"datasets": deepcopy(published),
|
||
"output_datasets": deepcopy(published),
|
||
"created": True,
|
||
}
|
||
|
||
|
||
def make_client(
|
||
tmp_path: Path,
|
||
) -> tuple[TestClient, FakeDataProcessStore, LocalDataProcessStorage]:
|
||
store = FakeDataProcessStore()
|
||
storage = LocalDataProcessStorage(tmp_path / "data-process")
|
||
app = FastAPI()
|
||
app.include_router(router, prefix="/modelTF")
|
||
app.dependency_overrides[get_data_process_store] = lambda: store
|
||
app.dependency_overrides[get_data_process_storage] = lambda: storage
|
||
return TestClient(app), store, storage
|
||
|
||
|
||
def _stored_files(storage: LocalDataProcessStorage) -> list[Path]:
|
||
return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()]
|
||
|
||
|
||
def _minimal_pdf_pages(*page_texts: str) -> bytes:
|
||
if not page_texts:
|
||
raise ValueError("at least one PDF page is required")
|
||
font_object_number = 3 + len(page_texts) * 2
|
||
page_object_numbers = [3 + index * 2 for index in range(len(page_texts))]
|
||
objects = [
|
||
b"<< /Type /Catalog /Pages 2 0 R >>",
|
||
(
|
||
b"<< /Type /Pages /Kids ["
|
||
+ b" ".join(f"{number} 0 R".encode() for number in page_object_numbers)
|
||
+ b"] /Count "
|
||
+ str(len(page_texts)).encode()
|
||
+ b" >>"
|
||
),
|
||
]
|
||
for index, text in enumerate(page_texts):
|
||
content_object_number = page_object_numbers[index] + 1
|
||
commands = [b"BT /F1 12 Tf 72 720 Td"]
|
||
for line_index, line in enumerate(text.splitlines()):
|
||
escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
||
if line_index:
|
||
commands.append(b"0 -16 Td")
|
||
commands.append(f"({escaped}) Tj".encode("ascii"))
|
||
commands.append(b"ET")
|
||
stream = b" ".join(commands)
|
||
objects.extend(
|
||
[
|
||
(
|
||
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
|
||
b"/Resources << /Font << /F1 "
|
||
+ str(font_object_number).encode()
|
||
+ b" 0 R >> >> /Contents "
|
||
+ str(content_object_number).encode()
|
||
+ b" 0 R >>"
|
||
),
|
||
b"<< /Length "
|
||
+ str(len(stream)).encode()
|
||
+ b" >>\nstream\n"
|
||
+ stream
|
||
+ b"\nendstream",
|
||
]
|
||
)
|
||
objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
|
||
result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
|
||
offsets = [0]
|
||
for object_number, value in enumerate(objects, start=1):
|
||
offsets.append(len(result))
|
||
result.extend(f"{object_number} 0 obj\n".encode())
|
||
result.extend(value)
|
||
result.extend(b"\nendobj\n")
|
||
xref_offset = len(result)
|
||
result.extend(f"xref\n0 {len(objects) + 1}\n".encode())
|
||
result.extend(b"0000000000 65535 f \n")
|
||
for offset in offsets[1:]:
|
||
result.extend(f"{offset:010d} 00000 n \n".encode())
|
||
result.extend(
|
||
(
|
||
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
|
||
f"startxref\n{xref_offset}\n%%EOF\n"
|
||
).encode()
|
||
)
|
||
return bytes(result)
|
||
|
||
|
||
def _minimal_pdf(text: str = "Hello PDF") -> bytes:
|
||
return _minimal_pdf_pages(text)
|
||
|
||
|
||
def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
created = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "客服问答处理",
|
||
"process_type": "structured",
|
||
"config": {"dataset_split": {"train": 80, "validation": 10, "test": 10}},
|
||
},
|
||
)
|
||
assert created.status_code == 200
|
||
task_id = created.json()["data"]["id"]
|
||
|
||
source_content = (
|
||
'{"question":"如何修改地址？",'
|
||
'"answer":"订单发货前可在订单详情申请修改收货地址。"}\n'
|
||
'{"question":"如何申请退款？",'
|
||
'"answer":"请在订单详情提交退款申请并等待审核处理。"}\n'
|
||
)
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("customer.jsonl", source_content.encode(), "application/jsonl")},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
source = uploaded.json()["data"]["files"][0]
|
||
assert len(source["checksum_sha256"]) == 64
|
||
assert source["version_no"] == 1
|
||
|
||
window = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||
params={"offset": 0, "limit": 20},
|
||
)
|
||
assert window.status_code == 200
|
||
assert window.json()["data"]["has_more"] is True
|
||
line_window = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||
params={"start_line": 2, "line_count": 1},
|
||
)
|
||
assert line_window.json()["data"]["start_line"] == 2
|
||
assert line_window.json()["data"]["end_line"] == 2
|
||
assert line_window.json()["data"]["total_lines"] == 2
|
||
|
||
preview = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_ids": [source["id"]]},
|
||
)
|
||
assert preview.status_code == 200
|
||
assert preview.json()["data"]["total"] == 2
|
||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||
assert listed_preview.json()["data"]["total"] == 2
|
||
preview_item = listed_preview.json()["data"]["items"][0]
|
||
updated_preview = client.put(
|
||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||
json={
|
||
"edited_content": preview_item["edited_content"],
|
||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||
},
|
||
)
|
||
assert "quality_score" in updated_preview.json()["data"]
|
||
|
||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
assert generated.status_code == 200
|
||
progress = client.get(f"/modelTF/data-process/{task_id}/progress")
|
||
assert progress.json()["data"]["status"] == "completed"
|
||
result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
|
||
assert result_page["total"] == 2
|
||
keyword_page = client.get(
|
||
f"/modelTF/data-process/{task_id}/results", params={"keyword": "地址"}
|
||
).json()["data"]
|
||
assert keyword_page["total"] == 1
|
||
|
||
result = result_page["items"][0]
|
||
edited = client.put(
|
||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||
json={
|
||
"output": "人工修改后的完整答案。",
|
||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||
},
|
||
)
|
||
assert edited.json()["data"]["status"] == "modified"
|
||
assert "quality_score" in edited.json()["data"]
|
||
invalid_edit = client.put(
|
||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||
json={"output": ""},
|
||
)
|
||
assert invalid_edit.json()["data"]["status"] == "invalid"
|
||
assert store.tasks[task_id]["error_count"] == 1
|
||
restored = client.post(
|
||
f"/modelTF/data-process/{task_id}/results/{result['id']}/restore"
|
||
)
|
||
assert restored.json()["data"]["output"] == result["original_output"]
|
||
assert restored.json()["data"]["status"] == "valid"
|
||
assert store.tasks[task_id]["error_count"] == 0
|
||
|
||
publish_payload = {"dataset_name": "客服问答清洗集"}
|
||
first_publish = client.post(
|
||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||
)
|
||
second_publish = client.post(
|
||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||
)
|
||
assert first_publish.json()["data"]["created"] is True
|
||
assert second_publish.json()["data"]["created"] is False
|
||
assert (
|
||
first_publish.json()["data"]["dataset"]["id"]
|
||
== second_publish.json()["data"]["dataset"]["id"]
|
||
)
|
||
|
||
|
||
def test_task_list_exposes_document_and_generation_counts(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "列表契约", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.sources[task_id] = [{"id": "source-1"}, {"id": "source-2"}]
|
||
store.tasks[task_id].update(
|
||
{
|
||
"status": "pending",
|
||
"output_count": 17,
|
||
"output_dataset_id": None,
|
||
}
|
||
)
|
||
|
||
response = client.get("/modelTF/data-process")
|
||
|
||
assert response.status_code == 200
|
||
item = response.json()["data"]["items"][0]
|
||
assert item["status"] == "pending"
|
||
assert item["source_file_count"] == 2
|
||
assert item["output_count"] == 17
|
||
assert item["output_dataset_id"] is None
|
||
|
||
|
||
def test_task_detail_uses_returned_source_files_as_document_count(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "详情文档数", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["source_file_count"] = 99
|
||
store.sources[task_id] = [
|
||
{"id": "source-1", "name": "一.pdf", "content": "正文一"},
|
||
{"id": "source-2", "name": "二.pdf", "content": "正文二"},
|
||
]
|
||
|
||
response = client.get(f"/modelTF/data-process/{task_id}")
|
||
|
||
assert response.status_code == 200
|
||
detail = response.json()["data"]
|
||
assert len(detail["source_files"]) == 2
|
||
assert detail["source_file_count"] == 2
|
||
|
||
|
||
def test_generation_start_response_clears_previous_output_count(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "新一轮生成", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["output_count"] = 28
|
||
store.previews[task_id] = [
|
||
{
|
||
"id": "preview-1",
|
||
"source_file_id": None,
|
||
"original_content": '{"question":"新问题","answer":"新答案"}',
|
||
"edited_content": '{"question":"新问题","answer":"新答案"}',
|
||
"status": "original",
|
||
}
|
||
]
|
||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *args: None)
|
||
|
||
response = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
|
||
assert response.status_code == 200
|
||
progress = response.json()["data"]
|
||
assert progress["status"] == "running"
|
||
assert progress["output_count"] == 0
|
||
assert store.tasks[task_id]["output_count"] == 0
|
||
|
||
|
||
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "逐文件预览", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("first.jsonl", b'{"id":1}\n', "application/jsonl")),
|
||
(
|
||
"files",
|
||
("second.jsonl", b'{"id":2}\n{"id":3}\n', "application/jsonl"),
|
||
),
|
||
],
|
||
)
|
||
assert uploaded.status_code == 200
|
||
first_source, second_source = uploaded.json()["data"]["files"]
|
||
|
||
first_build = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_ids": [first_source["id"]]},
|
||
)
|
||
assert first_build.status_code == 200
|
||
first_data = first_build.json()["data"]
|
||
assert first_data["file_counts"] == {first_source["id"]: 1}
|
||
assert first_data["files"] == [
|
||
{
|
||
"source_file_id": first_source["id"],
|
||
"preview_count": 1,
|
||
"status": "completed",
|
||
}
|
||
]
|
||
first_item = first_data["items"][0]
|
||
edited = client.put(
|
||
f"/modelTF/data-process/{task_id}/preview/{first_item['id']}",
|
||
json={"edited_content": "人工确认后的第一文件预览"},
|
||
)
|
||
assert edited.status_code == 200
|
||
|
||
second_build = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_id": second_source["id"]},
|
||
)
|
||
assert second_build.status_code == 200
|
||
second_data = second_build.json()["data"]
|
||
assert second_data["file_counts"] == {second_source["id"]: 2}
|
||
assert second_data["files"] == [
|
||
{
|
||
"source_file_id": second_source["id"],
|
||
"preview_count": 2,
|
||
"status": "completed",
|
||
}
|
||
]
|
||
assert {item["source_file_id"] for item in store.previews[task_id]} == {
|
||
first_source["id"],
|
||
second_source["id"],
|
||
}
|
||
preserved_first = next(
|
||
item
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == first_source["id"]
|
||
)
|
||
assert preserved_first["id"] == first_item["id"]
|
||
assert preserved_first["edited_content"] == "人工确认后的第一文件预览"
|
||
|
||
previous_second_ids = {
|
||
item["id"]
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == second_source["id"]
|
||
}
|
||
next(
|
||
source
|
||
for source in store.sources[task_id]
|
||
if source["id"] == second_source["id"]
|
||
)["content"] = '{"id":4}\n'
|
||
rebuilt = client.post(
|
||
f"/modelTF/data-process/{task_id}/preview/build",
|
||
json={"source_file_ids": [second_source["id"]]},
|
||
)
|
||
assert rebuilt.status_code == 200
|
||
assert rebuilt.json()["data"]["file_counts"] == {second_source["id"]: 1}
|
||
current_second_ids = {
|
||
item["id"]
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == second_source["id"]
|
||
}
|
||
assert current_second_ids.isdisjoint(previous_second_ids)
|
||
assert len(current_second_ids) == 1
|
||
assert next(
|
||
item
|
||
for item in store.previews[task_id]
|
||
if item["source_file_id"] == first_source["id"]
|
||
)["id"] == first_item["id"]
|
||
|
||
|
||
def test_preview_build_rejects_unknown_and_cross_task_source_file_ids(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
first_task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "归属任务一", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
second_task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "归属任务二", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
foreign_source = client.post(
|
||
f"/modelTF/data-process/{second_task_id}/source-files",
|
||
files={"files": ("foreign.jsonl", b'{"id":2}\n', "application/jsonl")},
|
||
).json()["data"]["files"][0]
|
||
|
||
unknown = client.post(
|
||
f"/modelTF/data-process/{first_task_id}/preview/build",
|
||
json={"source_file_ids": ["dpsf_not_found"]},
|
||
)
|
||
assert unknown.status_code == 404
|
||
foreign = client.post(
|
||
f"/modelTF/data-process/{first_task_id}/preview/build",
|
||
json={"source_file_id": foreign_source["id"]},
|
||
)
|
||
assert foreign.status_code == 404
|
||
ambiguous = client.post(
|
||
f"/modelTF/data-process/{first_task_id}/preview/build",
|
||
json={
|
||
"source_file_id": foreign_source["id"],
|
||
"source_file_ids": [foreign_source["id"]],
|
||
},
|
||
)
|
||
assert ambiguous.status_code == 422
|
||
|
||
|
||
def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "外部数据", "process_type": "external", "config": {}},
|
||
).json()["data"]["id"]
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/external/test",
|
||
json={"type": "mysql", "url": "mysql://db.example/test"},
|
||
)
|
||
assert response.status_code == 501
|
||
assert response.json()["detail"]["code"] == 501
|
||
|
||
|
||
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "待重新生成", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
{
|
||
"status": "completed",
|
||
"updated_at": "2026-07-25T19:00:00Z",
|
||
"output_dataset_id": "dataset_train",
|
||
"output_count": 2,
|
||
}
|
||
)
|
||
store.datasets["dataset_train"] = {
|
||
"id": "dataset_train",
|
||
"name": "原训练集",
|
||
"type": "train",
|
||
"source_task_id": task_id,
|
||
"deleted_at": None,
|
||
}
|
||
store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}]
|
||
store.results[task_id] = [{"id": "result_1"}]
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/regenerate",
|
||
json={
|
||
"name": "重新生成后名称",
|
||
"description": "更换生成模型",
|
||
"process_type": "structured",
|
||
"config": {"generation_model_id": "model_2"},
|
||
"expected_updated_at": "2026-07-25T19:00:00Z",
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
data = response.json()["data"]
|
||
assert data["task"]["status"] == "completed"
|
||
assert data["task"]["output_dataset_id"] == "dataset_train"
|
||
assert data["task"]["output_count"] == 2
|
||
assert data["preview_invalidated"] is False
|
||
assert data["published_outputs_preserved"] is True
|
||
assert store.results[task_id] == [{"id": "result_1"}]
|
||
assert store.previews[task_id][0]["id"] == "preview_1"
|
||
|
||
detail = client.get(f"/modelTF/data-process/{task_id}").json()["data"]
|
||
assert detail["status"] == "completed"
|
||
assert detail["output_dataset_id"] == "dataset_train"
|
||
assert detail["output_count"] == 2
|
||
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
|
||
|
||
|
||
def test_published_split_datasets_remain_in_detail_after_regeneration(
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "保留旧发布数据", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id].update(
|
||
{
|
||
"status": "completed",
|
||
"updated_at": "2026-07-27T09:00:00Z",
|
||
"output_count": 1,
|
||
}
|
||
)
|
||
store.results[task_id] = [
|
||
{
|
||
"id": "result_1",
|
||
"status": "valid",
|
||
"instruction": "问题",
|
||
"input": "",
|
||
"output": "答案",
|
||
}
|
||
]
|
||
store.previews[task_id] = [{"id": "preview_1", "edited_content": "原切片"}]
|
||
|
||
published = client.post(
|
||
f"/modelTF/data-process/{task_id}/publish",
|
||
json={"dataset_name": "保留旧发布数据集"},
|
||
)
|
||
assert published.status_code == 200
|
||
published_datasets = published.json()["data"]["datasets"]
|
||
assert len(published_datasets) == 3
|
||
published_ids = {item["id"] for item in published_datasets}
|
||
assert store.tasks[task_id]["output_dataset_id"] in published_ids
|
||
|
||
regenerated = client.post(
|
||
f"/modelTF/data-process/{task_id}/regenerate",
|
||
json={
|
||
"name": "保留旧发布数据",
|
||
"description": "更换生成配置后退出",
|
||
"process_type": "structured",
|
||
"config": {"generation_model_id": "model_2"},
|
||
"expected_updated_at": "2026-07-27T09:00:00Z",
|
||
},
|
||
)
|
||
assert regenerated.status_code == 200
|
||
original_output_dataset_id = store.tasks[task_id]["output_dataset_id"]
|
||
prepared_task = regenerated.json()["data"]["task"]
|
||
assert prepared_task["status"] == "completed"
|
||
assert prepared_task["output_dataset_id"] == original_output_dataset_id
|
||
assert prepared_task["output_count"] == 1
|
||
assert store.results[task_id][0]["id"] == "result_1"
|
||
|
||
detail = client.get(f"/modelTF/data-process/{task_id}")
|
||
assert detail.status_code == 200
|
||
detail_data = detail.json()["data"]
|
||
assert detail_data["status"] == "completed"
|
||
assert detail_data["output_dataset_id"] == original_output_dataset_id
|
||
assert detail_data["output_count"] == 1
|
||
assert len(detail_data["output_datasets"]) == 3
|
||
assert {item["id"] for item in detail_data["output_datasets"]} == published_ids
|
||
assert set(store.datasets) == published_ids
|
||
retained_results = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
|
||
assert retained_results["total"] == 1
|
||
assert retained_results["items"][0]["id"] == "result_1"
|
||
|
||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *args: None)
|
||
started = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
assert started.status_code == 200
|
||
running = started.json()["data"]
|
||
assert running["status"] == "running"
|
||
assert running["output_count"] == 0
|
||
assert store.results[task_id] == []
|
||
assert set(store.datasets) == published_ids
|
||
|
||
running_detail = client.get(f"/modelTF/data-process/{task_id}").json()["data"]
|
||
assert running_detail["status"] == "running"
|
||
assert running_detail["output_dataset_id"] is None
|
||
assert running_detail["output_count"] == 0
|
||
assert {item["id"] for item in running_detail["output_datasets"]} == published_ids
|
||
|
||
|
||
def test_regenerate_endpoint_validates_snapshot_and_locked_process_type(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "并发校验", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["updated_at"] = "2026-07-25T19:00:00Z"
|
||
payload = {
|
||
"name": "并发校验",
|
||
"description": "",
|
||
"process_type": "structured",
|
||
"config": {},
|
||
"expected_updated_at": "stale",
|
||
}
|
||
|
||
stale = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
|
||
assert stale.status_code == 409
|
||
|
||
payload.update(
|
||
{
|
||
"process_type": "unstructured",
|
||
"expected_updated_at": "2026-07-25T19:00:00Z",
|
||
}
|
||
)
|
||
locked_type = client.post(f"/modelTF/data-process/{task_id}/regenerate", json=payload)
|
||
assert locked_type.status_code == 409
|
||
|
||
for missing_field in ("expected_updated_at", "description", "config"):
|
||
missing_required_field = client.post(
|
||
f"/modelTF/data-process/{task_id}/regenerate",
|
||
json={key: value for key, value in payload.items() if key != missing_field},
|
||
)
|
||
assert missing_required_field.status_code == 422
|
||
|
||
|
||
def test_config_validation_and_stop_state(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
invalid = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "错误切片配置",
|
||
"process_type": "unstructured",
|
||
"config": {
|
||
"dataset_split": {"train": 80, "validation": 30, "test": 0},
|
||
"chunk_size": 100,
|
||
"chunk_overlap": 90,
|
||
"min_chunk_size": 20,
|
||
},
|
||
},
|
||
)
|
||
assert invalid.status_code == 422
|
||
|
||
semantic = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "语义切分策略",
|
||
"process_type": "unstructured",
|
||
"config": {"chunk_method": "semantic"},
|
||
},
|
||
)
|
||
assert semantic.status_code == 200
|
||
|
||
removed_custom_method = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "已移除的自定义分隔符",
|
||
"process_type": "unstructured",
|
||
"config": {"chunk_method": "custom"},
|
||
},
|
||
)
|
||
assert removed_custom_method.status_code == 422
|
||
assert "chunk_method" in removed_custom_method.text
|
||
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "可停止任务", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
store.tasks[task_id]["status"] = "running"
|
||
stopped = client.post(f"/modelTF/data-process/{task_id}/stop")
|
||
assert stopped.status_code == 200
|
||
assert stopped.json()["data"]["status"] == "stopped"
|
||
|
||
|
||
@pytest.mark.parametrize("config_key", ["qa_pairs_per_row", "qa_pairs_per_chunk"])
|
||
@pytest.mark.parametrize("count", [1, 50])
|
||
def test_qa_pair_config_accepts_supported_boundaries(
|
||
tmp_path: Path,
|
||
config_key: str,
|
||
count: int,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
response = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "问答数量边界",
|
||
"process_type": "unstructured",
|
||
"config": {config_key: count},
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
|
||
|
||
@pytest.mark.parametrize("config_key", ["qa_pairs_per_row", "qa_pairs_per_chunk"])
|
||
@pytest.mark.parametrize("count", [0, 51])
|
||
def test_qa_pair_config_rejects_out_of_range_boundaries(
|
||
tmp_path: Path,
|
||
config_key: str,
|
||
count: int,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
response = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "问答数量越界",
|
||
"process_type": "unstructured",
|
||
"config": {config_key: count},
|
||
},
|
||
)
|
||
|
||
assert response.status_code == 422
|
||
assert "[1, 50]" in response.text
|
||
|
||
|
||
def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "批量上传", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
|
||
duplicate_batch = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("first.txt", b"same content", "text/plain")),
|
||
("files", ("second.txt", b"same content", "text/plain")),
|
||
],
|
||
)
|
||
assert duplicate_batch.status_code == 400
|
||
assert store.sources[task_id] == []
|
||
assert _stored_files(storage) == []
|
||
|
||
parse_failure = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("valid.txt", "先暂存的内容".encode(), "text/plain")),
|
||
("files", ("broken.txt", b"\xff", "text/plain")),
|
||
],
|
||
)
|
||
assert parse_failure.status_code == 400
|
||
assert store.sources[task_id] == []
|
||
assert _stored_files(storage) == []
|
||
|
||
empty = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("empty.txt", b"", "text/plain")},
|
||
)
|
||
assert empty.status_code == 400
|
||
assert store.sources[task_id] == []
|
||
assert _stored_files(storage) == []
|
||
|
||
|
||
def test_incremental_upload_keeps_existing_file_previews(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "增量上传预览", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
first_batch = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files=[
|
||
("files", ("first.txt", "第一个文件内容".encode(), "text/plain")),
|
||
("files", ("second.txt", "第二个文件内容".encode(), "text/plain")),
|
||
],
|
||
)
|
||
assert first_batch.status_code == 200
|
||
first_source_ids = {item["id"] for item in first_batch.json()["data"]["files"]}
|
||
built = client.post(f"/modelTF/data-process/{task_id}/preview/build", json={})
|
||
assert built.status_code == 200
|
||
assert {item["source_file_id"] for item in built.json()["data"]["items"]} == first_source_ids
|
||
store.results[task_id] = [{"id": "old_result"}]
|
||
|
||
third = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("third.txt", "第三个文件内容".encode(), "text/plain")},
|
||
)
|
||
|
||
assert third.status_code == 200
|
||
previews = client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]
|
||
assert previews["total"] == 2
|
||
assert {item["source_file_id"] for item in previews["items"]} == first_source_ids
|
||
assert store.results[task_id] == []
|
||
assert store.tasks[task_id]["progress"] == 20
|
||
|
||
|
||
def test_upload_preserves_store_error_when_storage_rollback_fails(
|
||
tmp_path: Path,
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "回滚异常", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
cleanup_attempts: list[str] = []
|
||
|
||
def fail_store(*_: Any, **__: Any) -> list[dict[str, Any]]:
|
||
raise ValueError("simulated database transaction failure")
|
||
|
||
def fail_cleanup(reference: str, **_: Any) -> bool:
|
||
cleanup_attempts.append(reference)
|
||
raise OSError("simulated storage cleanup failure")
|
||
|
||
monkeypatch.setattr(store, "add_source_files", fail_store)
|
||
monkeypatch.setattr(storage, "delete", fail_cleanup)
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("rollback.txt", b"rollback payload", "text/plain")},
|
||
)
|
||
|
||
assert response.status_code == 400
|
||
assert response.json()["detail"]["message"] == "simulated database transaction failure"
|
||
assert len(cleanup_attempts) == 1
|
||
assert store.sources[task_id] == []
|
||
|
||
|
||
def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "去重与质量筛选",
|
||
"process_type": "structured",
|
||
"config": {
|
||
"preprocess_options": ["clean_invalid", "deduplicate"],
|
||
"quality_filter_enabled": True,
|
||
"filter_low_quality": False,
|
||
"filter_short_content": True,
|
||
"min_output_length": 100,
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
content = (
|
||
'{"question":"问题","answer":"短答案"}\n'
|
||
'{"question":"问题","answer":"短答案"}\n'
|
||
).encode()
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("duplicates.jsonl", content, "application/jsonl")},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
assert preview.json()["data"]["total"] == 1
|
||
|
||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||
assert generated.status_code == 200
|
||
progress = client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]
|
||
assert progress["status"] == "completed"
|
||
assert progress["filtered_count"] == 1
|
||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||
|
||
|
||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||
store = FakeDataProcessStore()
|
||
task = store.create_task(
|
||
{"name": "并发代次", "process_type": "structured", "config": {}}
|
||
)
|
||
task_id = task["id"]
|
||
store.replace_preview_items(
|
||
task_id,
|
||
[
|
||
{
|
||
"source_file_id": None,
|
||
"original_content": "来源内容",
|
||
"edited_content": "来源内容",
|
||
"status": "manual",
|
||
}
|
||
],
|
||
)
|
||
first = store.start_generation(task_id, replace_existing=True)
|
||
first_run_id = first["generation_run_id"]
|
||
second_run_id = ""
|
||
|
||
def restart_while_old_worker_runs(*_: Any, **__: Any) -> list[dict[str, Any]]:
|
||
nonlocal second_run_id
|
||
store.stop_task(task_id)
|
||
second = store.start_generation(task_id, replace_existing=True)
|
||
second_run_id = second["generation_run_id"]
|
||
return []
|
||
|
||
monkeypatch.setattr(
|
||
data_process_endpoint,
|
||
"generate_standard_records",
|
||
restart_while_old_worker_runs,
|
||
)
|
||
data_process_endpoint._run_generation(store, task_id, first_run_id)
|
||
|
||
assert second_run_id and second_run_id != first_run_id
|
||
assert store.tasks[task_id]["status"] == "running"
|
||
assert store.tasks[task_id]["generation_run_id"] == second_run_id
|
||
assert store.results[task_id] == []
|
||
store.mark_failed(task_id, "old failure", generation_run_id=first_run_id)
|
||
assert store.tasks[task_id]["status"] == "running"
|
||
|
||
|
||
def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "状态保护", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
response = client.put(
|
||
f"/modelTF/data-process/{task_id}/results/not-created",
|
||
json={"instruction": "", "output": "", "status": "valid"},
|
||
)
|
||
assert response.status_code == 422
|
||
|
||
|
||
def test_start_rebuilds_preview_and_generates_in_one_request(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "一键处理", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"one.jsonl",
|
||
b'{"question":"What is one?","answer":"One."}\n',
|
||
"application/jsonl",
|
||
)
|
||
},
|
||
)
|
||
assert uploaded.status_code == 200
|
||
|
||
started = client.post(f"/modelTF/data-process/{task_id}/start")
|
||
assert started.status_code == 200
|
||
assert started.json()["data"]["task_id"] == task_id
|
||
assert started.json()["data"]["status"] == "running"
|
||
assert client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]["status"] == "completed"
|
||
assert client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]["total"] == 1
|
||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1
|
||
|
||
|
||
def test_unsupported_upload_format_returns_415(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "格式限制", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("payload.exe", b"not supported", "application/octet-stream")},
|
||
)
|
||
assert response.status_code == 415
|
||
|
||
legacy = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("document.doc", b"legacy", "application/msword")},
|
||
)
|
||
assert legacy.status_code == 415
|
||
assert "convert the file to .docx" in legacy.json()["detail"]["message"]
|
||
|
||
|
||
def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "XLSX 上传", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
workbook = Workbook()
|
||
worksheet = workbook.active
|
||
worksheet.append(["question", "answer"])
|
||
worksheet.append(["问题一", "答案一"])
|
||
worksheet.append(["问题二", "答案二"])
|
||
output = BytesIO()
|
||
workbook.save(output)
|
||
workbook.close()
|
||
original_bytes = output.getvalue()
|
||
|
||
response = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={
|
||
"files": (
|
||
"records.xlsx",
|
||
original_bytes,
|
||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||
)
|
||
},
|
||
)
|
||
assert response.status_code == 200
|
||
source = response.json()["data"]["files"][0]
|
||
assert source["file_format"] == "xlsx"
|
||
assert source["record_count"] == 2
|
||
assert source["size_bytes"] == len(original_bytes)
|
||
assert source["storage_object_id"].startswith("local://data-process/")
|
||
assert str(storage.root) not in response.text
|
||
assert storage.read(source["storage_object_id"]) == original_bytes
|
||
|
||
stored_source = store.get_source_file(task_id, source["id"])
|
||
assert stored_source["id"] == source["id"]
|
||
assert stored_source["storage_object_id"] == source["storage_object_id"]
|
||
assert stored_source["metadata"]["storage_backend"] == "local"
|
||
assert stored_source["metadata"]["original_size_bytes"] == len(original_bytes)
|
||
assert '"question":"问题一"' in stored_source["content"]
|
||
|
||
content = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content"
|
||
)
|
||
assert content.status_code == 200
|
||
assert '"answer":"答案二"' in content.json()["data"]["content"]
|
||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
assert preview.status_code == 200
|
||
assert preview.json()["data"]["total"] == 2
|
||
|
||
|
||
def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None:
|
||
client, store, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "PDF 原件预览", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
original_pdf = _minimal_pdf()
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("说明 文档.pdf", original_pdf, "application/pdf")},
|
||
).json()["data"]["files"][0]
|
||
raw_url = f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
|
||
|
||
full = client.get(raw_url)
|
||
assert full.status_code == 200
|
||
assert full.content == original_pdf
|
||
assert full.headers["content-type"] == "application/pdf"
|
||
assert full.headers["accept-ranges"] == "bytes"
|
||
assert full.headers["cache-control"] == "private, no-store"
|
||
assert full.headers["content-length"] == str(len(original_pdf))
|
||
assert full.headers["content-disposition"].startswith("inline;")
|
||
assert "%E8%AF%B4%E6%98%8E%20%E6%96%87%E6%A1%A3.pdf" in full.headers[
|
||
"content-disposition"
|
||
]
|
||
assert full.headers["etag"] == f'"{uploaded["checksum_sha256"]}"'
|
||
|
||
partial = client.get(raw_url, headers={"Range": "bytes=5-14"})
|
||
assert partial.status_code == 206
|
||
assert partial.content == original_pdf[5:15]
|
||
assert partial.headers["content-range"] == f"bytes 5-14/{len(original_pdf)}"
|
||
assert partial.headers["content-length"] == "10"
|
||
|
||
suffix = client.get(raw_url, headers={"Range": "bytes=-8"})
|
||
assert suffix.status_code == 206
|
||
assert suffix.content == original_pdf[-8:]
|
||
|
||
invalid = client.get(raw_url, headers={"Range": "bytes=0-1,4-5"})
|
||
assert invalid.status_code == 416
|
||
assert invalid.headers["content-range"] == f"bytes */{len(original_pdf)}"
|
||
|
||
pages_url = (
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages"
|
||
)
|
||
pages = client.get(pages_url)
|
||
assert pages.status_code == 200
|
||
assert pages.json()["data"] == {
|
||
"page_count": 1,
|
||
"pages": [
|
||
{
|
||
"page_number": 1,
|
||
"source_start": 0,
|
||
"source_end": len("Hello PDF"),
|
||
}
|
||
],
|
||
}
|
||
|
||
legacy_id = "dpsf_legacy_pdf"
|
||
store.add_source_file(
|
||
task_id,
|
||
id=legacy_id,
|
||
storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1",
|
||
name="legacy.pdf",
|
||
content="legacy extracted PDF text",
|
||
raw_size=len(original_pdf),
|
||
checksum_sha256="a" * 64,
|
||
file_format="pdf",
|
||
record_count=1,
|
||
metadata={"legacy": True},
|
||
)
|
||
legacy = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/raw"
|
||
)
|
||
assert legacy.status_code == 410
|
||
legacy_pages = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/pdf-pages"
|
||
)
|
||
assert legacy_pages.status_code == 410
|
||
|
||
|
||
def test_pdf_preview_build_cleans_stored_document_noise_without_offset_drift(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={
|
||
"name": "PDF 文档噪声清理",
|
||
"process_type": "unstructured",
|
||
"config": {
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 20,
|
||
"preprocess_options": ["clean_invalid_content"],
|
||
},
|
||
},
|
||
).json()["data"]["id"]
|
||
raw = _minimal_pdf_pages(
|
||
"ACME Internal Manual\nBody page one keeps this guidance and explanation.",
|
||
"ACME Internal Manual\nContents\n"
|
||
"Chapter One........3\nChapter Two........4\nAppendix........5",
|
||
"ACME Internal Manual\n1.1 Policy........6\n1.2 Approval........7\n1.3 Archive........8",
|
||
"ACME Internal Manual\nBody page four keeps operational details and examples.",
|
||
"ACME Internal Manual\nBody page five keeps the final effective-date clause.",
|
||
)
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("manual.pdf", raw, "application/pdf")},
|
||
).json()["data"]["files"][0]
|
||
source_content = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/content"
|
||
).json()["data"]["content"]
|
||
|
||
built = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||
|
||
assert built.status_code == 200
|
||
items = built.json()["data"]["items"]
|
||
assert items
|
||
edited = "\n".join(item["edited_content"] for item in items)
|
||
assert "ACME Internal Manual" not in edited
|
||
assert "Contents" not in edited
|
||
assert "Chapter One" not in edited
|
||
assert "1.2 Approval" not in edited
|
||
assert "Body page one" in edited
|
||
assert "Body page five" in edited
|
||
assert all(
|
||
item["original_content"]
|
||
== source_content[item["source_start"] : item["source_end"]]
|
||
for item in items
|
||
)
|
||
|
||
|
||
def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "非 PDF 原件", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("notes.txt", b"plain source text", "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
|
||
response = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
|
||
)
|
||
assert response.status_code == 415
|
||
pages_response = client.get(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages"
|
||
)
|
||
assert pages_response.status_code == 415
|
||
|
||
|
||
def test_delete_source_removes_owned_local_object_and_accepts_legacy_db_reference(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "删除原件", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("原件.txt", "本地原始内容".encode(), "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
reference = uploaded["storage_object_id"]
|
||
assert storage.read(reference) == "本地原始内容".encode()
|
||
|
||
deleted = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}"
|
||
)
|
||
assert deleted.status_code == 200
|
||
assert deleted.json()["data"]["storage_cleanup_pending"] is False
|
||
with pytest.raises(DataProcessStorageError, match="does not exist"):
|
||
storage.read(reference)
|
||
|
||
legacy_id = "dpsf_legacy"
|
||
store.add_source_file(
|
||
task_id,
|
||
id=legacy_id,
|
||
storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1",
|
||
name="legacy.txt",
|
||
content="旧记录正文",
|
||
raw_size=len("旧记录正文".encode()),
|
||
checksum_sha256="a" * 64,
|
||
file_format="txt",
|
||
record_count=1,
|
||
metadata={"legacy": True},
|
||
)
|
||
legacy_deleted = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}"
|
||
)
|
||
assert legacy_deleted.status_code == 200
|
||
assert legacy_deleted.json()["data"]["storage_cleanup_pending"] is False
|
||
|
||
|
||
def test_delete_reports_pending_cleanup_after_database_soft_delete(
|
||
tmp_path: Path,
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "待清理原件", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("pending.txt", b"pending cleanup", "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
|
||
def fail_cleanup(*_: Any, **__: Any) -> bool:
|
||
raise OSError("simulated storage failure")
|
||
|
||
monkeypatch.setattr(storage, "delete", fail_cleanup)
|
||
response = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}"
|
||
)
|
||
|
||
assert response.status_code == 200
|
||
assert response.json()["data"]["storage_cleanup_pending"] is True
|
||
with pytest.raises(NotFoundError):
|
||
store.get_source_file(task_id, uploaded["id"])
|
||
assert storage.read(uploaded["storage_object_id"]) == b"pending cleanup"
|
||
|
||
|
||
def test_delete_rejects_polluted_reference_owned_by_another_source(tmp_path: Path) -> None:
|
||
client, store, storage = make_client(tmp_path)
|
||
task_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "归属校验", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
uploaded = client.post(
|
||
f"/modelTF/data-process/{task_id}/source-files",
|
||
files={"files": ("safe.txt", b"owned content", "text/plain")},
|
||
).json()["data"]["files"][0]
|
||
target_reference = uploaded["storage_object_id"]
|
||
|
||
polluted_id = "dpsf_polluted"
|
||
store.add_source_file(
|
||
task_id,
|
||
id=polluted_id,
|
||
storage_object_id=target_reference,
|
||
name="polluted.txt",
|
||
content="polluted",
|
||
raw_size=8,
|
||
checksum_sha256="b" * 64,
|
||
file_format="txt",
|
||
record_count=1,
|
||
metadata={},
|
||
)
|
||
rejected = client.delete(
|
||
f"/modelTF/data-process/{task_id}/source-files/{polluted_id}"
|
||
)
|
||
assert rejected.status_code == 400
|
||
assert storage.read(target_reference) == b"owned content"
|
||
assert store.get_source_file(task_id, polluted_id)["id"] == polluted_id
|
||
|
||
|
||
def test_upload_format_must_match_process_type(tmp_path: Path) -> None:
|
||
client, _, _ = make_client(tmp_path)
|
||
structured_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "结构化格式约束", "process_type": "structured", "config": {}},
|
||
).json()["data"]["id"]
|
||
structured_pdf = client.post(
|
||
f"/modelTF/data-process/{structured_id}/source-files",
|
||
files={"files": ("manual.pdf", b"not parsed", "application/pdf")},
|
||
)
|
||
assert structured_pdf.status_code == 415
|
||
|
||
unstructured_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "非结构化格式约束", "process_type": "unstructured", "config": {}},
|
||
).json()["data"]["id"]
|
||
unstructured_xlsx = client.post(
|
||
f"/modelTF/data-process/{unstructured_id}/source-files",
|
||
files={"files": ("records.xlsx", b"not parsed", "application/octet-stream")},
|
||
)
|
||
assert unstructured_xlsx.status_code == 415
|
||
|
||
external_id = client.post(
|
||
"/modelTF/data-process",
|
||
json={"name": "外部数据格式约束", "process_type": "external", "config": {}},
|
||
).json()["data"]["id"]
|
||
external_upload = client.post(
|
||
f"/modelTF/data-process/{external_id}/source-files",
|
||
files={"files": ("records.jsonl", b'{"id":1}', "application/jsonl")},
|
||
)
|
||
assert external_upload.status_code == 409
|
||
|
||
|
||
def _preview_task(
|
||
content: str,
|
||
*,
|
||
options: list[str],
|
||
config: dict[str, Any] | None = None,
|
||
source_id: str = "source-1",
|
||
file_format: str = "txt",
|
||
) -> list[dict[str, Any]]:
|
||
task_config = {
|
||
"preprocess_options": options,
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 20,
|
||
"min_chunk_size": 20,
|
||
**(config or {}),
|
||
}
|
||
return data_process_endpoint._build_preview_items(
|
||
{"process_type": "unstructured", "config": task_config},
|
||
[
|
||
{
|
||
"id": source_id,
|
||
"name": f"{source_id}.{file_format}",
|
||
"file_format": file_format,
|
||
"content": content,
|
||
}
|
||
],
|
||
)
|
||
|
||
|
||
def test_fixed_preview_preserves_source_offsets() -> None:
|
||
content = (
|
||
"# 第一章\n"
|
||
+ " ".join(f"alpha{index}" for index in range(18))
|
||
+ "\n# 第二章\n"
|
||
+ " ".join(f"beta{index}" for index in range(18))
|
||
)
|
||
normalized = normalize_text(content)
|
||
second_chapter_start = normalized.index("# 第二章")
|
||
common_config = {"chunk_size": 10, "chunk_overlap": 3, "min_chunk_size": 4}
|
||
|
||
default_items = _preview_task(
|
||
content,
|
||
options=["preserve_context"],
|
||
config=common_config,
|
||
)
|
||
fixed_items = _preview_task(
|
||
content,
|
||
options=["preserve_context"],
|
||
config={**common_config, "chunk_method": "fixed"},
|
||
)
|
||
|
||
def snapshot(items: list[dict[str, Any]]) -> list[tuple[Any, ...]]:
|
||
return [
|
||
(
|
||
item["original_content"],
|
||
item["source_start"],
|
||
item["source_end"],
|
||
item["source_start_line"],
|
||
item["source_end_line"],
|
||
)
|
||
for item in items
|
||
]
|
||
|
||
assert snapshot(default_items) == snapshot(fixed_items)
|
||
assert all(
|
||
item["original_content"]
|
||
== normalized[item["source_start"] : item["source_end"]]
|
||
for item in fixed_items
|
||
)
|
||
second_chapter_items = [
|
||
item for item in fixed_items if item["source_start"] >= second_chapter_start
|
||
]
|
||
assert second_chapter_items
|
||
|
||
|
||
def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None:
|
||
repeated = "@" * 120
|
||
assert len(_preview_task(repeated, options=[])) == 1
|
||
assert _preview_task(repeated, options=["clean_invalid_content"]) == []
|
||
|
||
mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。"
|
||
assert len(_preview_task(mojibake, options=[])) == 1
|
||
assert _preview_task(mojibake, options=["filter_low_quality"]) == []
|
||
|
||
first = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"
|
||
second = "alpha beta gamma, delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"
|
||
sources = [
|
||
{"id": "near-1", "name": "one.txt", "file_format": "txt", "content": first},
|
||
{"id": "near-2", "name": "two.txt", "file_format": "txt", "content": second},
|
||
]
|
||
base_task = {
|
||
"process_type": "unstructured",
|
||
"config": {
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 1,
|
||
"preprocess_options": [],
|
||
},
|
||
}
|
||
assert len(data_process_endpoint._build_preview_items(base_task, sources)) == 2
|
||
deduplicated_task = deepcopy(base_task)
|
||
deduplicated_task["config"]["preprocess_options"] = ["deduplicate_content"]
|
||
assert len(data_process_endpoint._build_preview_items(deduplicated_task, sources)) == 1
|
||
|
||
context_text = " ".join(f"token{index}" for index in range(45))
|
||
no_context = _preview_task(
|
||
context_text,
|
||
options=[],
|
||
config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5},
|
||
)
|
||
with_context = _preview_task(
|
||
context_text,
|
||
options=["preserve_context"],
|
||
config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5},
|
||
)
|
||
assert no_context[1]["source_start"] >= no_context[0]["source_end"]
|
||
assert with_context[1]["source_start"] < with_context[0]["source_end"]
|
||
|
||
sensitive = "联系人：张三，手机 13800138000，邮箱 user@example.com。"
|
||
plain = _preview_task(sensitive, options=[])[0]
|
||
masked = _preview_task(sensitive, options=["desensitize"])[0]
|
||
assert "张三" in plain["edited_content"]
|
||
assert "联系人:[NAME]" in masked["edited_content"]
|
||
assert "[PHONE]" in masked["edited_content"]
|
||
assert "[EMAIL]" in masked["edited_content"]
|
||
|
||
|
||
def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled() -> None:
|
||
source_text = normalize_text("重复页眉\n这是应保留的 PDF 正文内容，用于生成训练数据。")
|
||
source = {
|
||
"id": "pdf-source",
|
||
"name": "manual.pdf",
|
||
"file_format": "pdf",
|
||
"content": source_text,
|
||
"document_noise_spans": (
|
||
DocumentNoiseSpan(0, len("重复页眉"), "repeated_margin"),
|
||
),
|
||
}
|
||
config = {
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 200,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 1,
|
||
}
|
||
|
||
cleaned_items = data_process_endpoint._build_preview_items(
|
||
{
|
||
"process_type": "unstructured",
|
||
"config": {**config, "preprocess_options": ["clean_invalid_content"]},
|
||
},
|
||
[source],
|
||
)
|
||
original_items = data_process_endpoint._build_preview_items(
|
||
{
|
||
"process_type": "unstructured",
|
||
"config": {**config, "preprocess_options": []},
|
||
},
|
||
[source],
|
||
)
|
||
|
||
assert len(cleaned_items) == 1
|
||
cleaned = cleaned_items[0]
|
||
assert cleaned["original_content"] == source_text[
|
||
cleaned["source_start"] : cleaned["source_end"]
|
||
]
|
||
assert "重复页眉" not in cleaned["edited_content"]
|
||
assert "PDF 正文内容" in cleaned["edited_content"]
|
||
assert cleaned["status"] == "modified"
|
||
assert "document_noise_removed" in cleaned["quality_score"]["preprocess_flags"]
|
||
assert "重复页眉" in original_items[0]["edited_content"]
|
||
|
||
|
||
def test_merge_short_content_applies_across_adjacent_fixed_chunks() -> None:
|
||
content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
|
||
items = _preview_task(
|
||
content,
|
||
options=["merge_short_content"],
|
||
config={
|
||
"chunk_method": "fixed",
|
||
"chunk_size": 40,
|
||
"chunk_overlap": 0,
|
||
"min_chunk_size": 20,
|
||
},
|
||
)
|
||
|
||
assert len(items) == 3
|
||
assert all(item["token_count"] <= 40 for item in items)
|
||
assert items[0]["source_start_line"] == 1
|
||
assert items[-1]["source_end_line"] == 16
|
||
|
||
|
||
def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None:
|
||
for file_format in ("pdf", "docx", "pptx"):
|
||
items = _preview_task(
|
||
f"{file_format.upper()} 已抽取正文，可直接进入切片处理。",
|
||
options=[],
|
||
file_format=file_format,
|
||
)
|
||
assert len(items) == 1
|
||
assert "已抽取正文" in items[0]["edited_content"]
|