YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 完善文件解析与切分存储链路

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-24 11:27:51 +08:00
parent 6d4bf85284
commit 33d0ed2e01
12 changed files with 4813 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

808
backend/tests/test_data_process_api.py
View File

@@ -1,13 +1,23 @@
from __future__ import annotations
from copy import deepcopy
from io import BytesIO
from pathlib import Path
from typing import Any
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from openpyxl import Workbook
from app.api.v1.endpoints import data_process as data_process_endpoint
from app.api.v1.endpoints.data_process import router
from app.modules.data_process.algorithms import normalize_text
from app.modules.data_process.storage import (
DataProcessStorageError,
LocalDataProcessStorage,
get_data_process_storage,
)
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
@@ -87,11 +97,26 @@ class FakeDataProcessStore:
def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]:
self.get_task(task_id)
values = deepcopy(payload)
source_id = str(values.pop("id", None) or self._id("dpsf"))
storage_object_id = str(
values.pop("storage_object_id", None)
or f"db://data-process/{task_id}/{source_id}/v1"
)
raw_size = int(values.pop("raw_size"))
metadata = deepcopy(values.pop("metadata", {}))
metadata.setdefault(
"storage_backend",
"local" if storage_object_id.startswith("local://data-process/") else "database",
)
source = {
"id": self._id("dpsf"),
"id": source_id,
"task_id": task_id,
"version_no": 1,
**deepcopy(payload),
"storage_object_id": storage_object_id,
"size_bytes": raw_size,
"metadata": metadata,
**values,
}
self.sources[task_id].append(source)
self.tasks[task_id]["input_count"] += payload["record_count"]
@@ -163,14 +188,27 @@ class FakeDataProcessStore:
self.results[task_id] = []
def replace_preview_items(
self, task_id: str, items: list[dict[str, Any]]
self,
task_id: str,
items: list[dict[str, Any]],
*,
source_file_ids: list[str] | None = None,
) -> list[dict[str, Any]]:
self.previews[task_id] = [
created = [
{"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items
]
if source_file_ids is None:
self.previews[task_id] = created
else:
selected = set(source_file_ids)
self.previews[task_id] = [
item
for item in self.previews[task_id]
if item["source_file_id"] not in selected
] + created
self.results[task_id] = []
self.tasks[task_id]["progress"] = 20
return deepcopy(self.previews[task_id])
return deepcopy(created)
def list_preview_items(
self,
@@ -389,16 +427,59 @@ class FakeDataProcessStore:
return {"dataset": deepcopy(dataset), "created": True}
def make_client() -> tuple[TestClient, FakeDataProcessStore]:
def make_client(
tmp_path: Path,
) -> tuple[TestClient, FakeDataProcessStore, LocalDataProcessStorage]:
store = FakeDataProcessStore()
storage = LocalDataProcessStorage(tmp_path / "data-process")
app = FastAPI()
app.include_router(router, prefix="/modelTF")
app.dependency_overrides[get_data_process_store] = lambda: store
return TestClient(app), store
app.dependency_overrides[get_data_process_storage] = lambda: storage
return TestClient(app), store, storage
def test_data_process_full_contract_without_database() -> None:
client, store = make_client()
def _stored_files(storage: LocalDataProcessStorage) -> list[Path]:
return [path for path in storage.root.rglob("*") if path.is_file() or path.is_symlink()]
def _minimal_pdf(text: str = "Hello PDF") -> bytes:
stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("ascii")
objects = [
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
(
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>"
),
b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n"
+ stream
+ b"\nendstream",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
]
result = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
offsets = [0]
for object_number, value in enumerate(objects, start=1):
offsets.append(len(result))
result.extend(f"{object_number} 0 obj\n".encode())
result.extend(value)
result.extend(b"\nendobj\n")
xref_offset = len(result)
result.extend(f"xref\n0 {len(objects) + 1}\n".encode())
result.extend(b"0000000000 65535 f \n")
for offset in offsets[1:]:
result.extend(f"{offset:010d} 00000 n \n".encode())
result.extend(
(
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
f"startxref\n{xref_offset}\n%%EOF\n"
).encode()
)
return bytes(result)
def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
created = client.post(
"/modelTF/data-process",
json={
@@ -506,8 +587,143 @@ def test_data_process_full_contract_without_database() -> None:
)
def test_external_source_never_returns_fake_success() -> None:
client, _ = make_client()
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "逐文件预览", "process_type": "structured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("first.jsonl", b'{"id":1}\n', "application/jsonl")),
(
"files",
("second.jsonl", b'{"id":2}\n{"id":3}\n', "application/jsonl"),
),
],
)
assert uploaded.status_code == 200
first_source, second_source = uploaded.json()["data"]["files"]
first_build = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"source_file_ids": [first_source["id"]]},
)
assert first_build.status_code == 200
first_data = first_build.json()["data"]
assert first_data["file_counts"] == {first_source["id"]: 1}
assert first_data["files"] == [
{
"source_file_id": first_source["id"],
"preview_count": 1,
"status": "completed",
}
]
first_item = first_data["items"][0]
edited = client.put(
f"/modelTF/data-process/{task_id}/preview/{first_item['id']}",
json={"edited_content": "人工确认后的第一文件预览"},
)
assert edited.status_code == 200
second_build = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"source_file_id": second_source["id"]},
)
assert second_build.status_code == 200
second_data = second_build.json()["data"]
assert second_data["file_counts"] == {second_source["id"]: 2}
assert second_data["files"] == [
{
"source_file_id": second_source["id"],
"preview_count": 2,
"status": "completed",
}
]
assert {item["source_file_id"] for item in store.previews[task_id]} == {
first_source["id"],
second_source["id"],
}
preserved_first = next(
item
for item in store.previews[task_id]
if item["source_file_id"] == first_source["id"]
)
assert preserved_first["id"] == first_item["id"]
assert preserved_first["edited_content"] == "人工确认后的第一文件预览"
previous_second_ids = {
item["id"]
for item in store.previews[task_id]
if item["source_file_id"] == second_source["id"]
}
next(
source
for source in store.sources[task_id]
if source["id"] == second_source["id"]
)["content"] = '{"id":4}\n'
rebuilt = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"source_file_ids": [second_source["id"]]},
)
assert rebuilt.status_code == 200
assert rebuilt.json()["data"]["file_counts"] == {second_source["id"]: 1}
current_second_ids = {
item["id"]
for item in store.previews[task_id]
if item["source_file_id"] == second_source["id"]
}
assert current_second_ids.isdisjoint(previous_second_ids)
assert len(current_second_ids) == 1
assert next(
item
for item in store.previews[task_id]
if item["source_file_id"] == first_source["id"]
)["id"] == first_item["id"]
def test_preview_build_rejects_unknown_and_cross_task_source_file_ids(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
first_task_id = client.post(
"/modelTF/data-process",
json={"name": "归属任务一", "process_type": "structured", "config": {}},
).json()["data"]["id"]
second_task_id = client.post(
"/modelTF/data-process",
json={"name": "归属任务二", "process_type": "structured", "config": {}},
).json()["data"]["id"]
foreign_source = client.post(
f"/modelTF/data-process/{second_task_id}/source-files",
files={"files": ("foreign.jsonl", b'{"id":2}\n', "application/jsonl")},
).json()["data"]["files"][0]
unknown = client.post(
f"/modelTF/data-process/{first_task_id}/preview/build",
json={"source_file_ids": ["dpsf_not_found"]},
)
assert unknown.status_code == 404
foreign = client.post(
f"/modelTF/data-process/{first_task_id}/preview/build",
json={"source_file_id": foreign_source["id"]},
)
assert foreign.status_code == 404
ambiguous = client.post(
f"/modelTF/data-process/{first_task_id}/preview/build",
json={
"source_file_id": foreign_source["id"],
"source_file_ids": [foreign_source["id"]],
},
)
assert ambiguous.status_code == 422
def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "外部数据", "process_type": "external", "config": {}},
@@ -520,8 +736,8 @@ def test_external_source_never_returns_fake_success() -> None:
assert response.json()["detail"]["code"] == 501
def test_config_validation_and_stop_state() -> None:
client, store = make_client()
def test_config_validation_and_stop_state(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
invalid = client.post(
"/modelTF/data-process",
json={
@@ -537,6 +753,28 @@ def test_config_validation_and_stop_state() -> None:
)
assert invalid.status_code == 422
legacy_semantic = client.post(
"/modelTF/data-process",
json={
"name": "旧切分策略",
"process_type": "unstructured",
"config": {"chunk_method": "semantic"},
},
)
assert legacy_semantic.status_code == 422
assert "chunk_method" in legacy_semantic.text
missing_custom_delimiter = client.post(
"/modelTF/data-process",
json={
"name": "缺少自定义分隔符",
"process_type": "unstructured",
"config": {"chunk_method": "custom"},
},
)
assert missing_custom_delimiter.status_code == 422
assert "custom_delimiter" in missing_custom_delimiter.text
task_id = client.post(
"/modelTF/data-process",
json={"name": "可停止任务", "process_type": "structured", "config": {}},
@@ -547,11 +785,11 @@ def test_config_validation_and_stop_state() -> None:
assert stopped.json()["data"]["status"] == "stopped"
def test_upload_batch_is_atomic_and_empty_files_are_rejected() -> None:
client, store = make_client()
def test_upload_batch_is_atomic_and_empty_files_are_rejected(tmp_path: Path) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "批量上传", "process_type": "structured", "config": {}},
json={"name": "批量上传", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
duplicate_batch = client.post(
@@ -563,6 +801,18 @@ def test_upload_batch_is_atomic_and_empty_files_are_rejected() -> None:
)
assert duplicate_batch.status_code == 400
assert store.sources[task_id] == []
assert _stored_files(storage) == []
parse_failure = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("valid.txt", "先暂存的内容".encode(), "text/plain")),
("files", ("broken.txt", b"\xff", "text/plain")),
],
)
assert parse_failure.status_code == 400
assert store.sources[task_id] == []
assert _stored_files(storage) == []
empty = client.post(
f"/modelTF/data-process/{task_id}/source-files",
@@ -570,10 +820,45 @@ def test_upload_batch_is_atomic_and_empty_files_are_rejected() -> None:
)
assert empty.status_code == 400
assert store.sources[task_id] == []
assert _stored_files(storage) == []
def test_preprocess_deduplicates_and_quality_filter_removes_short_results() -> None:
client, _ = make_client()
def test_upload_preserves_store_error_when_storage_rollback_fails(
tmp_path: Path,
monkeypatch: Any,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "回滚异常", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
cleanup_attempts: list[str] = []
def fail_store(*_: Any, **__: Any) -> list[dict[str, Any]]:
raise ValueError("simulated database transaction failure")
def fail_cleanup(reference: str, **_: Any) -> bool:
cleanup_attempts.append(reference)
raise OSError("simulated storage cleanup failure")
monkeypatch.setattr(store, "add_source_files", fail_store)
monkeypatch.setattr(storage, "delete", fail_cleanup)
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("rollback.txt", b"rollback payload", "text/plain")},
)
assert response.status_code == 400
assert response.json()["detail"]["message"] == "simulated database transaction failure"
assert len(cleanup_attempts) == 1
assert store.sources[task_id] == []
def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
@@ -651,8 +936,8 @@ def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> N
assert store.tasks[task_id]["status"] == "running"
def test_result_status_cannot_be_forged_by_client() -> None:
client, _ = make_client()
def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "状态保护", "process_type": "structured", "config": {}},
@@ -664,8 +949,8 @@ def test_result_status_cannot_be_forged_by_client() -> None:
assert response.status_code == 422
def test_start_rebuilds_preview_and_generates_in_one_request() -> None:
client, _ = make_client()
def test_start_rebuilds_preview_and_generates_in_one_request(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "一键处理", "process_type": "structured", "config": {}},
@@ -691,14 +976,487 @@ def test_start_rebuilds_preview_and_generates_in_one_request() -> None:
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1
def test_unsupported_upload_format_returns_415() -> None:
client, _ = make_client()
def test_unsupported_upload_format_returns_415(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "格式限制", "process_type": "structured", "config": {}},
).json()["data"]["id"]
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("document.pdf", b"not a pdf", "application/pdf")},
files={"files": ("payload.exe", b"not supported", "application/octet-stream")},
)
assert response.status_code == 415
legacy = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("document.doc", b"legacy", "application/msword")},
)
assert legacy.status_code == 415
assert "convert the file to .docx" in legacy.json()["detail"]["message"]
def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "XLSX 上传", "process_type": "structured", "config": {}},
).json()["data"]["id"]
workbook = Workbook()
worksheet = workbook.active
worksheet.append(["question", "answer"])
worksheet.append(["问题一", "答案一"])
worksheet.append(["问题二", "答案二"])
output = BytesIO()
workbook.save(output)
workbook.close()
original_bytes = output.getvalue()
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={
"files": (
"records.xlsx",
original_bytes,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)
},
)
assert response.status_code == 200
source = response.json()["data"]["files"][0]
assert source["file_format"] == "xlsx"
assert source["record_count"] == 2
assert source["size_bytes"] == len(original_bytes)
assert source["storage_object_id"].startswith("local://data-process/")
assert str(storage.root) not in response.text
assert storage.read(source["storage_object_id"]) == original_bytes
stored_source = store.get_source_file(task_id, source["id"])
assert stored_source["id"] == source["id"]
assert stored_source["storage_object_id"] == source["storage_object_id"]
assert stored_source["metadata"]["storage_backend"] == "local"
assert stored_source["metadata"]["original_size_bytes"] == len(original_bytes)
assert '"question":"问题一"' in stored_source["content"]
content = client.get(
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content"
)
assert content.status_code == 200
assert '"answer":"答案二"' in content.json()["data"]["content"]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["total"] == 2
def test_pdf_raw_preview_streams_original_file_and_supports_ranges(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "PDF 原件预览", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
original_pdf = _minimal_pdf()
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("说明 文档.pdf", original_pdf, "application/pdf")},
).json()["data"]["files"][0]
raw_url = f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
full = client.get(raw_url)
assert full.status_code == 200
assert full.content == original_pdf
assert full.headers["content-type"] == "application/pdf"
assert full.headers["accept-ranges"] == "bytes"
assert full.headers["cache-control"] == "private, no-store"
assert full.headers["content-length"] == str(len(original_pdf))
assert full.headers["content-disposition"].startswith("inline;")
assert "%E8%AF%B4%E6%98%8E%20%E6%96%87%E6%A1%A3.pdf" in full.headers[
"content-disposition"
]
assert full.headers["etag"] == f'"{uploaded["checksum_sha256"]}"'
partial = client.get(raw_url, headers={"Range": "bytes=5-14"})
assert partial.status_code == 206
assert partial.content == original_pdf[5:15]
assert partial.headers["content-range"] == f"bytes 5-14/{len(original_pdf)}"
assert partial.headers["content-length"] == "10"
suffix = client.get(raw_url, headers={"Range": "bytes=-8"})
assert suffix.status_code == 206
assert suffix.content == original_pdf[-8:]
invalid = client.get(raw_url, headers={"Range": "bytes=0-1,4-5"})
assert invalid.status_code == 416
assert invalid.headers["content-range"] == f"bytes */{len(original_pdf)}"
pages_url = (
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages"
)
pages = client.get(pages_url)
assert pages.status_code == 200
assert pages.json()["data"] == {
"page_count": 1,
"pages": [
{
"page_number": 1,
"source_start": 0,
"source_end": len("Hello PDF"),
}
],
}
legacy_id = "dpsf_legacy_pdf"
store.add_source_file(
task_id,
id=legacy_id,
storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1",
name="legacy.pdf",
content="legacy extracted PDF text",
raw_size=len(original_pdf),
checksum_sha256="a" * 64,
file_format="pdf",
record_count=1,
metadata={"legacy": True},
)
legacy = client.get(
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/raw"
)
assert legacy.status_code == 410
legacy_pages = client.get(
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}/pdf-pages"
)
assert legacy_pages.status_code == 410
def test_raw_inline_preview_rejects_non_pdf_source(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "非 PDF 原件", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("notes.txt", b"plain source text", "text/plain")},
).json()["data"]["files"][0]
response = client.get(
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/raw"
)
assert response.status_code == 415
pages_response = client.get(
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}/pdf-pages"
)
assert pages_response.status_code == 415
def test_delete_source_removes_owned_local_object_and_accepts_legacy_db_reference(
tmp_path: Path,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "删除原件", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("原件.txt", "本地原始内容".encode(), "text/plain")},
).json()["data"]["files"][0]
reference = uploaded["storage_object_id"]
assert storage.read(reference) == "本地原始内容".encode()
deleted = client.delete(
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}"
)
assert deleted.status_code == 200
assert deleted.json()["data"]["storage_cleanup_pending"] is False
with pytest.raises(DataProcessStorageError, match="does not exist"):
storage.read(reference)
legacy_id = "dpsf_legacy"
store.add_source_file(
task_id,
id=legacy_id,
storage_object_id=f"db://data-process/{task_id}/{legacy_id}/v1",
name="legacy.txt",
content="旧记录正文",
raw_size=len("旧记录正文".encode()),
checksum_sha256="a" * 64,
file_format="txt",
record_count=1,
metadata={"legacy": True},
)
legacy_deleted = client.delete(
f"/modelTF/data-process/{task_id}/source-files/{legacy_id}"
)
assert legacy_deleted.status_code == 200
assert legacy_deleted.json()["data"]["storage_cleanup_pending"] is False
def test_delete_reports_pending_cleanup_after_database_soft_delete(
tmp_path: Path,
monkeypatch: Any,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "待清理原件", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("pending.txt", b"pending cleanup", "text/plain")},
).json()["data"]["files"][0]
def fail_cleanup(*_: Any, **__: Any) -> bool:
raise OSError("simulated storage failure")
monkeypatch.setattr(storage, "delete", fail_cleanup)
response = client.delete(
f"/modelTF/data-process/{task_id}/source-files/{uploaded['id']}"
)
assert response.status_code == 200
assert response.json()["data"]["storage_cleanup_pending"] is True
with pytest.raises(NotFoundError):
store.get_source_file(task_id, uploaded["id"])
assert storage.read(uploaded["storage_object_id"]) == b"pending cleanup"
def test_delete_rejects_polluted_reference_owned_by_another_source(tmp_path: Path) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "归属校验", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("safe.txt", b"owned content", "text/plain")},
).json()["data"]["files"][0]
target_reference = uploaded["storage_object_id"]
polluted_id = "dpsf_polluted"
store.add_source_file(
task_id,
id=polluted_id,
storage_object_id=target_reference,
name="polluted.txt",
content="polluted",
raw_size=8,
checksum_sha256="b" * 64,
file_format="txt",
record_count=1,
metadata={},
)
rejected = client.delete(
f"/modelTF/data-process/{task_id}/source-files/{polluted_id}"
)
assert rejected.status_code == 400
assert storage.read(target_reference) == b"owned content"
assert store.get_source_file(task_id, polluted_id)["id"] == polluted_id
def test_upload_format_must_match_process_type(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
structured_id = client.post(
"/modelTF/data-process",
json={"name": "结构化格式约束", "process_type": "structured", "config": {}},
).json()["data"]["id"]
structured_pdf = client.post(
f"/modelTF/data-process/{structured_id}/source-files",
files={"files": ("manual.pdf", b"not parsed", "application/pdf")},
)
assert structured_pdf.status_code == 415
unstructured_id = client.post(
"/modelTF/data-process",
json={"name": "非结构化格式约束", "process_type": "unstructured", "config": {}},
).json()["data"]["id"]
unstructured_xlsx = client.post(
f"/modelTF/data-process/{unstructured_id}/source-files",
files={"files": ("records.xlsx", b"not parsed", "application/octet-stream")},
)
assert unstructured_xlsx.status_code == 415
external_id = client.post(
"/modelTF/data-process",
json={"name": "外部数据格式约束", "process_type": "external", "config": {}},
).json()["data"]["id"]
external_upload = client.post(
f"/modelTF/data-process/{external_id}/source-files",
files={"files": ("records.jsonl", b'{"id":1}', "application/jsonl")},
)
assert external_upload.status_code == 409
def _preview_task(
content: str,
*,
options: list[str],
config: dict[str, Any] | None = None,
source_id: str = "source-1",
file_format: str = "txt",
) -> list[dict[str, Any]]:
task_config = {
"preprocess_options": options,
"chunk_size": 200,
"chunk_overlap": 20,
"min_chunk_size": 20,
**(config or {}),
}
return data_process_endpoint._build_preview_items(
{"process_type": "unstructured", "config": task_config},
[
{
"id": source_id,
"name": f"{source_id}.{file_format}",
"file_format": file_format,
"content": content,
}
],
)
def test_default_and_structure_preview_split_headings_without_cross_section_overlap() -> None:
content = (
"# 第一章\n"
+ " ".join(f"alpha{index}" for index in range(18))
+ "\n# 第二章\n"
+ " ".join(f"beta{index}" for index in range(18))
)
normalized = normalize_text(content)
second_chapter_start = normalized.index("# 第二章")
common_config = {"chunk_size": 10, "chunk_overlap": 3, "min_chunk_size": 4}
default_items = _preview_task(
content,
options=["preserve_context"],
config=common_config,
)
structure_items = _preview_task(
content,
options=["preserve_context"],
config={**common_config, "chunk_method": "structure"},
)
def snapshot(items: list[dict[str, Any]]) -> list[tuple[Any, ...]]:
return [
(
item["original_content"],
item["source_start"],
item["source_end"],
item["source_start_line"],
item["source_end_line"],
)
for item in items
]
assert snapshot(default_items) == snapshot(structure_items)
assert all(
item["original_content"]
== normalized[item["source_start"] : item["source_end"]]
for item in structure_items
)
assert all(
not (item["source_start"] < second_chapter_start < item["source_end"])
for item in structure_items
)
second_chapter_items = [
item for item in structure_items if item["source_start"] >= second_chapter_start
]
assert second_chapter_items[0]["source_start"] == second_chapter_start
assert second_chapter_items[0]["original_content"].startswith("# 第二章")
def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None:
repeated = "@" * 120
assert len(_preview_task(repeated, options=[])) == 1
assert _preview_task(repeated, options=["clean_invalid_content"]) == []
structured_text = "# 第一章\n" + "甲。" * 30 + "\n# 第二章\n" + "乙。" * 30
detected = _preview_task(
structured_text,
options=["detect_document_structure"],
config={"chunk_method": "fixed", "chunk_size": 20, "min_chunk_size": 5},
)
undetected = _preview_task(
structured_text,
options=[],
config={"chunk_method": "fixed", "chunk_size": 20, "min_chunk_size": 5},
)
assert all("heading_path" in item["quality_score"] for item in detected)
assert {tuple(item["quality_score"]["heading_path"]) for item in detected} == {
("第一章",),
("第二章",),
}
assert all("heading_path" not in item["quality_score"] for item in undetected)
assert all(not ("第一章" in item["edited_content"] and "第二章" in item["edited_content"]) for item in detected)
short_lead = "a b. c d e f g h i j k l m n o p q r s t u v w x y z"
without_merge = _preview_task(
short_lead,
options=[],
config={"chunk_method": "structure", "chunk_size": 12, "min_chunk_size": 5},
)
with_merge = _preview_task(
short_lead,
options=["merge_short_content"],
config={"chunk_method": "structure", "chunk_size": 12, "min_chunk_size": 5},
)
assert without_merge[0]["token_count"] < 5
assert with_merge[0]["token_count"] >= 5
mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。"
assert len(_preview_task(mojibake, options=[])) == 1
assert _preview_task(mojibake, options=["filter_low_quality"]) == []
first = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"
second = "alpha beta gamma, delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"
sources = [
{"id": "near-1", "name": "one.txt", "file_format": "txt", "content": first},
{"id": "near-2", "name": "two.txt", "file_format": "txt", "content": second},
]
base_task = {
"process_type": "unstructured",
"config": {
"chunk_method": "fixed",
"chunk_size": 200,
"chunk_overlap": 0,
"min_chunk_size": 1,
"preprocess_options": [],
},
}
assert len(data_process_endpoint._build_preview_items(base_task, sources)) == 2
deduplicated_task = deepcopy(base_task)
deduplicated_task["config"]["preprocess_options"] = ["deduplicate_content"]
assert len(data_process_endpoint._build_preview_items(deduplicated_task, sources)) == 1
context_text = " ".join(f"token{index}" for index in range(45))
no_context = _preview_task(
context_text,
options=[],
config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5},
)
with_context = _preview_task(
context_text,
options=["preserve_context"],
config={"chunk_method": "fixed", "chunk_size": 20, "chunk_overlap": 5},
)
assert no_context[1]["source_start"] >= no_context[0]["source_end"]
assert with_context[1]["source_start"] < with_context[0]["source_end"]
sensitive = "联系人：张三，手机 13800138000，邮箱 user@example.com。"
plain = _preview_task(sensitive, options=[])[0]
masked = _preview_task(sensitive, options=["desensitize"])[0]
assert "张三" in plain["edited_content"]
assert "联系人:[NAME]" in masked["edited_content"]
assert "[PHONE]" in masked["edited_content"]
assert "[EMAIL]" in masked["edited_content"]
def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None:
for file_format in ("pdf", "docx", "pptx"):
items = _preview_task(
f"{file_format.upper()} 已抽取正文，可直接进入切片处理。",
options=[],
file_format=file_format,
)
assert len(items) == 1
assert "已抽取正文" in items[0]["edited_content"]