feat: 完成数据处理接口与前端接入
This commit is contained in:
caoxiaozhu
1008
backend/app/api/v1/endpoints/data_process.py
Normal file
1008
backend/app/api/v1/endpoints/data_process.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,10 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.v1.endpoints.data_process import router as data_process_router
|
||||
from app.api.v1.endpoints.platform import router as platform_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(health_router, tags=["health"])
|
||||
api_router.include_router(data_process_router, tags=["data-process"])
|
||||
api_router.include_router(platform_router, tags=["platform"])
|
||||
|
||||
|
||||
235
backend/app/db/sql/002_data_process.sql
Normal file
235
backend/app/db/sql/002_data_process.sql
Normal file
@@ -0,0 +1,235 @@
|
||||
-- Data processing migration.
|
||||
--
|
||||
-- IMPORTANT: This file is intentionally NOT wired into application startup.
|
||||
-- Apply it explicitly in a controlled deployment, or call
|
||||
-- DataProcessStore.ensure_schema() from an administrative command.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- This migration targets the current runtime schema created by
|
||||
-- 001_platform_runtime.sql. Refuse the UUID/JSONB target-design schema instead
|
||||
-- of partially altering it with incompatible TEXT foreign keys.
|
||||
DO $$
|
||||
DECLARE
|
||||
datasets_id_type TEXT;
|
||||
BEGIN
|
||||
SELECT format_type(a.atttypid, a.atttypmod)
|
||||
INTO datasets_id_type
|
||||
FROM pg_attribute a
|
||||
JOIN pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.relname = 'datasets'
|
||||
AND a.attname = 'id'
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped;
|
||||
IF datasets_id_type IS NULL THEN
|
||||
RAISE EXCEPTION '002_data_process.sql requires 001_platform_runtime.sql first';
|
||||
END IF;
|
||||
IF datasets_id_type <> 'text' THEN
|
||||
RAISE EXCEPTION
|
||||
'002_data_process.sql supports only the current TEXT runtime schema; found datasets.id type %',
|
||||
datasets_id_type;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name VARCHAR(150) NOT NULL,
|
||||
description TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
|
||||
process_type VARCHAR(20) NOT NULL
|
||||
CHECK (process_type IN ('structured', 'unstructured', 'external')),
|
||||
source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
config TEXT NOT NULL DEFAULT '{}',
|
||||
progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
|
||||
input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
|
||||
output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
|
||||
filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
|
||||
duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
|
||||
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
|
||||
failure_reason TEXT,
|
||||
generation_run_id TEXT,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
owner_id TEXT,
|
||||
approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
|
||||
created_by TEXT,
|
||||
updated_by TEXT,
|
||||
deleted_by TEXT,
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
|
||||
ON data_process_tasks(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
|
||||
ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
|
||||
WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
|
||||
ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_source_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
storage_object_id TEXT,
|
||||
name TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
file_format VARCHAR(40),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
|
||||
content TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
|
||||
ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
|
||||
ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_preview_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
|
||||
original_content TEXT NOT NULL DEFAULT '',
|
||||
edited_content TEXT NOT NULL DEFAULT '',
|
||||
source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
|
||||
source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
|
||||
source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
|
||||
source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
|
||||
token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'original'
|
||||
CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
|
||||
CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
|
||||
ON data_process_preview_items(task_id, source_file_id, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
instruction TEXT NOT NULL,
|
||||
input TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL,
|
||||
original_instruction TEXT,
|
||||
original_input TEXT,
|
||||
original_output TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
error TEXT,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
||||
ON data_process_results(task_id, status, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
||||
ON data_process_results(task_id, split);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_file_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_no INTEGER NOT NULL CHECK (version_no > 0),
|
||||
storage_object_id TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
description TEXT,
|
||||
base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
|
||||
ON dataset_file_versions(dataset_file_id, version_no);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
|
||||
ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
|
||||
line_no INTEGER,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
instruction TEXT,
|
||||
input TEXT,
|
||||
output TEXT,
|
||||
raw TEXT NOT NULL DEFAULT '{}',
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
|
||||
ON dataset_records(dataset_id, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
|
||||
ON dataset_records(source_task_id, source_result_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
|
||||
ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
|
||||
ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
912
backend/app/modules/data_process/algorithms.py
Normal file
912
backend/app/modules/data_process/algorithms.py
Normal file
@@ -0,0 +1,912 @@
|
||||
"""数据处理模块使用的无副作用算法。
|
||||
|
||||
本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
|
||||
所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
|
||||
``source[chunk.start:chunk.end]``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from bisect import bisect_left
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
TextFormat = Literal["json", "jsonl", "csv", "markdown", "txt"]
|
||||
ChunkMethod = Literal["semantic", "heading", "fixed", "custom"]
|
||||
DatasetSplit = Literal["train", "validation", "test"]
|
||||
|
||||
SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
|
||||
"json",
|
||||
"jsonl",
|
||||
"csv",
|
||||
"markdown",
|
||||
"txt",
|
||||
)
|
||||
|
||||
_FORMAT_ALIASES: dict[str, TextFormat] = {
|
||||
"json": "json",
|
||||
"jsonl": "jsonl",
|
||||
"ndjson": "jsonl",
|
||||
"csv": "csv",
|
||||
"tsv": "csv",
|
||||
"md": "markdown",
|
||||
"markdown": "markdown",
|
||||
"txt": "txt",
|
||||
"text": "txt",
|
||||
}
|
||||
|
||||
_EMAIL_PATTERN = re.compile(
|
||||
r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
|
||||
r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
|
||||
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
|
||||
)
|
||||
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
|
||||
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
|
||||
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
|
||||
_HEADING_PATTERN = re.compile(
|
||||
r"(?m)^(?:#{1,6}\s+|第[一二三四五六七八九十百千万0-9]+[章节篇部分]\s*|"
|
||||
r"\d+(?:\.\d+)*[、.\s]+)"
|
||||
)
|
||||
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedText:
|
||||
"""UTF-8 文本的解析结果。"""
|
||||
|
||||
format: TextFormat
|
||||
text: str
|
||||
records: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TextChunk:
|
||||
"""带有可追溯来源位置的非结构化文本切片。"""
|
||||
|
||||
content: str
|
||||
start: int
|
||||
end: int
|
||||
start_line: int
|
||||
end_line: int
|
||||
token_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QualityScore:
|
||||
"""标准 instruction/input/output 记录的可解释质量分。"""
|
||||
|
||||
overall: float
|
||||
completeness: float
|
||||
length: float
|
||||
readability: float
|
||||
relevance: float
|
||||
duplicate: float
|
||||
is_valid: bool
|
||||
flags: tuple[str, ...]
|
||||
fingerprint: str
|
||||
|
||||
|
||||
def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""严格解码 UTF-8 文本，并移除可选 BOM。
|
||||
|
||||
不使用 ``errors='replace'``，避免上传内容损坏后仍被静默接收。
|
||||
"""
|
||||
|
||||
if isinstance(raw, str):
|
||||
return raw.removeprefix("\ufeff")
|
||||
if not isinstance(raw, (bytes, bytearray, memoryview)):
|
||||
raise TypeError("raw must be bytes-like or str")
|
||||
try:
|
||||
return bytes(raw).decode("utf-8-sig")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc
|
||||
|
||||
|
||||
def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""``decode_utf8`` 的语义化别名，供上传服务直接调用。"""
|
||||
|
||||
return decode_utf8(raw)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""规范 Unicode、换行和行尾空白，同时保留段落结构。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
|
||||
normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
|
||||
normalized = "".join(
|
||||
char
|
||||
for char in normalized
|
||||
if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
|
||||
)
|
||||
lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _normalize_format(value: str | None) -> TextFormat | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip().lower().removeprefix(".")
|
||||
try:
|
||||
return _FORMAT_ALIASES[normalized]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unsupported text format: {value}") from exc
|
||||
|
||||
|
||||
def detect_text_format(
|
||||
*,
|
||||
filename: str | None = None,
|
||||
text: str = "",
|
||||
file_format: str | None = None,
|
||||
) -> TextFormat:
|
||||
"""按显式格式、扩展名和内容特征依次识别文本格式。"""
|
||||
|
||||
explicit = _normalize_format(file_format)
|
||||
if explicit:
|
||||
return explicit
|
||||
|
||||
if filename:
|
||||
suffix = Path(filename).suffix.lower().removeprefix(".")
|
||||
detected = _FORMAT_ALIASES.get(suffix)
|
||||
if detected:
|
||||
return detected
|
||||
|
||||
stripped = text.strip()
|
||||
if stripped:
|
||||
if stripped[0] in "[{":
|
||||
try:
|
||||
json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "json"
|
||||
|
||||
nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
|
||||
if len(nonempty_lines) > 1:
|
||||
try:
|
||||
for line in nonempty_lines:
|
||||
json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "jsonl"
|
||||
|
||||
if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
|
||||
r"(?m)^\s*\|.+\|\s*$", stripped
|
||||
):
|
||||
return "markdown"
|
||||
|
||||
sample = stripped[:8192]
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
|
||||
rows = list(csv.reader(io.StringIO(sample), dialect))
|
||||
if len(rows) >= 2 and len(rows[0]) >= 2:
|
||||
return "csv"
|
||||
except csv.Error:
|
||||
pass
|
||||
|
||||
return "txt"
|
||||
|
||||
|
||||
def _normalize_value(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return normalize_text(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_normalize_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _record_from_value(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return dict(_normalize_value(value))
|
||||
return {"value": _normalize_value(value)}
|
||||
|
||||
|
||||
def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
|
||||
"""从 JSON、JSONL 或 CSV 中提取规范化记录。
|
||||
|
||||
JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组；
|
||||
其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
|
||||
"""
|
||||
|
||||
normalized_format = _normalize_format(file_format)
|
||||
if normalized_format not in {"json", "jsonl", "csv"}:
|
||||
raise ValueError("structured record extraction only supports JSON, JSONL and CSV")
|
||||
|
||||
normalized_text = normalize_text(text)
|
||||
if not normalized_text:
|
||||
return []
|
||||
|
||||
if normalized_format == "json":
|
||||
try:
|
||||
payload = json.loads(normalized_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
|
||||
values: Sequence[Any]
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("records", "data", "items", "rows")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
values = [payload]
|
||||
return [_record_from_value(value) for value in values]
|
||||
|
||||
if normalized_format == "jsonl":
|
||||
records: list[dict[str, Any]] = []
|
||||
for line_number, line in enumerate(normalized_text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSONL at line {line_number}, "
|
||||
f"column {exc.colno}: {exc.msg}"
|
||||
) from exc
|
||||
records.append(_record_from_value(value))
|
||||
return records
|
||||
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
|
||||
except csv.Error:
|
||||
dialect = csv.excel
|
||||
reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV header is required")
|
||||
headers = [normalize_text(header or "") for header in reader.fieldnames]
|
||||
if any(not header for header in headers):
|
||||
raise ValueError("CSV header cannot be empty")
|
||||
if len(set(headers)) != len(headers):
|
||||
raise ValueError("CSV headers must be unique")
|
||||
reader.fieldnames = headers
|
||||
|
||||
records = []
|
||||
for row in reader:
|
||||
if None in row:
|
||||
raise ValueError("CSV row has more fields than the header")
|
||||
normalized_row = {
|
||||
key: normalize_text(value or "")
|
||||
for key, value in row.items()
|
||||
}
|
||||
if any(value for value in normalized_row.values()):
|
||||
records.append(normalized_row)
|
||||
return records
|
||||
|
||||
|
||||
def parse_text_content(
|
||||
raw: bytes | bytearray | memoryview | str,
|
||||
*,
|
||||
filename: str | None = None,
|
||||
file_format: str | None = None,
|
||||
) -> ParsedText:
|
||||
"""严格解码并解析支持的 UTF-8 文本格式。"""
|
||||
|
||||
text = normalize_text(decode_utf8(raw))
|
||||
detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
|
||||
records: list[dict[str, Any]] = []
|
||||
if detected_format in {"json", "jsonl", "csv"}:
|
||||
records = extract_structured_records(text, detected_format)
|
||||
return ParsedText(format=detected_format, text=text, records=tuple(records))
|
||||
|
||||
|
||||
def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
|
||||
"""掩码邮箱、中国大陆手机号和 15/18 位身份证号，并返回命中统计。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}
|
||||
|
||||
def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
|
||||
def replacer(_: re.Match[str]) -> str:
|
||||
counts[kind] += 1
|
||||
return replacement
|
||||
|
||||
return pattern.sub(replacer, value)
|
||||
|
||||
masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
|
||||
masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
|
||||
masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)
|
||||
counts["total"] = sum(counts.values())
|
||||
return masked, counts
|
||||
|
||||
|
||||
def estimate_token_count(text: str) -> int:
|
||||
"""无分词器依赖的确定性 token 估算，用于预览与保护性限流。"""
|
||||
|
||||
return len(_TOKEN_PATTERN.findall(text))
|
||||
|
||||
|
||||
def _token_spans(text: str) -> list[tuple[int, int]]:
|
||||
return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
|
||||
|
||||
|
||||
def _line_number(newline_offsets: list[int], offset: int) -> int:
|
||||
# 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
|
||||
return bisect_left(newline_offsets, offset) + 1
|
||||
|
||||
|
||||
def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
|
||||
starts = [span[0] for span in spans]
|
||||
return bisect_left(starts, offset)
|
||||
|
||||
|
||||
def _protected_markdown_ranges(
|
||||
text: str,
|
||||
*,
|
||||
preserve_code_blocks: bool,
|
||||
preserve_tables: bool,
|
||||
preserve_lists: bool,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""
|
||||
|
||||
lines: list[tuple[int, int, str]] = []
|
||||
cursor = 0
|
||||
for raw_line in text.splitlines(keepends=True):
|
||||
end = cursor + len(raw_line)
|
||||
lines.append((cursor, end, raw_line.rstrip("\r\n")))
|
||||
cursor = end
|
||||
if cursor < len(text) or not lines:
|
||||
lines.append((cursor, len(text), text[cursor:]))
|
||||
|
||||
ranges: list[tuple[int, int]] = []
|
||||
code_line_indexes: set[int] = set()
|
||||
if preserve_code_blocks:
|
||||
open_block: tuple[int, str, int] | None = None
|
||||
for index, (start, end, content) in enumerate(lines):
|
||||
fence = re.match(r"^\s*(`{3,}|~{3,})", content)
|
||||
if not fence:
|
||||
continue
|
||||
marker = fence.group(1)[0]
|
||||
length = len(fence.group(1))
|
||||
if open_block is None:
|
||||
open_block = (index, marker, length)
|
||||
continue
|
||||
first_index, open_marker, open_length = open_block
|
||||
if marker == open_marker and length >= open_length:
|
||||
ranges.append((lines[first_index][0], end))
|
||||
code_line_indexes.update(range(first_index, index + 1))
|
||||
open_block = None
|
||||
if open_block is not None:
|
||||
first_index = open_block[0]
|
||||
ranges.append((lines[first_index][0], len(text)))
|
||||
code_line_indexes.update(range(first_index, len(lines)))
|
||||
|
||||
if preserve_tables:
|
||||
index = 0
|
||||
while index + 1 < len(lines):
|
||||
if index in code_line_indexes:
|
||||
index += 1
|
||||
continue
|
||||
header = lines[index][2].strip()
|
||||
separator = lines[index + 1][2].strip().strip("|")
|
||||
cells = [cell.strip() for cell in separator.split("|")]
|
||||
if (
|
||||
"|" not in header
|
||||
or len(cells) < 2
|
||||
or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index + 1
|
||||
while (
|
||||
end_index + 1 < len(lines)
|
||||
and end_index + 1 not in code_line_indexes
|
||||
and lines[end_index + 1][2].strip()
|
||||
and "|" in lines[end_index + 1][2]
|
||||
):
|
||||
end_index += 1
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
if preserve_lists:
|
||||
list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
|
||||
continuation_pattern = re.compile(r"^\s{2,}\S")
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
if index in code_line_indexes or not list_pattern.match(lines[index][2]):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index
|
||||
item_count = 1
|
||||
while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
|
||||
next_line = lines[end_index + 1][2]
|
||||
if list_pattern.match(next_line):
|
||||
item_count += 1
|
||||
end_index += 1
|
||||
elif continuation_pattern.match(next_line):
|
||||
end_index += 1
|
||||
else:
|
||||
break
|
||||
if item_count >= 2:
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
merged: list[tuple[int, int]] = []
|
||||
for start, end in sorted(ranges):
|
||||
if merged and start < merged[-1][1]:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
return merged
|
||||
|
||||
|
||||
def _range_containing(
|
||||
ranges: Sequence[tuple[int, int]], offset: int
|
||||
) -> tuple[int, int] | None:
|
||||
return next((item for item in ranges if item[0] < offset < item[1]), None)
|
||||
|
||||
|
||||
def _boundary_for_method(
|
||||
text: str,
|
||||
spans: list[tuple[int, int]],
|
||||
start_index: int,
|
||||
ideal_end_index: int,
|
||||
minimum_end_index: int,
|
||||
method: ChunkMethod,
|
||||
custom_delimiter: str,
|
||||
) -> tuple[int, int | None]:
|
||||
if method == "fixed":
|
||||
return ideal_end_index, None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
ideal_end_offset = spans[ideal_end_index - 1][1]
|
||||
minimum_end_offset = spans[minimum_end_index - 1][1]
|
||||
search_text = text[start_offset:ideal_end_offset]
|
||||
|
||||
if method == "custom":
|
||||
delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
|
||||
if not delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
relative_minimum = max(0, minimum_end_offset - start_offset)
|
||||
delimiter_start = search_text.rfind(delimiter, relative_minimum)
|
||||
if delimiter_start >= 0:
|
||||
boundary_offset = start_offset + delimiter_start + len(delimiter)
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
if method == "heading":
|
||||
heading_offsets = [
|
||||
start_offset + match.start()
|
||||
for match in _HEADING_PATTERN.finditer(search_text)
|
||||
if start_offset + match.start() >= minimum_end_offset
|
||||
]
|
||||
if heading_offsets:
|
||||
boundary_offset = heading_offsets[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if start_index < boundary_index <= ideal_end_index:
|
||||
return boundary_index, boundary_offset
|
||||
|
||||
semantic_boundaries = [
|
||||
start_offset + match.end()
|
||||
for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(search_text)
|
||||
if start_offset + match.end() >= minimum_end_offset
|
||||
]
|
||||
if semantic_boundaries:
|
||||
boundary_offset = semantic_boundaries[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
|
||||
def chunk_unstructured(
|
||||
text: str,
|
||||
*,
|
||||
method: ChunkMethod = "semantic",
|
||||
chunk_size: int = 800,
|
||||
chunk_overlap: int = 100,
|
||||
min_chunk_size: int = 100,
|
||||
custom_delimiter: str = "",
|
||||
preserve_code_blocks: bool = False,
|
||||
preserve_tables: bool = False,
|
||||
preserve_lists: bool = False,
|
||||
) -> list[TextChunk]:
|
||||
"""按估算 token 切分非结构化文本。
|
||||
|
||||
overlap 足够时精确保留配置数量；短边界下会自动收缩，并且每轮至少推进
|
||||
一个 token，避免异常配置或分隔符造成死循环。
|
||||
"""
|
||||
|
||||
if method not in {"semantic", "heading", "fixed", "custom"}:
|
||||
raise ValueError(f"unsupported chunk method: {method}")
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be greater than 0")
|
||||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if min_chunk_size <= 0 or min_chunk_size > chunk_size:
|
||||
raise ValueError("min_chunk_size must be in [1, chunk_size]")
|
||||
if chunk_overlap + min_chunk_size > chunk_size:
|
||||
raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
|
||||
if method == "custom" and not custom_delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
spans = _token_spans(normalized)
|
||||
if not spans:
|
||||
return []
|
||||
|
||||
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
||||
protected_ranges = _protected_markdown_ranges(
|
||||
normalized,
|
||||
preserve_code_blocks=preserve_code_blocks,
|
||||
preserve_tables=preserve_tables,
|
||||
preserve_lists=preserve_lists,
|
||||
)
|
||||
chunks: list[TextChunk] = []
|
||||
start_index = 0
|
||||
|
||||
while start_index < len(spans):
|
||||
ideal_end_index = min(len(spans), start_index + chunk_size)
|
||||
if ideal_end_index == len(spans):
|
||||
end_index, end_override = ideal_end_index, len(normalized)
|
||||
else:
|
||||
minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
|
||||
end_index, end_override = _boundary_for_method(
|
||||
normalized,
|
||||
spans,
|
||||
start_index,
|
||||
ideal_end_index,
|
||||
minimum_end_index,
|
||||
method,
|
||||
custom_delimiter,
|
||||
)
|
||||
if end_index <= start_index:
|
||||
end_index = min(len(spans), start_index + chunk_size)
|
||||
end_override = None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
end_offset = end_override if end_override is not None else spans[end_index - 1][1]
|
||||
end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
|
||||
split_range = _range_containing(protected_ranges, end_offset)
|
||||
if split_range:
|
||||
before_index = _token_index_at_or_after(spans, split_range[0])
|
||||
if before_index - start_index >= min_chunk_size:
|
||||
end_index = before_index
|
||||
end_offset = split_range[0]
|
||||
else:
|
||||
end_index = min(
|
||||
len(spans),
|
||||
max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
|
||||
)
|
||||
end_offset = split_range[1]
|
||||
content = normalized[start_offset:end_offset]
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
content=content,
|
||||
start=start_offset,
|
||||
end=end_offset,
|
||||
start_line=_line_number(newline_offsets, start_offset),
|
||||
end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
|
||||
token_count=end_index - start_index,
|
||||
)
|
||||
)
|
||||
|
||||
if end_index >= len(spans):
|
||||
break
|
||||
next_start = max(start_index + 1, end_index - chunk_overlap)
|
||||
overlap_range = _range_containing(protected_ranges, spans[next_start][0])
|
||||
if overlap_range:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[0])
|
||||
if candidate <= start_index:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[1])
|
||||
next_start = min(len(spans), max(start_index + 1, candidate))
|
||||
start_index = next_start
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
||||
"""计算与字典键顺序无关的稳定记录指纹。"""
|
||||
|
||||
canonical = {
|
||||
"instruction": normalize_text(str(record.get("instruction") or "")),
|
||||
"input": normalize_text(str(record.get("input") or "")),
|
||||
"output": normalize_text(str(record.get("output") or "")),
|
||||
}
|
||||
raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _readability_score(text: str) -> float:
|
||||
if not text:
|
||||
return 0.0
|
||||
nonspace = [char for char in text if not char.isspace()]
|
||||
if not nonspace:
|
||||
return 0.0
|
||||
printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
|
||||
useful_ratio = sum(
|
||||
char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
|
||||
for char in nonspace
|
||||
) / len(nonspace)
|
||||
return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)
|
||||
|
||||
|
||||
def _internal_duplicate_score(text: str) -> float:
|
||||
units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
|
||||
if len(units) <= 1:
|
||||
return 100.0
|
||||
return round(100 * len(set(units)) / len(units), 2)
|
||||
|
||||
|
||||
def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
|
||||
"""估算结果与来源文本的词元覆盖率。
|
||||
|
||||
这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
|
||||
时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
|
||||
"""
|
||||
|
||||
source = normalize_text(source_content)
|
||||
if not source:
|
||||
return 100.0
|
||||
candidate = normalize_text(
|
||||
"\n".join(
|
||||
str(record.get(field) or "") for field in ("instruction", "input", "output")
|
||||
)
|
||||
)
|
||||
|
||||
def semantic_tokens(text: str) -> set[str]:
|
||||
return {
|
||||
token.lower()
|
||||
for token in _TOKEN_PATTERN.findall(text)
|
||||
if token.isalnum() or "\u3400" <= token <= "\u9fff"
|
||||
}
|
||||
|
||||
source_tokens = semantic_tokens(source)
|
||||
candidate_tokens = semantic_tokens(candidate)
|
||||
if not candidate_tokens:
|
||||
return 0.0
|
||||
if not source_tokens:
|
||||
return 0.0
|
||||
return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)
|
||||
|
||||
|
||||
def score_quality(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
min_output_length: int = 20,
|
||||
source_content: str = "",
|
||||
known_fingerprints: Iterable[str] = (),
|
||||
threshold: float = 60.0,
|
||||
) -> QualityScore:
|
||||
"""按完整性、长度、可读性、来源相关性和重复度计算质量分。"""
|
||||
|
||||
if min_output_length <= 0:
|
||||
raise ValueError("min_output_length must be greater than 0")
|
||||
if not 0 <= threshold <= 100:
|
||||
raise ValueError("threshold must be in [0, 100]")
|
||||
|
||||
instruction = normalize_text(str(record.get("instruction") or ""))
|
||||
input_text = normalize_text(str(record.get("input") or ""))
|
||||
output = normalize_text(str(record.get("output") or ""))
|
||||
flags: list[str] = []
|
||||
|
||||
completeness = 100.0
|
||||
if not instruction:
|
||||
completeness -= 50
|
||||
flags.append("missing_instruction")
|
||||
if not output:
|
||||
completeness -= 50
|
||||
flags.append("missing_output")
|
||||
|
||||
output_length = len(output)
|
||||
length_score = round(min(100.0, output_length / min_output_length * 100), 2)
|
||||
if output_length < min_output_length:
|
||||
flags.append("output_too_short")
|
||||
|
||||
readability = _readability_score("\n".join((instruction, input_text, output)))
|
||||
if readability < 70:
|
||||
flags.append("low_readability")
|
||||
|
||||
relevance = _source_relevance_score(record, source_content)
|
||||
if source_content and relevance < 30:
|
||||
flags.append("low_source_relevance")
|
||||
|
||||
fingerprint = record_fingerprint(record)
|
||||
known = set(known_fingerprints)
|
||||
duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
|
||||
if duplicate == 0:
|
||||
flags.append("duplicate_record")
|
||||
elif duplicate < 70:
|
||||
flags.append("repetitive_output")
|
||||
|
||||
overall = round(
|
||||
completeness * 0.35
|
||||
+ length_score * 0.20
|
||||
+ readability * 0.20
|
||||
+ relevance * 0.15
|
||||
+ duplicate * 0.10,
|
||||
2,
|
||||
)
|
||||
hard_valid = bool(instruction and output)
|
||||
return QualityScore(
|
||||
overall=overall,
|
||||
completeness=completeness,
|
||||
length=length_score,
|
||||
readability=readability,
|
||||
relevance=relevance,
|
||||
duplicate=duplicate,
|
||||
is_valid=hard_valid and overall >= threshold,
|
||||
flags=tuple(flags),
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def stable_split(
|
||||
value: str | int,
|
||||
split: Mapping[str, int] | None = None,
|
||||
*,
|
||||
seed: str = "",
|
||||
) -> DatasetSplit:
|
||||
"""按稳定哈希将记录划分到 train/validation/test。"""
|
||||
|
||||
ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
|
||||
required = {"train", "validation", "test"}
|
||||
if set(ratios) != required:
|
||||
raise ValueError("split must contain exactly train, validation and test")
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
|
||||
raise ValueError("split ratios must be non-negative integers")
|
||||
if sum(ratios.values()) != 100:
|
||||
raise ValueError("split ratios must sum to 100")
|
||||
|
||||
digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
|
||||
bucket = int.from_bytes(digest[:8], "big") % 10_000
|
||||
train_boundary = ratios["train"] * 100
|
||||
validation_boundary = train_boundary + ratios["validation"] * 100
|
||||
if bucket < train_boundary:
|
||||
return "train"
|
||||
if bucket < validation_boundary:
|
||||
return "validation"
|
||||
return "test"
|
||||
|
||||
|
||||
def _preview_content(item: Mapping[str, Any]) -> str:
|
||||
for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
|
||||
value = item.get(field)
|
||||
if value is not None:
|
||||
return normalize_text(str(value))
|
||||
return ""
|
||||
|
||||
|
||||
def _standard_fields(content: str) -> tuple[str, str, str]:
|
||||
if not content:
|
||||
return "", "", ""
|
||||
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, Mapping):
|
||||
instruction = next(
|
||||
(
|
||||
str(payload[key])
|
||||
for key in ("instruction", "question", "prompt")
|
||||
if payload.get(key) is not None
|
||||
),
|
||||
"",
|
||||
)
|
||||
input_text = next(
|
||||
(str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
output = next(
|
||||
(str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
if instruction or output:
|
||||
return normalize_text(instruction), normalize_text(input_text), normalize_text(output)
|
||||
|
||||
question_answer = re.match(
|
||||
r"^\s*(?:问|question)\s*[:：]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[:：]\s*(.+)\s*$",
|
||||
content,
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if question_answer:
|
||||
return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))
|
||||
|
||||
lines = [line.strip() for line in content.splitlines() if line.strip()]
|
||||
first_line = re.sub(r"^(?:问|question)\s*[:：]\s*", "", lines[0], flags=re.IGNORECASE)
|
||||
output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
|
||||
return normalize_text(first_line), "", output
|
||||
|
||||
|
||||
def generate_standard_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
qa_pairs_per_item: int = 1,
|
||||
semantic_enrichment: bool = False,
|
||||
split: Mapping[str, int] | None = None,
|
||||
split_seed: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""把预览内容确定性转换为标准 instruction/input/output 记录。
|
||||
|
||||
该函数只负责本地标准化，不冒充 LLM；服务层可将其作为无模型模式或
|
||||
LLM 响应解析后的统一落库步骤。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ValueError("qa_pairs_per_item must be in [1, 5]")
|
||||
prefixes = (
|
||||
"请结合实际情况说明：",
|
||||
"请用通俗易懂的方式说明：",
|
||||
"请从实际应用角度说明：",
|
||||
"请简洁自然地说明：",
|
||||
"请详细解答：",
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for item_index, item in enumerate(preview_items):
|
||||
content = _preview_content(item)
|
||||
instruction, input_text, output = _standard_fields(content)
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
for variant_index in range(qa_pairs_per_item):
|
||||
variant_instruction = instruction
|
||||
if variant_index:
|
||||
if semantic_enrichment:
|
||||
variant_instruction = f"{prefixes[variant_index]}{instruction}"
|
||||
else:
|
||||
variant_instruction = f"{instruction}(问法 {variant_index + 1})"
|
||||
raw_id = f"{preview_id}:{variant_index + 1}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
|
||||
status = "valid" if variant_instruction and output else "invalid"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": variant_instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": variant_instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": status,
|
||||
"split": stable_split(result_id, split, seed=split_seed),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChunkMethod",
|
||||
"DatasetSplit",
|
||||
"ParsedText",
|
||||
"QualityScore",
|
||||
"SUPPORTED_TEXT_FORMATS",
|
||||
"TextChunk",
|
||||
"TextFormat",
|
||||
"chunk_unstructured",
|
||||
"decode_utf8",
|
||||
"desensitize_pii",
|
||||
"detect_text_format",
|
||||
"estimate_token_count",
|
||||
"extract_structured_records",
|
||||
"generate_standard_records",
|
||||
"normalize_text",
|
||||
"parse_text_content",
|
||||
"parse_utf8_text",
|
||||
"record_fingerprint",
|
||||
"score_quality",
|
||||
"stable_split",
|
||||
]
|
||||
250
backend/app/modules/data_process/generation.py
Normal file
250
backend/app/modules/data_process/generation.py
Normal file
@@ -0,0 +1,250 @@
|
||||
"""数据处理任务的大模型生成适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from app.modules.data_process.algorithms import normalize_text, stable_split
|
||||
|
||||
|
||||
class ModelGenerationError(ValueError):
|
||||
"""模型配置、响应或调用失败。"""
|
||||
|
||||
|
||||
def chat_completions_url(value: str) -> str:
|
||||
"""把域名、基础 URL 或完整地址统一为 chat completions 地址。"""
|
||||
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
raise ModelGenerationError("generation model api_url is required")
|
||||
if "://" not in raw:
|
||||
raw = f"https://{raw}"
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ModelGenerationError("generation model api_url must be an HTTP(S) host or URL")
|
||||
if parsed.username or parsed.password:
|
||||
raise ModelGenerationError("generation model api_url must not contain credentials")
|
||||
|
||||
path = parsed.path.rstrip("/")
|
||||
if path.endswith("/chat/completions"):
|
||||
target_path = path
|
||||
elif path.endswith("/v1"):
|
||||
target_path = f"{path}/chat/completions"
|
||||
elif not path:
|
||||
target_path = "/v1/chat/completions"
|
||||
else:
|
||||
target_path = f"{path}/v1/chat/completions"
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, target_path, "", ""))
|
||||
|
||||
|
||||
def _message_content(payload: Mapping[str, Any]) -> str:
|
||||
try:
|
||||
content = payload["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ModelGenerationError("model response does not contain choices[0].message.content") from exc
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [
|
||||
str(item.get("text") or "")
|
||||
for item in content
|
||||
if isinstance(item, Mapping) and item.get("type") in {None, "text", "output_text"}
|
||||
]
|
||||
if parts:
|
||||
return "".join(parts)
|
||||
raise ModelGenerationError("model response content must be text")
|
||||
|
||||
|
||||
def _json_payload(content: str) -> Any:
|
||||
cleaned = re.sub(r"<think>[\s\S]*?</think>", "", content, flags=re.IGNORECASE).strip()
|
||||
fenced = re.fullmatch(r"```(?:json)?\s*([\s\S]*?)\s*```", cleaned, flags=re.IGNORECASE)
|
||||
if fenced:
|
||||
cleaned = fenced.group(1).strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ModelGenerationError(
|
||||
f"model response is not valid JSON at line {exc.lineno}, column {exc.colno}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _result_items(payload: Any) -> list[Mapping[str, Any]]:
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("items", "results", "data", "records")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
raise ModelGenerationError("model JSON must be an object or array")
|
||||
items = [item for item in values if isinstance(item, Mapping)]
|
||||
if not items:
|
||||
raise ModelGenerationError("model JSON does not contain result objects")
|
||||
return items
|
||||
|
||||
|
||||
def _prompt_messages(prompt: str, content: str, count: int) -> list[dict[str, str]]:
|
||||
schema_instruction = (
|
||||
f"必须只返回 JSON 对象，格式为 {{\"items\":[{{\"instruction\":\"...\","
|
||||
f"\"input\":\"...\",\"output\":\"...\"}}]}}；items 必须包含 {count} 条。"
|
||||
"instruction 和 output 不得为空，不要输出 Markdown 代码围栏或分析过程。"
|
||||
)
|
||||
base_prompt = (
|
||||
normalize_text(prompt)
|
||||
or "请根据来源内容生成可用于监督微调的问答数据。"
|
||||
)
|
||||
if "{{ content }}" in base_prompt:
|
||||
user_prompt = base_prompt.replace("{{ content }}", content)
|
||||
return [
|
||||
{"role": "system", "content": schema_instruction},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
return [
|
||||
{"role": "system", "content": f"{base_prompt}\n{schema_instruction}"},
|
||||
{"role": "user", "content": f"来源内容：\n{content}"},
|
||||
]
|
||||
|
||||
|
||||
def generate_model_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
model: Mapping[str, Any],
|
||||
config: Mapping[str, Any],
|
||||
task_id: str,
|
||||
split: Mapping[str, int],
|
||||
qa_pairs_per_item: int,
|
||||
client: httpx.Client | None = None,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""调用 OpenAI 兼容接口，将预览切片生成标准训练记录。
|
||||
|
||||
单条调用失败会产生可人工修复的 invalid 结果，不会丢弃整批任务。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ModelGenerationError("qa_pairs_per_item must be in [1, 5]")
|
||||
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
||||
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||
if not model_name:
|
||||
raise ModelGenerationError("generation model name is required")
|
||||
|
||||
temperature = float(config.get("temperature", 0.7))
|
||||
max_tokens = int(config.get("max_tokens", 1024))
|
||||
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60))))
|
||||
retries = max(0, min(5, int(config.get("generation_retries", 2))))
|
||||
headers = {"Content-Type": "application/json"}
|
||||
api_key = str(model.get("api_key") or "").strip()
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
owns_client = client is None
|
||||
http_client = client or httpx.Client(timeout=timeout)
|
||||
results: list[dict[str, Any]] = []
|
||||
try:
|
||||
preview_list = list(preview_items)
|
||||
total_items = len(preview_list)
|
||||
for item_index, item in enumerate(preview_list):
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
content = normalize_text(
|
||||
str(item.get("edited_content") or item.get("original_content") or "")
|
||||
)
|
||||
request_payload: dict[str, Any] = {
|
||||
"model": model_name,
|
||||
"messages": _prompt_messages(
|
||||
str(config.get("generation_prompt") or ""),
|
||||
content,
|
||||
qa_pairs_per_item,
|
||||
),
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if bool(config.get("json_mode", False)):
|
||||
request_payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
last_error: Exception | None = None
|
||||
generated_items: list[Mapping[str, Any]] | None = None
|
||||
for _ in range(retries + 1):
|
||||
try:
|
||||
response = http_client.post(endpoint, headers=headers, json=request_payload)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if not isinstance(body, Mapping):
|
||||
raise ModelGenerationError("model response body must be a JSON object")
|
||||
generated_items = _result_items(_json_payload(_message_content(body)))
|
||||
break
|
||||
except (httpx.HTTPError, json.JSONDecodeError, ModelGenerationError) as exc:
|
||||
last_error = exc
|
||||
|
||||
if generated_items is None:
|
||||
error_message = str(last_error or "model generation failed")[:2000]
|
||||
result_id = f"result_{hashlib.sha256(f'{preview_id}:error'.encode()).hexdigest()[:16]}"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": "模型生成失败，请人工补充",
|
||||
"input": content,
|
||||
"output": "",
|
||||
"original_instruction": "模型生成失败，请人工补充",
|
||||
"original_input": content,
|
||||
"original_output": "",
|
||||
"status": "invalid",
|
||||
"error": error_message,
|
||||
"split": stable_split(result_id, split, seed=task_id),
|
||||
}
|
||||
)
|
||||
if on_progress:
|
||||
on_progress(item_index + 1, total_items)
|
||||
continue
|
||||
|
||||
for variant_index, value in enumerate(generated_items[:qa_pairs_per_item]):
|
||||
instruction = normalize_text(str(value.get("instruction") or value.get("question") or ""))
|
||||
input_text = normalize_text(str(value.get("input") or value.get("context") or ""))
|
||||
output = normalize_text(
|
||||
str(
|
||||
value.get("output")
|
||||
or value.get("answer")
|
||||
or value.get("response")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}"
|
||||
valid = bool(instruction and output)
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": "valid" if valid else "invalid",
|
||||
"error": None if valid else "model result is missing instruction or output",
|
||||
"split": stable_split(result_id, split, seed=task_id),
|
||||
}
|
||||
)
|
||||
if on_progress:
|
||||
on_progress(item_index + 1, total_items)
|
||||
finally:
|
||||
if owns_client:
|
||||
http_client.close()
|
||||
return results
|
||||
|
||||
|
||||
__all__ = ["ModelGenerationError", "chat_completions_url", "generate_model_records"]
|
||||
65
backend/app/modules/data_process/schema_cli.py
Normal file
65
backend/app/modules/data_process/schema_cli.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""数据处理运行表的显式检查与安装命令。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.modules.data_process.store import DataProcessStore
|
||||
|
||||
|
||||
def _target_label(database_url: str) -> str:
|
||||
parsed = urlsplit(database_url)
|
||||
database = parsed.path.strip("/") or "(unknown)"
|
||||
return f"{parsed.hostname or '(unknown)'}:{parsed.port or 5432}/{database}"
|
||||
|
||||
|
||||
def _schema_ready(store: DataProcessStore) -> bool:
|
||||
with store.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema=current_schema()
|
||||
AND table_name='data_process_tasks'
|
||||
AND column_name='generation_run_id'
|
||||
) AS ready
|
||||
"""
|
||||
).fetchone()
|
||||
return bool(row and row["ready"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="检查或显式安装数据处理运行表（不会由应用启动自动执行）"
|
||||
)
|
||||
action = parser.add_mutually_exclusive_group(required=True)
|
||||
action.add_argument("--check", action="store_true", help="只读检查迁移是否已安装")
|
||||
action.add_argument("--apply", action="store_true", help="执行 002 数据处理迁移")
|
||||
parser.add_argument(
|
||||
"--yes",
|
||||
action="store_true",
|
||||
help="确认允许修改 DATABASE_URL 指向的数据库；与 --apply 同时使用",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
store = DataProcessStore()
|
||||
target = _target_label(store.database_url)
|
||||
if args.check:
|
||||
ready = _schema_ready(store)
|
||||
print(f"数据处理 schema：{'已安装' if ready else '未安装'}；目标：{target}")
|
||||
return 0 if ready else 1
|
||||
if not args.yes:
|
||||
parser.error("--apply 必须同时提供 --yes，确认修改目标数据库")
|
||||
|
||||
print(f"正在安装数据处理 schema；目标：{target}")
|
||||
store.ensure_schema()
|
||||
if not _schema_ready(store):
|
||||
raise RuntimeError("迁移执行后仍未检测到 generation_run_id")
|
||||
print("数据处理 schema 安装完成")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1333
backend/app/modules/data_process/store.py
Normal file
1333
backend/app/modules/data_process/store.py
Normal file
File diff suppressed because it is too large
Load Diff
245
backend/app/schemas/data_process.py
Normal file
245
backend/app/schemas/data_process.py
Normal file
@@ -0,0 +1,245 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, default: Any) -> Any:
|
||||
if snake_name in config:
|
||||
return config[snake_name]
|
||||
return config.get(camel_name, default)
|
||||
|
||||
|
||||
def _validate_process_config(config: dict[str, Any]) -> None:
|
||||
split = _config_value(config, "dataset_split", "datasetSplit", None)
|
||||
if split is not None:
|
||||
if not isinstance(split, dict) or set(split) != {"train", "validation", "test"}:
|
||||
raise ValueError("dataset_split must contain train, validation and test")
|
||||
values = list(split.values())
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) for value in values):
|
||||
raise ValueError("dataset_split values must be integers")
|
||||
if any(value < 0 or value > 100 for value in values) or sum(values) != 100:
|
||||
raise ValueError("dataset_split values must be in [0, 100] and total 100")
|
||||
|
||||
chunk_fields = {
|
||||
"chunk_size",
|
||||
"chunkSize",
|
||||
"chunk_overlap",
|
||||
"chunkOverlap",
|
||||
"min_chunk_size",
|
||||
"minChunkSize",
|
||||
}
|
||||
if chunk_fields.intersection(config):
|
||||
chunk_size = _config_value(config, "chunk_size", "chunkSize", 800)
|
||||
overlap = _config_value(config, "chunk_overlap", "chunkOverlap", 100)
|
||||
minimum = _config_value(config, "min_chunk_size", "minChunkSize", 100)
|
||||
if any(
|
||||
isinstance(value, bool) or not isinstance(value, int)
|
||||
for value in (chunk_size, overlap, minimum)
|
||||
):
|
||||
raise ValueError("chunk_size, chunk_overlap and min_chunk_size must be integers")
|
||||
if not 16 <= chunk_size <= 32_768:
|
||||
raise ValueError("chunk_size must be in [16, 32768]")
|
||||
if overlap < 0 or overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if minimum <= 0 or minimum > chunk_size or overlap + minimum > chunk_size:
|
||||
raise ValueError("min_chunk_size and chunk_overlap exceed chunk_size")
|
||||
|
||||
temperature = _config_value(config, "temperature", "temperature", None)
|
||||
if temperature is not None:
|
||||
if isinstance(temperature, bool) or not isinstance(temperature, (int, float)):
|
||||
raise ValueError("temperature must be a number")
|
||||
if not 0 <= float(temperature) <= 2:
|
||||
raise ValueError("temperature must be in [0, 2]")
|
||||
|
||||
max_tokens = _config_value(config, "max_tokens", "maxTokens", None)
|
||||
if max_tokens is not None:
|
||||
if isinstance(max_tokens, bool) or not isinstance(max_tokens, int):
|
||||
raise ValueError("max_tokens must be an integer")
|
||||
if not 1 <= max_tokens <= 32_768:
|
||||
raise ValueError("max_tokens must be in [1, 32768]")
|
||||
|
||||
for snake_name, camel_name in (
|
||||
("qa_pairs_per_row", "qaPairsPerRow"),
|
||||
("qa_pairs_per_chunk", "qaPairsPerChunk"),
|
||||
):
|
||||
pairs = _config_value(config, snake_name, camel_name, None)
|
||||
if pairs is None:
|
||||
continue
|
||||
if isinstance(pairs, bool) or not isinstance(pairs, int) or not 1 <= pairs <= 5:
|
||||
raise ValueError(f"{snake_name} must be an integer in [1, 5]")
|
||||
|
||||
|
||||
class DataProcessStatus(StrEnum):
|
||||
pending = "pending"
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
stopped = "stopped"
|
||||
|
||||
|
||||
class ProcessType(StrEnum):
|
||||
structured = "structured"
|
||||
unstructured = "unstructured"
|
||||
external = "external"
|
||||
|
||||
|
||||
class DataProcessTaskCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str = Field(min_length=1, max_length=150)
|
||||
description: str = ""
|
||||
process_type: ProcessType
|
||||
source_dataset_id: str | None = None
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def normalize_name(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("task name cannot be empty")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_config(self) -> "DataProcessTaskCreate":
|
||||
_validate_process_config(self.config)
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessTaskUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=150)
|
||||
description: str | None = None
|
||||
process_type: ProcessType | None = None
|
||||
source_dataset_id: str | None = None
|
||||
config: dict[str, Any] | None = None
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def normalize_name(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("task name cannot be empty")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_config(self) -> "DataProcessTaskUpdate":
|
||||
if self.config is not None:
|
||||
_validate_process_config(self.config)
|
||||
return self
|
||||
|
||||
|
||||
class PreviewBuildRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
source_file_ids: list[str] | None = None
|
||||
|
||||
|
||||
class PreviewItemCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_file_id: str | None = None
|
||||
original_content: str = ""
|
||||
edited_content: str = ""
|
||||
source_start: int | None = Field(default=None, ge=0)
|
||||
source_end: int | None = Field(default=None, ge=0)
|
||||
source_start_line: int | None = Field(default=None, ge=1)
|
||||
source_end_line: int | None = Field(default=None, ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_ranges(self) -> "PreviewItemCreate":
|
||||
if self.source_start is not None and self.source_end is not None:
|
||||
if self.source_end < self.source_start:
|
||||
raise ValueError("source_end must be greater than or equal to source_start")
|
||||
if self.source_start_line is not None and self.source_end_line is not None:
|
||||
if self.source_end_line < self.source_start_line:
|
||||
raise ValueError(
|
||||
"source_end_line must be greater than or equal to source_start_line"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class PreviewItemUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
edited_content: str
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
|
||||
|
||||
class ExternalSourceRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
type: str = Field(min_length=1, max_length=30)
|
||||
url: str = Field(min_length=1, max_length=2048)
|
||||
auth_mode: Literal["none", "basic"] = "none"
|
||||
username: str | None = Field(default=None, max_length=150)
|
||||
password: str | None = Field(default=None, max_length=500)
|
||||
limit: int = Field(default=1000, ge=1, le=100_000)
|
||||
|
||||
|
||||
class ExternalPullRequest(ExternalSourceRequest):
|
||||
query: str | None = Field(default=None, max_length=20_000)
|
||||
file_name: str = Field(default="external-data.jsonl", min_length=1, max_length=255)
|
||||
|
||||
@field_validator("file_name")
|
||||
@classmethod
|
||||
def validate_file_name(cls, value: str) -> str:
|
||||
name = value.strip()
|
||||
if not name.lower().endswith((".jsonl", ".ndjson")):
|
||||
raise ValueError("external pull file_name must end with .jsonl or .ndjson")
|
||||
return name
|
||||
|
||||
|
||||
class ResultUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
instruction: str | None = None
|
||||
input: str | None = None
|
||||
output: str | None = None
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class DatasetSplit(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
train: int = Field(default=80, ge=0, le=100)
|
||||
validation: int = Field(default=10, ge=0, le=100)
|
||||
test: int = Field(default=10, ge=0, le=100)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_total(self) -> "DatasetSplit":
|
||||
if self.train + self.validation + self.test != 100:
|
||||
raise ValueError("dataset split must total 100")
|
||||
return self
|
||||
|
||||
|
||||
class PublishRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dataset_name: str = Field(min_length=1, max_length=150)
|
||||
dataset_type: Literal["train", "test", "eval", "val", "other"] = "train"
|
||||
storage_type: Literal["local"] = "local"
|
||||
split: DatasetSplit = Field(default_factory=DatasetSplit)
|
||||
format: Literal["alpaca_jsonl", "jsonl"] = "alpaca_jsonl"
|
||||
description: str = ""
|
||||
|
||||
@field_validator("dataset_name")
|
||||
@classmethod
|
||||
def normalize_dataset_name(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("dataset name cannot be empty")
|
||||
return value
|
||||
279
backend/tests/test_data_process_algorithms.py
Normal file
279
backend/tests/test_data_process_algorithms.py
Normal file
@@ -0,0 +1,279 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.modules.data_process.algorithms import (
|
||||
chunk_unstructured,
|
||||
desensitize_pii,
|
||||
detect_text_format,
|
||||
extract_structured_records,
|
||||
generate_standard_records,
|
||||
normalize_text,
|
||||
parse_text_content,
|
||||
record_fingerprint,
|
||||
score_quality,
|
||||
stable_split,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
|
||||
parsed_json = parse_text_content(
|
||||
b'\xef\xbb\xbf{"data":[{"name":"\xe5\xbc\xa0\xe4\xb8\x89"}]}',
|
||||
filename="records.json",
|
||||
)
|
||||
assert parsed_json.format == "json"
|
||||
assert parsed_json.records == ({"name": "张三"},)
|
||||
|
||||
parsed_jsonl = parse_text_content('{"id":1}\n\n{"id":2}\n', filename="records.jsonl")
|
||||
assert parsed_jsonl.format == "jsonl"
|
||||
assert parsed_jsonl.records == ({"id": 1}, {"id": 2})
|
||||
|
||||
parsed_csv = parse_text_content("name,answer\r\nAlice,yes\r\nBob,no", filename="records.csv")
|
||||
assert parsed_csv.format == "csv"
|
||||
assert parsed_csv.text == "name,answer\nAlice,yes\nBob,no"
|
||||
assert parsed_csv.records[1] == {"name": "Bob", "answer": "no"}
|
||||
|
||||
parsed_markdown = parse_text_content("# 标题\n\n正文", filename="README.md")
|
||||
assert parsed_markdown.format == "markdown"
|
||||
assert parsed_markdown.records == ()
|
||||
|
||||
parsed_txt = parse_text_content("普通文本", filename="note.txt")
|
||||
assert parsed_txt.format == "txt"
|
||||
assert parsed_txt.text == "普通文本"
|
||||
|
||||
|
||||
def test_invalid_utf8_and_malformed_structured_content_fail_loudly() -> None:
|
||||
with pytest.raises(ValueError, match="not valid UTF-8"):
|
||||
parse_text_content(b"\xff\xfe", filename="broken.txt")
|
||||
with pytest.raises(ValueError, match="invalid JSONL at line 2"):
|
||||
extract_structured_records('{"id":1}\nnot-json', "jsonl")
|
||||
with pytest.raises(ValueError, match="more fields"):
|
||||
extract_structured_records("a,b\n1,2,3", "csv")
|
||||
|
||||
|
||||
def test_detect_format_from_content_and_normalize() -> None:
|
||||
assert detect_text_format(text='{"id":1}\n{"id":2}') == "jsonl"
|
||||
assert detect_text_format(text="# Heading\ntext") == "markdown"
|
||||
assert detect_text_format(text="a,b\n1,2") == "csv"
|
||||
assert normalize_text("\ufeffＡＢＣ \r\n第二\x00行\u200b\t \r\n") == "ABC\n第二行"
|
||||
|
||||
|
||||
def test_extract_json_scalar_and_nested_values_are_stable() -> None:
|
||||
assert extract_structured_records("[1, true, null]", "json") == [
|
||||
{"value": 1},
|
||||
{"value": True},
|
||||
{"value": None},
|
||||
]
|
||||
result = extract_structured_records(
|
||||
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
|
||||
"json",
|
||||
)
|
||||
assert result == [{"text": "内容"}]
|
||||
|
||||
|
||||
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
|
||||
source = "邮箱 a.user+tag@example.com，手机 +86 13800138000，身份证 11010519491231002X。"
|
||||
masked, counts = desensitize_pii(source)
|
||||
assert masked == "邮箱 [EMAIL]，手机 [PHONE]，身份证 [ID_CARD]。"
|
||||
assert counts == {"email": 1, "phone": 1, "id_card": 1, "total": 3}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["semantic", "heading", "fixed", "custom"])
|
||||
def test_chunk_methods_preserve_offsets_and_always_advance(method: str) -> None:
|
||||
text = "# 第一章\n" + "甲。" * 18 + "\n# 第二章\n" + "乙。" * 18
|
||||
kwargs = {"custom_delimiter": "\\n"} if method == "custom" else {}
|
||||
chunks = chunk_unstructured(
|
||||
text,
|
||||
method=method, # type: ignore[arg-type]
|
||||
chunk_size=12,
|
||||
chunk_overlap=2,
|
||||
min_chunk_size=4,
|
||||
**kwargs,
|
||||
)
|
||||
assert len(chunks) > 1
|
||||
assert all(chunk.content == normalize_text(text)[chunk.start : chunk.end] for chunk in chunks)
|
||||
assert all(chunk.end > chunk.start for chunk in chunks)
|
||||
assert all(left.start < right.start for left, right in zip(chunks, chunks[1:]))
|
||||
assert all(chunk.start_line <= chunk.end_line for chunk in chunks)
|
||||
|
||||
|
||||
def test_fixed_chunk_overlap_is_exact_when_chunks_are_large_enough() -> None:
|
||||
text = " ".join(f"token{i}" for i in range(30))
|
||||
chunks = chunk_unstructured(
|
||||
text,
|
||||
method="fixed",
|
||||
chunk_size=10,
|
||||
chunk_overlap=3,
|
||||
min_chunk_size=4,
|
||||
)
|
||||
first_tokens = chunks[0].content.split()
|
||||
second_tokens = chunks[1].content.split()
|
||||
assert first_tokens[-3:] == second_tokens[:3]
|
||||
assert chunks[0].token_count == 10
|
||||
|
||||
|
||||
def test_chunk_line_numbers_treat_newline_as_previous_line_boundary() -> None:
|
||||
chunks = chunk_unstructured(
|
||||
"第一行。\n第二行。\n第三行。",
|
||||
method="custom",
|
||||
chunk_size=8,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=2,
|
||||
custom_delimiter="\\n",
|
||||
)
|
||||
assert chunks[0].content.endswith("\n")
|
||||
assert chunks[0].start_line == 1
|
||||
assert chunks[0].end_line == 1
|
||||
assert chunks[1].start_line == 2
|
||||
|
||||
|
||||
def test_heading_and_custom_boundaries_are_respected() -> None:
|
||||
heading_text = "前言 " * 8 + "\n# 第二章\n" + "正文 " * 12
|
||||
heading_chunks = chunk_unstructured(
|
||||
heading_text,
|
||||
method="heading",
|
||||
chunk_size=20,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=4,
|
||||
)
|
||||
assert "# 第二章" not in heading_chunks[0].content
|
||||
assert heading_chunks[1].content.startswith("#")
|
||||
|
||||
custom_chunks = chunk_unstructured(
|
||||
"a b c d <CUT> e f g h i j",
|
||||
method="custom",
|
||||
chunk_size=8,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=2,
|
||||
custom_delimiter="<CUT>",
|
||||
)
|
||||
assert custom_chunks[0].content.endswith("<CUT>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "block"),
|
||||
[
|
||||
(
|
||||
"preserve_code_blocks",
|
||||
"```python\n" + "\n".join(f"value_{i} = {i}" for i in range(30)) + "\n```",
|
||||
),
|
||||
(
|
||||
"preserve_tables",
|
||||
"| 字段 | 说明 |\n| --- | --- |\n"
|
||||
+ "\n".join(f"| field_{i} | value_{i} |" for i in range(30)),
|
||||
),
|
||||
(
|
||||
"preserve_lists",
|
||||
"\n".join(f"- 第 {i} 项需要完整保留" for i in range(30)),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_markdown_protected_blocks_are_not_split(field: str, block: str) -> None:
|
||||
text = "前言。" * 15 + "\n" + block + "\n" + "结尾。" * 40
|
||||
chunks = chunk_unstructured(
|
||||
text,
|
||||
method="fixed",
|
||||
chunk_size=40,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=10,
|
||||
**{field: True},
|
||||
)
|
||||
assert any(block in chunk.content for chunk in chunks)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "message"),
|
||||
[
|
||||
({"chunk_size": 0}, "chunk_size"),
|
||||
({"chunk_size": 10, "chunk_overlap": 10}, "chunk_overlap"),
|
||||
({"chunk_size": 10, "chunk_overlap": 0, "min_chunk_size": 11}, "min_chunk_size"),
|
||||
(
|
||||
{"chunk_size": 10, "chunk_overlap": 5, "min_chunk_size": 6},
|
||||
"cannot exceed",
|
||||
),
|
||||
({"method": "custom", "custom_delimiter": ""}, "custom_delimiter"),
|
||||
],
|
||||
)
|
||||
def test_chunk_configuration_validation(kwargs: dict[str, object], message: str) -> None:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
chunk_unstructured("some text", **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
|
||||
valid = {
|
||||
"instruction": "如何修改收货地址？",
|
||||
"input": "订单尚未发货",
|
||||
"output": "可以在订单详情页申请修改收货地址。",
|
||||
}
|
||||
source = "订单尚未发货时，可以在订单详情页申请修改收货地址。"
|
||||
first_score = score_quality(valid, min_output_length=10, source_content=source)
|
||||
assert first_score.is_valid
|
||||
assert first_score.completeness == 100
|
||||
assert first_score.length == 100
|
||||
assert first_score.readability >= 90
|
||||
assert first_score.relevance >= 70
|
||||
assert first_score.duplicate == 100
|
||||
|
||||
duplicate_score = score_quality(valid, known_fingerprints={first_score.fingerprint})
|
||||
assert duplicate_score.duplicate == 0
|
||||
assert "duplicate_record" in duplicate_score.flags
|
||||
|
||||
unrelated_score = score_quality(
|
||||
valid,
|
||||
min_output_length=10,
|
||||
source_content="量子计算使用量子比特处理信息。",
|
||||
)
|
||||
assert unrelated_score.relevance < first_score.relevance
|
||||
assert "low_source_relevance" in unrelated_score.flags
|
||||
|
||||
invalid_score = score_quality({"instruction": "", "output": "短"}, min_output_length=10)
|
||||
assert not invalid_score.is_valid
|
||||
assert {"missing_instruction", "output_too_short"}.issubset(invalid_score.flags)
|
||||
assert record_fingerprint(valid) == record_fingerprint(dict(reversed(list(valid.items()))))
|
||||
|
||||
|
||||
def test_stable_split_is_reproducible_and_validates_ratios() -> None:
|
||||
first = stable_split("record-42", seed="task-1")
|
||||
assert stable_split("record-42", seed="task-1") == first
|
||||
assert first in {"train", "validation", "test"}
|
||||
assert stable_split("record-42", {"train": 100, "validation": 0, "test": 0}) == "train"
|
||||
with pytest.raises(ValueError, match="sum to 100"):
|
||||
stable_split("record", {"train": 80, "validation": 10, "test": 9})
|
||||
|
||||
|
||||
def test_generate_standard_records_supports_json_qa_and_stable_variants() -> None:
|
||||
previews = [
|
||||
{
|
||||
"id": "preview-json",
|
||||
"edited_content": json.dumps(
|
||||
{"instruction": "问题", "input": "上下文", "output": "答案"},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
},
|
||||
{"id": "preview-qa", "editedContent": "问：如何操作？\n答：按步骤操作。"},
|
||||
]
|
||||
records = generate_standard_records(
|
||||
previews,
|
||||
qa_pairs_per_item=2,
|
||||
semantic_enrichment=True,
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
split_seed="task-1",
|
||||
)
|
||||
assert len(records) == 4
|
||||
assert records[0]["instruction"] == "问题"
|
||||
assert records[0]["input"] == "上下文"
|
||||
assert records[0]["output"] == "答案"
|
||||
assert records[1]["instruction"].endswith("问题")
|
||||
assert records[2]["instruction"] == "如何操作?"
|
||||
assert records[2]["output"] == "按步骤操作。"
|
||||
assert all(record["status"] == "valid" for record in records)
|
||||
assert all(record["split"] == "train" for record in records)
|
||||
assert records == generate_standard_records(
|
||||
previews,
|
||||
qa_pairs_per_item=2,
|
||||
semantic_enrichment=True,
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
split_seed="task-1",
|
||||
)
|
||||
704
backend/tests/test_data_process_api.py
Normal file
704
backend/tests/test_data_process_api.py
Normal file
@@ -0,0 +1,704 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.v1.endpoints import data_process as data_process_endpoint
|
||||
from app.api.v1.endpoints.data_process import router
|
||||
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
|
||||
|
||||
|
||||
class FakeDataProcessStore:
|
||||
"""接口测试专用内存实现，确保测试不会连接或迁移真实数据库。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.tasks: dict[str, dict[str, Any]] = {}
|
||||
self.sources: dict[str, list[dict[str, Any]]] = {}
|
||||
self.previews: dict[str, list[dict[str, Any]]] = {}
|
||||
self.results: dict[str, list[dict[str, Any]]] = {}
|
||||
self.datasets: dict[str, dict[str, Any]] = {}
|
||||
self.sequence = 0
|
||||
|
||||
def _id(self, prefix: str) -> str:
|
||||
self.sequence += 1
|
||||
return f"{prefix}_{self.sequence}"
|
||||
|
||||
def list_tasks(self, *, page: int, page_size: int, **filters: Any) -> dict[str, Any]:
|
||||
items = list(self.tasks.values())
|
||||
for field in ("status", "process_type", "tenant_id", "project_id"):
|
||||
if filters.get(field):
|
||||
items = [item for item in items if item.get(field) == filters[field]]
|
||||
keyword = filters.get("keyword")
|
||||
if keyword:
|
||||
items = [item for item in items if keyword in item["name"]]
|
||||
return {
|
||||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||||
"total": len(items),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def create_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = self._id("dpt")
|
||||
task = {
|
||||
"id": task_id,
|
||||
**deepcopy(payload),
|
||||
"status": "pending",
|
||||
"progress": 0,
|
||||
"input_count": 0,
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"output_dataset_id": None,
|
||||
}
|
||||
self.tasks[task_id] = task
|
||||
self.sources[task_id] = []
|
||||
self.previews[task_id] = []
|
||||
self.results[task_id] = []
|
||||
return deepcopy(task)
|
||||
|
||||
def get_task(self, task_id: str) -> dict[str, Any]:
|
||||
if task_id not in self.tasks:
|
||||
raise NotFoundError("data process task not found")
|
||||
return deepcopy(self.tasks[task_id])
|
||||
|
||||
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self.get_task(task_id)
|
||||
self.tasks[task_id].update(deepcopy(payload))
|
||||
return self.get_task(task_id)
|
||||
|
||||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||||
self.get_task(task_id)
|
||||
if self.tasks[task_id]["status"] == "running":
|
||||
raise InvalidStateError("running task must be stopped before deletion")
|
||||
del self.tasks[task_id]
|
||||
|
||||
def list_source_files(self, task_id: str) -> list[dict[str, Any]]:
|
||||
self.get_task(task_id)
|
||||
return [
|
||||
{key: value for key, value in item.items() if key != "content"}
|
||||
for item in self.sources[task_id]
|
||||
]
|
||||
|
||||
def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]:
|
||||
self.get_task(task_id)
|
||||
source = {
|
||||
"id": self._id("dpsf"),
|
||||
"task_id": task_id,
|
||||
"version_no": 1,
|
||||
**deepcopy(payload),
|
||||
}
|
||||
self.sources[task_id].append(source)
|
||||
self.tasks[task_id]["input_count"] += payload["record_count"]
|
||||
return {key: value for key, value in deepcopy(source).items() if key != "content"}
|
||||
|
||||
def add_source_files(
|
||||
self, task_id: str, files: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
# 先验证整个批次，模拟数据库事务的 all-or-nothing 语义。
|
||||
checksums = {item["checksum_sha256"] for item in self.sources.get(task_id, [])}
|
||||
incoming: set[str] = set()
|
||||
for payload in files:
|
||||
checksum = payload["checksum_sha256"]
|
||||
if checksum in checksums or checksum in incoming:
|
||||
raise ValueError("the same source file content is already attached to this task")
|
||||
incoming.add(checksum)
|
||||
return [self.add_source_file(task_id, **payload) for payload in files]
|
||||
|
||||
def get_source_file(
|
||||
self, task_id: str, file_id: str, *, include_content: bool = True
|
||||
) -> dict[str, Any]:
|
||||
source = next(
|
||||
(item for item in self.sources.get(task_id, []) if item["id"] == file_id),
|
||||
None,
|
||||
)
|
||||
if not source:
|
||||
raise NotFoundError("source file not found")
|
||||
result = deepcopy(source)
|
||||
if not include_content:
|
||||
result.pop("content", None)
|
||||
return result
|
||||
|
||||
def source_content_window(
|
||||
self, task_id: str, file_id: str, offset: int, limit: int
|
||||
) -> dict[str, Any]:
|
||||
source = self.get_source_file(task_id, file_id)
|
||||
content = source.pop("content")
|
||||
return {
|
||||
"file": source,
|
||||
"content": content[offset : offset + limit],
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"total_chars": len(content),
|
||||
"has_more": offset + limit < len(content),
|
||||
}
|
||||
|
||||
def source_content_lines(
|
||||
self, task_id: str, file_id: str, start_line: int, line_count: int
|
||||
) -> dict[str, Any]:
|
||||
source = self.get_source_file(task_id, file_id)
|
||||
lines = source.pop("content").splitlines(keepends=True)
|
||||
selected = lines[start_line - 1 : start_line - 1 + line_count]
|
||||
return {
|
||||
"file": source,
|
||||
"content": "".join(selected),
|
||||
"start_line": start_line,
|
||||
"end_line": start_line - 1 + len(selected),
|
||||
"line_count": len(selected),
|
||||
"total_lines": len(lines),
|
||||
"has_more": start_line - 1 + len(selected) < len(lines),
|
||||
}
|
||||
|
||||
def delete_source_file(self, task_id: str, file_id: str) -> None:
|
||||
self.get_source_file(task_id, file_id)
|
||||
self.sources[task_id] = [item for item in self.sources[task_id] if item["id"] != file_id]
|
||||
self.previews[task_id] = [
|
||||
item for item in self.previews[task_id] if item["source_file_id"] != file_id
|
||||
]
|
||||
self.results[task_id] = []
|
||||
|
||||
def replace_preview_items(
|
||||
self, task_id: str, items: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
self.previews[task_id] = [
|
||||
{"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items
|
||||
]
|
||||
self.results[task_id] = []
|
||||
self.tasks[task_id]["progress"] = 20
|
||||
return deepcopy(self.previews[task_id])
|
||||
|
||||
def list_preview_items(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
source_file_id: str | None = None,
|
||||
keyword: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
items = self.previews[task_id]
|
||||
if source_file_id:
|
||||
items = [item for item in items if item["source_file_id"] == source_file_id]
|
||||
if keyword:
|
||||
items = [item for item in items if keyword in item["edited_content"]]
|
||||
return {
|
||||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||||
"total": len(items),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]:
|
||||
item = next(
|
||||
(item for item in self.previews.get(task_id, []) if item["id"] == preview_id),
|
||||
None,
|
||||
)
|
||||
if not item:
|
||||
raise NotFoundError("preview item not found")
|
||||
return deepcopy(item)
|
||||
|
||||
def create_preview_item(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
item = {"id": self._id("dpp"), "task_id": task_id, **deepcopy(payload)}
|
||||
self.previews[task_id].append(item)
|
||||
self.results[task_id] = []
|
||||
return deepcopy(item)
|
||||
|
||||
def update_preview_item(
|
||||
self, task_id: str, preview_id: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
item = next(
|
||||
(item for item in self.previews[task_id] if item["id"] == preview_id),
|
||||
None,
|
||||
)
|
||||
if not item:
|
||||
raise NotFoundError("preview item not found")
|
||||
item.update(deepcopy(payload))
|
||||
self.results[task_id] = []
|
||||
return deepcopy(item)
|
||||
|
||||
def delete_preview_item(self, task_id: str, preview_id: str) -> None:
|
||||
before = len(self.previews[task_id])
|
||||
self.previews[task_id] = [
|
||||
item for item in self.previews[task_id] if item["id"] != preview_id
|
||||
]
|
||||
if len(self.previews[task_id]) == before:
|
||||
raise NotFoundError("preview item not found")
|
||||
|
||||
def start_generation(self, task_id: str, *, replace_existing: bool) -> dict[str, Any]:
|
||||
if not self.previews[task_id]:
|
||||
raise InvalidStateError("preview must be built before generation")
|
||||
if replace_existing:
|
||||
self.results[task_id] = []
|
||||
self.tasks[task_id].update(
|
||||
status="running",
|
||||
progress=30,
|
||||
generation_run_id=self._id("dprun"),
|
||||
)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def generation_is_running(self, task_id: str, generation_run_id: str) -> bool:
|
||||
return (
|
||||
self.tasks[task_id]["status"] == "running"
|
||||
and self.tasks[task_id].get("generation_run_id") == generation_run_id
|
||||
)
|
||||
|
||||
def update_generation_progress(
|
||||
self,
|
||||
task_id: str,
|
||||
generation_run_id: str,
|
||||
processed_count: int,
|
||||
total_count: int,
|
||||
) -> bool:
|
||||
if not self.generation_is_running(task_id, generation_run_id):
|
||||
return False
|
||||
self.tasks[task_id]["progress"] = min(
|
||||
95,
|
||||
30 + processed_count / max(1, total_count) * 65,
|
||||
)
|
||||
return True
|
||||
|
||||
def complete_generation(
|
||||
self,
|
||||
task_id: str,
|
||||
results: list[dict[str, Any]],
|
||||
*,
|
||||
generation_run_id: str,
|
||||
**counts: Any,
|
||||
) -> dict[str, Any]:
|
||||
if not self.generation_is_running(task_id, generation_run_id):
|
||||
return self.get_task(task_id)
|
||||
self.results[task_id] = deepcopy(results)
|
||||
self.tasks[task_id].update(
|
||||
status="completed",
|
||||
progress=100,
|
||||
output_count=len(results),
|
||||
generation_run_id=None,
|
||||
**counts,
|
||||
)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def mark_failed(
|
||||
self, task_id: str, reason: str, *, generation_run_id: str
|
||||
) -> dict[str, Any]:
|
||||
if self.generation_is_running(task_id, generation_run_id):
|
||||
self.tasks[task_id].update(
|
||||
status="failed",
|
||||
failure_reason=reason,
|
||||
generation_run_id=None,
|
||||
)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||
if self.tasks[task_id]["status"] != "running":
|
||||
raise InvalidStateError("only a running task can be stopped")
|
||||
self.tasks[task_id].update(status="stopped", generation_run_id=None)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def progress(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.get_task(task_id)
|
||||
result = {key: task.get(key) for key in (
|
||||
"status", "progress", "input_count", "output_count",
|
||||
"filtered_count", "duplicate_count", "error_count", "failure_reason",
|
||||
)}
|
||||
result["task_id"] = task["id"]
|
||||
return result
|
||||
|
||||
def list_results(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
status: str | None = None,
|
||||
split: str | None = None,
|
||||
keyword: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
items = self.results[task_id]
|
||||
if status:
|
||||
items = [item for item in items if item["status"] == status]
|
||||
if split:
|
||||
items = [item for item in items if item["split"] == split]
|
||||
if keyword:
|
||||
items = [
|
||||
item
|
||||
for item in items
|
||||
if any(keyword in item[field] for field in ("instruction", "input", "output"))
|
||||
]
|
||||
return {
|
||||
"items": deepcopy(items),
|
||||
"total": len(items),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def update_result(
|
||||
self, task_id: str, result_id: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||||
if not item:
|
||||
raise NotFoundError("data process result not found")
|
||||
for field in ("instruction", "input", "output", "quality_score"):
|
||||
if field in payload:
|
||||
item[field] = deepcopy(payload[field])
|
||||
hard_valid = bool(item["instruction"].strip() and item["output"].strip())
|
||||
quality_valid = bool((item.get("quality_score") or {}).get("is_valid", hard_valid))
|
||||
changed = any(
|
||||
item[field] != item[f"original_{field}"]
|
||||
for field in ("instruction", "input", "output")
|
||||
)
|
||||
item["status"] = (
|
||||
"invalid"
|
||||
if not hard_valid or not quality_valid
|
||||
else "modified" if changed else "valid"
|
||||
)
|
||||
self.tasks[task_id]["error_count"] = sum(
|
||||
result["status"] == "invalid" for result in self.results[task_id]
|
||||
)
|
||||
return deepcopy(item)
|
||||
|
||||
def get_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||||
if not item:
|
||||
raise NotFoundError("data process result not found")
|
||||
return deepcopy(item)
|
||||
|
||||
def restore_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||||
if not item:
|
||||
raise NotFoundError("data process result not found")
|
||||
for field in ("instruction", "input", "output"):
|
||||
item[field] = item[f"original_{field}"]
|
||||
item["status"] = "valid"
|
||||
return deepcopy(item)
|
||||
|
||||
def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task = self.tasks[task_id]
|
||||
if task.get("output_dataset_id"):
|
||||
return {"dataset": deepcopy(self.datasets[task["output_dataset_id"]]), "created": False}
|
||||
if task["status"] != "completed":
|
||||
raise InvalidStateError("only a completed task can be published")
|
||||
dataset_id = self._id("dataset")
|
||||
dataset = {"id": dataset_id, "name": payload["dataset_name"], "source_task_id": task_id}
|
||||
self.datasets[dataset_id] = dataset
|
||||
task["output_dataset_id"] = dataset_id
|
||||
return {"dataset": deepcopy(dataset), "created": True}
|
||||
|
||||
|
||||
def make_client() -> tuple[TestClient, FakeDataProcessStore]:
|
||||
store = FakeDataProcessStore()
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/modelTF")
|
||||
app.dependency_overrides[get_data_process_store] = lambda: store
|
||||
return TestClient(app), store
|
||||
|
||||
|
||||
def test_data_process_full_contract_without_database() -> None:
|
||||
client, store = make_client()
|
||||
created = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "客服问答处理",
|
||||
"process_type": "structured",
|
||||
"config": {"dataset_split": {"train": 80, "validation": 10, "test": 10}},
|
||||
},
|
||||
)
|
||||
assert created.status_code == 200
|
||||
task_id = created.json()["data"]["id"]
|
||||
|
||||
source_content = (
|
||||
'{"question":"如何修改地址？",'
|
||||
'"answer":"订单发货前可在订单详情申请修改收货地址。"}\n'
|
||||
'{"question":"如何申请退款？",'
|
||||
'"answer":"请在订单详情提交退款申请并等待审核处理。"}\n'
|
||||
)
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("customer.jsonl", source_content.encode(), "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
source = uploaded.json()["data"]["files"][0]
|
||||
assert len(source["checksum_sha256"]) == 64
|
||||
assert source["version_no"] == 1
|
||||
|
||||
window = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||||
params={"offset": 0, "limit": 20},
|
||||
)
|
||||
assert window.status_code == 200
|
||||
assert window.json()["data"]["has_more"] is True
|
||||
line_window = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||||
params={"start_line": 2, "line_count": 1},
|
||||
)
|
||||
assert line_window.json()["data"]["start_line"] == 2
|
||||
assert line_window.json()["data"]["end_line"] == 2
|
||||
assert line_window.json()["data"]["total_lines"] == 2
|
||||
|
||||
preview = client.post(
|
||||
f"/modelTF/data-process/{task_id}/preview/build",
|
||||
json={"source_file_ids": [source["id"]]},
|
||||
)
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["total"] == 2
|
||||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||||
assert listed_preview.json()["data"]["total"] == 2
|
||||
preview_item = listed_preview.json()["data"]["items"][0]
|
||||
updated_preview = client.put(
|
||||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||||
json={
|
||||
"edited_content": preview_item["edited_content"],
|
||||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||||
},
|
||||
)
|
||||
assert "quality_score" in updated_preview.json()["data"]
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
progress = client.get(f"/modelTF/data-process/{task_id}/progress")
|
||||
assert progress.json()["data"]["status"] == "completed"
|
||||
result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
|
||||
assert result_page["total"] == 2
|
||||
keyword_page = client.get(
|
||||
f"/modelTF/data-process/{task_id}/results", params={"keyword": "地址"}
|
||||
).json()["data"]
|
||||
assert keyword_page["total"] == 1
|
||||
|
||||
result = result_page["items"][0]
|
||||
edited = client.put(
|
||||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||||
json={
|
||||
"output": "人工修改后的完整答案。",
|
||||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||||
},
|
||||
)
|
||||
assert edited.json()["data"]["status"] == "modified"
|
||||
assert "quality_score" in edited.json()["data"]
|
||||
invalid_edit = client.put(
|
||||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||||
json={"output": ""},
|
||||
)
|
||||
assert invalid_edit.json()["data"]["status"] == "invalid"
|
||||
assert store.tasks[task_id]["error_count"] == 1
|
||||
restored = client.post(
|
||||
f"/modelTF/data-process/{task_id}/results/{result['id']}/restore"
|
||||
)
|
||||
assert restored.json()["data"]["output"] == result["original_output"]
|
||||
assert restored.json()["data"]["status"] == "valid"
|
||||
assert store.tasks[task_id]["error_count"] == 0
|
||||
|
||||
publish_payload = {"dataset_name": "客服问答清洗集"}
|
||||
first_publish = client.post(
|
||||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||||
)
|
||||
second_publish = client.post(
|
||||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||||
)
|
||||
assert first_publish.json()["data"]["created"] is True
|
||||
assert second_publish.json()["data"]["created"] is False
|
||||
assert (
|
||||
first_publish.json()["data"]["dataset"]["id"]
|
||||
== second_publish.json()["data"]["dataset"]["id"]
|
||||
)
|
||||
|
||||
|
||||
def test_external_source_never_returns_fake_success() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "外部数据", "process_type": "external", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/external/test",
|
||||
json={"type": "mysql", "url": "mysql://db.example/test"},
|
||||
)
|
||||
assert response.status_code == 501
|
||||
assert response.json()["detail"]["code"] == 501
|
||||
|
||||
|
||||
def test_config_validation_and_stop_state() -> None:
|
||||
client, store = make_client()
|
||||
invalid = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "错误切片配置",
|
||||
"process_type": "unstructured",
|
||||
"config": {
|
||||
"dataset_split": {"train": 80, "validation": 30, "test": 0},
|
||||
"chunk_size": 100,
|
||||
"chunk_overlap": 90,
|
||||
"min_chunk_size": 20,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert invalid.status_code == 422
|
||||
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "可停止任务", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id]["status"] = "running"
|
||||
stopped = client.post(f"/modelTF/data-process/{task_id}/stop")
|
||||
assert stopped.status_code == 200
|
||||
assert stopped.json()["data"]["status"] == "stopped"
|
||||
|
||||
|
||||
def test_upload_batch_is_atomic_and_empty_files_are_rejected() -> None:
|
||||
client, store = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "批量上传", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
|
||||
duplicate_batch = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("first.txt", b"same content", "text/plain")),
|
||||
("files", ("second.txt", b"same content", "text/plain")),
|
||||
],
|
||||
)
|
||||
assert duplicate_batch.status_code == 400
|
||||
assert store.sources[task_id] == []
|
||||
|
||||
empty = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("empty.txt", b"", "text/plain")},
|
||||
)
|
||||
assert empty.status_code == 400
|
||||
assert store.sources[task_id] == []
|
||||
|
||||
|
||||
def test_preprocess_deduplicates_and_quality_filter_removes_short_results() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "去重与质量筛选",
|
||||
"process_type": "structured",
|
||||
"config": {
|
||||
"preprocess_options": ["clean_invalid", "deduplicate"],
|
||||
"quality_filter_enabled": True,
|
||||
"filter_low_quality": False,
|
||||
"filter_short_content": True,
|
||||
"min_output_length": 100,
|
||||
},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
content = (
|
||||
'{"question":"问题","answer":"短答案"}\n'
|
||||
'{"question":"问题","answer":"短答案"}\n'
|
||||
).encode()
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("duplicates.jsonl", content, "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.json()["data"]["total"] == 1
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
progress = client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]
|
||||
assert progress["status"] == "completed"
|
||||
assert progress["filtered_count"] == 1
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||||
|
||||
|
||||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||||
store = FakeDataProcessStore()
|
||||
task = store.create_task(
|
||||
{"name": "并发代次", "process_type": "structured", "config": {}}
|
||||
)
|
||||
task_id = task["id"]
|
||||
store.replace_preview_items(
|
||||
task_id,
|
||||
[
|
||||
{
|
||||
"source_file_id": None,
|
||||
"original_content": "来源内容",
|
||||
"edited_content": "来源内容",
|
||||
"status": "manual",
|
||||
}
|
||||
],
|
||||
)
|
||||
first = store.start_generation(task_id, replace_existing=True)
|
||||
first_run_id = first["generation_run_id"]
|
||||
second_run_id = ""
|
||||
|
||||
def restart_while_old_worker_runs(*_: Any, **__: Any) -> list[dict[str, Any]]:
|
||||
nonlocal second_run_id
|
||||
store.stop_task(task_id)
|
||||
second = store.start_generation(task_id, replace_existing=True)
|
||||
second_run_id = second["generation_run_id"]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
data_process_endpoint,
|
||||
"generate_standard_records",
|
||||
restart_while_old_worker_runs,
|
||||
)
|
||||
data_process_endpoint._run_generation(store, task_id, first_run_id)
|
||||
|
||||
assert second_run_id and second_run_id != first_run_id
|
||||
assert store.tasks[task_id]["status"] == "running"
|
||||
assert store.tasks[task_id]["generation_run_id"] == second_run_id
|
||||
assert store.results[task_id] == []
|
||||
store.mark_failed(task_id, "old failure", generation_run_id=first_run_id)
|
||||
assert store.tasks[task_id]["status"] == "running"
|
||||
|
||||
|
||||
def test_result_status_cannot_be_forged_by_client() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "状态保护", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
response = client.put(
|
||||
f"/modelTF/data-process/{task_id}/results/not-created",
|
||||
json={"instruction": "", "output": "", "status": "valid"},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_start_rebuilds_preview_and_generates_in_one_request() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "一键处理", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={
|
||||
"files": (
|
||||
"one.jsonl",
|
||||
b'{"question":"What is one?","answer":"One."}\n',
|
||||
"application/jsonl",
|
||||
)
|
||||
},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
|
||||
started = client.post(f"/modelTF/data-process/{task_id}/start")
|
||||
assert started.status_code == 200
|
||||
assert started.json()["data"]["task_id"] == task_id
|
||||
assert started.json()["data"]["status"] == "running"
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]["status"] == "completed"
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]["total"] == 1
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1
|
||||
|
||||
|
||||
def test_unsupported_upload_format_returns_415() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "格式限制", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("document.pdf", b"not a pdf", "application/pdf")},
|
||||
)
|
||||
assert response.status_code == 415
|
||||
102
backend/tests/test_data_process_generation.py
Normal file
102
backend/tests/test_data_process_generation.py
Normal file
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
from app.modules.data_process.generation import chat_completions_url, generate_model_records
|
||||
|
||||
|
||||
def test_chat_completions_url_accepts_host_base_and_complete_url() -> None:
|
||||
assert chat_completions_url("www.caoxiaozhu.com") == (
|
||||
"https://www.caoxiaozhu.com/v1/chat/completions"
|
||||
)
|
||||
assert chat_completions_url("https://model.example/v1") == (
|
||||
"https://model.example/v1/chat/completions"
|
||||
)
|
||||
complete = "https://model.example/openai/v1/chat/completions"
|
||||
assert chat_completions_url(complete) == complete
|
||||
|
||||
|
||||
def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
|
||||
requests: list[httpx.Request] = []
|
||||
progress_updates: list[tuple[int, int]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
payload = json.loads(request.content)
|
||||
assert payload["model"] == "qwen-plus"
|
||||
assert payload["response_format"] == {"type": "json_object"}
|
||||
assert "客户反馈页面加载慢" in payload["messages"][1]["content"]
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps(
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"instruction": "请生成简洁客服回复",
|
||||
"input": "客户反馈页面加载慢",
|
||||
"output": "已收到反馈，我们正在排查。",
|
||||
}
|
||||
]
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-1", "edited_content": "客户反馈页面加载慢"}],
|
||||
model={
|
||||
"name": "Qwen",
|
||||
"online_model_name": "qwen-plus",
|
||||
"api_url": "model.example",
|
||||
"api_key": "test-secret",
|
||||
},
|
||||
config={
|
||||
"generation_prompt": "请处理：{{ content }}",
|
||||
"json_mode": True,
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 512,
|
||||
},
|
||||
task_id="task-1",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
on_progress=lambda processed, total: progress_updates.append((processed, total)),
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["status"] == "valid"
|
||||
assert records[0]["split"] == "train"
|
||||
assert requests[0].headers["Authorization"] == "Bearer test-secret"
|
||||
assert progress_updates == [(1, 1)]
|
||||
|
||||
|
||||
def test_generate_model_records_keeps_partial_failure_for_manual_repair() -> None:
|
||||
client = httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda _: httpx.Response(200, json={"choices": [{"message": {"content": "not-json"}}]})
|
||||
)
|
||||
)
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-1", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 1},
|
||||
task_id="task-1",
|
||||
split={"train": 80, "validation": 10, "test": 10},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert records[0]["error"]
|
||||
29
backend/tests/test_data_process_migration.py
Normal file
29
backend/tests/test_data_process_migration.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.modules.data_process.schema_cli import _target_label
|
||||
|
||||
|
||||
def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
|
||||
sql_path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "app"
|
||||
/ "db"
|
||||
/ "sql"
|
||||
/ "002_data_process.sql"
|
||||
)
|
||||
sql = sql_path.read_text(encoding="utf-8")
|
||||
|
||||
assert "requires 001_platform_runtime.sql first" in sql
|
||||
assert "supports only the current TEXT runtime schema" in sql
|
||||
assert "generation_run_id" in sql
|
||||
assert "CREATE TABLE IF NOT EXISTS data_process_results" in sql
|
||||
assert sql.count("BEGIN;") == 1
|
||||
assert sql.rstrip().endswith("COMMIT;")
|
||||
|
||||
|
||||
def test_schema_cli_target_label_never_contains_credentials() -> None:
|
||||
label = _target_label("postgresql://secret-user:secret-password@db.example:5433/yg_ft")
|
||||
assert label == "db.example:5433/yg_ft"
|
||||
assert "secret" not in label
|
||||
Reference in New Issue
Block a user