feat: 完成数据处理接口与前端接入
This commit is contained in:
caoxiaozhu
1008
backend/app/api/v1/endpoints/data_process.py
Normal file
1008
backend/app/api/v1/endpoints/data_process.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,10 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.v1.endpoints.data_process import router as data_process_router
|
||||
from app.api.v1.endpoints.platform import router as platform_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(health_router, tags=["health"])
|
||||
api_router.include_router(data_process_router, tags=["data-process"])
|
||||
api_router.include_router(platform_router, tags=["platform"])
|
||||
|
||||
|
||||
235
backend/app/db/sql/002_data_process.sql
Normal file
235
backend/app/db/sql/002_data_process.sql
Normal file
@@ -0,0 +1,235 @@
|
||||
-- Data processing migration.
|
||||
--
|
||||
-- IMPORTANT: This file is intentionally NOT wired into application startup.
|
||||
-- Apply it explicitly in a controlled deployment, or call
|
||||
-- DataProcessStore.ensure_schema() from an administrative command.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- This migration targets the current runtime schema created by
|
||||
-- 001_platform_runtime.sql. Refuse the UUID/JSONB target-design schema instead
|
||||
-- of partially altering it with incompatible TEXT foreign keys.
|
||||
DO $$
|
||||
DECLARE
|
||||
datasets_id_type TEXT;
|
||||
BEGIN
|
||||
SELECT format_type(a.atttypid, a.atttypmod)
|
||||
INTO datasets_id_type
|
||||
FROM pg_attribute a
|
||||
JOIN pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.relname = 'datasets'
|
||||
AND a.attname = 'id'
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped;
|
||||
IF datasets_id_type IS NULL THEN
|
||||
RAISE EXCEPTION '002_data_process.sql requires 001_platform_runtime.sql first';
|
||||
END IF;
|
||||
IF datasets_id_type <> 'text' THEN
|
||||
RAISE EXCEPTION
|
||||
'002_data_process.sql supports only the current TEXT runtime schema; found datasets.id type %',
|
||||
datasets_id_type;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name VARCHAR(150) NOT NULL,
|
||||
description TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
|
||||
process_type VARCHAR(20) NOT NULL
|
||||
CHECK (process_type IN ('structured', 'unstructured', 'external')),
|
||||
source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
config TEXT NOT NULL DEFAULT '{}',
|
||||
progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
|
||||
input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
|
||||
output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
|
||||
filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
|
||||
duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
|
||||
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
|
||||
failure_reason TEXT,
|
||||
generation_run_id TEXT,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
owner_id TEXT,
|
||||
approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
|
||||
created_by TEXT,
|
||||
updated_by TEXT,
|
||||
deleted_by TEXT,
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
|
||||
ON data_process_tasks(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
|
||||
ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
|
||||
WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
|
||||
ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_source_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
storage_object_id TEXT,
|
||||
name TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
file_format VARCHAR(40),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
|
||||
content TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
|
||||
ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
|
||||
ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_preview_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
|
||||
original_content TEXT NOT NULL DEFAULT '',
|
||||
edited_content TEXT NOT NULL DEFAULT '',
|
||||
source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
|
||||
source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
|
||||
source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
|
||||
source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
|
||||
token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'original'
|
||||
CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
|
||||
CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
|
||||
ON data_process_preview_items(task_id, source_file_id, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
instruction TEXT NOT NULL,
|
||||
input TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL,
|
||||
original_instruction TEXT,
|
||||
original_input TEXT,
|
||||
original_output TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
error TEXT,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
||||
ON data_process_results(task_id, status, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
||||
ON data_process_results(task_id, split);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_file_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_no INTEGER NOT NULL CHECK (version_no > 0),
|
||||
storage_object_id TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
description TEXT,
|
||||
base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
|
||||
ON dataset_file_versions(dataset_file_id, version_no);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
|
||||
ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
|
||||
line_no INTEGER,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
instruction TEXT,
|
||||
input TEXT,
|
||||
output TEXT,
|
||||
raw TEXT NOT NULL DEFAULT '{}',
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
|
||||
ON dataset_records(dataset_id, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
|
||||
ON dataset_records(source_task_id, source_result_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
|
||||
ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
|
||||
ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
912
backend/app/modules/data_process/algorithms.py
Normal file
912
backend/app/modules/data_process/algorithms.py
Normal file
@@ -0,0 +1,912 @@
|
||||
"""数据处理模块使用的无副作用算法。
|
||||
|
||||
本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
|
||||
所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
|
||||
``source[chunk.start:chunk.end]``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from bisect import bisect_left
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
TextFormat = Literal["json", "jsonl", "csv", "markdown", "txt"]
|
||||
ChunkMethod = Literal["semantic", "heading", "fixed", "custom"]
|
||||
DatasetSplit = Literal["train", "validation", "test"]
|
||||
|
||||
SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
|
||||
"json",
|
||||
"jsonl",
|
||||
"csv",
|
||||
"markdown",
|
||||
"txt",
|
||||
)
|
||||
|
||||
_FORMAT_ALIASES: dict[str, TextFormat] = {
|
||||
"json": "json",
|
||||
"jsonl": "jsonl",
|
||||
"ndjson": "jsonl",
|
||||
"csv": "csv",
|
||||
"tsv": "csv",
|
||||
"md": "markdown",
|
||||
"markdown": "markdown",
|
||||
"txt": "txt",
|
||||
"text": "txt",
|
||||
}
|
||||
|
||||
_EMAIL_PATTERN = re.compile(
|
||||
r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
|
||||
r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
|
||||
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
|
||||
)
|
||||
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
|
||||
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
|
||||
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
|
||||
_HEADING_PATTERN = re.compile(
|
||||
r"(?m)^(?:#{1,6}\s+|第[一二三四五六七八九十百千万0-9]+[章节篇部分]\s*|"
|
||||
r"\d+(?:\.\d+)*[、.\s]+)"
|
||||
)
|
||||
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedText:
|
||||
"""UTF-8 文本的解析结果。"""
|
||||
|
||||
format: TextFormat
|
||||
text: str
|
||||
records: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TextChunk:
|
||||
"""带有可追溯来源位置的非结构化文本切片。"""
|
||||
|
||||
content: str
|
||||
start: int
|
||||
end: int
|
||||
start_line: int
|
||||
end_line: int
|
||||
token_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QualityScore:
|
||||
"""标准 instruction/input/output 记录的可解释质量分。"""
|
||||
|
||||
overall: float
|
||||
completeness: float
|
||||
length: float
|
||||
readability: float
|
||||
relevance: float
|
||||
duplicate: float
|
||||
is_valid: bool
|
||||
flags: tuple[str, ...]
|
||||
fingerprint: str
|
||||
|
||||
|
||||
def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""严格解码 UTF-8 文本，并移除可选 BOM。
|
||||
|
||||
不使用 ``errors='replace'``，避免上传内容损坏后仍被静默接收。
|
||||
"""
|
||||
|
||||
if isinstance(raw, str):
|
||||
return raw.removeprefix("\ufeff")
|
||||
if not isinstance(raw, (bytes, bytearray, memoryview)):
|
||||
raise TypeError("raw must be bytes-like or str")
|
||||
try:
|
||||
return bytes(raw).decode("utf-8-sig")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc
|
||||
|
||||
|
||||
def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""``decode_utf8`` 的语义化别名，供上传服务直接调用。"""
|
||||
|
||||
return decode_utf8(raw)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""规范 Unicode、换行和行尾空白，同时保留段落结构。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
|
||||
normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
|
||||
normalized = "".join(
|
||||
char
|
||||
for char in normalized
|
||||
if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
|
||||
)
|
||||
lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _normalize_format(value: str | None) -> TextFormat | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip().lower().removeprefix(".")
|
||||
try:
|
||||
return _FORMAT_ALIASES[normalized]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unsupported text format: {value}") from exc
|
||||
|
||||
|
||||
def detect_text_format(
|
||||
*,
|
||||
filename: str | None = None,
|
||||
text: str = "",
|
||||
file_format: str | None = None,
|
||||
) -> TextFormat:
|
||||
"""按显式格式、扩展名和内容特征依次识别文本格式。"""
|
||||
|
||||
explicit = _normalize_format(file_format)
|
||||
if explicit:
|
||||
return explicit
|
||||
|
||||
if filename:
|
||||
suffix = Path(filename).suffix.lower().removeprefix(".")
|
||||
detected = _FORMAT_ALIASES.get(suffix)
|
||||
if detected:
|
||||
return detected
|
||||
|
||||
stripped = text.strip()
|
||||
if stripped:
|
||||
if stripped[0] in "[{":
|
||||
try:
|
||||
json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "json"
|
||||
|
||||
nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
|
||||
if len(nonempty_lines) > 1:
|
||||
try:
|
||||
for line in nonempty_lines:
|
||||
json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "jsonl"
|
||||
|
||||
if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
|
||||
r"(?m)^\s*\|.+\|\s*$", stripped
|
||||
):
|
||||
return "markdown"
|
||||
|
||||
sample = stripped[:8192]
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
|
||||
rows = list(csv.reader(io.StringIO(sample), dialect))
|
||||
if len(rows) >= 2 and len(rows[0]) >= 2:
|
||||
return "csv"
|
||||
except csv.Error:
|
||||
pass
|
||||
|
||||
return "txt"
|
||||
|
||||
|
||||
def _normalize_value(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return normalize_text(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_normalize_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _record_from_value(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return dict(_normalize_value(value))
|
||||
return {"value": _normalize_value(value)}
|
||||
|
||||
|
||||
def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
|
||||
"""从 JSON、JSONL 或 CSV 中提取规范化记录。
|
||||
|
||||
JSON 顶层对象若包含 ``records/data/items/rows`` 数组，则提取该数组；
|
||||
其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
|
||||
"""
|
||||
|
||||
normalized_format = _normalize_format(file_format)
|
||||
if normalized_format not in {"json", "jsonl", "csv"}:
|
||||
raise ValueError("structured record extraction only supports JSON, JSONL and CSV")
|
||||
|
||||
normalized_text = normalize_text(text)
|
||||
if not normalized_text:
|
||||
return []
|
||||
|
||||
if normalized_format == "json":
|
||||
try:
|
||||
payload = json.loads(normalized_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
|
||||
values: Sequence[Any]
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("records", "data", "items", "rows")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
values = [payload]
|
||||
return [_record_from_value(value) for value in values]
|
||||
|
||||
if normalized_format == "jsonl":
|
||||
records: list[dict[str, Any]] = []
|
||||
for line_number, line in enumerate(normalized_text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSONL at line {line_number}, "
|
||||
f"column {exc.colno}: {exc.msg}"
|
||||
) from exc
|
||||
records.append(_record_from_value(value))
|
||||
return records
|
||||
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
|
||||
except csv.Error:
|
||||
dialect = csv.excel
|
||||
reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV header is required")
|
||||
headers = [normalize_text(header or "") for header in reader.fieldnames]
|
||||
if any(not header for header in headers):
|
||||
raise ValueError("CSV header cannot be empty")
|
||||
if len(set(headers)) != len(headers):
|
||||
raise ValueError("CSV headers must be unique")
|
||||
reader.fieldnames = headers
|
||||
|
||||
records = []
|
||||
for row in reader:
|
||||
if None in row:
|
||||
raise ValueError("CSV row has more fields than the header")
|
||||
normalized_row = {
|
||||
key: normalize_text(value or "")
|
||||
for key, value in row.items()
|
||||
}
|
||||
if any(value for value in normalized_row.values()):
|
||||
records.append(normalized_row)
|
||||
return records
|
||||
|
||||
|
||||
def parse_text_content(
|
||||
raw: bytes | bytearray | memoryview | str,
|
||||
*,
|
||||
filename: str | None = None,
|
||||
file_format: str | None = None,
|
||||
) -> ParsedText:
|
||||
"""严格解码并解析支持的 UTF-8 文本格式。"""
|
||||
|
||||
text = normalize_text(decode_utf8(raw))
|
||||
detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
|
||||
records: list[dict[str, Any]] = []
|
||||
if detected_format in {"json", "jsonl", "csv"}:
|
||||
records = extract_structured_records(text, detected_format)
|
||||
return ParsedText(format=detected_format, text=text, records=tuple(records))
|
||||
|
||||
|
||||
def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
|
||||
"""掩码邮箱、中国大陆手机号和 15/18 位身份证号，并返回命中统计。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}
|
||||
|
||||
def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
|
||||
def replacer(_: re.Match[str]) -> str:
|
||||
counts[kind] += 1
|
||||
return replacement
|
||||
|
||||
return pattern.sub(replacer, value)
|
||||
|
||||
masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
|
||||
masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
|
||||
masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)
|
||||
counts["total"] = sum(counts.values())
|
||||
return masked, counts
|
||||
|
||||
|
||||
def estimate_token_count(text: str) -> int:
|
||||
"""无分词器依赖的确定性 token 估算，用于预览与保护性限流。"""
|
||||
|
||||
return len(_TOKEN_PATTERN.findall(text))
|
||||
|
||||
|
||||
def _token_spans(text: str) -> list[tuple[int, int]]:
|
||||
return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
|
||||
|
||||
|
||||
def _line_number(newline_offsets: list[int], offset: int) -> int:
|
||||
# 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
|
||||
return bisect_left(newline_offsets, offset) + 1
|
||||
|
||||
|
||||
def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
|
||||
starts = [span[0] for span in spans]
|
||||
return bisect_left(starts, offset)
|
||||
|
||||
|
||||
def _protected_markdown_ranges(
|
||||
text: str,
|
||||
*,
|
||||
preserve_code_blocks: bool,
|
||||
preserve_tables: bool,
|
||||
preserve_lists: bool,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""
|
||||
|
||||
lines: list[tuple[int, int, str]] = []
|
||||
cursor = 0
|
||||
for raw_line in text.splitlines(keepends=True):
|
||||
end = cursor + len(raw_line)
|
||||
lines.append((cursor, end, raw_line.rstrip("\r\n")))
|
||||
cursor = end
|
||||
if cursor < len(text) or not lines:
|
||||
lines.append((cursor, len(text), text[cursor:]))
|
||||
|
||||
ranges: list[tuple[int, int]] = []
|
||||
code_line_indexes: set[int] = set()
|
||||
if preserve_code_blocks:
|
||||
open_block: tuple[int, str, int] | None = None
|
||||
for index, (start, end, content) in enumerate(lines):
|
||||
fence = re.match(r"^\s*(`{3,}|~{3,})", content)
|
||||
if not fence:
|
||||
continue
|
||||
marker = fence.group(1)[0]
|
||||
length = len(fence.group(1))
|
||||
if open_block is None:
|
||||
open_block = (index, marker, length)
|
||||
continue
|
||||
first_index, open_marker, open_length = open_block
|
||||
if marker == open_marker and length >= open_length:
|
||||
ranges.append((lines[first_index][0], end))
|
||||
code_line_indexes.update(range(first_index, index + 1))
|
||||
open_block = None
|
||||
if open_block is not None:
|
||||
first_index = open_block[0]
|
||||
ranges.append((lines[first_index][0], len(text)))
|
||||
code_line_indexes.update(range(first_index, len(lines)))
|
||||
|
||||
if preserve_tables:
|
||||
index = 0
|
||||
while index + 1 < len(lines):
|
||||
if index in code_line_indexes:
|
||||
index += 1
|
||||
continue
|
||||
header = lines[index][2].strip()
|
||||
separator = lines[index + 1][2].strip().strip("|")
|
||||
cells = [cell.strip() for cell in separator.split("|")]
|
||||
if (
|
||||
"|" not in header
|
||||
or len(cells) < 2
|
||||
or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index + 1
|
||||
while (
|
||||
end_index + 1 < len(lines)
|
||||
and end_index + 1 not in code_line_indexes
|
||||
and lines[end_index + 1][2].strip()
|
||||
and "|" in lines[end_index + 1][2]
|
||||
):
|
||||
end_index += 1
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
if preserve_lists:
|
||||
list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
|
||||
continuation_pattern = re.compile(r"^\s{2,}\S")
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
if index in code_line_indexes or not list_pattern.match(lines[index][2]):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index
|
||||
item_count = 1
|
||||
while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
|
||||
next_line = lines[end_index + 1][2]
|
||||
if list_pattern.match(next_line):
|
||||
item_count += 1
|
||||
end_index += 1
|
||||
elif continuation_pattern.match(next_line):
|
||||
end_index += 1
|
||||
else:
|
||||
break
|
||||
if item_count >= 2:
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
merged: list[tuple[int, int]] = []
|
||||
for start, end in sorted(ranges):
|
||||
if merged and start < merged[-1][1]:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
return merged
|
||||
|
||||
|
||||
def _range_containing(
|
||||
ranges: Sequence[tuple[int, int]], offset: int
|
||||
) -> tuple[int, int] | None:
|
||||
return next((item for item in ranges if item[0] < offset < item[1]), None)
|
||||
|
||||
|
||||
def _boundary_for_method(
|
||||
text: str,
|
||||
spans: list[tuple[int, int]],
|
||||
start_index: int,
|
||||
ideal_end_index: int,
|
||||
minimum_end_index: int,
|
||||
method: ChunkMethod,
|
||||
custom_delimiter: str,
|
||||
) -> tuple[int, int | None]:
|
||||
if method == "fixed":
|
||||
return ideal_end_index, None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
ideal_end_offset = spans[ideal_end_index - 1][1]
|
||||
minimum_end_offset = spans[minimum_end_index - 1][1]
|
||||
search_text = text[start_offset:ideal_end_offset]
|
||||
|
||||
if method == "custom":
|
||||
delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
|
||||
if not delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
relative_minimum = max(0, minimum_end_offset - start_offset)
|
||||
delimiter_start = search_text.rfind(delimiter, relative_minimum)
|
||||
if delimiter_start >= 0:
|
||||
boundary_offset = start_offset + delimiter_start + len(delimiter)
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
if method == "heading":
|
||||
heading_offsets = [
|
||||
start_offset + match.start()
|
||||
for match in _HEADING_PATTERN.finditer(search_text)
|
||||
if start_offset + match.start() >= minimum_end_offset
|
||||
]
|
||||
if heading_offsets:
|
||||
boundary_offset = heading_offsets[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if start_index < boundary_index <= ideal_end_index:
|
||||
return boundary_index, boundary_offset
|
||||
|
||||
semantic_boundaries = [
|
||||
start_offset + match.end()
|
||||
for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(search_text)
|
||||
if start_offset + match.end() >= minimum_end_offset
|
||||
]
|
||||
if semantic_boundaries:
|
||||
boundary_offset = semantic_boundaries[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
|
||||
def chunk_unstructured(
|
||||
text: str,
|
||||
*,
|
||||
method: ChunkMethod = "semantic",
|
||||
chunk_size: int = 800,
|
||||
chunk_overlap: int = 100,
|
||||
min_chunk_size: int = 100,
|
||||
custom_delimiter: str = "",
|
||||
preserve_code_blocks: bool = False,
|
||||
preserve_tables: bool = False,
|
||||
preserve_lists: bool = False,
|
||||
) -> list[TextChunk]:
|
||||
"""按估算 token 切分非结构化文本。
|
||||
|
||||
overlap 足够时精确保留配置数量；短边界下会自动收缩，并且每轮至少推进
|
||||
一个 token，避免异常配置或分隔符造成死循环。
|
||||
"""
|
||||
|
||||
if method not in {"semantic", "heading", "fixed", "custom"}:
|
||||
raise ValueError(f"unsupported chunk method: {method}")
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be greater than 0")
|
||||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if min_chunk_size <= 0 or min_chunk_size > chunk_size:
|
||||
raise ValueError("min_chunk_size must be in [1, chunk_size]")
|
||||
if chunk_overlap + min_chunk_size > chunk_size:
|
||||
raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
|
||||
if method == "custom" and not custom_delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
spans = _token_spans(normalized)
|
||||
if not spans:
|
||||
return []
|
||||
|
||||
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
||||
protected_ranges = _protected_markdown_ranges(
|
||||
normalized,
|
||||
preserve_code_blocks=preserve_code_blocks,
|
||||
preserve_tables=preserve_tables,
|
||||
preserve_lists=preserve_lists,
|
||||
)
|
||||
chunks: list[TextChunk] = []
|
||||
start_index = 0
|
||||
|
||||
while start_index < len(spans):
|
||||
ideal_end_index = min(len(spans), start_index + chunk_size)
|
||||
if ideal_end_index == len(spans):
|
||||
end_index, end_override = ideal_end_index, len(normalized)
|
||||
else:
|
||||
minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
|
||||
end_index, end_override = _boundary_for_method(
|
||||
normalized,
|
||||
spans,
|
||||
start_index,
|
||||
ideal_end_index,
|
||||
minimum_end_index,
|
||||
method,
|
||||
custom_delimiter,
|
||||
)
|
||||
if end_index <= start_index:
|
||||
end_index = min(len(spans), start_index + chunk_size)
|
||||
end_override = None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
end_offset = end_override if end_override is not None else spans[end_index - 1][1]
|
||||
end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
|
||||
split_range = _range_containing(protected_ranges, end_offset)
|
||||
if split_range:
|
||||
before_index = _token_index_at_or_after(spans, split_range[0])
|
||||
if before_index - start_index >= min_chunk_size:
|
||||
end_index = before_index
|
||||
end_offset = split_range[0]
|
||||
else:
|
||||
end_index = min(
|
||||
len(spans),
|
||||
max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
|
||||
)
|
||||
end_offset = split_range[1]
|
||||
content = normalized[start_offset:end_offset]
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
content=content,
|
||||
start=start_offset,
|
||||
end=end_offset,
|
||||
start_line=_line_number(newline_offsets, start_offset),
|
||||
end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
|
||||
token_count=end_index - start_index,
|
||||
)
|
||||
)
|
||||
|
||||
if end_index >= len(spans):
|
||||
break
|
||||
next_start = max(start_index + 1, end_index - chunk_overlap)
|
||||
overlap_range = _range_containing(protected_ranges, spans[next_start][0])
|
||||
if overlap_range:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[0])
|
||||
if candidate <= start_index:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[1])
|
||||
next_start = min(len(spans), max(start_index + 1, candidate))
|
||||
start_index = next_start
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
||||
"""计算与字典键顺序无关的稳定记录指纹。"""
|
||||
|
||||
canonical = {
|
||||
"instruction": normalize_text(str(record.get("instruction") or "")),
|
||||
"input": normalize_text(str(record.get("input") or "")),
|
||||
"output": normalize_text(str(record.get("output") or "")),
|
||||
}
|
||||
raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _readability_score(text: str) -> float:
|
||||
if not text:
|
||||
return 0.0
|
||||
nonspace = [char for char in text if not char.isspace()]
|
||||
if not nonspace:
|
||||
return 0.0
|
||||
printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
|
||||
useful_ratio = sum(
|
||||
char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
|
||||
for char in nonspace
|
||||
) / len(nonspace)
|
||||
return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)
|
||||
|
||||
|
||||
def _internal_duplicate_score(text: str) -> float:
|
||||
units = [unit.strip().lower() for unit in re.split(r"[\n。！？!?；;]+", text) if unit.strip()]
|
||||
if len(units) <= 1:
|
||||
return 100.0
|
||||
return round(100 * len(set(units)) / len(units), 2)
|
||||
|
||||
|
||||
def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
|
||||
"""估算结果与来源文本的词元覆盖率。
|
||||
|
||||
这是无外部模型依赖、可重复的首版评分。没有来源文本（例如人工新增结果）
|
||||
时不扣分；存在来源时，以结果中的有效词元被来源覆盖的比例计分。
|
||||
"""
|
||||
|
||||
source = normalize_text(source_content)
|
||||
if not source:
|
||||
return 100.0
|
||||
candidate = normalize_text(
|
||||
"\n".join(
|
||||
str(record.get(field) or "") for field in ("instruction", "input", "output")
|
||||
)
|
||||
)
|
||||
|
||||
def semantic_tokens(text: str) -> set[str]:
|
||||
return {
|
||||
token.lower()
|
||||
for token in _TOKEN_PATTERN.findall(text)
|
||||
if token.isalnum() or "\u3400" <= token <= "\u9fff"
|
||||
}
|
||||
|
||||
source_tokens = semantic_tokens(source)
|
||||
candidate_tokens = semantic_tokens(candidate)
|
||||
if not candidate_tokens:
|
||||
return 0.0
|
||||
if not source_tokens:
|
||||
return 0.0
|
||||
return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)
|
||||
|
||||
|
||||
def score_quality(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
min_output_length: int = 20,
|
||||
source_content: str = "",
|
||||
known_fingerprints: Iterable[str] = (),
|
||||
threshold: float = 60.0,
|
||||
) -> QualityScore:
|
||||
"""按完整性、长度、可读性、来源相关性和重复度计算质量分。"""
|
||||
|
||||
if min_output_length <= 0:
|
||||
raise ValueError("min_output_length must be greater than 0")
|
||||
if not 0 <= threshold <= 100:
|
||||
raise ValueError("threshold must be in [0, 100]")
|
||||
|
||||
instruction = normalize_text(str(record.get("instruction") or ""))
|
||||
input_text = normalize_text(str(record.get("input") or ""))
|
||||
output = normalize_text(str(record.get("output") or ""))
|
||||
flags: list[str] = []
|
||||
|
||||
completeness = 100.0
|
||||
if not instruction:
|
||||
completeness -= 50
|
||||
flags.append("missing_instruction")
|
||||
if not output:
|
||||
completeness -= 50
|
||||
flags.append("missing_output")
|
||||
|
||||
output_length = len(output)
|
||||
length_score = round(min(100.0, output_length / min_output_length * 100), 2)
|
||||
if output_length < min_output_length:
|
||||
flags.append("output_too_short")
|
||||
|
||||
readability = _readability_score("\n".join((instruction, input_text, output)))
|
||||
if readability < 70:
|
||||
flags.append("low_readability")
|
||||
|
||||
relevance = _source_relevance_score(record, source_content)
|
||||
if source_content and relevance < 30:
|
||||
flags.append("low_source_relevance")
|
||||
|
||||
fingerprint = record_fingerprint(record)
|
||||
known = set(known_fingerprints)
|
||||
duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
|
||||
if duplicate == 0:
|
||||
flags.append("duplicate_record")
|
||||
elif duplicate < 70:
|
||||
flags.append("repetitive_output")
|
||||
|
||||
overall = round(
|
||||
completeness * 0.35
|
||||
+ length_score * 0.20
|
||||
+ readability * 0.20
|
||||
+ relevance * 0.15
|
||||
+ duplicate * 0.10,
|
||||
2,
|
||||
)
|
||||
hard_valid = bool(instruction and output)
|
||||
return QualityScore(
|
||||
overall=overall,
|
||||
completeness=completeness,
|
||||
length=length_score,
|
||||
readability=readability,
|
||||
relevance=relevance,
|
||||
duplicate=duplicate,
|
||||
is_valid=hard_valid and overall >= threshold,
|
||||
flags=tuple(flags),
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def stable_split(
|
||||
value: str | int,
|
||||
split: Mapping[str, int] | None = None,
|
||||
*,
|
||||
seed: str = "",
|
||||
) -> DatasetSplit:
|
||||
"""按稳定哈希将记录划分到 train/validation/test。"""
|
||||
|
||||
ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
|
||||
required = {"train", "validation", "test"}
|
||||
if set(ratios) != required:
|
||||
raise ValueError("split must contain exactly train, validation and test")
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
|
||||
raise ValueError("split ratios must be non-negative integers")
|
||||
if sum(ratios.values()) != 100:
|
||||
raise ValueError("split ratios must sum to 100")
|
||||
|
||||
digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
|
||||
bucket = int.from_bytes(digest[:8], "big") % 10_000
|
||||
train_boundary = ratios["train"] * 100
|
||||
validation_boundary = train_boundary + ratios["validation"] * 100
|
||||
if bucket < train_boundary:
|
||||
return "train"
|
||||
if bucket < validation_boundary:
|
||||
return "validation"
|
||||
return "test"
|
||||
|
||||
|
||||
def _preview_content(item: Mapping[str, Any]) -> str:
|
||||
for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
|
||||
value = item.get(field)
|
||||
if value is not None:
|
||||
return normalize_text(str(value))
|
||||
return ""
|
||||
|
||||
|
||||
def _standard_fields(content: str) -> tuple[str, str, str]:
|
||||
if not content:
|
||||
return "", "", ""
|
||||
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, Mapping):
|
||||
instruction = next(
|
||||
(
|
||||
str(payload[key])
|
||||
for key in ("instruction", "question", "prompt")
|
||||
if payload.get(key) is not None
|
||||
),
|
||||
"",
|
||||
)
|
||||
input_text = next(
|
||||
(str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
output = next(
|
||||
(str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
if instruction or output:
|
||||
return normalize_text(instruction), normalize_text(input_text), normalize_text(output)
|
||||
|
||||
question_answer = re.match(
|
||||
r"^\s*(?:问|question)\s*[:：]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[:：]\s*(.+)\s*$",
|
||||
content,
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if question_answer:
|
||||
return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))
|
||||
|
||||
lines = [line.strip() for line in content.splitlines() if line.strip()]
|
||||
first_line = re.sub(r"^(?:问|question)\s*[:：]\s*", "", lines[0], flags=re.IGNORECASE)
|
||||
output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
|
||||
return normalize_text(first_line), "", output
|
||||
|
||||
|
||||
def generate_standard_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
qa_pairs_per_item: int = 1,
|
||||
semantic_enrichment: bool = False,
|
||||
split: Mapping[str, int] | None = None,
|
||||
split_seed: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""把预览内容确定性转换为标准 instruction/input/output 记录。
|
||||
|
||||
该函数只负责本地标准化，不冒充 LLM；服务层可将其作为无模型模式或
|
||||
LLM 响应解析后的统一落库步骤。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ValueError("qa_pairs_per_item must be in [1, 5]")
|
||||
prefixes = (
|
||||
"请结合实际情况说明：",
|
||||
"请用通俗易懂的方式说明：",
|
||||
"请从实际应用角度说明：",
|
||||
"请简洁自然地说明：",
|
||||
"请详细解答：",
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for item_index, item in enumerate(preview_items):
|
||||
content = _preview_content(item)
|
||||
instruction, input_text, output = _standard_fields(content)
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
for variant_index in range(qa_pairs_per_item):
|
||||
variant_instruction = instruction
|
||||
if variant_index:
|
||||
if semantic_enrichment:
|
||||
variant_instruction = f"{prefixes[variant_index]}{instruction}"
|
||||
else:
|
||||
variant_instruction = f"{instruction}(问法 {variant_index + 1})"
|
||||
raw_id = f"{preview_id}:{variant_index + 1}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
|
||||
status = "valid" if variant_instruction and output else "invalid"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": variant_instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": variant_instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": status,
|
||||
"split": stable_split(result_id, split, seed=split_seed),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChunkMethod",
|
||||
"DatasetSplit",
|
||||
"ParsedText",
|
||||
"QualityScore",
|
||||
"SUPPORTED_TEXT_FORMATS",
|
||||
"TextChunk",
|
||||
"TextFormat",
|
||||
"chunk_unstructured",
|
||||
"decode_utf8",
|
||||
"desensitize_pii",
|
||||
"detect_text_format",
|
||||
"estimate_token_count",
|
||||
"extract_structured_records",
|
||||
"generate_standard_records",
|
||||
"normalize_text",
|
||||
"parse_text_content",
|
||||
"parse_utf8_text",
|
||||
"record_fingerprint",
|
||||
"score_quality",
|
||||
"stable_split",
|
||||
]
|
||||
250
backend/app/modules/data_process/generation.py
Normal file
250
backend/app/modules/data_process/generation.py
Normal file
@@ -0,0 +1,250 @@
|
||||
"""数据处理任务的大模型生成适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from app.modules.data_process.algorithms import normalize_text, stable_split
|
||||
|
||||
|
||||
class ModelGenerationError(ValueError):
|
||||
"""模型配置、响应或调用失败。"""
|
||||
|
||||
|
||||
def chat_completions_url(value: str) -> str:
|
||||
"""把域名、基础 URL 或完整地址统一为 chat completions 地址。"""
|
||||
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
raise ModelGenerationError("generation model api_url is required")
|
||||
if "://" not in raw:
|
||||
raw = f"https://{raw}"
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ModelGenerationError("generation model api_url must be an HTTP(S) host or URL")
|
||||
if parsed.username or parsed.password:
|
||||
raise ModelGenerationError("generation model api_url must not contain credentials")
|
||||
|
||||
path = parsed.path.rstrip("/")
|
||||
if path.endswith("/chat/completions"):
|
||||
target_path = path
|
||||
elif path.endswith("/v1"):
|
||||
target_path = f"{path}/chat/completions"
|
||||
elif not path:
|
||||
target_path = "/v1/chat/completions"
|
||||
else:
|
||||
target_path = f"{path}/v1/chat/completions"
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, target_path, "", ""))
|
||||
|
||||
|
||||
def _message_content(payload: Mapping[str, Any]) -> str:
|
||||
try:
|
||||
content = payload["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ModelGenerationError("model response does not contain choices[0].message.content") from exc
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [
|
||||
str(item.get("text") or "")
|
||||
for item in content
|
||||
if isinstance(item, Mapping) and item.get("type") in {None, "text", "output_text"}
|
||||
]
|
||||
if parts:
|
||||
return "".join(parts)
|
||||
raise ModelGenerationError("model response content must be text")
|
||||
|
||||
|
||||
def _json_payload(content: str) -> Any:
|
||||
cleaned = re.sub(r"<think>[\s\S]*?</think>", "", content, flags=re.IGNORECASE).strip()
|
||||
fenced = re.fullmatch(r"```(?:json)?\s*([\s\S]*?)\s*```", cleaned, flags=re.IGNORECASE)
|
||||
if fenced:
|
||||
cleaned = fenced.group(1).strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ModelGenerationError(
|
||||
f"model response is not valid JSON at line {exc.lineno}, column {exc.colno}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _result_items(payload: Any) -> list[Mapping[str, Any]]:
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("items", "results", "data", "records")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
raise ModelGenerationError("model JSON must be an object or array")
|
||||
items = [item for item in values if isinstance(item, Mapping)]
|
||||
if not items:
|
||||
raise ModelGenerationError("model JSON does not contain result objects")
|
||||
return items
|
||||
|
||||
|
||||
def _prompt_messages(prompt: str, content: str, count: int) -> list[dict[str, str]]:
|
||||
schema_instruction = (
|
||||
f"必须只返回 JSON 对象，格式为 {{\"items\":[{{\"instruction\":\"...\","
|
||||
f"\"input\":\"...\",\"output\":\"...\"}}]}}；items 必须包含 {count} 条。"
|
||||
"instruction 和 output 不得为空，不要输出 Markdown 代码围栏或分析过程。"
|
||||
)
|
||||
base_prompt = (
|
||||
normalize_text(prompt)
|
||||
or "请根据来源内容生成可用于监督微调的问答数据。"
|
||||
)
|
||||
if "{{ content }}" in base_prompt:
|
||||
user_prompt = base_prompt.replace("{{ content }}", content)
|
||||
return [
|
||||
{"role": "system", "content": schema_instruction},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
return [
|
||||
{"role": "system", "content": f"{base_prompt}\n{schema_instruction}"},
|
||||
{"role": "user", "content": f"来源内容：\n{content}"},
|
||||
]
|
||||
|
||||
|
||||
def generate_model_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
model: Mapping[str, Any],
|
||||
config: Mapping[str, Any],
|
||||
task_id: str,
|
||||
split: Mapping[str, int],
|
||||
qa_pairs_per_item: int,
|
||||
client: httpx.Client | None = None,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""调用 OpenAI 兼容接口，将预览切片生成标准训练记录。
|
||||
|
||||
单条调用失败会产生可人工修复的 invalid 结果，不会丢弃整批任务。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ModelGenerationError("qa_pairs_per_item must be in [1, 5]")
|
||||
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
||||
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||
if not model_name:
|
||||
raise ModelGenerationError("generation model name is required")
|
||||
|
||||
temperature = float(config.get("temperature", 0.7))
|
||||
max_tokens = int(config.get("max_tokens", 1024))
|
||||
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60))))
|
||||
retries = max(0, min(5, int(config.get("generation_retries", 2))))
|
||||
headers = {"Content-Type": "application/json"}
|
||||
api_key = str(model.get("api_key") or "").strip()
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
owns_client = client is None
|
||||
http_client = client or httpx.Client(timeout=timeout)
|
||||
results: list[dict[str, Any]] = []
|
||||
try:
|
||||
preview_list = list(preview_items)
|
||||
total_items = len(preview_list)
|
||||
for item_index, item in enumerate(preview_list):
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
content = normalize_text(
|
||||
str(item.get("edited_content") or item.get("original_content") or "")
|
||||
)
|
||||
request_payload: dict[str, Any] = {
|
||||
"model": model_name,
|
||||
"messages": _prompt_messages(
|
||||
str(config.get("generation_prompt") or ""),
|
||||
content,
|
||||
qa_pairs_per_item,
|
||||
),
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if bool(config.get("json_mode", False)):
|
||||
request_payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
last_error: Exception | None = None
|
||||
generated_items: list[Mapping[str, Any]] | None = None
|
||||
for _ in range(retries + 1):
|
||||
try:
|
||||
response = http_client.post(endpoint, headers=headers, json=request_payload)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if not isinstance(body, Mapping):
|
||||
raise ModelGenerationError("model response body must be a JSON object")
|
||||
generated_items = _result_items(_json_payload(_message_content(body)))
|
||||
break
|
||||
except (httpx.HTTPError, json.JSONDecodeError, ModelGenerationError) as exc:
|
||||
last_error = exc
|
||||
|
||||
if generated_items is None:
|
||||
error_message = str(last_error or "model generation failed")[:2000]
|
||||
result_id = f"result_{hashlib.sha256(f'{preview_id}:error'.encode()).hexdigest()[:16]}"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": "模型生成失败，请人工补充",
|
||||
"input": content,
|
||||
"output": "",
|
||||
"original_instruction": "模型生成失败，请人工补充",
|
||||
"original_input": content,
|
||||
"original_output": "",
|
||||
"status": "invalid",
|
||||
"error": error_message,
|
||||
"split": stable_split(result_id, split, seed=task_id),
|
||||
}
|
||||
)
|
||||
if on_progress:
|
||||
on_progress(item_index + 1, total_items)
|
||||
continue
|
||||
|
||||
for variant_index, value in enumerate(generated_items[:qa_pairs_per_item]):
|
||||
instruction = normalize_text(str(value.get("instruction") or value.get("question") or ""))
|
||||
input_text = normalize_text(str(value.get("input") or value.get("context") or ""))
|
||||
output = normalize_text(
|
||||
str(
|
||||
value.get("output")
|
||||
or value.get("answer")
|
||||
or value.get("response")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}"
|
||||
valid = bool(instruction and output)
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": "valid" if valid else "invalid",
|
||||
"error": None if valid else "model result is missing instruction or output",
|
||||
"split": stable_split(result_id, split, seed=task_id),
|
||||
}
|
||||
)
|
||||
if on_progress:
|
||||
on_progress(item_index + 1, total_items)
|
||||
finally:
|
||||
if owns_client:
|
||||
http_client.close()
|
||||
return results
|
||||
|
||||
|
||||
__all__ = ["ModelGenerationError", "chat_completions_url", "generate_model_records"]
|
||||
65
backend/app/modules/data_process/schema_cli.py
Normal file
65
backend/app/modules/data_process/schema_cli.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""数据处理运行表的显式检查与安装命令。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.modules.data_process.store import DataProcessStore
|
||||
|
||||
|
||||
def _target_label(database_url: str) -> str:
|
||||
parsed = urlsplit(database_url)
|
||||
database = parsed.path.strip("/") or "(unknown)"
|
||||
return f"{parsed.hostname or '(unknown)'}:{parsed.port or 5432}/{database}"
|
||||
|
||||
|
||||
def _schema_ready(store: DataProcessStore) -> bool:
|
||||
with store.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema=current_schema()
|
||||
AND table_name='data_process_tasks'
|
||||
AND column_name='generation_run_id'
|
||||
) AS ready
|
||||
"""
|
||||
).fetchone()
|
||||
return bool(row and row["ready"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="检查或显式安装数据处理运行表（不会由应用启动自动执行）"
|
||||
)
|
||||
action = parser.add_mutually_exclusive_group(required=True)
|
||||
action.add_argument("--check", action="store_true", help="只读检查迁移是否已安装")
|
||||
action.add_argument("--apply", action="store_true", help="执行 002 数据处理迁移")
|
||||
parser.add_argument(
|
||||
"--yes",
|
||||
action="store_true",
|
||||
help="确认允许修改 DATABASE_URL 指向的数据库；与 --apply 同时使用",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
store = DataProcessStore()
|
||||
target = _target_label(store.database_url)
|
||||
if args.check:
|
||||
ready = _schema_ready(store)
|
||||
print(f"数据处理 schema：{'已安装' if ready else '未安装'}；目标：{target}")
|
||||
return 0 if ready else 1
|
||||
if not args.yes:
|
||||
parser.error("--apply 必须同时提供 --yes，确认修改目标数据库")
|
||||
|
||||
print(f"正在安装数据处理 schema；目标：{target}")
|
||||
store.ensure_schema()
|
||||
if not _schema_ready(store):
|
||||
raise RuntimeError("迁移执行后仍未检测到 generation_run_id")
|
||||
print("数据处理 schema 安装完成")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1333
backend/app/modules/data_process/store.py
Normal file
1333
backend/app/modules/data_process/store.py
Normal file
File diff suppressed because it is too large
Load Diff
245
backend/app/schemas/data_process.py
Normal file
245
backend/app/schemas/data_process.py
Normal file
@@ -0,0 +1,245 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, default: Any) -> Any:
|
||||
if snake_name in config:
|
||||
return config[snake_name]
|
||||
return config.get(camel_name, default)
|
||||
|
||||
|
||||
def _validate_process_config(config: dict[str, Any]) -> None:
|
||||
split = _config_value(config, "dataset_split", "datasetSplit", None)
|
||||
if split is not None:
|
||||
if not isinstance(split, dict) or set(split) != {"train", "validation", "test"}:
|
||||
raise ValueError("dataset_split must contain train, validation and test")
|
||||
values = list(split.values())
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) for value in values):
|
||||
raise ValueError("dataset_split values must be integers")
|
||||
if any(value < 0 or value > 100 for value in values) or sum(values) != 100:
|
||||
raise ValueError("dataset_split values must be in [0, 100] and total 100")
|
||||
|
||||
chunk_fields = {
|
||||
"chunk_size",
|
||||
"chunkSize",
|
||||
"chunk_overlap",
|
||||
"chunkOverlap",
|
||||
"min_chunk_size",
|
||||
"minChunkSize",
|
||||
}
|
||||
if chunk_fields.intersection(config):
|
||||
chunk_size = _config_value(config, "chunk_size", "chunkSize", 800)
|
||||
overlap = _config_value(config, "chunk_overlap", "chunkOverlap", 100)
|
||||
minimum = _config_value(config, "min_chunk_size", "minChunkSize", 100)
|
||||
if any(
|
||||
isinstance(value, bool) or not isinstance(value, int)
|
||||
for value in (chunk_size, overlap, minimum)
|
||||
):
|
||||
raise ValueError("chunk_size, chunk_overlap and min_chunk_size must be integers")
|
||||
if not 16 <= chunk_size <= 32_768:
|
||||
raise ValueError("chunk_size must be in [16, 32768]")
|
||||
if overlap < 0 or overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if minimum <= 0 or minimum > chunk_size or overlap + minimum > chunk_size:
|
||||
raise ValueError("min_chunk_size and chunk_overlap exceed chunk_size")
|
||||
|
||||
temperature = _config_value(config, "temperature", "temperature", None)
|
||||
if temperature is not None:
|
||||
if isinstance(temperature, bool) or not isinstance(temperature, (int, float)):
|
||||
raise ValueError("temperature must be a number")
|
||||
if not 0 <= float(temperature) <= 2:
|
||||
raise ValueError("temperature must be in [0, 2]")
|
||||
|
||||
max_tokens = _config_value(config, "max_tokens", "maxTokens", None)
|
||||
if max_tokens is not None:
|
||||
if isinstance(max_tokens, bool) or not isinstance(max_tokens, int):
|
||||
raise ValueError("max_tokens must be an integer")
|
||||
if not 1 <= max_tokens <= 32_768:
|
||||
raise ValueError("max_tokens must be in [1, 32768]")
|
||||
|
||||
for snake_name, camel_name in (
|
||||
("qa_pairs_per_row", "qaPairsPerRow"),
|
||||
("qa_pairs_per_chunk", "qaPairsPerChunk"),
|
||||
):
|
||||
pairs = _config_value(config, snake_name, camel_name, None)
|
||||
if pairs is None:
|
||||
continue
|
||||
if isinstance(pairs, bool) or not isinstance(pairs, int) or not 1 <= pairs <= 5:
|
||||
raise ValueError(f"{snake_name} must be an integer in [1, 5]")
|
||||
|
||||
|
||||
class DataProcessStatus(StrEnum):
|
||||
pending = "pending"
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
stopped = "stopped"
|
||||
|
||||
|
||||
class ProcessType(StrEnum):
|
||||
structured = "structured"
|
||||
unstructured = "unstructured"
|
||||
external = "external"
|
||||
|
||||
|
||||
class DataProcessTaskCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str = Field(min_length=1, max_length=150)
|
||||
description: str = ""
|
||||
process_type: ProcessType
|
||||
source_dataset_id: str | None = None
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def normalize_name(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("task name cannot be empty")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_config(self) -> "DataProcessTaskCreate":
|
||||
_validate_process_config(self.config)
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessTaskUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=150)
|
||||
description: str | None = None
|
||||
process_type: ProcessType | None = None
|
||||
source_dataset_id: str | None = None
|
||||
config: dict[str, Any] | None = None
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def normalize_name(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("task name cannot be empty")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_config(self) -> "DataProcessTaskUpdate":
|
||||
if self.config is not None:
|
||||
_validate_process_config(self.config)
|
||||
return self
|
||||
|
||||
|
||||
class PreviewBuildRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
source_file_ids: list[str] | None = None
|
||||
|
||||
|
||||
class PreviewItemCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_file_id: str | None = None
|
||||
original_content: str = ""
|
||||
edited_content: str = ""
|
||||
source_start: int | None = Field(default=None, ge=0)
|
||||
source_end: int | None = Field(default=None, ge=0)
|
||||
source_start_line: int | None = Field(default=None, ge=1)
|
||||
source_end_line: int | None = Field(default=None, ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_ranges(self) -> "PreviewItemCreate":
|
||||
if self.source_start is not None and self.source_end is not None:
|
||||
if self.source_end < self.source_start:
|
||||
raise ValueError("source_end must be greater than or equal to source_start")
|
||||
if self.source_start_line is not None and self.source_end_line is not None:
|
||||
if self.source_end_line < self.source_start_line:
|
||||
raise ValueError(
|
||||
"source_end_line must be greater than or equal to source_start_line"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class PreviewItemUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
edited_content: str
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
|
||||
|
||||
class ExternalSourceRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
type: str = Field(min_length=1, max_length=30)
|
||||
url: str = Field(min_length=1, max_length=2048)
|
||||
auth_mode: Literal["none", "basic"] = "none"
|
||||
username: str | None = Field(default=None, max_length=150)
|
||||
password: str | None = Field(default=None, max_length=500)
|
||||
limit: int = Field(default=1000, ge=1, le=100_000)
|
||||
|
||||
|
||||
class ExternalPullRequest(ExternalSourceRequest):
|
||||
query: str | None = Field(default=None, max_length=20_000)
|
||||
file_name: str = Field(default="external-data.jsonl", min_length=1, max_length=255)
|
||||
|
||||
@field_validator("file_name")
|
||||
@classmethod
|
||||
def validate_file_name(cls, value: str) -> str:
|
||||
name = value.strip()
|
||||
if not name.lower().endswith((".jsonl", ".ndjson")):
|
||||
raise ValueError("external pull file_name must end with .jsonl or .ndjson")
|
||||
return name
|
||||
|
||||
|
||||
class ResultUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
instruction: str | None = None
|
||||
input: str | None = None
|
||||
output: str | None = None
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class DatasetSplit(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
train: int = Field(default=80, ge=0, le=100)
|
||||
validation: int = Field(default=10, ge=0, le=100)
|
||||
test: int = Field(default=10, ge=0, le=100)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_total(self) -> "DatasetSplit":
|
||||
if self.train + self.validation + self.test != 100:
|
||||
raise ValueError("dataset split must total 100")
|
||||
return self
|
||||
|
||||
|
||||
class PublishRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dataset_name: str = Field(min_length=1, max_length=150)
|
||||
dataset_type: Literal["train", "test", "eval", "val", "other"] = "train"
|
||||
storage_type: Literal["local"] = "local"
|
||||
split: DatasetSplit = Field(default_factory=DatasetSplit)
|
||||
format: Literal["alpaca_jsonl", "jsonl"] = "alpaca_jsonl"
|
||||
description: str = ""
|
||||
|
||||
@field_validator("dataset_name")
|
||||
@classmethod
|
||||
def normalize_dataset_name(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("dataset name cannot be empty")
|
||||
return value
|
||||
Reference in New Issue
Block a user