chore: 忽略离线部署包，提交安全加固、数据库初始化与文档
- .gitignore: 忽略 docker/offline 离线部署包（镜像/运行时等大文件） - 安全加固: 新增 compute/api/security.py 及各端安全测试，补充 docs/security-hardening.md - 数据库: 新增完整初始化 SQL 与 docs/database-config.md - 数据转换与评测: 修复类型检查、增强校验并补充测试 - Docker 配置与环境变量更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
@@ -1,6 +1,7 @@
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from pathlib import Path as _Path
|
||||
@@ -29,6 +30,24 @@ def _list_env(name: str, default: list[str]) -> list[str]:
|
||||
return [item.strip() for item in raw.split(",") if item.strip()]
|
||||
|
||||
|
||||
def _bool_env(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def docs_kwargs(enabled: bool) -> dict[str, Any]:
|
||||
"""Swagger UI / ReDoc / OpenAPI schema 路由开关。
|
||||
|
||||
关闭时 FastAPI 不注册 /docs、/redoc、/openapi.json，访问一律返回 404，
|
||||
避免未授权访问泄露 API 结构。
|
||||
"""
|
||||
if enabled:
|
||||
return {}
|
||||
return {"docs_url": None, "redoc_url": None, "openapi_url": None}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
app_name: str = os.getenv("APP_NAME", "YG Fine-Tune Platform API")
|
||||
@@ -48,6 +67,7 @@ class Settings:
|
||||
log_error_file_prefix: str = os.getenv("LOG_ERROR_FILE_PREFIX", "error")
|
||||
log_max_bytes: int = _int_env("LOG_MAX_BYTES", 20 * 1024 * 1024)
|
||||
log_retention_days: int = _int_env("LOG_RETENTION_DAYS", 10)
|
||||
enable_docs: bool = None # type: ignore[assignment]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(
|
||||
@@ -63,6 +83,15 @@ class Settings:
|
||||
],
|
||||
),
|
||||
)
|
||||
# Swagger UI / ReDoc / OpenAPI 文档路由开关：
|
||||
# 未显式配置 ENABLE_DOCS 时，仅本地/开发环境开放，生产环境默认关闭，
|
||||
# 避免未授权访问泄露 API 结构。从运行时环境读取 APP_ENV，而非类定义时
|
||||
# 缓存的默认值，保证生产默认关闭始终生效且便于测试。
|
||||
object.__setattr__(
|
||||
self,
|
||||
"enable_docs",
|
||||
_bool_env("ENABLE_DOCS", os.getenv("APP_ENV", "local") != "prod"),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -239,10 +239,18 @@ def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str
|
||||
fmt = str(formatting).lower()
|
||||
for key, file_name in zip(llama_dataset_keys(dataset_key, file_names), file_names):
|
||||
if fmt == "sharegpt":
|
||||
# 平台校验按 OpenAI 风格消息（role/content），故 tags 用 role/content
|
||||
# 与 LLaMA-Factory 默认的 from/value 不同，需显式声明避免解析失败。
|
||||
result[key] = {
|
||||
"file_name": file_name,
|
||||
"formatting": "sharegpt",
|
||||
"columns": {"messages": "messages"},
|
||||
"tags": {
|
||||
"role_tag": "role",
|
||||
"content_tag": "content",
|
||||
"user_tag": "user",
|
||||
"assistant_tag": "assistant",
|
||||
},
|
||||
}
|
||||
elif fmt == "dpo":
|
||||
result[key] = {
|
||||
@@ -273,6 +281,47 @@ def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str
|
||||
return result
|
||||
|
||||
|
||||
def _sniff_dataset_format(sample_text: str, max_samples: int = 20) -> str:
|
||||
"""嗅探数据集内容格式（兼容 jsonl），返回 sharegpt / dpo / cpt / alpaca。
|
||||
|
||||
按内容而非文件名判断，纯 jsonl 数据集（如 ShareGPT messages、缺省 input 的
|
||||
Alpaca）都能被正确识别，避免训练任务误按 alpaca 解析而失败。
|
||||
"""
|
||||
text = (sample_text or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
records: list[dict[str, Any]] = []
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
value = None
|
||||
if isinstance(value, list):
|
||||
records = [item for item in value[:max_samples] if isinstance(item, dict)]
|
||||
elif isinstance(value, dict):
|
||||
records = [value]
|
||||
else:
|
||||
for line in text.splitlines()[:max_samples]:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
records.append(obj)
|
||||
records = records[:max_samples]
|
||||
if not records:
|
||||
return ""
|
||||
if all("messages" in record for record in records):
|
||||
return "sharegpt"
|
||||
if all(record.get("chosen") and record.get("rejected") for record in records):
|
||||
return "dpo"
|
||||
if all(record.get("text") and not (record.get("instruction") or record.get("output")) for record in records):
|
||||
return "cpt"
|
||||
return "alpaca"
|
||||
|
||||
|
||||
PASSWORD_HASH_ITERATIONS = 390_000
|
||||
|
||||
|
||||
@@ -2076,18 +2125,38 @@ class PlatformStore:
|
||||
runtime_keys = llama_dataset_keys(dataset_key, runtime_file_names)
|
||||
training_keys = runtime_keys[: len(training_files)]
|
||||
validation_keys = runtime_keys[len(training_files) :]
|
||||
dataset_format = str(task.get("dataset_format") or (dataset and dataset.get("formatting")) or "alpaca").lower()
|
||||
# P0-2: Validate dataset content against declared format
|
||||
# P0-2: 推导数据集格式并校验内容（兼容 jsonl：按内容嗅探 ShareGPT/DPO/CPT/Alpaca）
|
||||
train_type = str(task.get("train_type", task.get("train_method", ""))).upper()
|
||||
expected_format = {
|
||||
"DPO": "dpo",
|
||||
"CPT": "cpt",
|
||||
}.get(train_type)
|
||||
expected_format = {"DPO": "dpo", "CPT": "cpt"}.get(train_type)
|
||||
raw_format = str(
|
||||
task.get("dataset_format")
|
||||
or dataset_metadata.get("format")
|
||||
or (dataset and dataset.get("formatting"))
|
||||
or "alpaca"
|
||||
).lower()
|
||||
sniffed_format = ""
|
||||
content_samples: dict[str, str] = {}
|
||||
if training_files:
|
||||
with self.connect() as conn:
|
||||
for file_entry in training_files:
|
||||
sample_row = conn.execute(
|
||||
"SELECT substr(content, 1, 400000) AS sample FROM dataset_files WHERE id=?",
|
||||
(str(file_entry["id"]),),
|
||||
).fetchone()
|
||||
sample = (sample_row or {}).get("sample") or ""
|
||||
content_samples[str(file_entry["id"])] = sample
|
||||
if not sniffed_format:
|
||||
sniffed_format = _sniff_dataset_format(sample)
|
||||
known_formats = {"sharegpt", "dpo", "cpt", "pt", "pretrain"}
|
||||
if expected_format:
|
||||
dataset_format = expected_format
|
||||
elif raw_format in known_formats:
|
||||
dataset_format = raw_format
|
||||
else:
|
||||
dataset_format = sniffed_format or raw_format or "alpaca"
|
||||
format_errors: list[str] = []
|
||||
for file_entry in training_files:
|
||||
content = file_entry.get("content") or ""
|
||||
content = content_samples.get(str(file_entry["id"])) or ""
|
||||
if content:
|
||||
from app.modules.data_process.dataset_format import validate_dataset_format
|
||||
file_errors = validate_dataset_format(dataset_format, content=content)
|
||||
|
||||
755
backend/app/db/sql/000_full_init.sql
Normal file
755
backend/app/db/sql/000_full_init.sql
Normal file
@@ -0,0 +1,755 @@
|
||||
-- ============================================================================
|
||||
-- YG Fine-Tune Platform — PostgreSQL 完整初始化脚本（一键建库建表）
|
||||
-- ============================================================================
|
||||
-- 用途：切换到新的 PG 数据集时，一次性创建平台运行所需的全部数据库对象与
|
||||
-- 基础种子数据（幂等，可重复执行）。
|
||||
--
|
||||
-- 覆盖范围（与运行时代码实际使用的表一致）：
|
||||
-- 001_platform_runtime.sql 平台核心表
|
||||
-- 002_governance.sql 治理表（租户 / 审批 / 审计 / 留存）
|
||||
-- 003_tenant_quota.sql 租户配额列
|
||||
-- 003_model_path_governance.sql 模型可训练标识列
|
||||
-- 002_data_process.sql 数据处理表 + 数据集扩展列
|
||||
-- 本文件补充：data_convert_tasks（数据转换任务，运行时代码引用但原脚本缺失）
|
||||
-- 种子数据：admin / operator 两个初始用户
|
||||
--
|
||||
-- 说明：
|
||||
-- * 本脚本通过 psql 执行，包含 DO $$ ... $$ 块与事务，不能用应用的
|
||||
-- executescript()（按分号切分）执行。
|
||||
-- * 应用启动时 PlatformStore.ensure_schema() 只会自动执行
|
||||
-- 001 / 002_governance / 003_tenant_quota；数据处理表需另跑
|
||||
-- 002_data_process.sql（本脚本已包含）。应用首次启动还会自动补充
|
||||
-- admin/operator 种子用户（本脚本已包含，二选一即可）。
|
||||
-- * 脚本内所有 DDL 均使用 IF NOT EXISTS / ADD COLUMN IF NOT EXISTS，
|
||||
-- 可在已初始化的库上安全重复执行。
|
||||
--
|
||||
-- 执行步骤（详见 docs/database-config.md）：
|
||||
-- 1. 以超级用户创建角色与数据库（必须单独执行，不能放进事务）：
|
||||
-- CREATE ROLE yg_ft LOGIN PASSWORD '请改为强密码';
|
||||
-- CREATE DATABASE yg_ft OWNER yg_ft;
|
||||
-- 2. 连接目标库执行本脚本：
|
||||
-- psql "postgresql://yg_ft:密码@<host>:5432/yg_ft" -f backend/app/db/sql/000_full_init.sql
|
||||
-- 3. 可选：为 superuser 授权
|
||||
-- ALTER ROLE yg_ft SUPERUSER; -- 仅当需要执行 CREATE EXTENSION 等
|
||||
-- ============================================================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ============================================================================
|
||||
-- 一、平台核心表（来源：001_platform_runtime.sql）
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
permissions TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
last_login TEXT,
|
||||
protected INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS models (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
type TEXT NOT NULL,
|
||||
purpose TEXT NOT NULL,
|
||||
model_source TEXT NOT NULL,
|
||||
description TEXT,
|
||||
path TEXT,
|
||||
api_url TEXT,
|
||||
api_key TEXT,
|
||||
online_model_name TEXT,
|
||||
can_train INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trained_models (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
train_methods TEXT NOT NULL,
|
||||
base_model_path TEXT,
|
||||
create_time TEXT NOT NULL,
|
||||
merged INTEGER NOT NULL DEFAULT 0,
|
||||
merging INTEGER NOT NULL DEFAULT 0,
|
||||
merged_path TEXT,
|
||||
artifact_dir TEXT,
|
||||
compute_node_id TEXT,
|
||||
compute_node_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_lineage (
|
||||
id TEXT PRIMARY KEY,
|
||||
child_resource_type TEXT NOT NULL,
|
||||
child_resource_id TEXT NOT NULL,
|
||||
parent_resource_type TEXT NOT NULL,
|
||||
parent_resource_id TEXT NOT NULL,
|
||||
relation_type TEXT NOT NULL,
|
||||
compute_job_id TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_artifacts (
|
||||
id TEXT PRIMARY KEY,
|
||||
model_id TEXT NOT NULL,
|
||||
model_kind TEXT NOT NULL,
|
||||
artifact_type TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
checksum_sha256 TEXT,
|
||||
metadata TEXT NOT NULL,
|
||||
compute_job_id TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_export_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
trained_model_id TEXT,
|
||||
compute_job_id TEXT NOT NULL,
|
||||
node_id TEXT,
|
||||
export_type TEXT NOT NULL,
|
||||
quantization_bit INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL,
|
||||
output_dir TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS datasets (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
type TEXT NOT NULL,
|
||||
storage_type TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
task_id TEXT,
|
||||
size TEXT,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
description TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
size TEXT,
|
||||
content TEXT NOT NULL,
|
||||
active_version_id TEXT NOT NULL,
|
||||
versions TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compute_nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
api_base_url TEXT NOT NULL,
|
||||
file_gateway_url TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
scheduler_status TEXT NOT NULL,
|
||||
scheduler_weight INTEGER NOT NULL DEFAULT 100,
|
||||
tags TEXT NOT NULL,
|
||||
gpu_count INTEGER NOT NULL DEFAULT 0,
|
||||
current_running_jobs INTEGER NOT NULL DEFAULT 0,
|
||||
max_parallel_jobs INTEGER NOT NULL DEFAULT 2,
|
||||
data_root TEXT NOT NULL,
|
||||
model_root TEXT NOT NULL,
|
||||
log_root TEXT NOT NULL,
|
||||
api_version TEXT NOT NULL DEFAULT 'v1',
|
||||
capabilities TEXT NOT NULL DEFAULT '[]',
|
||||
description TEXT,
|
||||
last_health_check_at TEXT,
|
||||
health_detail TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gpus (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER NOT NULL,
|
||||
uuid TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
memory_total_gb DOUBLE PRECISION NOT NULL,
|
||||
power_limit_w DOUBLE PRECISION NOT NULL,
|
||||
base_temperature INTEGER NOT NULL,
|
||||
last_seen_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
process_id INTEGER,
|
||||
create_time TEXT NOT NULL,
|
||||
start_time TEXT,
|
||||
completed_at TEXT,
|
||||
compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
|
||||
gpus TEXT NOT NULL,
|
||||
sync_job_id TEXT,
|
||||
compute_job_id TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
step INTEGER NOT NULL,
|
||||
epoch DOUBLE PRECISION,
|
||||
loss DOUBLE PRECISION,
|
||||
grad_norm DOUBLE PRECISION,
|
||||
learning_rate DOUBLE PRECISION,
|
||||
raw TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
step INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compute_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE SET NULL,
|
||||
node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
|
||||
engine TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
command TEXT NOT NULL,
|
||||
output_dir TEXT,
|
||||
log_file TEXT,
|
||||
payload TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
update_time TEXT NOT NULL,
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gpu_allocations (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
compute_job_id TEXT,
|
||||
node_id TEXT REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
released_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scheduler_locks (
|
||||
lock_key TEXT PRIMARY KEY,
|
||||
owner TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
update_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resource_replicas (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
local_path TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
sync_status TEXT NOT NULL,
|
||||
checksum_sha256 TEXT,
|
||||
byte_size BIGINT NOT NULL DEFAULT 0,
|
||||
last_checked_at TEXT,
|
||||
last_error TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resource_sync_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_node_id TEXT NOT NULL,
|
||||
resources TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL,
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_dimensions (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 1,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compare_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_node_status ON fine_tune_tasks(compute_node_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_lineage_child ON model_lineage(child_resource_type, child_resource_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_lineage_parent ON model_lineage(parent_resource_type, parent_resource_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_artifacts_model ON model_artifacts(model_kind, model_id, artifact_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_export_jobs_model ON model_export_jobs(trained_model_id, create_time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_export_jobs_compute ON model_export_jobs(compute_job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_metrics_task_step ON fine_tune_metrics(task_id, step);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_fine_tune_metrics_task_step_epoch ON fine_tune_metrics(task_id, step, epoch);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_checkpoints_task_step ON fine_tune_checkpoints(task_id, step);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_fine_tune_checkpoints_task_path ON fine_tune_checkpoints(task_id, path);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node_status ON compute_jobs(node_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_allocations_active ON gpu_allocations(node_id, gpu_index) WHERE status IN ('allocated','running');
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpus_node_index ON gpus(node_id, gpu_index);
|
||||
CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_type, resource_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_replicas_node_resource ON resource_replicas(node_id, resource_type, resource_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sync_jobs_node_status ON resource_sync_jobs(target_node_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
|
||||
|
||||
-- ---- 项目 / 租户 ----
|
||||
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||
name TEXT NOT NULL,
|
||||
code TEXT NOT NULL,
|
||||
description TEXT,
|
||||
quota TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
create_time TEXT NOT NULL,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS project_members (
|
||||
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL DEFAULT 'member',
|
||||
create_time TEXT NOT NULL,
|
||||
PRIMARY KEY (project_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
permissions TEXT NOT NULL DEFAULT '[]',
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
issued_at TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
ip TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acls (
|
||||
id TEXT PRIMARY KEY,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
principal_type TEXT NOT NULL,
|
||||
principal_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 二、治理表（来源：002_governance.sql）
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tenants (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
code TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
owner_user_id TEXT,
|
||||
quota TEXT,
|
||||
retention_policy_id TEXT,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_templates (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
steps TEXT,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_instances (
|
||||
id TEXT PRIMARY KEY,
|
||||
template_id TEXT,
|
||||
resource_type TEXT,
|
||||
resource_id TEXT,
|
||||
applicant_id TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
current_step INTEGER DEFAULT 0,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_steps (
|
||||
id TEXT PRIMARY KEY,
|
||||
instance_id TEXT,
|
||||
step_index INTEGER,
|
||||
approver_id TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
comment TEXT,
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
actor_id TEXT,
|
||||
action TEXT,
|
||||
target_type TEXT,
|
||||
target_id TEXT,
|
||||
detail TEXT,
|
||||
client_ip TEXT,
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_logs(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS retention_policies (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
scope TEXT,
|
||||
rule TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
create_time TEXT,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 三、租户配额扩展（来源：003_tenant_quota.sql）
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS gpu_quota TEXT;
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;
|
||||
|
||||
-- ============================================================================
|
||||
-- 四、模型路径治理（来源：003_model_path_governance.sql）
|
||||
-- models.can_train 已在建表语句中声明；以下为兼容旧库的幂等语句。
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
|
||||
|
||||
-- 按规则推定已有模型的 can_train（新库为空表，此语句为 no-op）
|
||||
UPDATE models
|
||||
SET can_train = CASE
|
||||
WHEN path IS NOT NULL AND path != '' AND model_source IS NOT NULL AND model_source != 'api' THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
|
||||
-- ============================================================================
|
||||
-- 五、数据处理（来源：002_data_process.sql，去掉其外层 BEGIN/COMMIT）
|
||||
-- 数据集扩展列
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name VARCHAR(150) NOT NULL,
|
||||
description TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
|
||||
process_type VARCHAR(20) NOT NULL
|
||||
CHECK (process_type IN ('structured', 'unstructured', 'external')),
|
||||
source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
config TEXT NOT NULL DEFAULT '{}',
|
||||
progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
|
||||
input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
|
||||
output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
|
||||
filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
|
||||
duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
|
||||
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
|
||||
failure_reason TEXT,
|
||||
generation_run_id TEXT,
|
||||
results_confirmed BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
workflow_step VARCHAR(20) NOT NULL DEFAULT 'create'
|
||||
CHECK (workflow_step IN ('create', 'model', 'upload', 'preview', 'generate', 'results')),
|
||||
preview_status VARCHAR(20) NOT NULL DEFAULT 'idle'
|
||||
CHECK (preview_status IN ('idle', 'queued', 'running', 'completed', 'failed', 'cancelled')),
|
||||
preview_progress NUMERIC(5,2) NOT NULL DEFAULT 0
|
||||
CHECK (preview_progress >= 0 AND preview_progress <= 100),
|
||||
preview_run_id TEXT,
|
||||
preview_failure_reason TEXT,
|
||||
preview_total_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_total_files >= 0),
|
||||
preview_completed_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_completed_files >= 0),
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
owner_id TEXT,
|
||||
approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
|
||||
created_by TEXT,
|
||||
updated_by TEXT,
|
||||
deleted_by TEXT,
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS results_confirmed BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS workflow_step VARCHAR(20);
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_status VARCHAR(20);
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_progress NUMERIC(5,2);
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_run_id TEXT;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_failure_reason TEXT;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_total_files INTEGER;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_completed_files INTEGER;
|
||||
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET DEFAULT 'create';
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET DEFAULT 'idle';
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET DEFAULT 0;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET DEFAULT 0;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET DEFAULT 0;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET NOT NULL;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
|
||||
ON data_process_tasks(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
|
||||
ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
|
||||
WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
|
||||
ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_source_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
storage_object_id TEXT,
|
||||
name TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
file_format VARCHAR(40),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
|
||||
content TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
|
||||
ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
|
||||
ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_preview_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
|
||||
original_content TEXT NOT NULL DEFAULT '',
|
||||
edited_content TEXT NOT NULL DEFAULT '',
|
||||
source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
|
||||
source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
|
||||
source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
|
||||
source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
|
||||
token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'original'
|
||||
CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
|
||||
CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
|
||||
ON data_process_preview_items(task_id, source_file_id, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
instruction TEXT NOT NULL,
|
||||
input TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL,
|
||||
original_instruction TEXT,
|
||||
original_input TEXT,
|
||||
original_output TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
error TEXT,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
||||
ON data_process_results(task_id, status, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
||||
ON data_process_results(task_id, split);
|
||||
|
||||
-- ---- 数据集版本 / 记录 ----
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_file_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_no INTEGER NOT NULL CHECK (version_no > 0),
|
||||
storage_object_id TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
description TEXT,
|
||||
base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
|
||||
ON dataset_file_versions(dataset_file_id, version_no);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
|
||||
ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
|
||||
line_no INTEGER,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
instruction TEXT,
|
||||
input TEXT,
|
||||
output TEXT,
|
||||
raw TEXT NOT NULL DEFAULT '{}',
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
|
||||
ON dataset_records(dataset_id, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
|
||||
ON dataset_records(source_task_id, source_result_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
|
||||
ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
|
||||
ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
-- ============================================================================
|
||||
-- 六、数据转换任务（data_convert_tasks）
|
||||
-- 运行时 router（app/modules/data_convert/router.py）引用但原脚本缺失，本文件补齐。
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_convert_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
output_filename TEXT DEFAULT 'converted-data.jsonl',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
input_count INTEGER NOT NULL DEFAULT 0,
|
||||
output_count INTEGER NOT NULL DEFAULT 0,
|
||||
error_message TEXT,
|
||||
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- 七、种子数据：初始管理员 / 操作员
|
||||
-- 应用首次启动（ensure_seed_data）也会自动创建；此处提供以便脱离应用直接初始化。
|
||||
-- 密码：admin / admin123，operator / operator123（上线前请改密）。
|
||||
-- ============================================================================
|
||||
|
||||
INSERT INTO users
|
||||
(id, username, password_hash, display_name, role, status, permissions, create_time, protected)
|
||||
VALUES
|
||||
(
|
||||
'u_admin', 'admin', 'pbkdf2_sha256$390000$ygft_init_salt_admin$2b6f31f22968c4f5a30bcf0acf066b7a0f58d4773d15c5ab898ba715ea87b5bd',
|
||||
'Platform Admin', 'admin', 'active',
|
||||
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]',
|
||||
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1
|
||||
),
|
||||
(
|
||||
'u_operator', 'operator', 'pbkdf2_sha256$390000$ygft_init_salt_op$525bf35d02ed26f37952cbd6862b0ae358b9d1a7fa0cbbf0217aa2b5dd544125',
|
||||
'Platform Operator', 'operator', 'active',
|
||||
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]',
|
||||
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0
|
||||
)
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
|
||||
COMMIT;
|
||||
@@ -5,7 +5,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.api.v1.router import api_router
|
||||
from app.core.config import get_settings
|
||||
from app.core.config import docs_kwargs, get_settings
|
||||
from app.core.logging import configure_logging, setup_request_logging
|
||||
from app.workers.compute_poller import run_compute_poller
|
||||
|
||||
@@ -14,7 +14,7 @@ def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
configure_logging(settings)
|
||||
|
||||
app = FastAPI(title=settings.app_name)
|
||||
app = FastAPI(title=settings.app_name, **docs_kwargs(settings.enable_docs))
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_allow_origins,
|
||||
|
||||
@@ -5,10 +5,11 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, UploadFile, File
|
||||
from fastapi import APIRouter, Body, Depends, File, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.core.auth import get_current_user
|
||||
from app.db.platform_store import get_platform_store, new_id
|
||||
|
||||
|
||||
@@ -18,6 +19,30 @@ router = APIRouter(prefix="/data-convert", tags=["data-convert"])
|
||||
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
|
||||
|
||||
|
||||
def _safe_output_filename(value: Any) -> str:
|
||||
"""输出文件名白名单校验：仅允许普通文件名，阻断 ``../``、``/``、``\\`` 等路径穿越。
|
||||
|
||||
转换结果始终写入 ``STORAGE_ROOT/<task_id>/output/<output_filename>``，
|
||||
若文件名可被注入路径分隔符，将导致任意文件读写/删除。
|
||||
"""
|
||||
name = str(value or "converted-data.jsonl").strip()
|
||||
if (
|
||||
not name
|
||||
or name in {".", ".."}
|
||||
or name != Path(name).name
|
||||
or "/" in name
|
||||
or "\\" in name
|
||||
or any(ord(character) < 32 or ord(character) == 127 for character in name)
|
||||
):
|
||||
raise fail(400, "output filename must be a plain file name")
|
||||
return name
|
||||
|
||||
|
||||
def _task_output_path(task: dict[str, Any]) -> Path:
|
||||
"""返回经过白名单校验的转换输出文件路径（始终位于任务 output 目录内）。"""
|
||||
return _output_dir(task["id"]) / _safe_output_filename(task.get("output_filename"))
|
||||
|
||||
|
||||
def _task_dir(task_id: str) -> Path:
|
||||
return STORAGE_ROOT / task_id
|
||||
|
||||
@@ -31,7 +56,11 @@ def _output_dir(task_id: str) -> Path:
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_tasks(page: int = 1, page_size: int = 20) -> dict[str, Any]:
|
||||
def list_tasks(
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
@@ -46,12 +75,15 @@ def list_tasks(page: int = 1, page_size: int = 20) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_task(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
def create_task(
|
||||
payload: dict[str, Any] = Body(...),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
name = str(payload.get("name") or "").strip()
|
||||
if not name:
|
||||
raise fail(400, "name is required")
|
||||
task_id = new_id("dct")
|
||||
output_filename = str(payload.get("output_filename") or "converted-data.jsonl").strip()
|
||||
output_filename = _safe_output_filename(payload.get("output_filename"))
|
||||
description = str(payload.get("description") or "").strip()
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
@@ -67,7 +99,10 @@ def create_task(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/{task_id}")
|
||||
def get_task(task_id: str) -> dict[str, Any]:
|
||||
def get_task(
|
||||
task_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
task = _get_task(task_id)
|
||||
if not task:
|
||||
raise fail(404, "task not found")
|
||||
@@ -86,6 +121,7 @@ def get_task(task_id: str) -> dict[str, Any]:
|
||||
async def upload_source_files(
|
||||
task_id: str,
|
||||
files: list[UploadFile] = File(...),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
task = _get_task(task_id)
|
||||
if not task:
|
||||
@@ -114,7 +150,7 @@ async def upload_source_files(
|
||||
try:
|
||||
output_dir = _output_dir(task_id)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_dir / (task["output_filename"] or "converted-data.jsonl")
|
||||
output_path = _task_output_path(task)
|
||||
# 清空旧输出（如果重新上传）
|
||||
if output_path.exists():
|
||||
output_path.unlink()
|
||||
@@ -157,7 +193,7 @@ async def upload_source_files(
|
||||
})
|
||||
dataset_id = dataset["id"]
|
||||
with store.connect() as conn:
|
||||
store.add_dataset_file(conn, dataset_id, task["output_filename"] or "converted-data.jsonl", content)
|
||||
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
|
||||
return ok({
|
||||
"staged_files": staged,
|
||||
"auto_converted": True,
|
||||
@@ -175,7 +211,10 @@ async def upload_source_files(
|
||||
|
||||
|
||||
@router.post("/{task_id}/run")
|
||||
def run_convert(task_id: str) -> dict[str, Any]:
|
||||
def run_convert(
|
||||
task_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
task = _get_task(task_id)
|
||||
if not task:
|
||||
raise fail(404, "task not found")
|
||||
@@ -192,7 +231,7 @@ def run_convert(task_id: str) -> dict[str, Any]:
|
||||
input_dir = _input_dir(task_id)
|
||||
output_dir = _output_dir(task_id)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_dir / (task["output_filename"] or "converted-data.jsonl")
|
||||
output_path = _task_output_path(task)
|
||||
input_count = 0
|
||||
output_count = 0
|
||||
for json_file in sorted(input_dir.iterdir()):
|
||||
@@ -229,19 +268,22 @@ def run_convert(task_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/{task_id}/download")
|
||||
def download_result(task_id: str):
|
||||
def download_result(
|
||||
task_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
task = _get_task(task_id)
|
||||
if not task:
|
||||
raise fail(404, "task not found")
|
||||
if task["status"] != "completed":
|
||||
raise fail(400, "task is not completed")
|
||||
output_path = _output_dir(task_id) / (task["output_filename"] or "converted-data.jsonl")
|
||||
output_path = _task_output_path(task)
|
||||
if not output_path.exists():
|
||||
raise fail(404, "output file not found")
|
||||
return FileResponse(
|
||||
str(output_path),
|
||||
media_type="application/octet-stream",
|
||||
filename=task["output_filename"] or "converted-data.jsonl",
|
||||
filename=_safe_output_filename(task.get("output_filename")),
|
||||
)
|
||||
|
||||
|
||||
@@ -249,6 +291,7 @@ def download_result(task_id: str):
|
||||
def import_as_dataset(
|
||||
task_id: str,
|
||||
payload: dict[str, Any] = Body(default={}),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
"""把已转换的 JSONL 文件导入为数据集管理中的上传任务记录（source='task'）。"""
|
||||
task = _get_task(task_id)
|
||||
@@ -256,7 +299,7 @@ def import_as_dataset(
|
||||
raise fail(404, "task not found")
|
||||
if task["status"] != "completed":
|
||||
raise fail(400, "task is not completed")
|
||||
output_path = _output_dir(task_id) / (task["output_filename"] or "converted-data.jsonl")
|
||||
output_path = _task_output_path(task)
|
||||
if not output_path.exists():
|
||||
raise fail(404, "output file not found")
|
||||
content = output_path.read_text(encoding="utf-8")
|
||||
@@ -277,12 +320,15 @@ def import_as_dataset(
|
||||
})
|
||||
dataset_id = dataset["id"]
|
||||
with store.connect() as conn:
|
||||
store.add_dataset_file(conn, dataset_id, task["output_filename"] or "converted-data.jsonl", content)
|
||||
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
|
||||
return ok({"dataset_id": dataset_id, "name": dataset_name})
|
||||
|
||||
|
||||
@router.delete("/{task_id}")
|
||||
def delete_task(task_id: str) -> dict[str, Any]:
|
||||
def delete_task(
|
||||
task_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
task = _get_task(task_id)
|
||||
if not task:
|
||||
raise fail(404, "task not found")
|
||||
|
||||
@@ -26,6 +26,16 @@ def _load_sample(path: str | None, content: str | None = None, max_samples: int
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# 先按整文件 JSON（数组/单对象）解析，兼容 .json；失败再按 jsonl 逐行解析
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
value = None
|
||||
if isinstance(value, list):
|
||||
return [item for item in value[:max_samples] if isinstance(item, dict)]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
|
||||
lines = text.splitlines()[:max_samples]
|
||||
records: list[dict[str, Any]] = []
|
||||
for line in lines:
|
||||
|
||||
57
backend/tests/test_data_convert_security.py
Normal file
57
backend/tests/test_data_convert_security.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""data_convert 模块安全回归测试：输出文件名路径穿越与鉴权。
|
||||
|
||||
- ``output_filename`` 必须通过白名单校验，阻断 ``../``、``/``、``\\`` 及控制字符，
|
||||
否则转换结果可被写出到存储根目录之外（任意文件读写/删除）。
|
||||
- 所有 data_convert 路由必须挂载 ``get_current_user`` 鉴权依赖。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.core.auth import get_current_user
|
||||
from app.modules.data_convert.router import _safe_output_filename, router
|
||||
|
||||
|
||||
def test_safe_output_filename_defaults() -> None:
|
||||
assert _safe_output_filename(None) == "converted-data.jsonl"
|
||||
assert _safe_output_filename("") == "converted-data.jsonl"
|
||||
|
||||
|
||||
def test_safe_output_filename_valid() -> None:
|
||||
assert _safe_output_filename("converted-data.jsonl") == "converted-data.jsonl"
|
||||
assert _safe_output_filename("my-data.v1.jsonl") == "my-data.v1.jsonl"
|
||||
assert _safe_output_filename(" 报告.jsonl ") == "报告.jsonl"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad",
|
||||
[
|
||||
"../../etc/passwd",
|
||||
"../x.jsonl",
|
||||
"a/b.jsonl",
|
||||
r"a\b.jsonl",
|
||||
"a\\b.jsonl",
|
||||
"..",
|
||||
".",
|
||||
"x\x00.jsonl",
|
||||
"x\n.jsonl",
|
||||
"x\t.jsonl",
|
||||
],
|
||||
)
|
||||
def test_safe_output_filename_rejects_traversal(bad: str) -> None:
|
||||
with pytest.raises(HTTPException):
|
||||
_safe_output_filename(bad)
|
||||
|
||||
|
||||
def test_all_data_convert_routes_require_auth() -> None:
|
||||
for route in router.routes:
|
||||
node = getattr(route, "dependant", None)
|
||||
assert node is not None, f"route {route.path} has no dependency graph"
|
||||
stack = list(node.dependencies)
|
||||
calls: list = []
|
||||
while stack:
|
||||
dep = stack.pop()
|
||||
stack.extend(getattr(dep, "dependencies", []))
|
||||
calls.append(getattr(dep, "call", None))
|
||||
assert get_current_user in calls, f"route {route.path} is missing get_current_user auth"
|
||||
76
backend/tests/test_docs_security.py
Normal file
76
backend/tests/test_docs_security.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""Swagger / ReDoc / OpenAPI 文档路由安全开关测试。
|
||||
|
||||
生产环境（APP_ENV=prod）默认关闭 /docs、/redoc、/openapi.json，
|
||||
避免未授权访问泄露 API 结构；本地开发环境默认开放，可用 ENABLE_DOCS 覆盖。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import docs_kwargs, get_settings
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_settings_cache():
|
||||
"""每次测试前后清空 get_settings 的 lru_cache，避免环境变量互相污染。"""
|
||||
get_settings.cache_clear()
|
||||
yield
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_docs_kwargs_enabled() -> None:
|
||||
assert docs_kwargs(True) == {}
|
||||
|
||||
|
||||
def test_docs_kwargs_disabled() -> None:
|
||||
assert docs_kwargs(False) == {"docs_url": None, "redoc_url": None, "openapi_url": None}
|
||||
|
||||
|
||||
def test_docs_disabled_by_default_in_prod(monkeypatch) -> None:
|
||||
monkeypatch.delenv("ENABLE_DOCS", raising=False)
|
||||
monkeypatch.setenv("APP_ENV", "prod")
|
||||
assert get_settings().enable_docs is False
|
||||
|
||||
|
||||
def test_docs_enabled_by_default_outside_prod(monkeypatch) -> None:
|
||||
monkeypatch.delenv("ENABLE_DOCS", raising=False)
|
||||
monkeypatch.setenv("APP_ENV", "local")
|
||||
assert get_settings().enable_docs is True
|
||||
|
||||
|
||||
def test_docs_env_override_enables_in_prod(monkeypatch) -> None:
|
||||
monkeypatch.setenv("ENABLE_DOCS", "true")
|
||||
monkeypatch.setenv("APP_ENV", "prod")
|
||||
assert get_settings().enable_docs is True
|
||||
|
||||
|
||||
def test_docs_env_override_disables_outside_prod(monkeypatch) -> None:
|
||||
monkeypatch.setenv("ENABLE_DOCS", "false")
|
||||
monkeypatch.setenv("APP_ENV", "local")
|
||||
assert get_settings().enable_docs is False
|
||||
|
||||
|
||||
def test_create_app_disables_docs_in_prod(monkeypatch, tmp_path) -> None:
|
||||
pytest.importorskip("fastapi")
|
||||
from app.main import create_app
|
||||
|
||||
monkeypatch.delenv("ENABLE_DOCS", raising=False)
|
||||
monkeypatch.setenv("APP_ENV", "prod")
|
||||
monkeypatch.setenv("LOG_DIR", str(tmp_path))
|
||||
app = create_app()
|
||||
assert app.docs_url is None
|
||||
assert app.redoc_url is None
|
||||
assert app.openapi_url is None
|
||||
|
||||
|
||||
def test_create_app_enables_docs_outside_prod(monkeypatch, tmp_path) -> None:
|
||||
pytest.importorskip("fastapi")
|
||||
from app.main import create_app
|
||||
|
||||
monkeypatch.delenv("ENABLE_DOCS", raising=False)
|
||||
monkeypatch.setenv("APP_ENV", "local")
|
||||
monkeypatch.setenv("LOG_DIR", str(tmp_path))
|
||||
app = create_app()
|
||||
assert app.docs_url == "/docs"
|
||||
assert app.redoc_url == "/redoc"
|
||||
assert app.openapi_url == "/openapi.json"
|
||||
Reference in New Issue
Block a user