第一次提交
This commit is contained in:
wangjiming
1
backend/app/db/__init__.py
Normal file
1
backend/app/db/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Database infrastructure package."""
|
||||
1946
backend/app/db/platform_store.py
Normal file
1946
backend/app/db/platform_store.py
Normal file
File diff suppressed because it is too large
Load Diff
40
backend/app/db/session.py
Normal file
40
backend/app/db/session.py
Normal file
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+psycopg://yg_ft:change_me@localhost:15432/yg_ft")
|
||||
|
||||
engine = create_engine(
|
||||
DATABASE_URL,
|
||||
pool_pre_ping=True,
|
||||
future=True,
|
||||
)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False, future=True)
|
||||
|
||||
|
||||
def get_db() -> Generator[Session, None, None]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Generator[Session, None, None]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
214
backend/app/db/sql/001_platform_runtime.sql
Normal file
214
backend/app/db/sql/001_platform_runtime.sql
Normal file
@@ -0,0 +1,214 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
permissions TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL,
|
||||
last_login TEXT,
|
||||
protected INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS models (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
type TEXT NOT NULL,
|
||||
purpose TEXT NOT NULL,
|
||||
model_source TEXT NOT NULL,
|
||||
description TEXT,
|
||||
path TEXT,
|
||||
api_url TEXT,
|
||||
api_key TEXT,
|
||||
online_model_name TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trained_models (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
train_methods TEXT NOT NULL,
|
||||
base_model_path TEXT,
|
||||
create_time TEXT NOT NULL,
|
||||
merged INTEGER NOT NULL DEFAULT 0,
|
||||
merging INTEGER NOT NULL DEFAULT 0,
|
||||
merged_path TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS datasets (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
type TEXT NOT NULL,
|
||||
storage_type TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
task_id TEXT,
|
||||
size TEXT,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
description TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
size TEXT,
|
||||
content TEXT NOT NULL,
|
||||
active_version_id TEXT NOT NULL,
|
||||
versions TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compute_nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
api_base_url TEXT NOT NULL,
|
||||
file_gateway_url TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
scheduler_status TEXT NOT NULL,
|
||||
scheduler_weight INTEGER NOT NULL DEFAULT 100,
|
||||
tags TEXT NOT NULL,
|
||||
gpu_count INTEGER NOT NULL DEFAULT 0,
|
||||
current_running_jobs INTEGER NOT NULL DEFAULT 0,
|
||||
max_parallel_jobs INTEGER NOT NULL DEFAULT 2,
|
||||
data_root TEXT NOT NULL,
|
||||
model_root TEXT NOT NULL,
|
||||
log_root TEXT NOT NULL,
|
||||
last_health_check_at TEXT,
|
||||
health_detail TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gpus (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER NOT NULL,
|
||||
uuid TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
memory_total_gb DOUBLE PRECISION NOT NULL,
|
||||
power_limit_w DOUBLE PRECISION NOT NULL,
|
||||
base_temperature INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
process_id INTEGER,
|
||||
create_time TEXT NOT NULL,
|
||||
start_time TEXT,
|
||||
completed_at TEXT,
|
||||
compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
|
||||
gpus TEXT NOT NULL,
|
||||
sync_job_id TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resource_replicas (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
local_path TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
sync_status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resource_sync_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_node_id TEXT NOT NULL,
|
||||
resources TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL,
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
-- ===================== Project / Tenant =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||
name TEXT NOT NULL,
|
||||
code TEXT NOT NULL,
|
||||
description TEXT,
|
||||
quota TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
create_time TEXT NOT NULL,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS project_members (
|
||||
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL DEFAULT 'member',
|
||||
create_time TEXT NOT NULL,
|
||||
PRIMARY KEY (project_id, user_id)
|
||||
);
|
||||
|
||||
-- ===================== Fine-tune Checkpoints =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
step INTEGER NOT NULL DEFAULT 0,
|
||||
loss DOUBLE PRECISION,
|
||||
is_best INTEGER NOT NULL DEFAULT 0,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- ===================== Compute Jobs (internal) =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compute_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
|
||||
node_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL DEFAULT 'train',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
command TEXT,
|
||||
gpu_count INTEGER NOT NULL DEFAULT 1,
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
timeout_seconds INTEGER,
|
||||
progress REAL DEFAULT 0,
|
||||
result TEXT,
|
||||
error_message TEXT,
|
||||
create_time TEXT NOT NULL,
|
||||
start_time TEXT,
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
-- ===================== Indexes =====================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_type, resource_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_projects_tenant ON projects(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_project_members_user ON project_members(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_checkpoints_task ON fine_tune_checkpoints(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node ON compute_jobs(node_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_status ON compute_jobs(status);
|
||||
|
||||
-- ===================== Migrations: extend fine_tune_tasks =====================
|
||||
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS dataset_name TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS model_name TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS trained_model_name TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS checkpoint_retention_policy TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS error_message TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS retry_count INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS last_retry_at TEXT;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS resumed_from_checkpoint_id TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_project ON fine_tune_tasks(project_id);
|
||||
127
backend/app/db/sql/002_governance.sql
Normal file
127
backend/app/db/sql/002_governance.sql
Normal file
@@ -0,0 +1,127 @@
|
||||
-- A. 平台基础与企业治理：权限 / 角色 / 租户 / 审批 / 审计 / 留存
|
||||
-- 沿用 001 的约定：时间戳存 TEXT，布尔用 INTEGER，列表用 TEXT(JSON)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS permissions (
|
||||
code TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
group_name TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
permissions TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS role_permissions (
|
||||
role_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
PRIMARY KEY (role_id, permission)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_permission_overrides (
|
||||
user_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
granted INTEGER NOT NULL,
|
||||
PRIMARY KEY (user_id, permission)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tenants (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL,
|
||||
owner_user_id TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tenant_users (
|
||||
tenant_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
PRIMARY KEY (tenant_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resource_acl (
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
principal_type TEXT NOT NULL,
|
||||
principal_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
granted INTEGER NOT NULL,
|
||||
PRIMARY KEY (resource_type, resource_id, principal_type, principal_id, permission)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_templates (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
steps TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_instances (
|
||||
id TEXT PRIMARY KEY,
|
||||
template_id TEXT,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
applicant_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
current_step INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_steps (
|
||||
instance_id TEXT NOT NULL,
|
||||
step_index INTEGER NOT NULL,
|
||||
approver_id TEXT,
|
||||
status TEXT NOT NULL,
|
||||
comment TEXT,
|
||||
time TEXT,
|
||||
PRIMARY KEY (instance_id, step_index)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
actor_id TEXT,
|
||||
action TEXT NOT NULL,
|
||||
target_type TEXT,
|
||||
target_id TEXT,
|
||||
detail TEXT,
|
||||
client_ip TEXT,
|
||||
time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS retention_policies (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
resource_type TEXT NOT NULL,
|
||||
retention_days INTEGER NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- ===================== 种子数据 =====================
|
||||
INSERT INTO permissions (code, name, group_name, create_time) VALUES
|
||||
('dashboard', '仪表盘', '概览', '2026-01-01T00:00:00Z'),
|
||||
('fine-tune', '模型微调', '模型', '2026-01-01T00:00:00Z'),
|
||||
('model-eval', '模型评估', '模型', '2026-01-01T00:00:00Z'),
|
||||
('model-inference', '模型推理', '模型', '2026-01-01T00:00:00Z'),
|
||||
('model-manage', '模型管理', '模型', '2026-01-01T00:00:00Z'),
|
||||
('dataset', '数据集', '数据', '2026-01-01T00:00:00Z'),
|
||||
('data-process', '数据处理', '数据', '2026-01-01T00:00:00Z'),
|
||||
('data-convert', '数据转换', '数据', '2026-01-01T00:00:00Z'),
|
||||
('compute', '算力管理', '算力', '2026-01-01T00:00:00Z'),
|
||||
('hardware', '硬件监控', '算力', '2026-01-01T00:00:00Z'),
|
||||
('logs', '日志查看', '运维', '2026-01-01T00:00:00Z'),
|
||||
('user-settings', '用户设置', '运维', '2026-01-01T00:00:00Z')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
|
||||
INSERT INTO roles (id, name, display_name, permissions, create_time) VALUES
|
||||
('role_admin', 'admin', '管理员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]', '2026-01-01T00:00:00Z'),
|
||||
('role_operator','operator','操作员', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', '2026-01-01T00:00:00Z'),
|
||||
('role_viewer', 'viewer', '访客', '["dashboard"]', '2026-01-01T00:00:00Z')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
17
backend/app/db/sql/003_tenant_quota.sql
Normal file
17
backend/app/db/sql/003_tenant_quota.sql
Normal file
@@ -0,0 +1,17 @@
|
||||
-- 003: tenants 补充 quota / retention_policy 列（接口契约 13.1 需要）
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'tenants' AND column_name = 'quota'
|
||||
) THEN
|
||||
ALTER TABLE tenants ADD COLUMN quota TEXT;
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'tenants' AND column_name = 'retention_policy_id'
|
||||
) THEN
|
||||
ALTER TABLE tenants ADD COLUMN retention_policy_id TEXT;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
Reference in New Issue
Block a user