YG_FT/backend/app/db/sql/000_full_init.sql

-- ============================================================================
-- YG Fine-Tune Platform — PostgreSQL 完整初始化脚本（一键建库建表）
-- ============================================================================
-- 用途：切换到新的 PG 数据集时，一次性创建平台运行所需的全部数据库对象与
--       基础种子数据（幂等，可重复执行）。
--
-- 覆盖范围（与运行时代码实际使用的表一致）：
--   001_platform_runtime.sql       平台核心表
--   002_governance.sql             治理表（租户 / 审批 / 审计 / 留存）
--   003_tenant_quota.sql           租户配额列
--   003_model_path_governance.sql  模型可训练标识列
--   002_data_process.sql           数据处理表 + 数据集扩展列
--   本文件补充：data_convert_tasks（数据转换任务，运行时代码引用但原脚本缺失）
--   种子数据：admin / operator 两个初始用户
--
-- 说明：
--   * 本脚本通过 psql 执行，包含 DO $$ ... $$ 块与事务，不能用应用的
--     executescript()（按分号切分）执行。
--   * 应用启动时 PlatformStore.ensure_schema() 只会自动执行
--     001 / 002_governance / 003_tenant_quota；数据处理表需另跑
--     002_data_process.sql（本脚本已包含）。应用首次启动还会自动补充
--     admin/operator 种子用户（本脚本已包含，二选一即可）。
--   * 脚本内所有 DDL 均使用 IF NOT EXISTS / ADD COLUMN IF NOT EXISTS，
--     可在已初始化的库上安全重复执行。
--
-- 执行步骤（详见 docs/database-config.md）：
--   1. 以超级用户创建角色与数据库（必须单独执行，不能放进事务）：
--        CREATE ROLE  yg_ft  LOGIN PASSWORD '请改为强密码';
--        CREATE DATABASE yg_ft OWNER yg_ft;
--   2. 连接目标库执行本脚本：
--        psql "postgresql://yg_ft:密码@<host>:5432/yg_ft" -f backend/app/db/sql/000_full_init.sql
--   3. 可选：为 superuser 授权
--        ALTER ROLE yg_ft SUPERUSER;   -- 仅当需要执行 CREATE EXTENSION 等
-- ============================================================================

BEGIN;

-- ============================================================================
-- 一、平台核心表（来源：001_platform_runtime.sql）
-- ============================================================================

CREATE TABLE IF NOT EXISTS users (
    id TEXT PRIMARY KEY,
    username TEXT NOT NULL UNIQUE,
    password_hash TEXT NOT NULL,
    display_name TEXT NOT NULL,
    role TEXT NOT NULL,
    status TEXT NOT NULL,
    permissions TEXT NOT NULL,
    create_time TEXT NOT NULL,
    last_login TEXT,
    protected INTEGER NOT NULL DEFAULT 0
);

CREATE TABLE IF NOT EXISTS models (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL UNIQUE,
    type TEXT NOT NULL,
    purpose TEXT NOT NULL,
    model_source TEXT NOT NULL,
    description TEXT,
    path TEXT,
    api_url TEXT,
    api_key TEXT,
    online_model_name TEXT,
    can_train INTEGER NOT NULL DEFAULT 0,
    create_time TEXT NOT NULL,
    created_by TEXT,
    tenant_id TEXT,
    project_id TEXT,
    deleted_at TEXT,
    deleted_by TEXT
);

CREATE TABLE IF NOT EXISTS trained_models (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL UNIQUE,
    train_methods TEXT NOT NULL,
    base_model_path TEXT,
    create_time TEXT NOT NULL,
    merged INTEGER NOT NULL DEFAULT 0,
    merging INTEGER NOT NULL DEFAULT 0,
    merged_path TEXT,
    artifact_dir TEXT,
    compute_node_id TEXT,
    compute_node_name TEXT,
    created_by TEXT,
    tenant_id TEXT,
    project_id TEXT,
    deleted_at TEXT,
    deleted_by TEXT
);

CREATE TABLE IF NOT EXISTS model_lineage (
    id TEXT PRIMARY KEY,
    child_resource_type TEXT NOT NULL,
    child_resource_id TEXT NOT NULL,
    parent_resource_type TEXT NOT NULL,
    parent_resource_id TEXT NOT NULL,
    relation_type TEXT NOT NULL,
    compute_job_id TEXT,
    payload TEXT NOT NULL,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS model_artifacts (
    id TEXT PRIMARY KEY,
    model_id TEXT NOT NULL,
    model_kind TEXT NOT NULL,
    artifact_type TEXT NOT NULL,
    path TEXT NOT NULL,
    size_bytes BIGINT NOT NULL DEFAULT 0,
    checksum_sha256 TEXT,
    storage_object_id TEXT,
    storage_backend TEXT NOT NULL DEFAULT 'minio',
    metadata TEXT NOT NULL,
    compute_job_id TEXT,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS model_export_jobs (
    id TEXT PRIMARY KEY,
    trained_model_id TEXT,
    compute_job_id TEXT NOT NULL,
    node_id TEXT,
    export_type TEXT NOT NULL,
    quantization_bit INTEGER NOT NULL DEFAULT 0,
    status TEXT NOT NULL,
    output_dir TEXT,
    payload TEXT NOT NULL,
    create_time TEXT NOT NULL,
    completed_at TEXT
);

CREATE TABLE IF NOT EXISTS datasets (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL UNIQUE,
    type TEXT NOT NULL,
    storage_type TEXT NOT NULL,
    source TEXT NOT NULL,
    task_id TEXT,
    size TEXT,
    count INTEGER NOT NULL DEFAULT 0,
    description TEXT,
    create_time TEXT NOT NULL,
    created_by TEXT,
    tenant_id TEXT,
    project_id TEXT,
    deleted_at TEXT,
    deleted_by TEXT
);

CREATE TABLE IF NOT EXISTS dataset_files (
    id TEXT PRIMARY KEY,
    dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
    name TEXT NOT NULL,
    size TEXT,
    content TEXT NOT NULL,
    active_version_id TEXT NOT NULL,
    versions TEXT NOT NULL,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS compute_nodes (
    id TEXT PRIMARY KEY,
    code TEXT NOT NULL UNIQUE,
    name TEXT NOT NULL,
    api_base_url TEXT NOT NULL,
    file_gateway_url TEXT NOT NULL,
    enabled INTEGER NOT NULL DEFAULT 1,
    scheduler_status TEXT NOT NULL,
    scheduler_weight INTEGER NOT NULL DEFAULT 100,
    tags TEXT NOT NULL,
    gpu_count INTEGER NOT NULL DEFAULT 0,
    current_running_jobs INTEGER NOT NULL DEFAULT 0,
    max_parallel_jobs INTEGER NOT NULL DEFAULT 2,
    data_root TEXT NOT NULL,
    model_root TEXT NOT NULL,
    log_root TEXT NOT NULL,
    api_version TEXT NOT NULL DEFAULT 'v1',
    capabilities TEXT NOT NULL DEFAULT '[]',
    description TEXT,
    last_health_check_at TEXT,
    health_detail TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS gpus (
    id TEXT PRIMARY KEY,
    node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
    gpu_index INTEGER NOT NULL,
    uuid TEXT NOT NULL,
    name TEXT NOT NULL,
    memory_total_gb DOUBLE PRECISION NOT NULL,
    power_limit_w DOUBLE PRECISION NOT NULL,
    base_temperature INTEGER NOT NULL,
    last_seen_at TEXT
);

CREATE TABLE IF NOT EXISTS fine_tune_tasks (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL UNIQUE,
    payload TEXT NOT NULL,
    status TEXT NOT NULL,
    progress INTEGER NOT NULL DEFAULT 0,
    process_id INTEGER,
    create_time TEXT NOT NULL,
    start_time TEXT,
    completed_at TEXT,
    compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
    gpus TEXT NOT NULL,
    sync_job_id TEXT,
    compute_job_id TEXT
);

CREATE TABLE IF NOT EXISTS fine_tune_metrics (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
    step INTEGER NOT NULL,
    epoch DOUBLE PRECISION,
    loss DOUBLE PRECISION,
    grad_norm DOUBLE PRECISION,
    learning_rate DOUBLE PRECISION,
    raw TEXT NOT NULL,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS fine_tune_checkpoints (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
    step INTEGER NOT NULL,
    name TEXT NOT NULL,
    path TEXT NOT NULL,
    size_bytes BIGINT NOT NULL DEFAULT 0,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS compute_jobs (
    id TEXT PRIMARY KEY,
    task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE SET NULL,
    node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
    engine TEXT NOT NULL,
    status TEXT NOT NULL,
    command TEXT NOT NULL,
    output_dir TEXT,
    log_file TEXT,
    payload TEXT NOT NULL,
    create_time TEXT NOT NULL,
    update_time TEXT NOT NULL,
    completed_at TEXT
);

CREATE TABLE IF NOT EXISTS gpu_allocations (
    id TEXT PRIMARY KEY,
    task_id TEXT REFERENCES fine_tune_tasks(id) ON DELETE CASCADE,
    compute_job_id TEXT,
    node_id TEXT REFERENCES compute_nodes(id) ON DELETE CASCADE,
    gpu_index INTEGER NOT NULL,
    status TEXT NOT NULL,
    create_time TEXT NOT NULL,
    released_at TEXT
);

CREATE TABLE IF NOT EXISTS scheduler_locks (
    lock_key TEXT PRIMARY KEY,
    owner TEXT NOT NULL,
    expires_at TEXT NOT NULL,
    create_time TEXT NOT NULL,
    update_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS resource_replicas (
    id TEXT PRIMARY KEY,
    node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
    resource_type TEXT NOT NULL,
    resource_id TEXT NOT NULL,
    local_path TEXT NOT NULL,
    status TEXT NOT NULL,
    sync_status TEXT NOT NULL,
    checksum_sha256 TEXT,
    byte_size BIGINT NOT NULL DEFAULT 0,
    last_checked_at TEXT,
    last_error TEXT,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS resource_sync_jobs (
    id TEXT PRIMARY KEY,
    target_node_id TEXT NOT NULL,
    resources TEXT NOT NULL,
    status TEXT NOT NULL,
    progress INTEGER NOT NULL DEFAULT 0,
    create_time TEXT NOT NULL,
    completed_at TEXT
);

CREATE TABLE IF NOT EXISTS storage_objects (
    id TEXT PRIMARY KEY,
    resource_type TEXT NOT NULL,
    resource_id TEXT NOT NULL,
    version_id TEXT NOT NULL,
    bucket TEXT NOT NULL,
    object_key TEXT NOT NULL,
    file_name TEXT,
    content_type TEXT,
    checksum_sha256 TEXT,
    byte_size BIGINT NOT NULL DEFAULT 0,
    status TEXT NOT NULL DEFAULT 'pending',
    metadata TEXT NOT NULL DEFAULT '{}',
    created_by TEXT,
    create_time TEXT NOT NULL,
    UNIQUE (resource_type, resource_id, version_id, object_key)
);

CREATE TABLE IF NOT EXISTS storage_cache_jobs (
    id TEXT PRIMARY KEY,
    storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
    node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
    direction TEXT NOT NULL,
    status TEXT NOT NULL,
    progress INTEGER NOT NULL DEFAULT 0,
    local_path TEXT,
    error TEXT,
    create_time TEXT NOT NULL,
    completed_at TEXT
);

CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);

CREATE TABLE IF NOT EXISTS eval_tasks (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    payload TEXT NOT NULL,
    status TEXT NOT NULL,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS eval_dimensions (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    payload TEXT NOT NULL,
    is_active INTEGER NOT NULL DEFAULT 1,
    is_default INTEGER NOT NULL DEFAULT 0,
    create_time TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS compare_tasks (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    payload TEXT NOT NULL,
    status TEXT NOT NULL,
    create_time TEXT NOT NULL
);

CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id);
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_node_status ON fine_tune_tasks(compute_node_id, status);
CREATE INDEX IF NOT EXISTS idx_model_lineage_child ON model_lineage(child_resource_type, child_resource_id);
CREATE INDEX IF NOT EXISTS idx_model_lineage_parent ON model_lineage(parent_resource_type, parent_resource_id);
CREATE INDEX IF NOT EXISTS idx_model_artifacts_model ON model_artifacts(model_kind, model_id, artifact_type);
CREATE INDEX IF NOT EXISTS idx_model_export_jobs_model ON model_export_jobs(trained_model_id, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_model_export_jobs_compute ON model_export_jobs(compute_job_id);
CREATE INDEX IF NOT EXISTS idx_fine_tune_metrics_task_step ON fine_tune_metrics(task_id, step);
CREATE UNIQUE INDEX IF NOT EXISTS uq_fine_tune_metrics_task_step_epoch ON fine_tune_metrics(task_id, step, epoch);
CREATE INDEX IF NOT EXISTS idx_fine_tune_checkpoints_task_step ON fine_tune_checkpoints(task_id, step);
CREATE UNIQUE INDEX IF NOT EXISTS uq_fine_tune_checkpoints_task_path ON fine_tune_checkpoints(task_id, path);
CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node_status ON compute_jobs(node_id, status);
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_allocations_active ON gpu_allocations(node_id, gpu_index) WHERE status IN ('allocated','running');
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpus_node_index ON gpus(node_id, gpu_index);
CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_type, resource_id);
CREATE UNIQUE INDEX IF NOT EXISTS uq_replicas_node_resource ON resource_replicas(node_id, resource_type, resource_id);
CREATE INDEX IF NOT EXISTS idx_sync_jobs_node_status ON resource_sync_jobs(target_node_id, status);
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);

-- ---- 项目 / 租户 ----

CREATE TABLE IF NOT EXISTS projects (
    id TEXT PRIMARY KEY,
    tenant_id TEXT NOT NULL DEFAULT 'default',
    name TEXT NOT NULL,
    code TEXT NOT NULL,
    description TEXT,
    quota TEXT,
    status TEXT NOT NULL DEFAULT 'active',
    create_time TEXT NOT NULL,
    create_by TEXT,
    updated_at TEXT
);

CREATE TABLE IF NOT EXISTS project_members (
    project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
    user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
    role TEXT NOT NULL DEFAULT 'member',
    create_time TEXT NOT NULL,
    PRIMARY KEY (project_id, user_id)
);

CREATE TABLE IF NOT EXISTS roles (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    permissions TEXT NOT NULL DEFAULT '[]',
    create_time TEXT
);

CREATE TABLE IF NOT EXISTS sessions (
    id TEXT PRIMARY KEY,
    user_id TEXT NOT NULL,
    username TEXT,
    login_at TEXT,
    logout_at TEXT,
    duration_seconds INTEGER,
    issued_at TEXT,
    expires_at TEXT,
    ip TEXT,
    create_time TEXT
);

CREATE TABLE IF NOT EXISTS acls (
    id TEXT PRIMARY KEY,
    resource_type TEXT NOT NULL,
    resource_id TEXT NOT NULL,
    principal_type TEXT NOT NULL,
    principal_id TEXT NOT NULL,
    permission TEXT NOT NULL,
    create_time TEXT
);
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE UNIQUE INDEX IF NOT EXISTS uq_acls_subject_permission
    ON acls(resource_type, resource_id, principal_type, principal_id, permission);
CREATE INDEX IF NOT EXISTS idx_acls_resource ON acls(resource_type, resource_id);
CREATE INDEX IF NOT EXISTS idx_acls_principal ON acls(principal_type, principal_id);

-- ---- 权限扩展（来源：004_permissions.sql） ----

CREATE TABLE IF NOT EXISTS gpu_assignments (
    id TEXT PRIMARY KEY,
    node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
    gpu_index INTEGER NOT NULL,
    user_id TEXT NOT NULL,
    assigned_by TEXT,
    assigned_at TEXT NOT NULL,
    UNIQUE (node_id, gpu_index, user_id)
);
CREATE INDEX IF NOT EXISTS idx_gpu_assignments_user ON gpu_assignments(user_id);
CREATE INDEX IF NOT EXISTS idx_gpu_assignments_gpu ON gpu_assignments(node_id, gpu_index);

ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;

-- ============================================================================
-- 二、治理表（来源：002_governance.sql）
-- ============================================================================

CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    code TEXT,
    status TEXT DEFAULT 'active',
    owner_user_id TEXT,
    quota TEXT,
    retention_policy_id TEXT,
    create_time TEXT
);

CREATE TABLE IF NOT EXISTS approval_templates (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    steps TEXT,
    create_time TEXT
);

CREATE TABLE IF NOT EXISTS approval_instances (
    id TEXT PRIMARY KEY,
    template_id TEXT,
    resource_type TEXT,
    resource_id TEXT,
    applicant_id TEXT,
    status TEXT DEFAULT 'pending',
    current_step INTEGER DEFAULT 0,
    create_time TEXT
);

CREATE TABLE IF NOT EXISTS approval_steps (
    id TEXT PRIMARY KEY,
    instance_id TEXT,
    step_index INTEGER,
    approver_id TEXT,
    status TEXT DEFAULT 'pending',
    comment TEXT,
    time TEXT
);
CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instances(applicant_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);

-- Compatibility for databases created from an older approval_steps definition.
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS id TEXT;
UPDATE approval_steps
SET id = 'astep_' || md5(concat_ws(':', instance_id, step_index, coalesce(approver_id, ''), coalesce(time, '')))
WHERE id IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_approval_steps_id ON approval_steps(id);

CREATE TABLE IF NOT EXISTS audit_logs (
    id TEXT PRIMARY KEY,
    tenant_id TEXT,
    project_id TEXT,
    actor_id TEXT,
    action TEXT,
    target_type TEXT,
    target_id TEXT,
    detail TEXT,
    client_ip TEXT,
    time TEXT
);

CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_logs(tenant_id);
CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);

CREATE TABLE IF NOT EXISTS retention_policies (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    scope TEXT,
    rule TEXT,
    status TEXT DEFAULT 'active',
    create_time TEXT,
    create_by TEXT,
    updated_at TEXT
);

-- Compatibility for the earlier retention policy schema
-- (resource_type/retention_days). Keep legacy columns if they exist.
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS scope TEXT;
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS rule TEXT;
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'active';
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS create_by TEXT;
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS updated_at TEXT;
DO $$
BEGIN
    IF EXISTS (
        SELECT 1 FROM information_schema.columns
        WHERE table_schema='public' AND table_name='retention_policies' AND column_name='resource_type'
    ) AND EXISTS (
        SELECT 1 FROM information_schema.columns
        WHERE table_schema='public' AND table_name='retention_policies' AND column_name='retention_days'
    ) THEN
        EXECUTE $migration$
            UPDATE retention_policies
            SET scope = COALESCE(scope, resource_type),
                rule = COALESCE(rule, retention_days::text),
                status = COALESCE(status, 'active'),
                updated_at = COALESCE(updated_at, create_time)
            WHERE scope IS NULL OR rule IS NULL OR status IS NULL OR updated_at IS NULL
        $migration$;
    ELSE
        UPDATE retention_policies
        SET status = COALESCE(status, 'active'),
            updated_at = COALESCE(updated_at, create_time)
        WHERE status IS NULL OR updated_at IS NULL;
    END IF;
END $$;

-- ============================================================================
-- 三、租户配额扩展（来源：003_tenant_quota.sql）
-- ============================================================================

ALTER TABLE tenants ADD COLUMN IF NOT EXISTS gpu_quota TEXT;
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;

-- ============================================================================
-- 四、模型路径治理（来源：003_model_path_governance.sql）
--   models.can_train 已在建表语句中声明；以下为兼容旧库的幂等语句。
-- ============================================================================

ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_id TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_name TEXT;

-- 按规则推定已有模型的 can_train（新库为空表，此语句为 no-op）
UPDATE models
SET can_train = CASE
    WHEN path IS NOT NULL AND path != '' AND model_source IS NOT NULL AND model_source != 'api' THEN 1
    ELSE 0
END;

-- ============================================================================
-- 五、数据处理（来源：002_data_process.sql，去掉其外层 BEGIN/COMMIT）
--   数据集扩展列
-- ============================================================================

ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;

ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';

-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----

CREATE TABLE IF NOT EXISTS data_process_tasks (
    id TEXT PRIMARY KEY,
    name VARCHAR(150) NOT NULL,
    description TEXT,
    status VARCHAR(20) NOT NULL DEFAULT 'pending'
        CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
    process_type VARCHAR(20) NOT NULL
        CHECK (process_type IN ('structured', 'unstructured', 'external')),
    source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
    output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
    config TEXT NOT NULL DEFAULT '{}',
    progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
    input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
    output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
    filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
    duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
    error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
    failure_reason TEXT,
    generation_run_id TEXT,
    results_confirmed BOOLEAN NOT NULL DEFAULT TRUE,
    workflow_step VARCHAR(20) NOT NULL DEFAULT 'create'
        CHECK (workflow_step IN ('create', 'model', 'upload', 'preview', 'generate', 'results')),
    preview_status VARCHAR(20) NOT NULL DEFAULT 'idle'
        CHECK (preview_status IN ('idle', 'queued', 'running', 'completed', 'failed', 'cancelled')),
    preview_progress NUMERIC(5,2) NOT NULL DEFAULT 0
        CHECK (preview_progress >= 0 AND preview_progress <= 100),
    preview_run_id TEXT,
    preview_failure_reason TEXT,
    preview_total_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_total_files >= 0),
    preview_completed_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_completed_files >= 0),
    tenant_id TEXT,
    project_id TEXT,
    owner_id TEXT,
    approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
    created_by TEXT,
    updated_by TEXT,
    deleted_by TEXT,
    started_at TIMESTAMPTZ,
    completed_at TIMESTAMPTZ,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    deleted_at TIMESTAMPTZ
);

ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS results_confirmed BOOLEAN NOT NULL DEFAULT TRUE;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS workflow_step VARCHAR(20);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_status VARCHAR(20);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_progress NUMERIC(5,2);
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_run_id TEXT;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_failure_reason TEXT;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_total_files INTEGER;
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_completed_files INTEGER;

ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET DEFAULT 'create';
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET DEFAULT 'idle';
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET DEFAULT 0;
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET DEFAULT 0;
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET NOT NULL;
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET DEFAULT 0;
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET NOT NULL;

CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
    ON data_process_tasks(name) WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
    ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
    WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
    ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;

CREATE TABLE IF NOT EXISTS data_process_source_files (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
    storage_object_id TEXT,
    name TEXT NOT NULL,
    size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
    record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
    file_format VARCHAR(40),
    checksum_sha256 CHAR(64) NOT NULL,
    version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
    content TEXT NOT NULL,
    content_preview TEXT,
    metadata TEXT NOT NULL DEFAULT '{}',
    tenant_id TEXT,
    project_id TEXT,
    created_by TEXT,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    deleted_at TIMESTAMPTZ
);

CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
    ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
    ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;

CREATE TABLE IF NOT EXISTS data_process_preview_items (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
    source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
    original_content TEXT NOT NULL DEFAULT '',
    edited_content TEXT NOT NULL DEFAULT '',
    source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
    source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
    source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
    source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
    token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
    status VARCHAR(20) NOT NULL DEFAULT 'original'
        CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
    quality_score TEXT NOT NULL DEFAULT '{}',
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
    CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
);

CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
    ON data_process_preview_items(task_id, source_file_id, created_at);

CREATE TABLE IF NOT EXISTS data_process_results (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
    preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
    instruction TEXT NOT NULL,
    input TEXT NOT NULL DEFAULT '',
    output TEXT NOT NULL,
    chosen TEXT NOT NULL DEFAULT '',
    rejected TEXT NOT NULL DEFAULT '',
    original_instruction TEXT,
    original_input TEXT,
    original_output TEXT,
    original_chosen TEXT,
    original_rejected TEXT,
    status VARCHAR(20) NOT NULL DEFAULT 'valid'
        CHECK (status IN ('valid', 'modified', 'invalid')),
    error TEXT,
    split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
    quality_score TEXT NOT NULL DEFAULT '{}',
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

-- Keep existing databases compatible with the current generation result model.
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS chosen TEXT NOT NULL DEFAULT '';
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS rejected TEXT NOT NULL DEFAULT '';
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_chosen TEXT;
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_rejected TEXT;

CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
    ON data_process_results(task_id, status, id);
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
    ON data_process_results(task_id, split);

-- ---- 数据集版本 / 记录 ----

CREATE TABLE IF NOT EXISTS dataset_file_versions (
    id TEXT PRIMARY KEY,
    dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
    version_no INTEGER NOT NULL CHECK (version_no > 0),
    storage_object_id TEXT NOT NULL,
    content_preview TEXT,
    description TEXT,
    base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
    size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
    record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
    checksum_sha256 CHAR(64) NOT NULL,
    source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
    metadata TEXT NOT NULL DEFAULT '{}',
    created_by TEXT,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
    ON dataset_file_versions(dataset_file_id, version_no);
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
    ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;

CREATE TABLE IF NOT EXISTS dataset_records (
    id TEXT PRIMARY KEY,
    dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
    dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
    version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
    line_no INTEGER,
    split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
    instruction TEXT,
    input TEXT,
    output TEXT,
    raw TEXT NOT NULL DEFAULT '{}',
    status VARCHAR(20) NOT NULL DEFAULT 'valid'
        CHECK (status IN ('valid', 'modified', 'invalid')),
    source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
    source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
    preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
    ON dataset_records(dataset_id, id);
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
    ON dataset_records(source_task_id, source_result_id);
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
    ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
    ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;

-- ============================================================================
-- 六、数据转换任务（data_convert_tasks）
--   运行时 router（app/modules/data_convert/router.py）引用但原脚本缺失，本文件补齐。
-- ============================================================================

CREATE TABLE IF NOT EXISTS data_convert_tasks (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    description TEXT,
    output_filename TEXT DEFAULT 'converted-data.jsonl',
    status TEXT NOT NULL DEFAULT 'pending',
    input_count INTEGER NOT NULL DEFAULT 0,
    output_count INTEGER NOT NULL DEFAULT 0,
    error_message TEXT,
    output_content TEXT,
    create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
    update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
    deleted_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;

-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员
--   应用首次启动（ensure_seed_data）也会自动创建；此处提供以便脱离应用直接初始化。
--   密码：admin / admin123，operator / operator123（上线前请改密）。
-- ============================================================================

INSERT INTO users
    (id, username, password_hash, display_name, role, status, permissions, create_time, protected)
VALUES
    (
        'u_admin', 'admin', 'pbkdf2_sha256$390000$ygft_init_salt_admin$2b6f31f22968c4f5a30bcf0acf066b7a0f58d4773d15c5ab898ba715ea87b5bd',
        'Platform Admin', 'admin', 'active',
        '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]',
        to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1
    ),
    (
        'u_operator', 'operator', 'pbkdf2_sha256$390000$ygft_init_salt_op$525bf35d02ed26f37952cbd6862b0ae358b9d1a7fa0cbbf0217aa2b5dd544125',
        'Platform Operator', 'operator', 'active',
        '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]',
        to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0
    )
ON CONFLICT (username) DO NOTHING;

COMMIT;