-- Data processing migration.
--
-- IMPORTANT: This file is intentionally NOT wired into application startup.
-- Apply it explicitly in a controlled deployment, or call
-- DataProcessStore.ensure_schema() from an administrative command.

BEGIN;

-- This migration targets the current runtime schema created by
-- 001_platform_runtime.sql. Refuse the UUID/JSONB target-design schema instead
-- of partially altering it with incompatible TEXT foreign keys.
DO $$
DECLARE
    datasets_id_type TEXT;
BEGIN
    SELECT format_type(a.atttypid, a.atttypmod)
      INTO datasets_id_type
      FROM pg_attribute a
      JOIN pg_class c ON c.oid = a.attrelid
      JOIN pg_namespace n ON n.oid = c.relnamespace
     WHERE n.nspname = current_schema()
       AND c.relname = 'datasets'
       AND a.attname = 'id'
       AND a.attnum > 0
       AND NOT a.attisdropped;
    IF datasets_id_type IS NULL THEN
        RAISE EXCEPTION '002_data_process.sql requires 001_platform_runtime.sql first';
    END IF;
    IF datasets_id_type <> 'text' THEN
        RAISE EXCEPTION
          '002_data_process.sql supports only the current TEXT runtime schema; found datasets.id type %',
          datasets_id_type;
    END IF;
END $$;

ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;

ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;

CREATE TABLE IF NOT EXISTS data_process_tasks (
    id TEXT PRIMARY KEY,
    name VARCHAR(150) NOT NULL,
    description TEXT,
    status VARCHAR(20) NOT NULL DEFAULT 'pending'
        CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
    process_type VARCHAR(20) NOT NULL
        CHECK (process_type IN ('structured', 'unstructured', 'external')),
    source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
    output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
    config TEXT NOT NULL DEFAULT '{}',
    progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
    input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
    output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
    filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
    duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
    error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
    failure_reason TEXT,
    generation_run_id TEXT,
    tenant_id TEXT,
    project_id TEXT,
    owner_id TEXT,
    approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
    created_by TEXT,
    updated_by TEXT,
    deleted_by TEXT,
    started_at TIMESTAMPTZ,
    completed_at TIMESTAMPTZ,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    deleted_at TIMESTAMPTZ
);

ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;

CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
    ON data_process_tasks(name) WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
    ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
    WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
    ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;

CREATE TABLE IF NOT EXISTS data_process_source_files (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
    storage_object_id TEXT,
    name TEXT NOT NULL,
    size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
    record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
    file_format VARCHAR(40),
    checksum_sha256 CHAR(64) NOT NULL,
    version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
    content TEXT NOT NULL,
    content_preview TEXT,
    metadata TEXT NOT NULL DEFAULT '{}',
    tenant_id TEXT,
    project_id TEXT,
    created_by TEXT,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    deleted_at TIMESTAMPTZ
);

CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
    ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
    ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;

CREATE TABLE IF NOT EXISTS data_process_preview_items (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
    source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
    original_content TEXT NOT NULL DEFAULT '',
    edited_content TEXT NOT NULL DEFAULT '',
    source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
    source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
    source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
    source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
    token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
    status VARCHAR(20) NOT NULL DEFAULT 'original'
        CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
    quality_score TEXT NOT NULL DEFAULT '{}',
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
    CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
);

CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
    ON data_process_preview_items(task_id, source_file_id, created_at);

CREATE TABLE IF NOT EXISTS data_process_results (
    id TEXT PRIMARY KEY,
    task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
    preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
    instruction TEXT NOT NULL,
    input TEXT NOT NULL DEFAULT '',
    output TEXT NOT NULL,
    original_instruction TEXT,
    original_input TEXT,
    original_output TEXT,
    status VARCHAR(20) NOT NULL DEFAULT 'valid'
        CHECK (status IN ('valid', 'modified', 'invalid')),
    error TEXT,
    split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
    quality_score TEXT NOT NULL DEFAULT '{}',
    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
    ON data_process_results(task_id, status, id);
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
    ON data_process_results(task_id, split);

CREATE TABLE IF NOT EXISTS dataset_file_versions (
    id TEXT PRIMARY KEY,
    dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
    version_no INTEGER NOT NULL CHECK (version_no > 0),
    storage_object_id TEXT NOT NULL,
    content_preview TEXT,
    description TEXT,
    base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
    size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
    record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
    checksum_sha256 CHAR(64) NOT NULL,
    source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
    metadata TEXT NOT NULL DEFAULT '{}',
    created_by TEXT,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
    ON dataset_file_versions(dataset_file_id, version_no);
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
    ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;

CREATE TABLE IF NOT EXISTS dataset_records (
    id TEXT PRIMARY KEY,
    dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
    dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
    version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
    line_no INTEGER,
    split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
    instruction TEXT,
    input TEXT,
    output TEXT,
    raw TEXT NOT NULL DEFAULT '{}',
    status VARCHAR(20) NOT NULL DEFAULT 'valid'
        CHECK (status IN ('valid', 'modified', 'invalid')),
    source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
    source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
    preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
    ON dataset_records(dataset_id, id);
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
    ON dataset_records(source_task_id, source_result_id);
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
    ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
    ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;

COMMIT;