236 lines
11 KiB
PL/PgSQL
236 lines
11 KiB
PL/PgSQL
-- Data processing migration.
|
|
--
|
|
-- IMPORTANT: This file is intentionally NOT wired into application startup.
|
|
-- Apply it explicitly in a controlled deployment, or call
|
|
-- DataProcessStore.ensure_schema() from an administrative command.
|
|
|
|
BEGIN;
|
|
|
|
-- This migration targets the current runtime schema created by
|
|
-- 001_platform_runtime.sql. Refuse the UUID/JSONB target-design schema instead
|
|
-- of partially altering it with incompatible TEXT foreign keys.
|
|
DO $$
|
|
DECLARE
|
|
datasets_id_type TEXT;
|
|
BEGIN
|
|
SELECT format_type(a.atttypid, a.atttypmod)
|
|
INTO datasets_id_type
|
|
FROM pg_attribute a
|
|
JOIN pg_class c ON c.oid = a.attrelid
|
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
WHERE n.nspname = current_schema()
|
|
AND c.relname = 'datasets'
|
|
AND a.attname = 'id'
|
|
AND a.attnum > 0
|
|
AND NOT a.attisdropped;
|
|
IF datasets_id_type IS NULL THEN
|
|
RAISE EXCEPTION '002_data_process.sql requires 001_platform_runtime.sql first';
|
|
END IF;
|
|
IF datasets_id_type <> 'text' THEN
|
|
RAISE EXCEPTION
|
|
'002_data_process.sql supports only the current TEXT runtime schema; found datasets.id type %',
|
|
datasets_id_type;
|
|
END IF;
|
|
END $$;
|
|
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
|
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
|
|
|
CREATE TABLE IF NOT EXISTS data_process_tasks (
|
|
id TEXT PRIMARY KEY,
|
|
name VARCHAR(150) NOT NULL,
|
|
description TEXT,
|
|
status VARCHAR(20) NOT NULL DEFAULT 'pending'
|
|
CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
|
|
process_type VARCHAR(20) NOT NULL
|
|
CHECK (process_type IN ('structured', 'unstructured', 'external')),
|
|
source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
|
output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
|
config TEXT NOT NULL DEFAULT '{}',
|
|
progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
|
|
input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
|
|
output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
|
|
filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
|
|
duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
|
|
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
|
|
failure_reason TEXT,
|
|
generation_run_id TEXT,
|
|
tenant_id TEXT,
|
|
project_id TEXT,
|
|
owner_id TEXT,
|
|
approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
|
|
created_by TEXT,
|
|
updated_by TEXT,
|
|
deleted_by TEXT,
|
|
started_at TIMESTAMPTZ,
|
|
completed_at TIMESTAMPTZ,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
deleted_at TIMESTAMPTZ
|
|
);
|
|
|
|
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
|
|
ON data_process_tasks(name) WHERE deleted_at IS NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
|
|
ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
|
|
WHERE deleted_at IS NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
|
|
ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;
|
|
|
|
CREATE TABLE IF NOT EXISTS data_process_source_files (
|
|
id TEXT PRIMARY KEY,
|
|
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
|
storage_object_id TEXT,
|
|
name TEXT NOT NULL,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
|
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
|
file_format VARCHAR(40),
|
|
checksum_sha256 CHAR(64) NOT NULL,
|
|
version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
|
|
content TEXT NOT NULL,
|
|
content_preview TEXT,
|
|
metadata TEXT NOT NULL DEFAULT '{}',
|
|
tenant_id TEXT,
|
|
project_id TEXT,
|
|
created_by TEXT,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
deleted_at TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
|
|
ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
|
|
ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;
|
|
|
|
CREATE TABLE IF NOT EXISTS data_process_preview_items (
|
|
id TEXT PRIMARY KEY,
|
|
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
|
source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
|
|
original_content TEXT NOT NULL DEFAULT '',
|
|
edited_content TEXT NOT NULL DEFAULT '',
|
|
source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
|
|
source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
|
|
source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
|
|
source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
|
|
token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
|
|
status VARCHAR(20) NOT NULL DEFAULT 'original'
|
|
CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
|
|
quality_score TEXT NOT NULL DEFAULT '{}',
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
|
|
CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
|
|
ON data_process_preview_items(task_id, source_file_id, created_at);
|
|
|
|
CREATE TABLE IF NOT EXISTS data_process_results (
|
|
id TEXT PRIMARY KEY,
|
|
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
|
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
|
instruction TEXT NOT NULL,
|
|
input TEXT NOT NULL DEFAULT '',
|
|
output TEXT NOT NULL,
|
|
original_instruction TEXT,
|
|
original_input TEXT,
|
|
original_output TEXT,
|
|
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
|
CHECK (status IN ('valid', 'modified', 'invalid')),
|
|
error TEXT,
|
|
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
|
quality_score TEXT NOT NULL DEFAULT '{}',
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
|
ON data_process_results(task_id, status, id);
|
|
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
|
ON data_process_results(task_id, split);
|
|
|
|
CREATE TABLE IF NOT EXISTS dataset_file_versions (
|
|
id TEXT PRIMARY KEY,
|
|
dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
|
|
version_no INTEGER NOT NULL CHECK (version_no > 0),
|
|
storage_object_id TEXT NOT NULL,
|
|
content_preview TEXT,
|
|
description TEXT,
|
|
base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
|
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
|
checksum_sha256 CHAR(64) NOT NULL,
|
|
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
|
metadata TEXT NOT NULL DEFAULT '{}',
|
|
created_by TEXT,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
|
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
|
|
ON dataset_file_versions(dataset_file_id, version_no);
|
|
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
|
|
ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;
|
|
|
|
CREATE TABLE IF NOT EXISTS dataset_records (
|
|
id TEXT PRIMARY KEY,
|
|
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
|
dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
|
|
version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
|
|
line_no INTEGER,
|
|
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
|
instruction TEXT,
|
|
input TEXT,
|
|
output TEXT,
|
|
raw TEXT NOT NULL DEFAULT '{}',
|
|
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
|
CHECK (status IN ('valid', 'modified', 'invalid')),
|
|
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
|
source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
|
|
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
|
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
|
|
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
|
|
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
|
|
ON dataset_records(dataset_id, id);
|
|
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
|
|
ON dataset_records(source_task_id, source_result_id);
|
|
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
|
|
ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
|
|
ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;
|
|
|
|
COMMIT;
|