feat: 数据集下载鉴权、MinIO 发布与数据库兼容增强
- 数据集下载接口鉴权：单文件直接返回、多文件打包 ZIP，前端统一走请求客户端携带 Token - 预检/同步支持 MinIO 对象补建与资源副本记录，兼容接入 MinIO 前的历史数据集 - create_dataset 增加名称重复校验，软删记录释放名称并保留墓碑 - 数据处理发布结果支持写入 MinIO storage_objects 并关联 dataset_file - 数据存储根目录支持 YG_FT_DATA_ROOT 环境变量 - SQL 迁移兼容旧版 approval_steps / retention_policies / data_process_results - 训练日志图表按曲线独立过滤缺失采样点 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
@@ -1607,7 +1607,24 @@ class PlatformStore:
|
||||
|
||||
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
dataset_id = payload.get("id") or new_id("ds")
|
||||
name = str(payload.get("name") or "").strip()
|
||||
if not name:
|
||||
raise ValueError("dataset name is required")
|
||||
with self.connect() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT id, deleted_at FROM datasets WHERE name=?",
|
||||
(name,),
|
||||
).fetchone()
|
||||
if existing and not existing.get("deleted_at"):
|
||||
raise ValueError(f"dataset name already exists: {name}")
|
||||
# Soft-deleted records remain in the database for audit/history and
|
||||
# still participate in the legacy unique constraint. Free the name
|
||||
# while retaining a traceable tombstone before creating the new row.
|
||||
if existing:
|
||||
conn.execute(
|
||||
"UPDATE datasets SET name=? WHERE id=?",
|
||||
(f"{name}__deleted__{existing['id']}", existing["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO datasets
|
||||
@@ -1616,7 +1633,7 @@ class PlatformStore:
|
||||
""",
|
||||
(
|
||||
dataset_id,
|
||||
payload["name"],
|
||||
name,
|
||||
payload.get("type", "train"),
|
||||
payload.get("storage_type", "local"),
|
||||
payload.get("source", "upload"),
|
||||
@@ -2965,6 +2982,27 @@ class PlatformStore:
|
||||
).fetchone()
|
||||
return dict(row)
|
||||
|
||||
def link_dataset_file_storage_object(self, file_id: str, storage_object_id: str) -> None:
|
||||
"""Link an uploaded dataset file to its canonical MinIO object."""
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE dataset_files
|
||||
SET storage_object_id=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(storage_object_id, file_id),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT dataset_id FROM dataset_files WHERE id=?",
|
||||
(file_id,),
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE datasets SET storage_type='minio' WHERE id=?",
|
||||
(row["dataset_id"],),
|
||||
)
|
||||
|
||||
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
|
||||
@@ -506,6 +506,13 @@ CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instance
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
|
||||
|
||||
-- Compatibility for databases created from an older approval_steps definition.
|
||||
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS id TEXT;
|
||||
UPDATE approval_steps
|
||||
SET id = 'astep_' || md5(concat_ws(':', instance_id, step_index, coalesce(approver_id, ''), coalesce(time, '')))
|
||||
WHERE id IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_approval_steps_id ON approval_steps(id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT,
|
||||
@@ -535,6 +542,38 @@ CREATE TABLE IF NOT EXISTS retention_policies (
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
-- Compatibility for the earlier retention policy schema
|
||||
-- (resource_type/retention_days). Keep legacy columns if they exist.
|
||||
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS scope TEXT;
|
||||
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS rule TEXT;
|
||||
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'active';
|
||||
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS create_by TEXT;
|
||||
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS updated_at TEXT;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='retention_policies' AND column_name='resource_type'
|
||||
) AND EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='retention_policies' AND column_name='retention_days'
|
||||
) THEN
|
||||
EXECUTE $migration$
|
||||
UPDATE retention_policies
|
||||
SET scope = COALESCE(scope, resource_type),
|
||||
rule = COALESCE(rule, retention_days::text),
|
||||
status = COALESCE(status, 'active'),
|
||||
updated_at = COALESCE(updated_at, create_time)
|
||||
WHERE scope IS NULL OR rule IS NULL OR status IS NULL OR updated_at IS NULL
|
||||
$migration$;
|
||||
ELSE
|
||||
UPDATE retention_policies
|
||||
SET status = COALESCE(status, 'active'),
|
||||
updated_at = COALESCE(updated_at, create_time)
|
||||
WHERE status IS NULL OR updated_at IS NULL;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- ============================================================================
|
||||
-- 三、租户配额扩展（来源：003_tenant_quota.sql）
|
||||
-- ============================================================================
|
||||
@@ -723,9 +762,13 @@ CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
instruction TEXT NOT NULL,
|
||||
input TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL,
|
||||
chosen TEXT NOT NULL DEFAULT '',
|
||||
rejected TEXT NOT NULL DEFAULT '',
|
||||
original_instruction TEXT,
|
||||
original_input TEXT,
|
||||
original_output TEXT,
|
||||
original_chosen TEXT,
|
||||
original_rejected TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
error TEXT,
|
||||
@@ -735,6 +778,12 @@ CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Keep existing databases compatible with the current generation result model.
|
||||
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS chosen TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS rejected TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_chosen TEXT;
|
||||
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_rejected TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
||||
ON data_process_results(task_id, status, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
||||
|
||||
Reference in New Issue
Block a user