feat: 数据集下载鉴权、MinIO 发布与数据库兼容增强
- 数据集下载接口鉴权:单文件直接返回、多文件打包 ZIP,前端统一走请求客户端携带 Token - 预检/同步支持 MinIO 对象补建与资源副本记录,兼容接入 MinIO 前的历史数据集 - create_dataset 增加名称重复校验,软删记录释放名称并保留墓碑 - 数据处理发布结果支持写入 MinIO storage_objects 并关联 dataset_file - 数据存储根目录支持 YG_FT_DATA_ROOT 环境变量 - SQL 迁移兼容旧版 approval_steps / retention_policies / data_process_results - 训练日志图表按曲线独立过滤缺失采样点 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1607,7 +1607,24 @@ class PlatformStore:
|
||||
|
||||
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
dataset_id = payload.get("id") or new_id("ds")
|
||||
name = str(payload.get("name") or "").strip()
|
||||
if not name:
|
||||
raise ValueError("dataset name is required")
|
||||
with self.connect() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT id, deleted_at FROM datasets WHERE name=?",
|
||||
(name,),
|
||||
).fetchone()
|
||||
if existing and not existing.get("deleted_at"):
|
||||
raise ValueError(f"dataset name already exists: {name}")
|
||||
# Soft-deleted records remain in the database for audit/history and
|
||||
# still participate in the legacy unique constraint. Free the name
|
||||
# while retaining a traceable tombstone before creating the new row.
|
||||
if existing:
|
||||
conn.execute(
|
||||
"UPDATE datasets SET name=? WHERE id=?",
|
||||
(f"{name}__deleted__{existing['id']}", existing["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO datasets
|
||||
@@ -1616,7 +1633,7 @@ class PlatformStore:
|
||||
""",
|
||||
(
|
||||
dataset_id,
|
||||
payload["name"],
|
||||
name,
|
||||
payload.get("type", "train"),
|
||||
payload.get("storage_type", "local"),
|
||||
payload.get("source", "upload"),
|
||||
@@ -2965,6 +2982,27 @@ class PlatformStore:
|
||||
).fetchone()
|
||||
return dict(row)
|
||||
|
||||
def link_dataset_file_storage_object(self, file_id: str, storage_object_id: str) -> None:
|
||||
"""Link an uploaded dataset file to its canonical MinIO object."""
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE dataset_files
|
||||
SET storage_object_id=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(storage_object_id, file_id),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT dataset_id FROM dataset_files WHERE id=?",
|
||||
(file_id,),
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE datasets SET storage_type='minio' WHERE id=?",
|
||||
(row["dataset_id"],),
|
||||
)
|
||||
|
||||
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
|
||||
Reference in New Issue
Block a user