feat: 数据集下载鉴权、MinIO 发布与数据库兼容增强

- 数据集下载接口鉴权:单文件直接返回、多文件打包 ZIP,前端统一走请求客户端携带 Token
- 预检/同步支持 MinIO 对象补建与资源副本记录,兼容接入 MinIO 前的历史数据集
- create_dataset 增加名称重复校验,软删记录释放名称并保留墓碑
- 数据处理发布结果支持写入 MinIO storage_objects 并关联 dataset_file
- 数据存储根目录支持 YG_FT_DATA_ROOT 环境变量
- SQL 迁移兼容旧版 approval_steps / retention_policies / data_process_results
- 训练日志图表按曲线独立过滤缺失采样点

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-18 14:45:16 +08:00
parent 6c1bf61ff7
commit 2adf78a6ed
10 changed files with 313 additions and 36 deletions

View File

@@ -1607,7 +1607,24 @@ class PlatformStore:
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
dataset_id = payload.get("id") or new_id("ds")
name = str(payload.get("name") or "").strip()
if not name:
raise ValueError("dataset name is required")
with self.connect() as conn:
existing = conn.execute(
"SELECT id, deleted_at FROM datasets WHERE name=?",
(name,),
).fetchone()
if existing and not existing.get("deleted_at"):
raise ValueError(f"dataset name already exists: {name}")
# Soft-deleted records remain in the database for audit/history and
# still participate in the legacy unique constraint. Free the name
# while retaining a traceable tombstone before creating the new row.
if existing:
conn.execute(
"UPDATE datasets SET name=? WHERE id=?",
(f"{name}__deleted__{existing['id']}", existing["id"]),
)
conn.execute(
"""
INSERT INTO datasets
@@ -1616,7 +1633,7 @@ class PlatformStore:
""",
(
dataset_id,
payload["name"],
name,
payload.get("type", "train"),
payload.get("storage_type", "local"),
payload.get("source", "upload"),
@@ -2965,6 +2982,27 @@ class PlatformStore:
).fetchone()
return dict(row)
def link_dataset_file_storage_object(self, file_id: str, storage_object_id: str) -> None:
"""Link an uploaded dataset file to its canonical MinIO object."""
with self.connect() as conn:
conn.execute(
"""
UPDATE dataset_files
SET storage_object_id=?
WHERE id=?
""",
(storage_object_id, file_id),
)
row = conn.execute(
"SELECT dataset_id FROM dataset_files WHERE id=?",
(file_id,),
).fetchone()
if row:
conn.execute(
"UPDATE datasets SET storage_type='minio' WHERE id=?",
(row["dataset_id"],),
)
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(