feat: 数据集下载鉴权、MinIO 发布与数据库兼容增强
- 数据集下载接口鉴权：单文件直接返回、多文件打包 ZIP，前端统一走请求客户端携带 Token - 预检/同步支持 MinIO 对象补建与资源副本记录，兼容接入 MinIO 前的历史数据集 - create_dataset 增加名称重复校验，软删记录释放名称并保留墓碑 - 数据处理发布结果支持写入 MinIO storage_objects 并关联 dataset_file - 数据存储根目录支持 YG_FT_DATA_ROOT 环境变量 - SQL 迁移兼容旧版 approval_steps / retention_policies / data_process_results - 训练日志图表按曲线独立过滤缺失采样点 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
@@ -28,6 +28,9 @@ class StagedSourceObject:
|
||||
|
||||
|
||||
def _default_storage_root() -> Path:
|
||||
data_root = os.getenv("YG_FT_DATA_ROOT", "").strip()
|
||||
if data_root:
|
||||
return Path(data_root).expanduser() / "data-process"
|
||||
return Path(__file__).resolve().parents[3] / "storage" / "data-process"
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,9 @@ import hashlib
|
||||
|
||||
import psycopg
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.modules.storage.minio_store import get_object_storage
|
||||
|
||||
from .base import (
|
||||
StoreBase,
|
||||
utcnow,
|
||||
@@ -165,6 +168,7 @@ class DatasetsMixin:
|
||||
split_name: assignments.count(split_name) for split_name in split_order
|
||||
}
|
||||
split_specs: list[dict[str, Any]] = []
|
||||
use_minio = bool(get_settings().minio_enabled)
|
||||
for split_name in split_order:
|
||||
split_records = [
|
||||
(source_row, record)
|
||||
@@ -194,7 +198,7 @@ class DatasetsMixin:
|
||||
source_result_ids = [row["id"] for row in rows]
|
||||
common_metadata = {
|
||||
"source": "data_process",
|
||||
"storage_backend": "database",
|
||||
"storage_backend": "minio" if use_minio else "database",
|
||||
"source_task_id": task_id,
|
||||
"output_type": _task_output_type(task),
|
||||
"reasoning_detail": _task_reasoning_detail(task),
|
||||
@@ -268,6 +272,53 @@ class DatasetsMixin:
|
||||
for spec in split_specs:
|
||||
split_name = str(spec["split"])
|
||||
dataset_id = dataset_ids[split_name]
|
||||
storage_object_id = str(spec["storage_object_id"])
|
||||
if use_minio:
|
||||
file_name = f"{base_dataset_name}.{split_name}.jsonl"
|
||||
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
|
||||
uploaded = get_object_storage().put_bytes(
|
||||
object_key,
|
||||
spec["raw"],
|
||||
"application/jsonl",
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO storage_objects
|
||||
(id, resource_type, resource_id, version_id, bucket, object_key,
|
||||
file_name, content_type, checksum_sha256, byte_size, status,
|
||||
created_by, create_time)
|
||||
VALUES (%s, 'dataset', %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
'available', %s, %s)
|
||||
ON CONFLICT (resource_type, resource_id, version_id, object_key)
|
||||
DO UPDATE SET file_name=EXCLUDED.file_name,
|
||||
content_type=EXCLUDED.content_type,
|
||||
checksum_sha256=EXCLUDED.checksum_sha256,
|
||||
byte_size=EXCLUDED.byte_size,
|
||||
status='available',
|
||||
created_by=EXCLUDED.created_by
|
||||
""",
|
||||
(
|
||||
new_id("object"),
|
||||
dataset_id,
|
||||
spec["version_id"],
|
||||
uploaded["bucket"],
|
||||
object_key,
|
||||
file_name,
|
||||
"application/jsonl",
|
||||
spec["checksum"],
|
||||
len(spec["raw"]),
|
||||
payload.get("created_by") or task.get("created_by"),
|
||||
now,
|
||||
),
|
||||
)
|
||||
storage_object_id = conn.execute(
|
||||
"""
|
||||
SELECT id FROM storage_objects
|
||||
WHERE resource_type='dataset' AND resource_id=%s
|
||||
AND version_id=%s AND object_key=%s
|
||||
""",
|
||||
(dataset_id, spec["version_id"], object_key),
|
||||
).fetchone()["id"]
|
||||
existing_dataset = existing_by_split.get(split_name)
|
||||
dataset_metadata = {
|
||||
**common_metadata,
|
||||
@@ -305,7 +356,7 @@ class DatasetsMixin:
|
||||
(
|
||||
dataset_name,
|
||||
dataset_types[split_name],
|
||||
payload.get("storage_type") or "local",
|
||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||
f"{len(spec['raw'])} B",
|
||||
len(spec["raw"]),
|
||||
len(spec["records"]),
|
||||
@@ -335,7 +386,7 @@ class DatasetsMixin:
|
||||
dataset_id,
|
||||
dataset_name,
|
||||
dataset_types[split_name],
|
||||
payload.get("storage_type") or "local",
|
||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||
task_id,
|
||||
task_id,
|
||||
f"{len(spec['raw'])} B",
|
||||
@@ -366,7 +417,7 @@ class DatasetsMixin:
|
||||
"created_at": now,
|
||||
"create_time": now,
|
||||
"source_task_id": task_id,
|
||||
"storage_object_id": spec["storage_object_id"],
|
||||
"storage_object_id": storage_object_id,
|
||||
}
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -383,7 +434,7 @@ class DatasetsMixin:
|
||||
spec["file_id"],
|
||||
dataset_id,
|
||||
f"{base_dataset_name}.{split_name}.jsonl",
|
||||
spec["storage_object_id"],
|
||||
storage_object_id,
|
||||
f"{len(spec['raw'])} B",
|
||||
spec["content"],
|
||||
spec["version_id"],
|
||||
@@ -414,7 +465,7 @@ class DatasetsMixin:
|
||||
(
|
||||
spec["version_id"],
|
||||
spec["file_id"],
|
||||
spec["storage_object_id"],
|
||||
storage_object_id,
|
||||
spec["content"][:2000],
|
||||
f"data process {split_name} publish",
|
||||
len(spec["raw"]),
|
||||
|
||||
Reference in New Issue
Block a user