feat: 平台治理与对象存储增强,审批中心与运行日志整合
- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚 - 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts - 数据转换任务输入输出对象化,支持从 MinIO 读写 - 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import psycopg
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.modules.storage.minio_store import get_object_storage
|
||||
from app.modules.storage.policy import should_store_in_minio
|
||||
|
||||
from .base import (
|
||||
StoreBase,
|
||||
@@ -168,7 +169,11 @@ class DatasetsMixin:
|
||||
split_name: assignments.count(split_name) for split_name in split_order
|
||||
}
|
||||
split_specs: list[dict[str, Any]] = []
|
||||
use_minio = bool(get_settings().minio_enabled)
|
||||
# Explicit local is retained for old callers/tests that request the
|
||||
# legacy backend; all normal platform requests default to MinIO.
|
||||
allow_minio = bool(get_settings().minio_enabled) and str(
|
||||
payload.get("storage_type") or "minio"
|
||||
).lower() != "local"
|
||||
for split_name in split_order:
|
||||
split_records = [
|
||||
(source_row, record)
|
||||
@@ -193,12 +198,15 @@ class DatasetsMixin:
|
||||
"storage_object_id": (
|
||||
f"db://data-process/{task_id}/{file_id}/v1"
|
||||
),
|
||||
"store_in_minio": allow_minio and should_store_in_minio(
|
||||
len(raw), content_type="application/jsonl", file_format="jsonl"
|
||||
),
|
||||
}
|
||||
)
|
||||
source_result_ids = [row["id"] for row in rows]
|
||||
common_metadata = {
|
||||
"source": "data_process",
|
||||
"storage_backend": "minio" if use_minio else "database",
|
||||
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
|
||||
"source_task_id": task_id,
|
||||
"output_type": _task_output_type(task),
|
||||
"reasoning_detail": _task_reasoning_detail(task),
|
||||
@@ -273,7 +281,7 @@ class DatasetsMixin:
|
||||
split_name = str(spec["split"])
|
||||
dataset_id = dataset_ids[split_name]
|
||||
storage_object_id = str(spec["storage_object_id"])
|
||||
if use_minio:
|
||||
if spec["store_in_minio"]:
|
||||
file_name = f"{base_dataset_name}.{split_name}.jsonl"
|
||||
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
|
||||
uploaded = get_object_storage().put_bytes(
|
||||
@@ -356,7 +364,7 @@ class DatasetsMixin:
|
||||
(
|
||||
dataset_name,
|
||||
dataset_types[split_name],
|
||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||
"minio" if spec["store_in_minio"] else "database",
|
||||
f"{len(spec['raw'])} B",
|
||||
len(spec["raw"]),
|
||||
len(spec["records"]),
|
||||
@@ -386,7 +394,7 @@ class DatasetsMixin:
|
||||
dataset_id,
|
||||
dataset_name,
|
||||
dataset_types[split_name],
|
||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||
"minio" if spec["store_in_minio"] else "database",
|
||||
task_id,
|
||||
task_id,
|
||||
f"{len(spec['raw'])} B",
|
||||
@@ -405,7 +413,11 @@ class DatasetsMixin:
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
file_metadata = {**dataset_metadata, "file_split": split_name}
|
||||
file_metadata = {
|
||||
**dataset_metadata,
|
||||
"file_split": split_name,
|
||||
"storage_backend": "minio" if spec["store_in_minio"] else "database",
|
||||
}
|
||||
version = {
|
||||
"id": spec["version_id"],
|
||||
"version_no": 1,
|
||||
|
||||
Reference in New Issue
Block a user