feat: 平台治理与对象存储增强,审批中心与运行日志整合
- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚 - 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts - 数据转换任务输入输出对象化,支持从 MinIO 读写 - 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -247,14 +247,19 @@ def _source_storage_descriptor(
|
||||
or f"db://data-process/{task_id}/{file_id}/v1"
|
||||
)
|
||||
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
|
||||
expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/"
|
||||
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
|
||||
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
|
||||
expected_local_prefix
|
||||
):
|
||||
storage_backend = "local"
|
||||
elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len(
|
||||
expected_minio_prefix
|
||||
):
|
||||
storage_backend = "minio"
|
||||
elif storage_object_id == expected_database_reference:
|
||||
storage_backend = "database"
|
||||
elif storage_object_id.startswith(("local://data-process/", "db://data-process/")):
|
||||
elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")):
|
||||
raise DataProcessStoreError("source storage object owner mismatch")
|
||||
else:
|
||||
raise DataProcessStoreError("unsupported source storage object reference")
|
||||
|
||||
@@ -11,6 +11,7 @@ import psycopg
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.modules.storage.minio_store import get_object_storage
|
||||
from app.modules.storage.policy import should_store_in_minio
|
||||
|
||||
from .base import (
|
||||
StoreBase,
|
||||
@@ -168,7 +169,11 @@ class DatasetsMixin:
|
||||
split_name: assignments.count(split_name) for split_name in split_order
|
||||
}
|
||||
split_specs: list[dict[str, Any]] = []
|
||||
use_minio = bool(get_settings().minio_enabled)
|
||||
# Explicit local is retained for old callers/tests that request the
|
||||
# legacy backend; all normal platform requests default to MinIO.
|
||||
allow_minio = bool(get_settings().minio_enabled) and str(
|
||||
payload.get("storage_type") or "minio"
|
||||
).lower() != "local"
|
||||
for split_name in split_order:
|
||||
split_records = [
|
||||
(source_row, record)
|
||||
@@ -193,12 +198,15 @@ class DatasetsMixin:
|
||||
"storage_object_id": (
|
||||
f"db://data-process/{task_id}/{file_id}/v1"
|
||||
),
|
||||
"store_in_minio": allow_minio and should_store_in_minio(
|
||||
len(raw), content_type="application/jsonl", file_format="jsonl"
|
||||
),
|
||||
}
|
||||
)
|
||||
source_result_ids = [row["id"] for row in rows]
|
||||
common_metadata = {
|
||||
"source": "data_process",
|
||||
"storage_backend": "minio" if use_minio else "database",
|
||||
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
|
||||
"source_task_id": task_id,
|
||||
"output_type": _task_output_type(task),
|
||||
"reasoning_detail": _task_reasoning_detail(task),
|
||||
@@ -273,7 +281,7 @@ class DatasetsMixin:
|
||||
split_name = str(spec["split"])
|
||||
dataset_id = dataset_ids[split_name]
|
||||
storage_object_id = str(spec["storage_object_id"])
|
||||
if use_minio:
|
||||
if spec["store_in_minio"]:
|
||||
file_name = f"{base_dataset_name}.{split_name}.jsonl"
|
||||
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
|
||||
uploaded = get_object_storage().put_bytes(
|
||||
@@ -356,7 +364,7 @@ class DatasetsMixin:
|
||||
(
|
||||
dataset_name,
|
||||
dataset_types[split_name],
|
||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||
"minio" if spec["store_in_minio"] else "database",
|
||||
f"{len(spec['raw'])} B",
|
||||
len(spec["raw"]),
|
||||
len(spec["records"]),
|
||||
@@ -386,7 +394,7 @@ class DatasetsMixin:
|
||||
dataset_id,
|
||||
dataset_name,
|
||||
dataset_types[split_name],
|
||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||
"minio" if spec["store_in_minio"] else "database",
|
||||
task_id,
|
||||
task_id,
|
||||
f"{len(spec['raw'])} B",
|
||||
@@ -405,7 +413,11 @@ class DatasetsMixin:
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
file_metadata = {**dataset_metadata, "file_split": split_name}
|
||||
file_metadata = {
|
||||
**dataset_metadata,
|
||||
"file_split": split_name,
|
||||
"storage_backend": "minio" if spec["store_in_minio"] else "database",
|
||||
}
|
||||
version = {
|
||||
"id": spec["version_id"],
|
||||
"version_no": 1,
|
||||
|
||||
@@ -137,7 +137,7 @@ class SourceFilesMixin:
|
||||
payload["record_count"],
|
||||
payload["file_format"],
|
||||
payload["checksum_sha256"],
|
||||
payload["content"],
|
||||
"" if str(storage_object_id or "").startswith("minio://") else payload["content"],
|
||||
str(payload["content"])[:2000],
|
||||
json_dumps(metadata_payload),
|
||||
task.get("tenant_id"),
|
||||
@@ -223,7 +223,17 @@ class SourceFilesMixin:
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise NotFoundError("source file not found")
|
||||
return _decode_row(row) or {}
|
||||
decoded = _decode_row(row) or {}
|
||||
# New source files keep only a preview in PostgreSQL. Load the
|
||||
# authoritative body from MinIO on demand for existing processing code.
|
||||
reference = str(decoded.get("storage_object_id") or "")
|
||||
if include_content and not decoded.get("content") and reference.startswith("minio://"):
|
||||
from app.modules.data_process.storage import get_data_process_storage
|
||||
|
||||
decoded["content"] = (get_data_process_storage().read(reference) or b"").decode(
|
||||
"utf-8", errors="replace"
|
||||
)
|
||||
return decoded
|
||||
|
||||
def source_content_window(
|
||||
self, task_id: str, file_id: str, offset: int, limit: int
|
||||
|
||||
Reference in New Issue
Block a user