feat: 平台治理与对象存储增强,审批中心与运行日志整合

- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚
- 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts
- 数据转换任务输入输出对象化,支持从 MinIO 读写
- 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions

View File

@@ -0,0 +1,54 @@
"""Storage placement rules shared by dataset and data-processing flows."""
from __future__ import annotations
from app.core.config import get_settings
_INLINE_TEXT_FORMATS = {
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
"yaml", "yml", "xml", "html", "text/plain", "application/json",
"application/jsonl", "text/csv",
}
def should_store_in_minio(
size_bytes: int | None,
*,
content_type: str | None = None,
file_format: str | None = None,
) -> bool:
"""Return whether a file is large enough to use the shared object store.
Small files remain inline in PostgreSQL so page previews and metadata reads
do not pay an object-storage round trip. MinIO is still mandatory for
large files when it is enabled.
"""
if not get_settings().minio_enabled:
return False
try:
size = max(0, int(size_bytes or 0))
except (TypeError, ValueError):
size = 0
if size > get_settings().minio_inline_max_bytes:
return True
# Binary office/document files remain in MinIO even when small because
# their original bytes cannot be safely represented by a text DB column.
normalized_format = str(file_format or "").strip().lower().lstrip(".")
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
if normalized_format or normalized_type:
return not (
normalized_format in _INLINE_TEXT_FORMATS
or normalized_type in _INLINE_TEXT_FORMATS
or normalized_type.startswith("text/")
)
return False
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
"""Return ``minio`` or ``database`` for a managed file."""
if str(requested or "").strip().lower() == "local":
return "database"
return "minio" if should_store_in_minio(size_bytes) else "database"