feat: 平台治理与对象存储增强,审批中心与运行日志整合

- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚
- 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts
- 数据转换任务输入输出对象化,支持从 MinIO 读写
- 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import timedelta
from functools import lru_cache
from io import BytesIO
from collections.abc import Iterator
from typing import Any
from minio import Minio
@@ -53,6 +54,64 @@ class MinioObjectStorage:
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def get_bytes(self, object_key: str) -> bytes:
"""Read an object through the backend for small API responses and workers."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
return response.read()
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
"""Stream an object without loading the complete file into memory."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
while True:
chunk = response.read(chunk_size)
if not chunk:
break
yield chunk
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
self._ensure_enabled()
self.ensure_bucket()
try:
return [
{
"object_key": item.object_name,
"byte_size": item.size or 0,
"etag": item.etag,
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
}
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
]
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def delete(self, object_key: str) -> None:
self._ensure_enabled()
self.ensure_bucket()
try:
self.client.remove_object(self.bucket, object_key)
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
self._ensure_enabled()
self.ensure_bucket()