feat: 平台治理与对象存储增强,审批中心与运行日志整合
- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚 - 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts - 数据转换任务输入输出对象化,支持从 MinIO 读写 - 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
from datetime import timedelta
|
||||
from functools import lru_cache
|
||||
from io import BytesIO
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from minio import Minio
|
||||
@@ -53,6 +54,64 @@ class MinioObjectStorage:
|
||||
except S3Error as exc:
|
||||
raise ObjectStorageError(str(exc)) from exc
|
||||
|
||||
def get_bytes(self, object_key: str) -> bytes:
|
||||
"""Read an object through the backend for small API responses and workers."""
|
||||
self._ensure_enabled()
|
||||
self.ensure_bucket()
|
||||
response = None
|
||||
try:
|
||||
response = self.client.get_object(self.bucket, object_key)
|
||||
return response.read()
|
||||
except S3Error as exc:
|
||||
raise ObjectStorageError(str(exc)) from exc
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
|
||||
"""Stream an object without loading the complete file into memory."""
|
||||
self._ensure_enabled()
|
||||
self.ensure_bucket()
|
||||
response = None
|
||||
try:
|
||||
response = self.client.get_object(self.bucket, object_key)
|
||||
while True:
|
||||
chunk = response.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
except S3Error as exc:
|
||||
raise ObjectStorageError(str(exc)) from exc
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
|
||||
self._ensure_enabled()
|
||||
self.ensure_bucket()
|
||||
try:
|
||||
return [
|
||||
{
|
||||
"object_key": item.object_name,
|
||||
"byte_size": item.size or 0,
|
||||
"etag": item.etag,
|
||||
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
|
||||
}
|
||||
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
||||
]
|
||||
except S3Error as exc:
|
||||
raise ObjectStorageError(str(exc)) from exc
|
||||
|
||||
def delete(self, object_key: str) -> None:
|
||||
self._ensure_enabled()
|
||||
self.ensure_bucket()
|
||||
try:
|
||||
self.client.remove_object(self.bucket, object_key)
|
||||
except S3Error as exc:
|
||||
raise ObjectStorageError(str(exc)) from exc
|
||||
|
||||
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
|
||||
self._ensure_enabled()
|
||||
self.ensure_bucket()
|
||||
|
||||
54
backend/app/modules/storage/policy.py
Normal file
54
backend/app/modules/storage/policy.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Storage placement rules shared by dataset and data-processing flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
|
||||
_INLINE_TEXT_FORMATS = {
|
||||
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
|
||||
"yaml", "yml", "xml", "html", "text/plain", "application/json",
|
||||
"application/jsonl", "text/csv",
|
||||
}
|
||||
|
||||
|
||||
def should_store_in_minio(
|
||||
size_bytes: int | None,
|
||||
*,
|
||||
content_type: str | None = None,
|
||||
file_format: str | None = None,
|
||||
) -> bool:
|
||||
"""Return whether a file is large enough to use the shared object store.
|
||||
|
||||
Small files remain inline in PostgreSQL so page previews and metadata reads
|
||||
do not pay an object-storage round trip. MinIO is still mandatory for
|
||||
large files when it is enabled.
|
||||
"""
|
||||
|
||||
if not get_settings().minio_enabled:
|
||||
return False
|
||||
try:
|
||||
size = max(0, int(size_bytes or 0))
|
||||
except (TypeError, ValueError):
|
||||
size = 0
|
||||
if size > get_settings().minio_inline_max_bytes:
|
||||
return True
|
||||
# Binary office/document files remain in MinIO even when small because
|
||||
# their original bytes cannot be safely represented by a text DB column.
|
||||
normalized_format = str(file_format or "").strip().lower().lstrip(".")
|
||||
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
|
||||
if normalized_format or normalized_type:
|
||||
return not (
|
||||
normalized_format in _INLINE_TEXT_FORMATS
|
||||
or normalized_type in _INLINE_TEXT_FORMATS
|
||||
or normalized_type.startswith("text/")
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
|
||||
"""Return ``minio`` or ``database`` for a managed file."""
|
||||
|
||||
if str(requested or "").strip().lower() == "local":
|
||||
return "database"
|
||||
return "minio" if should_store_in_minio(size_bytes) else "database"
|
||||
Reference in New Issue
Block a user