78e3baa9ba
- 新增 storage/policy.py 落盘策略：按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects，支持失败回滚 - 算力节点训练产物按版本归档到 MinIO，登记 model_artifacts - 数据转换任务输入输出对象化，支持从 MinIO 读写 - 新增审批中心（申请/我的/策略）、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
128 lines
4.8 KiB
Python
128 lines
4.8 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
from functools import lru_cache
|
|
from io import BytesIO
|
|
from collections.abc import Iterator
|
|
from typing import Any
|
|
|
|
from minio import Minio
|
|
from minio.error import S3Error
|
|
|
|
from app.core.config import get_settings
|
|
|
|
|
|
class ObjectStorageError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class MinioObjectStorage:
|
|
def __init__(self) -> None:
|
|
settings = get_settings()
|
|
endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/")
|
|
self.client = Minio(endpoint, access_key=settings.minio_access_key, secret_key=settings.minio_secret_key, secure=settings.minio_secure)
|
|
self.bucket = settings.minio_bucket
|
|
|
|
def _ensure_enabled(self) -> None:
|
|
if not get_settings().minio_enabled:
|
|
raise ObjectStorageError("MinIO object storage is disabled")
|
|
|
|
def ensure_bucket(self) -> None:
|
|
self._ensure_enabled()
|
|
try:
|
|
if not self.client.bucket_exists(self.bucket):
|
|
self.client.make_bucket(self.bucket)
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
|
|
def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str:
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
return self.client.presigned_put_object(self.bucket, object_key, expires=timedelta(seconds=expires_seconds))
|
|
|
|
def presigned_get(self, object_key: str, expires_seconds: int = 3600) -> str:
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
return self.client.presigned_get_object(self.bucket, object_key, expires=timedelta(seconds=expires_seconds))
|
|
|
|
def stat(self, object_key: str) -> dict[str, Any]:
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
try:
|
|
result = self.client.stat_object(self.bucket, object_key)
|
|
return {"object_key": object_key, "byte_size": result.size, "etag": result.etag, "last_modified": result.last_modified.isoformat() if result.last_modified else None}
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
|
|
def get_bytes(self, object_key: str) -> bytes:
|
|
"""Read an object through the backend for small API responses and workers."""
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
response = None
|
|
try:
|
|
response = self.client.get_object(self.bucket, object_key)
|
|
return response.read()
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
finally:
|
|
if response is not None:
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
|
|
"""Stream an object without loading the complete file into memory."""
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
response = None
|
|
try:
|
|
response = self.client.get_object(self.bucket, object_key)
|
|
while True:
|
|
chunk = response.read(chunk_size)
|
|
if not chunk:
|
|
break
|
|
yield chunk
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
finally:
|
|
if response is not None:
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
try:
|
|
return [
|
|
{
|
|
"object_key": item.object_name,
|
|
"byte_size": item.size or 0,
|
|
"etag": item.etag,
|
|
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
|
|
}
|
|
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
|
]
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
|
|
def delete(self, object_key: str) -> None:
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
try:
|
|
self.client.remove_object(self.bucket, object_key)
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
|
|
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
|
|
self._ensure_enabled()
|
|
self.ensure_bucket()
|
|
try:
|
|
result = self.client.put_object(self.bucket, object_key, BytesIO(content), len(content), content_type=content_type)
|
|
return {"bucket": self.bucket, "object_key": object_key, "etag": result.etag, "byte_size": len(content)}
|
|
except S3Error as exc:
|
|
raise ObjectStorageError(str(exc)) from exc
|
|
|
|
|
|
@lru_cache
|
|
def get_object_storage() -> MinioObjectStorage:
|
|
return MinioObjectStorage()
|