YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增 MinIO 对象存储与算力节点缓存预下载

Browse Source 
- 后端新增 storage 模块（minio_store），支持 MinIO 预签名 URL 上传与对象管理
- config 新增 MinIO 及存储等待相关配置项
- 算力节点新增 /compute/cache/prepare 缓存预下载接口（带校验和原子落盘）
- 算力节点健康接口增加存储可用性探针
- SQL 迁移补充资源存储相关表结构
- Docker 新增 minio 服务与后端 minio 配置
- 补充 minio-compute-cache-plan 设计文档

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-11 16:25:18 +08:00
parent a12f80492d
commit b5d2cd7935
20 changed files with 830 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
backend/app/api/v1/endpoints/platform.py
View File

@@ -1,6 +1,8 @@
from __future__ import annotations
import json
import asyncio
import hashlib
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
@@ -16,6 +18,7 @@ from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
router = APIRouter()
@@ -33,6 +36,22 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
return None
async def _wait_for_object_storage() -> None:
"""Wait for MinIO before starting a resource task."""
settings = get_settings()
deadline = datetime.now(timezone.utc).timestamp() + max(0, settings.storage_wait_seconds)
last_error = "MinIO unavailable"
while True:
try:
get_object_storage().ensure_bucket()
return
except Exception as exc: # noqa: BLE001 - retry until the configured deadline
last_error = str(exc)
if datetime.now(timezone.utc).timestamp() >= deadline:
raise RuntimeError(f"MinIO unavailable after {settings.storage_wait_seconds}s: {last_error}")
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
"""Select the compute node for an eval job.
@@ -330,6 +349,11 @@ async def _fine_tune_preflight_with_job_payload(
)
except Exception as exc: # noqa: BLE001 - return as preflight error for page visibility
sync_errors.append(str(exc))
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
sync_errors.append(f"shared storage health check failed: {exc}")
if get_settings().compute_mode == "simulator":
preview = {
"valid": True,
@@ -877,6 +901,11 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
raise fail(400, "adapter_path is required")
requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id"))
node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []})
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
except RuntimeError as exc:
raise fail(503, str(exc))
health = node.get("health_detail") or {}
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
@@ -981,7 +1010,7 @@ async def _sync_dataset_file_to_compute_nodes(
content: bytes,
) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
if get_settings().compute_mode == "simulator":
if get_settings().compute_mode == "simulator" or get_settings().minio_enabled:
return results
target_name = Path(filename or f"{file_id}.jsonl").name
target_relative_path = f"datasets/{dataset_id}/{target_name}"
@@ -1029,6 +1058,27 @@ async def _sync_training_dataset_to_compute_node(
node: dict[str, Any],
dataset_id: str,
) -> list[dict[str, Any]]:
if get_settings().minio_enabled:
files = store.training_dataset_files(dataset_id)
objects = store.storage_objects_for_resource("dataset", dataset_id)
object_by_name = {Path(str(item.get("file_name") or item.get("object_key") or "")).name: item for item in objects}
results: list[dict[str, Any]] = []
client = ComputeNodeClient(node["api_base_url"])
for item in files:
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
obj = object_by_name.get(target_name)
if not obj:
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
url = get_object_storage().presigned_get(obj["object_key"])
result = await client.prepare_cache({
"resource_id": dataset_id,
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"relative_path": f"datasets/{dataset_id}/{target_name}",
})
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
return results
if not dataset_id:
raise RuntimeError("train_dataset_id is required")
files = store.training_dataset_files(dataset_id)
@@ -1092,6 +1142,17 @@ async def upload_dataset_files(
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
created.append(created_file)
pending_sync.append((created_file["id"], created_file["name"], raw))
if get_settings().minio_enabled:
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
get_platform_store().create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": created_file.get("active_version_id") or created_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": created_file["name"], "content_type": file.content_type,
"byte_size": len(raw), "checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
if sync_to_compute:
for file_id, file_name, raw in pending_sync:
compute_sync.extend(
@@ -1865,6 +1926,8 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
errors = []
for node in _candidate_online_nodes(store, preferred_node_id):
try:
if get_settings().minio_enabled:
await _wait_for_object_storage()
client = ComputeNodeClient(node["api_base_url"])
await client.inference_load(load_payload)
store.mark_inference_loaded(node["id"])
@@ -2035,6 +2098,73 @@ async def compute_nodes() -> dict[str, Any]:
return ok(get_platform_store().compute_nodes())
@router.post("/storage/objects/presign")
async def presign_storage_object(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Create a short-lived MinIO upload/download URL for a platform resource."""
if not get_settings().minio_enabled:
raise fail(503, "MinIO object storage is disabled")
resource_type = str(payload.get("resource_type") or "")
resource_id = str(payload.get("resource_id") or "")
version_id = str(payload.get("version_id") or uuid.uuid4().hex)
object_key = str(payload.get("object_key") or f"{resource_type}/{resource_id}/versions/{version_id}/resource")
if not resource_type or not resource_id:
raise fail(400, "resource_type and resource_id are required")
if payload.get("method", "put").lower() == "get" and not has_resource_access(resource_type, resource_id, current_user, "read"):
raise fail(403, "no permission to read this resource")
try:
storage = get_object_storage()
url = storage.presigned_get(object_key) if payload.get("method", "put").lower() == "get" else storage.presigned_put(object_key)
record = get_platform_store().create_storage_object({
"resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
"bucket": storage.bucket, "object_key": object_key, "file_name": payload.get("file_name"),
"content_type": payload.get("content_type"), "created_by": current_user.get("id"),
})
return ok({"url": url, "method": payload.get("method", "put").lower(), "expires_seconds": 3600, "object": record})
except ObjectStorageError as exc:
raise fail(503, str(exc))
@router.get("/storage/resources/{resource_type}/{resource_id}")
async def storage_resource_objects(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access(resource_type, resource_id, current_user, "read"):
raise fail(403, "no permission to read this resource")
return ok(get_platform_store().storage_objects_for_resource(resource_type, resource_id))
@router.post("/storage/resources/{resource_type}/{resource_id}/prepare/{node_id}")
async def prepare_storage_resource(
resource_type: str,
resource_id: str,
node_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not has_resource_access(resource_type, resource_id, current_user, "execute"):
raise fail(403, "no permission to execute this resource")
store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
if not get_settings().minio_enabled:
raise fail(503, "MinIO object storage is disabled")
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
raise fail(404, "resource has no MinIO objects")
client = ComputeNodeClient(node["api_base_url"])
prepared = []
for obj in objects:
url = get_object_storage().presigned_get(obj["object_key"])
filename = Path(str(obj.get("file_name") or obj["object_key"])).name
result = await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
})
prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})
@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str) -> dict[str, Any]:
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)