feat: 新增 MinIO 对象存储与算力节点缓存预下载

- 后端新增 storage 模块（minio_store），支持 MinIO 预签名 URL 上传与对象管理
- config 新增 MinIO 及存储等待相关配置项
- 算力节点新增 /compute/cache/prepare 缓存预下载接口（带校验和原子落盘）
- 算力节点健康接口增加存储可用性探针
- SQL 迁移补充资源存储相关表结构
- Docker 新增 minio 服务与后端 minio 配置
- 补充 minio-compute-cache-plan 设计文档

Co-Authored-By: Claude <noreply@anthropic.com>

compute/api/main.py

@@ -11,6 +11,8 @@ import time
 from pathlib import Path
 from typing import Any
 
+import httpx
+
 from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
 from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
 
@@ -511,13 +513,27 @@ def create_app() -> FastAPI:
         llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
         gpu_items = gpu_resources()
         torch_cuda = torch_cuda_status()
+        storage_available = False
+        storage_error = ""
+        try:
+            data_root.mkdir(parents=True, exist_ok=True)
+            probe = data_root / ".yg-ft-storage-healthcheck"
+            probe.write_text(host_id(), encoding="utf-8")
+            storage_available = probe.read_text(encoding="utf-8").strip() == host_id()
+            probe.unlink(missing_ok=True)
+        except Exception as exc:  # noqa: BLE001 - health endpoint must remain available
+            storage_error = str(exc)
         return {
-            "status": "ok",
+            "status": "ok" if storage_available else "storage_unavailable",
             "api_version": "v1",
             "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
             "app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true",
             "data_root": str(data_root),
             "data_root_exists": data_root.exists(),
+            "storage_mode": os.getenv("STORAGE_MODE", "minio-cache"),
+            "storage_available": storage_available,
+            "storage_error": storage_error,
+            "storage_root": str(data_root),
             "model_root": os.getenv("YG_FT_MODEL_ROOT", str(data_root / "models")),
             "dataset_root": str(dataset_root),
             "dataset_root_exists": dataset_root.exists(),
@@ -532,7 +548,7 @@ def create_app() -> FastAPI:
             "nvidia_gpu_count": len(gpu_items),
             "torch_cuda": torch_cuda,
             "gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus",
-            "capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling"],
+            "capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling", "storage_health"],
         }
 
     @app.get(f"{route_prefix}/v1/compute/jobs")
@@ -810,6 +826,82 @@ def create_app() -> FastAPI:
             "checksum_sha256": hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "",
         }
 
+    @app.post(f"{route_prefix}/compute/cache/prepare")
+    async def prepare_cache(payload: dict[str, Any]) -> dict[str, Any]:
+        """Download one MinIO object into the node-local cache atomically."""
+        download_url = str(payload.get("download_url") or "")
+        resource_id = str(payload.get("resource_id") or "")
+        version_id = str(payload.get("version_id") or "latest")
+        if not download_url or not resource_id:
+            raise HTTPException(status_code=400, detail="download_url and resource_id are required")
+        data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
+        cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
+        relative_path = str(payload.get("relative_path") or f"resources/{resource_id}/{version_id}/resource")
+        target = (cache_root / relative_path.lstrip("/\\")).resolve()
+        if not _path_inside(cache_root, target):
+            raise HTTPException(status_code=400, detail="cache path must stay inside cache root")
+        target.parent.mkdir(parents=True, exist_ok=True)
+        temp_target = target.with_name(f".{target.name}.part")
+        expected_checksum = str(payload.get("checksum_sha256") or "").lower()
+        digest = hashlib.sha256()
+        byte_size = 0
+        try:
+            async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
+                async with client.stream("GET", download_url) as response:
+                    response.raise_for_status()
+                    with temp_target.open("wb") as output:
+                        async for chunk in response.aiter_bytes(1024 * 1024):
+                            output.write(chunk)
+                            digest.update(chunk)
+                            byte_size += len(chunk)
+            checksum = digest.hexdigest()
+            if expected_checksum and checksum != expected_checksum:
+                temp_target.unlink(missing_ok=True)
+                raise HTTPException(status_code=502, detail="cache checksum mismatch")
+            temp_target.replace(target)
+        except HTTPException:
+            raise
+        except Exception as exc:
+            temp_target.unlink(missing_ok=True)
+            raise HTTPException(status_code=502, detail=f"cache download failed: {exc}") from exc
+        return {
+            "resource_id": resource_id,
+            "version_id": version_id,
+            "status": "ready",
+            "local_path": str(target),
+            "byte_size": byte_size,
+            "checksum_sha256": checksum,
+        }
+
+    @app.get(f"{route_prefix}/compute/cache/status")
+    async def cache_status(resource_id: str = Query(...), version_id: str = Query(default="latest")) -> dict[str, Any]:
+        data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
+        cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
+        target = cache_root / "resources" / resource_id / version_id / "resource"
+        return {
+            "resource_id": resource_id,
+            "version_id": version_id,
+            "status": "ready" if target.is_file() else "missing",
+            "local_path": str(target),
+            "byte_size": target.stat().st_size if target.is_file() else 0,
+        }
+
+    @app.delete(f"{route_prefix}/compute/cache")
+    async def clear_cache(resource_id: str | None = Query(default=None), version_id: str | None = Query(default=None)) -> dict[str, Any]:
+        data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
+        cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
+        target = cache_root / "resources"
+        if resource_id:
+            target = target / resource_id
+            if version_id:
+                target = target / version_id
+        target = target.resolve()
+        if not _path_inside(cache_root, target):
+            raise HTTPException(status_code=400, detail="cache path must stay inside cache root")
+        if target.exists():
+            shutil.rmtree(target)
+        return {"status": "cleared", "resource_id": resource_id, "version_id": version_id}
+
     @app.post(f"{route_prefix}/compute/files/import-local")
     async def import_local_file(payload: dict[str, Any]) -> dict[str, Any]:
         source = Path(str(payload.get("source_path") or ""))

compute/requirements.txt

@@ -4,6 +4,7 @@ python-multipart>=0.0.9
 pydantic>=2.7.0
 python-dotenv>=1.0.1
 httpx>=0.27.0
+# Compute Agent downloads MinIO objects through presigned HTTP URLs; no MinIO SDK is required.
 # 模型评测指标
 sacrebleu>=2.4.0
 rouge-score>=0.1.2