feat: 新增 MinIO 对象存储与算力节点缓存预下载

- 后端新增 storage 模块(minio_store),支持 MinIO 预签名 URL 上传与对象管理
- config 新增 MinIO 及存储等待相关配置项
- 算力节点新增 /compute/cache/prepare 缓存预下载接口(带校验和原子落盘)
- 算力节点健康接口增加存储可用性探针
- SQL 迁移补充资源存储相关表结构
- Docker 新增 minio 服务与后端 minio 配置
- 补充 minio-compute-cache-plan 设计文档

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-11 16:25:18 +08:00
parent a12f80492d
commit b5d2cd7935
20 changed files with 830 additions and 11 deletions

View File

@@ -1,6 +1,8 @@
from __future__ import annotations from __future__ import annotations
import json import json
import asyncio
import hashlib
import uuid import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
@@ -16,6 +18,7 @@ from app.core.config import get_settings
from app.db.platform_store import get_platform_store from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
router = APIRouter() router = APIRouter()
@@ -33,6 +36,22 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
return None return None
async def _wait_for_object_storage() -> None:
"""Wait for MinIO before starting a resource task."""
settings = get_settings()
deadline = datetime.now(timezone.utc).timestamp() + max(0, settings.storage_wait_seconds)
last_error = "MinIO unavailable"
while True:
try:
get_object_storage().ensure_bucket()
return
except Exception as exc: # noqa: BLE001 - retry until the configured deadline
last_error = str(exc)
if datetime.now(timezone.utc).timestamp() >= deadline:
raise RuntimeError(f"MinIO unavailable after {settings.storage_wait_seconds}s: {last_error}")
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None: def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
"""Select the compute node for an eval job. """Select the compute node for an eval job.
@@ -330,6 +349,11 @@ async def _fine_tune_preflight_with_job_payload(
) )
except Exception as exc: # noqa: BLE001 - return as preflight error for page visibility except Exception as exc: # noqa: BLE001 - return as preflight error for page visibility
sync_errors.append(str(exc)) sync_errors.append(str(exc))
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
sync_errors.append(f"shared storage health check failed: {exc}")
if get_settings().compute_mode == "simulator": if get_settings().compute_mode == "simulator":
preview = { preview = {
"valid": True, "valid": True,
@@ -877,6 +901,11 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
raise fail(400, "adapter_path is required") raise fail(400, "adapter_path is required")
requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id")) requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id"))
node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []}) node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []})
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
except RuntimeError as exc:
raise fail(503, str(exc))
health = node.get("health_detail") or {} health = node.get("health_detail") or {}
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs") output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged") output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
@@ -981,7 +1010,7 @@ async def _sync_dataset_file_to_compute_nodes(
content: bytes, content: bytes,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = [] results: list[dict[str, Any]] = []
if get_settings().compute_mode == "simulator": if get_settings().compute_mode == "simulator" or get_settings().minio_enabled:
return results return results
target_name = Path(filename or f"{file_id}.jsonl").name target_name = Path(filename or f"{file_id}.jsonl").name
target_relative_path = f"datasets/{dataset_id}/{target_name}" target_relative_path = f"datasets/{dataset_id}/{target_name}"
@@ -1029,6 +1058,27 @@ async def _sync_training_dataset_to_compute_node(
node: dict[str, Any], node: dict[str, Any],
dataset_id: str, dataset_id: str,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
if get_settings().minio_enabled:
files = store.training_dataset_files(dataset_id)
objects = store.storage_objects_for_resource("dataset", dataset_id)
object_by_name = {Path(str(item.get("file_name") or item.get("object_key") or "")).name: item for item in objects}
results: list[dict[str, Any]] = []
client = ComputeNodeClient(node["api_base_url"])
for item in files:
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
obj = object_by_name.get(target_name)
if not obj:
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
url = get_object_storage().presigned_get(obj["object_key"])
result = await client.prepare_cache({
"resource_id": dataset_id,
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"relative_path": f"datasets/{dataset_id}/{target_name}",
})
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
return results
if not dataset_id: if not dataset_id:
raise RuntimeError("train_dataset_id is required") raise RuntimeError("train_dataset_id is required")
files = store.training_dataset_files(dataset_id) files = store.training_dataset_files(dataset_id)
@@ -1092,6 +1142,17 @@ async def upload_dataset_files(
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content) created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
created.append(created_file) created.append(created_file)
pending_sync.append((created_file["id"], created_file["name"], raw)) pending_sync.append((created_file["id"], created_file["name"], raw))
if get_settings().minio_enabled:
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
get_platform_store().create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": created_file.get("active_version_id") or created_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": created_file["name"], "content_type": file.content_type,
"byte_size": len(raw), "checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
if sync_to_compute: if sync_to_compute:
for file_id, file_name, raw in pending_sync: for file_id, file_name, raw in pending_sync:
compute_sync.extend( compute_sync.extend(
@@ -1865,6 +1926,8 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
errors = [] errors = []
for node in _candidate_online_nodes(store, preferred_node_id): for node in _candidate_online_nodes(store, preferred_node_id):
try: try:
if get_settings().minio_enabled:
await _wait_for_object_storage()
client = ComputeNodeClient(node["api_base_url"]) client = ComputeNodeClient(node["api_base_url"])
await client.inference_load(load_payload) await client.inference_load(load_payload)
store.mark_inference_loaded(node["id"]) store.mark_inference_loaded(node["id"])
@@ -2035,6 +2098,73 @@ async def compute_nodes() -> dict[str, Any]:
return ok(get_platform_store().compute_nodes()) return ok(get_platform_store().compute_nodes())
@router.post("/storage/objects/presign")
async def presign_storage_object(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Create a short-lived MinIO upload/download URL for a platform resource."""
if not get_settings().minio_enabled:
raise fail(503, "MinIO object storage is disabled")
resource_type = str(payload.get("resource_type") or "")
resource_id = str(payload.get("resource_id") or "")
version_id = str(payload.get("version_id") or uuid.uuid4().hex)
object_key = str(payload.get("object_key") or f"{resource_type}/{resource_id}/versions/{version_id}/resource")
if not resource_type or not resource_id:
raise fail(400, "resource_type and resource_id are required")
if payload.get("method", "put").lower() == "get" and not has_resource_access(resource_type, resource_id, current_user, "read"):
raise fail(403, "no permission to read this resource")
try:
storage = get_object_storage()
url = storage.presigned_get(object_key) if payload.get("method", "put").lower() == "get" else storage.presigned_put(object_key)
record = get_platform_store().create_storage_object({
"resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
"bucket": storage.bucket, "object_key": object_key, "file_name": payload.get("file_name"),
"content_type": payload.get("content_type"), "created_by": current_user.get("id"),
})
return ok({"url": url, "method": payload.get("method", "put").lower(), "expires_seconds": 3600, "object": record})
except ObjectStorageError as exc:
raise fail(503, str(exc))
@router.get("/storage/resources/{resource_type}/{resource_id}")
async def storage_resource_objects(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access(resource_type, resource_id, current_user, "read"):
raise fail(403, "no permission to read this resource")
return ok(get_platform_store().storage_objects_for_resource(resource_type, resource_id))
@router.post("/storage/resources/{resource_type}/{resource_id}/prepare/{node_id}")
async def prepare_storage_resource(
resource_type: str,
resource_id: str,
node_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not has_resource_access(resource_type, resource_id, current_user, "execute"):
raise fail(403, "no permission to execute this resource")
store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
if not get_settings().minio_enabled:
raise fail(503, "MinIO object storage is disabled")
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
raise fail(404, "resource has no MinIO objects")
client = ComputeNodeClient(node["api_base_url"])
prepared = []
for obj in objects:
url = get_object_storage().presigned_get(obj["object_key"])
filename = Path(str(obj.get("file_name") or obj["object_key"])).name
result = await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
})
prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})
@router.get("/compute/nodes/{node_id}") @router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str) -> dict[str, Any]: async def compute_node_detail(node_id: str) -> dict[str, Any]:
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None) node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)

View File

@@ -60,6 +60,14 @@ class Settings:
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling") compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3) compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5) compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5)
minio_enabled: bool = _bool_env("MINIO_ENABLED", False)
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://minio:9000")
minio_access_key: str = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
minio_secure: bool = _bool_env("MINIO_SECURE", False)
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "") compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
log_level: str = os.getenv("LOG_LEVEL", "INFO") log_level: str = os.getenv("LOG_LEVEL", "INFO")
log_dir: str = os.getenv("LOG_DIR", "./logs") log_dir: str = os.getenv("LOG_DIR", "./logs")

View File

@@ -505,6 +505,20 @@ class PlatformStore:
) )
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"}) self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"}) self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
self._ensure_columns(
conn,
"sessions",
{
"username": "TEXT",
"login_at": "TEXT",
"logout_at": "TEXT",
"duration_seconds": "INTEGER",
"issued_at": "TEXT",
"expires_at": "TEXT",
"ip": "TEXT",
"create_time": "TEXT",
},
)
self._ensure_columns( self._ensure_columns(
conn, conn,
"trained_models", "trained_models",
@@ -525,7 +539,12 @@ class PlatformStore:
}, },
) )
schema_dir = Path(__file__).with_name("sql") schema_dir = Path(__file__).with_name("sql")
for extra in ("002_governance.sql", "003_tenant_quota.sql", "004_permissions.sql"): for extra in (
"002_governance.sql",
"003_model_path_governance.sql",
"003_tenant_quota.sql",
"004_permissions.sql",
):
extra_path = schema_dir / extra extra_path = schema_dir / extra
if extra_path.exists(): if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8")) conn.executescript(extra_path.read_text(encoding="utf-8"))
@@ -2863,6 +2882,54 @@ class PlatformStore:
result.append(dict(row)) result.append(dict(row))
return result return result
def create_storage_object(self, payload: dict[str, Any]) -> dict[str, Any]:
object_id = str(payload.get("id") or new_id("object"))
with self.connect() as conn:
conn.execute(
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
row = conn.execute(
"SELECT * FROM storage_objects WHERE resource_type=? AND resource_id=? AND version_id=? AND object_key=?",
(payload["resource_type"], payload["resource_id"], payload["version_id"], payload["object_key"]),
).fetchone()
return dict(row)
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM storage_objects WHERE resource_type=? AND resource_id=? AND status='available' ORDER BY version_id, object_key",
(resource_type, resource_id),
).fetchall()
return [dict(row) for row in rows]
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
if fields:
assignments = ", ".join(f"{key}=?" for key in fields)
with self.connect() as conn:
conn.execute(f"UPDATE storage_objects SET {assignments} WHERE id=?", (*fields.values(), object_id))
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_resource_replica_sync_result( def update_resource_replica_sync_result(
self, self,
replica_id: str, replica_id: str,

View File

@@ -276,6 +276,39 @@ CREATE TABLE IF NOT EXISTS resource_sync_jobs (
completed_at TEXT completed_at TEXT
); );
CREATE TABLE IF NOT EXISTS storage_objects (
id TEXT PRIMARY KEY,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
version_id TEXT NOT NULL,
bucket TEXT NOT NULL,
object_key TEXT NOT NULL,
file_name TEXT,
content_type TEXT,
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
);
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
id TEXT PRIMARY KEY,
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
direction TEXT NOT NULL,
status TEXT NOT NULL,
progress INTEGER NOT NULL DEFAULT 0,
local_path TEXT,
error TEXT,
create_time TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
CREATE TABLE IF NOT EXISTS eval_tasks ( CREATE TABLE IF NOT EXISTS eval_tasks (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
name TEXT NOT NULL, name TEXT NOT NULL,
@@ -361,9 +394,14 @@ CREATE TABLE IF NOT EXISTS roles (
CREATE TABLE IF NOT EXISTS sessions ( CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
issued_at TEXT NOT NULL, username TEXT,
expires_at TEXT NOT NULL, login_at TEXT,
ip TEXT logout_at TEXT,
duration_seconds INTEGER,
issued_at TEXT,
expires_at TEXT,
ip TEXT,
create_time TEXT
); );
CREATE TABLE IF NOT EXISTS acls ( CREATE TABLE IF NOT EXISTS acls (
@@ -376,6 +414,25 @@ CREATE TABLE IF NOT EXISTS acls (
create_time TEXT create_time TEXT
); );
-- ---- 权限扩展来源004_permissions.sql ----
CREATE TABLE IF NOT EXISTS gpu_assignments (
id TEXT PRIMARY KEY,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
gpu_index INTEGER NOT NULL,
user_id TEXT NOT NULL,
assigned_by TEXT,
assigned_at TEXT NOT NULL,
UNIQUE (node_id, gpu_index, user_id)
);
CREATE INDEX IF NOT EXISTS idx_gpu_assignments_user ON gpu_assignments(user_id);
CREATE INDEX IF NOT EXISTS idx_gpu_assignments_gpu ON gpu_assignments(node_id, gpu_index);
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
-- ============================================================================ -- ============================================================================
-- 二、治理表来源002_governance.sql -- 二、治理表来源002_governance.sql
-- ============================================================================ -- ============================================================================
@@ -462,6 +519,8 @@ ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0; ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT; ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_id TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_name TEXT;
-- 按规则推定已有模型的 can_train新库为空表此语句为 no-op -- 按规则推定已有模型的 can_train新库为空表此语句为 no-op
UPDATE models UPDATE models

View File

@@ -234,6 +234,39 @@ CREATE TABLE IF NOT EXISTS resource_sync_jobs (
completed_at TEXT completed_at TEXT
); );
CREATE TABLE IF NOT EXISTS storage_objects (
id TEXT PRIMARY KEY,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
version_id TEXT NOT NULL,
bucket TEXT NOT NULL,
object_key TEXT NOT NULL,
file_name TEXT,
content_type TEXT,
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
);
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
id TEXT PRIMARY KEY,
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
direction TEXT NOT NULL,
status TEXT NOT NULL,
progress INTEGER NOT NULL DEFAULT 0,
local_path TEXT,
error TEXT,
create_time TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
CREATE TABLE IF NOT EXISTS eval_tasks ( CREATE TABLE IF NOT EXISTS eval_tasks (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
name TEXT NOT NULL, name TEXT NOT NULL,
@@ -319,9 +352,14 @@ CREATE TABLE IF NOT EXISTS roles (
CREATE TABLE IF NOT EXISTS sessions ( CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
issued_at TEXT NOT NULL, username TEXT,
expires_at TEXT NOT NULL, login_at TEXT,
ip TEXT logout_at TEXT,
duration_seconds INTEGER,
issued_at TEXT,
expires_at TEXT,
ip TEXT,
create_time TEXT
); );
CREATE TABLE IF NOT EXISTS acls ( CREATE TABLE IF NOT EXISTS acls (

View File

@@ -16,3 +16,5 @@ END;
-- 3. 给 trained_models 增加 artifact_dir训练产物目录扫描结果目录 -- 3. 给 trained_models 增加 artifact_dir训练产物目录扫描结果目录
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT; ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_id TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_name TEXT;

View File

@@ -192,6 +192,27 @@ class ComputeNodeClient:
response.raise_for_status() response.raise_for_status()
return _unwrap_dict(response.json()) return _unwrap_dict(response.json())
async def prepare_cache(self, payload: dict[str, Any]) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), headers=self.headers()) as client:
response = await client.post(
_join_url(self.api_base_url, f"{self.route_prefix}/compute/cache/prepare"),
json=payload,
)
response.raise_for_status()
return _unwrap_dict(response.json())
async def cache_status(self, resource_id: str, version_id: str | None = None) -> dict[str, Any]:
params = {"resource_id": resource_id}
if version_id:
params["version_id"] = version_id
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
response = await client.get(
_join_url(self.api_base_url, f"{self.route_prefix}/compute/cache/status"),
params=params,
)
response.raise_for_status()
return _unwrap_dict(response.json())
async def _request( async def _request(
self, self,
method: str, method: str,

View File

@@ -0,0 +1 @@
"""Central object storage integration."""

View File

@@ -0,0 +1,68 @@
from __future__ import annotations
from datetime import timedelta
from functools import lru_cache
from io import BytesIO
from typing import Any
from minio import Minio
from minio.error import S3Error
from app.core.config import get_settings
class ObjectStorageError(RuntimeError):
pass
class MinioObjectStorage:
def __init__(self) -> None:
settings = get_settings()
endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/")
self.client = Minio(endpoint, access_key=settings.minio_access_key, secret_key=settings.minio_secret_key, secure=settings.minio_secure)
self.bucket = settings.minio_bucket
def _ensure_enabled(self) -> None:
if not get_settings().minio_enabled:
raise ObjectStorageError("MinIO object storage is disabled")
def ensure_bucket(self) -> None:
self._ensure_enabled()
try:
if not self.client.bucket_exists(self.bucket):
self.client.make_bucket(self.bucket)
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str:
self._ensure_enabled()
self.ensure_bucket()
return self.client.presigned_put_object(self.bucket, object_key, expires=timedelta(seconds=expires_seconds))
def presigned_get(self, object_key: str, expires_seconds: int = 3600) -> str:
self._ensure_enabled()
self.ensure_bucket()
return self.client.presigned_get_object(self.bucket, object_key, expires=timedelta(seconds=expires_seconds))
def stat(self, object_key: str) -> dict[str, Any]:
self._ensure_enabled()
self.ensure_bucket()
try:
result = self.client.stat_object(self.bucket, object_key)
return {"object_key": object_key, "byte_size": result.size, "etag": result.etag, "last_modified": result.last_modified.isoformat() if result.last_modified else None}
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
self._ensure_enabled()
self.ensure_bucket()
try:
result = self.client.put_object(self.bucket, object_key, BytesIO(content), len(content), content_type=content_type)
return {"bucket": self.bucket, "object_key": object_key, "etag": result.etag, "byte_size": len(content)}
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
@lru_cache
def get_object_storage() -> MinioObjectStorage:
return MinioObjectStorage()

View File

@@ -8,6 +8,7 @@ psycopg-pool>=3.2.1
alembic>=1.13.1 alembic>=1.13.1
redis>=5.0.4 redis>=5.0.4
httpx>=0.27.0 httpx>=0.27.0
minio>=7.2.7
PyJWT>=2.8.0 PyJWT>=2.8.0
passlib[bcrypt]>=1.7.4 passlib[bcrypt]>=1.7.4
python-dotenv>=1.0.1 python-dotenv>=1.0.1

View File

@@ -11,6 +11,8 @@ import time
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
import httpx
from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
@@ -511,13 +513,27 @@ def create_app() -> FastAPI:
llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory")) llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
gpu_items = gpu_resources() gpu_items = gpu_resources()
torch_cuda = torch_cuda_status() torch_cuda = torch_cuda_status()
storage_available = False
storage_error = ""
try:
data_root.mkdir(parents=True, exist_ok=True)
probe = data_root / ".yg-ft-storage-healthcheck"
probe.write_text(host_id(), encoding="utf-8")
storage_available = probe.read_text(encoding="utf-8").strip() == host_id()
probe.unlink(missing_ok=True)
except Exception as exc: # noqa: BLE001 - health endpoint must remain available
storage_error = str(exc)
return { return {
"status": "ok", "status": "ok" if storage_available else "storage_unavailable",
"api_version": "v1", "api_version": "v1",
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"), "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
"app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true", "app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true",
"data_root": str(data_root), "data_root": str(data_root),
"data_root_exists": data_root.exists(), "data_root_exists": data_root.exists(),
"storage_mode": os.getenv("STORAGE_MODE", "minio-cache"),
"storage_available": storage_available,
"storage_error": storage_error,
"storage_root": str(data_root),
"model_root": os.getenv("YG_FT_MODEL_ROOT", str(data_root / "models")), "model_root": os.getenv("YG_FT_MODEL_ROOT", str(data_root / "models")),
"dataset_root": str(dataset_root), "dataset_root": str(dataset_root),
"dataset_root_exists": dataset_root.exists(), "dataset_root_exists": dataset_root.exists(),
@@ -532,7 +548,7 @@ def create_app() -> FastAPI:
"nvidia_gpu_count": len(gpu_items), "nvidia_gpu_count": len(gpu_items),
"torch_cuda": torch_cuda, "torch_cuda": torch_cuda,
"gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus", "gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus",
"capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling"], "capabilities": ["gpu_discovery", "torch_cuda_diagnostics", "llama_factory", "file_gateway", "job_polling", "storage_health"],
} }
@app.get(f"{route_prefix}/v1/compute/jobs") @app.get(f"{route_prefix}/v1/compute/jobs")
@@ -810,6 +826,82 @@ def create_app() -> FastAPI:
"checksum_sha256": hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "", "checksum_sha256": hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "",
} }
@app.post(f"{route_prefix}/compute/cache/prepare")
async def prepare_cache(payload: dict[str, Any]) -> dict[str, Any]:
"""Download one MinIO object into the node-local cache atomically."""
download_url = str(payload.get("download_url") or "")
resource_id = str(payload.get("resource_id") or "")
version_id = str(payload.get("version_id") or "latest")
if not download_url or not resource_id:
raise HTTPException(status_code=400, detail="download_url and resource_id are required")
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
relative_path = str(payload.get("relative_path") or f"resources/{resource_id}/{version_id}/resource")
target = (cache_root / relative_path.lstrip("/\\")).resolve()
if not _path_inside(cache_root, target):
raise HTTPException(status_code=400, detail="cache path must stay inside cache root")
target.parent.mkdir(parents=True, exist_ok=True)
temp_target = target.with_name(f".{target.name}.part")
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
digest = hashlib.sha256()
byte_size = 0
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
async with client.stream("GET", download_url) as response:
response.raise_for_status()
with temp_target.open("wb") as output:
async for chunk in response.aiter_bytes(1024 * 1024):
output.write(chunk)
digest.update(chunk)
byte_size += len(chunk)
checksum = digest.hexdigest()
if expected_checksum and checksum != expected_checksum:
temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail="cache checksum mismatch")
temp_target.replace(target)
except HTTPException:
raise
except Exception as exc:
temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail=f"cache download failed: {exc}") from exc
return {
"resource_id": resource_id,
"version_id": version_id,
"status": "ready",
"local_path": str(target),
"byte_size": byte_size,
"checksum_sha256": checksum,
}
@app.get(f"{route_prefix}/compute/cache/status")
async def cache_status(resource_id: str = Query(...), version_id: str = Query(default="latest")) -> dict[str, Any]:
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
target = cache_root / "resources" / resource_id / version_id / "resource"
return {
"resource_id": resource_id,
"version_id": version_id,
"status": "ready" if target.is_file() else "missing",
"local_path": str(target),
"byte_size": target.stat().st_size if target.is_file() else 0,
}
@app.delete(f"{route_prefix}/compute/cache")
async def clear_cache(resource_id: str | None = Query(default=None), version_id: str | None = Query(default=None)) -> dict[str, Any]:
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
target = cache_root / "resources"
if resource_id:
target = target / resource_id
if version_id:
target = target / version_id
target = target.resolve()
if not _path_inside(cache_root, target):
raise HTTPException(status_code=400, detail="cache path must stay inside cache root")
if target.exists():
shutil.rmtree(target)
return {"status": "cleared", "resource_id": resource_id, "version_id": version_id}
@app.post(f"{route_prefix}/compute/files/import-local") @app.post(f"{route_prefix}/compute/files/import-local")
async def import_local_file(payload: dict[str, Any]) -> dict[str, Any]: async def import_local_file(payload: dict[str, Any]) -> dict[str, Any]:
source = Path(str(payload.get("source_path") or "")) source = Path(str(payload.get("source_path") or ""))

View File

@@ -4,6 +4,7 @@ python-multipart>=0.0.9
pydantic>=2.7.0 pydantic>=2.7.0
python-dotenv>=1.0.1 python-dotenv>=1.0.1
httpx>=0.27.0 httpx>=0.27.0
# Compute Agent downloads MinIO objects through presigned HTTP URLs; no MinIO SDK is required.
# 模型评测指标 # 模型评测指标
sacrebleu>=2.4.0 sacrebleu>=2.4.0
rouge-score>=0.1.2 rouge-score>=0.1.2

View File

@@ -48,3 +48,13 @@ COMPUTE_STATUS_SYNC_MODE=polling
COMPUTE_POLL_INTERVAL_SECONDS=10 COMPUTE_POLL_INTERVAL_SECONDS=10
COMPUTE_POLL_BATCH_SIZE=100 COMPUTE_POLL_BATCH_SIZE=100
COMPUTE_REQUEST_TIMEOUT_SECONDS=5 COMPUTE_REQUEST_TIMEOUT_SECONDS=5
# MinIO object storage. Enable after the MinIO service is reachable.
MINIO_ENABLED=false
MINIO_ENDPOINT=http://minio:9000
MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=minioadmin
MINIO_BUCKET=yg-ft-resources
MINIO_SECURE=false
STORAGE_WAIT_SECONDS=300
STORAGE_CHECK_INTERVAL_SECONDS=10

View File

@@ -12,7 +12,7 @@ RUN pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple \
&& pip install -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple \ && pip install -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple \
&& rm -f /tmp/requirements.txt && rm -f /tmp/requirements.txt
RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis, jwt, passlib, httpx, alembic; print('backend dependency check ok')" RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis, jwt, passlib, httpx, minio, alembic; print('backend dependency check ok')"
RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft \ RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft \
&& chmod -R 0775 /opt/yg-ft /data/yg-ft && chmod -R 0775 /opt/yg-ft /data/yg-ft

View File

@@ -62,6 +62,14 @@ services:
COMPUTE_POLL_INTERVAL_SECONDS: ${COMPUTE_POLL_INTERVAL_SECONDS:-3} COMPUTE_POLL_INTERVAL_SECONDS: ${COMPUTE_POLL_INTERVAL_SECONDS:-3}
COMPUTE_POLL_BATCH_SIZE: ${COMPUTE_POLL_BATCH_SIZE:-100} COMPUTE_POLL_BATCH_SIZE: ${COMPUTE_POLL_BATCH_SIZE:-100}
COMPUTE_REQUEST_TIMEOUT_SECONDS: ${COMPUTE_REQUEST_TIMEOUT_SECONDS:-5} COMPUTE_REQUEST_TIMEOUT_SECONDS: ${COMPUTE_REQUEST_TIMEOUT_SECONDS:-5}
MINIO_ENABLED: ${MINIO_ENABLED:-false}
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-http://minio:9000}
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
MINIO_SECURE: ${MINIO_SECURE:-false}
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
PYTHONPATH: /app PYTHONPATH: /app
volumes: volumes:
- ../../backend:/app:ro - ../../backend:/app:ro

View File

@@ -8,6 +8,7 @@ ENABLE_DOCS=false
COMPUTE_API_PORT=19100 COMPUTE_API_PORT=19100
FILE_GATEWAY_PORT=19101 FILE_GATEWAY_PORT=19101
COMPUTE_API_IMAGE=yg-ft-compute-api:latest COMPUTE_API_IMAGE=yg-ft-compute-api:latest
YG_FT_CACHE_ROOT=/data/yg-ft
# The application server actively polls Compute API; compute server does not need reverse access. # The application server actively polls Compute API; compute server does not need reverse access.
COMPUTE_AUTH_ENABLED=true COMPUTE_AUTH_ENABLED=true

View File

@@ -30,6 +30,7 @@ services:
CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-all} CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-all}
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility} NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
YG_FT_CACHE_ROOT: ${YG_FT_CACHE_ROOT:-/data/yg-ft}
PYTHONPATH: /app PYTHONPATH: /app
volumes: volumes:
- ../../compute:/app/compute:ro - ../../compute:/app/compute:ro

View File

@@ -0,0 +1,5 @@
MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=change_me_minio_secret
MINIO_API_PORT=9000
MINIO_CONSOLE_PORT=9001
MINIO_DATA_ROOT_HOST=./data

View File

@@ -0,0 +1,19 @@
services:
minio:
image: minio/minio:RELEASE.2025-02-28T09-55-16Z
container_name: yg-ft-minio
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-change_me_minio_secret}
ports:
- "${MINIO_API_PORT:-9000}:9000"
- "${MINIO_CONSOLE_PORT:-9001}:9001"
volumes:
- ${MINIO_DATA_ROOT_HOST:-./data}:/data
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/minio/health/live || exit 1"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped

View File

@@ -0,0 +1,287 @@
# MinIO + Compute Agent 本地缓存方案与开发计划
## 1. 方案结论
本方案用 MinIO 作为模型、数据集、checkpoint、评测结果和训练产物的唯一正式存储算力节点不挂载 NFS也不要求安装 NFS 客户端。
算力节点只保留任务运行所需的本地缓存:
```text
MinIO
-> Backend API 生成资源授权和版本信息
-> Compute Agent 按任务下载到本地缓存
-> LLaMA-Factory / 推理 / 合并任务使用本地路径
-> 任务产物上传回 MinIO
-> PostgreSQL 更新资源和任务状态
```
MinIO 是唯一正式数据源,本地缓存不是正式资源,节点失效或缓存被清理不会丢失模型和数据。
## 2. 为什么适合当前项目
当前项目的 Compute API 已经负责:
- 训练、评测、推理和权重合并任务启动;
- 本地文件网关;
- GPU 和任务状态管理;
- 训练日志和产物路径管理。
因此不需要把 LLaMA-Factory 改成直接读取远程对象,只需要在 Compute Agent 启动任务前准备本地路径,继续把原来的 `model_name_or_path``dataset_dir``output_dir` 传给训练引擎。
## 3. MinIO 部署
MinIO 独立部署在 Linux 存储服务器或专用存储节点:
```text
9000 S3 API
9001 MinIO Console仅管理员网络开放
```
建议创建 bucket
```text
yg-ft-resources
```
对象前缀建议:
```text
models/{model_id}/versions/{version_id}/...
datasets/{dataset_id}/versions/{version_id}/...
outputs/{task_id}/...
evaluations/{task_id}/...
logs/{task_id}/...
```
MinIO 可以使用官方 Docker 镜像,不需要在 Linux 主机安装 MinIO 软件包。需要持久化挂载 MinIO 的数据目录。
## 4. 是否需要额外安装包
### MinIO 服务端
不需要安装额外系统包,使用 Docker 镜像即可:
```text
minio/minio
```
### Backend API
建议增加 Python SDK
```text
minio>=7.2.0
```
Backend 用 SDK 生成预签名上传、下载 URL并负责 bucket、对象元数据和权限控制。
### Compute API / Compute Agent
推荐第一版只使用现有 `httpx` 访问预签名 URL不额外安装 MinIO SDK。流程是
```text
Backend -> 返回预签名 URL
Compute Agent -> httpx 下载/上传
```
这样算力节点不需要 MinIO 客户端、AWS CLI 或 NFS 客户端。
如果后续需要 Agent 直接操作 bucket、列目录或分片上传再增加
```text
minio>=7.2.0
```
但不建议第一阶段让算力节点持有 MinIO 管理密钥。
## 5. 权限模型
继续沿用当前项目的:
- `projects`
- `project_members`
- `acls`
MinIO 只负责对象访问凭证Backend 负责业务授权:
1. 用户请求模型或数据集;
2. Backend 校验项目成员关系和资源 ACL
3. 校验通过后生成短时预签名 URL
4. Compute Agent 使用 URL 下载;
5. URL 过期后自动失效。
MinIO bucket 不直接向前端或普通算力节点开放长期 Access Key。
## 6. 本地缓存目录
Compute API 容器继续以 root 运行,本地缓存挂载到:
```text
/data/yg-ft/cache/models
/data/yg-ft/cache/datasets
/data/yg-ft/cache/adapters
/data/yg-ft/cache/outputs
```
每个缓存资源必须包含:
```text
resource_id
version_id
sha256
byte_size
last_used_at
status
```
缓存状态:
- `missing`:本地不存在;
- `downloading`:正在下载;
- `ready`:校验成功;
- `corrupted`:校验失败;
- `evicting`:正在清理。
任务只能使用 `ready` 状态的缓存。
## 7. 任务流程
### 7.1 训练
```text
校验项目/用户权限
-> 获取基座模型版本和数据集版本
-> 检查本地缓存
-> 缺失则下载并校验 SHA256
-> 启动 LLaMA-Factory
-> checkpoint 写入本地临时目录
-> 任务完成后上传 outputs 到 MinIO
-> MinIO 上传完成并校验后更新数据库
```
### 7.2 推理
```text
校验模型权限
-> 下载或复用本地模型缓存
-> 使用本地模型路径加载
-> 推理服务只绑定当前节点缓存
```
### 7.3 权重合并
```text
下载 base model 和 adapter/checkpoint
-> 在指定节点执行 CPU 合并
-> 上传 merged model 到 MinIO
-> 数据库记录新的模型版本
```
### 7.4 NFS 故障规则对应关系
MinIO 不可达时,节点不再依赖本地残留文件直接启动新任务:
- 已有完整缓存且资源版本仍有效:允许继续执行当前任务;
- 新任务无法确认资源版本:等待 MinIO 恢复;
- 等待超过配置窗口:任务失败;
- 产物无法上传:任务不得标记为最终成功,进入 `storage_error`
如果严格执行“共享存储故障时节点不能正常工作”,则即使本地缓存完整,也应禁止启动新任务。建议当前项目采用这一严格规则。
## 8. 数据库建议
现有 `resource_replicas` 可扩展为缓存索引,建议增加:
```text
storage_backend -- minio
storage_bucket
storage_object_key
version_id
cache_path
cache_status
last_used_at
download_progress
```
`resource_sync_jobs` 可继续用于下载和上传任务,但建议增加方向字段:
```text
direction -- download / upload
```
模型、数据集、checkpoint 和评测结果均使用 `resource_id + version_id`,不再把节点本地路径作为唯一资源标识。
## 9. 开发计划
### 阶段一MinIO 服务和配置
- 增加 `docker/minio/docker-compose.yml`
- 配置 MinIO endpoint、bucket、Access Key 和 Secret Key
- 增加 Backend `minio` 依赖;
- 增加 MinIO 健康检查;
- 创建统一 bucket 和对象前缀规则。
### 阶段二Backend 资源服务
- 实现对象上传、下载、删除和 HEAD 校验;
- 生成短时预签名 URL
- 接入项目成员和 ACL 校验;
- 建立资源版本、SHA256 和大小记录;
- 上传成功后再更新数据库资源状态。
### 阶段三Compute Agent 缓存服务
- 增加缓存目录管理器;
- 实现预签名 URL 下载;
- 支持临时文件下载和原子改名;
- 实现 SHA256 校验、失败重试和断点续传;
- 增加缓存状态查询和清理接口。
### 阶段四:接入业务任务
- 训练前准备基座模型和数据集;
- 推理前准备模型和 adapter
- 权重合并前准备 base model 和 checkpoint
- 评测前准备模型和数据集;
- 训练产物、合并模型和评测结果上传 MinIO
- MinIO 故障时统一等待并超时失败。
### 阶段五:前端和管理员功能
- 显示资源版本和对象存储状态;
- 显示节点缓存状态;
- 支持手动预热模型;
- 支持缓存清理;
- 显示下载、上传和校验失败原因。
### 阶段六:测试和切换
- 单节点下载和缓存复用测试;
- 多节点同时下载同一模型测试;
- MinIO 重启和网络中断测试;
- SHA256 损坏文件测试;
- 训练、推理、权重合并全流程测试;
- 关闭旧的逐节点上传逻辑。
## 10. 预计工作量
```text
MinIO 部署和配置 12 人日
Backend 对象存储服务 47 人日
Compute Agent 缓存 610 人日
训练/推理/合并/评测接入 815 人日
权限、数据库和前端 510 人日
故障和回归测试 58 人日
总计 2952 人日
```
## 11. 推荐结论
当前项目建议采用 MinIO + HTTP 预签名 URL + Compute Agent 本地缓存:
- MinIO 服务端使用 Docker不安装主机软件包
- Backend 增加 `minio` Python SDK
- Compute Agent 第一阶段继续使用现有 `httpx`,不增加 MinIO SDK
- 算力节点不安装 NFS 客户端;
- Compute API 继续以 root 运行;
- 训练、推理和权重合并继续使用本地路径,改造风险低于直接让训练框架读取对象存储。