YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增 MinIO 对象存储与算力节点缓存预下载

Browse Source 
- 后端新增 storage 模块（minio_store），支持 MinIO 预签名 URL 上传与对象管理
- config 新增 MinIO 及存储等待相关配置项
- 算力节点新增 /compute/cache/prepare 缓存预下载接口（带校验和原子落盘）
- 算力节点健康接口增加存储可用性探针
- SQL 迁移补充资源存储相关表结构
- Docker 新增 minio 服务与后端 minio 配置
- 补充 minio-compute-cache-plan 设计文档

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-11 16:25:18 +08:00
parent a12f80492d
commit b5d2cd7935
20 changed files with 830 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
backend/app/db/platform_store.py
View File

@@ -505,6 +505,20 @@ class PlatformStore:
)
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
self._ensure_columns(
conn,
"sessions",
{
"username": "TEXT",
"login_at": "TEXT",
"logout_at": "TEXT",
"duration_seconds": "INTEGER",
"issued_at": "TEXT",
"expires_at": "TEXT",
"ip": "TEXT",
"create_time": "TEXT",
},
)
self._ensure_columns(
conn,
"trained_models",
@@ -525,7 +539,12 @@ class PlatformStore:
},
)
schema_dir = Path(__file__).with_name("sql")
for extra in ("002_governance.sql", "003_tenant_quota.sql", "004_permissions.sql"):
for extra in (
"002_governance.sql",
"003_model_path_governance.sql",
"003_tenant_quota.sql",
"004_permissions.sql",
):
extra_path = schema_dir / extra
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
@@ -2863,6 +2882,54 @@ class PlatformStore:
result.append(dict(row))
return result
def create_storage_object(self, payload: dict[str, Any]) -> dict[str, Any]:
object_id = str(payload.get("id") or new_id("object"))
with self.connect() as conn:
conn.execute(
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
row = conn.execute(
"SELECT * FROM storage_objects WHERE resource_type=? AND resource_id=? AND version_id=? AND object_key=?",
(payload["resource_type"], payload["resource_id"], payload["version_id"], payload["object_key"]),
).fetchone()
return dict(row)
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM storage_objects WHERE resource_type=? AND resource_id=? AND status='available' ORDER BY version_id, object_key",
(resource_type, resource_id),
).fetchall()
return [dict(row) for row in rows]
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
if fields:
assignments = ", ".join(f"{key}=?" for key in fields)
with self.connect() as conn:
conn.execute(f"UPDATE storage_objects SET {assignments} WHERE id=?", (*fields.values(), object_id))
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_resource_replica_sync_result(
self,
replica_id: str,

65
backend/app/db/sql/000_full_init.sql
View File

@@ -276,6 +276,39 @@ CREATE TABLE IF NOT EXISTS resource_sync_jobs (
completed_at TEXT
);
CREATE TABLE IF NOT EXISTS storage_objects (
id TEXT PRIMARY KEY,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
version_id TEXT NOT NULL,
bucket TEXT NOT NULL,
object_key TEXT NOT NULL,
file_name TEXT,
content_type TEXT,
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
);
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
id TEXT PRIMARY KEY,
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
direction TEXT NOT NULL,
status TEXT NOT NULL,
progress INTEGER NOT NULL DEFAULT 0,
local_path TEXT,
error TEXT,
create_time TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
CREATE TABLE IF NOT EXISTS eval_tasks (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -361,9 +394,14 @@ CREATE TABLE IF NOT EXISTS roles (
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
issued_at TEXT NOT NULL,
expires_at TEXT NOT NULL,
ip TEXT
username TEXT,
login_at TEXT,
logout_at TEXT,
duration_seconds INTEGER,
issued_at TEXT,
expires_at TEXT,
ip TEXT,
create_time TEXT
);
CREATE TABLE IF NOT EXISTS acls (
@@ -376,6 +414,25 @@ CREATE TABLE IF NOT EXISTS acls (
create_time TEXT
);
-- ---- 权限扩展（来源：004_permissions.sql） ----
CREATE TABLE IF NOT EXISTS gpu_assignments (
id TEXT PRIMARY KEY,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
gpu_index INTEGER NOT NULL,
user_id TEXT NOT NULL,
assigned_by TEXT,
assigned_at TEXT NOT NULL,
UNIQUE (node_id, gpu_index, user_id)
);
CREATE INDEX IF NOT EXISTS idx_gpu_assignments_user ON gpu_assignments(user_id);
CREATE INDEX IF NOT EXISTS idx_gpu_assignments_gpu ON gpu_assignments(node_id, gpu_index);
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
-- ============================================================================
-- 二、治理表（来源：002_governance.sql）
-- ============================================================================
@@ -462,6 +519,8 @@ ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_id TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_name TEXT;
-- 按规则推定已有模型的 can_train（新库为空表，此语句为 no-op）
UPDATE models

44
backend/app/db/sql/001_platform_runtime.sql
View File

@@ -234,6 +234,39 @@ CREATE TABLE IF NOT EXISTS resource_sync_jobs (
completed_at TEXT
);
CREATE TABLE IF NOT EXISTS storage_objects (
id TEXT PRIMARY KEY,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
version_id TEXT NOT NULL,
bucket TEXT NOT NULL,
object_key TEXT NOT NULL,
file_name TEXT,
content_type TEXT,
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
);
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
id TEXT PRIMARY KEY,
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
direction TEXT NOT NULL,
status TEXT NOT NULL,
progress INTEGER NOT NULL DEFAULT 0,
local_path TEXT,
error TEXT,
create_time TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
CREATE TABLE IF NOT EXISTS eval_tasks (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -319,9 +352,14 @@ CREATE TABLE IF NOT EXISTS roles (
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
issued_at TEXT NOT NULL,
expires_at TEXT NOT NULL,
ip TEXT
username TEXT,
login_at TEXT,
logout_at TEXT,
duration_seconds INTEGER,
issued_at TEXT,
expires_at TEXT,
ip TEXT,
create_time TEXT
);
CREATE TABLE IF NOT EXISTS acls (

2
backend/app/db/sql/003_model_path_governance.sql
View File

@@ -16,3 +16,5 @@ END;
-- 3. 给 trained_models 增加 artifact_dir（训练产物目录扫描结果目录）
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_id TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS compute_node_name TEXT;