YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 权限与日志治理完善，MinIO 独立部署与 tiktoken 离线打包适配

Browse Source 
- 后端：强化平台/审批/资源/系统接口权限校验与操作日志，更新权限设计文档与测试用例
- 存储：新增 MinIO 独立部署适配（端口 19000/19001），外部端点与 host-gateway 互通
- 离线：打包 tiktoken cl100k_base 词表进镜像，避免无网环境联网下载
- 其他：算力节点接口微调，前端微调创建页小修，忽略 MinIO 运行时数据

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-12 15:21:23 +08:00
parent 2f64086177
commit 5ecca9f0bc
26 changed files with 101600 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
backend/app/db/platform_store.py
View File

@@ -526,8 +526,15 @@ class PlatformStore:
"artifact_dir": "TEXT",
"compute_node_id": "TEXT",
"compute_node_name": "TEXT",
"created_by": "TEXT",
"tenant_id": "TEXT",
"project_id": "TEXT",
"deleted_at": "TEXT",
"deleted_by": "TEXT",
},
)
for table in ("models", "datasets", "eval_tasks"):
self._ensure_columns(conn, table, {"deleted_at": "TEXT", "deleted_by": "TEXT", "tenant_id": "TEXT", "project_id": "TEXT"})
self._ensure_columns(
conn,
"resource_replicas",
@@ -1470,12 +1477,12 @@ class PlatformStore:
def delete_model(self, model_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM models WHERE id=?", (model_id,))
conn.execute("UPDATE models SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", model_id))
def trained_models(self) -> list[dict[str, Any]]:
self.refresh_runtime_state()
with self.connect() as conn:
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
rows = conn.execute("SELECT * FROM trained_models WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall()
items = []
for row in rows:
item = {
@@ -1498,7 +1505,7 @@ class PlatformStore:
def delete_trained_model(self, model_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM trained_models WHERE id=? OR name=?", (model_id, model_id))
conn.execute("UPDATE trained_models SET deleted_at=?, deleted_by=? WHERE id=? OR name=?", (utcnow(), "system", model_id, model_id))
def datasets(self) -> list[dict[str, Any]]:
with self.connect() as conn:
@@ -1507,6 +1514,7 @@ class PlatformStore:
FROM datasets dataset
LEFT JOIN data_process_tasks task
ON task.id=COALESCE(dataset.source_task_id, dataset.task_id)
WHERE dataset.deleted_at IS NULL
ORDER BY dataset.create_time DESC"""
).fetchall()
return [self._dataset(conn, row) for row in rows]
@@ -1646,8 +1654,7 @@ class PlatformStore:
def delete_dataset(self, dataset_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,))
conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,))
conn.execute("UPDATE datasets SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", dataset_id))
def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]:
now = utcnow()
@@ -2004,13 +2011,14 @@ class PlatformStore:
task_id = str(payload.get("task_id") or payload.get("id"))
current = self.task(task_id)
merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8}
selected_gpus = payload.get("gpus") or merged.get("gpus") or [0]
selected_gpus = payload.get("gpus") or merged.get("gpus") or []
process_id = int(43000 + (time.time() % 10000))
with self.connect() as conn:
owner = f"start:{task_id}:{uuid.uuid4().hex[:8]}"
if not self._acquire_scheduler_lock(conn, "compute-scheduler", owner):
raise RuntimeError("compute scheduler is busy, please retry")
node = self._schedule_node_locked(conn, payload)
selected_gpus = list(node.get("selected_gpus") or selected_gpus)
sync_job_id = new_id("sync")
conn.execute(
"""
@@ -2094,7 +2102,7 @@ class PlatformStore:
task = self.task(task_id)
merged = {**task, **(payload or {}), "id": task_id}
node = self.select_compute_node(merged)
selected_gpus = merged.get("gpus") or [0]
selected_gpus = merged.get("gpus") or []
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
def prepare_compute_job_payload_from_payload(self, payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -2108,7 +2116,7 @@ class PlatformStore:
"progress": int(payload.get("progress") or 0),
}
node = self.select_compute_node(transient_task)
selected_gpus = transient_task.get("gpus") or [0]
selected_gpus = transient_task.get("gpus") or []
return node, self._compute_job_payload_from_task_node(transient_task, node, selected_gpus)
def _compute_job_payload_from_task_node(
@@ -2260,7 +2268,7 @@ class PlatformStore:
for item in runtime_files
],
"output_dir": output_dir,
"gpus": selected_gpus or task.get("gpus") or [0],
"gpus": selected_gpus if selected_gpus is not None else (task.get("gpus") or []),
"compute_node_id": node["id"],
"compute_node_code": node["code"],
}
@@ -2270,7 +2278,7 @@ class PlatformStore:
node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None)
if not node:
raise RuntimeError("compute node not found")
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [0])
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [])
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
status_map = {
@@ -2493,7 +2501,7 @@ class PlatformStore:
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running."""
@@ -2712,27 +2720,71 @@ class PlatformStore:
def _node_capacity(self, node: dict[str, Any]) -> int:
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
@staticmethod
def _payload_gpu_indexes(payload: dict[str, Any]) -> list[int]:
raw = payload.get("gpu_indices")
if raw is None:
raw = payload.get("gpus")
if raw is None:
return []
try:
values = [int(item) for item in raw]
except (TypeError, ValueError) as exc:
raise RuntimeError("invalid GPU index") from exc
if any(item < 0 for item in values):
raise RuntimeError("GPU index must be non-negative")
return sorted(set(values))
def _select_node_gpus(
self,
conn: PgConnection,
node: dict[str, Any],
requested: list[int],
payload: dict[str, Any],
) -> list[int]:
available = self._node_gpu_indexes(conn, node)
active = self._active_gpu_indexes(conn, node["id"])
allowed = payload.get("allowed_gpu_indices")
if allowed is not None:
available &= {int(item) for item in allowed}
if requested:
selected = set(requested)
if not selected.issubset(available):
raise RuntimeError(f"requested GPU is not available on compute node {node['code']}")
if selected.intersection(active):
raise RuntimeError(f"requested GPU is busy on compute node {node['code']}")
return sorted(selected)
if payload.get("allow_cpu") or payload.get("device") == "cpu":
return []
count = max(1, int(payload.get("gpu_count") or 1))
free = sorted(available - active)
if len(free) < count:
raise RuntimeError(f"compute node {node['code']} has only {len(free)} available GPU(s)")
return free[:count]
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
requested_gpus = [int(item) for item in payload.get("gpus") or []]
requested_gpus = self._payload_gpu_indexes(payload)
nodes = self._compute_nodes_locked(conn)
candidates = [
n
for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
]
if requested_gpus:
requested_gpu_set = set(requested_gpus)
candidates = [
node
for node in candidates
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
]
if requested:
selected = next((n for n in candidates if n["id"] == requested), None)
if selected:
return selected
if not selected:
raise RuntimeError("selected compute node is unavailable")
selected["selected_gpus"] = self._select_node_gpus(conn, selected, requested_gpus, payload)
return selected
filtered = []
for node in candidates:
try:
node["selected_gpus"] = self._select_node_gpus(conn, node, requested_gpus, payload)
filtered.append(node)
except RuntimeError:
continue
candidates = filtered
if not candidates:
if not nodes:
raise RuntimeError("no available compute node: no compute node configured")
@@ -2744,6 +2796,8 @@ class PlatformStore:
reason = f"status={node['scheduler_status']}"
elif node["current_running_jobs"] >= self._node_capacity(node):
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
elif requested:
reason = "selected node unavailable"
else:
reason = "not selected"
reasons.append(f"{node['code']}({reason})")
@@ -2930,6 +2984,43 @@ class PlatformStore:
raise KeyError(object_id)
return dict(row)
def create_storage_cache_job(self, payload: dict[str, Any]) -> dict[str, Any]:
job_id = str(payload.get("id") or new_id("cache"))
with self.connect() as conn:
conn.execute(
"""
INSERT INTO storage_cache_jobs
(id, storage_object_id, node_id, direction, status, progress, local_path, error, create_time, completed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(job_id, payload["storage_object_id"], payload["node_id"], payload.get("direction", "download"),
payload.get("status", "running"), int(payload.get("progress", 0)), payload.get("local_path"),
payload.get("error"), payload.get("create_time") or utcnow(), payload.get("completed_at")),
)
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
return dict(row)
def update_storage_cache_job(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "progress", "local_path", "error", "completed_at"}
fields = {key: value for key, value in payload.items() if key in allowed}
if fields:
assignments = ", ".join(f"{key}=?" for key in fields)
with self.connect() as conn:
conn.execute(f"UPDATE storage_cache_jobs SET {assignments} WHERE id=?", (*fields.values(), job_id))
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
if not row:
raise KeyError(job_id)
return dict(row)
def storage_cache_jobs_for_node(self, node_id: str, limit: int = 100) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM storage_cache_jobs WHERE node_id=? ORDER BY create_time DESC LIMIT ?",
(node_id, max(1, min(limit, 500))),
).fetchall()
return [dict(row) for row in rows]
def update_resource_replica_sync_result(
self,
replica_id: str,
@@ -3613,9 +3704,9 @@ class PlatformStore:
login_at = utcnow()
with self.connect() as conn:
conn.execute(
"INSERT INTO sessions (id, user_id, username, login_at, create_time) "
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s)",
(sid, user_id, user_id, login_at, login_at),
"INSERT INTO sessions (id, user_id, username, login_at, issued_at, expires_at, create_time) "
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s, %s, %s)",
(sid, user_id, user_id, login_at, login_at, datetime.fromtimestamp(time.time() + 1800, timezone.utc).isoformat(), login_at),
)
return {"session_id": sid, "user_id": user_id, "login_at": login_at}
@@ -4263,6 +4354,14 @@ class PlatformStore:
assigned = {r["gpu_index"] for r in rows}
return all(idx in assigned for idx in gpu_indices)
def assigned_gpu_indexes(self, user_id: str, node_id: str) -> list[int]:
with self.connect() as conn:
rows = conn.execute(
"SELECT gpu_index FROM gpu_assignments WHERE user_id=? AND node_id=? ORDER BY gpu_index",
(user_id, node_id),
).fetchall()
return [int(row["gpu_index"]) for row in rows]
# ===================== 平台治理：资源可见性过滤 =====================
def _filter_accessible_ids(

36
backend/app/db/sql/000_full_init.sql
View File

@@ -64,7 +64,12 @@ CREATE TABLE IF NOT EXISTS models (
api_key TEXT,
online_model_name TEXT,
can_train INTEGER NOT NULL DEFAULT 0,
create_time TEXT NOT NULL
create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE TABLE IF NOT EXISTS trained_models (
@@ -78,7 +83,12 @@ CREATE TABLE IF NOT EXISTS trained_models (
merged_path TEXT,
artifact_dir TEXT,
compute_node_id TEXT,
compute_node_name TEXT
compute_node_name TEXT,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE TABLE IF NOT EXISTS model_lineage (
@@ -130,7 +140,12 @@ CREATE TABLE IF NOT EXISTS datasets (
size TEXT,
count INTEGER NOT NULL DEFAULT 0,
description TEXT,
create_time TEXT NOT NULL
create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE TABLE IF NOT EXISTS dataset_files (
@@ -413,6 +428,18 @@ CREATE TABLE IF NOT EXISTS acls (
permission TEXT NOT NULL,
create_time TEXT
);
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE UNIQUE INDEX IF NOT EXISTS uq_acls_subject_permission
ON acls(resource_type, resource_id, principal_type, principal_id, permission);
CREATE INDEX IF NOT EXISTS idx_acls_resource ON acls(resource_type, resource_id);
CREATE INDEX IF NOT EXISTS idx_acls_principal ON acls(principal_type, principal_id);
-- ---- 权限扩展（来源：004_permissions.sql） ----
@@ -475,6 +502,9 @@ CREATE TABLE IF NOT EXISTS approval_steps (
comment TEXT,
time TEXT
);
CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instances(applicant_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
CREATE TABLE IF NOT EXISTS audit_logs (
id TEXT PRIMARY KEY,