YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 推理与评测支持多卡 GPU 选择，训练任务实时 GPU 监控

Browse Source 
- 新增 GPU 选择归一化 helper，统一前端各形态的选择（gpu_indices/gpus/gpu_id）
- 评测与推理支持同一节点内多卡选择，校验所选 GPU 空闲后再派发
- 新增 /fine-tune/{id}/gpu-status 接口，训练日志页展示实时 GPU 指标
- 算力节点推理加载支持 CUDA_VISIBLE_DEVICES 多卡可见，nvidia-smi 进程级监控
- GPU 占用跟踪细化为按卡记录，覆盖评测任务、推理模型与对比任务

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 10:44:56 +08:00
parent c3e96ae61b
commit 9379f93633
13 changed files with 360 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
backend/app/db/platform_store.py
View File

@@ -454,15 +454,21 @@ class PlatformStore:
self.ensure_seed_data()
# Track which compute nodes have an active inference model loaded
self._inference_nodes: set[str] = set()
self._inference_gpu_indexes: dict[str, set[int]] = {}
self._last_runtime_refresh = 0.0
# ── inference node tracking ────────────────────────────────────
def mark_inference_loaded(self, node_id: str) -> None:
def mark_inference_loaded(self, node_id: str, gpu_indexes: list[int] | None = None) -> None:
self._inference_nodes.add(node_id)
if gpu_indexes is not None:
self._inference_gpu_indexes[node_id] = {int(item) for item in gpu_indexes}
else:
self._inference_gpu_indexes.pop(node_id, None)
def mark_inference_unloaded(self, node_id: str) -> None:
self._inference_nodes.discard(node_id)
self._inference_gpu_indexes.pop(node_id, None)
def is_inference_loaded(self, node_id: str) -> bool:
return node_id in self._inference_nodes
@@ -2766,7 +2772,35 @@ class PlatformStore:
"SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')",
(node_id,),
).fetchall()
return {int(row["gpu_index"]) for row in rows}
active = {int(row["gpu_index"]) for row in rows}
# Evaluation jobs use the same Compute ProcessManager GPU lock but do
# not have fine-tune allocation rows; derive their selected cards here
# so a training task cannot race onto an evaluation GPU.
for row in conn.execute(
"SELECT payload FROM eval_tasks WHERE status IN ('syncing','queued','running')"
).fetchall():
payload = json_loads(row["payload"], {})
if payload.get("compute_node_id") != node_id:
continue
selected = payload.get("gpu_indices") or payload.get("gpus")
if selected is None and payload.get("gpu_id") is not None:
selected = [payload.get("gpu_id")]
active.update(int(item) for item in selected or [])
# Loaded inference models also reserve only their selected cards.
active.update(self._inference_gpu_indexes.get(node_id, set()))
for row in conn.execute("SELECT payload FROM compare_tasks").fetchall():
payload = json_loads(row["payload"], {})
load_status = payload.get("load_status") or {}
if isinstance(load_status, str):
load_status = json_loads(load_status, {})
for item in load_status.get("loaded_models") or []:
if item.get("node_id") != node_id or item.get("status") not in {"starting", "ready", "running"}:
continue
selected = item.get("gpu_indices") or item.get("gpus")
if selected is None and item.get("gpu_id") is not None:
selected = [item.get("gpu_id")]
active.update(int(gpu) for gpu in selected or [])
return active
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
@@ -3195,6 +3229,9 @@ class PlatformStore:
# 推理模型占用算力节点同样计入：优先从 compare_tasks 持久化状态派生
# （重启后仍准确），并用内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3204,8 +3241,13 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
for nid in inference_node_ids:
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
for nid in set(inference_node_ids) | set(inference_gpu_indexes):
running_map[nid] = running_map.get(nid, 0) + 1
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
return [
@@ -3434,6 +3476,9 @@ class PlatformStore:
# 推理模型占用的节点：优先从 compare_tasks 持久化状态派生（重启后仍准确），
# 内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3443,7 +3488,12 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
items = []
for row in rows:
task = next(
@@ -3459,7 +3509,10 @@ class PlatformStore:
t
for t in eval_running
if t.get("compute_node_id") == row["node_id"]
and row["gpu_index"] == (int(t["gpu_id"]) if t.get("gpu_id") is not None else -1)
and row["gpu_index"] in {
int(item)
for item in (t.get("gpu_indices") or t.get("gpus") or ([t["gpu_id"]] if t.get("gpu_id") is not None else []))
}
),
None,
)
@@ -3468,7 +3521,8 @@ class PlatformStore:
eval_task is not None and eval_task.get("status") in {"syncing", "queued"}
)
# Also mark GPU as busy if an inference model is loaded on this node
if row["node_id"] in inference_node_ids and not busy:
inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set())
if inference_on_gpu and not busy:
busy = True
reserved = False
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)