fix: 修复 GPU 选择索引错误及 CUDA 不可用问题,优化 GPU 硬件单选

- 修复 FineTuneCreateView GPU 选择使用 v-for idx 替代真实 gpu.id 导致
  多节点环境下 GPU 索引错误(如 gpu-node-02 仅 GPU 0 但请求 GPU 1)
- GPU 选择改为单选模式,已离线节点自动过滤不展示
- GPU 卡片增加节点编号展示
- 修复 CUDA_VISIBLE_DEVICES=all 无效值导致 torch.cuda.is_available() False
  改为 CUDA_VISIBLE_DEVICES=0
- .gitignore 新增 .claude/ CLAUDE.md 排除规则
- GpuInfo 类型增加 node_id/node_code/node_name 字段

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-07-28 12:43:20 +08:00
parent 3fd9cf9100
commit 525fc55cef
4 changed files with 43 additions and 23 deletions

2
.gitignore vendored
View File

@@ -157,6 +157,8 @@ backend/config.yaml
.codex-backups/
.pnpm-store/
.zcode/
.claude/
CLAUDE.md
# Spyder project settings
.spyderproject

View File

@@ -35,6 +35,6 @@ COMPUTE_GPU_MEMORY_GB=80
COMPUTE_GPU_POWER_LIMIT_W=300
LOG_DIR=/opt/yg-ft/logs/compute
CUDA_VISIBLE_DEVICES=all
CUDA_VISIBLE_DEVICES=0
NVIDIA_VISIBLE_DEVICES=all
NVIDIA_DRIVER_CAPABILITIES=compute,utility

View File

@@ -362,6 +362,9 @@ export interface GpuInfo {
fan_speed?: number
clock_mhz?: number
driver_version?: string
node_id?: string
node_code?: string
node_name?: string
}
export interface SystemInfo {

View File

@@ -15,6 +15,7 @@ import {
import { getModelList } from '@/api/modules/model'
import { getDatasetList } from '@/api/modules/dataset'
import { getSystemInfo } from '@/api/modules/system'
import { getComputeNodes } from '@/api/modules/compute'
import { TEMPLATE_GROUPS, LR_SCHEDULER_OPTIONS, QUANTIZATION_BIT_OPTIONS, QUANT_METHOD_OPTIONS, GGUF_FORMAT_OPTIONS } from '@/constants'
import {
DEFAULT_TRAINING_PARAMS,
@@ -35,7 +36,18 @@ const preflightResult = ref<FineTunePreflightResult | null>(null)
const models = ref<ModelItem[]>([])
const datasets = ref<DatasetItem[]>([])
const gpus = ref<GpuInfo[]>([])
const selectedGpus = ref<number[]>([])
const computeNodes = ref<Array<{ id: string; scheduler_status?: string }>>([])
const selectedGpuId = ref<number | null>(null)
/** Only show GPUs from nodes that are online or draining */
const availableGpus = computed(() => {
const onlineNodeIds = new Set(
computeNodes.value
.filter((n) => n.scheduler_status === 'online' || n.scheduler_status === 'draining')
.map((n) => n.id),
)
return gpus.value.filter((gpu) => !gpu.node_id || onlineNodeIds.has(gpu.node_id))
})
const modelDialogVisible = ref(false)
const form = reactive(createDefaultFineTuneForm())
@@ -62,7 +74,8 @@ const selectedModel = computed(() => models.value.find((model) => model.id === f
const modelDialogTitle = computed(() => selectedModel.value?.name || '')
/** 训练命令与提交载荷共用同一份表单模型。 */
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpus.value))
const selectedGpuIds = computed(() => (selectedGpuId.value != null ? [selectedGpuId.value] : []))
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpuIds.value))
const remoteCommandPreview = computed(() => {
const command = preflightResult.value?.preview?.command
@@ -70,11 +83,9 @@ const remoteCommandPreview = computed(() => {
return preflightResult.value?.preview?.command_text || ''
})
/** GPU 选切换 */
function toggleGpu(index: number) {
const idx = selectedGpus.value.indexOf(index)
if (idx === -1) selectedGpus.value.push(index)
else selectedGpus.value.splice(idx, 1)
/** GPU 选切换(每次只选中一张 GPU */
function toggleGpu(gpuId: number) {
selectedGpuId.value = selectedGpuId.value === gpuId ? null : gpuId
}
function gpuUsageWidth(percent: number) {
@@ -164,10 +175,11 @@ async function loadDatasets() {
async function loadGpus() {
try {
const sys = await getSystemInfo()
const [sys, nodes] = await Promise.all([getSystemInfo(), getComputeNodes().catch(() => [])])
gpus.value = sys?.gpu || []
// 默认选中第一个
if (gpus.value.length > 0) selectedGpus.value = [0]
computeNodes.value = nodes || []
// Default select first available GPU
if (availableGpus.value.length > 0) selectedGpuId.value = availableGpus.value[0].id ?? null
} catch {
gpus.value = []
}
@@ -177,8 +189,8 @@ async function handleSubmit() {
if (!formRef.value) return
await formRef.value.validate(async (valid) => {
if (!valid) return
if (selectedGpus.value.length === 0) {
ElMessage.warning('请至少选择一个 GPU')
if (selectedGpuId.value == null) {
ElMessage.warning('请选择一个 GPU')
return
}
submitting.value = true
@@ -195,7 +207,7 @@ async function handleSubmit() {
return
}
const payload = buildFineTunePayload(form, selectedGpus.value)
const payload = buildFineTunePayload(form, selectedGpuIds.value)
const preflight = await runPreflight(payload)
if (!preflight?.valid) {
ElMessage.error('训练预检未通过,请先处理预检问题')
@@ -220,7 +232,7 @@ async function handleSubmit() {
})
}
async function runPreflight(payload = buildFineTunePayload(form, selectedGpus.value)) {
async function runPreflight(payload = buildFineTunePayload(form, selectedGpuIds.value)) {
preflightLoading.value = true
try {
const result = await preflightFineTune(payload)
@@ -249,8 +261,8 @@ async function handlePreflightClick() {
if (!formRef.value) return
await formRef.value.validate(async (valid) => {
if (!valid) return
if (selectedGpus.value.length === 0) {
ElMessage.warning('请至少选择一个 GPU')
if (selectedGpuId.value == null) {
ElMessage.warning('请选择一个 GPU')
return
}
await runPreflight()
@@ -286,15 +298,18 @@ onMounted(() => {
<el-form-item label="GPU 硬件">
<div class="gpu-list">
<div
v-for="(gpu, idx) in gpus"
:key="idx"
v-for="gpu in availableGpus"
:key="gpu.id"
class="gpu-card"
:class="{ active: selectedGpus.includes(idx), 'is-busy': gpu.gpu_percent > 80 }"
@click="toggleGpu(idx)"
:class="{ active: selectedGpuId === gpu.id, 'is-busy': gpu.gpu_percent > 80 }"
@click="toggleGpu(gpu.id!)"
>
<div class="gpu-card-top">
<div class="gpu-title">
<span class="gpu-index">GPU-{{ idx }}</span>
<span class="gpu-index">
GPU-{{ gpu.id }}
<template v-if="gpu.node_code"> · {{ gpu.node_code }}</template>
</span>
<span class="gpu-name">{{ gpu.name }}</span>
</div>
<span class="gpu-usage">{{ gpu.gpu_percent }}%</span>
@@ -308,7 +323,7 @@ onMounted(() => {
<span>{{ gpu.power_w }}W</span>
</div>
</div>
<div v-if="!gpus.length" class="gpu-empty">暂无 GPU 信息</div>
<div v-if="!availableGpus.length" class="gpu-empty">暂无可用 GPU请检查算力节点是否在线</div>
</div>
</el-form-item>