fix: 修复 GPU 选择索引错误及 CUDA 不可用问题,优化 GPU 硬件单选
- 修复 FineTuneCreateView GPU 选择使用 v-for idx 替代真实 gpu.id 导致 多节点环境下 GPU 索引错误(如 gpu-node-02 仅 GPU 0 但请求 GPU 1) - GPU 选择改为单选模式,已离线节点自动过滤不展示 - GPU 卡片增加节点编号展示 - 修复 CUDA_VISIBLE_DEVICES=all 无效值导致 torch.cuda.is_available() False 改为 CUDA_VISIBLE_DEVICES=0 - .gitignore 新增 .claude/ CLAUDE.md 排除规则 - GpuInfo 类型增加 node_id/node_code/node_name 字段 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -157,6 +157,8 @@ backend/config.yaml
|
||||
.codex-backups/
|
||||
.pnpm-store/
|
||||
.zcode/
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
||||
@@ -35,6 +35,6 @@ COMPUTE_GPU_MEMORY_GB=80
|
||||
COMPUTE_GPU_POWER_LIMIT_W=300
|
||||
|
||||
LOG_DIR=/opt/yg-ft/logs/compute
|
||||
CUDA_VISIBLE_DEVICES=all
|
||||
CUDA_VISIBLE_DEVICES=0
|
||||
NVIDIA_VISIBLE_DEVICES=all
|
||||
NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
|
||||
@@ -362,6 +362,9 @@ export interface GpuInfo {
|
||||
fan_speed?: number
|
||||
clock_mhz?: number
|
||||
driver_version?: string
|
||||
node_id?: string
|
||||
node_code?: string
|
||||
node_name?: string
|
||||
}
|
||||
|
||||
export interface SystemInfo {
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import { getModelList } from '@/api/modules/model'
|
||||
import { getDatasetList } from '@/api/modules/dataset'
|
||||
import { getSystemInfo } from '@/api/modules/system'
|
||||
import { getComputeNodes } from '@/api/modules/compute'
|
||||
import { TEMPLATE_GROUPS, LR_SCHEDULER_OPTIONS, QUANTIZATION_BIT_OPTIONS, QUANT_METHOD_OPTIONS, GGUF_FORMAT_OPTIONS } from '@/constants'
|
||||
import {
|
||||
DEFAULT_TRAINING_PARAMS,
|
||||
@@ -35,7 +36,18 @@ const preflightResult = ref<FineTunePreflightResult | null>(null)
|
||||
const models = ref<ModelItem[]>([])
|
||||
const datasets = ref<DatasetItem[]>([])
|
||||
const gpus = ref<GpuInfo[]>([])
|
||||
const selectedGpus = ref<number[]>([])
|
||||
const computeNodes = ref<Array<{ id: string; scheduler_status?: string }>>([])
|
||||
const selectedGpuId = ref<number | null>(null)
|
||||
|
||||
/** Only show GPUs from nodes that are online or draining */
|
||||
const availableGpus = computed(() => {
|
||||
const onlineNodeIds = new Set(
|
||||
computeNodes.value
|
||||
.filter((n) => n.scheduler_status === 'online' || n.scheduler_status === 'draining')
|
||||
.map((n) => n.id),
|
||||
)
|
||||
return gpus.value.filter((gpu) => !gpu.node_id || onlineNodeIds.has(gpu.node_id))
|
||||
})
|
||||
const modelDialogVisible = ref(false)
|
||||
|
||||
const form = reactive(createDefaultFineTuneForm())
|
||||
@@ -62,7 +74,8 @@ const selectedModel = computed(() => models.value.find((model) => model.id === f
|
||||
const modelDialogTitle = computed(() => selectedModel.value?.name || '')
|
||||
|
||||
/** 训练命令与提交载荷共用同一份表单模型。 */
|
||||
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpus.value))
|
||||
const selectedGpuIds = computed(() => (selectedGpuId.value != null ? [selectedGpuId.value] : []))
|
||||
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpuIds.value))
|
||||
|
||||
const remoteCommandPreview = computed(() => {
|
||||
const command = preflightResult.value?.preview?.command
|
||||
@@ -70,11 +83,9 @@ const remoteCommandPreview = computed(() => {
|
||||
return preflightResult.value?.preview?.command_text || ''
|
||||
})
|
||||
|
||||
/** GPU 多选切换 */
|
||||
function toggleGpu(index: number) {
|
||||
const idx = selectedGpus.value.indexOf(index)
|
||||
if (idx === -1) selectedGpus.value.push(index)
|
||||
else selectedGpus.value.splice(idx, 1)
|
||||
/** GPU 单选切换(每次只选中一张 GPU) */
|
||||
function toggleGpu(gpuId: number) {
|
||||
selectedGpuId.value = selectedGpuId.value === gpuId ? null : gpuId
|
||||
}
|
||||
|
||||
function gpuUsageWidth(percent: number) {
|
||||
@@ -164,10 +175,11 @@ async function loadDatasets() {
|
||||
|
||||
async function loadGpus() {
|
||||
try {
|
||||
const sys = await getSystemInfo()
|
||||
const [sys, nodes] = await Promise.all([getSystemInfo(), getComputeNodes().catch(() => [])])
|
||||
gpus.value = sys?.gpu || []
|
||||
// 默认选中第一个
|
||||
if (gpus.value.length > 0) selectedGpus.value = [0]
|
||||
computeNodes.value = nodes || []
|
||||
// Default select first available GPU
|
||||
if (availableGpus.value.length > 0) selectedGpuId.value = availableGpus.value[0].id ?? null
|
||||
} catch {
|
||||
gpus.value = []
|
||||
}
|
||||
@@ -177,8 +189,8 @@ async function handleSubmit() {
|
||||
if (!formRef.value) return
|
||||
await formRef.value.validate(async (valid) => {
|
||||
if (!valid) return
|
||||
if (selectedGpus.value.length === 0) {
|
||||
ElMessage.warning('请至少选择一个 GPU')
|
||||
if (selectedGpuId.value == null) {
|
||||
ElMessage.warning('请选择一个 GPU')
|
||||
return
|
||||
}
|
||||
submitting.value = true
|
||||
@@ -195,7 +207,7 @@ async function handleSubmit() {
|
||||
return
|
||||
}
|
||||
|
||||
const payload = buildFineTunePayload(form, selectedGpus.value)
|
||||
const payload = buildFineTunePayload(form, selectedGpuIds.value)
|
||||
const preflight = await runPreflight(payload)
|
||||
if (!preflight?.valid) {
|
||||
ElMessage.error('训练预检未通过,请先处理预检问题')
|
||||
@@ -220,7 +232,7 @@ async function handleSubmit() {
|
||||
})
|
||||
}
|
||||
|
||||
async function runPreflight(payload = buildFineTunePayload(form, selectedGpus.value)) {
|
||||
async function runPreflight(payload = buildFineTunePayload(form, selectedGpuIds.value)) {
|
||||
preflightLoading.value = true
|
||||
try {
|
||||
const result = await preflightFineTune(payload)
|
||||
@@ -249,8 +261,8 @@ async function handlePreflightClick() {
|
||||
if (!formRef.value) return
|
||||
await formRef.value.validate(async (valid) => {
|
||||
if (!valid) return
|
||||
if (selectedGpus.value.length === 0) {
|
||||
ElMessage.warning('请至少选择一个 GPU')
|
||||
if (selectedGpuId.value == null) {
|
||||
ElMessage.warning('请选择一个 GPU')
|
||||
return
|
||||
}
|
||||
await runPreflight()
|
||||
@@ -286,15 +298,18 @@ onMounted(() => {
|
||||
<el-form-item label="GPU 硬件">
|
||||
<div class="gpu-list">
|
||||
<div
|
||||
v-for="(gpu, idx) in gpus"
|
||||
:key="idx"
|
||||
v-for="gpu in availableGpus"
|
||||
:key="gpu.id"
|
||||
class="gpu-card"
|
||||
:class="{ active: selectedGpus.includes(idx), 'is-busy': gpu.gpu_percent > 80 }"
|
||||
@click="toggleGpu(idx)"
|
||||
:class="{ active: selectedGpuId === gpu.id, 'is-busy': gpu.gpu_percent > 80 }"
|
||||
@click="toggleGpu(gpu.id!)"
|
||||
>
|
||||
<div class="gpu-card-top">
|
||||
<div class="gpu-title">
|
||||
<span class="gpu-index">GPU-{{ idx }}</span>
|
||||
<span class="gpu-index">
|
||||
GPU-{{ gpu.id }}
|
||||
<template v-if="gpu.node_code"> · {{ gpu.node_code }}</template>
|
||||
</span>
|
||||
<span class="gpu-name">{{ gpu.name }}</span>
|
||||
</div>
|
||||
<span class="gpu-usage">{{ gpu.gpu_percent }}%</span>
|
||||
@@ -308,7 +323,7 @@ onMounted(() => {
|
||||
<span>{{ gpu.power_w }}W</span>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="!gpus.length" class="gpu-empty">暂无 GPU 信息</div>
|
||||
<div v-if="!availableGpus.length" class="gpu-empty">暂无可用 GPU(请检查算力节点是否在线)</div>
|
||||
</div>
|
||||
</el-form-item>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user