feat: 推理与评测支持多卡 GPU 选择，训练任务实时 GPU 监控

- 新增 GPU 选择归一化 helper，统一前端各形态的选择（gpu_indices/gpus/gpu_id）
- 评测与推理支持同一节点内多卡选择，校验所选 GPU 空闲后再派发
- 新增 /fine-tune/{id}/gpu-status 接口，训练日志页展示实时 GPU 指标
- 算力节点推理加载支持 CUDA_VISIBLE_DEVICES 多卡可见，nvidia-smi 进程级监控
- GPU 占用跟踪细化为按卡记录，覆盖评测任务、推理模型与对比任务

Co-Authored-By: Claude <noreply@anthropic.com>

frontend/src/api/modules/fineTune.ts

@@ -42,12 +42,23 @@ export interface FineTunePreflightResult {
   sync_results?: Array<Record<string, unknown>>
 }
 
+export interface FineTuneGpuStatus {
+  source: string
+  items: Array<Record<string, unknown>>
+  selected_gpus: number[]
+  error?: string
+}
+
 /** 训练任务列表 */
 export const getFineTuneList = () => get<FineTuneTask[]>('/fine-tune')
 
 /** 训练任务详情 */
 export const getFineTune = (id: string | number) => get<FineTuneTask>(`/fine-tune/${id}`)
 
+/** 获取任务所在 Compute 节点的实时 GPU 指标 */
+export const getFineTuneGpuStatus = (id: string | number) =>
+  get<FineTuneGpuStatus>(`/fine-tune/${id}/gpu-status`)
+
 /** 任务名查重 */
 export const checkFineTuneName = (name: string) =>
   get<{ exists: boolean }>('/fine-tune/check-name', { name })

frontend/src/types/index.ts

@@ -218,6 +218,8 @@ export interface LoadedModel {
   port?: number
   node_id?: string
   node_name?: string
+  gpu_indices?: number[]
+  gpus?: number[]
   error?: string
 }
 
@@ -239,6 +241,8 @@ export interface CompareModelRef {
   gpu_id: number
   node_id?: string
   node_name?: string
+  gpu_indices?: number[]
+  gpus?: number[]
   source?: string
   port?: number
 }
@@ -282,7 +286,9 @@ export interface StartEvalPayload {
   eval_task_name: string
   eval_type: EvalType
   model_id: string | number
-  gpu_id: string | number
+  gpu_id: string | number | string[]
+  gpu_indices?: number[]
+  gpus?: number[]
   compute_node_id?: string
   dataset_id: string | number
   dimension_id: string | number

frontend/src/views/eval/EvalCreateView.vue

@@ -39,7 +39,7 @@ const createdDimensionId = ref<string | number>('')
 const taskForm = ref<EvalTaskSetupDraft>({
   eval_task_name: '',
   model_id: '',
-  gpu_id: '',
+  gpu_id: [],
   data_source: 'dataset',
   dataset_id: '',
   leaderboard: false,
@@ -145,12 +145,24 @@ async function handleSubmit() {
     const dimensionId = await resolveDimensionId()
     // GPU 选择为「节点:GPU序号」复合值，解析出节点与 GPU 序号，
     // 多算力节点时必须把节点信息传给后端，否则会派发到错误的算力节点
-    const [gpuNodeId, gpuIndex] = String(taskForm.value.gpu_id).split(':')
+    const selectedGpuKeys = Array.isArray(taskForm.value.gpu_id)
+      ? taskForm.value.gpu_id
+      : [String(taskForm.value.gpu_id)]
+    const gpuSelections = selectedGpuKeys
+      .map((key) => {
+        const [nodeId, gpuIndex] = String(key).split(':')
+        return { nodeId, gpuIndex: Number(gpuIndex) }
+      })
+      .filter((item) => item.nodeId && Number.isInteger(item.gpuIndex) && item.gpuIndex >= 0)
+    const gpuNodeId = gpuSelections[0]?.nodeId || ''
+    const gpuIndices = gpuSelections.map((item) => item.gpuIndex)
     const evalResult: any = await startEval({
       eval_task_name: taskForm.value.eval_task_name,
       eval_type: 'custom',
       model_id: taskForm.value.model_id,
-      gpu_id: Number(gpuIndex) || 0,
+      gpu_id: gpuIndices[0] ?? 0,
+      gpu_indices: gpuIndices,
+      gpus: gpuIndices,
       compute_node_id: gpuNodeId || '',
       dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
       dimension_id: dimensionId,

frontend/src/views/eval/create/EvalTaskSetupStep.vue

@@ -6,7 +6,7 @@ import type { DatasetItem, GpuInfo, TrainedModel } from '@/types'
 export interface EvalTaskSetupDraft {
   eval_task_name: string
   model_id: string | number
-  gpu_id: string | number
+  gpu_id: string | number | string[]
   data_source: 'dataset' | 'inference'
   dataset_id: string | number
   leaderboard: boolean
@@ -23,6 +23,13 @@ defineProps<{
 const form = defineModel<EvalTaskSetupDraft>({ required: true })
 const formRef = ref<FormInstance>()
 
+function handleGpuChange(value: string | number | string[]) {
+  const keys = Array.isArray(value) ? value.map(String) : [String(value || '')]
+  const nodeId = keys[0]?.split(':', 1)[0]
+  if (!nodeId || !Array.isArray(form.value.gpu_id)) return
+  form.value.gpu_id = keys.filter((key) => key.split(':', 1)[0] === nodeId)
+}
+
 const rules: FormRules<EvalTaskSetupDraft> = {
   eval_task_name: [
     { required: true, message: '请输入任务名称', trigger: 'blur' },
@@ -101,7 +108,16 @@ defineExpose({ validate })
     </el-form-item>
 
     <el-form-item label="选择 GPU" prop="gpu_id">
-      <el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
+      <el-select
+        v-model="form.gpu_id"
+        multiple
+        collapse-tags
+        collapse-tags-tooltip
+        placeholder="请选择同一算力节点内的一张或多张 GPU"
+        style="width: 100%"
+        :loading="loading"
+        @change="handleGpuChange"
+      >
         <el-option
           v-for="gpu in gpus"
           :key="`${gpu.node_id || ''}:${gpu.id ?? 0}`"

frontend/src/views/eval/create/StartEvalStep.vue

@@ -21,7 +21,7 @@ function nameOf<T extends { id: string | number; name?: string }>(items: T[], id
 
 /** GPU 选择为「节点:GPU序号」复合值，解析并展示为可读标签 */
 const gpuLabel = computed(() => {
-  const key = String(props.task.gpu_id || '')
+  const key = Array.isArray(props.task.gpu_id) ? String(props.task.gpu_id[0] || '') : String(props.task.gpu_id || '')
   const gpu = props.gpus.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === key)
   if (gpu) return `${gpu.node_name || gpu.node_code || '算力节点'} / GPU ${gpu.id ?? 0}`
   const [nodeId, idx] = key.split(':')

frontend/src/views/inference/InferenceCreateView.vue

@@ -4,8 +4,7 @@ import { useRouter } from 'vue-router'
 import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
 import PageCard from '@/components/PageCard.vue'
 import { getModelList, getTrainedModels } from '@/api/modules/model'
-import { getSystemInfo } from '@/api/modules/system'
-import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
+import { getComputeGpus, getComputeNodes, type ComputeNode } from '@/api/modules/compute'
 import { createCompare, loadCompare } from '@/api/modules/compare'
 import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
 
@@ -83,8 +82,8 @@ const form = reactive({
   description: '',
   /** 选中的模型 key（单选） */
   model_key: '',
-  /** 使用的 GPU */
-  gpu_key: '',
+  /** 使用的 GPU（同一节点内可多选） */
+  gpu_keys: [] as string[],
 })
 
 const rules: FormRules = {
@@ -94,12 +93,26 @@ const rules: FormRules = {
 
 /** 当前选中的模型对象 */
 const selectedModel = computed(() => modelMap.value[form.model_key])
-const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key))
+const selectedGpus = computed(() => idleGpus.value.filter((gpu) => form.gpu_keys.includes(gpuKey(gpu))))
+
+function gpuKey(gpu: GpuInfo) {
+  return `${gpu.node_id || ''}:${gpu.id ?? 0}`
+}
+
+function handleGpuChange(keys: string[]) {
+  const nodeId = keys[0]?.split(':', 1)[0]
+  if (!nodeId) return
+  const filtered = keys.filter((key) => key.split(':', 1)[0] === nodeId)
+  if (filtered.length !== keys.length) {
+    ElMessage.info('一次推理只能使用同一算力节点内的 GPU，已忽略其它节点的选择')
+  }
+  form.gpu_keys = filtered
+}
 
 watch(selectedModel, (model) => {
   if (!model?.compute_node_id) return
   const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
-  if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}`
+  if (gpu) form.gpu_keys = [gpuKey(gpu)]
 })
 
 async function handleSubmit() {
@@ -111,6 +124,10 @@ async function handleSubmit() {
       ElMessage.warning('请选择模型')
       return
     }
+    if (!selectedGpus.value.length) {
+      ElMessage.warning('请至少选择一张空闲 GPU')
+      return
+    }
     submitting.value = true
     startupStatus.value = '正在创建推理任务...'
     try {
@@ -130,9 +147,11 @@ async function handleSubmit() {
             model_name: m.name,
             model_path: m.model_path,
             source: m.source,
-            gpu_id: selectedGpu.value?.id ?? 0,
-            node_id: selectedGpu.value?.node_id || m.compute_node_id,
-            node_name: selectedGpu.value?.node_name || m.compute_node_name,
+            gpu_id: selectedGpus.value[0]?.id ?? 0,
+            gpu_indices: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)),
+            gpus: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)),
+            node_id: selectedGpus.value[0]?.node_id || m.compute_node_id,
+            node_name: selectedGpus.value[0]?.node_name || m.compute_node_name,
           },
         ],
       })
@@ -169,17 +188,17 @@ async function loadData() {
     const [db, trained, sys, nodes] = await Promise.all([
       getModelList(),
       getTrainedModels(),
-      getSystemInfo(),
+      getComputeGpus(),
       getComputeNodes(),
     ])
     dbModels.value = db || []
     trainedModels.value = trained?.models || []
-    gpus.value = sys?.gpu || []
+    gpus.value = (sys || []) as unknown as GpuInfo[]
     computeNodes.value = nodes || []
     // 默认选中第一个空闲 GPU
     if (idleGpus.value.length > 0) {
       const firstGpu = idleGpus.value[0]
-      form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}`
+      form.gpu_keys = [gpuKey(firstGpu)]
     }
   } catch {
     // ignore
@@ -228,12 +247,20 @@ onMounted(loadData)
       </el-form-item>
 
       <el-form-item label="GPU">
-        <el-select v-model="form.gpu_key" style="width: 400px">
+        <el-select
+          v-model="form.gpu_keys"
+          multiple
+          collapse-tags
+          collapse-tags-tooltip
+          style="width: 400px"
+          placeholder="请选择同一算力节点内的一张或多张 GPU"
+          @change="handleGpuChange"
+        >
           <el-option
             v-for="g in idleGpus"
-            :key="`${g.node_id || ''}:${g.id ?? 0}`"
+            :key="gpuKey(g)"
             :label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
-            :value="`${g.node_id || ''}:${g.id ?? 0}`"
+            :value="gpuKey(g)"
           />
         </el-select>
       </el-form-item>

frontend/src/views/system/TrainingLogView.vue

@@ -8,10 +8,9 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
 import { usePolling } from '@/composables/usePolling'
 import '@/plugins/echarts-training-log'
 import { useModelsStore } from '@/stores/models'
-import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
+import { getFineTune, getFineTuneDiagnostics, getFineTuneGpuStatus, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
 import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
 import { getDataset } from '@/api/modules/dataset'
-import { getSystemInfo } from '@/api/modules/system'
 import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
 import {
   buildMetricChartOption,
@@ -205,15 +204,20 @@ async function loadDataset(datasetId: string | number) {
   }
 }
 
-async function loadGpuStatus() {
+async function loadGpuStatus(currentTask: FineTuneTask) {
   try {
-    const systemInfo = await getSystemInfo()
-    gpuPool.value = systemInfo.gpu ?? []
-    gpuUpdatedAt.value = new Date()
-    gpuLoadError.value = ''
+    const live = await getFineTuneGpuStatus(currentTask.id)
+    if (live.source === 'compute' && live.items.length) {
+      gpuPool.value = live.items as unknown as GpuInfo[]
+      gpuUpdatedAt.value = new Date()
+      gpuLoadError.value = ''
+      return
+    }
+    gpuPool.value = []
+    gpuLoadError.value = live.error || 'Compute 节点暂未返回实时 GPU 指标'
   } catch {
     gpuLoadError.value = 'GPU 监控数据暂时不可用'
-    if (!gpuUpdatedAt.value) gpuPool.value = []
+    gpuPool.value = []
   }
 }
 
@@ -306,7 +310,7 @@ async function refreshAll() {
     const datasetPromise = currentTask.train_dataset_id
       ? loadDataset(currentTask.train_dataset_id)
       : Promise.resolve()
-    await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
+    await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(currentTask), loadDiagnostics(currentTask)])
     await loadMetrics(currentTask)
   } finally {
     loading.value = false