YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 推理/评测结果同步、数据集统计与算力节点管理增强

Browse Source 
后端:
- 抽取 fetch_eval_result_content 复用函数，model_eval_detail 直接应用评测任务结果
- health 接口移除数据库依赖，返回静态指标
- 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理
- 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout
- 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error

评测引擎:
- _load_dataset 支持 JSON/JSONL 文件
- 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制

前端:
- 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode
- 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
wuyongtao
2026-08-03 15:49:21 +08:00
parent cc08b164d0
commit 5cc306eb0a
15 changed files with 402 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
frontend/src/api/modules/compute.ts
View File

@@ -1,4 +1,4 @@
import { get, post, put } from '../request'
import { del, get, post, put } from '../request'
export interface ComputeNode {
id: string
@@ -95,6 +95,9 @@ export const createComputeNode = (data: ComputeNodePayload) =>
export const updateComputeNode = (id: string, data: Partial<ComputeNode>) =>
put<ComputeNode>(`/compute/nodes/${id}`, data)
export const deleteComputeNode = (id: string) =>
del<{ deleted: string }>(`/compute/nodes/${id}`)
export const testComputeNode = (id: string) =>
post<{ node_id: string; success: boolean; latency_ms: number; gpu_count: number; error?: string }>(`/compute/nodes/${id}/test-connection`)

1
frontend/src/api/modules/dataset.ts
View File

@@ -29,6 +29,7 @@ export const uploadDatasetFiles = (datasetId: string | number, files: File[]) =>
files.forEach((f) => formData.append('files', f))
return post(`/dataset-manage/upload/${datasetId}`, formData, {
headers: { 'Content-Type': 'multipart/form-data' },
timeout: 120000,
})
}

3
frontend/src/types/index.ts
View File

@@ -132,6 +132,7 @@ export interface FineTuneTask {
train_dataset_id?: number | string
auto_merge?: boolean
output_model_name?: string
compute_node_id?: string
gpus?: number[]
batch_size?: number
learning_rate?: number
@@ -355,7 +356,7 @@ export interface GpuInfo {
power_w: number
id?: number
uuid?: string
status?: 'idle' | 'busy' | 'warning' | 'offline'
status?: 'idle' | 'busy' | 'reserved' | 'warning' | 'offline'
memory_percent?: number
power_limit_w?: number
processes?: GpuProcess[]

30
frontend/src/views/compute/ComputeNodesView.vue
View File

@@ -1,12 +1,12 @@
<script setup lang="ts">
import { computed, onMounted, onUnmounted, reactive, ref, watch } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { ElMessage } from 'element-plus'
import { ElMessage, ElMessageBox } from 'element-plus'
import {
checkNodeReplicaDrift,
createComputeNode,
deleteComputeNode,
disableComputeNode,
drainComputeNode,
enableComputeNode,
getComputeGpus,
getComputeNodes,
@@ -129,11 +129,10 @@ async function changeTab(name: string | number) {
await router.replace({ path: '/compute', query: { tab: String(name) } })
}
async function handleNodeAction(action: 'enable' | 'disable' | 'drain' | 'test', node: ComputeNode) {
async function handleNodeAction(action: 'enable' | 'disable' | 'test', node: ComputeNode) {
const nodeId = String(node.id)
if (action === 'enable') await enableComputeNode(nodeId)
if (action === 'disable') await disableComputeNode(nodeId)
if (action === 'drain') await drainComputeNode(nodeId)
if (action === 'test') {
const result = await testComputeNode(nodeId)
if (result.success) {
@@ -145,6 +144,27 @@ async function handleNodeAction(action: 'enable' | 'disable' | 'drain' | 'test',
await load()
}
async function handleDeleteNode(node: ComputeNode) {
try {
await ElMessageBox.confirm(
`确定删除算力节点「${node.name || node.code}」吗？节点删除后，其 GPU 设备和资源副本记录也会一并移除。`,
'删除算力节点',
{ type: 'warning', confirmButtonText: '删除', cancelButtonText: '取消' },
)
} catch {
return
}
try {
await deleteComputeNode(String(node.id))
ElMessage.success('算力节点已删除')
if (selectedNodeId.value === node.id) selectedNodeId.value = ''
await load({ showButtonLoading: true })
} catch (err: any) {
const message = err?.response?.data?.detail?.message || err?.response?.data?.message || '删除算力节点失败'
ElMessage.error(message)
}
}
async function handleReplicaDriftCheck() {
if (!selectedNodeId.value) return
checkingReplicas.value = true
@@ -355,7 +375,7 @@ onUnmounted(() => {
<el-button size="small" @click="handleNodeAction('test', asComputeNode(row))">测试</el-button>
<el-button v-if="row.enabled" size="small" @click="handleNodeAction('disable', asComputeNode(row))">停用</el-button>
<el-button v-else size="small" type="primary" @click="handleNodeAction('enable', asComputeNode(row))">启用</el-button>
<el-button size="small" type="warning" plain @click="handleNodeAction('drain', asComputeNode(row))">维护</el-button>
<el-button size="small" type="danger" plain @click="handleDeleteNode(asComputeNode(row))">删除</el-button>
</template>
</el-table-column>
</el-table>

34
frontend/src/views/dataset/DatasetCreateView.vue
View File

@@ -50,6 +50,20 @@ const rules: FormRules = {
}
/** 处理文件选择（替换模式：新文件覆盖旧文件） */
function parseDatasetRecordValues(text: string, fileName: string): unknown[] {
const content = text.trim()
if (!content) return []
if (fileName.toLowerCase().endsWith('.json')) {
const parsed = JSON.parse(content)
return Array.isArray(parsed) ? parsed : [parsed]
}
return content
.split('\n')
.map((line) => line.trim())
.filter(Boolean)
.map((line) => JSON.parse(line))
}
async function handleFileChange(uploadFile: UploadFile) {
const raw = uploadFile.raw
if (!raw) return
@@ -70,25 +84,19 @@ async function handleFileChange(uploadFile: UploadFile) {
async function analyzeFile(file: File) {
try {
const text = await file.text()
const lines = text.trim().split('\n').filter(Boolean)
fileCount.value = lines.length
const records = parseDatasetRecordValues(text, file.name)
fileCount.value = records.length
// Alpaca 格式校验：每行 JSON 须含 instruction 字段
let validCount = 0
for (const line of lines) {
try {
const obj = JSON.parse(line)
if (obj.instruction !== undefined) validCount++
} catch {
// 非 JSON 行（如纯 JSONL 多行结构）
}
}
if (validCount > 0 && validCount === lines.length) {
const validCount = records.filter(
(obj) => obj && typeof obj === 'object' && 'instruction' in obj,
).length
if (validCount > 0 && validCount === records.length) {
formatValid.value = true
formatMessage.value = `符合 Alpaca 格式（含 instruction 字段）`
} else if (validCount > 0) {
formatValid.value = true
formatMessage.value = `部分符合 Alpaca 格式（${validCount}/${lines.length}）`
formatMessage.value = `部分符合 Alpaca 格式（${validCount}/${records.length}）`
} else {
formatValid.value = false
formatMessage.value = '未检测到标准 Alpaca 格式（缺少 instruction 字段），仍可上传'

32
frontend/src/views/eval/EvalDetailView.vue
View File

@@ -1,9 +1,10 @@
<script setup lang="ts">
import { computed, onMounted, ref } from 'vue'
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { useRoute } from 'vue-router'
import PageCard from '@/components/PageCard.vue'
import ModelStatusTag from '@/components/ModelStatusTag.vue'
import { getEvalDetail } from '@/api/modules/eval'
import { usePolling } from '@/composables/usePolling'
import type { EvalSampleResult, EvalTaskDetail } from '@/types'
const route = useRoute()
@@ -16,6 +17,7 @@ const keyword = ref('')
const judgementFilter = ref('')
const currentPage = ref(1)
const pageSize = ref(10)
const ACTIVE_STATUSES = new Set(['pending', 'queued', 'running'])
const filteredSamples = computed(() => {
const normalizedKeyword = keyword.value.trim().toLowerCase()
@@ -74,8 +76,8 @@ function resetPage() {
currentPage.value = 1
}
async function loadDetail() {
loading.value = true
async function loadDetail(options: { silent?: boolean } = {}) {
if (!options.silent) loading.value = true
loadError.value = ''
try {
detail.value = await getEvalDetail(taskId)
@@ -83,11 +85,29 @@ async function loadDetail() {
detail.value = null
loadError.value = '评测详情加载失败，请稍后重试。'
} finally {
loading.value = false
if (!options.silent) loading.value = false
}
}
onMounted(loadDetail)
const { start: startPolling, stop: stopPolling } = usePolling(
async () => {
await loadDetail({ silent: true })
if (!ACTIVE_STATUSES.has(String(detail.value?.status || ''))) {
stopPolling()
}
},
5000,
{ immediate: false },
)
onMounted(async () => {
await loadDetail()
if (ACTIVE_STATUSES.has(String(detail.value?.status || ''))) {
startPolling()
}
})
onUnmounted(stopPolling)
</script>
<template>
@@ -113,7 +133,7 @@ onMounted(loadDetail)
<i class="fa fa-exclamation-circle" aria-hidden="true" />
<h2>无法加载评测详情</h2>
<p>{{ loadError }}</p>
<el-button type="primary" @click="loadDetail">重新加载</el-button>
<el-button type="primary" @click="() => loadDetail()">重新加载</el-button>
</div>
<template v-else-if="detail">

19
frontend/src/views/eval/EvalView.vue
View File

@@ -24,14 +24,16 @@ const leaderboard = ref([
{ rank: 3, name: 'Qwen-Max', score: 85.3 },
])
async function loadEvalList() {
evalLoading.value = true
const ACTIVE_STATUSES = new Set(['pending', 'queued', 'running'])
async function loadEvalList(options: { silent?: boolean } = {}) {
if (!options.silent) evalLoading.value = true
try {
evalList.value = (await getEvalList()) || []
} catch {
evalList.value = []
} finally {
evalLoading.value = false
if (!options.silent) evalLoading.value = false
}
}
@@ -56,14 +58,21 @@ function handleViewDetail(row: any) {
}
const { start: startPolling, stop: stopPolling } = usePolling(
() => loadEvalList(),
async () => {
await loadEvalList({ silent: true })
if (!evalList.value.some((item) => ACTIVE_STATUSES.has(String(item.status || '')))) {
stopPolling()
}
},
5000,
{ immediate: false },
)
onMounted(async () => {
await loadEvalList()
startPolling()
if (evalList.value.some((item) => ACTIVE_STATUSES.has(String(item.status || '')))) {
startPolling()
}
})
onUnmounted(() => {

79
frontend/src/views/fine-tune/FineTuneCreateView.vue
View File

@@ -37,7 +37,7 @@ const models = ref<ModelItem[]>([])
const datasets = ref<DatasetItem[]>([])
const gpus = ref<GpuInfo[]>([])
const computeNodes = ref<Array<{ id: string; scheduler_status?: string }>>([])
const selectedGpuId = ref<number | null>(null)
const selectedGpuKeys = ref<string[]>([])
/** Only show GPUs from nodes that are online or draining */
const availableGpus = computed(() => {
@@ -74,7 +74,13 @@ const selectedModel = computed(() => models.value.find((model) => model.id === f
const modelDialogTitle = computed(() => selectedModel.value?.name || '')
/** 训练命令与提交载荷共用同一份表单模型。 */
const selectedGpuIds = computed(() => (selectedGpuId.value != null ? [selectedGpuId.value] : []))
const selectedGpus = computed(() =>
selectedGpuKeys.value
.map((key) => availableGpus.value.find((gpu) => gpuKey(gpu) === key))
.filter((gpu): gpu is GpuInfo => Boolean(gpu)),
)
const selectedComputeNodeId = computed(() => selectedGpus.value[0]?.node_id)
const selectedGpuIds = computed(() => selectedGpus.value.map((gpu) => Number(gpu.id)))
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpuIds.value))
const remoteCommandPreview = computed(() => {
@@ -83,9 +89,32 @@ const remoteCommandPreview = computed(() => {
return preflightResult.value?.preview?.command_text || ''
})
/** GPU 单选切换（每次只选中一张 GPU） */
function toggleGpu(gpuId: number) {
selectedGpuId.value = selectedGpuId.value === gpuId ? null : gpuId
function gpuKey(gpu: GpuInfo) {
return `${gpu.node_id || 'local'}:${gpu.id ?? gpu.uuid ?? gpu.name}`
}
function isGpuUnavailable(gpu: GpuInfo) {
return gpu.status === 'busy' || gpu.status === 'reserved' || gpu.status === 'offline'
}
function isGpuSelected(gpu: GpuInfo) {
return selectedGpuKeys.value.includes(gpuKey(gpu))
}
/** GPU 多选切换：单个任务只允许选择同一算力节点内的空闲卡。 */
function toggleGpu(gpu: GpuInfo) {
if (isGpuUnavailable(gpu) || gpu.id == null) return
const key = gpuKey(gpu)
if (isGpuSelected(gpu)) {
selectedGpuKeys.value = selectedGpuKeys.value.filter((item) => item !== key)
return
}
if (selectedComputeNodeId.value && gpu.node_id && selectedComputeNodeId.value !== gpu.node_id) {
selectedGpuKeys.value = [key]
ElMessage.info('已切换到新的算力节点，之前选择的 GPU 已清空')
return
}
selectedGpuKeys.value = [...selectedGpuKeys.value, key]
}
function gpuUsageWidth(percent: number) {
@@ -178,8 +207,8 @@ async function loadGpus() {
const [sys, nodes] = await Promise.all([getSystemInfo(), getComputeNodes().catch(() => [])])
gpus.value = sys?.gpu || []
computeNodes.value = nodes || []
// Default select first available GPU
if (availableGpus.value.length > 0) selectedGpuId.value = availableGpus.value[0].id ?? null
const firstIdle = availableGpus.value.find((gpu) => !isGpuUnavailable(gpu) && gpu.id != null)
if (firstIdle) selectedGpuKeys.value = [gpuKey(firstIdle)]
} catch {
gpus.value = []
}
@@ -189,8 +218,8 @@ async function handleSubmit() {
if (!formRef.value) return
await formRef.value.validate(async (valid) => {
if (!valid) return
if (selectedGpuId.value == null) {
ElMessage.warning('请选择一个 GPU')
if (!selectedGpuIds.value.length) {
ElMessage.warning('请至少选择一张空闲 GPU')
return
}
submitting.value = true
@@ -207,7 +236,7 @@ async function handleSubmit() {
return
}
const payload = buildFineTunePayload(form, selectedGpuIds.value)
const payload = buildFineTunePayload(form, selectedGpuIds.value, selectedComputeNodeId.value)
const preflight = await runPreflight(payload)
if (!preflight?.valid) {
ElMessage.error('训练预检未通过，请先处理预检问题')
@@ -232,7 +261,7 @@ async function handleSubmit() {
})
}
async function runPreflight(payload = buildFineTunePayload(form, selectedGpuIds.value)) {
async function runPreflight(payload = buildFineTunePayload(form, selectedGpuIds.value, selectedComputeNodeId.value)) {
preflightLoading.value = true
try {
const result = await preflightFineTune(payload)
@@ -261,8 +290,8 @@ async function handlePreflightClick() {
if (!formRef.value) return
await formRef.value.validate(async (valid) => {
if (!valid) return
if (selectedGpuId.value == null) {
ElMessage.warning('请选择一个 GPU')
if (!selectedGpuIds.value.length) {
ElMessage.warning('请至少选择一张空闲 GPU')
return
}
await runPreflight()
@@ -297,12 +326,16 @@ onMounted(() => {
<el-divider content-position="left">训练配置</el-divider>
<el-form-item label="GPU 硬件">
<div class="gpu-list">
<div class="gpu-selection-summary">
已选择 {{ selectedGpuIds.length }} 张 GPU
<template v-if="selectedGpus[0]?.node_code"> · {{ selectedGpus[0].node_code }}</template>
</div>
<div
v-for="gpu in availableGpus"
:key="gpu.id"
:key="gpuKey(gpu)"
class="gpu-card"
:class="{ active: selectedGpuId === gpu.id, 'is-busy': gpu.gpu_percent > 80 }"
@click="toggleGpu(gpu.id!)"
:class="{ active: isGpuSelected(gpu), 'is-busy': isGpuUnavailable(gpu), 'is-disabled': isGpuUnavailable(gpu) }"
@click="toggleGpu(gpu)"
>
<div class="gpu-card-top">
<div class="gpu-title">
@@ -312,7 +345,7 @@ onMounted(() => {
</span>
<span class="gpu-name">{{ gpu.name }}</span>
</div>
<span class="gpu-usage">{{ gpu.gpu_percent }}%</span>
<span class="gpu-usage">{{ isGpuUnavailable(gpu) ? gpu.status : `${gpu.gpu_percent}%` }}</span>
</div>
<div class="gpu-usage-bar">
<span :style="{ width: gpuUsageWidth(gpu.gpu_percent) }" />
@@ -578,6 +611,13 @@ onMounted(() => {
width: 100%;
}
.gpu-selection-summary {
grid-column: 1 / -1;
color: #64748b;
font-size: 12px;
line-height: 20px;
}
.gpu-card {
border: 1px solid #e5e7eb;
border-radius: 6px;
@@ -627,6 +667,11 @@ onMounted(() => {
background: #dc2626;
}
}
&.is-disabled {
cursor: not-allowed;
opacity: 0.72;
}
}
.gpu-card-top {

2
frontend/src/views/fine-tune/fineTuneFormModel.ts
View File

@@ -65,6 +65,7 @@ export function createDefaultFineTuneForm(): FineTuneFormModel {
export function buildFineTunePayload(
form: FineTuneFormModel,
gpus: number[],
computeNodeId?: string,
): Omit<FineTuneStartPayload, 'task_id'> {
return {
name: form.name,
@@ -77,6 +78,7 @@ export function buildFineTunePayload(
train_dataset_id: form.train_dataset_id,
auto_merge: form.train_type === 'SFT' && form.auto_merge,
output_model_name: form.name,
compute_node_id: computeNodeId,
batch_size: form.batch_size,
learning_rate: form.learning_rate,
n_epochs: form.n_epochs,