feat: 模型推理异步加载与对话链路修复,同步基线
模型推理全异步化改造: - 计算节点 InferenceSession 改为后台线程异步加载模型,load 立即返回, 加载期间事件循环保持响应(/inference/status 与 /health 不阻塞) - 后端模型加载改为异步派发 + 轮询对账器(reconcile_inference_loads), 任务状态由 starting 自动推进到 ready/error,解决多节点启动超时 (timeout of 120000ms exceeded) - 推理删除/卸载改为任务感知 + 短超时,删除先删记录再 best-effort 卸载, 不再被不可达节点阻塞;同节点新模型替换旧任务标记失效 - 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点, useStreamChat 解析 SSE 错误帧以干净文案展示 - 对话历史按任务 id 本地持久化,退出重进可恢复;移除页脚提示文本 - 新增后端推理异步加载与计算节点异步状态机单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import { get, post, del } from '../request'
|
||||
import type { CompareTask, CompareModelRef } from '@/types'
|
||||
|
||||
const INFERENCE_START_TIMEOUT_MS = 15 * 60 * 1000
|
||||
|
||||
/** 推理/对比任务列表 */
|
||||
export const getCompareList = () => get<CompareTask[]>('/model-compare')
|
||||
|
||||
@@ -12,7 +14,7 @@ export const createCompare = (data: Partial<CompareTask>) =>
|
||||
post<{ id: string | number }>('/model-compare', data)
|
||||
|
||||
/** 删除任务 */
|
||||
export const deleteCompare = (id: string | number) => del(`/model-compare/${id}`)
|
||||
export const deleteCompare = (id: string | number) => del(`/model-compare/${id}`, undefined, { timeout: 60_000 })
|
||||
|
||||
/** 更新任务加载状态 */
|
||||
export const updateLoadStatus = (id: string | number, load_status: any) =>
|
||||
@@ -34,7 +36,8 @@ export const stopModelByPid = (pid: number) =>
|
||||
post('/model-compare/stop-by-pid', { pid })
|
||||
|
||||
/** 加载任务 */
|
||||
export const loadCompare = (id: string | number) => post(`/model-compare/${id}/load`)
|
||||
export const loadCompare = (id: string | number) =>
|
||||
post(`/model-compare/${id}/load`, undefined, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||
|
||||
/** 卸载任务 */
|
||||
export const unloadCompare = (id: string | number) => post(`/model-compare/${id}/unload`)
|
||||
@@ -81,6 +84,10 @@ export const streamChatReal = (data: any): Promise<Response> => {
|
||||
temperature: data.temperature ?? 0.7,
|
||||
top_p: data.top_p ?? 0.95,
|
||||
max_tokens: data.max_tokens ?? 2048,
|
||||
// 透传 task_id/node_id,让后端按 load_status 路由到真正加载了模型的算力节点,
|
||||
// 避免在多节点时回退到“第一个在线节点”导致连接失败
|
||||
task_id: data.task_id,
|
||||
node_id: data.node_id,
|
||||
}),
|
||||
})
|
||||
}
|
||||
@@ -94,8 +101,8 @@ export const batchChat = (data: any) => post('/model-chat/batch', data)
|
||||
/** 本地 transformers 模型对话 */
|
||||
export const localChat = (data: any) => post('/model-chat/local/chat', data)
|
||||
|
||||
/** 预加载本地模型(模型加载耗时长,超时 5 分钟) */
|
||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data, { timeout: 300000 })
|
||||
/** 预加载本地模型(模型加载耗时长,超时 15 分钟) */
|
||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||
|
||||
/** 预加载已训练模型(超时 5 分钟) */
|
||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data, { timeout: 300000 })
|
||||
/** 预加载已训练模型(超时 15 分钟) */
|
||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
import { get, post, put, del } from '../request'
|
||||
import type { FineTuneStartPayload, FineTuneTask, TrainingProgress, LogContent } from '@/types'
|
||||
|
||||
export interface FineTuneMetricPoint {
|
||||
step: number
|
||||
epoch?: number | null
|
||||
loss?: number | null
|
||||
grad_norm?: number | null
|
||||
learning_rate?: number | null
|
||||
raw?: string
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
export interface TrainingDiagnostic {
|
||||
level: string
|
||||
title: string
|
||||
@@ -80,6 +90,10 @@ export const getFineTuneLogs = (
|
||||
params: { tail_lines?: number; offset?: number; limit?: number } = {},
|
||||
) => get<LogContent & { job_id?: string; source?: string }>(`/fine-tune/${id}/logs`, params)
|
||||
|
||||
/** 获取训练指标曲线数据 */
|
||||
export const getFineTuneMetrics = (id: string | number) =>
|
||||
get<FineTuneMetricPoint[]>(`/fine-tune/${id}/metrics`)
|
||||
|
||||
/** 启动 TensorBoard */
|
||||
export const startTensorboard = () => post('/fine-tune/tensorboard/start')
|
||||
|
||||
|
||||
@@ -88,10 +88,14 @@ export const updateModelPurpose = (id: string | number, purpose: string) =>
|
||||
|
||||
/** 合并 LoRA 权重 */
|
||||
export const mergeModel = (data: {
|
||||
trained_model_id?: string | number
|
||||
model_name: string
|
||||
train_method: string
|
||||
base_model_path: string
|
||||
}) => post('/model-manage/merge', data)
|
||||
adapter_path?: string
|
||||
compute_node_id?: string
|
||||
output_model_name?: string
|
||||
}) => post('/model-manage/merge', data, { timeout: 15 * 60 * 1000 })
|
||||
|
||||
/** 导出已训练模型权重 */
|
||||
export const exportModelUrl = (modelName: string) =>
|
||||
|
||||
@@ -44,6 +44,24 @@ export function useStreamChat() {
|
||||
})
|
||||
const loading = ref(false)
|
||||
|
||||
/** 从 SSE 帧中提取错误信息(后端/计算节点错误以 data: {"error": "..."} 形式下发) */
|
||||
function extractSseError(buffer: string): string | null {
|
||||
const trimmed = buffer.trim()
|
||||
if (!trimmed.startsWith('data: ')) return null
|
||||
const lines = trimmed.split(/\r?\n/)
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
const line = lines[i].trim()
|
||||
if (!line.startsWith('data: ')) continue
|
||||
try {
|
||||
const obj = JSON.parse(line.slice(6))
|
||||
if (obj && typeof obj.error === 'string' && obj.error) return obj.error
|
||||
} catch {
|
||||
/* 非 JSON 的 data 行忽略 */
|
||||
}
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
/** 从内容中解析 think 标签 */
|
||||
function parseContent(content: string) {
|
||||
const thinkRegex = /<think>([\s\S]*?)(<\/think>)?/g
|
||||
@@ -121,6 +139,16 @@ export function useStreamChat() {
|
||||
}
|
||||
|
||||
// 最终更新
|
||||
// 若整段响应是 SSE 错误帧,提取 error 字段以干净文案展示
|
||||
const sseError = extractSseError(buffer)
|
||||
if (sseError) {
|
||||
message.value.isThinking = false
|
||||
message.value.isStreaming = false
|
||||
message.value.done = true
|
||||
message.value.error = sseError
|
||||
message.value.displayContent = sseError
|
||||
return
|
||||
}
|
||||
const parsed = parseContent(buffer)
|
||||
message.value.thinkContent = parsed.think
|
||||
message.value.displayContent = parsed.display
|
||||
|
||||
@@ -34,6 +34,10 @@ export interface TrainedModel {
|
||||
name: string
|
||||
train_methods?: TrainMethod[]
|
||||
base_model_path?: string
|
||||
artifact_dir?: string
|
||||
adapter_path?: string
|
||||
compute_node_id?: string
|
||||
compute_node_name?: string
|
||||
create_time?: string
|
||||
merged?: boolean
|
||||
merging?: boolean
|
||||
@@ -212,6 +216,9 @@ export interface LoadedModel {
|
||||
status?: string
|
||||
pid?: number
|
||||
port?: number
|
||||
node_id?: string
|
||||
node_name?: string
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface CompareTask {
|
||||
@@ -230,6 +237,8 @@ export interface CompareModelRef {
|
||||
model_name: string
|
||||
model_path: string
|
||||
gpu_id: number
|
||||
node_id?: string
|
||||
node_name?: string
|
||||
source?: string
|
||||
port?: number
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, reactive, nextTick, onMounted, watch } from 'vue'
|
||||
import { ref, reactive, nextTick, onMounted, onUnmounted, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import MarkdownView from '@/components/MarkdownView.vue'
|
||||
import { useStreamChat } from '@/composables/useStreamChat'
|
||||
import { getCompare } from '@/api/modules/compare'
|
||||
import { getCompare, getLoadStatus } from '@/api/modules/compare'
|
||||
import type { CompareTask, LoadedModel } from '@/types'
|
||||
|
||||
const route = useRoute()
|
||||
@@ -37,6 +37,10 @@ const contentRef = ref<HTMLElement>()
|
||||
let activeAssistant: ChatMessage | null = null
|
||||
/** 设置面板抽屉 */
|
||||
const showSettings = ref(false)
|
||||
/** 模型仍在加载中(直接 URL 进入 chat 时兜底轮询就绪状态) */
|
||||
const taskLoading = ref(false)
|
||||
const taskError = ref('')
|
||||
let statusTimer: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
/** 获取任务信息,定位已启动的模型(mock 模式跳过) */
|
||||
async function loadTask() {
|
||||
@@ -45,6 +49,14 @@ async function loadTask() {
|
||||
task.value = await getCompare(taskId)
|
||||
const models = parseLoadedModels(task.value)
|
||||
if (models[0]?.model_name) modelName.value = models[0].model_name
|
||||
// 恢复本地保存的历史对话
|
||||
restoreHistory()
|
||||
// 模型仍在上次加载中:启动轮询等待就绪
|
||||
if (models.some((m) => m.status === 'starting')) {
|
||||
taskLoading.value = true
|
||||
await pollTaskStatus()
|
||||
statusTimer = setInterval(pollTaskStatus, 3000)
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
@@ -60,6 +72,80 @@ function parseLoadedModels(t: CompareTask | null): LoadedModel[] {
|
||||
}
|
||||
}
|
||||
|
||||
/** 对话历史本地持久化(按任务 id 存储,退出重进可恢复) */
|
||||
const STORAGE_PREFIX = 'ygft_chat_history_'
|
||||
|
||||
function historyKey(id: string | number): string {
|
||||
return `${STORAGE_PREFIX}${id}`
|
||||
}
|
||||
|
||||
function saveHistory() {
|
||||
if (isMock) return
|
||||
try {
|
||||
const snapshot = messages.value.map((m) => ({
|
||||
role: m.role,
|
||||
content: m.content,
|
||||
think: m.think,
|
||||
done: true,
|
||||
}))
|
||||
localStorage.setItem(historyKey(taskId), JSON.stringify(snapshot))
|
||||
} catch {
|
||||
// 存储失败忽略
|
||||
}
|
||||
}
|
||||
|
||||
function restoreHistory() {
|
||||
if (isMock) return
|
||||
try {
|
||||
const raw = localStorage.getItem(historyKey(taskId))
|
||||
if (!raw) return
|
||||
const parsed = JSON.parse(raw)
|
||||
if (Array.isArray(parsed)) {
|
||||
messages.value = parsed.map((m) => ({
|
||||
role: m.role === 'user' ? 'user' : 'assistant',
|
||||
content: m.content || '',
|
||||
think: m.think || '',
|
||||
isThinking: false,
|
||||
isStreaming: false,
|
||||
done: true,
|
||||
}))
|
||||
}
|
||||
} catch {
|
||||
// 恢复失败忽略
|
||||
}
|
||||
}
|
||||
|
||||
/** 停止就绪状态轮询 */
|
||||
function stopStatusPolling() {
|
||||
if (statusTimer) {
|
||||
clearInterval(statusTimer)
|
||||
statusTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
/** 轮询任务加载状态:starting → ready/error */
|
||||
async function pollTaskStatus() {
|
||||
try {
|
||||
const st = await getLoadStatus(taskId)
|
||||
const items = st.loaded_models || []
|
||||
const anyReady = items.some((m) => m.status === 'ready' || m.status === 'running')
|
||||
const anyError = items.some((m) => m.status === 'error')
|
||||
if (anyReady) {
|
||||
taskLoading.value = false
|
||||
taskError.value = ''
|
||||
stopStatusPolling()
|
||||
} else if (anyError) {
|
||||
taskLoading.value = false
|
||||
taskError.value = items.find((m) => m.status === 'error')?.error || '模型加载失败'
|
||||
stopStatusPolling()
|
||||
} else {
|
||||
taskLoading.value = true
|
||||
}
|
||||
} catch {
|
||||
// 轮询失败忽略,下次再试
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSend() {
|
||||
const question = inputQuestion.value.trim()
|
||||
if (!question || loading.value) return
|
||||
@@ -76,6 +162,7 @@ async function handleSend() {
|
||||
done: false,
|
||||
})
|
||||
messages.value.push(assistantMsg)
|
||||
saveHistory()
|
||||
|
||||
inputQuestion.value = ''
|
||||
await nextTick()
|
||||
@@ -85,6 +172,7 @@ async function handleSend() {
|
||||
// mock 模式:直接用假数据逐字填充
|
||||
if (isMock) {
|
||||
await mockReply(assistantMsg, question)
|
||||
saveHistory()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -94,6 +182,7 @@ async function handleSend() {
|
||||
await send(
|
||||
{
|
||||
model_path: route.query.model_path as string || '',
|
||||
task_id: taskId,
|
||||
system_prompt: systemPrompt.value,
|
||||
user_question: question,
|
||||
temperature: temperature.value,
|
||||
@@ -111,6 +200,7 @@ async function handleSend() {
|
||||
assistantMsg.done = true
|
||||
activeAssistant = null
|
||||
reset()
|
||||
saveHistory()
|
||||
await nextTick()
|
||||
scrollToBottom()
|
||||
}
|
||||
@@ -169,6 +259,11 @@ function handleNewChat() {
|
||||
activeAssistant = null
|
||||
messages.value = []
|
||||
reset()
|
||||
try {
|
||||
localStorage.removeItem(historyKey(taskId))
|
||||
} catch {
|
||||
// 忽略
|
||||
}
|
||||
}
|
||||
|
||||
/** 输入框自适应高度 */
|
||||
@@ -185,6 +280,7 @@ function resetInputHeight() {
|
||||
}
|
||||
|
||||
onMounted(loadTask)
|
||||
onUnmounted(stopStatusPolling)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -252,6 +348,12 @@ onMounted(loadTask)
|
||||
|
||||
<!-- 输入栏 -->
|
||||
<footer class="chat-input-container">
|
||||
<div v-if="taskLoading" class="loading-hint">
|
||||
<i class="fa fa-spinner fa-spin" style="margin-right: 6px" />模型加载中,就绪后即可对话...
|
||||
</div>
|
||||
<div v-else-if="taskError" class="loading-hint error">
|
||||
<i class="fa fa-exclamation-circle" style="margin-right: 6px" />{{ taskError }}
|
||||
</div>
|
||||
<div class="chat-input-inner">
|
||||
<button class="clear-btn" title="清空对话" @click="handleNewChat">
|
||||
<i class="fa fa-eraser" />
|
||||
@@ -261,22 +363,21 @@ onMounted(loadTask)
|
||||
v-model="inputQuestion"
|
||||
class="input-box"
|
||||
rows="1"
|
||||
:disabled="loading"
|
||||
:disabled="loading || taskLoading"
|
||||
placeholder="给模型发送消息..."
|
||||
@keydown.enter.exact.prevent="handleSend"
|
||||
@input="autoResize"
|
||||
/>
|
||||
<button
|
||||
class="send-btn"
|
||||
:class="{ active: inputQuestion.trim() && !loading }"
|
||||
:disabled="!inputQuestion.trim() || loading"
|
||||
:class="{ active: inputQuestion.trim() && !loading && !taskLoading }"
|
||||
:disabled="!inputQuestion.trim() || loading || taskLoading"
|
||||
@click="handleSend"
|
||||
>
|
||||
<i class="fa fa-arrow-up" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer-hint">内容由 AI 生成,请仔细甄别。</div>
|
||||
</footer>
|
||||
|
||||
<!-- 设置抽屉(系统提示词等) -->
|
||||
@@ -703,9 +804,18 @@ onMounted(loadTask)
|
||||
}
|
||||
}
|
||||
|
||||
.footer-hint {
|
||||
margin-top: 12px;
|
||||
font-size: 12px;
|
||||
color: #9ca3af;
|
||||
.loading-hint {
|
||||
margin-bottom: 10px;
|
||||
padding: 6px 14px;
|
||||
font-size: 13px;
|
||||
color: #b45309;
|
||||
background: #fef3c7;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
|
||||
&.error {
|
||||
color: #b91c1c;
|
||||
background: #fee2e2;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, reactive, computed, onMounted } from 'vue'
|
||||
import { ref, reactive, computed, onMounted, watch } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
|
||||
import PageCard from '@/components/PageCard.vue'
|
||||
import { getModelList, getTrainedModels } from '@/api/modules/model'
|
||||
import { getSystemInfo } from '@/api/modules/system'
|
||||
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
||||
import { createCompare, preloadLocalModel, preloadTrainedModel } from '@/api/modules/compare'
|
||||
import { createCompare, loadCompare } from '@/api/modules/compare'
|
||||
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
|
||||
|
||||
const router = useRouter()
|
||||
@@ -27,6 +27,8 @@ interface SelectableModel {
|
||||
name: string
|
||||
source: 'database' | 'trained'
|
||||
model_path: string
|
||||
compute_node_id?: string
|
||||
compute_node_name?: string
|
||||
merged?: boolean
|
||||
merging?: boolean
|
||||
disabled?: boolean
|
||||
@@ -51,6 +53,8 @@ const trainedOptions = computed<SelectableModel[]>(() =>
|
||||
name: m.name,
|
||||
source: 'trained',
|
||||
model_path: m.merged_path || m.base_model_path || '',
|
||||
compute_node_id: m.compute_node_id,
|
||||
compute_node_name: m.compute_node_name,
|
||||
merged: m.merged,
|
||||
merging: m.merging,
|
||||
disabled: m.merged === false,
|
||||
@@ -80,7 +84,7 @@ const form = reactive({
|
||||
/** 选中的模型 key(单选) */
|
||||
model_key: '',
|
||||
/** 使用的 GPU */
|
||||
gpu_id: 0,
|
||||
gpu_key: '',
|
||||
})
|
||||
|
||||
const rules: FormRules = {
|
||||
@@ -90,6 +94,13 @@ const rules: FormRules = {
|
||||
|
||||
/** 当前选中的模型对象 */
|
||||
const selectedModel = computed(() => modelMap.value[form.model_key])
|
||||
const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key))
|
||||
|
||||
watch(selectedModel, (model) => {
|
||||
if (!model?.compute_node_id) return
|
||||
const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
|
||||
if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}`
|
||||
})
|
||||
|
||||
async function handleSubmit() {
|
||||
if (!formRef.value) return
|
||||
@@ -101,62 +112,47 @@ async function handleSubmit() {
|
||||
return
|
||||
}
|
||||
submitting.value = true
|
||||
startupStatus.value = '正在启动模型服务...'
|
||||
startupStatus.value = '正在创建推理任务...'
|
||||
try {
|
||||
// Step 1: 将模型加载到算力节点
|
||||
const preloadPayload = {
|
||||
model_name_or_path: m.model_path,
|
||||
model_name: m.name,
|
||||
template: 'qwen',
|
||||
}
|
||||
let preloadResult: any
|
||||
if (m.source === 'trained') {
|
||||
preloadResult = await preloadTrainedModel(preloadPayload)
|
||||
} else {
|
||||
preloadResult = await preloadLocalModel(preloadPayload)
|
||||
}
|
||||
|
||||
if (preloadResult && (preloadResult as any).error) {
|
||||
ElMessage.warning(`模型加载失败:${(preloadResult as any).error}`)
|
||||
submitting.value = false
|
||||
startupStatus.value = ''
|
||||
if (!m.model_path) {
|
||||
ElMessage.warning('当前模型未配置算力节点可访问路径,请先在模型管理中维护模型路径')
|
||||
return
|
||||
}
|
||||
|
||||
// Step 2: 创建推理任务记录
|
||||
// Step 1: 创建推理任务记录
|
||||
const taskResult = await createCompare({
|
||||
name: form.name || m.name,
|
||||
description: form.description,
|
||||
status: 'pending',
|
||||
models: [
|
||||
{
|
||||
model_id: String(m.id),
|
||||
model_name: m.name,
|
||||
model_path: m.model_path,
|
||||
source: m.source,
|
||||
gpu_id: form.gpu_id,
|
||||
gpu_id: selectedGpu.value?.id ?? 0,
|
||||
node_id: selectedGpu.value?.node_id || m.compute_node_id,
|
||||
node_name: selectedGpu.value?.node_name || m.compute_node_name,
|
||||
},
|
||||
],
|
||||
})
|
||||
const taskId = taskResult?.id || 'unknown'
|
||||
|
||||
ElMessage.success('模型已启动')
|
||||
router.push({
|
||||
path: `/model-inference/chat/${taskId}`,
|
||||
query: {
|
||||
model: m.name,
|
||||
source: m.source,
|
||||
model_path: m.model_path,
|
||||
},
|
||||
})
|
||||
// Step 2: 统一通过推理任务加载接口异步派发模型加载,状态会落到列表记录中。
|
||||
startupStatus.value = '正在启动模型服务,首次加载可能需要数分钟...'
|
||||
const loadResult: any = await loadCompare(taskId)
|
||||
if (loadResult?.status === 'failed' || loadResult?.error) {
|
||||
ElMessage.warning(`模型加载失败:${loadResult?.error || '请检查算力节点日志'}`)
|
||||
router.push('/model-inference')
|
||||
return
|
||||
}
|
||||
|
||||
// 加载为异步派发,回到列表页可看到“启动中 → 已就绪”的状态流转
|
||||
ElMessage.success('模型加载中,就绪后即可对话')
|
||||
router.push('/model-inference')
|
||||
} catch (e: any) {
|
||||
// 真实 API 失败时回退到 mock 模式(方便无算力节点的开发调试)
|
||||
const m = selectedModel.value!
|
||||
const reason = e?.message || e?.toString() || '未知错误'
|
||||
ElMessage.warning(`推理服务启动失败:${reason},进入 mock 演示模式`)
|
||||
router.push({
|
||||
path: '/model-inference/chat/mock',
|
||||
query: { model: m.name },
|
||||
})
|
||||
ElMessage.warning(`推理服务启动失败:${reason}`)
|
||||
} finally {
|
||||
submitting.value = false
|
||||
startupStatus.value = ''
|
||||
@@ -181,7 +177,10 @@ async function loadData() {
|
||||
gpus.value = sys?.gpu || []
|
||||
computeNodes.value = nodes || []
|
||||
// 默认选中第一个空闲 GPU
|
||||
if (idleGpus.value.length > 0) form.gpu_id = idleGpus.value[0].id ?? 0
|
||||
if (idleGpus.value.length > 0) {
|
||||
const firstGpu = idleGpus.value[0]
|
||||
form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}`
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
@@ -229,12 +228,12 @@ onMounted(loadData)
|
||||
</el-form-item>
|
||||
|
||||
<el-form-item label="GPU">
|
||||
<el-select v-model="form.gpu_id" style="width: 400px">
|
||||
<el-select v-model="form.gpu_key" style="width: 400px">
|
||||
<el-option
|
||||
v-for="g in idleGpus"
|
||||
:key="g.id ?? 0"
|
||||
:label="`${g.name} (GPU${g.id ?? 0}) [空闲]`"
|
||||
:value="g.id ?? 0"
|
||||
:key="`${g.node_id || ''}:${g.id ?? 0}`"
|
||||
:label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
|
||||
:value="`${g.node_id || ''}:${g.id ?? 0}`"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
|
||||
@@ -92,11 +92,9 @@ async function handleUnload(row: any) {
|
||||
loadData()
|
||||
}
|
||||
|
||||
/** 删除(先释放算力节点再删除记录) */
|
||||
/** 删除(后端删除内部会 best-effort 释放算力节点,这里直接删记录) */
|
||||
async function handleDelete(row: any) {
|
||||
await ElMessageBox.confirm('确定要删除该推理记录吗?将先释放算力节点再删除。', '确认删除', { type: 'warning' })
|
||||
// 先释放算力节点上的模型
|
||||
await unloadCompare(row.id).catch(() => {})
|
||||
await deleteCompare(row.id)
|
||||
dataList.value = dataList.value.filter((item) => item.id !== row.id)
|
||||
await loadData(true)
|
||||
|
||||
@@ -18,9 +18,12 @@ const trainedModels = ref<TrainedModel[]>([])
|
||||
const currentModel = computed(() => trainedModels.value.find((m) => m.name === modelName.value))
|
||||
|
||||
const form = reactive({
|
||||
trained_model_id: '',
|
||||
model_name: modelName.value,
|
||||
train_method: method.value,
|
||||
base_model_path: '',
|
||||
adapter_path: '',
|
||||
compute_node_id: '',
|
||||
})
|
||||
|
||||
async function loadModel() {
|
||||
@@ -28,23 +31,30 @@ async function loadModel() {
|
||||
const res = await getTrainedModels()
|
||||
trainedModels.value = res?.models || []
|
||||
const target = trainedModels.value.find((m) => m.name === modelName.value)
|
||||
form.trained_model_id = target?.id == null ? '' : String(target.id)
|
||||
form.base_model_path = target?.base_model_path || ''
|
||||
form.adapter_path = target?.artifact_dir || target?.adapter_path || target?.merged_path || ''
|
||||
form.compute_node_id = target?.compute_node_id || ''
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
async function handleMerge() {
|
||||
if (!form.model_name || !form.base_model_path) {
|
||||
if (!form.model_name || !form.base_model_path || !form.adapter_path) {
|
||||
ElMessage.warning('缺少模型信息')
|
||||
return
|
||||
}
|
||||
merging.value = true
|
||||
try {
|
||||
await mergeModel({
|
||||
trained_model_id: form.trained_model_id || form.model_name,
|
||||
model_name: form.model_name,
|
||||
train_method: form.train_method,
|
||||
base_model_path: form.base_model_path,
|
||||
adapter_path: form.adapter_path,
|
||||
compute_node_id: form.compute_node_id,
|
||||
output_model_name: `${form.model_name}-merged`,
|
||||
})
|
||||
ElMessage.success('合并成功')
|
||||
router.push('/model-manage')
|
||||
@@ -82,6 +92,9 @@ onMounted(loadModel)
|
||||
<el-form-item label="基座模型路径">
|
||||
<el-input v-model="form.base_model_path" placeholder="基座模型路径" />
|
||||
</el-form-item>
|
||||
<el-form-item label="Adapter 路径">
|
||||
<el-input v-model="form.adapter_path" placeholder="LoRA Adapter 权重目录" />
|
||||
</el-form-item>
|
||||
|
||||
<el-form-item>
|
||||
<el-button type="primary" :loading="merging" @click="handleMerge">
|
||||
|
||||
@@ -8,13 +8,14 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
import '@/plugins/echarts-training-log'
|
||||
import { useModelsStore } from '@/stores/models'
|
||||
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, type TrainingDiagnostic } from '@/api/modules/fineTune'
|
||||
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
|
||||
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
|
||||
import { getDataset } from '@/api/modules/dataset'
|
||||
import { getSystemInfo } from '@/api/modules/system'
|
||||
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
|
||||
import {
|
||||
buildMetricChartOption,
|
||||
metricsFromApi,
|
||||
parseTrainingLog,
|
||||
resolveTrainingLogFile,
|
||||
} from './training-log/trainingLogModel'
|
||||
@@ -42,6 +43,7 @@ const loading = ref(true)
|
||||
|
||||
// 训练指标数据(ECharts 接收 number[],下标即 step)
|
||||
const metricData = reactive({
|
||||
steps: [] as number[],
|
||||
loss: [] as number[],
|
||||
gradNorm: [] as number[],
|
||||
lr: [] as number[],
|
||||
@@ -62,9 +64,9 @@ const gpuExpanded = ref(false)
|
||||
let refreshInFlight = false
|
||||
|
||||
/** 三个曲线的 ECharts 配置(响应式,数据变化自动重绘) */
|
||||
const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, '#4f46e5'))
|
||||
const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, '#3b82f6'))
|
||||
const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, '#14b8a6', true))
|
||||
const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, metricData.steps, '#4f46e5'))
|
||||
const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, metricData.steps, '#3b82f6'))
|
||||
const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, metricData.steps, '#14b8a6', true))
|
||||
const baseModelName = computed(() => task.value?.base_model != null
|
||||
? modelsStore.getModelName(task.value.base_model)
|
||||
: '未配置')
|
||||
@@ -77,10 +79,10 @@ const trainingMethodName = computed(() => task.value?.train_method
|
||||
const taskGpuLabel = computed(() => task.value?.gpus?.length
|
||||
? task.value.gpus.map((gpuId) => `GPU ${gpuId}`).join('、')
|
||||
: '未配置')
|
||||
const latestLoss = computed(() => metricData.loss[metricData.loss.length - 1])
|
||||
const latestGradNorm = computed(() => metricData.gradNorm[metricData.gradNorm.length - 1])
|
||||
const latestLearningRate = computed(() => metricData.lr[metricData.lr.length - 1])
|
||||
const latestEpoch = computed(() => metricData.epoch[metricData.epoch.length - 1])
|
||||
const latestLoss = computed(() => lastFinite(metricData.loss))
|
||||
const latestGradNorm = computed(() => lastFinite(metricData.gradNorm))
|
||||
const latestLearningRate = computed(() => lastFinite(metricData.lr))
|
||||
const latestEpoch = computed(() => lastFinite(metricData.epoch))
|
||||
const logLineCount = computed(() => logContent.value ? logContent.value.split(/\r?\n/).length : 0)
|
||||
const taskGpuItems = computed<TaskGpuItem[]>(() => (task.value?.gpus ?? []).map((gpuId) => {
|
||||
const index = Number(gpuId)
|
||||
@@ -123,7 +125,7 @@ const gpuRefreshState = computed(() => {
|
||||
})
|
||||
return gpuLoadError.value
|
||||
? `更新失败 · 最后更新 ${updateTime}`
|
||||
: `${updateTime} 更新 · 每 5 秒刷新`
|
||||
: `${updateTime} 更新 · 每 3 秒刷新`
|
||||
})
|
||||
|
||||
function formatMetric(value?: number, scientific = false) {
|
||||
@@ -131,6 +133,13 @@ function formatMetric(value?: number, scientific = false) {
|
||||
return scientific ? value.toExponential(2) : value.toFixed(4).replace(/0+$/, '').replace(/\.$/, '')
|
||||
}
|
||||
|
||||
function lastFinite(values: number[]) {
|
||||
for (let index = values.length - 1; index >= 0; index -= 1) {
|
||||
if (Number.isFinite(values[index])) return values[index]
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function safePercent(value?: number) {
|
||||
return Math.round(Math.min(100, Math.max(0, Number(value || 0))))
|
||||
}
|
||||
@@ -216,6 +225,7 @@ const isLoraMethod = computed(() =>
|
||||
function applyLogContent(content: string) {
|
||||
const parsed = parseTrainingLog(content)
|
||||
logContent.value = content
|
||||
metricData.steps = parsed.metrics.steps
|
||||
metricData.loss = parsed.metrics.loss
|
||||
metricData.gradNorm = parsed.metrics.gradNorm
|
||||
metricData.lr = parsed.metrics.lr
|
||||
@@ -223,6 +233,26 @@ function applyLogContent(content: string) {
|
||||
Object.assign(summary, parsed.summary)
|
||||
}
|
||||
|
||||
function applyMetricData(metrics = { steps: [] as number[], loss: [] as number[], gradNorm: [] as number[], lr: [] as number[], epoch: [] as number[] }) {
|
||||
metricData.steps = metrics.steps
|
||||
metricData.loss = metrics.loss
|
||||
metricData.gradNorm = metrics.gradNorm
|
||||
metricData.lr = metrics.lr
|
||||
metricData.epoch = metrics.epoch
|
||||
}
|
||||
|
||||
async function loadMetrics(currentTask: FineTuneTask) {
|
||||
try {
|
||||
const points = await getFineTuneMetrics(currentTask.id)
|
||||
const parsed = metricsFromApi(points || [])
|
||||
if (parsed.loss.length || parsed.gradNorm.length || parsed.lr.length) {
|
||||
applyMetricData(parsed)
|
||||
}
|
||||
} catch {
|
||||
// 日志解析结果会作为兜底曲线数据。
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLog(currentTask: FineTuneTask) {
|
||||
try {
|
||||
const runtime = await getFineTuneLogs(currentTask.id, { tail_lines: 800 })
|
||||
@@ -277,6 +307,7 @@ async function refreshAll() {
|
||||
? loadDataset(currentTask.train_dataset_id)
|
||||
: Promise.resolve()
|
||||
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
|
||||
await loadMetrics(currentTask)
|
||||
} finally {
|
||||
loading.value = false
|
||||
refreshInFlight = false
|
||||
@@ -523,7 +554,7 @@ onMounted(async () => {
|
||||
|
||||
<!-- 训练曲线 -->
|
||||
<PageCard class="metrics-panel" title="训练曲线" subtitle="持续监控模型收敛情况与学习率变化">
|
||||
<template #extra><span class="refresh-state">每 5 秒刷新</span></template>
|
||||
<template #extra><span class="refresh-state">每 3 秒刷新</span></template>
|
||||
<div class="chart-list" aria-label="训练指标曲线">
|
||||
<section class="chart-section">
|
||||
<div class="chart-section-header">
|
||||
@@ -560,7 +591,7 @@ onMounted(async () => {
|
||||
|
||||
<!-- 原始日志 -->
|
||||
<PageCard class="log-card" title="训练日志" subtitle="查看训练任务的原始运行输出">
|
||||
<template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 5 秒刷新</span></template>
|
||||
<template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 3 秒刷新</span></template>
|
||||
<pre class="log-pre">{{ logContent || '暂无日志' }}</pre>
|
||||
</PageCard>
|
||||
</template>
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import type { EChartsOption } from 'echarts'
|
||||
import type { FineTuneTask, TrainingLogFile } from '@/types'
|
||||
import type { FineTuneMetricPoint } from '@/api/modules/fineTune'
|
||||
|
||||
export interface TrainingMetricData {
|
||||
steps: number[]
|
||||
loss: number[]
|
||||
gradNorm: number[]
|
||||
lr: number[]
|
||||
@@ -26,7 +28,7 @@ function escapeRegExp(value: string) {
|
||||
}
|
||||
|
||||
function extractNumber(source: string, key: string) {
|
||||
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*:\\s*(${NUMBER_SOURCE})`, 'i'))
|
||||
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*(?:=|:)\\s*(${NUMBER_SOURCE})`, 'i'))
|
||||
return match ? Number(match[1]) : undefined
|
||||
}
|
||||
|
||||
@@ -57,24 +59,44 @@ export function resolveTrainingLogFile(
|
||||
|
||||
/** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
|
||||
export function parseTrainingMetrics(text: string): TrainingMetricData {
|
||||
const metrics: TrainingMetricData = { loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||
const blocks = text.match(/\{[^{}\r\n]*\}/g) || []
|
||||
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||
const candidates = text
|
||||
.split(/\r?\n/)
|
||||
.flatMap((line) => {
|
||||
const blocks = line.match(/\{[^{}\r\n]*\}/g)
|
||||
return blocks?.length ? blocks.map((block) => `${line} ${block}`) : [line]
|
||||
})
|
||||
|
||||
for (const block of blocks) {
|
||||
const loss = extractNumber(block, 'loss')
|
||||
const gradNorm = extractNumber(block, 'grad_norm')
|
||||
const learningRate = extractNumber(block, 'learning_rate')
|
||||
const epoch = extractNumber(block, 'epoch')
|
||||
if (loss == null || gradNorm == null || learningRate == null) continue
|
||||
metrics.loss.push(loss)
|
||||
metrics.gradNorm.push(gradNorm)
|
||||
metrics.lr.push(learningRate)
|
||||
if (epoch != null) metrics.epoch.push(epoch)
|
||||
for (const [index, line] of candidates.entries()) {
|
||||
const loss = extractNumber(line, 'loss')
|
||||
const gradNorm = extractNumber(line, 'grad_norm')
|
||||
const learningRate = extractNumber(line, 'learning_rate')
|
||||
const epoch = extractNumber(line, 'epoch')
|
||||
if (loss == null && gradNorm == null && learningRate == null) continue
|
||||
metrics.steps.push(extractNumber(line, 'step') ?? metrics.steps.length + index + 1)
|
||||
metrics.loss.push(loss ?? Number.NaN)
|
||||
metrics.gradNorm.push(gradNorm ?? Number.NaN)
|
||||
metrics.lr.push(learningRate ?? Number.NaN)
|
||||
metrics.epoch.push(epoch ?? Number.NaN)
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
export function metricsFromApi(points: FineTuneMetricPoint[]): TrainingMetricData {
|
||||
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||
for (const [index, point] of points.entries()) {
|
||||
const hasMetric = point.loss != null || point.grad_norm != null || point.learning_rate != null
|
||||
if (!hasMetric) continue
|
||||
metrics.steps.push(Number(point.step || index + 1))
|
||||
metrics.loss.push(point.loss == null ? Number.NaN : Number(point.loss))
|
||||
metrics.gradNorm.push(point.grad_norm == null ? Number.NaN : Number(point.grad_norm))
|
||||
metrics.lr.push(point.learning_rate == null ? Number.NaN : Number(point.learning_rate))
|
||||
metrics.epoch.push(point.epoch == null ? Number.NaN : Number(point.epoch))
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
/** 每次都返回新对象,日志截断或切换时不会残留上一轮汇总。 */
|
||||
export function parseTrainingSummary(text: string): TrainingSummary {
|
||||
const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
|
||||
@@ -102,11 +124,23 @@ export function parseTrainingLog(text: string): ParsedTrainingLog {
|
||||
export function buildMetricChartOption(
|
||||
label: string,
|
||||
data: number[],
|
||||
steps: number[],
|
||||
color: string,
|
||||
logScale = false,
|
||||
): EChartsOption {
|
||||
const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
|
||||
return {
|
||||
grid: { top: 24, right: 20, bottom: 56, left: 56 },
|
||||
graphic: visibleData.some((value) => value != null)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
type: 'text',
|
||||
left: 'center',
|
||||
top: 'middle',
|
||||
style: { text: '暂无训练指标数据', fill: '#94a3b8', fontSize: 13 },
|
||||
},
|
||||
],
|
||||
tooltip: {
|
||||
trigger: 'axis',
|
||||
axisPointer: { type: 'cross' },
|
||||
@@ -116,6 +150,7 @@ export function buildMetricChartOption(
|
||||
},
|
||||
xAxis: {
|
||||
type: 'category',
|
||||
data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
|
||||
boundaryGap: false,
|
||||
name: 'Step',
|
||||
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
|
||||
@@ -142,7 +177,7 @@ export function buildMetricChartOption(
|
||||
{
|
||||
name: label,
|
||||
type: 'line',
|
||||
data,
|
||||
data: visibleData,
|
||||
smooth: true,
|
||||
symbol: 'none',
|
||||
lineStyle: { width: 2, color },
|
||||
|
||||
Reference in New Issue
Block a user