YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 模型推理异步加载与对话链路修复，同步基线

Browse Source 
模型推理全异步化改造：
- 计算节点 InferenceSession 改为后台线程异步加载模型，load 立即返回，
  加载期间事件循环保持响应（/inference/status 与 /health 不阻塞）
- 后端模型加载改为异步派发 + 轮询对账器（reconcile_inference_loads），
  任务状态由 starting 自动推进到 ready/error，解决多节点启动超时
  （timeout of 120000ms exceeded）
- 推理删除/卸载改为任务感知 + 短超时，删除先删记录再 best-effort 卸载，
  不再被不可达节点阻塞；同节点新模型替换旧任务标记失效
- 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点，
  useStreamChat 解析 SSE 错误帧以干净文案展示
- 对话历史按任务 id 本地持久化，退出重进可恢复；移除页脚提示文本
- 新增后端推理异步加载与计算节点异步状态机单元测试

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-04 16:59:34 +08:00
parent 250e060271
commit 0271942ba5
21 changed files with 1272 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
frontend/src/views/inference/InferenceChatView.vue
View File

@@ -1,10 +1,10 @@
<script setup lang="ts">
import { ref, reactive, nextTick, onMounted, watch } from 'vue'
import { ref, reactive, nextTick, onMounted, onUnmounted, watch } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { ElMessage } from 'element-plus'
import MarkdownView from '@/components/MarkdownView.vue'
import { useStreamChat } from '@/composables/useStreamChat'
import { getCompare } from '@/api/modules/compare'
import { getCompare, getLoadStatus } from '@/api/modules/compare'
import type { CompareTask, LoadedModel } from '@/types'
const route = useRoute()
@@ -37,6 +37,10 @@ const contentRef = ref<HTMLElement>()
let activeAssistant: ChatMessage | null = null
/** 设置面板抽屉 */
const showSettings = ref(false)
/** 模型仍在加载中（直接 URL 进入 chat 时兜底轮询就绪状态） */
const taskLoading = ref(false)
const taskError = ref('')
let statusTimer: ReturnType<typeof setInterval> | null = null
/** 获取任务信息，定位已启动的模型（mock 模式跳过） */
async function loadTask() {
@@ -45,6 +49,14 @@ async function loadTask() {
task.value = await getCompare(taskId)
const models = parseLoadedModels(task.value)
if (models[0]?.model_name) modelName.value = models[0].model_name
// 恢复本地保存的历史对话
restoreHistory()
// 模型仍在上次加载中：启动轮询等待就绪
if (models.some((m) => m.status === 'starting')) {
taskLoading.value = true
await pollTaskStatus()
statusTimer = setInterval(pollTaskStatus, 3000)
}
} catch {
// ignore
}
@@ -60,6 +72,80 @@ function parseLoadedModels(t: CompareTask | null): LoadedModel[] {
}
}
/** 对话历史本地持久化（按任务 id 存储，退出重进可恢复） */
const STORAGE_PREFIX = 'ygft_chat_history_'
function historyKey(id: string | number): string {
return `${STORAGE_PREFIX}${id}`
}
function saveHistory() {
if (isMock) return
try {
const snapshot = messages.value.map((m) => ({
role: m.role,
content: m.content,
think: m.think,
done: true,
}))
localStorage.setItem(historyKey(taskId), JSON.stringify(snapshot))
} catch {
// 存储失败忽略
}
}
function restoreHistory() {
if (isMock) return
try {
const raw = localStorage.getItem(historyKey(taskId))
if (!raw) return
const parsed = JSON.parse(raw)
if (Array.isArray(parsed)) {
messages.value = parsed.map((m) => ({
role: m.role === 'user' ? 'user' : 'assistant',
content: m.content || '',
think: m.think || '',
isThinking: false,
isStreaming: false,
done: true,
}))
}
} catch {
// 恢复失败忽略
}
}
/** 停止就绪状态轮询 */
function stopStatusPolling() {
if (statusTimer) {
clearInterval(statusTimer)
statusTimer = null
}
}
/** 轮询任务加载状态：starting → ready/error */
async function pollTaskStatus() {
try {
const st = await getLoadStatus(taskId)
const items = st.loaded_models || []
const anyReady = items.some((m) => m.status === 'ready' || m.status === 'running')
const anyError = items.some((m) => m.status === 'error')
if (anyReady) {
taskLoading.value = false
taskError.value = ''
stopStatusPolling()
} else if (anyError) {
taskLoading.value = false
taskError.value = items.find((m) => m.status === 'error')?.error || '模型加载失败'
stopStatusPolling()
} else {
taskLoading.value = true
}
} catch {
// 轮询失败忽略，下次再试
}
}
async function handleSend() {
const question = inputQuestion.value.trim()
if (!question || loading.value) return
@@ -76,6 +162,7 @@ async function handleSend() {
done: false,
})
messages.value.push(assistantMsg)
saveHistory()
inputQuestion.value = ''
await nextTick()
@@ -85,6 +172,7 @@ async function handleSend() {
// mock 模式：直接用假数据逐字填充
if (isMock) {
await mockReply(assistantMsg, question)
saveHistory()
return
}
@@ -94,6 +182,7 @@ async function handleSend() {
await send(
{
model_path: route.query.model_path as string || '',
task_id: taskId,
system_prompt: systemPrompt.value,
user_question: question,
temperature: temperature.value,
@@ -111,6 +200,7 @@ async function handleSend() {
assistantMsg.done = true
activeAssistant = null
reset()
saveHistory()
await nextTick()
scrollToBottom()
}
@@ -169,6 +259,11 @@ function handleNewChat() {
activeAssistant = null
messages.value = []
reset()
try {
localStorage.removeItem(historyKey(taskId))
} catch {
// 忽略
}
}
/** 输入框自适应高度 */
@@ -185,6 +280,7 @@ function resetInputHeight() {
}
onMounted(loadTask)
onUnmounted(stopStatusPolling)
</script>
<template>
@@ -252,6 +348,12 @@ onMounted(loadTask)
<!-- 输入栏 -->
<footer class="chat-input-container">
<div v-if="taskLoading" class="loading-hint">
<i class="fa fa-spinner fa-spin" style="margin-right: 6px" />模型加载中，就绪后即可对话...
</div>
<div v-else-if="taskError" class="loading-hint error">
<i class="fa fa-exclamation-circle" style="margin-right: 6px" />{{ taskError }}
</div>
<div class="chat-input-inner">
<button class="clear-btn" title="清空对话" @click="handleNewChat">
<i class="fa fa-eraser" />
@@ -261,22 +363,21 @@ onMounted(loadTask)
v-model="inputQuestion"
class="input-box"
rows="1"
:disabled="loading"
:disabled="loading || taskLoading"
placeholder="给模型发送消息..."
@keydown.enter.exact.prevent="handleSend"
@input="autoResize"
/>
<button
class="send-btn"
:class="{ active: inputQuestion.trim() && !loading }"
:disabled="!inputQuestion.trim() || loading"
:class="{ active: inputQuestion.trim() && !loading && !taskLoading }"
:disabled="!inputQuestion.trim() || loading || taskLoading"
@click="handleSend"
>
<i class="fa fa-arrow-up" />
</button>
</div>
</div>
<div class="footer-hint">内容由 AI 生成，请仔细甄别。</div>
</footer>
<!-- 设置抽屉（系统提示词等） -->
@@ -703,9 +804,18 @@ onMounted(loadTask)
}
}
.footer-hint {
margin-top: 12px;
font-size: 12px;
color: #9ca3af;
.loading-hint {
margin-bottom: 10px;
padding: 6px 14px;
font-size: 13px;
color: #b45309;
background: #fef3c7;
border-radius: 8px;
text-align: center;
&.error {
color: #b91c1c;
background: #fee2e2;
}
}
</style>

87
frontend/src/views/inference/InferenceCreateView.vue
View File

@@ -1,12 +1,12 @@
<script setup lang="ts">
import { ref, reactive, computed, onMounted } from 'vue'
import { ref, reactive, computed, onMounted, watch } from 'vue'
import { useRouter } from 'vue-router'
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
import PageCard from '@/components/PageCard.vue'
import { getModelList, getTrainedModels } from '@/api/modules/model'
import { getSystemInfo } from '@/api/modules/system'
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
import { createCompare, preloadLocalModel, preloadTrainedModel } from '@/api/modules/compare'
import { createCompare, loadCompare } from '@/api/modules/compare'
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
const router = useRouter()
@@ -27,6 +27,8 @@ interface SelectableModel {
name: string
source: 'database' | 'trained'
model_path: string
compute_node_id?: string
compute_node_name?: string
merged?: boolean
merging?: boolean
disabled?: boolean
@@ -51,6 +53,8 @@ const trainedOptions = computed<SelectableModel[]>(() =>
name: m.name,
source: 'trained',
model_path: m.merged_path || m.base_model_path || '',
compute_node_id: m.compute_node_id,
compute_node_name: m.compute_node_name,
merged: m.merged,
merging: m.merging,
disabled: m.merged === false,
@@ -80,7 +84,7 @@ const form = reactive({
/** 选中的模型 key（单选） */
model_key: '',
/** 使用的 GPU */
gpu_id: 0,
gpu_key: '',
})
const rules: FormRules = {
@@ -90,6 +94,13 @@ const rules: FormRules = {
/** 当前选中的模型对象 */
const selectedModel = computed(() => modelMap.value[form.model_key])
const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key))
watch(selectedModel, (model) => {
if (!model?.compute_node_id) return
const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}`
})
async function handleSubmit() {
if (!formRef.value) return
@@ -101,62 +112,47 @@ async function handleSubmit() {
return
}
submitting.value = true
startupStatus.value = '正在启动模型服务...'
startupStatus.value = '正在创建推理任务...'
try {
// Step 1: 将模型加载到算力节点
const preloadPayload = {
model_name_or_path: m.model_path,
model_name: m.name,
template: 'qwen',
}
let preloadResult: any
if (m.source === 'trained') {
preloadResult = await preloadTrainedModel(preloadPayload)
} else {
preloadResult = await preloadLocalModel(preloadPayload)
}
if (preloadResult && (preloadResult as any).error) {
ElMessage.warning(`模型加载失败：${(preloadResult as any).error}`)
submitting.value = false
startupStatus.value = ''
if (!m.model_path) {
ElMessage.warning('当前模型未配置算力节点可访问路径，请先在模型管理中维护模型路径')
return
}
// Step 2: 创建推理任务记录
// Step 1: 创建推理任务记录
const taskResult = await createCompare({
name: form.name || m.name,
description: form.description,
status: 'pending',
models: [
{
model_id: String(m.id),
model_name: m.name,
model_path: m.model_path,
source: m.source,
gpu_id: form.gpu_id,
gpu_id: selectedGpu.value?.id ?? 0,
node_id: selectedGpu.value?.node_id || m.compute_node_id,
node_name: selectedGpu.value?.node_name || m.compute_node_name,
},
],
})
const taskId = taskResult?.id || 'unknown'
ElMessage.success('模型已启动')
router.push({
path: `/model-inference/chat/${taskId}`,
query: {
model: m.name,
source: m.source,
model_path: m.model_path,
},
})
// Step 2: 统一通过推理任务加载接口异步派发模型加载，状态会落到列表记录中。
startupStatus.value = '正在启动模型服务，首次加载可能需要数分钟...'
const loadResult: any = await loadCompare(taskId)
if (loadResult?.status === 'failed' || loadResult?.error) {
ElMessage.warning(`模型加载失败：${loadResult?.error || '请检查算力节点日志'}`)
router.push('/model-inference')
return
}
// 加载为异步派发，回到列表页可看到“启动中 → 已就绪”的状态流转
ElMessage.success('模型加载中，就绪后即可对话')
router.push('/model-inference')
} catch (e: any) {
// 真实 API 失败时回退到 mock 模式（方便无算力节点的开发调试）
const m = selectedModel.value!
const reason = e?.message || e?.toString() || '未知错误'
ElMessage.warning(`推理服务启动失败：${reason}，进入 mock 演示模式`)
router.push({
path: '/model-inference/chat/mock',
query: { model: m.name },
})
ElMessage.warning(`推理服务启动失败：${reason}`)
} finally {
submitting.value = false
startupStatus.value = ''
@@ -181,7 +177,10 @@ async function loadData() {
gpus.value = sys?.gpu || []
computeNodes.value = nodes || []
// 默认选中第一个空闲 GPU
if (idleGpus.value.length > 0) form.gpu_id = idleGpus.value[0].id ?? 0
if (idleGpus.value.length > 0) {
const firstGpu = idleGpus.value[0]
form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}`
}
} catch {
// ignore
}
@@ -229,12 +228,12 @@ onMounted(loadData)
</el-form-item>
<el-form-item label="GPU">
<el-select v-model="form.gpu_id" style="width: 400px">
<el-select v-model="form.gpu_key" style="width: 400px">
<el-option
v-for="g in idleGpus"
:key="g.id ?? 0"
:label="`${g.name} (GPU${g.id ?? 0}) [空闲]`"
:value="g.id ?? 0"
:key="`${g.node_id || ''}:${g.id ?? 0}`"
:label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
:value="`${g.node_id || ''}:${g.id ?? 0}`"
/>
</el-select>
</el-form-item>

4
frontend/src/views/inference/InferenceListView.vue
View File

@@ -92,11 +92,9 @@ async function handleUnload(row: any) {
loadData()
}
/** 删除（先释放算力节点再删除记录） */
/** 删除（后端删除内部会 best-effort 释放算力节点，这里直接删记录） */
async function handleDelete(row: any) {
await ElMessageBox.confirm('确定要删除该推理记录吗？将先释放算力节点再删除。', '确认删除', { type: 'warning' })
// 先释放算力节点上的模型
await unloadCompare(row.id).catch(() => {})
await deleteCompare(row.id)
dataList.value = dataList.value.filter((item) => item.id !== row.id)
await loadData(true)

15
frontend/src/views/model/MergeWeightsView.vue
View File

@@ -18,9 +18,12 @@ const trainedModels = ref<TrainedModel[]>([])
const currentModel = computed(() => trainedModels.value.find((m) => m.name === modelName.value))
const form = reactive({
trained_model_id: '',
model_name: modelName.value,
train_method: method.value,
base_model_path: '',
adapter_path: '',
compute_node_id: '',
})
async function loadModel() {
@@ -28,23 +31,30 @@ async function loadModel() {
const res = await getTrainedModels()
trainedModels.value = res?.models || []
const target = trainedModels.value.find((m) => m.name === modelName.value)
form.trained_model_id = target?.id == null ? '' : String(target.id)
form.base_model_path = target?.base_model_path || ''
form.adapter_path = target?.artifact_dir || target?.adapter_path || target?.merged_path || ''
form.compute_node_id = target?.compute_node_id || ''
} catch {
// ignore
}
}
async function handleMerge() {
if (!form.model_name || !form.base_model_path) {
if (!form.model_name || !form.base_model_path || !form.adapter_path) {
ElMessage.warning('缺少模型信息')
return
}
merging.value = true
try {
await mergeModel({
trained_model_id: form.trained_model_id || form.model_name,
model_name: form.model_name,
train_method: form.train_method,
base_model_path: form.base_model_path,
adapter_path: form.adapter_path,
compute_node_id: form.compute_node_id,
output_model_name: `${form.model_name}-merged`,
})
ElMessage.success('合并成功')
router.push('/model-manage')
@@ -82,6 +92,9 @@ onMounted(loadModel)
<el-form-item label="基座模型路径">
<el-input v-model="form.base_model_path" placeholder="基座模型路径" />
</el-form-item>
<el-form-item label="Adapter 路径">
<el-input v-model="form.adapter_path" placeholder="LoRA Adapter 权重目录" />
</el-form-item>
<el-form-item>
<el-button type="primary" :loading="merging" @click="handleMerge">

53
frontend/src/views/system/TrainingLogView.vue
View File

@@ -8,13 +8,14 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
import { usePolling } from '@/composables/usePolling'
import '@/plugins/echarts-training-log'
import { useModelsStore } from '@/stores/models'
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, type TrainingDiagnostic } from '@/api/modules/fineTune'
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
import { getDataset } from '@/api/modules/dataset'
import { getSystemInfo } from '@/api/modules/system'
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
import {
buildMetricChartOption,
metricsFromApi,
parseTrainingLog,
resolveTrainingLogFile,
} from './training-log/trainingLogModel'
@@ -42,6 +43,7 @@ const loading = ref(true)
// 训练指标数据（ECharts 接收 number[]，下标即 step）
const metricData = reactive({
steps: [] as number[],
loss: [] as number[],
gradNorm: [] as number[],
lr: [] as number[],
@@ -62,9 +64,9 @@ const gpuExpanded = ref(false)
let refreshInFlight = false
/** 三个曲线的 ECharts 配置（响应式，数据变化自动重绘） */
const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, '#4f46e5'))
const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, '#3b82f6'))
const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, '#14b8a6', true))
const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, metricData.steps, '#4f46e5'))
const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, metricData.steps, '#3b82f6'))
const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, metricData.steps, '#14b8a6', true))
const baseModelName = computed(() => task.value?.base_model != null
? modelsStore.getModelName(task.value.base_model)
: '未配置')
@@ -77,10 +79,10 @@ const trainingMethodName = computed(() => task.value?.train_method
const taskGpuLabel = computed(() => task.value?.gpus?.length
? task.value.gpus.map((gpuId) => `GPU ${gpuId}`).join('、')
: '未配置')
const latestLoss = computed(() => metricData.loss[metricData.loss.length - 1])
const latestGradNorm = computed(() => metricData.gradNorm[metricData.gradNorm.length - 1])
const latestLearningRate = computed(() => metricData.lr[metricData.lr.length - 1])
const latestEpoch = computed(() => metricData.epoch[metricData.epoch.length - 1])
const latestLoss = computed(() => lastFinite(metricData.loss))
const latestGradNorm = computed(() => lastFinite(metricData.gradNorm))
const latestLearningRate = computed(() => lastFinite(metricData.lr))
const latestEpoch = computed(() => lastFinite(metricData.epoch))
const logLineCount = computed(() => logContent.value ? logContent.value.split(/\r?\n/).length : 0)
const taskGpuItems = computed<TaskGpuItem[]>(() => (task.value?.gpus ?? []).map((gpuId) => {
const index = Number(gpuId)
@@ -123,7 +125,7 @@ const gpuRefreshState = computed(() => {
})
return gpuLoadError.value
? `更新失败 · 最后更新 ${updateTime}`
: `${updateTime} 更新 · 每 5 秒刷新`
: `${updateTime} 更新 · 每 3 秒刷新`
})
function formatMetric(value?: number, scientific = false) {
@@ -131,6 +133,13 @@ function formatMetric(value?: number, scientific = false) {
return scientific ? value.toExponential(2) : value.toFixed(4).replace(/0+$/, '').replace(/\.$/, '')
}
function lastFinite(values: number[]) {
for (let index = values.length - 1; index >= 0; index -= 1) {
if (Number.isFinite(values[index])) return values[index]
}
return undefined
}
function safePercent(value?: number) {
return Math.round(Math.min(100, Math.max(0, Number(value || 0))))
}
@@ -216,6 +225,7 @@ const isLoraMethod = computed(() =>
function applyLogContent(content: string) {
const parsed = parseTrainingLog(content)
logContent.value = content
metricData.steps = parsed.metrics.steps
metricData.loss = parsed.metrics.loss
metricData.gradNorm = parsed.metrics.gradNorm
metricData.lr = parsed.metrics.lr
@@ -223,6 +233,26 @@ function applyLogContent(content: string) {
Object.assign(summary, parsed.summary)
}
function applyMetricData(metrics = { steps: [] as number[], loss: [] as number[], gradNorm: [] as number[], lr: [] as number[], epoch: [] as number[] }) {
metricData.steps = metrics.steps
metricData.loss = metrics.loss
metricData.gradNorm = metrics.gradNorm
metricData.lr = metrics.lr
metricData.epoch = metrics.epoch
}
async function loadMetrics(currentTask: FineTuneTask) {
try {
const points = await getFineTuneMetrics(currentTask.id)
const parsed = metricsFromApi(points || [])
if (parsed.loss.length || parsed.gradNorm.length || parsed.lr.length) {
applyMetricData(parsed)
}
} catch {
// 日志解析结果会作为兜底曲线数据。
}
}
async function loadLog(currentTask: FineTuneTask) {
try {
const runtime = await getFineTuneLogs(currentTask.id, { tail_lines: 800 })
@@ -277,6 +307,7 @@ async function refreshAll() {
? loadDataset(currentTask.train_dataset_id)
: Promise.resolve()
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
await loadMetrics(currentTask)
} finally {
loading.value = false
refreshInFlight = false
@@ -523,7 +554,7 @@ onMounted(async () => {
<!-- 训练曲线 -->
<PageCard class="metrics-panel" title="训练曲线" subtitle="持续监控模型收敛情况与学习率变化">
<template #extra><span class="refresh-state">每 5 秒刷新</span></template>
<template #extra><span class="refresh-state">每 3 秒刷新</span></template>
<div class="chart-list" aria-label="训练指标曲线">
<section class="chart-section">
<div class="chart-section-header">
@@ -560,7 +591,7 @@ onMounted(async () => {
<!-- 原始日志 -->
<PageCard class="log-card" title="训练日志" subtitle="查看训练任务的原始运行输出">
<template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 5 秒刷新</span></template>
<template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 3 秒刷新</span></template>
<pre class="log-pre">{{ logContent || '暂无日志' }}</pre>
</PageCard>
</template>

63
frontend/src/views/system/training-log/trainingLogModel.ts
View File

@@ -1,7 +1,9 @@
import type { EChartsOption } from 'echarts'
import type { FineTuneTask, TrainingLogFile } from '@/types'
import type { FineTuneMetricPoint } from '@/api/modules/fineTune'
export interface TrainingMetricData {
steps: number[]
loss: number[]
gradNorm: number[]
lr: number[]
@@ -26,7 +28,7 @@ function escapeRegExp(value: string) {
}
function extractNumber(source: string, key: string) {
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*:\\s*(${NUMBER_SOURCE})`, 'i'))
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*(?:=|:)\\s*(${NUMBER_SOURCE})`, 'i'))
return match ? Number(match[1]) : undefined
}
@@ -57,24 +59,44 @@ export function resolveTrainingLogFile(
/** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
export function parseTrainingMetrics(text: string): TrainingMetricData {
const metrics: TrainingMetricData = { loss: [], gradNorm: [], lr: [], epoch: [] }
const blocks = text.match(/\{[^{}\r\n]*\}/g) || []
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
const candidates = text
.split(/\r?\n/)
.flatMap((line) => {
const blocks = line.match(/\{[^{}\r\n]*\}/g)
return blocks?.length ? blocks.map((block) => `${line} ${block}`) : [line]
})
for (const block of blocks) {
const loss = extractNumber(block, 'loss')
const gradNorm = extractNumber(block, 'grad_norm')
const learningRate = extractNumber(block, 'learning_rate')
const epoch = extractNumber(block, 'epoch')
if (loss == null || gradNorm == null || learningRate == null) continue
metrics.loss.push(loss)
metrics.gradNorm.push(gradNorm)
metrics.lr.push(learningRate)
if (epoch != null) metrics.epoch.push(epoch)
for (const [index, line] of candidates.entries()) {
const loss = extractNumber(line, 'loss')
const gradNorm = extractNumber(line, 'grad_norm')
const learningRate = extractNumber(line, 'learning_rate')
const epoch = extractNumber(line, 'epoch')
if (loss == null && gradNorm == null && learningRate == null) continue
metrics.steps.push(extractNumber(line, 'step') ?? metrics.steps.length + index + 1)
metrics.loss.push(loss ?? Number.NaN)
metrics.gradNorm.push(gradNorm ?? Number.NaN)
metrics.lr.push(learningRate ?? Number.NaN)
metrics.epoch.push(epoch ?? Number.NaN)
}
return metrics
}
export function metricsFromApi(points: FineTuneMetricPoint[]): TrainingMetricData {
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
for (const [index, point] of points.entries()) {
const hasMetric = point.loss != null || point.grad_norm != null || point.learning_rate != null
if (!hasMetric) continue
metrics.steps.push(Number(point.step || index + 1))
metrics.loss.push(point.loss == null ? Number.NaN : Number(point.loss))
metrics.gradNorm.push(point.grad_norm == null ? Number.NaN : Number(point.grad_norm))
metrics.lr.push(point.learning_rate == null ? Number.NaN : Number(point.learning_rate))
metrics.epoch.push(point.epoch == null ? Number.NaN : Number(point.epoch))
}
return metrics
}
/** 每次都返回新对象，日志截断或切换时不会残留上一轮汇总。 */
export function parseTrainingSummary(text: string): TrainingSummary {
const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
@@ -102,11 +124,23 @@ export function parseTrainingLog(text: string): ParsedTrainingLog {
export function buildMetricChartOption(
label: string,
data: number[],
steps: number[],
color: string,
logScale = false,
): EChartsOption {
const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
return {
grid: { top: 24, right: 20, bottom: 56, left: 56 },
graphic: visibleData.some((value) => value != null)
? []
: [
{
type: 'text',
left: 'center',
top: 'middle',
style: { text: '暂无训练指标数据', fill: '#94a3b8', fontSize: 13 },
},
],
tooltip: {
trigger: 'axis',
axisPointer: { type: 'cross' },
@@ -116,6 +150,7 @@ export function buildMetricChartOption(
},
xAxis: {
type: 'category',
data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
boundaryGap: false,
name: 'Step',
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
@@ -142,7 +177,7 @@ export function buildMetricChartOption(
{
name: label,
type: 'line',
data,
data: visibleData,
smooth: true,
symbol: 'none',
lineStyle: { width: 2, color },