feat: 模型推理异步加载与对话链路修复，同步基线

模型推理全异步化改造： - 计算节点 InferenceSession 改为后台线程异步加载模型，load 立即返回，加载期间事件循环保持响应（/inference/status 与 /health 不阻塞） - 后端模型加载改为异步派发 + 轮询对账器（reconcile_inference_loads），任务状态由 starting 自动推进到 ready/error，解决多节点启动超时（timeout of 120000ms exceeded） - 推理删除/卸载改为任务感知 + 短超时，删除先删记录再 best-effort 卸载，不再被不可达节点阻塞；同节点新模型替换旧任务标记失效 - 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点， useStreamChat 解析 SSE 错误帧以干净文案展示 - 对话历史按任务 id 本地持久化，退出重进可恢复；移除页脚提示文本 - 新增后端推理异步加载与计算节点异步状态机单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-04 16:59:34 +08:00
parent 250e060271
commit 0271942ba5
21 changed files with 1272 additions and 245 deletions
--- a/frontend/src/views/system/TrainingLogView.vue
+++ b/frontend/src/views/system/TrainingLogView.vue
@@ -8,13 +8,14 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
 import { usePolling } from '@/composables/usePolling'
 import '@/plugins/echarts-training-log'
 import { useModelsStore } from '@/stores/models'
-import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, type TrainingDiagnostic } from '@/api/modules/fineTune'
+import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
 import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
 import { getDataset } from '@/api/modules/dataset'
 import { getSystemInfo } from '@/api/modules/system'
 import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
 import {
  buildMetricChartOption,
+  metricsFromApi,
  parseTrainingLog,
  resolveTrainingLogFile,
 } from './training-log/trainingLogModel'
@@ -42,6 +43,7 @@ const loading = ref(true)

 // 训练指标数据（ECharts 接收 number[]，下标即 step）
 const metricData = reactive({
+  steps: [] as number[],
  loss: [] as number[],
  gradNorm: [] as number[],
  lr: [] as number[],
@@ -62,9 +64,9 @@ const gpuExpanded = ref(false)
 let refreshInFlight = false

 /** 三个曲线的 ECharts 配置（响应式，数据变化自动重绘） */
-const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, '#4f46e5'))
-const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, '#3b82f6'))
-const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, '#14b8a6', true))
+const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, metricData.steps, '#4f46e5'))
+const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, metricData.steps, '#3b82f6'))
+const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, metricData.steps, '#14b8a6', true))
 const baseModelName = computed(() => task.value?.base_model != null
  ? modelsStore.getModelName(task.value.base_model)
  : '未配置')
@@ -77,10 +79,10 @@ const trainingMethodName = computed(() => task.value?.train_method
 const taskGpuLabel = computed(() => task.value?.gpus?.length
  ? task.value.gpus.map((gpuId) => `GPU ${gpuId}`).join('、')
  : '未配置')
-const latestLoss = computed(() => metricData.loss[metricData.loss.length - 1])
-const latestGradNorm = computed(() => metricData.gradNorm[metricData.gradNorm.length - 1])
-const latestLearningRate = computed(() => metricData.lr[metricData.lr.length - 1])
-const latestEpoch = computed(() => metricData.epoch[metricData.epoch.length - 1])
+const latestLoss = computed(() => lastFinite(metricData.loss))
+const latestGradNorm = computed(() => lastFinite(metricData.gradNorm))
+const latestLearningRate = computed(() => lastFinite(metricData.lr))
+const latestEpoch = computed(() => lastFinite(metricData.epoch))
 const logLineCount = computed(() => logContent.value ? logContent.value.split(/\r?\n/).length : 0)
 const taskGpuItems = computed<TaskGpuItem[]>(() => (task.value?.gpus ?? []).map((gpuId) => {
  const index = Number(gpuId)
@@ -123,7 +125,7 @@ const gpuRefreshState = computed(() => {
  })
  return gpuLoadError.value
    ? `更新失败 · 最后更新 ${updateTime}`
-    : `${updateTime} 更新 · 每 5 秒刷新`
+    : `${updateTime} 更新 · 每 3 秒刷新`
 })

 function formatMetric(value?: number, scientific = false) {
@@ -131,6 +133,13 @@ function formatMetric(value?: number, scientific = false) {
  return scientific ? value.toExponential(2) : value.toFixed(4).replace(/0+$/, '').replace(/\.$/, '')
 }

+function lastFinite(values: number[]) {
+  for (let index = values.length - 1; index >= 0; index -= 1) {
+    if (Number.isFinite(values[index])) return values[index]
+  }
+  return undefined
+}
+
 function safePercent(value?: number) {
  return Math.round(Math.min(100, Math.max(0, Number(value || 0))))
 }
@@ -216,6 +225,7 @@ const isLoraMethod = computed(() =>
 function applyLogContent(content: string) {
  const parsed = parseTrainingLog(content)
  logContent.value = content
+  metricData.steps = parsed.metrics.steps
  metricData.loss = parsed.metrics.loss
  metricData.gradNorm = parsed.metrics.gradNorm
  metricData.lr = parsed.metrics.lr
@@ -223,6 +233,26 @@ function applyLogContent(content: string) {
  Object.assign(summary, parsed.summary)
 }

+function applyMetricData(metrics = { steps: [] as number[], loss: [] as number[], gradNorm: [] as number[], lr: [] as number[], epoch: [] as number[] }) {
+  metricData.steps = metrics.steps
+  metricData.loss = metrics.loss
+  metricData.gradNorm = metrics.gradNorm
+  metricData.lr = metrics.lr
+  metricData.epoch = metrics.epoch
+}
+
+async function loadMetrics(currentTask: FineTuneTask) {
+  try {
+    const points = await getFineTuneMetrics(currentTask.id)
+    const parsed = metricsFromApi(points || [])
+    if (parsed.loss.length || parsed.gradNorm.length || parsed.lr.length) {
+      applyMetricData(parsed)
+    }
+  } catch {
+    // 日志解析结果会作为兜底曲线数据。
+  }
+}
+
 async function loadLog(currentTask: FineTuneTask) {
  try {
    const runtime = await getFineTuneLogs(currentTask.id, { tail_lines: 800 })
@@ -277,6 +307,7 @@ async function refreshAll() {
      ? loadDataset(currentTask.train_dataset_id)
      : Promise.resolve()
    await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
+    await loadMetrics(currentTask)
  } finally {
    loading.value = false
    refreshInFlight = false
@@ -523,7 +554,7 @@ onMounted(async () => {

    <!-- 训练曲线 -->
    <PageCard class="metrics-panel" title="训练曲线" subtitle="持续监控模型收敛情况与学习率变化">
-      <template #extra><span class="refresh-state">每 5 秒刷新</span></template>
+      <template #extra><span class="refresh-state">每 3 秒刷新</span></template>
      <div class="chart-list" aria-label="训练指标曲线">
        <section class="chart-section">
          <div class="chart-section-header">
@@ -560,7 +591,7 @@ onMounted(async () => {

    <!-- 原始日志 -->
    <PageCard class="log-card" title="训练日志" subtitle="查看训练任务的原始运行输出">
-      <template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 5 秒刷新</span></template>
+      <template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 3 秒刷新</span></template>
      <pre class="log-pre">{{ logContent || '暂无日志' }}</pre>
    </PageCard>
    </template>
--- a/frontend/src/views/system/training-log/trainingLogModel.ts
+++ b/frontend/src/views/system/training-log/trainingLogModel.ts
@@ -1,7 +1,9 @@
 import type { EChartsOption } from 'echarts'
 import type { FineTuneTask, TrainingLogFile } from '@/types'
+import type { FineTuneMetricPoint } from '@/api/modules/fineTune'

 export interface TrainingMetricData {
+  steps: number[]
  loss: number[]
  gradNorm: number[]
  lr: number[]
@@ -26,7 +28,7 @@ function escapeRegExp(value: string) {
 }

 function extractNumber(source: string, key: string) {
-  const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*:\\s*(${NUMBER_SOURCE})`, 'i'))
+  const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*(?:=|:)\\s*(${NUMBER_SOURCE})`, 'i'))
  return match ? Number(match[1]) : undefined
 }

@@ -57,24 +59,44 @@ export function resolveTrainingLogFile(

 /** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
 export function parseTrainingMetrics(text: string): TrainingMetricData {
-  const metrics: TrainingMetricData = { loss: [], gradNorm: [], lr: [], epoch: [] }
-  const blocks = text.match(/\{[^{}\r\n]*\}/g) || []
+  const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
+  const candidates = text
+    .split(/\r?\n/)
+    .flatMap((line) => {
+      const blocks = line.match(/\{[^{}\r\n]*\}/g)
+      return blocks?.length ? blocks.map((block) => `${line} ${block}`) : [line]
+    })

-  for (const block of blocks) {
-    const loss = extractNumber(block, 'loss')
-    const gradNorm = extractNumber(block, 'grad_norm')
-    const learningRate = extractNumber(block, 'learning_rate')
-    const epoch = extractNumber(block, 'epoch')
-    if (loss == null || gradNorm == null || learningRate == null) continue
-    metrics.loss.push(loss)
-    metrics.gradNorm.push(gradNorm)
-    metrics.lr.push(learningRate)
-    if (epoch != null) metrics.epoch.push(epoch)
+  for (const [index, line] of candidates.entries()) {
+    const loss = extractNumber(line, 'loss')
+    const gradNorm = extractNumber(line, 'grad_norm')
+    const learningRate = extractNumber(line, 'learning_rate')
+    const epoch = extractNumber(line, 'epoch')
+    if (loss == null && gradNorm == null && learningRate == null) continue
+    metrics.steps.push(extractNumber(line, 'step') ?? metrics.steps.length + index + 1)
+    metrics.loss.push(loss ?? Number.NaN)
+    metrics.gradNorm.push(gradNorm ?? Number.NaN)
+    metrics.lr.push(learningRate ?? Number.NaN)
+    metrics.epoch.push(epoch ?? Number.NaN)
  }

  return metrics
 }

+export function metricsFromApi(points: FineTuneMetricPoint[]): TrainingMetricData {
+  const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
+  for (const [index, point] of points.entries()) {
+    const hasMetric = point.loss != null || point.grad_norm != null || point.learning_rate != null
+    if (!hasMetric) continue
+    metrics.steps.push(Number(point.step || index + 1))
+    metrics.loss.push(point.loss == null ? Number.NaN : Number(point.loss))
+    metrics.gradNorm.push(point.grad_norm == null ? Number.NaN : Number(point.grad_norm))
+    metrics.lr.push(point.learning_rate == null ? Number.NaN : Number(point.learning_rate))
+    metrics.epoch.push(point.epoch == null ? Number.NaN : Number(point.epoch))
+  }
+  return metrics
+}
+
 /** 每次都返回新对象，日志截断或切换时不会残留上一轮汇总。 */
 export function parseTrainingSummary(text: string): TrainingSummary {
  const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
@@ -102,11 +124,23 @@ export function parseTrainingLog(text: string): ParsedTrainingLog {
 export function buildMetricChartOption(
  label: string,
  data: number[],
+  steps: number[],
  color: string,
  logScale = false,
 ): EChartsOption {
+  const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
  return {
    grid: { top: 24, right: 20, bottom: 56, left: 56 },
+    graphic: visibleData.some((value) => value != null)
+      ? []
+      : [
+          {
+            type: 'text',
+            left: 'center',
+            top: 'middle',
+            style: { text: '暂无训练指标数据', fill: '#94a3b8', fontSize: 13 },
+          },
+        ],
    tooltip: {
      trigger: 'axis',
      axisPointer: { type: 'cross' },
@@ -116,6 +150,7 @@ export function buildMetricChartOption(
    },
    xAxis: {
      type: 'category',
+      data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
      boundaryGap: false,
      name: 'Step',
      nameTextStyle: { color: '#94a3b8', fontSize: 11 },
@@ -142,7 +177,7 @@ export function buildMetricChartOption(
      {
        name: label,
        type: 'line',
-        data,
+        data: visibleData,
        smooth: true,
        symbol: 'none',
        lineStyle: { width: 2, color },