feat: 模型推理异步加载与对话链路修复,同步基线

模型推理全异步化改造:
- 计算节点 InferenceSession 改为后台线程异步加载模型,load 立即返回,
  加载期间事件循环保持响应(/inference/status 与 /health 不阻塞)
- 后端模型加载改为异步派发 + 轮询对账器(reconcile_inference_loads),
  任务状态由 starting 自动推进到 ready/error,解决多节点启动超时
  (timeout of 120000ms exceeded)
- 推理删除/卸载改为任务感知 + 短超时,删除先删记录再 best-effort 卸载,
  不再被不可达节点阻塞;同节点新模型替换旧任务标记失效
- 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点,
  useStreamChat 解析 SSE 错误帧以干净文案展示
- 对话历史按任务 id 本地持久化,退出重进可恢复;移除页脚提示文本
- 新增后端推理异步加载与计算节点异步状态机单元测试

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-04 16:59:34 +08:00
parent 250e060271
commit 0271942ba5
21 changed files with 1272 additions and 245 deletions

View File

@@ -1,7 +1,9 @@
import type { EChartsOption } from 'echarts'
import type { FineTuneTask, TrainingLogFile } from '@/types'
import type { FineTuneMetricPoint } from '@/api/modules/fineTune'
export interface TrainingMetricData {
steps: number[]
loss: number[]
gradNorm: number[]
lr: number[]
@@ -26,7 +28,7 @@ function escapeRegExp(value: string) {
}
function extractNumber(source: string, key: string) {
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*:\\s*(${NUMBER_SOURCE})`, 'i'))
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*(?:=|:)\\s*(${NUMBER_SOURCE})`, 'i'))
return match ? Number(match[1]) : undefined
}
@@ -57,24 +59,44 @@ export function resolveTrainingLogFile(
/** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
export function parseTrainingMetrics(text: string): TrainingMetricData {
const metrics: TrainingMetricData = { loss: [], gradNorm: [], lr: [], epoch: [] }
const blocks = text.match(/\{[^{}\r\n]*\}/g) || []
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
const candidates = text
.split(/\r?\n/)
.flatMap((line) => {
const blocks = line.match(/\{[^{}\r\n]*\}/g)
return blocks?.length ? blocks.map((block) => `${line} ${block}`) : [line]
})
for (const block of blocks) {
const loss = extractNumber(block, 'loss')
const gradNorm = extractNumber(block, 'grad_norm')
const learningRate = extractNumber(block, 'learning_rate')
const epoch = extractNumber(block, 'epoch')
if (loss == null || gradNorm == null || learningRate == null) continue
metrics.loss.push(loss)
metrics.gradNorm.push(gradNorm)
metrics.lr.push(learningRate)
if (epoch != null) metrics.epoch.push(epoch)
for (const [index, line] of candidates.entries()) {
const loss = extractNumber(line, 'loss')
const gradNorm = extractNumber(line, 'grad_norm')
const learningRate = extractNumber(line, 'learning_rate')
const epoch = extractNumber(line, 'epoch')
if (loss == null && gradNorm == null && learningRate == null) continue
metrics.steps.push(extractNumber(line, 'step') ?? metrics.steps.length + index + 1)
metrics.loss.push(loss ?? Number.NaN)
metrics.gradNorm.push(gradNorm ?? Number.NaN)
metrics.lr.push(learningRate ?? Number.NaN)
metrics.epoch.push(epoch ?? Number.NaN)
}
return metrics
}
export function metricsFromApi(points: FineTuneMetricPoint[]): TrainingMetricData {
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
for (const [index, point] of points.entries()) {
const hasMetric = point.loss != null || point.grad_norm != null || point.learning_rate != null
if (!hasMetric) continue
metrics.steps.push(Number(point.step || index + 1))
metrics.loss.push(point.loss == null ? Number.NaN : Number(point.loss))
metrics.gradNorm.push(point.grad_norm == null ? Number.NaN : Number(point.grad_norm))
metrics.lr.push(point.learning_rate == null ? Number.NaN : Number(point.learning_rate))
metrics.epoch.push(point.epoch == null ? Number.NaN : Number(point.epoch))
}
return metrics
}
/** 每次都返回新对象,日志截断或切换时不会残留上一轮汇总。 */
export function parseTrainingSummary(text: string): TrainingSummary {
const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
@@ -102,11 +124,23 @@ export function parseTrainingLog(text: string): ParsedTrainingLog {
export function buildMetricChartOption(
label: string,
data: number[],
steps: number[],
color: string,
logScale = false,
): EChartsOption {
const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
return {
grid: { top: 24, right: 20, bottom: 56, left: 56 },
graphic: visibleData.some((value) => value != null)
? []
: [
{
type: 'text',
left: 'center',
top: 'middle',
style: { text: '暂无训练指标数据', fill: '#94a3b8', fontSize: 13 },
},
],
tooltip: {
trigger: 'axis',
axisPointer: { type: 'cross' },
@@ -116,6 +150,7 @@ export function buildMetricChartOption(
},
xAxis: {
type: 'category',
data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
boundaryGap: false,
name: 'Step',
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
@@ -142,7 +177,7 @@ export function buildMetricChartOption(
{
name: label,
type: 'line',
data,
data: visibleData,
smooth: true,
symbol: 'none',
lineStyle: { width: 2, color },