feat: 模型推理异步加载与对话链路修复,同步基线
模型推理全异步化改造: - 计算节点 InferenceSession 改为后台线程异步加载模型,load 立即返回, 加载期间事件循环保持响应(/inference/status 与 /health 不阻塞) - 后端模型加载改为异步派发 + 轮询对账器(reconcile_inference_loads), 任务状态由 starting 自动推进到 ready/error,解决多节点启动超时 (timeout of 120000ms exceeded) - 推理删除/卸载改为任务感知 + 短超时,删除先删记录再 best-effort 卸载, 不再被不可达节点阻塞;同节点新模型替换旧任务标记失效 - 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点, useStreamChat 解析 SSE 错误帧以干净文案展示 - 对话历史按任务 id 本地持久化,退出重进可恢复;移除页脚提示文本 - 新增后端推理异步加载与计算节点异步状态机单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,9 @@
|
||||
import type { EChartsOption } from 'echarts'
|
||||
import type { FineTuneTask, TrainingLogFile } from '@/types'
|
||||
import type { FineTuneMetricPoint } from '@/api/modules/fineTune'
|
||||
|
||||
export interface TrainingMetricData {
|
||||
steps: number[]
|
||||
loss: number[]
|
||||
gradNorm: number[]
|
||||
lr: number[]
|
||||
@@ -26,7 +28,7 @@ function escapeRegExp(value: string) {
|
||||
}
|
||||
|
||||
function extractNumber(source: string, key: string) {
|
||||
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*:\\s*(${NUMBER_SOURCE})`, 'i'))
|
||||
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*(?:=|:)\\s*(${NUMBER_SOURCE})`, 'i'))
|
||||
return match ? Number(match[1]) : undefined
|
||||
}
|
||||
|
||||
@@ -57,24 +59,44 @@ export function resolveTrainingLogFile(
|
||||
|
||||
/** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
|
||||
export function parseTrainingMetrics(text: string): TrainingMetricData {
|
||||
const metrics: TrainingMetricData = { loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||
const blocks = text.match(/\{[^{}\r\n]*\}/g) || []
|
||||
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||
const candidates = text
|
||||
.split(/\r?\n/)
|
||||
.flatMap((line) => {
|
||||
const blocks = line.match(/\{[^{}\r\n]*\}/g)
|
||||
return blocks?.length ? blocks.map((block) => `${line} ${block}`) : [line]
|
||||
})
|
||||
|
||||
for (const block of blocks) {
|
||||
const loss = extractNumber(block, 'loss')
|
||||
const gradNorm = extractNumber(block, 'grad_norm')
|
||||
const learningRate = extractNumber(block, 'learning_rate')
|
||||
const epoch = extractNumber(block, 'epoch')
|
||||
if (loss == null || gradNorm == null || learningRate == null) continue
|
||||
metrics.loss.push(loss)
|
||||
metrics.gradNorm.push(gradNorm)
|
||||
metrics.lr.push(learningRate)
|
||||
if (epoch != null) metrics.epoch.push(epoch)
|
||||
for (const [index, line] of candidates.entries()) {
|
||||
const loss = extractNumber(line, 'loss')
|
||||
const gradNorm = extractNumber(line, 'grad_norm')
|
||||
const learningRate = extractNumber(line, 'learning_rate')
|
||||
const epoch = extractNumber(line, 'epoch')
|
||||
if (loss == null && gradNorm == null && learningRate == null) continue
|
||||
metrics.steps.push(extractNumber(line, 'step') ?? metrics.steps.length + index + 1)
|
||||
metrics.loss.push(loss ?? Number.NaN)
|
||||
metrics.gradNorm.push(gradNorm ?? Number.NaN)
|
||||
metrics.lr.push(learningRate ?? Number.NaN)
|
||||
metrics.epoch.push(epoch ?? Number.NaN)
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
export function metricsFromApi(points: FineTuneMetricPoint[]): TrainingMetricData {
|
||||
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||
for (const [index, point] of points.entries()) {
|
||||
const hasMetric = point.loss != null || point.grad_norm != null || point.learning_rate != null
|
||||
if (!hasMetric) continue
|
||||
metrics.steps.push(Number(point.step || index + 1))
|
||||
metrics.loss.push(point.loss == null ? Number.NaN : Number(point.loss))
|
||||
metrics.gradNorm.push(point.grad_norm == null ? Number.NaN : Number(point.grad_norm))
|
||||
metrics.lr.push(point.learning_rate == null ? Number.NaN : Number(point.learning_rate))
|
||||
metrics.epoch.push(point.epoch == null ? Number.NaN : Number(point.epoch))
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
/** 每次都返回新对象,日志截断或切换时不会残留上一轮汇总。 */
|
||||
export function parseTrainingSummary(text: string): TrainingSummary {
|
||||
const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
|
||||
@@ -102,11 +124,23 @@ export function parseTrainingLog(text: string): ParsedTrainingLog {
|
||||
export function buildMetricChartOption(
|
||||
label: string,
|
||||
data: number[],
|
||||
steps: number[],
|
||||
color: string,
|
||||
logScale = false,
|
||||
): EChartsOption {
|
||||
const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
|
||||
return {
|
||||
grid: { top: 24, right: 20, bottom: 56, left: 56 },
|
||||
graphic: visibleData.some((value) => value != null)
|
||||
? []
|
||||
: [
|
||||
{
|
||||
type: 'text',
|
||||
left: 'center',
|
||||
top: 'middle',
|
||||
style: { text: '暂无训练指标数据', fill: '#94a3b8', fontSize: 13 },
|
||||
},
|
||||
],
|
||||
tooltip: {
|
||||
trigger: 'axis',
|
||||
axisPointer: { type: 'cross' },
|
||||
@@ -116,6 +150,7 @@ export function buildMetricChartOption(
|
||||
},
|
||||
xAxis: {
|
||||
type: 'category',
|
||||
data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
|
||||
boundaryGap: false,
|
||||
name: 'Step',
|
||||
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
|
||||
@@ -142,7 +177,7 @@ export function buildMetricChartOption(
|
||||
{
|
||||
name: label,
|
||||
type: 'line',
|
||||
data,
|
||||
data: visibleData,
|
||||
smooth: true,
|
||||
symbol: 'none',
|
||||
lineStyle: { width: 2, color },
|
||||
|
||||
Reference in New Issue
Block a user