Files
YG_FT/frontend/src/composables/useStreamChat.ts
wuyongtao c7c9ed925b feat: 模型推理端到端闭环 — 真实流式推理 + 释放/删除 + GPU 状态同步
后端 (platform.py + platform_store.py):
- 新增 _build_messages_payload() 转换前端格式为 OpenAI messages
- 新增 _stream_chat_proxy() SSE 流式代理到算力节点
- 新增 _unload_from_compute_node() 真正释放算力节点 GPU 显存
- 重写 model_compare_load: 从假 PID/端口改为真正调用算力节点加载模型
- 修复 model_compare_unload: 调用 _unload_from_compute_node 释放 GPU
- 修复 model_compare_delete: 先释放 GPU 再删除记录
- 修复 model_compare_stream_chat: 从 mock 改为 StreamingResponse 代理
- 修复 model_chat_local/stream: 消息格式转换 + 路径修正
- PlatformStore 新增 _inference_nodes 追踪,gpus() 同步推理占用状态
- preload/unload 端点标记/清除推理节点占用

算力节点 (compute):
- inference.py: 适配新版 LLaMA-Factory API (get_infer_args 4 返回值、ChatModel args dict、stream_chat 新签名)
- inference.py: unload() 增加 gc.collect + torch.cuda.empty_cache + synchronize 彻底释放显存
- main.py: inference/load 移除 HTTPException(500),错误以 200 正常返回

前端:
- InferenceChatView: 真实模式下走 SSE 流式推理,mock 模式保留兼容
- InferenceCreateView: 调用 preloadLocalModel + createCompare 真实创建推理任务,失败回退 mock
- InferenceListView: 「停止」改为「释放」,删除前先释放算力节点,改进错误提示
- compare.ts: 新增 streamChatReal() fetch SSE,preload 超时提升至 5 分钟
- useStreamChat.ts: send() 支持 useMock 参数,真实模式调用 streamChatReal
- GPU 选择过滤: 仅显示在线算力节点上的空闲 GPU

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-28 17:29:16 +08:00

154 lines
4.2 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { ref } from 'vue'
import { streamChat, streamChatReal } from '@/api/modules/compare'
export interface StreamMessage {
/** 用户问题 */
question: string
/** 完整回答(含 think 标签原始内容) */
fullContent: string
/** 去除 think 标签后的展示内容 */
displayContent: string
/** 思考过程内容 */
thinkContent: string
/** 是否正在思考think 标签未闭合) */
isThinking: boolean
/** 是否流式中 */
isStreaming: boolean
/** 是否已完成 */
done: boolean
/** 错误信息 */
error?: string
}
export interface SendOptions {
/** 是否使用 mock 模式(默认 true向后兼容 */
useMock?: boolean
}
/**
* 流式对话 composable
* 移植自原 model-chat.html
* - fetch + body.getReader() + TextDecoder
* - <think>...</think> 标签解析(思考过程可折叠)
* - 50ms 节流更新
*/
export function useStreamChat() {
const message = ref<StreamMessage>({
question: '',
fullContent: '',
displayContent: '',
thinkContent: '',
isThinking: false,
isStreaming: false,
done: false,
})
const loading = ref(false)
/** 从内容中解析 think 标签 */
function parseContent(content: string) {
const thinkRegex = /<think>([\s\S]*?)(<\/think>)?/g
let think = ''
let display = content
let isThinking = false
let match
// 检查是否有未闭合的 think 标签
const openTags = (content.match(/<think>/g) || []).length
const closeTags = (content.match(/<\/think>/g) || []).length
isThinking = openTags > closeTags
// 提取所有 think 内容
while ((match = thinkRegex.exec(content)) !== null) {
think += match[1]
}
// 去除 think 标签得到展示内容
display = content.replace(/<think>[\s\S]*?(<\/think>|$)/g, '').trim()
return { think: think.trim(), display, isThinking }
}
/**
* 发起流式对话
* @param payload 后端请求体 { port, model_name, model_path, system_prompt, user_question, ... }
* @param options 可选配置 { useMock?: boolean }
*/
async function send(payload: any, options?: SendOptions) {
const useMock = options?.useMock ?? true
loading.value = true
message.value = {
question: payload.user_question || '',
fullContent: '',
displayContent: '',
thinkContent: '',
isThinking: false,
isStreaming: true,
done: false,
}
let lastUpdate = 0
const UPDATE_INTERVAL = 50 // 50ms 节流
try {
const response = useMock
? await streamChat(payload)
: await streamChatReal(payload)
if (!response.ok) {
throw new Error(`HTTP ${response.status}`)
}
const reader = response.body?.getReader()
if (!reader) throw new Error('无法读取响应流')
const decoder = new TextDecoder('utf-8')
let buffer = ''
while (true) {
const { done, value } = await reader.read()
if (done) break
buffer += decoder.decode(value, { stream: true })
message.value.fullContent = buffer
const now = Date.now()
if (now - lastUpdate >= UPDATE_INTERVAL) {
lastUpdate = now
const parsed = parseContent(buffer)
message.value.thinkContent = parsed.think
message.value.displayContent = parsed.display
message.value.isThinking = parsed.isThinking
}
}
// 最终更新
const parsed = parseContent(buffer)
message.value.thinkContent = parsed.think
message.value.displayContent = parsed.display
message.value.isThinking = false
message.value.isStreaming = false
message.value.done = true
} catch (e: any) {
message.value.isStreaming = false
message.value.done = true
message.value.error = e.message || '流式请求失败'
message.value.displayContent = message.value.fullContent || message.value.error || '请求失败'
} finally {
loading.value = false
}
}
function reset() {
message.value = {
question: '',
fullContent: '',
displayContent: '',
thinkContent: '',
isThinking: false,
isStreaming: false,
done: false,
}
}
return { message, loading, send, reset }
}