后端 (platform.py + platform_store.py): - 新增 _build_messages_payload() 转换前端格式为 OpenAI messages - 新增 _stream_chat_proxy() SSE 流式代理到算力节点 - 新增 _unload_from_compute_node() 真正释放算力节点 GPU 显存 - 重写 model_compare_load: 从假 PID/端口改为真正调用算力节点加载模型 - 修复 model_compare_unload: 调用 _unload_from_compute_node 释放 GPU - 修复 model_compare_delete: 先释放 GPU 再删除记录 - 修复 model_compare_stream_chat: 从 mock 改为 StreamingResponse 代理 - 修复 model_chat_local/stream: 消息格式转换 + 路径修正 - PlatformStore 新增 _inference_nodes 追踪,gpus() 同步推理占用状态 - preload/unload 端点标记/清除推理节点占用 算力节点 (compute): - inference.py: 适配新版 LLaMA-Factory API (get_infer_args 4 返回值、ChatModel args dict、stream_chat 新签名) - inference.py: unload() 增加 gc.collect + torch.cuda.empty_cache + synchronize 彻底释放显存 - main.py: inference/load 移除 HTTPException(500),错误以 200 正常返回 前端: - InferenceChatView: 真实模式下走 SSE 流式推理,mock 模式保留兼容 - InferenceCreateView: 调用 preloadLocalModel + createCompare 真实创建推理任务,失败回退 mock - InferenceListView: 「停止」改为「释放」,删除前先释放算力节点,改进错误提示 - compare.ts: 新增 streamChatReal() fetch SSE,preload 超时提升至 5 分钟 - useStreamChat.ts: send() 支持 useMock 参数,真实模式调用 streamChatReal - GPU 选择过滤: 仅显示在线算力节点上的空闲 GPU Co-Authored-By: Claude <noreply@anthropic.com>
154 lines
4.2 KiB
TypeScript
154 lines
4.2 KiB
TypeScript
import { ref } from 'vue'
|
||
import { streamChat, streamChatReal } from '@/api/modules/compare'
|
||
|
||
export interface StreamMessage {
|
||
/** 用户问题 */
|
||
question: string
|
||
/** 完整回答(含 think 标签原始内容) */
|
||
fullContent: string
|
||
/** 去除 think 标签后的展示内容 */
|
||
displayContent: string
|
||
/** 思考过程内容 */
|
||
thinkContent: string
|
||
/** 是否正在思考(think 标签未闭合) */
|
||
isThinking: boolean
|
||
/** 是否流式中 */
|
||
isStreaming: boolean
|
||
/** 是否已完成 */
|
||
done: boolean
|
||
/** 错误信息 */
|
||
error?: string
|
||
}
|
||
|
||
export interface SendOptions {
|
||
/** 是否使用 mock 模式(默认 true,向后兼容) */
|
||
useMock?: boolean
|
||
}
|
||
|
||
/**
|
||
* 流式对话 composable
|
||
* 移植自原 model-chat.html:
|
||
* - fetch + body.getReader() + TextDecoder
|
||
* - <think>...</think> 标签解析(思考过程可折叠)
|
||
* - 50ms 节流更新
|
||
*/
|
||
export function useStreamChat() {
|
||
const message = ref<StreamMessage>({
|
||
question: '',
|
||
fullContent: '',
|
||
displayContent: '',
|
||
thinkContent: '',
|
||
isThinking: false,
|
||
isStreaming: false,
|
||
done: false,
|
||
})
|
||
const loading = ref(false)
|
||
|
||
/** 从内容中解析 think 标签 */
|
||
function parseContent(content: string) {
|
||
const thinkRegex = /<think>([\s\S]*?)(<\/think>)?/g
|
||
let think = ''
|
||
let display = content
|
||
let isThinking = false
|
||
|
||
let match
|
||
// 检查是否有未闭合的 think 标签
|
||
const openTags = (content.match(/<think>/g) || []).length
|
||
const closeTags = (content.match(/<\/think>/g) || []).length
|
||
isThinking = openTags > closeTags
|
||
|
||
// 提取所有 think 内容
|
||
while ((match = thinkRegex.exec(content)) !== null) {
|
||
think += match[1]
|
||
}
|
||
// 去除 think 标签得到展示内容
|
||
display = content.replace(/<think>[\s\S]*?(<\/think>|$)/g, '').trim()
|
||
|
||
return { think: think.trim(), display, isThinking }
|
||
}
|
||
|
||
/**
|
||
* 发起流式对话
|
||
* @param payload 后端请求体 { port, model_name, model_path, system_prompt, user_question, ... }
|
||
* @param options 可选配置 { useMock?: boolean }
|
||
*/
|
||
async function send(payload: any, options?: SendOptions) {
|
||
const useMock = options?.useMock ?? true
|
||
loading.value = true
|
||
message.value = {
|
||
question: payload.user_question || '',
|
||
fullContent: '',
|
||
displayContent: '',
|
||
thinkContent: '',
|
||
isThinking: false,
|
||
isStreaming: true,
|
||
done: false,
|
||
}
|
||
|
||
let lastUpdate = 0
|
||
const UPDATE_INTERVAL = 50 // 50ms 节流
|
||
|
||
try {
|
||
const response = useMock
|
||
? await streamChat(payload)
|
||
: await streamChatReal(payload)
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`HTTP ${response.status}`)
|
||
}
|
||
|
||
const reader = response.body?.getReader()
|
||
if (!reader) throw new Error('无法读取响应流')
|
||
|
||
const decoder = new TextDecoder('utf-8')
|
||
let buffer = ''
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read()
|
||
if (done) break
|
||
|
||
buffer += decoder.decode(value, { stream: true })
|
||
message.value.fullContent = buffer
|
||
|
||
const now = Date.now()
|
||
if (now - lastUpdate >= UPDATE_INTERVAL) {
|
||
lastUpdate = now
|
||
const parsed = parseContent(buffer)
|
||
message.value.thinkContent = parsed.think
|
||
message.value.displayContent = parsed.display
|
||
message.value.isThinking = parsed.isThinking
|
||
}
|
||
}
|
||
|
||
// 最终更新
|
||
const parsed = parseContent(buffer)
|
||
message.value.thinkContent = parsed.think
|
||
message.value.displayContent = parsed.display
|
||
message.value.isThinking = false
|
||
message.value.isStreaming = false
|
||
message.value.done = true
|
||
} catch (e: any) {
|
||
message.value.isStreaming = false
|
||
message.value.done = true
|
||
message.value.error = e.message || '流式请求失败'
|
||
message.value.displayContent = message.value.fullContent || message.value.error || '请求失败'
|
||
} finally {
|
||
loading.value = false
|
||
}
|
||
}
|
||
|
||
function reset() {
|
||
message.value = {
|
||
question: '',
|
||
fullContent: '',
|
||
displayContent: '',
|
||
thinkContent: '',
|
||
isThinking: false,
|
||
isStreaming: false,
|
||
done: false,
|
||
}
|
||
}
|
||
|
||
return { message, loading, send, reset }
|
||
}
|