feat: 模型推理端到端闭环 — 真实流式推理 + 释放/删除 + GPU 状态同步
后端 (platform.py + platform_store.py): - 新增 _build_messages_payload() 转换前端格式为 OpenAI messages - 新增 _stream_chat_proxy() SSE 流式代理到算力节点 - 新增 _unload_from_compute_node() 真正释放算力节点 GPU 显存 - 重写 model_compare_load: 从假 PID/端口改为真正调用算力节点加载模型 - 修复 model_compare_unload: 调用 _unload_from_compute_node 释放 GPU - 修复 model_compare_delete: 先释放 GPU 再删除记录 - 修复 model_compare_stream_chat: 从 mock 改为 StreamingResponse 代理 - 修复 model_chat_local/stream: 消息格式转换 + 路径修正 - PlatformStore 新增 _inference_nodes 追踪,gpus() 同步推理占用状态 - preload/unload 端点标记/清除推理节点占用 算力节点 (compute): - inference.py: 适配新版 LLaMA-Factory API (get_infer_args 4 返回值、ChatModel args dict、stream_chat 新签名) - inference.py: unload() 增加 gc.collect + torch.cuda.empty_cache + synchronize 彻底释放显存 - main.py: inference/load 移除 HTTPException(500),错误以 200 正常返回 前端: - InferenceChatView: 真实模式下走 SSE 流式推理,mock 模式保留兼容 - InferenceCreateView: 调用 preloadLocalModel + createCompare 真实创建推理任务,失败回退 mock - InferenceListView: 「停止」改为「释放」,删除前先释放算力节点,改进错误提示 - compare.ts: 新增 streamChatReal() fetch SSE,preload 超时提升至 5 分钟 - useStreamChat.ts: send() 支持 useMock 参数,真实模式调用 streamChatReal - GPU 选择过滤: 仅显示在线算力节点上的空闲 GPU Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -64,6 +64,27 @@ export const streamChat = async (data: any): Promise<any> => {
|
||||
}
|
||||
}
|
||||
|
||||
/** 真实流式对话 — 使用 fetch 调用后端 SSE 端点,返回 Response 供 ReadableStream 消费 */
|
||||
export const streamChatReal = (data: any): Promise<Response> => {
|
||||
const messages = data.messages || []
|
||||
if (!messages.length && data.user_question) {
|
||||
if (data.system_prompt) {
|
||||
messages.push({ role: 'system', content: data.system_prompt })
|
||||
}
|
||||
messages.push({ role: 'user', content: data.user_question })
|
||||
}
|
||||
return fetch('/modelTF/model-compare/stream-chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
messages,
|
||||
temperature: data.temperature ?? 0.7,
|
||||
top_p: data.top_p ?? 0.95,
|
||||
max_tokens: data.max_tokens ?? 2048,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/** 非流式对话(按端口代理) */
|
||||
export const chatWithPort = (data: any) => post('/model-compare/chat-with-port', data)
|
||||
|
||||
@@ -73,8 +94,8 @@ export const batchChat = (data: any) => post('/model-chat/batch', data)
|
||||
/** 本地 transformers 模型对话 */
|
||||
export const localChat = (data: any) => post('/model-chat/local/chat', data)
|
||||
|
||||
/** 预加载本地模型 */
|
||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data)
|
||||
/** 预加载本地模型(模型加载耗时长,超时 5 分钟) */
|
||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data, { timeout: 300000 })
|
||||
|
||||
/** 预加载已训练模型 */
|
||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data)
|
||||
/** 预加载已训练模型(超时 5 分钟) */
|
||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data, { timeout: 300000 })
|
||||
|
||||
Reference in New Issue
Block a user