feat: 模型推理异步加载与对话链路修复,同步基线

模型推理全异步化改造:
- 计算节点 InferenceSession 改为后台线程异步加载模型,load 立即返回,
  加载期间事件循环保持响应(/inference/status 与 /health 不阻塞)
- 后端模型加载改为异步派发 + 轮询对账器(reconcile_inference_loads),
  任务状态由 starting 自动推进到 ready/error,解决多节点启动超时
  (timeout of 120000ms exceeded)
- 推理删除/卸载改为任务感知 + 短超时,删除先删记录再 best-effort 卸载,
  不再被不可达节点阻塞;同节点新模型替换旧任务标记失效
- 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点,
  useStreamChat 解析 SSE 错误帧以干净文案展示
- 对话历史按任务 id 本地持久化,退出重进可恢复;移除页脚提示文本
- 新增后端推理异步加载与计算节点异步状态机单元测试

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-04 16:59:34 +08:00
parent 250e060271
commit 0271942ba5
21 changed files with 1272 additions and 245 deletions

View File

@@ -33,6 +33,16 @@ def _unwrap_dict(payload: Any) -> dict[str, Any]:
return payload if isinstance(payload, dict) else {}
# Inference calls are intentionally short-timeout:
# - load dispatch only confirms the compute node accepted the request
# (the actual model load now runs asynchronously on the node).
# - status/unload must never block the platform for long when a node is
# unreachable but still marked online.
INFERENCE_LOAD_TIMEOUT = httpx.Timeout(30, connect=10)
INFERENCE_STATUS_TIMEOUT = httpx.Timeout(30, connect=5)
INFERENCE_UNLOAD_TIMEOUT = httpx.Timeout(30, connect=5)
class ComputeNodeClient:
"""Application-side client for one compute node.
@@ -182,10 +192,16 @@ class ComputeNodeClient:
response.raise_for_status()
return _unwrap_dict(response.json())
async def _request(self, method: str, path: str, json_data: dict[str, Any] | None = None) -> dict[str, Any]:
async def _request(
self,
method: str,
path: str,
json_data: dict[str, Any] | None = None,
timeout: float | None = None,
) -> dict[str, Any]:
"""Generic request method for compute API endpoints."""
url = _join_url(self.api_base_url, f"{self.route_prefix}{path}")
async with httpx.AsyncClient(timeout=300, headers=self.headers()) as client:
async with httpx.AsyncClient(timeout=timeout or 300, headers=self.headers()) as client:
if method.upper() == "GET":
response = await client.get(url)
else:
@@ -193,6 +209,19 @@ class ComputeNodeClient:
response.raise_for_status()
return _unwrap_dict(response.json())
# ── Inference helpers (short timeouts — see module constants) ──────────
async def inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
"""Dispatch a model load. Returns as soon as the node accepts the
request; the node now loads asynchronously (status goes 'loading')."""
return await self._request("POST", "/inference/load", json_data=payload, timeout=INFERENCE_LOAD_TIMEOUT)
async def inference_status(self) -> dict[str, Any]:
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
async def inference_unload(self) -> dict[str, Any]:
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)
async def upload_file(
self,
filename: str,