fix: 模型评测异步加载等待与多节点路由修复

- eval_runner 等待异步模型加载完成(InferenceSession.wait_until_loaded),
  修复 "model load failed: unknown"
- 评测算力节点选择:优先页面选择的节点 / 模型所在节点(_select_eval_node),
  多节点时不再派发到不可达节点导致连接超时
- 前端评测 GPU 选择改为节点感知(节点:GPU 复合值),透传 compute_node_id,
  并检查 startEval 结果展示真实错误
- 大模型评价(judge)使用模型记录的真实 API 模型名(api_model),
  避免用平台内部名调用 LLM API 导致 HTTP 400
- 新增后端节点选择与 compute wait_until_loaded 单元测试

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-04 18:21:16 +08:00
parent 0271942ba5
commit 0292bf5138
9 changed files with 131 additions and 14 deletions

View File

@@ -183,6 +183,9 @@ def _judge_sample(
api_url = (config.get("api_url") or "").strip().rstrip("/")
api_key = (config.get("api_key") or "").strip()
eval_model = (config.get("eval_model") or "").strip()
# 优先使用模型记录里配置的真实 API 模型名(如 deepseek-chat
# 否则回退到平台内部模型名
api_model = (config.get("api_model") or "").strip() or eval_model
eval_prompt = (config.get("eval_prompt") or "").strip()
score_min = float(config.get("score_min", 0))
score_max = float(config.get("score_max", 5))
@@ -208,7 +211,7 @@ def _judge_sample(
import urllib.error
body = json.dumps({
"model": eval_model,
"model": api_model,
"messages": [
{"role": "system", "content": system_msg},
{"role": "user", "content": user_msg},
@@ -308,13 +311,15 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
print(f"[eval] loading model: {model_path}")
from compute.engines.llama_factory.inference import InferenceSession
session = InferenceSession()
load_result = session.load(
session.load(
model_name_or_path=model_path,
adapter_name_or_path=adapter_path,
template=template,
infer_backend=config.get("infer_backend", "huggingface"),
infer_dtype=config.get("infer_dtype", "auto"),
)
# load() 为异步加载(立即返回 loading必须等待后台线程完成后再进行推理
load_result = session.wait_until_loaded(timeout=float(config.get("load_timeout", 1800)))
if not load_result.get("loaded"):
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
print(f"[eval] model loaded OK")

View File

@@ -55,6 +55,31 @@ class InferenceSession:
"error": self._error,
}
def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]:
"""Wait for an in-flight async load to finish and return its outcome.
供同步消费方(如 eval_runner 子进程)使用:``load()`` 立即返回 loading 后,
调用本方法等待后台加载线程完成,拿到最终的 loaded/error 结果。
若在 timeout 秒内仍未加载完成,返回 ``status == "loading"`` 并附上超时提示。
"""
with self._state_lock:
thread = self._load_thread
if thread is not None and thread.is_alive():
thread.join(timeout=timeout)
with self._state_lock:
loaded = self._status == "ready"
status = self._status
error = self._error
if not loaded and status == "loading":
error = error or f"model load timed out after {timeout or 'N/A'}s"
return {
"loaded": loaded,
"status": status,
"model_name": self._model_name,
"adapter_path": self._adapter_path,
"error": error,
}
def load(
self,
model_name_or_path,