fix: 模型评测异步加载等待与多节点路由修复

- eval_runner 等待异步模型加载完成(InferenceSession.wait_until_loaded),
  修复 "model load failed: unknown"
- 评测算力节点选择:优先页面选择的节点 / 模型所在节点(_select_eval_node),
  多节点时不再派发到不可达节点导致连接超时
- 前端评测 GPU 选择改为节点感知(节点:GPU 复合值),透传 compute_node_id,
  并检查 startEval 结果展示真实错误
- 大模型评价(judge)使用模型记录的真实 API 模型名(api_model),
  避免用平台内部名调用 LLM API 导致 HTTP 400
- 新增后端节点选择与 compute wait_until_loaded 单元测试

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-04 18:21:16 +08:00
parent 0271942ba5
commit 0292bf5138
9 changed files with 131 additions and 14 deletions

View File

@@ -33,6 +33,22 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
return None
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
"""Select the compute node for an eval job.
被评测模型是节点相关的(训练/合并产物只存在于对应算力节点),因此优先使用
页面选择的节点或模型所在节点;若该节点不可用则明确失败,绝不派发到其它
可能没有模型路径的节点(多算力节点场景下这是评测失败的主因)。
"""
if preferred_node_id:
node = next((n for n in store.compute_nodes() if n.get("id") == preferred_node_id), None)
if node:
if node.get("enabled") and node.get("scheduler_status") == "online":
return node
return None
return _select_first_online_node(store)
def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) -> list[dict[str, Any]]:
nodes = [node for node in store.compute_nodes() if node.get("enabled") and node.get("scheduler_status") == "online"]
if not preferred_node_id:
@@ -1382,13 +1398,16 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
model_id = str(payload.get("model_id", ""))
model_path = ""
adapter_path = payload.get("adapter_path", "")
model_node_id = ""
try:
db_model = store.model(model_id)
model_path = db_model.get("path", "")
model_node_id = db_model.get("compute_node_id") or ""
except KeyError:
# Try trained_models table (IDs prefixed with tm_)
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
if trained:
model_node_id = trained.get("compute_node_id") or ""
merged_path = trained.get("merged_path", "")
base_path = trained.get("base_model_path", "")
if trained.get("merged") and merged_path:
@@ -1438,16 +1457,22 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
eval_model_name = dim.get("eval_model", "")
api_url = ""
api_key = ""
api_model_name = ""
if eval_model_name:
try:
eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name)
api_url = eval_model.get("api_url", "")
api_key = eval_model.get("api_key", "")
if isinstance(eval_model, dict):
api_url = eval_model.get("api_url", "")
api_key = eval_model.get("api_key", "")
# 模型记录里的 model_name 是真实 API 模型名(如 deepseek-chat
# 优先传给评测器,避免用平台内部名称调用 LLM API
api_model_name = eval_model.get("model_name") or ""
except (KeyError, Exception):
pass
dimension_cfg = {
"type": dim.get("type", ""),
"eval_model": eval_model_name,
"api_model": api_model_name or eval_model_name,
"eval_method": dim.get("eval_method", ""),
"eval_prompt": dim.get("eval_prompt", ""),
"api_url": api_url,
@@ -1459,11 +1484,13 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
except KeyError:
pass
# 5. Select compute node
node = _select_first_online_node(store)
# 5. Select compute node: 优先页面选择的节点 / 模型所在节点,避免多节点时选错
preferred_node_id = payload.get("compute_node_id") or payload.get("node_id") or model_node_id
node = _select_eval_node(store, preferred_node_id)
if not node:
store.update_eval_task(task["id"], {"status": "failed", "error": "no online compute node"})
return ok({"task_id": task["id"], "status": "failed", "error": "no online compute node"})
message = "no online compute node" if not preferred_node_id else f"model compute node not schedulable: {preferred_node_id}"
store.update_eval_task(task["id"], {"status": "failed", "error": message})
return ok({"task_id": task["id"], "status": "failed", "error": message})
# 6. Build eval job payload
output_dir = f"/data/yg-ft/outputs/{task['id']}"

View File

@@ -100,6 +100,28 @@ def _patch_store(monkeypatch, store: FakeInferenceStore) -> None:
monkeypatch.setattr(platform, "get_settings", lambda: SimpleNamespace(compute_mode="real"))
def test_select_eval_node_prefers_model_node(monkeypatch) -> None:
from app.api.v1.endpoints.platform import _select_eval_node
store = FakeInferenceStore(nodes=[_node("n1"), _node("n2")])
# 指定模型所在节点时优先返回该节点
assert _select_eval_node(store, "n2")["id"] == "n2"
# 无指定节点时回退到第一个在线节点
assert _select_eval_node(store, None)["id"] == "n1"
def test_select_eval_node_returns_none_when_model_node_offline(monkeypatch) -> None:
from app.api.v1.endpoints.platform import _select_eval_node
nodes = [_node("n1"), _node("n2")]
nodes[1]["enabled"] = False
store = FakeInferenceStore(nodes=nodes)
# 模型所在节点不可用 → 明确失败,不派发到其它节点
assert _select_eval_node(store, "n2") is None
# 无指定节点时仍回退第一个在线节点
assert _select_eval_node(store, None)["id"] == "n1"
def test_model_compare_load_dispatches_and_returns_starting(monkeypatch) -> None:
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
_patch_store(monkeypatch, store)