fix: 模型评测异步加载等待与多节点路由修复
- eval_runner 等待异步模型加载完成(InferenceSession.wait_until_loaded), 修复 "model load failed: unknown" - 评测算力节点选择:优先页面选择的节点 / 模型所在节点(_select_eval_node), 多节点时不再派发到不可达节点导致连接超时 - 前端评测 GPU 选择改为节点感知(节点:GPU 复合值),透传 compute_node_id, 并检查 startEval 结果展示真实错误 - 大模型评价(judge)使用模型记录的真实 API 模型名(api_model), 避免用平台内部名调用 LLM API 导致 HTTP 400 - 新增后端节点选择与 compute wait_until_loaded 单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,22 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
|
||||
"""Select the compute node for an eval job.
|
||||
|
||||
被评测模型是节点相关的(训练/合并产物只存在于对应算力节点),因此优先使用
|
||||
页面选择的节点或模型所在节点;若该节点不可用则明确失败,绝不派发到其它
|
||||
可能没有模型路径的节点(多算力节点场景下这是评测失败的主因)。
|
||||
"""
|
||||
if preferred_node_id:
|
||||
node = next((n for n in store.compute_nodes() if n.get("id") == preferred_node_id), None)
|
||||
if node:
|
||||
if node.get("enabled") and node.get("scheduler_status") == "online":
|
||||
return node
|
||||
return None
|
||||
return _select_first_online_node(store)
|
||||
|
||||
|
||||
def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) -> list[dict[str, Any]]:
|
||||
nodes = [node for node in store.compute_nodes() if node.get("enabled") and node.get("scheduler_status") == "online"]
|
||||
if not preferred_node_id:
|
||||
@@ -1382,13 +1398,16 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
|
||||
model_id = str(payload.get("model_id", ""))
|
||||
model_path = ""
|
||||
adapter_path = payload.get("adapter_path", "")
|
||||
model_node_id = ""
|
||||
try:
|
||||
db_model = store.model(model_id)
|
||||
model_path = db_model.get("path", "")
|
||||
model_node_id = db_model.get("compute_node_id") or ""
|
||||
except KeyError:
|
||||
# Try trained_models table (IDs prefixed with tm_)
|
||||
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
|
||||
if trained:
|
||||
model_node_id = trained.get("compute_node_id") or ""
|
||||
merged_path = trained.get("merged_path", "")
|
||||
base_path = trained.get("base_model_path", "")
|
||||
if trained.get("merged") and merged_path:
|
||||
@@ -1438,16 +1457,22 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
|
||||
eval_model_name = dim.get("eval_model", "")
|
||||
api_url = ""
|
||||
api_key = ""
|
||||
api_model_name = ""
|
||||
if eval_model_name:
|
||||
try:
|
||||
eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name)
|
||||
api_url = eval_model.get("api_url", "")
|
||||
api_key = eval_model.get("api_key", "")
|
||||
if isinstance(eval_model, dict):
|
||||
api_url = eval_model.get("api_url", "")
|
||||
api_key = eval_model.get("api_key", "")
|
||||
# 模型记录里的 model_name 是真实 API 模型名(如 deepseek-chat),
|
||||
# 优先传给评测器,避免用平台内部名称调用 LLM API
|
||||
api_model_name = eval_model.get("model_name") or ""
|
||||
except (KeyError, Exception):
|
||||
pass
|
||||
dimension_cfg = {
|
||||
"type": dim.get("type", ""),
|
||||
"eval_model": eval_model_name,
|
||||
"api_model": api_model_name or eval_model_name,
|
||||
"eval_method": dim.get("eval_method", ""),
|
||||
"eval_prompt": dim.get("eval_prompt", ""),
|
||||
"api_url": api_url,
|
||||
@@ -1459,11 +1484,13 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# 5. Select compute node
|
||||
node = _select_first_online_node(store)
|
||||
# 5. Select compute node: 优先页面选择的节点 / 模型所在节点,避免多节点时选错
|
||||
preferred_node_id = payload.get("compute_node_id") or payload.get("node_id") or model_node_id
|
||||
node = _select_eval_node(store, preferred_node_id)
|
||||
if not node:
|
||||
store.update_eval_task(task["id"], {"status": "failed", "error": "no online compute node"})
|
||||
return ok({"task_id": task["id"], "status": "failed", "error": "no online compute node"})
|
||||
message = "no online compute node" if not preferred_node_id else f"model compute node not schedulable: {preferred_node_id}"
|
||||
store.update_eval_task(task["id"], {"status": "failed", "error": message})
|
||||
return ok({"task_id": task["id"], "status": "failed", "error": message})
|
||||
|
||||
# 6. Build eval job payload
|
||||
output_dir = f"/data/yg-ft/outputs/{task['id']}"
|
||||
|
||||
Reference in New Issue
Block a user