feat: 更新 request.ts
This commit is contained in:
@@ -33,6 +33,15 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) -> list[dict[str, Any]]:
|
||||||
|
nodes = [node for node in store.compute_nodes() if node.get("enabled") and node.get("scheduler_status") == "online"]
|
||||||
|
if not preferred_node_id:
|
||||||
|
return nodes
|
||||||
|
preferred = [node for node in nodes if node.get("id") == preferred_node_id]
|
||||||
|
others = [node for node in nodes if node.get("id") != preferred_node_id]
|
||||||
|
return preferred + others
|
||||||
|
|
||||||
|
|
||||||
def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
"""Convert frontend inference payload to compute API messages format.
|
"""Convert frontend inference payload to compute API messages format.
|
||||||
|
|
||||||
@@ -61,10 +70,47 @@ def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _node_for_inference_payload(store: Any, payload: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
|
node_id = payload.get("node_id") or payload.get("compute_node_id")
|
||||||
|
task_id = payload.get("task_id") or payload.get("compare_task_id")
|
||||||
|
if task_id and not node_id:
|
||||||
|
try:
|
||||||
|
task = store.compare_task(str(task_id))
|
||||||
|
load_status = task.get("load_status") or {}
|
||||||
|
if isinstance(load_status, str):
|
||||||
|
load_status = json.loads(load_status)
|
||||||
|
loaded_models = load_status.get("loaded_models") or []
|
||||||
|
ready_model = next((item for item in loaded_models if item.get("status") in {"ready", "running"} and item.get("node_id")), None)
|
||||||
|
if ready_model:
|
||||||
|
node_id = ready_model.get("node_id")
|
||||||
|
except Exception:
|
||||||
|
node_id = None
|
||||||
|
if node_id:
|
||||||
|
return next((node for node in store.compute_nodes() if node.get("id") == node_id), None)
|
||||||
|
return _select_first_online_node(store)
|
||||||
|
|
||||||
|
|
||||||
async def _stream_chat_proxy(payload: dict[str, Any]) -> StreamingResponse:
|
async def _stream_chat_proxy(payload: dict[str, Any]) -> StreamingResponse:
|
||||||
"""Common SSE streaming proxy: convert payload → forward to compute node → stream back."""
|
"""Common SSE streaming proxy: convert payload → forward to compute node → stream back."""
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
node = _select_first_online_node(store)
|
# 任务仍在加载中时,直接返回明确的加载中提示,避免转发到尚未就绪的节点
|
||||||
|
task_id = payload.get("task_id") or payload.get("compare_task_id")
|
||||||
|
if task_id:
|
||||||
|
try:
|
||||||
|
task = store.compare_task(str(task_id))
|
||||||
|
load_status = task.get("load_status") or {}
|
||||||
|
if isinstance(load_status, str):
|
||||||
|
load_status = json.loads(load_status)
|
||||||
|
items = load_status.get("loaded_models") or []
|
||||||
|
if items and not any(item.get("status") in {"ready", "running"} for item in items):
|
||||||
|
if any(item.get("status") == "starting" for item in items):
|
||||||
|
return StreamingResponse(
|
||||||
|
iter(['data: {"error": "模型加载中,请稍候再试"}\n\n']),
|
||||||
|
media_type="text/event-stream",
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 - fall through to normal routing on lookup errors
|
||||||
|
pass
|
||||||
|
node = _node_for_inference_payload(store, payload)
|
||||||
if not node:
|
if not node:
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
iter(['data: {"error": "no online compute node available for inference"}\n\n']),
|
iter(['data: {"error": "no online compute node available for inference"}\n\n']),
|
||||||
@@ -421,7 +467,7 @@ async def dashboard_stats() -> dict[str, Any]:
|
|||||||
("数据类型转换", "/data-convert", "数据类型转换"),
|
("数据类型转换", "/data-convert", "数据类型转换"),
|
||||||
]
|
]
|
||||||
service_status = []
|
service_status = []
|
||||||
for svc_type, path, _label in service_checks:
|
for svc_type, _path, _label in service_checks:
|
||||||
try:
|
try:
|
||||||
svc_count = 0
|
svc_count = 0
|
||||||
if svc_type == "模型训练":
|
if svc_type == "模型训练":
|
||||||
@@ -729,12 +775,17 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
|
base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
|
||||||
adapter_path = payload.get("adapter_path") or payload.get("adapter_name_or_path") or (trained_model and trained_model.get("merged_path"))
|
adapter_path = (
|
||||||
|
payload.get("adapter_path")
|
||||||
|
or payload.get("adapter_name_or_path")
|
||||||
|
or (trained_model and (trained_model.get("artifact_dir") or trained_model.get("adapter_path") or trained_model.get("merged_path")))
|
||||||
|
)
|
||||||
if not base_model_path:
|
if not base_model_path:
|
||||||
raise fail(400, "base_model_path is required")
|
raise fail(400, "base_model_path is required")
|
||||||
if not adapter_path:
|
if not adapter_path:
|
||||||
raise fail(400, "adapter_path is required")
|
raise fail(400, "adapter_path is required")
|
||||||
node = store.schedule_node({**payload, "gpus": payload.get("gpus") or []})
|
requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id"))
|
||||||
|
node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []})
|
||||||
health = node.get("health_detail") or {}
|
health = node.get("health_detail") or {}
|
||||||
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
|
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
|
||||||
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
|
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
|
||||||
@@ -753,11 +804,13 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|||||||
"gpus": payload.get("gpus") or [],
|
"gpus": payload.get("gpus") or [],
|
||||||
"trained_model_id": trained_model["id"] if trained_model else trained_model_id,
|
"trained_model_id": trained_model["id"] if trained_model else trained_model_id,
|
||||||
"model_name": trained_model["name"] if trained_model else payload.get("model_name"),
|
"model_name": trained_model["name"] if trained_model else payload.get("model_name"),
|
||||||
|
"compute_node_id": node["id"],
|
||||||
|
"compute_node_code": node.get("code"),
|
||||||
}
|
}
|
||||||
if get_settings().compute_mode == "simulator":
|
if get_settings().compute_mode == "simulator":
|
||||||
job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir}
|
job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir}
|
||||||
else:
|
else:
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"], timeout=900)
|
||||||
preview = await client.validate_job(job_payload)
|
preview = await client.validate_job(job_payload)
|
||||||
if not preview.get("valid", False):
|
if not preview.get("valid", False):
|
||||||
raise fail(409, "; ".join(preview.get("errors") or ["merge preflight failed"]))
|
raise fail(409, "; ".join(preview.get("errors") or ["merge preflight failed"]))
|
||||||
@@ -1537,28 +1590,49 @@ async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
|||||||
raise fail(404, "compare task not found")
|
raise fail(404, "compare task not found")
|
||||||
|
|
||||||
|
|
||||||
async def _unload_from_compute_node() -> dict[str, Any]:
|
async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||||
"""Best-effort unload the inference model from the first online compute node."""
|
"""Best-effort unload the inference model from the node(s) that hold it.
|
||||||
store = get_platform_store()
|
|
||||||
# Clear all inference tracking — only one model can be loaded at a time
|
任务感知:优先卸载 ``task.load_status.loaded_models`` 中记录的节点;
|
||||||
for node in store.compute_nodes():
|
无任务时回退到平台记录的已加载推理的节点。每个节点使用短超时,
|
||||||
store.mark_inference_unloaded(node["id"])
|
保证卸载永远不会长时间阻塞调用方(例如删除操作)。
|
||||||
node = _select_first_online_node(store)
|
"""
|
||||||
if not node:
|
node_ids: set[str] = set()
|
||||||
return {"unloaded": False, "error": "no online compute node"}
|
if task:
|
||||||
|
load_status = task.get("load_status") or {}
|
||||||
|
if isinstance(load_status, str):
|
||||||
try:
|
try:
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
load_status = json.loads(load_status)
|
||||||
result = await client._request("POST", "/inference/unload", json_data={})
|
except json.JSONDecodeError:
|
||||||
return result
|
load_status = {}
|
||||||
except Exception as exc:
|
node_ids = {item.get("node_id") for item in load_status.get("loaded_models") or [] if item.get("node_id")}
|
||||||
return {"unloaded": False, "error": str(exc)}
|
if not node_ids:
|
||||||
|
node_ids = {node["id"] for node in store.compute_nodes() if store.is_inference_loaded(node["id"])}
|
||||||
|
nodes = [node for node in store.compute_nodes() if node["id"] in node_ids]
|
||||||
|
results: list[dict[str, Any]] = []
|
||||||
|
for node in nodes:
|
||||||
|
try:
|
||||||
|
result = await ComputeNodeClient(node["api_base_url"]).inference_unload()
|
||||||
|
results.append({"node_id": node["id"], "node_code": node.get("code"), "success": True, "result": result})
|
||||||
|
except Exception as exc: # noqa: BLE001 - best-effort unload must not raise
|
||||||
|
results.append({"node_id": node["id"], "node_code": node.get("code"), "success": False, "error": str(exc)})
|
||||||
|
finally:
|
||||||
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
return {"unloaded": bool(results), "nodes": results}
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/model-compare/{task_id}")
|
@router.delete("/model-compare/{task_id}")
|
||||||
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
||||||
# 删除前先释放算力节点上的模型
|
# 先删记录(快),再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞
|
||||||
await _unload_from_compute_node()
|
try:
|
||||||
|
task = get_platform_store().compare_task(task_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "compare task not found")
|
||||||
get_platform_store().delete_compare_task(task_id)
|
get_platform_store().delete_compare_task(task_id)
|
||||||
|
try:
|
||||||
|
await _unload_from_compute_node(get_platform_store(), task=task)
|
||||||
|
except Exception: # noqa: BLE001 - deletion must succeed even if unload fails
|
||||||
|
pass
|
||||||
return ok({"deleted": task_id})
|
return ok({"deleted": task_id})
|
||||||
|
|
||||||
|
|
||||||
@@ -1585,9 +1659,44 @@ async def model_compare_update_load_status(task_id: str, payload: dict[str, Any]
|
|||||||
raise fail(404, "compare task not found")
|
raise fail(404, "compare task not found")
|
||||||
|
|
||||||
|
|
||||||
|
def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[dict[str, Any]]) -> None:
|
||||||
|
"""同一计算节点同一时刻只能加载一个推理模型。
|
||||||
|
|
||||||
|
当新任务把模型派发到了某节点后,把其它任务中在该节点上 ready/running
|
||||||
|
的模型标记为已被替换,保持平台 DB 与计算节点实际状态一致。
|
||||||
|
"""
|
||||||
|
taken_node_ids = {m.get("node_id") for m in loaded_models if m.get("node_id") and m.get("status") == "starting"}
|
||||||
|
if not taken_node_ids:
|
||||||
|
return
|
||||||
|
for other in store.compare_tasks():
|
||||||
|
if str(other.get("id")) == str(task_id):
|
||||||
|
continue
|
||||||
|
load_status = other.get("load_status") or {}
|
||||||
|
if isinstance(load_status, str):
|
||||||
|
try:
|
||||||
|
load_status = json.loads(load_status)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
load_status = {}
|
||||||
|
items = load_status.get("loaded_models") or []
|
||||||
|
changed = False
|
||||||
|
for item in items:
|
||||||
|
if item.get("node_id") in taken_node_ids and item.get("status") in {"ready", "running"}:
|
||||||
|
item["status"] = "error"
|
||||||
|
item["error"] = "模型已被其他推理任务替换"
|
||||||
|
changed = True
|
||||||
|
if changed:
|
||||||
|
new_status = "loaded" if any(i.get("status") in {"ready", "running"} for i in items) else "failed"
|
||||||
|
store.update_compare_task(other["id"], {"status": new_status, "load_status": {"loaded_models": items}})
|
||||||
|
|
||||||
|
|
||||||
@router.post("/model-compare/{task_id}/load")
|
@router.post("/model-compare/{task_id}/load")
|
||||||
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||||
"""真正加载模型到算力节点(不再使用假 PID/端口)。"""
|
"""异步派发模型加载到算力节点,立即返回。
|
||||||
|
|
||||||
|
加载进度由轮询对账器(compute_poller → reconcile_inference_loads)推进:
|
||||||
|
任务项先以 status=starting 记录,对账器查询节点 /inference/status 后
|
||||||
|
推进到 ready/error。这里只负责把加载请求派发出去,绝不同步等待加载完成。
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
task = store.compare_task(task_id)
|
task = store.compare_task(task_id)
|
||||||
@@ -1597,15 +1706,14 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
|
|||||||
models = json.loads(models)
|
models = json.loads(models)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
models = []
|
models = []
|
||||||
# 选取在线算力节点
|
online_nodes = _candidate_online_nodes(store)
|
||||||
node = _select_first_online_node(store)
|
if not online_nodes:
|
||||||
if not node:
|
|
||||||
return ok({"status": "failed", "error": "no online compute node"})
|
return ok({"status": "failed", "error": "no online compute node"})
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
|
||||||
loaded_models = []
|
loaded_models = []
|
||||||
for item in models:
|
for item in models:
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
continue
|
continue
|
||||||
|
preferred_node_id = item.get("node_id") or item.get("compute_node_id")
|
||||||
model_path = item.get("model_path", "")
|
model_path = item.get("model_path", "")
|
||||||
if not model_path:
|
if not model_path:
|
||||||
# 尝试从模型库获取路径
|
# 尝试从模型库获取路径
|
||||||
@@ -1614,28 +1722,46 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
|
|||||||
db_model = store.model(model_id)
|
db_model = store.model(model_id)
|
||||||
model_path = db_model.get("path", "")
|
model_path = db_model.get("path", "")
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
trained_model = next((m for m in store.trained_models() if str(m.get("id")) == str(model_id)), None)
|
||||||
|
if trained_model:
|
||||||
|
model_path = trained_model.get("merged_path") or trained_model.get("artifact_dir") or ""
|
||||||
|
preferred_node_id = preferred_node_id or trained_model.get("compute_node_id")
|
||||||
if not model_path:
|
if not model_path:
|
||||||
loaded_models.append({**item, "status": "error", "error": "model_path not found"})
|
loaded_models.append({**item, "status": "error", "error": "model_path not found"})
|
||||||
continue
|
continue
|
||||||
# 真正调用算力节点加载模型
|
|
||||||
load_payload = {
|
load_payload = {
|
||||||
"model_name_or_path": model_path,
|
"model_name_or_path": model_path,
|
||||||
"template": item.get("template", "qwen"),
|
"template": item.get("template", "qwen"),
|
||||||
}
|
}
|
||||||
if item.get("adapter_path"):
|
if item.get("adapter_path"):
|
||||||
load_payload["adapter_name_or_path"] = item["adapter_path"]
|
load_payload["adapter_name_or_path"] = item["adapter_path"]
|
||||||
|
if get_settings().compute_mode == "simulator":
|
||||||
|
loaded_models.append({**item, "status": "ready", "node_id": "", "node_name": ""})
|
||||||
|
continue
|
||||||
|
# 只派发:HTTP 响应成功即视为已接受(节点会异步加载),loaded 字段忽略
|
||||||
|
item_dispatched = False
|
||||||
|
errors = []
|
||||||
|
for node in _candidate_online_nodes(store, preferred_node_id):
|
||||||
try:
|
try:
|
||||||
result = await client._request("POST", "/inference/load", json_data=load_payload)
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
if result.get("loaded"):
|
await client.inference_load(load_payload)
|
||||||
store.mark_inference_loaded(node["id"])
|
store.mark_inference_loaded(node["id"])
|
||||||
loaded_models.append({**item, "status": "ready"})
|
loaded_models.append({**item, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
|
||||||
|
item_dispatched = True
|
||||||
|
break
|
||||||
|
except Exception as exc: # noqa: BLE001 - try next candidate node
|
||||||
|
errors.append(f"{node.get('name') or node.get('code')}: {exc}")
|
||||||
|
if not item_dispatched:
|
||||||
|
loaded_models.append({**item, "status": "error", "error": "; ".join(errors) or "load dispatch failed"})
|
||||||
|
if any(m.get("status") == "starting" for m in loaded_models):
|
||||||
|
status = "starting"
|
||||||
|
elif any(m.get("status") == "error" for m in loaded_models):
|
||||||
|
status = "failed"
|
||||||
else:
|
else:
|
||||||
loaded_models.append({**item, "status": "error", "error": result.get("error", "load failed")})
|
status = "loaded"
|
||||||
except Exception as exc:
|
|
||||||
loaded_models.append({**item, "status": "error", "error": str(exc)})
|
|
||||||
status = "loaded" if any(m.get("status") == "ready" for m in loaded_models) else "failed"
|
|
||||||
updated = store.update_compare_task(task_id, {"status": status, "load_status": {"loaded_models": loaded_models}})
|
updated = store.update_compare_task(task_id, {"status": status, "load_status": {"loaded_models": loaded_models}})
|
||||||
|
# 同一节点同一时刻只能有一个推理模型;新任务占用了节点后,把其它任务上该节点的模型标记为已被替换
|
||||||
|
_invalidate_superseded_models(store, task_id, loaded_models)
|
||||||
return ok(updated)
|
return ok(updated)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise fail(404, "compare task not found")
|
raise fail(404, "compare task not found")
|
||||||
@@ -1645,8 +1771,9 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
|
|||||||
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
# 真正释放算力节点上的模型资源
|
task = store.compare_task(task_id)
|
||||||
unload_result = await _unload_from_compute_node()
|
# 任务感知卸载:只释放该任务实际加载到的节点,短超时快速返回
|
||||||
|
unload_result = await _unload_from_compute_node(store, task=task)
|
||||||
updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
|
updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
|
||||||
return ok({"task": updated, "unload": unload_result})
|
return ok({"task": updated, "unload": unload_result})
|
||||||
except KeyError:
|
except KeyError:
|
||||||
@@ -1662,7 +1789,7 @@ async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body
|
|||||||
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
"""Proxy non-streaming chat to the compute node running the inference model."""
|
"""Proxy non-streaming chat to the compute node running the inference model."""
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
node = _select_first_online_node(store)
|
node = _node_for_inference_payload(store, payload)
|
||||||
if not node:
|
if not node:
|
||||||
return ok({"response": "no online compute node available for inference", "request": payload})
|
return ok({"response": "no online compute node available for inference", "request": payload})
|
||||||
try:
|
try:
|
||||||
@@ -1717,8 +1844,9 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[
|
|||||||
return ok({"loaded": False, "error": "no online compute node"})
|
return ok({"loaded": False, "error": "no online compute node"})
|
||||||
try:
|
try:
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
result = await client._request("POST", "/inference/load", json_data=payload)
|
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
||||||
if result.get("loaded"):
|
result = await client.inference_load(payload)
|
||||||
|
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
||||||
store.mark_inference_loaded(node["id"])
|
store.mark_inference_loaded(node["id"])
|
||||||
return ok(result)
|
return ok(result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -1729,18 +1857,19 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[
|
|||||||
async def model_chat_local_unload() -> dict[str, Any]:
|
async def model_chat_local_unload() -> dict[str, Any]:
|
||||||
"""Unload the inference model from the compute node."""
|
"""Unload the inference model from the compute node."""
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
# Clear all inference tracking
|
# 释放所有已加载推理的节点(短超时,best-effort)
|
||||||
|
results: list[dict[str, Any]] = []
|
||||||
for n in store.compute_nodes():
|
for n in store.compute_nodes():
|
||||||
store.mark_inference_unloaded(n["id"])
|
if not store.is_inference_loaded(n["id"]):
|
||||||
node = _select_first_online_node(store)
|
continue
|
||||||
if not node:
|
|
||||||
return ok({"unloaded": False, "error": "no online compute node"})
|
|
||||||
try:
|
try:
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
result = await ComputeNodeClient(n["api_base_url"]).inference_unload()
|
||||||
result = await client._request("POST", "/inference/unload", json_data={})
|
results.append({"node_id": n["id"], "success": True, "result": result})
|
||||||
return ok(result)
|
except Exception as exc: # noqa: BLE001 - best-effort unload
|
||||||
except Exception as exc:
|
results.append({"node_id": n["id"], "success": False, "error": str(exc)})
|
||||||
return ok({"unloaded": False, "error": str(exc)})
|
finally:
|
||||||
|
store.mark_inference_unloaded(n["id"])
|
||||||
|
return ok({"unloaded": True, "nodes": results})
|
||||||
|
|
||||||
|
|
||||||
@router.get("/model-chat/local/status")
|
@router.get("/model-chat/local/status")
|
||||||
@@ -1752,7 +1881,7 @@ async def model_chat_local_status() -> dict[str, Any]:
|
|||||||
return ok({"loaded": False, "error": "no online compute node"})
|
return ok({"loaded": False, "error": "no online compute node"})
|
||||||
try:
|
try:
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
result = await client._request("GET", "/inference/status")
|
result = await client.inference_status()
|
||||||
return ok(result)
|
return ok(result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return ok({"loaded": False, "error": str(exc)})
|
return ok({"loaded": False, "error": str(exc)})
|
||||||
@@ -1770,8 +1899,9 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dic
|
|||||||
return ok({"loaded": False, "error": "no online compute node"})
|
return ok({"loaded": False, "error": "no online compute node"})
|
||||||
try:
|
try:
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
result = await client._request("POST", "/inference/load", json_data=payload)
|
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
||||||
if result.get("loaded"):
|
result = await client.inference_load(payload)
|
||||||
|
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
||||||
store.mark_inference_loaded(node["id"])
|
store.mark_inference_loaded(node["id"])
|
||||||
return ok(result)
|
return ok(result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
@@ -104,6 +104,68 @@ def count_dataset_records(content: str) -> int:
|
|||||||
return len([line for line in text.splitlines() if line.strip()])
|
return len([line for line in text.splitlines() if line.strip()])
|
||||||
|
|
||||||
|
|
||||||
|
_EVAL_METHOD_LABELS = {
|
||||||
|
"standard": "标准匹配",
|
||||||
|
"metric_standard": "综合评测",
|
||||||
|
"semantic": "语义相似度",
|
||||||
|
"sentiment": "情感分析",
|
||||||
|
"accuracy": "准确性评估",
|
||||||
|
"safety": "安全性评估",
|
||||||
|
"relevance": "相关性评估",
|
||||||
|
"fluency": "流畅性评估",
|
||||||
|
"factuality": "事实性评估",
|
||||||
|
"custom": "自定义评估",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _eval_method_label(value: Any) -> str:
|
||||||
|
if isinstance(value, list):
|
||||||
|
return "、".join(_eval_method_label(item) for item in value if item)
|
||||||
|
text = str(value or "").strip()
|
||||||
|
return _EVAL_METHOD_LABELS.get(text, text)
|
||||||
|
|
||||||
|
|
||||||
|
def _basic_metric_labels(config: dict[str, Any] | None) -> list[str]:
|
||||||
|
cfg = config or {}
|
||||||
|
labels: list[str] = []
|
||||||
|
bleu = cfg.get("bleu") or {}
|
||||||
|
if bleu.get("enabled"):
|
||||||
|
labels.append(f"BLEU-{int(bleu.get('ngram') or 4)}")
|
||||||
|
rouge = cfg.get("rouge") or {}
|
||||||
|
if rouge.get("enabled"):
|
||||||
|
methods = rouge.get("methods") or []
|
||||||
|
method_labels = {
|
||||||
|
"rouge1": "ROUGE-1",
|
||||||
|
"rouge2": "ROUGE-2",
|
||||||
|
"rougeL": "ROUGE-L",
|
||||||
|
"rouge_1": "ROUGE-1",
|
||||||
|
"rouge_2": "ROUGE-2",
|
||||||
|
"rouge_l": "ROUGE-L",
|
||||||
|
}
|
||||||
|
labels.extend(method_labels.get(str(item), str(item)) for item in methods)
|
||||||
|
cosine = cfg.get("cosine") or {}
|
||||||
|
if cosine.get("enabled"):
|
||||||
|
labels.append("Cosine")
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def _build_eval_metric_label(payload: dict[str, Any], dimension: dict[str, Any] | None = None) -> str:
|
||||||
|
parts: list[str] = []
|
||||||
|
dim = dimension or {}
|
||||||
|
dim_type = str(dim.get("type") or payload.get("dimension_type") or "").strip()
|
||||||
|
method_label = _eval_method_label(dim.get("eval_method") or payload.get("eval_method"))
|
||||||
|
if dim_type in {"classification", "metric"} and method_label:
|
||||||
|
parts.append(f"LLM:{method_label}")
|
||||||
|
elif dim_type == "text_similarity" and method_label:
|
||||||
|
parts.append(method_label)
|
||||||
|
|
||||||
|
parts.extend(_basic_metric_labels(payload.get("basic_metrics") or {}))
|
||||||
|
if not parts:
|
||||||
|
metric = str(payload.get("metric") or payload.get("eval_type") or "").strip()
|
||||||
|
return "自定义评测" if metric == "custom" else (metric or "-")
|
||||||
|
return " + ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
def version_number(value: Any, default: int = 0) -> int:
|
def version_number(value: Any, default: int = 0) -> int:
|
||||||
try:
|
try:
|
||||||
number = int(value)
|
number = int(value)
|
||||||
@@ -146,8 +208,12 @@ def parse_training_metric_line(line: str) -> dict[str, float] | None:
|
|||||||
if "loss" not in line and "learning_rate" not in line:
|
if "loss" not in line and "learning_rate" not in line:
|
||||||
return None
|
return None
|
||||||
result: dict[str, float] = {}
|
result: dict[str, float] = {}
|
||||||
|
number_pattern = r"([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)"
|
||||||
|
step_match = re.search(rf"(?:^|[\s,{{])['\"]?step['\"]?\s*(?:=|:)\s*{number_pattern}", line, re.I)
|
||||||
|
if step_match:
|
||||||
|
result["step"] = float(step_match.group(1))
|
||||||
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
|
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
|
||||||
match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
|
match = re.search(rf"['\"]?{key}['\"]?\s*(?:=|:)\s*{number_pattern}", line, re.I)
|
||||||
if match:
|
if match:
|
||||||
result[key] = float(match.group(1))
|
result[key] = float(match.group(1))
|
||||||
return result or None
|
return result or None
|
||||||
@@ -339,6 +405,7 @@ class PlatformStore:
|
|||||||
self.ensure_seed_data()
|
self.ensure_seed_data()
|
||||||
# Track which compute nodes have an active inference model loaded
|
# Track which compute nodes have an active inference model loaded
|
||||||
self._inference_nodes: set[str] = set()
|
self._inference_nodes: set[str] = set()
|
||||||
|
self._last_runtime_refresh = 0.0
|
||||||
|
|
||||||
# ── inference node tracking ────────────────────────────────────
|
# ── inference node tracking ────────────────────────────────────
|
||||||
|
|
||||||
@@ -389,6 +456,15 @@ class PlatformStore:
|
|||||||
)
|
)
|
||||||
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
|
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
|
||||||
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
|
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
|
||||||
|
self._ensure_columns(
|
||||||
|
conn,
|
||||||
|
"trained_models",
|
||||||
|
{
|
||||||
|
"artifact_dir": "TEXT",
|
||||||
|
"compute_node_id": "TEXT",
|
||||||
|
"compute_node_name": "TEXT",
|
||||||
|
},
|
||||||
|
)
|
||||||
self._ensure_columns(
|
self._ensure_columns(
|
||||||
conn,
|
conn,
|
||||||
"resource_replicas",
|
"resource_replicas",
|
||||||
@@ -463,6 +539,10 @@ class PlatformStore:
|
|||||||
def refresh_runtime_state(self) -> None:
|
def refresh_runtime_state(self) -> None:
|
||||||
if get_settings().compute_mode != "simulator":
|
if get_settings().compute_mode != "simulator":
|
||||||
return
|
return
|
||||||
|
now_ts = time.monotonic()
|
||||||
|
if now_ts - self._last_runtime_refresh < 2:
|
||||||
|
return
|
||||||
|
self._last_runtime_refresh = now_ts
|
||||||
|
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
@@ -522,6 +602,15 @@ class PlatformStore:
|
|||||||
name = task.get("output_model_name") or f"{task['name']}-lora"
|
name = task.get("output_model_name") or f"{task['name']}-lora"
|
||||||
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
|
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
|
||||||
if exists:
|
if exists:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE trained_models
|
||||||
|
SET compute_node_id=COALESCE(compute_node_id, ?),
|
||||||
|
compute_node_name=COALESCE(compute_node_name, ?)
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
(task.get("compute_node_id"), task.get("compute_node_code") or task.get("compute_node_name"), exists["id"]),
|
||||||
|
)
|
||||||
return
|
return
|
||||||
model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
|
model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
|
||||||
output_dir = task.get("output_dir") or f"/data/yg-ft/outputs/{task['name']}"
|
output_dir = task.get("output_dir") or f"/data/yg-ft/outputs/{task['name']}"
|
||||||
@@ -529,8 +618,8 @@ class PlatformStore:
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO trained_models
|
INSERT INTO trained_models
|
||||||
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir)
|
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir, compute_node_id, compute_node_name)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
trained_model_id,
|
trained_model_id,
|
||||||
@@ -542,6 +631,8 @@ class PlatformStore:
|
|||||||
0,
|
0,
|
||||||
output_dir,
|
output_dir,
|
||||||
output_dir,
|
output_dir,
|
||||||
|
task.get("compute_node_id"),
|
||||||
|
task.get("compute_node_code") or task.get("compute_node_name"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
# Use real artifact data from compute node when available
|
# Use real artifact data from compute node when available
|
||||||
@@ -808,7 +899,7 @@ class PlatformStore:
|
|||||||
(
|
(
|
||||||
new_id("metric"),
|
new_id("metric"),
|
||||||
task_id,
|
task_id,
|
||||||
line_number,
|
int(metric.get("step") or line_number),
|
||||||
metric.get("epoch"),
|
metric.get("epoch"),
|
||||||
metric.get("loss"),
|
metric.get("loss"),
|
||||||
metric.get("grad_norm"),
|
metric.get("grad_norm"),
|
||||||
@@ -1275,15 +1366,25 @@ class PlatformStore:
|
|||||||
self.refresh_runtime_state()
|
self.refresh_runtime_state()
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
|
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
|
||||||
return [
|
items = []
|
||||||
{
|
for row in rows:
|
||||||
|
item = {
|
||||||
**dict(row),
|
**dict(row),
|
||||||
"train_methods": json_loads(row["train_methods"], []),
|
"train_methods": json_loads(row["train_methods"], []),
|
||||||
"merged": bool(row["merged"]),
|
"merged": bool(row["merged"]),
|
||||||
"merging": bool(row["merging"]),
|
"merging": bool(row["merging"]),
|
||||||
}
|
}
|
||||||
for row in rows
|
if not item.get("compute_node_id"):
|
||||||
]
|
task_rows = conn.execute("SELECT payload FROM fine_tune_tasks ORDER BY create_time DESC").fetchall()
|
||||||
|
for task in task_rows:
|
||||||
|
task_payload = json_loads(task["payload"], {})
|
||||||
|
output_name = task_payload.get("output_model_name") or f"{task_payload.get('name')}-lora"
|
||||||
|
if output_name == item["name"]:
|
||||||
|
item["compute_node_id"] = task_payload.get("compute_node_id")
|
||||||
|
item["compute_node_name"] = task_payload.get("compute_node_code") or task_payload.get("compute_node_name")
|
||||||
|
break
|
||||||
|
items.append(item)
|
||||||
|
return items
|
||||||
|
|
||||||
def delete_trained_model(self, model_id: str) -> None:
|
def delete_trained_model(self, model_id: str) -> None:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
@@ -2146,19 +2247,54 @@ class PlatformStore:
|
|||||||
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
||||||
payload = json_loads(row["payload"], {})
|
payload = json_loads(row["payload"], {})
|
||||||
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
||||||
|
if not payload.get("metric_label"):
|
||||||
|
payload["metric_label"] = _build_eval_metric_label(payload)
|
||||||
|
if not payload.get("metric") or payload.get("metric") == "custom":
|
||||||
|
payload["metric"] = payload["metric_label"]
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
def _enrich_eval_payload(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
data = dict(payload)
|
||||||
|
model_id = str(data.get("model_id") or "")
|
||||||
|
if model_id and not data.get("model_name"):
|
||||||
|
model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone()
|
||||||
|
if not model:
|
||||||
|
model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone()
|
||||||
|
if model:
|
||||||
|
data["model_name"] = model["name"]
|
||||||
|
|
||||||
|
dataset_id = str(data.get("dataset_id") or "")
|
||||||
|
if dataset_id and not data.get("dataset"):
|
||||||
|
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (dataset_id,)).fetchone()
|
||||||
|
if dataset:
|
||||||
|
data["dataset"] = dataset["name"]
|
||||||
|
|
||||||
|
dimension = None
|
||||||
|
dimension_id = str(data.get("dimension_id") or "")
|
||||||
|
if dimension_id:
|
||||||
|
dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||||
|
if dimension_row:
|
||||||
|
dimension = json_loads(dimension_row["payload"], {})
|
||||||
|
data.setdefault("dimension_type", dimension.get("type"))
|
||||||
|
data.setdefault("eval_method", dimension.get("eval_method"))
|
||||||
|
data.setdefault("evaluator_model", dimension.get("eval_model"))
|
||||||
|
|
||||||
|
data["metric_label"] = _build_eval_metric_label(data, dimension)
|
||||||
|
if not data.get("metric") or data.get("metric") in {"custom", "自定义评测"}:
|
||||||
|
data["metric"] = data["metric_label"]
|
||||||
|
return data
|
||||||
|
|
||||||
def eval_tasks(self) -> list[dict[str, Any]]:
|
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
||||||
return [self._json_payload_row(row) for row in rows]
|
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
|
||||||
|
|
||||||
def eval_task(self, task_id: str) -> dict[str, Any]:
|
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
||||||
if not row:
|
if not row:
|
||||||
raise KeyError(task_id)
|
raise KeyError(task_id)
|
||||||
payload = self._json_payload_row(row)
|
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
|
||||||
payload.setdefault("sample_count", 0)
|
payload.setdefault("sample_count", 0)
|
||||||
payload.setdefault("completed_count", 0)
|
payload.setdefault("completed_count", 0)
|
||||||
payload.setdefault("passed_count", 0)
|
payload.setdefault("passed_count", 0)
|
||||||
@@ -2184,12 +2320,29 @@ class PlatformStore:
|
|||||||
"metric": payload.get("metric") or "custom",
|
"metric": payload.get("metric") or "custom",
|
||||||
}
|
}
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
|
model_id = str(payload.get("model_id") or "")
|
||||||
|
model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone()
|
||||||
|
trained_model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone()
|
||||||
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
||||||
|
dimension = None
|
||||||
|
dimension_id = str(payload.get("dimension_id") or "")
|
||||||
|
if dimension_id:
|
||||||
|
dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||||
|
if dimension_row:
|
||||||
|
dimension = json_loads(dimension_row["payload"], {})
|
||||||
if model:
|
if model:
|
||||||
data.setdefault("model_name", model["name"])
|
data.setdefault("model_name", model["name"])
|
||||||
|
elif trained_model:
|
||||||
|
data.setdefault("model_name", trained_model["name"])
|
||||||
if dataset:
|
if dataset:
|
||||||
data.setdefault("dataset", dataset["name"])
|
data.setdefault("dataset", dataset["name"])
|
||||||
|
if dimension:
|
||||||
|
data.setdefault("dimension_type", dimension.get("type"))
|
||||||
|
data.setdefault("eval_method", dimension.get("eval_method"))
|
||||||
|
data.setdefault("evaluator_model", dimension.get("eval_model"))
|
||||||
|
data["metric_label"] = _build_eval_metric_label(data, dimension)
|
||||||
|
if data.get("metric") == "custom":
|
||||||
|
data["metric"] = data["metric_label"]
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||||
(task_id, name, json_dumps(data), status, now),
|
(task_id, name, json_dumps(data), status, now),
|
||||||
|
|||||||
@@ -33,7 +33,10 @@ CREATE TABLE IF NOT EXISTS trained_models (
|
|||||||
create_time TEXT NOT NULL,
|
create_time TEXT NOT NULL,
|
||||||
merged INTEGER NOT NULL DEFAULT 0,
|
merged INTEGER NOT NULL DEFAULT 0,
|
||||||
merging INTEGER NOT NULL DEFAULT 0,
|
merging INTEGER NOT NULL DEFAULT 0,
|
||||||
merged_path TEXT
|
merged_path TEXT,
|
||||||
|
artifact_dir TEXT,
|
||||||
|
compute_node_id TEXT,
|
||||||
|
compute_node_name TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS model_lineage (
|
CREATE TABLE IF NOT EXISTS model_lineage (
|
||||||
|
|||||||
@@ -33,6 +33,16 @@ def _unwrap_dict(payload: Any) -> dict[str, Any]:
|
|||||||
return payload if isinstance(payload, dict) else {}
|
return payload if isinstance(payload, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
# Inference calls are intentionally short-timeout:
|
||||||
|
# - load dispatch only confirms the compute node accepted the request
|
||||||
|
# (the actual model load now runs asynchronously on the node).
|
||||||
|
# - status/unload must never block the platform for long when a node is
|
||||||
|
# unreachable but still marked online.
|
||||||
|
INFERENCE_LOAD_TIMEOUT = httpx.Timeout(30, connect=10)
|
||||||
|
INFERENCE_STATUS_TIMEOUT = httpx.Timeout(30, connect=5)
|
||||||
|
INFERENCE_UNLOAD_TIMEOUT = httpx.Timeout(30, connect=5)
|
||||||
|
|
||||||
|
|
||||||
class ComputeNodeClient:
|
class ComputeNodeClient:
|
||||||
"""Application-side client for one compute node.
|
"""Application-side client for one compute node.
|
||||||
|
|
||||||
@@ -182,10 +192,16 @@ class ComputeNodeClient:
|
|||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return _unwrap_dict(response.json())
|
return _unwrap_dict(response.json())
|
||||||
|
|
||||||
async def _request(self, method: str, path: str, json_data: dict[str, Any] | None = None) -> dict[str, Any]:
|
async def _request(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
json_data: dict[str, Any] | None = None,
|
||||||
|
timeout: float | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
"""Generic request method for compute API endpoints."""
|
"""Generic request method for compute API endpoints."""
|
||||||
url = _join_url(self.api_base_url, f"{self.route_prefix}{path}")
|
url = _join_url(self.api_base_url, f"{self.route_prefix}{path}")
|
||||||
async with httpx.AsyncClient(timeout=300, headers=self.headers()) as client:
|
async with httpx.AsyncClient(timeout=timeout or 300, headers=self.headers()) as client:
|
||||||
if method.upper() == "GET":
|
if method.upper() == "GET":
|
||||||
response = await client.get(url)
|
response = await client.get(url)
|
||||||
else:
|
else:
|
||||||
@@ -193,6 +209,19 @@ class ComputeNodeClient:
|
|||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return _unwrap_dict(response.json())
|
return _unwrap_dict(response.json())
|
||||||
|
|
||||||
|
# ── Inference helpers (short timeouts — see module constants) ──────────
|
||||||
|
|
||||||
|
async def inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Dispatch a model load. Returns as soon as the node accepts the
|
||||||
|
request; the node now loads asynchronously (status goes 'loading')."""
|
||||||
|
return await self._request("POST", "/inference/load", json_data=payload, timeout=INFERENCE_LOAD_TIMEOUT)
|
||||||
|
|
||||||
|
async def inference_status(self) -> dict[str, Any]:
|
||||||
|
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
|
||||||
|
|
||||||
|
async def inference_unload(self) -> dict[str, Any]:
|
||||||
|
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)
|
||||||
|
|
||||||
async def upload_file(
|
async def upload_file(
|
||||||
self,
|
self,
|
||||||
filename: str,
|
filename: str,
|
||||||
|
|||||||
@@ -1,15 +1,101 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
|
||||||
|
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
|
||||||
|
MAX_STARTING_ATTEMPTS = 40
|
||||||
|
|
||||||
|
|
||||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_inference_load_status(task: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||||
|
load_status = task.get("load_status") or {}
|
||||||
|
if isinstance(load_status, str):
|
||||||
|
try:
|
||||||
|
load_status = json.loads(load_status)
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
load_status = {}
|
||||||
|
return load_status.get("loaded_models") or [], load_status
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
||||||
|
"""推进处于 starting 状态的推理加载。
|
||||||
|
|
||||||
|
模型加载已改为异步派发:/model-compare/{id}/load 立即返回,这里在每次
|
||||||
|
轮询时查询对应计算节点的 /inference/status,把任务从 starting 推进到
|
||||||
|
ready/error。使用短超时,单节点不可达不会阻塞整轮轮询。
|
||||||
|
"""
|
||||||
|
reconciled: list[dict[str, Any]] = []
|
||||||
|
now = time.time()
|
||||||
|
for task in store.compare_tasks():
|
||||||
|
items, _ = _parse_inference_load_status(task)
|
||||||
|
if not any(item.get("status") == "starting" for item in items):
|
||||||
|
continue
|
||||||
|
# dirty 只要处理过任一 starting 项就置位:load_attempts / last_polled_at
|
||||||
|
# 必须落库,否则节点不可达时计数不会累积,封顶逻辑永远触发不了
|
||||||
|
dirty = False
|
||||||
|
for item in items:
|
||||||
|
if item.get("status") != "starting":
|
||||||
|
continue
|
||||||
|
# 节流:同一 item 每 3s 只查询一次
|
||||||
|
if now - float(item.get("last_polled_at") or 0) < 3:
|
||||||
|
continue
|
||||||
|
item["last_polled_at"] = now
|
||||||
|
item["load_attempts"] = int(item.get("load_attempts") or 0) + 1
|
||||||
|
dirty = True
|
||||||
|
node = next((n for n in store.compute_nodes() if n["id"] == item.get("node_id")), None)
|
||||||
|
if not node:
|
||||||
|
item["status"] = "error"
|
||||||
|
item["error"] = "compute node deleted"
|
||||||
|
store.mark_inference_unloaded(item.get("node_id") or "")
|
||||||
|
continue
|
||||||
|
if not node.get("enabled") or node.get("scheduler_status") != "online":
|
||||||
|
item["status"] = "error"
|
||||||
|
item["error"] = "compute node offline"
|
||||||
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
|
||||||
|
except Exception as exc: # noqa: BLE001 - node unreachable; keep retrying until cap
|
||||||
|
if int(item.get("load_attempts") or 0) >= MAX_STARTING_ATTEMPTS:
|
||||||
|
item["status"] = "error"
|
||||||
|
item["error"] = f"compute node unreachable: {exc}"
|
||||||
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
continue
|
||||||
|
node_status = status.get("status")
|
||||||
|
if node_status == "ready":
|
||||||
|
item["status"] = "ready"
|
||||||
|
item.pop("error", None)
|
||||||
|
store.mark_inference_loaded(node["id"])
|
||||||
|
elif node_status == "error":
|
||||||
|
item["status"] = "error"
|
||||||
|
item["error"] = status.get("error") or "model load failed on compute node"
|
||||||
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
elif node_status == "idle":
|
||||||
|
# 节点重启导致已加载模型丢失
|
||||||
|
item["status"] = "error"
|
||||||
|
item["error"] = "model disappeared from compute node (node may have restarted)"
|
||||||
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
# node_status == "loading" -> 保持 starting,下轮再查
|
||||||
|
if dirty:
|
||||||
|
if any(i.get("status") in {"ready", "running"} for i in items):
|
||||||
|
new_status = "loaded"
|
||||||
|
elif any(i.get("status") == "starting" for i in items):
|
||||||
|
new_status = "starting" # 仍在加载中,保持 starting
|
||||||
|
else:
|
||||||
|
new_status = "failed"
|
||||||
|
store.update_compare_task(task["id"], {"status": new_status, "load_status": {"loaded_models": items}})
|
||||||
|
reconciled.append({"task_id": task["id"], "status": new_status})
|
||||||
|
return reconciled
|
||||||
|
|
||||||
|
|
||||||
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
|
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
output_dir = job.get("output_dir")
|
output_dir = job.get("output_dir")
|
||||||
if not output_dir:
|
if not output_dir:
|
||||||
@@ -95,5 +181,13 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc: # noqa: BLE001
|
||||||
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
|
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
|
||||||
|
|
||||||
|
# ── Inference load reconciliation ─────────────────────────────────────
|
||||||
|
try:
|
||||||
|
inference_reconciled = await reconcile_inference_loads(store)
|
||||||
|
except Exception as exc: # noqa: BLE001 - keep polling alive
|
||||||
|
failed.append({"inference_reconcile": str(exc)})
|
||||||
|
inference_reconciled = []
|
||||||
|
|
||||||
return {"synced": len(synced) + len(standalone_synced) + eval_synced, "failed": failed,
|
return {"synced": len(synced) + len(standalone_synced) + eval_synced, "failed": failed,
|
||||||
"items": synced, "standalone": standalone_synced, "eval_synced": eval_synced}
|
"items": synced, "standalone": standalone_synced, "eval_synced": eval_synced,
|
||||||
|
"inference_reconciled": inference_reconciled}
|
||||||
|
|||||||
254
backend/tests/test_compare_inference_async.py
Normal file
254
backend/tests/test_compare_inference_async.py
Normal file
@@ -0,0 +1,254 @@
|
|||||||
|
"""
|
||||||
|
模型推理异步加载改造的单元测试。
|
||||||
|
|
||||||
|
覆盖:
|
||||||
|
- model_compare_load:异步派发,立即返回 starting + 节点信息(不等待加载完成)
|
||||||
|
- model_compare_delete:先删记录,卸载失败也不阻塞删除
|
||||||
|
- reconcile_inference_loads:starting -> ready/error/idle/不可达的状态迁移与封顶
|
||||||
|
- _unload_from_compute_node:任务感知,只命中记录中的节点
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.api.v1.endpoints.platform import model_compare_delete, model_compare_load
|
||||||
|
import app.api.v1.endpoints.platform as platform
|
||||||
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
from app.modules.compute_gateway.sync import MAX_STARTING_ATTEMPTS, reconcile_inference_loads
|
||||||
|
|
||||||
|
|
||||||
|
class FakeInferenceStore:
|
||||||
|
"""内存 store,仅实现推理加载/对账用到的接口。"""
|
||||||
|
|
||||||
|
def __init__(self, tasks: list[dict[str, Any]] | None = None, nodes: list[dict[str, Any]] | None = None) -> None:
|
||||||
|
self._tasks: dict[str, dict[str, Any]] = {t["id"]: dict(t) for t in (tasks or [])}
|
||||||
|
self._nodes = nodes or []
|
||||||
|
self._inference_nodes: set[str] = set()
|
||||||
|
|
||||||
|
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||||
|
if task_id not in self._tasks:
|
||||||
|
raise KeyError(task_id)
|
||||||
|
return dict(self._tasks[task_id])
|
||||||
|
|
||||||
|
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||||
|
return [dict(t) for t in self._tasks.values()]
|
||||||
|
|
||||||
|
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
current = self._tasks[task_id]
|
||||||
|
merged = {**current, **payload, "id": task_id}
|
||||||
|
self._tasks[task_id] = merged
|
||||||
|
return dict(merged)
|
||||||
|
|
||||||
|
def delete_compare_task(self, task_id: str) -> None:
|
||||||
|
self._tasks.pop(task_id, None)
|
||||||
|
|
||||||
|
def compute_nodes(self) -> list[dict[str, Any]]:
|
||||||
|
return [dict(n) for n in self._nodes]
|
||||||
|
|
||||||
|
def model(self, model_id: str) -> dict[str, Any]:
|
||||||
|
raise KeyError(model_id)
|
||||||
|
|
||||||
|
def trained_models(self) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def mark_inference_loaded(self, node_id: str) -> None:
|
||||||
|
self._inference_nodes.add(node_id)
|
||||||
|
|
||||||
|
def mark_inference_unloaded(self, node_id: str) -> None:
|
||||||
|
self._inference_nodes.discard(node_id)
|
||||||
|
|
||||||
|
def is_inference_loaded(self, node_id: str) -> bool:
|
||||||
|
return node_id in self._inference_nodes
|
||||||
|
|
||||||
|
|
||||||
|
def _node(node_id: str, code: str = "") -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": node_id,
|
||||||
|
"code": code or node_id,
|
||||||
|
"name": code or node_id,
|
||||||
|
"api_base_url": f"http://{code or node_id}:19100",
|
||||||
|
"enabled": True,
|
||||||
|
"scheduler_status": "online",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _task(task_id: str, *, node_id: str | None = None, load_status: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"id": task_id,
|
||||||
|
"name": f"task-{task_id}",
|
||||||
|
"status": "pending",
|
||||||
|
"models": [
|
||||||
|
{"model_id": "m_1", "model_name": "qwen", "model_path": "/models/qwen", "node_id": node_id}
|
||||||
|
],
|
||||||
|
"load_status": load_status or {"loaded_models": []},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _fake_inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {"loaded": False, "status": "loading", "request_id": "req-1"}
|
||||||
|
|
||||||
|
|
||||||
|
async def _fake_inference_unload(self) -> dict[str, Any]:
|
||||||
|
return {"unloaded": True, "status": "idle"}
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_store(monkeypatch, store: FakeInferenceStore) -> None:
|
||||||
|
monkeypatch.setattr(platform, "get_platform_store", lambda: store)
|
||||||
|
monkeypatch.setattr(platform, "get_settings", lambda: SimpleNamespace(compute_mode="real"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_compare_load_dispatches_and_returns_starting(monkeypatch) -> None:
|
||||||
|
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
|
||||||
|
_patch_store(monkeypatch, store)
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_load", _fake_inference_load)
|
||||||
|
|
||||||
|
result = asyncio.run(model_compare_load("t1"))
|
||||||
|
assert result["code"] == 0
|
||||||
|
updated = result["data"]
|
||||||
|
assert updated["status"] == "starting"
|
||||||
|
items = updated["load_status"]["loaded_models"]
|
||||||
|
assert items[0]["status"] == "starting"
|
||||||
|
assert items[0]["node_id"] == "n1"
|
||||||
|
assert "n1" in store._inference_nodes
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_compare_load_marks_error_when_all_nodes_fail(monkeypatch) -> None:
|
||||||
|
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
|
||||||
|
_patch_store(monkeypatch, store)
|
||||||
|
|
||||||
|
async def _raise(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
raise RuntimeError("conn refused")
|
||||||
|
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_load", _raise)
|
||||||
|
|
||||||
|
result = asyncio.run(model_compare_load("t1"))
|
||||||
|
updated = result["data"]
|
||||||
|
assert updated["status"] == "failed"
|
||||||
|
assert updated["load_status"]["loaded_models"][0]["status"] == "error"
|
||||||
|
assert "conn refused" in updated["load_status"]["loaded_models"][0]["error"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_compare_delete_removes_record_even_if_unload_raises(monkeypatch) -> None:
|
||||||
|
task = _task(
|
||||||
|
"t1",
|
||||||
|
node_id="n1",
|
||||||
|
load_status={"loaded_models": [{"model_id": "m_1", "status": "ready", "node_id": "n1"}]},
|
||||||
|
)
|
||||||
|
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||||
|
_patch_store(monkeypatch, store)
|
||||||
|
|
||||||
|
async def _raise(self) -> dict[str, Any]:
|
||||||
|
raise RuntimeError("unload boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_unload", _raise)
|
||||||
|
|
||||||
|
result = asyncio.run(model_compare_delete("t1"))
|
||||||
|
assert result["data"] == {"deleted": "t1"}
|
||||||
|
assert "t1" not in store._tasks
|
||||||
|
# finally 中仍清掉了节点标记
|
||||||
|
assert "n1" not in store._inference_nodes
|
||||||
|
|
||||||
|
|
||||||
|
def test_unload_from_compute_node_only_hits_recorded_node(monkeypatch) -> None:
|
||||||
|
task = _task(
|
||||||
|
"t1",
|
||||||
|
load_status={"loaded_models": [{"model_id": "m_1", "status": "ready", "node_id": "n1"}]},
|
||||||
|
)
|
||||||
|
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1"), _node("n2")])
|
||||||
|
_patch_store(monkeypatch, store)
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_unload", _fake_inference_unload)
|
||||||
|
|
||||||
|
from app.api.v1.endpoints.platform import _unload_from_compute_node
|
||||||
|
|
||||||
|
result = asyncio.run(_unload_from_compute_node(store, task=task))
|
||||||
|
assert result["unloaded"] is True
|
||||||
|
# 只命中任务记录中的节点 n1,n2 未被卸载
|
||||||
|
assert [r["node_id"] for r in result["nodes"]] == ["n1"]
|
||||||
|
assert "n1" not in store._inference_nodes
|
||||||
|
|
||||||
|
|
||||||
|
async def _status_ready(self) -> dict[str, Any]:
|
||||||
|
return {"loaded": True, "status": "ready", "model_name": "qwen"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_transitions_starting_to_ready(monkeypatch) -> None:
|
||||||
|
task = _task(
|
||||||
|
"t1",
|
||||||
|
node_id="n1",
|
||||||
|
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||||
|
)
|
||||||
|
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_ready)
|
||||||
|
|
||||||
|
reconciled = asyncio.run(reconcile_inference_loads(store))
|
||||||
|
assert reconciled == [{"task_id": "t1", "status": "loaded"}]
|
||||||
|
updated = store._tasks["t1"]
|
||||||
|
assert updated["status"] == "loaded"
|
||||||
|
assert updated["load_status"]["loaded_models"][0]["status"] == "ready"
|
||||||
|
assert "n1" in store._inference_nodes
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_transitions_to_error_and_failed(monkeypatch) -> None:
|
||||||
|
async def _status_error(self) -> dict[str, Any]:
|
||||||
|
return {"loaded": False, "status": "error", "error": "CUDA out of memory"}
|
||||||
|
|
||||||
|
task = _task(
|
||||||
|
"t1",
|
||||||
|
node_id="n1",
|
||||||
|
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||||
|
)
|
||||||
|
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_error)
|
||||||
|
|
||||||
|
reconciled = asyncio.run(reconcile_inference_loads(store))
|
||||||
|
assert reconciled == [{"task_id": "t1", "status": "failed"}]
|
||||||
|
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||||
|
assert item["status"] == "error"
|
||||||
|
assert "CUDA out of memory" in item["error"]
|
||||||
|
assert "n1" not in store._inference_nodes
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_idle_marks_model_disappeared(monkeypatch) -> None:
|
||||||
|
async def _status_idle(self) -> dict[str, Any]:
|
||||||
|
return {"loaded": False, "status": "idle"}
|
||||||
|
|
||||||
|
task = _task(
|
||||||
|
"t1",
|
||||||
|
node_id="n1",
|
||||||
|
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||||
|
)
|
||||||
|
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_idle)
|
||||||
|
|
||||||
|
asyncio.run(reconcile_inference_loads(store))
|
||||||
|
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||||
|
assert item["status"] == "error"
|
||||||
|
assert "disappeared" in item["error"]
|
||||||
|
assert store._tasks["t1"]["status"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_unreachable_node_flips_to_error_after_cap(monkeypatch) -> None:
|
||||||
|
async def _raise(self) -> dict[str, Any]:
|
||||||
|
raise RuntimeError("conn refused")
|
||||||
|
|
||||||
|
task = _task(
|
||||||
|
"t1",
|
||||||
|
node_id="n1",
|
||||||
|
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||||
|
)
|
||||||
|
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||||
|
monkeypatch.setattr(ComputeNodeClient, "inference_status", _raise)
|
||||||
|
|
||||||
|
# 每次轮询前重置节流时间戳,逐次推进 load_attempts 到封顶
|
||||||
|
for _ in range(MAX_STARTING_ATTEMPTS):
|
||||||
|
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||||
|
item["last_polled_at"] = 0
|
||||||
|
asyncio.run(reconcile_inference_loads(store))
|
||||||
|
|
||||||
|
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||||
|
assert item["status"] == "error"
|
||||||
|
assert "unreachable" in item["error"]
|
||||||
|
assert store._tasks["t1"]["status"] == "failed"
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import math
|
import math
|
||||||
@@ -717,7 +718,9 @@ def create_app() -> FastAPI:
|
|||||||
@app.post(f"{route_prefix}/inference/unload")
|
@app.post(f"{route_prefix}/inference/unload")
|
||||||
async def inference_unload() -> dict[str, Any]:
|
async def inference_unload() -> dict[str, Any]:
|
||||||
"""Unload the currently loaded model and free GPU memory."""
|
"""Unload the currently loaded model and free GPU memory."""
|
||||||
return get_inference_session().unload()
|
# Teardown (gc.collect + cuda.empty_cache) can take a while; run it off
|
||||||
|
# the event loop so /health and /inference/status stay responsive.
|
||||||
|
return await asyncio.to_thread(get_inference_session().unload)
|
||||||
|
|
||||||
@app.get(f"{route_prefix}/inference/status")
|
@app.get(f"{route_prefix}/inference/status")
|
||||||
async def inference_status() -> dict[str, Any]:
|
async def inference_status() -> dict[str, Any]:
|
||||||
@@ -737,7 +740,10 @@ def create_app() -> FastAPI:
|
|||||||
messages = payload.get("messages") or []
|
messages = payload.get("messages") or []
|
||||||
if not messages:
|
if not messages:
|
||||||
raise HTTPException(status_code=400, detail="messages is required")
|
raise HTTPException(status_code=400, detail="messages is required")
|
||||||
result = get_inference_session().chat(
|
# Generation is long-running; run it in a thread so the event loop keeps
|
||||||
|
# serving /inference/status and /health during inference.
|
||||||
|
result = await asyncio.to_thread(
|
||||||
|
get_inference_session().chat,
|
||||||
messages=messages,
|
messages=messages,
|
||||||
temperature=float(payload.get("temperature", 0.95)),
|
temperature=float(payload.get("temperature", 0.95)),
|
||||||
top_p=float(payload.get("top_p", 0.7)),
|
top_p=float(payload.get("top_p", 0.7)),
|
||||||
|
|||||||
@@ -2,114 +2,208 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from typing import Any
|
import uuid
|
||||||
|
from typing import Any, Iterator
|
||||||
|
|
||||||
|
|
||||||
class InferenceSession:
|
class InferenceSession:
|
||||||
"""Manages a loaded model for inference with LLaMA-Factory ChatModel."""
|
"""Manages a loaded model for inference with LLaMA-Factory ChatModel.
|
||||||
|
|
||||||
|
Model loading is asynchronous: ``load()`` spawns a background daemon thread
|
||||||
|
and returns immediately with ``status == "loading"``. ``info()`` (served by
|
||||||
|
``/inference/status``) is always responsive, so the platform backend can
|
||||||
|
poll loading progress without being blocked by a minutes-long model load —
|
||||||
|
which previously froze the whole compute node event loop.
|
||||||
|
|
||||||
|
State machine: idle -> loading -> ready | error, ready -> idle (unload),
|
||||||
|
loading -> idle (cancelled). Long operations (ChatModel build, teardown,
|
||||||
|
generation) never run while holding ``_state_lock``; they either run in the
|
||||||
|
worker thread or under ``_chat_lock`` only.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
self._state_lock = threading.Lock() # brief state transitions only
|
||||||
|
self._chat_lock = threading.Lock() # serialize chat/teardown
|
||||||
|
self._status: str = "idle"
|
||||||
|
self._error: str = ""
|
||||||
|
self._request_id: str = ""
|
||||||
|
self._load_args: dict[str, Any] = {}
|
||||||
|
self._teardown_old = False # load-while-ready: unload old before loading new
|
||||||
|
self._cancel_requested = False # unload-while-loading: tear down after load finishes
|
||||||
|
self._load_thread: threading.Thread | None = None
|
||||||
self._model: Any = None
|
self._model: Any = None
|
||||||
self._tokenizer: Any = None
|
self._tokenizer: Any = None
|
||||||
self._generating_args: dict[str, Any] = {}
|
self._generating_args: dict[str, Any] = {}
|
||||||
self._model_name: str = ""
|
self._model_name: str = ""
|
||||||
self._adapter_path: str = ""
|
self._adapter_path: str = ""
|
||||||
self._lock = threading.Lock()
|
|
||||||
self._loaded_at: float = 0.0
|
self._loaded_at: float = 0.0
|
||||||
self._status: str = "idle"
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def status(self) -> str:
|
def status(self) -> str:
|
||||||
|
with self._state_lock:
|
||||||
return self._status
|
return self._status
|
||||||
|
|
||||||
@property
|
|
||||||
def model_name(self) -> str:
|
|
||||||
return self._model_name
|
|
||||||
|
|
||||||
@property
|
|
||||||
def adapter_path(self) -> str:
|
|
||||||
return self._adapter_path
|
|
||||||
|
|
||||||
@property
|
|
||||||
def loaded_at(self) -> float:
|
|
||||||
return self._loaded_at
|
|
||||||
|
|
||||||
def info(self) -> dict[str, Any]:
|
def info(self) -> dict[str, Any]:
|
||||||
|
with self._state_lock:
|
||||||
return {
|
return {
|
||||||
"loaded": self._status == "ready",
|
"loaded": self._status == "ready",
|
||||||
"status": self._status,
|
"status": self._status,
|
||||||
"model_name": self._model_name,
|
"model_name": self._model_name,
|
||||||
"adapter_path": self._adapter_path,
|
"adapter_path": self._adapter_path,
|
||||||
"loaded_at": self._loaded_at,
|
"loaded_at": self._loaded_at,
|
||||||
|
"request_id": self._request_id,
|
||||||
|
"error": self._error,
|
||||||
}
|
}
|
||||||
|
|
||||||
def load(self, model_name_or_path, adapter_name_or_path="", template="qwen", infer_backend="huggingface", infer_dtype="auto", **kwargs):
|
def load(
|
||||||
with self._lock:
|
self,
|
||||||
|
model_name_or_path,
|
||||||
|
adapter_name_or_path="",
|
||||||
|
template="qwen",
|
||||||
|
infer_backend="huggingface",
|
||||||
|
infer_dtype="auto",
|
||||||
|
**kwargs,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
with self._state_lock:
|
||||||
if self._status == "loading":
|
if self._status == "loading":
|
||||||
return {"loaded": False, "error": "model is already loading"}
|
# A model is already loading — dedupe, reuse the same request id.
|
||||||
if self._status == "ready":
|
return {"loaded": False, "status": "loading", "request_id": self._request_id}
|
||||||
self.unload()
|
self._teardown_old = self._status == "ready"
|
||||||
self._status = "loading"
|
self._status = "loading"
|
||||||
|
self._error = ""
|
||||||
|
self._request_id = uuid.uuid4().hex[:12]
|
||||||
|
self._cancel_requested = False
|
||||||
|
self._load_args = {
|
||||||
|
"model_name_or_path": model_name_or_path,
|
||||||
|
"template": template,
|
||||||
|
"infer_backend": infer_backend,
|
||||||
|
"infer_dtype": infer_dtype,
|
||||||
|
}
|
||||||
|
if adapter_name_or_path:
|
||||||
|
self._load_args["adapter_name_or_path"] = adapter_name_or_path
|
||||||
|
self._load_args.update(kwargs)
|
||||||
self._model_name = model_name_or_path
|
self._model_name = model_name_or_path
|
||||||
self._adapter_path = adapter_name_or_path
|
self._adapter_path = adapter_name_or_path
|
||||||
|
self._load_thread = threading.Thread(target=self._load_worker, daemon=True)
|
||||||
|
self._load_thread.start()
|
||||||
|
return {"loaded": False, "status": "loading", "request_id": self._request_id}
|
||||||
|
|
||||||
|
def _load_worker(self) -> None:
|
||||||
|
"""Build the ChatModel off the state lock so info() never blocks."""
|
||||||
|
model = None
|
||||||
|
tokenizer = None
|
||||||
|
generating_args: dict[str, Any] = {}
|
||||||
|
error = ""
|
||||||
try:
|
try:
|
||||||
|
if self._teardown_old:
|
||||||
|
self._release_model()
|
||||||
from llamafactory.chat import ChatModel
|
from llamafactory.chat import ChatModel
|
||||||
from llamafactory.hparams import get_infer_args
|
from llamafactory.hparams import get_infer_args
|
||||||
args = {"model_name_or_path": model_name_or_path, "template": template, "infer_backend": infer_backend, "infer_dtype": infer_dtype}
|
|
||||||
if adapter_name_or_path:
|
|
||||||
args["adapter_name_or_path"] = adapter_name_or_path
|
|
||||||
args.update(kwargs)
|
|
||||||
infer_result = get_infer_args(args)
|
|
||||||
# ChatModel internally re-parses the args dict via get_infer_args,
|
|
||||||
# so pass the original args (not the parsed dataclass objects).
|
|
||||||
self._model = ChatModel(args)
|
|
||||||
self._tokenizer = getattr(self._model, 'tokenizer', None) or self._model.engine.tokenizer
|
|
||||||
# Extract generating_args (last element) for later use in chat()
|
|
||||||
generating_args = infer_result[-1]
|
|
||||||
if hasattr(generating_args, '__dataclass_fields__'):
|
|
||||||
self._generating_args = {k: v for k, v in vars(generating_args).items()
|
|
||||||
if not k.startswith('_')}
|
|
||||||
else:
|
|
||||||
self._generating_args = dict(generating_args)
|
|
||||||
self._loaded_at = time.time()
|
|
||||||
self._status = "ready"
|
|
||||||
return {"loaded": True, "status": "ready"}
|
|
||||||
except Exception as exc:
|
|
||||||
self._status = "error"
|
|
||||||
self._model = None
|
|
||||||
return {"loaded": False, "status": "error", "error": str(exc)}
|
|
||||||
|
|
||||||
def unload(self):
|
args = dict(self._load_args)
|
||||||
with self._lock:
|
infer_result = get_infer_args(args)
|
||||||
if self._model is not None:
|
model = ChatModel(args)
|
||||||
try:
|
tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer
|
||||||
del self._model
|
generating_args = infer_result[-1]
|
||||||
except Exception:
|
if hasattr(generating_args, "__dataclass_fields__"):
|
||||||
pass
|
generating_args = {
|
||||||
|
k: v for k, v in vars(generating_args).items() if not k.startswith("_")
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
generating_args = dict(generating_args)
|
||||||
|
except Exception as exc: # noqa: BLE001 - surface load failure via status
|
||||||
|
error = str(exc)
|
||||||
|
with self._state_lock:
|
||||||
|
if error:
|
||||||
self._model = None
|
self._model = None
|
||||||
self._tokenizer = None
|
self._tokenizer = None
|
||||||
|
self._status = "error"
|
||||||
|
self._error = error
|
||||||
|
return
|
||||||
|
if self._cancel_requested:
|
||||||
|
# Unload was requested while loading — drop the fresh model.
|
||||||
|
model = None
|
||||||
|
tokenizer = None
|
||||||
|
self._model = None
|
||||||
|
self._tokenizer = None
|
||||||
|
self._status = "idle"
|
||||||
|
return
|
||||||
|
self._model = model
|
||||||
|
self._tokenizer = tokenizer
|
||||||
|
self._generating_args = generating_args
|
||||||
|
self._loaded_at = time.time()
|
||||||
|
self._status = "ready"
|
||||||
|
|
||||||
|
def _release_model(self) -> None:
|
||||||
|
with self._chat_lock:
|
||||||
|
with self._state_lock:
|
||||||
|
self._status = "unloading"
|
||||||
|
model = self._model
|
||||||
|
self._model = None
|
||||||
|
self._tokenizer = None
|
||||||
|
if model is not None:
|
||||||
|
try:
|
||||||
|
del model
|
||||||
|
except Exception: # noqa: BLE001 - best-effort teardown
|
||||||
|
pass
|
||||||
# 强制释放 PyTorch CUDA 缓存,真正归还 GPU 显存
|
# 强制释放 PyTorch CUDA 缓存,真正归还 GPU 显存
|
||||||
try:
|
try:
|
||||||
import gc
|
import gc
|
||||||
|
|
||||||
gc.collect()
|
gc.collect()
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
if torch.cuda.is_available():
|
if torch.cuda.is_available():
|
||||||
torch.cuda.empty_cache()
|
torch.cuda.empty_cache()
|
||||||
torch.cuda.synchronize()
|
torch.cuda.synchronize()
|
||||||
except Exception:
|
except Exception: # noqa: BLE001 - teardown must not raise
|
||||||
pass
|
pass
|
||||||
|
with self._state_lock:
|
||||||
self._status = "idle"
|
self._status = "idle"
|
||||||
self._model_name = ""
|
self._model_name = ""
|
||||||
self._adapter_path = ""
|
self._adapter_path = ""
|
||||||
self._loaded_at = 0.0
|
self._loaded_at = 0.0
|
||||||
return {"unloaded": True}
|
self._error = ""
|
||||||
|
|
||||||
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs):
|
def unload(self) -> dict[str, Any]:
|
||||||
with self._lock:
|
with self._state_lock:
|
||||||
|
if self._status == "loading":
|
||||||
|
# Ask the worker to tear down right after the load finishes.
|
||||||
|
self._cancel_requested = True
|
||||||
|
return {"unloaded": False, "status": "cancelling", "request_id": self._request_id}
|
||||||
|
was_ready = self._status == "ready"
|
||||||
|
if was_ready:
|
||||||
|
self._release_model()
|
||||||
|
else:
|
||||||
|
with self._state_lock:
|
||||||
|
self._model = None
|
||||||
|
self._tokenizer = None
|
||||||
|
self._status = "idle"
|
||||||
|
self._model_name = ""
|
||||||
|
self._adapter_path = ""
|
||||||
|
self._loaded_at = 0.0
|
||||||
|
self._error = ""
|
||||||
|
return {"unloaded": True, "status": "idle"}
|
||||||
|
|
||||||
|
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]:
|
||||||
|
with self._chat_lock:
|
||||||
|
with self._state_lock:
|
||||||
|
if self._status == "loading":
|
||||||
|
return {
|
||||||
|
"error": f"model is still loading (request_id={self._request_id}); please retry",
|
||||||
|
"response": "",
|
||||||
|
}
|
||||||
|
if self._status == "error":
|
||||||
|
return {"error": f"model load failed: {self._error}", "response": ""}
|
||||||
if self._status != "ready" or self._model is None:
|
if self._status != "ready" or self._model is None:
|
||||||
return {"error": "model not loaded", "response": ""}
|
return {"error": "model not loaded", "response": ""}
|
||||||
try:
|
try:
|
||||||
generate_kwargs = {"temperature": temperature, "top_p": top_p, "max_new_tokens": max_new_tokens, "do_sample": do_sample}
|
generate_kwargs = {
|
||||||
|
"temperature": temperature,
|
||||||
|
"top_p": top_p,
|
||||||
|
"max_new_tokens": max_new_tokens,
|
||||||
|
"do_sample": do_sample,
|
||||||
|
}
|
||||||
generate_kwargs.update(kwargs)
|
generate_kwargs.update(kwargs)
|
||||||
system = next((m["content"] for m in messages if m["role"] == "system"), None)
|
system = next((m["content"] for m in messages if m["role"] == "system"), None)
|
||||||
user_messages = [m for m in messages if m["role"] != "system"]
|
user_messages = [m for m in messages if m["role"] != "system"]
|
||||||
@@ -118,11 +212,18 @@ class InferenceSession:
|
|||||||
responses.append(response)
|
responses.append(response)
|
||||||
full_response = "".join(str(r) for r in responses)
|
full_response = "".join(str(r) for r in responses)
|
||||||
return {"response": full_response}
|
return {"response": full_response}
|
||||||
except Exception as exc:
|
except Exception as exc: # noqa: BLE001 - return generation error to caller
|
||||||
return {"error": str(exc), "response": ""}
|
return {"error": str(exc), "response": ""}
|
||||||
|
|
||||||
def chat_stream(self, messages, **kwargs):
|
def chat_stream(self, messages, **kwargs) -> Iterator[str]:
|
||||||
with self._lock:
|
with self._chat_lock:
|
||||||
|
with self._state_lock:
|
||||||
|
if self._status == "loading":
|
||||||
|
yield 'data: {"error": "model is still loading; please retry"}\n\n'
|
||||||
|
return
|
||||||
|
if self._status == "error":
|
||||||
|
yield 'data: {"error": "model load failed: ' + str(self._error) + '"}\n\n'
|
||||||
|
return
|
||||||
if self._status != "ready" or self._model is None:
|
if self._status != "ready" or self._model is None:
|
||||||
yield 'data: {"error": "model not loaded"}\n\n'
|
yield 'data: {"error": "model not loaded"}\n\n'
|
||||||
return
|
return
|
||||||
@@ -132,13 +233,14 @@ class InferenceSession:
|
|||||||
user_messages = [m for m in messages if m["role"] != "system"]
|
user_messages = [m for m in messages if m["role"] != "system"]
|
||||||
for new_text in self._model.stream_chat(user_messages, system=system, **generate_kwargs):
|
for new_text in self._model.stream_chat(user_messages, system=system, **generate_kwargs):
|
||||||
yield new_text
|
yield new_text
|
||||||
except Exception as exc:
|
except Exception as exc: # noqa: BLE001 - stream error as SSE event
|
||||||
yield 'data: {"error": "' + str(exc) + '"}\n\n'
|
yield 'data: {"error": "' + str(exc) + '"}\n\n'
|
||||||
|
|
||||||
|
|
||||||
_inference_session = None
|
_inference_session = None
|
||||||
|
|
||||||
def get_inference_session():
|
|
||||||
|
def get_inference_session() -> InferenceSession:
|
||||||
global _inference_session
|
global _inference_session
|
||||||
if _inference_session is None:
|
if _inference_session is None:
|
||||||
_inference_session = InferenceSession()
|
_inference_session = InferenceSession()
|
||||||
|
|||||||
127
compute/tests/test_inference_session.py
Normal file
127
compute/tests/test_inference_session.py
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import types
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from compute.engines.llama_factory.inference import InferenceSession
|
||||||
|
|
||||||
|
# 模拟模型加载耗时,用于验证 load() 立即返回、info() 不阻塞
|
||||||
|
LOAD_DELAY = 0.2
|
||||||
|
|
||||||
|
|
||||||
|
class FakeChatModel:
|
||||||
|
def __init__(self, args: dict[str, Any]) -> None:
|
||||||
|
time.sleep(LOAD_DELAY)
|
||||||
|
self.tokenizer = object()
|
||||||
|
self.engine = types.SimpleNamespace(tokenizer=object())
|
||||||
|
self._output = "hello from model"
|
||||||
|
|
||||||
|
def stream_chat(self, *args, **kwargs):
|
||||||
|
for _ in range(1):
|
||||||
|
yield self._output
|
||||||
|
|
||||||
|
|
||||||
|
class FailingChatModel:
|
||||||
|
def __init__(self, args: dict[str, Any]) -> None:
|
||||||
|
time.sleep(LOAD_DELAY)
|
||||||
|
raise RuntimeError("boom: fake load failure")
|
||||||
|
|
||||||
|
|
||||||
|
def _get_infer_args(args: dict[str, Any]) -> list[Any]:
|
||||||
|
# 最后一个元素为 generating_args,worker 会转成 dict
|
||||||
|
return [None, None, {"temperature": 0.7}]
|
||||||
|
|
||||||
|
|
||||||
|
def _install_llamafactory(monkeypatch, chat_model: type) -> None:
|
||||||
|
llmf = types.ModuleType("llamafactory")
|
||||||
|
chat_mod = types.ModuleType("llamafactory.chat")
|
||||||
|
hparams_mod = types.ModuleType("llamafactory.hparams")
|
||||||
|
chat_mod.ChatModel = chat_model
|
||||||
|
hparams_mod.get_infer_args = _get_infer_args
|
||||||
|
llmf.chat = chat_mod
|
||||||
|
llmf.hparams = hparams_mod
|
||||||
|
monkeypatch.setitem(sys.modules, "llamafactory", llmf)
|
||||||
|
monkeypatch.setitem(sys.modules, "llamafactory.chat", chat_mod)
|
||||||
|
monkeypatch.setitem(sys.modules, "llamafactory.hparams", hparams_mod)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def stub_llamafactory(monkeypatch) -> None:
|
||||||
|
_install_llamafactory(monkeypatch, FakeChatModel)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def stub_failing_llamafactory(monkeypatch) -> None:
|
||||||
|
_install_llamafactory(monkeypatch, FailingChatModel)
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_status(session: InferenceSession, status: str, timeout: float = 3.0) -> bool:
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
while time.time() < deadline:
|
||||||
|
if session.info()["status"] == status:
|
||||||
|
return True
|
||||||
|
time.sleep(0.02)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_returns_immediately_then_ready(stub_llamafactory) -> None:
|
||||||
|
session = InferenceSession()
|
||||||
|
started = time.time()
|
||||||
|
result = session.load("/models/qwen")
|
||||||
|
assert result["status"] == "loading"
|
||||||
|
assert result["loaded"] is False
|
||||||
|
assert result["request_id"]
|
||||||
|
# 在慢加载完成前就返回,且 info() 加载期间可响应
|
||||||
|
assert time.time() - started < LOAD_DELAY
|
||||||
|
assert session.info()["status"] == "loading"
|
||||||
|
assert _wait_for_status(session, "ready")
|
||||||
|
info = session.info()
|
||||||
|
assert info["loaded"] is True
|
||||||
|
assert info["status"] == "ready"
|
||||||
|
assert info["model_name"] == "/models/qwen"
|
||||||
|
|
||||||
|
|
||||||
|
def test_second_load_while_loading_deduped(stub_llamafactory) -> None:
|
||||||
|
session = InferenceSession()
|
||||||
|
r1 = session.load("/models/a")
|
||||||
|
r2 = session.load("/models/b")
|
||||||
|
assert r2["status"] == "loading"
|
||||||
|
assert r2["request_id"] == r1["request_id"]
|
||||||
|
assert _wait_for_status(session, "ready")
|
||||||
|
assert session.info()["status"] == "ready"
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_error_surfaces_in_status(stub_failing_llamafactory) -> None:
|
||||||
|
session = InferenceSession()
|
||||||
|
session.load("/models/bad")
|
||||||
|
assert _wait_for_status(session, "error")
|
||||||
|
assert "boom" in session.info()["error"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_unload_while_loading_cancels(stub_llamafactory) -> None:
|
||||||
|
session = InferenceSession()
|
||||||
|
session.load("/models/qwen")
|
||||||
|
result = session.unload()
|
||||||
|
assert result["status"] == "cancelling"
|
||||||
|
assert _wait_for_status(session, "idle")
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_while_loading_returns_loading_error(stub_llamafactory) -> None:
|
||||||
|
session = InferenceSession()
|
||||||
|
session.load("/models/qwen")
|
||||||
|
out = session.chat([{"role": "user", "content": "hi"}])
|
||||||
|
assert "still loading" in (out.get("error") or "")
|
||||||
|
assert _wait_for_status(session, "ready")
|
||||||
|
out = session.chat([{"role": "user", "content": "hi"}])
|
||||||
|
assert out.get("response") == "hello from model"
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_stream_while_loading_yields_error(stub_llamafactory) -> None:
|
||||||
|
session = InferenceSession()
|
||||||
|
session.load("/models/qwen")
|
||||||
|
chunks = list(session.chat_stream([{"role": "user", "content": "hi"}]))
|
||||||
|
assert any("still loading" in c for c in chunks)
|
||||||
@@ -14,8 +14,8 @@ server {
|
|||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
proxy_read_timeout 300s;
|
proxy_read_timeout 900s;
|
||||||
proxy_send_timeout 300s;
|
proxy_send_timeout 900s;
|
||||||
}
|
}
|
||||||
|
|
||||||
location = /modelTF {
|
location = /modelTF {
|
||||||
@@ -25,8 +25,8 @@ server {
|
|||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
proxy_read_timeout 300s;
|
proxy_read_timeout 900s;
|
||||||
proxy_send_timeout 300s;
|
proxy_send_timeout 900s;
|
||||||
}
|
}
|
||||||
|
|
||||||
location ~* \.(?:js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf)$ {
|
location ~* \.(?:js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf)$ {
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import { get, post, del } from '../request'
|
import { get, post, del } from '../request'
|
||||||
import type { CompareTask, CompareModelRef } from '@/types'
|
import type { CompareTask, CompareModelRef } from '@/types'
|
||||||
|
|
||||||
|
const INFERENCE_START_TIMEOUT_MS = 15 * 60 * 1000
|
||||||
|
|
||||||
/** 推理/对比任务列表 */
|
/** 推理/对比任务列表 */
|
||||||
export const getCompareList = () => get<CompareTask[]>('/model-compare')
|
export const getCompareList = () => get<CompareTask[]>('/model-compare')
|
||||||
|
|
||||||
@@ -12,7 +14,7 @@ export const createCompare = (data: Partial<CompareTask>) =>
|
|||||||
post<{ id: string | number }>('/model-compare', data)
|
post<{ id: string | number }>('/model-compare', data)
|
||||||
|
|
||||||
/** 删除任务 */
|
/** 删除任务 */
|
||||||
export const deleteCompare = (id: string | number) => del(`/model-compare/${id}`)
|
export const deleteCompare = (id: string | number) => del(`/model-compare/${id}`, undefined, { timeout: 60_000 })
|
||||||
|
|
||||||
/** 更新任务加载状态 */
|
/** 更新任务加载状态 */
|
||||||
export const updateLoadStatus = (id: string | number, load_status: any) =>
|
export const updateLoadStatus = (id: string | number, load_status: any) =>
|
||||||
@@ -34,7 +36,8 @@ export const stopModelByPid = (pid: number) =>
|
|||||||
post('/model-compare/stop-by-pid', { pid })
|
post('/model-compare/stop-by-pid', { pid })
|
||||||
|
|
||||||
/** 加载任务 */
|
/** 加载任务 */
|
||||||
export const loadCompare = (id: string | number) => post(`/model-compare/${id}/load`)
|
export const loadCompare = (id: string | number) =>
|
||||||
|
post(`/model-compare/${id}/load`, undefined, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||||
|
|
||||||
/** 卸载任务 */
|
/** 卸载任务 */
|
||||||
export const unloadCompare = (id: string | number) => post(`/model-compare/${id}/unload`)
|
export const unloadCompare = (id: string | number) => post(`/model-compare/${id}/unload`)
|
||||||
@@ -81,6 +84,10 @@ export const streamChatReal = (data: any): Promise<Response> => {
|
|||||||
temperature: data.temperature ?? 0.7,
|
temperature: data.temperature ?? 0.7,
|
||||||
top_p: data.top_p ?? 0.95,
|
top_p: data.top_p ?? 0.95,
|
||||||
max_tokens: data.max_tokens ?? 2048,
|
max_tokens: data.max_tokens ?? 2048,
|
||||||
|
// 透传 task_id/node_id,让后端按 load_status 路由到真正加载了模型的算力节点,
|
||||||
|
// 避免在多节点时回退到“第一个在线节点”导致连接失败
|
||||||
|
task_id: data.task_id,
|
||||||
|
node_id: data.node_id,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -94,8 +101,8 @@ export const batchChat = (data: any) => post('/model-chat/batch', data)
|
|||||||
/** 本地 transformers 模型对话 */
|
/** 本地 transformers 模型对话 */
|
||||||
export const localChat = (data: any) => post('/model-chat/local/chat', data)
|
export const localChat = (data: any) => post('/model-chat/local/chat', data)
|
||||||
|
|
||||||
/** 预加载本地模型(模型加载耗时长,超时 5 分钟) */
|
/** 预加载本地模型(模型加载耗时长,超时 15 分钟) */
|
||||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data, { timeout: 300000 })
|
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||||
|
|
||||||
/** 预加载已训练模型(超时 5 分钟) */
|
/** 预加载已训练模型(超时 15 分钟) */
|
||||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data, { timeout: 300000 })
|
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||||
|
|||||||
@@ -1,6 +1,16 @@
|
|||||||
import { get, post, put, del } from '../request'
|
import { get, post, put, del } from '../request'
|
||||||
import type { FineTuneStartPayload, FineTuneTask, TrainingProgress, LogContent } from '@/types'
|
import type { FineTuneStartPayload, FineTuneTask, TrainingProgress, LogContent } from '@/types'
|
||||||
|
|
||||||
|
export interface FineTuneMetricPoint {
|
||||||
|
step: number
|
||||||
|
epoch?: number | null
|
||||||
|
loss?: number | null
|
||||||
|
grad_norm?: number | null
|
||||||
|
learning_rate?: number | null
|
||||||
|
raw?: string
|
||||||
|
create_time?: string
|
||||||
|
}
|
||||||
|
|
||||||
export interface TrainingDiagnostic {
|
export interface TrainingDiagnostic {
|
||||||
level: string
|
level: string
|
||||||
title: string
|
title: string
|
||||||
@@ -80,6 +90,10 @@ export const getFineTuneLogs = (
|
|||||||
params: { tail_lines?: number; offset?: number; limit?: number } = {},
|
params: { tail_lines?: number; offset?: number; limit?: number } = {},
|
||||||
) => get<LogContent & { job_id?: string; source?: string }>(`/fine-tune/${id}/logs`, params)
|
) => get<LogContent & { job_id?: string; source?: string }>(`/fine-tune/${id}/logs`, params)
|
||||||
|
|
||||||
|
/** 获取训练指标曲线数据 */
|
||||||
|
export const getFineTuneMetrics = (id: string | number) =>
|
||||||
|
get<FineTuneMetricPoint[]>(`/fine-tune/${id}/metrics`)
|
||||||
|
|
||||||
/** 启动 TensorBoard */
|
/** 启动 TensorBoard */
|
||||||
export const startTensorboard = () => post('/fine-tune/tensorboard/start')
|
export const startTensorboard = () => post('/fine-tune/tensorboard/start')
|
||||||
|
|
||||||
|
|||||||
@@ -88,10 +88,14 @@ export const updateModelPurpose = (id: string | number, purpose: string) =>
|
|||||||
|
|
||||||
/** 合并 LoRA 权重 */
|
/** 合并 LoRA 权重 */
|
||||||
export const mergeModel = (data: {
|
export const mergeModel = (data: {
|
||||||
|
trained_model_id?: string | number
|
||||||
model_name: string
|
model_name: string
|
||||||
train_method: string
|
train_method: string
|
||||||
base_model_path: string
|
base_model_path: string
|
||||||
}) => post('/model-manage/merge', data)
|
adapter_path?: string
|
||||||
|
compute_node_id?: string
|
||||||
|
output_model_name?: string
|
||||||
|
}) => post('/model-manage/merge', data, { timeout: 15 * 60 * 1000 })
|
||||||
|
|
||||||
/** 导出已训练模型权重 */
|
/** 导出已训练模型权重 */
|
||||||
export const exportModelUrl = (modelName: string) =>
|
export const exportModelUrl = (modelName: string) =>
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import type {
|
|||||||
UpdateUserAccessPayload,
|
UpdateUserAccessPayload,
|
||||||
} from '@/types'
|
} from '@/types'
|
||||||
|
|
||||||
|
export type { SystemUser } from '@/types'
|
||||||
|
|
||||||
/** 系统信息(CPU/内存/磁盘/GPU/网络/系统) */
|
/** 系统信息(CPU/内存/磁盘/GPU/网络/系统) */
|
||||||
export const getSystemInfo = () => get<SystemInfo>('/system-info')
|
export const getSystemInfo = () => get<SystemInfo>('/system-info')
|
||||||
|
|
||||||
|
|||||||
@@ -92,8 +92,6 @@ service.interceptors.response.use(
|
|||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
if (res.code === 0) {
|
if (res.code === 0) {
|
||||||
// 记录业务模块访问(用于看板用户操作分布统计)
|
|
||||||
trackVisit(response.config.url)
|
|
||||||
return res.data
|
return res.data
|
||||||
}
|
}
|
||||||
// 业务错误
|
// 业务错误
|
||||||
|
|||||||
@@ -44,6 +44,24 @@ export function useStreamChat() {
|
|||||||
})
|
})
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
|
|
||||||
|
/** 从 SSE 帧中提取错误信息(后端/计算节点错误以 data: {"error": "..."} 形式下发) */
|
||||||
|
function extractSseError(buffer: string): string | null {
|
||||||
|
const trimmed = buffer.trim()
|
||||||
|
if (!trimmed.startsWith('data: ')) return null
|
||||||
|
const lines = trimmed.split(/\r?\n/)
|
||||||
|
for (let i = lines.length - 1; i >= 0; i--) {
|
||||||
|
const line = lines[i].trim()
|
||||||
|
if (!line.startsWith('data: ')) continue
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(line.slice(6))
|
||||||
|
if (obj && typeof obj.error === 'string' && obj.error) return obj.error
|
||||||
|
} catch {
|
||||||
|
/* 非 JSON 的 data 行忽略 */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
/** 从内容中解析 think 标签 */
|
/** 从内容中解析 think 标签 */
|
||||||
function parseContent(content: string) {
|
function parseContent(content: string) {
|
||||||
const thinkRegex = /<think>([\s\S]*?)(<\/think>)?/g
|
const thinkRegex = /<think>([\s\S]*?)(<\/think>)?/g
|
||||||
@@ -121,6 +139,16 @@ export function useStreamChat() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 最终更新
|
// 最终更新
|
||||||
|
// 若整段响应是 SSE 错误帧,提取 error 字段以干净文案展示
|
||||||
|
const sseError = extractSseError(buffer)
|
||||||
|
if (sseError) {
|
||||||
|
message.value.isThinking = false
|
||||||
|
message.value.isStreaming = false
|
||||||
|
message.value.done = true
|
||||||
|
message.value.error = sseError
|
||||||
|
message.value.displayContent = sseError
|
||||||
|
return
|
||||||
|
}
|
||||||
const parsed = parseContent(buffer)
|
const parsed = parseContent(buffer)
|
||||||
message.value.thinkContent = parsed.think
|
message.value.thinkContent = parsed.think
|
||||||
message.value.displayContent = parsed.display
|
message.value.displayContent = parsed.display
|
||||||
|
|||||||
@@ -390,6 +390,31 @@ router.beforeEach((to, _from, next) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 路由切换时记录业务模块访问(用于看板用户操作分布统计)
|
||||||
|
const ROUTE_TO_MODULE: Record<string, string> = {
|
||||||
|
'/fine-tune': 'fine-tune',
|
||||||
|
'/model-eval': 'model-eval',
|
||||||
|
'/model-inference': 'model-inference',
|
||||||
|
'/model-compare': 'model-inference',
|
||||||
|
'/data-process': 'data-process',
|
||||||
|
'/data-convert': 'data-convert',
|
||||||
|
'/model-manage': 'model-manage',
|
||||||
|
'/dataset-manage': 'dataset',
|
||||||
|
}
|
||||||
|
for (const [prefix, module] of Object.entries(ROUTE_TO_MODULE)) {
|
||||||
|
if (to.path.startsWith(prefix)) {
|
||||||
|
const key = `route-visit:${module}`
|
||||||
|
const last = Number(sessionStorage.getItem(key) || 0)
|
||||||
|
if (Date.now() - last >= 60000) {
|
||||||
|
sessionStorage.setItem(key, String(Date.now()))
|
||||||
|
import('@/api/modules/audit-visit').then(({ recordModuleVisit }) => {
|
||||||
|
recordModuleVisit(module, to.fullPath).catch(() => {})
|
||||||
|
}).catch(() => {})
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
next()
|
next()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,10 @@ export interface TrainedModel {
|
|||||||
name: string
|
name: string
|
||||||
train_methods?: TrainMethod[]
|
train_methods?: TrainMethod[]
|
||||||
base_model_path?: string
|
base_model_path?: string
|
||||||
|
artifact_dir?: string
|
||||||
|
adapter_path?: string
|
||||||
|
compute_node_id?: string
|
||||||
|
compute_node_name?: string
|
||||||
create_time?: string
|
create_time?: string
|
||||||
merged?: boolean
|
merged?: boolean
|
||||||
merging?: boolean
|
merging?: boolean
|
||||||
@@ -212,6 +216,9 @@ export interface LoadedModel {
|
|||||||
status?: string
|
status?: string
|
||||||
pid?: number
|
pid?: number
|
||||||
port?: number
|
port?: number
|
||||||
|
node_id?: string
|
||||||
|
node_name?: string
|
||||||
|
error?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CompareTask {
|
export interface CompareTask {
|
||||||
@@ -230,6 +237,8 @@ export interface CompareModelRef {
|
|||||||
model_name: string
|
model_name: string
|
||||||
model_path: string
|
model_path: string
|
||||||
gpu_id: number
|
gpu_id: number
|
||||||
|
node_id?: string
|
||||||
|
node_name?: string
|
||||||
source?: string
|
source?: string
|
||||||
port?: number
|
port?: number
|
||||||
}
|
}
|
||||||
@@ -245,7 +254,9 @@ export interface EvalTask {
|
|||||||
model_name?: string
|
model_name?: string
|
||||||
model_id?: number | string
|
model_id?: number | string
|
||||||
dataset?: string
|
dataset?: string
|
||||||
|
dataset_id?: number | string
|
||||||
metric?: string
|
metric?: string
|
||||||
|
metric_label?: string
|
||||||
score?: number
|
score?: number
|
||||||
status?: string
|
status?: string
|
||||||
create_time?: string
|
create_time?: string
|
||||||
|
|||||||
@@ -66,8 +66,12 @@ const operationDistribution = ref<{ name: string; value: number }[]>([])
|
|||||||
const serviceIcon: Record<string, string> = {
|
const serviceIcon: Record<string, string> = {
|
||||||
'模型推理': 'fa-cube',
|
'模型推理': 'fa-cube',
|
||||||
'模型微调': 'fa-sliders',
|
'模型微调': 'fa-sliders',
|
||||||
|
'模型训练': 'fa-sliders',
|
||||||
'模型评测': 'fa-bar-chart',
|
'模型评测': 'fa-bar-chart',
|
||||||
|
'模型管理': 'fa-cubes',
|
||||||
|
'数据集管理': 'fa-file-text',
|
||||||
'数据处理': 'fa-filter',
|
'数据处理': 'fa-filter',
|
||||||
|
'数据类型转换': 'fa-exchange',
|
||||||
}
|
}
|
||||||
const roleLabel: Record<string, string> = {
|
const roleLabel: Record<string, string> = {
|
||||||
admin: '超级管理员',
|
admin: '超级管理员',
|
||||||
@@ -115,7 +119,7 @@ const chartOption = computed<EChartsOption>(() => ({
|
|||||||
borderWidth: 0,
|
borderWidth: 0,
|
||||||
padding: [10, 12],
|
padding: [10, 12],
|
||||||
textStyle: { color: '#ffffff', fontSize: 12 },
|
textStyle: { color: '#ffffff', fontSize: 12 },
|
||||||
valueFormatter: (value) => `${value}`,
|
valueFormatter: (value) => String(value ?? ''),
|
||||||
},
|
},
|
||||||
xAxis: {
|
xAxis: {
|
||||||
type: 'category',
|
type: 'category',
|
||||||
@@ -252,7 +256,7 @@ const loginDurationChartOption = computed<EChartsOption>(() => {
|
|||||||
tooltip: {
|
tooltip: {
|
||||||
trigger: 'axis',
|
trigger: 'axis',
|
||||||
axisPointer: { type: 'shadow' },
|
axisPointer: { type: 'shadow' },
|
||||||
valueFormatter: (value: number) => `${value} 小时`,
|
valueFormatter: (value: unknown) => String(Number(Array.isArray(value) ? value[0] : value) || 0) + ' 小时',
|
||||||
},
|
},
|
||||||
xAxis: {
|
xAxis: {
|
||||||
type: 'value',
|
type: 'value',
|
||||||
@@ -738,10 +742,11 @@ function viewTask(task: DashboardTask) {
|
|||||||
|
|
||||||
.service-table {
|
.service-table {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-rows: 36px repeat(4, minmax(48px, 1fr));
|
grid-auto-rows: minmax(44px, auto);
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
margin-top: 12px;
|
margin-top: 12px;
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
|
overflow-y: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
.service-row {
|
.service-row {
|
||||||
@@ -935,7 +940,7 @@ function viewTask(task: DashboardTask) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.service-table {
|
.service-table {
|
||||||
grid-template-rows: 32px repeat(4, minmax(40px, 1fr));
|
grid-auto-rows: minmax(38px, auto);
|
||||||
margin-top: 8px;
|
margin-top: 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, ref, watch } from 'vue'
|
import { onMounted, ref, watch } from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter } from 'vue-router'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
@@ -10,8 +10,7 @@ import StartEvalStep from './create/StartEvalStep.vue'
|
|||||||
import { createDimension, startEval } from '@/api/modules/eval'
|
import { createDimension, startEval } from '@/api/modules/eval'
|
||||||
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
||||||
import { getDatasetList } from '@/api/modules/dataset'
|
import { getDatasetList } from '@/api/modules/dataset'
|
||||||
import { getSystemInfo } from '@/api/modules/system'
|
import { getComputeGpus } from '@/api/modules/compute'
|
||||||
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
|
||||||
import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types'
|
import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types'
|
||||||
|
|
||||||
type StepExposed = { validate: () => Promise<boolean> }
|
type StepExposed = { validate: () => Promise<boolean> }
|
||||||
@@ -83,9 +82,8 @@ async function loadData() {
|
|||||||
const results = await Promise.allSettled([
|
const results = await Promise.allSettled([
|
||||||
getTrainedModels(),
|
getTrainedModels(),
|
||||||
getDatasetList(),
|
getDatasetList(),
|
||||||
getSystemInfo(),
|
|
||||||
getModelList(),
|
getModelList(),
|
||||||
getComputeNodes(),
|
getComputeGpus(),
|
||||||
])
|
])
|
||||||
|
|
||||||
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
|
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
|
||||||
@@ -93,18 +91,12 @@ async function loadData() {
|
|||||||
evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
|
evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
|
||||||
}
|
}
|
||||||
if (results[2].status === 'fulfilled') {
|
if (results[2].status === 'fulfilled') {
|
||||||
const allGpus: GpuInfo[] = results[2].value?.gpu || []
|
evalModels.value = (results[2].value || []).filter(
|
||||||
const nodes: ComputeNode[] = (results[4].status === 'fulfilled' ? results[4].value : []) || []
|
(model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url),
|
||||||
const onlineIds = new Set(nodes.filter((n) => n.enabled && n.scheduler_status === 'online').map((n) => n.id))
|
|
||||||
// Only show idle GPUs from online compute nodes
|
|
||||||
gpus.value = allGpus.filter(
|
|
||||||
(g) => g.status === 'idle' && (!g.node_id || onlineIds.has(g.node_id)),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
if (results[3].status === 'fulfilled') {
|
if (results[3].status === 'fulfilled') {
|
||||||
evalModels.value = (results[3].value || []).filter(
|
gpus.value = ((results[3].value || []) as unknown as GpuInfo[]).filter((g) => g.status === 'idle')
|
||||||
(model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const failedCount = results.filter((result) => result.status === 'rejected').length
|
const failedCount = results.filter((result) => result.status === 'rejected').length
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
import { useRoute } from 'vue-router'
|
import { useRoute } from 'vue-router'
|
||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
@@ -50,6 +50,8 @@ const passRate = computed(() => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
const overallScore = computed(() => formatScore(detail.value?.overall_score, detail.value?.overall_score_max))
|
const overallScore = computed(() => formatScore(detail.value?.overall_score, detail.value?.overall_score_max))
|
||||||
|
const displayModelName = computed(() => detail.value?.model_name || String(detail.value?.model_id || '-'))
|
||||||
|
const displayMetric = computed(() => detail.value?.metric_label || detail.value?.metric || '-')
|
||||||
|
|
||||||
function formatDateTime(value?: string) {
|
function formatDateTime(value?: string) {
|
||||||
if (!value) return '-'
|
if (!value) return '-'
|
||||||
@@ -121,9 +123,9 @@ onUnmounted(stopPolling)
|
|||||||
</div>
|
</div>
|
||||||
<dl class="task-meta">
|
<dl class="task-meta">
|
||||||
<div><dt>任务 ID</dt><dd>{{ detail?.id || taskId }}</dd></div>
|
<div><dt>任务 ID</dt><dd>{{ detail?.id || taskId }}</dd></div>
|
||||||
<div><dt>评测模型</dt><dd>{{ detail?.model_name || '-' }}</dd></div>
|
<div><dt>评测模型</dt><dd>{{ displayModelName }}</dd></div>
|
||||||
<div><dt>测试集</dt><dd>{{ detail?.dataset || '-' }}</dd></div>
|
<div><dt>测试集</dt><dd>{{ detail?.dataset || '-' }}</dd></div>
|
||||||
<div><dt>评测指标</dt><dd>{{ detail?.metric || '-' }}</dd></div>
|
<div><dt>评测指标</dt><dd>{{ displayMetric }}</dd></div>
|
||||||
</dl>
|
</dl>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -141,7 +143,7 @@ onUnmounted(stopPolling)
|
|||||||
<div class="overview-item score-hero">
|
<div class="overview-item score-hero">
|
||||||
<span>综合得分</span>
|
<span>综合得分</span>
|
||||||
<strong>{{ overallScore }}</strong>
|
<strong>{{ overallScore }}</strong>
|
||||||
<small>大模型综合评分</small>
|
<small>模型综合评分</small>
|
||||||
</div>
|
</div>
|
||||||
<div class="overview-item">
|
<div class="overview-item">
|
||||||
<span>样本通过率</span>
|
<span>样本通过率</span>
|
||||||
@@ -164,7 +166,7 @@ onUnmounted(stopPolling)
|
|||||||
<div class="review-copy">
|
<div class="review-copy">
|
||||||
<div class="section-heading">
|
<div class="section-heading">
|
||||||
<div>
|
<div>
|
||||||
<h2 id="overall-review-title">大模型综合评价</h2>
|
<h2 id="overall-review-title">综合评价</h2>
|
||||||
<p>基于全部已评测样本生成的总体结论</p>
|
<p>基于全部已评测样本生成的总体结论</p>
|
||||||
</div>
|
</div>
|
||||||
<el-tag v-if="detail.evaluator_model" type="primary" size="small">
|
<el-tag v-if="detail.evaluator_model" type="primary" size="small">
|
||||||
@@ -172,7 +174,7 @@ onUnmounted(stopPolling)
|
|||||||
</el-tag>
|
</el-tag>
|
||||||
</div>
|
</div>
|
||||||
<p class="review-text">
|
<p class="review-text">
|
||||||
{{ detail.overall_evaluation || (detail.status === 'running' ? '评测仍在进行,综合评价将在样本评分完成后生成。' : '暂无综合评价。') }}
|
{{ detail.overall_evaluation || (detail.status === 'running' ? '评测正在进行,综合评价将在样本完成后生成。' : '暂无综合评价。') }}
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<div class="suggestion-block">
|
<div class="suggestion-block">
|
||||||
@@ -190,8 +192,8 @@ onUnmounted(stopPolling)
|
|||||||
<section v-if="detail.dimension_summary?.length" class="dimension-summary" aria-labelledby="dimension-title">
|
<section v-if="detail.dimension_summary?.length" class="dimension-summary" aria-labelledby="dimension-title">
|
||||||
<div class="section-heading compact-heading">
|
<div class="section-heading compact-heading">
|
||||||
<div>
|
<div>
|
||||||
<h2 id="dimension-title">维度表现</h2>
|
<h2 id="dimension-title">指标表现</h2>
|
||||||
<p>查看各评测维度的得分与样本通过率</p>
|
<p>查看各评测指标的得分与通过率</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="dimension-grid">
|
<div class="dimension-grid">
|
||||||
@@ -212,22 +214,10 @@ onUnmounted(stopPolling)
|
|||||||
<p>共 {{ filteredSamples.length }} 条结果,展开行可查看评分依据与子维度分数</p>
|
<p>共 {{ filteredSamples.length }} 条结果,展开行可查看评分依据与子维度分数</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="sample-filters" aria-label="样本筛选">
|
<div class="sample-filters" aria-label="样本筛选">
|
||||||
<el-input
|
<el-input v-model="keyword" clearable placeholder="搜索问题、回答或评价" aria-label="搜索样本" @input="resetPage">
|
||||||
v-model="keyword"
|
|
||||||
clearable
|
|
||||||
placeholder="搜索问题、回答或评价"
|
|
||||||
aria-label="搜索样本"
|
|
||||||
@input="resetPage"
|
|
||||||
>
|
|
||||||
<template #prefix><i class="fa fa-search" aria-hidden="true" /></template>
|
<template #prefix><i class="fa fa-search" aria-hidden="true" /></template>
|
||||||
</el-input>
|
</el-input>
|
||||||
<el-select
|
<el-select v-model="judgementFilter" clearable placeholder="全部判定" aria-label="按判定筛选" @change="resetPage">
|
||||||
v-model="judgementFilter"
|
|
||||||
clearable
|
|
||||||
placeholder="全部判定"
|
|
||||||
aria-label="按判定筛选"
|
|
||||||
@change="resetPage"
|
|
||||||
>
|
|
||||||
<el-option label="正确" value="正确" />
|
<el-option label="正确" value="正确" />
|
||||||
<el-option label="部分正确" value="部分正确" />
|
<el-option label="部分正确" value="部分正确" />
|
||||||
<el-option label="错误" value="错误" />
|
<el-option label="错误" value="错误" />
|
||||||
@@ -235,18 +225,12 @@ onUnmounted(stopPolling)
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<el-table
|
<el-table v-if="filteredSamples.length" class="sample-results-table" :data="paginatedSamples" row-key="id" table-layout="fixed">
|
||||||
v-if="filteredSamples.length"
|
|
||||||
class="sample-results-table"
|
|
||||||
:data="paginatedSamples"
|
|
||||||
row-key="id"
|
|
||||||
table-layout="fixed"
|
|
||||||
>
|
|
||||||
<el-table-column type="expand" width="48">
|
<el-table-column type="expand" width="48">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<div class="sample-detail-grid">
|
<div class="sample-detail-grid">
|
||||||
<div class="evaluation-reason">
|
<div class="evaluation-reason">
|
||||||
<span>大模型评分依据</span>
|
<span>评分依据</span>
|
||||||
<p>{{ row.evaluation_reason || '暂无评分依据。' }}</p>
|
<p>{{ row.evaluation_reason || '暂无评分依据。' }}</p>
|
||||||
</div>
|
</div>
|
||||||
<div v-if="row.error_type" class="error-type">
|
<div v-if="row.error_type" class="error-type">
|
||||||
@@ -275,9 +259,7 @@ onUnmounted(stopPolling)
|
|||||||
<template #default="{ row }"><p class="cell-copy">{{ row.model_output || '等待生成' }}</p></template>
|
<template #default="{ row }"><p class="cell-copy">{{ row.model_output || '等待生成' }}</p></template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="得分" width="90" align="center">
|
<el-table-column label="得分" width="90" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }"><span class="sample-score">{{ formatScore(row.score, row.max_score) }}</span></template>
|
||||||
<span class="sample-score">{{ formatScore(row.score, row.max_score) }}</span>
|
|
||||||
</template>
|
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="判定" width="96" align="center">
|
<el-table-column label="判定" width="96" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
@@ -294,21 +276,11 @@ onUnmounted(stopPolling)
|
|||||||
<p>{{ detail.status === 'running' ? '任务正在运行,结果生成后会显示在这里。' : '请调整筛选条件或稍后重试。' }}</p>
|
<p>{{ detail.status === 'running' ? '任务正在运行,结果生成后会显示在这里。' : '请调整筛选条件或稍后重试。' }}</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<el-pagination
|
<el-pagination v-if="filteredSamples.length > pageSize" v-model:current-page="currentPage" v-model:page-size="pageSize" background layout="total, sizes, prev, pager, next" :page-sizes="[10, 20, 50]" :total="filteredSamples.length" aria-label="样本结果分页" />
|
||||||
v-if="filteredSamples.length > pageSize"
|
|
||||||
v-model:current-page="currentPage"
|
|
||||||
v-model:page-size="pageSize"
|
|
||||||
background
|
|
||||||
layout="total, sizes, prev, pager, next"
|
|
||||||
:page-sizes="[10, 20, 50]"
|
|
||||||
:total="filteredSamples.length"
|
|
||||||
aria-label="样本结果分页"
|
|
||||||
/>
|
|
||||||
</section>
|
</section>
|
||||||
</template>
|
</template>
|
||||||
</PageCard>
|
</PageCard>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<style scoped lang="scss">
|
<style scoped lang="scss">
|
||||||
.eval-detail-page {
|
.eval-detail-page {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
@@ -767,3 +739,6 @@ onUnmounted(stopPolling)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref, onMounted, onUnmounted } from 'vue'
|
import { ref, onMounted, onUnmounted } from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter } from 'vue-router'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
@@ -57,6 +57,14 @@ function handleViewDetail(row: any) {
|
|||||||
router.push({ name: 'model-eval-detail', params: { id: row.id } })
|
router.push({ name: 'model-eval-detail', params: { id: row.id } })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function displayModelName(row: Partial<EvalTask>) {
|
||||||
|
return row.model_name || String(row.model_id || '-')
|
||||||
|
}
|
||||||
|
|
||||||
|
function displayMetric(row: Partial<EvalTask>) {
|
||||||
|
return row.metric_label || row.metric || '-'
|
||||||
|
}
|
||||||
|
|
||||||
const { start: startPolling, stop: stopPolling } = usePolling(
|
const { start: startPolling, stop: stopPolling } = usePolling(
|
||||||
async () => {
|
async () => {
|
||||||
await loadEvalList({ silent: true })
|
await loadEvalList({ silent: true })
|
||||||
@@ -102,9 +110,21 @@ onUnmounted(() => {
|
|||||||
</template>
|
</template>
|
||||||
<template #columns>
|
<template #columns>
|
||||||
<el-table-column label="任务名称" prop="eval_task_name" align="center" />
|
<el-table-column label="任务名称" prop="eval_task_name" align="center" />
|
||||||
<el-table-column label="评测模型" prop="model_name" align="center" />
|
<el-table-column label="评测模型" align="center" min-width="160">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-tooltip :content="displayModelName(row)" placement="top" :disabled="displayModelName(row).length < 18">
|
||||||
|
<span class="cell-ellipsis">{{ displayModelName(row) }}</span>
|
||||||
|
</el-tooltip>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="数据集" prop="dataset" align="center" />
|
<el-table-column label="数据集" prop="dataset" align="center" />
|
||||||
<el-table-column label="指标" prop="metric" align="center" />
|
<el-table-column label="指标" align="center" min-width="220">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-tooltip :content="displayMetric(row)" placement="top" :disabled="displayMetric(row).length < 24">
|
||||||
|
<span class="cell-ellipsis">{{ displayMetric(row) }}</span>
|
||||||
|
</el-tooltip>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="评分" prop="score" width="100" align="center" />
|
<el-table-column label="评分" prop="score" width="100" align="center" />
|
||||||
<el-table-column label="状态" width="100" align="center">
|
<el-table-column label="状态" width="100" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
@@ -159,7 +179,7 @@ onUnmounted(() => {
|
|||||||
min-height: 0;
|
min-height: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* 胶囊切换栏样式 */
|
/* 胶囊切换栏 */
|
||||||
.capsule-tabs {
|
.capsule-tabs {
|
||||||
display: flex;
|
display: flex;
|
||||||
background: #f1f5f9;
|
background: #f1f5f9;
|
||||||
@@ -193,4 +213,13 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.cell-ellipsis {
|
||||||
|
display: inline-block;
|
||||||
|
max-width: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
vertical-align: middle;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref, reactive, nextTick, onMounted, watch } from 'vue'
|
import { ref, reactive, nextTick, onMounted, onUnmounted, watch } from 'vue'
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
import MarkdownView from '@/components/MarkdownView.vue'
|
import MarkdownView from '@/components/MarkdownView.vue'
|
||||||
import { useStreamChat } from '@/composables/useStreamChat'
|
import { useStreamChat } from '@/composables/useStreamChat'
|
||||||
import { getCompare } from '@/api/modules/compare'
|
import { getCompare, getLoadStatus } from '@/api/modules/compare'
|
||||||
import type { CompareTask, LoadedModel } from '@/types'
|
import type { CompareTask, LoadedModel } from '@/types'
|
||||||
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
@@ -37,6 +37,10 @@ const contentRef = ref<HTMLElement>()
|
|||||||
let activeAssistant: ChatMessage | null = null
|
let activeAssistant: ChatMessage | null = null
|
||||||
/** 设置面板抽屉 */
|
/** 设置面板抽屉 */
|
||||||
const showSettings = ref(false)
|
const showSettings = ref(false)
|
||||||
|
/** 模型仍在加载中(直接 URL 进入 chat 时兜底轮询就绪状态) */
|
||||||
|
const taskLoading = ref(false)
|
||||||
|
const taskError = ref('')
|
||||||
|
let statusTimer: ReturnType<typeof setInterval> | null = null
|
||||||
|
|
||||||
/** 获取任务信息,定位已启动的模型(mock 模式跳过) */
|
/** 获取任务信息,定位已启动的模型(mock 模式跳过) */
|
||||||
async function loadTask() {
|
async function loadTask() {
|
||||||
@@ -45,6 +49,14 @@ async function loadTask() {
|
|||||||
task.value = await getCompare(taskId)
|
task.value = await getCompare(taskId)
|
||||||
const models = parseLoadedModels(task.value)
|
const models = parseLoadedModels(task.value)
|
||||||
if (models[0]?.model_name) modelName.value = models[0].model_name
|
if (models[0]?.model_name) modelName.value = models[0].model_name
|
||||||
|
// 恢复本地保存的历史对话
|
||||||
|
restoreHistory()
|
||||||
|
// 模型仍在上次加载中:启动轮询等待就绪
|
||||||
|
if (models.some((m) => m.status === 'starting')) {
|
||||||
|
taskLoading.value = true
|
||||||
|
await pollTaskStatus()
|
||||||
|
statusTimer = setInterval(pollTaskStatus, 3000)
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
@@ -60,6 +72,80 @@ function parseLoadedModels(t: CompareTask | null): LoadedModel[] {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 对话历史本地持久化(按任务 id 存储,退出重进可恢复) */
|
||||||
|
const STORAGE_PREFIX = 'ygft_chat_history_'
|
||||||
|
|
||||||
|
function historyKey(id: string | number): string {
|
||||||
|
return `${STORAGE_PREFIX}${id}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveHistory() {
|
||||||
|
if (isMock) return
|
||||||
|
try {
|
||||||
|
const snapshot = messages.value.map((m) => ({
|
||||||
|
role: m.role,
|
||||||
|
content: m.content,
|
||||||
|
think: m.think,
|
||||||
|
done: true,
|
||||||
|
}))
|
||||||
|
localStorage.setItem(historyKey(taskId), JSON.stringify(snapshot))
|
||||||
|
} catch {
|
||||||
|
// 存储失败忽略
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function restoreHistory() {
|
||||||
|
if (isMock) return
|
||||||
|
try {
|
||||||
|
const raw = localStorage.getItem(historyKey(taskId))
|
||||||
|
if (!raw) return
|
||||||
|
const parsed = JSON.parse(raw)
|
||||||
|
if (Array.isArray(parsed)) {
|
||||||
|
messages.value = parsed.map((m) => ({
|
||||||
|
role: m.role === 'user' ? 'user' : 'assistant',
|
||||||
|
content: m.content || '',
|
||||||
|
think: m.think || '',
|
||||||
|
isThinking: false,
|
||||||
|
isStreaming: false,
|
||||||
|
done: true,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// 恢复失败忽略
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 停止就绪状态轮询 */
|
||||||
|
function stopStatusPolling() {
|
||||||
|
if (statusTimer) {
|
||||||
|
clearInterval(statusTimer)
|
||||||
|
statusTimer = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 轮询任务加载状态:starting → ready/error */
|
||||||
|
async function pollTaskStatus() {
|
||||||
|
try {
|
||||||
|
const st = await getLoadStatus(taskId)
|
||||||
|
const items = st.loaded_models || []
|
||||||
|
const anyReady = items.some((m) => m.status === 'ready' || m.status === 'running')
|
||||||
|
const anyError = items.some((m) => m.status === 'error')
|
||||||
|
if (anyReady) {
|
||||||
|
taskLoading.value = false
|
||||||
|
taskError.value = ''
|
||||||
|
stopStatusPolling()
|
||||||
|
} else if (anyError) {
|
||||||
|
taskLoading.value = false
|
||||||
|
taskError.value = items.find((m) => m.status === 'error')?.error || '模型加载失败'
|
||||||
|
stopStatusPolling()
|
||||||
|
} else {
|
||||||
|
taskLoading.value = true
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// 轮询失败忽略,下次再试
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function handleSend() {
|
async function handleSend() {
|
||||||
const question = inputQuestion.value.trim()
|
const question = inputQuestion.value.trim()
|
||||||
if (!question || loading.value) return
|
if (!question || loading.value) return
|
||||||
@@ -76,6 +162,7 @@ async function handleSend() {
|
|||||||
done: false,
|
done: false,
|
||||||
})
|
})
|
||||||
messages.value.push(assistantMsg)
|
messages.value.push(assistantMsg)
|
||||||
|
saveHistory()
|
||||||
|
|
||||||
inputQuestion.value = ''
|
inputQuestion.value = ''
|
||||||
await nextTick()
|
await nextTick()
|
||||||
@@ -85,6 +172,7 @@ async function handleSend() {
|
|||||||
// mock 模式:直接用假数据逐字填充
|
// mock 模式:直接用假数据逐字填充
|
||||||
if (isMock) {
|
if (isMock) {
|
||||||
await mockReply(assistantMsg, question)
|
await mockReply(assistantMsg, question)
|
||||||
|
saveHistory()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,6 +182,7 @@ async function handleSend() {
|
|||||||
await send(
|
await send(
|
||||||
{
|
{
|
||||||
model_path: route.query.model_path as string || '',
|
model_path: route.query.model_path as string || '',
|
||||||
|
task_id: taskId,
|
||||||
system_prompt: systemPrompt.value,
|
system_prompt: systemPrompt.value,
|
||||||
user_question: question,
|
user_question: question,
|
||||||
temperature: temperature.value,
|
temperature: temperature.value,
|
||||||
@@ -111,6 +200,7 @@ async function handleSend() {
|
|||||||
assistantMsg.done = true
|
assistantMsg.done = true
|
||||||
activeAssistant = null
|
activeAssistant = null
|
||||||
reset()
|
reset()
|
||||||
|
saveHistory()
|
||||||
await nextTick()
|
await nextTick()
|
||||||
scrollToBottom()
|
scrollToBottom()
|
||||||
}
|
}
|
||||||
@@ -169,6 +259,11 @@ function handleNewChat() {
|
|||||||
activeAssistant = null
|
activeAssistant = null
|
||||||
messages.value = []
|
messages.value = []
|
||||||
reset()
|
reset()
|
||||||
|
try {
|
||||||
|
localStorage.removeItem(historyKey(taskId))
|
||||||
|
} catch {
|
||||||
|
// 忽略
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 输入框自适应高度 */
|
/** 输入框自适应高度 */
|
||||||
@@ -185,6 +280,7 @@ function resetInputHeight() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
onMounted(loadTask)
|
onMounted(loadTask)
|
||||||
|
onUnmounted(stopStatusPolling)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
@@ -252,6 +348,12 @@ onMounted(loadTask)
|
|||||||
|
|
||||||
<!-- 输入栏 -->
|
<!-- 输入栏 -->
|
||||||
<footer class="chat-input-container">
|
<footer class="chat-input-container">
|
||||||
|
<div v-if="taskLoading" class="loading-hint">
|
||||||
|
<i class="fa fa-spinner fa-spin" style="margin-right: 6px" />模型加载中,就绪后即可对话...
|
||||||
|
</div>
|
||||||
|
<div v-else-if="taskError" class="loading-hint error">
|
||||||
|
<i class="fa fa-exclamation-circle" style="margin-right: 6px" />{{ taskError }}
|
||||||
|
</div>
|
||||||
<div class="chat-input-inner">
|
<div class="chat-input-inner">
|
||||||
<button class="clear-btn" title="清空对话" @click="handleNewChat">
|
<button class="clear-btn" title="清空对话" @click="handleNewChat">
|
||||||
<i class="fa fa-eraser" />
|
<i class="fa fa-eraser" />
|
||||||
@@ -261,22 +363,21 @@ onMounted(loadTask)
|
|||||||
v-model="inputQuestion"
|
v-model="inputQuestion"
|
||||||
class="input-box"
|
class="input-box"
|
||||||
rows="1"
|
rows="1"
|
||||||
:disabled="loading"
|
:disabled="loading || taskLoading"
|
||||||
placeholder="给模型发送消息..."
|
placeholder="给模型发送消息..."
|
||||||
@keydown.enter.exact.prevent="handleSend"
|
@keydown.enter.exact.prevent="handleSend"
|
||||||
@input="autoResize"
|
@input="autoResize"
|
||||||
/>
|
/>
|
||||||
<button
|
<button
|
||||||
class="send-btn"
|
class="send-btn"
|
||||||
:class="{ active: inputQuestion.trim() && !loading }"
|
:class="{ active: inputQuestion.trim() && !loading && !taskLoading }"
|
||||||
:disabled="!inputQuestion.trim() || loading"
|
:disabled="!inputQuestion.trim() || loading || taskLoading"
|
||||||
@click="handleSend"
|
@click="handleSend"
|
||||||
>
|
>
|
||||||
<i class="fa fa-arrow-up" />
|
<i class="fa fa-arrow-up" />
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="footer-hint">内容由 AI 生成,请仔细甄别。</div>
|
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
<!-- 设置抽屉(系统提示词等) -->
|
<!-- 设置抽屉(系统提示词等) -->
|
||||||
@@ -703,9 +804,18 @@ onMounted(loadTask)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.footer-hint {
|
.loading-hint {
|
||||||
margin-top: 12px;
|
margin-bottom: 10px;
|
||||||
font-size: 12px;
|
padding: 6px 14px;
|
||||||
color: #9ca3af;
|
font-size: 13px;
|
||||||
|
color: #b45309;
|
||||||
|
background: #fef3c7;
|
||||||
|
border-radius: 8px;
|
||||||
|
text-align: center;
|
||||||
|
|
||||||
|
&.error {
|
||||||
|
color: #b91c1c;
|
||||||
|
background: #fee2e2;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref, reactive, computed, onMounted } from 'vue'
|
import { ref, reactive, computed, onMounted, watch } from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter } from 'vue-router'
|
||||||
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
|
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
|
||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
import { getModelList, getTrainedModels } from '@/api/modules/model'
|
import { getModelList, getTrainedModels } from '@/api/modules/model'
|
||||||
import { getSystemInfo } from '@/api/modules/system'
|
import { getSystemInfo } from '@/api/modules/system'
|
||||||
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
||||||
import { createCompare, preloadLocalModel, preloadTrainedModel } from '@/api/modules/compare'
|
import { createCompare, loadCompare } from '@/api/modules/compare'
|
||||||
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
|
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
|
||||||
|
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
@@ -27,6 +27,8 @@ interface SelectableModel {
|
|||||||
name: string
|
name: string
|
||||||
source: 'database' | 'trained'
|
source: 'database' | 'trained'
|
||||||
model_path: string
|
model_path: string
|
||||||
|
compute_node_id?: string
|
||||||
|
compute_node_name?: string
|
||||||
merged?: boolean
|
merged?: boolean
|
||||||
merging?: boolean
|
merging?: boolean
|
||||||
disabled?: boolean
|
disabled?: boolean
|
||||||
@@ -51,6 +53,8 @@ const trainedOptions = computed<SelectableModel[]>(() =>
|
|||||||
name: m.name,
|
name: m.name,
|
||||||
source: 'trained',
|
source: 'trained',
|
||||||
model_path: m.merged_path || m.base_model_path || '',
|
model_path: m.merged_path || m.base_model_path || '',
|
||||||
|
compute_node_id: m.compute_node_id,
|
||||||
|
compute_node_name: m.compute_node_name,
|
||||||
merged: m.merged,
|
merged: m.merged,
|
||||||
merging: m.merging,
|
merging: m.merging,
|
||||||
disabled: m.merged === false,
|
disabled: m.merged === false,
|
||||||
@@ -80,7 +84,7 @@ const form = reactive({
|
|||||||
/** 选中的模型 key(单选) */
|
/** 选中的模型 key(单选) */
|
||||||
model_key: '',
|
model_key: '',
|
||||||
/** 使用的 GPU */
|
/** 使用的 GPU */
|
||||||
gpu_id: 0,
|
gpu_key: '',
|
||||||
})
|
})
|
||||||
|
|
||||||
const rules: FormRules = {
|
const rules: FormRules = {
|
||||||
@@ -90,6 +94,13 @@ const rules: FormRules = {
|
|||||||
|
|
||||||
/** 当前选中的模型对象 */
|
/** 当前选中的模型对象 */
|
||||||
const selectedModel = computed(() => modelMap.value[form.model_key])
|
const selectedModel = computed(() => modelMap.value[form.model_key])
|
||||||
|
const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key))
|
||||||
|
|
||||||
|
watch(selectedModel, (model) => {
|
||||||
|
if (!model?.compute_node_id) return
|
||||||
|
const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
|
||||||
|
if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}`
|
||||||
|
})
|
||||||
|
|
||||||
async function handleSubmit() {
|
async function handleSubmit() {
|
||||||
if (!formRef.value) return
|
if (!formRef.value) return
|
||||||
@@ -101,62 +112,47 @@ async function handleSubmit() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
submitting.value = true
|
submitting.value = true
|
||||||
startupStatus.value = '正在启动模型服务...'
|
startupStatus.value = '正在创建推理任务...'
|
||||||
try {
|
try {
|
||||||
// Step 1: 将模型加载到算力节点
|
if (!m.model_path) {
|
||||||
const preloadPayload = {
|
ElMessage.warning('当前模型未配置算力节点可访问路径,请先在模型管理中维护模型路径')
|
||||||
model_name_or_path: m.model_path,
|
|
||||||
model_name: m.name,
|
|
||||||
template: 'qwen',
|
|
||||||
}
|
|
||||||
let preloadResult: any
|
|
||||||
if (m.source === 'trained') {
|
|
||||||
preloadResult = await preloadTrainedModel(preloadPayload)
|
|
||||||
} else {
|
|
||||||
preloadResult = await preloadLocalModel(preloadPayload)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (preloadResult && (preloadResult as any).error) {
|
|
||||||
ElMessage.warning(`模型加载失败:${(preloadResult as any).error}`)
|
|
||||||
submitting.value = false
|
|
||||||
startupStatus.value = ''
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 2: 创建推理任务记录
|
// Step 1: 创建推理任务记录
|
||||||
const taskResult = await createCompare({
|
const taskResult = await createCompare({
|
||||||
name: form.name || m.name,
|
name: form.name || m.name,
|
||||||
description: form.description,
|
description: form.description,
|
||||||
|
status: 'pending',
|
||||||
models: [
|
models: [
|
||||||
{
|
{
|
||||||
model_id: String(m.id),
|
model_id: String(m.id),
|
||||||
model_name: m.name,
|
model_name: m.name,
|
||||||
model_path: m.model_path,
|
model_path: m.model_path,
|
||||||
source: m.source,
|
source: m.source,
|
||||||
gpu_id: form.gpu_id,
|
gpu_id: selectedGpu.value?.id ?? 0,
|
||||||
|
node_id: selectedGpu.value?.node_id || m.compute_node_id,
|
||||||
|
node_name: selectedGpu.value?.node_name || m.compute_node_name,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
const taskId = taskResult?.id || 'unknown'
|
const taskId = taskResult?.id || 'unknown'
|
||||||
|
|
||||||
ElMessage.success('模型已启动')
|
// Step 2: 统一通过推理任务加载接口异步派发模型加载,状态会落到列表记录中。
|
||||||
router.push({
|
startupStatus.value = '正在启动模型服务,首次加载可能需要数分钟...'
|
||||||
path: `/model-inference/chat/${taskId}`,
|
const loadResult: any = await loadCompare(taskId)
|
||||||
query: {
|
if (loadResult?.status === 'failed' || loadResult?.error) {
|
||||||
model: m.name,
|
ElMessage.warning(`模型加载失败:${loadResult?.error || '请检查算力节点日志'}`)
|
||||||
source: m.source,
|
router.push('/model-inference')
|
||||||
model_path: m.model_path,
|
return
|
||||||
},
|
}
|
||||||
})
|
|
||||||
|
// 加载为异步派发,回到列表页可看到“启动中 → 已就绪”的状态流转
|
||||||
|
ElMessage.success('模型加载中,就绪后即可对话')
|
||||||
|
router.push('/model-inference')
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
// 真实 API 失败时回退到 mock 模式(方便无算力节点的开发调试)
|
|
||||||
const m = selectedModel.value!
|
|
||||||
const reason = e?.message || e?.toString() || '未知错误'
|
const reason = e?.message || e?.toString() || '未知错误'
|
||||||
ElMessage.warning(`推理服务启动失败:${reason},进入 mock 演示模式`)
|
ElMessage.warning(`推理服务启动失败:${reason}`)
|
||||||
router.push({
|
|
||||||
path: '/model-inference/chat/mock',
|
|
||||||
query: { model: m.name },
|
|
||||||
})
|
|
||||||
} finally {
|
} finally {
|
||||||
submitting.value = false
|
submitting.value = false
|
||||||
startupStatus.value = ''
|
startupStatus.value = ''
|
||||||
@@ -181,7 +177,10 @@ async function loadData() {
|
|||||||
gpus.value = sys?.gpu || []
|
gpus.value = sys?.gpu || []
|
||||||
computeNodes.value = nodes || []
|
computeNodes.value = nodes || []
|
||||||
// 默认选中第一个空闲 GPU
|
// 默认选中第一个空闲 GPU
|
||||||
if (idleGpus.value.length > 0) form.gpu_id = idleGpus.value[0].id ?? 0
|
if (idleGpus.value.length > 0) {
|
||||||
|
const firstGpu = idleGpus.value[0]
|
||||||
|
form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}`
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
@@ -229,12 +228,12 @@ onMounted(loadData)
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item label="GPU">
|
<el-form-item label="GPU">
|
||||||
<el-select v-model="form.gpu_id" style="width: 400px">
|
<el-select v-model="form.gpu_key" style="width: 400px">
|
||||||
<el-option
|
<el-option
|
||||||
v-for="g in idleGpus"
|
v-for="g in idleGpus"
|
||||||
:key="g.id ?? 0"
|
:key="`${g.node_id || ''}:${g.id ?? 0}`"
|
||||||
:label="`${g.name} (GPU${g.id ?? 0}) [空闲]`"
|
:label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
|
||||||
:value="g.id ?? 0"
|
:value="`${g.node_id || ''}:${g.id ?? 0}`"
|
||||||
/>
|
/>
|
||||||
</el-select>
|
</el-select>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|||||||
@@ -92,11 +92,9 @@ async function handleUnload(row: any) {
|
|||||||
loadData()
|
loadData()
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 删除(先释放算力节点再删除记录) */
|
/** 删除(后端删除内部会 best-effort 释放算力节点,这里直接删记录) */
|
||||||
async function handleDelete(row: any) {
|
async function handleDelete(row: any) {
|
||||||
await ElMessageBox.confirm('确定要删除该推理记录吗?将先释放算力节点再删除。', '确认删除', { type: 'warning' })
|
await ElMessageBox.confirm('确定要删除该推理记录吗?将先释放算力节点再删除。', '确认删除', { type: 'warning' })
|
||||||
// 先释放算力节点上的模型
|
|
||||||
await unloadCompare(row.id).catch(() => {})
|
|
||||||
await deleteCompare(row.id)
|
await deleteCompare(row.id)
|
||||||
dataList.value = dataList.value.filter((item) => item.id !== row.id)
|
dataList.value = dataList.value.filter((item) => item.id !== row.id)
|
||||||
await loadData(true)
|
await loadData(true)
|
||||||
|
|||||||
@@ -18,9 +18,12 @@ const trainedModels = ref<TrainedModel[]>([])
|
|||||||
const currentModel = computed(() => trainedModels.value.find((m) => m.name === modelName.value))
|
const currentModel = computed(() => trainedModels.value.find((m) => m.name === modelName.value))
|
||||||
|
|
||||||
const form = reactive({
|
const form = reactive({
|
||||||
|
trained_model_id: '',
|
||||||
model_name: modelName.value,
|
model_name: modelName.value,
|
||||||
train_method: method.value,
|
train_method: method.value,
|
||||||
base_model_path: '',
|
base_model_path: '',
|
||||||
|
adapter_path: '',
|
||||||
|
compute_node_id: '',
|
||||||
})
|
})
|
||||||
|
|
||||||
async function loadModel() {
|
async function loadModel() {
|
||||||
@@ -28,23 +31,30 @@ async function loadModel() {
|
|||||||
const res = await getTrainedModels()
|
const res = await getTrainedModels()
|
||||||
trainedModels.value = res?.models || []
|
trainedModels.value = res?.models || []
|
||||||
const target = trainedModels.value.find((m) => m.name === modelName.value)
|
const target = trainedModels.value.find((m) => m.name === modelName.value)
|
||||||
|
form.trained_model_id = target?.id == null ? '' : String(target.id)
|
||||||
form.base_model_path = target?.base_model_path || ''
|
form.base_model_path = target?.base_model_path || ''
|
||||||
|
form.adapter_path = target?.artifact_dir || target?.adapter_path || target?.merged_path || ''
|
||||||
|
form.compute_node_id = target?.compute_node_id || ''
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleMerge() {
|
async function handleMerge() {
|
||||||
if (!form.model_name || !form.base_model_path) {
|
if (!form.model_name || !form.base_model_path || !form.adapter_path) {
|
||||||
ElMessage.warning('缺少模型信息')
|
ElMessage.warning('缺少模型信息')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
merging.value = true
|
merging.value = true
|
||||||
try {
|
try {
|
||||||
await mergeModel({
|
await mergeModel({
|
||||||
|
trained_model_id: form.trained_model_id || form.model_name,
|
||||||
model_name: form.model_name,
|
model_name: form.model_name,
|
||||||
train_method: form.train_method,
|
train_method: form.train_method,
|
||||||
base_model_path: form.base_model_path,
|
base_model_path: form.base_model_path,
|
||||||
|
adapter_path: form.adapter_path,
|
||||||
|
compute_node_id: form.compute_node_id,
|
||||||
|
output_model_name: `${form.model_name}-merged`,
|
||||||
})
|
})
|
||||||
ElMessage.success('合并成功')
|
ElMessage.success('合并成功')
|
||||||
router.push('/model-manage')
|
router.push('/model-manage')
|
||||||
@@ -82,6 +92,9 @@ onMounted(loadModel)
|
|||||||
<el-form-item label="基座模型路径">
|
<el-form-item label="基座模型路径">
|
||||||
<el-input v-model="form.base_model_path" placeholder="基座模型路径" />
|
<el-input v-model="form.base_model_path" placeholder="基座模型路径" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
<el-form-item label="Adapter 路径">
|
||||||
|
<el-input v-model="form.adapter_path" placeholder="LoRA Adapter 权重目录" />
|
||||||
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item>
|
<el-form-item>
|
||||||
<el-button type="primary" :loading="merging" @click="handleMerge">
|
<el-button type="primary" :loading="merging" @click="handleMerge">
|
||||||
|
|||||||
@@ -99,23 +99,23 @@ onMounted(() => {
|
|||||||
<el-button type="primary" :icon="Plus" @click="showCreate = true">新建项目</el-button>
|
<el-button type="primary" :icon="Plus" @click="showCreate = true">新建项目</el-button>
|
||||||
</template>
|
</template>
|
||||||
<template #columns>
|
<template #columns>
|
||||||
<el-table-column prop="name" label="项目名" min-width="140" />
|
<el-table-column prop="name" label="项目名称" min-width="140" />
|
||||||
<el-table-column prop="code" label="编码ID" min-width="100" />
|
<el-table-column prop="code" label="编码 ID" min-width="100" />
|
||||||
<el-table-column prop="status" label="状态" min-width="100" />
|
<el-table-column prop="status" label="状态" min-width="100" />
|
||||||
<el-table-column prop="description" label="描述" min-width="200" show-overflow-tooltip />
|
<el-table-column prop="description" label="描述" min-width="200" show-overflow-tooltip />
|
||||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||||
</template>
|
</template>
|
||||||
<template #actions="{ row }">
|
<template #actions="{ row }">
|
||||||
<el-button link type="primary" @click="openDetail(row.id)">详情</el-button>
|
<el-button link type="primary" @click="openDetail(row.id)">详情</el-button>
|
||||||
<el-button link type="danger" @click="handleDelete(row)">删除</el-button>
|
<el-button link type="danger" @click="handleDelete(asProject(row))">删除</el-button>
|
||||||
</template>
|
</template>
|
||||||
</DataTablePage>
|
</DataTablePage>
|
||||||
<el-dialog v-model="showCreate" title="新建项目" width="520px">
|
<el-dialog v-model="showCreate" title="新建项目" width="520px">
|
||||||
<el-form label-width="90px">
|
<el-form label-width="90px">
|
||||||
<el-form-item label="名称" required>
|
<el-form-item label="名称" required>
|
||||||
<el-input v-model="form.name" placeholder="项目名" />
|
<el-input v-model="form.name" placeholder="项目名称" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="编码ID" required>
|
<el-form-item label="编码 ID" required>
|
||||||
<el-select v-model="form.tenant_id" style="width: 100%" placeholder="选择租户编码">
|
<el-select v-model="form.tenant_id" style="width: 100%" placeholder="选择租户编码">
|
||||||
<el-option v-for="t in tenantCodeOptions" :key="t.value" :label="t.label" :value="t.tenantId" />
|
<el-option v-for="t in tenantCodeOptions" :key="t.value" :label="t.label" :value="t.tenantId" />
|
||||||
</el-select>
|
</el-select>
|
||||||
|
|||||||
@@ -8,13 +8,14 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
|
|||||||
import { usePolling } from '@/composables/usePolling'
|
import { usePolling } from '@/composables/usePolling'
|
||||||
import '@/plugins/echarts-training-log'
|
import '@/plugins/echarts-training-log'
|
||||||
import { useModelsStore } from '@/stores/models'
|
import { useModelsStore } from '@/stores/models'
|
||||||
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, type TrainingDiagnostic } from '@/api/modules/fineTune'
|
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
|
||||||
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
|
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
|
||||||
import { getDataset } from '@/api/modules/dataset'
|
import { getDataset } from '@/api/modules/dataset'
|
||||||
import { getSystemInfo } from '@/api/modules/system'
|
import { getSystemInfo } from '@/api/modules/system'
|
||||||
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
|
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
|
||||||
import {
|
import {
|
||||||
buildMetricChartOption,
|
buildMetricChartOption,
|
||||||
|
metricsFromApi,
|
||||||
parseTrainingLog,
|
parseTrainingLog,
|
||||||
resolveTrainingLogFile,
|
resolveTrainingLogFile,
|
||||||
} from './training-log/trainingLogModel'
|
} from './training-log/trainingLogModel'
|
||||||
@@ -42,6 +43,7 @@ const loading = ref(true)
|
|||||||
|
|
||||||
// 训练指标数据(ECharts 接收 number[],下标即 step)
|
// 训练指标数据(ECharts 接收 number[],下标即 step)
|
||||||
const metricData = reactive({
|
const metricData = reactive({
|
||||||
|
steps: [] as number[],
|
||||||
loss: [] as number[],
|
loss: [] as number[],
|
||||||
gradNorm: [] as number[],
|
gradNorm: [] as number[],
|
||||||
lr: [] as number[],
|
lr: [] as number[],
|
||||||
@@ -62,9 +64,9 @@ const gpuExpanded = ref(false)
|
|||||||
let refreshInFlight = false
|
let refreshInFlight = false
|
||||||
|
|
||||||
/** 三个曲线的 ECharts 配置(响应式,数据变化自动重绘) */
|
/** 三个曲线的 ECharts 配置(响应式,数据变化自动重绘) */
|
||||||
const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, '#4f46e5'))
|
const lossChartOption = computed(() => buildMetricChartOption('Loss', metricData.loss, metricData.steps, '#4f46e5'))
|
||||||
const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, '#3b82f6'))
|
const gradChartOption = computed(() => buildMetricChartOption('Grad Norm', metricData.gradNorm, metricData.steps, '#3b82f6'))
|
||||||
const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, '#14b8a6', true))
|
const lrChartOption = computed(() => buildMetricChartOption('Learning Rate', metricData.lr, metricData.steps, '#14b8a6', true))
|
||||||
const baseModelName = computed(() => task.value?.base_model != null
|
const baseModelName = computed(() => task.value?.base_model != null
|
||||||
? modelsStore.getModelName(task.value.base_model)
|
? modelsStore.getModelName(task.value.base_model)
|
||||||
: '未配置')
|
: '未配置')
|
||||||
@@ -77,10 +79,10 @@ const trainingMethodName = computed(() => task.value?.train_method
|
|||||||
const taskGpuLabel = computed(() => task.value?.gpus?.length
|
const taskGpuLabel = computed(() => task.value?.gpus?.length
|
||||||
? task.value.gpus.map((gpuId) => `GPU ${gpuId}`).join('、')
|
? task.value.gpus.map((gpuId) => `GPU ${gpuId}`).join('、')
|
||||||
: '未配置')
|
: '未配置')
|
||||||
const latestLoss = computed(() => metricData.loss[metricData.loss.length - 1])
|
const latestLoss = computed(() => lastFinite(metricData.loss))
|
||||||
const latestGradNorm = computed(() => metricData.gradNorm[metricData.gradNorm.length - 1])
|
const latestGradNorm = computed(() => lastFinite(metricData.gradNorm))
|
||||||
const latestLearningRate = computed(() => metricData.lr[metricData.lr.length - 1])
|
const latestLearningRate = computed(() => lastFinite(metricData.lr))
|
||||||
const latestEpoch = computed(() => metricData.epoch[metricData.epoch.length - 1])
|
const latestEpoch = computed(() => lastFinite(metricData.epoch))
|
||||||
const logLineCount = computed(() => logContent.value ? logContent.value.split(/\r?\n/).length : 0)
|
const logLineCount = computed(() => logContent.value ? logContent.value.split(/\r?\n/).length : 0)
|
||||||
const taskGpuItems = computed<TaskGpuItem[]>(() => (task.value?.gpus ?? []).map((gpuId) => {
|
const taskGpuItems = computed<TaskGpuItem[]>(() => (task.value?.gpus ?? []).map((gpuId) => {
|
||||||
const index = Number(gpuId)
|
const index = Number(gpuId)
|
||||||
@@ -123,7 +125,7 @@ const gpuRefreshState = computed(() => {
|
|||||||
})
|
})
|
||||||
return gpuLoadError.value
|
return gpuLoadError.value
|
||||||
? `更新失败 · 最后更新 ${updateTime}`
|
? `更新失败 · 最后更新 ${updateTime}`
|
||||||
: `${updateTime} 更新 · 每 5 秒刷新`
|
: `${updateTime} 更新 · 每 3 秒刷新`
|
||||||
})
|
})
|
||||||
|
|
||||||
function formatMetric(value?: number, scientific = false) {
|
function formatMetric(value?: number, scientific = false) {
|
||||||
@@ -131,6 +133,13 @@ function formatMetric(value?: number, scientific = false) {
|
|||||||
return scientific ? value.toExponential(2) : value.toFixed(4).replace(/0+$/, '').replace(/\.$/, '')
|
return scientific ? value.toExponential(2) : value.toFixed(4).replace(/0+$/, '').replace(/\.$/, '')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function lastFinite(values: number[]) {
|
||||||
|
for (let index = values.length - 1; index >= 0; index -= 1) {
|
||||||
|
if (Number.isFinite(values[index])) return values[index]
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
function safePercent(value?: number) {
|
function safePercent(value?: number) {
|
||||||
return Math.round(Math.min(100, Math.max(0, Number(value || 0))))
|
return Math.round(Math.min(100, Math.max(0, Number(value || 0))))
|
||||||
}
|
}
|
||||||
@@ -216,6 +225,7 @@ const isLoraMethod = computed(() =>
|
|||||||
function applyLogContent(content: string) {
|
function applyLogContent(content: string) {
|
||||||
const parsed = parseTrainingLog(content)
|
const parsed = parseTrainingLog(content)
|
||||||
logContent.value = content
|
logContent.value = content
|
||||||
|
metricData.steps = parsed.metrics.steps
|
||||||
metricData.loss = parsed.metrics.loss
|
metricData.loss = parsed.metrics.loss
|
||||||
metricData.gradNorm = parsed.metrics.gradNorm
|
metricData.gradNorm = parsed.metrics.gradNorm
|
||||||
metricData.lr = parsed.metrics.lr
|
metricData.lr = parsed.metrics.lr
|
||||||
@@ -223,6 +233,26 @@ function applyLogContent(content: string) {
|
|||||||
Object.assign(summary, parsed.summary)
|
Object.assign(summary, parsed.summary)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function applyMetricData(metrics = { steps: [] as number[], loss: [] as number[], gradNorm: [] as number[], lr: [] as number[], epoch: [] as number[] }) {
|
||||||
|
metricData.steps = metrics.steps
|
||||||
|
metricData.loss = metrics.loss
|
||||||
|
metricData.gradNorm = metrics.gradNorm
|
||||||
|
metricData.lr = metrics.lr
|
||||||
|
metricData.epoch = metrics.epoch
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadMetrics(currentTask: FineTuneTask) {
|
||||||
|
try {
|
||||||
|
const points = await getFineTuneMetrics(currentTask.id)
|
||||||
|
const parsed = metricsFromApi(points || [])
|
||||||
|
if (parsed.loss.length || parsed.gradNorm.length || parsed.lr.length) {
|
||||||
|
applyMetricData(parsed)
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// 日志解析结果会作为兜底曲线数据。
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadLog(currentTask: FineTuneTask) {
|
async function loadLog(currentTask: FineTuneTask) {
|
||||||
try {
|
try {
|
||||||
const runtime = await getFineTuneLogs(currentTask.id, { tail_lines: 800 })
|
const runtime = await getFineTuneLogs(currentTask.id, { tail_lines: 800 })
|
||||||
@@ -277,6 +307,7 @@ async function refreshAll() {
|
|||||||
? loadDataset(currentTask.train_dataset_id)
|
? loadDataset(currentTask.train_dataset_id)
|
||||||
: Promise.resolve()
|
: Promise.resolve()
|
||||||
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
|
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
|
||||||
|
await loadMetrics(currentTask)
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
loading.value = false
|
||||||
refreshInFlight = false
|
refreshInFlight = false
|
||||||
@@ -523,7 +554,7 @@ onMounted(async () => {
|
|||||||
|
|
||||||
<!-- 训练曲线 -->
|
<!-- 训练曲线 -->
|
||||||
<PageCard class="metrics-panel" title="训练曲线" subtitle="持续监控模型收敛情况与学习率变化">
|
<PageCard class="metrics-panel" title="训练曲线" subtitle="持续监控模型收敛情况与学习率变化">
|
||||||
<template #extra><span class="refresh-state">每 5 秒刷新</span></template>
|
<template #extra><span class="refresh-state">每 3 秒刷新</span></template>
|
||||||
<div class="chart-list" aria-label="训练指标曲线">
|
<div class="chart-list" aria-label="训练指标曲线">
|
||||||
<section class="chart-section">
|
<section class="chart-section">
|
||||||
<div class="chart-section-header">
|
<div class="chart-section-header">
|
||||||
@@ -560,7 +591,7 @@ onMounted(async () => {
|
|||||||
|
|
||||||
<!-- 原始日志 -->
|
<!-- 原始日志 -->
|
||||||
<PageCard class="log-card" title="训练日志" subtitle="查看训练任务的原始运行输出">
|
<PageCard class="log-card" title="训练日志" subtitle="查看训练任务的原始运行输出">
|
||||||
<template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 5 秒刷新</span></template>
|
<template #extra><span class="log-meta">{{ logLineCount }} 行 · 每 3 秒刷新</span></template>
|
||||||
<pre class="log-pre">{{ logContent || '暂无日志' }}</pre>
|
<pre class="log-pre">{{ logContent || '暂无日志' }}</pre>
|
||||||
</PageCard>
|
</PageCard>
|
||||||
</template>
|
</template>
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import type { EChartsOption } from 'echarts'
|
import type { EChartsOption } from 'echarts'
|
||||||
import type { FineTuneTask, TrainingLogFile } from '@/types'
|
import type { FineTuneTask, TrainingLogFile } from '@/types'
|
||||||
|
import type { FineTuneMetricPoint } from '@/api/modules/fineTune'
|
||||||
|
|
||||||
export interface TrainingMetricData {
|
export interface TrainingMetricData {
|
||||||
|
steps: number[]
|
||||||
loss: number[]
|
loss: number[]
|
||||||
gradNorm: number[]
|
gradNorm: number[]
|
||||||
lr: number[]
|
lr: number[]
|
||||||
@@ -26,7 +28,7 @@ function escapeRegExp(value: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function extractNumber(source: string, key: string) {
|
function extractNumber(source: string, key: string) {
|
||||||
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*:\\s*(${NUMBER_SOURCE})`, 'i'))
|
const match = source.match(new RegExp(`['"]?${escapeRegExp(key)}['"]?\\s*(?:=|:)\\s*(${NUMBER_SOURCE})`, 'i'))
|
||||||
return match ? Number(match[1]) : undefined
|
return match ? Number(match[1]) : undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,24 +59,44 @@ export function resolveTrainingLogFile(
|
|||||||
|
|
||||||
/** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
|
/** 解析日志中的逐步指标。字段顺序和常见数值格式均不受限制。 */
|
||||||
export function parseTrainingMetrics(text: string): TrainingMetricData {
|
export function parseTrainingMetrics(text: string): TrainingMetricData {
|
||||||
const metrics: TrainingMetricData = { loss: [], gradNorm: [], lr: [], epoch: [] }
|
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||||
const blocks = text.match(/\{[^{}\r\n]*\}/g) || []
|
const candidates = text
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.flatMap((line) => {
|
||||||
|
const blocks = line.match(/\{[^{}\r\n]*\}/g)
|
||||||
|
return blocks?.length ? blocks.map((block) => `${line} ${block}`) : [line]
|
||||||
|
})
|
||||||
|
|
||||||
for (const block of blocks) {
|
for (const [index, line] of candidates.entries()) {
|
||||||
const loss = extractNumber(block, 'loss')
|
const loss = extractNumber(line, 'loss')
|
||||||
const gradNorm = extractNumber(block, 'grad_norm')
|
const gradNorm = extractNumber(line, 'grad_norm')
|
||||||
const learningRate = extractNumber(block, 'learning_rate')
|
const learningRate = extractNumber(line, 'learning_rate')
|
||||||
const epoch = extractNumber(block, 'epoch')
|
const epoch = extractNumber(line, 'epoch')
|
||||||
if (loss == null || gradNorm == null || learningRate == null) continue
|
if (loss == null && gradNorm == null && learningRate == null) continue
|
||||||
metrics.loss.push(loss)
|
metrics.steps.push(extractNumber(line, 'step') ?? metrics.steps.length + index + 1)
|
||||||
metrics.gradNorm.push(gradNorm)
|
metrics.loss.push(loss ?? Number.NaN)
|
||||||
metrics.lr.push(learningRate)
|
metrics.gradNorm.push(gradNorm ?? Number.NaN)
|
||||||
if (epoch != null) metrics.epoch.push(epoch)
|
metrics.lr.push(learningRate ?? Number.NaN)
|
||||||
|
metrics.epoch.push(epoch ?? Number.NaN)
|
||||||
}
|
}
|
||||||
|
|
||||||
return metrics
|
return metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function metricsFromApi(points: FineTuneMetricPoint[]): TrainingMetricData {
|
||||||
|
const metrics: TrainingMetricData = { steps: [], loss: [], gradNorm: [], lr: [], epoch: [] }
|
||||||
|
for (const [index, point] of points.entries()) {
|
||||||
|
const hasMetric = point.loss != null || point.grad_norm != null || point.learning_rate != null
|
||||||
|
if (!hasMetric) continue
|
||||||
|
metrics.steps.push(Number(point.step || index + 1))
|
||||||
|
metrics.loss.push(point.loss == null ? Number.NaN : Number(point.loss))
|
||||||
|
metrics.gradNorm.push(point.grad_norm == null ? Number.NaN : Number(point.grad_norm))
|
||||||
|
metrics.lr.push(point.learning_rate == null ? Number.NaN : Number(point.learning_rate))
|
||||||
|
metrics.epoch.push(point.epoch == null ? Number.NaN : Number(point.epoch))
|
||||||
|
}
|
||||||
|
return metrics
|
||||||
|
}
|
||||||
|
|
||||||
/** 每次都返回新对象,日志截断或切换时不会残留上一轮汇总。 */
|
/** 每次都返回新对象,日志截断或切换时不会残留上一轮汇总。 */
|
||||||
export function parseTrainingSummary(text: string): TrainingSummary {
|
export function parseTrainingSummary(text: string): TrainingSummary {
|
||||||
const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
|
const emptySummary: TrainingSummary = { epoch: '', trainLoss: '', runtime: '' }
|
||||||
@@ -102,11 +124,23 @@ export function parseTrainingLog(text: string): ParsedTrainingLog {
|
|||||||
export function buildMetricChartOption(
|
export function buildMetricChartOption(
|
||||||
label: string,
|
label: string,
|
||||||
data: number[],
|
data: number[],
|
||||||
|
steps: number[],
|
||||||
color: string,
|
color: string,
|
||||||
logScale = false,
|
logScale = false,
|
||||||
): EChartsOption {
|
): EChartsOption {
|
||||||
|
const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
|
||||||
return {
|
return {
|
||||||
grid: { top: 24, right: 20, bottom: 56, left: 56 },
|
grid: { top: 24, right: 20, bottom: 56, left: 56 },
|
||||||
|
graphic: visibleData.some((value) => value != null)
|
||||||
|
? []
|
||||||
|
: [
|
||||||
|
{
|
||||||
|
type: 'text',
|
||||||
|
left: 'center',
|
||||||
|
top: 'middle',
|
||||||
|
style: { text: '暂无训练指标数据', fill: '#94a3b8', fontSize: 13 },
|
||||||
|
},
|
||||||
|
],
|
||||||
tooltip: {
|
tooltip: {
|
||||||
trigger: 'axis',
|
trigger: 'axis',
|
||||||
axisPointer: { type: 'cross' },
|
axisPointer: { type: 'cross' },
|
||||||
@@ -116,6 +150,7 @@ export function buildMetricChartOption(
|
|||||||
},
|
},
|
||||||
xAxis: {
|
xAxis: {
|
||||||
type: 'category',
|
type: 'category',
|
||||||
|
data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
|
||||||
boundaryGap: false,
|
boundaryGap: false,
|
||||||
name: 'Step',
|
name: 'Step',
|
||||||
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
|
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
|
||||||
@@ -142,7 +177,7 @@ export function buildMetricChartOption(
|
|||||||
{
|
{
|
||||||
name: label,
|
name: label,
|
||||||
type: 'line',
|
type: 'line',
|
||||||
data,
|
data: visibleData,
|
||||||
smooth: true,
|
smooth: true,
|
||||||
symbol: 'none',
|
symbol: 'none',
|
||||||
lineStyle: { width: 2, color },
|
lineStyle: { width: 2, color },
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ function formatQuota(quota: Record<string, unknown> | undefined | null) {
|
|||||||
if (q.gpu > 0) parts.push(`GPU ${q.gpu}`)
|
if (q.gpu > 0) parts.push(`GPU ${q.gpu}`)
|
||||||
if (q.storage > 0) parts.push(`存储 ${q.storage}GB`)
|
if (q.storage > 0) parts.push(`存储 ${q.storage}GB`)
|
||||||
if (q.maxProjects > 0) parts.push(`项目 ${q.maxProjects}`)
|
if (q.maxProjects > 0) parts.push(`项目 ${q.maxProjects}`)
|
||||||
return parts.length ? parts.join(' | ') : '—'
|
return parts.length ? parts.join(' | ') : '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
async function load() {
|
async function load() {
|
||||||
@@ -52,7 +52,7 @@ function asTenant(row: unknown): Tenant {
|
|||||||
|
|
||||||
function quotaText(row: unknown): string {
|
function quotaText(row: unknown): string {
|
||||||
const quota = asTenant(row).quota || {}
|
const quota = asTenant(row).quota || {}
|
||||||
return Object.keys(quota).length ? JSON.stringify(quota) : '—'
|
return Object.keys(quota).length ? JSON.stringify(quota) : '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
async function submitCreate() {
|
async function submitCreate() {
|
||||||
@@ -118,13 +118,13 @@ onMounted(load)
|
|||||||
</template>
|
</template>
|
||||||
<template #columns>
|
<template #columns>
|
||||||
<el-table-column prop="name" label="租户名称" min-width="140" />
|
<el-table-column prop="name" label="租户名称" min-width="140" />
|
||||||
<el-table-column prop="code" label="用户ID" min-width="100" />
|
<el-table-column prop="code" label="租户 ID" min-width="100" />
|
||||||
<el-table-column prop="status" label="状态" min-width="100" />
|
<el-table-column prop="status" label="状态" min-width="100" />
|
||||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||||
</template>
|
</template>
|
||||||
<template #actions="{ row }">
|
<template #actions="{ row }">
|
||||||
<el-button link type="primary" @click="openDetail(row.id)">详情</el-button>
|
<el-button link type="primary" @click="openDetail(row.id)">详情</el-button>
|
||||||
<el-button link type="danger" @click="handleDelete(row)">删除</el-button>
|
<el-button link type="danger" @click="handleDelete(asTenant(row))">删除</el-button>
|
||||||
</template>
|
</template>
|
||||||
</DataTablePage>
|
</DataTablePage>
|
||||||
|
|
||||||
|
|||||||
353
测试脚本.md
Normal file
353
测试脚本.md
Normal file
@@ -0,0 +1,353 @@
|
|||||||
|
# YG_FT 模型微调平台测试脚本
|
||||||
|
|
||||||
|
## 环境准备
|
||||||
|
|
||||||
|
### 启动顺序
|
||||||
|
|
||||||
|
1. **数据库**:远程 PostgreSQL `www.caoxiaozhu.com:5432`(已就绪)
|
||||||
|
2. **后端**:
|
||||||
|
```cmd
|
||||||
|
cd /d E:\yg_ft\backend
|
||||||
|
set COMPUTE_STATUS_SYNC_MODE=off
|
||||||
|
.\.venv\Scripts\python.exe -m uvicorn app.main:app --reload --port 17861
|
||||||
|
```
|
||||||
|
3. **算力服务**(Docker):
|
||||||
|
```cmd
|
||||||
|
cd /d E:\yg_ft\docker\compute
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
4. **前端**:
|
||||||
|
```cmd
|
||||||
|
cd /d E:\yg_ft\frontend
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
### 验证服务可用性
|
||||||
|
|
||||||
|
| 服务 | 验证命令 | 期望结果 |
|
||||||
|
|------|---------|---------|
|
||||||
|
| 后端 | `curl http://localhost:17861/modelTF/health` | `{"code":0,...,"data":{"cpu_percent":x,...}}` |
|
||||||
|
| 算力 | `curl http://localhost:19100/health` | `{"status":"ok","compute_host_id":"gpu-node-01"}` |
|
||||||
|
| 前端 | 浏览器打开 `http://localhost:16801` | 跳转登录页 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、登录与认证
|
||||||
|
|
||||||
|
### 1.1 登录
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 打开 `http://localhost:16801` | 显示登录页,不自动登录 |
|
||||||
|
| 2 | 输入账号 `admin`,密码 `admin123`,点击登录 | 提示"登录成功",跳转看板页 |
|
||||||
|
| 3 | F12 控制台执行 `localStorage.getItem('sessionId')` | 有值(非 null) |
|
||||||
|
| 4 | 浏览器 Network 查看 `/modelTF/login` 响应 | 返回 `token`、`user`、`session_id` |
|
||||||
|
|
||||||
|
### 1.2 退出登录
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击侧边栏底部"退出登录"按钮 | 跳转登录页 |
|
||||||
|
| 2 | F12 控制台执行 `localStorage.getItem('currentUser')` | 返回 null |
|
||||||
|
| 3 | 浏览器 Network 查看 `/modelTF/logout` 请求 | 状态 200 |
|
||||||
|
|
||||||
|
### 1.3 会话超时
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 登录后切到其他软件,等待 5 分钟以上 | — |
|
||||||
|
| 2 | 切回浏览器 | 提示"登录已过期",跳转登录页 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、看板
|
||||||
|
|
||||||
|
### 2.1 服务状态
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 登录后进入看板页 | 显示 7 个服务:模型训练/模型评测/模型推理/模型管理/数据集管理/数据处理/数据类型转换 |
|
||||||
|
| 2 | 检查每个服务状态 | 全部显示"正常" |
|
||||||
|
| 3 | 检查实例数 | 模型管理有数量、数据集管理有数量等 |
|
||||||
|
|
||||||
|
### 2.2 运行中任务
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 查看顶部"运行中任务"数字 | 显示训练+评测+数据处理中 running 状态的任务总数 |
|
||||||
|
| 2 | 无运行任务时 | 显示 0 |
|
||||||
|
|
||||||
|
### 2.3 用户操作分布
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 查看饼图 | 4 个分类(数据处理/模型训练/模型评测/模型推理)各一色 |
|
||||||
|
| 2 | 无数据时 | 4 等分(各 25%) |
|
||||||
|
| 3 | 点击侧边栏"模型训练"菜单,回到看板 | 模型训练权重增加 |
|
||||||
|
| 4 | 点击"模型推理"菜单,回到看板 | 模型推理权重增加 |
|
||||||
|
| 5 | 浏览器控制台执行 `sessionStorage.clear()` 后重试 | 去重缓存清除,可再次记录 |
|
||||||
|
|
||||||
|
### 2.4 登录时长排行
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 查看柱状图 | 显示当前登录用户,时长 > 0 |
|
||||||
|
| 2 | 条形图上无文字标签 | 只在鼠标悬浮时显示 tooltip |
|
||||||
|
| 3 | Y 轴用户名清晰可读 | 字体颜色 `#1f2937`,非模糊 |
|
||||||
|
|
||||||
|
### 2.5 顶部状态栏
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 查看顶部 CPU/内存/磁盘 | 显示真实数值(非 0) |
|
||||||
|
| 2 | 等待 30 秒 | 数值自动刷新 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 三、租户管理
|
||||||
|
|
||||||
|
### 3.1 列表
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 平台治理 → 租户管理 | 显示列表(名称/用户ID/状态/创建时间) |
|
||||||
|
| 2 | 检查列名 | "编码"已改为"用户ID" |
|
||||||
|
| 3 | 检查操作列 | 有"详情"和"删除"按钮,无"配额"按钮 |
|
||||||
|
|
||||||
|
### 3.2 新建租户
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"新建租户" | 弹窗显示:名称、用户ID、GPU数量、存储配额、最大项目数 |
|
||||||
|
| 2 | 填写名称"测试租户",GPU=2,存储=100,项目=5,点击创建 | 提示"租户创建成功",列表新增一行 |
|
||||||
|
|
||||||
|
### 3.3 租户详情与配额
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"详情" | 跳转详情页,显示基本信息 + 配额 |
|
||||||
|
| 2 | 配额显示格式 | `GPU 2 | 存储 100GB | 项目 5` |
|
||||||
|
| 3 | 修改配额数字,点击"保存配额" | 提示"配额已保存" |
|
||||||
|
| 4 | 检查详情页无"项目空间"卡片 | 已移除 |
|
||||||
|
|
||||||
|
### 3.4 删除租户
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 在列表页点击"删除" | 弹出确认框 |
|
||||||
|
| 2 | 点击"确定删除" | 提示"租户已删除",列表刷新 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 四、项目空间
|
||||||
|
|
||||||
|
### 4.1 列表
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 项目空间 | 显示列表(项目名/编码ID/状态/描述/创建时间) |
|
||||||
|
| 2 | 检查列名 | "编码"已改为"编码ID" |
|
||||||
|
| 3 | 顶部有租户筛选下拉框 | 可按租户过滤 |
|
||||||
|
|
||||||
|
### 4.2 新建项目
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"新建项目" | 弹窗显示:名称、编码ID(下拉)、描述 |
|
||||||
|
| 2 | 编码ID下拉选项 | 显示已有租户的编码 |
|
||||||
|
| 3 | 选择编码ID后提交 | 提示"项目创建成功" |
|
||||||
|
|
||||||
|
### 4.3 项目详情
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"详情" | 显示基本信息 |
|
||||||
|
| 2 | 查看成员管理区域 | 显示成员列表 + 添加成员按钮 |
|
||||||
|
| 3 | 点击"资源授权(ACL)" | 弹窗显示 ACL 编辑器 |
|
||||||
|
|
||||||
|
### 4.4 ACL 授权
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"添加授权项" | 新增一行 |
|
||||||
|
| 2 | 主体类型选"用户" | 显示用户下拉框(非文本输入) |
|
||||||
|
| 3 | 主体类型选"项目角色" | 显示角色下拉框(member/admin/viewer) |
|
||||||
|
| 4 | 勾选权限后保存 | 提示"ACL 已保存" |
|
||||||
|
|
||||||
|
### 4.5 删除项目
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"删除" | 确认弹窗 |
|
||||||
|
| 2 | 确认删除 | 提示"项目已删除" |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 五、算力节点
|
||||||
|
|
||||||
|
### 5.1 节点列表
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 算力节点 | 显示节点列表 |
|
||||||
|
| 2 | 检查 gpu-node-01 | 状态为启用 |
|
||||||
|
|
||||||
|
### 5.2 测试连接
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击 gpu-node-01 的"测试"按钮 | 提示"连接成功,发现 X 张 GPU,延迟 Xms" |
|
||||||
|
| 2 | 检查 GPU Tab | 显示 GPU 资源列表 |
|
||||||
|
|
||||||
|
### 5.3 节点启停
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"停用" | 节点状态变为停用 |
|
||||||
|
| 2 | 点击"启用" | 节点状态变为启用 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 六、用户设置
|
||||||
|
|
||||||
|
### 6.1 用户列表
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 系统设置 → 用户设置 | 显示用户列表 |
|
||||||
|
| 2 | admin 用户 | 不可停用、不可删除 |
|
||||||
|
|
||||||
|
### 6.2 创建用户
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"创建用户" | 跳转创建页 |
|
||||||
|
| 2 | 填写账号/显示名/密码/角色/权限 | 表单正常 |
|
||||||
|
| 3 | 提交 | 提示创建成功,列表新增 |
|
||||||
|
|
||||||
|
### 6.3 权限分配
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击某用户"页面权限" | 弹窗显示 12 个权限 checkbox |
|
||||||
|
| 2 | 取消某权限,保存 | 该用户侧边栏不再显示对应菜单 |
|
||||||
|
|
||||||
|
### 6.4 重置密码
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 点击"重置密码" | 弹窗显示默认密码 `Platform@123` |
|
||||||
|
| 2 | 确认 | 提示重置成功 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 七、审批管理
|
||||||
|
|
||||||
|
### 7.1 审批模板
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 审批模板 | 显示模板列表 |
|
||||||
|
| 2 | 点击新建 | 弹窗显示名称 + 步骤 JSON |
|
||||||
|
| 3 | 填写并提交 | 列表新增 |
|
||||||
|
|
||||||
|
### 7.2 审批中心
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 审批中心 | 显示实例列表 |
|
||||||
|
| 2 | 按状态筛选 | 列表过滤正常 |
|
||||||
|
| 3 | 对 pending 实例点"通过/拒绝" | 弹窗填写意见,提交后状态变更 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 八、审计日志
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 审计日志 | 显示日志列表 |
|
||||||
|
| 2 | 填写筛选条件查询 | 列表过滤 |
|
||||||
|
| 3 | 点击"导出" | 下载 CSV 文件 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 九、平台性能
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 平台性能 | 显示 CPU/内存/磁盘/GPU 卡片 |
|
||||||
|
| 2 | 数据非 0 | 与本机实际使用率一致 |
|
||||||
|
| 3 | 开启自动刷新(3秒) | 趋势图实时更新 |
|
||||||
|
| 4 | 点击 GPU 卡片 | 弹出详情抽屉 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 十、查看日志
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 侧边栏 → 查看日志 | 显示系统日志 Tab |
|
||||||
|
| 2 | 选择日期和文件 | 显示日志内容 |
|
||||||
|
| 3 | 输入关键词搜索 | 高亮匹配 |
|
||||||
|
| 4 | 切换到训练日志 Tab | 显示训练日志列表 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 十一、数据流主链路
|
||||||
|
|
||||||
|
### 11.1 数据集管理
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 进入数据集管理 | 显示列表 |
|
||||||
|
| 2 | 新建数据集 | 创建成功 |
|
||||||
|
| 3 | 上传文件 | 文件上传成功 |
|
||||||
|
| 4 | 预览文件 | 显示内容 |
|
||||||
|
|
||||||
|
### 11.2 数据处理
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 进入数据处理 | 显示任务列表 |
|
||||||
|
| 2 | 创建数据处理任务 | 创建成功 |
|
||||||
|
| 3 | 启动任务 | 状态变为 running |
|
||||||
|
| 4 | 回到看板 | "运行中任务"数字 +1 |
|
||||||
|
|
||||||
|
### 11.3 模型训练
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 进入模型训练 | 显示训练任务列表 |
|
||||||
|
| 2 | 创建训练任务 | 选择数据集 + 模型 + GPU |
|
||||||
|
| 3 | 启动训练 | 状态变为 running |
|
||||||
|
| 4 | 查看训练日志 | 显示实时日志 |
|
||||||
|
| 5 | 训练完成 | 产出模型 |
|
||||||
|
|
||||||
|
### 11.4 模型评测
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 进入模型评测 | 显示评测任务列表 |
|
||||||
|
| 2 | 创建评测任务 | 选择模型 + 数据集 |
|
||||||
|
| 3 | 启动评测 | 状态变为 running |
|
||||||
|
| 4 | 评测完成 | 显示指标结果 |
|
||||||
|
|
||||||
|
### 11.5 模型推理
|
||||||
|
|
||||||
|
| 步骤 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 1 | 进入模型推理 | 显示推理会话列表 |
|
||||||
|
| 2 | 创建推理任务 | 选择模型 + 加载 |
|
||||||
|
| 3 | 开始对话 | 返回模型回复 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 十二、异常场景
|
||||||
|
|
||||||
|
| 场景 | 操作 | 预期结果 |
|
||||||
|
|------|------|---------|
|
||||||
|
| 数据库断开 | 停止远程数据库连接 | 所有接口返回 500 |
|
||||||
|
| 算力服务停掉 | `docker compose down` | 算力测试返回失败 |
|
||||||
|
| 未登录访问 | 直接访问 `/dashboard` | 跳转登录页 |
|
||||||
|
| 权限不足 | 普通用户访问用户设置 | 跳转 403 页面 |
|
||||||
|
| Token 过期 | 清掉 localStorage 后刷新 | 跳转登录页 |
|
||||||
Reference in New Issue
Block a user