feat: 新增 compute_gateway、compute_poller、agent 模块，重构前端 dist
- 新增 backend/app/modules/compute_gateway（client/sync）计算网关模块 - 新增 backend/app/workers/compute_poller 计算轮询 worker - 新增 compute/agent/process_manager 进程管理器 - 新增 scripts/ 脚本目录 - 更新 Docker 部署配置（app/compute/nginx） - 更新后端平台 API、数据库 SQL、core 配置 - 更新前端多个视图组件及 API 模块 - 重构 frontend/dist 构建产物（新 hash） - 更新多项文档 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
266
scripts/verify_platform.py
Normal file
266
scripts/verify_platform.py
Normal file
@@ -0,0 +1,266 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
def request(method: str, url: str, payload: dict[str, Any] | None = None, timeout: int = 20) -> tuple[int, Any]:
|
||||
data = None
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
try:
|
||||
body: Any = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
body = raw
|
||||
return resp.status, body
|
||||
|
||||
|
||||
def check(name: str, fn) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
data = fn()
|
||||
return {
|
||||
"name": name,
|
||||
"ok": True,
|
||||
"duration_ms": int((time.perf_counter() - started) * 1000),
|
||||
"data": data,
|
||||
}
|
||||
except Exception as exc: # noqa: BLE001 - verification should report every failure shape.
|
||||
error = str(exc)
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
error = f"HTTP Error {exc.code}: {body or exc.reason}"
|
||||
return {
|
||||
"name": name,
|
||||
"ok": False,
|
||||
"duration_ms": int((time.perf_counter() - started) * 1000),
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
def unwrap(payload: Any) -> Any:
|
||||
if isinstance(payload, dict) and payload.get("code") == 0:
|
||||
return payload.get("data")
|
||||
return payload
|
||||
|
||||
|
||||
def post_json(url: str, payload: dict[str, Any], timeout: int = 20) -> Any:
|
||||
return unwrap(request("POST", url, payload, timeout=timeout)[1])
|
||||
|
||||
|
||||
def get_json(url: str, timeout: int = 20) -> Any:
|
||||
return unwrap(request("GET", url, timeout=timeout)[1])
|
||||
|
||||
|
||||
def run_training_smoke(backend_url: str) -> dict[str, Any]:
|
||||
suffix = str(int(time.time()))
|
||||
nodes = get_json(f"{backend_url}/compute/nodes", timeout=10) or []
|
||||
if not nodes:
|
||||
raise RuntimeError("no compute node found")
|
||||
node = next((item for item in nodes if item.get("enabled") and item.get("scheduler_status") == "online"), nodes[0])
|
||||
gpus = get_json(f"{backend_url}/compute/gpus", timeout=10) or []
|
||||
gpu_id = int((gpus[0] or {}).get("id", 0)) if gpus else 0
|
||||
|
||||
model = post_json(
|
||||
f"{backend_url}/model-manage",
|
||||
{
|
||||
"name": f"smoke-base-{suffix}",
|
||||
"type": "LLM",
|
||||
"purpose": "training",
|
||||
"model_source": "local",
|
||||
"path": "/data/yg-ft/models/smoke-base",
|
||||
"description": "Smoke verification model path.",
|
||||
},
|
||||
timeout=10,
|
||||
)
|
||||
dataset_id = post_json(
|
||||
f"{backend_url}/dataset-manage",
|
||||
{
|
||||
"name": f"smoke-dataset-{suffix}",
|
||||
"type": "train",
|
||||
"storage_type": "local",
|
||||
"source": "smoke",
|
||||
"count": 2,
|
||||
"description": "Smoke verification dataset.",
|
||||
},
|
||||
timeout=10,
|
||||
)["id"]
|
||||
task_id = post_json(
|
||||
f"{backend_url}/fine-tune",
|
||||
{
|
||||
"name": f"smoke-train-{suffix}",
|
||||
"description": "Automated smoke fine-tune job.",
|
||||
"engine": "smoke",
|
||||
"train_type": "SFT",
|
||||
"train_method": "lora",
|
||||
"template": "qwen",
|
||||
"base_model": model["id"],
|
||||
"train_dataset_id": dataset_id,
|
||||
"requested_node_id": node["id"],
|
||||
"gpus": [gpu_id],
|
||||
"batch_size": 1,
|
||||
"learning_rate": 0.0002,
|
||||
"n_epochs": 1,
|
||||
"save_steps": 2,
|
||||
"quantization_bit": 0,
|
||||
},
|
||||
timeout=10,
|
||||
)["id"]
|
||||
command_preview = post_json(
|
||||
f"{backend_url}/fine-tune/{task_id}/command-preview",
|
||||
{"requested_node_id": node["id"], "gpus": [gpu_id]},
|
||||
timeout=30,
|
||||
)
|
||||
if not command_preview.get("preview", {}).get("command"):
|
||||
raise RuntimeError(f"command preview is empty: {command_preview}")
|
||||
preflight = post_json(
|
||||
f"{backend_url}/fine-tune/{task_id}/preflight",
|
||||
{"requested_node_id": node["id"], "gpus": [gpu_id]},
|
||||
timeout=30,
|
||||
)
|
||||
if not preflight.get("valid"):
|
||||
raise RuntimeError(f"fine-tune preflight failed: {preflight}")
|
||||
started = post_json(
|
||||
f"{backend_url}/fine-tune/start",
|
||||
{"task_id": task_id, "requested_node_id": node["id"], "gpus": [gpu_id]},
|
||||
timeout=30,
|
||||
)
|
||||
compute_job_id = started.get("compute_job_id")
|
||||
if not compute_job_id:
|
||||
raise RuntimeError(f"task did not create compute_job_id: {started}")
|
||||
|
||||
task = started
|
||||
poll_items: list[dict[str, Any]] = []
|
||||
deadline = time.time() + 45
|
||||
while time.time() < deadline:
|
||||
with_context = post_json(f"{backend_url}/internal/compute-sync/jobs/poll", {}, timeout=20)
|
||||
task = get_json(f"{backend_url}/fine-tune/{task_id}", timeout=10)
|
||||
poll_items.append(
|
||||
{
|
||||
"status": task.get("status"),
|
||||
"progress": task.get("progress"),
|
||||
"compute_job_id": task.get("compute_job_id"),
|
||||
"synced": with_context.get("synced") if isinstance(with_context, dict) else None,
|
||||
}
|
||||
)
|
||||
if task.get("status") in {"completed", "failed", "stopped"}:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
logs = get_json(f"{backend_url}/compute/jobs/{compute_job_id}/logs?tail_lines=80", timeout=20)
|
||||
if task.get("status") != "completed":
|
||||
raise RuntimeError({"task": task, "poll": poll_items[-8:], "logs": logs})
|
||||
if not logs.get("content"):
|
||||
raise RuntimeError("compute job logs are empty")
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"compute_job_id": compute_job_id,
|
||||
"node_id": node["id"],
|
||||
"gpu_id": gpu_id,
|
||||
"status": task.get("status"),
|
||||
"progress": task.get("progress"),
|
||||
"artifact_count": len(task.get("artifacts") or []),
|
||||
"log_lines": logs.get("total_lines"),
|
||||
"metric_count": len(logs.get("metrics") or []),
|
||||
"preflight_valid": preflight.get("valid"),
|
||||
"command": command_preview.get("preview", {}).get("command"),
|
||||
"poll": poll_items[-8:],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Verify YG Fine-Tune platform deployment.")
|
||||
parser.add_argument("--frontend-url", default="http://localhost:16801", help="Frontend base URL.")
|
||||
parser.add_argument("--backend-url", default="http://localhost:17861/modelTF", help="Backend API base URL with /modelTF.")
|
||||
parser.add_argument("--username", default="admin", help="Login username.")
|
||||
parser.add_argument("--password", default="admin123", help="Login password.")
|
||||
parser.add_argument("--skip-compute-test", action="store_true", help="Skip active compute node connection tests.")
|
||||
parser.add_argument("--run-training-smoke", action="store_true", help="Create and run a short smoke fine-tune job.")
|
||||
args = parser.parse_args()
|
||||
|
||||
frontend_url = args.frontend_url.rstrip("/")
|
||||
backend_url = args.backend_url.rstrip("/")
|
||||
report: list[dict[str, Any]] = []
|
||||
|
||||
report.append(
|
||||
check(
|
||||
"frontend index",
|
||||
lambda: {"status": request("GET", f"{frontend_url}/index.html", timeout=5)[0]},
|
||||
)
|
||||
)
|
||||
report.append(
|
||||
check(
|
||||
"backend health",
|
||||
lambda: unwrap(request("GET", f"{backend_url}/health", timeout=10)[1]),
|
||||
)
|
||||
)
|
||||
report.append(
|
||||
check(
|
||||
"admin login",
|
||||
lambda: unwrap(
|
||||
request(
|
||||
"POST",
|
||||
f"{backend_url}/login",
|
||||
{"username": args.username, "password": args.password},
|
||||
timeout=10,
|
||||
)[1]
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
nodes_result: dict[str, Any] = {"nodes": []}
|
||||
|
||||
def load_nodes() -> dict[str, Any]:
|
||||
nodes = unwrap(request("GET", f"{backend_url}/compute/nodes", timeout=10)[1]) or []
|
||||
nodes_result["nodes"] = nodes
|
||||
return {"count": len(nodes), "nodes": [{"id": n.get("id"), "code": n.get("code"), "api_base_url": n.get("api_base_url")} for n in nodes]}
|
||||
|
||||
report.append(check("compute nodes", load_nodes))
|
||||
|
||||
if not args.skip_compute_test:
|
||||
for node in nodes_result["nodes"]:
|
||||
node_id = node.get("id")
|
||||
node_code = node.get("code")
|
||||
if not node_id:
|
||||
continue
|
||||
report.append(
|
||||
check(
|
||||
f"compute node test {node_code}",
|
||||
lambda node_id=node_id: unwrap(
|
||||
request("POST", f"{backend_url}/compute/nodes/{node_id}/test-connection", timeout=30)[1]
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
report.append(
|
||||
check(
|
||||
"compute gpus",
|
||||
lambda: {"count": len(unwrap(request("GET", f"{backend_url}/compute/gpus", timeout=10)[1]) or [])},
|
||||
)
|
||||
)
|
||||
report.append(
|
||||
check(
|
||||
"compute queue",
|
||||
lambda: {"count": len(unwrap(request("GET", f"{backend_url}/compute/queue", timeout=10)[1]) or [])},
|
||||
)
|
||||
)
|
||||
|
||||
if args.run_training_smoke:
|
||||
report.append(check("fine-tune smoke job", lambda: run_training_smoke(backend_url)))
|
||||
|
||||
ok = all(item["ok"] for item in report)
|
||||
print(json.dumps({"ok": ok, "checks": report}, ensure_ascii=False, indent=2))
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user