update
This commit is contained in:
@@ -16,13 +16,25 @@ class LoginBody(BaseModel):
|
||||
password: str
|
||||
|
||||
|
||||
class LogoutBody(BaseModel):
|
||||
session_id: str | None = None
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
def login(body: LoginBody) -> dict:
|
||||
user = get_platform_store().login(body.username, body.password)
|
||||
store = get_platform_store()
|
||||
user = store.login(body.username, body.password)
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
|
||||
token = create_access_token(user["id"])
|
||||
return {"code": 0, "message": "ok", "data": {"token": token, "user": user}}
|
||||
session_id = store.create_session(user)
|
||||
return {"code": 0, "message": "ok", "data": {"token": token, "user": user, "session_id": session_id}}
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
def logout(body: LogoutBody) -> dict:
|
||||
get_platform_store().close_session(body.session_id)
|
||||
return {"code": 0, "message": "ok", "data": None}
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
"""Application-side compute platform gateway module."""
|
||||
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
|
||||
__all__ = ["ComputeNodeClient", "poll_compute_jobs_once"]
|
||||
|
||||
180
backend/app/modules/compute_gateway/client.py
Normal file
180
backend/app/modules/compute_gateway/client.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""Client for talking to a compute node's REST API.
|
||||
|
||||
This is the application-side bridge: the application backend never runs GPU
|
||||
workloads itself; it dispatches them to a compute node and reads back status,
|
||||
logs and metrics through this client.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
from app.core.config import get_settings
|
||||
|
||||
|
||||
def _compute_timeout() -> float:
|
||||
return float(get_settings().compute_request_timeout_seconds or 30.0)
|
||||
|
||||
|
||||
def _auth_headers() -> dict:
|
||||
token = get_settings().compute_service_token
|
||||
if not token:
|
||||
return {}
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
class ComputeNodeClient:
|
||||
"""Thin wrapper over a single compute node's HTTP API."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float | None = None):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout or _compute_timeout()
|
||||
|
||||
# ---- health -------------------------------------------------------
|
||||
def health(self) -> dict:
|
||||
last_err = None
|
||||
for path in ("/v1/compute/health", "/health"):
|
||||
try:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(f"{self.base_url}{path}", headers=_auth_headers())
|
||||
if r.status_code == 200:
|
||||
return r.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_err = str(exc)
|
||||
raise RuntimeError(f"compute node unhealthy: {last_err}")
|
||||
|
||||
# ---- gpus ---------------------------------------------------------
|
||||
def gpus(self) -> list:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/resources/gpus",
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json().get("data", [])
|
||||
|
||||
# ---- jobs ---------------------------------------------------------
|
||||
def create_job(self, payload: dict) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs",
|
||||
json=payload,
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def preview_job(self, payload: dict) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs/preview",
|
||||
json=payload,
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def validate_job(self, payload: dict) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs/validate",
|
||||
json=payload,
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def get_job(self, job_id: str) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/jobs/{job_id}",
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def stop_job(self, job_id: str) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/jobs/{job_id}/stop",
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def job_logs(self, job_id: str, cursor: int = 0, limit: int = 200) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/jobs/{job_id}/logs",
|
||||
params={"cursor": cursor, "limit": limit},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
# ---- files --------------------------------------------------------
|
||||
def check_paths(self, paths: list) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/files/check-paths",
|
||||
json={"paths": paths},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def list_files(self, path: str = "/") -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.get(
|
||||
f"{self.base_url}/compute/files/list",
|
||||
params={"path": path},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def import_local_file(self, src_path: str, dest_name: str | None = None) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/files/import-local",
|
||||
json={"src_path": src_path, "dest_name": dest_name},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def upload_file(self, filename: str, content: bytes, content_type: str | None = None) -> dict:
|
||||
with httpx.Client(timeout=self.timeout, verify=False) as c:
|
||||
r = c.post(
|
||||
f"{self.base_url}/compute/files/upload",
|
||||
files={"file": (filename, content, content_type)},
|
||||
headers=_auth_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
# ---- inference (async) -------------------------------------------
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
"""Return auth headers for compute node requests."""
|
||||
token = get_settings().compute_service_token
|
||||
if not token:
|
||||
return {}
|
||||
return {"X-Compute-Token": token}
|
||||
|
||||
@property
|
||||
def route_prefix(self) -> str:
|
||||
return get_settings().route_prefix.rstrip("/") or "/modelTF"
|
||||
|
||||
async def _request(self, method: str, path: str, json_data: dict | None = None) -> dict:
|
||||
"""Generic async request method for compute API endpoints."""
|
||||
prefix = self.route_prefix
|
||||
url = f"{self.base_url.rstrip('/')}{prefix}{path}"
|
||||
async with httpx.AsyncClient(timeout=300, verify=False, headers=self.headers()) as client:
|
||||
if method.upper() == "GET":
|
||||
response = await client.get(url)
|
||||
else:
|
||||
response = await client.post(url, json=json_data)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if isinstance(data, dict) and isinstance(data.get("data"), dict):
|
||||
return data["data"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
50
backend/app/modules/compute_gateway/sync.py
Normal file
50
backend/app/modules/compute_gateway/sync.py
Normal file
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
synced: list[dict[str, Any]] = []
|
||||
failed: list[dict[str, str]] = []
|
||||
for task in store.running_compute_tasks():
|
||||
node = _node_for_task(task)
|
||||
if not node:
|
||||
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
job = client.get_job(task["compute_job_id"])
|
||||
try:
|
||||
logs = client.job_logs(task["compute_job_id"], tail_lines=5000)
|
||||
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
|
||||
except Exception:
|
||||
pass
|
||||
if job.get("status") in {"failed", "stopped"}:
|
||||
try:
|
||||
last_logs = client.job_logs(task["compute_job_id"], tail_lines=200)
|
||||
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||
except Exception:
|
||||
pass
|
||||
synced.append(store.apply_compute_job(task["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||
standalone_synced: list[dict[str, Any]] = []
|
||||
for record in store.active_standalone_compute_jobs():
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
|
||||
if not node:
|
||||
failed.append({"job_id": record["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
job = ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
||||
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
|
||||
@@ -3,13 +3,17 @@
|
||||
|
||||
- preset 参数预设(quick / standard / high)
|
||||
- train_type → stage 映射(sft/dpo/cpt/cot)
|
||||
- 训练任务的启动 / 暂停 / 恢复 / 取消(委托 runner 真实执行)
|
||||
- 训练任务的启动 / 暂停 / 恢复 / 取消
|
||||
- compute_gateway 状态轮询线程(当任务派发到算力时自动同步状态)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.fine_tune import runner
|
||||
|
||||
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
|
||||
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
|
||||
}
|
||||
|
||||
# ── compute sync 轮询线程 ──────────────────────────────────────────────
|
||||
_sync_thread: threading.Thread | None = None
|
||||
_sync_thread_stop = threading.Event()
|
||||
|
||||
|
||||
def _compute_sync_loop() -> None:
|
||||
"""后台线程:周期性轮询算力节点,同步训练任务状态/日志/指标。"""
|
||||
interval = get_settings().compute_poll_interval_seconds or 3
|
||||
while not _sync_thread_stop.is_set():
|
||||
try:
|
||||
store = get_platform_store()
|
||||
running = store.running_compute_tasks()
|
||||
if running:
|
||||
asyncio.run(_poll_once())
|
||||
except Exception: # noqa: BLE001 - keep polling loop alive
|
||||
pass
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
async def _poll_once() -> None:
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
await poll_compute_jobs_once()
|
||||
|
||||
|
||||
def start_compute_sync_worker() -> None:
|
||||
"""启动后台轮询线程(幂等,多次调用安全)。"""
|
||||
global _sync_thread
|
||||
if _sync_thread is not None and _sync_thread.is_alive():
|
||||
return
|
||||
_sync_thread_stop.clear()
|
||||
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
|
||||
_sync_thread.start()
|
||||
|
||||
|
||||
def stop_compute_sync_worker() -> None:
|
||||
"""停止后台轮询线程。"""
|
||||
_sync_thread_stop.set()
|
||||
|
||||
|
||||
# ── preset / config ────────────────────────────────────────────────────
|
||||
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""根据 preset 字段补全缺失的超参;preset=custom 时不覆盖。"""
|
||||
payload = dict(payload)
|
||||
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
|
||||
def launch_training(task_id: str) -> None:
|
||||
"""在后台线程启动真实训练。"""
|
||||
"""在后台线程启动真实训练(本机 subprocess fallback,当算力节点不可用时使用)。
|
||||
|
||||
架构原则:GPU 计算应派发到算力服务进程执行。
|
||||
当 platform_store.start_task 检测到在线算力节点时,会走 _dispatch_to_compute 派发路径;
|
||||
仅当无可用算力节点且非 simulator 模式时,降级到本机 runner(违反 §1.1,待移除)。
|
||||
"""
|
||||
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user