This commit is contained in:
wangjiming
2026-07-31 16:10:34 +08:00
parent 945b4ace86
commit 242407b676
34 changed files with 3847 additions and 717 deletions

View File

@@ -16,13 +16,25 @@ class LoginBody(BaseModel):
password: str
class LogoutBody(BaseModel):
session_id: str | None = None
@router.post("/login")
def login(body: LoginBody) -> dict:
user = get_platform_store().login(body.username, body.password)
store = get_platform_store()
user = store.login(body.username, body.password)
if not user:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
token = create_access_token(user["id"])
return {"code": 0, "message": "ok", "data": {"token": token, "user": user}}
session_id = store.create_session(user)
return {"code": 0, "message": "ok", "data": {"token": token, "user": user, "session_id": session_id}}
@router.post("/logout")
def logout(body: LogoutBody) -> dict:
get_platform_store().close_session(body.session_id)
return {"code": 0, "message": "ok", "data": None}
@router.get("/me")

View File

@@ -1 +1,6 @@
"""Application-side compute platform gateway module."""
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import poll_compute_jobs_once
__all__ = ["ComputeNodeClient", "poll_compute_jobs_once"]

View File

@@ -0,0 +1,180 @@
"""Client for talking to a compute node's REST API.
This is the application-side bridge: the application backend never runs GPU
workloads itself; it dispatches them to a compute node and reads back status,
logs and metrics through this client.
"""
from __future__ import annotations
import httpx
from app.core.config import get_settings
def _compute_timeout() -> float:
return float(get_settings().compute_request_timeout_seconds or 30.0)
def _auth_headers() -> dict:
token = get_settings().compute_service_token
if not token:
return {}
return {"Authorization": f"Bearer {token}"}
class ComputeNodeClient:
"""Thin wrapper over a single compute node's HTTP API."""
def __init__(self, base_url: str, timeout: float | None = None):
self.base_url = base_url.rstrip("/")
self.timeout = timeout or _compute_timeout()
# ---- health -------------------------------------------------------
def health(self) -> dict:
last_err = None
for path in ("/v1/compute/health", "/health"):
try:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(f"{self.base_url}{path}", headers=_auth_headers())
if r.status_code == 200:
return r.json()
except Exception as exc: # noqa: BLE001
last_err = str(exc)
raise RuntimeError(f"compute node unhealthy: {last_err}")
# ---- gpus ---------------------------------------------------------
def gpus(self) -> list:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/resources/gpus",
headers=_auth_headers(),
)
r.raise_for_status()
return r.json().get("data", [])
# ---- jobs ---------------------------------------------------------
def create_job(self, payload: dict) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs",
json=payload,
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def preview_job(self, payload: dict) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs/preview",
json=payload,
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def validate_job(self, payload: dict) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs/validate",
json=payload,
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def get_job(self, job_id: str) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/jobs/{job_id}",
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def stop_job(self, job_id: str) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/jobs/{job_id}/stop",
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def job_logs(self, job_id: str, cursor: int = 0, limit: int = 200) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/jobs/{job_id}/logs",
params={"cursor": cursor, "limit": limit},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
# ---- files --------------------------------------------------------
def check_paths(self, paths: list) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/files/check-paths",
json={"paths": paths},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def list_files(self, path: str = "/") -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.get(
f"{self.base_url}/compute/files/list",
params={"path": path},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def import_local_file(self, src_path: str, dest_name: str | None = None) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/files/import-local",
json={"src_path": src_path, "dest_name": dest_name},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
def upload_file(self, filename: str, content: bytes, content_type: str | None = None) -> dict:
with httpx.Client(timeout=self.timeout, verify=False) as c:
r = c.post(
f"{self.base_url}/compute/files/upload",
files={"file": (filename, content, content_type)},
headers=_auth_headers(),
)
r.raise_for_status()
return r.json()
# ---- inference (async) -------------------------------------------
def headers(self) -> dict[str, str]:
"""Return auth headers for compute node requests."""
token = get_settings().compute_service_token
if not token:
return {}
return {"X-Compute-Token": token}
@property
def route_prefix(self) -> str:
return get_settings().route_prefix.rstrip("/") or "/modelTF"
async def _request(self, method: str, path: str, json_data: dict | None = None) -> dict:
"""Generic async request method for compute API endpoints."""
prefix = self.route_prefix
url = f"{self.base_url.rstrip('/')}{prefix}{path}"
async with httpx.AsyncClient(timeout=300, verify=False, headers=self.headers()) as client:
if method.upper() == "GET":
response = await client.get(url)
else:
response = await client.post(url, json=json_data)
response.raise_for_status()
data = response.json()
if isinstance(data, dict) and isinstance(data.get("data"), dict):
return data["data"]
return data if isinstance(data, dict) else {}

View File

@@ -0,0 +1,50 @@
from __future__ import annotations
from typing import Any
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
async def poll_compute_jobs_once() -> dict[str, Any]:
store = get_platform_store()
synced: list[dict[str, Any]] = []
failed: list[dict[str, str]] = []
for task in store.running_compute_tasks():
node = _node_for_task(task)
if not node:
failed.append({"task_id": task["id"], "error": "compute node not found"})
continue
try:
client = ComputeNodeClient(node["api_base_url"])
job = client.get_job(task["compute_job_id"])
try:
logs = client.job_logs(task["compute_job_id"], tail_lines=5000)
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
except Exception:
pass
if job.get("status") in {"failed", "stopped"}:
try:
last_logs = client.job_logs(task["compute_job_id"], tail_lines=200)
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
for record in store.active_standalone_compute_jobs():
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
if not node:
failed.append({"job_id": record["id"], "error": "compute node not found"})
continue
try:
job = ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"job_id": record["id"], "error": str(exc)})
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}

View File

@@ -3,13 +3,17 @@
- preset 参数预设quick / standard / high
- train_type → stage 映射sft/dpo/cpt/cot
- 训练任务的启动 / 暂停 / 恢复 / 取消(委托 runner 真实执行)
- 训练任务的启动 / 暂停 / 恢复 / 取消
- compute_gateway 状态轮询线程(当任务派发到算力时自动同步状态)
"""
from __future__ import annotations
import asyncio
import threading
import time
from typing import Any
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.fine_tune import runner
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
}
# ── compute sync 轮询线程 ──────────────────────────────────────────────
_sync_thread: threading.Thread | None = None
_sync_thread_stop = threading.Event()
def _compute_sync_loop() -> None:
"""后台线程:周期性轮询算力节点,同步训练任务状态/日志/指标。"""
interval = get_settings().compute_poll_interval_seconds or 3
while not _sync_thread_stop.is_set():
try:
store = get_platform_store()
running = store.running_compute_tasks()
if running:
asyncio.run(_poll_once())
except Exception: # noqa: BLE001 - keep polling loop alive
pass
time.sleep(interval)
async def _poll_once() -> None:
from app.modules.compute_gateway.sync import poll_compute_jobs_once
await poll_compute_jobs_once()
def start_compute_sync_worker() -> None:
"""启动后台轮询线程(幂等,多次调用安全)。"""
global _sync_thread
if _sync_thread is not None and _sync_thread.is_alive():
return
_sync_thread_stop.clear()
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
_sync_thread.start()
def stop_compute_sync_worker() -> None:
"""停止后台轮询线程。"""
_sync_thread_stop.set()
# ── preset / config ────────────────────────────────────────────────────
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
"""根据 preset 字段补全缺失的超参preset=custom 时不覆盖。"""
payload = dict(payload)
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
def launch_training(task_id: str) -> None:
"""在后台线程启动真实训练"""
"""在后台线程启动真实训练(本机 subprocess fallback当算力节点不可用时使用
架构原则GPU 计算应派发到算力服务进程执行。
当 platform_store.start_task 检测到在线算力节点时,会走 _dispatch_to_compute 派发路径;
仅当无可用算力节点且非 simulator 模式时,降级到本机 runner违反 §1.1,待移除)。
"""
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()