This commit is contained in:
wangjiming
2026-07-31 16:10:34 +08:00
parent 945b4ace86
commit 242407b676
34 changed files with 3847 additions and 717 deletions

View File

@@ -3,13 +3,17 @@
- preset 参数预设quick / standard / high
- train_type → stage 映射sft/dpo/cpt/cot
- 训练任务的启动 / 暂停 / 恢复 / 取消(委托 runner 真实执行)
- 训练任务的启动 / 暂停 / 恢复 / 取消
- compute_gateway 状态轮询线程(当任务派发到算力时自动同步状态)
"""
from __future__ import annotations
import asyncio
import threading
import time
from typing import Any
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.fine_tune import runner
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
}
# ── compute sync 轮询线程 ──────────────────────────────────────────────
_sync_thread: threading.Thread | None = None
_sync_thread_stop = threading.Event()
def _compute_sync_loop() -> None:
"""后台线程:周期性轮询算力节点,同步训练任务状态/日志/指标。"""
interval = get_settings().compute_poll_interval_seconds or 3
while not _sync_thread_stop.is_set():
try:
store = get_platform_store()
running = store.running_compute_tasks()
if running:
asyncio.run(_poll_once())
except Exception: # noqa: BLE001 - keep polling loop alive
pass
time.sleep(interval)
async def _poll_once() -> None:
from app.modules.compute_gateway.sync import poll_compute_jobs_once
await poll_compute_jobs_once()
def start_compute_sync_worker() -> None:
"""启动后台轮询线程(幂等,多次调用安全)。"""
global _sync_thread
if _sync_thread is not None and _sync_thread.is_alive():
return
_sync_thread_stop.clear()
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
_sync_thread.start()
def stop_compute_sync_worker() -> None:
"""停止后台轮询线程。"""
_sync_thread_stop.set()
# ── preset / config ────────────────────────────────────────────────────
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
"""根据 preset 字段补全缺失的超参preset=custom 时不覆盖。"""
payload = dict(payload)
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
def launch_training(task_id: str) -> None:
"""在后台线程启动真实训练"""
"""在后台线程启动真实训练(本机 subprocess fallback当算力节点不可用时使用
架构原则GPU 计算应派发到算力服务进程执行。
当 platform_store.start_task 检测到在线算力节点时,会走 _dispatch_to_compute 派发路径;
仅当无可用算力节点且非 simulator 模式时,降级到本机 runner违反 §1.1,待移除)。
"""
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()