update
This commit is contained in:
@@ -3,13 +3,17 @@
|
||||
|
||||
- preset 参数预设(quick / standard / high)
|
||||
- train_type → stage 映射(sft/dpo/cpt/cot)
|
||||
- 训练任务的启动 / 暂停 / 恢复 / 取消(委托 runner 真实执行)
|
||||
- 训练任务的启动 / 暂停 / 恢复 / 取消
|
||||
- compute_gateway 状态轮询线程(当任务派发到算力时自动同步状态)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.fine_tune import runner
|
||||
|
||||
@@ -19,7 +23,46 @@ PRESETS: dict[str, dict[str, Any]] = {
|
||||
"high": {"learning_rate": "1e-5", "n_epochs": 5, "batch_size": 4, "lora_rank": 32},
|
||||
}
|
||||
|
||||
# ── compute sync 轮询线程 ──────────────────────────────────────────────
|
||||
_sync_thread: threading.Thread | None = None
|
||||
_sync_thread_stop = threading.Event()
|
||||
|
||||
|
||||
def _compute_sync_loop() -> None:
|
||||
"""后台线程:周期性轮询算力节点,同步训练任务状态/日志/指标。"""
|
||||
interval = get_settings().compute_poll_interval_seconds or 3
|
||||
while not _sync_thread_stop.is_set():
|
||||
try:
|
||||
store = get_platform_store()
|
||||
running = store.running_compute_tasks()
|
||||
if running:
|
||||
asyncio.run(_poll_once())
|
||||
except Exception: # noqa: BLE001 - keep polling loop alive
|
||||
pass
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
async def _poll_once() -> None:
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
await poll_compute_jobs_once()
|
||||
|
||||
|
||||
def start_compute_sync_worker() -> None:
|
||||
"""启动后台轮询线程(幂等,多次调用安全)。"""
|
||||
global _sync_thread
|
||||
if _sync_thread is not None and _sync_thread.is_alive():
|
||||
return
|
||||
_sync_thread_stop.clear()
|
||||
_sync_thread = threading.Thread(target=_compute_sync_loop, daemon=True)
|
||||
_sync_thread.start()
|
||||
|
||||
|
||||
def stop_compute_sync_worker() -> None:
|
||||
"""停止后台轮询线程。"""
|
||||
_sync_thread_stop.set()
|
||||
|
||||
|
||||
# ── preset / config ────────────────────────────────────────────────────
|
||||
def apply_presets(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""根据 preset 字段补全缺失的超参;preset=custom 时不覆盖。"""
|
||||
payload = dict(payload)
|
||||
@@ -63,7 +106,12 @@ def build_training_config(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
|
||||
def launch_training(task_id: str) -> None:
|
||||
"""在后台线程启动真实训练。"""
|
||||
"""在后台线程启动真实训练(本机 subprocess fallback,当算力节点不可用时使用)。
|
||||
|
||||
架构原则:GPU 计算应派发到算力服务进程执行。
|
||||
当 platform_store.start_task 检测到在线算力节点时,会走 _dispatch_to_compute 派发路径;
|
||||
仅当无可用算力节点且非 simulator 模式时,降级到本机 runner(违反 §1.1,待移除)。
|
||||
"""
|
||||
threading.Thread(target=runner.run_training, args=(task_id,), daemon=True).start()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user