YG_FT/backend/app/workers/compute_poller.py

from __future__ import annotations

import asyncio
import time

from app.core.config import get_settings
from app.core.logging import get_logger
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.sync import poll_compute_jobs_once


logger = get_logger(__name__)


async def run_compute_poller() -> None:
    settings = get_settings()
    if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
        logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode})
        return

    interval = max(3, settings.compute_poll_interval_seconds)
    logger.info("compute poller started", extra={"interval_seconds": interval})
    # PlatformStore may run additive schema checks against a remote PostgreSQL
    # server on first use. Keep that startup work off the Uvicorn event loop so
    # health checks and normal API requests can still respond while the DB is
    # unavailable or slow.
    store = None
    last_failure_signature = ""
    last_failure_logged_at = 0.0
    await asyncio.sleep(1)
    while True:
        try:
            if store is None:
                store = await asyncio.to_thread(get_platform_store)
            result = await poll_compute_jobs_once(store)
            if result["failed"]:
                signature = "|".join(sorted({
                    str(item.get("error") or "")[:120]
                    for item in result["failed"]
                }))
                now = time.monotonic()
                if signature != last_failure_signature or now - last_failure_logged_at >= 300:
                    logger.warning(
                        "compute polling reported failures count=%d first_error=%s",
                        len(result["failed"]),
                        signature[:500],
                    )
                    last_failure_signature = signature
                    last_failure_logged_at = now
            elif result["synced"]:
                logger.debug("compute jobs synchronized", extra={"result": result})
        except asyncio.CancelledError:
            logger.info("compute poller stopped")
            raise
        except Exception as exc:  # noqa: BLE001 - keep background polling alive
            logger.exception("compute poller failed", extra={"error": str(exc)})
            if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)):
                store = None
        await asyncio.sleep(interval)