6f0e82f351
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请 - 存储: MinIO 存储进度迁移、对象存储安全加固与测试 - 计算: GPU 资源预留、compute 轮询与同步增强 - 权限: permission v2 迁移、权限安全验收测试 - 日志: 后端运行日志中文说明、操作日志整合 - 数据处理/评测: 数据转换与模型评测优化 Co-Authored-By: Claude <noreply@anthropic.com>
60 lines
2.4 KiB
Python
60 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
|
|
from app.core.config import get_settings
|
|
from app.core.logging import get_logger
|
|
from app.db.platform_store import get_platform_store
|
|
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
|
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
async def run_compute_poller() -> None:
|
|
settings = get_settings()
|
|
if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
|
|
logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode})
|
|
return
|
|
|
|
interval = max(3, settings.compute_poll_interval_seconds)
|
|
logger.info("compute poller started", extra={"interval_seconds": interval})
|
|
# PlatformStore may run additive schema checks against a remote PostgreSQL
|
|
# server on first use. Keep that startup work off the Uvicorn event loop so
|
|
# health checks and normal API requests can still respond while the DB is
|
|
# unavailable or slow.
|
|
store = None
|
|
last_failure_signature = ""
|
|
last_failure_logged_at = 0.0
|
|
await asyncio.sleep(1)
|
|
while True:
|
|
try:
|
|
if store is None:
|
|
store = await asyncio.to_thread(get_platform_store)
|
|
result = await poll_compute_jobs_once(store)
|
|
if result["failed"]:
|
|
signature = "|".join(sorted({
|
|
str(item.get("error") or "")[:120]
|
|
for item in result["failed"]
|
|
}))
|
|
now = time.monotonic()
|
|
if signature != last_failure_signature or now - last_failure_logged_at >= 300:
|
|
logger.warning(
|
|
"compute polling reported failures count=%d first_error=%s",
|
|
len(result["failed"]),
|
|
signature[:500],
|
|
)
|
|
last_failure_signature = signature
|
|
last_failure_logged_at = now
|
|
elif result["synced"]:
|
|
logger.debug("compute jobs synchronized", extra={"result": result})
|
|
except asyncio.CancelledError:
|
|
logger.info("compute poller stopped")
|
|
raise
|
|
except Exception as exc: # noqa: BLE001 - keep background polling alive
|
|
logger.exception("compute poller failed", extra={"error": str(exc)})
|
|
if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)):
|
|
store = None
|
|
await asyncio.sleep(interval)
|