X-Financial/server/tests/test_agent_asset_release_monitor.py

from __future__ import annotations

import inspect
from collections.abc import Generator
from datetime import UTC, datetime, timedelta

import pytest
from sqlalchemy import create_engine, select, update
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool

from app.core.agent_enums import AgentAssetDomain, AgentAssetStatus, AgentAssetType
from app.db.base import Base
from app.models.agent_asset import AgentAsset, AgentAssetTestRun
from app.models.agent_asset_release_telemetry import AgentAssetReleaseObservation
from app.services.agent_asset_release_monitor import AgentAssetReleaseMonitor
from app.services.agent_asset_release_review import AgentAssetReleaseReviewService
from app.services.agent_asset_release_telemetry import (
    AgentAssetReleaseTelemetryService,
    ReleaseObservationInput,
)


@pytest.fixture
def db() -> Generator[Session, None, None]:
    engine = create_engine(
        "sqlite+pysqlite:///:memory:",
        connect_args={"check_same_thread": False},
        poolclass=StaticPool,
    )
    Base.metadata.create_all(engine)
    factory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
    with factory() as session:
        yield session
    engine.dispose()


def _policy(
    *,
    min_precision: float = 0.8,
    max_error_rate: float = 0.25,
) -> dict:
    return {
        "shadow_min_samples": 1,
        "canary_min_samples": 1,
        "max_error_rate": max_error_rate,
        "min_precision": min_precision,
        "max_precision_drop": 0.2,
        "canary_traffic_percent": 10,
    }


def _seed_asset(
    db: Session,
    *,
    asset_id: str = "release-monitor-asset",
    tenant_id: str | None = "tenant-a",
    stage: str = "shadow",
    candidate_version: str = "v2",
    min_precision: float = 0.8,
    max_error_rate: float = 0.25,
    detail_mode: str = "json_risk",
) -> AgentAsset:
    previous_config = {
        "detail_mode": detail_mode,
        "enabled": True,
        "stable_marker": f"stable:{asset_id}",
    }
    if tenant_id is not None:
        previous_config["tenant_id"] = tenant_id
    state = {
        "release_id": f"release-{asset_id}",
        "stage": stage,
        "candidate_version": candidate_version,
        "previous_version": "v1",
        "policy": _policy(
            min_precision=min_precision,
            max_error_rate=max_error_rate,
        ),
        "previous_config": dict(previous_config),
        "history": [],
    }
    asset = AgentAsset(
        id=asset_id,
        tenant_id=tenant_id or "platform",
        scope="tenant" if tenant_id is not None else "platform",
        asset_type=AgentAssetType.RULE.value,
        code=f"risk.{asset_id}",
        name="发布监控规则",
        description="",
        domain=AgentAssetDomain.EXPENSE.value,
        scenario_json=["travel"],
        owner="finance",
        status=AgentAssetStatus.ACTIVE.value,
        current_version="v1",
        published_version="v1",
        working_version="v2",
        config_json={
            **previous_config,
            "release_guard": state,
            # 外部或客户端放入配置的数字不是发布评测输入。
            "untrusted_external_metrics": {
                "total": 999999,
                "precision": 1.0,
            },
        },
    )
    db.add(asset)
    db.commit()
    return asset


def _record(
    db: Session,
    asset: AgentAsset,
    *,
    source: str,
    candidate_hit: bool,
    baseline_hit: bool | None = True,
    label: str | None = None,
    failed: bool = False,
) -> str:
    state = dict((asset.config_json or {})["release_guard"])
    service = AgentAssetReleaseTelemetryService(db)
    observation = service.record_observation(
        ReleaseObservationInput(
            tenant_id=str((asset.config_json or {}).get("tenant_id") or "tenant-a"),
            asset_id=asset.id,
            release_id=str(state["release_id"]),
            stage=str(state["stage"]),  # type: ignore[arg-type]
            version=str(state["candidate_version"]),
            rule_code=asset.code,
            source_key=source,
            candidate_hit=candidate_hit,
            baseline_hit=baseline_hit,
            runtime_status="failed" if failed else "completed",
            failure_code="evaluator_error" if failed else "none",
        )
    )
    if label is not None:
        service.record_review_label(
            tenant_id=observation.tenant_id,
            observation_id=observation.id,
            label=label,  # type: ignore[arg-type]
            request_id=f"review:{source}",
            actor_id="trusted-auditor",
        )
    db.commit()
    return observation.id


class _NeverCalledGuard:
    def __init__(self) -> None:
        self.calls = 0

    def record_evaluation(self, *_args, **_kwargs):
        self.calls += 1
        raise AssertionError("collecting telemetry must never call Release Guard")


def test_collecting_never_calls_guard_or_rolls_back(db: Session) -> None:
    asset = _seed_asset(db)
    _record(
        db,
        asset,
        source="claim-unlabeled",
        candidate_hit=True,
        label=None,
    )
    guard = _NeverCalledGuard()
    monitor = AgentAssetReleaseMonitor(db, guard_service=guard)  # type: ignore[arg-type]

    result = monitor.evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )

    refreshed = db.get(AgentAsset, asset.id)
    assert result["status"] == "collecting"
    assert result["telemetry_status"] == "collecting"
    assert result["evaluation_submitted"] is False
    assert result["metrics"]["candidate_pending_label_count"] == 1
    assert result["metrics"]["precision"] is None
    assert result["rolled_back"] is False
    assert guard.calls == 0
    assert refreshed.config_json["release_guard"]["stage"] == "shadow"
    assert db.scalar(select(AgentAssetTestRun)) is None


def test_ready_metrics_are_evaluated_and_passed(db: Session) -> None:
    asset = _seed_asset(db)
    _record(
        db,
        asset,
        source="claim-confirmed",
        candidate_hit=True,
        label="confirmed",
    )

    result = AgentAssetReleaseMonitor(db).evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )
    run = db.scalar(select(AgentAssetTestRun).where(AgentAssetTestRun.asset_id == asset.id))

    assert result["telemetry_status"] == "ready"
    assert result["status"] == "passed"
    assert result["evaluation_submitted"] is True
    assert result["rolled_back"] is False
    assert result["metrics"]["observed_count"] == 1
    assert result["metrics"]["precision"] == 1.0
    assert result["metrics"]["baseline_precision"] == 1.0
    assert result["metrics"]["recall"] == 1.0
    assert result["metrics"]["recall_lower_bound"] == 1.0
    assert run is not None
    assert run.input_json["total"] == 1
    assert run.input_json["failure_count"] == 0
    assert run.input_json["precision"] == 1.0
    assert run.result_json["details"]["metric_source"] == "release_runtime_telemetry"


def test_same_ready_snapshot_reuses_test_run(db: Session) -> None:
    asset = _seed_asset(db)
    _record(
        db,
        asset,
        source="claim-idempotent",
        candidate_hit=True,
        label="confirmed",
    )
    monitor = AgentAssetReleaseMonitor(db)

    first = monitor.evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )
    second = monitor.evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor-retry",
    )
    runs = list(
        db.scalars(
            select(AgentAssetTestRun).where(AgentAssetTestRun.asset_id == asset.id)
        ).all()
    )

    assert second == first
    assert len(runs) == 1


def test_real_low_precision_automatically_rolls_back(db: Session) -> None:
    asset = _seed_asset(db, min_precision=0.9, max_error_rate=1.0)
    _record(db, asset, source="claim-true", candidate_hit=True, label="confirmed")
    _record(
        db,
        asset,
        source="claim-false-positive",
        candidate_hit=True,
        label="false_positive",
    )

    result = AgentAssetReleaseMonitor(db).evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )
    refreshed = db.get(AgentAsset, asset.id)

    assert result["status"] == "failed"
    assert result["rolled_back"] is True
    assert result["release_stage"] == "rolled_back"
    assert result["metrics"]["precision"] == 0.5
    assert "precision_below_threshold" in result["reasons"]
    assert refreshed.published_version == "v1"
    assert refreshed.config_json["stable_marker"] == f"stable:{asset.id}"
    assert refreshed.config_json["release_guard"]["rollback"]["automatic"] is True


def test_real_runtime_failure_automatically_rolls_back(db: Session) -> None:
    asset = _seed_asset(db, min_precision=0.5, max_error_rate=0.1)
    _record(db, asset, source="claim-good", candidate_hit=True, label="confirmed")
    _record(
        db,
        asset,
        source="claim-evaluator-failed",
        candidate_hit=False,
        baseline_hit=None,
        failed=True,
    )

    result = AgentAssetReleaseMonitor(db).evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )

    assert result["status"] == "failed"
    assert result["rolled_back"] is True
    assert result["metrics"]["observed_count"] == 2
    assert result["metrics"]["runtime_failure_count"] == 1
    assert result["metrics"]["runtime_failure_rate"] == 0.5
    assert "error_rate_exceeded" in result["reasons"]
    assert {item["code"] for item in result["alerts"]} >= {
        "runtime_failures_detected",
        "release_auto_rolled_back",
    }


def test_all_runtime_failures_without_hits_automatically_roll_back(db: Session) -> None:
    asset = _seed_asset(db, min_precision=0.5, max_error_rate=0.1)
    _record(
        db,
        asset,
        source="claim-only-evaluator-failed",
        candidate_hit=False,
        baseline_hit=None,
        failed=True,
    )

    result = AgentAssetReleaseMonitor(db).evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )

    assert result["telemetry_status"] == "ready"
    assert result["status"] == "failed"
    assert result["rolled_back"] is True
    assert result["metrics"]["candidate_hit_count"] == 0
    assert result["metrics"]["precision"] is None
    assert result["metrics"]["runtime_failure_count"] == 1
    assert "error_rate_exceeded" in result["reasons"]


def test_incomplete_labels_with_tolerated_failure_cannot_pass(db: Session) -> None:
    asset = _seed_asset(db, min_precision=0.5, max_error_rate=1.0)
    _record(
        db,
        asset,
        source="claim-unlabeled-hit",
        candidate_hit=True,
        label=None,
    )
    _record(
        db,
        asset,
        source="claim-tolerated-evaluator-failed",
        candidate_hit=False,
        baseline_hit=None,
        failed=True,
    )

    result = AgentAssetReleaseMonitor(db).evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )

    assert result["telemetry_status"] == "ready"
    assert result["status"] == "collecting"
    assert result["rolled_back"] is False
    assert result["metrics"]["candidate_pending_label_count"] == 1
    assert "precision_metric_missing" in result["reasons"]
    assert "release_labels_pending" in {item["code"] for item in result["alerts"]}


def test_cross_tenant_and_global_asset_scope_are_enforced(db: Session) -> None:
    tenant_asset = _seed_asset(db)
    _record(db, tenant_asset, source="tenant-a-claim", candidate_hit=True, label="confirmed")
    monitor = AgentAssetReleaseMonitor(db)

    with pytest.raises(LookupError):
        monitor.evaluate_current(
            tenant_id="tenant-b",
            asset_id=tenant_asset.id,
            actor="foreign-monitor",
        )
    assert monitor.batch_evaluate(tenant_id="tenant-b", actor="foreign-monitor")["scanned"] == 0

    global_asset = _seed_asset(
        db,
        asset_id="global-release-monitor",
        tenant_id=None,
    )
    with pytest.raises(LookupError):
        _record(db, global_asset, source="global-claim", candidate_hit=True, label="confirmed")
    with pytest.raises(LookupError):
        monitor.evaluate_current(
            tenant_id="tenant-a",
            asset_id=global_asset.id,
            actor="tenant-monitor",
        )
    with pytest.raises(ValueError, match="cross-tenant aggregate"):
        monitor.evaluate_current(
            tenant_id="tenant-a",
            asset_id=global_asset.id,
            actor="platform-monitor",
            allow_global_management=True,
        )
    assert global_asset.config_json["release_guard"]["stage"] == "shadow"


def test_batch_scan_is_bounded_tenant_safe_and_failure_isolated(db: Session) -> None:
    collecting_asset = _seed_asset(db, asset_id="batch-collecting")
    _record(db, collecting_asset, source="collecting", candidate_hit=True)

    broken_asset = _seed_asset(
        db,
        asset_id="batch-error",
        candidate_version="",
    )
    passed_asset = _seed_asset(db, asset_id="batch-passed")
    _record(db, passed_asset, source="passed", candidate_hit=True, label="confirmed")
    rolled_asset = _seed_asset(
        db,
        asset_id="batch-rolled-back",
        min_precision=0.9,
        max_error_rate=1.0,
    )
    _record(db, rolled_asset, source="roll-true", candidate_hit=True, label="confirmed")
    _record(
        db,
        rolled_asset,
        source="roll-false",
        candidate_hit=True,
        label="false_positive",
    )
    foreign_asset = _seed_asset(db, asset_id="foreign-ready", tenant_id="tenant-b")
    _record(
        db,
        foreign_asset,
        source="foreign",
        candidate_hit=True,
        label="confirmed",
    )
    _seed_asset(db, asset_id="not-json", detail_mode="markdown")
    _seed_asset(db, asset_id="not-current", stage="rolled_back")
    monitor = AgentAssetReleaseMonitor(db)

    result = monitor.batch_evaluate(
        tenant_id="tenant-a",
        actor="batch-monitor",
        limit=10,
    )

    assert result["scanned"] == 4
    assert result["evaluated"] == 2
    assert result["collecting"] == 1
    assert result["rolled_back"] == 1
    assert len(result["errors"]) == 1
    assert result["errors"][0]["asset_id"] == broken_asset.id
    assert result["errors"][0]["alerts"][0]["code"] == "release_aggregation_failed"
    assert {item["asset_id"] for item in result["results"]} == {
        collecting_asset.id,
        passed_asset.id,
        rolled_asset.id,
    }
    assert foreign_asset.id not in str(result)
    assert db.get(AgentAsset, passed_asset.id).config_json["release_guard"]["stage"] == "shadow"
    assert db.get(AgentAsset, rolled_asset.id).config_json["release_guard"]["stage"] == (
        "rolled_back"
    )
    with pytest.raises(ValueError, match="between 1 and 500"):
        monitor.batch_evaluate(tenant_id="tenant-a", actor="batch-monitor", limit=501)


def test_overdue_review_sample_emits_actionable_alert(db: Session) -> None:
    asset = _seed_asset(db)
    observation_id = _record(
        db,
        asset,
        source="claim-overdue-review",
        candidate_hit=True,
    )
    # SQLite 测试库没有 PostgreSQL append-only 触发器；用 Core 语句只调整
    # fixture 时间，避免把“运营超时”测试误写成 ORM 可更新契约。
    db.execute(
        update(AgentAssetReleaseObservation)
        .where(AgentAssetReleaseObservation.id == observation_id)
        .values(created_at=datetime.now(UTC) - timedelta(hours=25))
    )
    db.commit()

    result = AgentAssetReleaseReviewService(db).list_pending(
        tenant_id="tenant-a",
        asset_id=asset.id,
    )

    assert result["telemetry_status"] == "collecting"
    assert result["metrics"]["candidate_oldest_pending_age_seconds"] >= 86_400
    assert "release_labels_overdue" in {item["code"] for item in result["alerts"]}


def test_monitor_contract_cannot_accept_fabricated_quality_metrics(db: Session) -> None:
    asset = _seed_asset(db, min_precision=0.9, max_error_rate=1.0)
    _record(
        db,
        asset,
        source="actual-false-positive",
        candidate_hit=True,
        label="false_positive",
    )
    signature = inspect.signature(AgentAssetReleaseMonitor.evaluate_current)

    assert {"total", "failure_count", "precision", "baseline_precision"}.isdisjoint(
        signature.parameters
    )
    with pytest.raises(TypeError):
        AgentAssetReleaseMonitor(db).evaluate_current(
            tenant_id="tenant-a",
            asset_id=asset.id,
            actor="malicious-monitor",
            precision=1.0,  # type: ignore[call-arg]
        )

    result = AgentAssetReleaseMonitor(db).evaluate_current(
        tenant_id="tenant-a",
        asset_id=asset.id,
        actor="release-monitor",
    )
    assert result["metrics"]["precision"] == 0.0
    assert result["rolled_back"] is True