X-Financial/server/tests/test_agent_asset_release_guard.py

from __future__ import annotations

import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool

from app.db.base import Base
from app.models.agent_asset import AgentAsset, AgentAssetTestRun, AgentAssetVersion
from app.services.agent_asset_release_guard import (
    AgentAssetReleaseGuardService,
    ReleaseEvaluationInput,
    ReleaseGuardPolicy,
)


def _session() -> Session:
    engine = create_engine(
        "sqlite+pysqlite:///:memory:",
        connect_args={"check_same_thread": False},
        poolclass=StaticPool,
    )
    Base.metadata.create_all(engine)
    return sessionmaker(bind=engine, autoflush=False, autocommit=False)()


def _seed_asset(db: Session) -> AgentAsset:
    asset = AgentAsset(
        id="asset-release",
        asset_type="task",
        code="task.release.guard",
        name="发布门禁任务",
        domain="expense",
        owner="tester",
        status="active",
        current_version="v1",
        working_version="v2",
        published_version="v1",
        config_json={"business_config": "preserved"},
    )
    db.add(asset)
    db.add_all(
        [
            AgentAssetVersion(
                asset_id=asset.id,
                version=version,
                content="{}",
                content_type="json",
                created_by="tester",
            )
            for version in ("v1", "v2")
        ]
    )
    db.commit()
    return asset


def _policy() -> ReleaseGuardPolicy:
    return ReleaseGuardPolicy(
        shadow_min_samples=2,
        canary_min_samples=3,
        max_error_rate=0.1,
        min_precision=0.9,
        max_precision_drop=0.05,
        canary_traffic_percent=10,
        recall_gate_enabled=False,
    )


def test_release_moves_shadow_canary_active_and_preserves_previous_version() -> None:
    with _session() as db:
        asset = _seed_asset(db)
        service = AgentAssetReleaseGuardService(db)

        shadow = service.start_shadow(asset.id, "v2", actor="manager", policy=_policy())
        assert shadow["stage"] == "shadow"
        assert shadow["previous_version"] == "v1"
        assert db.get(AgentAsset, asset.id).config_json["business_config"] == "preserved"
        assert service.get_serving_plan(asset.id) == {
            "stage": "shadow",
            "primary_version": "v1",
            "candidate_version": "v2",
            "candidate_traffic_percent": 0,
            "shadow_evaluation": True,
        }

        service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(total=2, failure_count=0, precision=0.99),
            actor="evaluator",
        )
        canary = service.promote(asset.id, actor="manager")
        assert canary["stage"] == "canary"
        assert canary["previous_version"] == "v1"
        assert service.get_serving_plan(asset.id)["candidate_traffic_percent"] == 10

        service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(
                total=3,
                failure_count=0,
                precision=0.98,
                baseline_precision=0.99,
            ),
            actor="evaluator",
        )
        active = service.promote(asset.id, actor="manager")

        refreshed = db.get(AgentAsset, asset.id)
        assert active["stage"] == "active"
        assert active["previous_version"] == "v1"
        assert refreshed.published_version == "v2"
        assert service.get_serving_plan(asset.id)["primary_version"] == "v2"


def test_release_cannot_promote_while_samples_are_collecting() -> None:
    with _session() as db:
        asset = _seed_asset(db)
        service = AgentAssetReleaseGuardService(db)
        service.start_shadow(asset.id, "v2", actor="manager", policy=_policy())

        result = service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(total=1, failure_count=0, precision=0.99),
            actor="evaluator",
        )

        assert result["status"] == "collecting"
        try:
            service.promote(asset.id, actor="manager")
        except PermissionError as exc:
            assert "尚无通过" in str(exc)
        else:
            raise AssertionError("collecting 状态不应允许晋级")


def test_active_quality_regression_automatically_restores_previous_version() -> None:
    with _session() as db:
        asset = _seed_asset(db)
        service = AgentAssetReleaseGuardService(db)
        service.start_shadow(asset.id, "v2", actor="manager", policy=_policy())
        service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(total=2, failure_count=0, precision=0.99),
            actor="evaluator",
        )
        service.promote(asset.id, actor="manager")
        service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(total=3, failure_count=0, precision=0.99),
            actor="evaluator",
        )
        service.promote(asset.id, actor="manager")

        result = service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(
                total=10,
                failure_count=2,
                precision=0.8,
                baseline_precision=0.99,
            ),
            actor="monitor",
        )

        refreshed = db.get(AgentAsset, asset.id)
        assert result["status"] == "failed"
        assert result["release_stage"] == "rolled_back"
        assert refreshed.published_version == "v1"
        assert refreshed.config_json["release_guard"]["previous_version"] == "v1"
        assert refreshed.config_json["release_guard"]["rollback"]["automatic"] is True
        assert service.get_serving_plan(asset.id)["primary_version"] == "v1"


def test_invalid_or_missing_precision_fails_closed_and_writes_test_run() -> None:
    with _session() as db:
        asset = _seed_asset(db)
        service = AgentAssetReleaseGuardService(db)
        service.start_shadow(asset.id, "v2", actor="manager", policy=_policy())

        result = service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(total=2, failure_count=0, precision=None),
            actor="evaluator",
        )

        run = db.query(AgentAssetTestRun).filter_by(asset_id=asset.id).one()
        assert result["status"] == "failed"
        assert result["release_stage"] == "rolled_back"
        assert run.passed is False
        assert "precision_metric_missing" in run.result_json["reasons"]


def test_malformed_policy_and_metric_counts_fail_closed() -> None:
    with _session() as db:
        asset = _seed_asset(db)
        service = AgentAssetReleaseGuardService(db)
        service.start_shadow(
            asset.id,
            "v2",
            actor="manager",
            policy=ReleaseGuardPolicy(
                shadow_min_samples="invalid",  # type: ignore[arg-type]
                min_precision="invalid",  # type: ignore[arg-type]
                reviewer_quorum=99,
            ),
        )

        state = service.get_state(asset.id)
        assert state["policy"]["shadow_min_samples"] == 20
        assert state["policy"]["min_precision"] == 0.98
        assert state["policy"]["reviewer_quorum"] == 2
        assert ReleaseGuardPolicy(reviewer_quorum=0).to_dict()["reviewer_quorum"] == 1

        result = service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(
                total="invalid",  # type: ignore[arg-type]
                failure_count=0,
                precision=0.99,
            ),
            actor="evaluator",
        )

        assert result["status"] == "failed"
        assert result["release_stage"] == "rolled_back"
        assert "invalid_evaluation_metrics" in result["reasons"]


def test_restarted_release_cannot_reuse_previous_attempt_quality_gate() -> None:
    with _session() as db:
        asset = _seed_asset(db)
        service = AgentAssetReleaseGuardService(db)
        first = service.start_shadow(asset.id, "v2", actor="manager", policy=_policy())
        service.record_evaluation(
            asset.id,
            ReleaseEvaluationInput(total=2, failure_count=0, precision=0.99),
            actor="evaluator",
        )
        service.rollback(asset.id, actor="manager", reason="restart")
        second = service.start_shadow(asset.id, "v2", actor="manager", policy=_policy())

        assert first["release_id"] != second["release_id"]
        with pytest.raises(PermissionError, match="尚无通过"):
            service.promote(asset.id, actor="manager")