from __future__ import annotations from collections.abc import Generator import pytest from sqlalchemy import create_engine, select from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import StaticPool from app.core.agent_enums import AgentAssetDomain, AgentAssetStatus, AgentAssetType from app.db.base import Base from app.models.agent_asset import AgentAsset from app.models.agent_asset_release_telemetry import ( AgentAssetReleaseAuditSample, AgentAssetReleaseLabel, AgentAssetReleaseObservation, ) from app.models.risk_disposition import RiskDisposition, RiskDispositionEvent from app.models.risk_observation import RiskObservation from app.services.agent_asset_release_disposition_labels import ( AgentAssetReleaseDispositionLabelService, ) from app.services.agent_asset_release_review import AgentAssetReleaseReviewService from app.services.agent_asset_release_telemetry import ( AgentAssetReleaseTelemetryService, ReleaseObservationInput, ReleaseTelemetryCollecting, ReleaseTelemetryIdempotencyConflict, ReleaseTelemetryStaleRelease, ) @pytest.fixture def db() -> Generator[Session, None, None]: engine = create_engine( "sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all( engine, tables=[ AgentAsset.__table__, RiskObservation.__table__, RiskDisposition.__table__, RiskDispositionEvent.__table__, AgentAssetReleaseObservation.__table__, AgentAssetReleaseLabel.__table__, AgentAssetReleaseAuditSample.__table__, ], ) factory = sessionmaker(bind=engine, autoflush=False, autocommit=False) with factory() as session: yield session engine.dispose() def _seed_asset( db: Session, *, stage: str = "shadow", tenant_id: str = "tenant-a", release_id: str = "release-1", reviewer_quorum: int = 1, recall_gate_enabled: bool = False, negative_sample_percent: int = 20, negative_min_reviewed: int = 5, min_recall: float = 0.95, ) -> AgentAsset: asset = AgentAsset( id="release-asset", asset_type=AgentAssetType.RULE.value, code="risk.release.telemetry", name="发布遥测规则", description="", domain=AgentAssetDomain.EXPENSE.value, scenario_json=["travel"], owner="finance", status=AgentAssetStatus.ACTIVE.value, current_version="v1", published_version="v1", working_version="v2", config_json={ "tenant_id": tenant_id, "detail_mode": "json_risk", "release_guard": { "release_id": release_id, "stage": stage, "candidate_version": "v2", "previous_version": "v1", "started_by": "publisher", "policy": { "reviewer_quorum": reviewer_quorum, "recall_gate_enabled": recall_gate_enabled, "negative_sample_percent": negative_sample_percent, "negative_min_reviewed": negative_min_reviewed, "min_recall": min_recall, "recall_confidence_level": 0.95, }, }, }, ) db.add(asset) db.commit() return asset def _shadow_result(*, candidate_hit: bool, baseline_hit: bool) -> dict: flags = [] if baseline_hit: flags.append( { "rule_code": "risk.release.telemetry", "rule_version": "v1", "release_stage": "shadow", "release_mode": "enforced", } ) return { "flags": flags, "shadow_evaluations": [ { "asset_id": "release-asset", "rule_code": "risk.release.telemetry", "rule_version": "v2", "release_stage": "shadow", "hit": candidate_hit, "severity": "high" if candidate_hit else "none", } ], } def _record_shadow( db: Session, *, claim_id: str, candidate_hit: bool = True, baseline_hit: bool = True, ) -> AgentAssetReleaseObservation: return AgentAssetReleaseTelemetryService(db).record_expense_risk_result( tenant_id="tenant-a", claim_id=claim_id, result=_shadow_result( candidate_hit=candidate_hit, baseline_hit=baseline_hit, ), )[0] def test_shadow_runtime_samples_and_labels_build_conservative_release_input( db: Session, ) -> None: _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) candidate = _record_shadow(db, claim_id="claim-secret-001") baseline_only = _record_shadow( db, claim_id="claim-secret-002", candidate_hit=False, baseline_hit=True, ) replay = _record_shadow(db, claim_id="claim-secret-001") db.commit() assert replay.id == candidate.id assert candidate.candidate_hit is True assert candidate.baseline_hit is True assert baseline_only.candidate_hit is False stored_text = str( { column.name: getattr(candidate, column.name) for column in AgentAssetReleaseObservation.__table__.columns } ) assert "claim-secret" not in stored_text collecting = service.aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) assert collecting.status == "collecting" assert collecting.precision is None assert collecting.recall is None assert collecting.false_negative_count is None assert collecting.negative_ground_truth_status == "collecting_candidate_labels" with pytest.raises(ReleaseTelemetryCollecting): collecting.to_release_evaluation_input() candidate_label = service.record_review_label( tenant_id="tenant-a", observation_id=candidate.id, label="confirmed", request_id="review-001", actor_id="auditor-secret-account", ) service.record_review_label( tenant_id="tenant-a", observation_id=baseline_only.id, label="false_positive", request_id="review-002", actor_id="auditor-secret-account", ) service.record_review_label( tenant_id="tenant-a", observation_id=baseline_only.id, label="false_positive", request_id="review-002-second", actor_id="second-auditor-secret-account", ) db.commit() assert candidate_label.actor_fingerprint != "auditor-secret-account" assert "auditor-secret-account" not in str(candidate_label.__dict__) ready = service.aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) assert ready.status == "ready" assert ready.observed_count == 2 assert ready.candidate_hit_count == 1 assert ready.candidate_labeled_count == 1 assert ready.precision == 1.0 assert ready.baseline_hit_count == 2 assert ready.baseline_labeled_count == 2 assert ready.baseline_precision == 0.5 evaluation = ready.to_release_evaluation_input() assert evaluation.total == 2 assert evaluation.failure_count == 0 assert evaluation.precision == 1.0 assert evaluation.baseline_precision == 0.5 assert evaluation.details["metric_source"] == "release_runtime_telemetry" assert evaluation.details["recall"] == 1.0 assert evaluation.details["recall_lower_bound"] == 1.0 def test_unlabeled_candidate_hit_never_counts_as_release_success(db: Session) -> None: _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) first = _record_shadow(db, claim_id="claim-1") _record_shadow(db, claim_id="claim-2") service.record_review_label( tenant_id="tenant-a", observation_id=first.id, label="confirmed", request_id="review-first", actor_id="auditor", ) db.commit() aggregate = service.aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) assert aggregate.status == "collecting" assert aggregate.candidate_hit_count == 2 assert aggregate.candidate_labeled_count == 1 assert aggregate.candidate_pending_label_count == 1 assert aggregate.baseline_precision is None assert "candidate_labels_pending" in aggregate.reasons with pytest.raises(ReleaseTelemetryCollecting): aggregate.to_release_evaluation_input() def test_observation_and_label_idempotency_reject_changed_payload(db: Session) -> None: _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) observation = _record_shadow(db, claim_id="claim-1") with pytest.raises(ReleaseTelemetryIdempotencyConflict): service.record_observation( ReleaseObservationInput( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", rule_code="risk.release.telemetry", source_key="claim-1", candidate_hit=False, baseline_hit=True, ) ) service.record_review_label( tenant_id="tenant-a", observation_id=observation.id, label="confirmed", request_id="same-review-request", actor_id="auditor", ) with pytest.raises(ReleaseTelemetryIdempotencyConflict): service.record_review_label( tenant_id="tenant-a", observation_id=observation.id, label="false_positive", request_id="same-review-request", actor_id="auditor", ) def test_release_label_values_cannot_cross_verification_sources(db: Session) -> None: _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) observation = _record_shadow(db, claim_id="claim-label-semantics") with pytest.raises(ValueError, match="release reviews require confirmed"): service.record_review_label( tenant_id="tenant-a", observation_id=observation.id, label="risk_present", # type: ignore[arg-type] request_id="wrong-release-review-semantics", actor_id="auditor", ) with pytest.raises(ValueError, match="Blind release review requires"): service.record_blind_review_label( tenant_id="tenant-a", observation_id=observation.id, ground_truth="confirmed", # type: ignore[arg-type] request_id="wrong-blind-review-semantics", actor_id="auditor", ) def test_observation_rolls_back_when_blind_sample_cannot_be_protected( db: Session, monkeypatch: pytest.MonkeyPatch, ) -> None: _seed_asset(db) def fail_encryption(_value: str) -> str: raise ValueError("sample encryption unavailable") monkeypatch.setattr( "app.services.agent_asset_release_sampling.encrypt_secret", fail_encryption, ) with pytest.raises(ValueError, match="sample encryption unavailable"): _record_shadow(db, claim_id="claim-must-not-leak") assert db.query(AgentAssetReleaseObservation).count() == 0 assert db.query(AgentAssetReleaseAuditSample).count() == 0 def test_cross_tenant_and_stale_release_labels_are_rejected(db: Session) -> None: asset = _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) observation = _record_shadow(db, claim_id="claim-1") db.commit() with pytest.raises(LookupError): service.record_review_label( tenant_id="tenant-b", observation_id=observation.id, label="confirmed", request_id="foreign-review", actor_id="foreign-auditor", ) config = dict(asset.config_json or {}) state = dict(config["release_guard"]) state["release_id"] = "release-2" config["release_guard"] = state asset.config_json = config db.add(asset) db.commit() with pytest.raises(ReleaseTelemetryStaleRelease): service.record_review_label( tenant_id="tenant-a", observation_id=observation.id, label="confirmed", request_id="stale-review", actor_id="auditor", ) def test_typed_risk_disposition_event_is_a_trusted_release_label(db: Session) -> None: _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) telemetry = _record_shadow(db, claim_id="claim-typed") risk_observation = RiskObservation( id="risk-observation-1", tenant_id="tenant-a", observation_key="risk:claim-typed:platform:risk.release.telemetry", subject_type="expense_claim", subject_key="claim:claim-typed", claim_id="claim-typed", claim_no="REDACTED", risk_type="policy", risk_signal="risk.release.telemetry", risk_score=82, risk_level="high", algorithm_version="v1", decision_trace_json={"rule_code": "risk.release.telemetry"}, ) disposition = RiskDisposition( id="disposition-1", tenant_id="tenant-a", observation_id=risk_observation.id, adjudication="confirmed", lifecycle_status="open", version=1, ) event = RiskDispositionEvent( id="disposition-event-1", tenant_id="tenant-a", disposition_id=disposition.id, observation_id=risk_observation.id, version=1, action="confirm", actor_id="real-auditor-secret", actor_name="真实审计员", request_id="typed-action-1", payload_fingerprint="f" * 64, before_json={"adjudication": "unreviewed"}, after_json={"adjudication": "confirmed"}, ) db.add_all([risk_observation, disposition, event]) db.commit() label = service.record_risk_disposition_label( tenant_id="tenant-a", observation_id=telemetry.id, disposition_event_id=event.id, ) db.commit() assert label.label == "confirmed" assert label.verification_source == "typed_risk_disposition" assert "real-auditor-secret" not in str(label.__dict__) aggregate = service.aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) assert aggregate.status == "ready" assert aggregate.precision == 1.0 assert aggregate.baseline_precision == 1.0 def test_typed_disposition_resolves_current_release_sample_without_client_observation_id( db: Session, ) -> None: _seed_asset(db) telemetry = _record_shadow(db, claim_id="claim-current-label") risk_observation = RiskObservation( id="risk-observation-current", tenant_id="tenant-a", observation_key="risk:claim-current-label:platform:risk.release.telemetry", subject_type="expense_claim", subject_key="claim:claim-current-label", claim_id="claim-current-label", claim_no="REDACTED", risk_type="policy", risk_signal="risk.release.telemetry", risk_score=82, risk_level="high", control_stage="reimbursement", algorithm_version="v1", decision_trace_json={"rule_code": "risk.release.telemetry"}, ) disposition = RiskDisposition( id="disposition-current", tenant_id="tenant-a", observation_id=risk_observation.id, adjudication="false_positive", lifecycle_status="open", version=1, ) event = RiskDispositionEvent( id="disposition-event-current", tenant_id="tenant-a", disposition_id=disposition.id, observation_id=risk_observation.id, version=1, action="false_positive", actor_id="auditor-secret", actor_name="审计员", request_id="typed-current-action", payload_fingerprint="e" * 64, before_json={"adjudication": "unreviewed"}, after_json={"adjudication": "false_positive"}, ) db.add_all([risk_observation, disposition, event]) db.commit() labels = AgentAssetReleaseDispositionLabelService(db).record_current_labels( tenant_id="tenant-a", disposition_event_id=event.id, ) db.commit() assert [item.observation_id for item in labels] == [telemetry.id] assert labels[0].label == "false_positive" assert "auditor-secret" not in str(labels[0].__dict__) def test_current_disposition_does_not_label_stale_release_sample(db: Session) -> None: asset = _seed_asset(db) _record_shadow(db, claim_id="claim-stale-label") risk_observation = RiskObservation( id="risk-observation-stale-current", tenant_id="tenant-a", observation_key="risk:claim-stale-label:platform:risk.release.telemetry", subject_type="expense_claim", subject_key="claim:claim-stale-label", claim_id="claim-stale-label", claim_no="REDACTED", risk_type="policy", risk_signal="risk.release.telemetry", risk_score=82, risk_level="high", control_stage="reimbursement", algorithm_version="v1", decision_trace_json={"rule_code": "risk.release.telemetry"}, ) disposition = RiskDisposition( id="disposition-stale-current", tenant_id="tenant-a", observation_id=risk_observation.id, adjudication="confirmed", lifecycle_status="open", version=1, ) event = RiskDispositionEvent( id="disposition-event-stale-current", tenant_id="tenant-a", disposition_id=disposition.id, observation_id=risk_observation.id, version=1, action="confirm", actor_id="auditor", actor_name="审计员", request_id="typed-stale-current-action", payload_fingerprint="d" * 64, before_json={"adjudication": "unreviewed"}, after_json={"adjudication": "confirmed"}, ) db.add_all([risk_observation, disposition, event]) db.commit() config = dict(asset.config_json or {}) state = dict(config["release_guard"]) state["release_id"] = "release-2" config["release_guard"] = state asset.config_json = config db.add(asset) db.commit() labels = AgentAssetReleaseDispositionLabelService(db).record_current_labels( tenant_id="tenant-a", disposition_event_id=event.id, ) assert labels == [] def test_canary_hits_and_structured_runtime_failures_are_real_samples(db: Session) -> None: _seed_asset(db, stage="canary") service = AgentAssetReleaseTelemetryService(db) records = service.record_expense_risk_result( tenant_id="tenant-a", claim_id="canary-claim-1", result={ "shadow_evaluations": [], "flags": [ { "rule_code": "risk.release.telemetry", "rule_version": "v2", "release_stage": "canary", "release_mode": "enforced", } ], }, ) miss = service.record_manifest_evaluation( tenant_id="tenant-a", claim_id="canary-claim-miss", manifest={ "_rule_asset_id": "release-asset", "_release_stage": "canary", "_release_mode": "enforced", "_rule_version": "v2", "rule_code": "risk.release.telemetry", }, hit=False, ) failure = service.record_observation( ReleaseObservationInput( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="canary", version="v2", rule_code="risk.release.telemetry", source_key="canary-claim-2", candidate_hit=False, baseline_hit=None, runtime_status="failed", failure_code="evaluator_error", ) ) service.record_review_label( tenant_id="tenant-a", observation_id=records[0].id, label="false_positive", request_id="canary-review", actor_id="auditor", ) with pytest.raises(ValueError, match="independent ground truth"): service.record_review_label( tenant_id="tenant-a", observation_id=miss.id, label="confirmed", request_id="unsupported-false-negative-label", actor_id="auditor", ) db.commit() aggregate = service.aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="canary", version="v2", ) evaluation = aggregate.to_release_evaluation_input() assert records[0].candidate_hit is True assert miss.candidate_hit is False assert failure.runtime_status == "failed" assert aggregate.status == "ready" assert aggregate.runtime_failure_count == 1 assert aggregate.precision == 0.0 assert aggregate.baseline_precision is None assert evaluation.total == 3 assert evaluation.failure_count == 1 def test_dedicated_review_queue_is_tenant_safe_and_separates_publisher( db: Session, ) -> None: _seed_asset(db) observation = _record_shadow(db, claim_id="claim-review-queue") service = AgentAssetReleaseReviewService(db) with pytest.raises(LookupError): service.list_pending(tenant_id="tenant-b", asset_id="release-asset") queue = service.list_pending(tenant_id="tenant-a", asset_id="release-asset") assert queue["pending_total"] == 1 assert queue["items"][0]["observation_id"] == observation.id assert "source_fingerprint" not in queue["items"][0] with pytest.raises(PermissionError, match="发布发起人"): service.record_label( tenant_id="tenant-a", asset_id="release-asset", observation_id=observation.id, label="confirmed", actor_id="publisher", request_id="publisher-self-review", ) label = service.record_label( tenant_id="tenant-a", asset_id="release-asset", observation_id=observation.id, label="confirmed", actor_id="independent-reviewer", request_id="independent-review", ) db.commit() assert label.label == "risk_present" assert service.list_pending(tenant_id="tenant-a", asset_id="release-asset")[ "pending_total" ] == 0 def test_release_review_quorum_requires_distinct_reviewers(db: Session) -> None: _seed_asset(db, reviewer_quorum=2) observation = _record_shadow(db, claim_id="claim-two-reviewers") service = AgentAssetReleaseReviewService(db) service.record_label( tenant_id="tenant-a", asset_id="release-asset", observation_id=observation.id, label="confirmed", actor_id="reviewer-one", request_id="review-one", ) db.commit() first_queue = service.list_pending(tenant_id="tenant-a", asset_id="release-asset") first_aggregate = AgentAssetReleaseTelemetryService(db).aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) assert first_queue["pending_total"] == 1 assert first_queue["items"][0]["reviewer_count"] == 1 assert first_queue["items"][0]["required_reviewers"] == 2 assert first_aggregate.status == "collecting" service.record_label( tenant_id="tenant-a", asset_id="release-asset", observation_id=observation.id, label="confirmed", actor_id="reviewer-two", request_id="review-two", ) db.commit() second_aggregate = AgentAssetReleaseTelemetryService(db).aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) assert service.list_pending(tenant_id="tenant-a", asset_id="release-asset")[ "pending_total" ] == 0 assert second_aggregate.status == "ready" assert second_aggregate.precision == 1.0 def test_blind_negative_audit_builds_real_false_negative_and_recall_evidence( db: Session, ) -> None: _seed_asset( db, recall_gate_enabled=True, negative_sample_percent=100, negative_min_reviewed=2, min_recall=0.3, ) positive = _record_shadow( db, claim_id="claim-blind-positive", candidate_hit=True, baseline_hit=False, ) false_negative = _record_shadow( db, claim_id="claim-blind-missed-risk", candidate_hit=False, baseline_hit=False, ) true_negative = _record_shadow( db, claim_id="claim-blind-no-risk", candidate_hit=False, baseline_hit=False, ) db.commit() review = AgentAssetReleaseReviewService(db) queue = review.list_pending(tenant_id="tenant-a", asset_id="release-asset") assert queue["pending_total"] == 3 assert {item["source_document_id"] for item in queue["items"]} == { "claim-blind-positive", "claim-blind-missed-risk", "claim-blind-no-risk", } assert all(item["prediction_blinded"] is True for item in queue["items"]) assert all("candidate_hit" not in item for item in queue["items"]) assert all("baseline_hit" not in item for item in queue["items"]) quorum_by_source = { item["source_document_id"]: item["required_reviewers"] for item in queue["items"] } assert quorum_by_source["claim-blind-positive"] == 1 assert quorum_by_source["claim-blind-missed-risk"] == 2 assert quorum_by_source["claim-blind-no-risk"] == 2 stored_samples = list(db.scalars(select(AgentAssetReleaseAuditSample)).all()) assert len(stored_samples) == 3 assert "claim-blind" not in str( [item.source_reference_encrypted for item in stored_samples] ) for observation, label in ( (positive, "confirmed"), (false_negative, "confirmed"), (true_negative, "false_positive"), ): review.record_label( tenant_id="tenant-a", asset_id="release-asset", observation_id=observation.id, label=label, actor_id="independent-auditor", request_id=f"blind-review:{observation.id}", ) if not observation.candidate_hit: review.record_label( tenant_id="tenant-a", asset_id="release-asset", observation_id=observation.id, label=label, actor_id="second-independent-auditor", request_id=f"blind-review-second:{observation.id}", ) db.commit() aggregate = AgentAssetReleaseTelemetryService(db).aggregate( tenant_id="tenant-a", asset_id="release-asset", release_id="release-1", stage="shadow", version="v2", ) evaluation = aggregate.to_release_evaluation_input() assert aggregate.status == "ready" assert aggregate.false_negative_count == 1 assert aggregate.random_negative_population_count == 2 assert aggregate.random_negative_labeled_count == 2 assert aggregate.recall == 0.5 assert aggregate.recall_lower_bound is not None assert aggregate.recall_lower_bound < aggregate.recall assert ( aggregate.negative_ground_truth_status == "available_stratified_random_audit" ) assert evaluation.details is not None assert evaluation.details["recall"] == 0.5 assert evaluation.details["false_negative_count"] == 1 def test_telemetry_rows_are_append_only(db: Session) -> None: _seed_asset(db) service = AgentAssetReleaseTelemetryService(db) observation = _record_shadow(db, claim_id="claim-immutable") label = service.record_review_label( tenant_id="tenant-a", observation_id=observation.id, label="confirmed", request_id="immutable-review", actor_id="auditor", ) db.commit() observation.candidate_hit = False with pytest.raises(ValueError, match="append-only"): db.flush() db.rollback() persisted_label = db.scalar( select(AgentAssetReleaseLabel).where(AgentAssetReleaseLabel.id == label.id) ) assert persisted_label is not None db.delete(persisted_label) with pytest.raises(ValueError, match="append-only"): db.flush()