feat(platform): close AI expense value loop
Add tenant-safe value, telemetry, connector, commercial, and production-readiness foundations.
This commit is contained in:
caoxiaozhu
871
server/tests/test_agent_asset_release_telemetry.py
Normal file
871
server/tests/test_agent_asset_release_telemetry.py
Normal file
@@ -0,0 +1,871 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.core.agent_enums import AgentAssetDomain, AgentAssetStatus, AgentAssetType
|
||||
from app.db.base import Base
|
||||
from app.models.agent_asset import AgentAsset
|
||||
from app.models.agent_asset_release_telemetry import (
|
||||
AgentAssetReleaseAuditSample,
|
||||
AgentAssetReleaseLabel,
|
||||
AgentAssetReleaseObservation,
|
||||
)
|
||||
from app.models.risk_disposition import RiskDisposition, RiskDispositionEvent
|
||||
from app.models.risk_observation import RiskObservation
|
||||
from app.services.agent_asset_release_disposition_labels import (
|
||||
AgentAssetReleaseDispositionLabelService,
|
||||
)
|
||||
from app.services.agent_asset_release_review import AgentAssetReleaseReviewService
|
||||
from app.services.agent_asset_release_telemetry import (
|
||||
AgentAssetReleaseTelemetryService,
|
||||
ReleaseObservationInput,
|
||||
ReleaseTelemetryCollecting,
|
||||
ReleaseTelemetryIdempotencyConflict,
|
||||
ReleaseTelemetryStaleRelease,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db() -> Generator[Session, None, None]:
|
||||
engine = create_engine(
|
||||
"sqlite+pysqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(
|
||||
engine,
|
||||
tables=[
|
||||
AgentAsset.__table__,
|
||||
RiskObservation.__table__,
|
||||
RiskDisposition.__table__,
|
||||
RiskDispositionEvent.__table__,
|
||||
AgentAssetReleaseObservation.__table__,
|
||||
AgentAssetReleaseLabel.__table__,
|
||||
AgentAssetReleaseAuditSample.__table__,
|
||||
],
|
||||
)
|
||||
factory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
with factory() as session:
|
||||
yield session
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _seed_asset(
|
||||
db: Session,
|
||||
*,
|
||||
stage: str = "shadow",
|
||||
tenant_id: str = "tenant-a",
|
||||
release_id: str = "release-1",
|
||||
reviewer_quorum: int = 1,
|
||||
recall_gate_enabled: bool = False,
|
||||
negative_sample_percent: int = 20,
|
||||
negative_min_reviewed: int = 5,
|
||||
min_recall: float = 0.95,
|
||||
) -> AgentAsset:
|
||||
asset = AgentAsset(
|
||||
id="release-asset",
|
||||
asset_type=AgentAssetType.RULE.value,
|
||||
code="risk.release.telemetry",
|
||||
name="发布遥测规则",
|
||||
description="",
|
||||
domain=AgentAssetDomain.EXPENSE.value,
|
||||
scenario_json=["travel"],
|
||||
owner="finance",
|
||||
status=AgentAssetStatus.ACTIVE.value,
|
||||
current_version="v1",
|
||||
published_version="v1",
|
||||
working_version="v2",
|
||||
config_json={
|
||||
"tenant_id": tenant_id,
|
||||
"detail_mode": "json_risk",
|
||||
"release_guard": {
|
||||
"release_id": release_id,
|
||||
"stage": stage,
|
||||
"candidate_version": "v2",
|
||||
"previous_version": "v1",
|
||||
"started_by": "publisher",
|
||||
"policy": {
|
||||
"reviewer_quorum": reviewer_quorum,
|
||||
"recall_gate_enabled": recall_gate_enabled,
|
||||
"negative_sample_percent": negative_sample_percent,
|
||||
"negative_min_reviewed": negative_min_reviewed,
|
||||
"min_recall": min_recall,
|
||||
"recall_confidence_level": 0.95,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
db.add(asset)
|
||||
db.commit()
|
||||
return asset
|
||||
|
||||
|
||||
def _shadow_result(*, candidate_hit: bool, baseline_hit: bool) -> dict:
|
||||
flags = []
|
||||
if baseline_hit:
|
||||
flags.append(
|
||||
{
|
||||
"rule_code": "risk.release.telemetry",
|
||||
"rule_version": "v1",
|
||||
"release_stage": "shadow",
|
||||
"release_mode": "enforced",
|
||||
}
|
||||
)
|
||||
return {
|
||||
"flags": flags,
|
||||
"shadow_evaluations": [
|
||||
{
|
||||
"asset_id": "release-asset",
|
||||
"rule_code": "risk.release.telemetry",
|
||||
"rule_version": "v2",
|
||||
"release_stage": "shadow",
|
||||
"hit": candidate_hit,
|
||||
"severity": "high" if candidate_hit else "none",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _record_shadow(
|
||||
db: Session,
|
||||
*,
|
||||
claim_id: str,
|
||||
candidate_hit: bool = True,
|
||||
baseline_hit: bool = True,
|
||||
) -> AgentAssetReleaseObservation:
|
||||
return AgentAssetReleaseTelemetryService(db).record_expense_risk_result(
|
||||
tenant_id="tenant-a",
|
||||
claim_id=claim_id,
|
||||
result=_shadow_result(
|
||||
candidate_hit=candidate_hit,
|
||||
baseline_hit=baseline_hit,
|
||||
),
|
||||
)[0]
|
||||
|
||||
|
||||
def test_shadow_runtime_samples_and_labels_build_conservative_release_input(
|
||||
db: Session,
|
||||
) -> None:
|
||||
_seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
|
||||
candidate = _record_shadow(db, claim_id="claim-secret-001")
|
||||
baseline_only = _record_shadow(
|
||||
db,
|
||||
claim_id="claim-secret-002",
|
||||
candidate_hit=False,
|
||||
baseline_hit=True,
|
||||
)
|
||||
replay = _record_shadow(db, claim_id="claim-secret-001")
|
||||
db.commit()
|
||||
|
||||
assert replay.id == candidate.id
|
||||
assert candidate.candidate_hit is True
|
||||
assert candidate.baseline_hit is True
|
||||
assert baseline_only.candidate_hit is False
|
||||
stored_text = str(
|
||||
{
|
||||
column.name: getattr(candidate, column.name)
|
||||
for column in AgentAssetReleaseObservation.__table__.columns
|
||||
}
|
||||
)
|
||||
assert "claim-secret" not in stored_text
|
||||
|
||||
collecting = service.aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
assert collecting.status == "collecting"
|
||||
assert collecting.precision is None
|
||||
assert collecting.recall is None
|
||||
assert collecting.false_negative_count is None
|
||||
assert collecting.negative_ground_truth_status == "collecting_candidate_labels"
|
||||
with pytest.raises(ReleaseTelemetryCollecting):
|
||||
collecting.to_release_evaluation_input()
|
||||
|
||||
candidate_label = service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=candidate.id,
|
||||
label="confirmed",
|
||||
request_id="review-001",
|
||||
actor_id="auditor-secret-account",
|
||||
)
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=baseline_only.id,
|
||||
label="false_positive",
|
||||
request_id="review-002",
|
||||
actor_id="auditor-secret-account",
|
||||
)
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=baseline_only.id,
|
||||
label="false_positive",
|
||||
request_id="review-002-second",
|
||||
actor_id="second-auditor-secret-account",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
assert candidate_label.actor_fingerprint != "auditor-secret-account"
|
||||
assert "auditor-secret-account" not in str(candidate_label.__dict__)
|
||||
ready = service.aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
assert ready.status == "ready"
|
||||
assert ready.observed_count == 2
|
||||
assert ready.candidate_hit_count == 1
|
||||
assert ready.candidate_labeled_count == 1
|
||||
assert ready.precision == 1.0
|
||||
assert ready.baseline_hit_count == 2
|
||||
assert ready.baseline_labeled_count == 2
|
||||
assert ready.baseline_precision == 0.5
|
||||
|
||||
evaluation = ready.to_release_evaluation_input()
|
||||
assert evaluation.total == 2
|
||||
assert evaluation.failure_count == 0
|
||||
assert evaluation.precision == 1.0
|
||||
assert evaluation.baseline_precision == 0.5
|
||||
assert evaluation.details["metric_source"] == "release_runtime_telemetry"
|
||||
assert evaluation.details["recall"] == 1.0
|
||||
assert evaluation.details["recall_lower_bound"] == 1.0
|
||||
|
||||
|
||||
def test_unlabeled_candidate_hit_never_counts_as_release_success(db: Session) -> None:
|
||||
_seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
first = _record_shadow(db, claim_id="claim-1")
|
||||
_record_shadow(db, claim_id="claim-2")
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=first.id,
|
||||
label="confirmed",
|
||||
request_id="review-first",
|
||||
actor_id="auditor",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
aggregate = service.aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
|
||||
assert aggregate.status == "collecting"
|
||||
assert aggregate.candidate_hit_count == 2
|
||||
assert aggregate.candidate_labeled_count == 1
|
||||
assert aggregate.candidate_pending_label_count == 1
|
||||
assert aggregate.baseline_precision is None
|
||||
assert "candidate_labels_pending" in aggregate.reasons
|
||||
with pytest.raises(ReleaseTelemetryCollecting):
|
||||
aggregate.to_release_evaluation_input()
|
||||
|
||||
|
||||
def test_observation_and_label_idempotency_reject_changed_payload(db: Session) -> None:
|
||||
_seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
observation = _record_shadow(db, claim_id="claim-1")
|
||||
|
||||
with pytest.raises(ReleaseTelemetryIdempotencyConflict):
|
||||
service.record_observation(
|
||||
ReleaseObservationInput(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
rule_code="risk.release.telemetry",
|
||||
source_key="claim-1",
|
||||
candidate_hit=False,
|
||||
baseline_hit=True,
|
||||
)
|
||||
)
|
||||
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
request_id="same-review-request",
|
||||
actor_id="auditor",
|
||||
)
|
||||
with pytest.raises(ReleaseTelemetryIdempotencyConflict):
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=observation.id,
|
||||
label="false_positive",
|
||||
request_id="same-review-request",
|
||||
actor_id="auditor",
|
||||
)
|
||||
|
||||
|
||||
def test_release_label_values_cannot_cross_verification_sources(db: Session) -> None:
|
||||
_seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
observation = _record_shadow(db, claim_id="claim-label-semantics")
|
||||
|
||||
with pytest.raises(ValueError, match="release reviews require confirmed"):
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=observation.id,
|
||||
label="risk_present", # type: ignore[arg-type]
|
||||
request_id="wrong-release-review-semantics",
|
||||
actor_id="auditor",
|
||||
)
|
||||
with pytest.raises(ValueError, match="Blind release review requires"):
|
||||
service.record_blind_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=observation.id,
|
||||
ground_truth="confirmed", # type: ignore[arg-type]
|
||||
request_id="wrong-blind-review-semantics",
|
||||
actor_id="auditor",
|
||||
)
|
||||
|
||||
|
||||
def test_observation_rolls_back_when_blind_sample_cannot_be_protected(
|
||||
db: Session,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_seed_asset(db)
|
||||
|
||||
def fail_encryption(_value: str) -> str:
|
||||
raise ValueError("sample encryption unavailable")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.agent_asset_release_sampling.encrypt_secret",
|
||||
fail_encryption,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="sample encryption unavailable"):
|
||||
_record_shadow(db, claim_id="claim-must-not-leak")
|
||||
|
||||
assert db.query(AgentAssetReleaseObservation).count() == 0
|
||||
assert db.query(AgentAssetReleaseAuditSample).count() == 0
|
||||
|
||||
|
||||
def test_cross_tenant_and_stale_release_labels_are_rejected(db: Session) -> None:
|
||||
asset = _seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
observation = _record_shadow(db, claim_id="claim-1")
|
||||
db.commit()
|
||||
|
||||
with pytest.raises(LookupError):
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-b",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
request_id="foreign-review",
|
||||
actor_id="foreign-auditor",
|
||||
)
|
||||
|
||||
config = dict(asset.config_json or {})
|
||||
state = dict(config["release_guard"])
|
||||
state["release_id"] = "release-2"
|
||||
config["release_guard"] = state
|
||||
asset.config_json = config
|
||||
db.add(asset)
|
||||
db.commit()
|
||||
|
||||
with pytest.raises(ReleaseTelemetryStaleRelease):
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
request_id="stale-review",
|
||||
actor_id="auditor",
|
||||
)
|
||||
|
||||
|
||||
def test_typed_risk_disposition_event_is_a_trusted_release_label(db: Session) -> None:
|
||||
_seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
telemetry = _record_shadow(db, claim_id="claim-typed")
|
||||
risk_observation = RiskObservation(
|
||||
id="risk-observation-1",
|
||||
tenant_id="tenant-a",
|
||||
observation_key="risk:claim-typed:platform:risk.release.telemetry",
|
||||
subject_type="expense_claim",
|
||||
subject_key="claim:claim-typed",
|
||||
claim_id="claim-typed",
|
||||
claim_no="REDACTED",
|
||||
risk_type="policy",
|
||||
risk_signal="risk.release.telemetry",
|
||||
risk_score=82,
|
||||
risk_level="high",
|
||||
algorithm_version="v1",
|
||||
decision_trace_json={"rule_code": "risk.release.telemetry"},
|
||||
)
|
||||
disposition = RiskDisposition(
|
||||
id="disposition-1",
|
||||
tenant_id="tenant-a",
|
||||
observation_id=risk_observation.id,
|
||||
adjudication="confirmed",
|
||||
lifecycle_status="open",
|
||||
version=1,
|
||||
)
|
||||
event = RiskDispositionEvent(
|
||||
id="disposition-event-1",
|
||||
tenant_id="tenant-a",
|
||||
disposition_id=disposition.id,
|
||||
observation_id=risk_observation.id,
|
||||
version=1,
|
||||
action="confirm",
|
||||
actor_id="real-auditor-secret",
|
||||
actor_name="真实审计员",
|
||||
request_id="typed-action-1",
|
||||
payload_fingerprint="f" * 64,
|
||||
before_json={"adjudication": "unreviewed"},
|
||||
after_json={"adjudication": "confirmed"},
|
||||
)
|
||||
db.add_all([risk_observation, disposition, event])
|
||||
db.commit()
|
||||
|
||||
label = service.record_risk_disposition_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=telemetry.id,
|
||||
disposition_event_id=event.id,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
assert label.label == "confirmed"
|
||||
assert label.verification_source == "typed_risk_disposition"
|
||||
assert "real-auditor-secret" not in str(label.__dict__)
|
||||
aggregate = service.aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
assert aggregate.status == "ready"
|
||||
assert aggregate.precision == 1.0
|
||||
assert aggregate.baseline_precision == 1.0
|
||||
|
||||
|
||||
def test_typed_disposition_resolves_current_release_sample_without_client_observation_id(
|
||||
db: Session,
|
||||
) -> None:
|
||||
_seed_asset(db)
|
||||
telemetry = _record_shadow(db, claim_id="claim-current-label")
|
||||
risk_observation = RiskObservation(
|
||||
id="risk-observation-current",
|
||||
tenant_id="tenant-a",
|
||||
observation_key="risk:claim-current-label:platform:risk.release.telemetry",
|
||||
subject_type="expense_claim",
|
||||
subject_key="claim:claim-current-label",
|
||||
claim_id="claim-current-label",
|
||||
claim_no="REDACTED",
|
||||
risk_type="policy",
|
||||
risk_signal="risk.release.telemetry",
|
||||
risk_score=82,
|
||||
risk_level="high",
|
||||
control_stage="reimbursement",
|
||||
algorithm_version="v1",
|
||||
decision_trace_json={"rule_code": "risk.release.telemetry"},
|
||||
)
|
||||
disposition = RiskDisposition(
|
||||
id="disposition-current",
|
||||
tenant_id="tenant-a",
|
||||
observation_id=risk_observation.id,
|
||||
adjudication="false_positive",
|
||||
lifecycle_status="open",
|
||||
version=1,
|
||||
)
|
||||
event = RiskDispositionEvent(
|
||||
id="disposition-event-current",
|
||||
tenant_id="tenant-a",
|
||||
disposition_id=disposition.id,
|
||||
observation_id=risk_observation.id,
|
||||
version=1,
|
||||
action="false_positive",
|
||||
actor_id="auditor-secret",
|
||||
actor_name="审计员",
|
||||
request_id="typed-current-action",
|
||||
payload_fingerprint="e" * 64,
|
||||
before_json={"adjudication": "unreviewed"},
|
||||
after_json={"adjudication": "false_positive"},
|
||||
)
|
||||
db.add_all([risk_observation, disposition, event])
|
||||
db.commit()
|
||||
|
||||
labels = AgentAssetReleaseDispositionLabelService(db).record_current_labels(
|
||||
tenant_id="tenant-a",
|
||||
disposition_event_id=event.id,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
assert [item.observation_id for item in labels] == [telemetry.id]
|
||||
assert labels[0].label == "false_positive"
|
||||
assert "auditor-secret" not in str(labels[0].__dict__)
|
||||
|
||||
|
||||
def test_current_disposition_does_not_label_stale_release_sample(db: Session) -> None:
|
||||
asset = _seed_asset(db)
|
||||
_record_shadow(db, claim_id="claim-stale-label")
|
||||
risk_observation = RiskObservation(
|
||||
id="risk-observation-stale-current",
|
||||
tenant_id="tenant-a",
|
||||
observation_key="risk:claim-stale-label:platform:risk.release.telemetry",
|
||||
subject_type="expense_claim",
|
||||
subject_key="claim:claim-stale-label",
|
||||
claim_id="claim-stale-label",
|
||||
claim_no="REDACTED",
|
||||
risk_type="policy",
|
||||
risk_signal="risk.release.telemetry",
|
||||
risk_score=82,
|
||||
risk_level="high",
|
||||
control_stage="reimbursement",
|
||||
algorithm_version="v1",
|
||||
decision_trace_json={"rule_code": "risk.release.telemetry"},
|
||||
)
|
||||
disposition = RiskDisposition(
|
||||
id="disposition-stale-current",
|
||||
tenant_id="tenant-a",
|
||||
observation_id=risk_observation.id,
|
||||
adjudication="confirmed",
|
||||
lifecycle_status="open",
|
||||
version=1,
|
||||
)
|
||||
event = RiskDispositionEvent(
|
||||
id="disposition-event-stale-current",
|
||||
tenant_id="tenant-a",
|
||||
disposition_id=disposition.id,
|
||||
observation_id=risk_observation.id,
|
||||
version=1,
|
||||
action="confirm",
|
||||
actor_id="auditor",
|
||||
actor_name="审计员",
|
||||
request_id="typed-stale-current-action",
|
||||
payload_fingerprint="d" * 64,
|
||||
before_json={"adjudication": "unreviewed"},
|
||||
after_json={"adjudication": "confirmed"},
|
||||
)
|
||||
db.add_all([risk_observation, disposition, event])
|
||||
db.commit()
|
||||
config = dict(asset.config_json or {})
|
||||
state = dict(config["release_guard"])
|
||||
state["release_id"] = "release-2"
|
||||
config["release_guard"] = state
|
||||
asset.config_json = config
|
||||
db.add(asset)
|
||||
db.commit()
|
||||
|
||||
labels = AgentAssetReleaseDispositionLabelService(db).record_current_labels(
|
||||
tenant_id="tenant-a",
|
||||
disposition_event_id=event.id,
|
||||
)
|
||||
|
||||
assert labels == []
|
||||
|
||||
|
||||
def test_canary_hits_and_structured_runtime_failures_are_real_samples(db: Session) -> None:
|
||||
_seed_asset(db, stage="canary")
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
records = service.record_expense_risk_result(
|
||||
tenant_id="tenant-a",
|
||||
claim_id="canary-claim-1",
|
||||
result={
|
||||
"shadow_evaluations": [],
|
||||
"flags": [
|
||||
{
|
||||
"rule_code": "risk.release.telemetry",
|
||||
"rule_version": "v2",
|
||||
"release_stage": "canary",
|
||||
"release_mode": "enforced",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
miss = service.record_manifest_evaluation(
|
||||
tenant_id="tenant-a",
|
||||
claim_id="canary-claim-miss",
|
||||
manifest={
|
||||
"_rule_asset_id": "release-asset",
|
||||
"_release_stage": "canary",
|
||||
"_release_mode": "enforced",
|
||||
"_rule_version": "v2",
|
||||
"rule_code": "risk.release.telemetry",
|
||||
},
|
||||
hit=False,
|
||||
)
|
||||
failure = service.record_observation(
|
||||
ReleaseObservationInput(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="canary",
|
||||
version="v2",
|
||||
rule_code="risk.release.telemetry",
|
||||
source_key="canary-claim-2",
|
||||
candidate_hit=False,
|
||||
baseline_hit=None,
|
||||
runtime_status="failed",
|
||||
failure_code="evaluator_error",
|
||||
)
|
||||
)
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=records[0].id,
|
||||
label="false_positive",
|
||||
request_id="canary-review",
|
||||
actor_id="auditor",
|
||||
)
|
||||
with pytest.raises(ValueError, match="independent ground truth"):
|
||||
service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=miss.id,
|
||||
label="confirmed",
|
||||
request_id="unsupported-false-negative-label",
|
||||
actor_id="auditor",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
aggregate = service.aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="canary",
|
||||
version="v2",
|
||||
)
|
||||
evaluation = aggregate.to_release_evaluation_input()
|
||||
assert records[0].candidate_hit is True
|
||||
assert miss.candidate_hit is False
|
||||
assert failure.runtime_status == "failed"
|
||||
assert aggregate.status == "ready"
|
||||
assert aggregate.runtime_failure_count == 1
|
||||
assert aggregate.precision == 0.0
|
||||
assert aggregate.baseline_precision is None
|
||||
assert evaluation.total == 3
|
||||
assert evaluation.failure_count == 1
|
||||
|
||||
|
||||
def test_dedicated_review_queue_is_tenant_safe_and_separates_publisher(
|
||||
db: Session,
|
||||
) -> None:
|
||||
_seed_asset(db)
|
||||
observation = _record_shadow(db, claim_id="claim-review-queue")
|
||||
service = AgentAssetReleaseReviewService(db)
|
||||
|
||||
with pytest.raises(LookupError):
|
||||
service.list_pending(tenant_id="tenant-b", asset_id="release-asset")
|
||||
queue = service.list_pending(tenant_id="tenant-a", asset_id="release-asset")
|
||||
assert queue["pending_total"] == 1
|
||||
assert queue["items"][0]["observation_id"] == observation.id
|
||||
assert "source_fingerprint" not in queue["items"][0]
|
||||
|
||||
with pytest.raises(PermissionError, match="发布发起人"):
|
||||
service.record_label(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
actor_id="publisher",
|
||||
request_id="publisher-self-review",
|
||||
)
|
||||
label = service.record_label(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
actor_id="independent-reviewer",
|
||||
request_id="independent-review",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
assert label.label == "risk_present"
|
||||
assert service.list_pending(tenant_id="tenant-a", asset_id="release-asset")[
|
||||
"pending_total"
|
||||
] == 0
|
||||
|
||||
|
||||
def test_release_review_quorum_requires_distinct_reviewers(db: Session) -> None:
|
||||
_seed_asset(db, reviewer_quorum=2)
|
||||
observation = _record_shadow(db, claim_id="claim-two-reviewers")
|
||||
service = AgentAssetReleaseReviewService(db)
|
||||
|
||||
service.record_label(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
actor_id="reviewer-one",
|
||||
request_id="review-one",
|
||||
)
|
||||
db.commit()
|
||||
first_queue = service.list_pending(tenant_id="tenant-a", asset_id="release-asset")
|
||||
first_aggregate = AgentAssetReleaseTelemetryService(db).aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
assert first_queue["pending_total"] == 1
|
||||
assert first_queue["items"][0]["reviewer_count"] == 1
|
||||
assert first_queue["items"][0]["required_reviewers"] == 2
|
||||
assert first_aggregate.status == "collecting"
|
||||
|
||||
service.record_label(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
actor_id="reviewer-two",
|
||||
request_id="review-two",
|
||||
)
|
||||
db.commit()
|
||||
second_aggregate = AgentAssetReleaseTelemetryService(db).aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
assert service.list_pending(tenant_id="tenant-a", asset_id="release-asset")[
|
||||
"pending_total"
|
||||
] == 0
|
||||
assert second_aggregate.status == "ready"
|
||||
assert second_aggregate.precision == 1.0
|
||||
|
||||
|
||||
def test_blind_negative_audit_builds_real_false_negative_and_recall_evidence(
|
||||
db: Session,
|
||||
) -> None:
|
||||
_seed_asset(
|
||||
db,
|
||||
recall_gate_enabled=True,
|
||||
negative_sample_percent=100,
|
||||
negative_min_reviewed=2,
|
||||
min_recall=0.3,
|
||||
)
|
||||
positive = _record_shadow(
|
||||
db,
|
||||
claim_id="claim-blind-positive",
|
||||
candidate_hit=True,
|
||||
baseline_hit=False,
|
||||
)
|
||||
false_negative = _record_shadow(
|
||||
db,
|
||||
claim_id="claim-blind-missed-risk",
|
||||
candidate_hit=False,
|
||||
baseline_hit=False,
|
||||
)
|
||||
true_negative = _record_shadow(
|
||||
db,
|
||||
claim_id="claim-blind-no-risk",
|
||||
candidate_hit=False,
|
||||
baseline_hit=False,
|
||||
)
|
||||
db.commit()
|
||||
review = AgentAssetReleaseReviewService(db)
|
||||
|
||||
queue = review.list_pending(tenant_id="tenant-a", asset_id="release-asset")
|
||||
|
||||
assert queue["pending_total"] == 3
|
||||
assert {item["source_document_id"] for item in queue["items"]} == {
|
||||
"claim-blind-positive",
|
||||
"claim-blind-missed-risk",
|
||||
"claim-blind-no-risk",
|
||||
}
|
||||
assert all(item["prediction_blinded"] is True for item in queue["items"])
|
||||
assert all("candidate_hit" not in item for item in queue["items"])
|
||||
assert all("baseline_hit" not in item for item in queue["items"])
|
||||
quorum_by_source = {
|
||||
item["source_document_id"]: item["required_reviewers"]
|
||||
for item in queue["items"]
|
||||
}
|
||||
assert quorum_by_source["claim-blind-positive"] == 1
|
||||
assert quorum_by_source["claim-blind-missed-risk"] == 2
|
||||
assert quorum_by_source["claim-blind-no-risk"] == 2
|
||||
stored_samples = list(db.scalars(select(AgentAssetReleaseAuditSample)).all())
|
||||
assert len(stored_samples) == 3
|
||||
assert "claim-blind" not in str(
|
||||
[item.source_reference_encrypted for item in stored_samples]
|
||||
)
|
||||
|
||||
for observation, label in (
|
||||
(positive, "confirmed"),
|
||||
(false_negative, "confirmed"),
|
||||
(true_negative, "false_positive"),
|
||||
):
|
||||
review.record_label(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
observation_id=observation.id,
|
||||
label=label,
|
||||
actor_id="independent-auditor",
|
||||
request_id=f"blind-review:{observation.id}",
|
||||
)
|
||||
if not observation.candidate_hit:
|
||||
review.record_label(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
observation_id=observation.id,
|
||||
label=label,
|
||||
actor_id="second-independent-auditor",
|
||||
request_id=f"blind-review-second:{observation.id}",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
aggregate = AgentAssetReleaseTelemetryService(db).aggregate(
|
||||
tenant_id="tenant-a",
|
||||
asset_id="release-asset",
|
||||
release_id="release-1",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
)
|
||||
evaluation = aggregate.to_release_evaluation_input()
|
||||
|
||||
assert aggregate.status == "ready"
|
||||
assert aggregate.false_negative_count == 1
|
||||
assert aggregate.random_negative_population_count == 2
|
||||
assert aggregate.random_negative_labeled_count == 2
|
||||
assert aggregate.recall == 0.5
|
||||
assert aggregate.recall_lower_bound is not None
|
||||
assert aggregate.recall_lower_bound < aggregate.recall
|
||||
assert (
|
||||
aggregate.negative_ground_truth_status
|
||||
== "available_stratified_random_audit"
|
||||
)
|
||||
assert evaluation.details is not None
|
||||
assert evaluation.details["recall"] == 0.5
|
||||
assert evaluation.details["false_negative_count"] == 1
|
||||
|
||||
|
||||
def test_telemetry_rows_are_append_only(db: Session) -> None:
|
||||
_seed_asset(db)
|
||||
service = AgentAssetReleaseTelemetryService(db)
|
||||
observation = _record_shadow(db, claim_id="claim-immutable")
|
||||
label = service.record_review_label(
|
||||
tenant_id="tenant-a",
|
||||
observation_id=observation.id,
|
||||
label="confirmed",
|
||||
request_id="immutable-review",
|
||||
actor_id="auditor",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
observation.candidate_hit = False
|
||||
with pytest.raises(ValueError, match="append-only"):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
persisted_label = db.scalar(
|
||||
select(AgentAssetReleaseLabel).where(AgentAssetReleaseLabel.id == label.id)
|
||||
)
|
||||
assert persisted_label is not None
|
||||
db.delete(persisted_label)
|
||||
with pytest.raises(ValueError, match="append-only"):
|
||||
db.flush()
|
||||
Reference in New Issue
Block a user