438 lines
15 KiB
Python
438 lines
15 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import threading
|
||
|
|
import uuid
|
||
|
|
from concurrent.futures import ThreadPoolExecutor
|
||
|
|
from dataclasses import replace
|
||
|
|
|
||
|
|
from savings_postgres_testkit import ( # noqa: F401 - 注册 pg_factory fixture
|
||
|
|
_pg_factory_fixture,
|
||
|
|
)
|
||
|
|
from sqlalchemy import func, select
|
||
|
|
from sqlalchemy.orm import Session, sessionmaker
|
||
|
|
|
||
|
|
from app.core.agent_enums import AgentAssetDomain, AgentAssetStatus, AgentAssetType
|
||
|
|
from app.models.agent_asset import AgentAsset
|
||
|
|
from app.models.agent_asset_release_telemetry import (
|
||
|
|
AgentAssetReleaseAuditSample,
|
||
|
|
AgentAssetReleaseLabel,
|
||
|
|
AgentAssetReleaseObservation,
|
||
|
|
)
|
||
|
|
from app.models.financial_record import ExpenseClaim
|
||
|
|
from app.services.agent_asset_release_review import AgentAssetReleaseReviewService
|
||
|
|
from app.services.agent_asset_release_scheduler import AgentAssetReleaseScheduler
|
||
|
|
from app.services.agent_asset_release_telemetry import (
|
||
|
|
AgentAssetReleaseTelemetryService,
|
||
|
|
ReleaseObservationInput,
|
||
|
|
ReleaseTelemetryIdempotencyConflict,
|
||
|
|
ReleaseTelemetryStaleRelease,
|
||
|
|
)
|
||
|
|
from app.services.expense_claim_release_telemetry import (
|
||
|
|
ExpenseClaimReleaseTelemetryRecorder,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_concurrent_observation_and_label_replay_create_one_fact(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||
|
|
payload = _payload(tenant_id, asset_id, rule_code, source="claim-same")
|
||
|
|
ready = threading.Barrier(2)
|
||
|
|
|
||
|
|
def record_observation() -> str:
|
||
|
|
with pg_factory() as db:
|
||
|
|
ready.wait(timeout=5)
|
||
|
|
row = AgentAssetReleaseTelemetryService(db).record_observation(payload)
|
||
|
|
db.commit()
|
||
|
|
return row.id
|
||
|
|
|
||
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
|
|
observation_ids = [
|
||
|
|
future.result(timeout=10)
|
||
|
|
for future in (
|
||
|
|
pool.submit(record_observation),
|
||
|
|
pool.submit(record_observation),
|
||
|
|
)
|
||
|
|
]
|
||
|
|
assert len(set(observation_ids)) == 1
|
||
|
|
observation_id = observation_ids[0]
|
||
|
|
|
||
|
|
label_ready = threading.Barrier(2)
|
||
|
|
|
||
|
|
def record_label() -> str:
|
||
|
|
with pg_factory() as db:
|
||
|
|
label_ready.wait(timeout=5)
|
||
|
|
row = AgentAssetReleaseTelemetryService(db).record_review_label(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
observation_id=observation_id,
|
||
|
|
label="confirmed",
|
||
|
|
request_id="review-concurrent-same",
|
||
|
|
actor_id="trusted-reviewer",
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
return row.id
|
||
|
|
|
||
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
|
|
label_ids = [
|
||
|
|
future.result(timeout=10)
|
||
|
|
for future in (pool.submit(record_label), pool.submit(record_label))
|
||
|
|
]
|
||
|
|
assert len(set(label_ids)) == 1
|
||
|
|
|
||
|
|
with pg_factory() as db:
|
||
|
|
assert db.scalar(
|
||
|
|
select(func.count())
|
||
|
|
.select_from(AgentAssetReleaseObservation)
|
||
|
|
.where(
|
||
|
|
AgentAssetReleaseObservation.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseObservation.id == observation_id,
|
||
|
|
)
|
||
|
|
) == 1
|
||
|
|
assert db.scalar(
|
||
|
|
select(func.count())
|
||
|
|
.select_from(AgentAssetReleaseAuditSample)
|
||
|
|
.where(
|
||
|
|
AgentAssetReleaseAuditSample.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseAuditSample.observation_id == observation_id,
|
||
|
|
)
|
||
|
|
) == 1
|
||
|
|
assert db.scalar(
|
||
|
|
select(func.count())
|
||
|
|
.select_from(AgentAssetReleaseLabel)
|
||
|
|
.where(
|
||
|
|
AgentAssetReleaseLabel.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseLabel.observation_id == observation_id,
|
||
|
|
)
|
||
|
|
) == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_stage_transition_wins_before_label_and_rejects_stale_fact(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||
|
|
with pg_factory() as db:
|
||
|
|
observation = AgentAssetReleaseTelemetryService(db).record_observation(
|
||
|
|
_payload(tenant_id, asset_id, rule_code, source="claim-stage-race")
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
observation_id = observation.id
|
||
|
|
|
||
|
|
label_started = threading.Event()
|
||
|
|
|
||
|
|
def record_stale_label() -> str:
|
||
|
|
with pg_factory() as db:
|
||
|
|
label_started.set()
|
||
|
|
try:
|
||
|
|
AgentAssetReleaseTelemetryService(db).record_review_label(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
observation_id=observation_id,
|
||
|
|
label="confirmed",
|
||
|
|
request_id="review-after-stage-transition",
|
||
|
|
actor_id="trusted-reviewer",
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
return "created"
|
||
|
|
except ReleaseTelemetryStaleRelease:
|
||
|
|
db.rollback()
|
||
|
|
return "stale"
|
||
|
|
|
||
|
|
with pg_factory() as transition_db:
|
||
|
|
asset = transition_db.scalar(
|
||
|
|
select(AgentAsset).where(AgentAsset.id == asset_id).with_for_update()
|
||
|
|
)
|
||
|
|
assert asset is not None
|
||
|
|
config = dict(asset.config_json or {})
|
||
|
|
state = dict(config["release_guard"])
|
||
|
|
state["stage"] = "canary"
|
||
|
|
config["release_guard"] = state
|
||
|
|
asset.config_json = config
|
||
|
|
transition_db.flush()
|
||
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
||
|
|
future = pool.submit(record_stale_label)
|
||
|
|
assert label_started.wait(timeout=5)
|
||
|
|
transition_db.commit()
|
||
|
|
outcome = future.result(timeout=10)
|
||
|
|
|
||
|
|
assert outcome == "stale"
|
||
|
|
with pg_factory() as db:
|
||
|
|
assert db.scalar(
|
||
|
|
select(func.count())
|
||
|
|
.select_from(AgentAssetReleaseLabel)
|
||
|
|
.where(
|
||
|
|
AgentAssetReleaseLabel.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseLabel.observation_id == observation_id,
|
||
|
|
)
|
||
|
|
) == 0
|
||
|
|
|
||
|
|
|
||
|
|
def test_concurrent_conflicting_observation_payload_has_one_winner(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||
|
|
first = _payload(tenant_id, asset_id, rule_code, source="claim-conflict")
|
||
|
|
second = replace(first, candidate_hit=False)
|
||
|
|
ready = threading.Barrier(2)
|
||
|
|
|
||
|
|
def record_once(payload: ReleaseObservationInput) -> str:
|
||
|
|
with pg_factory() as db:
|
||
|
|
ready.wait(timeout=5)
|
||
|
|
try:
|
||
|
|
AgentAssetReleaseTelemetryService(db).record_observation(payload)
|
||
|
|
db.commit()
|
||
|
|
return "created"
|
||
|
|
except ReleaseTelemetryIdempotencyConflict:
|
||
|
|
db.rollback()
|
||
|
|
return "conflict"
|
||
|
|
|
||
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
|
|
outcomes = [
|
||
|
|
future.result(timeout=10)
|
||
|
|
for future in (
|
||
|
|
pool.submit(record_once, first),
|
||
|
|
pool.submit(record_once, second),
|
||
|
|
)
|
||
|
|
]
|
||
|
|
assert sorted(outcomes) == ["conflict", "created"]
|
||
|
|
with pg_factory() as db:
|
||
|
|
assert db.scalar(
|
||
|
|
select(func.count())
|
||
|
|
.select_from(AgentAssetReleaseObservation)
|
||
|
|
.where(
|
||
|
|
AgentAssetReleaseObservation.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseObservation.asset_id == asset_id,
|
||
|
|
)
|
||
|
|
) == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_concurrent_conflicting_label_payload_has_one_winner(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||
|
|
with pg_factory() as db:
|
||
|
|
observation = AgentAssetReleaseTelemetryService(db).record_observation(
|
||
|
|
_payload(tenant_id, asset_id, rule_code, source="claim-label-conflict")
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
observation_id = observation.id
|
||
|
|
ready = threading.Barrier(2)
|
||
|
|
|
||
|
|
def record_once(label: str) -> str:
|
||
|
|
with pg_factory() as db:
|
||
|
|
ready.wait(timeout=5)
|
||
|
|
try:
|
||
|
|
AgentAssetReleaseTelemetryService(db).record_review_label(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
observation_id=observation_id,
|
||
|
|
label=label, # type: ignore[arg-type]
|
||
|
|
request_id="review-conflicting-same-request",
|
||
|
|
actor_id="trusted-reviewer",
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
return "created"
|
||
|
|
except ReleaseTelemetryIdempotencyConflict:
|
||
|
|
db.rollback()
|
||
|
|
return "conflict"
|
||
|
|
|
||
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
|
|
outcomes = [
|
||
|
|
future.result(timeout=10)
|
||
|
|
for future in (
|
||
|
|
pool.submit(record_once, "confirmed"),
|
||
|
|
pool.submit(record_once, "false_positive"),
|
||
|
|
)
|
||
|
|
]
|
||
|
|
assert sorted(outcomes) == ["conflict", "created"]
|
||
|
|
with pg_factory() as db:
|
||
|
|
assert db.scalar(
|
||
|
|
select(func.count())
|
||
|
|
.select_from(AgentAssetReleaseLabel)
|
||
|
|
.where(
|
||
|
|
AgentAssetReleaseLabel.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseLabel.observation_id == observation_id,
|
||
|
|
)
|
||
|
|
) == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_negative_blind_review_requires_two_distinct_actors_under_concurrency(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||
|
|
with pg_factory() as db:
|
||
|
|
observation = AgentAssetReleaseTelemetryService(db).record_observation(
|
||
|
|
replace(
|
||
|
|
_payload(
|
||
|
|
tenant_id,
|
||
|
|
asset_id,
|
||
|
|
rule_code,
|
||
|
|
source="claim-negative-double-review",
|
||
|
|
),
|
||
|
|
candidate_hit=False,
|
||
|
|
baseline_hit=True,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
observation_id = observation.id
|
||
|
|
|
||
|
|
ready = threading.Barrier(2)
|
||
|
|
|
||
|
|
def record_same_actor(request_id: str) -> str:
|
||
|
|
with pg_factory() as db:
|
||
|
|
ready.wait(timeout=5)
|
||
|
|
row = AgentAssetReleaseReviewService(db).record_label(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
asset_id=asset_id,
|
||
|
|
observation_id=observation_id,
|
||
|
|
label="risk_present",
|
||
|
|
actor_id="same-independent-reviewer",
|
||
|
|
request_id=request_id,
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
return row.id
|
||
|
|
|
||
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
|
|
same_actor_label_ids = [
|
||
|
|
future.result(timeout=10)
|
||
|
|
for future in (
|
||
|
|
pool.submit(record_same_actor, "same-actor-review-a"),
|
||
|
|
pool.submit(record_same_actor, "same-actor-review-b"),
|
||
|
|
)
|
||
|
|
]
|
||
|
|
assert len(set(same_actor_label_ids)) == 2
|
||
|
|
|
||
|
|
with pg_factory() as db:
|
||
|
|
queue = AgentAssetReleaseReviewService(db).list_pending(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
asset_id=asset_id,
|
||
|
|
)
|
||
|
|
assert queue["pending_total"] == 1
|
||
|
|
assert queue["items"][0]["reviewer_count"] == 1
|
||
|
|
assert queue["items"][0]["required_reviewers"] == 2
|
||
|
|
|
||
|
|
AgentAssetReleaseReviewService(db).record_label(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
asset_id=asset_id,
|
||
|
|
observation_id=observation_id,
|
||
|
|
label="risk_present",
|
||
|
|
actor_id="second-independent-reviewer",
|
||
|
|
request_id="second-actor-review",
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
|
||
|
|
resolved = AgentAssetReleaseReviewService(db).list_pending(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
asset_id=asset_id,
|
||
|
|
)
|
||
|
|
assert resolved["pending_total"] == 0
|
||
|
|
|
||
|
|
|
||
|
|
def test_failed_observation_survives_business_transaction_rollback(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||
|
|
claim = ExpenseClaim(id="claim-durable-failure", claim_no="BX-DURABLE-FAILURE")
|
||
|
|
manifest = {
|
||
|
|
"_rule_asset_id": asset_id,
|
||
|
|
"_release_stage": "shadow",
|
||
|
|
"_release_mode": "shadow",
|
||
|
|
"_rule_version": "v2",
|
||
|
|
"rule_code": rule_code,
|
||
|
|
}
|
||
|
|
with pg_factory() as business_db:
|
||
|
|
asset = business_db.get(AgentAsset, asset_id)
|
||
|
|
assert asset is not None
|
||
|
|
asset.description = "this business mutation must roll back"
|
||
|
|
business_db.flush()
|
||
|
|
|
||
|
|
ExpenseClaimReleaseTelemetryRecorder(business_db).record_failure_durably(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
claim=claim,
|
||
|
|
manifest=manifest,
|
||
|
|
failure_code="evaluator_error",
|
||
|
|
business_stage="reimbursement",
|
||
|
|
)
|
||
|
|
business_db.rollback()
|
||
|
|
|
||
|
|
with pg_factory() as db:
|
||
|
|
asset = db.get(AgentAsset, asset_id)
|
||
|
|
observation = db.scalar(
|
||
|
|
select(AgentAssetReleaseObservation).where(
|
||
|
|
AgentAssetReleaseObservation.tenant_id == tenant_id,
|
||
|
|
AgentAssetReleaseObservation.asset_id == asset_id,
|
||
|
|
AgentAssetReleaseObservation.runtime_status == "failed",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
assert asset is not None and asset.description == ""
|
||
|
|
assert observation is not None
|
||
|
|
assert observation.failure_code == "evaluator_error"
|
||
|
|
|
||
|
|
|
||
|
|
def test_release_scheduler_advisory_lease_has_single_owner_and_transfers(
|
||
|
|
pg_factory: sessionmaker[Session],
|
||
|
|
) -> None:
|
||
|
|
scheduler = AgentAssetReleaseScheduler(session_factory=pg_factory)
|
||
|
|
with pg_factory() as first, pg_factory() as second:
|
||
|
|
assert scheduler._try_acquire_lease(first) is True
|
||
|
|
assert scheduler._try_acquire_lease(second) is False
|
||
|
|
|
||
|
|
scheduler._release_lease(first)
|
||
|
|
|
||
|
|
assert scheduler._try_acquire_lease(second) is True
|
||
|
|
scheduler._release_lease(second)
|
||
|
|
|
||
|
|
|
||
|
|
def _seed_asset(factory: sessionmaker[Session]) -> tuple[str, str, str]:
|
||
|
|
suffix = uuid.uuid4().hex
|
||
|
|
tenant_id = f"tenant-release-{suffix}"
|
||
|
|
asset_id = str(uuid.uuid4())
|
||
|
|
rule_code = f"risk.release.{suffix}"
|
||
|
|
with factory() as db:
|
||
|
|
db.add(
|
||
|
|
AgentAsset(
|
||
|
|
id=asset_id,
|
||
|
|
asset_type=AgentAssetType.RULE.value,
|
||
|
|
code=rule_code,
|
||
|
|
name="发布遥测并发规则",
|
||
|
|
description="",
|
||
|
|
domain=AgentAssetDomain.EXPENSE.value,
|
||
|
|
scenario_json=["travel"],
|
||
|
|
owner="postgres-test",
|
||
|
|
status=AgentAssetStatus.ACTIVE.value,
|
||
|
|
current_version="v1",
|
||
|
|
published_version="v1",
|
||
|
|
working_version="v2",
|
||
|
|
config_json={
|
||
|
|
"tenant_id": tenant_id,
|
||
|
|
"detail_mode": "json_risk",
|
||
|
|
"enabled": True,
|
||
|
|
"release_guard": {
|
||
|
|
"release_id": f"release-{suffix}",
|
||
|
|
"stage": "shadow",
|
||
|
|
"candidate_version": "v2",
|
||
|
|
"previous_version": "v1",
|
||
|
|
},
|
||
|
|
},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
db.commit()
|
||
|
|
return tenant_id, asset_id, rule_code
|
||
|
|
|
||
|
|
|
||
|
|
def _payload(
|
||
|
|
tenant_id: str,
|
||
|
|
asset_id: str,
|
||
|
|
rule_code: str,
|
||
|
|
*,
|
||
|
|
source: str,
|
||
|
|
) -> ReleaseObservationInput:
|
||
|
|
return ReleaseObservationInput(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
asset_id=asset_id,
|
||
|
|
release_id=f"release-{rule_code.removeprefix('risk.release.')}",
|
||
|
|
stage="shadow",
|
||
|
|
version="v2",
|
||
|
|
rule_code=rule_code,
|
||
|
|
source_key=source,
|
||
|
|
candidate_hit=True,
|
||
|
|
baseline_hit=True,
|
||
|
|
)
|