feat(platform): close AI expense value loop
Add tenant-safe value, telemetry, connector, commercial, and production-readiness foundations.
This commit is contained in:
caoxiaozhu
@@ -0,0 +1,437 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
|
||||
from savings_postgres_testkit import ( # noqa: F401 - 注册 pg_factory fixture
|
||||
_pg_factory_fixture,
|
||||
)
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.core.agent_enums import AgentAssetDomain, AgentAssetStatus, AgentAssetType
|
||||
from app.models.agent_asset import AgentAsset
|
||||
from app.models.agent_asset_release_telemetry import (
|
||||
AgentAssetReleaseAuditSample,
|
||||
AgentAssetReleaseLabel,
|
||||
AgentAssetReleaseObservation,
|
||||
)
|
||||
from app.models.financial_record import ExpenseClaim
|
||||
from app.services.agent_asset_release_review import AgentAssetReleaseReviewService
|
||||
from app.services.agent_asset_release_scheduler import AgentAssetReleaseScheduler
|
||||
from app.services.agent_asset_release_telemetry import (
|
||||
AgentAssetReleaseTelemetryService,
|
||||
ReleaseObservationInput,
|
||||
ReleaseTelemetryIdempotencyConflict,
|
||||
ReleaseTelemetryStaleRelease,
|
||||
)
|
||||
from app.services.expense_claim_release_telemetry import (
|
||||
ExpenseClaimReleaseTelemetryRecorder,
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_observation_and_label_replay_create_one_fact(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||||
payload = _payload(tenant_id, asset_id, rule_code, source="claim-same")
|
||||
ready = threading.Barrier(2)
|
||||
|
||||
def record_observation() -> str:
|
||||
with pg_factory() as db:
|
||||
ready.wait(timeout=5)
|
||||
row = AgentAssetReleaseTelemetryService(db).record_observation(payload)
|
||||
db.commit()
|
||||
return row.id
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
observation_ids = [
|
||||
future.result(timeout=10)
|
||||
for future in (
|
||||
pool.submit(record_observation),
|
||||
pool.submit(record_observation),
|
||||
)
|
||||
]
|
||||
assert len(set(observation_ids)) == 1
|
||||
observation_id = observation_ids[0]
|
||||
|
||||
label_ready = threading.Barrier(2)
|
||||
|
||||
def record_label() -> str:
|
||||
with pg_factory() as db:
|
||||
label_ready.wait(timeout=5)
|
||||
row = AgentAssetReleaseTelemetryService(db).record_review_label(
|
||||
tenant_id=tenant_id,
|
||||
observation_id=observation_id,
|
||||
label="confirmed",
|
||||
request_id="review-concurrent-same",
|
||||
actor_id="trusted-reviewer",
|
||||
)
|
||||
db.commit()
|
||||
return row.id
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
label_ids = [
|
||||
future.result(timeout=10)
|
||||
for future in (pool.submit(record_label), pool.submit(record_label))
|
||||
]
|
||||
assert len(set(label_ids)) == 1
|
||||
|
||||
with pg_factory() as db:
|
||||
assert db.scalar(
|
||||
select(func.count())
|
||||
.select_from(AgentAssetReleaseObservation)
|
||||
.where(
|
||||
AgentAssetReleaseObservation.tenant_id == tenant_id,
|
||||
AgentAssetReleaseObservation.id == observation_id,
|
||||
)
|
||||
) == 1
|
||||
assert db.scalar(
|
||||
select(func.count())
|
||||
.select_from(AgentAssetReleaseAuditSample)
|
||||
.where(
|
||||
AgentAssetReleaseAuditSample.tenant_id == tenant_id,
|
||||
AgentAssetReleaseAuditSample.observation_id == observation_id,
|
||||
)
|
||||
) == 1
|
||||
assert db.scalar(
|
||||
select(func.count())
|
||||
.select_from(AgentAssetReleaseLabel)
|
||||
.where(
|
||||
AgentAssetReleaseLabel.tenant_id == tenant_id,
|
||||
AgentAssetReleaseLabel.observation_id == observation_id,
|
||||
)
|
||||
) == 1
|
||||
|
||||
|
||||
def test_stage_transition_wins_before_label_and_rejects_stale_fact(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||||
with pg_factory() as db:
|
||||
observation = AgentAssetReleaseTelemetryService(db).record_observation(
|
||||
_payload(tenant_id, asset_id, rule_code, source="claim-stage-race")
|
||||
)
|
||||
db.commit()
|
||||
observation_id = observation.id
|
||||
|
||||
label_started = threading.Event()
|
||||
|
||||
def record_stale_label() -> str:
|
||||
with pg_factory() as db:
|
||||
label_started.set()
|
||||
try:
|
||||
AgentAssetReleaseTelemetryService(db).record_review_label(
|
||||
tenant_id=tenant_id,
|
||||
observation_id=observation_id,
|
||||
label="confirmed",
|
||||
request_id="review-after-stage-transition",
|
||||
actor_id="trusted-reviewer",
|
||||
)
|
||||
db.commit()
|
||||
return "created"
|
||||
except ReleaseTelemetryStaleRelease:
|
||||
db.rollback()
|
||||
return "stale"
|
||||
|
||||
with pg_factory() as transition_db:
|
||||
asset = transition_db.scalar(
|
||||
select(AgentAsset).where(AgentAsset.id == asset_id).with_for_update()
|
||||
)
|
||||
assert asset is not None
|
||||
config = dict(asset.config_json or {})
|
||||
state = dict(config["release_guard"])
|
||||
state["stage"] = "canary"
|
||||
config["release_guard"] = state
|
||||
asset.config_json = config
|
||||
transition_db.flush()
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(record_stale_label)
|
||||
assert label_started.wait(timeout=5)
|
||||
transition_db.commit()
|
||||
outcome = future.result(timeout=10)
|
||||
|
||||
assert outcome == "stale"
|
||||
with pg_factory() as db:
|
||||
assert db.scalar(
|
||||
select(func.count())
|
||||
.select_from(AgentAssetReleaseLabel)
|
||||
.where(
|
||||
AgentAssetReleaseLabel.tenant_id == tenant_id,
|
||||
AgentAssetReleaseLabel.observation_id == observation_id,
|
||||
)
|
||||
) == 0
|
||||
|
||||
|
||||
def test_concurrent_conflicting_observation_payload_has_one_winner(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||||
first = _payload(tenant_id, asset_id, rule_code, source="claim-conflict")
|
||||
second = replace(first, candidate_hit=False)
|
||||
ready = threading.Barrier(2)
|
||||
|
||||
def record_once(payload: ReleaseObservationInput) -> str:
|
||||
with pg_factory() as db:
|
||||
ready.wait(timeout=5)
|
||||
try:
|
||||
AgentAssetReleaseTelemetryService(db).record_observation(payload)
|
||||
db.commit()
|
||||
return "created"
|
||||
except ReleaseTelemetryIdempotencyConflict:
|
||||
db.rollback()
|
||||
return "conflict"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
outcomes = [
|
||||
future.result(timeout=10)
|
||||
for future in (
|
||||
pool.submit(record_once, first),
|
||||
pool.submit(record_once, second),
|
||||
)
|
||||
]
|
||||
assert sorted(outcomes) == ["conflict", "created"]
|
||||
with pg_factory() as db:
|
||||
assert db.scalar(
|
||||
select(func.count())
|
||||
.select_from(AgentAssetReleaseObservation)
|
||||
.where(
|
||||
AgentAssetReleaseObservation.tenant_id == tenant_id,
|
||||
AgentAssetReleaseObservation.asset_id == asset_id,
|
||||
)
|
||||
) == 1
|
||||
|
||||
|
||||
def test_concurrent_conflicting_label_payload_has_one_winner(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||||
with pg_factory() as db:
|
||||
observation = AgentAssetReleaseTelemetryService(db).record_observation(
|
||||
_payload(tenant_id, asset_id, rule_code, source="claim-label-conflict")
|
||||
)
|
||||
db.commit()
|
||||
observation_id = observation.id
|
||||
ready = threading.Barrier(2)
|
||||
|
||||
def record_once(label: str) -> str:
|
||||
with pg_factory() as db:
|
||||
ready.wait(timeout=5)
|
||||
try:
|
||||
AgentAssetReleaseTelemetryService(db).record_review_label(
|
||||
tenant_id=tenant_id,
|
||||
observation_id=observation_id,
|
||||
label=label, # type: ignore[arg-type]
|
||||
request_id="review-conflicting-same-request",
|
||||
actor_id="trusted-reviewer",
|
||||
)
|
||||
db.commit()
|
||||
return "created"
|
||||
except ReleaseTelemetryIdempotencyConflict:
|
||||
db.rollback()
|
||||
return "conflict"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
outcomes = [
|
||||
future.result(timeout=10)
|
||||
for future in (
|
||||
pool.submit(record_once, "confirmed"),
|
||||
pool.submit(record_once, "false_positive"),
|
||||
)
|
||||
]
|
||||
assert sorted(outcomes) == ["conflict", "created"]
|
||||
with pg_factory() as db:
|
||||
assert db.scalar(
|
||||
select(func.count())
|
||||
.select_from(AgentAssetReleaseLabel)
|
||||
.where(
|
||||
AgentAssetReleaseLabel.tenant_id == tenant_id,
|
||||
AgentAssetReleaseLabel.observation_id == observation_id,
|
||||
)
|
||||
) == 1
|
||||
|
||||
|
||||
def test_negative_blind_review_requires_two_distinct_actors_under_concurrency(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||||
with pg_factory() as db:
|
||||
observation = AgentAssetReleaseTelemetryService(db).record_observation(
|
||||
replace(
|
||||
_payload(
|
||||
tenant_id,
|
||||
asset_id,
|
||||
rule_code,
|
||||
source="claim-negative-double-review",
|
||||
),
|
||||
candidate_hit=False,
|
||||
baseline_hit=True,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
observation_id = observation.id
|
||||
|
||||
ready = threading.Barrier(2)
|
||||
|
||||
def record_same_actor(request_id: str) -> str:
|
||||
with pg_factory() as db:
|
||||
ready.wait(timeout=5)
|
||||
row = AgentAssetReleaseReviewService(db).record_label(
|
||||
tenant_id=tenant_id,
|
||||
asset_id=asset_id,
|
||||
observation_id=observation_id,
|
||||
label="risk_present",
|
||||
actor_id="same-independent-reviewer",
|
||||
request_id=request_id,
|
||||
)
|
||||
db.commit()
|
||||
return row.id
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
same_actor_label_ids = [
|
||||
future.result(timeout=10)
|
||||
for future in (
|
||||
pool.submit(record_same_actor, "same-actor-review-a"),
|
||||
pool.submit(record_same_actor, "same-actor-review-b"),
|
||||
)
|
||||
]
|
||||
assert len(set(same_actor_label_ids)) == 2
|
||||
|
||||
with pg_factory() as db:
|
||||
queue = AgentAssetReleaseReviewService(db).list_pending(
|
||||
tenant_id=tenant_id,
|
||||
asset_id=asset_id,
|
||||
)
|
||||
assert queue["pending_total"] == 1
|
||||
assert queue["items"][0]["reviewer_count"] == 1
|
||||
assert queue["items"][0]["required_reviewers"] == 2
|
||||
|
||||
AgentAssetReleaseReviewService(db).record_label(
|
||||
tenant_id=tenant_id,
|
||||
asset_id=asset_id,
|
||||
observation_id=observation_id,
|
||||
label="risk_present",
|
||||
actor_id="second-independent-reviewer",
|
||||
request_id="second-actor-review",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
resolved = AgentAssetReleaseReviewService(db).list_pending(
|
||||
tenant_id=tenant_id,
|
||||
asset_id=asset_id,
|
||||
)
|
||||
assert resolved["pending_total"] == 0
|
||||
|
||||
|
||||
def test_failed_observation_survives_business_transaction_rollback(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
tenant_id, asset_id, rule_code = _seed_asset(pg_factory)
|
||||
claim = ExpenseClaim(id="claim-durable-failure", claim_no="BX-DURABLE-FAILURE")
|
||||
manifest = {
|
||||
"_rule_asset_id": asset_id,
|
||||
"_release_stage": "shadow",
|
||||
"_release_mode": "shadow",
|
||||
"_rule_version": "v2",
|
||||
"rule_code": rule_code,
|
||||
}
|
||||
with pg_factory() as business_db:
|
||||
asset = business_db.get(AgentAsset, asset_id)
|
||||
assert asset is not None
|
||||
asset.description = "this business mutation must roll back"
|
||||
business_db.flush()
|
||||
|
||||
ExpenseClaimReleaseTelemetryRecorder(business_db).record_failure_durably(
|
||||
tenant_id=tenant_id,
|
||||
claim=claim,
|
||||
manifest=manifest,
|
||||
failure_code="evaluator_error",
|
||||
business_stage="reimbursement",
|
||||
)
|
||||
business_db.rollback()
|
||||
|
||||
with pg_factory() as db:
|
||||
asset = db.get(AgentAsset, asset_id)
|
||||
observation = db.scalar(
|
||||
select(AgentAssetReleaseObservation).where(
|
||||
AgentAssetReleaseObservation.tenant_id == tenant_id,
|
||||
AgentAssetReleaseObservation.asset_id == asset_id,
|
||||
AgentAssetReleaseObservation.runtime_status == "failed",
|
||||
)
|
||||
)
|
||||
assert asset is not None and asset.description == ""
|
||||
assert observation is not None
|
||||
assert observation.failure_code == "evaluator_error"
|
||||
|
||||
|
||||
def test_release_scheduler_advisory_lease_has_single_owner_and_transfers(
|
||||
pg_factory: sessionmaker[Session],
|
||||
) -> None:
|
||||
scheduler = AgentAssetReleaseScheduler(session_factory=pg_factory)
|
||||
with pg_factory() as first, pg_factory() as second:
|
||||
assert scheduler._try_acquire_lease(first) is True
|
||||
assert scheduler._try_acquire_lease(second) is False
|
||||
|
||||
scheduler._release_lease(first)
|
||||
|
||||
assert scheduler._try_acquire_lease(second) is True
|
||||
scheduler._release_lease(second)
|
||||
|
||||
|
||||
def _seed_asset(factory: sessionmaker[Session]) -> tuple[str, str, str]:
|
||||
suffix = uuid.uuid4().hex
|
||||
tenant_id = f"tenant-release-{suffix}"
|
||||
asset_id = str(uuid.uuid4())
|
||||
rule_code = f"risk.release.{suffix}"
|
||||
with factory() as db:
|
||||
db.add(
|
||||
AgentAsset(
|
||||
id=asset_id,
|
||||
asset_type=AgentAssetType.RULE.value,
|
||||
code=rule_code,
|
||||
name="发布遥测并发规则",
|
||||
description="",
|
||||
domain=AgentAssetDomain.EXPENSE.value,
|
||||
scenario_json=["travel"],
|
||||
owner="postgres-test",
|
||||
status=AgentAssetStatus.ACTIVE.value,
|
||||
current_version="v1",
|
||||
published_version="v1",
|
||||
working_version="v2",
|
||||
config_json={
|
||||
"tenant_id": tenant_id,
|
||||
"detail_mode": "json_risk",
|
||||
"enabled": True,
|
||||
"release_guard": {
|
||||
"release_id": f"release-{suffix}",
|
||||
"stage": "shadow",
|
||||
"candidate_version": "v2",
|
||||
"previous_version": "v1",
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return tenant_id, asset_id, rule_code
|
||||
|
||||
|
||||
def _payload(
|
||||
tenant_id: str,
|
||||
asset_id: str,
|
||||
rule_code: str,
|
||||
*,
|
||||
source: str,
|
||||
) -> ReleaseObservationInput:
|
||||
return ReleaseObservationInput(
|
||||
tenant_id=tenant_id,
|
||||
asset_id=asset_id,
|
||||
release_id=f"release-{rule_code.removeprefix('risk.release.')}",
|
||||
stage="shadow",
|
||||
version="v2",
|
||||
rule_code=rule_code,
|
||||
source_key=source,
|
||||
candidate_hit=True,
|
||||
baseline_hit=True,
|
||||
)
|
||||
Reference in New Issue
Block a user