78 lines
2.8 KiB
Python
78 lines
2.8 KiB
Python
|
"""Control effect analysis for risk rules, sampling, and digital employees."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
HIGH_LEVELS = {"high", "critical"}
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(slots=True)
|
||
|
|
class ControlEffectSummary:
|
||
|
|
before_count: int
|
||
|
|
after_count: int
|
||
|
|
risk_count_delta: int
|
||
|
|
average_score_delta: float
|
||
|
|
high_rate_delta: float
|
||
|
|
confirmation_rate_delta: float
|
||
|
|
false_positive_rate_delta: float
|
||
|
|
|
||
|
|
def as_dict(self) -> dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"before_count": self.before_count,
|
||
|
|
"after_count": self.after_count,
|
||
|
|
"risk_count_delta": self.risk_count_delta,
|
||
|
|
"average_score_delta": self.average_score_delta,
|
||
|
|
"high_rate_delta": self.high_rate_delta,
|
||
|
|
"confirmation_rate_delta": self.confirmation_rate_delta,
|
||
|
|
"false_positive_rate_delta": self.false_positive_rate_delta,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
class ControlEffectAnalyzer:
|
||
|
|
def compare(
|
||
|
|
self,
|
||
|
|
before: list[dict[str, Any]],
|
||
|
|
after: list[dict[str, Any]],
|
||
|
|
) -> ControlEffectSummary:
|
||
|
|
before_metrics = _metrics(before)
|
||
|
|
after_metrics = _metrics(after)
|
||
|
|
return ControlEffectSummary(
|
||
|
|
before_count=before_metrics["count"],
|
||
|
|
after_count=after_metrics["count"],
|
||
|
|
risk_count_delta=after_metrics["count"] - before_metrics["count"],
|
||
|
|
average_score_delta=round(after_metrics["average_score"] - before_metrics["average_score"], 4),
|
||
|
|
high_rate_delta=round(after_metrics["high_rate"] - before_metrics["high_rate"], 4),
|
||
|
|
confirmation_rate_delta=round(
|
||
|
|
after_metrics["confirmation_rate"] - before_metrics["confirmation_rate"],
|
||
|
|
4,
|
||
|
|
),
|
||
|
|
false_positive_rate_delta=round(
|
||
|
|
after_metrics["false_positive_rate"] - before_metrics["false_positive_rate"],
|
||
|
|
4,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _metrics(items: list[dict[str, Any]]) -> dict[str, Any]:
|
||
|
|
count = len(items)
|
||
|
|
if count == 0:
|
||
|
|
return {
|
||
|
|
"count": 0,
|
||
|
|
"average_score": 0.0,
|
||
|
|
"high_rate": 0.0,
|
||
|
|
"confirmation_rate": 0.0,
|
||
|
|
"false_positive_rate": 0.0,
|
||
|
|
}
|
||
|
|
confirmed = sum(1 for item in items if item.get("feedback_status") == "confirmed")
|
||
|
|
false_positive = sum(1 for item in items if item.get("feedback_status") == "false_positive")
|
||
|
|
reviewed = confirmed + false_positive
|
||
|
|
return {
|
||
|
|
"count": count,
|
||
|
|
"average_score": sum(int(item.get("risk_score") or 0) for item in items) / count,
|
||
|
|
"high_rate": sum(1 for item in items if item.get("risk_level") in HIGH_LEVELS) / count,
|
||
|
|
"confirmation_rate": confirmed / reviewed if reviewed else 0.0,
|
||
|
|
"false_positive_rate": false_positive / reviewed if reviewed else 0.0,
|
||
|
|
}
|