feat(skills): enhance skills system with matching and evaluation
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
58
backend/app/agents/skills/evaluator.py
Normal file
58
backend/app/agents/skills/evaluator.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from app.agents.schemas.learning import SessionRetrospective, SkillCandidate
|
||||
from app.agents.skills.models import SkillLifecycleDecision
|
||||
from app.services.skill_service import SkillService
|
||||
|
||||
|
||||
class SkillPromotionEvaluator:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
self.skill_service = SkillService(db)
|
||||
|
||||
async def sync_retrospective(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
retrospective: SessionRetrospective,
|
||||
) -> list[SkillLifecycleDecision]:
|
||||
decisions: list[SkillLifecycleDecision] = []
|
||||
|
||||
for candidate in retrospective.skill_candidates:
|
||||
decisions.append(
|
||||
await self.skill_service.upsert_learned_candidate(
|
||||
user_id=user_id,
|
||||
candidate=candidate,
|
||||
primary_agent=retrospective.primary_agent,
|
||||
evidence_refs=candidate.evidence_refs,
|
||||
)
|
||||
)
|
||||
|
||||
outcome_score = self._derive_outcome_score(retrospective)
|
||||
for skill_name in retrospective.used_skill_names:
|
||||
decision = await self.skill_service.record_activation_feedback(
|
||||
user_id=user_id,
|
||||
skill_name=skill_name,
|
||||
outcome_score=outcome_score,
|
||||
evidence_refs=retrospective.evidence_refs,
|
||||
)
|
||||
if decision is not None:
|
||||
decisions.append(decision)
|
||||
|
||||
return decisions
|
||||
|
||||
@staticmethod
|
||||
def _derive_outcome_score(retrospective: SessionRetrospective) -> float:
|
||||
if retrospective.verification_status == "passed":
|
||||
return 0.9
|
||||
if retrospective.verification_status == "skipped":
|
||||
return 0.55
|
||||
if retrospective.verification_status == "failed":
|
||||
return 0.15
|
||||
return 0.7 if retrospective.outcome == "completed" else 0.2
|
||||
|
||||
|
||||
def next_review_after(days: int = 7) -> datetime:
|
||||
return datetime.now(UTC) + timedelta(days=days)
|
||||
Reference in New Issue
Block a user