feat(expenses): backfill historical claims into expense cases

2026-07-14 10:09:09 +08:00
parent 11275e4ba6
commit 5ed34c2b8f
11 changed files with 2053 additions and 3 deletions
--- a/server/src/app/services/expense_case_legacy_backfill.py
+++ b/server/src/app/services/expense_case_legacy_backfill.py
@@ -0,0 +1,391 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from decimal import Decimal
+from enum import StrEnum
+
+from sqlalchemy import and_, or_, select
+from sqlalchemy.orm import Session
+
+from app.models.expense_case import BusinessEvent, ExpenseCase, ExpenseCaseLink
+from app.models.financial_record import ExpenseClaim
+from app.services.expense_cases import ExpenseCaseService
+
+HISTORICAL_CLAIM_IMPORTED_EVENT = "historical_claim_imported"
+HISTORICAL_IMPORT_VERSION = 1
+HISTORICAL_IMPORT_DELIVERY_STATUS = "suppressed"
+DEFAULT_BATCH_SIZE = 100
+MAX_BATCH_SIZE = 1000
+
+
+class LegacyBackfillDisposition(StrEnum):
+    ELIGIBLE = "eligible"
+    LINKED = "linked"
+    CONFLICT = "conflict"
+
+
+@dataclass(frozen=True, slots=True)
+class LegacyBackfillCursor:
+    created_at: datetime
+    claim_id: str
+
+
+@dataclass(frozen=True, slots=True)
+class LegacyBackfillItem:
+    claim_id: str
+    claim_no: str
+    disposition: LegacyBackfillDisposition
+    source_fingerprint: str
+    reason: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class LegacyBackfillPreview:
+    tenant_id: str
+    cutoff: datetime
+    inspected: int
+    eligible: int
+    linked: int
+    conflicts: int
+    has_more: bool
+    next_cursor: LegacyBackfillCursor | None
+    items: tuple[LegacyBackfillItem, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class LegacyBackfillBatchResult:
+    tenant_id: str
+    cutoff: datetime
+    run_id: str
+    inspected: int
+    created: int
+    skipped_linked: int
+    conflicts: int
+    has_more: bool
+    next_cursor: LegacyBackfillCursor | None
+    items: tuple[LegacyBackfillItem, ...]
+
+
+class ExpenseCaseLegacyBackfillService:
+    """把旧费用单诚实地接入费用事件，不虚构迁移前的逐节点历史。"""
+
+    def __init__(self, db: Session, *, tenant_id: str, cutoff: datetime) -> None:
+        self.db = db
+        self.tenant_id = self._require_text(tenant_id, field_name="tenant_id", max_length=64)
+        self.cutoff = self._require_aware_datetime(cutoff, field_name="cutoff")
+        self.expense_cases = ExpenseCaseService(db)
+
+    def preview(
+        self,
+        *,
+        batch_size: int = DEFAULT_BATCH_SIZE,
+        after: LegacyBackfillCursor | None = None,
+    ) -> LegacyBackfillPreview:
+        """预览一批旧单；该方法只查询，不 flush、不 commit。"""
+
+        with self.db.no_autoflush:
+            claims, has_more = self._load_claims(
+                batch_size=self._normalize_batch_size(batch_size),
+                after=after,
+                lock_rows=False,
+            )
+            items = self._classify(claims)
+        return LegacyBackfillPreview(
+            tenant_id=self.tenant_id,
+            cutoff=self.cutoff,
+            inspected=len(items),
+            eligible=self._count(items, LegacyBackfillDisposition.ELIGIBLE),
+            linked=self._count(items, LegacyBackfillDisposition.LINKED),
+            conflicts=self._count(items, LegacyBackfillDisposition.CONFLICT),
+            has_more=has_more,
+            next_cursor=self._next_cursor(claims),
+            items=items,
+        )
+
+    def apply_batch(
+        self,
+        *,
+        run_id: str,
+        batch_size: int = DEFAULT_BATCH_SIZE,
+        after: LegacyBackfillCursor | None = None,
+        backfilled_at: datetime | None = None,
+    ) -> LegacyBackfillBatchResult:
+        """应用一批回填但不提交；调用方拥有完整的批次事务边界。"""
+
+        normalized_run_id = self._require_text(run_id, field_name="run_id", max_length=64)
+        normalized_backfilled_at = self._require_aware_datetime(
+            backfilled_at or datetime.now(UTC),
+            field_name="backfilled_at",
+        )
+        claims, has_more = self._load_claims(
+            batch_size=self._normalize_batch_size(batch_size),
+            after=after,
+            lock_rows=True,
+        )
+        classified = self._classify(claims)
+        claims_by_id = {claim.id: claim for claim in claims}
+        result_items: list[LegacyBackfillItem] = []
+        created = 0
+
+        for item in classified:
+            if item.disposition is not LegacyBackfillDisposition.ELIGIBLE:
+                result_items.append(item)
+                continue
+
+            claim = claims_by_id[item.claim_id]
+            _expense_case, event = self.expense_cases.record_claim_event(
+                claim,
+                event_type=HISTORICAL_CLAIM_IMPORTED_EVENT,
+                actor_id="system",
+                tenant_id=self.tenant_id,
+                correlation_id=normalized_run_id,
+                idempotency_key=self.idempotency_key(claim.id),
+                previous_status="",
+                previous_approval_stage="",
+                extra_payload=self._event_payload(
+                    claim,
+                    run_id=normalized_run_id,
+                    backfilled_at=normalized_backfilled_at,
+                    source_fingerprint=item.source_fingerprint,
+                ),
+                delivery_status=HISTORICAL_IMPORT_DELIVERY_STATUS,
+            )
+            # 事件发生时间表达真实回填动作；旧单业务时间只保留在 payload 中。
+            event.occurred_at = normalized_backfilled_at
+            created += 1
+            result_items.append(item)
+
+        return LegacyBackfillBatchResult(
+            tenant_id=self.tenant_id,
+            cutoff=self.cutoff,
+            run_id=normalized_run_id,
+            inspected=len(result_items),
+            created=created,
+            skipped_linked=self._count(result_items, LegacyBackfillDisposition.LINKED),
+            conflicts=self._count(result_items, LegacyBackfillDisposition.CONFLICT),
+            has_more=has_more,
+            next_cursor=self._next_cursor(claims),
+            items=tuple(result_items),
+        )
+
+    def idempotency_key(self, claim_id: str) -> str:
+        normalized_claim_id = str(claim_id or "").strip()
+        if not normalized_claim_id:
+            raise ValueError("claim_id must not be empty")
+        key = (
+            f"historical-import:v{HISTORICAL_IMPORT_VERSION}:{self.tenant_id}:{normalized_claim_id}"
+        )
+        if len(key) <= 120:
+            return key
+        digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
+        return f"historical-import:v{HISTORICAL_IMPORT_VERSION}:sha256:{digest}"
+
+    @classmethod
+    def source_fingerprint(cls, claim: ExpenseClaim) -> str:
+        encoded = json.dumps(
+            cls._source_snapshot(claim),
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode("utf-8")
+        return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
+
+    def _load_claims(
+        self,
+        *,
+        batch_size: int,
+        after: LegacyBackfillCursor | None,
+        lock_rows: bool,
+    ) -> tuple[list[ExpenseClaim], bool]:
+        stmt = select(ExpenseClaim).where(ExpenseClaim.created_at < self.cutoff)
+        if after is not None:
+            normalized_claim_id = self._require_text(
+                after.claim_id,
+                field_name="after.claim_id",
+                max_length=36,
+            )
+            stmt = stmt.where(
+                or_(
+                    ExpenseClaim.created_at > after.created_at,
+                    and_(
+                        ExpenseClaim.created_at == after.created_at,
+                        ExpenseClaim.id > normalized_claim_id,
+                    ),
+                )
+            )
+        stmt = stmt.order_by(ExpenseClaim.created_at.asc(), ExpenseClaim.id.asc()).limit(
+            batch_size + 1
+        )
+        if lock_rows:
+            stmt = stmt.with_for_update()
+
+        claims = list(self.db.scalars(stmt).all())
+        return claims[:batch_size], len(claims) > batch_size
+
+    def _classify(self, claims: list[ExpenseClaim]) -> tuple[LegacyBackfillItem, ...]:
+        if not claims:
+            return ()
+
+        claim_ids = [claim.id for claim in claims]
+        links_by_claim_id = {
+            link.resource_id: (link, expense_case)
+            for link, expense_case in self.db.execute(
+                select(ExpenseCaseLink, ExpenseCase)
+                .outerjoin(ExpenseCase, ExpenseCase.id == ExpenseCaseLink.expense_case_id)
+                .where(
+                    ExpenseCaseLink.resource_type == "expense_claim",
+                    ExpenseCaseLink.resource_id.in_(claim_ids),
+                )
+            ).all()
+        }
+        event_claim_ids = set(
+            self.db.scalars(
+                select(BusinessEvent.aggregate_id).where(
+                    BusinessEvent.aggregate_type == "expense_claim",
+                    BusinessEvent.aggregate_id.in_(claim_ids),
+                )
+            ).all()
+        )
+        expected_case_nos = {f"CASE-{str(claim.claim_no or claim.id).strip()}" for claim in claims}
+        existing_case_nos = set(
+            self.db.scalars(
+                select(ExpenseCase.case_no).where(ExpenseCase.case_no.in_(expected_case_nos))
+            ).all()
+        )
+
+        items: list[LegacyBackfillItem] = []
+        for claim in claims:
+            fingerprint = self.source_fingerprint(claim)
+            linked_record = links_by_claim_id.get(claim.id)
+            expected_case_no = f"CASE-{str(claim.claim_no or claim.id).strip()}"
+            if linked_record is not None:
+                link, expense_case = linked_record
+                if link.tenant_id == self.tenant_id:
+                    if expense_case is None or expense_case.tenant_id != self.tenant_id:
+                        disposition = LegacyBackfillDisposition.CONFLICT
+                        reason = "expense claim link points to an invalid tenant expense case"
+                    else:
+                        disposition = LegacyBackfillDisposition.LINKED
+                        reason = "expense claim already belongs to an expense case"
+                else:
+                    disposition = LegacyBackfillDisposition.CONFLICT
+                    reason = "expense claim is linked under another tenant"
+            elif claim.id in event_claim_ids:
+                disposition = LegacyBackfillDisposition.CONFLICT
+                reason = "business event exists without an expense case link"
+            elif expected_case_no in existing_case_nos:
+                disposition = LegacyBackfillDisposition.CONFLICT
+                reason = "expense case exists without an expense claim link"
+            else:
+                disposition = LegacyBackfillDisposition.ELIGIBLE
+                reason = ""
+            items.append(
+                LegacyBackfillItem(
+                    claim_id=claim.id,
+                    claim_no=str(claim.claim_no or ""),
+                    disposition=disposition,
+                    source_fingerprint=fingerprint,
+                    reason=reason,
+                )
+            )
+        return tuple(items)
+
+    @classmethod
+    def _source_snapshot(cls, claim: ExpenseClaim) -> dict[str, object]:
+        return {
+            "id": str(claim.id or ""),
+            "claim_no": str(claim.claim_no or ""),
+            "employee_id": str(claim.employee_id or ""),
+            "expense_type": str(claim.expense_type or ""),
+            "amount": cls._money_text(claim.amount),
+            "currency": str(claim.currency or "CNY"),
+            "status": str(claim.status or ""),
+            "approval_stage": str(claim.approval_stage or ""),
+            "source_times": cls._source_times(claim),
+        }
+
+    @classmethod
+    def _event_payload(
+        cls,
+        claim: ExpenseClaim,
+        *,
+        run_id: str,
+        backfilled_at: datetime,
+        source_fingerprint: str,
+    ) -> dict[str, object]:
+        return {
+            "schema_version": HISTORICAL_IMPORT_VERSION,
+            "backfill_version": HISTORICAL_IMPORT_VERSION,
+            "source": "legacy_expense_claim",
+            "history_reconstructed": False,
+            "backfill_run_id": run_id,
+            "backfilled_at": cls._isoformat(backfilled_at),
+            "performed_by": "expense_case_legacy_backfill_cli",
+            "source_times": cls._source_times(claim),
+            "source_fingerprint": source_fingerprint,
+            "reason": "该单据已纳入统一费用事件；迁移前的逐节点办理明细未重建。",
+        }
+
+    @classmethod
+    def _source_times(cls, claim: ExpenseClaim) -> dict[str, str | None]:
+        return {
+            "occurred_at": cls._optional_isoformat(claim.occurred_at),
+            "submitted_at": cls._optional_isoformat(claim.submitted_at),
+            "created_at": cls._optional_isoformat(claim.created_at),
+            "updated_at": cls._optional_isoformat(claim.updated_at),
+        }
+
+    @staticmethod
+    def _money_text(value: Decimal | None) -> str:
+        return f"{Decimal(value or Decimal('0.00')).quantize(Decimal('0.01')):.2f}"
+
+    @classmethod
+    def _optional_isoformat(cls, value: datetime | None) -> str | None:
+        return cls._isoformat(value) if value is not None else None
+
+    @staticmethod
+    def _isoformat(value: datetime) -> str:
+        normalized = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+        return normalized.isoformat().replace("+00:00", "Z")
+
+    @staticmethod
+    def _require_aware_datetime(value: datetime, *, field_name: str) -> datetime:
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise ValueError(f"{field_name} must include a timezone")
+        return value.astimezone(UTC)
+
+    @staticmethod
+    def _require_text(value: str, *, field_name: str, max_length: int) -> str:
+        normalized = str(value or "").strip()
+        if not normalized:
+            raise ValueError(f"{field_name} must not be empty")
+        if len(normalized) > max_length:
+            raise ValueError(f"{field_name} must be at most {max_length} characters")
+        return normalized
+
+    @staticmethod
+    def _normalize_batch_size(value: int) -> int:
+        normalized = int(value)
+        if normalized < 1 or normalized > MAX_BATCH_SIZE:
+            raise ValueError(f"batch_size must be between 1 and {MAX_BATCH_SIZE}")
+        return normalized
+
+    @staticmethod
+    def _count(
+        items: tuple[LegacyBackfillItem, ...] | list[LegacyBackfillItem],
+        disposition: LegacyBackfillDisposition,
+    ) -> int:
+        return sum(1 for item in items if item.disposition is disposition)
+
+    @staticmethod
+    def _next_cursor(claims: list[ExpenseClaim]) -> LegacyBackfillCursor | None:
+        if not claims:
+            return None
+        last_claim = claims[-1]
+        return LegacyBackfillCursor(
+            created_at=last_claim.created_at,
+            claim_id=last_claim.id,
+        )
--- a/server/src/app/services/expense_cases.py
+++ b/server/src/app/services/expense_cases.py
@@ -151,6 +151,7 @@ class ExpenseCaseService:
        expense_case: ExpenseCase | None = None,
        relation_type: str | None = None,
        update_case_state: bool = True,
+        delivery_status: str = "pending",
    ) -> tuple[ExpenseCase, BusinessEvent]:
        normalized_tenant = self.normalize_tenant_id(tenant_id)
        if expense_case is None:
@@ -215,7 +216,7 @@ class ExpenseCaseService:
            actor_id=str(actor_id or "system").strip() or "system",
            actor_type="system" if str(actor_id or "").strip() == "system" else "user",
            payload_json=payload,
-            delivery_status="pending",
+            delivery_status=str(delivery_status or "pending").strip() or "pending",
            occurred_at=datetime.now(UTC),
        )
        self.db.add(event)