feat(expenses): backfill historical claims into expense cases

This commit is contained in:
caoxiaozhu
2026-07-14 10:09:09 +08:00
parent 11275e4ba6
commit 5ed34c2b8f
11 changed files with 2053 additions and 3 deletions

View File

@@ -0,0 +1,237 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Any
from sqlalchemy.engine import URL, make_url
from sqlalchemy.exc import ArgumentError
DEFAULT_POSTGRESQL_PORT = 5432
DISPOSABLE_TARGET_MARKERS = ("migration-probe", "disposable-probe")
SENSITIVE_QUERY_KEY_PARTS = (
"credential",
"passfile",
"password",
"secret",
"token",
)
FORBIDDEN_ROUTING_QUERY_KEYS = frozenset(
{
"database",
"dbname",
"host",
"hostaddr",
"options",
"port",
"service",
"servicefile",
"user",
"username",
}
)
class MaintenanceDatabaseTargetError(ValueError):
"""维护命令数据库目标不满足安全约束。"""
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
@dataclass(frozen=True, slots=True)
class MaintenanceDatabaseTarget:
host: str
port: int
database: str
username: str | None
sanitized_url: str
exact_target: str
is_disposable: bool
def _raise_target_error(code: str, message: str) -> None:
raise MaintenanceDatabaseTargetError(code, message)
def _normalize_host(value: str) -> str:
normalized = str(value or "").strip().lower()
if normalized.startswith("[") and normalized.endswith("]"):
normalized = normalized[1:-1]
return normalized.rstrip(".")
def _normalize_probe_component(value: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", str(value or "").lower()).strip("-")
def _matches_disposable_marker(value: str) -> bool:
normalized = _normalize_probe_component(value)
return any(
normalized == marker
or normalized.startswith(f"{marker}-")
or normalized.startswith(f"x-financial-{marker}-")
for marker in DISPOSABLE_TARGET_MARKERS
)
def is_disposable_maintenance_target(*, host: str, database: str) -> bool:
"""只有主机名和数据库名都带显式 probe 标记时才视为一次性目标。"""
return _matches_disposable_marker(host) and _matches_disposable_marker(database)
def _sanitize_query(url: URL) -> URL:
sanitized_query: dict[str, Any] = {}
for key, value in url.query.items():
normalized_key = str(key).lower()
if any(part in normalized_key for part in SENSITIVE_QUERY_KEY_PARTS):
if isinstance(value, tuple):
sanitized_query[key] = tuple("***" for _ in value)
else:
sanitized_query[key] = "***"
continue
sanitized_query[key] = value
return url.set(query=sanitized_query)
def _format_target_host(host: str) -> str:
return f"[{host}]" if ":" in host else host
def parse_maintenance_database_target(
database_url: str,
*,
expected_host: str,
expected_database: str,
) -> MaintenanceDatabaseTarget:
"""解析显式数据库 URL并核对操作人声明的目标主机和数据库。"""
raw_url = str(database_url or "").strip()
if not raw_url:
_raise_target_error(
"database_url_required",
"维护命令必须显式提供 DATABASE_URL禁止回退到环境文件或默认配置。",
)
try:
parsed_url = make_url(raw_url)
except (ArgumentError, TypeError, ValueError) as exc:
raise MaintenanceDatabaseTargetError(
"invalid_database_url",
"DATABASE_URL 不是有效的 SQLAlchemy 数据库 URL。",
) from exc
if parsed_url.get_backend_name() != "postgresql":
_raise_target_error(
"postgresql_required",
"维护命令只允许连接 PostgreSQL 数据库。",
)
routing_query_keys = sorted(
str(key).lower()
for key in parsed_url.query
if str(key).lower() in FORBIDDEN_ROUTING_QUERY_KEYS
)
if routing_query_keys:
_raise_target_error(
"database_routing_query_forbidden",
f"DATABASE_URL 查询参数不得覆盖连接目标或 schema{', '.join(routing_query_keys)}",
)
host = _normalize_host(parsed_url.host or "")
if not host:
_raise_target_error(
"database_host_required",
"DATABASE_URL 必须包含显式 PostgreSQL 主机名。",
)
database = str(parsed_url.database or "").strip()
if not database:
_raise_target_error(
"database_name_required",
"DATABASE_URL 必须包含显式数据库名。",
)
normalized_expected_host = _normalize_host(expected_host)
if not normalized_expected_host:
_raise_target_error(
"expected_host_required",
"必须通过 expected_host 声明预期数据库主机。",
)
if host != normalized_expected_host:
_raise_target_error(
"expected_host_mismatch",
f"DATABASE_URL 主机与预期不一致actual={host}; expected={normalized_expected_host}",
)
normalized_expected_database = str(expected_database or "").strip()
if not normalized_expected_database:
_raise_target_error(
"expected_database_required",
"必须通过 expected_database 声明预期数据库名。",
)
if database != normalized_expected_database:
_raise_target_error(
"expected_database_mismatch",
"DATABASE_URL 数据库名与预期不一致:"
f"actual={database}; expected={normalized_expected_database}",
)
port = int(parsed_url.port or DEFAULT_POSTGRESQL_PORT)
exact_target = f"{_format_target_host(host)}:{port}/{database}"
sanitized_url = _sanitize_query(parsed_url).render_as_string(hide_password=True)
return MaintenanceDatabaseTarget(
host=host,
port=port,
database=database,
username=parsed_url.username,
sanitized_url=sanitized_url,
exact_target=exact_target,
is_disposable=is_disposable_maintenance_target(host=host, database=database),
)
def validate_maintenance_database_target(
database_url: str,
*,
expected_host: str,
expected_database: str,
apply: bool = False,
allow_non_disposable: bool = False,
confirm_target: str | None = None,
) -> MaintenanceDatabaseTarget:
"""校验维护目标;非一次性数据库 apply 必须显式放行并精确确认目标。"""
target = parse_maintenance_database_target(
database_url,
expected_host=expected_host,
expected_database=expected_database,
)
if not apply:
return target
normalized_confirmation = str(confirm_target or "").strip()
if target.is_disposable:
if normalized_confirmation and normalized_confirmation != target.exact_target:
_raise_target_error(
"confirm_target_mismatch",
"confirm_target 与解析后的数据库目标不一致:"
f"actual={normalized_confirmation}; expected={target.exact_target}",
)
return target
if not allow_non_disposable:
_raise_target_error(
"non_disposable_apply_forbidden",
"非一次性数据库 apply 必须显式启用 allow_non_disposable。",
)
if normalized_confirmation != target.exact_target:
_raise_target_error(
"confirm_target_mismatch",
"非一次性数据库 apply 必须通过 confirm_target 精确确认:"
f"expected={target.exact_target}",
)
return target

View File

@@ -0,0 +1,391 @@
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from datetime import UTC, datetime
from decimal import Decimal
from enum import StrEnum
from sqlalchemy import and_, or_, select
from sqlalchemy.orm import Session
from app.models.expense_case import BusinessEvent, ExpenseCase, ExpenseCaseLink
from app.models.financial_record import ExpenseClaim
from app.services.expense_cases import ExpenseCaseService
HISTORICAL_CLAIM_IMPORTED_EVENT = "historical_claim_imported"
HISTORICAL_IMPORT_VERSION = 1
HISTORICAL_IMPORT_DELIVERY_STATUS = "suppressed"
DEFAULT_BATCH_SIZE = 100
MAX_BATCH_SIZE = 1000
class LegacyBackfillDisposition(StrEnum):
ELIGIBLE = "eligible"
LINKED = "linked"
CONFLICT = "conflict"
@dataclass(frozen=True, slots=True)
class LegacyBackfillCursor:
created_at: datetime
claim_id: str
@dataclass(frozen=True, slots=True)
class LegacyBackfillItem:
claim_id: str
claim_no: str
disposition: LegacyBackfillDisposition
source_fingerprint: str
reason: str = ""
@dataclass(frozen=True, slots=True)
class LegacyBackfillPreview:
tenant_id: str
cutoff: datetime
inspected: int
eligible: int
linked: int
conflicts: int
has_more: bool
next_cursor: LegacyBackfillCursor | None
items: tuple[LegacyBackfillItem, ...]
@dataclass(frozen=True, slots=True)
class LegacyBackfillBatchResult:
tenant_id: str
cutoff: datetime
run_id: str
inspected: int
created: int
skipped_linked: int
conflicts: int
has_more: bool
next_cursor: LegacyBackfillCursor | None
items: tuple[LegacyBackfillItem, ...]
class ExpenseCaseLegacyBackfillService:
"""把旧费用单诚实地接入费用事件,不虚构迁移前的逐节点历史。"""
def __init__(self, db: Session, *, tenant_id: str, cutoff: datetime) -> None:
self.db = db
self.tenant_id = self._require_text(tenant_id, field_name="tenant_id", max_length=64)
self.cutoff = self._require_aware_datetime(cutoff, field_name="cutoff")
self.expense_cases = ExpenseCaseService(db)
def preview(
self,
*,
batch_size: int = DEFAULT_BATCH_SIZE,
after: LegacyBackfillCursor | None = None,
) -> LegacyBackfillPreview:
"""预览一批旧单;该方法只查询,不 flush、不 commit。"""
with self.db.no_autoflush:
claims, has_more = self._load_claims(
batch_size=self._normalize_batch_size(batch_size),
after=after,
lock_rows=False,
)
items = self._classify(claims)
return LegacyBackfillPreview(
tenant_id=self.tenant_id,
cutoff=self.cutoff,
inspected=len(items),
eligible=self._count(items, LegacyBackfillDisposition.ELIGIBLE),
linked=self._count(items, LegacyBackfillDisposition.LINKED),
conflicts=self._count(items, LegacyBackfillDisposition.CONFLICT),
has_more=has_more,
next_cursor=self._next_cursor(claims),
items=items,
)
def apply_batch(
self,
*,
run_id: str,
batch_size: int = DEFAULT_BATCH_SIZE,
after: LegacyBackfillCursor | None = None,
backfilled_at: datetime | None = None,
) -> LegacyBackfillBatchResult:
"""应用一批回填但不提交;调用方拥有完整的批次事务边界。"""
normalized_run_id = self._require_text(run_id, field_name="run_id", max_length=64)
normalized_backfilled_at = self._require_aware_datetime(
backfilled_at or datetime.now(UTC),
field_name="backfilled_at",
)
claims, has_more = self._load_claims(
batch_size=self._normalize_batch_size(batch_size),
after=after,
lock_rows=True,
)
classified = self._classify(claims)
claims_by_id = {claim.id: claim for claim in claims}
result_items: list[LegacyBackfillItem] = []
created = 0
for item in classified:
if item.disposition is not LegacyBackfillDisposition.ELIGIBLE:
result_items.append(item)
continue
claim = claims_by_id[item.claim_id]
_expense_case, event = self.expense_cases.record_claim_event(
claim,
event_type=HISTORICAL_CLAIM_IMPORTED_EVENT,
actor_id="system",
tenant_id=self.tenant_id,
correlation_id=normalized_run_id,
idempotency_key=self.idempotency_key(claim.id),
previous_status="",
previous_approval_stage="",
extra_payload=self._event_payload(
claim,
run_id=normalized_run_id,
backfilled_at=normalized_backfilled_at,
source_fingerprint=item.source_fingerprint,
),
delivery_status=HISTORICAL_IMPORT_DELIVERY_STATUS,
)
# 事件发生时间表达真实回填动作;旧单业务时间只保留在 payload 中。
event.occurred_at = normalized_backfilled_at
created += 1
result_items.append(item)
return LegacyBackfillBatchResult(
tenant_id=self.tenant_id,
cutoff=self.cutoff,
run_id=normalized_run_id,
inspected=len(result_items),
created=created,
skipped_linked=self._count(result_items, LegacyBackfillDisposition.LINKED),
conflicts=self._count(result_items, LegacyBackfillDisposition.CONFLICT),
has_more=has_more,
next_cursor=self._next_cursor(claims),
items=tuple(result_items),
)
def idempotency_key(self, claim_id: str) -> str:
normalized_claim_id = str(claim_id or "").strip()
if not normalized_claim_id:
raise ValueError("claim_id must not be empty")
key = (
f"historical-import:v{HISTORICAL_IMPORT_VERSION}:{self.tenant_id}:{normalized_claim_id}"
)
if len(key) <= 120:
return key
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
return f"historical-import:v{HISTORICAL_IMPORT_VERSION}:sha256:{digest}"
@classmethod
def source_fingerprint(cls, claim: ExpenseClaim) -> str:
encoded = json.dumps(
cls._source_snapshot(claim),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
def _load_claims(
self,
*,
batch_size: int,
after: LegacyBackfillCursor | None,
lock_rows: bool,
) -> tuple[list[ExpenseClaim], bool]:
stmt = select(ExpenseClaim).where(ExpenseClaim.created_at < self.cutoff)
if after is not None:
normalized_claim_id = self._require_text(
after.claim_id,
field_name="after.claim_id",
max_length=36,
)
stmt = stmt.where(
or_(
ExpenseClaim.created_at > after.created_at,
and_(
ExpenseClaim.created_at == after.created_at,
ExpenseClaim.id > normalized_claim_id,
),
)
)
stmt = stmt.order_by(ExpenseClaim.created_at.asc(), ExpenseClaim.id.asc()).limit(
batch_size + 1
)
if lock_rows:
stmt = stmt.with_for_update()
claims = list(self.db.scalars(stmt).all())
return claims[:batch_size], len(claims) > batch_size
def _classify(self, claims: list[ExpenseClaim]) -> tuple[LegacyBackfillItem, ...]:
if not claims:
return ()
claim_ids = [claim.id for claim in claims]
links_by_claim_id = {
link.resource_id: (link, expense_case)
for link, expense_case in self.db.execute(
select(ExpenseCaseLink, ExpenseCase)
.outerjoin(ExpenseCase, ExpenseCase.id == ExpenseCaseLink.expense_case_id)
.where(
ExpenseCaseLink.resource_type == "expense_claim",
ExpenseCaseLink.resource_id.in_(claim_ids),
)
).all()
}
event_claim_ids = set(
self.db.scalars(
select(BusinessEvent.aggregate_id).where(
BusinessEvent.aggregate_type == "expense_claim",
BusinessEvent.aggregate_id.in_(claim_ids),
)
).all()
)
expected_case_nos = {f"CASE-{str(claim.claim_no or claim.id).strip()}" for claim in claims}
existing_case_nos = set(
self.db.scalars(
select(ExpenseCase.case_no).where(ExpenseCase.case_no.in_(expected_case_nos))
).all()
)
items: list[LegacyBackfillItem] = []
for claim in claims:
fingerprint = self.source_fingerprint(claim)
linked_record = links_by_claim_id.get(claim.id)
expected_case_no = f"CASE-{str(claim.claim_no or claim.id).strip()}"
if linked_record is not None:
link, expense_case = linked_record
if link.tenant_id == self.tenant_id:
if expense_case is None or expense_case.tenant_id != self.tenant_id:
disposition = LegacyBackfillDisposition.CONFLICT
reason = "expense claim link points to an invalid tenant expense case"
else:
disposition = LegacyBackfillDisposition.LINKED
reason = "expense claim already belongs to an expense case"
else:
disposition = LegacyBackfillDisposition.CONFLICT
reason = "expense claim is linked under another tenant"
elif claim.id in event_claim_ids:
disposition = LegacyBackfillDisposition.CONFLICT
reason = "business event exists without an expense case link"
elif expected_case_no in existing_case_nos:
disposition = LegacyBackfillDisposition.CONFLICT
reason = "expense case exists without an expense claim link"
else:
disposition = LegacyBackfillDisposition.ELIGIBLE
reason = ""
items.append(
LegacyBackfillItem(
claim_id=claim.id,
claim_no=str(claim.claim_no or ""),
disposition=disposition,
source_fingerprint=fingerprint,
reason=reason,
)
)
return tuple(items)
@classmethod
def _source_snapshot(cls, claim: ExpenseClaim) -> dict[str, object]:
return {
"id": str(claim.id or ""),
"claim_no": str(claim.claim_no or ""),
"employee_id": str(claim.employee_id or ""),
"expense_type": str(claim.expense_type or ""),
"amount": cls._money_text(claim.amount),
"currency": str(claim.currency or "CNY"),
"status": str(claim.status or ""),
"approval_stage": str(claim.approval_stage or ""),
"source_times": cls._source_times(claim),
}
@classmethod
def _event_payload(
cls,
claim: ExpenseClaim,
*,
run_id: str,
backfilled_at: datetime,
source_fingerprint: str,
) -> dict[str, object]:
return {
"schema_version": HISTORICAL_IMPORT_VERSION,
"backfill_version": HISTORICAL_IMPORT_VERSION,
"source": "legacy_expense_claim",
"history_reconstructed": False,
"backfill_run_id": run_id,
"backfilled_at": cls._isoformat(backfilled_at),
"performed_by": "expense_case_legacy_backfill_cli",
"source_times": cls._source_times(claim),
"source_fingerprint": source_fingerprint,
"reason": "该单据已纳入统一费用事件;迁移前的逐节点办理明细未重建。",
}
@classmethod
def _source_times(cls, claim: ExpenseClaim) -> dict[str, str | None]:
return {
"occurred_at": cls._optional_isoformat(claim.occurred_at),
"submitted_at": cls._optional_isoformat(claim.submitted_at),
"created_at": cls._optional_isoformat(claim.created_at),
"updated_at": cls._optional_isoformat(claim.updated_at),
}
@staticmethod
def _money_text(value: Decimal | None) -> str:
return f"{Decimal(value or Decimal('0.00')).quantize(Decimal('0.01')):.2f}"
@classmethod
def _optional_isoformat(cls, value: datetime | None) -> str | None:
return cls._isoformat(value) if value is not None else None
@staticmethod
def _isoformat(value: datetime) -> str:
normalized = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
return normalized.isoformat().replace("+00:00", "Z")
@staticmethod
def _require_aware_datetime(value: datetime, *, field_name: str) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError(f"{field_name} must include a timezone")
return value.astimezone(UTC)
@staticmethod
def _require_text(value: str, *, field_name: str, max_length: int) -> str:
normalized = str(value or "").strip()
if not normalized:
raise ValueError(f"{field_name} must not be empty")
if len(normalized) > max_length:
raise ValueError(f"{field_name} must be at most {max_length} characters")
return normalized
@staticmethod
def _normalize_batch_size(value: int) -> int:
normalized = int(value)
if normalized < 1 or normalized > MAX_BATCH_SIZE:
raise ValueError(f"batch_size must be between 1 and {MAX_BATCH_SIZE}")
return normalized
@staticmethod
def _count(
items: tuple[LegacyBackfillItem, ...] | list[LegacyBackfillItem],
disposition: LegacyBackfillDisposition,
) -> int:
return sum(1 for item in items if item.disposition is disposition)
@staticmethod
def _next_cursor(claims: list[ExpenseClaim]) -> LegacyBackfillCursor | None:
if not claims:
return None
last_claim = claims[-1]
return LegacyBackfillCursor(
created_at=last_claim.created_at,
claim_id=last_claim.id,
)

View File

@@ -151,6 +151,7 @@ class ExpenseCaseService:
expense_case: ExpenseCase | None = None,
relation_type: str | None = None,
update_case_state: bool = True,
delivery_status: str = "pending",
) -> tuple[ExpenseCase, BusinessEvent]:
normalized_tenant = self.normalize_tenant_id(tenant_id)
if expense_case is None:
@@ -215,7 +216,7 @@ class ExpenseCaseService:
actor_id=str(actor_id or "system").strip() or "system",
actor_type="system" if str(actor_id or "").strip() == "system" else "user",
payload_json=payload,
delivery_status="pending",
delivery_status=str(delivery_status or "pending").strip() or "pending",
occurred_at=datetime.now(UTC),
)
self.db.add(event)