YG-Soft/X-Financial
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(expenses): backfill historical claims into expense cases

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-14 10:09:09 +08:00
parent 11275e4ba6
commit 5ed34c2b8f
11 changed files with 2053 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
server/tests/test_backfill_legacy_expense_claim_cases_cli.py Normal file
View File

@@ -0,0 +1,183 @@
from __future__ import annotations
import argparse
import importlib.util
from datetime import UTC, datetime
from pathlib import Path
import pytest
def load_script_module():
script_path = (
Path(__file__).resolve().parents[1] / "scripts" / "backfill_legacy_expense_claim_cases.py"
)
spec = importlib.util.spec_from_file_location(
"backfill_legacy_expense_claim_cases_cli",
script_path,
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
MODULE = load_script_module()
def test_created_before_requires_timezone_and_normalizes_to_utc() -> None:
assert MODULE.parse_created_before("2026-07-14T08:00:00+08:00") == datetime(
2026,
7,
14,
tzinfo=UTC,
)
with pytest.raises(argparse.ArgumentTypeError, match="时区"):
MODULE.parse_created_before("2026-07-14T00:00:00")
@pytest.mark.parametrize("value", ["0", "-1", "not-a-number"])
def test_positive_int_rejects_invalid_values(value: str) -> None:
with pytest.raises(argparse.ArgumentTypeError):
MODULE.positive_int(value)
def test_parser_defaults_to_dry_run_and_requires_explicit_scope() -> None:
args = MODULE.build_parser().parse_args(
[
"--tenant-id",
"default",
"--created-before",
"2026-07-14T00:00:00Z",
"--expected-host",
"migration-probe",
"--expected-database",
"migration_probe",
]
)
assert args.apply is False
assert args.dry_run is False
assert args.batch_size == 100
assert args.tenant_id == "default"
def test_parser_rejects_apply_and_dry_run_together() -> None:
with pytest.raises(SystemExit):
MODULE.build_parser().parse_args(
[
"--apply",
"--dry-run",
"--tenant-id",
"default",
"--created-before",
"2026-07-14T00:00:00Z",
"--expected-host",
"migration-probe",
"--expected-database",
"migration_probe",
]
)
def test_parser_rejects_blank_tenant() -> None:
with pytest.raises(SystemExit):
MODULE.build_parser().parse_args(
[
"--tenant-id",
" ",
"--created-before",
"2026-07-14T00:00:00Z",
"--expected-host",
"migration-probe",
"--expected-database",
"migration_probe",
]
)
def test_apply_requires_exact_target_confirmation_before_connecting(monkeypatch) -> None:
monkeypatch.setenv(
"DATABASE_URL",
"postgresql://finance:secret@migration-probe/migration_probe",
)
args = MODULE.build_parser().parse_args(
[
"--apply",
"--tenant-id",
"default",
"--created-before",
"2026-07-14T00:00:00Z",
"--expected-host",
"migration-probe",
"--expected-database",
"migration_probe",
]
)
with pytest.raises(MODULE.BackfillCommandError) as exc_info:
MODULE.run(args)
assert exc_info.value.code == "confirm_target_required"
assert exc_info.value.exit_code == MODULE.EXIT_SAFETY
def test_missing_database_url_is_rejected_without_configuration_fallback(monkeypatch) -> None:
monkeypatch.delenv("DATABASE_URL", raising=False)
args = MODULE.build_parser().parse_args(
[
"--tenant-id",
"default",
"--created-before",
"2026-07-14T00:00:00Z",
"--expected-host",
"migration-probe",
"--expected-database",
"migration_probe",
]
)
with pytest.raises(MODULE.MaintenanceDatabaseTargetError) as exc_info:
MODULE.run(args)
assert exc_info.value.code == "database_url_required"
def test_partial_failure_payload_discloses_committed_progress() -> None:
summary = {
"run_id": "historical-import-run-1",
"batches": 2,
"inspected": 4,
"created": 3,
"already_linked": 1,
"last_cursor": {
"created_at": "2026-07-10T00:00:00Z",
"claim_id": "claim-4",
},
}
details = MODULE._execution_progress(
summary,
current_batch_conflict_ids=["claim-5"],
)
error = MODULE.BackfillCommandError(
"current batch rolled back",
exit_code=MODULE.EXIT_CONFLICT,
code="legacy_claim_conflict",
details=details,
)
payload = MODULE._error_payload(error, code=error.code, details=error.details)
assert payload["details"] == {
"run_id": "historical-import-run-1",
"partial_commit": True,
"committed_batches": 2,
"committed_claims": 4,
"created": 3,
"already_linked": 1,
"last_committed_cursor": {
"created_at": "2026-07-10T00:00:00Z",
"claim_id": "claim-4",
},
"current_batch_conflict_ids": ["claim-5"],
}

385
server/tests/test_expense_case_legacy_backfill.py Normal file
View File

@@ -0,0 +1,385 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from decimal import Decimal
import pytest
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from app.db.base import Base
from app.models.expense_case import BusinessEvent, ExpenseCase, ExpenseCaseLink
from app.models.financial_record import ExpenseClaim
from app.services.expense_case_legacy_backfill import (
HISTORICAL_CLAIM_IMPORTED_EVENT,
HISTORICAL_IMPORT_DELIVERY_STATUS,
ExpenseCaseLegacyBackfillService,
LegacyBackfillDisposition,
)
CUTOFF = datetime(2026, 7, 14, tzinfo=UTC)
def build_session() -> Session:
engine = create_engine(
"sqlite+pysqlite:///:memory:",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
Base.metadata.create_all(bind=engine)
return sessionmaker(bind=engine, autoflush=False, autocommit=False)()
def build_claim(claim_no: str, *, created_at: datetime | None = None) -> ExpenseClaim:
return ExpenseClaim(
claim_no=claim_no,
employee_name="张三",
department_name="市场部",
project_code="PRJ-BACKFILL",
expense_type="travel",
reason="历史客户拜访",
location="上海",
amount=Decimal("128.50"),
currency="CNY",
invoice_count=1,
occurred_at=datetime(2026, 7, 10, 9, 0, tzinfo=UTC),
submitted_at=datetime(2026, 7, 11, 10, 0, tzinfo=UTC),
status="submitted",
approval_stage="直属领导审批",
risk_flags_json=[],
created_at=created_at or datetime(2026, 7, 10, 8, 0, tzinfo=UTC),
updated_at=datetime(2026, 7, 11, 10, 0, tzinfo=UTC),
)
def seed_link(db: Session, claim: ExpenseClaim, *, tenant_id: str = "default") -> None:
expense_case = ExpenseCase(
tenant_id=tenant_id,
case_no=f"CASE-{claim.claim_no}",
scene_code="travel",
title=claim.reason,
current_stage="reviewing",
status="active",
)
db.add(expense_case)
db.flush()
db.add(
ExpenseCaseLink(
tenant_id=tenant_id,
expense_case_id=expense_case.id,
resource_type="expense_claim",
resource_id=claim.id,
relation_type="claim",
)
)
db.flush()
def seed_event_without_link(db: Session, claim: ExpenseClaim) -> None:
expense_case = ExpenseCase(
tenant_id="default",
case_no=f"CASE-ORPHAN-{claim.claim_no}",
scene_code="travel",
title=claim.reason,
current_stage="reviewing",
status="active",
)
db.add(expense_case)
db.flush()
db.add(
BusinessEvent(
tenant_id="default",
expense_case_id=expense_case.id,
aggregate_type="expense_claim",
aggregate_id=claim.id,
event_type="claim_submitted",
event_version=1,
idempotency_key=f"orphan:{claim.id}",
correlation_id="orphan-event",
actor_id="system",
actor_type="system",
payload_json={},
delivery_status="pending",
occurred_at=datetime(2026, 7, 11, tzinfo=UTC),
)
)
db.flush()
def table_counts(db: Session) -> tuple[int, int, int]:
return (
db.scalar(select(func.count()).select_from(ExpenseCase)) or 0,
db.scalar(select(func.count()).select_from(ExpenseCaseLink)) or 0,
db.scalar(select(func.count()).select_from(BusinessEvent)) or 0,
)
def test_requires_explicit_tenant_and_timezone_aware_cutoff() -> None:
with build_session() as db:
with pytest.raises(ValueError, match="tenant_id"):
ExpenseCaseLegacyBackfillService(db, tenant_id="", cutoff=CUTOFF)
with pytest.raises(ValueError, match="timezone"):
ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=datetime(2026, 7, 14),
)
def test_fingerprint_is_stable_and_idempotency_key_is_tenant_scoped() -> None:
with build_session() as db:
claim = build_claim("EXP-HISTORY-FINGERPRINT")
db.add(claim)
db.flush()
default_service = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
)
tenant_service = ExpenseCaseLegacyBackfillService(
db,
tenant_id="tenant-b",
cutoff=CUTOFF,
)
first_fingerprint = default_service.source_fingerprint(claim)
assert default_service.source_fingerprint(claim) == first_fingerprint
assert default_service.idempotency_key(claim.id) == (
f"historical-import:v1:default:{claim.id}"
)
assert tenant_service.idempotency_key(claim.id) == (
f"historical-import:v1:tenant-b:{claim.id}"
)
claim.status = "approved"
assert default_service.source_fingerprint(claim) != first_fingerprint
def test_long_tenant_idempotency_key_is_stably_hashed_to_database_limit() -> None:
with build_session() as db:
service = ExpenseCaseLegacyBackfillService(
db,
tenant_id="t" * 64,
cutoff=CUTOFF,
)
first = service.idempotency_key("c" * 36)
second = service.idempotency_key("c" * 36)
assert first == second
assert first.startswith("historical-import:v1:sha256:")
assert len(first) <= 120
def test_preview_is_read_only_and_classifies_linked_and_orphan_event_conflicts() -> None:
with build_session() as db:
eligible = build_claim("EXP-HISTORY-ELIGIBLE")
linked = build_claim("EXP-HISTORY-LINKED", created_at=CUTOFF - timedelta(days=3))
conflict = build_claim("EXP-HISTORY-CONFLICT", created_at=CUTOFF - timedelta(days=2))
future = build_claim("EXP-HISTORY-FUTURE", created_at=CUTOFF + timedelta(seconds=1))
db.add_all([eligible, linked, conflict, future])
db.flush()
seed_link(db, linked)
seed_event_without_link(db, conflict)
db.commit()
counts_before = table_counts(db)
preview = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
).preview(batch_size=10)
assert preview.inspected == 3
assert preview.eligible == 1
assert preview.linked == 1
assert preview.conflicts == 1
assert not preview.has_more
assert table_counts(db) == counts_before
dispositions = {item.claim_no: item.disposition for item in preview.items}
assert dispositions == {
"EXP-HISTORY-ELIGIBLE": LegacyBackfillDisposition.ELIGIBLE,
"EXP-HISTORY-LINKED": LegacyBackfillDisposition.LINKED,
"EXP-HISTORY-CONFLICT": LegacyBackfillDisposition.CONFLICT,
}
def test_apply_batch_writes_honest_snapshot_without_committing_and_is_idempotent() -> None:
with build_session() as db:
claim = build_claim("EXP-HISTORY-APPLY")
db.add(claim)
db.commit()
backfilled_at = datetime(2026, 7, 14, 12, 30, tzinfo=UTC)
service = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
)
result = service.apply_batch(
run_id="legacy-backfill-run-1",
batch_size=10,
backfilled_at=backfilled_at,
)
assert result.created == 1
assert result.skipped_linked == 0
assert result.conflicts == 0
event = db.scalar(select(BusinessEvent))
assert event is not None
assert event.event_type == HISTORICAL_CLAIM_IMPORTED_EVENT
assert event.idempotency_key == (f"historical-import:v1:default:{claim.id}")
assert event.idempotency_key == service.idempotency_key(claim.id)
assert event.actor_id == "system"
assert event.actor_type == "system"
assert event.delivery_status == HISTORICAL_IMPORT_DELIVERY_STATUS
assert event.occurred_at == backfilled_at
assert event.payload_json["history_reconstructed"] is False
assert event.payload_json["backfill_version"] == 1
assert event.payload_json["backfilled_at"] == "2026-07-14T12:30:00Z"
assert event.payload_json["performed_by"] == "expense_case_legacy_backfill_cli"
assert event.payload_json["source_times"] == {
"occurred_at": "2026-07-10T09:00:00Z",
"submitted_at": "2026-07-11T10:00:00Z",
"created_at": "2026-07-10T08:00:00Z",
"updated_at": "2026-07-11T10:00:00Z",
}
assert event.payload_json["source_fingerprint"].startswith("sha256:")
assert "未重建" in event.payload_json["reason"]
db.rollback()
assert table_counts(db) == (0, 0, 0)
first_apply = service.apply_batch(
run_id="legacy-backfill-run-2",
batch_size=10,
backfilled_at=backfilled_at,
)
db.commit()
repeated_apply = service.apply_batch(
run_id="legacy-backfill-run-3",
batch_size=10,
backfilled_at=backfilled_at + timedelta(minutes=1),
)
assert first_apply.created == 1
assert repeated_apply.created == 0
assert repeated_apply.skipped_linked == 1
assert table_counts(db) == (1, 1, 1)
def test_apply_batch_reports_event_without_link_as_conflict() -> None:
with build_session() as db:
claim = build_claim("EXP-HISTORY-ORPHAN")
db.add(claim)
db.flush()
seed_event_without_link(db, claim)
db.commit()
result = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
).apply_batch(
run_id="legacy-backfill-conflict",
batch_size=10,
)
assert result.created == 0
assert result.conflicts == 1
assert result.items[0].disposition is LegacyBackfillDisposition.CONFLICT
assert "without an expense case link" in result.items[0].reason
assert db.scalar(select(func.count()).select_from(ExpenseCaseLink)) == 0
def test_preview_rejects_damaged_link_tenant_and_unlinked_case() -> None:
with build_session() as db:
damaged_link_claim = build_claim("EXP-HISTORY-DAMAGED-LINK")
unlinked_case_claim = build_claim(
"EXP-HISTORY-UNLINKED-CASE",
created_at=CUTOFF - timedelta(days=2),
)
db.add_all([damaged_link_claim, unlinked_case_claim])
db.flush()
foreign_case = ExpenseCase(
tenant_id="other-tenant",
case_no=f"CASE-{damaged_link_claim.claim_no}",
scene_code="travel",
title=damaged_link_claim.reason,
current_stage="reviewing",
status="active",
)
orphan_case = ExpenseCase(
tenant_id="default",
case_no=f"CASE-{unlinked_case_claim.claim_no}",
scene_code="travel",
title=unlinked_case_claim.reason,
current_stage="reviewing",
status="active",
)
db.add_all([foreign_case, orphan_case])
db.flush()
db.add(
ExpenseCaseLink(
tenant_id="default",
expense_case_id=foreign_case.id,
resource_type="expense_claim",
resource_id=damaged_link_claim.id,
relation_type="claim",
)
)
db.commit()
preview = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
).preview(batch_size=10)
assert preview.conflicts == 2
reasons = {item.claim_no: item.reason for item in preview.items}
assert "invalid tenant" in reasons["EXP-HISTORY-DAMAGED-LINK"]
assert "without an expense claim link" in reasons["EXP-HISTORY-UNLINKED-CASE"]
def test_preview_cursor_provides_stable_keyset_batches() -> None:
with build_session() as db:
first = build_claim("EXP-HISTORY-001", created_at=CUTOFF - timedelta(days=3))
second = build_claim("EXP-HISTORY-002", created_at=CUTOFF - timedelta(days=2))
db.add_all([first, second])
db.commit()
service = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
)
page_one = service.preview(batch_size=1)
page_two = service.preview(batch_size=1, after=page_one.next_cursor)
assert page_one.has_more
assert page_one.items[0].claim_no == "EXP-HISTORY-001"
assert not page_two.has_more
assert page_two.items[0].claim_no == "EXP-HISTORY-002"
def test_standard_event_delivery_status_remains_pending_by_default() -> None:
with build_session() as db:
claim = build_claim("EXP-NORMAL-EVENT")
db.add(claim)
db.commit()
_expense_case, event = ExpenseCaseLegacyBackfillService(
db,
tenant_id="default",
cutoff=CUTOFF,
).expense_cases.record_claim_event(
claim,
event_type="claim_submitted",
actor_id="owner@example.com",
tenant_id="default",
)
assert event.delivery_status == "pending"

225
server/tests/test_maintenance_database_target.py Normal file
View File

@@ -0,0 +1,225 @@
from __future__ import annotations
import pytest
from app.db.maintenance_database_target import (
MaintenanceDatabaseTargetError,
is_disposable_maintenance_target,
parse_maintenance_database_target,
validate_maintenance_database_target,
)
def assert_target_error(code: str, callback) -> None:
with pytest.raises(MaintenanceDatabaseTargetError) as exc_info:
callback()
assert exc_info.value.code == code
def test_parse_target_supports_hostname_and_hides_password() -> None:
target = parse_maintenance_database_target(
"postgresql+psycopg://finance:super-secret@db.internal:5544/x_financial"
"?application_name=backfill",
expected_host="DB.INTERNAL.",
expected_database="x_financial",
)
assert target.host == "db.internal"
assert target.port == 5544
assert target.database == "x_financial"
assert target.username == "finance"
assert target.exact_target == "db.internal:5544/x_financial"
assert "super-secret" not in target.sanitized_url
assert "***" in target.sanitized_url
assert "application_name=backfill" in target.sanitized_url
assert target.is_disposable is False
def test_parse_target_supports_ipv4_and_defaults_postgresql_port() -> None:
target = parse_maintenance_database_target(
"postgresql://finance:password@127.0.0.1/x_financial",
expected_host="127.0.0.1",
expected_database="x_financial",
)
assert target.port == 5432
assert target.exact_target == "127.0.0.1:5432/x_financial"
assert "password" not in target.sanitized_url
def test_parse_target_redacts_sensitive_query_values() -> None:
target = parse_maintenance_database_target(
"postgresql://finance:authority-secret@db.internal/x_financial"
"?sslpassword=query-secret&application_name=safe-name",
expected_host="db.internal",
expected_database="x_financial",
)
assert "authority-secret" not in target.sanitized_url
assert "query-secret" not in target.sanitized_url
assert "application_name=safe-name" in target.sanitized_url
@pytest.mark.parametrize(
"routing_query",
[
"host=x-financial-local-postgres",
"port=6432",
"dbname=production",
"user=other-user",
"service=production-service",
"options=-csearch_path%3Dother_schema",
],
)
def test_parse_target_rejects_query_parameters_that_can_override_route(
routing_query: str,
) -> None:
assert_target_error(
"database_routing_query_forbidden",
lambda: parse_maintenance_database_target(
f"postgresql://finance:password@migration-probe/migration_probe?{routing_query}",
expected_host="migration-probe",
expected_database="migration_probe",
),
)
@pytest.mark.parametrize(
("database_url", "code"),
[
("", "database_url_required"),
("not a database url", "invalid_database_url"),
("sqlite:///tmp/test.db", "postgresql_required"),
("postgresql:///x_financial", "database_host_required"),
("postgresql://finance@db.internal", "database_name_required"),
],
)
def test_parse_target_rejects_invalid_or_incomplete_urls(database_url: str, code: str) -> None:
assert_target_error(
code,
lambda: parse_maintenance_database_target(
database_url,
expected_host="db.internal",
expected_database="x_financial",
),
)
@pytest.mark.parametrize(
("expected_host", "expected_database", "code"),
[
("", "x_financial", "expected_host_required"),
("other-db", "x_financial", "expected_host_mismatch"),
("db.internal", "", "expected_database_required"),
("db.internal", "other_database", "expected_database_mismatch"),
],
)
def test_parse_target_rejects_expected_target_mismatches(
expected_host: str,
expected_database: str,
code: str,
) -> None:
assert_target_error(
code,
lambda: parse_maintenance_database_target(
"postgresql://finance:password@db.internal/x_financial",
expected_host=expected_host,
expected_database=expected_database,
),
)
@pytest.mark.parametrize(
("host", "database", "expected"),
[
("migration-probe", "migration_probe", True),
("x-financial-migration-probe-123", "migration_probe_clone_123", True),
("disposable-probe-7", "disposable_probe_7", True),
("x-financial-local-postgres", "migration_probe", False),
("migration-probe-7", "x_financial", False),
],
)
def test_disposable_target_requires_markers_on_host_and_database(
host: str,
database: str,
expected: bool,
) -> None:
assert is_disposable_maintenance_target(host=host, database=database) is expected
def test_dry_run_allows_non_disposable_target_without_apply_override() -> None:
target = validate_maintenance_database_target(
"postgresql://finance:password@db.internal/x_financial",
expected_host="db.internal",
expected_database="x_financial",
apply=False,
)
assert target.is_disposable is False
def test_apply_allows_disposable_target_without_non_disposable_override() -> None:
target = validate_maintenance_database_target(
"postgresql://finance:password@x-financial-migration-probe-1/migration_probe_1",
expected_host="x-financial-migration-probe-1",
expected_database="migration_probe_1",
apply=True,
)
assert target.is_disposable is True
def test_apply_rejects_wrong_optional_confirmation_for_disposable_target() -> None:
assert_target_error(
"confirm_target_mismatch",
lambda: validate_maintenance_database_target(
"postgresql://finance:password@migration-probe/migration_probe",
expected_host="migration-probe",
expected_database="migration_probe",
apply=True,
confirm_target="other:5432/migration_probe",
),
)
def test_apply_rejects_non_disposable_target_without_explicit_override() -> None:
assert_target_error(
"non_disposable_apply_forbidden",
lambda: validate_maintenance_database_target(
"postgresql://finance:password@db.internal/x_financial",
expected_host="db.internal",
expected_database="x_financial",
apply=True,
confirm_target="db.internal:5432/x_financial",
),
)
@pytest.mark.parametrize("confirm_target", [None, "", "db.internal:5432/other_database"])
def test_apply_rejects_non_disposable_target_without_exact_confirmation(
confirm_target: str | None,
) -> None:
assert_target_error(
"confirm_target_mismatch",
lambda: validate_maintenance_database_target(
"postgresql://finance:password@db.internal/x_financial",
expected_host="db.internal",
expected_database="x_financial",
apply=True,
allow_non_disposable=True,
confirm_target=confirm_target,
),
)
def test_apply_allows_non_disposable_target_with_override_and_exact_confirmation() -> None:
target = validate_maintenance_database_target(
"postgresql://finance:password@db.internal/x_financial",
expected_host="db.internal",
expected_database="x_financial",
apply=True,
allow_non_disposable=True,
confirm_target="db.internal:5432/x_financial",
)
assert target.exact_target == "db.internal:5432/x_financial"