feat(ai): add tenant-safe hierarchical expense learning

2026-07-16 14:30:41 +08:00
parent 6bdf65bc24
commit ee88a36baf
65 changed files with 6909 additions and 232 deletions
--- a/server/tests/test_alembic_migrations.py
+++ b/server/tests/test_alembic_migrations.py
@@ -3,7 +3,10 @@ from __future__ import annotations
 import os
 import re
 from collections.abc import Iterator
+from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
+from types import SimpleNamespace
+from typing import Any

 import pytest
 from alembic.config import Config
@@ -14,16 +17,44 @@ from sqlalchemy.pool import NullPool

 from alembic import command
 from app.core.config import get_settings
-from app.db.migration_preflight import MigrationPreflightError, validate_migration_state
+from app.db.migration_preflight import (
+    LEGACY_ADOPTABLE_HISTORICAL_CASE_TABLES,
+    MigrationPreflightError,
+    validate_migration_state,
+)
 from app.db.schema_ownership import MIGRATION_OWNED_TABLES, create_legacy_schema
+from app.models.ai_memory import MemoryEntry
+from app.models.risk_observation import RiskObservation

 MIGRATION_TEST_DATABASE_URL = os.getenv("MIGRATION_TEST_DATABASE_URL", "").strip()
 LEGACY_PROBE_TABLE = "legacy_migration_probe_records"
-HEAD_REVISION = "20260716_0006"
+HEAD_REVISION = "20260716_0009"
 SERVER_DIR = Path(__file__).resolve().parents[1]
 ALEMBIC_INI_PATH = SERVER_DIR / "alembic.ini"


+class _UnsupportedDialectOperationGuard:
+    def __init__(self, dialect_name: str = "sqlite") -> None:
+        self.bind = SimpleNamespace(dialect=SimpleNamespace(name=dialect_name))
+        self.mutation_calls: list[str] = []
+
+    def get_bind(self) -> SimpleNamespace:
+        return self.bind
+
+    def __getattr__(self, name: str) -> Any:
+        self.mutation_calls.append(name)
+        raise AssertionError(f"unsupported dialect attempted migration operation: {name}")
+
+
+def _load_migration_module(filename: str) -> Any:
+    path = SERVER_DIR / "alembic" / "versions" / filename
+    spec = spec_from_file_location(f"migration_test_{path.stem}", path)
+    assert spec is not None and spec.loader is not None
+    module = module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 def _normalize_probe_component(value: str) -> str:
    return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")

@@ -53,13 +84,9 @@ def _require_disposable_probe_url(raw_url: str) -> str:
    host = _normalize_probe_component(parsed.host or "")
    database = _normalize_probe_component(parsed.database or "")
    if not _is_disposable_probe_host(host):
-        raise RuntimeError(
-            "迁移测试数据库主机名必须使用 migration-probe 或 disposable-probe 前缀"
-        )
+        raise RuntimeError("迁移测试数据库主机名必须使用 migration-probe 或 disposable-probe 前缀")
    if not _is_disposable_probe_database(database):
-        raise RuntimeError(
-            "迁移测试数据库名必须使用 migration-probe 或 disposable-probe 前缀"
-        )
+        raise RuntimeError("迁移测试数据库名必须使用 migration-probe 或 disposable-probe 前缀")

    return raw_url

@@ -93,13 +120,21 @@ def _alembic_config(database_url: str) -> Config:


 def _upgrade_head(database_url: str) -> None:
+    _upgrade_revision(database_url, "head")
+
+
+def _upgrade_revision(database_url: str, revision: str) -> None:
    get_settings.cache_clear()
-    command.upgrade(_alembic_config(database_url), "head")
+    command.upgrade(_alembic_config(database_url), revision)


 def _downgrade_base(database_url: str) -> None:
+    _downgrade_revision(database_url, "base")
+
+
+def _downgrade_revision(database_url: str, revision: str) -> None:
    get_settings.cache_clear()
-    command.downgrade(_alembic_config(database_url), "base")
+    command.downgrade(_alembic_config(database_url), revision)


 def _table_names(engine: Engine) -> set[str]:
@@ -132,6 +167,28 @@ def _assert_indexes(
        assert indexes.get(index_name) == expected_columns


+def _assert_postgresql_index_predicate(
+    engine: Engine,
+    table_name: str,
+    index_name: str,
+    *expected_fragments: str,
+) -> None:
+    with engine.connect() as connection:
+        index_definition = str(
+            connection.scalar(
+                text(
+                    "SELECT indexdef FROM pg_indexes "
+                    "WHERE schemaname = 'public' "
+                    "AND tablename = :table_name AND indexname = :index_name"
+                ),
+                {"table_name": table_name, "index_name": index_name},
+            )
+            or ""
+        ).lower()
+    for fragment in expected_fragments:
+        assert fragment.lower() in index_definition
+
+
 def _assert_check_constraint(
    engine: Engine,
    table_name: str,
@@ -176,6 +233,20 @@ def _assert_composite_foreign_key(
    assert str(matching[0].get("options", {}).get("ondelete", "")).upper() == "RESTRICT"


+def _assert_no_foreign_key(
+    engine: Engine,
+    table_name: str,
+    constrained_columns: tuple[str, ...],
+    referred_table: str,
+) -> None:
+    foreign_keys = inspect(engine).get_foreign_keys(table_name, schema="public")
+    assert not any(
+        tuple(item["constrained_columns"]) == constrained_columns
+        and item["referred_table"] == referred_table
+        for item in foreign_keys
+    )
+
+
 def _assert_head_schema(engine: Engine) -> None:
    names = _table_names(engine)
    assert MIGRATION_OWNED_TABLES.issubset(names)
@@ -280,11 +351,48 @@ def _assert_head_schema(engine: Engine) -> None:
        "uq_attachment_association_jobs_owner_dedupe",
        ("tenant_id", "owner_username", "dedupe_key", "generation"),
    )
+    _assert_unique_constraint(
+        engine,
+        "risk_observations",
+        "uq_risk_observations_tenant_key",
+        ("tenant_id", "observation_key"),
+    )
+    _assert_unique_constraint(
+        engine,
+        "few_shot_samples",
+        "uq_few_shot_samples_tenant_key",
+        ("tenant_id", "sample_key"),
+    )
    _assert_check_constraint(
        engine,
        "memory_entries",
        "ck_memory_entries_expired_fields",
    )
+    _assert_check_constraint(
+        engine,
+        "memory_entries",
+        "ck_memory_entries_scope_origin",
+    )
+    _assert_check_constraint(
+        engine,
+        "memory_entries",
+        "ck_memory_entries_enterprise_scope",
+    )
+    _assert_check_constraint(
+        engine,
+        "memory_entries",
+        "ck_memory_entries_management_audit",
+    )
+    _assert_check_constraint(
+        engine,
+        "memory_entries",
+        "ck_memory_entries_management_idempotency_pair",
+    )
+    _assert_check_constraint(
+        engine,
+        "memory_entries",
+        "ck_memory_entries_revoke_idempotency_pair",
+    )
    _assert_check_constraint(
        engine,
        "attachment_association_jobs",
@@ -304,6 +412,16 @@ def _assert_head_schema(engine: Engine) -> None:
            "ix_expense_cases_tenant_status": ("tenant_id", "status"),
        },
    )
+    _assert_postgresql_index_predicate(
+        engine,
+        "memory_entries",
+        "uq_memory_entries_active_scope",
+        "status",
+        "'active'",
+        "scope_type",
+        "department",
+        "enterprise",
+    )
    _assert_indexes(
        engine,
        "expense_case_links",
@@ -416,6 +534,21 @@ def _assert_head_schema(engine: Engine) -> None:
                "field_key",
                "status",
            ),
+            "uq_memory_entries_management_request": (
+                "tenant_id",
+                "management_request_id",
+            ),
+            "uq_memory_entries_revoke_request": (
+                "tenant_id",
+                "revoke_request_id",
+            ),
+            "uq_memory_entries_active_scope": (
+                "tenant_id",
+                "scope_type",
+                "scope_id",
+                "scene",
+                "field_key",
+            ),
        },
    )
    _assert_indexes(
@@ -429,6 +562,30 @@ def _assert_head_schema(engine: Engine) -> None:
            ),
        },
    )
+    _assert_indexes(
+        engine,
+        "risk_observations",
+        {
+            "ix_risk_observations_tenant_status": (
+                "tenant_id",
+                "status",
+                "created_at",
+            ),
+        },
+    )
+    _assert_indexes(
+        engine,
+        "few_shot_samples",
+        {
+            "ix_few_shot_samples_tenant_rule_lookup": (
+                "tenant_id",
+                "scene",
+                "policy_ref",
+                "rule_version",
+                "status",
+            ),
+        },
+    )
    _assert_cascade_foreign_key(engine, "expense_case_links")
    _assert_cascade_foreign_key(engine, "business_events")
    _assert_composite_foreign_key(
@@ -443,6 +600,12 @@ def _assert_head_schema(engine: Engine) -> None:
        ("tenant_id", "expense_case_id"),
        "expense_cases",
    )
+    _assert_no_foreign_key(
+        engine,
+        "risk_observations",
+        ("claim_id",),
+        "expense_claims",
+    )
    _assert_composite_foreign_key(
        engine,
        "ai_decisions",
@@ -556,15 +719,19 @@ def _assert_runtime_cascade(engine: Engine) -> None:
                """
            )
        )
-        connection.execute(
-            text("DELETE FROM expense_cases WHERE id = 'migration-probe-case'")
+        connection.execute(text("DELETE FROM expense_cases WHERE id = 'migration-probe-case'"))
+        assert (
+            connection.scalar(
+                text("SELECT COUNT(*) FROM expense_case_links WHERE id = 'migration-probe-link'")
+            )
+            == 0
+        )
+        assert (
+            connection.scalar(
+                text("SELECT COUNT(*) FROM business_events WHERE id = 'migration-probe-event'")
+            )
+            == 0
        )
-        assert connection.scalar(
-            text("SELECT COUNT(*) FROM expense_case_links WHERE id = 'migration-probe-link'")
-        ) == 0
-        assert connection.scalar(
-            text("SELECT COUNT(*) FROM business_events WHERE id = 'migration-probe-event'")
-        ) == 0


 def _assert_learning_ledger_tenant_boundary(engine: Engine) -> None:
@@ -777,26 +944,273 @@ def _create_legacy_sentinel(engine: Engine) -> None:
        )


+def _create_hierarchical_memory_downgrade_probe(engine: Engine) -> None:
+    """验证 0007 在真实组织记忆存在时仍可安全降级。"""
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                INSERT INTO memory_entries (
+                    id, tenant_id, scope_type, scope_id, origin_type,
+                    managed_by, managed_at, management_reason, policy_version,
+                    scene, field_key, generation, value_json, value_fingerprint,
+                    status, evidence_count, approved_evidence_count, confidence,
+                    last_evidence_at, candidate_expires_at,
+                    activated_at, active_expires_at
+                ) VALUES (
+                    'hierarchical-memory-downgrade-probe',
+                    'migration-probe', 'enterprise', 'migration-probe',
+                    'admin_managed', 'migration-admin', CURRENT_TIMESTAMP,
+                    '验证组织记忆降级', 'expense_application_transport_org_memory.v1',
+                    'travel_application', 'transport_mode', 1,
+                    CAST('{"value":"火车"}' AS JSON),
+                    'sha256:hierarchical-memory-downgrade-probe',
+                    'active', 0, 0, 1,
+                    CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + INTERVAL '180 days',
+                    CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + INTERVAL '180 days'
+                )
+                """
+            )
+        )
+        assert connection.scalar(
+            text(
+                "SELECT COUNT(*) FROM memory_entries "
+                "WHERE id = 'hierarchical-memory-downgrade-probe'"
+            )
+        ) == 1
+
+
+def _create_duplicate_active_organization_memory_probe(engine: Engine) -> None:
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                INSERT INTO memory_entries (
+                    id, tenant_id, scope_type, scope_id, origin_type,
+                    managed_by, managed_at, management_reason, policy_version,
+                    scene, field_key, generation, value_json, value_fingerprint,
+                    status, evidence_count, approved_evidence_count, confidence,
+                    last_evidence_at, candidate_expires_at,
+                    activated_at, active_expires_at
+                ) VALUES
+                (
+                    'duplicate-active-organization-a',
+                    'migration-probe', 'department', 'department-probe',
+                    'admin_managed', 'migration-admin', CURRENT_TIMESTAMP,
+                    '验证组织生效记忆重复预检', 'expense_application_transport_org_memory.v1',
+                    'travel_application', 'transport_mode', 1,
+                    CAST('{"value":"火车"}' AS JSON),
+                    'sha256:duplicate-active-organization-a',
+                    'active', 0, 0, 1,
+                    CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + INTERVAL '180 days',
+                    CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + INTERVAL '180 days'
+                ),
+                (
+                    'duplicate-active-organization-b',
+                    'migration-probe', 'department', 'department-probe',
+                    'admin_managed', 'migration-admin', CURRENT_TIMESTAMP,
+                    '验证组织生效记忆重复预检', 'expense_application_transport_org_memory.v1',
+                    'travel_application', 'transport_mode', 2,
+                    CAST('{"value":"飞机"}' AS JSON),
+                    'sha256:duplicate-active-organization-b',
+                    'active', 0, 0, 1,
+                    CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + INTERVAL '180 days',
+                    CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + INTERVAL '180 days'
+                )
+                """
+            )
+        )
+
+
+def _delete_duplicate_active_organization_memory_probe(engine: Engine) -> None:
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                "DELETE FROM memory_entries "
+                "WHERE id IN ("
+                "'duplicate-active-organization-a', "
+                "'duplicate-active-organization-b'"
+                ")"
+            )
+        )
+
+
+def _create_enriched_few_shot_downgrade_probe(engine: Engine) -> None:
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                INSERT INTO few_shot_samples (
+                    id, tenant_id, sample_key, scene,
+                    policy_ref, rule_version, payload_json
+                ) VALUES (
+                    'enriched-few-shot-downgrade-probe', 'default',
+                    'enriched:few-shot:downgrade-probe', 'expense_reimbursement',
+                    'TRAVEL-001', 'v2', CAST('{}' AS JSON)
+                )
+                """
+            )
+        )
+
+
+def _delete_enriched_few_shot_downgrade_probe(engine: Engine) -> None:
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                "DELETE FROM few_shot_samples "
+                "WHERE id = 'enriched-few-shot-downgrade-probe'"
+            )
+        )
+
+
+def _create_historical_case_downgrade_probe(engine: Engine) -> None:
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                INSERT INTO risk_observations (
+                    id, tenant_id, observation_key, subject_type, subject_key,
+                    risk_type, risk_signal, risk_level,
+                    contribution_scores_json, baseline_json, evidence_json,
+                    graph_node_keys_json, graph_edge_keys_json, policy_refs_json,
+                    similar_case_claim_ids_json, ontology_json, decision_trace_json
+                ) VALUES (
+                    'historical-downgrade-observation', 'default',
+                    'historical:downgrade:observation', 'expense_claim',
+                    'claim:historical-downgrade', 'duplicate_invoice',
+                    'duplicate_invoice', 'high', CAST('{}' AS JSON), CAST('{}' AS JSON),
+                    CAST('[]' AS JSON), CAST('[]' AS JSON), CAST('[]' AS JSON),
+                    CAST('[]' AS JSON), CAST('[]' AS JSON), CAST('{}' AS JSON),
+                    CAST('{}' AS JSON)
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                INSERT INTO risk_observation_feedback (
+                    id, observation_id, feedback_type, payload_json
+                ) VALUES (
+                    'historical-downgrade-feedback',
+                    'historical-downgrade-observation', 'confirm', CAST('{}' AS JSON)
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                INSERT INTO few_shot_samples (
+                    id, tenant_id, sample_key, source_observation_id,
+                    scene, policy_ref, rule_version, payload_json
+                ) VALUES (
+                    'historical-downgrade-sample', 'default',
+                    'historical:downgrade:sample', 'historical-downgrade-observation',
+                    'expense_reimbursement', '', '', CAST('{}' AS JSON)
+                )
+                """
+            )
+        )
+
+
+def _assert_historical_case_downgrade_probe(engine: Engine) -> None:
+    inspector = inspect(engine)
+    risk_columns = {
+        str(item["name"]) for item in inspector.get_columns("risk_observations")
+    }
+    sample_columns = {
+        str(item["name"]) for item in inspector.get_columns("few_shot_samples")
+    }
+    assert "tenant_id" not in risk_columns
+    assert {"tenant_id", "policy_ref", "rule_version"}.isdisjoint(sample_columns)
+    assert not any(
+        item["constrained_columns"] == ["claim_id"]
+        and item["referred_table"] == "expense_claims"
+        and item["referred_columns"] == ["id"]
+        for item in inspector.get_foreign_keys("risk_observations")
+    )
+    with engine.connect() as connection:
+        assert connection.scalar(
+            text(
+                "SELECT COUNT(*) FROM risk_observations "
+                "WHERE id = 'historical-downgrade-observation'"
+            )
+        ) == 1
+        assert connection.scalar(
+            text(
+                "SELECT COUNT(*) FROM risk_observation_feedback "
+                "WHERE id = 'historical-downgrade-feedback'"
+            )
+        ) == 1
+        assert connection.scalar(
+            text(
+                "SELECT COUNT(*) FROM few_shot_samples "
+                "WHERE id = 'historical-downgrade-sample'"
+            )
+        ) == 1
+
+
 def _assert_legacy_sentinel(engine: Engine) -> None:
    assert LEGACY_PROBE_TABLE in _table_names(engine)
    with engine.connect() as connection:
        payload = connection.scalar(
-            text(
-                f"SELECT payload FROM {LEGACY_PROBE_TABLE} "
-                "WHERE id = 'legacy-sentinel'"
-            )
+            text(f"SELECT payload FROM {LEGACY_PROBE_TABLE} WHERE id = 'legacy-sentinel'")
        )
    assert payload == "must-survive-migration-cycle"


 def _assert_base_schema(engine: Engine) -> None:
    names = _table_names(engine)
-    assert not MIGRATION_OWNED_TABLES.intersection(names)
+    unsafe_owned_tables = (
+        MIGRATION_OWNED_TABLES - LEGACY_ADOPTABLE_HISTORICAL_CASE_TABLES
+    ).intersection(names)
+    assert not unsafe_owned_tables
+    assert LEGACY_ADOPTABLE_HISTORICAL_CASE_TABLES.issubset(names)
    assert "alembic_version" in names
    with engine.connect() as connection:
        assert connection.scalar(text("SELECT COUNT(*) FROM alembic_version")) == 0


+@pytest.mark.parametrize(
+    ("filename", "direction"),
+    [
+        ("20260716_0008_tenant_safe_historical_cases.py", "upgrade"),
+        ("20260716_0008_tenant_safe_historical_cases.py", "downgrade"),
+        ("20260716_0009_organization_memory_idempotency.py", "upgrade"),
+        ("20260716_0009_organization_memory_idempotency.py", "downgrade"),
+    ],
+)
+def test_postgresql_only_migrations_reject_other_dialects_before_mutation(
+    filename: str,
+    direction: str,
+) -> None:
+    migration = _load_migration_module(filename)
+    operation_guard = _UnsupportedDialectOperationGuard()
+    migration.op = operation_guard
+
+    with pytest.raises(RuntimeError, match="only supports PostgreSQL"):
+        getattr(migration, direction)()
+
+    assert operation_guard.mutation_calls == []
+
+
+def test_head_model_declares_soft_claim_reference_and_organization_only_active_index() -> None:
+    claim_column = RiskObservation.__table__.c.claim_id
+    assert not claim_column.foreign_keys
+    claim_relationship = inspect(RiskObservation).relationships["claim"]
+    assert claim_relationship.viewonly is True
+
+    active_scope_index = next(
+        index
+        for index in MemoryEntry.__table__.indexes
+        if index.name == "uq_memory_entries_active_scope"
+    )
+    predicate = str(active_scope_index.dialect_options["postgresql"]["where"])
+    assert "status = 'active'" in predicate
+    assert "scope_type IN ('department', 'enterprise')" in predicate
+
+
@pytest.mark.parametrize(
    "database_url",
    [
@@ -818,6 +1232,23 @@ def test_alembic_migration_cycle_on_disposable_postgres(
        assert _table_names(engine) == set(), "迁移测试必须从全新空库开始"
        assert validate_migration_state(engine).revision is None

+        _upgrade_revision(migration_database_url, "20260716_0008")
+        _create_duplicate_active_organization_memory_probe(engine)
+        with pytest.raises(RuntimeError, match="duplicate active memories"):
+            _upgrade_head(migration_database_url)
+        assert validate_migration_state(engine).revision == "20260716_0008"
+        memory_columns = {
+            str(item["name"])
+            for item in inspect(engine).get_columns("memory_entries", schema="public")
+        }
+        assert {
+            "management_request_id",
+            "management_payload_fingerprint",
+            "revoke_request_id",
+            "revoke_payload_fingerprint",
+        }.isdisjoint(memory_columns)
+        _delete_duplicate_active_organization_memory_probe(engine)
+
        _upgrade_head(migration_database_url)
        _assert_head_schema(engine)
        assert validate_migration_state(engine).revision == HEAD_REVISION
@@ -830,10 +1261,28 @@ def test_alembic_migration_cycle_on_disposable_postgres(
        _assert_runtime_cascade(engine)

        _create_legacy_sentinel(engine)
+        _downgrade_revision(migration_database_url, "20260716_0008")
+        _create_enriched_few_shot_downgrade_probe(engine)
+        with pytest.raises(RuntimeError, match="contain policy_ref or rule_version data"):
+            _downgrade_revision(migration_database_url, "20260716_0007")
+        assert validate_migration_state(engine).revision == "20260716_0008"
+        with engine.connect() as connection:
+            assert connection.execute(
+                text(
+                    "SELECT policy_ref, rule_version FROM few_shot_samples "
+                    "WHERE id = 'enriched-few-shot-downgrade-probe'"
+                )
+            ).one() == ("TRAVEL-001", "v2")
+        _delete_enriched_few_shot_downgrade_probe(engine)
+        _upgrade_head(migration_database_url)
+
+        _create_hierarchical_memory_downgrade_probe(engine)
+        _create_historical_case_downgrade_probe(engine)
        _downgrade_base(migration_database_url)

        _assert_base_schema(engine)
        _assert_legacy_sentinel(engine)
+        _assert_historical_case_downgrade_probe(engine)
        assert validate_migration_state(engine).revision is None

        with engine.begin() as connection: