feat(platform): close AI expense value loop

Add tenant-safe value, telemetry, connector, commercial, and production-readiness foundations.

server/tests/test_runtime_chat_attempts.py

@@ -0,0 +1,716 @@
+from __future__ import annotations
+
+import pytest
+from runtime_chat_testkit import (
+    DenyingAttemptObserver,
+    FailingCompletionObserver,
+    FailingPermitObserver,
+    RecordingAttemptObserver,
+    build_operation_context,
+    patch_single_slot,
+)
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.db.base import Base
+from app.services import runtime_chat as runtime_chat_module
+from app.services.model_connectivity import ConnectivityCheckError
+from app.services.runtime_chat import (
+    RuntimeChatOperationContext,
+    RuntimeChatService,
+)
+
+
+def build_session_factory() -> sessionmaker[Session]:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(bind=engine)
+    return sessionmaker(bind=engine, autoflush=False, autocommit=False)
+
+
+def _clear_runtime_chat_cooldown() -> None:
+    runtime_chat_module._slot_failure_until.clear()
+
+
+@pytest.mark.parametrize(
+    ("provider", "configured_model", "response_payload", "expected_usage"),
+    [
+        (
+            "OpenAI Compatible",
+            "gpt-configured",
+            {
+                "id": "chatcmpl-openai-001",
+                "model": "gpt-response",
+                "choices": [{"message": {"content": "openai answer"}}],
+                "usage": {
+                    "prompt_tokens": 17,
+                    "completion_tokens": 5,
+                    "total_tokens": 22,
+                },
+            },
+            {
+                "prompt_tokens": 17,
+                "completion_tokens": 5,
+                "total_tokens": 22,
+                "prompt_eval_count": None,
+                "eval_count": None,
+            },
+        ),
+        (
+            "Azure OpenAI",
+            "azure-deployment",
+            {
+                "id": "chatcmpl-azure-001",
+                "model": "azure-response-model",
+                "choices": [{"message": {"content": "azure answer"}}],
+                "usage": {
+                    "prompt_tokens": 23,
+                    "completion_tokens": 7,
+                    "total_tokens": 30,
+                },
+            },
+            {
+                "prompt_tokens": 23,
+                "completion_tokens": 7,
+                "total_tokens": 30,
+                "prompt_eval_count": None,
+                "eval_count": None,
+            },
+        ),
+        (
+            "Ollama",
+            "llama-configured",
+            {
+                "model": "llama-response",
+                "message": {"content": "ollama answer"},
+                "prompt_eval_count": 31,
+                "eval_count": 11,
+            },
+            {
+                "prompt_tokens": None,
+                "completion_tokens": None,
+                "total_tokens": None,
+                "prompt_eval_count": 31,
+                "eval_count": 11,
+            },
+        ),
+    ],
+)
+def test_runtime_chat_preserves_authoritative_provider_usage(
+    monkeypatch,
+    provider: str,
+    configured_model: str,
+    response_payload: dict[str, object],
+    expected_usage: dict[str, int | None],
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    observer = RecordingAttemptObserver()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider=provider,
+            model=configured_model,
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (200, response_payload),
+        )
+
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.text is not None
+    assert len(result.calls) == 1
+    trace = result.calls[0]
+    assert trace.provider == provider
+    assert trace.model == configured_model
+    assert trace.attempt == 1
+    assert trace.response_id == response_payload.get("id")
+    assert trace.response_model == response_payload.get("model")
+    assert trace.outcome == "succeeded"
+    assert trace.started_at is not None
+    assert trace.completed_at is not None
+    assert trace.completed_at >= trace.started_at
+    assert trace.usage.availability == "available"
+    for field_name, expected_value in expected_usage.items():
+        assert getattr(trace.usage, field_name) == expected_value
+    assert trace.observer_status == "completed_notified"
+    assert len(observer.permits) == 1
+    assert len(observer.completed) == 1
+    assert observer.completed[0].identity.tenant_id == "tenant-runtime-chat"
+    assert observer.completed[0].identity.invocation_seq == 7
+    assert observer.completed[0].usage == trace.usage
+
+
+def test_runtime_chat_keeps_missing_usage_unavailable_without_estimation(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    observer = RecordingAttemptObserver()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-no-usage",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (
+                200,
+                {
+                    "id": "chatcmpl-no-usage",
+                    "model": "gpt-no-usage",
+                    "choices": [{"message": {"content": "answer"}}],
+                },
+            ),
+        )
+
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_tokens=9876,
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    usage = result.calls[0].usage
+    assert usage.availability == "unavailable"
+    assert usage.source == "unavailable"
+    assert usage.prompt_tokens is None
+    assert usage.completion_tokens is None
+    assert usage.total_tokens is None
+    assert observer.completed[0].usage == usage
+
+
+def test_runtime_chat_notifies_every_real_retry_attempt_and_marks_timeout_unknown(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    timeline: list[str] = []
+    observer = RecordingAttemptObserver(timeline)
+    request_count = 0
+
+    def fake_send_json_request(*_args, **_kwargs):
+        nonlocal request_count
+        request_count += 1
+        timeline.append(f"send:{request_count}")
+        if request_count == 1:
+            raise TimeoutError("provider response timed out")
+        return 200, {
+            "id": "chatcmpl-retry-002",
+            "model": "gpt-retry-response",
+            "choices": [
+                {
+                    "message": {
+                        "tool_calls": [
+                            {
+                                "id": "call-retry-002",
+                                "function": {
+                                    "name": "submit_plan",
+                                    "arguments": "{}",
+                                },
+                            }
+                        ]
+                    }
+                }
+            ],
+            "usage": {
+                "prompt_tokens": 41,
+                "completion_tokens": 3,
+                "total_tokens": 44,
+            },
+        }
+
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-retry",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            fake_send_json_request,
+        )
+        monkeypatch.setattr("app.services.runtime_chat.sleep", lambda *_args: None)
+
+        result = service.complete_with_tool_call(
+            [{"role": "user", "content": "build plan"}],
+            tools=[{"type": "function", "function": {"name": "submit_plan"}}],
+            slot_priority=("main",),
+            max_attempts=2,
+            use_failure_cooldown=False,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.tool_call is not None
+    assert [item.attempt for item in result.calls] == [1, 2]
+    assert [item.outcome for item in result.calls] == [
+        "outcome_unknown",
+        "succeeded",
+    ]
+    assert result.calls[0].usage.availability == "unavailable"
+    assert result.calls[1].usage.total_tokens == 44
+    assert [item.identity.attempt for item in observer.permits] == [1, 2]
+    assert [item.identity.attempt for item in observer.completed] == [1, 2]
+    assert observer.completed[0].request_may_have_been_sent is True
+    assert observer.completed[0].requires_reconciliation is True
+    assert result.calls[0].requires_reconciliation is True
+    assert observer.permits[0].identity.attempt_key != observer.permits[1].identity.attempt_key
+    assert timeline == [
+        "permit:1",
+        "send:1",
+        "completed:1:outcome_unknown",
+        "permit:2",
+        "send:2",
+        "completed:2:succeeded",
+    ]
+
+
+def test_runtime_chat_preserves_usage_when_tool_response_postprocessing_fails(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    observer = RecordingAttemptObserver()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-postprocess",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (
+                200,
+                {
+                    "id": "chatcmpl-postprocess",
+                    "model": "gpt-postprocess-response",
+                    "choices": [
+                        {
+                            "message": {
+                                "tool_calls": [
+                                    {
+                                        "id": "call-invalid-json",
+                                        "function": {
+                                            "name": "submit_plan",
+                                            "arguments": "{invalid-json",
+                                        },
+                                    }
+                                ]
+                            }
+                        }
+                    ],
+                    "usage": {
+                        "prompt_tokens": 13,
+                        "completion_tokens": 2,
+                        "total_tokens": 15,
+                    },
+                },
+            ),
+        )
+
+        result = service.complete_with_tool_call(
+            [{"role": "user", "content": "build plan"}],
+            tools=[{"type": "function", "function": {"name": "submit_plan"}}],
+            slot_priority=("main",),
+            max_attempts=1,
+            use_failure_cooldown=False,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.tool_call is None
+    assert result.calls[0].outcome == "postprocess_failed"
+    assert result.calls[0].response_id == "chatcmpl-postprocess"
+    assert result.calls[0].usage.total_tokens == 15
+    assert observer.completed[0].outcome == "postprocess_failed"
+    assert observer.completed[0].usage.total_tokens == 15
+
+
+def test_runtime_chat_observer_failure_keeps_success_and_reports_reconciliation(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    observer = FailingCompletionObserver()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-observer-failure",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (
+                200,
+                {
+                    "id": "chatcmpl-observer-failure",
+                    "model": "gpt-observer-response",
+                    "choices": [{"message": {"content": "valid answer"}}],
+                    "usage": {
+                        "prompt_tokens": 8,
+                        "completion_tokens": 2,
+                        "total_tokens": 10,
+                    },
+                },
+            ),
+        )
+
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.text == "valid answer"
+    trace = result.calls[0]
+    assert trace.outcome == "succeeded"
+    assert trace.usage.total_tokens == 10
+    assert trace.observer_status == "reconciliation_required"
+    assert [item.phase for item in trace.observer_failures] == ["completion"]
+    assert len(observer.failures) == 1
+    assert observer.failures[0].identity.operation_id == "operation-runtime-chat-001"
+
+
+def test_runtime_chat_without_operation_context_does_not_invoke_observer(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    observer = RecordingAttemptObserver()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-no-context",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (
+                200,
+                {
+                    "id": "chatcmpl-no-context",
+                    "choices": [{"message": {"content": "legacy answer"}}],
+                    "usage": {
+                        "prompt_tokens": 5,
+                        "completion_tokens": 2,
+                        "total_tokens": 7,
+                    },
+                },
+            ),
+        )
+
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+        )
+
+    assert result.text == "legacy answer"
+    assert result.calls[0].usage.total_tokens == 7
+    assert result.calls[0].observer_status == "not_applicable"
+    assert observer.permits == []
+    assert observer.completed == []
+    assert observer.failures == []
+
+
+def test_runtime_chat_explicit_permit_denial_is_not_sent(monkeypatch) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    observer = DenyingAttemptObserver()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-denied",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: pytest.fail("permit 拒绝后不应发送请求"),
+        )
+
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.text is None
+    assert result.calls[0].status == "blocked"
+    assert result.calls[0].outcome == "not_sent"
+    assert observer.completed[0].request_may_have_been_sent is False
+    assert observer.completed[0].usage.availability == "unavailable"
+
+
+def test_runtime_chat_provider_http_error_is_rejected_with_status(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    observer = RecordingAttemptObserver()
+    session_factory = build_session_factory()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-rate-limited",
+        )
+
+        def reject_request(*_args, **_kwargs):
+            raise ConnectivityCheckError("rate limited", status_code=429)
+
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            reject_request,
+        )
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.text is None
+    trace = result.calls[0]
+    assert trace.outcome == "provider_rejected"
+    assert trace.provider_status_code == 429
+    assert trace.requires_reconciliation is False
+    assert observer.completed[0].provider_status_code == 429
+    assert observer.completed[0].request_may_have_been_sent is True
+
+
+def test_runtime_chat_rejected_response_preserves_authoritative_usage(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    observer = RecordingAttemptObserver()
+    session_factory = build_session_factory()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-rejected-with-usage",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (
+                429,
+                {
+                    "id": "chatcmpl-rejected",
+                    "model": "gpt-rejected-response",
+                    "usage": {
+                        "prompt_tokens": 9,
+                        "completion_tokens": 0,
+                        "total_tokens": 9,
+                    },
+                },
+            ),
+        )
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    trace = result.calls[0]
+    assert trace.outcome == "provider_rejected"
+    assert trace.provider_status_code == 429
+    assert trace.response_id == "chatcmpl-rejected"
+    assert trace.response_model == "gpt-rejected-response"
+    assert trace.usage.availability == "available"
+    assert trace.usage.total_tokens == 9
+    assert observer.completed[0].usage == trace.usage
+
+
+def test_runtime_chat_pre_send_adapter_failure_is_not_sent(monkeypatch) -> None:
+    _clear_runtime_chat_cooldown()
+    observer = RecordingAttemptObserver()
+    session_factory = build_session_factory()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-invalid-endpoint",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat.request_openai_compatible_completion",
+            lambda **_kwargs: (_ for _ in ()).throw(ValueError("invalid endpoint")),
+        )
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.calls[0].outcome == "not_sent"
+    assert result.calls[0].provider_status_code is None
+    assert observer.completed[0].request_may_have_been_sent is False
+
+
+def test_runtime_chat_permit_observer_failure_blocks_provider_and_reconciles(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    observer = FailingPermitObserver()
+    session_factory = build_session_factory()
+    with session_factory() as db:
+        service = RuntimeChatService(db, attempt_observer=observer)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-permit-failure",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: pytest.fail(
+                "permit 状态未知时不应发送 provider 请求"
+            ),
+        )
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    trace = result.calls[0]
+    assert result.text is None
+    assert trace.status == "blocked"
+    assert trace.outcome == "not_sent"
+    assert trace.observer_status == "reconciliation_required"
+    assert trace.requires_reconciliation is True
+    assert [failure.phase for failure in trace.observer_failures] == ["permit"]
+    assert len(observer.completed) == 1
+    assert observer.completed[0].request_may_have_been_sent is False
+
+
+def test_runtime_chat_trusted_context_without_observer_requires_reconciliation(
+    monkeypatch,
+) -> None:
+    _clear_runtime_chat_cooldown()
+    session_factory = build_session_factory()
+    with session_factory() as db:
+        service = RuntimeChatService(db)
+        patch_single_slot(
+            monkeypatch,
+            service,
+            provider="OpenAI Compatible",
+            model="gpt-unobserved",
+        )
+        monkeypatch.setattr(
+            "app.services.runtime_chat._send_json_request",
+            lambda *_args, **_kwargs: (
+                200,
+                {
+                    "id": "chatcmpl-unobserved",
+                    "choices": [{"message": {"content": "answer"}}],
+                    "usage": {
+                        "prompt_tokens": 3,
+                        "completion_tokens": 1,
+                        "total_tokens": 4,
+                    },
+                },
+            ),
+        )
+        result = service.complete_with_trace(
+            [{"role": "user", "content": "hello"}],
+            slot_priority=("main",),
+            max_attempts=1,
+            operation_context=build_operation_context(),
+        )
+
+    assert result.text == "answer"
+    assert result.calls[0].observer_status == "not_configured"
+    assert result.calls[0].requires_reconciliation is True
+
+
+@pytest.mark.parametrize(
+    "context_kwargs",
+    [
+        {"tenant_id": ""},
+        {"operation_id": ""},
+        {"invocation_seq": 0},
+        {"invocation_seq": True},
+        {"invocation_seq": "1"},
+        {"attempt_scope": ""},
+    ],
+)
+def test_runtime_chat_operation_context_rejects_untrusted_empty_identity(
+    context_kwargs,
+) -> None:
+    values = {
+        "tenant_id": "tenant-trusted",
+        "operation_id": "operation-trusted",
+        "invocation_seq": 1,
+        "attempt_scope": "trusted-entry",
+    }
+    values.update(context_kwargs)
+    with pytest.raises(ValueError):
+        RuntimeChatOperationContext(**values)
+
+
+def test_runtime_chat_attempt_key_has_unambiguous_opaque_identity() -> None:
+    common = {
+        "run_id": "run",
+        "invocation_seq": 1,
+        "attempt_scope": "scope",
+    }
+    first = RuntimeChatOperationContext(
+        tenant_id="tenant|operation",
+        operation_id="id",
+        **common,
+    ).build_attempt_identity(
+        slot="main",
+        provider="OpenAI Compatible",
+        model="gpt-test",
+        attempt=1,
+    )
+    second = RuntimeChatOperationContext(
+        tenant_id="tenant",
+        operation_id="operation|id",
+        **common,
+    ).build_attempt_identity(
+        slot="main",
+        provider="OpenAI Compatible",
+        model="gpt-test",
+        attempt=1,
+    )
+
+    assert first.attempt_key != second.attempt_key
+    assert first.attempt_key.startswith("runtime-chat-attempt:v1:")
+    assert "tenant" not in first.attempt_key
+    assert "operation" not in first.attempt_key