from __future__ import annotations import pytest from runtime_chat_testkit import ( DenyingAttemptObserver, FailingCompletionObserver, FailingPermitObserver, RecordingAttemptObserver, build_operation_context, patch_single_slot, ) from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import StaticPool from app.db.base import Base from app.services import runtime_chat as runtime_chat_module from app.services.model_connectivity import ConnectivityCheckError from app.services.runtime_chat import ( RuntimeChatOperationContext, RuntimeChatService, ) def build_session_factory() -> sessionmaker[Session]: engine = create_engine( "sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all(bind=engine) return sessionmaker(bind=engine, autoflush=False, autocommit=False) def _clear_runtime_chat_cooldown() -> None: runtime_chat_module._slot_failure_until.clear() @pytest.mark.parametrize( ("provider", "configured_model", "response_payload", "expected_usage"), [ ( "OpenAI Compatible", "gpt-configured", { "id": "chatcmpl-openai-001", "model": "gpt-response", "choices": [{"message": {"content": "openai answer"}}], "usage": { "prompt_tokens": 17, "completion_tokens": 5, "total_tokens": 22, }, }, { "prompt_tokens": 17, "completion_tokens": 5, "total_tokens": 22, "prompt_eval_count": None, "eval_count": None, }, ), ( "Azure OpenAI", "azure-deployment", { "id": "chatcmpl-azure-001", "model": "azure-response-model", "choices": [{"message": {"content": "azure answer"}}], "usage": { "prompt_tokens": 23, "completion_tokens": 7, "total_tokens": 30, }, }, { "prompt_tokens": 23, "completion_tokens": 7, "total_tokens": 30, "prompt_eval_count": None, "eval_count": None, }, ), ( "Ollama", "llama-configured", { "model": "llama-response", "message": {"content": "ollama answer"}, "prompt_eval_count": 31, "eval_count": 11, }, { "prompt_tokens": None, "completion_tokens": None, "total_tokens": None, "prompt_eval_count": 31, "eval_count": 11, }, ), ], ) def test_runtime_chat_preserves_authoritative_provider_usage( monkeypatch, provider: str, configured_model: str, response_payload: dict[str, object], expected_usage: dict[str, int | None], ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() observer = RecordingAttemptObserver() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider=provider, model=configured_model, ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: (200, response_payload), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) assert result.text is not None assert len(result.calls) == 1 trace = result.calls[0] assert trace.provider == provider assert trace.model == configured_model assert trace.attempt == 1 assert trace.response_id == response_payload.get("id") assert trace.response_model == response_payload.get("model") assert trace.outcome == "succeeded" assert trace.started_at is not None assert trace.completed_at is not None assert trace.completed_at >= trace.started_at assert trace.usage.availability == "available" for field_name, expected_value in expected_usage.items(): assert getattr(trace.usage, field_name) == expected_value assert trace.observer_status == "completed_notified" assert len(observer.permits) == 1 assert len(observer.completed) == 1 assert observer.completed[0].identity.tenant_id == "tenant-runtime-chat" assert observer.completed[0].identity.invocation_seq == 7 assert observer.completed[0].usage == trace.usage def test_runtime_chat_keeps_missing_usage_unavailable_without_estimation( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() observer = RecordingAttemptObserver() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-no-usage", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: ( 200, { "id": "chatcmpl-no-usage", "model": "gpt-no-usage", "choices": [{"message": {"content": "answer"}}], }, ), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_tokens=9876, max_attempts=1, operation_context=build_operation_context(), ) usage = result.calls[0].usage assert usage.availability == "unavailable" assert usage.source == "unavailable" assert usage.prompt_tokens is None assert usage.completion_tokens is None assert usage.total_tokens is None assert observer.completed[0].usage == usage def test_runtime_chat_notifies_every_real_retry_attempt_and_marks_timeout_unknown( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() timeline: list[str] = [] observer = RecordingAttemptObserver(timeline) request_count = 0 def fake_send_json_request(*_args, **_kwargs): nonlocal request_count request_count += 1 timeline.append(f"send:{request_count}") if request_count == 1: raise TimeoutError("provider response timed out") return 200, { "id": "chatcmpl-retry-002", "model": "gpt-retry-response", "choices": [ { "message": { "tool_calls": [ { "id": "call-retry-002", "function": { "name": "submit_plan", "arguments": "{}", }, } ] } } ], "usage": { "prompt_tokens": 41, "completion_tokens": 3, "total_tokens": 44, }, } with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-retry", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", fake_send_json_request, ) monkeypatch.setattr("app.services.runtime_chat.sleep", lambda *_args: None) result = service.complete_with_tool_call( [{"role": "user", "content": "build plan"}], tools=[{"type": "function", "function": {"name": "submit_plan"}}], slot_priority=("main",), max_attempts=2, use_failure_cooldown=False, operation_context=build_operation_context(), ) assert result.tool_call is not None assert [item.attempt for item in result.calls] == [1, 2] assert [item.outcome for item in result.calls] == [ "outcome_unknown", "succeeded", ] assert result.calls[0].usage.availability == "unavailable" assert result.calls[1].usage.total_tokens == 44 assert [item.identity.attempt for item in observer.permits] == [1, 2] assert [item.identity.attempt for item in observer.completed] == [1, 2] assert observer.completed[0].request_may_have_been_sent is True assert observer.completed[0].requires_reconciliation is True assert result.calls[0].requires_reconciliation is True assert observer.permits[0].identity.attempt_key != observer.permits[1].identity.attempt_key assert timeline == [ "permit:1", "send:1", "completed:1:outcome_unknown", "permit:2", "send:2", "completed:2:succeeded", ] def test_runtime_chat_preserves_usage_when_tool_response_postprocessing_fails( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() observer = RecordingAttemptObserver() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-postprocess", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: ( 200, { "id": "chatcmpl-postprocess", "model": "gpt-postprocess-response", "choices": [ { "message": { "tool_calls": [ { "id": "call-invalid-json", "function": { "name": "submit_plan", "arguments": "{invalid-json", }, } ] } } ], "usage": { "prompt_tokens": 13, "completion_tokens": 2, "total_tokens": 15, }, }, ), ) result = service.complete_with_tool_call( [{"role": "user", "content": "build plan"}], tools=[{"type": "function", "function": {"name": "submit_plan"}}], slot_priority=("main",), max_attempts=1, use_failure_cooldown=False, operation_context=build_operation_context(), ) assert result.tool_call is None assert result.calls[0].outcome == "postprocess_failed" assert result.calls[0].response_id == "chatcmpl-postprocess" assert result.calls[0].usage.total_tokens == 15 assert observer.completed[0].outcome == "postprocess_failed" assert observer.completed[0].usage.total_tokens == 15 def test_runtime_chat_observer_failure_keeps_success_and_reports_reconciliation( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() observer = FailingCompletionObserver() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-observer-failure", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: ( 200, { "id": "chatcmpl-observer-failure", "model": "gpt-observer-response", "choices": [{"message": {"content": "valid answer"}}], "usage": { "prompt_tokens": 8, "completion_tokens": 2, "total_tokens": 10, }, }, ), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) assert result.text == "valid answer" trace = result.calls[0] assert trace.outcome == "succeeded" assert trace.usage.total_tokens == 10 assert trace.observer_status == "reconciliation_required" assert [item.phase for item in trace.observer_failures] == ["completion"] assert len(observer.failures) == 1 assert observer.failures[0].identity.operation_id == "operation-runtime-chat-001" def test_runtime_chat_without_operation_context_does_not_invoke_observer( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() observer = RecordingAttemptObserver() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-no-context", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: ( 200, { "id": "chatcmpl-no-context", "choices": [{"message": {"content": "legacy answer"}}], "usage": { "prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7, }, }, ), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, ) assert result.text == "legacy answer" assert result.calls[0].usage.total_tokens == 7 assert result.calls[0].observer_status == "not_applicable" assert observer.permits == [] assert observer.completed == [] assert observer.failures == [] def test_runtime_chat_explicit_permit_denial_is_not_sent(monkeypatch) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() observer = DenyingAttemptObserver() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-denied", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: pytest.fail("permit 拒绝后不应发送请求"), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) assert result.text is None assert result.calls[0].status == "blocked" assert result.calls[0].outcome == "not_sent" assert observer.completed[0].request_may_have_been_sent is False assert observer.completed[0].usage.availability == "unavailable" def test_runtime_chat_provider_http_error_is_rejected_with_status( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() observer = RecordingAttemptObserver() session_factory = build_session_factory() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-rate-limited", ) def reject_request(*_args, **_kwargs): raise ConnectivityCheckError("rate limited", status_code=429) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", reject_request, ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) assert result.text is None trace = result.calls[0] assert trace.outcome == "provider_rejected" assert trace.provider_status_code == 429 assert trace.requires_reconciliation is False assert observer.completed[0].provider_status_code == 429 assert observer.completed[0].request_may_have_been_sent is True def test_runtime_chat_rejected_response_preserves_authoritative_usage( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() observer = RecordingAttemptObserver() session_factory = build_session_factory() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-rejected-with-usage", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: ( 429, { "id": "chatcmpl-rejected", "model": "gpt-rejected-response", "usage": { "prompt_tokens": 9, "completion_tokens": 0, "total_tokens": 9, }, }, ), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) trace = result.calls[0] assert trace.outcome == "provider_rejected" assert trace.provider_status_code == 429 assert trace.response_id == "chatcmpl-rejected" assert trace.response_model == "gpt-rejected-response" assert trace.usage.availability == "available" assert trace.usage.total_tokens == 9 assert observer.completed[0].usage == trace.usage def test_runtime_chat_pre_send_adapter_failure_is_not_sent(monkeypatch) -> None: _clear_runtime_chat_cooldown() observer = RecordingAttemptObserver() session_factory = build_session_factory() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-invalid-endpoint", ) monkeypatch.setattr( "app.services.runtime_chat.request_openai_compatible_completion", lambda **_kwargs: (_ for _ in ()).throw(ValueError("invalid endpoint")), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) assert result.calls[0].outcome == "not_sent" assert result.calls[0].provider_status_code is None assert observer.completed[0].request_may_have_been_sent is False def test_runtime_chat_permit_observer_failure_blocks_provider_and_reconciles( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() observer = FailingPermitObserver() session_factory = build_session_factory() with session_factory() as db: service = RuntimeChatService(db, attempt_observer=observer) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-permit-failure", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: pytest.fail( "permit 状态未知时不应发送 provider 请求" ), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) trace = result.calls[0] assert result.text is None assert trace.status == "blocked" assert trace.outcome == "not_sent" assert trace.observer_status == "reconciliation_required" assert trace.requires_reconciliation is True assert [failure.phase for failure in trace.observer_failures] == ["permit"] assert len(observer.completed) == 1 assert observer.completed[0].request_may_have_been_sent is False def test_runtime_chat_trusted_context_without_observer_requires_reconciliation( monkeypatch, ) -> None: _clear_runtime_chat_cooldown() session_factory = build_session_factory() with session_factory() as db: service = RuntimeChatService(db) patch_single_slot( monkeypatch, service, provider="OpenAI Compatible", model="gpt-unobserved", ) monkeypatch.setattr( "app.services.runtime_chat._send_json_request", lambda *_args, **_kwargs: ( 200, { "id": "chatcmpl-unobserved", "choices": [{"message": {"content": "answer"}}], "usage": { "prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, }, }, ), ) result = service.complete_with_trace( [{"role": "user", "content": "hello"}], slot_priority=("main",), max_attempts=1, operation_context=build_operation_context(), ) assert result.text == "answer" assert result.calls[0].observer_status == "not_configured" assert result.calls[0].requires_reconciliation is True @pytest.mark.parametrize( "context_kwargs", [ {"tenant_id": ""}, {"operation_id": ""}, {"invocation_seq": 0}, {"invocation_seq": True}, {"invocation_seq": "1"}, {"attempt_scope": ""}, ], ) def test_runtime_chat_operation_context_rejects_untrusted_empty_identity( context_kwargs, ) -> None: values = { "tenant_id": "tenant-trusted", "operation_id": "operation-trusted", "invocation_seq": 1, "attempt_scope": "trusted-entry", } values.update(context_kwargs) with pytest.raises(ValueError): RuntimeChatOperationContext(**values) def test_runtime_chat_attempt_key_has_unambiguous_opaque_identity() -> None: common = { "run_id": "run", "invocation_seq": 1, "attempt_scope": "scope", } first = RuntimeChatOperationContext( tenant_id="tenant|operation", operation_id="id", **common, ).build_attempt_identity( slot="main", provider="OpenAI Compatible", model="gpt-test", attempt=1, ) second = RuntimeChatOperationContext( tenant_id="tenant", operation_id="operation|id", **common, ).build_attempt_identity( slot="main", provider="OpenAI Compatible", model="gpt-test", attempt=1, ) assert first.attempt_key != second.attempt_key assert first.attempt_key.startswith("runtime-chat-attempt:v1:") assert "tenant" not in first.attempt_key assert "operation" not in first.attempt_key