717 lines
24 KiB
Python
717 lines
24 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
from runtime_chat_testkit import (
|
||
|
|
DenyingAttemptObserver,
|
||
|
|
FailingCompletionObserver,
|
||
|
|
FailingPermitObserver,
|
||
|
|
RecordingAttemptObserver,
|
||
|
|
build_operation_context,
|
||
|
|
patch_single_slot,
|
||
|
|
)
|
||
|
|
from sqlalchemy import create_engine
|
||
|
|
from sqlalchemy.orm import Session, sessionmaker
|
||
|
|
from sqlalchemy.pool import StaticPool
|
||
|
|
|
||
|
|
from app.db.base import Base
|
||
|
|
from app.services import runtime_chat as runtime_chat_module
|
||
|
|
from app.services.model_connectivity import ConnectivityCheckError
|
||
|
|
from app.services.runtime_chat import (
|
||
|
|
RuntimeChatOperationContext,
|
||
|
|
RuntimeChatService,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def build_session_factory() -> sessionmaker[Session]:
|
||
|
|
engine = create_engine(
|
||
|
|
"sqlite+pysqlite:///:memory:",
|
||
|
|
connect_args={"check_same_thread": False},
|
||
|
|
poolclass=StaticPool,
|
||
|
|
)
|
||
|
|
Base.metadata.create_all(bind=engine)
|
||
|
|
return sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||
|
|
|
||
|
|
|
||
|
|
def _clear_runtime_chat_cooldown() -> None:
|
||
|
|
runtime_chat_module._slot_failure_until.clear()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
("provider", "configured_model", "response_payload", "expected_usage"),
|
||
|
|
[
|
||
|
|
(
|
||
|
|
"OpenAI Compatible",
|
||
|
|
"gpt-configured",
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-openai-001",
|
||
|
|
"model": "gpt-response",
|
||
|
|
"choices": [{"message": {"content": "openai answer"}}],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 17,
|
||
|
|
"completion_tokens": 5,
|
||
|
|
"total_tokens": 22,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt_tokens": 17,
|
||
|
|
"completion_tokens": 5,
|
||
|
|
"total_tokens": 22,
|
||
|
|
"prompt_eval_count": None,
|
||
|
|
"eval_count": None,
|
||
|
|
},
|
||
|
|
),
|
||
|
|
(
|
||
|
|
"Azure OpenAI",
|
||
|
|
"azure-deployment",
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-azure-001",
|
||
|
|
"model": "azure-response-model",
|
||
|
|
"choices": [{"message": {"content": "azure answer"}}],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 23,
|
||
|
|
"completion_tokens": 7,
|
||
|
|
"total_tokens": 30,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt_tokens": 23,
|
||
|
|
"completion_tokens": 7,
|
||
|
|
"total_tokens": 30,
|
||
|
|
"prompt_eval_count": None,
|
||
|
|
"eval_count": None,
|
||
|
|
},
|
||
|
|
),
|
||
|
|
(
|
||
|
|
"Ollama",
|
||
|
|
"llama-configured",
|
||
|
|
{
|
||
|
|
"model": "llama-response",
|
||
|
|
"message": {"content": "ollama answer"},
|
||
|
|
"prompt_eval_count": 31,
|
||
|
|
"eval_count": 11,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt_tokens": None,
|
||
|
|
"completion_tokens": None,
|
||
|
|
"total_tokens": None,
|
||
|
|
"prompt_eval_count": 31,
|
||
|
|
"eval_count": 11,
|
||
|
|
},
|
||
|
|
),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_runtime_chat_preserves_authoritative_provider_usage(
|
||
|
|
monkeypatch,
|
||
|
|
provider: str,
|
||
|
|
configured_model: str,
|
||
|
|
response_payload: dict[str, object],
|
||
|
|
expected_usage: dict[str, int | None],
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider=provider,
|
||
|
|
model=configured_model,
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (200, response_payload),
|
||
|
|
)
|
||
|
|
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.text is not None
|
||
|
|
assert len(result.calls) == 1
|
||
|
|
trace = result.calls[0]
|
||
|
|
assert trace.provider == provider
|
||
|
|
assert trace.model == configured_model
|
||
|
|
assert trace.attempt == 1
|
||
|
|
assert trace.response_id == response_payload.get("id")
|
||
|
|
assert trace.response_model == response_payload.get("model")
|
||
|
|
assert trace.outcome == "succeeded"
|
||
|
|
assert trace.started_at is not None
|
||
|
|
assert trace.completed_at is not None
|
||
|
|
assert trace.completed_at >= trace.started_at
|
||
|
|
assert trace.usage.availability == "available"
|
||
|
|
for field_name, expected_value in expected_usage.items():
|
||
|
|
assert getattr(trace.usage, field_name) == expected_value
|
||
|
|
assert trace.observer_status == "completed_notified"
|
||
|
|
assert len(observer.permits) == 1
|
||
|
|
assert len(observer.completed) == 1
|
||
|
|
assert observer.completed[0].identity.tenant_id == "tenant-runtime-chat"
|
||
|
|
assert observer.completed[0].identity.invocation_seq == 7
|
||
|
|
assert observer.completed[0].usage == trace.usage
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_keeps_missing_usage_unavailable_without_estimation(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-no-usage",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (
|
||
|
|
200,
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-no-usage",
|
||
|
|
"model": "gpt-no-usage",
|
||
|
|
"choices": [{"message": {"content": "answer"}}],
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_tokens=9876,
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
usage = result.calls[0].usage
|
||
|
|
assert usage.availability == "unavailable"
|
||
|
|
assert usage.source == "unavailable"
|
||
|
|
assert usage.prompt_tokens is None
|
||
|
|
assert usage.completion_tokens is None
|
||
|
|
assert usage.total_tokens is None
|
||
|
|
assert observer.completed[0].usage == usage
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_notifies_every_real_retry_attempt_and_marks_timeout_unknown(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
timeline: list[str] = []
|
||
|
|
observer = RecordingAttemptObserver(timeline)
|
||
|
|
request_count = 0
|
||
|
|
|
||
|
|
def fake_send_json_request(*_args, **_kwargs):
|
||
|
|
nonlocal request_count
|
||
|
|
request_count += 1
|
||
|
|
timeline.append(f"send:{request_count}")
|
||
|
|
if request_count == 1:
|
||
|
|
raise TimeoutError("provider response timed out")
|
||
|
|
return 200, {
|
||
|
|
"id": "chatcmpl-retry-002",
|
||
|
|
"model": "gpt-retry-response",
|
||
|
|
"choices": [
|
||
|
|
{
|
||
|
|
"message": {
|
||
|
|
"tool_calls": [
|
||
|
|
{
|
||
|
|
"id": "call-retry-002",
|
||
|
|
"function": {
|
||
|
|
"name": "submit_plan",
|
||
|
|
"arguments": "{}",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 41,
|
||
|
|
"completion_tokens": 3,
|
||
|
|
"total_tokens": 44,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-retry",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
fake_send_json_request,
|
||
|
|
)
|
||
|
|
monkeypatch.setattr("app.services.runtime_chat.sleep", lambda *_args: None)
|
||
|
|
|
||
|
|
result = service.complete_with_tool_call(
|
||
|
|
[{"role": "user", "content": "build plan"}],
|
||
|
|
tools=[{"type": "function", "function": {"name": "submit_plan"}}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=2,
|
||
|
|
use_failure_cooldown=False,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.tool_call is not None
|
||
|
|
assert [item.attempt for item in result.calls] == [1, 2]
|
||
|
|
assert [item.outcome for item in result.calls] == [
|
||
|
|
"outcome_unknown",
|
||
|
|
"succeeded",
|
||
|
|
]
|
||
|
|
assert result.calls[0].usage.availability == "unavailable"
|
||
|
|
assert result.calls[1].usage.total_tokens == 44
|
||
|
|
assert [item.identity.attempt for item in observer.permits] == [1, 2]
|
||
|
|
assert [item.identity.attempt for item in observer.completed] == [1, 2]
|
||
|
|
assert observer.completed[0].request_may_have_been_sent is True
|
||
|
|
assert observer.completed[0].requires_reconciliation is True
|
||
|
|
assert result.calls[0].requires_reconciliation is True
|
||
|
|
assert observer.permits[0].identity.attempt_key != observer.permits[1].identity.attempt_key
|
||
|
|
assert timeline == [
|
||
|
|
"permit:1",
|
||
|
|
"send:1",
|
||
|
|
"completed:1:outcome_unknown",
|
||
|
|
"permit:2",
|
||
|
|
"send:2",
|
||
|
|
"completed:2:succeeded",
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_preserves_usage_when_tool_response_postprocessing_fails(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-postprocess",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (
|
||
|
|
200,
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-postprocess",
|
||
|
|
"model": "gpt-postprocess-response",
|
||
|
|
"choices": [
|
||
|
|
{
|
||
|
|
"message": {
|
||
|
|
"tool_calls": [
|
||
|
|
{
|
||
|
|
"id": "call-invalid-json",
|
||
|
|
"function": {
|
||
|
|
"name": "submit_plan",
|
||
|
|
"arguments": "{invalid-json",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 13,
|
||
|
|
"completion_tokens": 2,
|
||
|
|
"total_tokens": 15,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
result = service.complete_with_tool_call(
|
||
|
|
[{"role": "user", "content": "build plan"}],
|
||
|
|
tools=[{"type": "function", "function": {"name": "submit_plan"}}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
use_failure_cooldown=False,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.tool_call is None
|
||
|
|
assert result.calls[0].outcome == "postprocess_failed"
|
||
|
|
assert result.calls[0].response_id == "chatcmpl-postprocess"
|
||
|
|
assert result.calls[0].usage.total_tokens == 15
|
||
|
|
assert observer.completed[0].outcome == "postprocess_failed"
|
||
|
|
assert observer.completed[0].usage.total_tokens == 15
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_observer_failure_keeps_success_and_reports_reconciliation(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
observer = FailingCompletionObserver()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-observer-failure",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (
|
||
|
|
200,
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-observer-failure",
|
||
|
|
"model": "gpt-observer-response",
|
||
|
|
"choices": [{"message": {"content": "valid answer"}}],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 8,
|
||
|
|
"completion_tokens": 2,
|
||
|
|
"total_tokens": 10,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.text == "valid answer"
|
||
|
|
trace = result.calls[0]
|
||
|
|
assert trace.outcome == "succeeded"
|
||
|
|
assert trace.usage.total_tokens == 10
|
||
|
|
assert trace.observer_status == "reconciliation_required"
|
||
|
|
assert [item.phase for item in trace.observer_failures] == ["completion"]
|
||
|
|
assert len(observer.failures) == 1
|
||
|
|
assert observer.failures[0].identity.operation_id == "operation-runtime-chat-001"
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_without_operation_context_does_not_invoke_observer(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-no-context",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (
|
||
|
|
200,
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-no-context",
|
||
|
|
"choices": [{"message": {"content": "legacy answer"}}],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 5,
|
||
|
|
"completion_tokens": 2,
|
||
|
|
"total_tokens": 7,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.text == "legacy answer"
|
||
|
|
assert result.calls[0].usage.total_tokens == 7
|
||
|
|
assert result.calls[0].observer_status == "not_applicable"
|
||
|
|
assert observer.permits == []
|
||
|
|
assert observer.completed == []
|
||
|
|
assert observer.failures == []
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_explicit_permit_denial_is_not_sent(monkeypatch) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
observer = DenyingAttemptObserver()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-denied",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: pytest.fail("permit 拒绝后不应发送请求"),
|
||
|
|
)
|
||
|
|
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.text is None
|
||
|
|
assert result.calls[0].status == "blocked"
|
||
|
|
assert result.calls[0].outcome == "not_sent"
|
||
|
|
assert observer.completed[0].request_may_have_been_sent is False
|
||
|
|
assert observer.completed[0].usage.availability == "unavailable"
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_provider_http_error_is_rejected_with_status(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-rate-limited",
|
||
|
|
)
|
||
|
|
|
||
|
|
def reject_request(*_args, **_kwargs):
|
||
|
|
raise ConnectivityCheckError("rate limited", status_code=429)
|
||
|
|
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
reject_request,
|
||
|
|
)
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.text is None
|
||
|
|
trace = result.calls[0]
|
||
|
|
assert trace.outcome == "provider_rejected"
|
||
|
|
assert trace.provider_status_code == 429
|
||
|
|
assert trace.requires_reconciliation is False
|
||
|
|
assert observer.completed[0].provider_status_code == 429
|
||
|
|
assert observer.completed[0].request_may_have_been_sent is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_rejected_response_preserves_authoritative_usage(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-rejected-with-usage",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (
|
||
|
|
429,
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-rejected",
|
||
|
|
"model": "gpt-rejected-response",
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 9,
|
||
|
|
"completion_tokens": 0,
|
||
|
|
"total_tokens": 9,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
trace = result.calls[0]
|
||
|
|
assert trace.outcome == "provider_rejected"
|
||
|
|
assert trace.provider_status_code == 429
|
||
|
|
assert trace.response_id == "chatcmpl-rejected"
|
||
|
|
assert trace.response_model == "gpt-rejected-response"
|
||
|
|
assert trace.usage.availability == "available"
|
||
|
|
assert trace.usage.total_tokens == 9
|
||
|
|
assert observer.completed[0].usage == trace.usage
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_pre_send_adapter_failure_is_not_sent(monkeypatch) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
observer = RecordingAttemptObserver()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-invalid-endpoint",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat.request_openai_compatible_completion",
|
||
|
|
lambda **_kwargs: (_ for _ in ()).throw(ValueError("invalid endpoint")),
|
||
|
|
)
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.calls[0].outcome == "not_sent"
|
||
|
|
assert result.calls[0].provider_status_code is None
|
||
|
|
assert observer.completed[0].request_may_have_been_sent is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_permit_observer_failure_blocks_provider_and_reconciles(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
observer = FailingPermitObserver()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db, attempt_observer=observer)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-permit-failure",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: pytest.fail(
|
||
|
|
"permit 状态未知时不应发送 provider 请求"
|
||
|
|
),
|
||
|
|
)
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
trace = result.calls[0]
|
||
|
|
assert result.text is None
|
||
|
|
assert trace.status == "blocked"
|
||
|
|
assert trace.outcome == "not_sent"
|
||
|
|
assert trace.observer_status == "reconciliation_required"
|
||
|
|
assert trace.requires_reconciliation is True
|
||
|
|
assert [failure.phase for failure in trace.observer_failures] == ["permit"]
|
||
|
|
assert len(observer.completed) == 1
|
||
|
|
assert observer.completed[0].request_may_have_been_sent is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_trusted_context_without_observer_requires_reconciliation(
|
||
|
|
monkeypatch,
|
||
|
|
) -> None:
|
||
|
|
_clear_runtime_chat_cooldown()
|
||
|
|
session_factory = build_session_factory()
|
||
|
|
with session_factory() as db:
|
||
|
|
service = RuntimeChatService(db)
|
||
|
|
patch_single_slot(
|
||
|
|
monkeypatch,
|
||
|
|
service,
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-unobserved",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(
|
||
|
|
"app.services.runtime_chat._send_json_request",
|
||
|
|
lambda *_args, **_kwargs: (
|
||
|
|
200,
|
||
|
|
{
|
||
|
|
"id": "chatcmpl-unobserved",
|
||
|
|
"choices": [{"message": {"content": "answer"}}],
|
||
|
|
"usage": {
|
||
|
|
"prompt_tokens": 3,
|
||
|
|
"completion_tokens": 1,
|
||
|
|
"total_tokens": 4,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
result = service.complete_with_trace(
|
||
|
|
[{"role": "user", "content": "hello"}],
|
||
|
|
slot_priority=("main",),
|
||
|
|
max_attempts=1,
|
||
|
|
operation_context=build_operation_context(),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.text == "answer"
|
||
|
|
assert result.calls[0].observer_status == "not_configured"
|
||
|
|
assert result.calls[0].requires_reconciliation is True
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"context_kwargs",
|
||
|
|
[
|
||
|
|
{"tenant_id": ""},
|
||
|
|
{"operation_id": ""},
|
||
|
|
{"invocation_seq": 0},
|
||
|
|
{"invocation_seq": True},
|
||
|
|
{"invocation_seq": "1"},
|
||
|
|
{"attempt_scope": ""},
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_runtime_chat_operation_context_rejects_untrusted_empty_identity(
|
||
|
|
context_kwargs,
|
||
|
|
) -> None:
|
||
|
|
values = {
|
||
|
|
"tenant_id": "tenant-trusted",
|
||
|
|
"operation_id": "operation-trusted",
|
||
|
|
"invocation_seq": 1,
|
||
|
|
"attempt_scope": "trusted-entry",
|
||
|
|
}
|
||
|
|
values.update(context_kwargs)
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
RuntimeChatOperationContext(**values)
|
||
|
|
|
||
|
|
|
||
|
|
def test_runtime_chat_attempt_key_has_unambiguous_opaque_identity() -> None:
|
||
|
|
common = {
|
||
|
|
"run_id": "run",
|
||
|
|
"invocation_seq": 1,
|
||
|
|
"attempt_scope": "scope",
|
||
|
|
}
|
||
|
|
first = RuntimeChatOperationContext(
|
||
|
|
tenant_id="tenant|operation",
|
||
|
|
operation_id="id",
|
||
|
|
**common,
|
||
|
|
).build_attempt_identity(
|
||
|
|
slot="main",
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-test",
|
||
|
|
attempt=1,
|
||
|
|
)
|
||
|
|
second = RuntimeChatOperationContext(
|
||
|
|
tenant_id="tenant",
|
||
|
|
operation_id="operation|id",
|
||
|
|
**common,
|
||
|
|
).build_attempt_identity(
|
||
|
|
slot="main",
|
||
|
|
provider="OpenAI Compatible",
|
||
|
|
model="gpt-test",
|
||
|
|
attempt=1,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert first.attempt_key != second.attempt_key
|
||
|
|
assert first.attempt_key.startswith("runtime-chat-attempt:v1:")
|
||
|
|
assert "tenant" not in first.attempt_key
|
||
|
|
assert "operation" not in first.attempt_key
|