From 52fb619084fbf5a8a02bfa0d0946725c07588946 Mon Sep 17 00:00:00 2001 From: "WIN-JHFT4D3SIVT\\caoxiaozhu" Date: Wed, 8 Apr 2026 00:12:50 +0800 Subject: [PATCH] test(backend): add tests for orchestration and learning runtimes Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../tests/backend/app/agents/test_graph.py | 85 +++++++++ .../app/agents/test_result_merge_runtime.py | 138 ++++++++++++++ .../app/agents/test_runtime_context.py | 170 ++++++++++++++++++ .../app/agents/test_scheduler_runtime.py | 66 +++++++ .../app/agents/test_task_graph_runtime.py | 59 ++++++ .../backend/app/agents/test_visibility_api.py | 24 ++- .../backend/app/test_conversation_router.py | 22 +++ .../app/test_database_schema_bootstrap.py | 115 ++++++++++++ .../tests/backend/app/test_skill_router.py | 14 ++ .../tests/backend/app/test_system_router.py | 130 ++++++++++++++ 10 files changed, 822 insertions(+), 1 deletion(-) create mode 100644 backend/tests/backend/app/agents/test_result_merge_runtime.py create mode 100644 backend/tests/backend/app/agents/test_runtime_context.py create mode 100644 backend/tests/backend/app/agents/test_scheduler_runtime.py create mode 100644 backend/tests/backend/app/agents/test_task_graph_runtime.py create mode 100644 backend/tests/backend/app/test_database_schema_bootstrap.py create mode 100644 backend/tests/backend/app/test_system_router.py diff --git a/backend/tests/backend/app/agents/test_graph.py b/backend/tests/backend/app/agents/test_graph.py index 1d955c0..11b0485 100644 --- a/backend/tests/backend/app/agents/test_graph.py +++ b/backend/tests/backend/app/agents/test_graph.py @@ -314,6 +314,22 @@ class FailIfCalledLLM: raise AssertionError('LLM should not be called for simple greetings') +class InternalMarkupRecoveryLLM: + def __init__(self, responses: list[str]): + self.responses = responses + self.calls = 0 + self._jarvis_provider_capabilities = SimpleNamespace( + provider='minimax', + supports_native_tools=False, + preferred_tool_strategy='json_fallback', + ) + + async def ainvoke(self, messages): + self.calls += 1 + index = min(self.calls - 1, len(self.responses) - 1) + return AIMessage(content=self.responses[index]) + + def test_initial_state_sets_structured_continuity_defaults(): state = initial_state('u1', 'c1') @@ -2047,6 +2063,75 @@ async def test_run_sub_commander_uses_web_search_in_json_fallback(monkeypatch): assert result['final_response'] == '我查了外部网页，下面是最新结果摘要。' +async def test_run_sub_commander_recovers_from_internal_tool_markup_after_tool_round(monkeypatch): + fake_llm = InternalMarkupRecoveryLLM([ + '{"mode":"tool_call","tool_calls":[{"name":"web_search","arguments":{"query":"武汉 介绍","top_k":2}}]}', + '我来让知识管理员为你整理武汉的详细介绍。\n\n分发说明：这个问题需要调用知识库信息，由 librarian（知识管理员）处理最合适。\n\n\ncity_introduction\n{"city":"武汉","word_count":2000,"language":"zh-CN"}\n\n', + '武汉是湖北省省会，位于长江与汉江交汇处，是中部重要的交通、科教和工业中心。', + ]) + fake_tool = FakeTool('web_search', 'found 2 web results') + + monkeypatch.setattr('app.agents.graph._get_llm_for_state', lambda state: fake_llm) + monkeypatch.setitem( + __import__('app.agents.graph', fromlist=['SUB_COMMANDER_TOOLSETS']).SUB_COMMANDER_TOOLSETS, + 'librarian_retrieval', + [fake_tool], + ) + + state = _base_state('请介绍一下武汉', {'provider': 'openai', 'model': 'MiniMax-M2.7-highspeed', 'base_url': 'https://api.minimaxi.com/v1'}) + state['current_agent'] = AgentRole.LIBRARIAN + state['max_retries'] = 1 + + result = await _run_sub_commander( + state, + AgentRole.LIBRARIAN, + 'manager prompt', + '请介绍一下武汉', + use_tools=True, + summary_target='knowledge_context', + ) + + assert fake_llm.calls == 3 + assert fake_tool.invocations == [{'query': '武汉 介绍', 'top_k': 2}] + assert result['fallback_parse_error'] is None + assert '', + ]) + fake_tool = FakeTool('web_search', 'found 2 web results') + + monkeypatch.setattr('app.agents.graph._get_llm_for_state', lambda state: fake_llm) + monkeypatch.setitem( + __import__('app.agents.graph', fromlist=['SUB_COMMANDER_TOOLSETS']).SUB_COMMANDER_TOOLSETS, + 'librarian_retrieval', + [fake_tool], + ) + + state = _base_state('请介绍一下武汉', {'provider': 'openai', 'model': 'MiniMax-M2.7-highspeed', 'base_url': 'https://api.minimaxi.com/v1'}) + state['current_agent'] = AgentRole.LIBRARIAN + state['max_retries'] = 0 + + result = await _run_sub_commander( + state, + AgentRole.LIBRARIAN, + 'manager prompt', + '请介绍一下武汉', + use_tools=True, + summary_target='knowledge_context', + ) + + assert fake_llm.calls == 2 + assert result['fallback_parse_error'] == 'internal_tool_markup' + assert result['final_response'] == '我已经完成检索，直接给您可用信息：\n\nfound 2 web results' + assert '= 2 + assert graph.entry_node_ids + assert any(node.execution_mode == "parallel" for node in graph.nodes[:-1]) + assert graph.nodes[-1].role == "master" + + +def test_runtime_request_context_summary_renders_task_graph(): + worthiness = assess_parallel_worthiness( + "先查资料、再分析风险、再安排下周计划", + retrospective_count=1, + skill_count=1, + ) + task_graph = build_bounded_task_graph( + query_text="先查资料、再分析风险、再安排下周计划", + parallel_worthiness=worthiness, + ) + context = RuntimeRequestContext( + user_id="u1", + session_id="c1", + query_text="先查资料、再分析风险、再安排下周计划", + parallel_worthiness=worthiness, + task_graph=task_graph, + recommended_runtime_mode="collaboration", + ) + + summary = render_runtime_request_context_summary(context) + + assert "任务图" in summary + assert "max_parallelism" in summary + + +def test_runtime_request_context_summary_renders_assembly_metrics(): + context = RuntimeRequestContext( + user_id="u1", + session_id="c1", + query_text="帮我分析一下资料", + assembly_metrics={"total_ms": 12.3}, + ) + + summary = render_runtime_request_context_summary(context) + + assert "上下文装配耗时" in summary diff --git a/backend/tests/backend/app/agents/test_visibility_api.py b/backend/tests/backend/app/agents/test_visibility_api.py index 6775d43..6ea94a6 100644 --- a/backend/tests/backend/app/agents/test_visibility_api.py +++ b/backend/tests/backend/app/agents/test_visibility_api.py @@ -503,7 +503,9 @@ async def test_visibility_tools_returns_governance_metadata_and_usage_counts(vis payload = response.json() assert payload['total_tools'] >= 1 assert payload['used_tools'] >= 1 - search_tool = next(item for item in payload['items'] if item['tool_name'] == 'search_web') + search_tool = next( + item for item in payload['items'] if item['tool_name'] in {'search_web', 'web_search'} + ) assert search_tool['permission_class'] == 'external' assert search_tool['side_effect_scope'] == 'network' assert search_tool['usage_count'] == 1 @@ -516,6 +518,26 @@ async def test_visibility_tools_returns_governance_metadata_and_usage_counts(vis ] +@pytest.mark.asyncio +async def test_visibility_debug_returns_observability_and_learning_views(visibility_env): + app, ids = visibility_env + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url='http://testserver') as client: + response = await client.get( + '/api/agents/visibility/debug', + params={'conversation_id': ids['conversation_id']}, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload['conversation_id'] == ids['conversation_id'] + assert 'observability' in payload + assert 'skill_shortlist' in payload + assert 'retrospective_shortlist' in payload + assert 'recent_retrospectives' in payload + assert 'recent_learning_artifacts' in payload + + @pytest.mark.asyncio async def test_visibility_events_reject_invalid_datetime(visibility_env): app, ids = visibility_env diff --git a/backend/tests/backend/app/test_conversation_router.py b/backend/tests/backend/app/test_conversation_router.py index f157beb..49afa16 100644 --- a/backend/tests/backend/app/test_conversation_router.py +++ b/backend/tests/backend/app/test_conversation_router.py @@ -73,3 +73,25 @@ async def test_list_conversations_succeeds_when_agent_state_column_was_missing(c assert len(payload) == 1 assert payload[0]['title'] == 'Existing conversation' assert payload[0]['message_count'] == 3 + + +@pytest.mark.asyncio +async def test_chat_stream_emits_error_event_when_agent_service_fails_before_stream_starts( + conversation_env, + monkeypatch, +): + async def fail_chat(*args, **kwargs): + raise RuntimeError('stream boot failed') + + monkeypatch.setattr('app.routers.conversation.AgentService.chat', fail_chat) + + transport = ASGITransport(app=conversation_env) + async with AsyncClient(transport=transport, base_url='http://testserver') as client: + response = await client.post( + '/api/conversations/chat/stream', + json={'message': 'hello'}, + ) + + assert response.status_code == 200 + assert 'event: error' in response.text + assert 'stream boot failed' in response.text diff --git a/backend/tests/backend/app/test_database_schema_bootstrap.py b/backend/tests/backend/app/test_database_schema_bootstrap.py new file mode 100644 index 0000000..a8ff6d7 --- /dev/null +++ b/backend/tests/backend/app/test_database_schema_bootstrap.py @@ -0,0 +1,115 @@ +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from app.database import ensure_learning_artifact_tables, ensure_memory_columns, ensure_skill_columns + + +@pytest.mark.anyio +async def test_ensure_memory_columns_adds_importance_tracking_fields_for_existing_user_memories_table(tmp_path): + db_path = tmp_path / 'test_user_memories.db' + engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}", future=True) + + async with engine.begin() as conn: + await conn.execute(text( + ''' + CREATE TABLE user_memories ( + id VARCHAR(36) PRIMARY KEY, + user_id VARCHAR(36) NOT NULL, + memory_type VARCHAR(50) NOT NULL, + content TEXT NOT NULL, + importance INTEGER, + is_recalled BOOLEAN, + recall_count INTEGER, + source_conversation_id VARCHAR(36), + extracted_at DATETIME, + last_recalled_at DATETIME, + created_at DATETIME, + updated_at DATETIME + ) + ''' + )) + result = await conn.execute(text("PRAGMA table_info(user_memories)")) + columns_before = {row[1] for row in result.fetchall()} + assert 'frequency_count' not in columns_before + assert 'importance_score' not in columns_before + assert 'decay_score' not in columns_before + + await ensure_memory_columns(conn) + + result = await conn.execute(text("PRAGMA table_info(user_memories)")) + columns_after = {row[1] for row in result.fetchall()} + assert 'frequency_count' in columns_after + assert 'emotion_tags' in columns_after + assert 'importance_score' in columns_after + assert 'importance_level' in columns_after + assert 'associated_topics' in columns_after + assert 'decay_score' in columns_after + assert 'is_archived' in columns_after + assert 'last_accessed_at' in columns_after + assert 'archive_at' in columns_after + + await engine.dispose() + + +@pytest.mark.anyio +async def test_ensure_skill_columns_adds_lifecycle_fields_for_existing_skills_table(tmp_path): + db_path = tmp_path / 'test_skills.db' + engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}", future=True) + + async with engine.begin() as conn: + await conn.execute(text( + ''' + CREATE TABLE skills ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(100) NOT NULL, + description TEXT, + instructions TEXT NOT NULL, + agent_type VARCHAR(50) NOT NULL, + visibility VARCHAR(20), + is_active BOOLEAN, + owner_id VARCHAR(36), + created_at DATETIME, + updated_at DATETIME + ) + ''' + )) + result = await conn.execute(text("PRAGMA table_info(skills)")) + columns_before = {row[1] for row in result.fetchall()} + assert 'status' not in columns_before + assert 'effectiveness' not in columns_before + + await ensure_skill_columns(conn) + + result = await conn.execute(text("PRAGMA table_info(skills)")) + columns_after = {row[1] for row in result.fetchall()} + assert 'status' in columns_after + assert 'scope' in columns_after + assert 'effectiveness' in columns_after + assert 'review_after' in columns_after + assert 'activation_count' in columns_after + assert 'last_activated_at' in columns_after + + await engine.dispose() + + +@pytest.mark.anyio +async def test_ensure_learning_artifact_tables_creates_table_and_indexes(tmp_path): + db_path = tmp_path / 'test_learning_artifacts.db' + engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}", future=True) + + async with engine.begin() as conn: + await ensure_learning_artifact_tables(conn) + result = await conn.execute(text("PRAGMA table_info(learning_artifacts)")) + columns = {row[1] for row in result.fetchall()} + assert 'artifact_type' in columns + assert 'artifact_key' in columns + assert 'summary_text' in columns + assert 'payload' in columns + + indexes = await conn.execute(text("PRAGMA index_list(learning_artifacts)")) + index_names = {row[1] for row in indexes.fetchall()} + assert 'ix_learning_artifacts_user_id' in index_names + assert 'ix_learning_artifacts_artifact_type' in index_names + + await engine.dispose() diff --git a/backend/tests/backend/app/test_skill_router.py b/backend/tests/backend/app/test_skill_router.py index 617ecc5..c621e3d 100644 --- a/backend/tests/backend/app/test_skill_router.py +++ b/backend/tests/backend/app/test_skill_router.py @@ -54,6 +54,9 @@ async def skill_env(tmp_path, monkeypatch): required_context=[], visibility='private', is_active=True, + status='active', + scope=['schedule_planner'], + effectiveness=0.88, owner_id=user.id, ), Skill( @@ -65,6 +68,9 @@ async def skill_env(tmp_path, monkeypatch): required_context=[], visibility='private', is_active=True, + status='shadow', + scope=['executor'], + effectiveness=0.41, owner_id=user.id, ), Skill( @@ -76,6 +82,8 @@ async def skill_env(tmp_path, monkeypatch): required_context=[], visibility='private', is_active=True, + status='active', + scope=['schedule_planner'], owner_id=other_user.id, ), ]) @@ -188,3 +196,9 @@ async def test_list_skills_without_agent_type_returns_current_user_skills(skill_ assert all(isinstance(item['updated_at'], str) for item in payload) assert all('is_builtin' in item for item in payload) assert all(item['is_builtin'] is False for item in payload) + assert all('status' in item for item in payload) + assert all('scope' in item for item in payload) + assert any(item['status'] == 'shadow' for item in payload) + executor = next(item for item in payload if item['name'] == 'Executor skill') + assert executor['scope'] == ['executor'] + assert executor['effectiveness'] == 0.41 diff --git a/backend/tests/backend/app/test_system_router.py b/backend/tests/backend/app/test_system_router.py new file mode 100644 index 0000000..7679866 --- /dev/null +++ b/backend/tests/backend/app/test_system_router.py @@ -0,0 +1,130 @@ +import httpx +import pytest +from httpx import ASGITransport, AsyncClient + +from app.main import app + + +@pytest.mark.asyncio +async def test_system_config_returns_location_and_weather(monkeypatch): + async def fake_get_config(self): + return { + 'location': 'wuhan', + 'weather_code': 3, + 'weather_summary': 'Overcast 22°C', + } + + monkeypatch.setattr('app.routers.system.SystemService.get_config', fake_get_config) + transport = ASGITransport(app=app) + + async with AsyncClient(transport=transport, base_url='http://testserver') as client: + response = await client.get('/api/system/config') + + assert response.status_code == 200 + assert response.json() == { + 'location': 'wuhan', + 'weather_code': 3, + 'weather_summary': 'Overcast 22°C', + } + + +@pytest.mark.asyncio +async def test_system_config_gracefully_returns_unavailable_weather(monkeypatch): + async def fake_get_config(self): + return { + 'location': 'wuhan', + 'weather_code': None, + 'weather_summary': 'Weather unavailable', + } + + monkeypatch.setattr('app.routers.system.SystemService.get_config', fake_get_config) + transport = ASGITransport(app=app) + + async with AsyncClient(transport=transport, base_url='http://testserver') as client: + response = await client.get('/api/system/config') + + assert response.status_code == 200 + assert response.json() == { + 'location': 'wuhan', + 'weather_code': None, + 'weather_summary': 'Weather unavailable', + } + + +class FakeWeatherResponse: + def __init__(self, payload: dict, status_code: int = 200): + self._payload = payload + self.status_code = status_code + + def raise_for_status(self): + if self.status_code >= 400: + raise httpx.HTTPStatusError( + 'request failed', + request=httpx.Request('GET', 'https://wttr.in/wuhan?format=j1'), + response=httpx.Response(self.status_code, request=httpx.Request('GET', 'https://wttr.in/wuhan?format=j1')), + ) + + def json(self): + return self._payload + + +class FakeAsyncClient: + def __init__(self, *, response=None, error=None, **kwargs): + self._response = response + self._error = error + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def get(self, url, *, params=None): + if self._error is not None: + raise self._error + return self._response + + +@pytest.mark.asyncio +async def test_system_service_get_config_fetches_weather(monkeypatch): + monkeypatch.setattr( + 'app.services.system_service.httpx.AsyncClient', + lambda **kwargs: FakeAsyncClient( + response=FakeWeatherResponse({'current_condition': [{'weatherCode': '61', 'temp_C': '18'}]}), + **kwargs, + ), + ) + + from app.services.system_service import SystemService + + service = SystemService() + monkeypatch.setattr(service._settings, 'LOCATION', 'wuhan') + + payload = await service.get_config() + + assert payload == { + 'location': 'wuhan', + 'weather_code': 61, + 'weather_summary': 'Rain 18°C', + } + + +@pytest.mark.asyncio +async def test_system_service_get_config_handles_weather_failure(monkeypatch): + monkeypatch.setattr( + 'app.services.system_service.httpx.AsyncClient', + lambda **kwargs: FakeAsyncClient(error=httpx.TimeoutException('timed out'), **kwargs), + ) + + from app.services.system_service import SystemService + + service = SystemService() + monkeypatch.setattr(service._settings, 'LOCATION', 'wuhan') + + payload = await service.get_config() + + assert payload == { + 'location': 'wuhan', + 'weather_code': None, + 'weather_summary': 'Weather unavailable', + }