"""Golden-set routing test for the structured router. 20 labelled tasks → expected agent. Live tests require ANTHROPIC_API_KEY + COMPOSIO_API_KEY; without them the parametrized cases skip at runtime but the module still collects (so `pytest --collect-only` in CI works without secrets). Run with: pytest tests/test_routing_golden.py -v """ import os import pytest from dotenv import load_dotenv load_dotenv(os.path.join(os.path.dirname(__file__), "..", ".env")) _REQUIRED_KEYS = ("ANTHROPIC_API_KEY", "COMPOSIO_API_KEY") _missing = [k for k in _REQUIRED_KEYS if not os.getenv(k)] requires_keys = pytest.mark.skipif( bool(_missing), reason=f"Requires {', '.join(_REQUIRED_KEYS)}; missing: {', '.join(_missing)}", ) _RETRIEVAL_KEYS = ("ANTHROPIC_API_KEY", "COMPOSIO_API_KEY", "OPENAI_API_KEY") _missing_retrieval = [k for k in _RETRIEVAL_KEYS if not os.getenv(k)] requires_keys_with_retrieval = pytest.mark.skipif( bool(_missing_retrieval), reason=f"Requires {', '.join(_RETRIEVAL_KEYS)}; missing: {', '.join(_missing_retrieval)}", ) GOLDEN_SET: list[tuple[str, str]] = [ # implementer (3) ( "Write a Python function that validates email addresses using a regex", "implementer", ), ( "Add a new Lambda handler in handlers/notify.py that publishes an SNS message", "implementer", ), ( "Fix the bug in our auth middleware where expired tokens are accepted as valid", "implementer", ), # reviewer (3) ( "Review this pull request diff for correctness, security, and maintainability concerns", "reviewer", ), ( "Code review the attached commit and categorize each issue as BLOCK, FIX, or NIT", "reviewer", ), ( "Review the following code changes and tell me what should block merge", "reviewer", ), # researcher (3) ( "What is the latest stable version of the langgraph Python package?", "researcher", ), ( "Look up the AWS Lambda maximum concurrent execution limit in us-east-1", "researcher", ), ("Find documentation on how to configure DynamoDB TTL", "researcher"), # cross_reviewer (2) ( "Get a cross-family second opinion on this diff using GPT to catch what Claude might miss", "cross_reviewer", ), ( "Run an independent cross-model review on this Lambda handler change", "cross_reviewer", ), # scanner (3) ( "Scan the entire monorepo to find inconsistent error handling patterns across 200+ files", "scanner", ), ( "Analyze the whole codebase for unused imports and dead code using a large-context model", "scanner", ), ("Audit every Lambda handler in this repo for hardcoded secrets", "scanner"), # fast_coder (3) ( "Quick small task using DeepSeek: write a 5-line Python helper that converts kebab-case to snake_case", "fast_coder", ), ( "Fast bounded coding job: implement a one-function utility that pads strings to a fixed width", "fast_coder", ), ( "Use the cheap fast coder to write a short Python snippet that parses a CSV row into a dict", "fast_coder", ), # connector (3) ("Send a Slack message to the #ops channel announcing the deploy", "connector"), ("Create a Notion page under the Engineering space called 'Q3 plan'", "connector"), ("Find a file named contracts.pdf in my Google Drive", "connector"), ] def test_golden_set_size(): assert len(GOLDEN_SET) == 20 @requires_keys @pytest.mark.parametrize("task,expected", GOLDEN_SET) def test_router_picks_expected_agent(task: str, expected: str): # Lazy import — graph.py loads composio_tools at module import and needs # COMPOSIO_API_KEY. Keep the import inside the skipif-guarded test body. from graph import router_node out = router_node({"task": task, "messages": []}) assert out["route"] == expected, ( f"task={task!r} got={out['route']} expected={expected}" ) # --- Retrieval-augmented routing tests (Phase 5 gate) --- RETRIEVAL_CASES: list[tuple[str, str]] = [ ("Send a Slack message to the #ops channel about the deploy", "connector"), ( "Review this pull request diff for correctness and security", "reviewer", ), ( "Scan the entire repo for hardcoded AWS credentials across all files", "scanner", ), ] @requires_keys_with_retrieval @pytest.mark.parametrize("task,expected", RETRIEVAL_CASES) def test_router_stable_with_live_retrieval(task: str, expected: str): """Route stays correct when real retrieved memories are injected.""" from graph import router_node from retriever import retrieve retrieved = retrieve(task, k=3) out = router_node({"task": task, "messages": [], "retrieved": retrieved}) assert out["route"] == expected, ( f"task={task!r} got={out['route']} expected={expected} " f"retrieved={[m['name'] for m in retrieved]}" ) @requires_keys @pytest.mark.parametrize("task,expected", RETRIEVAL_CASES) def test_router_stable_with_offtopic_memory(task: str, expected: str): """Off-topic retrieved memories don't mislead the router.""" from graph import router_node fake_retrieved = [ { "name": "feedback_emulator_coordinates", "score": 0.1, "content": "ADB input uses landscape coords; getevent uses portrait.", }, { "name": "project_unrelated_game", "score": 0.05, "content": "Unity project for a 2D platformer. Uses C# and .NET 8.", }, ] out = router_node({"task": task, "messages": [], "retrieved": fake_retrieved}) assert out["route"] == expected, ( f"task={task!r} got={out['route']} expected={expected} " f"with off-topic memories injected" )