diff --git a/agent/dashboard/agent_overrides.py b/agent/dashboard/agent_overrides.py index 6d572a5e..e672ff2e 100644 --- a/agent/dashboard/agent_overrides.py +++ b/agent/dashboard/agent_overrides.py @@ -9,7 +9,7 @@ import httpx from langgraph_sdk import get_client from ..utils.github_user_email_map import GITHUB_USER_EMAIL_MAP -from .options import SUPPORTED_MODEL_IDS, model_supports_effort +from .options import SUPPORTED_MODEL_IDS, model_supports_effort, provider_fallback_pair from .profiles import PROFILES_NAMESPACE logger = logging.getLogger(__name__) @@ -97,6 +97,14 @@ def _normalize_profile_model_pair( and model_supports_effort(model_id, effort) ): return model_id, effort + # A stored selection whose exact id dropped out of the supported set (e.g. an + # Opus minor-version bump) stays on its provider rather than being discarded + # and silently deferring to the team default. An absent/unknown-provider + # selection still returns (None, None) so the team default applies. + if isinstance(model_id, str): + provider_pair = provider_fallback_pair(model_id, effort) + if provider_pair is not None: + return provider_pair return None, None diff --git a/agent/dashboard/options.py b/agent/dashboard/options.py index 6a31e6a4..19181d06 100644 --- a/agent/dashboard/options.py +++ b/agent/dashboard/options.py @@ -14,8 +14,8 @@ class ModelOption(TypedDict): SUPPORTED_MODELS: list[ModelOption] = [ { - "id": "anthropic:claude-opus-4-7", - "label": "Opus 4.7", + "id": "anthropic:claude-opus-4-8", + "label": "Opus 4.8", "efforts": ["low", "medium", "high", "xhigh", "max"], "default_effort": "high", }, @@ -40,6 +40,32 @@ def model_supports_effort(model_id: str, effort: str) -> bool: return False +def _provider_of(model_id: str) -> str | None: + provider, _, rest = model_id.partition(":") + return provider if rest else None + + +def provider_fallback_pair(model_id: object, effort: object = None) -> tuple[str, str] | None: + """Newest supported ``(model_id, effort)`` for the same provider as ``model_id``. + + Keeps a stored selection on its original provider when its exact id has + dropped out of the supported set (e.g. an Opus minor-version bump), instead + of falling through to the cross-provider global default. Preserves ``effort`` + when the fallback model supports it, otherwise uses that model's default + effort. Returns ``None`` when no supported model shares the provider. + """ + if not isinstance(model_id, str): + return None + provider = _provider_of(model_id) + if provider is None: + return None + for m in SUPPORTED_MODELS: + if _provider_of(m["id"]) == provider: + new_effort = effort if (isinstance(effort, str) and effort in m["efforts"]) else None + return m["id"], new_effort or m["default_effort"] + return None + + def default_model_pair() -> tuple[str, str]: """Hardcoded fallback (model_id, reasoning_effort) used when no team default is set.""" if DEFAULT_MODEL_ID in SUPPORTED_MODEL_IDS and model_supports_effort( diff --git a/agent/dashboard/team_settings.py b/agent/dashboard/team_settings.py index 313ce64e..d6d05a8d 100644 --- a/agent/dashboard/team_settings.py +++ b/agent/dashboard/team_settings.py @@ -14,7 +14,12 @@ from typing import Any, Literal from langgraph_sdk import get_client from pydantic import BaseModel, model_validator -from .options import SUPPORTED_MODEL_IDS, default_model_pair, model_supports_effort +from .options import ( + SUPPORTED_MODEL_IDS, + default_model_pair, + model_supports_effort, + provider_fallback_pair, +) logger = logging.getLogger(__name__) @@ -145,10 +150,11 @@ async def get_team_default_model( ) -> tuple[str, str]: """Return the team-wide default ``(model_id, reasoning_effort)`` for ``role``. - Always returns a valid pair: the admin-configured pair if set, otherwise the - hardcoded fallback from :func:`agent.dashboard.options.default_model_pair`. - Invalid stored pairs (unsupported model or mismatched effort) fall back to - the hardcoded default rather than propagating bad data. + Always returns a valid pair, resolved in order: the admin-configured pair if + still supported; otherwise the newest supported model for the same provider + (so a stale Anthropic/OpenAI selection stays on its provider rather than + jumping cross-provider); otherwise the hardcoded global default from + :func:`agent.dashboard.options.default_model_pair`. """ settings = await get_team_settings() if role == "agent": @@ -157,14 +163,7 @@ async def get_team_default_model( else: model = settings.get("default_reviewer_model") effort = settings.get("default_reviewer_reasoning_effort") - if ( - isinstance(model, str) - and model in SUPPORTED_MODEL_IDS - and isinstance(effort, str) - and model_supports_effort(model, effort) - ): - return model, effort - return default_model_pair() + return _resolve_default_pair(model, effort) async def get_team_default_subagent_model( @@ -178,11 +177,19 @@ async def get_team_default_subagent_model( else: model = settings.get("default_reviewer_subagent_model") effort = settings.get("default_reviewer_subagent_reasoning_effort") + return _resolve_default_pair(model, effort) + + +def _resolve_default_pair(model: object, effort: object) -> tuple[str, str]: + """Supported pair if valid, else same-provider fallback, else global default.""" if ( isinstance(model, str) - and model in SUPPORTED_MODEL_IDS and isinstance(effort, str) + and model in SUPPORTED_MODEL_IDS and model_supports_effort(model, effort) ): return model, effort + provider_pair = provider_fallback_pair(model, effort) + if provider_pair is not None: + return provider_pair return default_model_pair() diff --git a/evals/reviewer/config.toml b/evals/reviewer/config.toml index e647f03e..ea920de2 100644 --- a/evals/reviewer/config.toml +++ b/evals/reviewer/config.toml @@ -5,7 +5,7 @@ max_concurrency = 5 # Leave blank to use LANGGRAPH_URL or local dev. langgraph_url = "" assistant_id = "reviewer" -# models: openai:gpt-5.5, anthropic:claude-opus-4-7, google_genai:gemini-3.5-flash +# models: openai:gpt-5.5, anthropic:claude-opus-4-8, google_genai:gemini-3.5-flash model_id = "google_genai:gemini-3.5-flash" reasoning_effort = "medium" diff --git a/tests/test_agent_subagent_models.py b/tests/test_agent_subagent_models.py index f53033f8..8af29244 100644 --- a/tests/test_agent_subagent_models.py +++ b/tests/test_agent_subagent_models.py @@ -61,7 +61,7 @@ async def test_agent_uses_profile_subagent_model_override() -> None: "agent.server.load_profile", new_callable=AsyncMock, return_value={ - "default_model": "anthropic:claude-opus-4-7", + "default_model": "anthropic:claude-opus-4-8", "reasoning_effort": "high", "default_subagent_model": "openai:gpt-5.5", "subagent_reasoning_effort": "xhigh", @@ -81,7 +81,7 @@ async def test_agent_uses_profile_subagent_model_override() -> None: assert subagents[0]["model"] is subagent_model main_call = make_model.call_args_list[0] - assert main_call.args == ("anthropic:claude-opus-4-7",) + assert main_call.args == ("anthropic:claude-opus-4-8",) assert main_call.kwargs["thinking"] == {"type": "adaptive"} assert main_call.kwargs["effort"] == "high" @@ -139,7 +139,7 @@ async def test_agent_subagent_inherits_profile_model_override_without_explicit_p "agent.server.load_profile", new_callable=AsyncMock, return_value={ - "default_model": "anthropic:claude-opus-4-7", + "default_model": "anthropic:claude-opus-4-8", "reasoning_effort": "high", }, ), @@ -153,7 +153,7 @@ async def test_agent_subagent_inherits_profile_model_override_without_explicit_p subagents = captured["subagents"] assert isinstance(subagents, list) assert subagents[0]["model"] is subagent_model - assert make_model.call_args_list[0].args == ("anthropic:claude-opus-4-7",) - assert make_model.call_args_list[1].args == ("anthropic:claude-opus-4-7",) + assert make_model.call_args_list[0].args == ("anthropic:claude-opus-4-8",) + assert make_model.call_args_list[1].args == ("anthropic:claude-opus-4-8",) assert make_model.call_args_list[1].kwargs["thinking"] == {"type": "adaptive"} assert make_model.call_args_list[1].kwargs["effort"] == "high" diff --git a/tests/test_model_fallback_resolution.py b/tests/test_model_fallback_resolution.py new file mode 100644 index 00000000..516d7bc3 --- /dev/null +++ b/tests/test_model_fallback_resolution.py @@ -0,0 +1,81 @@ +from unittest.mock import AsyncMock, patch + +import pytest + +from agent.dashboard.agent_overrides import normalize_profile_overrides +from agent.dashboard.options import ( + DEFAULT_MODEL_ID, + default_model_pair, + provider_fallback_pair, +) +from agent.dashboard.team_settings import get_team_default_model + +STALE_ANTHROPIC = "anthropic:claude-opus-4-7" +SUPPORTED_ANTHROPIC = "anthropic:claude-opus-4-8" + + +def test_provider_fallback_preserves_provider_and_effort() -> None: + assert provider_fallback_pair(STALE_ANTHROPIC, "xhigh") == (SUPPORTED_ANTHROPIC, "xhigh") + + +def test_provider_fallback_uses_default_effort_when_unsupported() -> None: + assert provider_fallback_pair(STALE_ANTHROPIC, "bogus") == (SUPPORTED_ANTHROPIC, "high") + assert provider_fallback_pair(STALE_ANTHROPIC, None) == (SUPPORTED_ANTHROPIC, "high") + + +def test_provider_fallback_resolves_openai_within_provider() -> None: + model, effort = provider_fallback_pair("openai:gpt-5-legacy", "low") + assert model.startswith("openai:") + assert effort == "low" + + +@pytest.mark.parametrize("model_id", ["unknown:model", "no-colon", "", None, 123]) +def test_provider_fallback_returns_none_without_provider_match(model_id: object) -> None: + assert provider_fallback_pair(model_id, "high") is None + + +@pytest.mark.asyncio +async def test_team_default_stale_anthropic_stays_on_provider() -> None: + settings = { + "default_agent_model": STALE_ANTHROPIC, + "default_agent_reasoning_effort": "xhigh", + } + with patch( + "agent.dashboard.team_settings.get_team_settings", + new_callable=AsyncMock, + return_value=settings, + ): + assert await get_team_default_model("agent") == (SUPPORTED_ANTHROPIC, "xhigh") + + +@pytest.mark.asyncio +async def test_team_default_unknown_provider_falls_back_to_global() -> None: + settings = { + "default_reviewer_model": "mystery:model", + "default_reviewer_reasoning_effort": "high", + } + with patch( + "agent.dashboard.team_settings.get_team_settings", + new_callable=AsyncMock, + return_value=settings, + ): + assert await get_team_default_model("reviewer") == default_model_pair() + + +def test_profile_stale_anthropic_upgrades_to_supported() -> None: + profile = {"default_model": STALE_ANTHROPIC, "reasoning_effort": "high"} + assert normalize_profile_overrides(profile) == (SUPPORTED_ANTHROPIC, "high") + + +def test_profile_without_model_defers_to_team_default() -> None: + assert normalize_profile_overrides({"reasoning_effort": "high"}) == (None, None) + + +def test_profile_unknown_provider_defers_to_team_default() -> None: + profile = {"default_model": "mystery:model", "reasoning_effort": "high"} + assert normalize_profile_overrides(profile) == (None, None) + + +def test_global_default_is_supported() -> None: + model, _ = default_model_pair() + assert model == DEFAULT_MODEL_ID diff --git a/tests/test_reviewer.py b/tests/test_reviewer.py index ada38abe..82508b71 100644 --- a/tests/test_reviewer.py +++ b/tests/test_reviewer.py @@ -113,7 +113,7 @@ async def test_reviewer_applies_eval_model_and_effort_overrides() -> None: "pr_url": "https://github.com/acme/repo/pull/1", "base_sha": "base", "head_sha": "head", - "reviewer_model_id": "anthropic:claude-opus-4-7", + "reviewer_model_id": "anthropic:claude-opus-4-8", "reviewer_reasoning_effort": "high", "reviewer_subagent_model_id": "openai:gpt-5.5", "reviewer_subagent_reasoning_effort": "low", @@ -144,7 +144,7 @@ async def test_reviewer_applies_eval_model_and_effort_overrides() -> None: await reviewer.get_reviewer_agent(config) main_model_call = make_model.call_args_list[0] - assert main_model_call.args == ("anthropic:claude-opus-4-7",) + assert main_model_call.args == ("anthropic:claude-opus-4-8",) assert main_model_call.kwargs["thinking"] == {"type": "adaptive"} assert main_model_call.kwargs["effort"] == "high" subagent_model_call = make_model.call_args_list[1] @@ -163,7 +163,7 @@ async def test_reviewer_subagent_inherits_eval_model_without_explicit_override() "pr_url": "https://github.com/acme/repo/pull/1", "base_sha": "base", "head_sha": "head", - "reviewer_model_id": "anthropic:claude-opus-4-7", + "reviewer_model_id": "anthropic:claude-opus-4-8", "reviewer_reasoning_effort": "high", }, "metadata": {}, @@ -192,11 +192,11 @@ async def test_reviewer_subagent_inherits_eval_model_without_explicit_override() await reviewer.get_reviewer_agent(config) main_model_call = make_model.call_args_list[0] - assert main_model_call.args == ("anthropic:claude-opus-4-7",) + assert main_model_call.args == ("anthropic:claude-opus-4-8",) assert main_model_call.kwargs["thinking"] == {"type": "adaptive"} assert main_model_call.kwargs["effort"] == "high" subagent_model_call = make_model.call_args_list[1] - assert subagent_model_call.args == ("anthropic:claude-opus-4-7",) + assert subagent_model_call.args == ("anthropic:claude-opus-4-8",) assert subagent_model_call.kwargs["thinking"] == {"type": "adaptive"} assert subagent_model_call.kwargs["effort"] == "high" diff --git a/tests/test_reviewer_eval_run.py b/tests/test_reviewer_eval_run.py index b546d1ad..2545b2f9 100644 --- a/tests/test_reviewer_eval_run.py +++ b/tests/test_reviewer_eval_run.py @@ -14,7 +14,7 @@ def test_reviewer_eval_config_coerces_known_values() -> None: "max_concurrency": 2, "langgraph_url": "https://example.test", "assistant_id": "reviewer", - "model_id": "anthropic:claude-opus-4-7", + "model_id": "anthropic:claude-opus-4-8", "reasoning_effort": "high", "score_mode": "surfaced_findings", "severity_threshold": "medium", @@ -29,7 +29,7 @@ def test_reviewer_eval_config_coerces_known_values() -> None: "max_concurrency": 2, "langgraph_url": "https://example.test", "assistant_id": "reviewer", - "model_id": "anthropic:claude-opus-4-7", + "model_id": "anthropic:claude-opus-4-8", "reasoning_effort": "high", "score_mode": "surfaced_findings", "severity_threshold": "medium", @@ -43,7 +43,7 @@ def test_reviewer_eval_config_sets_target_env() -> None: { "langgraph_url": "https://example.test", "assistant_id": "reviewer", - "model_id": "anthropic:claude-opus-4-7", + "model_id": "anthropic:claude-opus-4-8", "reasoning_effort": "high", "score_mode": "surfaced_findings", "severity_threshold": "high", @@ -53,7 +53,7 @@ def test_reviewer_eval_config_sets_target_env() -> None: assert os.environ["LANGGRAPH_URL"] == "https://example.test" assert os.environ["REVIEWER_ASSISTANT_ID"] == "reviewer" - assert os.environ["REVIEWER_EVAL_MODEL_ID"] == "anthropic:claude-opus-4-7" + assert os.environ["REVIEWER_EVAL_MODEL_ID"] == "anthropic:claude-opus-4-8" assert os.environ["REVIEWER_EVAL_REASONING_EFFORT"] == "high" assert os.environ["REVIEWER_EVAL_SCORE_MODE"] == "surfaced_findings" assert os.environ["REVIEWER_EVAL_SEVERITY_THRESHOLD"] == "high" diff --git a/tests/test_reviewer_eval_target.py b/tests/test_reviewer_eval_target.py index 18d6ea3b..78d8b968 100644 --- a/tests/test_reviewer_eval_target.py +++ b/tests/test_reviewer_eval_target.py @@ -29,7 +29,7 @@ def test_eval_target_marks_runs_as_eval_dry_run(monkeypatch: pytest.MonkeyPatch) def test_eval_target_passes_model_overrides(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("REVIEWER_EVAL_MODEL_ID", "anthropic:claude-opus-4-7") + monkeypatch.setenv("REVIEWER_EVAL_MODEL_ID", "anthropic:claude-opus-4-8") monkeypatch.setenv("REVIEWER_EVAL_REASONING_EFFORT", "high") configurable = target._build_configurable( @@ -43,7 +43,7 @@ def test_eval_target_passes_model_overrides(monkeypatch: pytest.MonkeyPatch) -> } ) - assert configurable["reviewer_model_id"] == "anthropic:claude-opus-4-7" + assert configurable["reviewer_model_id"] == "anthropic:claude-opus-4-8" assert configurable["reviewer_reasoning_effort"] == "high"