open-swe/tests/test_model_fallback_resolution.py

from unittest.mock import AsyncMock, patch

import pytest

from agent.dashboard.agent_overrides import normalize_profile_overrides
from agent.dashboard.options import (
    DEFAULT_MODEL_ID,
    FABLE_MODEL_IDS,
    default_model_pair,
    fable_disabled_fallback,
    gate_fable_model,
    provider_fallback_pair,
)
from agent.dashboard.team_settings import (
    TeamSettingsUpdate,
    get_team_default_model,
    normalize_team_settings_for_response,
)

STALE_ANTHROPIC = "bedrock_converse:us.anthropic.claude-opus-4-7"
SUPPORTED_ANTHROPIC = "bedrock_converse:us.anthropic.claude-opus-4-8"
STALE_SONNET = "bedrock_converse:us.anthropic.claude-sonnet-4-9"
SUPPORTED_SONNET = "bedrock_converse:us.anthropic.claude-sonnet-5"
RETIRED_DIRECT_ANTHROPIC = "anthropic:claude-opus-4-8"
RETIRED_FABLE = "anthropic:claude-fable-5"


def test_provider_fallback_preserves_provider_and_effort() -> None:
    assert provider_fallback_pair(STALE_ANTHROPIC, "xhigh") == (SUPPORTED_ANTHROPIC, "xhigh")


def test_provider_fallback_keeps_bedrock_sonnet_in_family() -> None:
    # A dropped Bedrock Sonnet must prefer the current Bedrock Sonnet, not cross to
    # the Bedrock Opus that happens to sit first in the provider's list. Requires
    # _claude_family_of to understand bedrock_converse ids, not just anthropic:.
    assert provider_fallback_pair(STALE_SONNET, "high") == (SUPPORTED_SONNET, "high")


def test_provider_fallback_uses_default_effort_when_unsupported() -> None:
    assert provider_fallback_pair(STALE_ANTHROPIC, "bogus") == (SUPPORTED_ANTHROPIC, "high")
    assert provider_fallback_pair(STALE_ANTHROPIC, None) == (SUPPORTED_ANTHROPIC, "high")


def test_provider_fallback_resolves_fireworks_within_provider() -> None:
    model, effort = provider_fallback_pair("fireworks:accounts/fireworks/models/legacy-old", "low")
    assert model.startswith("fireworks:")
    assert effort == "low"


@pytest.mark.parametrize("model_id", ["unknown:model", "no-colon", "", None, 123])
def test_provider_fallback_returns_none_without_provider_match(model_id: object) -> None:
    assert provider_fallback_pair(model_id, "high") is None


@pytest.mark.asyncio
async def test_team_default_stale_anthropic_stays_on_provider() -> None:
    settings = {
        "default_agent_model": STALE_ANTHROPIC,
        "default_agent_reasoning_effort": "xhigh",
    }
    with patch(
        "agent.dashboard.team_settings.get_team_settings",
        new_callable=AsyncMock,
        return_value=settings,
    ):
        assert await get_team_default_model("agent") == (SUPPORTED_ANTHROPIC, "xhigh")


@pytest.mark.asyncio
async def test_team_default_unknown_provider_falls_back_to_global() -> None:
    settings = {
        "default_reviewer_model": "mystery:model",
        "default_reviewer_reasoning_effort": "high",
    }
    with patch(
        "agent.dashboard.team_settings.get_team_settings",
        new_callable=AsyncMock,
        return_value=settings,
    ):
        assert await get_team_default_model("reviewer") == default_model_pair()


def test_profile_stale_anthropic_upgrades_to_supported() -> None:
    profile = {"default_model": STALE_ANTHROPIC, "reasoning_effort": "high"}
    assert normalize_profile_overrides(profile) == (SUPPORTED_ANTHROPIC, "high")


def test_team_settings_update_normalizes_retired_models() -> None:
    update = TeamSettingsUpdate(
        default_agent_model=SUPPORTED_ANTHROPIC,
        default_agent_reasoning_effort="medium",
        default_agent_subagent_model=RETIRED_DIRECT_ANTHROPIC,
        default_agent_subagent_reasoning_effort="medium",
        default_reviewer_model=RETIRED_DIRECT_ANTHROPIC,
        default_reviewer_reasoning_effort="medium",
        default_reviewer_subagent_model=RETIRED_DIRECT_ANTHROPIC,
        default_reviewer_subagent_reasoning_effort="low",
    )

    assert update.default_agent_subagent_model == SUPPORTED_ANTHROPIC
    assert update.default_reviewer_model == SUPPORTED_ANTHROPIC
    assert update.default_reviewer_subagent_model == SUPPORTED_ANTHROPIC


def test_team_settings_update_maps_retired_fable_to_opus_not_fable() -> None:
    update = TeamSettingsUpdate(
        fable_enabled=True,
        default_agent_model=RETIRED_FABLE,
        default_agent_reasoning_effort="high",
    )

    assert update.default_agent_model == SUPPORTED_ANTHROPIC
    assert update.default_agent_model not in FABLE_MODEL_IDS


def test_team_settings_update_rejects_unknown_model() -> None:
    with pytest.raises(ValueError, match="unsupported agent model"):
        TeamSettingsUpdate(
            default_agent_model="openai:gpt-6",
            default_agent_reasoning_effort="medium",
        )


def test_team_settings_update_rejects_invalid_effort_for_retired_model() -> None:
    with pytest.raises(ValueError, match="effort 'bogus' not supported"):
        TeamSettingsUpdate(
            default_agent_model=RETIRED_DIRECT_ANTHROPIC,
            default_agent_reasoning_effort="bogus",
        )


def test_team_settings_response_normalizes_retired_models() -> None:
    settings = normalize_team_settings_for_response(
        {
            "default_agent_subagent_model": RETIRED_DIRECT_ANTHROPIC,
            "default_agent_subagent_reasoning_effort": "medium",
            "default_reviewer_model": "openai:gpt-5.5",
            "default_reviewer_reasoning_effort": "medium",
            "default_reviewer_subagent_model": RETIRED_FABLE,
            "default_reviewer_subagent_reasoning_effort": "low",
        }
    )

    assert settings["default_agent_subagent_model"] == SUPPORTED_ANTHROPIC
    assert settings["default_reviewer_model"] == SUPPORTED_ANTHROPIC
    assert settings["default_reviewer_subagent_model"] == SUPPORTED_ANTHROPIC


def test_profile_without_model_defers_to_team_default() -> None:
    assert normalize_profile_overrides({"reasoning_effort": "high"}) == (None, None)


def test_profile_unknown_provider_defers_to_team_default() -> None:
    profile = {"default_model": "mystery:model", "reasoning_effort": "high"}
    assert normalize_profile_overrides(profile) == (None, None)


def test_global_default_is_supported() -> None:
    model, _ = default_model_pair()
    assert model == DEFAULT_MODEL_ID


def test_gate_fable_passthrough_when_enabled() -> None:
    assert gate_fable_model(
        "bedrock_converse:us.anthropic.claude-fable-5", "high", fable_enabled=True
    ) == (
        "bedrock_converse:us.anthropic.claude-fable-5",
        "high",
    )


def test_gate_fable_swaps_to_opus_when_disabled() -> None:
    assert gate_fable_model(
        "bedrock_converse:us.anthropic.claude-fable-5", "high", fable_enabled=False
    ) == (
        "bedrock_converse:us.anthropic.claude-opus-4-8",
        "high",
    )


def test_gate_fable_leaves_non_fable_ids_alone() -> None:
    assert gate_fable_model(
        "fireworks:accounts/fireworks/models/deepseek-v4-pro", "high", fable_enabled=False
    ) == (
        "fireworks:accounts/fireworks/models/deepseek-v4-pro",
        "high",
    )


def test_fable_disabled_fallback_is_non_fable_claude() -> None:
    model, effort = fable_disabled_fallback("high")
    assert model == "bedrock_converse:us.anthropic.claude-opus-4-8"
    assert model not in FABLE_MODEL_IDS
    assert effort == "high"