mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 17:23:15 +00:00
5350b63c85
Some checks are pending
CI / Lint (push) Waiting to run
CI / Format check (push) Waiting to run
CI / Typecheck (push) Waiting to run
CI / Unit tests (push) Waiting to run
CI / Playwright E2E (push) Waiting to run
CI / Docker build smoke (push) Waiting to run
CI / Triage ledger up to date (push) Waiting to run
CI / ui bun.lock in sync (push) Waiting to run
* feat(models): re-add Fable 5 with admin disable toggle (port of upstream #1677) * refactor(models): convert re-added Fable 5 to Bedrock model IDs * fix(open-swe): correct Fable copy to describe provider data sharing, not ZDR The ported admin toggle description and code comments described Fable 5 as incompatible with Zero Data Retention. That is backwards: Fable 5 requires the account to opt into Bedrock provider_data_share — prompts/completions are retained and shared with Anthropic (up to 30 days, incl. human review). The old UI copy would lead an admin to believe the opposite of what enabling the toggle does. Reword the toggle description and the gate_fable_model / team_settings comments accordingly. Still off by default. Refs #171.
127 lines
4.5 KiB
Python
127 lines
4.5 KiB
Python
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.dashboard.agent_overrides import normalize_profile_overrides
|
|
from agent.dashboard.options import (
|
|
DEFAULT_MODEL_ID,
|
|
FABLE_MODEL_IDS,
|
|
default_model_pair,
|
|
fable_disabled_fallback,
|
|
gate_fable_model,
|
|
provider_fallback_pair,
|
|
)
|
|
from agent.dashboard.team_settings import get_team_default_model
|
|
|
|
STALE_ANTHROPIC = "bedrock_converse:us.anthropic.claude-opus-4-7"
|
|
SUPPORTED_ANTHROPIC = "bedrock_converse:us.anthropic.claude-opus-4-8"
|
|
STALE_SONNET = "bedrock_converse:us.anthropic.claude-sonnet-4-9"
|
|
SUPPORTED_SONNET = "bedrock_converse:us.anthropic.claude-sonnet-5"
|
|
|
|
|
|
def test_provider_fallback_preserves_provider_and_effort() -> None:
|
|
assert provider_fallback_pair(STALE_ANTHROPIC, "xhigh") == (SUPPORTED_ANTHROPIC, "xhigh")
|
|
|
|
|
|
def test_provider_fallback_keeps_bedrock_sonnet_in_family() -> None:
|
|
# A dropped Bedrock Sonnet must prefer the current Bedrock Sonnet, not cross to
|
|
# the Bedrock Opus that happens to sit first in the provider's list. Requires
|
|
# _claude_family_of to understand bedrock_converse ids, not just anthropic:.
|
|
assert provider_fallback_pair(STALE_SONNET, "high") == (SUPPORTED_SONNET, "high")
|
|
|
|
|
|
def test_provider_fallback_uses_default_effort_when_unsupported() -> None:
|
|
assert provider_fallback_pair(STALE_ANTHROPIC, "bogus") == (SUPPORTED_ANTHROPIC, "high")
|
|
assert provider_fallback_pair(STALE_ANTHROPIC, None) == (SUPPORTED_ANTHROPIC, "high")
|
|
|
|
|
|
def test_provider_fallback_resolves_fireworks_within_provider() -> None:
|
|
model, effort = provider_fallback_pair("fireworks:accounts/fireworks/models/legacy-old", "low")
|
|
assert model.startswith("fireworks:")
|
|
assert effort == "low"
|
|
|
|
|
|
@pytest.mark.parametrize("model_id", ["unknown:model", "no-colon", "", None, 123])
|
|
def test_provider_fallback_returns_none_without_provider_match(model_id: object) -> None:
|
|
assert provider_fallback_pair(model_id, "high") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_team_default_stale_anthropic_stays_on_provider() -> None:
|
|
settings = {
|
|
"default_agent_model": STALE_ANTHROPIC,
|
|
"default_agent_reasoning_effort": "xhigh",
|
|
}
|
|
with patch(
|
|
"agent.dashboard.team_settings.get_team_settings",
|
|
new_callable=AsyncMock,
|
|
return_value=settings,
|
|
):
|
|
assert await get_team_default_model("agent") == (SUPPORTED_ANTHROPIC, "xhigh")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_team_default_unknown_provider_falls_back_to_global() -> None:
|
|
settings = {
|
|
"default_reviewer_model": "mystery:model",
|
|
"default_reviewer_reasoning_effort": "high",
|
|
}
|
|
with patch(
|
|
"agent.dashboard.team_settings.get_team_settings",
|
|
new_callable=AsyncMock,
|
|
return_value=settings,
|
|
):
|
|
assert await get_team_default_model("reviewer") == default_model_pair()
|
|
|
|
|
|
def test_profile_stale_anthropic_upgrades_to_supported() -> None:
|
|
profile = {"default_model": STALE_ANTHROPIC, "reasoning_effort": "high"}
|
|
assert normalize_profile_overrides(profile) == (SUPPORTED_ANTHROPIC, "high")
|
|
|
|
|
|
def test_profile_without_model_defers_to_team_default() -> None:
|
|
assert normalize_profile_overrides({"reasoning_effort": "high"}) == (None, None)
|
|
|
|
|
|
def test_profile_unknown_provider_defers_to_team_default() -> None:
|
|
profile = {"default_model": "mystery:model", "reasoning_effort": "high"}
|
|
assert normalize_profile_overrides(profile) == (None, None)
|
|
|
|
|
|
def test_global_default_is_supported() -> None:
|
|
model, _ = default_model_pair()
|
|
assert model == DEFAULT_MODEL_ID
|
|
|
|
|
|
def test_gate_fable_passthrough_when_enabled() -> None:
|
|
assert gate_fable_model(
|
|
"bedrock_converse:us.anthropic.claude-fable-5", "high", fable_enabled=True
|
|
) == (
|
|
"bedrock_converse:us.anthropic.claude-fable-5",
|
|
"high",
|
|
)
|
|
|
|
|
|
def test_gate_fable_swaps_to_opus_when_disabled() -> None:
|
|
assert gate_fable_model(
|
|
"bedrock_converse:us.anthropic.claude-fable-5", "high", fable_enabled=False
|
|
) == (
|
|
"bedrock_converse:us.anthropic.claude-opus-4-8",
|
|
"high",
|
|
)
|
|
|
|
|
|
def test_gate_fable_leaves_non_fable_ids_alone() -> None:
|
|
assert gate_fable_model(
|
|
"fireworks:accounts/fireworks/models/deepseek-v4-pro", "high", fable_enabled=False
|
|
) == (
|
|
"fireworks:accounts/fireworks/models/deepseek-v4-pro",
|
|
"high",
|
|
)
|
|
|
|
|
|
def test_fable_disabled_fallback_is_non_fable_claude() -> None:
|
|
model, effort = fable_disabled_fallback("high")
|
|
assert model == "bedrock_converse:us.anthropic.claude-opus-4-8"
|
|
assert model not in FABLE_MODEL_IDS
|
|
assert effort == "high"
|