fix: fall back to vision model for image threads (#1626)

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>

agent/dashboard/options.py

@@ -125,3 +125,18 @@ def default_model_pair() -> tuple[str, str]:
         return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT
     first = SUPPORTED_MODELS[0]
     return first["id"], first["default_effort"]
+
+
+def default_vision_model_pair() -> tuple[str, str]:
+    """Default OpenAI/Anthropic model pair to use when image input is required."""
+    if (
+        DEFAULT_MODEL_ID in SUPPORTED_MODEL_IDS
+        and model_supports_images(DEFAULT_MODEL_ID)
+        and model_supports_effort(DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT)
+        and DEFAULT_MODEL_ID.startswith(("openai:", "anthropic:"))
+    ):
+        return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT
+    for model in SUPPORTED_MODELS:
+        if model["id"].startswith(("openai:", "anthropic:")) and model["supports_images"]:
+            return model["id"], model["default_effort"]
+    return default_model_pair()

agent/dashboard/thread_api.py

@@ -26,7 +26,12 @@ from ..utils.thread_ops import (
     queue_message_for_thread,
 )
 from .agent_overrides import normalize_profile_overrides
-from .options import SUPPORTED_MODEL_IDS, model_supports_effort, model_supports_images
+from .options import (
+    SUPPORTED_MODEL_IDS,
+    default_vision_model_pair,
+    model_supports_effort,
+    model_supports_images,
+)
 from .pr_diff import build_pr_diff_files
 from .profiles import get_profile, get_valid_access_token
 from .team_settings import get_team_default_model
@@ -149,6 +154,19 @@ async def _resolve_agent_model_choice(
     return resolved_model, resolved_effort
 
 
+def _with_vision_fallback(model_id: str, effort: str, *, has_images: bool) -> tuple[str, str]:
+    if not has_images or model_supports_images(model_id):
+        return model_id, effort
+    fallback_model_id, fallback_effort = default_vision_model_pair()
+    logger.info(
+        "Using vision fallback model %s for dashboard image input; configured model %s "
+        "does not support images",
+        fallback_model_id,
+        model_id,
+    )
+    return fallback_model_id, fallback_effort
+
+
 def _now_ms() -> int:
     return int(datetime.now(UTC).timestamp() * 1000)
 
@@ -926,13 +944,18 @@ async def _create_dashboard_thread_record(
     now_ms = _now_ms()
     prompt = prompt.strip()
     resolved_model, resolved_effort = await _resolve_agent_model_choice(profile, model_id, effort)
-    # Validate any attached images against the resolved model (raises 422 for
+    resolved_model, resolved_effort = _with_vision_fallback(
-    # text-only models). The run itself is started client-side via the stream
+        resolved_model,
-    # commands endpoint, so we only need the validation side effect here.
+        resolved_effort,
+        has_images=bool(images),
+    )
     _user_message_content(prompt, images or [], model_id=resolved_model)
     chosen_model, chosen_effort = _normalize_model_choice(model_id, effort)
     metadata_model = chosen_model or profile.get("default_model") or "Default"
     metadata_effort = chosen_effort or profile.get("reasoning_effort")
+    if images and not model_supports_images(str(metadata_model)):
+        metadata_model = resolved_model
+        metadata_effort = resolved_effort
     has_repo = bool(repo_config.get("owner") and repo_config.get("name"))
     metadata: dict[str, Any] = {
         "source": _DASHBOARD_SOURCE,
@@ -1135,6 +1158,7 @@ async def _enrich_run_start_command(
     )
     plan_mode_requested = client_configurable.get("plan_mode") is True
     content = _command_message_content(params)
+    command_images = _dashboard_images_from_content(content)
     overrides: dict[str, Any] = {}
 
     if creating:
@@ -1150,22 +1174,45 @@ async def _enrich_run_start_command(
             repo_config=_parse_repo(client_configurable.get("repo")) or {},
             repo_explicitly_none=client_configurable.get("repo_explicitly_none") is True,
             prompt=_command_prompt_text(content),
-            images=_dashboard_images_from_content(content),
+            images=command_images,
             model_id=client_configurable.get("agent_model_id"),
             effort=client_configurable.get("agent_effort"),
             plan_mode=plan_mode_requested,
         )
         metadata = thread.get("metadata") if isinstance(thread.get("metadata"), dict) else metadata
-        if chosen_model and chosen_effort:
+        if command_images:
+            resolved_model = metadata.get("resolved_model")
+            resolved_effort = metadata.get("resolved_effort")
+            if isinstance(resolved_model, str) and isinstance(resolved_effort, str):
+                overrides["agent_model_id"] = resolved_model
+                overrides["agent_effort"] = resolved_effort
+        elif chosen_model and chosen_effort:
             overrides["agent_model_id"] = chosen_model
             overrides["agent_effort"] = chosen_effort
     else:
-        _validate_command_images(content, model_id=chosen_model or _metadata_model_id(metadata))
+        run_model = chosen_model or _metadata_model_id(metadata)
+        run_effort = chosen_effort
+        if not run_effort:
+            for key in ("resolved_effort", "effort"):
+                value = metadata.get(key)
+                if isinstance(value, str):
+                    run_effort = value
+                    break
+        if command_images and run_model and run_effort:
+            run_model, run_effort = _with_vision_fallback(run_model, run_effort, has_images=True)
+        _validate_command_images(content, model_id=run_model)
         prefix = _attribution_prefix(metadata, login, email)
         if prefix:
             _set_command_last_message_content(params, _prefix_message_content(content, prefix))
         metadata_update: dict[str, Any] = {"plan_mode": plan_mode_requested}
-        if chosen_model and chosen_effort:
+        if command_images and run_model and run_effort:
+            overrides["agent_model_id"] = run_model
+            overrides["agent_effort"] = run_effort
+            metadata_update["model"] = run_model
+            metadata_update["effort"] = run_effort
+            metadata_update["resolved_model"] = run_model
+            metadata_update["resolved_effort"] = run_effort
+        elif chosen_model and chosen_effort:
             overrides["agent_model_id"] = chosen_model
             overrides["agent_effort"] = chosen_effort
             metadata_update["model"] = chosen_model

agent/webapp.py

@@ -27,7 +27,7 @@ from .dashboard.agent_overrides import (
 )
 from .dashboard.enabled_repos import is_review_repo_enabled
 from .dashboard.oauth import build_settings_url
-from .dashboard.options import model_supports_images  # noqa: F401
+from .dashboard.options import default_vision_model_pair, model_supports_images  # noqa: F401
 from .dashboard.profiles import (  # noqa: F401
     get_profile,
     get_valid_access_token,

agent/webhooks/linear.py

@@ -163,31 +163,33 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
         f"When you're done, commit and push your changes. {tag_instruction}"
     )
     content_blocks: list[dict[str, Any]] = [create_text_block(prompt)]
+    image_model_override: tuple[str, str] | None = None
     if image_urls:
         image_urls = webapp.dedupe_urls(image_urls)
         linear_login = (
             await webapp.resolve_login_from_email_async(user_email) if user_email else None
         )
         resolved_model_id = await webapp.resolve_agent_model_id(linear_login)
-        if webapp.model_supports_images(resolved_model_id):
+        if not webapp.model_supports_images(resolved_model_id):
-            webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
+            fallback_model_id, fallback_effort = webapp.default_vision_model_pair()
-            webapp.logger.debug("Image URLs: %s", image_urls)
+            webapp.logger.info(
-
+                "Using vision fallback model %s for %d Linear image(s); configured model %s "
-            async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client:
+                "does not support images",
-                for image_url in image_urls:
+                fallback_model_id,
-                    image_block = await webapp.fetch_image_block(image_url, client)
-                    if image_block:
-                        content_blocks.append(image_block)
-            webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks))
-        else:
-            webapp.logger.warning(
-                "Skipping %d image(s) for Linear issue: model %s does not support images",
                 len(image_urls),
                 resolved_model_id,
             )
-            prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls))
+            resolved_model_id = fallback_model_id
-            content_blocks[0] = create_text_block(prompt)
+            image_model_override = (fallback_model_id, fallback_effort)
-            image_urls = []
+        webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
+        webapp.logger.debug("Image URLs: %s", image_urls)
+
+        async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client:
+            for image_url in image_urls:
+                image_block = await webapp.fetch_image_block(image_url, client)
+                if image_block:
+                    content_blocks.append(image_block)
+        webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks))
 
     linear_project_id = ""
     linear_issue_number = ""
@@ -210,6 +212,9 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
         "user_email": user_email,
         "source": "linear",
     }
+    if image_model_override:
+        configurable["agent_model_id"] = image_model_override[0]
+        configurable["agent_effort"] = image_model_override[1]
 
     await webapp.upsert_agent_thread_owner_metadata(
         thread_id,

agent/webhooks/slack.py

@@ -127,24 +127,26 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st
     if not mapped_login and user_email:
         mapped_login = await webapp.login_for_email(user_email)
 
+    image_model_override: tuple[str, str] | None = None
     if image_urls:
         resolved_model_id = await webapp.resolve_agent_model_id(mapped_login)
-        if webapp.model_supports_images(resolved_model_id):
+        if not webapp.model_supports_images(resolved_model_id):
-            webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls))
+            fallback_model_id, fallback_effort = webapp.default_vision_model_pair()
-            async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client:
+            webapp.logger.info(
-                for image_url in image_urls:
+                "Using vision fallback model %s for %d Slack image(s); configured model %s "
-                    image_block = await webapp.fetch_image_block(image_url, http_client)
+                "does not support images",
-                    if image_block:
+                fallback_model_id,
-                        content_blocks.append(image_block)
-        else:
-            webapp.logger.warning(
-                "Skipping %d image(s) for Slack mention: model %s does not support images",
                 len(image_urls),
                 resolved_model_id,
             )
-            prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls))
+            resolved_model_id = fallback_model_id
-            content_blocks[0] = create_text_block(prompt)
+            image_model_override = (fallback_model_id, fallback_effort)
-            image_urls = []
+        webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls))
+        async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client:
+            for image_url in image_urls:
+                image_block = await webapp.fetch_image_block(image_url, http_client)
+                if image_block:
+                    content_blocks.append(image_block)
 
     # Open SWE opens PRs as the triggering user, so a run only proceeds when we
     # have a valid user GitHub token. Users who have never signed in with
@@ -207,6 +209,9 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st
     }
     if mapped_login:
         configurable["github_login"] = mapped_login
+    if image_model_override:
+        configurable["agent_model_id"] = image_model_override[0]
+        configurable["agent_effort"] = image_model_override[1]
 
     thread_plan_mode = await webapp._get_thread_plan_mode(thread_id)
     if thread_plan_mode is not None:

tests/test_dashboard_thread_api.py

@@ -207,7 +207,7 @@ async def test_enrich_run_start_command_creates_and_stamps_new_thread(monkeypatc
     assert enriched["params"]["assistant_id"] == "agent"
 
 
-async def test_enrich_run_start_command_rejects_images_for_resolved_text_only_model(
+async def test_enrich_run_start_command_uses_vision_fallback_for_text_only_model(
     monkeypatch,
 ) -> None:
     created: dict[str, object] = {}
@@ -240,17 +240,23 @@ async def test_enrich_run_start_command_rejects_images_for_resolved_text_only_mo
         },
     }
 
-    with pytest.raises(HTTPException) as exc_info:
+    enriched = await thread_api._enrich_run_start_command(
-        await thread_api._enrich_run_start_command(
+        "new-tid",
-            "new-tid",
+        "octocat",
-            "octocat",
+        command,
-            command,
+        metadata={},
-            metadata={},
+        creating=True,
-            creating=True,
+    )
-        )
 
-    assert exc_info.value.status_code == 422
+    stamped = created["metadata"]
-    assert "does not support image input" in exc_info.value.detail
+    assert isinstance(stamped, dict)
+    assert stamped["model"] == _VISION_MODEL
+    assert stamped["effort"] == "medium"
+    assert stamped["resolved_model"] == _VISION_MODEL
+    assert stamped["resolved_effort"] == "medium"
+    configurable = enriched["params"]["config"]["configurable"]
+    assert configurable["agent_model_id"] == _VISION_MODEL
+    assert configurable["agent_effort"] == "medium"
 
 
 def _thread_with_metadata(metadata: dict) -> dict:

tests/test_slack_context.py

@@ -17,6 +17,9 @@ from agent.utils.slack import (
 )
 from agent.webapp import generate_thread_id_from_slack_thread
 
+_TEXT_ONLY_MODEL = "fireworks:accounts/fireworks/models/glm-5p2"
+_VISION_MODEL = "openai:gpt-5.5"
+
 
 class _FakeNotFoundError(Exception):
     status_code = 404
@@ -837,6 +840,69 @@ def test_process_slack_mention_bot_only_mode_runs_without_user_token(
     assert "prompt" not in captured
 
 
+def test_process_slack_mention_uses_vision_fallback_for_image_thread(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    captured: dict[str, object] = {}
+    _setup_slack_mention_fakes(monkeypatch, captured)
+
+    async def fake_thread_exists(thread_id: str) -> bool:
+        return False
+
+    async def fake_fetch_slack_thread_messages(channel_id: str, thread_ts: str) -> list[dict]:
+        return [
+            {
+                "ts": "1700000000.000100",
+                "text": "<@UBOT> please inspect this",
+                "user": "U123",
+                "files": [
+                    {
+                        "mimetype": "image/png",
+                        "url_private": "https://files.slack.com/screenshot.png",
+                    }
+                ],
+            }
+        ]
+
+    async def fake_resolve_agent_model_id(login: str | None) -> str:
+        assert login == "mason-gh"
+        return _TEXT_ONLY_MODEL
+
+    async def fake_fetch_image_block(image_url: str, client: object) -> dict[str, str]:
+        captured["image_url"] = image_url
+        return {"type": "image", "source_type": "base64", "mime_type": "image/png", "data": "abc"}
+
+    monkeypatch.setattr(webapp, "_thread_exists", fake_thread_exists)
+    monkeypatch.setattr(webapp, "fetch_slack_thread_messages", fake_fetch_slack_thread_messages)
+    monkeypatch.setattr(webapp, "resolve_agent_model_id", fake_resolve_agent_model_id)
+    monkeypatch.setattr(webapp, "fetch_image_block", fake_fetch_image_block)
+
+    asyncio.run(
+        webapp.process_slack_mention(
+            {
+                "channel_id": "C123",
+                "thread_ts": "1700000000.000100",
+                "event_ts": "1700000000.000100",
+                "user_id": "U123",
+                "text": "<@UBOT> please inspect this",
+                "bot_user_id": "UBOT",
+            },
+            {"owner": "langchain-ai", "name": "open-swe"},
+        )
+    )
+
+    assert captured["image_url"] == "https://files.slack.com/screenshot.png"
+    run_create = captured["run_create"]
+    assert isinstance(run_create, dict)
+    kwargs = run_create["kwargs"]
+    configurable = kwargs["config"]["configurable"]
+    assert configurable["agent_model_id"] == _VISION_MODEL
+    assert configurable["agent_effort"] == "medium"
+    content = kwargs["input"]["messages"][0]["content"]
+    assert any(block.get("type") == "image" for block in content)
+    assert "does not support image input" not in content[0]["text"]
+
+
 class _FakeResponse:
     def __init__(self, payload: dict) -> None:
         self._payload = payload