diff --git a/agent/dashboard/options.py b/agent/dashboard/options.py index 48e7932a..595ed66d 100644 --- a/agent/dashboard/options.py +++ b/agent/dashboard/options.py @@ -158,15 +158,20 @@ def default_model_pair() -> tuple[str, str]: def default_vision_model_pair() -> tuple[str, str]: - """Default OpenAI/Anthropic model pair to use when image input is required.""" + """Default (model_id, reasoning_effort) to use when image input is required. + + Prefers the configured default model when it is vision-capable (Opus 4.8), + otherwise the first vision-capable supported model. The fork ships only + Bedrock/Fireworks models, so this selects on ``supports_images`` rather than + upstream's ``openai:``/``anthropic:`` provider filter. + """ if ( DEFAULT_MODEL_ID in SUPPORTED_MODEL_IDS and model_supports_images(DEFAULT_MODEL_ID) and model_supports_effort(DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT) - and DEFAULT_MODEL_ID.startswith(("openai:", "anthropic:")) ): return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT for model in SUPPORTED_MODELS: - if model["id"].startswith(("openai:", "anthropic:")) and model["supports_images"]: + if model["supports_images"]: return model["id"], model["default_effort"] return default_model_pair() diff --git a/agent/webapp.py b/agent/webapp.py index fdb11950..56cffaa7 100644 --- a/agent/webapp.py +++ b/agent/webapp.py @@ -27,7 +27,7 @@ from .dashboard.agent_overrides import ( ) from .dashboard.enabled_repos import is_review_repo_enabled from .dashboard.oauth import build_settings_url -from .dashboard.options import model_supports_images # noqa: F401 +from .dashboard.options import default_vision_model_pair, model_supports_images # noqa: F401 from .dashboard.profiles import ( # noqa: F401 get_profile, get_valid_access_token, diff --git a/agent/webhooks/linear.py b/agent/webhooks/linear.py index d50366c4..f897bb1c 100644 --- a/agent/webhooks/linear.py +++ b/agent/webhooks/linear.py @@ -163,31 +163,33 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 f"When you're done, commit and push your changes. {tag_instruction}" ) content_blocks: list[dict[str, Any]] = [create_text_block(prompt)] + image_model_override: tuple[str, str] | None = None if image_urls: image_urls = webapp.dedupe_urls(image_urls) linear_login = ( await webapp.resolve_login_from_email_async(user_email) if user_email else None ) resolved_model_id = await webapp.resolve_agent_model_id(linear_login) - if webapp.model_supports_images(resolved_model_id): - webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls)) - webapp.logger.debug("Image URLs: %s", image_urls) - - async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client: - for image_url in image_urls: - image_block = await webapp.fetch_image_block(image_url, client) - if image_block: - content_blocks.append(image_block) - webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks)) - else: - webapp.logger.warning( - "Skipping %d image(s) for Linear issue: model %s does not support images", + if not webapp.model_supports_images(resolved_model_id): + fallback_model_id, fallback_effort = webapp.default_vision_model_pair() + webapp.logger.info( + "Using vision fallback model %s for %d Linear image(s); configured model %s " + "does not support images", + fallback_model_id, len(image_urls), resolved_model_id, ) - prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls)) - content_blocks[0] = create_text_block(prompt) - image_urls = [] + resolved_model_id = fallback_model_id + image_model_override = (fallback_model_id, fallback_effort) + webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls)) + webapp.logger.debug("Image URLs: %s", image_urls) + + async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client: + for image_url in image_urls: + image_block = await webapp.fetch_image_block(image_url, client) + if image_block: + content_blocks.append(image_block) + webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks)) linear_project_id = "" linear_issue_number = "" @@ -210,6 +212,9 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 "user_email": user_email, "source": "linear", } + if image_model_override: + configurable["agent_model_id"] = image_model_override[0] + configurable["agent_effort"] = image_model_override[1] await webapp.upsert_agent_thread_owner_metadata( thread_id, diff --git a/agent/webhooks/slack.py b/agent/webhooks/slack.py index 87ae821f..4e3021ff 100644 --- a/agent/webhooks/slack.py +++ b/agent/webhooks/slack.py @@ -127,24 +127,26 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st if not mapped_login and user_email: mapped_login = await webapp.login_for_email(user_email) + image_model_override: tuple[str, str] | None = None if image_urls: resolved_model_id = await webapp.resolve_agent_model_id(mapped_login) - if webapp.model_supports_images(resolved_model_id): - webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls)) - async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client: - for image_url in image_urls: - image_block = await webapp.fetch_image_block(image_url, http_client) - if image_block: - content_blocks.append(image_block) - else: - webapp.logger.warning( - "Skipping %d image(s) for Slack mention: model %s does not support images", + if not webapp.model_supports_images(resolved_model_id): + fallback_model_id, fallback_effort = webapp.default_vision_model_pair() + webapp.logger.info( + "Using vision fallback model %s for %d Slack image(s); configured model %s " + "does not support images", + fallback_model_id, len(image_urls), resolved_model_id, ) - prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls)) - content_blocks[0] = create_text_block(prompt) - image_urls = [] + resolved_model_id = fallback_model_id + image_model_override = (fallback_model_id, fallback_effort) + webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls)) + async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client: + for image_url in image_urls: + image_block = await webapp.fetch_image_block(image_url, http_client) + if image_block: + content_blocks.append(image_block) # Open SWE opens PRs as the triggering user, so a run only proceeds when we # have a valid user GitHub token. Users who have never signed in with @@ -207,6 +209,9 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st } if mapped_login: configurable["github_login"] = mapped_login + if image_model_override: + configurable["agent_model_id"] = image_model_override[0] + configurable["agent_effort"] = image_model_override[1] thread_plan_mode = await webapp._get_thread_plan_mode(thread_id) if thread_plan_mode is not None: diff --git a/tests/test_slack_context.py b/tests/test_slack_context.py index da21ee10..b045f866 100644 --- a/tests/test_slack_context.py +++ b/tests/test_slack_context.py @@ -17,6 +17,9 @@ from agent.utils.slack import ( ) from agent.webapp import generate_thread_id_from_slack_thread +_TEXT_ONLY_MODEL = "fireworks:accounts/fireworks/models/glm-5p2" +_VISION_MODEL = "bedrock_converse:us.anthropic.claude-opus-4-8" + class _FakeNotFoundError(Exception): status_code = 404 @@ -572,6 +575,69 @@ def test_process_slack_mention_creates_thread_first_run_with_trace_reply( assert "## Latest Mention Request\ncontinue on the branch" in prompt_block["text"] +def test_process_slack_mention_uses_vision_fallback_for_image_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + _setup_slack_mention_fakes(monkeypatch, captured) + + async def fake_thread_exists(thread_id: str) -> bool: + return False + + async def fake_fetch_slack_thread_messages(channel_id: str, thread_ts: str) -> list[dict]: + return [ + { + "ts": "1700000000.000100", + "text": "<@UBOT> please inspect this", + "user": "U123", + "files": [ + { + "mimetype": "image/png", + "url_private": "https://files.slack.com/screenshot.png", + } + ], + } + ] + + async def fake_resolve_agent_model_id(login: str | None) -> str: + assert login == "mason-gh" + return _TEXT_ONLY_MODEL + + async def fake_fetch_image_block(image_url: str, client: object) -> dict[str, str]: + captured["image_url"] = image_url + return {"type": "image", "source_type": "base64", "mime_type": "image/png", "data": "abc"} + + monkeypatch.setattr(webapp, "_thread_exists", fake_thread_exists) + monkeypatch.setattr(webapp, "fetch_slack_thread_messages", fake_fetch_slack_thread_messages) + monkeypatch.setattr(webapp, "resolve_agent_model_id", fake_resolve_agent_model_id) + monkeypatch.setattr(webapp, "fetch_image_block", fake_fetch_image_block) + + asyncio.run( + webapp.process_slack_mention( + { + "channel_id": "C123", + "thread_ts": "1700000000.000100", + "event_ts": "1700000000.000100", + "user_id": "U123", + "text": "<@UBOT> please inspect this", + "bot_user_id": "UBOT", + }, + {"owner": "langchain-ai", "name": "open-swe"}, + ) + ) + + assert captured["image_url"] == "https://files.slack.com/screenshot.png" + run_create = captured["run_create"] + assert isinstance(run_create, dict) + kwargs = run_create["kwargs"] + configurable = kwargs["config"]["configurable"] + assert configurable["agent_model_id"] == _VISION_MODEL + assert configurable["agent_effort"] == "medium" + content = kwargs["input"]["messages"][0]["content"] + assert any(block.get("type") == "image" for block in content) + assert "does not support image input" not in content[0]["text"] + + def test_process_slack_mention_skips_trace_reply_on_followup_mention( monkeypatch: pytest.MonkeyPatch, ) -> None: