diff --git a/agent/dashboard/options.py b/agent/dashboard/options.py index a617e493..ff7cb3e8 100644 --- a/agent/dashboard/options.py +++ b/agent/dashboard/options.py @@ -125,3 +125,18 @@ def default_model_pair() -> tuple[str, str]: return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT first = SUPPORTED_MODELS[0] return first["id"], first["default_effort"] + + +def default_vision_model_pair() -> tuple[str, str]: + """Default OpenAI/Anthropic model pair to use when image input is required.""" + if ( + DEFAULT_MODEL_ID in SUPPORTED_MODEL_IDS + and model_supports_images(DEFAULT_MODEL_ID) + and model_supports_effort(DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT) + and DEFAULT_MODEL_ID.startswith(("openai:", "anthropic:")) + ): + return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT + for model in SUPPORTED_MODELS: + if model["id"].startswith(("openai:", "anthropic:")) and model["supports_images"]: + return model["id"], model["default_effort"] + return default_model_pair() diff --git a/agent/dashboard/thread_api.py b/agent/dashboard/thread_api.py index c43af784..891cae68 100644 --- a/agent/dashboard/thread_api.py +++ b/agent/dashboard/thread_api.py @@ -26,7 +26,12 @@ from ..utils.thread_ops import ( queue_message_for_thread, ) from .agent_overrides import normalize_profile_overrides -from .options import SUPPORTED_MODEL_IDS, model_supports_effort, model_supports_images +from .options import ( + SUPPORTED_MODEL_IDS, + default_vision_model_pair, + model_supports_effort, + model_supports_images, +) from .pr_diff import build_pr_diff_files from .profiles import get_profile, get_valid_access_token from .team_settings import get_team_default_model @@ -149,6 +154,19 @@ async def _resolve_agent_model_choice( return resolved_model, resolved_effort +def _with_vision_fallback(model_id: str, effort: str, *, has_images: bool) -> tuple[str, str]: + if not has_images or model_supports_images(model_id): + return model_id, effort + fallback_model_id, fallback_effort = default_vision_model_pair() + logger.info( + "Using vision fallback model %s for dashboard image input; configured model %s " + "does not support images", + fallback_model_id, + model_id, + ) + return fallback_model_id, fallback_effort + + def _now_ms() -> int: return int(datetime.now(UTC).timestamp() * 1000) @@ -926,13 +944,18 @@ async def _create_dashboard_thread_record( now_ms = _now_ms() prompt = prompt.strip() resolved_model, resolved_effort = await _resolve_agent_model_choice(profile, model_id, effort) - # Validate any attached images against the resolved model (raises 422 for - # text-only models). The run itself is started client-side via the stream - # commands endpoint, so we only need the validation side effect here. + resolved_model, resolved_effort = _with_vision_fallback( + resolved_model, + resolved_effort, + has_images=bool(images), + ) _user_message_content(prompt, images or [], model_id=resolved_model) chosen_model, chosen_effort = _normalize_model_choice(model_id, effort) metadata_model = chosen_model or profile.get("default_model") or "Default" metadata_effort = chosen_effort or profile.get("reasoning_effort") + if images and not model_supports_images(str(metadata_model)): + metadata_model = resolved_model + metadata_effort = resolved_effort has_repo = bool(repo_config.get("owner") and repo_config.get("name")) metadata: dict[str, Any] = { "source": _DASHBOARD_SOURCE, @@ -1135,6 +1158,7 @@ async def _enrich_run_start_command( ) plan_mode_requested = client_configurable.get("plan_mode") is True content = _command_message_content(params) + command_images = _dashboard_images_from_content(content) overrides: dict[str, Any] = {} if creating: @@ -1150,22 +1174,45 @@ async def _enrich_run_start_command( repo_config=_parse_repo(client_configurable.get("repo")) or {}, repo_explicitly_none=client_configurable.get("repo_explicitly_none") is True, prompt=_command_prompt_text(content), - images=_dashboard_images_from_content(content), + images=command_images, model_id=client_configurable.get("agent_model_id"), effort=client_configurable.get("agent_effort"), plan_mode=plan_mode_requested, ) metadata = thread.get("metadata") if isinstance(thread.get("metadata"), dict) else metadata - if chosen_model and chosen_effort: + if command_images: + resolved_model = metadata.get("resolved_model") + resolved_effort = metadata.get("resolved_effort") + if isinstance(resolved_model, str) and isinstance(resolved_effort, str): + overrides["agent_model_id"] = resolved_model + overrides["agent_effort"] = resolved_effort + elif chosen_model and chosen_effort: overrides["agent_model_id"] = chosen_model overrides["agent_effort"] = chosen_effort else: - _validate_command_images(content, model_id=chosen_model or _metadata_model_id(metadata)) + run_model = chosen_model or _metadata_model_id(metadata) + run_effort = chosen_effort + if not run_effort: + for key in ("resolved_effort", "effort"): + value = metadata.get(key) + if isinstance(value, str): + run_effort = value + break + if command_images and run_model and run_effort: + run_model, run_effort = _with_vision_fallback(run_model, run_effort, has_images=True) + _validate_command_images(content, model_id=run_model) prefix = _attribution_prefix(metadata, login, email) if prefix: _set_command_last_message_content(params, _prefix_message_content(content, prefix)) metadata_update: dict[str, Any] = {"plan_mode": plan_mode_requested} - if chosen_model and chosen_effort: + if command_images and run_model and run_effort: + overrides["agent_model_id"] = run_model + overrides["agent_effort"] = run_effort + metadata_update["model"] = run_model + metadata_update["effort"] = run_effort + metadata_update["resolved_model"] = run_model + metadata_update["resolved_effort"] = run_effort + elif chosen_model and chosen_effort: overrides["agent_model_id"] = chosen_model overrides["agent_effort"] = chosen_effort metadata_update["model"] = chosen_model diff --git a/agent/webapp.py b/agent/webapp.py index 54b1f3a0..1ab4f5c1 100644 --- a/agent/webapp.py +++ b/agent/webapp.py @@ -27,7 +27,7 @@ from .dashboard.agent_overrides import ( ) from .dashboard.enabled_repos import is_review_repo_enabled from .dashboard.oauth import build_settings_url -from .dashboard.options import model_supports_images # noqa: F401 +from .dashboard.options import default_vision_model_pair, model_supports_images # noqa: F401 from .dashboard.profiles import ( # noqa: F401 get_profile, get_valid_access_token, diff --git a/agent/webhooks/linear.py b/agent/webhooks/linear.py index d50366c4..f897bb1c 100644 --- a/agent/webhooks/linear.py +++ b/agent/webhooks/linear.py @@ -163,31 +163,33 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 f"When you're done, commit and push your changes. {tag_instruction}" ) content_blocks: list[dict[str, Any]] = [create_text_block(prompt)] + image_model_override: tuple[str, str] | None = None if image_urls: image_urls = webapp.dedupe_urls(image_urls) linear_login = ( await webapp.resolve_login_from_email_async(user_email) if user_email else None ) resolved_model_id = await webapp.resolve_agent_model_id(linear_login) - if webapp.model_supports_images(resolved_model_id): - webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls)) - webapp.logger.debug("Image URLs: %s", image_urls) - - async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client: - for image_url in image_urls: - image_block = await webapp.fetch_image_block(image_url, client) - if image_block: - content_blocks.append(image_block) - webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks)) - else: - webapp.logger.warning( - "Skipping %d image(s) for Linear issue: model %s does not support images", + if not webapp.model_supports_images(resolved_model_id): + fallback_model_id, fallback_effort = webapp.default_vision_model_pair() + webapp.logger.info( + "Using vision fallback model %s for %d Linear image(s); configured model %s " + "does not support images", + fallback_model_id, len(image_urls), resolved_model_id, ) - prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls)) - content_blocks[0] = create_text_block(prompt) - image_urls = [] + resolved_model_id = fallback_model_id + image_model_override = (fallback_model_id, fallback_effort) + webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls)) + webapp.logger.debug("Image URLs: %s", image_urls) + + async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client: + for image_url in image_urls: + image_block = await webapp.fetch_image_block(image_url, client) + if image_block: + content_blocks.append(image_block) + webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks)) linear_project_id = "" linear_issue_number = "" @@ -210,6 +212,9 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 "user_email": user_email, "source": "linear", } + if image_model_override: + configurable["agent_model_id"] = image_model_override[0] + configurable["agent_effort"] = image_model_override[1] await webapp.upsert_agent_thread_owner_metadata( thread_id, diff --git a/agent/webhooks/slack.py b/agent/webhooks/slack.py index 87ae821f..4e3021ff 100644 --- a/agent/webhooks/slack.py +++ b/agent/webhooks/slack.py @@ -127,24 +127,26 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st if not mapped_login and user_email: mapped_login = await webapp.login_for_email(user_email) + image_model_override: tuple[str, str] | None = None if image_urls: resolved_model_id = await webapp.resolve_agent_model_id(mapped_login) - if webapp.model_supports_images(resolved_model_id): - webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls)) - async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client: - for image_url in image_urls: - image_block = await webapp.fetch_image_block(image_url, http_client) - if image_block: - content_blocks.append(image_block) - else: - webapp.logger.warning( - "Skipping %d image(s) for Slack mention: model %s does not support images", + if not webapp.model_supports_images(resolved_model_id): + fallback_model_id, fallback_effort = webapp.default_vision_model_pair() + webapp.logger.info( + "Using vision fallback model %s for %d Slack image(s); configured model %s " + "does not support images", + fallback_model_id, len(image_urls), resolved_model_id, ) - prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls)) - content_blocks[0] = create_text_block(prompt) - image_urls = [] + resolved_model_id = fallback_model_id + image_model_override = (fallback_model_id, fallback_effort) + webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls)) + async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client: + for image_url in image_urls: + image_block = await webapp.fetch_image_block(image_url, http_client) + if image_block: + content_blocks.append(image_block) # Open SWE opens PRs as the triggering user, so a run only proceeds when we # have a valid user GitHub token. Users who have never signed in with @@ -207,6 +209,9 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st } if mapped_login: configurable["github_login"] = mapped_login + if image_model_override: + configurable["agent_model_id"] = image_model_override[0] + configurable["agent_effort"] = image_model_override[1] thread_plan_mode = await webapp._get_thread_plan_mode(thread_id) if thread_plan_mode is not None: diff --git a/tests/test_dashboard_thread_api.py b/tests/test_dashboard_thread_api.py index 84c71c46..e90582af 100644 --- a/tests/test_dashboard_thread_api.py +++ b/tests/test_dashboard_thread_api.py @@ -207,7 +207,7 @@ async def test_enrich_run_start_command_creates_and_stamps_new_thread(monkeypatc assert enriched["params"]["assistant_id"] == "agent" -async def test_enrich_run_start_command_rejects_images_for_resolved_text_only_model( +async def test_enrich_run_start_command_uses_vision_fallback_for_text_only_model( monkeypatch, ) -> None: created: dict[str, object] = {} @@ -240,17 +240,23 @@ async def test_enrich_run_start_command_rejects_images_for_resolved_text_only_mo }, } - with pytest.raises(HTTPException) as exc_info: - await thread_api._enrich_run_start_command( - "new-tid", - "octocat", - command, - metadata={}, - creating=True, - ) + enriched = await thread_api._enrich_run_start_command( + "new-tid", + "octocat", + command, + metadata={}, + creating=True, + ) - assert exc_info.value.status_code == 422 - assert "does not support image input" in exc_info.value.detail + stamped = created["metadata"] + assert isinstance(stamped, dict) + assert stamped["model"] == _VISION_MODEL + assert stamped["effort"] == "medium" + assert stamped["resolved_model"] == _VISION_MODEL + assert stamped["resolved_effort"] == "medium" + configurable = enriched["params"]["config"]["configurable"] + assert configurable["agent_model_id"] == _VISION_MODEL + assert configurable["agent_effort"] == "medium" def _thread_with_metadata(metadata: dict) -> dict: diff --git a/tests/test_slack_context.py b/tests/test_slack_context.py index da21ee10..5354a481 100644 --- a/tests/test_slack_context.py +++ b/tests/test_slack_context.py @@ -17,6 +17,9 @@ from agent.utils.slack import ( ) from agent.webapp import generate_thread_id_from_slack_thread +_TEXT_ONLY_MODEL = "fireworks:accounts/fireworks/models/glm-5p2" +_VISION_MODEL = "openai:gpt-5.5" + class _FakeNotFoundError(Exception): status_code = 404 @@ -837,6 +840,69 @@ def test_process_slack_mention_bot_only_mode_runs_without_user_token( assert "prompt" not in captured +def test_process_slack_mention_uses_vision_fallback_for_image_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + _setup_slack_mention_fakes(monkeypatch, captured) + + async def fake_thread_exists(thread_id: str) -> bool: + return False + + async def fake_fetch_slack_thread_messages(channel_id: str, thread_ts: str) -> list[dict]: + return [ + { + "ts": "1700000000.000100", + "text": "<@UBOT> please inspect this", + "user": "U123", + "files": [ + { + "mimetype": "image/png", + "url_private": "https://files.slack.com/screenshot.png", + } + ], + } + ] + + async def fake_resolve_agent_model_id(login: str | None) -> str: + assert login == "mason-gh" + return _TEXT_ONLY_MODEL + + async def fake_fetch_image_block(image_url: str, client: object) -> dict[str, str]: + captured["image_url"] = image_url + return {"type": "image", "source_type": "base64", "mime_type": "image/png", "data": "abc"} + + monkeypatch.setattr(webapp, "_thread_exists", fake_thread_exists) + monkeypatch.setattr(webapp, "fetch_slack_thread_messages", fake_fetch_slack_thread_messages) + monkeypatch.setattr(webapp, "resolve_agent_model_id", fake_resolve_agent_model_id) + monkeypatch.setattr(webapp, "fetch_image_block", fake_fetch_image_block) + + asyncio.run( + webapp.process_slack_mention( + { + "channel_id": "C123", + "thread_ts": "1700000000.000100", + "event_ts": "1700000000.000100", + "user_id": "U123", + "text": "<@UBOT> please inspect this", + "bot_user_id": "UBOT", + }, + {"owner": "langchain-ai", "name": "open-swe"}, + ) + ) + + assert captured["image_url"] == "https://files.slack.com/screenshot.png" + run_create = captured["run_create"] + assert isinstance(run_create, dict) + kwargs = run_create["kwargs"] + configurable = kwargs["config"]["configurable"] + assert configurable["agent_model_id"] == _VISION_MODEL + assert configurable["agent_effort"] == "medium" + content = kwargs["input"]["messages"][0]["content"] + assert any(block.get("type") == "image" for block in content) + assert "does not support image input" not in content[0]["text"] + + class _FakeResponse: def __init__(self, payload: dict) -> None: self._payload = payload