adam/open-swe
1
0
Fork 0
mirror of https://github.com/Sea-Haven-Industries/open-swe.git synced 2026-09-30 15:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: fall back to vision model for image threads (#1626)

Browse source 
Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
This commit is contained in:
Johannes du Plessis 2026-06-29 09:37:33 -07:00 • committed by GitHub
parent 7ee3e05724
commit 2f237b53e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 193 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
agent/dashboard/options.py
View file

@ -125,3 +125,18 @@ def default_model_pair() -> tuple[str, str]:
return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT
first = SUPPORTED_MODELS[0]
return first["id"], first["default_effort"]
def default_vision_model_pair() -> tuple[str, str]:
"""Default OpenAI/Anthropic model pair to use when image input is required."""
if (
DEFAULT_MODEL_ID in SUPPORTED_MODEL_IDS
and model_supports_images(DEFAULT_MODEL_ID)
and model_supports_effort(DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT)
and DEFAULT_MODEL_ID.startswith(("openai:", "anthropic:"))
):
return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT
for model in SUPPORTED_MODELS:
if model["id"].startswith(("openai:", "anthropic:")) and model["supports_images"]:
return model["id"], model["default_effort"]
return default_model_pair()

63
agent/dashboard/thread_api.py
View file

@ -26,7 +26,12 @@ from ..utils.thread_ops import (
queue_message_for_thread,
)
from .agent_overrides import normalize_profile_overrides
from .options import SUPPORTED_MODEL_IDS, model_supports_effort, model_supports_images
from .options import (
SUPPORTED_MODEL_IDS,
default_vision_model_pair,
model_supports_effort,
model_supports_images,
)
from .pr_diff import build_pr_diff_files
from .profiles import get_profile, get_valid_access_token
from .team_settings import get_team_default_model
@ -149,6 +154,19 @@ async def _resolve_agent_model_choice(
return resolved_model, resolved_effort
def _with_vision_fallback(model_id: str, effort: str, *, has_images: bool) -> tuple[str, str]:
if not has_images or model_supports_images(model_id):
return model_id, effort
fallback_model_id, fallback_effort = default_vision_model_pair()
logger.info(
"Using vision fallback model %s for dashboard image input; configured model %s "
"does not support images",
fallback_model_id,
model_id,
)
return fallback_model_id, fallback_effort
def _now_ms() -> int:
return int(datetime.now(UTC).timestamp() * 1000)
@ -926,13 +944,18 @@ async def _create_dashboard_thread_record(
now_ms = _now_ms()
prompt = prompt.strip()
resolved_model, resolved_effort = await _resolve_agent_model_choice(profile, model_id, effort)
# Validate any attached images against the resolved model (raises 422 for
# text-only models). The run itself is started client-side via the stream
# commands endpoint, so we only need the validation side effect here.
resolved_model, resolved_effort = _with_vision_fallback(
resolved_model,
resolved_effort,
has_images=bool(images),
)
_user_message_content(prompt, images or [], model_id=resolved_model)
chosen_model, chosen_effort = _normalize_model_choice(model_id, effort)
metadata_model = chosen_model or profile.get("default_model") or "Default"
metadata_effort = chosen_effort or profile.get("reasoning_effort")
if images and not model_supports_images(str(metadata_model)):
metadata_model = resolved_model
metadata_effort = resolved_effort
has_repo = bool(repo_config.get("owner") and repo_config.get("name"))
metadata: dict[str, Any] = {
"source": _DASHBOARD_SOURCE,
@ -1135,6 +1158,7 @@ async def _enrich_run_start_command(
)
plan_mode_requested = client_configurable.get("plan_mode") is True
content = _command_message_content(params)
command_images = _dashboard_images_from_content(content)
overrides: dict[str, Any] = {}
if creating:
@ -1150,22 +1174,45 @@ async def _enrich_run_start_command(
repo_config=_parse_repo(client_configurable.get("repo")) or {},
repo_explicitly_none=client_configurable.get("repo_explicitly_none") is True,
prompt=_command_prompt_text(content),
images=_dashboard_images_from_content(content),
images=command_images,
model_id=client_configurable.get("agent_model_id"),
effort=client_configurable.get("agent_effort"),
plan_mode=plan_mode_requested,
)
metadata = thread.get("metadata") if isinstance(thread.get("metadata"), dict) else metadata
if chosen_model and chosen_effort:
if command_images:
resolved_model = metadata.get("resolved_model")
resolved_effort = metadata.get("resolved_effort")
if isinstance(resolved_model, str) and isinstance(resolved_effort, str):
overrides["agent_model_id"] = resolved_model
overrides["agent_effort"] = resolved_effort
elif chosen_model and chosen_effort:
overrides["agent_model_id"] = chosen_model
overrides["agent_effort"] = chosen_effort
else:
_validate_command_images(content, model_id=chosen_model or _metadata_model_id(metadata))
run_model = chosen_model or _metadata_model_id(metadata)
run_effort = chosen_effort
if not run_effort:
for key in ("resolved_effort", "effort"):
value = metadata.get(key)
if isinstance(value, str):
run_effort = value
break
if command_images and run_model and run_effort:
run_model, run_effort = _with_vision_fallback(run_model, run_effort, has_images=True)
_validate_command_images(content, model_id=run_model)
prefix = _attribution_prefix(metadata, login, email)
if prefix:
_set_command_last_message_content(params, _prefix_message_content(content, prefix))
metadata_update: dict[str, Any] = {"plan_mode": plan_mode_requested}
if chosen_model and chosen_effort:
if command_images and run_model and run_effort:
overrides["agent_model_id"] = run_model
overrides["agent_effort"] = run_effort
metadata_update["model"] = run_model
metadata_update["effort"] = run_effort
metadata_update["resolved_model"] = run_model
metadata_update["resolved_effort"] = run_effort
elif chosen_model and chosen_effort:
overrides["agent_model_id"] = chosen_model
overrides["agent_effort"] = chosen_effort
metadata_update["model"] = chosen_model

2
agent/webapp.py
View file

@ -27,7 +27,7 @@ from .dashboard.agent_overrides import (
)
from .dashboard.enabled_repos import is_review_repo_enabled
from .dashboard.oauth import build_settings_url
from .dashboard.options import model_supports_images # noqa: F401
from .dashboard.options import default_vision_model_pair, model_supports_images # noqa: F401
from .dashboard.profiles import ( # noqa: F401
get_profile,
get_valid_access_token,

37
agent/webhooks/linear.py
View file

@ -163,31 +163,33 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915
f"When you're done, commit and push your changes. {tag_instruction}"
)
content_blocks: list[dict[str, Any]] = [create_text_block(prompt)]
image_model_override: tuple[str, str] | None = None
if image_urls:
image_urls = webapp.dedupe_urls(image_urls)
linear_login = (
await webapp.resolve_login_from_email_async(user_email) if user_email else None
)
resolved_model_id = await webapp.resolve_agent_model_id(linear_login)
if webapp.model_supports_images(resolved_model_id):
webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
webapp.logger.debug("Image URLs: %s", image_urls)
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, client)
if image_block:
content_blocks.append(image_block)
webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks))
else:
webapp.logger.warning(
"Skipping %d image(s) for Linear issue: model %s does not support images",
if not webapp.model_supports_images(resolved_model_id):
fallback_model_id, fallback_effort = webapp.default_vision_model_pair()
webapp.logger.info(
"Using vision fallback model %s for %d Linear image(s); configured model %s "
"does not support images",
fallback_model_id,
len(image_urls),
resolved_model_id,
)
prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls))
content_blocks[0] = create_text_block(prompt)
image_urls = []
resolved_model_id = fallback_model_id
image_model_override = (fallback_model_id, fallback_effort)
webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
webapp.logger.debug("Image URLs: %s", image_urls)
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, client)
if image_block:
content_blocks.append(image_block)
webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks))
linear_project_id = ""
linear_issue_number = ""
@ -210,6 +212,9 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915
"user_email": user_email,
"source": "linear",
}
if image_model_override:
configurable["agent_model_id"] = image_model_override[0]
configurable["agent_effort"] = image_model_override[1]
await webapp.upsert_agent_thread_owner_metadata(
thread_id,

31
agent/webhooks/slack.py
View file

@ -127,24 +127,26 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st
if not mapped_login and user_email:
mapped_login = await webapp.login_for_email(user_email)
image_model_override: tuple[str, str] | None = None
if image_urls:
resolved_model_id = await webapp.resolve_agent_model_id(mapped_login)
if webapp.model_supports_images(resolved_model_id):
webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls))
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, http_client)
if image_block:
content_blocks.append(image_block)
else:
webapp.logger.warning(
"Skipping %d image(s) for Slack mention: model %s does not support images",
if not webapp.model_supports_images(resolved_model_id):
fallback_model_id, fallback_effort = webapp.default_vision_model_pair()
webapp.logger.info(
"Using vision fallback model %s for %d Slack image(s); configured model %s "
"does not support images",
fallback_model_id,
len(image_urls),
resolved_model_id,
)
prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls))
content_blocks[0] = create_text_block(prompt)
image_urls = []
resolved_model_id = fallback_model_id
image_model_override = (fallback_model_id, fallback_effort)
webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls))
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, http_client)
if image_block:
content_blocks.append(image_block)
# Open SWE opens PRs as the triggering user, so a run only proceeds when we
# have a valid user GitHub token. Users who have never signed in with
@ -207,6 +209,9 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st
}
if mapped_login:
configurable["github_login"] = mapped_login
if image_model_override:
configurable["agent_model_id"] = image_model_override[0]
configurable["agent_effort"] = image_model_override[1]
thread_plan_mode = await webapp._get_thread_plan_mode(thread_id)
if thread_plan_mode is not None:

28
tests/test_dashboard_thread_api.py
View file

@ -207,7 +207,7 @@ async def test_enrich_run_start_command_creates_and_stamps_new_thread(monkeypatc
assert enriched["params"]["assistant_id"] == "agent"
async def test_enrich_run_start_command_rejects_images_for_resolved_text_only_model(
async def test_enrich_run_start_command_uses_vision_fallback_for_text_only_model(
monkeypatch,
) -> None:
created: dict[str, object] = {}
@ -240,17 +240,23 @@ async def test_enrich_run_start_command_rejects_images_for_resolved_text_only_mo
},
}
with pytest.raises(HTTPException) as exc_info:
await thread_api._enrich_run_start_command(
"new-tid",
"octocat",
command,
metadata={},
creating=True,
)
enriched = await thread_api._enrich_run_start_command(
"new-tid",
"octocat",
command,
metadata={},
creating=True,
)
assert exc_info.value.status_code == 422
assert "does not support image input" in exc_info.value.detail
stamped = created["metadata"]
assert isinstance(stamped, dict)
assert stamped["model"] == _VISION_MODEL
assert stamped["effort"] == "medium"
assert stamped["resolved_model"] == _VISION_MODEL
assert stamped["resolved_effort"] == "medium"
configurable = enriched["params"]["config"]["configurable"]
assert configurable["agent_model_id"] == _VISION_MODEL
assert configurable["agent_effort"] == "medium"
def _thread_with_metadata(metadata: dict) -> dict:

66
tests/test_slack_context.py
View file

@ -17,6 +17,9 @@ from agent.utils.slack import (
)
from agent.webapp import generate_thread_id_from_slack_thread
_TEXT_ONLY_MODEL = "fireworks:accounts/fireworks/models/glm-5p2"
_VISION_MODEL = "openai:gpt-5.5"
class _FakeNotFoundError(Exception):
status_code = 404
@ -837,6 +840,69 @@ def test_process_slack_mention_bot_only_mode_runs_without_user_token(
assert "prompt" not in captured
def test_process_slack_mention_uses_vision_fallback_for_image_thread(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: dict[str, object] = {}
_setup_slack_mention_fakes(monkeypatch, captured)
async def fake_thread_exists(thread_id: str) -> bool:
return False
async def fake_fetch_slack_thread_messages(channel_id: str, thread_ts: str) -> list[dict]:
return [
{
"ts": "1700000000.000100",
"text": "<@UBOT> please inspect this",
"user": "U123",
"files": [
{
"mimetype": "image/png",
"url_private": "https://files.slack.com/screenshot.png",
}
],
}
]
async def fake_resolve_agent_model_id(login: str | None) -> str:
assert login == "mason-gh"
return _TEXT_ONLY_MODEL
async def fake_fetch_image_block(image_url: str, client: object) -> dict[str, str]:
captured["image_url"] = image_url
return {"type": "image", "source_type": "base64", "mime_type": "image/png", "data": "abc"}
monkeypatch.setattr(webapp, "_thread_exists", fake_thread_exists)
monkeypatch.setattr(webapp, "fetch_slack_thread_messages", fake_fetch_slack_thread_messages)
monkeypatch.setattr(webapp, "resolve_agent_model_id", fake_resolve_agent_model_id)
monkeypatch.setattr(webapp, "fetch_image_block", fake_fetch_image_block)
asyncio.run(
webapp.process_slack_mention(
{
"channel_id": "C123",
"thread_ts": "1700000000.000100",
"event_ts": "1700000000.000100",
"user_id": "U123",
"text": "<@UBOT> please inspect this",
"bot_user_id": "UBOT",
},
{"owner": "langchain-ai", "name": "open-swe"},
)
)
assert captured["image_url"] == "https://files.slack.com/screenshot.png"
run_create = captured["run_create"]
assert isinstance(run_create, dict)
kwargs = run_create["kwargs"]
configurable = kwargs["config"]["configurable"]
assert configurable["agent_model_id"] == _VISION_MODEL
assert configurable["agent_effort"] == "medium"
content = kwargs["input"]["messages"][0]["content"]
assert any(block.get("type") == "image" for block in content)
assert "does not support image input" not in content[0]["text"]
class _FakeResponse:
def __init__(self, payload: dict) -> None:
self._payload = payload