adam/open-swe
1
0
Fork 0
mirror of https://github.com/Sea-Haven-Industries/open-swe.git synced 2026-10-07 16:19:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(webhooks): fall back to vision model for Slack/Linear image threads

Browse source 
Re-land upstream #1626 onto the modular webhook structure. When a
Slack mention or Linear issue carries images but the resolved model is
text-only, fall back to a vision-capable model instead of dropping the
images. Re-points default_vision_model_pair at the fork's image-capable
models (Opus 4.8 default, else any supports_images model) rather than
upstream's openai:/anthropic: provider filter.

Refs #80, upstream #1626
This commit is contained in:
Adam Moussa 2026-07-08 16:41:51 -04:00
parent 2a226e2cd1
commit ea7da4220a
No known key found for this signature in database
5 changed files with 114 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
agent/dashboard/options.py
View file

@ -158,15 +158,20 @@ def default_model_pair() -> tuple[str, str]:
def default_vision_model_pair() -> tuple[str, str]:
"""Default OpenAI/Anthropic model pair to use when image input is required."""
"""Default (model_id, reasoning_effort) to use when image input is required.
Prefers the configured default model when it is vision-capable (Opus 4.8),
otherwise the first vision-capable supported model. The fork ships only
Bedrock/Fireworks models, so this selects on ``supports_images`` rather than
upstream's ``openai:``/``anthropic:`` provider filter.
"""
if (
DEFAULT_MODEL_ID in SUPPORTED_MODEL_IDS
and model_supports_images(DEFAULT_MODEL_ID)
and model_supports_effort(DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT)
and DEFAULT_MODEL_ID.startswith(("openai:", "anthropic:"))
):
return DEFAULT_MODEL_ID, DEFAULT_MODEL_EFFORT
for model in SUPPORTED_MODELS:
if model["id"].startswith(("openai:", "anthropic:")) and model["supports_images"]:
if model["supports_images"]:
return model["id"], model["default_effort"]
return default_model_pair()

2
agent/webapp.py
View file

@ -27,7 +27,7 @@ from .dashboard.agent_overrides import (
)
from .dashboard.enabled_repos import is_review_repo_enabled
from .dashboard.oauth import build_settings_url
from .dashboard.options import model_supports_images # noqa: F401
from .dashboard.options import default_vision_model_pair, model_supports_images # noqa: F401
from .dashboard.profiles import ( # noqa: F401
get_profile,
get_valid_access_token,

37
agent/webhooks/linear.py
View file

@ -163,31 +163,33 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915
f"When you're done, commit and push your changes. {tag_instruction}"
)
content_blocks: list[dict[str, Any]] = [create_text_block(prompt)]
image_model_override: tuple[str, str] | None = None
if image_urls:
image_urls = webapp.dedupe_urls(image_urls)
linear_login = (
await webapp.resolve_login_from_email_async(user_email) if user_email else None
)
resolved_model_id = await webapp.resolve_agent_model_id(linear_login)
if webapp.model_supports_images(resolved_model_id):
webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
webapp.logger.debug("Image URLs: %s", image_urls)
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, client)
if image_block:
content_blocks.append(image_block)
webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks))
else:
webapp.logger.warning(
"Skipping %d image(s) for Linear issue: model %s does not support images",
if not webapp.model_supports_images(resolved_model_id):
fallback_model_id, fallback_effort = webapp.default_vision_model_pair()
webapp.logger.info(
"Using vision fallback model %s for %d Linear image(s); configured model %s "
"does not support images",
fallback_model_id,
len(image_urls),
resolved_model_id,
)
prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls))
content_blocks[0] = create_text_block(prompt)
image_urls = []
resolved_model_id = fallback_model_id
image_model_override = (fallback_model_id, fallback_effort)
webapp.logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
webapp.logger.debug("Image URLs: %s", image_urls)
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, client)
if image_block:
content_blocks.append(image_block)
webapp.logger.info("Built %d content block(s) for prompt", len(content_blocks))
linear_project_id = ""
linear_issue_number = ""
@ -210,6 +212,9 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915
"user_email": user_email,
"source": "linear",
}
if image_model_override:
configurable["agent_model_id"] = image_model_override[0]
configurable["agent_effort"] = image_model_override[1]
await webapp.upsert_agent_thread_owner_metadata(
thread_id,

31
agent/webhooks/slack.py
View file

@ -127,24 +127,26 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st
if not mapped_login and user_email:
mapped_login = await webapp.login_for_email(user_email)
image_model_override: tuple[str, str] | None = None
if image_urls:
resolved_model_id = await webapp.resolve_agent_model_id(mapped_login)
if webapp.model_supports_images(resolved_model_id):
webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls))
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, http_client)
if image_block:
content_blocks.append(image_block)
else:
webapp.logger.warning(
"Skipping %d image(s) for Slack mention: model %s does not support images",
if not webapp.model_supports_images(resolved_model_id):
fallback_model_id, fallback_effort = webapp.default_vision_model_pair()
webapp.logger.info(
"Using vision fallback model %s for %d Slack image(s); configured model %s "
"does not support images",
fallback_model_id,
len(image_urls),
resolved_model_id,
)
prompt += webapp.vision_not_supported_warning(resolved_model_id, len(image_urls))
content_blocks[0] = create_text_block(prompt)
image_urls = []
resolved_model_id = fallback_model_id
image_model_override = (fallback_model_id, fallback_effort)
webapp.logger.info("Preparing %d image(s) for Slack mention", len(image_urls))
async with httpx.AsyncClient(timeout=webapp.DEFAULT_HTTP_TIMEOUT) as http_client:
for image_url in image_urls:
image_block = await webapp.fetch_image_block(image_url, http_client)
if image_block:
content_blocks.append(image_block)
# Open SWE opens PRs as the triggering user, so a run only proceeds when we
# have a valid user GitHub token. Users who have never signed in with
@ -207,6 +209,9 @@ async def process_slack_mention(event_data: dict[str, Any], repo_config: dict[st
}
if mapped_login:
configurable["github_login"] = mapped_login
if image_model_override:
configurable["agent_model_id"] = image_model_override[0]
configurable["agent_effort"] = image_model_override[1]
thread_plan_mode = await webapp._get_thread_plan_mode(thread_id)
if thread_plan_mode is not None:

66
tests/test_slack_context.py
View file

@ -17,6 +17,9 @@ from agent.utils.slack import (
)
from agent.webapp import generate_thread_id_from_slack_thread
_TEXT_ONLY_MODEL = "fireworks:accounts/fireworks/models/glm-5p2"
_VISION_MODEL = "bedrock_converse:us.anthropic.claude-opus-4-8"
class _FakeNotFoundError(Exception):
status_code = 404
@ -572,6 +575,69 @@ def test_process_slack_mention_creates_thread_first_run_with_trace_reply(
assert "## Latest Mention Request\ncontinue on the branch" in prompt_block["text"]
def test_process_slack_mention_uses_vision_fallback_for_image_thread(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: dict[str, object] = {}
_setup_slack_mention_fakes(monkeypatch, captured)
async def fake_thread_exists(thread_id: str) -> bool:
return False
async def fake_fetch_slack_thread_messages(channel_id: str, thread_ts: str) -> list[dict]:
return [
{
"ts": "1700000000.000100",
"text": "<@UBOT> please inspect this",
"user": "U123",
"files": [
{
"mimetype": "image/png",
"url_private": "https://files.slack.com/screenshot.png",
}
],
}
]
async def fake_resolve_agent_model_id(login: str | None) -> str:
assert login == "mason-gh"
return _TEXT_ONLY_MODEL
async def fake_fetch_image_block(image_url: str, client: object) -> dict[str, str]:
captured["image_url"] = image_url
return {"type": "image", "source_type": "base64", "mime_type": "image/png", "data": "abc"}
monkeypatch.setattr(webapp, "_thread_exists", fake_thread_exists)
monkeypatch.setattr(webapp, "fetch_slack_thread_messages", fake_fetch_slack_thread_messages)
monkeypatch.setattr(webapp, "resolve_agent_model_id", fake_resolve_agent_model_id)
monkeypatch.setattr(webapp, "fetch_image_block", fake_fetch_image_block)
asyncio.run(
webapp.process_slack_mention(
{
"channel_id": "C123",
"thread_ts": "1700000000.000100",
"event_ts": "1700000000.000100",
"user_id": "U123",
"text": "<@UBOT> please inspect this",
"bot_user_id": "UBOT",
},
{"owner": "langchain-ai", "name": "open-swe"},
)
)
assert captured["image_url"] == "https://files.slack.com/screenshot.png"
run_create = captured["run_create"]
assert isinstance(run_create, dict)
kwargs = run_create["kwargs"]
configurable = kwargs["config"]["configurable"]
assert configurable["agent_model_id"] == _VISION_MODEL
assert configurable["agent_effort"] == "medium"
content = kwargs["input"]["messages"][0]["content"]
assert any(block.get("type") == "image" for block in content)
assert "does not support image input" not in content[0]["text"]
def test_process_slack_mention_skips_trace_reply_on_followup_mention(
monkeypatch: pytest.MonkeyPatch,
) -> None: