mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 17:23:15 +00:00
b19804536c
* feat: handle images sent to non-vision models in Slack, Linear, and web UI Add vision capability checks across all image input paths. When a user sends images to a text-only model (e.g. GLM 5.2, DeepSeek V4 Pro), the images are now skipped and a warning is injected into the prompt instead of sending unsupported content to the model. - Slack: resolve model at webhook time, skip image fetch + add warning - Linear: same pattern as Slack - Queued message middleware: read resolved model from thread metadata, strip images from queued payloads for text-only models - Web UI: disable submit + show inline warning when images are attached to a non-vision model selection - Shared: resolve_agent_model_id helper + vision_not_supported_warning Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * test: mock resolve_agent_model_id in Slack mention test The test_process_slack_mention_queues_active_thread_message test was missing a mock for the new resolve_agent_model_id call added to the Slack webhook handler, causing a TypeError when image URLs triggered the model resolution path. Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * fix: include vision warning in queued payload for text-only models Update the prompt variable (not just content_blocks) before clearing image_urls so the queued payload also carries the warning text when a Slack/Linear follow-up arrives while the thread is busy. Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.middleware.check_message_queue import (
|
|
DASHBOARD_HANDOFF_MARKER,
|
|
_build_blocks_from_payload,
|
|
check_message_queue_before_model,
|
|
)
|
|
|
|
|
|
class _QueuedItem:
|
|
def __init__(self, value: dict[str, Any]) -> None:
|
|
self.value = value
|
|
|
|
|
|
class _FakeStore:
|
|
def __init__(self, value: dict[str, Any]) -> None:
|
|
self.value = value
|
|
self.deleted: list[tuple[tuple[str, ...], str]] = []
|
|
|
|
async def aget(self, namespace: tuple[str, ...], key: str) -> _QueuedItem:
|
|
return _QueuedItem(self.value)
|
|
|
|
async def adelete(self, namespace: tuple[str, ...], key: str) -> None:
|
|
self.deleted.append((namespace, key))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_message_queue_injects_dashboard_handoff_instruction() -> None:
|
|
store = _FakeStore(
|
|
{
|
|
"messages": [
|
|
{"content": {"text": "continue in web", "source": "dashboard"}},
|
|
]
|
|
}
|
|
)
|
|
|
|
with (
|
|
patch(
|
|
"agent.middleware.check_message_queue.get_config",
|
|
return_value={"configurable": {"thread_id": "thread-1"}},
|
|
),
|
|
patch("agent.middleware.check_message_queue.get_store", return_value=store),
|
|
):
|
|
result = await check_message_queue_before_model.abefore_model({}, MagicMock())
|
|
|
|
assert result is not None
|
|
message = result["messages"][0]
|
|
assert message["role"] == "user"
|
|
assert DASHBOARD_HANDOFF_MARKER in message["content"][0]["text"]
|
|
assert message["content"][1] == {"type": "text", "text": "continue in web"}
|
|
assert store.deleted == [(("queue", "thread-1"), "pending_messages")]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_build_blocks_skips_images_for_text_only_model() -> None:
|
|
payload = {
|
|
"text": "see this screenshot",
|
|
"image_urls": ["https://files.slack.com/fake.png"],
|
|
}
|
|
blocks = await _build_blocks_from_payload(
|
|
payload, model_id="fireworks:accounts/fireworks/models/glm-5p2"
|
|
)
|
|
assert len(blocks) == 1
|
|
assert blocks[0]["type"] == "text"
|
|
assert "does not support image input" in blocks[0]["text"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_build_blocks_includes_images_for_vision_model() -> None:
|
|
payload: dict[str, Any] = {"text": "see this", "image_urls": []}
|
|
blocks = await _build_blocks_from_payload(payload, model_id="openai:gpt-5.5")
|
|
assert blocks == [{"type": "text", "text": "see this"}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_build_blocks_no_model_check_fetches_images() -> None:
|
|
payload: dict[str, Any] = {"text": "see this", "image_urls": []}
|
|
blocks = await _build_blocks_from_payload(payload)
|
|
assert blocks == [{"type": "text", "text": "see this"}]
|