mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 09:13:14 +00:00
b19804536c
* feat: handle images sent to non-vision models in Slack, Linear, and web UI Add vision capability checks across all image input paths. When a user sends images to a text-only model (e.g. GLM 5.2, DeepSeek V4 Pro), the images are now skipped and a warning is injected into the prompt instead of sending unsupported content to the model. - Slack: resolve model at webhook time, skip image fetch + add warning - Linear: same pattern as Slack - Queued message middleware: read resolved model from thread metadata, strip images from queued payloads for text-only models - Web UI: disable submit + show inline warning when images are attached to a non-vision model selection - Shared: resolve_agent_model_id helper + vision_not_supported_warning Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * test: mock resolve_agent_model_id in Slack mention test The test_process_slack_mention_queues_active_thread_message test was missing a mock for the new resolve_agent_model_id call added to the Slack webhook handler, causing a TypeError when image URLs triggered the model resolution path. Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * fix: include vision warning in queued payload for text-only models Update the prompt variable (not just content_blocks) before clearing image_urls so the queued payload also carries the warning text when a Slack/Linear follow-up arrives while the thread is busy. Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
110 lines
3.5 KiB
Python
110 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
from agent.utils.multimodal import extract_image_urls, vision_not_supported_warning
|
|
|
|
|
|
def test_extract_image_urls_empty() -> None:
|
|
assert extract_image_urls("") == []
|
|
|
|
|
|
def test_extract_image_urls_markdown_and_direct_dedupes() -> None:
|
|
text = (
|
|
"Here is an image  and another "
|
|
"![https://example.com/b.JPG?size=large plus a repeat https://example.com/a.png"
|
|
)
|
|
|
|
assert extract_image_urls(text) == [
|
|
"https://example.com/a.png",
|
|
"https://example.com/b.JPG?size=large",
|
|
]
|
|
|
|
|
|
def test_extract_image_urls_ignores_non_images() -> None:
|
|
text = "Not images: https://example.com/file.pdf and https://example.com/noext"
|
|
|
|
assert extract_image_urls(text) == []
|
|
|
|
|
|
def test_extract_image_urls_markdown_syntax() -> None:
|
|
text = "Check out this screenshot: "
|
|
|
|
assert extract_image_urls(text) == ["https://example.com/screenshot.png"]
|
|
|
|
|
|
def test_extract_image_urls_direct_links() -> None:
|
|
text = "Direct link: https://example.com/photo.jpg and another https://example.com/image.gif"
|
|
|
|
assert extract_image_urls(text) == [
|
|
"https://example.com/photo.jpg",
|
|
"https://example.com/image.gif",
|
|
]
|
|
|
|
|
|
def test_extract_image_urls_various_formats() -> None:
|
|
text = (
|
|
"Multiple formats: "
|
|
"https://example.com/image.png "
|
|
"https://example.com/photo.jpeg "
|
|
"https://example.com/pic.gif "
|
|
"https://example.com/img.webp "
|
|
"https://example.com/bitmap.bmp "
|
|
"https://example.com/scan.tiff"
|
|
)
|
|
|
|
assert extract_image_urls(text) == [
|
|
"https://example.com/image.png",
|
|
"https://example.com/photo.jpeg",
|
|
"https://example.com/pic.gif",
|
|
"https://example.com/img.webp",
|
|
"https://example.com/bitmap.bmp",
|
|
"https://example.com/scan.tiff",
|
|
]
|
|
|
|
|
|
def test_extract_image_urls_with_query_params() -> None:
|
|
text = "Image with params: https://cdn.example.com/image.png?width=800&height=600"
|
|
|
|
assert extract_image_urls(text) == ["https://cdn.example.com/image.png?width=800&height=600"]
|
|
|
|
|
|
def test_extract_image_urls_case_insensitive() -> None:
|
|
text = "Mixed case: https://example.com/Image.PNG and https://example.com/photo.JpEg"
|
|
|
|
assert extract_image_urls(text) == [
|
|
"https://example.com/Image.PNG",
|
|
"https://example.com/photo.JpEg",
|
|
]
|
|
|
|
|
|
def test_extract_image_urls_deduplication() -> None:
|
|
text = "Same URL twice: https://example.com/image.png and again https://example.com/image.png"
|
|
|
|
assert extract_image_urls(text) == ["https://example.com/image.png"]
|
|
|
|
|
|
def test_extract_image_urls_mixed_markdown_and_direct() -> None:
|
|
text = (
|
|
"Markdown:  "
|
|
"and direct: https://example.com/direct.jpg "
|
|
"and another markdown "
|
|
)
|
|
|
|
result = extract_image_urls(text)
|
|
assert set(result) == {
|
|
"https://example.com/markdown.png",
|
|
"https://example.com/direct.jpg",
|
|
"https://example.com/another.gif",
|
|
}
|
|
assert len(result) == 3
|
|
|
|
|
|
def test_vision_not_supported_warning_includes_model_and_count() -> None:
|
|
warning = vision_not_supported_warning("fireworks:.../glm-5p2", 2)
|
|
assert "glm-5p2" in warning
|
|
assert "2 image(s)" in warning
|
|
assert "does not support image input" in warning
|
|
|
|
|
|
def test_vision_not_supported_warning_singular() -> None:
|
|
warning = vision_not_supported_warning("fireworks:.../glm-5p2", 1)
|
|
assert "1 image(s)" in warning
|