feat: add multimodal image support for Linear comments and queued run

apps/agent/agent/middleware/check_message_queue.py

@@ -8,12 +8,16 @@ human messages before the next model call.
 from __future__ import annotations
 
 import logging
+import os
 from typing import Any
 
+import httpx
 from langchain.agents.middleware import AgentState, before_model
 from langgraph.config import get_config, get_store
 from langgraph.runtime import Runtime
 
+from ..utils.multimodal import fetch_image_block
+
 logger = logging.getLogger(__name__)
 
 
@@ -80,11 +84,55 @@ async def check_message_queue_before_model(  # noqa: PLR0911
             thread_id,
         )
 
-        content_blocks = [
+        async def _build_blocks_from_payload(
-            {"type": "text", "text": msg.get("content", "")}
+            payload: dict[str, Any],
-            for msg in queued_messages
+        ) -> list[dict[str, Any]]:
-            if msg.get("content")
+            text = payload.get("text", "")
-        ]
+            image_urls = payload.get("image_urls", []) or []
+            blocks: list[dict[str, Any]] = []
+            if text:
+                blocks.append({"type": "text", "text": text})
+
+            if not image_urls:
+                return blocks
+            linear_api_key = os.environ.get("LINEAR_API_KEY", "")
+            async with httpx.AsyncClient() as client:
+                for image_url in image_urls:
+                    headers = None
+                    if "uploads.linear.app" in image_url:
+                        if linear_api_key:
+                            headers = {"Authorization": linear_api_key}
+                        else:
+                            logger.warning(
+                                "LINEAR_API_KEY not set; cannot authenticate image fetch for %s",
+                                image_url,
+                            )
+                    image_block = await fetch_image_block(
+                        image_url, client, headers=headers
+                    )
+                    if image_block:
+                        blocks.append(image_block)
+            return blocks
+
+        content_blocks: list[dict[str, Any]] = []
+        for msg in queued_messages:
+            content = msg.get("content")
+            if isinstance(content, dict) and (
+                "text" in content or "image_urls" in content
+            ):
+                logger.debug("Queued message contains text + image URLs")
+                blocks = await _build_blocks_from_payload(content)
+                content_blocks.extend(blocks)
+                continue
+            if isinstance(content, list):
+                logger.debug(
+                    "Queued message contains %d content block(s)", len(content)
+                )
+                content_blocks.extend(content)
+                continue
+            if isinstance(content, str) and content:
+                logger.debug("Queued message contains text content")
+                content_blocks.append({"type": "text", "text": content})
 
         if not content_blocks:
             return None

apps/agent/agent/utils/multimodal.py

@@ -0,0 +1,86 @@
+"""Utilities for building multimodal content blocks."""
+
+from __future__ import annotations
+
+import base64
+import logging
+import mimetypes
+import re
+from typing import Any
+
+import httpx
+from langchain_core.messages.content import create_image_block
+
+logger = logging.getLogger(__name__)
+
+IMAGE_MARKDOWN_RE = re.compile(r"!\[[^\]]*\]\((https?://[^\s)]+)\)")
+IMAGE_URL_RE = re.compile(
+    r"(https?://[^\s)]+\.(?:png|jpe?g|gif|webp|bmp|tiff)(?:\?[^\s)]+)?)",
+    re.IGNORECASE,
+)
+
+
+def extract_image_urls(text: str) -> list[str]:
+    """Extract image URLs from markdown image syntax and direct image links."""
+    if not text:
+        return []
+
+    urls: list[str] = []
+    urls.extend(IMAGE_MARKDOWN_RE.findall(text))
+    urls.extend(IMAGE_URL_RE.findall(text))
+
+    deduped = _dedupe_urls(urls)
+    if deduped:
+        logger.debug("Extracted %d image URL(s)", len(deduped))
+    return deduped
+
+
+def strip_image_urls(text: str, image_urls: list[str]) -> str:
+    """Remove markdown image links and direct image URLs from text."""
+    if not text or not image_urls:
+        return text
+
+    logger.debug("Stripping %d image URL(s) from text", len(image_urls))
+    text = IMAGE_MARKDOWN_RE.sub("", text)
+    for url in image_urls:
+        text = text.replace(url, "")
+    return text
+
+
+async def fetch_image_block(
+    image_url: str,
+    client: httpx.AsyncClient,
+    headers: dict[str, str] | None = None,
+) -> dict[str, Any] | None:
+    """Fetch image bytes and build an image content block."""
+    try:
+        logger.debug("Fetching image from %s", image_url)
+        response = await client.get(image_url, headers=headers)
+        response.raise_for_status()
+        content_type = response.headers.get("Content-Type", "").split(";")[0].strip()
+        if not content_type:
+            guessed, _ = mimetypes.guess_type(image_url)
+            content_type = guessed or "application/octet-stream"
+
+        encoded = base64.b64encode(response.content).decode("ascii")
+        logger.info(
+            "Fetched image %s (%s, %d bytes)",
+            image_url,
+            content_type,
+            len(response.content),
+        )
+        return create_image_block(base64=encoded, mime_type=content_type)
+    except Exception:
+        logger.exception("Failed to fetch image from %s", image_url)
+        return None
+
+
+def _dedupe_urls(urls: list[str]) -> list[str]:
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for url in urls:
+        if url in seen:
+            continue
+        seen.add(url)
+        deduped.append(url)
+    return deduped

apps/agent/agent/webapp.py

@@ -15,6 +15,7 @@ from langgraph_sdk import get_client
 
 # Local import for encryption
 from .encryption import encrypt_token
+from .utils.multimodal import extract_image_urls, fetch_image_block, strip_image_urls
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +37,7 @@ LINEAR_API_KEY = os.environ.get("LINEAR_API_KEY", "")
 X_SERVICE_AUTH_JWT_SECRET = os.environ.get("X_SERVICE_AUTH_JWT_SECRET", "")
 
 
+
 def get_service_jwt_token_for_user(
     user_id: str, tenant_id: str, expiration_seconds: int = 300
 ) -> str:
@@ -131,6 +133,8 @@ def get_repo_config_from_team_mapping(
     return {"owner": "langchain-ai", "name": "langchainplus"}
 
 
+
+
 async def get_ls_user_id_from_email(email: str) -> dict[str, str | None]:
     """Get the LangSmith user ID and tenant ID from a user's email.
 
@@ -426,7 +430,9 @@ async def is_thread_active(thread_id: str) -> bool:
     return status == "busy"
 
 
-async def queue_message_for_thread(thread_id: str, message_content: str) -> bool:
+async def queue_message_for_thread(
+    thread_id: str, message_content: str | list[dict[str, Any]] | dict[str, Any]
+) -> bool:
     """Queue a message for a thread that is currently active.
 
     Stores the message in the langgraph store, namespaced to the thread.
@@ -435,7 +441,7 @@ async def queue_message_for_thread(thread_id: str, message_content: str) -> bool
 
     Args:
         thread_id: The LangGraph thread ID
-        message_content: The message content to queue
+        message_content: The message content to queue (text or content blocks)
 
     Returns:
         True if successfully queued, False otherwise
@@ -571,9 +577,20 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
 
     title = full_issue.get("title", "No title")
     description = full_issue.get("description") or "No description"
+    image_urls: list[str] = []
+    description_image_urls = extract_image_urls(description)
+    if description_image_urls:
+        image_urls.extend(description_image_urls)
+        description = strip_image_urls(description, description_image_urls)
+        logger.debug(
+            "Found %d image URL(s) in issue description",
+            len(description_image_urls),
+        )
 
     comments = full_issue.get("comments", {}).get("nodes", [])
     comments_text = ""
+    triggering_comment = issue_data.get("triggering_comment", "")
+    triggering_comment_id = issue_data.get("triggering_comment_id", "")
 
     bot_message_prefixes = (
         "🔐 **GitHub Authentication Required**",
@@ -585,32 +602,77 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
         "❌ **Agent Error**",
     )
 
+    comment_ids: set[str] = set()
+    comment_id_to_index: dict[str, int] = {}
     if comments:
         last_bot_comment_idx = -1
         for i, comment in enumerate(comments):
+            comment_id = comment.get("id", "")
+            if comment_id:
+                comment_ids.add(comment_id)
+                comment_id_to_index[comment_id] = i
             body = comment.get("body", "")
             if any(body.startswith(prefix) for prefix in bot_message_prefixes):
                 last_bot_comment_idx = i
 
         relevant_comments = []
-        for i, comment in enumerate(comments):
+        trigger_index = None
-            if i <= last_bot_comment_idx:
+        if triggering_comment_id:
-                continue
+            trigger_index = comment_id_to_index.get(triggering_comment_id)
-            body = comment.get("body", "")
+        if trigger_index is not None:
-            if "@openswe" in body.lower():
+            relevant_comments = comments[trigger_index:]
-                relevant_comments.append(comment)
+            logger.debug(
-                relevant_comments.extend(comments[i + 1 :])
+                "Using triggering comment index %d to build relevant comments",
-                break
+                trigger_index,
+            )
+        else:
+            for i, comment in enumerate(comments):
+                if i <= last_bot_comment_idx:
+                    continue
+                body = comment.get("body", "")
+                if "@openswe" in body.lower():
+                    relevant_comments.append(comment)
+                    relevant_comments.extend(comments[i + 1 :])
+                    break
 
         if relevant_comments:
             comments_text = "\n\n## Comments:\n"
             for comment in relevant_comments:
                 author = comment.get("user", {}).get("name", "Unknown")
                 body = comment.get("body", "")
+                body_image_urls = extract_image_urls(body)
+                if body_image_urls:
+                    image_urls.extend(body_image_urls)
+                    body = strip_image_urls(body, body_image_urls)
+                    logger.debug(
+                        "Found %d image URL(s) in comment by %s",
+                        len(body_image_urls),
+                        author,
+                    )
                 if any(body.startswith(prefix) for prefix in bot_message_prefixes):
                     continue
                 comments_text += f"\n**{author}:** {body}\n"
 
+    if triggering_comment and triggering_comment_id not in comment_ids:
+        if not comments_text:
+            comments_text = "\n\n## Comments:\n"
+        trigger_author = comment_author.get("name", "Unknown")
+        trigger_body = triggering_comment
+        trigger_image_urls = extract_image_urls(trigger_body)
+        if trigger_image_urls:
+            image_urls.extend(trigger_image_urls)
+            trigger_body = strip_image_urls(trigger_body, trigger_image_urls)
+            logger.debug(
+                "Found %d image URL(s) in triggering comment by %s",
+                len(trigger_image_urls),
+                trigger_author,
+            )
+        comments_text += f"\n**{trigger_author}:** {trigger_body}\n"
+        logger.debug(
+            "Appended triggering comment %s not present in issue comments list",
+            triggering_comment_id or "<missing-id>",
+        )
+
     prompt = (
         f"Please work on the following issue:\n\n"
         f"## Title: {title}\n\n"
@@ -619,6 +681,34 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
         "Please analyze this issue and implement the necessary changes. "
         "When you're done, commit and push your changes."
     )
+    content_blocks: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
+    if image_urls:
+        seen_urls: set[str] = set()
+        deduped_urls: list[str] = []
+        for url in image_urls:
+            if url in seen_urls:
+                continue
+            seen_urls.add(url)
+            deduped_urls.append(url)
+        image_urls = deduped_urls
+        logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
+        logger.debug("Image URLs: %s", image_urls)
+
+        async with httpx.AsyncClient() as client:
+            for image_url in image_urls:
+                headers = None
+                if "uploads.linear.app" in image_url:
+                    if LINEAR_API_KEY:
+                        headers = {"Authorization": LINEAR_API_KEY}
+                    else:
+                        logger.warning(
+                            "LINEAR_API_KEY not set; cannot authenticate image fetch for %s",
+                            image_url,
+                        )
+                image_block = await fetch_image_block(image_url, client, headers=headers)
+                if image_block:
+                    content_blocks.append(image_block)
+        logger.info("Built %d content block(s) for prompt", len(content_blocks))
 
     identifier = full_issue.get("identifier", "") or issue_data.get("identifier", "")
     linear_project_id = ""
@@ -652,9 +742,10 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
                 thread_id,
             )
 
+            queued_payload = {"text": prompt, "image_urls": image_urls}
             queued = await queue_message_for_thread(
                 thread_id=thread_id,
-                message_content=prompt,
+                message_content=queued_payload,
             )
 
             if queued:
@@ -669,7 +760,7 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
             await langgraph_client.runs.create(
                 thread_id,
                 "agent",
-                input={"messages": [{"role": "user", "content": prompt}]},
+                input={"messages": [{"role": "user", "content": content_blocks}]},
                 config={"configurable": configurable},
                 if_not_exists="create",
             )