From bec99b106ea97164ed2dd6bccac993a31ad41955 Mon Sep 17 00:00:00 2001 From: aran-yogesh Date: Thu, 19 Feb 2026 14:06:06 -0800 Subject: [PATCH] feat: add multimodal image support for Linear comments and queued run --- .../agent/middleware/check_message_queue.py | 58 ++++++++- apps/agent/agent/utils/multimodal.py | 86 +++++++++++++ apps/agent/agent/webapp.py | 115 ++++++++++++++++-- 3 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 apps/agent/agent/utils/multimodal.py diff --git a/apps/agent/agent/middleware/check_message_queue.py b/apps/agent/agent/middleware/check_message_queue.py index b85248fd..89cc49a3 100644 --- a/apps/agent/agent/middleware/check_message_queue.py +++ b/apps/agent/agent/middleware/check_message_queue.py @@ -8,12 +8,16 @@ human messages before the next model call. from __future__ import annotations import logging +import os from typing import Any +import httpx from langchain.agents.middleware import AgentState, before_model from langgraph.config import get_config, get_store from langgraph.runtime import Runtime +from ..utils.multimodal import fetch_image_block + logger = logging.getLogger(__name__) @@ -80,11 +84,55 @@ async def check_message_queue_before_model( # noqa: PLR0911 thread_id, ) - content_blocks = [ - {"type": "text", "text": msg.get("content", "")} - for msg in queued_messages - if msg.get("content") - ] + async def _build_blocks_from_payload( + payload: dict[str, Any], + ) -> list[dict[str, Any]]: + text = payload.get("text", "") + image_urls = payload.get("image_urls", []) or [] + blocks: list[dict[str, Any]] = [] + if text: + blocks.append({"type": "text", "text": text}) + + if not image_urls: + return blocks + linear_api_key = os.environ.get("LINEAR_API_KEY", "") + async with httpx.AsyncClient() as client: + for image_url in image_urls: + headers = None + if "uploads.linear.app" in image_url: + if linear_api_key: + headers = {"Authorization": linear_api_key} + else: + logger.warning( + "LINEAR_API_KEY not set; cannot authenticate image fetch for %s", + image_url, + ) + image_block = await fetch_image_block( + image_url, client, headers=headers + ) + if image_block: + blocks.append(image_block) + return blocks + + content_blocks: list[dict[str, Any]] = [] + for msg in queued_messages: + content = msg.get("content") + if isinstance(content, dict) and ( + "text" in content or "image_urls" in content + ): + logger.debug("Queued message contains text + image URLs") + blocks = await _build_blocks_from_payload(content) + content_blocks.extend(blocks) + continue + if isinstance(content, list): + logger.debug( + "Queued message contains %d content block(s)", len(content) + ) + content_blocks.extend(content) + continue + if isinstance(content, str) and content: + logger.debug("Queued message contains text content") + content_blocks.append({"type": "text", "text": content}) if not content_blocks: return None diff --git a/apps/agent/agent/utils/multimodal.py b/apps/agent/agent/utils/multimodal.py new file mode 100644 index 00000000..a8cb05cb --- /dev/null +++ b/apps/agent/agent/utils/multimodal.py @@ -0,0 +1,86 @@ +"""Utilities for building multimodal content blocks.""" + +from __future__ import annotations + +import base64 +import logging +import mimetypes +import re +from typing import Any + +import httpx +from langchain_core.messages.content import create_image_block + +logger = logging.getLogger(__name__) + +IMAGE_MARKDOWN_RE = re.compile(r"!\[[^\]]*\]\((https?://[^\s)]+)\)") +IMAGE_URL_RE = re.compile( + r"(https?://[^\s)]+\.(?:png|jpe?g|gif|webp|bmp|tiff)(?:\?[^\s)]+)?)", + re.IGNORECASE, +) + + +def extract_image_urls(text: str) -> list[str]: + """Extract image URLs from markdown image syntax and direct image links.""" + if not text: + return [] + + urls: list[str] = [] + urls.extend(IMAGE_MARKDOWN_RE.findall(text)) + urls.extend(IMAGE_URL_RE.findall(text)) + + deduped = _dedupe_urls(urls) + if deduped: + logger.debug("Extracted %d image URL(s)", len(deduped)) + return deduped + + +def strip_image_urls(text: str, image_urls: list[str]) -> str: + """Remove markdown image links and direct image URLs from text.""" + if not text or not image_urls: + return text + + logger.debug("Stripping %d image URL(s) from text", len(image_urls)) + text = IMAGE_MARKDOWN_RE.sub("", text) + for url in image_urls: + text = text.replace(url, "") + return text + + +async def fetch_image_block( + image_url: str, + client: httpx.AsyncClient, + headers: dict[str, str] | None = None, +) -> dict[str, Any] | None: + """Fetch image bytes and build an image content block.""" + try: + logger.debug("Fetching image from %s", image_url) + response = await client.get(image_url, headers=headers) + response.raise_for_status() + content_type = response.headers.get("Content-Type", "").split(";")[0].strip() + if not content_type: + guessed, _ = mimetypes.guess_type(image_url) + content_type = guessed or "application/octet-stream" + + encoded = base64.b64encode(response.content).decode("ascii") + logger.info( + "Fetched image %s (%s, %d bytes)", + image_url, + content_type, + len(response.content), + ) + return create_image_block(base64=encoded, mime_type=content_type) + except Exception: + logger.exception("Failed to fetch image from %s", image_url) + return None + + +def _dedupe_urls(urls: list[str]) -> list[str]: + seen: set[str] = set() + deduped: list[str] = [] + for url in urls: + if url in seen: + continue + seen.add(url) + deduped.append(url) + return deduped diff --git a/apps/agent/agent/webapp.py b/apps/agent/agent/webapp.py index de40a6da..c3b308c5 100644 --- a/apps/agent/agent/webapp.py +++ b/apps/agent/agent/webapp.py @@ -15,6 +15,7 @@ from langgraph_sdk import get_client # Local import for encryption from .encryption import encrypt_token +from .utils.multimodal import extract_image_urls, fetch_image_block, strip_image_urls logger = logging.getLogger(__name__) @@ -36,6 +37,7 @@ LINEAR_API_KEY = os.environ.get("LINEAR_API_KEY", "") X_SERVICE_AUTH_JWT_SECRET = os.environ.get("X_SERVICE_AUTH_JWT_SECRET", "") + def get_service_jwt_token_for_user( user_id: str, tenant_id: str, expiration_seconds: int = 300 ) -> str: @@ -131,6 +133,8 @@ def get_repo_config_from_team_mapping( return {"owner": "langchain-ai", "name": "langchainplus"} + + async def get_ls_user_id_from_email(email: str) -> dict[str, str | None]: """Get the LangSmith user ID and tenant ID from a user's email. @@ -426,7 +430,9 @@ async def is_thread_active(thread_id: str) -> bool: return status == "busy" -async def queue_message_for_thread(thread_id: str, message_content: str) -> bool: +async def queue_message_for_thread( + thread_id: str, message_content: str | list[dict[str, Any]] | dict[str, Any] +) -> bool: """Queue a message for a thread that is currently active. Stores the message in the langgraph store, namespaced to the thread. @@ -435,7 +441,7 @@ async def queue_message_for_thread(thread_id: str, message_content: str) -> bool Args: thread_id: The LangGraph thread ID - message_content: The message content to queue + message_content: The message content to queue (text or content blocks) Returns: True if successfully queued, False otherwise @@ -571,9 +577,20 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 title = full_issue.get("title", "No title") description = full_issue.get("description") or "No description" + image_urls: list[str] = [] + description_image_urls = extract_image_urls(description) + if description_image_urls: + image_urls.extend(description_image_urls) + description = strip_image_urls(description, description_image_urls) + logger.debug( + "Found %d image URL(s) in issue description", + len(description_image_urls), + ) comments = full_issue.get("comments", {}).get("nodes", []) comments_text = "" + triggering_comment = issue_data.get("triggering_comment", "") + triggering_comment_id = issue_data.get("triggering_comment_id", "") bot_message_prefixes = ( "🔐 **GitHub Authentication Required**", @@ -585,32 +602,77 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 "❌ **Agent Error**", ) + comment_ids: set[str] = set() + comment_id_to_index: dict[str, int] = {} if comments: last_bot_comment_idx = -1 for i, comment in enumerate(comments): + comment_id = comment.get("id", "") + if comment_id: + comment_ids.add(comment_id) + comment_id_to_index[comment_id] = i body = comment.get("body", "") if any(body.startswith(prefix) for prefix in bot_message_prefixes): last_bot_comment_idx = i relevant_comments = [] - for i, comment in enumerate(comments): - if i <= last_bot_comment_idx: - continue - body = comment.get("body", "") - if "@openswe" in body.lower(): - relevant_comments.append(comment) - relevant_comments.extend(comments[i + 1 :]) - break + trigger_index = None + if triggering_comment_id: + trigger_index = comment_id_to_index.get(triggering_comment_id) + if trigger_index is not None: + relevant_comments = comments[trigger_index:] + logger.debug( + "Using triggering comment index %d to build relevant comments", + trigger_index, + ) + else: + for i, comment in enumerate(comments): + if i <= last_bot_comment_idx: + continue + body = comment.get("body", "") + if "@openswe" in body.lower(): + relevant_comments.append(comment) + relevant_comments.extend(comments[i + 1 :]) + break if relevant_comments: comments_text = "\n\n## Comments:\n" for comment in relevant_comments: author = comment.get("user", {}).get("name", "Unknown") body = comment.get("body", "") + body_image_urls = extract_image_urls(body) + if body_image_urls: + image_urls.extend(body_image_urls) + body = strip_image_urls(body, body_image_urls) + logger.debug( + "Found %d image URL(s) in comment by %s", + len(body_image_urls), + author, + ) if any(body.startswith(prefix) for prefix in bot_message_prefixes): continue comments_text += f"\n**{author}:** {body}\n" + if triggering_comment and triggering_comment_id not in comment_ids: + if not comments_text: + comments_text = "\n\n## Comments:\n" + trigger_author = comment_author.get("name", "Unknown") + trigger_body = triggering_comment + trigger_image_urls = extract_image_urls(trigger_body) + if trigger_image_urls: + image_urls.extend(trigger_image_urls) + trigger_body = strip_image_urls(trigger_body, trigger_image_urls) + logger.debug( + "Found %d image URL(s) in triggering comment by %s", + len(trigger_image_urls), + trigger_author, + ) + comments_text += f"\n**{trigger_author}:** {trigger_body}\n" + logger.debug( + "Appended triggering comment %s not present in issue comments list", + triggering_comment_id or "", + ) + prompt = ( f"Please work on the following issue:\n\n" f"## Title: {title}\n\n" @@ -619,6 +681,34 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 "Please analyze this issue and implement the necessary changes. " "When you're done, commit and push your changes." ) + content_blocks: list[dict[str, Any]] = [{"type": "text", "text": prompt}] + if image_urls: + seen_urls: set[str] = set() + deduped_urls: list[str] = [] + for url in image_urls: + if url in seen_urls: + continue + seen_urls.add(url) + deduped_urls.append(url) + image_urls = deduped_urls + logger.info("Preparing %d image(s) for multimodal content", len(image_urls)) + logger.debug("Image URLs: %s", image_urls) + + async with httpx.AsyncClient() as client: + for image_url in image_urls: + headers = None + if "uploads.linear.app" in image_url: + if LINEAR_API_KEY: + headers = {"Authorization": LINEAR_API_KEY} + else: + logger.warning( + "LINEAR_API_KEY not set; cannot authenticate image fetch for %s", + image_url, + ) + image_block = await fetch_image_block(image_url, client, headers=headers) + if image_block: + content_blocks.append(image_block) + logger.info("Built %d content block(s) for prompt", len(content_blocks)) identifier = full_issue.get("identifier", "") or issue_data.get("identifier", "") linear_project_id = "" @@ -652,9 +742,10 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 thread_id, ) + queued_payload = {"text": prompt, "image_urls": image_urls} queued = await queue_message_for_thread( thread_id=thread_id, - message_content=prompt, + message_content=queued_payload, ) if queued: @@ -669,7 +760,7 @@ async def process_linear_issue( # noqa: PLR0912, PLR0915 await langgraph_client.runs.create( thread_id, "agent", - input={"messages": [{"role": "user", "content": prompt}]}, + input={"messages": [{"role": "user", "content": content_blocks}]}, config={"configurable": configurable}, if_not_exists="create", )