Merge branch 'open-swe-v3' into yogesh/followup-comments

.github/workflows/agent-ci.yml

@@ -0,0 +1,61 @@
+name: Agent CI
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "apps/agent/**"
+  pull_request:
+    paths:
+      - "apps/agent/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Agent lint
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: apps/agent
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Run lint
+        run: make lint
+
+  format:
+    name: Agent format check
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: apps/agent
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Run format check
+        run: make format-check
+
+  unit-tests:
+    name: Agent unit tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: apps/agent
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --locked --extra dev
+      - name: Run unit tests
+        run: make test

.github/workflows/ci.yml

@@ -2,10 +2,23 @@
 
 name: CI
 
+permissions:
+  contents: read
+
 on:
   push:
     branches: ["main"]
+    paths:
+      - "apps/web/**"
+      - "packages/shared/**"
+      - "*"
+      - ".*"
   pull_request:
+    paths:
+      - "apps/web/**"
+      - "packages/shared/**"
+      - "*"
+      - ".*"
   workflow_dispatch: # Allows triggering the workflow manually in GitHub UI
 
 # If another push to the same PR or branch happens while this workflow is still running,

.github/workflows/unit-tests.yml

@@ -6,7 +6,17 @@ permissions:
 on:
   push:
     branches: ["main"]
+    paths:
+      - "apps/web/**"
+      - "packages/shared/**"
+      - "*"
+      - ".*"
   pull_request:
+    paths:
+      - "apps/web/**"
+      - "packages/shared/**"
+      - "*"
+      - ".*"
   workflow_dispatch: # Allows triggering the workflow manually in GitHub UI
 
 # If another push to the same PR or branch happens while this workflow is still running,

apps/agent/Makefile

@@ -1,4 +1,4 @@
-.PHONY: all format lint test tests integration_tests help run dev
+.PHONY: all format format-check lint test tests integration_tests help run dev
 
 # Default target executed when no arguments are given to make.
 all: help
@@ -23,10 +23,18 @@ install:
 TEST_FILE ?= tests/
 
 test tests:
-	uv run pytest -vvv $(TEST_FILE)
+	@if [ -d "$(TEST_FILE)" ] || [ -f "$(TEST_FILE)" ]; then \
+		uv run pytest -vvv $(TEST_FILE); \
+	else \
+		echo "Skipping tests: path not found: $(TEST_FILE)"; \
+	fi
 
 integration_tests:
-	uv run pytest -vvv tests/integration_tests/
+	@if [ -d "tests/integration_tests/" ] || [ -f "tests/integration_tests/" ]; then \
+		uv run pytest -vvv tests/integration_tests/; \
+	else \
+		echo "Skipping integration tests: path not found: tests/integration_tests/"; \
+	fi
 
 ######################
 # LINTING AND FORMATTING
@@ -42,6 +50,9 @@ format:
 	uv run ruff format $(PYTHON_FILES)
 	uv run ruff check --fix $(PYTHON_FILES)
 
+format-check:
+	uv run ruff format $(PYTHON_FILES) --check
+
 ######################
 # HELP
 ######################

apps/agent/agent/integrations/langsmith.py

@@ -43,7 +43,7 @@ def _get_sandbox_template_config() -> tuple[str | None, str | None]:
     return template_name, template_image
 
 
-def _create_langsmith_sandbox(
+def create_langsmith_sandbox(
     sandbox_id: str | None = None,
 ) -> SandboxBackendProtocol:
     """Create or connect to a LangSmith sandbox without automatic cleanup.
@@ -108,17 +108,28 @@ class LangSmithBackend(BaseSandbox):
 
     def __init__(self, sandbox: Sandbox) -> None:
         self._sandbox = sandbox
-        self._timeout: int = 30 * 60  # 30 mins default
+        self._default_timeout: int = 30 * 5  # 5 minute default
 
     @property
     def id(self) -> str:
         """Unique identifier for the sandbox backend."""
         return self._sandbox.name
 
-    def execute(self, command: str) -> ExecuteResponse:
+    def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse:
-        """Execute a command in the sandbox and return ExecuteResponse."""
+        """Execute a command in the sandbox and return ExecuteResponse.
-        result = self._sandbox.run(command, timeout=self._timeout)
 
+        Args:
+            command: Full shell command string to execute.
+            timeout: Maximum time in seconds to wait for the command to complete.
+                If None, uses the default timeout of 5 minutes.
+
+        Returns:
+            ExecuteResponse with combined output, exit code, and truncation flag.
+        """
+        effective_timeout = timeout if timeout is not None else self._default_timeout
+        result = self._sandbox.run(command, timeout=effective_timeout)
+
+        # Combine stdout and stderr (matching other backends' approach)
         output = result.stdout or ""
         if result.stderr:
             output += "\n" + result.stderr if output else result.stderr

apps/agent/agent/middleware/check_message_queue.py

@@ -10,10 +10,13 @@ from __future__ import annotations
 import logging
 from typing import Any
 
+import httpx
 from langchain.agents.middleware import AgentState, before_model
 from langgraph.config import get_config, get_store
 from langgraph.runtime import Runtime
 
+from ..utils.multimodal import fetch_image_block
+
 logger = logging.getLogger(__name__)
 
 
@@ -23,6 +26,25 @@ class LinearNotifyState(AgentState):
     linear_messages_sent_count: int
 
 
+async def _build_blocks_from_payload(
+    payload: dict[str, Any],
+) -> list[dict[str, Any]]:
+    text = payload.get("text", "")
+    image_urls = payload.get("image_urls", []) or []
+    blocks: list[dict[str, Any]] = []
+    if text:
+        blocks.append({"type": "text", "text": text})
+
+    if not image_urls:
+        return blocks
+    async with httpx.AsyncClient() as client:
+        for image_url in image_urls:
+            image_block = await fetch_image_block(image_url, client)
+            if image_block:
+                blocks.append(image_block)
+    return blocks
+
+
 @before_model(state_schema=LinearNotifyState)
 async def check_message_queue_before_model(  # noqa: PLR0911
     state: LinearNotifyState,  # noqa: ARG001
@@ -80,11 +102,21 @@ async def check_message_queue_before_model(  # noqa: PLR0911
             thread_id,
         )
 
-        content_blocks = [
+        content_blocks: list[dict[str, Any]] = []
-            {"type": "text", "text": msg.get("content", "")}
+        for msg in queued_messages:
-            for msg in queued_messages
+            content = msg.get("content")
-            if msg.get("content")
+            if isinstance(content, dict) and ("text" in content or "image_urls" in content):
-        ]
+                logger.debug("Queued message contains text + image URLs")
+                blocks = await _build_blocks_from_payload(content)
+                content_blocks.extend(blocks)
+                continue
+            if isinstance(content, list):
+                logger.debug("Queued message contains %d content block(s)", len(content))
+                content_blocks.extend(content)
+                continue
+            if isinstance(content, str) and content:
+                logger.debug("Queued message contains text content")
+                content_blocks.append({"type": "text", "text": content})
 
         if not content_blocks:
             return None

apps/agent/agent/prompt.py

@@ -4,9 +4,12 @@ You are operating in a **remote Linux sandbox** at `{working_dir}`.
 
 All code execution and file operations happen in this sandbox environment.
 
+{agents_md_section}
 
 **Important:**
 - Use `{working_dir}` as your working directory for all operations
+- The `execute` tool enforces a 5-minute timeout by default
+- If a command times out and needs longer, rerun it explicitly passing the `timeout` argument to the `execute` tool with a higher value in seconds.
 
 
 ---
@@ -72,9 +75,20 @@ def construct_system_prompt(
     working_dir: str,
     linear_project_id: str = "",
     linear_issue_number: str = "",
+    agents_md: str = "",
 ) -> str:
+    agents_md_section = ""
+    if agents_md:
+        agents_md_section = (
+            "\nThe following text is pulled from the repository's AGENTS.md file. "
+            "It may contain specific instructions and guidelines for the agent.\n"
+            "<agents_md>\n"
+            f"{agents_md}\n"
+            "</agents_md>\n"
+        )
     return SYSTEM_PROMPT.format(
         working_dir=working_dir,
         linear_project_id=linear_project_id or "<PROJECT_ID>",
         linear_issue_number=linear_issue_number or "<ISSUE_NUMBER>",
+        agents_md_section=agents_md_section,
     )

apps/agent/agent/server.py

@@ -26,7 +26,7 @@ from deepagents.backends.protocol import SandboxBackendProtocol
 from langchain_openai import ChatOpenAI
 
 from .encryption import decrypt_token
-from .integrations.langsmith import _create_langsmith_sandbox
+from .integrations.langsmith import create_langsmith_sandbox
 from .middleware import (
     ToolErrorMiddleware,
     check_message_queue_before_model,
@@ -42,6 +42,7 @@ SANDBOX_CREATING = "__creating__"
 SANDBOX_CREATION_TIMEOUT = 180
 SANDBOX_POLL_INTERVAL = 1.0
 
+from .utils.agents_md import read_agents_md_in_sandbox
 from .utils.github import (
     git_has_uncommitted_changes,
     is_valid_git_repo,
@@ -215,7 +216,6 @@ async def get_agent(config: RunnableConfig) -> Pregel:  # noqa: PLR0915
     encrypted_token = config["configurable"].get("github_token_encrypted")
     if encrypted_token:
         github_token = decrypt_token(encrypted_token)
-        logger.debug("Decrypted GitHub token")
 
     if thread_id is None or not graph_loaded_for_execution(config):
         logger.info("No thread_id or not for execution, returning agent without sandbox")
@@ -252,7 +252,7 @@ async def get_agent(config: RunnableConfig) -> Pregel:  # noqa: PLR0915
 
         try:
             # Create sandbox without context manager cleanup (sandbox persists)
-            sandbox_backend = await asyncio.to_thread(_create_langsmith_sandbox)
+            sandbox_backend = await asyncio.to_thread(create_langsmith_sandbox)
             logger.info("Sandbox created: %s", sandbox_backend.id)
 
             # Update metadata immediately after sandbox creation so other callers
@@ -286,7 +286,7 @@ async def get_agent(config: RunnableConfig) -> Pregel:  # noqa: PLR0915
         logger.info("Connecting to existing sandbox %s", sandbox_id)
         try:
             # Connect to existing sandbox without context manager cleanup
-            sandbox_backend = await asyncio.to_thread(_create_langsmith_sandbox, sandbox_id)
+            sandbox_backend = await asyncio.to_thread(create_langsmith_sandbox, sandbox_id)
             logger.info("Connected to existing sandbox %s", sandbox_id)
         except Exception:
             logger.warning("Failed to connect to existing sandbox %s, creating new one", sandbox_id)
@@ -297,7 +297,7 @@ async def get_agent(config: RunnableConfig) -> Pregel:  # noqa: PLR0915
             )
 
             try:
-                sandbox_backend = await asyncio.to_thread(_create_langsmith_sandbox)
+                sandbox_backend = await asyncio.to_thread(create_langsmith_sandbox)
                 logger.info("New sandbox created: %s", sandbox_backend.id)
 
                 await client.threads.update(
@@ -324,9 +324,14 @@ async def get_agent(config: RunnableConfig) -> Pregel:  # noqa: PLR0915
 
     SANDBOX_BACKENDS[thread_id] = sandbox_backend
 
+    if not repo_dir:
+        msg = "Cannot proceed: no repo was cloned. Set 'repo.owner' and 'repo.name' in the configurable config"
+        raise RuntimeError(msg)
+
     linear_issue = config["configurable"].get("linear_issue", {})
     linear_project_id = linear_issue.get("linear_project_id", "")
     linear_issue_number = linear_issue.get("linear_issue_number", "")
+    agents_md = await read_agents_md_in_sandbox(sandbox_backend, repo_dir)
 
     logger.info("Returning agent with sandbox for thread %s", thread_id)
     return create_deep_agent(
@@ -335,6 +340,7 @@ async def get_agent(config: RunnableConfig) -> Pregel:  # noqa: PLR0915
             repo_dir,
             linear_project_id=linear_project_id,
             linear_issue_number=linear_issue_number,
+            agents_md=agents_md,
         ),
         tools=[http_request, fetch_url, commit_and_open_pr],
         backend=sandbox_backend,

apps/agent/agent/utils/agents_md.py

@@ -0,0 +1,34 @@
+"""Helpers for reading agent instructions from AGENTS.md."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import shlex
+
+from deepagents.backends.protocol import SandboxBackendProtocol
+
+logger = logging.getLogger(__name__)
+
+
+async def read_agents_md_in_sandbox(
+    sandbox_backend: SandboxBackendProtocol,
+    repo_dir: str | None,
+) -> str | None:
+    """Read AGENTS.md from the repo root if it exists."""
+    if not repo_dir:
+        return None
+
+    safe_agents_path = shlex.quote(f"{repo_dir}/AGENTS.md")
+    loop = asyncio.get_event_loop()
+    result = await loop.run_in_executor(
+        None,
+        sandbox_backend.execute,
+        f"test -f {safe_agents_path} && cat {safe_agents_path}",
+    )
+    if result.exit_code != 0:
+        logger.debug("AGENTS.md not found at %s", safe_agents_path)
+        return None
+    content = result.output or ""
+    content = content.strip()
+    return content or None

apps/agent/agent/utils/github.py

@@ -134,7 +134,7 @@ async def create_github_pr(
     base_branch: str,
     body: str,
 ) -> tuple[str | None, int | None, bool]:
-    """Create a GitHub pull request via the API.
+    """Create a draft GitHub pull request via the API.
 
     Args:
         repo_owner: Repository owner (e.g., "langchain-ai")
@@ -153,6 +153,7 @@ async def create_github_pr(
         "head": head_branch,
         "base": base_branch,
         "body": body,
+        "draft": True,
     }
 
     logger.info(

apps/agent/agent/utils/multimodal.py

@@ -0,0 +1,83 @@
+"""Utilities for building multimodal content blocks."""
+
+from __future__ import annotations
+
+import base64
+import logging
+import mimetypes
+import os
+import re
+from typing import Any
+
+import httpx
+from langchain_core.messages.content import create_image_block
+
+logger = logging.getLogger(__name__)
+
+IMAGE_MARKDOWN_RE = re.compile(r"!\[[^\]]*\]\((https?://[^\s)]+)\)")
+IMAGE_URL_RE = re.compile(
+    r"(https?://[^\s)]+\.(?:png|jpe?g|gif|webp|bmp|tiff)(?:\?[^\s)]+)?)",
+    re.IGNORECASE,
+)
+
+
+def extract_image_urls(text: str) -> list[str]:
+    """Extract image URLs from markdown image syntax and direct image links."""
+    if not text:
+        return []
+
+    urls: list[str] = []
+    urls.extend(IMAGE_MARKDOWN_RE.findall(text))
+    urls.extend(IMAGE_URL_RE.findall(text))
+
+    deduped = dedupe_urls(urls)
+    if deduped:
+        logger.debug("Extracted %d image URL(s)", len(deduped))
+    return deduped
+
+
+async def fetch_image_block(
+    image_url: str,
+    client: httpx.AsyncClient,
+) -> dict[str, Any] | None:
+    """Fetch image bytes and build an image content block."""
+    try:
+        logger.debug("Fetching image from %s", image_url)
+        headers = None
+        if "uploads.linear.app" in image_url:
+            linear_api_key = os.environ.get("LINEAR_API_KEY", "")
+            if linear_api_key:
+                headers = {"Authorization": linear_api_key}
+            else:
+                logger.warning(
+                    "LINEAR_API_KEY not set; cannot authenticate image fetch for %s",
+                    image_url,
+                )
+        response = await client.get(image_url, headers=headers)
+        response.raise_for_status()
+        content_type = response.headers.get("Content-Type", "").split(";")[0].strip()
+        if not content_type:
+            guessed, _ = mimetypes.guess_type(image_url)
+            if not guessed:
+                logger.warning(
+                    "Could not determine content type for %s; skipping image",
+                    image_url,
+                )
+                return None
+            content_type = guessed
+
+        encoded = base64.b64encode(response.content).decode("ascii")
+        logger.info(
+            "Fetched image %s (%s, %d bytes)",
+            image_url,
+            content_type,
+            len(response.content),
+        )
+        return create_image_block(base64=encoded, mime_type=content_type)
+    except Exception:
+        logger.exception("Failed to fetch image from %s", image_url)
+        return None
+
+
+def dedupe_urls(urls: list[str]) -> list[str]:
+    return list(dict.fromkeys(urls))

apps/agent/agent/utils/sandbox_state.py

@@ -8,7 +8,7 @@ from typing import Any
 
 from langgraph.config import get_config
 
-from ..integrations.langsmith import _create_langsmith_sandbox
+from ..integrations.langsmith import create_langsmith_sandbox
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +36,7 @@ async def get_sandbox_backend(thread_id: str) -> Any | None:
     if not sandbox_id:
         raise ValueError(f"Missing sandbox_id in thread metadata for {thread_id}")
 
-    sandbox_backend = await asyncio.to_thread(_create_langsmith_sandbox, sandbox_id)
+    sandbox_backend = await asyncio.to_thread(create_langsmith_sandbox, sandbox_id)
     SANDBOX_BACKENDS[thread_id] = sandbox_backend
     return sandbox_backend
 

apps/agent/agent/webapp.py

@@ -11,11 +11,13 @@ from typing import Any
 import httpx
 import jwt
 from fastapi import BackgroundTasks, FastAPI, HTTPException, Request
+from langchain_core.messages.content import create_text_block
 from langgraph_sdk import get_client
 
 # Local import for encryption
 from .encryption import encrypt_token
 from .utils.comments import get_recent_comments
+from .utils.multimodal import dedupe_urls, extract_image_urls, fetch_image_block
 
 logger = logging.getLogger(__name__)
 
@@ -427,7 +429,9 @@ async def is_thread_active(thread_id: str) -> bool:
     return status == "busy"
 
 
-async def queue_message_for_thread(thread_id: str, message_content: str) -> bool:
+async def queue_message_for_thread(
+    thread_id: str, message_content: str | list[dict[str, Any]] | dict[str, Any]
+) -> bool:
     """Queue a message for a thread that is currently active.
 
     Stores the message in the langgraph store, namespaced to the thread.
@@ -436,7 +440,7 @@ async def queue_message_for_thread(thread_id: str, message_content: str) -> bool
 
     Args:
         thread_id: The LangGraph thread ID
-        message_content: The message content to queue
+        message_content: The message content to queue (text or content blocks)
 
     Returns:
         True if successfully queued, False otherwise
@@ -572,9 +576,19 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
 
     title = full_issue.get("title", "No title")
     description = full_issue.get("description") or "No description"
+    image_urls: list[str] = []
+    description_image_urls = extract_image_urls(description)
+    if description_image_urls:
+        image_urls.extend(description_image_urls)
+        logger.debug(
+            "Found %d image URL(s) in issue description",
+            len(description_image_urls),
+        )
 
     comments = full_issue.get("comments", {}).get("nodes", [])
     comments_text = ""
+    triggering_comment = issue_data.get("triggering_comment", "")
+    triggering_comment_id = issue_data.get("triggering_comment_id", "")
 
     bot_message_prefixes = (
         "🔐 **GitHub Authentication Required**",
@@ -586,15 +600,75 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
         "❌ **Agent Error**",
     )
 
+    comment_ids: set[str] = set()
+    comment_id_to_index: dict[str, int] = {}
     if comments:
-        recent_user_comments = get_recent_comments(comments, bot_message_prefixes)
+        last_bot_comment_idx = -1
-        if recent_user_comments:
+        for i, comment in enumerate(comments):
+            comment_id = comment.get("id", "")
+            if comment_id:
+                comment_ids.add(comment_id)
+                comment_id_to_index[comment_id] = i
+            body = comment.get("body", "")
+            if any(body.startswith(prefix) for prefix in bot_message_prefixes):
+                last_bot_comment_idx = i
+
+        relevant_comments = []
+        trigger_index = None
+        if triggering_comment_id:
+            trigger_index = comment_id_to_index.get(triggering_comment_id)
+        if trigger_index is not None:
+            relevant_comments = comments[trigger_index:]
+            logger.debug(
+                "Using triggering comment index %d to build relevant comments",
+                trigger_index,
+            )
+        else:
+            for i, comment in enumerate(comments):
+                if i <= last_bot_comment_idx:
+                    continue
+                body = comment.get("body", "")
+                if "@openswe" in body.lower():
+                    relevant_comments.append(comment)
+                    relevant_comments.extend(comments[i + 1 :])
+                    break
+
+        if relevant_comments:
             comments_text = "\n\n## Comments:\n"
             for comment in recent_user_comments:
                 author = comment.get("user", {}).get("name", "Unknown")
                 body = comment.get("body", "")
+                body_image_urls = extract_image_urls(body)
+                if body_image_urls:
+                    image_urls.extend(body_image_urls)
+                    logger.debug(
+                        "Found %d image URL(s) in comment by %s",
+                        len(body_image_urls),
+                        author,
+                    )
+                if any(body.startswith(prefix) for prefix in bot_message_prefixes):
+                    continue
                 comments_text += f"\n**{author}:** {body}\n"
 
+    if triggering_comment and triggering_comment_id not in comment_ids:
+        if not comments_text:
+            comments_text = "\n\n## Comments:\n"
+        trigger_author = comment_author.get("name", "Unknown")
+        trigger_body = triggering_comment
+        trigger_image_urls = extract_image_urls(trigger_body)
+        if trigger_image_urls:
+            image_urls.extend(trigger_image_urls)
+            logger.debug(
+                "Found %d image URL(s) in triggering comment by %s",
+                len(trigger_image_urls),
+                trigger_author,
+            )
+        comments_text += f"\n**{trigger_author}:** {trigger_body}\n"
+        logger.debug(
+            "Appended triggering comment %s not present in issue comments list",
+            triggering_comment_id or "<missing-id>",
+        )
+
     prompt = (
         f"Please work on the following issue:\n\n"
         f"## Title: {title}\n\n"
@@ -603,6 +677,18 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
         "Please analyze this issue and implement the necessary changes. "
         "When you're done, commit and push your changes."
     )
+    content_blocks: list[dict[str, Any]] = [create_text_block(prompt)]
+    if image_urls:
+        image_urls = dedupe_urls(image_urls)
+        logger.info("Preparing %d image(s) for multimodal content", len(image_urls))
+        logger.debug("Image URLs: %s", image_urls)
+
+        async with httpx.AsyncClient() as client:
+            for image_url in image_urls:
+                image_block = await fetch_image_block(image_url, client)
+                if image_block:
+                    content_blocks.append(image_block)
+        logger.info("Built %d content block(s) for prompt", len(content_blocks))
 
     identifier = full_issue.get("identifier", "") or issue_data.get("identifier", "")
     linear_project_id = ""
@@ -636,9 +722,10 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
                 thread_id,
             )
 
+            queued_payload = {"text": prompt, "image_urls": image_urls}
             queued = await queue_message_for_thread(
                 thread_id=thread_id,
-                message_content=prompt,
+                message_content=queued_payload,
             )
 
             if queued:
@@ -653,7 +740,7 @@ async def process_linear_issue(  # noqa: PLR0912, PLR0915
             await langgraph_client.runs.create(
                 thread_id,
                 "agent",
-                input={"messages": [{"role": "user", "content": prompt}]},
+                input={"messages": [{"role": "user", "content": content_blocks}]},
                 config={"configurable": configurable},
                 if_not_exists="create",
             )

apps/agent/pyproject.toml

@@ -6,26 +6,18 @@ readme = "README.md"
 requires-python = ">=3.11"
 license = { text = "MIT" }
 dependencies = [
-    # Core deepagents library
+    "deepagents>=0.4.3",
-    "deepagents>=0.4.0",
-    # FastAPI for webhook handling
     "fastapi>=0.104.0",
     "uvicorn>=0.24.0",
-    # HTTP client
     "httpx>=0.25.0",
-    # JWT for service authentication
     "PyJWT>=2.8.0",
-    # Encryption
     "cryptography>=41.0.0",
-    # LangGraph SDK for thread management
     "langgraph-sdk>=0.1.0",
-    # LangChain dependencies (will be pulled in by deepagents but listing for clarity)
     "langchain>=1.2.9",
     "langgraph>=1.0.8",
     "markdownify>=1.2.2",
     "langchain-anthropic>1.1.0",
     "langgraph-cli[inmem]>=0.4.12",
-    # LangSmith SDK for sandbox management
     "langsmith>=0.7.1",
     "langchain-openai==1.1.10",
 ]

apps/agent/tests/test_multimodal.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from agent.utils.multimodal import extract_image_urls
+
+
+def test_extract_image_urls_empty() -> None:
+    assert extract_image_urls("") == []
+
+
+def test_extract_image_urls_markdown_and_direct_dedupes() -> None:
+    text = (
+        "Here is an image ![alt](https://example.com/a.png) and another "
+        "![https://example.com/b.JPG?size=large plus a repeat https://example.com/a.png"
+    )
+
+    assert extract_image_urls(text) == [
+        "https://example.com/a.png",
+        "https://example.com/b.JPG?size=large",
+    ]
+
+
+def test_extract_image_urls_ignores_non_images() -> None:
+    text = "Not images: https://example.com/file.pdf and https://example.com/noext"
+
+    assert extract_image_urls(text) == []
+
+
+def test_extract_image_urls_markdown_syntax() -> None:
+    text = "Check out this screenshot: ![Screenshot](https://example.com/screenshot.png)"
+
+    assert extract_image_urls(text) == ["https://example.com/screenshot.png"]
+
+
+def test_extract_image_urls_direct_links() -> None:
+    text = "Direct link: https://example.com/photo.jpg and another https://example.com/image.gif"
+
+    assert extract_image_urls(text) == [
+        "https://example.com/photo.jpg",
+        "https://example.com/image.gif",
+    ]
+
+
+def test_extract_image_urls_various_formats() -> None:
+    text = (
+        "Multiple formats: "
+        "https://example.com/image.png "
+        "https://example.com/photo.jpeg "
+        "https://example.com/pic.gif "
+        "https://example.com/img.webp "
+        "https://example.com/bitmap.bmp "
+        "https://example.com/scan.tiff"
+    )
+
+    assert extract_image_urls(text) == [
+        "https://example.com/image.png",
+        "https://example.com/photo.jpeg",
+        "https://example.com/pic.gif",
+        "https://example.com/img.webp",
+        "https://example.com/bitmap.bmp",
+        "https://example.com/scan.tiff",
+    ]
+
+
+def test_extract_image_urls_with_query_params() -> None:
+    text = "Image with params: https://cdn.example.com/image.png?width=800&height=600"
+
+    assert extract_image_urls(text) == ["https://cdn.example.com/image.png?width=800&height=600"]
+
+
+def test_extract_image_urls_case_insensitive() -> None:
+    text = "Mixed case: https://example.com/Image.PNG and https://example.com/photo.JpEg"
+
+    assert extract_image_urls(text) == [
+        "https://example.com/Image.PNG",
+        "https://example.com/photo.JpEg",
+    ]
+
+
+def test_extract_image_urls_deduplication() -> None:
+    text = "Same URL twice: https://example.com/image.png and again https://example.com/image.png"
+
+    assert extract_image_urls(text) == ["https://example.com/image.png"]
+
+
+def test_extract_image_urls_mixed_markdown_and_direct() -> None:
+    text = (
+        "Markdown: ![alt text](https://example.com/markdown.png) "
+        "and direct: https://example.com/direct.jpg "
+        "and another markdown ![](https://example.com/another.gif)"
+    )
+
+    result = extract_image_urls(text)
+    assert set(result) == {
+        "https://example.com/markdown.png",
+        "https://example.com/direct.jpg",
+        "https://example.com/another.gif",
+    }
+    assert len(result) == 3

apps/agent/uv.lock

@@ -345,7 +345,7 @@ wheels = [
 
 [[package]]
 name = "deepagents"
-version = "0.4.0"
+version = "0.4.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "langchain" },
@@ -354,9 +354,9 @@ dependencies = [
     { name = "langchain-google-genai" },
     { name = "wcmatch" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/b2/ac/895c5efe77ee64f38af64146509b97220867fc149a9a376b2c74f266fd45/deepagents-0.4.0.tar.gz", hash = "sha256:ccfbb2394d2c50a3cf6f61457c5c1d3868354beee2177a57b8497c72c38f8e9b", size = 77614 }
+sdist = { url = "https://files.pythonhosted.org/packages/b4/30/5bba09d1c196a9e6e2e3a3406cd131bdf01e84ec67c4b6233f68a903978f/deepagents-0.4.3.tar.gz", hash = "sha256:88033c616c5ea481f2620dbb2d05533bc8fdcd48f376d713f9dba49a8157b6f8", size = 83210 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c4/c8/cbedac42e011889f151047cf22dbecef9a38330c2e90e517c2bd1b1e636d/deepagents-0.4.0-py3-none-any.whl", hash = "sha256:475af99429c7b6abe4c53d476d6b48f01960dc1bcaec7fa21093189acdfc2864", size = 87831 },
+    { url = "https://files.pythonhosted.org/packages/58/f8/c076a841b68cc13d89c395cc97965b37751ed008691a304119efa0f5717e/deepagents-0.4.3-py3-none-any.whl", hash = "sha256:298d19c5c0b4c6fc6a74b68049a7bfea0ba481aece7201ab21e7172b71ee61b9", size = 94882 },
 ]
 
 [[package]]
@@ -787,21 +787,21 @@ wheels = [
 
 [[package]]
 name = "langchain-anthropic"
-version = "1.3.2"
+version = "1.3.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "anthropic" },
     { name = "langchain-core" },
     { name = "pydantic" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e7/dd/c5e094079bdd748ca3f0bd0a09189ed2fa46bba56b5a8351198dc7c19e1f/langchain_anthropic-1.3.2.tar.gz", hash = "sha256:e551726a6ebf20229bde06022b5149d33bd48d28e34bd002a744953667b8ad48", size = 686239 }
+sdist = { url = "https://files.pythonhosted.org/packages/30/4e/7c1ffac126f5e62b0b9066f331f91ae69361e73476fd3ca1b19f8d8a3cc3/langchain_anthropic-1.3.4.tar.gz", hash = "sha256:000ed4c2d6fb8842b4ffeed22a74a3e84f9e9bcb63638e4abbb4a1d8ffa07211", size = 671858 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c9/6b/2da16c32308f79bb4588cec7095edbc770722ae4b3c3a1c135e05b0bdc2e/langchain_anthropic-1.3.2-py3-none-any.whl", hash = "sha256:35bc30862696a493680b898eb76bd6c866841f8e48a57d5eca1420a4fd807ac0", size = 46751 },
+    { url = "https://files.pythonhosted.org/packages/9b/cf/b7c7b7270efbb3db2edbf14b09ba9110a41628f3a85a11cae9527a35641c/langchain_anthropic-1.3.4-py3-none-any.whl", hash = "sha256:cd112dcc8049aef09f58b3c4338b2c9db5ee98105e08664954a4e40d8bf120b9", size = 47454 },
 ]
 
 [[package]]
 name = "langchain-core"
-version = "1.2.14"
+version = "1.2.15"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "jsonpatch" },
@@ -813,9 +813,9 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "uuid-utils" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/3f/ff/c5e3da8eca8a18719b300ef6c29e28208ee4e9da7f9749022b96292b6541/langchain_core-1.2.14.tar.gz", hash = "sha256:09549d838a2672781da3a9502f3b9c300863284b77b27e2a6dac4e6e650acfed", size = 833399 }
+sdist = { url = "https://files.pythonhosted.org/packages/cc/db/693d81b6c229aceb7c6e6809939c6ab1b023554227a25de438f00c0389c6/langchain_core-1.2.15.tar.gz", hash = "sha256:7d5f5d2daa8ddbe4054a96101dc5d509926f831b9914808c24640987d499758c", size = 835280 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/71/41/fe6ae9065b866b1397adbfc98db5e1648e8dcd78126b8e1266fcbe2d6395/langchain_core-1.2.14-py3-none-any.whl", hash = "sha256:b349ca28c057ac1f9b5280ea091bddb057db24d0f1c3c89bbb590713e1715838", size = 501411 },
+    { url = "https://files.pythonhosted.org/packages/48/e0/a6a83dde94400b43d9b091ecbb41a50d6f86c4fecacb81b13d8452a7712b/langchain_core-1.2.15-py3-none-any.whl", hash = "sha256:8d920d8a31d8c223966a3993d8c79fd6093b9665f2222fc878812f3a52072ab7", size = 502213 },
 ]
 
 [[package]]
@@ -1050,7 +1050,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "cryptography", specifier = ">=41.0.0" },
-    { name = "deepagents", specifier = ">=0.4.0" },
+    { name = "deepagents", specifier = ">=0.4.3" },
     { name = "fastapi", specifier = ">=0.104.0" },
     { name = "httpx", specifier = ">=0.25.0" },
     { name = "langchain", specifier = ">=1.2.9" },