open-swe/agent/tools/fetch_url.py

from typing import Any

import httpx
from markdownify import markdownify

from .http_request import _request_with_safe_redirects

FETCH_URL_MAX_CHARS = 100_000


async def fetch_url(url: str, timeout: int = 30) -> dict[str, Any]:
    """Fetch content from a URL and convert HTML to markdown format.

    This tool fetches web page content and converts it to clean markdown text,
    making it easy to read and process HTML content. After receiving the markdown,
    you MUST synthesize the information into a natural, helpful response for the user.

    Args:
        url: The URL to fetch (must be a valid HTTP/HTTPS URL)
        timeout: Request timeout in seconds (default: 30)

    Returns:
        Dictionary containing:
        - success: Whether the request succeeded
        - url: The final URL after redirects
        - markdown_content: The page content converted to markdown
        - status_code: HTTP status code
        - content_length: Length of the markdown content in characters

    IMPORTANT: After using this tool:
    1. Read through the markdown content
    2. Extract relevant information that answers the user's question
    3. Synthesize this into a clear, natural language response
    4. NEVER show the raw markdown to the user unless specifically requested
    """
    try:
        async with httpx.AsyncClient(timeout=timeout) as client:
            response, blocked = await _request_with_safe_redirects(
                client,
                "GET",
                url,
                headers={"User-Agent": "Mozilla/5.0 (compatible; DeepAgents/1.0)"},
            )
            if blocked:
                return {
                    "error": blocked["content"],
                    "status_code": blocked["status_code"],
                    "url": blocked["url"],
                }

            response.raise_for_status()

            # Convert HTML content to markdown
            markdown_content = markdownify(response.text)

        if len(markdown_content) > FETCH_URL_MAX_CHARS:
            markdown_content = (
                markdown_content[:FETCH_URL_MAX_CHARS] + "\n... [content truncated: "
                f"{FETCH_URL_MAX_CHARS}/{len(markdown_content)} chars]\n"
            )

        return {
            "url": str(response.url),
            "markdown_content": markdown_content,
            "status_code": response.status_code,
            "content_length": len(markdown_content),
        }
    except httpx.HTTPError as e:
        return {"error": f"Fetch URL error: {e!s}", "url": url}