open-swe/agent/utils/github_http.py

"""Shared GitHub HTTP helper with sane timeouts, retries, and rate-limit handling.

All GitHub API calls in the reviewer publish path (and gradually everywhere else)
should go through ``github_request`` instead of raw ``httpx.AsyncClient`` calls.
This centralises:

- **Timeouts**: httpx defaults to 5 s which is too aggressive for paginated
  GitHub/GraphQL fetches.  The default here is 30 s read / 10 s connect.
- **Retries**: exponential backoff with jitter for retryable HTTP status codes
  and transport errors, gated by method idempotency to prevent duplicate writes.
  See ``github_request`` for the full retry matrix.
- **Rate-limit awareness**: respects ``Retry-After`` headers and detects
  GitHub secondary rate limits (403 with ``X-RateLimit-Remaining: 0`` or a
  "secondary rate limit" body message), backing off before retrying.
"""

from __future__ import annotations

import asyncio
import logging
import random
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import Any

import httpx

logger = logging.getLogger(__name__)

GITHUB_API_BASE = "https://api.github.com"
GITHUB_GRAPHQL = "https://api.github.com/graphql"
GITHUB_HEADERS_VERSION = "2022-11-28"

DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0, pool=5.0)
DEFAULT_MAX_RETRIES = 3

_ALWAYS_RETRYABLE_STATUS = frozenset({429, 503})
_IDEMPOTENT_RETRYABLE_STATUS = frozenset({502, 504})
_SECONDARY_RATE_LIMIT_MARKERS = ("secondary rate limit", "rate limit")
_RETRYABLE_TRANSPORT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"})

_BASE_BACKOFF = 1.0
_BACKOFF_MULTIPLIER = 2.0
_MAX_BACKOFF = 60.0
_JITTER_FACTOR = 0.25


def github_headers(token: str) -> dict[str, str]:
    """Standard GitHub API headers for a bearer token."""
    return {
        "Authorization": f"Bearer {token}",
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": GITHUB_HEADERS_VERSION,
    }


def _is_secondary_rate_limit(response: httpx.Response) -> bool:
    if response.status_code != 403:
        return False
    if response.headers.get("X-RateLimit-Remaining") == "0":
        return True
    body = (response.text or "").lower()
    return any(marker in body for marker in _SECONDARY_RATE_LIMIT_MARKERS)


def _is_retryable_response(response: httpx.Response, method: str) -> bool:
    if response.status_code in _ALWAYS_RETRYABLE_STATUS:
        return True
    if response.status_code in _IDEMPOTENT_RETRYABLE_STATUS:
        return method.upper() in _RETRYABLE_TRANSPORT_METHODS
    return _is_secondary_rate_limit(response)


def _retry_after_seconds(response: httpx.Response) -> float | None:
    retry_after = response.headers.get("Retry-After")
    if retry_after:
        try:
            return float(retry_after)
        except ValueError:
            return None
    return None


def _compute_backoff(response: httpx.Response | None, attempt: int) -> float:
    if response is not None:
        retry_after = _retry_after_seconds(response)
        if retry_after is not None:
            return min(retry_after, _MAX_BACKOFF)
    base = _BASE_BACKOFF * (_BACKOFF_MULTIPLIER**attempt)
    jitter = base * random.uniform(-_JITTER_FACTOR, _JITTER_FACTOR)
    return min(base + jitter, _MAX_BACKOFF)


@asynccontextmanager
async def github_client(
    *,
    token: str | None = None,
    timeout: httpx.Timeout | float | None = None,
    headers: dict[str, str] | None = None,
) -> AsyncIterator[httpx.AsyncClient]:
    """Yield an ``httpx.AsyncClient`` with sane GitHub defaults.

    The token (when provided) is baked into the default headers so callers
    don't need to pass headers on every request.  A custom ``timeout`` can
    override the default 30 s / 10 s-connect timeout.
    """
    merged_headers: dict[str, str] = {}
    if token:
        merged_headers.update(github_headers(token))
    if headers:
        merged_headers.update(headers)
    async with httpx.AsyncClient(
        headers=merged_headers or None,
        timeout=timeout or DEFAULT_TIMEOUT,
    ) as client:
        yield client


async def github_request(
    client: httpx.AsyncClient,
    method: str,
    url: str,
    *,
    max_retries: int = DEFAULT_MAX_RETRIES,
    **kwargs: Any,
) -> httpx.Response:
    """Execute a single GitHub API request with retries and rate-limit handling.

    Returns the ``httpx.Response`` for non-retryable status codes and for
    retryable status codes that have exhausted retries (caller should call
    ``raise_for_status()``).

    Retry matrix:

    | Condition                         | Idempotent (GET, PUT, DELETE…) | Non-idempotent (POST, PATCH) |
    |-----------------------------------|--------------------------------|------------------------------|
    | Transport error (timeout/reset)   | Retry with backoff             | Raise immediately            |
    | 429 / 503 / secondary rate limit  | Retry with backoff             | Retry with backoff           |
    | 502 / 504 (ambiguous gateway)     | Retry with backoff             | Raise immediately            |

    429 and 503 are safe to retry for any method: the server explicitly did
    not process the request.  502/504 are ambiguous — the upstream may have
    processed the write before the gateway returned an error — so they are
    only retried for idempotent methods.  Transport errors are only retried
    for idempotent methods for the same reason.
    """
    method_upper = method.upper()
    retry_transport = method_upper in _RETRYABLE_TRANSPORT_METHODS
    method_func = getattr(client, method.lower())
    last_exc: Exception | None = None
    for attempt in range(max_retries + 1):
        try:
            response = await method_func(url, **kwargs)
        except (httpx.TimeoutException, httpx.TransportError) as exc:
            last_exc = exc
            if retry_transport and attempt < max_retries:
                delay = _compute_backoff(None, attempt)
                logger.warning(
                    "GitHub API %s %s raised %s, retrying in %.1fs (attempt %d/%d)",
                    method,
                    url,
                    type(exc).__name__,
                    delay,
                    attempt + 1,
                    max_retries,
                )
                await asyncio.sleep(delay)
                continue
            raise

        if _is_retryable_response(response, method):
            if attempt < max_retries:
                delay = _compute_backoff(response, attempt)
                logger.warning(
                    "GitHub API %s %s returned %d, retrying in %.1fs (attempt %d/%d)",
                    method,
                    url,
                    response.status_code,
                    delay,
                    attempt + 1,
                    max_retries,
                )
                await asyncio.sleep(delay)
                continue
            logger.warning(
                "GitHub API %s %s returned %d after %d retries, giving up",
                method,
                url,
                response.status_code,
                max_retries,
            )
        return response

    raise last_exc or httpx.HTTPError("Max retries exceeded")