open-swe/evals/reviewer/target.py

"""Target function for the reviewer eval.

Invokes the Open SWE Reviewer graph over the langgraph_sdk client and returns
the structured comments produced by the agent's `submit_review` tool call.

The reviewer graph itself is not part of this PR — wire `REVIEWER_ASSISTANT_ID`
to whatever graph id you want to evaluate once it exists.
"""

from __future__ import annotations

import os
from typing import Any

from langgraph_sdk import get_client

REVIEWER_ASSISTANT_ID = os.getenv("REVIEWER_ASSISTANT_ID", "reviewer")
LANGGRAPH_URL = os.getenv("LANGGRAPH_URL", "http://localhost:2024")


async def review_pr(inputs: dict[str, Any]) -> dict[str, Any]:
    """LangSmith target: run the reviewer agent on one PR.

    `inputs` carries: repo, pr_number, pr_url, base_sha, head_sha, base_ref,
    head_ref, pr_title. The reviewer graph is responsible for cloning the
    repo at base_sha, fetching the PR's head, and emitting structured review
    comments via a `submit_review` tool whose args become the graph output.

    Returns: {"comments": [{file, line, severity, body}, ...]}.
    """
    client = get_client(url=LANGGRAPH_URL)
    thread = await client.threads.create()
    result = await client.runs.wait(
        thread["thread_id"],
        assistant_id=REVIEWER_ASSISTANT_ID,
        input={"pr": inputs},
    )
    return {"comments": _extract_comments(result)}


def _extract_comments(result: Any) -> list[dict]:
    """Pull the submit_review payload out of the graph's final state.

    Supports two shapes:
      1. Graph state contains a top-level `review` field populated by the tool
         (preferred — wire the reviewer graph to set this).
      2. Last AI message includes a `submit_review` tool call; we parse args.
    """
    if isinstance(result, dict):
        if isinstance(result.get("review"), dict) and "comments" in result["review"]:
            return list(result["review"]["comments"])
        if isinstance(result.get("comments"), list):
            return list(result["comments"])

        messages = result.get("messages") or []
        for msg in reversed(messages):
            tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else None
            for tc in tool_calls or []:
                if tc.get("name") == "submit_review":
                    args = tc.get("args") or {}
                    if isinstance(args.get("comments"), list):
                        return list(args["comments"])
    return []