mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-10-02 01:33:15 +00:00
6cecf429c7
* feat: add reviewer eval harness Offline LangSmith eval scaffolding for the upcoming Open SWE Reviewer graph. Imports the 50 PRs from withmartian/code-review-benchmark goldens, resolves base/head SHAs via gh, and runs a claude-opus-4-5 LLM judge using martian's verbatim prompt so scores are directly comparable to their published Devin Review numbers. Reviewer graph itself is not part of this change. * fix: ruff lint and format on reviewer eval files
63 lines
2.4 KiB
Python
63 lines
2.4 KiB
Python
"""Target function for the reviewer eval.
|
|
|
|
Invokes the Open SWE Reviewer graph over the langgraph_sdk client and returns
|
|
the structured comments produced by the agent's `submit_review` tool call.
|
|
|
|
The reviewer graph itself is not part of this PR — wire `REVIEWER_ASSISTANT_ID`
|
|
to whatever graph id you want to evaluate once it exists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any
|
|
|
|
from langgraph_sdk import get_client
|
|
|
|
REVIEWER_ASSISTANT_ID = os.getenv("REVIEWER_ASSISTANT_ID", "reviewer")
|
|
LANGGRAPH_URL = os.getenv("LANGGRAPH_URL", "http://localhost:2024")
|
|
|
|
|
|
async def review_pr(inputs: dict[str, Any]) -> dict[str, Any]:
|
|
"""LangSmith target: run the reviewer agent on one PR.
|
|
|
|
`inputs` carries: repo, pr_number, pr_url, base_sha, head_sha, base_ref,
|
|
head_ref, pr_title. The reviewer graph is responsible for cloning the
|
|
repo at base_sha, fetching the PR's head, and emitting structured review
|
|
comments via a `submit_review` tool whose args become the graph output.
|
|
|
|
Returns: {"comments": [{file, line, severity, body}, ...]}.
|
|
"""
|
|
client = get_client(url=LANGGRAPH_URL)
|
|
thread = await client.threads.create()
|
|
result = await client.runs.wait(
|
|
thread["thread_id"],
|
|
assistant_id=REVIEWER_ASSISTANT_ID,
|
|
input={"pr": inputs},
|
|
)
|
|
return {"comments": _extract_comments(result)}
|
|
|
|
|
|
def _extract_comments(result: Any) -> list[dict]:
|
|
"""Pull the submit_review payload out of the graph's final state.
|
|
|
|
Supports two shapes:
|
|
1. Graph state contains a top-level `review` field populated by the tool
|
|
(preferred — wire the reviewer graph to set this).
|
|
2. Last AI message includes a `submit_review` tool call; we parse args.
|
|
"""
|
|
if isinstance(result, dict):
|
|
if isinstance(result.get("review"), dict) and "comments" in result["review"]:
|
|
return list(result["review"]["comments"])
|
|
if isinstance(result.get("comments"), list):
|
|
return list(result["comments"])
|
|
|
|
messages = result.get("messages") or []
|
|
for msg in reversed(messages):
|
|
tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else None
|
|
for tc in tool_calls or []:
|
|
if tc.get("name") == "submit_review":
|
|
args = tc.get("args") or {}
|
|
if isinstance(args.get("comments"), list):
|
|
return list(args["comments"])
|
|
return []
|