mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 08:03:15 +00:00
6cecf429c7
* feat: add reviewer eval harness Offline LangSmith eval scaffolding for the upcoming Open SWE Reviewer graph. Imports the 50 PRs from withmartian/code-review-benchmark goldens, resolves base/head SHAs via gh, and runs a claude-opus-4-5 LLM judge using martian's verbatim prompt so scores are directly comparable to their published Devin Review numbers. Reviewer graph itself is not part of this change. * fix: ruff lint and format on reviewer eval files
46 lines
1.3 KiB
Python
46 lines
1.3 KiB
Python
"""Run the reviewer eval against the LangSmith dataset.
|
|
|
|
Usage:
|
|
uv run python -m evals.reviewer.run_eval \\
|
|
--dataset-name openswe-reviewer-v1 \\
|
|
--experiment-prefix openswe-reviewer-baseline \\
|
|
--max-concurrency 5
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
|
|
from dotenv import load_dotenv
|
|
from langsmith import aevaluate
|
|
|
|
from evals.reviewer.judge import aggregate_pr, judge_match
|
|
from evals.reviewer.target import review_pr
|
|
|
|
load_dotenv()
|
|
|
|
|
|
async def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--dataset-name", default="openswe-reviewer-v1")
|
|
ap.add_argument("--experiment-prefix", default="openswe-reviewer-baseline")
|
|
ap.add_argument("--max-concurrency", type=int, default=5)
|
|
ap.add_argument("--limit", type=int, default=None, help="Run only the first N examples.")
|
|
args = ap.parse_args()
|
|
|
|
await aevaluate(
|
|
review_pr,
|
|
data=args.dataset_name,
|
|
evaluators=[judge_match],
|
|
summary_evaluators=[aggregate_pr],
|
|
experiment_prefix=args.experiment_prefix,
|
|
max_concurrency=args.max_concurrency,
|
|
num_repetitions=1,
|
|
**({"max_examples": args.limit} if args.limit else {}),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import asyncio
|
|
|
|
asyncio.run(main())
|