mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 17:23:15 +00:00
834efbc33c
* feat: tighten reviewer eval workflow Require the reviewer to verify and dedupe findings before recording them, and make benchmark runs safe to execute against deployed reviewer graphs without posting GitHub reviews. * chore: move reviewer eval settings to config Load reviewer benchmark settings from the default eval config file so deployed eval runs do not require a wide CLI surface. * feat: allow reviewer eval model overrides Pass reviewer model and reasoning effort from the eval config into reviewer runs so isolated benchmark deployments can test Opus 4.7 high thinking. * fix: use adaptive thinking for Opus 4.7 Switch Opus 4.7 model overrides to Anthropic adaptive thinking with effort instead of the deprecated budgeted thinking payload rejected by the API. * refactor: use latest Anthropic effort API Remove legacy Anthropic budget-token thinking support and route Anthropic efforts through adaptive thinking plus effort. * revert prompting
94 lines
2.6 KiB
Python
94 lines
2.6 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from agent.reviewer_findings import new_finding
|
|
from evals.reviewer import target
|
|
|
|
|
|
def test_eval_target_marks_runs_as_eval_dry_run(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("REVIEWER_EVAL_MODEL_ID", raising=False)
|
|
monkeypatch.delenv("REVIEWER_EVAL_REASONING_EFFORT", raising=False)
|
|
configurable = target._build_configurable(
|
|
{
|
|
"repo": "acme/repo",
|
|
"pr_number": 1,
|
|
"pr_url": "https://github.com/acme/repo/pull/1",
|
|
"base_sha": "base",
|
|
"head_sha": "head",
|
|
"head_ref": "branch",
|
|
}
|
|
)
|
|
|
|
assert configurable["reviewer_eval"] is True
|
|
assert configurable["eval"] is True
|
|
assert configurable["__is_for_execution__"] is True
|
|
assert "source" not in configurable
|
|
|
|
|
|
def test_eval_target_passes_model_overrides(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("REVIEWER_EVAL_MODEL_ID", "anthropic:claude-opus-4-7")
|
|
monkeypatch.setenv("REVIEWER_EVAL_REASONING_EFFORT", "high")
|
|
|
|
configurable = target._build_configurable(
|
|
{
|
|
"repo": "acme/repo",
|
|
"pr_number": 1,
|
|
"pr_url": "https://github.com/acme/repo/pull/1",
|
|
"base_sha": "base",
|
|
"head_sha": "head",
|
|
"head_ref": "branch",
|
|
}
|
|
)
|
|
|
|
assert configurable["reviewer_model_id"] == "anthropic:claude-opus-4-7"
|
|
assert configurable["reviewer_reasoning_effort"] == "high"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_surfaced_comments_uses_publish_filter(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
high = new_finding(
|
|
severity="high",
|
|
category="correctness",
|
|
file="a.py",
|
|
start_line=10,
|
|
end_line=10,
|
|
description="high signal",
|
|
sha="head",
|
|
finding_id="f_high",
|
|
)
|
|
low = new_finding(
|
|
severity="low",
|
|
category="style",
|
|
file="b.py",
|
|
start_line=20,
|
|
end_line=20,
|
|
description="low signal",
|
|
sha="head",
|
|
finding_id="f_low",
|
|
)
|
|
|
|
class Threads:
|
|
async def get(self, _thread_id: str) -> dict[str, Any]:
|
|
return {"metadata": {"findings": [high, low]}}
|
|
|
|
class Client:
|
|
threads = Threads()
|
|
|
|
monkeypatch.setenv("REVIEWER_EVAL_SEVERITY_THRESHOLD", "medium")
|
|
monkeypatch.setenv("REVIEWER_EVAL_CAP", "4")
|
|
|
|
comments = await target._extract_surfaced_comments(Client(), "tid")
|
|
|
|
assert comments == [
|
|
{
|
|
"file": "a.py",
|
|
"line": 10,
|
|
"body": "high signal",
|
|
"severity": "high",
|
|
}
|
|
]
|