mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 16:13:15 +00:00
Adam Moussa
ae1f883b4c
Applies the plan's C5 step: git mv every test per the domain-reorg
move-map (movemap-m50.txt) into tests/{agent,analyzer,auth,dashboard,
github,middleware,models,reviewer,sandbox,slack,tools,webhooks}/, plus
the 13 fork-only placements from the scoping report §2c (Atlassian
webhook tests -> tests/webhooks/, test_atlassian_connect.py and
test_auth_error_leak.py -> tests/auth/, jira/confluence util tests ->
tests/tools/, test_repo_binding_isolation.py -> tests/sandbox/,
bot-identity/autofix tests -> tests/github/).
Path-only move: the only content edits are parents[1] -> parents[2]
fixes in test_e2b_integration.py and test_daytona_integration.py,
required because their __file__-relative ROOT path gained one more
directory level in the move.
Monkeypatch retargets for these files were already completed in C4;
none remained outstanding here.
95 lines
3.6 KiB
Python
95 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from evals.reviewer import store_reporter
|
|
from evals.reviewer.store_reporter import StoreReporter, github_run_url, is_enabled
|
|
|
|
_CONFIG = {
|
|
"experiment_prefix": "openswe-review-confidence",
|
|
"langsmith_project": "open-swe-evals",
|
|
"model_id": "google_genai:gemini-3.5-flash",
|
|
}
|
|
|
|
|
|
def _make_reporter(
|
|
monkeypatch: pytest.MonkeyPatch, completed: int = 0
|
|
) -> tuple[StoreReporter, MagicMock]:
|
|
monkeypatch.setenv("LANGGRAPH_URL", "https://lg.test")
|
|
monkeypatch.setenv("GITHUB_SERVER_URL", "https://github.com")
|
|
monkeypatch.setenv("GITHUB_REPOSITORY", "langchain-ai/open-swe")
|
|
monkeypatch.setenv("GITHUB_RUN_ID", "12345")
|
|
monkeypatch.setenv("GITHUB_ACTOR", "octocat")
|
|
client = MagicMock()
|
|
client.store.put_item = AsyncMock()
|
|
with patch.object(store_reporter, "get_client", return_value=client):
|
|
reporter = StoreReporter(
|
|
config=dict(_CONFIG),
|
|
limit=3,
|
|
total=10,
|
|
created_by=None,
|
|
completed_getter=lambda: completed,
|
|
tail_getter=lambda: "tail",
|
|
experiment_url_getter=lambda: "https://smith.langchain.com/exp",
|
|
)
|
|
return reporter, client
|
|
|
|
|
|
def test_is_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("REVIEWER_EVAL_REPORT_STORE", raising=False)
|
|
monkeypatch.delenv("LANGGRAPH_URL", raising=False)
|
|
assert is_enabled() is False
|
|
monkeypatch.setenv("REVIEWER_EVAL_REPORT_STORE", "1")
|
|
assert is_enabled() is False # still needs LANGGRAPH_URL
|
|
monkeypatch.setenv("LANGGRAPH_URL", "https://lg.test")
|
|
assert is_enabled() is True
|
|
|
|
|
|
def test_github_run_url(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("GITHUB_SERVER_URL", "https://github.com")
|
|
monkeypatch.setenv("GITHUB_REPOSITORY", "langchain-ai/open-swe")
|
|
monkeypatch.setenv("GITHUB_RUN_ID", "999")
|
|
assert github_run_url() == "https://github.com/langchain-ai/open-swe/actions/runs/999"
|
|
monkeypatch.delenv("GITHUB_RUN_ID")
|
|
assert github_run_url() is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_writes_running_record(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
reporter, client = _make_reporter(monkeypatch, completed=2)
|
|
await reporter.start()
|
|
|
|
client.store.put_item.assert_awaited_once()
|
|
namespace, key, record = client.store.put_item.await_args.args
|
|
assert namespace == ["evals"]
|
|
assert key == "reviewer"
|
|
assert record["status"] == "running"
|
|
assert record["trigger"] == "github_action"
|
|
assert record["progress"] == {"completed": 2, "total": 10}
|
|
assert record["github_run_url"] == "https://github.com/langchain-ai/open-swe/actions/runs/12345"
|
|
assert record["created_by"] == "octocat" # falls back to GITHUB_ACTOR
|
|
assert record["worker_id"] == "12345"
|
|
assert record["run_name"] == "openswe-review-confidence"
|
|
assert record["limit"] == 3
|
|
assert record["heartbeat"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finish_writes_terminal_record(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
reporter, client = _make_reporter(monkeypatch)
|
|
await reporter.finish(status="failed", error="boom")
|
|
|
|
_, _, record = client.store.put_item.await_args.args
|
|
assert record["status"] == "failed"
|
|
assert record["error"] == "boom"
|
|
assert record["finished_at"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_put_swallows_store_errors(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
reporter, client = _make_reporter(monkeypatch)
|
|
client.store.put_item = AsyncMock(side_effect=RuntimeError("store down"))
|
|
# Should not raise — store failures must not crash the eval.
|
|
await reporter.start()
|