mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 22:03:14 +00:00
eb18b07b20
* feat: trigger reviewer evals from the admin page Add an admin-only "Reviewer eval" section + endpoints that launch the reviewer benchmark as an isolated subprocess against the running deployment, with live status and the LangSmith experiment link. Route eval traces to a dedicated open-swe-evals project so they stay out of the production tracing project. Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * fix: reconcile reviewer eval status via heartbeat, not local process The persisted record is shared across workers but _PROCS is process-local. The owning worker now refreshes a heartbeat while the subprocess runs, and status is only reconciled to failed once the heartbeat is stale, so a poll on a worker without the local handle no longer kills a live run (and a duplicate start is rejected across workers). Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
102 lines
3.8 KiB
Python
102 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.dashboard import eval_jobs
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clear_procs():
|
|
eval_jobs._PROCS.clear()
|
|
yield
|
|
eval_jobs._PROCS.clear()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_returns_idle_when_no_record() -> None:
|
|
with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "idle"
|
|
assert status["name"] == eval_jobs.REVIEWER_EVAL_KEY
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_reconciles_stale_running() -> None:
|
|
stale = (datetime.now(UTC) - timedelta(seconds=300)).isoformat()
|
|
record = {"name": "reviewer", "status": "running", "heartbeat": stale}
|
|
with (
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
|
|
):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "failed"
|
|
assert "no longer tracked" in status["error"]
|
|
put.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_keeps_running_with_fresh_heartbeat() -> None:
|
|
"""A poll on a worker without the local handle must not kill a live run."""
|
|
fresh = datetime.now(UTC).isoformat()
|
|
record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
|
|
with (
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
|
|
):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "running"
|
|
put.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_reviewer_eval_launches_subprocess() -> None:
|
|
proc = MagicMock()
|
|
proc.pid = 4321
|
|
proc.returncode = None
|
|
create = AsyncMock(return_value=proc)
|
|
|
|
def _consume(coro):
|
|
coro.close()
|
|
return MagicMock()
|
|
|
|
with (
|
|
patch.object(eval_jobs.asyncio, "create_subprocess_exec", new=create),
|
|
patch.object(eval_jobs.asyncio, "create_task", new=_consume),
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)),
|
|
patch.object(eval_jobs, "_resolve_langgraph_url", return_value="https://lg.test"),
|
|
):
|
|
record = await eval_jobs.start_reviewer_eval(limit=3, created_by="octo")
|
|
|
|
assert record["status"] == "running"
|
|
assert record["limit"] == 3
|
|
assert record["pid"] == 4321
|
|
assert record["heartbeat"] is not None
|
|
assert record["worker_id"] == eval_jobs._WORKER_ID
|
|
assert eval_jobs._PROCS[eval_jobs.REVIEWER_EVAL_KEY] is proc
|
|
|
|
args, kwargs = create.call_args
|
|
assert "--limit" in args and "3" in args
|
|
assert kwargs["env"]["LANGSMITH_PROJECT"] == eval_jobs.DEFAULT_EVAL_PROJECT
|
|
assert kwargs["env"]["LANGGRAPH_URL"] == "https://lg.test"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_reviewer_eval_rejects_when_running() -> None:
|
|
running = MagicMock()
|
|
running.returncode = None
|
|
eval_jobs._PROCS[eval_jobs.REVIEWER_EVAL_KEY] = running
|
|
with pytest.raises(RuntimeError):
|
|
await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_reviewer_eval_rejects_fresh_run_on_other_worker() -> None:
|
|
fresh = datetime.now(UTC).isoformat()
|
|
record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
|
|
with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)):
|
|
with pytest.raises(RuntimeError):
|
|
await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")
|