mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 10:23:14 +00:00
0927f2dd9c
* Run reviewer eval in a GitHub Action; make dashboard a read-only progress view The dashboard launched the eval as a subprocess inside the serving deployment worker, so a container recycle killed long runs and discarded results that had already completed server-side. Move the harness to a workflow_dispatch Action (run on prod). run_eval now publishes status/progress/log-tail to the LangGraph store record the dashboard reads, so /admin/evals stays a live view; a killed Action surfaces as failed via the stale-heartbeat reconcile. * reviewer_eval workflow: pass inputs via env, no shell interpolation Addresses the reviewer finding: workflow_dispatch string inputs were interpolated into the run: block (limit unquoted), allowing shell injection in a job holding LANGSMITH/ANTHROPIC keys. Pass inputs through env and reference quoted "$VARS"; validate limit is numeric and build its flag in bash. --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
68 lines
2.7 KiB
Python
68 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.dashboard import eval_jobs
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_returns_idle_when_no_record() -> None:
|
|
with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "idle"
|
|
assert status["name"] == eval_jobs.REVIEWER_EVAL_KEY
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_returns_terminal_record_unchanged() -> None:
|
|
record = {"name": "reviewer", "status": "completed", "experiment_url": "https://x"}
|
|
with (
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
|
|
):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status is record
|
|
put.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_reconciles_stale_running() -> None:
|
|
stale = (datetime.now(UTC) - timedelta(seconds=300)).isoformat()
|
|
record = {"name": "reviewer", "status": "running", "heartbeat": stale}
|
|
with (
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
|
|
):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "failed"
|
|
assert "no longer tracked" in status["error"]
|
|
put.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_keeps_running_with_fresh_heartbeat() -> None:
|
|
"""A poll must not kill a run whose Action is still heartbeating."""
|
|
fresh = datetime.now(UTC).isoformat()
|
|
record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
|
|
with (
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
|
|
):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "running"
|
|
put.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status_fails_running_without_heartbeat() -> None:
|
|
"""A running record with no heartbeat at all is treated as stale."""
|
|
record = {"name": "reviewer", "status": "running"}
|
|
with (
|
|
patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
|
|
patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)),
|
|
):
|
|
status = await eval_jobs.get_reviewer_eval_status()
|
|
assert status["status"] == "failed"
|