open-swe/tests/test_eval_jobs.py

from __future__ import annotations

from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from agent.dashboard import eval_jobs


@pytest.fixture(autouse=True)
def _clear_procs():
    eval_jobs._PROCS.clear()
    yield
    eval_jobs._PROCS.clear()


@pytest.mark.asyncio
async def test_get_status_returns_idle_when_no_record() -> None:
    with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)):
        status = await eval_jobs.get_reviewer_eval_status()
    assert status["status"] == "idle"
    assert status["name"] == eval_jobs.REVIEWER_EVAL_KEY


@pytest.mark.asyncio
async def test_get_status_reconciles_stale_running() -> None:
    stale = (datetime.now(UTC) - timedelta(seconds=300)).isoformat()
    record = {"name": "reviewer", "status": "running", "heartbeat": stale}
    with (
        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
    ):
        status = await eval_jobs.get_reviewer_eval_status()
    assert status["status"] == "failed"
    assert "no longer tracked" in status["error"]
    put.assert_awaited_once()


@pytest.mark.asyncio
async def test_get_status_keeps_running_with_fresh_heartbeat() -> None:
    """A poll on a worker without the local handle must not kill a live run."""
    fresh = datetime.now(UTC).isoformat()
    record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
    with (
        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
    ):
        status = await eval_jobs.get_reviewer_eval_status()
    assert status["status"] == "running"
    put.assert_not_awaited()


@pytest.mark.asyncio
async def test_start_reviewer_eval_launches_subprocess() -> None:
    proc = MagicMock()
    proc.pid = 4321
    proc.returncode = None
    create = AsyncMock(return_value=proc)

    def _consume(coro):
        coro.close()
        return MagicMock()

    with (
        patch.object(eval_jobs.asyncio, "create_subprocess_exec", new=create),
        patch.object(eval_jobs.asyncio, "create_task", new=_consume),
        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)),
        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)),
        patch.object(eval_jobs, "_resolve_langgraph_url", return_value="https://lg.test"),
    ):
        record = await eval_jobs.start_reviewer_eval(limit=3, created_by="octo")

    assert record["status"] == "running"
    assert record["limit"] == 3
    assert record["run_name"] == "openswe-review-confidence"
    assert record["config_snapshot"]["model_id"] == "google_genai:gemini-3.5-flash"
    assert record["pid"] == 4321
    assert record["heartbeat"] is not None
    assert record["worker_id"] == eval_jobs._WORKER_ID
    assert eval_jobs._PROCS[eval_jobs.REVIEWER_EVAL_KEY] is proc

    args, kwargs = create.call_args
    assert "--limit" in args and "3" in args
    assert "--experiment-prefix" in args and "openswe-review-confidence" in args
    assert "--model-id" in args and "google_genai:gemini-3.5-flash" in args
    assert "--reasoning-effort" in args and "medium" in args
    assert kwargs["env"]["LANGSMITH_PROJECT"] == eval_jobs.DEFAULT_EVAL_PROJECT
    assert kwargs["env"]["LANGGRAPH_URL"] == "https://lg.test"


@pytest.mark.asyncio
async def test_start_reviewer_eval_uses_config_snapshot() -> None:
    proc = MagicMock()
    proc.pid = 4321
    proc.returncode = None
    create = AsyncMock(return_value=proc)
    config: eval_jobs.ReviewerEvalConfig = {
        **eval_jobs.DEFAULT_REVIEWER_EVAL_CONFIG,
        "experiment_prefix": "custom-run",
        "langsmith_project": "custom-project",
        "model_id": "anthropic:claude-opus-4-8",
        "reasoning_effort": "high",
        "score_mode": "surfaced_findings",
        "severity_threshold": "high",
        "cap": 2,
    }

    def _consume(coro):
        coro.close()
        return MagicMock()

    with (
        patch.object(eval_jobs.asyncio, "create_subprocess_exec", new=create),
        patch.object(eval_jobs.asyncio, "create_task", new=_consume),
        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)),
        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)),
    ):
        record = await eval_jobs.start_reviewer_eval(
            limit=None,
            config=config,
            created_by="octo",
        )

    assert record["run_name"] == "custom-run"
    assert record["langsmith_project"] == "custom-project"
    assert record["config_snapshot"] == config

    args, kwargs = create.call_args
    assert "--experiment-prefix" in args and "custom-run" in args
    assert "--langsmith-project" in args and "custom-project" in args
    assert "--model-id" in args and "anthropic:claude-opus-4-8" in args
    assert "--score-mode" in args and "surfaced_findings" in args
    assert "--severity-threshold" in args and "high" in args
    assert "--cap" in args and "2" in args
    assert kwargs["env"]["LANGSMITH_PROJECT"] == "custom-project"


@pytest.mark.asyncio
async def test_start_reviewer_eval_rejects_when_running() -> None:
    running = MagicMock()
    running.returncode = None
    eval_jobs._PROCS[eval_jobs.REVIEWER_EVAL_KEY] = running
    with pytest.raises(RuntimeError):
        await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")


@pytest.mark.asyncio
async def test_stream_output_keeps_rolling_tail_and_experiment_url() -> None:
    url = "https://smith.langchain.com/o/x/experiments/abc"
    chunks = [
        f"starting eval {url}\n".encode(),
        *[f"row {i} done\n".encode() for i in range(2000)],
        b"",
    ]
    stdout = MagicMock()
    stdout.read = AsyncMock(side_effect=chunks)
    proc = MagicMock()
    proc.stdout = stdout

    tail, experiment_url = await eval_jobs._stream_output(proc)

    assert experiment_url == url
    assert len(tail) <= eval_jobs._LOG_TAIL_CHARS
    assert tail.endswith("row 1999 done\n")
    assert url not in tail  # scrolled out of the window but still captured
    assert eval_jobs.REVIEWER_EVAL_KEY not in tail


@pytest.mark.asyncio
async def test_start_reviewer_eval_rejects_fresh_run_on_other_worker() -> None:
    fresh = datetime.now(UTC).isoformat()
    record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
    with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)):
        with pytest.raises(RuntimeError):
            await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")