feat: trigger reviewer evals from the admin page (#1524)

* feat: trigger reviewer evals from the admin page

Add an admin-only "Reviewer eval" section + endpoints that launch the
reviewer benchmark as an isolated subprocess against the running
deployment, with live status and the LangSmith experiment link. Route
eval traces to a dedicated open-swe-evals project so they stay out of
the production tracing project.

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>

* fix: reconcile reviewer eval status via heartbeat, not local process

The persisted record is shared across workers but _PROCS is process-local.
The owning worker now refreshes a heartbeat while the subprocess runs, and
status is only reconciled to failed once the heartbeat is stale, so a poll on
a worker without the local handle no longer kills a live run (and a duplicate
start is rejected across workers).

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>

---------

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>

agent/dashboard/eval_jobs.py

@@ -0,0 +1,304 @@
+"""Kick off and track the reviewer eval from the admin dashboard.
+
+Runs ``evals.reviewer.run_eval`` as a subprocess so its LangSmith tracing
+project (``open-swe-evals``) stays isolated from the deployment's production
+project, and so a long eval does not block the server event loop. Status is
+persisted in the LangGraph store so it survives across dashboard requests.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import os
+import re
+import sys
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal
+
+from langgraph_sdk import get_client
+
+logger = logging.getLogger(__name__)
+
+EVALS_NAMESPACE: list[str] = ["evals"]
+REVIEWER_EVAL_KEY = "reviewer"
+DEFAULT_EVAL_PROJECT = "open-swe-evals"
+_MODULE = "evals.reviewer.run_eval"
+_LOG_TAIL_CHARS = 4000
+_EXPERIMENT_URL_RE = re.compile(r"https://\S*smith\.langchain\.com/\S+")
+# The owning worker refreshes the heartbeat this often while the subprocess
+# runs; a record is only reconciled as failed once its heartbeat is older than
+# the stale threshold, so polls on other workers don't kill a live run.
+_HEARTBEAT_INTERVAL_SECONDS = 10
+_HEARTBEAT_STALE_SECONDS = 60
+
+EvalStatus = Literal["idle", "running", "completed", "failed"]
+
+# Identifies this process so heartbeat ownership can be reasoned about across
+# workers that share the persisted record.
+_WORKER_ID = uuid.uuid4().hex
+
+# Live subprocess handles keyed by eval name, owned by the worker that launched
+# them. The store record is the source of truth across workers/requests.
+_PROCS: dict[str, asyncio.subprocess.Process] = {}
+
+
+def _client():
+    return get_client()
+
+
+def _now_iso() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def _resolve_langgraph_url() -> str | None:
+    return os.environ.get("LANGGRAPH_URL") or os.environ.get("LANGGRAPH_URL_PROD")
+
+
+def _eval_project() -> str:
+    return os.environ.get("EVAL_LANGSMITH_PROJECT") or DEFAULT_EVAL_PROJECT
+
+
+def _idle_record() -> dict[str, Any]:
+    return {
+        "name": REVIEWER_EVAL_KEY,
+        "status": "idle",
+        "langsmith_project": _eval_project(),
+        "limit": None,
+        "started_at": None,
+        "finished_at": None,
+        "created_by": None,
+        "pid": None,
+        "exit_code": None,
+        "experiment_url": None,
+        "error": None,
+        "log_tail": None,
+        "worker_id": None,
+        "heartbeat": None,
+        "updated_at": _now_iso(),
+    }
+
+
+async def _get_record() -> dict[str, Any] | None:
+    try:
+        item = await _client().store.get_item(EVALS_NAMESPACE, REVIEWER_EVAL_KEY)
+    except Exception as e:
+        logger.debug("store get_item failed for reviewer eval: %s", e)
+        return None
+    if item is None:
+        return None
+    value = item.get("value") if isinstance(item, dict) else getattr(item, "value", None)
+    return value if isinstance(value, dict) else None
+
+
+async def _put_record(record: dict[str, Any]) -> dict[str, Any]:
+    record = {**record, "updated_at": _now_iso()}
+    try:
+        await _client().store.put_item(EVALS_NAMESPACE, REVIEWER_EVAL_KEY, record)
+    except Exception:
+        logger.exception("Failed to persist reviewer eval status")
+    return record
+
+
+def _is_locally_running() -> bool:
+    proc = _PROCS.get(REVIEWER_EVAL_KEY)
+    return proc is not None and proc.returncode is None
+
+
+def _heartbeat_age_seconds(record: dict[str, Any]) -> float | None:
+    """Seconds since the record's heartbeat, or ``None`` if absent/unparseable."""
+    hb = record.get("heartbeat")
+    if not isinstance(hb, str) or not hb:
+        return None
+    try:
+        ts = datetime.fromisoformat(hb)
+    except ValueError:
+        return None
+    if ts.tzinfo is None:
+        ts = ts.replace(tzinfo=UTC)
+    return (datetime.now(UTC) - ts).total_seconds()
+
+
+def _is_heartbeat_fresh(record: dict[str, Any]) -> bool:
+    age = _heartbeat_age_seconds(record)
+    return age is not None and age <= _HEARTBEAT_STALE_SECONDS
+
+
+async def get_reviewer_eval_status() -> dict[str, Any]:
+    """Return the latest reviewer-eval status, reconciling stale ``running``.
+
+    The owning worker refreshes the record's heartbeat while its subprocess
+    runs. A poll from any worker only marks the run failed once the heartbeat
+    is stale, so a status check on a worker that doesn't own the process (no
+    local handle but a fresh heartbeat) leaves a live run untouched.
+    """
+    record = await _get_record()
+    if record is None:
+        return _idle_record()
+    if record.get("status") != "running":
+        return record
+    if _is_locally_running() or _is_heartbeat_fresh(record):
+        return record
+    return await _put_record(
+        {
+            **record,
+            "status": "failed",
+            "finished_at": record.get("finished_at") or _now_iso(),
+            "error": "Eval process is no longer tracked (server restarted?).",
+        }
+    )
+
+
+async def start_reviewer_eval(
+    *,
+    limit: int | None,
+    created_by: str,
+) -> dict[str, Any]:
+    """Launch the reviewer eval subprocess and persist a ``running`` record.
+
+    Raises ``RuntimeError`` if an eval is already running on this or another
+    worker (detected via a fresh heartbeat on the shared record).
+    """
+    if _is_locally_running():
+        raise RuntimeError("a reviewer eval is already running")
+    existing = await _get_record()
+    if existing and existing.get("status") == "running" and _is_heartbeat_fresh(existing):
+        raise RuntimeError("a reviewer eval is already running")
+
+    project = _eval_project()
+    cmd = [sys.executable, "-m", _MODULE]
+    if limit is not None and limit > 0:
+        cmd += ["--limit", str(limit)]
+
+    env = {
+        **os.environ,
+        "LANGSMITH_PROJECT": project,
+        "LANGCHAIN_PROJECT": project,
+        "LANGSMITH_TRACING": "true",
+    }
+    langgraph_url = _resolve_langgraph_url()
+    if langgraph_url:
+        env["LANGGRAPH_URL"] = langgraph_url
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=str(_repo_root()),
+            env=env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+        )
+    except Exception as exc:
+        logger.exception("Failed to launch reviewer eval subprocess")
+        return await _put_record(
+            {
+                **_idle_record(),
+                "status": "failed",
+                "finished_at": _now_iso(),
+                "created_by": created_by,
+                "error": f"Failed to launch eval: {exc}",
+            }
+        )
+
+    _PROCS[REVIEWER_EVAL_KEY] = proc
+    record = await _put_record(
+        {
+            **_idle_record(),
+            "status": "running",
+            "langsmith_project": project,
+            "limit": limit,
+            "started_at": _now_iso(),
+            "finished_at": None,
+            "created_by": created_by,
+            "pid": proc.pid,
+            "worker_id": _WORKER_ID,
+            "heartbeat": _now_iso(),
+        }
+    )
+    asyncio.create_task(_monitor(proc, created_by=created_by, limit=limit, project=project))
+    return record
+
+
+async def cancel_reviewer_eval() -> dict[str, Any]:
+    """Terminate a locally-running reviewer eval, if any."""
+    proc = _PROCS.get(REVIEWER_EVAL_KEY)
+    if proc is not None and proc.returncode is None:
+        try:
+            proc.terminate()
+        except ProcessLookupError:
+            pass
+    record = await _get_record() or _idle_record()
+    return await _put_record(
+        {
+            **record,
+            "status": "failed",
+            "finished_at": _now_iso(),
+            "error": "Eval cancelled by an admin.",
+        }
+    )
+
+
+async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None:
+    """Refresh the record heartbeat while the owned subprocess is alive."""
+    while proc.returncode is None:
+        await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS)
+        if proc.returncode is not None:
+            return
+        record = await _get_record()
+        if not record or record.get("status") != "running":
+            return
+        await _put_record({**record, "heartbeat": _now_iso()})
+
+
+async def _monitor(
+    proc: asyncio.subprocess.Process,
+    *,
+    created_by: str,
+    limit: int | None,
+    project: str,
+) -> None:
+    heartbeat = asyncio.create_task(_heartbeat_loop(proc))
+    output = b""
+    try:
+        if proc.stdout is not None:
+            output = await proc.stdout.read()
+        await proc.wait()
+    except Exception:
+        logger.exception("Error while monitoring reviewer eval subprocess")
+    finally:
+        heartbeat.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await heartbeat
+        _PROCS.pop(REVIEWER_EVAL_KEY, None)
+
+    text = output.decode("utf-8", errors="replace")
+    log_tail = text[-_LOG_TAIL_CHARS:] if text else None
+    urls = _EXPERIMENT_URL_RE.findall(text)
+    experiment_url = urls[-1] if urls else None
+    exit_code = proc.returncode
+    status: EvalStatus = "completed" if exit_code == 0 else "failed"
+    error = None if status == "completed" else f"Eval exited with code {exit_code}."
+
+    record = await _get_record() or _idle_record()
+    await _put_record(
+        {
+            **record,
+            "status": status,
+            "langsmith_project": project,
+            "limit": limit,
+            "created_by": created_by,
+            "finished_at": _now_iso(),
+            "pid": proc.pid,
+            "exit_code": exit_code,
+            "experiment_url": experiment_url,
+            "error": error,
+            "log_tail": log_tail,
+        }
+    )

agent/dashboard/routes.py

@@ -32,6 +32,11 @@ from .enabled_repos import (
     list_enabled_review_repos,
     set_review_repo_enabled,
 )
+from .eval_jobs import (
+    cancel_reviewer_eval,
+    get_reviewer_eval_status,
+    start_reviewer_eval,
+)
 from .oauth import (
     COOKIE_NAME,
     SESSION_TTL_SECONDS,
@@ -553,6 +558,38 @@ async def admin_delete_user_mapping(
     return {"deleted": deleted}
 
 
+class ReviewerEvalStartBody(BaseModel):
+    limit: int | None = None
+
+
+@router.get("/admin/evals/reviewer")
+async def admin_get_reviewer_eval(
+    _admin: dict[str, Any] = _ADMIN_DEP,
+) -> dict[str, Any]:
+    return await get_reviewer_eval_status()
+
+
+@router.post("/admin/evals/reviewer")
+async def admin_start_reviewer_eval(
+    body: ReviewerEvalStartBody,
+    session: dict[str, Any] = _ADMIN_DEP,
+) -> dict[str, Any]:
+    status = await get_reviewer_eval_status()
+    if status.get("status") == "running":
+        raise HTTPException(409, "a reviewer eval is already running")
+    try:
+        return await start_reviewer_eval(limit=body.limit, created_by=session["sub"])
+    except RuntimeError as exc:
+        raise HTTPException(409, str(exc)) from exc
+
+
+@router.delete("/admin/evals/reviewer")
+async def admin_cancel_reviewer_eval(
+    _admin: dict[str, Any] = _ADMIN_DEP,
+) -> dict[str, Any]:
+    return await cancel_reviewer_eval()
+
+
 def _next_link_url(link_header: str | None) -> str | None:
     if not link_header:
         return None

evals/reviewer/README.md

@@ -54,6 +54,23 @@ Smoke-test with 3 PRs first:
 uv run python -m evals.reviewer.run_eval --limit 3
 ```
 
+### From the admin dashboard
+
+Admins can also kick off the eval from the **Reviewer eval** section on the
+dashboard Admin page (no local shell needed). It launches the same
+`run_eval` runner as a subprocess against the running deployment, with an
+optional limit for a smoke test, and surfaces live status plus the LangSmith
+experiment link. The deployment must have `LANGSMITH_API_KEY` / `ANTHROPIC_API_KEY`
+in its environment.
+
+### Tracing project
+
+Eval traces are routed to the **`open-swe-evals`** LangSmith project (set via
+`langsmith_project` in `config.toml`, default `open-swe-evals`) so they stay out
+of the deployment's production tracing project. The admin-triggered run forces
+the same project via the `LANGSMITH_PROJECT` env var; override the default with
+`EVAL_LANGSMITH_PROJECT`.
+
 The runner reads benchmark settings from `evals/reviewer/config.toml`. Set the
 deployment URL there (or leave it blank to use `LANGGRAPH_URL` / local dev).
 The target sets `reviewer_eval` for every run, so `publish_review` does not post

evals/reviewer/config.toml

@@ -2,6 +2,10 @@ dataset_name = "openswe-reviewer-v1"
 experiment_prefix = "openswe-review-confidence"
 max_concurrency = 5
 
+# LangSmith tracing project for eval runs. Keeps eval traces out of the
+# deployment's production project.
+langsmith_project = "open-swe-evals"
+
 # Leave blank to use LANGGRAPH_URL or local dev.
 langgraph_url = ""
 assistant_id = "reviewer"

evals/reviewer/run_eval.py

@@ -25,6 +25,7 @@ from evals.reviewer.target import drain_thread_ids, get_langgraph_url, review_pr
 logger = logging.getLogger(__name__)
 
 CONFIG_PATH = Path(__file__).with_name("config.toml")
+DEFAULT_LANGSMITH_PROJECT = "open-swe-evals"
 ScoreMode = Literal["all_findings", "surfaced_findings"]
 Severity = Literal["low", "medium", "high", "critical"]
 
@@ -34,6 +35,7 @@ class ReviewerEvalConfig(TypedDict, total=False):
     experiment_prefix: str
     max_concurrency: int
     langgraph_url: str
+    langsmith_project: str
     assistant_id: str
     model_id: str
     reasoning_effort: str
@@ -64,6 +66,10 @@ def _coerce_config(raw: dict[str, Any]) -> ReviewerEvalConfig:
     if isinstance(langgraph_url, str) and langgraph_url:
         config["langgraph_url"] = langgraph_url
 
+    langsmith_project = raw.get("langsmith_project")
+    if isinstance(langsmith_project, str) and langsmith_project:
+        config["langsmith_project"] = langsmith_project
+
     assistant_id = raw.get("assistant_id")
     if isinstance(assistant_id, str) and assistant_id:
         config["assistant_id"] = assistant_id
@@ -108,6 +114,19 @@ def _apply_config_to_env(config: ReviewerEvalConfig) -> None:
         value = config.get(config_key)
         if value is not None:
             os.environ[env_key] = str(value)
+    _apply_langsmith_project(config.get("langsmith_project"))
+
+
+def _apply_langsmith_project(project: str | None) -> None:
+    """Route eval traces to a dedicated LangSmith project.
+
+    A project already set in the environment (e.g. by the admin-triggered job)
+    wins so callers can override the config default.
+    """
+    resolved = os.environ.get("LANGSMITH_PROJECT") or project or DEFAULT_LANGSMITH_PROJECT
+    os.environ["LANGSMITH_PROJECT"] = resolved
+    os.environ["LANGCHAIN_PROJECT"] = resolved
+    os.environ.setdefault("LANGSMITH_TRACING", "true")
 
 
 async def _cleanup_threads(thread_ids: Iterable[str]) -> None:

tests/test_eval_jobs.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from agent.dashboard import eval_jobs
+
+
+@pytest.fixture(autouse=True)
+def _clear_procs():
+    eval_jobs._PROCS.clear()
+    yield
+    eval_jobs._PROCS.clear()
+
+
+@pytest.mark.asyncio
+async def test_get_status_returns_idle_when_no_record() -> None:
+    with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)):
+        status = await eval_jobs.get_reviewer_eval_status()
+    assert status["status"] == "idle"
+    assert status["name"] == eval_jobs.REVIEWER_EVAL_KEY
+
+
+@pytest.mark.asyncio
+async def test_get_status_reconciles_stale_running() -> None:
+    stale = (datetime.now(UTC) - timedelta(seconds=300)).isoformat()
+    record = {"name": "reviewer", "status": "running", "heartbeat": stale}
+    with (
+        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
+        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
+    ):
+        status = await eval_jobs.get_reviewer_eval_status()
+    assert status["status"] == "failed"
+    assert "no longer tracked" in status["error"]
+    put.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_get_status_keeps_running_with_fresh_heartbeat() -> None:
+    """A poll on a worker without the local handle must not kill a live run."""
+    fresh = datetime.now(UTC).isoformat()
+    record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
+    with (
+        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)),
+        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)) as put,
+    ):
+        status = await eval_jobs.get_reviewer_eval_status()
+    assert status["status"] == "running"
+    put.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_start_reviewer_eval_launches_subprocess() -> None:
+    proc = MagicMock()
+    proc.pid = 4321
+    proc.returncode = None
+    create = AsyncMock(return_value=proc)
+
+    def _consume(coro):
+        coro.close()
+        return MagicMock()
+
+    with (
+        patch.object(eval_jobs.asyncio, "create_subprocess_exec", new=create),
+        patch.object(eval_jobs.asyncio, "create_task", new=_consume),
+        patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=None)),
+        patch.object(eval_jobs, "_put_record", new=AsyncMock(side_effect=lambda r: r)),
+        patch.object(eval_jobs, "_resolve_langgraph_url", return_value="https://lg.test"),
+    ):
+        record = await eval_jobs.start_reviewer_eval(limit=3, created_by="octo")
+
+    assert record["status"] == "running"
+    assert record["limit"] == 3
+    assert record["pid"] == 4321
+    assert record["heartbeat"] is not None
+    assert record["worker_id"] == eval_jobs._WORKER_ID
+    assert eval_jobs._PROCS[eval_jobs.REVIEWER_EVAL_KEY] is proc
+
+    args, kwargs = create.call_args
+    assert "--limit" in args and "3" in args
+    assert kwargs["env"]["LANGSMITH_PROJECT"] == eval_jobs.DEFAULT_EVAL_PROJECT
+    assert kwargs["env"]["LANGGRAPH_URL"] == "https://lg.test"
+
+
+@pytest.mark.asyncio
+async def test_start_reviewer_eval_rejects_when_running() -> None:
+    running = MagicMock()
+    running.returncode = None
+    eval_jobs._PROCS[eval_jobs.REVIEWER_EVAL_KEY] = running
+    with pytest.raises(RuntimeError):
+        await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")
+
+
+@pytest.mark.asyncio
+async def test_start_reviewer_eval_rejects_fresh_run_on_other_worker() -> None:
+    fresh = datetime.now(UTC).isoformat()
+    record = {"name": "reviewer", "status": "running", "heartbeat": fresh}
+    with patch.object(eval_jobs, "_get_record", new=AsyncMock(return_value=record)):
+        with pytest.raises(RuntimeError):
+            await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")

tests/test_reviewer_eval_run.py

@@ -3,7 +3,12 @@ from __future__ import annotations
 import os
 from unittest.mock import patch
 
-from evals.reviewer.run_eval import _apply_config_to_env, _coerce_config
+from evals.reviewer.run_eval import (
+    DEFAULT_LANGSMITH_PROJECT,
+    _apply_config_to_env,
+    _apply_langsmith_project,
+    _coerce_config,
+)
 
 
 def test_reviewer_eval_config_coerces_known_values() -> None:
@@ -58,3 +63,29 @@ def test_reviewer_eval_config_sets_target_env() -> None:
         assert os.environ["REVIEWER_EVAL_SCORE_MODE"] == "surfaced_findings"
         assert os.environ["REVIEWER_EVAL_SEVERITY_THRESHOLD"] == "high"
         assert os.environ["REVIEWER_EVAL_CAP"] == "3"
+
+
+def test_reviewer_eval_config_coerces_langsmith_project() -> None:
+    config = _coerce_config({"langsmith_project": "open-swe-evals"})
+    assert config == {"langsmith_project": "open-swe-evals"}
+
+
+def test_apply_langsmith_project_uses_config_default() -> None:
+    with patch.dict(os.environ, {}, clear=True):
+        _apply_langsmith_project("my-eval-project")
+        assert os.environ["LANGSMITH_PROJECT"] == "my-eval-project"
+        assert os.environ["LANGCHAIN_PROJECT"] == "my-eval-project"
+        assert os.environ["LANGSMITH_TRACING"] == "true"
+
+
+def test_apply_langsmith_project_falls_back_to_default() -> None:
+    with patch.dict(os.environ, {}, clear=True):
+        _apply_langsmith_project(None)
+        assert os.environ["LANGSMITH_PROJECT"] == DEFAULT_LANGSMITH_PROJECT
+
+
+def test_apply_langsmith_project_env_overrides_config() -> None:
+    with patch.dict(os.environ, {"LANGSMITH_PROJECT": "from-env"}, clear=True):
+        _apply_langsmith_project("from-config")
+        assert os.environ["LANGSMITH_PROJECT"] == "from-env"
+        assert os.environ["LANGCHAIN_PROJECT"] == "from-env"

ui/src/lib/api.ts

@@ -379,6 +379,22 @@ export interface ReviewDiffPayload {
   truncated: boolean;
 }
 
+export interface ReviewerEvalStatus {
+  name: string;
+  status: "idle" | "running" | "completed" | "failed";
+  langsmith_project: string;
+  limit: number | null;
+  started_at: string | null;
+  finished_at: string | null;
+  created_by: string | null;
+  pid: number | null;
+  exit_code: number | null;
+  experiment_url: string | null;
+  error: string | null;
+  log_tail: string | null;
+  updated_at: string;
+}
+
 export const api = {
   me: () => request<SessionUser>("/me"),
   options: () => request<OptionsPayload>("/options"),
@@ -487,6 +503,15 @@ export const api = {
       `/reviews/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${number}/re-review`,
       { method: "POST" },
     ),
+  getReviewerEval: () =>
+    request<ReviewerEvalStatus>("/admin/evals/reviewer"),
+  startReviewerEval: (limit: number | null) =>
+    request<ReviewerEvalStatus>("/admin/evals/reviewer", {
+      method: "POST",
+      body: JSON.stringify({ limit }),
+    }),
+  cancelReviewerEval: () =>
+    request<ReviewerEvalStatus>("/admin/evals/reviewer", { method: "DELETE" }),
   logout: () => request<void>("/auth/logout", { method: "POST" }),
 };
 

ui/src/routes/admin.tsx

@@ -6,6 +6,7 @@ import type {
   DatadogConnectBody,
   LangSmithConnectBody,
   ModelOption,
+  ReviewerEvalStatus,
   TeamSettings,
   UserMapping,
 } from "@/lib/api"
@@ -54,6 +55,8 @@ function AdminPage() {
 
       <TriggerReviewSection />
 
+      <ReviewerEvalSection />
+
       <ObservabilityCredentialsSection />
 
       <UserMappingsSection enabled={!!session.data.is_admin} />
@@ -147,6 +150,121 @@ function TriggerReviewSection() {
   )
 }
 
+function ReviewerEvalSection() {
+  const qc = useQueryClient()
+  const [limit, setLimit] = useState("")
+  const [error, setError] = useState<string | null>(null)
+
+  const status = useQuery({
+    queryKey: ["reviewerEval"],
+    queryFn: api.getReviewerEval,
+    refetchInterval: (query) =>
+      query.state.data?.status === "running" ? 5000 : false,
+  })
+
+  const data = status.data
+  const running = data?.status === "running"
+
+  const onSuccess = (next: ReviewerEvalStatus) => {
+    qc.setQueryData(["reviewerEval"], next)
+    setError(null)
+  }
+  const onError = (e: Error) => setError(e.message)
+
+  const start = useMutation({
+    mutationFn: () => {
+      const n = limit.trim() ? Number(limit.trim()) : null
+      if (n !== null && (!Number.isInteger(n) || n <= 0)) {
+        throw new Error("Limit must be a positive whole number")
+      }
+      return api.startReviewerEval(n)
+    },
+    onSuccess,
+    onError,
+  })
+  const cancel = useMutation({
+    mutationFn: () => api.cancelReviewerEval(),
+    onSuccess,
+    onError,
+  })
+
+  return (
+    <SettingsSection
+      title="Reviewer eval"
+      description="Run the offline reviewer benchmark against the LangSmith dataset. Traces are sent to the open-swe-evals project."
+    >
+      <div className="flex flex-col gap-3 p-4">
+        <div className="flex items-center gap-2">
+          <Input
+            className="w-48"
+            type="number"
+            min={1}
+            placeholder="Limit (optional)"
+            value={limit}
+            disabled={running}
+            onChange={(e) => setLimit(e.target.value)}
+          />
+          <Button
+            size="sm"
+            onClick={() => start.mutate()}
+            disabled={running || start.isPending}
+          >
+            {start.isPending ? "Starting…" : "Run eval"}
+          </Button>
+          {running && (
+            <Button
+              size="sm"
+              variant="outline"
+              onClick={() => cancel.mutate()}
+              disabled={cancel.isPending}
+            >
+              Cancel
+            </Button>
+          )}
+        </div>
+
+        <p className="text-xs text-muted-foreground">
+          Leave the limit blank for the full dataset, or enter N to run only the
+          first N PRs (smoke test).
+        </p>
+
+        {data && (
+          <div className="flex flex-col gap-1 text-xs text-muted-foreground">
+            <span>
+              Status: <span className="font-medium">{data.status}</span>
+              {data.langsmith_project ? ` · ${data.langsmith_project}` : ""}
+              {data.limit ? ` · limit ${data.limit}` : ""}
+            </span>
+            {data.started_at && (
+              <span>Started: {new Date(data.started_at).toLocaleString()}</span>
+            )}
+            {data.finished_at && (
+              <span>
+                Finished: {new Date(data.finished_at).toLocaleString()}
+              </span>
+            )}
+            {data.experiment_url && (
+              <a
+                href={data.experiment_url}
+                target="_blank"
+                rel="noreferrer"
+                className="underline hover:text-foreground"
+              >
+                View experiment in LangSmith
+              </a>
+            )}
+            {data.error && (
+              <span className="text-destructive">{data.error}</span>
+            )}
+          </div>
+        )}
+
+        {error && <p className="text-xs text-destructive">{error}</p>}
+      </div>
+    </SettingsSection>
+  )
+}
+
 const PAGE_SIZE = 20
 
 function UserMappingsSection({ enabled }: { enabled: boolean }) {