feat: live reviewer eval logs on a dedicated admin page (#1527)

* feat: stream reviewer eval logs on a dedicated admin page

Stream the eval subprocess output into a rolling log_tail and persist it
during the run (was only captured at exit), so the live output is visible
while the eval runs. Move the eval runner off the admin page onto its own
/admin/evals page (linked like Review Style Prompts) with a live log viewer.

* chore: drop unrelated SSR-register drift from generated route tree

* fix(ui): pre-bundle workbox-window to stop dev re-optimize reload

The PWA service worker (devOptions.enabled) pulls workbox-window, which
Vite discovers after first render and re-optimizes, forcing a reload that
cancels in-flight code-split route imports (Failed to fetch dynamically
imported module). Pre-bundling it via optimizeDeps.include avoids the
mid-session reload.

* fix(ui): suppress html hydration warning for pre-hydration theme script

The inline theme script sets class="dark"/color-scheme on <html> before
React hydrates, so the prerendered HTML never matches. suppressHydrationWarning
on <html> silences the (expected) one-level attribute mismatch.

---------

Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>

agent/dashboard/eval_jobs.py

@@ -45,6 +45,10 @@ _WORKER_ID = uuid.uuid4().hex
 # them. The store record is the source of truth across workers/requests.
 _PROCS: dict[str, asyncio.subprocess.Process] = {}
 
+# Rolling tail of subprocess output, kept by the owning worker so the heartbeat
+# loop can persist a live log tail to the store while the eval runs.
+_LOG_BUFFERS: dict[str, str] = {}
+
 
 def _client():
     return get_client()
@@ -246,7 +250,7 @@ async def cancel_reviewer_eval() -> dict[str, Any]:
 
 
 async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None:
-    """Refresh the record heartbeat while the owned subprocess is alive."""
+    """Refresh heartbeat and live log tail while the owned subprocess is alive."""
     while proc.returncode is None:
         await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS)
         if proc.returncode is not None:
@@ -254,7 +258,36 @@ async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None:
         record = await _get_record()
         if not record or record.get("status") != "running":
             return
-        await _put_record({**record, "heartbeat": _now_iso()})
+        await _put_record(
+            {
+                **record,
+                "heartbeat": _now_iso(),
+                "log_tail": _LOG_BUFFERS.get(REVIEWER_EVAL_KEY) or record.get("log_tail"),
+            }
+        )
+
+
+async def _stream_output(proc: asyncio.subprocess.Process) -> tuple[str, str | None]:
+    """Read stdout to EOF, keeping a rolling tail and the last experiment URL.
+
+    The tail is published to ``_LOG_BUFFERS`` as it grows so the heartbeat loop
+    can persist it mid-run. Reading in fixed chunks avoids the line-length cap
+    that ``StreamReader.readline`` would impose on long log lines.
+    """
+    tail = ""
+    experiment_url: str | None = None
+    if proc.stdout is None:
+        return tail, experiment_url
+    while True:
+        chunk = await proc.stdout.read(4096)
+        if not chunk:
+            break
+        tail = (tail + chunk.decode("utf-8", errors="replace"))[-_LOG_TAIL_CHARS:]
+        urls = _EXPERIMENT_URL_RE.findall(tail)
+        if urls:
+            experiment_url = urls[-1]
+        _LOG_BUFFERS[REVIEWER_EVAL_KEY] = tail
+    return tail, experiment_url
 
 
 async def _monitor(
@@ -265,10 +298,10 @@ async def _monitor(
     project: str,
 ) -> None:
     heartbeat = asyncio.create_task(_heartbeat_loop(proc))
-    output = b""
+    tail = ""
+    experiment_url: str | None = None
     try:
-        if proc.stdout is not None:
-            output = await proc.stdout.read()
+        tail, experiment_url = await _stream_output(proc)
         await proc.wait()
     except Exception:
         logger.exception("Error while monitoring reviewer eval subprocess")
@@ -277,11 +310,9 @@ async def _monitor(
         with contextlib.suppress(asyncio.CancelledError):
             await heartbeat
         _PROCS.pop(REVIEWER_EVAL_KEY, None)
+        _LOG_BUFFERS.pop(REVIEWER_EVAL_KEY, None)
 
-    text = output.decode("utf-8", errors="replace")
-    log_tail = text[-_LOG_TAIL_CHARS:] if text else None
-    urls = _EXPERIMENT_URL_RE.findall(text)
-    experiment_url = urls[-1] if urls else None
+    log_tail = tail or None
     exit_code = proc.returncode
     status: EvalStatus = "completed" if exit_code == 0 else "failed"
     error = None if status == "completed" else f"Eval exited with code {exit_code}."

tests/test_eval_jobs.py

@@ -93,6 +93,28 @@ async def test_start_reviewer_eval_rejects_when_running() -> None:
         await eval_jobs.start_reviewer_eval(limit=None, created_by="octo")
 
 
+@pytest.mark.asyncio
+async def test_stream_output_keeps_rolling_tail_and_experiment_url() -> None:
+    url = "https://smith.langchain.com/o/x/experiments/abc"
+    chunks = [
+        f"starting eval {url}\n".encode(),
+        *[f"row {i} done\n".encode() for i in range(2000)],
+        b"",
+    ]
+    stdout = MagicMock()
+    stdout.read = AsyncMock(side_effect=chunks)
+    proc = MagicMock()
+    proc.stdout = stdout
+
+    tail, experiment_url = await eval_jobs._stream_output(proc)
+
+    assert experiment_url == url
+    assert len(tail) <= eval_jobs._LOG_TAIL_CHARS
+    assert tail.endswith("row 1999 done\n")
+    assert url not in tail  # scrolled out of the window but still captured
+    assert eval_jobs.REVIEWER_EVAL_KEY not in tail
+
+
 @pytest.mark.asyncio
 async def test_start_reviewer_eval_rejects_fresh_run_on_other_worker() -> None:
     fresh = datetime.now(UTC).isoformat()

ui/src/routeTree.gen.ts

@@ -23,6 +23,7 @@ import { Route as ReviewStylesRouteImport } from './routes/review_.styles'
 import { Route as AgentsInstructionsRouteImport } from './routes/agents_.instructions'
 import { Route as AgentsThreadsRouteImport } from './routes/agents/threads'
 import { Route as AgentsThreadIdRouteImport } from './routes/agents/$threadId'
+import { Route as AdminEvalsRouteImport } from './routes/admin_.evals'
 import { Route as AgentsReviewsIndexRouteImport } from './routes/agents/reviews/index'
 import { Route as AgentsAutomationsIndexRouteImport } from './routes/agents/automations/index'
 import { Route as ReviewRepositoriesOwnerRouteImport } from './routes/review_.repositories.$owner'
@@ -100,6 +101,11 @@ const AgentsThreadIdRoute = AgentsThreadIdRouteImport.update({
   path: '/$threadId',
   getParentRoute: () => AgentsRoute,
 } as any)
+const AdminEvalsRoute = AdminEvalsRouteImport.update({
+  id: '/admin_/evals',
+  path: '/admin/evals',
+  getParentRoute: () => rootRouteImport,
+} as any)
 const AgentsReviewsIndexRoute = AgentsReviewsIndexRouteImport.update({
   id: '/reviews/',
   path: '/reviews/',
@@ -143,6 +149,7 @@ export interface FileRoutesByFullPath {
   '/my-settings': typeof MySettingsRoute
   '/review': typeof ReviewRoute
   '/usage': typeof UsageRoute
+  '/admin/evals': typeof AdminEvalsRoute
   '/agents/$threadId': typeof AgentsThreadIdRoute
   '/agents/threads': typeof AgentsThreadsRoute
   '/agents/instructions': typeof AgentsInstructionsRoute
@@ -164,6 +171,7 @@ export interface FileRoutesByTo {
   '/my-settings': typeof MySettingsRoute
   '/review': typeof ReviewRoute
   '/usage': typeof UsageRoute
+  '/admin/evals': typeof AdminEvalsRoute
   '/agents/$threadId': typeof AgentsThreadIdRoute
   '/agents/threads': typeof AgentsThreadsRoute
   '/agents/instructions': typeof AgentsInstructionsRoute
@@ -187,6 +195,7 @@ export interface FileRoutesById {
   '/my-settings': typeof MySettingsRoute
   '/review': typeof ReviewRoute
   '/usage': typeof UsageRoute
+  '/admin_/evals': typeof AdminEvalsRoute
   '/agents/$threadId': typeof AgentsThreadIdRoute
   '/agents/threads': typeof AgentsThreadsRoute
   '/agents_/instructions': typeof AgentsInstructionsRoute
@@ -211,6 +220,7 @@ export interface FileRouteTypes {
     | '/my-settings'
     | '/review'
     | '/usage'
+    | '/admin/evals'
     | '/agents/$threadId'
     | '/agents/threads'
     | '/agents/instructions'
@@ -232,6 +242,7 @@ export interface FileRouteTypes {
     | '/my-settings'
     | '/review'
     | '/usage'
+    | '/admin/evals'
     | '/agents/$threadId'
     | '/agents/threads'
     | '/agents/instructions'
@@ -254,6 +265,7 @@ export interface FileRouteTypes {
     | '/my-settings'
     | '/review'
     | '/usage'
+    | '/admin_/evals'
     | '/agents/$threadId'
     | '/agents/threads'
     | '/agents_/instructions'
@@ -277,6 +289,7 @@ export interface RootRouteChildren {
   MySettingsRoute: typeof MySettingsRoute
   ReviewRoute: typeof ReviewRoute
   UsageRoute: typeof UsageRoute
+  AdminEvalsRoute: typeof AdminEvalsRoute
   AgentsInstructionsRoute: typeof AgentsInstructionsRoute
   ReviewStylesRoute: typeof ReviewStylesRoute
   ReviewRepositoriesOwnerRoute: typeof ReviewRepositoriesOwnerRoute
@@ -382,6 +395,13 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof AgentsThreadIdRouteImport
       parentRoute: typeof AgentsRoute
     }
+    '/admin_/evals': {
+      id: '/admin_/evals'
+      path: '/admin/evals'
+      fullPath: '/admin/evals'
+      preLoaderRoute: typeof AdminEvalsRouteImport
+      parentRoute: typeof rootRouteImport
+    }
     '/agents/reviews/': {
       id: '/agents/reviews/'
       path: '/reviews'
@@ -462,6 +482,7 @@ const rootRouteChildren: RootRouteChildren = {
   MySettingsRoute: MySettingsRoute,
   ReviewRoute: ReviewRoute,
   UsageRoute: UsageRoute,
+  AdminEvalsRoute: AdminEvalsRoute,
   AgentsInstructionsRoute: AgentsInstructionsRoute,
   ReviewStylesRoute: ReviewStylesRoute,
   ReviewRepositoriesOwnerRoute: ReviewRepositoriesOwnerRoute,

ui/src/routes/__root.tsx

@@ -47,7 +47,7 @@ export const Route = createRootRoute({
 function RootDocument({ children }: { children: React.ReactNode }) {
   const [queryClient] = useState(() => makeQueryClient())
   return (
-    <html lang="en">
+    <html lang="en" suppressHydrationWarning>
       <head>
         <script dangerouslySetInnerHTML={{ __html: themeInitScript }} />
         <HeadContent />

ui/src/routes/admin.tsx

@@ -1,12 +1,12 @@
 import { Link, Navigate, createFileRoute } from "@tanstack/react-router"
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
+import { CaretRightIcon } from "@phosphor-icons/react"
 import { useEffect, useMemo, useState } from "react"
 
 import type {
   DatadogConnectBody,
   LangSmithConnectBody,
   ModelOption,
-  ReviewerEvalStatus,
   TeamSettings,
   UserMapping,
 } from "@/lib/api"
@@ -55,7 +55,20 @@ function AdminPage() {
 
       <TriggerReviewSection />
 
-      <ReviewerEvalSection />
+      <SettingsSection title="Evals">
+        <Link
+          to="/admin/evals"
+          className="flex items-center justify-between gap-6 px-4 py-3 hover:bg-muted/40"
+        >
+          <div className="flex flex-col gap-0.5">
+            <span className="text-xs font-medium text-foreground">Reviewer eval</span>
+            <span className="text-xs text-muted-foreground">
+              Run the offline reviewer benchmark and watch its output stream live.
+            </span>
+          </div>
+          <CaretRightIcon className="size-3.5 shrink-0 text-muted-foreground" />
+        </Link>
+      </SettingsSection>
 
       <ObservabilityCredentialsSection />
 
@@ -150,121 +163,6 @@ function TriggerReviewSection() {
   )
 }
 
-function ReviewerEvalSection() {
-  const qc = useQueryClient()
-  const [limit, setLimit] = useState("")
-  const [error, setError] = useState<string | null>(null)
-
-  const status = useQuery({
-    queryKey: ["reviewerEval"],
-    queryFn: api.getReviewerEval,
-    refetchInterval: (query) =>
-      query.state.data?.status === "running" ? 5000 : false,
-  })
-
-  const data = status.data
-  const running = data?.status === "running"
-
-  const onSuccess = (next: ReviewerEvalStatus) => {
-    qc.setQueryData(["reviewerEval"], next)
-    setError(null)
-  }
-  const onError = (e: Error) => setError(e.message)
-
-  const start = useMutation({
-    mutationFn: () => {
-      const n = limit.trim() ? Number(limit.trim()) : null
-      if (n !== null && (!Number.isInteger(n) || n <= 0)) {
-        throw new Error("Limit must be a positive whole number")
-      }
-      return api.startReviewerEval(n)
-    },
-    onSuccess,
-    onError,
-  })
-  const cancel = useMutation({
-    mutationFn: () => api.cancelReviewerEval(),
-    onSuccess,
-    onError,
-  })
-
-  return (
-    <SettingsSection
-      title="Reviewer eval"
-      description="Run the offline reviewer benchmark against the LangSmith dataset. Traces are sent to the open-swe-evals project."
-    >
-      <div className="flex flex-col gap-3 p-4">
-        <div className="flex items-center gap-2">
-          <Input
-            className="w-48"
-            type="number"
-            min={1}
-            placeholder="Limit (optional)"
-            value={limit}
-            disabled={running}
-            onChange={(e) => setLimit(e.target.value)}
-          />
-          <Button
-            size="sm"
-            onClick={() => start.mutate()}
-            disabled={running || start.isPending}
-          >
-            {start.isPending ? "Starting…" : "Run eval"}
-          </Button>
-          {running && (
-            <Button
-              size="sm"
-              variant="outline"
-              onClick={() => cancel.mutate()}
-              disabled={cancel.isPending}
-            >
-              Cancel
-            </Button>
-          )}
-        </div>
-
-        <p className="text-xs text-muted-foreground">
-          Leave the limit blank for the full dataset, or enter N to run only the
-          first N PRs (smoke test).
-        </p>
-
-        {data && (
-          <div className="flex flex-col gap-1 text-xs text-muted-foreground">
-            <span>
-              Status: <span className="font-medium">{data.status}</span>
-              {data.langsmith_project ? ` · ${data.langsmith_project}` : ""}
-              {data.limit ? ` · limit ${data.limit}` : ""}
-            </span>
-            {data.started_at && (
-              <span>Started: {new Date(data.started_at).toLocaleString()}</span>
-            )}
-            {data.finished_at && (
-              <span>
-                Finished: {new Date(data.finished_at).toLocaleString()}
-              </span>
-            )}
-            {data.experiment_url && (
-              <a
-                href={data.experiment_url}
-                target="_blank"
-                rel="noreferrer"
-                className="underline hover:text-foreground"
-              >
-                View experiment in LangSmith
-              </a>
-            )}
-            {data.error && (
-              <span className="text-destructive">{data.error}</span>
-            )}
-          </div>
-        )}
-
-        {error && <p className="text-xs text-destructive">{error}</p>}
-      </div>
-    </SettingsSection>
-  )
-}
-
 const PAGE_SIZE = 20
 
 function UserMappingsSection({ enabled }: { enabled: boolean }) {

ui/src/routes/admin_.evals.tsx

@@ -0,0 +1,200 @@
+import { Navigate, createFileRoute } from "@tanstack/react-router"
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
+import { useEffect, useRef, useState } from "react"
+
+import type { ReviewerEvalStatus } from "@/lib/api"
+import { AppShell, SettingsSection } from "@/components/AppShell"
+import { Button } from "@/components/ui/button"
+import { Input } from "@/components/ui/input"
+import { Skeleton } from "@/components/ui/skeleton"
+import { api } from "@/lib/api"
+import { useSession } from "@/lib/session"
+
+export const Route = createFileRoute("/admin_/evals")({ component: ReviewerEvalPage })
+
+function ReviewerEvalPage() {
+  const session = useSession()
+
+  if (session.isLoading) {
+    return (
+      <main className="p-6">
+        <Skeleton className="h-64 w-full" />
+      </main>
+    )
+  }
+  if (!session.data) return <Navigate to="/login" />
+  if (!session.data.is_admin) return <Navigate to="/my-settings" />
+
+  return (
+    <AppShell
+      user={session.data}
+      title="Reviewer eval"
+      description="Run the offline reviewer benchmark against the LangSmith dataset and watch its output stream live."
+      backTo={{ to: "/admin", label: "Back to Admin" }}
+    >
+      <ReviewerEvalRunner />
+      <ReviewerEvalLogs />
+    </AppShell>
+  )
+}
+
+function ReviewerEvalRunner() {
+  const qc = useQueryClient()
+  const [limit, setLimit] = useState("")
+  const [error, setError] = useState<string | null>(null)
+
+  const status = useQuery({
+    queryKey: ["reviewerEval"],
+    queryFn: api.getReviewerEval,
+    refetchInterval: (query) =>
+      query.state.data?.status === "running" ? 5000 : false,
+  })
+
+  const data = status.data
+  const running = data?.status === "running"
+
+  const onSuccess = (next: ReviewerEvalStatus) => {
+    qc.setQueryData(["reviewerEval"], next)
+    setError(null)
+  }
+  const onError = (e: Error) => setError(e.message)
+
+  const start = useMutation({
+    mutationFn: () => {
+      const n = limit.trim() ? Number(limit.trim()) : null
+      if (n !== null && (!Number.isInteger(n) || n <= 0)) {
+        throw new Error("Limit must be a positive whole number")
+      }
+      return api.startReviewerEval(n)
+    },
+    onSuccess,
+    onError,
+  })
+  const cancel = useMutation({
+    mutationFn: () => api.cancelReviewerEval(),
+    onSuccess,
+    onError,
+  })
+
+  return (
+    <SettingsSection
+      title="Run"
+      description="Traces are sent to the open-swe-evals project. Leave the limit blank for the full dataset, or enter N to run only the first N PRs (smoke test)."
+    >
+      <div className="flex flex-col gap-3 p-4">
+        <div className="flex items-center gap-2">
+          <Input
+            className="w-48"
+            type="number"
+            min={1}
+            placeholder="Limit (optional)"
+            value={limit}
+            disabled={running}
+            onChange={(e) => setLimit(e.target.value)}
+          />
+          <Button
+            size="sm"
+            onClick={() => start.mutate()}
+            disabled={running || start.isPending}
+          >
+            {start.isPending ? "Starting…" : "Run eval"}
+          </Button>
+          {running && (
+            <Button
+              size="sm"
+              variant="outline"
+              onClick={() => cancel.mutate()}
+              disabled={cancel.isPending}
+            >
+              Cancel
+            </Button>
+          )}
+        </div>
+
+        {data && (
+          <div className="flex flex-col gap-1 text-xs text-muted-foreground">
+            <span>
+              Status: <span className="font-medium">{data.status}</span>
+              {data.langsmith_project ? ` · ${data.langsmith_project}` : ""}
+              {data.limit ? ` · limit ${data.limit}` : ""}
+            </span>
+            {data.started_at && (
+              <span>Started: {new Date(data.started_at).toLocaleString()}</span>
+            )}
+            {data.finished_at && (
+              <span>
+                Finished: {new Date(data.finished_at).toLocaleString()}
+              </span>
+            )}
+            {data.experiment_url && (
+              <a
+                href={data.experiment_url}
+                target="_blank"
+                rel="noreferrer"
+                className="underline hover:text-foreground"
+              >
+                View experiment in LangSmith
+              </a>
+            )}
+            {data.error && <span className="text-destructive">{data.error}</span>}
+          </div>
+        )}
+
+        {error && <p className="text-xs text-destructive">{error}</p>}
+      </div>
+    </SettingsSection>
+  )
+}
+
+function ReviewerEvalLogs() {
+  const status = useQuery({ queryKey: ["reviewerEval"], queryFn: api.getReviewerEval })
+  const logTail = status.data?.log_tail ?? null
+  const running = status.data?.status === "running"
+
+  const scrollRef = useRef<HTMLPreElement>(null)
+  const [follow, setFollow] = useState(true)
+
+  useEffect(() => {
+    if (follow && scrollRef.current) {
+      scrollRef.current.scrollTop = scrollRef.current.scrollHeight
+    }
+  }, [logTail, follow])
+
+  return (
+    <SettingsSection
+      title="Output"
+      description="Last 4000 characters of the eval process output. Updates roughly every 10 seconds while running."
+      action={
+        <label className="flex items-center gap-1.5 text-xs text-muted-foreground">
+          <input
+            type="checkbox"
+            checked={follow}
+            onChange={(e) => setFollow(e.target.checked)}
+          />
+          Follow
+        </label>
+      }
+    >
+      <div className="p-4">
+        {logTail ? (
+          <pre
+            ref={scrollRef}
+            onScroll={(e) => {
+              const el = e.currentTarget
+              const atBottom =
+                el.scrollHeight - el.scrollTop - el.clientHeight < 24
+              setFollow(atBottom)
+            }}
+            className="max-h-[28rem] overflow-auto whitespace-pre-wrap break-words rounded-md bg-muted/50 p-3 font-mono text-xs text-foreground"
+          >
+            {logTail}
+          </pre>
+        ) : (
+          <p className="text-xs text-muted-foreground">
+            {running ? "Waiting for output…" : "No output yet. Run an eval to see logs here."}
+          </p>
+        )}
+      </div>
+    </SettingsSection>
+  )
+}

ui/vite.config.ts

@@ -8,6 +8,7 @@ import { nitro } from "nitro/vite"
 import { VitePWA } from "vite-plugin-pwa"
 
 const config = defineConfig({
+  optimizeDeps: { include: ["workbox-window"] },
   plugins: [
     devtools(),
     nitro(),