From be2fe7131ba107866811135b7b5a88fc14f9b460 Mon Sep 17 00:00:00 2001 From: Johannes du Plessis Date: Mon, 15 Jun 2026 11:29:16 -0700 Subject: [PATCH] feat: live reviewer eval logs on a dedicated admin page (#1527) * feat: stream reviewer eval logs on a dedicated admin page Stream the eval subprocess output into a rolling log_tail and persist it during the run (was only captured at exit), so the live output is visible while the eval runs. Move the eval runner off the admin page onto its own /admin/evals page (linked like Review Style Prompts) with a live log viewer. * chore: drop unrelated SSR-register drift from generated route tree * fix(ui): pre-bundle workbox-window to stop dev re-optimize reload The PWA service worker (devOptions.enabled) pulls workbox-window, which Vite discovers after first render and re-optimizes, forcing a reload that cancels in-flight code-split route imports (Failed to fetch dynamically imported module). Pre-bundling it via optimizeDeps.include avoids the mid-session reload. * fix(ui): suppress html hydration warning for pre-hydration theme script The inline theme script sets class="dark"/color-scheme on before React hydrates, so the prerendered HTML never matches. suppressHydrationWarning on silences the (expected) one-level attribute mismatch. --------- Co-authored-by: open-swe[bot] --- agent/dashboard/eval_jobs.py | 49 ++++++-- tests/test_eval_jobs.py | 22 ++++ ui/src/routeTree.gen.ts | 21 ++++ ui/src/routes/__root.tsx | 2 +- ui/src/routes/admin.tsx | 132 +++------------------- ui/src/routes/admin_.evals.tsx | 200 +++++++++++++++++++++++++++++++++ ui/vite.config.ts | 1 + 7 files changed, 300 insertions(+), 127 deletions(-) create mode 100644 ui/src/routes/admin_.evals.tsx diff --git a/agent/dashboard/eval_jobs.py b/agent/dashboard/eval_jobs.py index b0dd8627..9acf2092 100644 --- a/agent/dashboard/eval_jobs.py +++ b/agent/dashboard/eval_jobs.py @@ -45,6 +45,10 @@ _WORKER_ID = uuid.uuid4().hex # them. The store record is the source of truth across workers/requests. _PROCS: dict[str, asyncio.subprocess.Process] = {} +# Rolling tail of subprocess output, kept by the owning worker so the heartbeat +# loop can persist a live log tail to the store while the eval runs. +_LOG_BUFFERS: dict[str, str] = {} + def _client(): return get_client() @@ -246,7 +250,7 @@ async def cancel_reviewer_eval() -> dict[str, Any]: async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None: - """Refresh the record heartbeat while the owned subprocess is alive.""" + """Refresh heartbeat and live log tail while the owned subprocess is alive.""" while proc.returncode is None: await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS) if proc.returncode is not None: @@ -254,7 +258,36 @@ async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None: record = await _get_record() if not record or record.get("status") != "running": return - await _put_record({**record, "heartbeat": _now_iso()}) + await _put_record( + { + **record, + "heartbeat": _now_iso(), + "log_tail": _LOG_BUFFERS.get(REVIEWER_EVAL_KEY) or record.get("log_tail"), + } + ) + + +async def _stream_output(proc: asyncio.subprocess.Process) -> tuple[str, str | None]: + """Read stdout to EOF, keeping a rolling tail and the last experiment URL. + + The tail is published to ``_LOG_BUFFERS`` as it grows so the heartbeat loop + can persist it mid-run. Reading in fixed chunks avoids the line-length cap + that ``StreamReader.readline`` would impose on long log lines. + """ + tail = "" + experiment_url: str | None = None + if proc.stdout is None: + return tail, experiment_url + while True: + chunk = await proc.stdout.read(4096) + if not chunk: + break + tail = (tail + chunk.decode("utf-8", errors="replace"))[-_LOG_TAIL_CHARS:] + urls = _EXPERIMENT_URL_RE.findall(tail) + if urls: + experiment_url = urls[-1] + _LOG_BUFFERS[REVIEWER_EVAL_KEY] = tail + return tail, experiment_url async def _monitor( @@ -265,10 +298,10 @@ async def _monitor( project: str, ) -> None: heartbeat = asyncio.create_task(_heartbeat_loop(proc)) - output = b"" + tail = "" + experiment_url: str | None = None try: - if proc.stdout is not None: - output = await proc.stdout.read() + tail, experiment_url = await _stream_output(proc) await proc.wait() except Exception: logger.exception("Error while monitoring reviewer eval subprocess") @@ -277,11 +310,9 @@ async def _monitor( with contextlib.suppress(asyncio.CancelledError): await heartbeat _PROCS.pop(REVIEWER_EVAL_KEY, None) + _LOG_BUFFERS.pop(REVIEWER_EVAL_KEY, None) - text = output.decode("utf-8", errors="replace") - log_tail = text[-_LOG_TAIL_CHARS:] if text else None - urls = _EXPERIMENT_URL_RE.findall(text) - experiment_url = urls[-1] if urls else None + log_tail = tail or None exit_code = proc.returncode status: EvalStatus = "completed" if exit_code == 0 else "failed" error = None if status == "completed" else f"Eval exited with code {exit_code}." diff --git a/tests/test_eval_jobs.py b/tests/test_eval_jobs.py index 0e2b4acb..fe926b40 100644 --- a/tests/test_eval_jobs.py +++ b/tests/test_eval_jobs.py @@ -93,6 +93,28 @@ async def test_start_reviewer_eval_rejects_when_running() -> None: await eval_jobs.start_reviewer_eval(limit=None, created_by="octo") +@pytest.mark.asyncio +async def test_stream_output_keeps_rolling_tail_and_experiment_url() -> None: + url = "https://smith.langchain.com/o/x/experiments/abc" + chunks = [ + f"starting eval {url}\n".encode(), + *[f"row {i} done\n".encode() for i in range(2000)], + b"", + ] + stdout = MagicMock() + stdout.read = AsyncMock(side_effect=chunks) + proc = MagicMock() + proc.stdout = stdout + + tail, experiment_url = await eval_jobs._stream_output(proc) + + assert experiment_url == url + assert len(tail) <= eval_jobs._LOG_TAIL_CHARS + assert tail.endswith("row 1999 done\n") + assert url not in tail # scrolled out of the window but still captured + assert eval_jobs.REVIEWER_EVAL_KEY not in tail + + @pytest.mark.asyncio async def test_start_reviewer_eval_rejects_fresh_run_on_other_worker() -> None: fresh = datetime.now(UTC).isoformat() diff --git a/ui/src/routeTree.gen.ts b/ui/src/routeTree.gen.ts index 568ed87f..caa083da 100644 --- a/ui/src/routeTree.gen.ts +++ b/ui/src/routeTree.gen.ts @@ -23,6 +23,7 @@ import { Route as ReviewStylesRouteImport } from './routes/review_.styles' import { Route as AgentsInstructionsRouteImport } from './routes/agents_.instructions' import { Route as AgentsThreadsRouteImport } from './routes/agents/threads' import { Route as AgentsThreadIdRouteImport } from './routes/agents/$threadId' +import { Route as AdminEvalsRouteImport } from './routes/admin_.evals' import { Route as AgentsReviewsIndexRouteImport } from './routes/agents/reviews/index' import { Route as AgentsAutomationsIndexRouteImport } from './routes/agents/automations/index' import { Route as ReviewRepositoriesOwnerRouteImport } from './routes/review_.repositories.$owner' @@ -100,6 +101,11 @@ const AgentsThreadIdRoute = AgentsThreadIdRouteImport.update({ path: '/$threadId', getParentRoute: () => AgentsRoute, } as any) +const AdminEvalsRoute = AdminEvalsRouteImport.update({ + id: '/admin_/evals', + path: '/admin/evals', + getParentRoute: () => rootRouteImport, +} as any) const AgentsReviewsIndexRoute = AgentsReviewsIndexRouteImport.update({ id: '/reviews/', path: '/reviews/', @@ -143,6 +149,7 @@ export interface FileRoutesByFullPath { '/my-settings': typeof MySettingsRoute '/review': typeof ReviewRoute '/usage': typeof UsageRoute + '/admin/evals': typeof AdminEvalsRoute '/agents/$threadId': typeof AgentsThreadIdRoute '/agents/threads': typeof AgentsThreadsRoute '/agents/instructions': typeof AgentsInstructionsRoute @@ -164,6 +171,7 @@ export interface FileRoutesByTo { '/my-settings': typeof MySettingsRoute '/review': typeof ReviewRoute '/usage': typeof UsageRoute + '/admin/evals': typeof AdminEvalsRoute '/agents/$threadId': typeof AgentsThreadIdRoute '/agents/threads': typeof AgentsThreadsRoute '/agents/instructions': typeof AgentsInstructionsRoute @@ -187,6 +195,7 @@ export interface FileRoutesById { '/my-settings': typeof MySettingsRoute '/review': typeof ReviewRoute '/usage': typeof UsageRoute + '/admin_/evals': typeof AdminEvalsRoute '/agents/$threadId': typeof AgentsThreadIdRoute '/agents/threads': typeof AgentsThreadsRoute '/agents_/instructions': typeof AgentsInstructionsRoute @@ -211,6 +220,7 @@ export interface FileRouteTypes { | '/my-settings' | '/review' | '/usage' + | '/admin/evals' | '/agents/$threadId' | '/agents/threads' | '/agents/instructions' @@ -232,6 +242,7 @@ export interface FileRouteTypes { | '/my-settings' | '/review' | '/usage' + | '/admin/evals' | '/agents/$threadId' | '/agents/threads' | '/agents/instructions' @@ -254,6 +265,7 @@ export interface FileRouteTypes { | '/my-settings' | '/review' | '/usage' + | '/admin_/evals' | '/agents/$threadId' | '/agents/threads' | '/agents_/instructions' @@ -277,6 +289,7 @@ export interface RootRouteChildren { MySettingsRoute: typeof MySettingsRoute ReviewRoute: typeof ReviewRoute UsageRoute: typeof UsageRoute + AdminEvalsRoute: typeof AdminEvalsRoute AgentsInstructionsRoute: typeof AgentsInstructionsRoute ReviewStylesRoute: typeof ReviewStylesRoute ReviewRepositoriesOwnerRoute: typeof ReviewRepositoriesOwnerRoute @@ -382,6 +395,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AgentsThreadIdRouteImport parentRoute: typeof AgentsRoute } + '/admin_/evals': { + id: '/admin_/evals' + path: '/admin/evals' + fullPath: '/admin/evals' + preLoaderRoute: typeof AdminEvalsRouteImport + parentRoute: typeof rootRouteImport + } '/agents/reviews/': { id: '/agents/reviews/' path: '/reviews' @@ -462,6 +482,7 @@ const rootRouteChildren: RootRouteChildren = { MySettingsRoute: MySettingsRoute, ReviewRoute: ReviewRoute, UsageRoute: UsageRoute, + AdminEvalsRoute: AdminEvalsRoute, AgentsInstructionsRoute: AgentsInstructionsRoute, ReviewStylesRoute: ReviewStylesRoute, ReviewRepositoriesOwnerRoute: ReviewRepositoriesOwnerRoute, diff --git a/ui/src/routes/__root.tsx b/ui/src/routes/__root.tsx index de488a8f..8f0560eb 100644 --- a/ui/src/routes/__root.tsx +++ b/ui/src/routes/__root.tsx @@ -47,7 +47,7 @@ export const Route = createRootRoute({ function RootDocument({ children }: { children: React.ReactNode }) { const [queryClient] = useState(() => makeQueryClient()) return ( - +