diff --git a/agent/dashboard/eval_jobs.py b/agent/dashboard/eval_jobs.py index b0dd8627..9acf2092 100644 --- a/agent/dashboard/eval_jobs.py +++ b/agent/dashboard/eval_jobs.py @@ -45,6 +45,10 @@ _WORKER_ID = uuid.uuid4().hex # them. The store record is the source of truth across workers/requests. _PROCS: dict[str, asyncio.subprocess.Process] = {} +# Rolling tail of subprocess output, kept by the owning worker so the heartbeat +# loop can persist a live log tail to the store while the eval runs. +_LOG_BUFFERS: dict[str, str] = {} + def _client(): return get_client() @@ -246,7 +250,7 @@ async def cancel_reviewer_eval() -> dict[str, Any]: async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None: - """Refresh the record heartbeat while the owned subprocess is alive.""" + """Refresh heartbeat and live log tail while the owned subprocess is alive.""" while proc.returncode is None: await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS) if proc.returncode is not None: @@ -254,7 +258,36 @@ async def _heartbeat_loop(proc: asyncio.subprocess.Process) -> None: record = await _get_record() if not record or record.get("status") != "running": return - await _put_record({**record, "heartbeat": _now_iso()}) + await _put_record( + { + **record, + "heartbeat": _now_iso(), + "log_tail": _LOG_BUFFERS.get(REVIEWER_EVAL_KEY) or record.get("log_tail"), + } + ) + + +async def _stream_output(proc: asyncio.subprocess.Process) -> tuple[str, str | None]: + """Read stdout to EOF, keeping a rolling tail and the last experiment URL. + + The tail is published to ``_LOG_BUFFERS`` as it grows so the heartbeat loop + can persist it mid-run. Reading in fixed chunks avoids the line-length cap + that ``StreamReader.readline`` would impose on long log lines. + """ + tail = "" + experiment_url: str | None = None + if proc.stdout is None: + return tail, experiment_url + while True: + chunk = await proc.stdout.read(4096) + if not chunk: + break + tail = (tail + chunk.decode("utf-8", errors="replace"))[-_LOG_TAIL_CHARS:] + urls = _EXPERIMENT_URL_RE.findall(tail) + if urls: + experiment_url = urls[-1] + _LOG_BUFFERS[REVIEWER_EVAL_KEY] = tail + return tail, experiment_url async def _monitor( @@ -265,10 +298,10 @@ async def _monitor( project: str, ) -> None: heartbeat = asyncio.create_task(_heartbeat_loop(proc)) - output = b"" + tail = "" + experiment_url: str | None = None try: - if proc.stdout is not None: - output = await proc.stdout.read() + tail, experiment_url = await _stream_output(proc) await proc.wait() except Exception: logger.exception("Error while monitoring reviewer eval subprocess") @@ -277,11 +310,9 @@ async def _monitor( with contextlib.suppress(asyncio.CancelledError): await heartbeat _PROCS.pop(REVIEWER_EVAL_KEY, None) + _LOG_BUFFERS.pop(REVIEWER_EVAL_KEY, None) - text = output.decode("utf-8", errors="replace") - log_tail = text[-_LOG_TAIL_CHARS:] if text else None - urls = _EXPERIMENT_URL_RE.findall(text) - experiment_url = urls[-1] if urls else None + log_tail = tail or None exit_code = proc.returncode status: EvalStatus = "completed" if exit_code == 0 else "failed" error = None if status == "completed" else f"Eval exited with code {exit_code}." diff --git a/tests/test_eval_jobs.py b/tests/test_eval_jobs.py index 0e2b4acb..fe926b40 100644 --- a/tests/test_eval_jobs.py +++ b/tests/test_eval_jobs.py @@ -93,6 +93,28 @@ async def test_start_reviewer_eval_rejects_when_running() -> None: await eval_jobs.start_reviewer_eval(limit=None, created_by="octo") +@pytest.mark.asyncio +async def test_stream_output_keeps_rolling_tail_and_experiment_url() -> None: + url = "https://smith.langchain.com/o/x/experiments/abc" + chunks = [ + f"starting eval {url}\n".encode(), + *[f"row {i} done\n".encode() for i in range(2000)], + b"", + ] + stdout = MagicMock() + stdout.read = AsyncMock(side_effect=chunks) + proc = MagicMock() + proc.stdout = stdout + + tail, experiment_url = await eval_jobs._stream_output(proc) + + assert experiment_url == url + assert len(tail) <= eval_jobs._LOG_TAIL_CHARS + assert tail.endswith("row 1999 done\n") + assert url not in tail # scrolled out of the window but still captured + assert eval_jobs.REVIEWER_EVAL_KEY not in tail + + @pytest.mark.asyncio async def test_start_reviewer_eval_rejects_fresh_run_on_other_worker() -> None: fresh = datetime.now(UTC).isoformat() diff --git a/ui/src/routeTree.gen.ts b/ui/src/routeTree.gen.ts index 568ed87f..caa083da 100644 --- a/ui/src/routeTree.gen.ts +++ b/ui/src/routeTree.gen.ts @@ -23,6 +23,7 @@ import { Route as ReviewStylesRouteImport } from './routes/review_.styles' import { Route as AgentsInstructionsRouteImport } from './routes/agents_.instructions' import { Route as AgentsThreadsRouteImport } from './routes/agents/threads' import { Route as AgentsThreadIdRouteImport } from './routes/agents/$threadId' +import { Route as AdminEvalsRouteImport } from './routes/admin_.evals' import { Route as AgentsReviewsIndexRouteImport } from './routes/agents/reviews/index' import { Route as AgentsAutomationsIndexRouteImport } from './routes/agents/automations/index' import { Route as ReviewRepositoriesOwnerRouteImport } from './routes/review_.repositories.$owner' @@ -100,6 +101,11 @@ const AgentsThreadIdRoute = AgentsThreadIdRouteImport.update({ path: '/$threadId', getParentRoute: () => AgentsRoute, } as any) +const AdminEvalsRoute = AdminEvalsRouteImport.update({ + id: '/admin_/evals', + path: '/admin/evals', + getParentRoute: () => rootRouteImport, +} as any) const AgentsReviewsIndexRoute = AgentsReviewsIndexRouteImport.update({ id: '/reviews/', path: '/reviews/', @@ -143,6 +149,7 @@ export interface FileRoutesByFullPath { '/my-settings': typeof MySettingsRoute '/review': typeof ReviewRoute '/usage': typeof UsageRoute + '/admin/evals': typeof AdminEvalsRoute '/agents/$threadId': typeof AgentsThreadIdRoute '/agents/threads': typeof AgentsThreadsRoute '/agents/instructions': typeof AgentsInstructionsRoute @@ -164,6 +171,7 @@ export interface FileRoutesByTo { '/my-settings': typeof MySettingsRoute '/review': typeof ReviewRoute '/usage': typeof UsageRoute + '/admin/evals': typeof AdminEvalsRoute '/agents/$threadId': typeof AgentsThreadIdRoute '/agents/threads': typeof AgentsThreadsRoute '/agents/instructions': typeof AgentsInstructionsRoute @@ -187,6 +195,7 @@ export interface FileRoutesById { '/my-settings': typeof MySettingsRoute '/review': typeof ReviewRoute '/usage': typeof UsageRoute + '/admin_/evals': typeof AdminEvalsRoute '/agents/$threadId': typeof AgentsThreadIdRoute '/agents/threads': typeof AgentsThreadsRoute '/agents_/instructions': typeof AgentsInstructionsRoute @@ -211,6 +220,7 @@ export interface FileRouteTypes { | '/my-settings' | '/review' | '/usage' + | '/admin/evals' | '/agents/$threadId' | '/agents/threads' | '/agents/instructions' @@ -232,6 +242,7 @@ export interface FileRouteTypes { | '/my-settings' | '/review' | '/usage' + | '/admin/evals' | '/agents/$threadId' | '/agents/threads' | '/agents/instructions' @@ -254,6 +265,7 @@ export interface FileRouteTypes { | '/my-settings' | '/review' | '/usage' + | '/admin_/evals' | '/agents/$threadId' | '/agents/threads' | '/agents_/instructions' @@ -277,6 +289,7 @@ export interface RootRouteChildren { MySettingsRoute: typeof MySettingsRoute ReviewRoute: typeof ReviewRoute UsageRoute: typeof UsageRoute + AdminEvalsRoute: typeof AdminEvalsRoute AgentsInstructionsRoute: typeof AgentsInstructionsRoute ReviewStylesRoute: typeof ReviewStylesRoute ReviewRepositoriesOwnerRoute: typeof ReviewRepositoriesOwnerRoute @@ -382,6 +395,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AgentsThreadIdRouteImport parentRoute: typeof AgentsRoute } + '/admin_/evals': { + id: '/admin_/evals' + path: '/admin/evals' + fullPath: '/admin/evals' + preLoaderRoute: typeof AdminEvalsRouteImport + parentRoute: typeof rootRouteImport + } '/agents/reviews/': { id: '/agents/reviews/' path: '/reviews' @@ -462,6 +482,7 @@ const rootRouteChildren: RootRouteChildren = { MySettingsRoute: MySettingsRoute, ReviewRoute: ReviewRoute, UsageRoute: UsageRoute, + AdminEvalsRoute: AdminEvalsRoute, AgentsInstructionsRoute: AgentsInstructionsRoute, ReviewStylesRoute: ReviewStylesRoute, ReviewRepositoriesOwnerRoute: ReviewRepositoriesOwnerRoute, diff --git a/ui/src/routes/__root.tsx b/ui/src/routes/__root.tsx index de488a8f..8f0560eb 100644 --- a/ui/src/routes/__root.tsx +++ b/ui/src/routes/__root.tsx @@ -47,7 +47,7 @@ export const Route = createRootRoute({ function RootDocument({ children }: { children: React.ReactNode }) { const [queryClient] = useState(() => makeQueryClient()) return ( - +