diff --git a/agent/dashboard/agent_usage.py b/agent/dashboard/agent_usage.py index 0dadf9ae..a53ddd7e 100644 --- a/agent/dashboard/agent_usage.py +++ b/agent/dashboard/agent_usage.py @@ -5,22 +5,28 @@ from __future__ import annotations import asyncio import logging from collections import Counter +from collections.abc import Awaitable, Callable from datetime import UTC, datetime, timedelta from typing import Any, Literal import httpx from langgraph_sdk import get_client +from ..reviewer_findings import REVIEWER_THREAD_KIND from ..utils.github_app import get_github_app_installation_token USAGE_THREAD_NAMESPACE: list[str] = ["agent_usage", "threads"] USAGE_PR_NAMESPACE: list[str] = ["agent_usage", "prs"] +USAGE_LEADERBOARD_CACHE_NAMESPACE: list[str] = ["agent_usage", "leaderboard_cache"] +REVIEWER_STATS_CACHE_NAMESPACE: list[str] = ["agent_usage", "reviewer_stats_cache"] Period = Literal["7d", "30d", "all"] _AGENT_SOURCES = frozenset({"dashboard", "github", "slack", "linear"}) _PR_REFRESH_INTERVAL_MS = 10 * 60 * 1000 _MAX_PR_REFRESH_PER_REQUEST = 25 _PR_REFRESH_CONCURRENCY = 5 +_CACHE_TTL_MS = 10 * 60 * 1000 +_CACHE_SEARCH_LIMIT = 1000 _GITHUB_API = "https://api.github.com" logger = logging.getLogger(__name__) @@ -64,7 +70,9 @@ async def _get_value(namespace: list[str], key: str) -> dict[str, Any] | None: return _record_from_item(item) -async def _search_values(namespace: list[str], *, limit: int = 1000) -> list[dict[str, Any]]: +async def _search_values( + namespace: list[str], *, limit: int = _CACHE_SEARCH_LIMIT +) -> list[dict[str, Any]]: result = await _client().store.search_items(namespace, limit=limit) items = result.get("items") if isinstance(result, dict) else getattr(result, "items", []) values: list[dict[str, Any]] = [] @@ -95,6 +103,26 @@ def _coerce_int(value: object) -> int: return 0 +def _timestamp_ms(value: object) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int | float): + raw = int(value) + return raw if raw > 10_000_000_000 else raw * 1000 + if isinstance(value, str) and value.strip(): + raw = value.strip() + if raw.isdigit(): + return _timestamp_ms(int(raw)) + try: + parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return 0 + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return int(parsed.timestamp() * 1000) + return 0 + + def _in_period(record: dict[str, Any], cutoff_ms: int | None) -> bool: if cutoff_ms is None: return True @@ -309,17 +337,27 @@ async def _refresh_pr_records(records: list[dict[str, Any]]) -> list[dict[str, A return refreshed -async def list_agent_usage_leaderboard( - *, - period: str | None, - limit: int, - current_login: str | None, - current_email: str | None, -) -> dict[str, Any]: - """Build the Open SWE Agent usage leaderboard from recorded telemetry.""" - normalized_period = _normalize_period(period) - cutoff_ms = _period_cutoff_ms(normalized_period) - safe_limit = min(max(limit, 1), 100) +def _serialize_usage_user(index: int, user: dict[str, Any]) -> dict[str, Any]: + model_counts: Counter[str] = user.get("model_counts", Counter()) + favorite_model = model_counts.most_common(1)[0][0] if model_counts else "default" + return { + "rank": index, + "key": user["key"], + "name": user.get("name") or "Unknown user", + "github_login": user.get("github_login") or None, + "email": user.get("email") or None, + "favorite_model": favorite_model, + "agent_runs": user["agent_runs"], + "prs_opened": user["prs_opened"], + "merged_prs": user["merged_prs"], + "agent_loc": user["agent_loc"], + "additions": user["additions"], + "deletions": user["deletions"], + } + + +async def _build_usage_leaderboard_snapshot(period: Period) -> dict[str, Any]: + cutoff_ms = _period_cutoff_ms(period) users: dict[str, dict[str, Any]] = {} for thread in await _search_values(USAGE_THREAD_NAMESPACE): @@ -366,6 +404,236 @@ async def list_agent_usage_leaderboard( item.get("name") or "", ), ) + return { + "period": period, + "users": [_serialize_usage_user(index, user) for index, user in enumerate(sorted_users, 1)], + "total_members": len(sorted_users), + } + + +async def refresh_usage_leaderboard_cache(period: str | None = "30d") -> dict[str, Any]: + normalized_period = _normalize_period(period) + snapshot = await _build_usage_leaderboard_snapshot(normalized_period) + await _client().store.put_item( + USAGE_LEADERBOARD_CACHE_NAMESPACE, + normalized_period, + {"generated_at_ms": _now_ms(), "snapshot": snapshot}, + ) + return snapshot + + +def _is_finding_surfaced(finding: dict[str, Any]) -> bool: + surface = finding.get("surface") if isinstance(finding.get("surface"), dict) else {} + state = surface.get("state") + if state in {"surfaced", "resolve_pending", "resolved"}: + return True + if isinstance(finding.get("github_review_id"), int): + return True + if isinstance(finding.get("github_review_comment_id"), int): + return True + comment_ids = finding.get("github_review_comment_ids") + thread_ids = finding.get("github_review_thread_ids") + return bool(comment_ids or thread_ids) + + +def _is_finding_resolved_by_us(finding: dict[str, Any]) -> bool: + if finding.get("status") != "resolved" or not _is_finding_surfaced(finding): + return False + surface = finding.get("surface") if isinstance(finding.get("surface"), dict) else {} + return bool( + surface.get("state") == "resolved" + or finding.get("github_thread_resolved") + or finding.get("github_resolved_thread_ids") + or finding.get("resolution_note") + ) + + +def _thread_created_at_ms(thread: dict[str, Any], metadata: dict[str, Any]) -> int: + timestamp = _thread_explicit_created_at_ms(thread, metadata) + if timestamp: + return timestamp + for source in ( + metadata.get("updated_at_ms"), + metadata.get("updated_at"), + thread.get("updated_at"), + thread.get("updatedAt"), + ): + timestamp = _timestamp_ms(source) + if timestamp: + return timestamp + return _now_ms() + + +def _thread_explicit_created_at_ms(thread: dict[str, Any], metadata: dict[str, Any]) -> int: + for source in ( + metadata.get("created_at_ms"), + metadata.get("created_at"), + thread.get("created_at"), + thread.get("createdAt"), + ): + timestamp = _timestamp_ms(source) + if timestamp: + return timestamp + return 0 + + +def _counter_rows(counter: Counter[str], *, limit: int = 5) -> list[dict[str, Any]]: + return [{"name": name, "count": count} for name, count in counter.most_common(limit)] + + +async def _iter_reviewer_thread_pages(cutoff_ms: int | None): + client = _client() + offset = 0 + while True: + page = await client.threads.search( + metadata={"kind": REVIEWER_THREAD_KIND}, + limit=_CACHE_SEARCH_LIMIT, + offset=offset, + sort_by="created_at", + sort_order="desc", + ) + if not page: + return + yield page + if len(page) < _CACHE_SEARCH_LIMIT: + return + if cutoff_ms is not None: + last_thread = next( + (thread for thread in reversed(page) if isinstance(thread, dict)), None + ) + metadata = last_thread.get("metadata") if isinstance(last_thread, dict) else None + if isinstance(metadata, dict): + last_created_at_ms = _thread_explicit_created_at_ms(last_thread, metadata) + if last_created_at_ms and last_created_at_ms < cutoff_ms: + return + offset += len(page) + + +async def _build_reviewer_stats_snapshot(period: Period) -> dict[str, Any]: + cutoff_ms = _period_cutoff_ms(period) + + reviewed_prs = 0 + prs_with_findings = 0 + findings_recorded = 0 + surfaced_findings = 0 + addressed_findings = 0 + dismissed_findings = 0 + human_replies = 0 + resolved_after_update = 0 + severity_counts: Counter[str] = Counter() + category_counts: Counter[str] = Counter() + + async for page in _iter_reviewer_thread_pages(cutoff_ms): + for thread in page: + if not isinstance(thread, dict): + continue + metadata = thread.get("metadata") if isinstance(thread.get("metadata"), dict) else {} + if cutoff_ms is not None and _thread_created_at_ms(thread, metadata) < cutoff_ms: + continue + + reviewed_prs += 1 + findings = metadata.get("findings") + if not isinstance(findings, list): + continue + valid_findings = [finding for finding in findings if isinstance(finding, dict)] + if valid_findings: + prs_with_findings += 1 + for finding in valid_findings: + findings_recorded += 1 + severity = finding.get("severity") + if isinstance(severity, str) and severity: + severity_counts[severity] += 1 + category = finding.get("category") + if isinstance(category, str) and category: + category_counts[category] += 1 + interactions = finding.get("interactions") + if isinstance(interactions, list): + human_replies += sum( + 1 + for interaction in interactions + if isinstance(interaction, dict) + and interaction.get("kind") == "human_reply" + ) + elif finding.get("last_human_reply_at"): + human_replies += 1 + + surfaced = _is_finding_surfaced(finding) + if surfaced: + surfaced_findings += 1 + if _is_finding_resolved_by_us(finding): + addressed_findings += 1 + first_seen_sha = finding.get("first_seen_sha") + head_sha = metadata.get("head_sha") or finding.get("last_confirmed_sha") + if ( + isinstance(first_seen_sha, str) + and isinstance(head_sha, str) + and first_seen_sha != head_sha + ): + resolved_after_update += 1 + if finding.get("status") == "dismissed": + dismissed_findings += 1 + + unresolved_surfaced_findings = max( + 0, surfaced_findings - addressed_findings - dismissed_findings + ) + resolution_rate = addressed_findings / surfaced_findings if surfaced_findings else 0.0 + return { + "period": period, + "reviewed_prs": reviewed_prs, + "prs_with_findings": prs_with_findings, + "findings_recorded": findings_recorded, + "surfaced_findings": surfaced_findings, + "addressed_findings": addressed_findings, + "resolved_after_update": resolved_after_update, + "dismissed_findings": dismissed_findings, + "unresolved_surfaced_findings": unresolved_surfaced_findings, + "resolution_rate": resolution_rate, + "human_replies": human_replies, + "severity_counts": dict(severity_counts), + "top_categories": _counter_rows(category_counts), + } + + +async def refresh_reviewer_stats_cache(period: str | None = "30d") -> dict[str, Any]: + normalized_period = _normalize_period(period) + snapshot = await _build_reviewer_stats_snapshot(normalized_period) + await _client().store.put_item( + REVIEWER_STATS_CACHE_NAMESPACE, + normalized_period, + {"generated_at_ms": _now_ms(), "snapshot": snapshot}, + ) + return snapshot + + +async def _cached_snapshot( + namespace: list[str], + period: Period, + refresh: Callable[[str | None], Awaitable[dict[str, Any]]], + *, + schedule_refresh: Callable[[Period], None] | None = None, +) -> tuple[dict[str, Any], int | None]: + cached = await _get_value(namespace, period) + if cached: + snapshot = cached.get("snapshot") + generated_at_ms = _coerce_int(cached.get("generated_at_ms")) + if isinstance(snapshot, dict): + if not generated_at_ms or _now_ms() - generated_at_ms <= _CACHE_TTL_MS: + return snapshot, generated_at_ms or None + if schedule_refresh is not None: + schedule_refresh(period) + return snapshot, generated_at_ms + return await refresh(period), _now_ms() + + +def _usage_payload_from_snapshot( + snapshot: dict[str, Any], + *, + limit: int, + current_login: str | None, + current_email: str | None, + generated_at_ms: int | None, +) -> dict[str, Any]: + safe_limit = min(max(limit, 1), 100) current_keys = { _user_key(current_login, current_email), _user_key(current_login, None), @@ -373,25 +641,27 @@ async def list_agent_usage_leaderboard( } rows: list[dict[str, Any]] = [] current_user_row: dict[str, Any] | None = None - for index, user in enumerate(sorted_users, start=1): - model_counts: Counter[str] = user.pop("model_counts") - favorite_model = model_counts.most_common(1)[0][0] if model_counts else "default" - github_login = user.get("github_login") or None - is_current_user = user["key"] in current_keys + for user in snapshot.get("users", []): + if not isinstance(user, dict): + continue + github_login = ( + user.get("github_login") if isinstance(user.get("github_login"), str) else None + ) + is_current_user = user.get("key") in current_keys row = { - "rank": index, + "rank": _coerce_int(user.get("rank")), "user": { "name": user.get("name") if is_current_user or github_login else "Open SWE user", "github_login": github_login, "email": (user.get("email") or None) if is_current_user else None, }, - "favorite_model": favorite_model, - "agent_runs": user["agent_runs"], - "prs_opened": user["prs_opened"], - "merged_prs": user["merged_prs"], - "agent_loc": user["agent_loc"], - "additions": user["additions"], - "deletions": user["deletions"], + "favorite_model": user.get("favorite_model") or "default", + "agent_runs": _coerce_int(user.get("agent_runs")), + "prs_opened": _coerce_int(user.get("prs_opened")), + "merged_prs": _coerce_int(user.get("merged_prs")), + "agent_loc": _coerce_int(user.get("agent_loc")), + "additions": _coerce_int(user.get("additions")), + "deletions": _coerce_int(user.get("deletions")), } if is_current_user: current_user_row = row @@ -402,8 +672,43 @@ async def list_agent_usage_leaderboard( rows.append(current_user_row) return { - "period": normalized_period, + "period": snapshot.get("period") or "30d", "rows": rows, - "total_members": len(sorted_users), + "total_members": _coerce_int(snapshot.get("total_members")), "current_user_rank": current_user_row["rank"] if current_user_row else None, + "generated_at_ms": generated_at_ms, } + + +async def list_agent_usage_leaderboard( + *, + period: str | None, + limit: int, + current_login: str | None, + current_email: str | None, + schedule_usage_refresh: Callable[[Period], None] | None = None, + schedule_reviewer_refresh: Callable[[Period], None] | None = None, +) -> dict[str, Any]: + """Return cached Open SWE Agent and reviewer usage stats.""" + normalized_period = _normalize_period(period) + usage_snapshot, generated_at_ms = await _cached_snapshot( + USAGE_LEADERBOARD_CACHE_NAMESPACE, + normalized_period, + refresh_usage_leaderboard_cache, + schedule_refresh=schedule_usage_refresh, + ) + reviewer_stats, reviewer_generated_at_ms = await _cached_snapshot( + REVIEWER_STATS_CACHE_NAMESPACE, + normalized_period, + refresh_reviewer_stats_cache, + schedule_refresh=schedule_reviewer_refresh, + ) + payload = _usage_payload_from_snapshot( + usage_snapshot, + limit=limit, + current_login=current_login, + current_email=current_email, + generated_at_ms=generated_at_ms, + ) + payload["reviewer_stats"] = {**reviewer_stats, "generated_at_ms": reviewer_generated_at_ms} + return payload diff --git a/agent/dashboard/routes.py b/agent/dashboard/routes.py index 11752cbe..3c2b46c5 100644 --- a/agent/dashboard/routes.py +++ b/agent/dashboard/routes.py @@ -8,12 +8,16 @@ import os from typing import Any import httpx -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request from fastapi.responses import RedirectResponse, Response, StreamingResponse from pydantic import BaseModel from .admin import is_admin -from .agent_usage import list_agent_usage_leaderboard +from .agent_usage import ( + list_agent_usage_leaderboard, + refresh_reviewer_stats_cache, + refresh_usage_leaderboard_cache, +) from .analyzer_cron import remove_continual_cron from .enabled_repos import ( list_enabled_review_repos, @@ -706,6 +710,7 @@ async def api_delete_review_style( @router.get("/agent-usage-leaderboard") async def api_agent_usage_leaderboard( + background_tasks: BackgroundTasks, period: str | None = "30d", limit: int = 10, session: dict[str, Any] = _SESSION_DEP, @@ -715,6 +720,12 @@ async def api_agent_usage_leaderboard( limit=limit, current_login=session["sub"], current_email=session.get("email"), + schedule_usage_refresh=lambda cache_period: background_tasks.add_task( + refresh_usage_leaderboard_cache, cache_period + ), + schedule_reviewer_refresh=lambda cache_period: background_tasks.add_task( + refresh_reviewer_stats_cache, cache_period + ), ) diff --git a/tests/test_agent_usage.py b/tests/test_agent_usage.py new file mode 100644 index 00000000..b340558f --- /dev/null +++ b/tests/test_agent_usage.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import pytest + +from agent.dashboard import agent_usage + + +class FakeStore: + def __init__(self, values: dict[tuple[tuple[str, ...], str], dict] | None = None): + self.values = values or {} + self.puts: list[tuple[list[str], str, dict]] = [] + + async def get_item(self, namespace: list[str], key: str) -> dict | None: + value = self.values.get((tuple(namespace), key)) + return {"value": value} if value is not None else None + + async def put_item(self, namespace: list[str], key: str, value: dict) -> None: + self.puts.append((namespace, key, value)) + self.values[(tuple(namespace), key)] = value + + +class FakeThreads: + def __init__(self, threads: list[dict]): + self.threads = threads + self.calls: list[dict] = [] + + async def search(self, **kwargs) -> list[dict]: + self.calls.append(kwargs) + offset = kwargs.get("offset") or 0 + limit = kwargs.get("limit") or len(self.threads) + return self.threads[offset : offset + limit] + + +class FakeClient: + def __init__(self, *, store: FakeStore | None = None, threads: FakeThreads | None = None): + self.store = store or FakeStore() + self.threads = threads or FakeThreads([]) + + +@pytest.mark.asyncio +async def test_cached_usage_payload_returns_stale_snapshot_and_schedules_refresh(monkeypatch): + usage_snapshot = { + "period": "30d", + "total_members": 2, + "users": [ + { + "rank": 1, + "key": "github:octo", + "name": "octo", + "github_login": "octo", + "email": "octo@example.com", + "favorite_model": "claude", + "agent_runs": 3, + "prs_opened": 2, + "merged_prs": 1, + "agent_loc": 10, + "additions": 8, + "deletions": 2, + }, + { + "rank": 2, + "key": "email:private@example.com", + "name": "private", + "github_login": None, + "email": "private@example.com", + "favorite_model": "default", + "agent_runs": 1, + "prs_opened": 0, + "merged_prs": 0, + "agent_loc": 0, + "additions": 0, + "deletions": 0, + }, + ], + } + reviewer_snapshot = { + "period": "30d", + "reviewed_prs": 1, + "prs_with_findings": 1, + "findings_recorded": 1, + "surfaced_findings": 1, + "addressed_findings": 0, + "resolved_after_update": 0, + "dismissed_findings": 0, + "unresolved_surfaced_findings": 1, + "resolution_rate": 0.0, + "human_replies": 0, + "severity_counts": {"medium": 1}, + "top_categories": [{"name": "correctness", "count": 1}], + } + store = FakeStore( + { + (tuple(agent_usage.USAGE_LEADERBOARD_CACHE_NAMESPACE), "30d"): { + "generated_at_ms": 1, + "snapshot": usage_snapshot, + }, + (tuple(agent_usage.REVIEWER_STATS_CACHE_NAMESPACE), "30d"): { + "generated_at_ms": 1, + "snapshot": reviewer_snapshot, + }, + } + ) + monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(store=store)) + monkeypatch.setattr(agent_usage, "_now_ms", lambda: agent_usage._CACHE_TTL_MS + 2) + + usage_refreshes: list[str] = [] + reviewer_refreshes: list[str] = [] + payload = await agent_usage.list_agent_usage_leaderboard( + period="30d", + limit=1, + current_login="octo", + current_email="octo@example.com", + schedule_usage_refresh=usage_refreshes.append, + schedule_reviewer_refresh=reviewer_refreshes.append, + ) + + assert usage_refreshes == ["30d"] + assert reviewer_refreshes == ["30d"] + assert payload["rows"][0]["user"]["email"] == "octo@example.com" + assert payload["total_members"] == 2 + assert payload["reviewer_stats"]["surfaced_findings"] == 1 + assert store.puts == [] + + +@pytest.mark.asyncio +async def test_reviewer_stats_snapshot_counts_surfaced_and_resolved_findings(monkeypatch): + threads = [ + { + "created_at": "2025-01-01T00:00:00Z", + "metadata": { + "kind": "reviewer", + "head_sha": "fixed-sha", + "pr": {"owner": "langchain-ai", "name": "open-swe", "number": 1}, + "findings": [ + { + "id": "f_1", + "status": "resolved", + "severity": "high", + "category": "correctness", + "first_seen_sha": "buggy-sha", + "github_thread_resolved": True, + "github_review_comment_id": 10, + "resolution_note": "Fixed in a follow-up commit.", + "interactions": [{"kind": "human_reply"}], + }, + { + "id": "f_2", + "status": "open", + "severity": "medium", + "category": "performance", + "github_review_comment_id": 11, + }, + { + "id": "f_3", + "status": "dismissed", + "severity": "low", + "category": "style", + "github_review_id": 12, + }, + ], + }, + } + ] + monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(threads=FakeThreads(threads))) + + snapshot = await agent_usage._build_reviewer_stats_snapshot("all") + + assert snapshot["reviewed_prs"] == 1 + assert snapshot["prs_with_findings"] == 1 + assert snapshot["findings_recorded"] == 3 + assert snapshot["surfaced_findings"] == 3 + assert snapshot["addressed_findings"] == 1 + assert snapshot["resolved_after_update"] == 1 + assert snapshot["dismissed_findings"] == 1 + assert snapshot["unresolved_surfaced_findings"] == 1 + assert snapshot["human_replies"] == 1 + assert snapshot["severity_counts"] == {"high": 1, "medium": 1, "low": 1} + assert snapshot["top_categories"] == [ + {"name": "correctness", "count": 1}, + {"name": "performance", "count": 1}, + {"name": "style", "count": 1}, + ] + + +@pytest.mark.asyncio +async def test_reviewer_stats_paginates_reviewer_threads(monkeypatch): + threads = [ + {"created_at": "2025-01-03T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}}, + {"created_at": "2025-01-02T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}}, + {"created_at": "2025-01-01T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}}, + ] + fake_threads = FakeThreads(threads) + monkeypatch.setattr(agent_usage, "_CACHE_SEARCH_LIMIT", 2) + monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(threads=fake_threads)) + + snapshot = await agent_usage._build_reviewer_stats_snapshot("all") + + assert snapshot["reviewed_prs"] == 3 + assert [call["offset"] for call in fake_threads.calls] == [0, 2] + assert all(call["sort_by"] == "created_at" for call in fake_threads.calls) + + +@pytest.mark.asyncio +async def test_reviewer_stats_stops_after_page_older_than_cutoff(monkeypatch): + threads = [ + {"created_at": "2025-01-03T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}}, + {"created_at": "2025-01-01T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}}, + {"created_at": "2024-12-31T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}}, + ] + fake_threads = FakeThreads(threads) + monkeypatch.setattr(agent_usage, "_CACHE_SEARCH_LIMIT", 2) + monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(threads=fake_threads)) + cutoff_ms = agent_usage._timestamp_ms("2025-01-02T00:00:00Z") + + pages = [page async for page in agent_usage._iter_reviewer_thread_pages(cutoff_ms)] + + assert len(pages) == 1 + assert [call["offset"] for call in fake_threads.calls] == [0] diff --git a/ui/src/components/AppSidebar.tsx b/ui/src/components/AppSidebar.tsx index 41bef682..a7d8ec14 100644 --- a/ui/src/components/AppSidebar.tsx +++ b/ui/src/components/AppSidebar.tsx @@ -30,8 +30,8 @@ interface NavItem { const NAV: Array = [ { to: "/my-settings", label: "Profile Settings", icon: IoOptionsOutline }, { to: "/cloud-agents", label: "Open SWE Agent", icon: IoCloudOutline }, - { to: "/usage", label: "Usage Leaderboard", icon: IoStatsChartOutline }, { to: "/review", label: "Open SWE Review", icon: IoGitPullRequestOutline }, + { to: "/usage", label: "Usage", icon: IoStatsChartOutline }, { to: "/admin", label: "Admin", icon: IoSettingsOutline, adminOnly: true }, ]; diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 3c96e7c6..a3f02dcf 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -157,11 +157,35 @@ export interface UsageLeaderboardRow { deletions: number; } +export interface ReviewerStatsCounterRow { + name: string; + count: number; +} + +export interface ReviewerStatsPayload { + period: UsageLeaderboardPeriod; + reviewed_prs: number; + prs_with_findings: number; + findings_recorded: number; + surfaced_findings: number; + addressed_findings: number; + resolved_after_update: number; + dismissed_findings: number; + unresolved_surfaced_findings: number; + resolution_rate: number; + human_replies: number; + severity_counts: Record; + top_categories: Array; + generated_at_ms: number | null; +} + export interface UsageLeaderboardPayload { period: UsageLeaderboardPeriod; rows: Array; total_members: number; current_user_rank: number | null; + generated_at_ms: number | null; + reviewer_stats: ReviewerStatsPayload; } export interface Repository { diff --git a/ui/src/routes/usage.tsx b/ui/src/routes/usage.tsx index d3d33823..eb5333eb 100644 --- a/ui/src/routes/usage.tsx +++ b/ui/src/routes/usage.tsx @@ -1,7 +1,11 @@ import { Navigate, createFileRoute } from "@tanstack/react-router" import { useQuery } from "@tanstack/react-query" -import type { UsageLeaderboardPeriod, UsageLeaderboardRow } from "@/lib/api" +import type { + ReviewerStatsPayload, + UsageLeaderboardPeriod, + UsageLeaderboardRow, +} from "@/lib/api" import { AppShell, SettingsSection } from "@/components/AppShell" import { Avatar, AvatarFallback } from "@/components/ui/avatar" import { @@ -43,6 +47,7 @@ function UsagePage() { queryKey: ["usageLeaderboard", activePeriod], queryFn: () => api.usageLeaderboard(activePeriod, 10), enabled: !!session.data, + staleTime: 5 * 60 * 1000, }) if (session.isLoading) { @@ -55,13 +60,9 @@ function UsagePage() { if (!session.data) return return ( - + ) : leaderboard.isError ? (

- Failed to load usage leaderboard: {leaderboard.error.message} + Failed to load usage data: {leaderboard.error.message}

) : !leaderboard.data?.rows.length ? (
@@ -108,6 +109,31 @@ function UsagePage() { /> )} + + + {leaderboard.isLoading ? ( +
+ + + + +
+ ) : leaderboard.isError ? ( +

+ Failed to load reviewer stats: {leaderboard.error.message} +

+ ) : leaderboard.data?.reviewer_stats ? ( + + ) : ( +
+ No reviewer stats have been recorded for{" "} + {PERIOD_LABELS[activePeriod].toLowerCase()} yet. +
+ )} + ) } @@ -172,6 +198,105 @@ function UsageTable({ ) } +function ReviewerStats({ stats }: { stats: ReviewerStatsPayload }) { + const cards = [ + { + label: "Reviewed PRs", + value: stats.reviewed_prs, + helper: `${formatNumber(stats.prs_with_findings)} with findings`, + }, + { + label: "Issues surfaced", + value: stats.surfaced_findings, + helper: `${formatNumber(stats.findings_recorded)} recorded`, + }, + { + label: "Addressed & resolved", + value: stats.addressed_findings, + helper: `${formatPercent(stats.resolution_rate)} of surfaced`, + }, + { + label: "Resolved after update", + value: stats.resolved_after_update, + helper: "Resolved on a later PR head", + }, + { + label: "Awaiting follow-up", + value: stats.unresolved_surfaced_findings, + helper: "Surfaced but not resolved/dismissed", + }, + { + label: "Dismissed", + value: stats.dismissed_findings, + helper: `${formatNumber(stats.human_replies)} human replies tracked`, + }, + ] + + return ( +
+
+ {cards.map((card) => ( +
+
{card.label}
+
+ {formatNumber(card.value)} +
+
+ {card.helper} +
+
+ ))} +
+
+ + +
+
+ ) +} + +function severityRows( + stats: ReviewerStatsPayload +): Array<{ name: string; count: number }> { + return ["critical", "high", "medium", "low"] + .map((severity) => ({ + name: severity, + count: stats.severity_counts[severity] ?? 0, + })) + .filter((row) => row.count > 0) +} + +function CounterList({ + title, + rows, +}: { + title: string + rows: Array<{ name: string; count: number }> +}) { + return ( +
+

{title}

+ {rows.length ? ( +
    + {rows.map((row) => ( +
  • + {row.name} + + {formatNumber(row.count)} + +
    • + ))} +
+ ) : ( +

No data yet.

+ )} +
+ ) +} + function UserCell({ row }: { row: UsageLeaderboardRow }) { const initials = initialsFor(row.user.name) return ( @@ -203,3 +328,10 @@ function initialsFor(name: string): string { function formatNumber(value: number): string { return new Intl.NumberFormat().format(value) } + +function formatPercent(value: number): string { + return new Intl.NumberFormat(undefined, { + maximumFractionDigits: 0, + style: "percent", + }).format(value) +}