mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 13:53:15 +00:00
feat: cache usage stats and add reviewer metrics (#1432)
* feat: cache usage dashboard stats Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * chore: adjust usage nav placement Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * fix: paginate reviewer usage stats Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
This commit is contained in:
Johannes du Plessis
GitHub
parent
f3512841fc
commit
e128f2d6dd
6 changed files with 728 additions and 38 deletions
|
|
@ -5,22 +5,28 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from langgraph_sdk import get_client
|
||||
|
||||
from ..reviewer_findings import REVIEWER_THREAD_KIND
|
||||
from ..utils.github_app import get_github_app_installation_token
|
||||
|
||||
USAGE_THREAD_NAMESPACE: list[str] = ["agent_usage", "threads"]
|
||||
USAGE_PR_NAMESPACE: list[str] = ["agent_usage", "prs"]
|
||||
USAGE_LEADERBOARD_CACHE_NAMESPACE: list[str] = ["agent_usage", "leaderboard_cache"]
|
||||
REVIEWER_STATS_CACHE_NAMESPACE: list[str] = ["agent_usage", "reviewer_stats_cache"]
|
||||
|
||||
Period = Literal["7d", "30d", "all"]
|
||||
_AGENT_SOURCES = frozenset({"dashboard", "github", "slack", "linear"})
|
||||
_PR_REFRESH_INTERVAL_MS = 10 * 60 * 1000
|
||||
_MAX_PR_REFRESH_PER_REQUEST = 25
|
||||
_PR_REFRESH_CONCURRENCY = 5
|
||||
_CACHE_TTL_MS = 10 * 60 * 1000
|
||||
_CACHE_SEARCH_LIMIT = 1000
|
||||
_GITHUB_API = "https://api.github.com"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -64,7 +70,9 @@ async def _get_value(namespace: list[str], key: str) -> dict[str, Any] | None:
|
|||
return _record_from_item(item)
|
||||
|
||||
|
||||
async def _search_values(namespace: list[str], *, limit: int = 1000) -> list[dict[str, Any]]:
|
||||
async def _search_values(
|
||||
namespace: list[str], *, limit: int = _CACHE_SEARCH_LIMIT
|
||||
) -> list[dict[str, Any]]:
|
||||
result = await _client().store.search_items(namespace, limit=limit)
|
||||
items = result.get("items") if isinstance(result, dict) else getattr(result, "items", [])
|
||||
values: list[dict[str, Any]] = []
|
||||
|
|
@ -95,6 +103,26 @@ def _coerce_int(value: object) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def _timestamp_ms(value: object) -> int:
|
||||
if isinstance(value, bool):
|
||||
return 0
|
||||
if isinstance(value, int | float):
|
||||
raw = int(value)
|
||||
return raw if raw > 10_000_000_000 else raw * 1000
|
||||
if isinstance(value, str) and value.strip():
|
||||
raw = value.strip()
|
||||
if raw.isdigit():
|
||||
return _timestamp_ms(int(raw))
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return 0
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return int(parsed.timestamp() * 1000)
|
||||
return 0
|
||||
|
||||
|
||||
def _in_period(record: dict[str, Any], cutoff_ms: int | None) -> bool:
|
||||
if cutoff_ms is None:
|
||||
return True
|
||||
|
|
@ -309,17 +337,27 @@ async def _refresh_pr_records(records: list[dict[str, Any]]) -> list[dict[str, A
|
|||
return refreshed
|
||||
|
||||
|
||||
async def list_agent_usage_leaderboard(
|
||||
*,
|
||||
period: str | None,
|
||||
limit: int,
|
||||
current_login: str | None,
|
||||
current_email: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the Open SWE Agent usage leaderboard from recorded telemetry."""
|
||||
normalized_period = _normalize_period(period)
|
||||
cutoff_ms = _period_cutoff_ms(normalized_period)
|
||||
safe_limit = min(max(limit, 1), 100)
|
||||
def _serialize_usage_user(index: int, user: dict[str, Any]) -> dict[str, Any]:
|
||||
model_counts: Counter[str] = user.get("model_counts", Counter())
|
||||
favorite_model = model_counts.most_common(1)[0][0] if model_counts else "default"
|
||||
return {
|
||||
"rank": index,
|
||||
"key": user["key"],
|
||||
"name": user.get("name") or "Unknown user",
|
||||
"github_login": user.get("github_login") or None,
|
||||
"email": user.get("email") or None,
|
||||
"favorite_model": favorite_model,
|
||||
"agent_runs": user["agent_runs"],
|
||||
"prs_opened": user["prs_opened"],
|
||||
"merged_prs": user["merged_prs"],
|
||||
"agent_loc": user["agent_loc"],
|
||||
"additions": user["additions"],
|
||||
"deletions": user["deletions"],
|
||||
}
|
||||
|
||||
|
||||
async def _build_usage_leaderboard_snapshot(period: Period) -> dict[str, Any]:
|
||||
cutoff_ms = _period_cutoff_ms(period)
|
||||
users: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for thread in await _search_values(USAGE_THREAD_NAMESPACE):
|
||||
|
|
@ -366,6 +404,236 @@ async def list_agent_usage_leaderboard(
|
|||
item.get("name") or "",
|
||||
),
|
||||
)
|
||||
return {
|
||||
"period": period,
|
||||
"users": [_serialize_usage_user(index, user) for index, user in enumerate(sorted_users, 1)],
|
||||
"total_members": len(sorted_users),
|
||||
}
|
||||
|
||||
|
||||
async def refresh_usage_leaderboard_cache(period: str | None = "30d") -> dict[str, Any]:
|
||||
normalized_period = _normalize_period(period)
|
||||
snapshot = await _build_usage_leaderboard_snapshot(normalized_period)
|
||||
await _client().store.put_item(
|
||||
USAGE_LEADERBOARD_CACHE_NAMESPACE,
|
||||
normalized_period,
|
||||
{"generated_at_ms": _now_ms(), "snapshot": snapshot},
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def _is_finding_surfaced(finding: dict[str, Any]) -> bool:
|
||||
surface = finding.get("surface") if isinstance(finding.get("surface"), dict) else {}
|
||||
state = surface.get("state")
|
||||
if state in {"surfaced", "resolve_pending", "resolved"}:
|
||||
return True
|
||||
if isinstance(finding.get("github_review_id"), int):
|
||||
return True
|
||||
if isinstance(finding.get("github_review_comment_id"), int):
|
||||
return True
|
||||
comment_ids = finding.get("github_review_comment_ids")
|
||||
thread_ids = finding.get("github_review_thread_ids")
|
||||
return bool(comment_ids or thread_ids)
|
||||
|
||||
|
||||
def _is_finding_resolved_by_us(finding: dict[str, Any]) -> bool:
|
||||
if finding.get("status") != "resolved" or not _is_finding_surfaced(finding):
|
||||
return False
|
||||
surface = finding.get("surface") if isinstance(finding.get("surface"), dict) else {}
|
||||
return bool(
|
||||
surface.get("state") == "resolved"
|
||||
or finding.get("github_thread_resolved")
|
||||
or finding.get("github_resolved_thread_ids")
|
||||
or finding.get("resolution_note")
|
||||
)
|
||||
|
||||
|
||||
def _thread_created_at_ms(thread: dict[str, Any], metadata: dict[str, Any]) -> int:
|
||||
timestamp = _thread_explicit_created_at_ms(thread, metadata)
|
||||
if timestamp:
|
||||
return timestamp
|
||||
for source in (
|
||||
metadata.get("updated_at_ms"),
|
||||
metadata.get("updated_at"),
|
||||
thread.get("updated_at"),
|
||||
thread.get("updatedAt"),
|
||||
):
|
||||
timestamp = _timestamp_ms(source)
|
||||
if timestamp:
|
||||
return timestamp
|
||||
return _now_ms()
|
||||
|
||||
|
||||
def _thread_explicit_created_at_ms(thread: dict[str, Any], metadata: dict[str, Any]) -> int:
|
||||
for source in (
|
||||
metadata.get("created_at_ms"),
|
||||
metadata.get("created_at"),
|
||||
thread.get("created_at"),
|
||||
thread.get("createdAt"),
|
||||
):
|
||||
timestamp = _timestamp_ms(source)
|
||||
if timestamp:
|
||||
return timestamp
|
||||
return 0
|
||||
|
||||
|
||||
def _counter_rows(counter: Counter[str], *, limit: int = 5) -> list[dict[str, Any]]:
|
||||
return [{"name": name, "count": count} for name, count in counter.most_common(limit)]
|
||||
|
||||
|
||||
async def _iter_reviewer_thread_pages(cutoff_ms: int | None):
|
||||
client = _client()
|
||||
offset = 0
|
||||
while True:
|
||||
page = await client.threads.search(
|
||||
metadata={"kind": REVIEWER_THREAD_KIND},
|
||||
limit=_CACHE_SEARCH_LIMIT,
|
||||
offset=offset,
|
||||
sort_by="created_at",
|
||||
sort_order="desc",
|
||||
)
|
||||
if not page:
|
||||
return
|
||||
yield page
|
||||
if len(page) < _CACHE_SEARCH_LIMIT:
|
||||
return
|
||||
if cutoff_ms is not None:
|
||||
last_thread = next(
|
||||
(thread for thread in reversed(page) if isinstance(thread, dict)), None
|
||||
)
|
||||
metadata = last_thread.get("metadata") if isinstance(last_thread, dict) else None
|
||||
if isinstance(metadata, dict):
|
||||
last_created_at_ms = _thread_explicit_created_at_ms(last_thread, metadata)
|
||||
if last_created_at_ms and last_created_at_ms < cutoff_ms:
|
||||
return
|
||||
offset += len(page)
|
||||
|
||||
|
||||
async def _build_reviewer_stats_snapshot(period: Period) -> dict[str, Any]:
|
||||
cutoff_ms = _period_cutoff_ms(period)
|
||||
|
||||
reviewed_prs = 0
|
||||
prs_with_findings = 0
|
||||
findings_recorded = 0
|
||||
surfaced_findings = 0
|
||||
addressed_findings = 0
|
||||
dismissed_findings = 0
|
||||
human_replies = 0
|
||||
resolved_after_update = 0
|
||||
severity_counts: Counter[str] = Counter()
|
||||
category_counts: Counter[str] = Counter()
|
||||
|
||||
async for page in _iter_reviewer_thread_pages(cutoff_ms):
|
||||
for thread in page:
|
||||
if not isinstance(thread, dict):
|
||||
continue
|
||||
metadata = thread.get("metadata") if isinstance(thread.get("metadata"), dict) else {}
|
||||
if cutoff_ms is not None and _thread_created_at_ms(thread, metadata) < cutoff_ms:
|
||||
continue
|
||||
|
||||
reviewed_prs += 1
|
||||
findings = metadata.get("findings")
|
||||
if not isinstance(findings, list):
|
||||
continue
|
||||
valid_findings = [finding for finding in findings if isinstance(finding, dict)]
|
||||
if valid_findings:
|
||||
prs_with_findings += 1
|
||||
for finding in valid_findings:
|
||||
findings_recorded += 1
|
||||
severity = finding.get("severity")
|
||||
if isinstance(severity, str) and severity:
|
||||
severity_counts[severity] += 1
|
||||
category = finding.get("category")
|
||||
if isinstance(category, str) and category:
|
||||
category_counts[category] += 1
|
||||
interactions = finding.get("interactions")
|
||||
if isinstance(interactions, list):
|
||||
human_replies += sum(
|
||||
1
|
||||
for interaction in interactions
|
||||
if isinstance(interaction, dict)
|
||||
and interaction.get("kind") == "human_reply"
|
||||
)
|
||||
elif finding.get("last_human_reply_at"):
|
||||
human_replies += 1
|
||||
|
||||
surfaced = _is_finding_surfaced(finding)
|
||||
if surfaced:
|
||||
surfaced_findings += 1
|
||||
if _is_finding_resolved_by_us(finding):
|
||||
addressed_findings += 1
|
||||
first_seen_sha = finding.get("first_seen_sha")
|
||||
head_sha = metadata.get("head_sha") or finding.get("last_confirmed_sha")
|
||||
if (
|
||||
isinstance(first_seen_sha, str)
|
||||
and isinstance(head_sha, str)
|
||||
and first_seen_sha != head_sha
|
||||
):
|
||||
resolved_after_update += 1
|
||||
if finding.get("status") == "dismissed":
|
||||
dismissed_findings += 1
|
||||
|
||||
unresolved_surfaced_findings = max(
|
||||
0, surfaced_findings - addressed_findings - dismissed_findings
|
||||
)
|
||||
resolution_rate = addressed_findings / surfaced_findings if surfaced_findings else 0.0
|
||||
return {
|
||||
"period": period,
|
||||
"reviewed_prs": reviewed_prs,
|
||||
"prs_with_findings": prs_with_findings,
|
||||
"findings_recorded": findings_recorded,
|
||||
"surfaced_findings": surfaced_findings,
|
||||
"addressed_findings": addressed_findings,
|
||||
"resolved_after_update": resolved_after_update,
|
||||
"dismissed_findings": dismissed_findings,
|
||||
"unresolved_surfaced_findings": unresolved_surfaced_findings,
|
||||
"resolution_rate": resolution_rate,
|
||||
"human_replies": human_replies,
|
||||
"severity_counts": dict(severity_counts),
|
||||
"top_categories": _counter_rows(category_counts),
|
||||
}
|
||||
|
||||
|
||||
async def refresh_reviewer_stats_cache(period: str | None = "30d") -> dict[str, Any]:
|
||||
normalized_period = _normalize_period(period)
|
||||
snapshot = await _build_reviewer_stats_snapshot(normalized_period)
|
||||
await _client().store.put_item(
|
||||
REVIEWER_STATS_CACHE_NAMESPACE,
|
||||
normalized_period,
|
||||
{"generated_at_ms": _now_ms(), "snapshot": snapshot},
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
async def _cached_snapshot(
|
||||
namespace: list[str],
|
||||
period: Period,
|
||||
refresh: Callable[[str | None], Awaitable[dict[str, Any]]],
|
||||
*,
|
||||
schedule_refresh: Callable[[Period], None] | None = None,
|
||||
) -> tuple[dict[str, Any], int | None]:
|
||||
cached = await _get_value(namespace, period)
|
||||
if cached:
|
||||
snapshot = cached.get("snapshot")
|
||||
generated_at_ms = _coerce_int(cached.get("generated_at_ms"))
|
||||
if isinstance(snapshot, dict):
|
||||
if not generated_at_ms or _now_ms() - generated_at_ms <= _CACHE_TTL_MS:
|
||||
return snapshot, generated_at_ms or None
|
||||
if schedule_refresh is not None:
|
||||
schedule_refresh(period)
|
||||
return snapshot, generated_at_ms
|
||||
return await refresh(period), _now_ms()
|
||||
|
||||
|
||||
def _usage_payload_from_snapshot(
|
||||
snapshot: dict[str, Any],
|
||||
*,
|
||||
limit: int,
|
||||
current_login: str | None,
|
||||
current_email: str | None,
|
||||
generated_at_ms: int | None,
|
||||
) -> dict[str, Any]:
|
||||
safe_limit = min(max(limit, 1), 100)
|
||||
current_keys = {
|
||||
_user_key(current_login, current_email),
|
||||
_user_key(current_login, None),
|
||||
|
|
@ -373,25 +641,27 @@ async def list_agent_usage_leaderboard(
|
|||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
current_user_row: dict[str, Any] | None = None
|
||||
for index, user in enumerate(sorted_users, start=1):
|
||||
model_counts: Counter[str] = user.pop("model_counts")
|
||||
favorite_model = model_counts.most_common(1)[0][0] if model_counts else "default"
|
||||
github_login = user.get("github_login") or None
|
||||
is_current_user = user["key"] in current_keys
|
||||
for user in snapshot.get("users", []):
|
||||
if not isinstance(user, dict):
|
||||
continue
|
||||
github_login = (
|
||||
user.get("github_login") if isinstance(user.get("github_login"), str) else None
|
||||
)
|
||||
is_current_user = user.get("key") in current_keys
|
||||
row = {
|
||||
"rank": index,
|
||||
"rank": _coerce_int(user.get("rank")),
|
||||
"user": {
|
||||
"name": user.get("name") if is_current_user or github_login else "Open SWE user",
|
||||
"github_login": github_login,
|
||||
"email": (user.get("email") or None) if is_current_user else None,
|
||||
},
|
||||
"favorite_model": favorite_model,
|
||||
"agent_runs": user["agent_runs"],
|
||||
"prs_opened": user["prs_opened"],
|
||||
"merged_prs": user["merged_prs"],
|
||||
"agent_loc": user["agent_loc"],
|
||||
"additions": user["additions"],
|
||||
"deletions": user["deletions"],
|
||||
"favorite_model": user.get("favorite_model") or "default",
|
||||
"agent_runs": _coerce_int(user.get("agent_runs")),
|
||||
"prs_opened": _coerce_int(user.get("prs_opened")),
|
||||
"merged_prs": _coerce_int(user.get("merged_prs")),
|
||||
"agent_loc": _coerce_int(user.get("agent_loc")),
|
||||
"additions": _coerce_int(user.get("additions")),
|
||||
"deletions": _coerce_int(user.get("deletions")),
|
||||
}
|
||||
if is_current_user:
|
||||
current_user_row = row
|
||||
|
|
@ -402,8 +672,43 @@ async def list_agent_usage_leaderboard(
|
|||
rows.append(current_user_row)
|
||||
|
||||
return {
|
||||
"period": normalized_period,
|
||||
"period": snapshot.get("period") or "30d",
|
||||
"rows": rows,
|
||||
"total_members": len(sorted_users),
|
||||
"total_members": _coerce_int(snapshot.get("total_members")),
|
||||
"current_user_rank": current_user_row["rank"] if current_user_row else None,
|
||||
"generated_at_ms": generated_at_ms,
|
||||
}
|
||||
|
||||
|
||||
async def list_agent_usage_leaderboard(
|
||||
*,
|
||||
period: str | None,
|
||||
limit: int,
|
||||
current_login: str | None,
|
||||
current_email: str | None,
|
||||
schedule_usage_refresh: Callable[[Period], None] | None = None,
|
||||
schedule_reviewer_refresh: Callable[[Period], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return cached Open SWE Agent and reviewer usage stats."""
|
||||
normalized_period = _normalize_period(period)
|
||||
usage_snapshot, generated_at_ms = await _cached_snapshot(
|
||||
USAGE_LEADERBOARD_CACHE_NAMESPACE,
|
||||
normalized_period,
|
||||
refresh_usage_leaderboard_cache,
|
||||
schedule_refresh=schedule_usage_refresh,
|
||||
)
|
||||
reviewer_stats, reviewer_generated_at_ms = await _cached_snapshot(
|
||||
REVIEWER_STATS_CACHE_NAMESPACE,
|
||||
normalized_period,
|
||||
refresh_reviewer_stats_cache,
|
||||
schedule_refresh=schedule_reviewer_refresh,
|
||||
)
|
||||
payload = _usage_payload_from_snapshot(
|
||||
usage_snapshot,
|
||||
limit=limit,
|
||||
current_login=current_login,
|
||||
current_email=current_email,
|
||||
generated_at_ms=generated_at_ms,
|
||||
)
|
||||
payload["reviewer_stats"] = {**reviewer_stats, "generated_at_ms": reviewer_generated_at_ms}
|
||||
return payload
|
||||
|
|
|
|||
|
|
@ -8,12 +8,16 @@ import os
|
|||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from fastapi.responses import RedirectResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .admin import is_admin
|
||||
from .agent_usage import list_agent_usage_leaderboard
|
||||
from .agent_usage import (
|
||||
list_agent_usage_leaderboard,
|
||||
refresh_reviewer_stats_cache,
|
||||
refresh_usage_leaderboard_cache,
|
||||
)
|
||||
from .analyzer_cron import remove_continual_cron
|
||||
from .enabled_repos import (
|
||||
list_enabled_review_repos,
|
||||
|
|
@ -706,6 +710,7 @@ async def api_delete_review_style(
|
|||
|
||||
@router.get("/agent-usage-leaderboard")
|
||||
async def api_agent_usage_leaderboard(
|
||||
background_tasks: BackgroundTasks,
|
||||
period: str | None = "30d",
|
||||
limit: int = 10,
|
||||
session: dict[str, Any] = _SESSION_DEP,
|
||||
|
|
@ -715,6 +720,12 @@ async def api_agent_usage_leaderboard(
|
|||
limit=limit,
|
||||
current_login=session["sub"],
|
||||
current_email=session.get("email"),
|
||||
schedule_usage_refresh=lambda cache_period: background_tasks.add_task(
|
||||
refresh_usage_leaderboard_cache, cache_period
|
||||
),
|
||||
schedule_reviewer_refresh=lambda cache_period: background_tasks.add_task(
|
||||
refresh_reviewer_stats_cache, cache_period
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
218
tests/test_agent_usage.py
Normal file
218
tests/test_agent_usage.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.dashboard import agent_usage
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self, values: dict[tuple[tuple[str, ...], str], dict] | None = None):
|
||||
self.values = values or {}
|
||||
self.puts: list[tuple[list[str], str, dict]] = []
|
||||
|
||||
async def get_item(self, namespace: list[str], key: str) -> dict | None:
|
||||
value = self.values.get((tuple(namespace), key))
|
||||
return {"value": value} if value is not None else None
|
||||
|
||||
async def put_item(self, namespace: list[str], key: str, value: dict) -> None:
|
||||
self.puts.append((namespace, key, value))
|
||||
self.values[(tuple(namespace), key)] = value
|
||||
|
||||
|
||||
class FakeThreads:
|
||||
def __init__(self, threads: list[dict]):
|
||||
self.threads = threads
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def search(self, **kwargs) -> list[dict]:
|
||||
self.calls.append(kwargs)
|
||||
offset = kwargs.get("offset") or 0
|
||||
limit = kwargs.get("limit") or len(self.threads)
|
||||
return self.threads[offset : offset + limit]
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *, store: FakeStore | None = None, threads: FakeThreads | None = None):
|
||||
self.store = store or FakeStore()
|
||||
self.threads = threads or FakeThreads([])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cached_usage_payload_returns_stale_snapshot_and_schedules_refresh(monkeypatch):
|
||||
usage_snapshot = {
|
||||
"period": "30d",
|
||||
"total_members": 2,
|
||||
"users": [
|
||||
{
|
||||
"rank": 1,
|
||||
"key": "github:octo",
|
||||
"name": "octo",
|
||||
"github_login": "octo",
|
||||
"email": "octo@example.com",
|
||||
"favorite_model": "claude",
|
||||
"agent_runs": 3,
|
||||
"prs_opened": 2,
|
||||
"merged_prs": 1,
|
||||
"agent_loc": 10,
|
||||
"additions": 8,
|
||||
"deletions": 2,
|
||||
},
|
||||
{
|
||||
"rank": 2,
|
||||
"key": "email:private@example.com",
|
||||
"name": "private",
|
||||
"github_login": None,
|
||||
"email": "private@example.com",
|
||||
"favorite_model": "default",
|
||||
"agent_runs": 1,
|
||||
"prs_opened": 0,
|
||||
"merged_prs": 0,
|
||||
"agent_loc": 0,
|
||||
"additions": 0,
|
||||
"deletions": 0,
|
||||
},
|
||||
],
|
||||
}
|
||||
reviewer_snapshot = {
|
||||
"period": "30d",
|
||||
"reviewed_prs": 1,
|
||||
"prs_with_findings": 1,
|
||||
"findings_recorded": 1,
|
||||
"surfaced_findings": 1,
|
||||
"addressed_findings": 0,
|
||||
"resolved_after_update": 0,
|
||||
"dismissed_findings": 0,
|
||||
"unresolved_surfaced_findings": 1,
|
||||
"resolution_rate": 0.0,
|
||||
"human_replies": 0,
|
||||
"severity_counts": {"medium": 1},
|
||||
"top_categories": [{"name": "correctness", "count": 1}],
|
||||
}
|
||||
store = FakeStore(
|
||||
{
|
||||
(tuple(agent_usage.USAGE_LEADERBOARD_CACHE_NAMESPACE), "30d"): {
|
||||
"generated_at_ms": 1,
|
||||
"snapshot": usage_snapshot,
|
||||
},
|
||||
(tuple(agent_usage.REVIEWER_STATS_CACHE_NAMESPACE), "30d"): {
|
||||
"generated_at_ms": 1,
|
||||
"snapshot": reviewer_snapshot,
|
||||
},
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(store=store))
|
||||
monkeypatch.setattr(agent_usage, "_now_ms", lambda: agent_usage._CACHE_TTL_MS + 2)
|
||||
|
||||
usage_refreshes: list[str] = []
|
||||
reviewer_refreshes: list[str] = []
|
||||
payload = await agent_usage.list_agent_usage_leaderboard(
|
||||
period="30d",
|
||||
limit=1,
|
||||
current_login="octo",
|
||||
current_email="octo@example.com",
|
||||
schedule_usage_refresh=usage_refreshes.append,
|
||||
schedule_reviewer_refresh=reviewer_refreshes.append,
|
||||
)
|
||||
|
||||
assert usage_refreshes == ["30d"]
|
||||
assert reviewer_refreshes == ["30d"]
|
||||
assert payload["rows"][0]["user"]["email"] == "octo@example.com"
|
||||
assert payload["total_members"] == 2
|
||||
assert payload["reviewer_stats"]["surfaced_findings"] == 1
|
||||
assert store.puts == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reviewer_stats_snapshot_counts_surfaced_and_resolved_findings(monkeypatch):
|
||||
threads = [
|
||||
{
|
||||
"created_at": "2025-01-01T00:00:00Z",
|
||||
"metadata": {
|
||||
"kind": "reviewer",
|
||||
"head_sha": "fixed-sha",
|
||||
"pr": {"owner": "langchain-ai", "name": "open-swe", "number": 1},
|
||||
"findings": [
|
||||
{
|
||||
"id": "f_1",
|
||||
"status": "resolved",
|
||||
"severity": "high",
|
||||
"category": "correctness",
|
||||
"first_seen_sha": "buggy-sha",
|
||||
"github_thread_resolved": True,
|
||||
"github_review_comment_id": 10,
|
||||
"resolution_note": "Fixed in a follow-up commit.",
|
||||
"interactions": [{"kind": "human_reply"}],
|
||||
},
|
||||
{
|
||||
"id": "f_2",
|
||||
"status": "open",
|
||||
"severity": "medium",
|
||||
"category": "performance",
|
||||
"github_review_comment_id": 11,
|
||||
},
|
||||
{
|
||||
"id": "f_3",
|
||||
"status": "dismissed",
|
||||
"severity": "low",
|
||||
"category": "style",
|
||||
"github_review_id": 12,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(threads=FakeThreads(threads)))
|
||||
|
||||
snapshot = await agent_usage._build_reviewer_stats_snapshot("all")
|
||||
|
||||
assert snapshot["reviewed_prs"] == 1
|
||||
assert snapshot["prs_with_findings"] == 1
|
||||
assert snapshot["findings_recorded"] == 3
|
||||
assert snapshot["surfaced_findings"] == 3
|
||||
assert snapshot["addressed_findings"] == 1
|
||||
assert snapshot["resolved_after_update"] == 1
|
||||
assert snapshot["dismissed_findings"] == 1
|
||||
assert snapshot["unresolved_surfaced_findings"] == 1
|
||||
assert snapshot["human_replies"] == 1
|
||||
assert snapshot["severity_counts"] == {"high": 1, "medium": 1, "low": 1}
|
||||
assert snapshot["top_categories"] == [
|
||||
{"name": "correctness", "count": 1},
|
||||
{"name": "performance", "count": 1},
|
||||
{"name": "style", "count": 1},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reviewer_stats_paginates_reviewer_threads(monkeypatch):
|
||||
threads = [
|
||||
{"created_at": "2025-01-03T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}},
|
||||
{"created_at": "2025-01-02T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}},
|
||||
{"created_at": "2025-01-01T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}},
|
||||
]
|
||||
fake_threads = FakeThreads(threads)
|
||||
monkeypatch.setattr(agent_usage, "_CACHE_SEARCH_LIMIT", 2)
|
||||
monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(threads=fake_threads))
|
||||
|
||||
snapshot = await agent_usage._build_reviewer_stats_snapshot("all")
|
||||
|
||||
assert snapshot["reviewed_prs"] == 3
|
||||
assert [call["offset"] for call in fake_threads.calls] == [0, 2]
|
||||
assert all(call["sort_by"] == "created_at" for call in fake_threads.calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reviewer_stats_stops_after_page_older_than_cutoff(monkeypatch):
|
||||
threads = [
|
||||
{"created_at": "2025-01-03T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}},
|
||||
{"created_at": "2025-01-01T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}},
|
||||
{"created_at": "2024-12-31T00:00:00Z", "metadata": {"kind": "reviewer", "findings": []}},
|
||||
]
|
||||
fake_threads = FakeThreads(threads)
|
||||
monkeypatch.setattr(agent_usage, "_CACHE_SEARCH_LIMIT", 2)
|
||||
monkeypatch.setattr(agent_usage, "_client", lambda: FakeClient(threads=fake_threads))
|
||||
cutoff_ms = agent_usage._timestamp_ms("2025-01-02T00:00:00Z")
|
||||
|
||||
pages = [page async for page in agent_usage._iter_reviewer_thread_pages(cutoff_ms)]
|
||||
|
||||
assert len(pages) == 1
|
||||
assert [call["offset"] for call in fake_threads.calls] == [0]
|
||||
|
|
@ -30,8 +30,8 @@ interface NavItem {
|
|||
const NAV: Array<NavItem> = [
|
||||
{ to: "/my-settings", label: "Profile Settings", icon: IoOptionsOutline },
|
||||
{ to: "/cloud-agents", label: "Open SWE Agent", icon: IoCloudOutline },
|
||||
{ to: "/usage", label: "Usage Leaderboard", icon: IoStatsChartOutline },
|
||||
{ to: "/review", label: "Open SWE Review", icon: IoGitPullRequestOutline },
|
||||
{ to: "/usage", label: "Usage", icon: IoStatsChartOutline },
|
||||
{ to: "/admin", label: "Admin", icon: IoSettingsOutline, adminOnly: true },
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -157,11 +157,35 @@ export interface UsageLeaderboardRow {
|
|||
deletions: number;
|
||||
}
|
||||
|
||||
export interface ReviewerStatsCounterRow {
|
||||
name: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface ReviewerStatsPayload {
|
||||
period: UsageLeaderboardPeriod;
|
||||
reviewed_prs: number;
|
||||
prs_with_findings: number;
|
||||
findings_recorded: number;
|
||||
surfaced_findings: number;
|
||||
addressed_findings: number;
|
||||
resolved_after_update: number;
|
||||
dismissed_findings: number;
|
||||
unresolved_surfaced_findings: number;
|
||||
resolution_rate: number;
|
||||
human_replies: number;
|
||||
severity_counts: Record<string, number>;
|
||||
top_categories: Array<ReviewerStatsCounterRow>;
|
||||
generated_at_ms: number | null;
|
||||
}
|
||||
|
||||
export interface UsageLeaderboardPayload {
|
||||
period: UsageLeaderboardPeriod;
|
||||
rows: Array<UsageLeaderboardRow>;
|
||||
total_members: number;
|
||||
current_user_rank: number | null;
|
||||
generated_at_ms: number | null;
|
||||
reviewer_stats: ReviewerStatsPayload;
|
||||
}
|
||||
|
||||
export interface Repository {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import { Navigate, createFileRoute } from "@tanstack/react-router"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
|
||||
import type { UsageLeaderboardPeriod, UsageLeaderboardRow } from "@/lib/api"
|
||||
import type {
|
||||
ReviewerStatsPayload,
|
||||
UsageLeaderboardPeriod,
|
||||
UsageLeaderboardRow,
|
||||
} from "@/lib/api"
|
||||
import { AppShell, SettingsSection } from "@/components/AppShell"
|
||||
import { Avatar, AvatarFallback } from "@/components/ui/avatar"
|
||||
import {
|
||||
|
|
@ -43,6 +47,7 @@ function UsagePage() {
|
|||
queryKey: ["usageLeaderboard", activePeriod],
|
||||
queryFn: () => api.usageLeaderboard(activePeriod, 10),
|
||||
enabled: !!session.data,
|
||||
staleTime: 5 * 60 * 1000,
|
||||
})
|
||||
|
||||
if (session.isLoading) {
|
||||
|
|
@ -55,13 +60,9 @@ function UsagePage() {
|
|||
if (!session.data) return <Navigate to="/login" />
|
||||
|
||||
return (
|
||||
<AppShell
|
||||
user={session.data}
|
||||
title="Usage Leaderboard"
|
||||
description="Open SWE Agent usage recorded from this release onward. Reviewer-agent runs are excluded."
|
||||
>
|
||||
<AppShell user={session.data} title="Usage">
|
||||
<SettingsSection
|
||||
title="Leaderboard"
|
||||
title="Agent leaderboard"
|
||||
description="Ranked by agent lines of code, then PRs opened and agent runs."
|
||||
action={
|
||||
<Select
|
||||
|
|
@ -94,7 +95,7 @@ function UsagePage() {
|
|||
</div>
|
||||
) : leaderboard.isError ? (
|
||||
<p className="p-4 text-xs text-destructive">
|
||||
Failed to load usage leaderboard: {leaderboard.error.message}
|
||||
Failed to load usage data: {leaderboard.error.message}
|
||||
</p>
|
||||
) : !leaderboard.data?.rows.length ? (
|
||||
<div className="p-6 text-center text-sm text-muted-foreground">
|
||||
|
|
@ -108,6 +109,31 @@ function UsagePage() {
|
|||
/>
|
||||
)}
|
||||
</SettingsSection>
|
||||
|
||||
<SettingsSection
|
||||
title="Reviewer stats"
|
||||
description="Issues surfaced by Open SWE Review and how often users addressed them."
|
||||
>
|
||||
{leaderboard.isLoading ? (
|
||||
<div className="grid gap-3 p-4 sm:grid-cols-2">
|
||||
<Skeleton className="h-24 w-full" />
|
||||
<Skeleton className="h-24 w-full" />
|
||||
<Skeleton className="h-24 w-full" />
|
||||
<Skeleton className="h-24 w-full" />
|
||||
</div>
|
||||
) : leaderboard.isError ? (
|
||||
<p className="p-4 text-xs text-destructive">
|
||||
Failed to load reviewer stats: {leaderboard.error.message}
|
||||
</p>
|
||||
) : leaderboard.data?.reviewer_stats ? (
|
||||
<ReviewerStats stats={leaderboard.data.reviewer_stats} />
|
||||
) : (
|
||||
<div className="p-6 text-center text-sm text-muted-foreground">
|
||||
No reviewer stats have been recorded for{" "}
|
||||
{PERIOD_LABELS[activePeriod].toLowerCase()} yet.
|
||||
</div>
|
||||
)}
|
||||
</SettingsSection>
|
||||
</AppShell>
|
||||
)
|
||||
}
|
||||
|
|
@ -172,6 +198,105 @@ function UsageTable({
|
|||
)
|
||||
}
|
||||
|
||||
function ReviewerStats({ stats }: { stats: ReviewerStatsPayload }) {
|
||||
const cards = [
|
||||
{
|
||||
label: "Reviewed PRs",
|
||||
value: stats.reviewed_prs,
|
||||
helper: `${formatNumber(stats.prs_with_findings)} with findings`,
|
||||
},
|
||||
{
|
||||
label: "Issues surfaced",
|
||||
value: stats.surfaced_findings,
|
||||
helper: `${formatNumber(stats.findings_recorded)} recorded`,
|
||||
},
|
||||
{
|
||||
label: "Addressed & resolved",
|
||||
value: stats.addressed_findings,
|
||||
helper: `${formatPercent(stats.resolution_rate)} of surfaced`,
|
||||
},
|
||||
{
|
||||
label: "Resolved after update",
|
||||
value: stats.resolved_after_update,
|
||||
helper: "Resolved on a later PR head",
|
||||
},
|
||||
{
|
||||
label: "Awaiting follow-up",
|
||||
value: stats.unresolved_surfaced_findings,
|
||||
helper: "Surfaced but not resolved/dismissed",
|
||||
},
|
||||
{
|
||||
label: "Dismissed",
|
||||
value: stats.dismissed_findings,
|
||||
helper: `${formatNumber(stats.human_replies)} human replies tracked`,
|
||||
},
|
||||
]
|
||||
|
||||
return (
|
||||
<div className="space-y-4 p-4">
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-3">
|
||||
{cards.map((card) => (
|
||||
<div key={card.label} className="rounded-md border border-border p-3">
|
||||
<div className="text-xs text-muted-foreground">{card.label}</div>
|
||||
<div className="mt-1 text-xl font-medium tabular-nums">
|
||||
{formatNumber(card.value)}
|
||||
</div>
|
||||
<div className="mt-1 text-xs text-muted-foreground">
|
||||
{card.helper}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div className="grid gap-4 border-t border-border pt-4 sm:grid-cols-2">
|
||||
<CounterList title="Top categories" rows={stats.top_categories} />
|
||||
<CounterList title="Severity mix" rows={severityRows(stats)} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function severityRows(
|
||||
stats: ReviewerStatsPayload
|
||||
): Array<{ name: string; count: number }> {
|
||||
return ["critical", "high", "medium", "low"]
|
||||
.map((severity) => ({
|
||||
name: severity,
|
||||
count: stats.severity_counts[severity] ?? 0,
|
||||
}))
|
||||
.filter((row) => row.count > 0)
|
||||
}
|
||||
|
||||
function CounterList({
|
||||
title,
|
||||
rows,
|
||||
}: {
|
||||
title: string
|
||||
rows: Array<{ name: string; count: number }>
|
||||
}) {
|
||||
return (
|
||||
<div>
|
||||
<h3 className="text-xs font-medium text-muted-foreground">{title}</h3>
|
||||
{rows.length ? (
|
||||
<ul className="mt-2 space-y-2 text-sm">
|
||||
{rows.map((row) => (
|
||||
<li
|
||||
key={row.name}
|
||||
className="flex items-center justify-between gap-3"
|
||||
>
|
||||
<span className="truncate text-foreground">{row.name}</span>
|
||||
<span className="text-muted-foreground tabular-nums">
|
||||
{formatNumber(row.count)}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : (
|
||||
<p className="mt-2 text-xs text-muted-foreground">No data yet.</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function UserCell({ row }: { row: UsageLeaderboardRow }) {
|
||||
const initials = initialsFor(row.user.name)
|
||||
return (
|
||||
|
|
@ -203,3 +328,10 @@ function initialsFor(name: string): string {
|
|||
function formatNumber(value: number): string {
|
||||
return new Intl.NumberFormat().format(value)
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return new Intl.NumberFormat(undefined, {
|
||||
maximumFractionDigits: 0,
|
||||
style: "percent",
|
||||
}).format(value)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue