mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 15:03:16 +00:00
feat: add agent usage leaderboard (#1418)
* feat: add agent usage leaderboard Co-authored-by: open-swe[bot] <215916821+open-swe[bot]@users.noreply.github.com> * fix: bound leaderboard refresh and hide emails Co-authored-by: open-swe[bot] <215916821+open-swe[bot]@users.noreply.github.com> --------- Co-authored-by: open-swe[bot] <215916821+open-swe[bot]@users.noreply.github.com>
This commit is contained in:
Johannes du Plessis
GitHub
parent
02cfdbda5b
commit
a511add856
8 changed files with 823 additions and 0 deletions
409
agent/dashboard/agent_usage.py
Normal file
409
agent/dashboard/agent_usage.py
Normal file
|
|
@ -0,0 +1,409 @@
|
|||
"""Forward-looking Open SWE Agent usage telemetry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from langgraph_sdk import get_client
|
||||
|
||||
from ..utils.github_app import get_github_app_installation_token
|
||||
|
||||
USAGE_THREAD_NAMESPACE: list[str] = ["agent_usage", "threads"]
|
||||
USAGE_PR_NAMESPACE: list[str] = ["agent_usage", "prs"]
|
||||
|
||||
Period = Literal["7d", "30d", "all"]
|
||||
_AGENT_SOURCES = frozenset({"dashboard", "github", "slack", "linear"})
|
||||
_PR_REFRESH_INTERVAL_MS = 10 * 60 * 1000
|
||||
_MAX_PR_REFRESH_PER_REQUEST = 25
|
||||
_PR_REFRESH_CONCURRENCY = 5
|
||||
_GITHUB_API = "https://api.github.com"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _client():
|
||||
return get_client()
|
||||
|
||||
|
||||
def _now_ms() -> int:
|
||||
return int(datetime.now(UTC).timestamp() * 1000)
|
||||
|
||||
|
||||
def _period_cutoff_ms(period: str) -> int | None:
|
||||
now = datetime.now(UTC)
|
||||
if period == "7d":
|
||||
return int((now - timedelta(days=7)).timestamp() * 1000)
|
||||
if period == "30d":
|
||||
return int((now - timedelta(days=30)).timestamp() * 1000)
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_period(period: str | None) -> Period:
|
||||
return period if period in {"7d", "30d", "all"} else "30d"
|
||||
|
||||
|
||||
def _record_from_item(item: Any) -> dict[str, Any] | None:
|
||||
if item is None:
|
||||
return None
|
||||
value = item.get("value") if isinstance(item, dict) else getattr(item, "value", None)
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
async def _get_value(namespace: list[str], key: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
item = await _client().store.get_item(namespace, key)
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 404:
|
||||
return None
|
||||
raise
|
||||
return _record_from_item(item)
|
||||
|
||||
|
||||
async def _search_values(namespace: list[str], *, limit: int = 1000) -> list[dict[str, Any]]:
|
||||
result = await _client().store.search_items(namespace, limit=limit)
|
||||
items = result.get("items") if isinstance(result, dict) else getattr(result, "items", [])
|
||||
values: list[dict[str, Any]] = []
|
||||
for item in items or []:
|
||||
record = _record_from_item(item)
|
||||
if record:
|
||||
values.append(record)
|
||||
return values
|
||||
|
||||
|
||||
def _user_key(github_login: str | None, email: str | None) -> str:
|
||||
login = github_login.strip().lower() if isinstance(github_login, str) else ""
|
||||
if login:
|
||||
return f"github:{login}"
|
||||
norm_email = email.strip().lower() if isinstance(email, str) else ""
|
||||
if norm_email:
|
||||
return f"email:{norm_email}"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _coerce_int(value: object) -> int:
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
return 0
|
||||
|
||||
|
||||
def _in_period(record: dict[str, Any], cutoff_ms: int | None) -> bool:
|
||||
if cutoff_ms is None:
|
||||
return True
|
||||
created_at = _coerce_int(record.get("created_at_ms"))
|
||||
return created_at >= cutoff_ms
|
||||
|
||||
|
||||
def _display_name(github_login: str, email: str) -> str:
|
||||
if github_login:
|
||||
return github_login
|
||||
if email:
|
||||
return email.split("@", 1)[0]
|
||||
return "Unknown user"
|
||||
|
||||
|
||||
def _ensure_user(
|
||||
users: dict[str, dict[str, Any]],
|
||||
*,
|
||||
github_login: str | None,
|
||||
email: str | None,
|
||||
) -> dict[str, Any]:
|
||||
login = github_login.strip() if isinstance(github_login, str) else ""
|
||||
norm_email = email.strip().lower() if isinstance(email, str) else ""
|
||||
key = _user_key(login, norm_email)
|
||||
user = users.get(key)
|
||||
if user is None:
|
||||
user = {
|
||||
"key": key,
|
||||
"github_login": login,
|
||||
"email": norm_email,
|
||||
"name": _display_name(login, norm_email),
|
||||
"agent_runs": 0,
|
||||
"prs_opened": 0,
|
||||
"merged_prs": 0,
|
||||
"agent_loc": 0,
|
||||
"additions": 0,
|
||||
"deletions": 0,
|
||||
"model_counts": Counter(),
|
||||
}
|
||||
users[key] = user
|
||||
elif login and not user.get("github_login"):
|
||||
user["github_login"] = login
|
||||
user["name"] = _display_name(login, norm_email)
|
||||
if norm_email and not user.get("email"):
|
||||
user["email"] = norm_email
|
||||
return user
|
||||
|
||||
|
||||
async def record_agent_thread_usage(
|
||||
*,
|
||||
thread_id: str,
|
||||
github_login: str | None,
|
||||
user_email: str | None,
|
||||
model_id: str,
|
||||
effort: str | None,
|
||||
source: str | None,
|
||||
) -> None:
|
||||
"""Record one Open SWE Agent thread for leaderboard aggregation."""
|
||||
if not thread_id:
|
||||
return
|
||||
source_value = source if isinstance(source, str) and source in _AGENT_SOURCES else "dashboard"
|
||||
now_ms = _now_ms()
|
||||
existing = await _get_value(USAGE_THREAD_NAMESPACE, thread_id)
|
||||
value = {
|
||||
**(existing or {}),
|
||||
"thread_id": thread_id,
|
||||
"github_login": github_login.strip() if isinstance(github_login, str) else "",
|
||||
"user_email": user_email.strip().lower() if isinstance(user_email, str) else "",
|
||||
"model_id": model_id,
|
||||
"effort": effort or "",
|
||||
"source": source_value,
|
||||
"agent_kind": "agent",
|
||||
"updated_at_ms": now_ms,
|
||||
}
|
||||
if not existing:
|
||||
value["created_at_ms"] = now_ms
|
||||
elif not value.get("created_at_ms"):
|
||||
value["created_at_ms"] = existing.get("created_at_ms") or now_ms
|
||||
await _client().store.put_item(USAGE_THREAD_NAMESPACE, thread_id, value)
|
||||
|
||||
|
||||
async def record_agent_pr_usage(
|
||||
*,
|
||||
thread_id: str | None,
|
||||
github_login: str | None,
|
||||
user_email: str | None,
|
||||
owner: str,
|
||||
repo: str,
|
||||
pr_number: int,
|
||||
pr_url: str | None,
|
||||
head: str,
|
||||
base: str,
|
||||
additions: int = 0,
|
||||
deletions: int = 0,
|
||||
changed_files: int = 0,
|
||||
state: str | None = None,
|
||||
merged: bool = False,
|
||||
) -> None:
|
||||
"""Record one Open SWE Agent pull request for leaderboard aggregation."""
|
||||
if not owner or not repo or not pr_number:
|
||||
return
|
||||
key = f"{owner}/{repo}#{pr_number}"
|
||||
now_ms = _now_ms()
|
||||
existing = await _get_value(USAGE_PR_NAMESPACE, key)
|
||||
value = {
|
||||
**(existing or {}),
|
||||
"key": key,
|
||||
"thread_id": thread_id or "",
|
||||
"github_login": github_login.strip() if isinstance(github_login, str) else "",
|
||||
"user_email": user_email.strip().lower() if isinstance(user_email, str) else "",
|
||||
"owner": owner,
|
||||
"repo": repo,
|
||||
"pr_number": pr_number,
|
||||
"pr_url": pr_url or "",
|
||||
"head": head,
|
||||
"base": base,
|
||||
"additions": max(0, additions),
|
||||
"deletions": max(0, deletions),
|
||||
"changed_files": max(0, changed_files),
|
||||
"state": state or "open",
|
||||
"merged": bool(merged),
|
||||
"agent_kind": "agent",
|
||||
"updated_at_ms": now_ms,
|
||||
}
|
||||
if not existing:
|
||||
value["created_at_ms"] = now_ms
|
||||
elif not value.get("created_at_ms"):
|
||||
value["created_at_ms"] = existing.get("created_at_ms") or now_ms
|
||||
await _client().store.put_item(USAGE_PR_NAMESPACE, key, value)
|
||||
|
||||
|
||||
def _github_headers(token: str) -> dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
|
||||
|
||||
async def _refresh_pr_record(
|
||||
client: httpx.AsyncClient, token: str, record: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
owner = record.get("owner")
|
||||
repo = record.get("repo")
|
||||
pr_number = record.get("pr_number")
|
||||
if not isinstance(owner, str) or not isinstance(repo, str) or not isinstance(pr_number, int):
|
||||
return record
|
||||
resp = await client.get(
|
||||
f"{_GITHUB_API}/repos/{owner}/{repo}/pulls/{pr_number}",
|
||||
headers=_github_headers(token),
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.debug(
|
||||
"GitHub returned %s refreshing usage PR %s/%s#%s",
|
||||
resp.status_code,
|
||||
owner,
|
||||
repo,
|
||||
pr_number,
|
||||
)
|
||||
return record
|
||||
data = resp.json()
|
||||
if not isinstance(data, dict):
|
||||
return record
|
||||
updated = {
|
||||
**record,
|
||||
"pr_url": data.get("html_url") or record.get("pr_url") or "",
|
||||
"state": data.get("state") if isinstance(data.get("state"), str) else record.get("state"),
|
||||
"merged": bool(data.get("merged")),
|
||||
"additions": data.get("additions") if isinstance(data.get("additions"), int) else 0,
|
||||
"deletions": data.get("deletions") if isinstance(data.get("deletions"), int) else 0,
|
||||
"changed_files": data.get("changed_files")
|
||||
if isinstance(data.get("changed_files"), int)
|
||||
else 0,
|
||||
"updated_at_ms": _now_ms(),
|
||||
}
|
||||
key = updated.get("key")
|
||||
if isinstance(key, str) and key:
|
||||
await _client().store.put_item(USAGE_PR_NAMESPACE, key, updated)
|
||||
return updated
|
||||
|
||||
|
||||
async def _refresh_pr_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
now_ms = _now_ms()
|
||||
stale_indexes = [
|
||||
index
|
||||
for index, record in enumerate(records)
|
||||
if not (updated_at := _coerce_int(record.get("updated_at_ms")))
|
||||
or now_ms - updated_at >= _PR_REFRESH_INTERVAL_MS
|
||||
][:_MAX_PR_REFRESH_PER_REQUEST]
|
||||
if not stale_indexes:
|
||||
return records
|
||||
token = await get_github_app_installation_token()
|
||||
if not token:
|
||||
return records
|
||||
|
||||
refreshed = list(records)
|
||||
semaphore = asyncio.Semaphore(_PR_REFRESH_CONCURRENCY)
|
||||
|
||||
async def refresh_one(index: int, client: httpx.AsyncClient) -> tuple[int, dict[str, Any]]:
|
||||
async with semaphore:
|
||||
try:
|
||||
return index, await _refresh_pr_record(client, token, records[index])
|
||||
except Exception:
|
||||
logger.debug("Failed to refresh usage PR record", exc_info=True)
|
||||
return index, records[index]
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
for index, record in await asyncio.gather(
|
||||
*(refresh_one(index, client) for index in stale_indexes)
|
||||
):
|
||||
refreshed[index] = record
|
||||
return refreshed
|
||||
|
||||
|
||||
async def list_agent_usage_leaderboard(
|
||||
*,
|
||||
period: str | None,
|
||||
limit: int,
|
||||
current_login: str | None,
|
||||
current_email: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the Open SWE Agent usage leaderboard from recorded telemetry."""
|
||||
normalized_period = _normalize_period(period)
|
||||
cutoff_ms = _period_cutoff_ms(normalized_period)
|
||||
safe_limit = min(max(limit, 1), 100)
|
||||
users: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for thread in await _search_values(USAGE_THREAD_NAMESPACE):
|
||||
if thread.get("agent_kind") != "agent" or thread.get("source") not in _AGENT_SOURCES:
|
||||
continue
|
||||
if not _in_period(thread, cutoff_ms):
|
||||
continue
|
||||
user = _ensure_user(
|
||||
users,
|
||||
github_login=thread.get("github_login"),
|
||||
email=thread.get("user_email"),
|
||||
)
|
||||
user["agent_runs"] += 1
|
||||
model_id = thread.get("model_id")
|
||||
if isinstance(model_id, str) and model_id:
|
||||
user["model_counts"][model_id] += 1
|
||||
|
||||
pr_records = [
|
||||
pr
|
||||
for pr in await _search_values(USAGE_PR_NAMESPACE)
|
||||
if pr.get("agent_kind") == "agent" and _in_period(pr, cutoff_ms)
|
||||
]
|
||||
for pr in await _refresh_pr_records(pr_records):
|
||||
user = _ensure_user(
|
||||
users,
|
||||
github_login=pr.get("github_login"),
|
||||
email=pr.get("user_email"),
|
||||
)
|
||||
additions = _coerce_int(pr.get("additions"))
|
||||
deletions = _coerce_int(pr.get("deletions"))
|
||||
user["prs_opened"] += 1
|
||||
if pr.get("merged"):
|
||||
user["merged_prs"] += 1
|
||||
user["additions"] += additions
|
||||
user["deletions"] += deletions
|
||||
user["agent_loc"] += additions + deletions
|
||||
|
||||
sorted_users = sorted(
|
||||
users.values(),
|
||||
key=lambda item: (
|
||||
-item["agent_loc"],
|
||||
-item["prs_opened"],
|
||||
-item["agent_runs"],
|
||||
item.get("name") or "",
|
||||
),
|
||||
)
|
||||
current_keys = {
|
||||
_user_key(current_login, current_email),
|
||||
_user_key(current_login, None),
|
||||
_user_key(None, current_email),
|
||||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
current_user_row: dict[str, Any] | None = None
|
||||
for index, user in enumerate(sorted_users, start=1):
|
||||
model_counts: Counter[str] = user.pop("model_counts")
|
||||
favorite_model = model_counts.most_common(1)[0][0] if model_counts else "default"
|
||||
github_login = user.get("github_login") or None
|
||||
is_current_user = user["key"] in current_keys
|
||||
row = {
|
||||
"rank": index,
|
||||
"user": {
|
||||
"name": user.get("name") if is_current_user or github_login else "Open SWE user",
|
||||
"github_login": github_login,
|
||||
"email": (user.get("email") or None) if is_current_user else None,
|
||||
},
|
||||
"favorite_model": favorite_model,
|
||||
"agent_runs": user["agent_runs"],
|
||||
"prs_opened": user["prs_opened"],
|
||||
"merged_prs": user["merged_prs"],
|
||||
"agent_loc": user["agent_loc"],
|
||||
"additions": user["additions"],
|
||||
"deletions": user["deletions"],
|
||||
}
|
||||
if is_current_user:
|
||||
current_user_row = row
|
||||
if len(rows) < safe_limit:
|
||||
rows.append(row)
|
||||
|
||||
if current_user_row and all(row["rank"] != current_user_row["rank"] for row in rows):
|
||||
rows.append(current_user_row)
|
||||
|
||||
return {
|
||||
"period": normalized_period,
|
||||
"rows": rows,
|
||||
"total_members": len(sorted_users),
|
||||
"current_user_rank": current_user_row["rank"] if current_user_row else None,
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ from fastapi.responses import RedirectResponse, Response, StreamingResponse
|
|||
from pydantic import BaseModel
|
||||
|
||||
from .admin import is_admin
|
||||
from .agent_usage import list_agent_usage_leaderboard
|
||||
from .analyzer_cron import remove_continual_cron
|
||||
from .enabled_repos import (
|
||||
list_enabled_review_repos,
|
||||
|
|
@ -747,6 +748,20 @@ async def api_delete_review_style(
|
|||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.get("/agent-usage-leaderboard")
|
||||
async def api_agent_usage_leaderboard(
|
||||
period: str | None = "30d",
|
||||
limit: int = 10,
|
||||
session: dict[str, Any] = _SESSION_DEP,
|
||||
) -> dict[str, Any]:
|
||||
return await list_agent_usage_leaderboard(
|
||||
period=period,
|
||||
limit=limit,
|
||||
current_login=session["sub"],
|
||||
current_email=session.get("email"),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/threads")
|
||||
async def api_list_threads(
|
||||
session: dict[str, Any] = _SESSION_DEP,
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from .dashboard.agent_overrides import (
|
|||
profile_create_prs,
|
||||
resolve_github_login,
|
||||
)
|
||||
from .dashboard.agent_usage import record_agent_thread_usage
|
||||
from .dashboard.options import DEFAULT_MODEL_ID, SUPPORTED_MODEL_IDS, model_supports_effort
|
||||
from .dashboard.team_settings import get_team_default_model_pair
|
||||
from .integrations.langsmith import _configure_github_proxy
|
||||
|
|
@ -514,6 +515,32 @@ async def get_agent(config: RunnableConfig) -> Pregel:
|
|||
)
|
||||
logger.info("Configured model fallback %s -> %s", model_id, fallback_model_id)
|
||||
|
||||
source = (
|
||||
configurable.get("source") if isinstance(configurable.get("source"), str) else "dashboard"
|
||||
)
|
||||
user_email = configurable.get("user_email")
|
||||
user_email = user_email if isinstance(user_email, str) else ""
|
||||
try:
|
||||
await client.threads.update(
|
||||
thread_id=thread_id,
|
||||
metadata={
|
||||
"agent_kind": "agent",
|
||||
"model": model_id,
|
||||
"effort": profile_effort,
|
||||
"source": source,
|
||||
},
|
||||
)
|
||||
await record_agent_thread_usage(
|
||||
thread_id=thread_id,
|
||||
github_login=profile_login,
|
||||
user_email=user_email,
|
||||
model_id=model_id,
|
||||
effort=profile_effort,
|
||||
source=source,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to record agent usage for thread %s", thread_id, exc_info=True)
|
||||
|
||||
logger.info("Returning agent with sandbox for thread %s", thread_id)
|
||||
main_model = make_model(model_id, **model_kwargs)
|
||||
subagent_model = make_model(subagent_model_id, **subagent_model_kwargs)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ from typing import Any
|
|||
|
||||
import httpx
|
||||
from langgraph.config import get_config
|
||||
from langgraph_sdk import get_client
|
||||
|
||||
from ..dashboard.agent_usage import record_agent_pr_usage
|
||||
from ..utils.github_app import get_github_app_installation_token
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -67,6 +69,100 @@ async def _find_existing_pr(
|
|||
return items[0] if isinstance(items, list) and items else None
|
||||
|
||||
|
||||
async def _fetch_pr_details(
|
||||
client: httpx.AsyncClient, token: str, owner: str, repo: str, pr_number: int
|
||||
) -> dict[str, Any]:
|
||||
resp = await client.get(
|
||||
f"{GITHUB_API}/repos/{owner}/{repo}/pulls/{pr_number}",
|
||||
headers=_auth_headers(token),
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.debug(
|
||||
"GitHub returned %s fetching PR stats for %s/%s#%s: %s",
|
||||
resp.status_code,
|
||||
owner,
|
||||
repo,
|
||||
pr_number,
|
||||
resp.text,
|
||||
)
|
||||
return {}
|
||||
data = resp.json()
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
async def _record_pr_telemetry(
|
||||
*,
|
||||
client: httpx.AsyncClient,
|
||||
token: str,
|
||||
owner: str,
|
||||
repo: str,
|
||||
head: str,
|
||||
base: str,
|
||||
pr: dict[str, Any],
|
||||
) -> None:
|
||||
pr_number = pr.get("number")
|
||||
if not isinstance(pr_number, int):
|
||||
return
|
||||
try:
|
||||
details = await _fetch_pr_details(client, token, owner, repo, pr_number)
|
||||
config = get_config()
|
||||
configurable = config.get("configurable", {}) if isinstance(config, dict) else {}
|
||||
thread_id = configurable.get("thread_id")
|
||||
github_login = configurable.get("github_login")
|
||||
user_email = configurable.get("user_email")
|
||||
if not isinstance(github_login, str) or not github_login.strip():
|
||||
from ..dashboard.user_mappings import login_for_email
|
||||
|
||||
github_login = (
|
||||
await login_for_email(user_email if isinstance(user_email, str) else None) or ""
|
||||
)
|
||||
pr_url = details.get("html_url") or pr.get("html_url")
|
||||
merged = bool(details.get("merged"))
|
||||
state = details.get("state") if isinstance(details.get("state"), str) else "open"
|
||||
additions = details.get("additions") if isinstance(details.get("additions"), int) else 0
|
||||
deletions = details.get("deletions") if isinstance(details.get("deletions"), int) else 0
|
||||
changed_files = (
|
||||
details.get("changed_files") if isinstance(details.get("changed_files"), int) else 0
|
||||
)
|
||||
await record_agent_pr_usage(
|
||||
thread_id=thread_id if isinstance(thread_id, str) else None,
|
||||
github_login=github_login,
|
||||
user_email=user_email if isinstance(user_email, str) else None,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
pr_number=pr_number,
|
||||
pr_url=pr_url if isinstance(pr_url, str) else None,
|
||||
head=head,
|
||||
base=base,
|
||||
additions=additions,
|
||||
deletions=deletions,
|
||||
changed_files=changed_files,
|
||||
state=state,
|
||||
merged=merged,
|
||||
)
|
||||
if isinstance(thread_id, str) and thread_id:
|
||||
await get_client().threads.update(
|
||||
thread_id=thread_id,
|
||||
metadata={
|
||||
"agent_kind": "agent",
|
||||
"pr_url": pr_url if isinstance(pr_url, str) else "",
|
||||
"pr_number": pr_number,
|
||||
"pr_state": "merged" if merged else state,
|
||||
"branch_name": head,
|
||||
"base_branch": base,
|
||||
"diff_stats": {
|
||||
"files": changed_files,
|
||||
"additions": additions,
|
||||
"deletions": deletions,
|
||||
},
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to record PR usage for %s/%s#%s", owner, repo, pr_number, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
async def _open_pull_request(
|
||||
*,
|
||||
owner: str,
|
||||
|
|
@ -93,6 +189,16 @@ async def _open_pull_request(
|
|||
)
|
||||
if resp.status_code == 201:
|
||||
pr = resp.json()
|
||||
if isinstance(pr, dict):
|
||||
await _record_pr_telemetry(
|
||||
client=client,
|
||||
token=token,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
head=head,
|
||||
base=base,
|
||||
pr=pr,
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"created": True,
|
||||
|
|
@ -107,6 +213,15 @@ async def _open_pull_request(
|
|||
if resp.status_code == 422: # noqa: PLR2004
|
||||
existing = await _find_existing_pr(client, token, owner, repo, head)
|
||||
if existing is not None:
|
||||
await _record_pr_telemetry(
|
||||
client=client,
|
||||
token=token,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
head=head,
|
||||
base=base,
|
||||
pr=existing,
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"created": False,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
IoGitPullRequestOutline,
|
||||
IoOptionsOutline,
|
||||
IoSettingsOutline,
|
||||
IoStatsChartOutline,
|
||||
} from "react-icons/io5";
|
||||
import type { ComponentType, SVGProps } from "react";
|
||||
|
||||
|
|
@ -29,6 +30,7 @@ interface NavItem {
|
|||
const NAV: Array<NavItem> = [
|
||||
{ to: "/my-settings", label: "Profile Settings", icon: IoOptionsOutline },
|
||||
{ to: "/cloud-agents", label: "Open SWE Agent", icon: IoCloudOutline },
|
||||
{ to: "/usage", label: "Usage Leaderboard", icon: IoStatsChartOutline },
|
||||
{ to: "/review", label: "Open SWE Review", icon: IoGitPullRequestOutline },
|
||||
{ to: "/admin", label: "Admin", icon: IoSettingsOutline, adminOnly: true },
|
||||
];
|
||||
|
|
|
|||
|
|
@ -137,6 +137,31 @@ export interface UserMappingsPage {
|
|||
page_size: number;
|
||||
}
|
||||
|
||||
export type UsageLeaderboardPeriod = "7d" | "30d" | "all";
|
||||
|
||||
export interface UsageLeaderboardRow {
|
||||
rank: number;
|
||||
user: {
|
||||
name: string;
|
||||
github_login: string | null;
|
||||
email: string | null;
|
||||
};
|
||||
favorite_model: string;
|
||||
agent_runs: number;
|
||||
prs_opened: number;
|
||||
merged_prs: number;
|
||||
agent_loc: number;
|
||||
additions: number;
|
||||
deletions: number;
|
||||
}
|
||||
|
||||
export interface UsageLeaderboardPayload {
|
||||
period: UsageLeaderboardPeriod;
|
||||
rows: Array<UsageLeaderboardRow>;
|
||||
total_members: number;
|
||||
current_user_rank: number | null;
|
||||
}
|
||||
|
||||
export interface Repository {
|
||||
full_name: string;
|
||||
private: boolean;
|
||||
|
|
@ -215,6 +240,10 @@ export const api = {
|
|||
method: "PUT",
|
||||
body: JSON.stringify({ full_name, enabled }),
|
||||
}),
|
||||
usageLeaderboard: (period: UsageLeaderboardPeriod = "30d", limit = 10) =>
|
||||
request<UsageLeaderboardPayload>(
|
||||
`/agent-usage-leaderboard?period=${encodeURIComponent(period)}&limit=${limit}`,
|
||||
),
|
||||
myMapping: () => request<Partial<UserMapping>>("/my-mapping"),
|
||||
adminListUserMappings: (page = 1, pageSize = 20) =>
|
||||
request<UserMappingsPage>(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
|
||||
|
||||
import { Route as rootRouteImport } from './routes/__root'
|
||||
import { Route as UsageRouteImport } from './routes/usage'
|
||||
import { Route as ReviewRouteImport } from './routes/review'
|
||||
import { Route as MySettingsRouteImport } from './routes/my-settings'
|
||||
import { Route as LoginRouteImport } from './routes/login'
|
||||
|
|
@ -22,6 +23,11 @@ import { Route as ReviewStylesRouteImport } from './routes/review_.styles'
|
|||
import { Route as AgentsThreadIdRouteImport } from './routes/agents/$threadId'
|
||||
import { Route as ReviewRepositoriesOwnerRouteImport } from './routes/review_.repositories.$owner'
|
||||
|
||||
const UsageRoute = UsageRouteImport.update({
|
||||
id: '/usage',
|
||||
path: '/usage',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
const ReviewRoute = ReviewRouteImport.update({
|
||||
id: '/review',
|
||||
path: '/review',
|
||||
|
|
@ -92,6 +98,7 @@ export interface FileRoutesByFullPath {
|
|||
'/login': typeof LoginRoute
|
||||
'/my-settings': typeof MySettingsRoute
|
||||
'/review': typeof ReviewRoute
|
||||
'/usage': typeof UsageRoute
|
||||
'/agents/$threadId': typeof AgentsThreadIdRoute
|
||||
'/review/styles': typeof ReviewStylesRoute
|
||||
'/agents/': typeof AgentsIndexRoute
|
||||
|
|
@ -105,6 +112,7 @@ export interface FileRoutesByTo {
|
|||
'/login': typeof LoginRoute
|
||||
'/my-settings': typeof MySettingsRoute
|
||||
'/review': typeof ReviewRoute
|
||||
'/usage': typeof UsageRoute
|
||||
'/agents/$threadId': typeof AgentsThreadIdRoute
|
||||
'/review/styles': typeof ReviewStylesRoute
|
||||
'/agents': typeof AgentsIndexRoute
|
||||
|
|
@ -120,6 +128,7 @@ export interface FileRoutesById {
|
|||
'/login': typeof LoginRoute
|
||||
'/my-settings': typeof MySettingsRoute
|
||||
'/review': typeof ReviewRoute
|
||||
'/usage': typeof UsageRoute
|
||||
'/agents/$threadId': typeof AgentsThreadIdRoute
|
||||
'/review_/styles': typeof ReviewStylesRoute
|
||||
'/agents/': typeof AgentsIndexRoute
|
||||
|
|
@ -136,6 +145,7 @@ export interface FileRouteTypes {
|
|||
| '/login'
|
||||
| '/my-settings'
|
||||
| '/review'
|
||||
| '/usage'
|
||||
| '/agents/$threadId'
|
||||
| '/review/styles'
|
||||
| '/agents/'
|
||||
|
|
@ -149,6 +159,7 @@ export interface FileRouteTypes {
|
|||
| '/login'
|
||||
| '/my-settings'
|
||||
| '/review'
|
||||
| '/usage'
|
||||
| '/agents/$threadId'
|
||||
| '/review/styles'
|
||||
| '/agents'
|
||||
|
|
@ -163,6 +174,7 @@ export interface FileRouteTypes {
|
|||
| '/login'
|
||||
| '/my-settings'
|
||||
| '/review'
|
||||
| '/usage'
|
||||
| '/agents/$threadId'
|
||||
| '/review_/styles'
|
||||
| '/agents/'
|
||||
|
|
@ -178,6 +190,7 @@ export interface RootRouteChildren {
|
|||
LoginRoute: typeof LoginRoute
|
||||
MySettingsRoute: typeof MySettingsRoute
|
||||
ReviewRoute: typeof ReviewRoute
|
||||
UsageRoute: typeof UsageRoute
|
||||
ReviewStylesRoute: typeof ReviewStylesRoute
|
||||
ReviewRepositoriesOwnerRoute: typeof ReviewRepositoriesOwnerRoute
|
||||
}
|
||||
|
|
@ -191,6 +204,13 @@ declare module '@tanstack/react-router' {
|
|||
preLoaderRoute: typeof ReviewRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/usage': {
|
||||
id: '/usage'
|
||||
path: '/usage'
|
||||
fullPath: '/usage'
|
||||
preLoaderRoute: typeof UsageRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/my-settings': {
|
||||
id: '/my-settings'
|
||||
path: '/my-settings'
|
||||
|
|
@ -293,6 +313,7 @@ const rootRouteChildren: RootRouteChildren = {
|
|||
LoginRoute: LoginRoute,
|
||||
MySettingsRoute: MySettingsRoute,
|
||||
ReviewRoute: ReviewRoute,
|
||||
UsageRoute: UsageRoute,
|
||||
ReviewStylesRoute: ReviewStylesRoute,
|
||||
ReviewRepositoriesOwnerRoute: ReviewRepositoriesOwnerRoute,
|
||||
}
|
||||
|
|
|
|||
205
ui/src/routes/usage.tsx
Normal file
205
ui/src/routes/usage.tsx
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
import { Navigate, createFileRoute } from "@tanstack/react-router"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
|
||||
import type { UsageLeaderboardPeriod, UsageLeaderboardRow } from "@/lib/api"
|
||||
import { AppShell, SettingsSection } from "@/components/AppShell"
|
||||
import { Avatar, AvatarFallback } from "@/components/ui/avatar"
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select"
|
||||
import { Skeleton } from "@/components/ui/skeleton"
|
||||
import { api } from "@/lib/api"
|
||||
import { useSession } from "@/lib/session"
|
||||
|
||||
export const Route = createFileRoute("/usage")({
|
||||
validateSearch: (search: Record<string, unknown>) => ({
|
||||
period: typeof search.period === "string" ? search.period : undefined,
|
||||
}),
|
||||
component: UsagePage,
|
||||
})
|
||||
|
||||
const PERIOD_LABELS: Record<UsageLeaderboardPeriod, string> = {
|
||||
"7d": "Last 7 days",
|
||||
"30d": "Last 30 days",
|
||||
all: "All time",
|
||||
}
|
||||
|
||||
function UsagePage() {
|
||||
const session = useSession()
|
||||
const period =
|
||||
(Route.useSearch().period as UsageLeaderboardPeriod | undefined) ?? "30d"
|
||||
const navigate = Route.useNavigate()
|
||||
const activePeriod: UsageLeaderboardPeriod = ["7d", "30d", "all"].includes(
|
||||
period
|
||||
)
|
||||
? period
|
||||
: "30d"
|
||||
|
||||
const leaderboard = useQuery({
|
||||
queryKey: ["usageLeaderboard", activePeriod],
|
||||
queryFn: () => api.usageLeaderboard(activePeriod, 10),
|
||||
enabled: !!session.data,
|
||||
})
|
||||
|
||||
if (session.isLoading) {
|
||||
return (
|
||||
<main className="p-6">
|
||||
<Skeleton className="h-64 w-full" />
|
||||
</main>
|
||||
)
|
||||
}
|
||||
if (!session.data) return <Navigate to="/login" />
|
||||
|
||||
return (
|
||||
<AppShell
|
||||
user={session.data}
|
||||
title="Usage Leaderboard"
|
||||
description="Open SWE Agent usage recorded from this release onward. Reviewer-agent runs are excluded."
|
||||
>
|
||||
<SettingsSection
|
||||
title="Leaderboard"
|
||||
description="Ranked by agent lines of code, then PRs opened and agent runs."
|
||||
action={
|
||||
<Select
|
||||
value={activePeriod}
|
||||
onValueChange={(value) =>
|
||||
navigate({
|
||||
to: "/usage",
|
||||
search: { period: value as UsageLeaderboardPeriod },
|
||||
})
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-36">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{Object.entries(PERIOD_LABELS).map(([value, label]) => (
|
||||
<SelectItem key={value} value={value}>
|
||||
{label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
}
|
||||
>
|
||||
{leaderboard.isLoading ? (
|
||||
<div className="space-y-2 p-4">
|
||||
<Skeleton className="h-12 w-full" />
|
||||
<Skeleton className="h-12 w-full" />
|
||||
<Skeleton className="h-12 w-full" />
|
||||
</div>
|
||||
) : leaderboard.isError ? (
|
||||
<p className="p-4 text-xs text-destructive">
|
||||
Failed to load usage leaderboard: {leaderboard.error.message}
|
||||
</p>
|
||||
) : !leaderboard.data?.rows.length ? (
|
||||
<div className="p-6 text-center text-sm text-muted-foreground">
|
||||
No Open SWE Agent usage has been recorded for{" "}
|
||||
{PERIOD_LABELS[activePeriod].toLowerCase()} yet.
|
||||
</div>
|
||||
) : (
|
||||
<UsageTable
|
||||
rows={leaderboard.data.rows}
|
||||
totalMembers={leaderboard.data.total_members}
|
||||
/>
|
||||
)}
|
||||
</SettingsSection>
|
||||
</AppShell>
|
||||
)
|
||||
}
|
||||
|
||||
function UsageTable({
|
||||
rows,
|
||||
totalMembers,
|
||||
}: {
|
||||
rows: Array<UsageLeaderboardRow>
|
||||
totalMembers: number
|
||||
}) {
|
||||
return (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[760px] text-sm">
|
||||
<thead className="border-b border-border text-xs text-muted-foreground">
|
||||
<tr>
|
||||
<th className="w-14 px-4 py-3 text-left font-normal">Rank</th>
|
||||
<th className="px-2 py-3 text-left font-normal">User</th>
|
||||
<th className="px-2 py-3 text-left font-normal">Favorite Model</th>
|
||||
<th className="px-2 py-3 text-right font-normal">Agent Runs</th>
|
||||
<th className="px-2 py-3 text-right font-normal">PRs Opened</th>
|
||||
<th className="px-2 py-3 text-right font-normal">Merged PRs</th>
|
||||
<th className="px-4 py-3 text-right font-normal">Agent LOC</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-border">
|
||||
{rows.map((row) => (
|
||||
<tr
|
||||
key={`${row.rank}-${row.user.github_login ?? row.user.email ?? row.user.name}`}
|
||||
>
|
||||
<td className="px-4 py-3 text-muted-foreground">{row.rank}</td>
|
||||
<td className="px-2 py-3">
|
||||
<UserCell row={row} />
|
||||
</td>
|
||||
<td className="max-w-48 truncate px-2 py-3 text-muted-foreground">
|
||||
{row.favorite_model}
|
||||
</td>
|
||||
<td className="px-2 py-3 text-right tabular-nums">
|
||||
{formatNumber(row.agent_runs)}
|
||||
</td>
|
||||
<td className="px-2 py-3 text-right tabular-nums">
|
||||
{formatNumber(row.prs_opened)}
|
||||
</td>
|
||||
<td className="px-2 py-3 text-right tabular-nums">
|
||||
{formatNumber(row.merged_prs)}
|
||||
</td>
|
||||
<td
|
||||
className="px-4 py-3 text-right tabular-nums"
|
||||
title={`${formatNumber(row.additions)} additions, ${formatNumber(row.deletions)} deletions`}
|
||||
>
|
||||
{formatNumber(row.agent_loc)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<div className="border-t border-border px-4 py-3 text-xs text-muted-foreground">
|
||||
Top {Math.min(10, totalMembers)} of {formatNumber(totalMembers)} member
|
||||
{totalMembers === 1 ? "" : "s"}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function UserCell({ row }: { row: UsageLeaderboardRow }) {
|
||||
const initials = initialsFor(row.user.name)
|
||||
return (
|
||||
<div className="flex min-w-0 items-center gap-2.5">
|
||||
<Avatar>
|
||||
<AvatarFallback>{initials}</AvatarFallback>
|
||||
</Avatar>
|
||||
<div className="flex min-w-0 flex-col">
|
||||
<span className="truncate font-medium text-foreground">
|
||||
{row.user.name}
|
||||
</span>
|
||||
<span className="truncate text-xs text-muted-foreground">
|
||||
{row.user.email ?? row.user.github_login ?? "unknown"}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function initialsFor(name: string): string {
|
||||
const parts = name.trim().split(/\s+/).filter(Boolean)
|
||||
if (!parts.length) return "?"
|
||||
const first = parts[0] ?? "?"
|
||||
const second = parts[1]
|
||||
if (!second) return first.slice(0, 2).toUpperCase()
|
||||
return `${first[0] ?? ""}${second[0] ?? ""}`.toUpperCase()
|
||||
}
|
||||
|
||||
function formatNumber(value: number): string {
|
||||
return new Intl.NumberFormat().format(value)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue