5378bac69b
Follow-up to the Phases 1-2-4 modernization PR. - models.py: extend retry coverage to Gemini transient errors (google.api_core: ResourceExhausted, InternalServerError, ServiceUnavailable, DeadlineExceeded, GatewayTimeout). Before this, a 429 or 5xx from Google would crash the scanner route. Retriable count went from 6 to 11. - retriever.py: batch cache-miss embeddings into a single embed_documents() call instead of one embed_query() per memory. Cold rebuild of 87 memories went from ~30s to ~10s; one HTTPS round-trip instead of 87. - telemetry.py + scripts/weekly_summary.py: use UTC date for log filenames so they align with the UTC `timestamp` field inside each record. Eliminates the off-by-one near local midnight and matches Sea Haven's UTC-for-logs convention. - scripts/weekly_summary.py: include failed runs in the total cost estimate (they consumed tokens too) and surface a `(incl. $X on failed runs)` breakdown so outages are visible in the digest. Validated: ruff clean; weekly_summary.py still prints; retriever cold rebuild + cache hit both verified end-to-end.
95 lines
3 KiB
Python
95 lines
3 KiB
Python
"""JSONL telemetry for orchestrator runs.
|
|
|
|
Appends one JSON line per run to ~/.claude/logs/orchestrator/YYYY-MM-DD.jsonl.
|
|
Logging failures are swallowed — telemetry must never kill the orchestrator.
|
|
|
|
Fields (Phase 4):
|
|
timestamp ISO-8601 UTC
|
|
task_hash sha256(task)[:16] — never log raw task content
|
|
retrieved list[str] — memory names surfaced by the retriever
|
|
route str | None — the router's choice (or None on early failure)
|
|
risk_class null — placeholder for Phase 5
|
|
confidence null — placeholder for Phase 5
|
|
runtime_seconds float
|
|
tokens_in int
|
|
tokens_out int
|
|
success bool
|
|
error str | None — type+message if the run raised
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
LOG_DIR = Path(os.path.expanduser("~/.claude/logs/orchestrator"))
|
|
|
|
|
|
def task_hash(task: str) -> str:
|
|
return hashlib.sha256(task.encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
def extract_token_usage(messages: list[Any]) -> tuple[int, int]:
|
|
"""Sum input/output token usage across all AIMessages in the run.
|
|
|
|
Each LangChain AIMessage from a provider with usage telemetry exposes a
|
|
`usage_metadata` dict with at least input_tokens / output_tokens. Messages
|
|
without usage are skipped.
|
|
"""
|
|
in_tok = 0
|
|
out_tok = 0
|
|
for m in messages or []:
|
|
usage = getattr(m, "usage_metadata", None)
|
|
if not usage:
|
|
continue
|
|
in_tok += int(usage.get("input_tokens", 0) or 0)
|
|
out_tok += int(usage.get("output_tokens", 0) or 0)
|
|
return in_tok, out_tok
|
|
|
|
|
|
def build_record(
|
|
task: str,
|
|
started: float,
|
|
finished: float,
|
|
result: dict | None,
|
|
success: bool,
|
|
error: str | None,
|
|
) -> dict:
|
|
retrieved_names = [
|
|
m.get("name", "?") for m in (result or {}).get("retrieved", []) or []
|
|
]
|
|
tokens_in, tokens_out = extract_token_usage((result or {}).get("messages", []))
|
|
return {
|
|
"timestamp": datetime.datetime.now(datetime.UTC).isoformat(),
|
|
"task_hash": task_hash(task),
|
|
"retrieved": retrieved_names,
|
|
"route": (result or {}).get("route"),
|
|
"risk_class": None,
|
|
"confidence": None,
|
|
"runtime_seconds": round(finished - started, 3),
|
|
"tokens_in": tokens_in,
|
|
"tokens_out": tokens_out,
|
|
"success": success,
|
|
"error": error,
|
|
}
|
|
|
|
|
|
def log_run(record: dict, log_dir: Path = LOG_DIR) -> None:
|
|
"""Append a single record as one JSON line. Never raises.
|
|
|
|
Filename uses the UTC date so it stays aligned with the UTC `timestamp`
|
|
field inside the record (no off-by-one near local midnight).
|
|
"""
|
|
try:
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
date = datetime.datetime.now(datetime.UTC).date().isoformat()
|
|
path = log_dir / f"{date}.jsonl"
|
|
with path.open("a") as f:
|
|
f.write(json.dumps(record) + "\n")
|
|
except Exception:
|
|
# Telemetry never kills the run.
|
|
pass
|