f05a98dba1
Observability slim layer. Every full run appends one JSON line to ~/.claude/logs/orchestrator/YYYY-MM-DD.jsonl with timestamp, sha256-prefix task hash (raw task is never logged), retrieved memory names, router choice, runtime, tokens in/out, success/error. risk_class and confidence fields are reserved nulls for Phase 5. - telemetry.py: log_run(), build_record(), task_hash(), token-usage extraction from AIMessage.usage_metadata. log_run swallows all exceptions — telemetry never kills a run. - run.py: wraps app.invoke in try/except with a monotonic-clock window; logs on both success and failure. --route-only path is left unlogged (no agent work, doesn't represent a "run"). - scripts/weekly_summary.py: scans the last 7 days of JSONL and prints a markdown digest (routes, unknown rate, cross-review rate, success rate, total spend, mean tokens/route). Schedule via /schedule and pipe stdout to Slack from the scheduler. Cost rates per route are rough Sonnet/Haiku/GPT/Gemini/DeepSeek defaults suitable for spotting runaway prompts, not finance. Router tokens for structured-output calls aren't captured (they don't surface through the message trail); agent tokens are the dominant component anyway. Validated: golden-set 21/21 still passing; full-run smoke writes expected fields; weekly_summary.py prints clean markdown from a 2-run log.
91 lines
2.9 KiB
Python
91 lines
2.9 KiB
Python
"""JSONL telemetry for orchestrator runs.
|
|
|
|
Appends one JSON line per run to ~/.claude/logs/orchestrator/YYYY-MM-DD.jsonl.
|
|
Logging failures are swallowed — telemetry must never kill the orchestrator.
|
|
|
|
Fields (Phase 4):
|
|
timestamp ISO-8601 UTC
|
|
task_hash sha256(task)[:16] — never log raw task content
|
|
retrieved list[str] — memory names surfaced by the retriever
|
|
route str | None — the router's choice (or None on early failure)
|
|
risk_class null — placeholder for Phase 5
|
|
confidence null — placeholder for Phase 5
|
|
runtime_seconds float
|
|
tokens_in int
|
|
tokens_out int
|
|
success bool
|
|
error str | None — type+message if the run raised
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
LOG_DIR = Path(os.path.expanduser("~/.claude/logs/orchestrator"))
|
|
|
|
|
|
def task_hash(task: str) -> str:
|
|
return hashlib.sha256(task.encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
def extract_token_usage(messages: list[Any]) -> tuple[int, int]:
|
|
"""Sum input/output token usage across all AIMessages in the run.
|
|
|
|
Each LangChain AIMessage from a provider with usage telemetry exposes a
|
|
`usage_metadata` dict with at least input_tokens / output_tokens. Messages
|
|
without usage are skipped.
|
|
"""
|
|
in_tok = 0
|
|
out_tok = 0
|
|
for m in messages or []:
|
|
usage = getattr(m, "usage_metadata", None)
|
|
if not usage:
|
|
continue
|
|
in_tok += int(usage.get("input_tokens", 0) or 0)
|
|
out_tok += int(usage.get("output_tokens", 0) or 0)
|
|
return in_tok, out_tok
|
|
|
|
|
|
def build_record(
|
|
task: str,
|
|
started: float,
|
|
finished: float,
|
|
result: dict | None,
|
|
success: bool,
|
|
error: str | None,
|
|
) -> dict:
|
|
retrieved_names = [
|
|
m.get("name", "?") for m in (result or {}).get("retrieved", []) or []
|
|
]
|
|
tokens_in, tokens_out = extract_token_usage((result or {}).get("messages", []))
|
|
return {
|
|
"timestamp": datetime.datetime.now(datetime.UTC).isoformat(),
|
|
"task_hash": task_hash(task),
|
|
"retrieved": retrieved_names,
|
|
"route": (result or {}).get("route"),
|
|
"risk_class": None,
|
|
"confidence": None,
|
|
"runtime_seconds": round(finished - started, 3),
|
|
"tokens_in": tokens_in,
|
|
"tokens_out": tokens_out,
|
|
"success": success,
|
|
"error": error,
|
|
}
|
|
|
|
|
|
def log_run(record: dict, log_dir: Path = LOG_DIR) -> None:
|
|
"""Append a single record as one JSON line. Never raises."""
|
|
try:
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
date = datetime.date.today().isoformat()
|
|
path = log_dir / f"{date}.jsonl"
|
|
with path.open("a") as f:
|
|
f.write(json.dumps(record) + "\n")
|
|
except Exception:
|
|
# Telemetry never kills the run.
|
|
pass
|