From 2812390f8237025f9c9a373ba6e430db4150f1e2 Mon Sep 17 00:00:00 2001 From: Johannes du Plessis Date: Tue, 9 Jun 2026 12:52:57 -0700 Subject: [PATCH] fix(evals): call reviewer-eval judge directly instead of via gateway (#1472) --- evals/reviewer/judge.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/evals/reviewer/judge.py b/evals/reviewer/judge.py index f4d74c54..f6cc27c5 100644 --- a/evals/reviewer/judge.py +++ b/evals/reviewer/judge.py @@ -13,6 +13,7 @@ directly comparable to martian's published numbers. from __future__ import annotations import json +import os import threading from typing import Any from uuid import UUID @@ -22,6 +23,11 @@ from langsmith.schemas import Example, Run JUDGE_MODEL = "claude-opus-4-5" +# Call Anthropic directly. Without an explicit base_url the Anthropic SDK falls +# back to ANTHROPIC_BASE_URL, which in dev shells points at the LangSmith +# gateway and 403s for this model — silently nulling every judge score. +JUDGE_BASE_URL = os.environ.get("JUDGE_ANTHROPIC_BASE_URL", "https://api.anthropic.com") + JUDGE_SYSTEM = "You are a precise code review evaluator. Always respond with valid JSON." JUDGE_PROMPT = """You are evaluating AI code review tools. @@ -48,7 +54,20 @@ _judge: ChatAnthropic | None = None def _get_judge() -> ChatAnthropic: global _judge if _judge is None: - _judge = ChatAnthropic(model=JUDGE_MODEL, temperature=0.0, max_tokens=512) + api_key = os.environ.get("JUDGE_ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + raise RuntimeError( + "No Anthropic API key for the judge. Set JUDGE_ANTHROPIC_API_KEY or " + "ANTHROPIC_API_KEY (the judge calls Anthropic directly, not via a gateway)." + ) + _judge = ChatAnthropic( + model=JUDGE_MODEL, + temperature=0.0, + max_tokens=512, + base_url=JUDGE_BASE_URL, + api_key=api_key, + max_retries=3, + ) return _judge