diff --git a/CUSTOMIZATION.md b/CUSTOMIZATION.md index 33037489..6f2ca79f 100644 --- a/CUSTOMIZATION.md +++ b/CUSTOMIZATION.md @@ -193,6 +193,25 @@ async def get_agent(config: RunnableConfig) -> Pregel: return create_deep_agent(model=model, ...) ``` +### Routing through the LangSmith LLM Gateway + +Model calls can be proxied through the [LangSmith LLM Gateway](https://docs.langchain.com/langsmith/llm-gateway) (private beta) instead of hitting providers directly. The gateway authenticates with a **LangSmith API key** that has the `gateway:invoke` permission and resolves the real provider key from workspace Provider Secrets, so no provider API keys are needed at runtime — and it adds central spend limits, PII/secrets redaction, and tracing. Your org must have the gateway enabled with Provider Secrets configured. + +Routing is opt-in and off by default. Enable it either way: + +| Env var | Default | Purpose | +|---|---|---| +| `LANGSMITH_GATEWAY_ENABLED` | `false` | Deployment-level default for gateway routing. | +| `LANGSMITH_GATEWAY_API_KEY` | unset | Optional dedicated LangSmith key for Gateway calls. Prefer this in LangGraph Cloud if the platform-provided `LANGSMITH_API_KEY` lacks `gateway:invoke`. Falls back to `LANGSMITH_API_KEY_PROD`, then `LANGSMITH_API_KEY`. | +| `LANGSMITH_GATEWAY_BASE_URL` | `https://gateway.smith.langchain.com` | Override for a regional or self-hosted gateway host. | +| `LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES` | `true` | Use the OpenAI Responses API through the gateway. Set to `false` only to force Chat Completions for OpenAI models. | + +The admin panel (**Admin → LLM Gateway**) exposes a per-workspace toggle stored in team settings; when set it overrides the `LANGSMITH_GATEWAY_ENABLED` env default (a `None`/unset team value inherits the env default). + +Routing is applied centrally in `make_model` (`agent/utils/model.py`), which resolves the effective on/off and delegates URL/key wiring to `agent/utils/gateway.py`. **OpenAI, Anthropic, Fireworks, and Google Gemini** are routed (their LangChain integrations accept `base_url` + `api_key`); Google Vertex (service-account auth) and any other provider call the provider directly with a logged warning. + +**Caveat — OpenAI endpoint:** open-swe uses the OpenAI Responses API by default because OpenAI reasoning models with function tools reject `reasoning_effort` on Chat Completions. Direct OpenAI calls use a `wss://` base URL; gateway-routed OpenAI uses the HTTPS gateway base URL with Responses enabled. Set `LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES=false` only if you need to force Chat Completions. Anthropic and Fireworks are unaffected. + --- ## 3. Tools diff --git a/agent/analyzer.py b/agent/analyzer.py index 0f8cc1da..13680de8 100644 --- a/agent/analyzer.py +++ b/agent/analyzer.py @@ -29,6 +29,7 @@ from deepagents.backends.protocol import SandboxBackendProtocol from deepagents.backends.state import StateBackend from langchain.agents.middleware import ModelCallLimitMiddleware +from .dashboard.team_settings import get_effective_gateway_enabled from .integrations.langsmith import _configure_github_proxy from .middleware import SanitizeToolInputsMiddleware, ToolErrorMiddleware from .review_style_guidance import REVIEWER_STYLE_THEMES @@ -116,6 +117,7 @@ async def get_analyzer(config: RunnableConfig) -> Pregel: backend = CompositeBackend(default=sandbox_backend, routes={SKILLS_ROUTE: StateBackend()}) model_id = DEFAULT_LLM_MODEL_ID + use_gateway = await get_effective_gateway_enabled() model_kwargs = provider_model_kwargs( model_id, None, @@ -135,7 +137,7 @@ async def get_analyzer(config: RunnableConfig) -> Pregel: system_prompt = f"{system_prompt}\n\n{user_context}" return create_deep_agent( - model=make_model(model_id, **model_kwargs), + model=make_model(model_id, use_gateway=use_gateway, **model_kwargs), system_prompt=system_prompt, tools=[save_review_style_prompt, read_finding_outcomes], backend=backend, diff --git a/agent/chat.py b/agent/chat.py index efdba398..132f83d8 100644 --- a/agent/chat.py +++ b/agent/chat.py @@ -29,9 +29,11 @@ from deepagents import create_deep_agent from langchain.agents.middleware import ModelCallLimitMiddleware from .dashboard.options import SUPPORTED_MODEL_IDS, model_supports_effort -from .dashboard.team_settings import get_team_default_model +from .dashboard.team_settings import get_effective_gateway_enabled, get_team_default_model from .middleware import ( ExcludeToolsMiddleware, + SanitizeFireworksMessagesMiddleware, + SanitizeOpenAIResponsesMiddleware, SanitizeThinkingBlocksMiddleware, SanitizeToolInputsMiddleware, ToolErrorMiddleware, @@ -123,6 +125,7 @@ async def get_chat_agent(config: RunnableConfig) -> Pregel: configurable["chat_github_token"] = token model_id, effort = await _resolve_chat_model(configurable) + use_gateway = await get_effective_gateway_enabled() model_kwargs = provider_model_kwargs( model_id, effort, @@ -137,7 +140,7 @@ async def get_chat_agent(config: RunnableConfig) -> Pregel: ) return create_deep_agent( - model=make_model(model_id, **model_kwargs), + model=make_model(model_id, use_gateway=use_gateway, **model_kwargs), system_prompt=system_prompt, tools=[ read_repo_file, @@ -151,6 +154,8 @@ async def get_chat_agent(config: RunnableConfig) -> Pregel: ModelCallLimitMiddleware(run_limit=CHAT_MODEL_CALL_LIMIT, exit_behavior="end"), ToolErrorMiddleware(), ExcludeToolsMiddleware(excluded=_EXCLUDED_TOOLS), + SanitizeOpenAIResponsesMiddleware(), + SanitizeFireworksMessagesMiddleware(), SanitizeThinkingBlocksMiddleware(), ], ).with_config(config) diff --git a/agent/dashboard/team_settings.py b/agent/dashboard/team_settings.py index e81ff4b8..98299ed5 100644 --- a/agent/dashboard/team_settings.py +++ b/agent/dashboard/team_settings.py @@ -15,6 +15,7 @@ from typing import Any, Literal from langgraph_sdk import get_client from pydantic import BaseModel, field_validator, model_validator +from ..utils.gateway import resolve_gateway_enabled from .options import ( SUPPORTED_MODEL_IDS, default_model_pair, @@ -52,6 +53,9 @@ class TeamSettingsUpdate(BaseModel): review_draft_prs: bool = False pr_summaries: bool = True review_trace_links: bool = True + # Tri-state LLM Gateway toggle: True/False is authoritative, None inherits the + # LANGSMITH_GATEWAY_ENABLED deployment default. + gateway_enabled: bool | None = None review_tracing_project: str | None = None org_guidelines: str | None = None default_agent_model: str | None = None @@ -166,6 +170,7 @@ def _default_settings() -> dict[str, Any]: "review_draft_prs": False, "pr_summaries": True, "review_trace_links": True, + "gateway_enabled": None, "review_tracing_project": None, "org_guidelines": DEFAULT_ORG_REVIEW_GUIDELINES, "default_agent_model": fallback_model, @@ -220,6 +225,7 @@ async def upsert_team_settings(update: TeamSettingsUpdate) -> dict[str, Any]: "review_draft_prs": update.review_draft_prs, "pr_summaries": update.pr_summaries, "review_trace_links": update.review_trace_links, + "gateway_enabled": update.gateway_enabled, "review_tracing_project": update.review_tracing_project, "org_guidelines": update.org_guidelines, "default_agent_model": update.default_agent_model, @@ -340,6 +346,18 @@ async def get_team_review_trace_links_enabled() -> bool: return bool(settings.get("review_trace_links", True)) +async def get_team_gateway_enabled() -> bool | None: + """Return the stored LLM Gateway toggle (``None`` means inherit the env default).""" + settings = await get_team_settings() + value = settings.get("gateway_enabled") + return value if isinstance(value, bool) else None + + +async def get_effective_gateway_enabled() -> bool: + """Resolve whether LLM Gateway routing is on: team setting, else env default.""" + return resolve_gateway_enabled(await get_team_gateway_enabled()) + + async def get_team_review_tracing_project() -> str | None: """Return the LangSmith tracing project used for PR trace resolution.""" settings = await get_team_settings() diff --git a/agent/middleware/__init__.py b/agent/middleware/__init__.py index c40ce67b..d6d94403 100644 --- a/agent/middleware/__init__.py +++ b/agent/middleware/__init__.py @@ -8,6 +8,8 @@ from .refresh_github_proxy import refresh_github_proxy_before_model from .refresh_slack_status import SlackAssistantStatusMiddleware from .repair_orphaned_tool_calls import RepairOrphanedToolCallsMiddleware from .sandbox_circuit_breaker import SandboxCircuitBreakerMiddleware +from .sanitize_fireworks_messages import SanitizeFireworksMessagesMiddleware +from .sanitize_openai_responses import SanitizeOpenAIResponsesMiddleware from .sanitize_thinking_blocks import SanitizeThinkingBlocksMiddleware from .sanitize_tool_inputs import SanitizeToolInputsMiddleware from .settle_review_check import settle_review_check_on_exit @@ -21,6 +23,8 @@ __all__ = [ "ModelFallbackMiddleware", "PlanModeMiddleware", "RepairOrphanedToolCallsMiddleware", + "SanitizeFireworksMessagesMiddleware", + "SanitizeOpenAIResponsesMiddleware", "SanitizeThinkingBlocksMiddleware", "SanitizeToolInputsMiddleware", "SubdirAgentsReadMiddleware", diff --git a/agent/middleware/sanitize_fireworks_messages.py b/agent/middleware/sanitize_fireworks_messages.py new file mode 100644 index 00000000..79c4d0d3 --- /dev/null +++ b/agent/middleware/sanitize_fireworks_messages.py @@ -0,0 +1,79 @@ +"""Strip legacy ``function_call`` fields before Fireworks provider calls. + +The LangSmith LLM Gateway enforces strict request-schema validation and rejects +messages carrying the legacy OpenAI ``function_call`` field +(``Extra inputs are not permitted, field: 'messages[N].function_call'``). +``langchain_fireworks`` forwards ``function_call`` from +``AIMessage.additional_kwargs`` whenever it is present — even when the message +also carries modern ``tool_calls`` — so a message that was originally produced +by (or deserialized from) a provider that populated the legacy field will break +every subsequent Fireworks turn once routed through the gateway. + +This middleware drops the redundant legacy field from assistant messages before +they reach the Fireworks serializer, mirroring the thinking-block sanitizer. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from langchain.agents.middleware import AgentMiddleware +from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse +from langchain_core.messages import AIMessage + +try: + from langchain_fireworks.chat_models import ChatFireworks +except ImportError: # pragma: no cover + ChatFireworks = None # type: ignore[assignment, misc] + + +def _is_chat_fireworks(model: object) -> bool: + if ChatFireworks is None: + return False + seen: set[int] = set() + current = model + for _ in range(10): + if isinstance(current, ChatFireworks): + return True + current_id = id(current) + if current_id in seen: + return False + seen.add(current_id) + bound = getattr(current, "bound", None) + if bound is None or bound is current: + return False + current = bound + return False + + +def _sanitize_messages(messages: list[Any]) -> None: + for message in messages: + if not isinstance(message, AIMessage): + continue + additional_kwargs = message.additional_kwargs + if not isinstance(additional_kwargs, dict) or "function_call" not in additional_kwargs: + continue + additional_kwargs.pop("function_call", None) + + +class SanitizeFireworksMessagesMiddleware(AgentMiddleware): + """Drop legacy ``function_call`` fields before Fireworks provider calls.""" + + def wrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], ModelResponse], + ) -> ModelCallResult: + if _is_chat_fireworks(request.model): + _sanitize_messages(request.messages) + return handler(request) + + async def awrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], Awaitable[ModelResponse]], + ) -> Any: + if _is_chat_fireworks(request.model): + _sanitize_messages(request.messages) + return await handler(request) diff --git a/agent/middleware/sanitize_openai_responses.py b/agent/middleware/sanitize_openai_responses.py new file mode 100644 index 00000000..a68104b0 --- /dev/null +++ b/agent/middleware/sanitize_openai_responses.py @@ -0,0 +1,85 @@ +"""Middleware that removes stale OpenAI Responses reasoning references.""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from langchain.agents.middleware import AgentMiddleware +from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse +from langchain_core.messages import AIMessage + +try: + from langchain_openai import ChatOpenAI +except ImportError: # pragma: no cover + ChatOpenAI = None # type: ignore[assignment, misc] + +logger = logging.getLogger(__name__) + + +def _is_chat_openai(model: object) -> bool: + if ChatOpenAI is None: + return False + seen: set[int] = set() + current = model + for _ in range(10): + if isinstance(current, ChatOpenAI): + return True + current_id = id(current) + if current_id in seen: + return False + seen.add(current_id) + bound = getattr(current, "bound", None) + if bound is None or bound is current: + return False + current = bound + return False + + +def _is_stale_reasoning_reference(block: dict[str, Any]) -> bool: + if block.get("type") != "reasoning": + return False + if block.get("encrypted_content"): + return False + block_id = block.get("id") + return isinstance(block_id, str) and block_id.startswith("rs_") + + +def _sanitize_messages(messages: list[Any]) -> None: + removed = 0 + for message in messages: + if not isinstance(message, AIMessage) or not isinstance(message.content, list): + continue + content = [] + for block in message.content: + if isinstance(block, dict) and _is_stale_reasoning_reference(block): + removed += 1 + continue + content.append(block) + if len(content) != len(message.content): + message.content = content + if removed: + logger.warning("Removed %d stale OpenAI Responses reasoning reference(s)", removed) + + +class SanitizeOpenAIResponsesMiddleware(AgentMiddleware): + """Drop non-replayable OpenAI Responses reasoning item references.""" + + def wrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], ModelResponse], + ) -> ModelCallResult: + if _is_chat_openai(request.model): + _sanitize_messages(request.messages) + return handler(request) + + async def awrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], Awaitable[ModelResponse]], + ) -> Any: + if _is_chat_openai(request.model): + _sanitize_messages(request.messages) + return await handler(request) diff --git a/agent/reviewer.py b/agent/reviewer.py index 7704ef98..2bafe7ba 100644 --- a/agent/reviewer.py +++ b/agent/reviewer.py @@ -34,12 +34,15 @@ from langchain.agents.middleware import ModelCallLimitMiddleware from langchain_core.language_models.chat_models import BaseChatModel from .dashboard.team_settings import ( + get_effective_gateway_enabled, get_org_review_guidelines, get_team_default_grouping_model, get_team_default_model_pair, ) from .middleware import ( RepairOrphanedToolCallsMiddleware, + SanitizeFireworksMessagesMiddleware, + SanitizeOpenAIResponsesMiddleware, SanitizeThinkingBlocksMiddleware, SanitizeToolInputsMiddleware, SlackAssistantStatusMiddleware, @@ -801,7 +804,9 @@ def _on_background_task_done(task: asyncio.Task[None]) -> None: logger.warning("Background reviewer task failed: %s", exc) -async def _resolve_grouping_model(configurable: dict[str, object]) -> BaseChatModel: +async def _resolve_grouping_model( + configurable: dict[str, object], *, use_gateway: bool +) -> BaseChatModel: """Resolve the model for the diff-grouping pass. Per-run override (``grouping_model_id``/``grouping_reasoning_effort``) wins; @@ -821,7 +826,7 @@ async def _resolve_grouping_model(configurable: dict[str, object]) -> BaseChatMo max_tokens=DEFAULT_LLM_MAX_TOKENS, openai_reasoning_default=DEFAULT_LLM_REASONING, ) - return make_model(model_id, **model_kwargs) + return make_model(model_id, use_gateway=use_gateway, **model_kwargs) async def get_reviewer_agent(config: RunnableConfig) -> Pregel: @@ -1156,8 +1161,11 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: if review_context: system_prompt = f"{system_prompt}\n\n{review_context}" - reviewer_model = make_model(model_id, **model_kwargs) - reviewer_subagent_model = make_model(subagent_model_id, **subagent_model_kwargs) + use_gateway = await get_effective_gateway_enabled() + reviewer_model = make_model(model_id, use_gateway=use_gateway, **model_kwargs) + reviewer_subagent_model = make_model( + subagent_model_id, use_gateway=use_gateway, **subagent_model_kwargs + ) # Kick off the AI-sorted diff grouping pass at run start, concurrently with # the review, so it adds ~0 latency. First-review and re-review only — a @@ -1165,7 +1173,9 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: # its own errors and the UI falls back to the folder view when groups are # absent. if reviewer_event != "finding_reply" and pr_diff_text and thread_id: - grouping_model = await _resolve_grouping_model(config["configurable"]) + grouping_model = await _resolve_grouping_model( + config["configurable"], use_gateway=use_gateway + ) grouping_task = asyncio.create_task( maybe_generate_and_store_diff_groups( thread_id=thread_id, @@ -1201,6 +1211,8 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: refresh_github_proxy_before_model, check_message_queue_before_model, SlackAssistantStatusMiddleware(), + SanitizeOpenAIResponsesMiddleware(), + SanitizeFireworksMessagesMiddleware(), SanitizeThinkingBlocksMiddleware(), RepairOrphanedToolCallsMiddleware(), settle_review_check_on_exit, diff --git a/agent/server.py b/agent/server.py index 48afa4d0..5e8af637 100644 --- a/agent/server.py +++ b/agent/server.py @@ -44,6 +44,7 @@ from .dashboard.agent_usage import record_agent_thread_usage from .dashboard.options import DEFAULT_MODEL_ID, SUPPORTED_MODEL_IDS, model_supports_effort from .dashboard.repo_snapshots import resolve_repo_snapshot_id from .dashboard.team_settings import ( + get_effective_gateway_enabled, get_team_default_model_pair, get_team_default_repo, ) @@ -58,6 +59,8 @@ from .middleware import ( ModelFallbackMiddleware, PlanModeMiddleware, SandboxCircuitBreakerMiddleware, + SanitizeFireworksMessagesMiddleware, + SanitizeOpenAIResponsesMiddleware, SanitizeThinkingBlocksMiddleware, SanitizeToolInputsMiddleware, SlackAssistantStatusMiddleware, @@ -758,19 +761,31 @@ async def get_agent(config: RunnableConfig) -> Pregel: ensure_sandbox_for_thread(thread_id, repo=prompt_default_repo) ) team_defaults_task = asyncio.create_task(get_team_default_model_pair("agent")) + gateway_task = asyncio.create_task(get_effective_gateway_enabled()) profile_task = asyncio.create_task(load_profile(profile_login)) if profile_login else None try: - triggering_user_identity, sandbox_backend, team_defaults = await asyncio.gather( + ( + triggering_user_identity, + sandbox_backend, + team_defaults, + use_gateway, + ) = await asyncio.gather( triggering_user_identity_task, sandbox_task, team_defaults_task, + gateway_task, ) except SandboxRepoMismatchError as exc: # Repo-binding refusal at the run boundary: log for alarming and surface the # already-sanitized terminal error (no sandbox/token internals) to the caller, # rather than letting an opaque deep-stack exception crash-loop the worker. logger.error("Refusing agent run for thread %s: %s", thread_id, exc) - for pending in (triggering_user_identity_task, team_defaults_task, profile_task): + for pending in ( + triggering_user_identity_task, + team_defaults_task, + gateway_task, + profile_task, + ): if pending is not None and not pending.done(): pending.cancel() raise RuntimeError(str(exc)) from exc @@ -860,7 +875,9 @@ async def get_agent(config: RunnableConfig) -> Pregel: fallback_model_id, None, max_tokens=DEFAULT_LLM_MAX_TOKENS ) fallback_middleware.append( - ModelFallbackMiddleware(make_model(fallback_model_id, **fallback_kwargs)) + ModelFallbackMiddleware( + make_model(fallback_model_id, use_gateway=use_gateway, **fallback_kwargs) + ) ) logger.info("Configured model fallback %s -> %s", model_id, fallback_model_id) @@ -926,8 +943,8 @@ async def get_agent(config: RunnableConfig) -> Pregel: notion_tools = [] logger.info("Returning agent with sandbox for thread %s", thread_id) - main_model = make_model(model_id, **model_kwargs) - subagent_model = make_model(subagent_model_id, **subagent_model_kwargs) + main_model = make_model(model_id, use_gateway=use_gateway, **model_kwargs) + subagent_model = make_model(subagent_model_id, use_gateway=use_gateway, **subagent_model_kwargs) return create_deep_agent( model=main_model, system_prompt=construct_system_prompt( @@ -987,6 +1004,8 @@ async def get_agent(config: RunnableConfig) -> Pregel: SandboxCircuitBreakerMiddleware(), *fallback_middleware, *plan_mode_middleware, + SanitizeOpenAIResponsesMiddleware(), + SanitizeFireworksMessagesMiddleware(), SanitizeThinkingBlocksMiddleware(), ], ).with_config(config) diff --git a/agent/utils/gateway.py b/agent/utils/gateway.py new file mode 100644 index 00000000..065e6edb --- /dev/null +++ b/agent/utils/gateway.py @@ -0,0 +1,126 @@ +"""LangSmith LLM Gateway routing for model construction. + +The LLM Gateway (https://docs.langchain.com/langsmith/llm-gateway) proxies +provider calls through LangSmith: the client authenticates with a LangSmith API +key and the gateway resolves the real provider key from workspace Provider +Secrets, enforcing spend/PII/secrets policies and tracing every call. Routing is +opt-in via ``LANGSMITH_GATEWAY_ENABLED`` (deployment default) or the +``gateway_enabled`` team setting, and is applied centrally in +:func:`agent.utils.model.make_model`. +""" + +from __future__ import annotations + +import logging +import os + +logger = logging.getLogger(__name__) + +DEFAULT_GATEWAY_BASE_URL = "https://gateway.smith.langchain.com" + +# Provider prefix -> base-URL suffix appended to the gateway host. Each suffix +# matches the SDK's own path handling: the OpenAI SDK appends +# ``/chat/completions`` to a ``/v1`` base, Fireworks appends +# ``/v1/chat/completions`` to a bare provider host, Anthropic appends +# ``/v1/messages`` to a bare host, and google-genai appends +# ``//models/...`` to a bare host. Vertex (``google_vertexai``, which +# uses service-account auth rather than a bearer key) and any other provider are +# not routed and call the provider directly. +_GATEWAY_PROVIDER_PATHS: dict[str, str] = { + "openai": "/openai/v1", + "anthropic": "/anthropic", + "fireworks": "/fireworks", + "google_genai": "/gemini", +} + + +def _env_bool(value: str | None) -> bool: + return (value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _langsmith_api_key() -> str | None: + """LangSmith API key used to authenticate gateway calls. + + Prefer a gateway-specific key, then the prod LangSmith key. LangGraph Cloud + may inject ``LANGSMITH_API_KEY`` for tracing/platform APIs, and that key can + lack the ``gateway:invoke`` permission required by the LLM Gateway. + """ + return ( + os.environ.get("LANGSMITH_GATEWAY_API_KEY") + or os.environ.get("LANGSMITH_API_KEY_PROD") + or os.environ.get("LANGSMITH_API_KEY") + ) + + +def gateway_base_url() -> str: + """Gateway host, overridable via ``LANGSMITH_GATEWAY_BASE_URL`` (regional/self-hosted).""" + return (os.environ.get("LANGSMITH_GATEWAY_BASE_URL") or DEFAULT_GATEWAY_BASE_URL).rstrip("/") + + +def gateway_env_default() -> bool: + """Deployment-level default for gateway routing (``LANGSMITH_GATEWAY_ENABLED``).""" + return _env_bool(os.environ.get("LANGSMITH_GATEWAY_ENABLED")) + + +def gateway_openai_use_responses() -> bool: + """Whether gateway-routed OpenAI keeps the Responses API. + + Defaults to ``True`` because OpenAI reasoning models with tool calls reject + ``reasoning_effort`` on Chat Completions. Set + ``LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES=false`` only for deployments that + need to force Chat Completions through the gateway. + """ + raw = os.environ.get("LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES") + if raw is None: + return True + return _env_bool(raw) + + +def resolve_gateway_enabled(team_value: bool | None) -> bool: + """Combine the team-settings toggle with the env default. + + A team value of ``True``/``False`` is authoritative; ``None`` inherits the + ``LANGSMITH_GATEWAY_ENABLED`` deployment default. + """ + if team_value is None: + return gateway_env_default() + return team_value + + +def _provider_of(model_id: str) -> str: + return model_id.split(":", 1)[0] + + +def gateway_overrides(model_id: str) -> dict[str, object] | None: + """``init_chat_model`` kwargs that route ``model_id`` through the gateway. + + Returns ``None`` (so the caller keeps talking to the provider directly) when + the provider isn't routable through the gateway or no LangSmith API key is + available — both cases are logged rather than raised, so a run never fails + just because gateway routing couldn't be applied. + """ + provider = _provider_of(model_id) + path = _GATEWAY_PROVIDER_PATHS.get(provider) + if path is None: + logger.debug( + "LangSmith gateway enabled but provider %r is not routed; calling it directly", + provider, + ) + return None + api_key = _langsmith_api_key() + if not api_key: + logger.debug( + "LangSmith gateway enabled but no LANGSMITH_GATEWAY_API_KEY or " + "LANGSMITH_API_KEY(_PROD) is set; " + "calling the provider directly" + ) + return None + overrides: dict[str, object] = { + "base_url": f"{gateway_base_url()}{path}", + "api_key": api_key, + } + if provider == "openai": + # Use HTTPS Responses through the gateway by default; tool-calling OpenAI + # reasoning models reject reasoning_effort on Chat Completions. + overrides["use_responses_api"] = gateway_openai_use_responses() + return overrides diff --git a/agent/utils/model.py b/agent/utils/model.py index cadf1dbd..1313663c 100644 --- a/agent/utils/model.py +++ b/agent/utils/model.py @@ -4,6 +4,7 @@ from typing import Literal, TypedDict, Unpack from langchain.chat_models import init_chat_model from ..dashboard.options import DEFAULT_MODEL_ID +from .gateway import gateway_env_default, gateway_overrides OPENAI_RESPONSES_WS_BASE_URL = "wss://api.openai.com/v1" @@ -39,11 +40,14 @@ class AnthropicThinking(TypedDict, total=False): class ModelKwargs(TypedDict, total=False): max_tokens: int | None reasoning: OpenAIReasoning | None + reasoning_effort: OpenAIReasoningEffort | None thinking: AnthropicThinking | None effort: AnthropicEffort | None thinking_level: GoogleThinkingLevel | None temperature: float | None max_retries: int | None + store: bool | None + include: list[str] | None model_kwargs: dict[str, object] | None additional_model_request_fields: dict[str, object] | None region_name: str | None @@ -52,21 +56,62 @@ class ModelKwargs(TypedDict, total=False): _ANTHROPIC_EFFORTS: set[AnthropicEffort] = {"low", "medium", "high", "xhigh", "max"} -def make_model(model_id: str, **kwargs: Unpack[ModelKwargs]): +def _coerce_openai_chat_completions_kwargs(model_kwargs: dict[str, object]) -> None: + if model_kwargs.get("use_responses_api") is not False: + return + reasoning = model_kwargs.pop("reasoning", None) + if isinstance(reasoning, dict): + effort = reasoning.get("effort") + if isinstance(effort, str): + model_kwargs.setdefault("reasoning_effort", effort) + + +def _configure_openai_responses_kwargs(model_kwargs: dict[str, object]) -> None: + if model_kwargs.get("use_responses_api") is False: + return + model_kwargs.setdefault("store", False) + include = model_kwargs.get("include") + if include is None: + model_kwargs["include"] = ["reasoning.encrypted_content"] + elif isinstance(include, list) and "reasoning.encrypted_content" not in include: + include.append("reasoning.encrypted_content") + + +def make_model(model_id: str, *, use_gateway: bool | None = None, **kwargs: Unpack[ModelKwargs]): + """Build a chat model, optionally routed through the LangSmith LLM Gateway. + + ``use_gateway`` resolves the deployment default (``LANGSMITH_GATEWAY_ENABLED``) + when ``None``; async callers pass the team-settings-resolved value. When on, + gateway ``base_url``/``api_key``/``use_responses_api`` override the direct + provider defaults below (see :mod:`agent.utils.gateway`). + """ model_kwargs: dict[str, object] = kwargs.copy() model_kwargs.setdefault("max_retries", DEFAULT_MAX_RETRIES) if model_id.startswith("openai:"): + # Direct-provider default: Responses API over the OpenAI websocket base. + # Gateway routing overrides this below (an HTTP(S) proxy can't carry wss). model_kwargs["base_url"] = OPENAI_RESPONSES_WS_BASE_URL model_kwargs["use_responses_api"] = True elif model_id.startswith("bedrock_converse:"): - # Resolve region with the same precedence validate_local_dev_llm_config accepts - # (AWS_REGION or AWS_DEFAULT_REGION), so the validated value is the one actually used. + # Resolve region with the same precedence validate_local_dev_llm_config + # accepts (AWS_REGION or AWS_DEFAULT_REGION), so the validated value + # is the one actually used. model_kwargs.setdefault( "region_name", os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1", ) + enabled = gateway_env_default() if use_gateway is None else use_gateway + if enabled: + overrides = gateway_overrides(model_id) + if overrides is not None: + model_kwargs.update(overrides) + + if model_id.startswith("openai:"): + _configure_openai_responses_kwargs(model_kwargs) + _coerce_openai_chat_completions_kwargs(model_kwargs) + return init_chat_model(model=model_id, **model_kwargs) diff --git a/docs/upstream-sync/triage.jsonl b/docs/upstream-sync/triage.jsonl index 226e4b58..1f390645 100644 --- a/docs/upstream-sync/triage.jsonl +++ b/docs/upstream-sync/triage.jsonl @@ -32,10 +32,10 @@ {"sha": "4cd5fa5c", "pr": 1629, "subject": "avoid recapping Slack replies", "disposition": "deferred", "reason": "", "branch": "slack-tooling", "local_sha": null, "updated": "2026-07-02T00:00:00Z"} {"sha": "92dbf6f9", "pr": 1630, "subject": "update Slack trace reply on web handoff", "disposition": "landed", "reason": "ported in #128 (trace_message_ts on first-mention run mapping)", "branch": "slack-tooling", "local_sha": null, "updated": "2026-07-08T22:58:21Z"} {"sha": "bb36448b", "pr": 1627, "subject": "surface Slack thread errors", "disposition": "landed", "reason": "", "branch": "slack-tooling", "local_sha": "e8b6fb70508861e5742d31aaa722e79f4a9a320a", "updated": "2026-07-03T19:50:30Z"} -{"sha": "73b7d1c0", "pr": 1678, "subject": "fix OpenAI Responses reasoning replay", "disposition": "deferred", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-02T00:00:00Z"} -{"sha": "5f7c2f46", "pr": 1674, "subject": "fix Fireworks Gateway base URL", "disposition": "deferred", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-02T00:00:00Z"} -{"sha": "702ef908", "pr": 1673, "subject": "dedicated LangSmith gateway API key", "disposition": "deferred", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-02T00:00:00Z"} -{"sha": "e9dc6e01", "pr": 1671, "subject": "opt-in LangSmith LLM Gateway routing", "disposition": "deferred", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-02T00:00:00Z"} +{"sha": "73b7d1c0", "pr": 1678, "subject": "fix OpenAI Responses reasoning replay", "disposition": "landed", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-09T16:22:05Z"} +{"sha": "5f7c2f46", "pr": 1674, "subject": "fix Fireworks Gateway base URL", "disposition": "landed", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-09T16:22:05Z"} +{"sha": "702ef908", "pr": 1673, "subject": "dedicated LangSmith gateway API key", "disposition": "landed", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-09T16:22:05Z"} +{"sha": "e9dc6e01", "pr": 1671, "subject": "opt-in LangSmith LLM Gateway routing", "disposition": "landed", "reason": "", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-09T16:22:04Z"} {"sha": "289f5e3a", "pr": 1651, "subject": "add Sonnet 5 to model picker", "disposition": "landed", "reason": "already in dev; added Bedrock family fallback fix (c16fb915)", "branch": "gateway-routing", "local_sha": null, "updated": "2026-07-03T00:19:50Z"} {"sha": "5da3d0c6", "pr": 1624, "subject": "post reviewer resolution notes verbatim", "disposition": "landed", "reason": "cherry-picked (-x) in #127", "branch": "reviewer-misc", "local_sha": null, "updated": "2026-07-08T22:58:20Z"} {"sha": "69148f54", "pr": 1612, "subject": "add PR trace resolution", "disposition": "landed", "reason": "cherry-picked (-x) in #127", "branch": "reviewer-misc", "local_sha": null, "updated": "2026-07-08T22:58:20Z"} diff --git a/docs/upstream-sync/triage.md b/docs/upstream-sync/triage.md index 72c47736..ef9267de 100644 --- a/docs/upstream-sync/triage.md +++ b/docs/upstream-sync/triage.md @@ -29,6 +29,10 @@ Rows key on the **upstream SHA** (stable across local cherry-picks). Deferred ro | `27d90ef1` | #1633 | include Slack channel context in prompts | Landed | | slack-tooling | | `92dbf6f9` | #1630 | update Slack trace reply on web handoff | Landed | ported in #128 (trace_message_ts on first-mention run mapping) | slack-tooling | | `bb36448b` | #1627 | surface Slack thread errors | Landed | | slack-tooling | +| `73b7d1c0` | #1678 | fix OpenAI Responses reasoning replay | Landed | | gateway-routing | +| `5f7c2f46` | #1674 | fix Fireworks Gateway base URL | Landed | | gateway-routing | +| `702ef908` | #1673 | dedicated LangSmith gateway API key | Landed | | gateway-routing | +| `e9dc6e01` | #1671 | opt-in LangSmith LLM Gateway routing | Landed | | gateway-routing | | `289f5e3a` | #1651 | add Sonnet 5 to model picker | Landed | already in dev; added Bedrock family fallback fix (c16fb915) | gateway-routing | | `5da3d0c6` | #1624 | post reviewer resolution notes verbatim | Landed | cherry-picked (-x) in #127 | reviewer-misc | | `69148f54` | #1612 | add PR trace resolution | Landed | cherry-picked (-x) in #127 | reviewer-misc | @@ -63,10 +67,6 @@ Rows key on the **upstream SHA** (stable across local cherry-picks). Deferred ro | `546042a4` | #1652 | add workflow approval UI | Deferred | | plan-approval | | `c03a6be7` | #1634 | keep plan guidance high-level | Deferred | | plan-approval | | `4cd5fa5c` | #1629 | avoid recapping Slack replies | Deferred | | slack-tooling | -| `73b7d1c0` | #1678 | fix OpenAI Responses reasoning replay | Deferred | | gateway-routing | -| `5f7c2f46` | #1674 | fix Fireworks Gateway base URL | Deferred | | gateway-routing | -| `702ef908` | #1673 | dedicated LangSmith gateway API key | Deferred | | gateway-routing | -| `e9dc6e01` | #1671 | opt-in LangSmith LLM Gateway routing | Deferred | | gateway-routing | | `baf0c248` | #1617 | filter & grouping menu in threads sidebar | Deferred | ~998 LOC | own branch | | `f29868ff` | #1615 | recover thread work as patch | Deferred | ~495 LOC | own branch | | `8e0788dc` | #1631 | show queued dashboard follow-ups | Deferred | | own branch | diff --git a/tests/test_gateway.py b/tests/test_gateway.py new file mode 100644 index 00000000..1c9cb980 --- /dev/null +++ b/tests/test_gateway.py @@ -0,0 +1,437 @@ +"""Unit tests for LangSmith LLM Gateway routing (agent/utils/gateway.py + make_model).""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +import httpx +import pytest +from fireworks import AsyncFireworks +from langchain_core.messages import AIMessage, HumanMessage +from langchain_openai import ChatOpenAI + +from agent.utils import gateway, model + +_GATEWAY_ENV_VARS = ( + "LANGSMITH_API_KEY", + "LANGSMITH_API_KEY_PROD", + "LANGSMITH_GATEWAY_API_KEY", + "LANGSMITH_GATEWAY_ENABLED", + "LANGSMITH_GATEWAY_BASE_URL", + "LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES", +) + + +@pytest.fixture(autouse=True) +def _clean_gateway_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Start each test from a known env: no key, gateway off, default base URL.""" + for name in _GATEWAY_ENV_VARS: + monkeypatch.delenv(name, raising=False) + + +# --- gateway_overrides -------------------------------------------------------- + + +def test_openai_overrides_use_responses_by_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + overrides = gateway.gateway_overrides("openai:gpt-5.5") + assert overrides == { + "base_url": "https://gateway.smith.langchain.com/openai/v1", + "api_key": "ls-key", + "use_responses_api": True, + } + + +def test_openai_overrides_chat_completions_optout(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + monkeypatch.setenv("LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES", "false") + overrides = gateway.gateway_overrides("openai:gpt-5.5") + assert overrides is not None + assert overrides["use_responses_api"] is False + + +async def test_openai_sdk_uses_gateway_responses_path() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={ + "id": "resp_test", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-5.5", + "output": [ + { + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + chat_model = ChatOpenAI( + model="gpt-5.5", + api_key="dummy", + base_url="https://gateway.smith.langchain.com/openai/v1", + use_responses_api=True, + http_async_client=http_client, + max_retries=0, + ) + await chat_model.ainvoke([HumanMessage(content="hi")]) + finally: + await http_client.aclose() + + assert len(requests) == 1 + assert requests[0].url.path == "/openai/v1/responses" + + +def test_anthropic_overrides_have_no_responses_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + overrides = gateway.gateway_overrides("anthropic:claude-opus-4-8") + assert overrides == { + "base_url": "https://gateway.smith.langchain.com/anthropic", + "api_key": "ls-key", + } + + +def test_fireworks_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + overrides = gateway.gateway_overrides("fireworks:accounts/fireworks/models/glm-5p2") + assert overrides is not None + assert overrides["base_url"] == "https://gateway.smith.langchain.com/fireworks" + + +async def test_fireworks_sdk_uses_allowlisted_gateway_path() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "accounts/fireworks/models/glm-5p2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + client = AsyncFireworks( + api_key="dummy", + base_url="https://gateway.smith.langchain.com/fireworks", + http_client=http_client, + max_retries=0, + ) + await client.chat.completions.create( + model="accounts/fireworks/models/glm-5p2", + messages=[{"role": "user", "content": "hi"}], + ) + finally: + await http_client.aclose() + + assert len(requests) == 1 + assert requests[0].url.path == "/fireworks/v1/chat/completions" + + +async def test_fireworks_gateway_strips_legacy_function_call() -> None: + """The serializer must not emit ``function_call`` after sanitization. + + Reproduces the production 400 — ``Extra inputs are not permitted, field: + 'messages[N].function_call'`` — by routing an ``AIMessage`` that carries the + legacy ``function_call`` (alongside modern ``tool_calls``) through the + Fireworks serializer toward the gateway. Without the sanitizer middleware + the request body contains ``function_call``; with it, only ``tool_calls`` + survives. + """ + import json + + from fireworks import AsyncFireworks + from langchain_fireworks.chat_models import ChatFireworks + + from agent.middleware.sanitize_fireworks_messages import _sanitize_messages + + captured_bodies: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured_bodies.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "accounts/fireworks/models/glm-5p2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + chat_model = ChatFireworks( + model="accounts/fireworks/models/glm-5p2", + api_key="dummy", + base_url="https://gateway.smith.langchain.com/fireworks", + max_retries=0, + ) + # Inject a mock-transport client so no real network call is made. + mock_sdk = AsyncFireworks( + api_key="dummy", + base_url="https://gateway.smith.langchain.com/fireworks", + http_client=http_client, + max_retries=0, + ) + chat_model._async_sdk_client = mock_sdk # type: ignore[attr-defined] + chat_model.async_client = mock_sdk.chat.completions # type: ignore[attr-defined] + + ai_message = AIMessage( + content="", + tool_calls=[{"name": "read_file", "args": {"file_path": "/x"}, "id": "tc1"}], + additional_kwargs={"function_call": {"name": "read_file", "arguments": "{}"}}, + ) + messages = [HumanMessage(content="hi"), ai_message] + + # Apply the sanitizer the same way the middleware stack does. + _sanitize_messages(messages) + + await chat_model.ainvoke(messages) + await mock_sdk.close() + finally: + await http_client.aclose() + + assert len(captured_bodies) == 1 + body = captured_bodies[0] + for msg in body["messages"]: + assert "function_call" not in msg, msg + # The assistant message still carries tool_calls. + assistant_msgs = [m for m in body["messages"] if m["role"] == "assistant"] + assert assistant_msgs and "tool_calls" in assistant_msgs[0] + + +def test_google_genai_routes_to_gemini(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + overrides = gateway.gateway_overrides("google_genai:gemini-3.5-flash") + assert overrides == { + "base_url": "https://gateway.smith.langchain.com/gemini", + "api_key": "ls-key", + } + + +def test_unsupported_provider_passes_through(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + # Vertex authenticates with a service account, not a bearer key, so it isn't routed. + assert gateway.gateway_overrides("google_vertexai:gemini-2.5-pro") is None + + +def test_missing_api_key_passes_through(monkeypatch: pytest.MonkeyPatch) -> None: + assert gateway.gateway_overrides("openai:gpt-5.5") is None + + +def test_prod_key_used_as_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY_PROD", "ls-prod-key") + overrides = gateway.gateway_overrides("anthropic:claude-opus-4-8") + assert overrides is not None + assert overrides["api_key"] == "ls-prod-key" + + +def test_prod_key_preferred_over_platform_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-platform-key") + monkeypatch.setenv("LANGSMITH_API_KEY_PROD", "ls-prod-key") + overrides = gateway.gateway_overrides("anthropic:claude-opus-4-8") + assert overrides is not None + assert overrides["api_key"] == "ls-prod-key" + + +def test_gateway_key_preferred_over_prod_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-platform-key") + monkeypatch.setenv("LANGSMITH_API_KEY_PROD", "ls-prod-key") + monkeypatch.setenv("LANGSMITH_GATEWAY_API_KEY", "ls-gateway-key") + overrides = gateway.gateway_overrides("anthropic:claude-opus-4-8") + assert overrides is not None + assert overrides["api_key"] == "ls-gateway-key" + + +def test_base_url_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + monkeypatch.setenv("LANGSMITH_GATEWAY_BASE_URL", "https://gw.internal.example.com/") + overrides = gateway.gateway_overrides("anthropic:claude-opus-4-8") + assert overrides is not None + # Trailing slash is stripped, then the provider path is appended. + assert overrides["base_url"] == "https://gw.internal.example.com/anthropic" + + +# --- resolve_gateway_enabled -------------------------------------------------- + + +@pytest.mark.parametrize( + ("team_value", "env_enabled", "expected"), + [ + (True, False, True), # team True wins over env off + (False, True, False), # team False wins over env on + (None, True, True), # unset inherits env on + (None, False, False), # unset inherits env off + ], +) +def test_resolve_gateway_enabled_precedence( + monkeypatch: pytest.MonkeyPatch, + team_value: bool | None, + env_enabled: bool, + expected: bool, +) -> None: + if env_enabled: + monkeypatch.setenv("LANGSMITH_GATEWAY_ENABLED", "true") + assert gateway.resolve_gateway_enabled(team_value) is expected + + +# --- make_model integration --------------------------------------------------- + + +def _capture_init_chat_model() -> tuple[dict[str, Any], Any]: + """Patch init_chat_model to record the kwargs make_model builds.""" + captured: dict[str, Any] = {} + + def _fake(model: str, **kwargs: Any) -> str: + captured["model"] = model + captured.update(kwargs) + return "MODEL" + + return captured, _fake + + +def test_make_model_direct_openai_uses_responses_websocket() -> None: + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model("openai:gpt-5.5", use_gateway=False) + assert captured["base_url"] == model.OPENAI_RESPONSES_WS_BASE_URL + assert captured["use_responses_api"] is True + assert captured["store"] is False + assert captured["include"] == ["reasoning.encrypted_content"] + + +def test_make_model_gateway_openai_replaces_websocket( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model("openai:gpt-5.5", use_gateway=True) + assert captured["base_url"] == "https://gateway.smith.langchain.com/openai/v1" + assert captured["use_responses_api"] is True + assert captured["store"] is False + assert captured["include"] == ["reasoning.encrypted_content"] + assert captured["api_key"] == "ls-key" + + +def test_make_model_gateway_openai_chat_completions_optout_converts_reasoning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + monkeypatch.setenv("LANGSMITH_GATEWAY_OPENAI_USE_RESPONSES", "false") + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model( + "openai:gpt-5.5", + use_gateway=True, + reasoning={"effort": "high", "summary": "auto"}, + ) + assert captured["use_responses_api"] is False + assert captured["reasoning_effort"] == "high" + assert "reasoning" not in captured + assert "include" not in captured + assert "store" not in captured + + +def test_make_model_gateway_openai_preserves_reasoning_none( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model( + "openai:gpt-5.5", + use_gateway=True, + reasoning={"effort": "none"}, + ) + assert captured["use_responses_api"] is True + assert captured["store"] is False + assert captured["include"] == ["reasoning.encrypted_content"] + assert captured["reasoning"] == {"effort": "none"} + assert "reasoning_effort" not in captured + + +def test_make_model_gateway_openai_responses_keeps_reasoning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + reasoning = {"effort": "high", "summary": "auto"} + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model("openai:gpt-5.5", use_gateway=True, reasoning=reasoning) + assert captured["use_responses_api"] is True + assert captured["store"] is False + assert captured["include"] == ["reasoning.encrypted_content"] + assert captured["reasoning"] == reasoning + assert "reasoning_effort" not in captured + + +def test_make_model_gateway_follows_env_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + monkeypatch.setenv("LANGSMITH_GATEWAY_ENABLED", "true") + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model("anthropic:claude-opus-4-8") # use_gateway=None -> env default + assert captured["base_url"] == "https://gateway.smith.langchain.com/anthropic" + assert captured["api_key"] == "ls-key" + + +def test_make_model_gateway_google_genai(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key") + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model("google_genai:gemini-3.5-flash", use_gateway=True) + assert captured["base_url"] == "https://gateway.smith.langchain.com/gemini" + assert captured["api_key"] == "ls-key" + + +def test_make_model_gateway_without_key_falls_back_direct( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured, fake = _capture_init_chat_model() + with patch.object(model, "init_chat_model", fake): + model.make_model("openai:gpt-5.5", use_gateway=True) # no LangSmith key + # No key -> overrides skipped -> the direct-provider websocket base stands. + assert captured["base_url"] == model.OPENAI_RESPONSES_WS_BASE_URL + assert captured["use_responses_api"] is True + assert captured["store"] is False + assert captured["include"] == ["reasoning.encrypted_content"] + assert "api_key" not in captured diff --git a/tests/test_sanitize_fireworks_messages.py b/tests/test_sanitize_fireworks_messages.py new file mode 100644 index 00000000..9f0fb68e --- /dev/null +++ b/tests/test_sanitize_fireworks_messages.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + +from agent.middleware.sanitize_fireworks_messages import SanitizeFireworksMessagesMiddleware + + +def _make_request(messages: list[object], model: object | None = None) -> MagicMock: + request = MagicMock() + request.model = model + request.messages = messages + return request + + +def _fireworks_model() -> MagicMock: + """A mock that satisfies ``_is_chat_fireworks`` via spec'd ``ChatFireworks``.""" + try: + from langchain_fireworks.chat_models import ChatFireworks + except ImportError: # pragma: no cover + pytest.skip("langchain-fireworks not installed") + return MagicMock(spec=ChatFireworks) + + +class TestSanitizeFireworksMessagesMiddleware: + def test_drops_legacy_function_call(self) -> None: + message = AIMessage( + content="", + tool_calls=[{"name": "read_file", "args": {"file_path": "/x"}, "id": "tc1"}], + additional_kwargs={"function_call": {"name": "read_file", "arguments": "{}"}}, + ) + request = _make_request([message], model=_fireworks_model()) + response = MagicMock() + + def handler(req: object) -> object: + assert req is request + return response + + result = SanitizeFireworksMessagesMiddleware().wrap_model_call(request, handler) + + assert result is response + assert "function_call" not in message.additional_kwargs + # tool_calls are untouched + assert len(message.tool_calls) == 1 + + def test_preserves_message_without_function_call(self) -> None: + message = AIMessage( + content="ok", + tool_calls=[{"name": "read_file", "args": {"file_path": "/x"}, "id": "tc1"}], + ) + request = _make_request([message], model=_fireworks_model()) + + SanitizeFireworksMessagesMiddleware().wrap_model_call(request, lambda req: MagicMock()) + + assert "function_call" not in message.additional_kwargs + assert len(message.tool_calls) == 1 + + def test_drops_function_call_with_no_tool_calls(self) -> None: + message = AIMessage( + content="", + additional_kwargs={"function_call": {"name": "search", "arguments": "{}"}}, + ) + request = _make_request([message], model=_fireworks_model()) + + SanitizeFireworksMessagesMiddleware().wrap_model_call(request, lambda req: MagicMock()) + + assert "function_call" not in message.additional_kwargs + + @pytest.mark.asyncio + async def test_async_drops_legacy_function_call(self) -> None: + tool_result = ToolMessage(content="result", tool_call_id="tc1") + message = AIMessage( + content="", + tool_calls=[{"name": "read_file", "args": {"file_path": "/x"}, "id": "tc1"}], + additional_kwargs={"function_call": {"name": "read_file", "arguments": "{}"}}, + ) + request = _make_request( + [HumanMessage(content="hi"), message, tool_result], + model=_fireworks_model(), + ) + response = MagicMock() + + async def handler(req: object) -> object: + assert req is request + return response + + result = await SanitizeFireworksMessagesMiddleware().awrap_model_call(request, handler) + + assert result is response + assert "function_call" not in message.additional_kwargs + + def test_ignores_non_fireworks_models(self) -> None: + message = AIMessage( + content="", + tool_calls=[{"name": "read_file", "args": {"file_path": "/x"}, "id": "tc1"}], + additional_kwargs={"function_call": {"name": "read_file", "arguments": "{}"}}, + ) + # Non-Fireworks model (plain MagicMock, no ChatFireworks in its spec chain) + request = _make_request([message], model=MagicMock()) + + SanitizeFireworksMessagesMiddleware().wrap_model_call(request, lambda req: MagicMock()) + + # function_call preserved for non-Fireworks providers + assert "function_call" in message.additional_kwargs + + def test_skips_non_ai_messages(self) -> None: + messages = [ + HumanMessage(content="hi"), + ToolMessage(content="result", tool_call_id="tc1"), + ] + request = _make_request(messages, model=_fireworks_model()) + + SanitizeFireworksMessagesMiddleware().wrap_model_call(request, lambda req: MagicMock()) + + # No AIMessages to mutate — handler still called + assert all(not isinstance(m, AIMessage) for m in messages) diff --git a/tests/test_sanitize_openai_responses.py b/tests/test_sanitize_openai_responses.py new file mode 100644 index 00000000..6185431c --- /dev/null +++ b/tests/test_sanitize_openai_responses.py @@ -0,0 +1,55 @@ +from langchain_core.messages import AIMessage, HumanMessage + +from agent.middleware.sanitize_openai_responses import _sanitize_messages + + +def test_sanitize_messages_drops_stale_reasoning_references() -> None: + messages = [ + HumanMessage(content="review this"), + AIMessage( + content=[ + { + "type": "reasoning", + "id": "rs_123", + "summary": [], + "content": [], + }, + { + "type": "function_call", + "id": "fc_123", + "call_id": "call_123", + "name": "read_file", + "arguments": "{}", + }, + ], + tool_calls=[ + { + "name": "read_file", + "args": {}, + "id": "call_123", + } + ], + ), + ] + + _sanitize_messages(messages) + + content = messages[1].content + assert isinstance(content, list) + assert [block["type"] for block in content] == ["function_call"] + assert messages[1].tool_calls[0]["id"] == "call_123" + + +def test_sanitize_messages_preserves_encrypted_reasoning() -> None: + reasoning = { + "type": "reasoning", + "id": "rs_123", + "encrypted_content": "encrypted", + "summary": [], + "content": [], + } + messages = [AIMessage(content=[reasoning])] + + _sanitize_messages(messages) + + assert messages[0].content == [reasoning] diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index e9fd7705..569f7abe 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -164,6 +164,8 @@ export interface TeamSettings { review_draft_prs: boolean pr_summaries: boolean review_trace_links: boolean + /** Tri-state LLM Gateway toggle; null inherits the LANGSMITH_GATEWAY_ENABLED default. */ + gateway_enabled?: boolean | null review_tracing_project?: string | null org_guidelines?: string | null default_agent_model?: string | null diff --git a/ui/src/routes/admin.tsx b/ui/src/routes/admin.tsx index 69b99d13..7ec9b2b4 100644 --- a/ui/src/routes/admin.tsx +++ b/ui/src/routes/admin.tsx @@ -55,6 +55,8 @@ function AdminPage() { > + + @@ -552,6 +554,77 @@ function ObservabilityCredentialsSection() { ) } +type GatewayMode = "inherit" | "enabled" | "disabled" + +function gatewayMode(value: boolean | null | undefined): GatewayMode { + if (value === true) return "enabled" + if (value === false) return "disabled" + return "inherit" +} + +function gatewayModeValue(mode: GatewayMode): boolean | null { + if (mode === "enabled") return true + if (mode === "disabled") return false + return null +} + +function LLMGatewaySection() { + const qc = useQueryClient() + const settings = useQuery({ + queryKey: ["teamSettings"], + queryFn: api.getTeamSettings, + }) + const [error, setError] = useState(null) + + const save = useMutation({ + mutationFn: (body: TeamSettings) => api.saveTeamSettings(body), + onSuccess: (saved) => { + qc.setQueryData(["teamSettings"], saved) + setError(null) + }, + onError: (e: Error) => setError(e.message), + }) + + const mode = gatewayMode(settings.data?.gateway_enabled) + + return ( + +
+ + settings.data && + save.mutate({ + ...settings.data, + gateway_enabled: gatewayModeValue(next as GatewayMode), + }) + } + disabled={!settings.data || save.isPending} + > + + + + + Inherit deployment default + Enabled + Disabled + + + } + /> +
+ {error &&

{error}

} + + ) +} + function PRTraceResolutionSection() { const qc = useQueryClient() const settings = useQuery({