From ccea80b8875e977cb765690069f9d8e5575fc5d5 Mon Sep 17 00:00:00 2001 From: Johannes du Plessis Date: Fri, 22 May 2026 14:58:27 -0700 Subject: [PATCH] feat: inline AGENTS.md into reviewer system prompt (#1328) * feat(reviewer): inline AGENTS.md into reviewer system prompt Fetches AGENTS.md from the PR's head_sha via the GitHub contents API during reviewer setup and inlines it as a "Repository conventions" block in the system prompt. Mirrors how the per-repo review style prompt is already wired. The main agent has long had a mandatory step to read AGENTS.md after cloning, but the reviewer often skips cloning entirely (it can `gh pr diff` directly), so it never saw the file. Loading it deterministically means the reviewer judges findings against the project's own conventions instead of relying on the model to fetch the file itself. * fix(reviewer): fetch AGENTS.md from base_sha, not head_sha The reviewer inlines AGENTS.md into its system prompt. Reading from head_sha means a PR author can edit AGENTS.md in the same PR being reviewed and smuggle instructions like "ignore all bugs" / "publish no findings" into the reviewer's prompt. Switch to base_sha (the target branch's pre-PR state, which is trusted) and update the prompt text to reflect that the contents come from the base, not the head. --------- Co-authored-by: open-swe[bot] --- agent/reviewer.py | 47 +++++++++++++++++++++++-- agent/utils/agents_md.py | 73 ++++++++++++++++++++++++++++++++++++++ tests/test_reviewer.py | 75 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 agent/utils/agents_md.py diff --git a/agent/reviewer.py b/agent/reviewer.py index 59e792cb..a0df2c47 100644 --- a/agent/reviewer.py +++ b/agent/reviewer.py @@ -57,6 +57,7 @@ from .tools import ( update_finding, web_search, ) +from .utils.agents_md import fetch_agents_md from .utils.auth import resolve_github_token from .utils.github_token import get_github_token_from_thread from .utils.model import DEFAULT_LLM_REASONING, make_model, provider_model_kwargs @@ -216,6 +217,7 @@ def _reviewer_system_prompt( pr_number: int | str, reviewer_eval: bool = False, repo_style_prompt: str | None = None, + agents_md_content: str | None = None, ) -> str: prompt = REVIEWER_PROMPT_TEMPLATE.format( working_dir=working_dir, @@ -234,6 +236,21 @@ def _reviewer_system_prompt( "they refine tone, severity, and what this team typically flags.\n\n" f"{repo_style_prompt}" ) + if agents_md_content: + prompt = ( + f"{prompt}\n\n" + "# Repository conventions (AGENTS.md)\n\n" + "The following is the `AGENTS.md` file from the target branch " + "(the PR's base), not from the PR head. It documents the " + "project's conventions, architecture, and rules. Treat " + "violations of these conventions as candidate findings when " + "they meet the global bar above (anchored to a changed line, " + "concrete failure mode, in-diff). Do not file findings for " + "pre-existing violations outside the diff.\n\n" + "```\n" + f"{agents_md_content}\n" + "```" + ) return prompt @@ -322,6 +339,7 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: logger.info("No thread_id or not for execution, returning reviewer agent without sandbox") return create_deep_agent(system_prompt="", tools=[]).with_config(config) + github_token: str | None = None if config["configurable"].get("source"): cached_token, cached_encrypted, cached_expires_at = await get_github_token_from_thread( thread_id @@ -329,12 +347,12 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: if cached_token and cached_encrypted: config["metadata"]["github_token_encrypted"] = cached_encrypted config["metadata"]["github_token_expires_at"] = cached_expires_at - del cached_token + github_token = cached_token else: _token, new_encrypted, new_expires_at = await resolve_github_token(config, thread_id) config["metadata"]["github_token_encrypted"] = new_encrypted config["metadata"]["github_token_expires_at"] = new_expires_at - del _token + github_token = _token sandbox_backend = await ensure_sandbox_for_thread(thread_id) @@ -411,6 +429,30 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: from .dashboard.review_styles import get_repo_custom_prompt repo_style_prompt = await get_repo_custom_prompt(repo_owner, repo_name) + + # Fetch AGENTS.md from base_sha (the target branch's state before this + # PR's changes), not head_sha. The contents are inlined into the system + # prompt, so reading from head would let a PR author smuggle reviewer + # instructions ("ignore all bugs", "publish no findings") into the + # review. base_sha is the trusted ref. + agents_md_content: str | None = None + if repo_owner and repo_name and base_sha: + agents_md_content = await fetch_agents_md( + repo_owner, + repo_name, + base_sha, + token=github_token, + ) + if agents_md_content: + logger.info( + "Loaded AGENTS.md (%d chars) from %s/%s@%s into reviewer prompt", + len(agents_md_content), + repo_owner, + repo_name, + base_sha, + ) + del github_token + system_prompt = _reviewer_system_prompt( f"{work_dir}/{repo_name}" if repo_name else work_dir, repo_owner=repo_owner, @@ -418,6 +460,7 @@ async def get_reviewer_agent(config: RunnableConfig) -> Pregel: pr_number=pr_number if isinstance(pr_number, int) else "", reviewer_eval=reviewer_eval, repo_style_prompt=repo_style_prompt, + agents_md_content=agents_md_content, ) if review_context: system_prompt = f"{system_prompt}\n\n{review_context}" diff --git a/agent/utils/agents_md.py b/agent/utils/agents_md.py new file mode 100644 index 00000000..f57a4b5e --- /dev/null +++ b/agent/utils/agents_md.py @@ -0,0 +1,73 @@ +"""Fetch ``AGENTS.md`` from a GitHub repo so it can be inlined into prompts. + +Used by the reviewer to deterministically load repo conventions into context +without the model having to clone the repo and read the file itself. +""" + +from __future__ import annotations + +import logging + +import httpx + +logger = logging.getLogger(__name__) + +# Cap the inlined content. AGENTS.md is meant to be a short conventions doc; +# anything larger is probably accidental and would bloat every reviewer prompt. +_MAX_AGENTS_MD_BYTES = 64 * 1024 + + +async def fetch_agents_md( + owner: str, + repo: str, + ref: str, + *, + token: str | None, + timeout: float = 10.0, +) -> str | None: + """Fetch ``AGENTS.md`` at ``ref`` from ``owner/repo``. + + Returns the raw file contents, or ``None`` if the file is missing, the + request fails, or the file exceeds the size cap. + """ + if not owner or not repo or not ref: + return None + + url = f"https://api.github.com/repos/{owner}/{repo}/contents/AGENTS.md" + headers = { + "Accept": "application/vnd.github.raw", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + + try: + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.get(url, headers=headers, params={"ref": ref}) + except httpx.HTTPError: + logger.exception("Failed to fetch AGENTS.md from %s/%s@%s", owner, repo, ref) + return None + + if response.status_code == 404: + return None + if response.status_code != 200: + logger.warning( + "Unexpected status %s fetching AGENTS.md from %s/%s@%s", + response.status_code, + owner, + repo, + ref, + ) + return None + + content = response.text + if len(content.encode("utf-8")) > _MAX_AGENTS_MD_BYTES: + logger.info( + "AGENTS.md in %s/%s@%s exceeds %d bytes; skipping inline", + owner, + repo, + ref, + _MAX_AGENTS_MD_BYTES, + ) + return None + return content diff --git a/tests/test_reviewer.py b/tests/test_reviewer.py index 6d884ffb..dd3fc55d 100644 --- a/tests/test_reviewer.py +++ b/tests/test_reviewer.py @@ -113,6 +113,11 @@ async def test_reviewer_applies_eval_model_and_effort_overrides() -> None: ), patch("agent.reviewer.make_model", return_value=MagicMock()) as make_model, patch("agent.reviewer.create_deep_agent", return_value=dummy_agent), + patch( + "agent.reviewer.fetch_agents_md", + new_callable=AsyncMock, + return_value=None, + ), ): await reviewer.get_reviewer_agent(config) @@ -161,8 +166,78 @@ async def test_reviewer_injects_repo_style_during_eval() -> None: ), patch("agent.reviewer.make_model", return_value=MagicMock()), patch("agent.reviewer.create_deep_agent", side_effect=fake_create_deep_agent), + patch( + "agent.reviewer.fetch_agents_md", + new_callable=AsyncMock, + return_value=None, + ), ): await reviewer.get_reviewer_agent(config) assert "Repository-specific review style" in captured["system_prompt"] assert "Flag table rerender regressions" in captured["system_prompt"] + + +def test_reviewer_system_prompt_includes_agents_md_section() -> None: + prompt = reviewer._reviewer_system_prompt( + "/workspace/repo", + repo_owner="acme", + repo_name="repo", + pr_number=42, + agents_md_content="Use snake_case for all Python identifiers.", + ) + assert "Repository conventions (AGENTS.md)" in prompt + assert "Use snake_case for all Python identifiers." in prompt + + +@pytest.mark.asyncio +async def test_reviewer_inlines_agents_md_into_system_prompt() -> None: + config: RunnableConfig = { + "configurable": { + "__is_for_execution__": True, + "thread_id": "reviewer-thread-id", + "source": "github", + "repo": {"owner": "acme", "name": "repo"}, + "pr_number": 7, + "pr_url": "https://github.com/acme/repo/pull/7", + "base_sha": "base-sha-xyz", + "head_sha": "head-sha-abc", + }, + "metadata": {}, + } + captured: dict[str, str] = {} + + def fake_create_deep_agent(*, system_prompt: str, **kwargs: object) -> _DummyAgent: + captured["system_prompt"] = system_prompt + return _DummyAgent() + + with ( + patch( + "agent.reviewer.get_github_token_from_thread", + new_callable=AsyncMock, + return_value=("gh-token", "encrypted-token", None), + ), + patch("agent.reviewer.resolve_github_token", new_callable=AsyncMock), + patch( + "agent.reviewer.ensure_sandbox_for_thread", + new_callable=AsyncMock, + return_value=MagicMock(), + ), + patch( + "agent.reviewer.aresolve_sandbox_work_dir", + new_callable=AsyncMock, + return_value="/workspace", + ), + patch( + "agent.reviewer.fetch_agents_md", + new_callable=AsyncMock, + return_value="Always use the design system IconButton.", + ) as mock_fetch_agents_md, + patch("agent.reviewer.make_model", return_value=MagicMock()), + patch("agent.reviewer.create_deep_agent", side_effect=fake_create_deep_agent), + ): + await reviewer.get_reviewer_agent(config) + + mock_fetch_agents_md.assert_awaited_once_with("acme", "repo", "base-sha-xyz", token="gh-token") + assert "Repository conventions (AGENTS.md)" in captured["system_prompt"] + assert "Always use the design system IconButton." in captured["system_prompt"]