From 259329a971e8f04952f0fc4f4249cb1cc2fbfee3 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Fri, 31 Jul 2026 13:48:03 -0400 Subject: [PATCH] fix(reviewer): preserve explicit verdict authority --- agent/prompt.py | 2 +- agent/reviewer.py | 4 +++- agent/tools/request_pr_review.py | 11 ++++++----- tests/github/test_github_comment_prompts.py | 6 ++++-- tests/github/test_github_issue_webhook.py | 2 ++ tests/reviewer/test_reviewer.py | 6 +++++- 6 files changed, 21 insertions(+), 10 deletions(-) diff --git a/agent/prompt.py b/agent/prompt.py index d1c5700f..286690fb 100644 --- a/agent/prompt.py +++ b/agent/prompt.py @@ -203,7 +203,7 @@ TASK_EXECUTION_SECTION = """--- First decide: is the user asking for code/repository changes, or for information only? Do not create commits, branches, or pull requests for questions, explanations, or status checks that can be answered without changing files. -If a Slack- or GitHub-triggered request asks you to review a GitHub pull request, do not clone/edit/commit/push/open a PR — call `request_pr_review` once with the PR URL, reply in the source channel saying whether the review started or why not, and stop. Pass the user's review instructions VERBATIM via `instructions=` (do not paraphrase or add your own). Review dispatch decides whether the reviewer is authorized to submit a verdict. Never approve or request changes on a PR yourself via `gh pr review` or the GitHub API; that path is blocked, and verdicts are the reviewer run's job. +If a Slack- or GitHub-triggered request asks you to review a GitHub pull request, do not clone/edit/commit/push/open a PR — call `request_pr_review` once with the PR URL, reply in the source channel saying whether the review started or why not, and stop. Pass the user's review instructions VERBATIM via `instructions=` (do not paraphrase or add your own). Set `request_verdict=True` only when the human explicitly asked for APPROVE/REQUEST_CHANGES authority; never infer that elevated requested authority from tone or context. Review dispatch may independently authorize a verdict, so `request_verdict=False` does not make every review comment-only. Never approve or request changes on a PR yourself via `gh pr review` or the GitHub API; that path is blocked, and verdicts are the reviewer run's job. **For code-change tasks:** Understand the task and explore relevant files first. Make focused, minimal changes — do not touch code outside the task's scope or add implementations in other languages/packages. Verify with linters and only the tests related to your changes. Then commit, push, and (when a PR is warranted) open/update the draft PR — see Committing below. diff --git a/agent/reviewer.py b/agent/reviewer.py index adb5a8ae..6640535f 100644 --- a/agent/reviewer.py +++ b/agent/reviewer.py @@ -360,7 +360,9 @@ REVIEWER_COMMENT_ONLY_PROMPT_SUFFIX = """ This run is not authorized to submit APPROVE or REQUEST_CHANGES. Reconcile all findings and call `publish_review` without `verdict`. Publish an advisory comment -review only, and do not claim that GitHub recorded a verdict. +review only, and do not claim that GitHub recorded a verdict. If `publish_review` +returns `verdict_ignored`, report the outcome as COMMENTED, state the +`verdict_ignored_reason`, and never claim a verdict. """ diff --git a/agent/tools/request_pr_review.py b/agent/tools/request_pr_review.py index 0e72b4a4..e7be9244 100644 --- a/agent/tools/request_pr_review.py +++ b/agent/tools/request_pr_review.py @@ -47,11 +47,12 @@ async def request_pr_review( verdict under repository policy. request_verdict: Set True only for an explicit human request for an approve/request-changes decision. This grants direct verdict - authority in addition to dispatch-side policy. False does not force - a comment-only review because dispatch may authorize verdicts from - repository and pull-request context. Never submit a verdict - yourself through ``gh pr review`` or the GitHub API; that path is - blocked. + authority in addition to dispatch-side policy; never infer that + elevated requested authority from tone or context. False does not + force a comment-only review because dispatch may independently + authorize verdicts from repository and pull-request context. Never + submit a verdict yourself through ``gh pr review`` or the GitHub + API; that path is blocked. """ pr_ref = parse_github_pr_url(pr_url) if not pr_ref: diff --git a/tests/github/test_github_comment_prompts.py b/tests/github/test_github_comment_prompts.py index cb6f014a..c60b4440 100644 --- a/tests/github/test_github_comment_prompts.py +++ b/tests/github/test_github_comment_prompts.py @@ -92,8 +92,10 @@ def test_agent_review_prompt_defers_verdict_authorization_to_dispatch() -> None: prompt = construct_system_prompt(working_dir="/workspace") assert "Pass the user's review instructions VERBATIM" in prompt - assert "Review dispatch decides whether the reviewer is authorized" in prompt - assert "never infer it" not in prompt + assert "Set `request_verdict=True` only when the human explicitly asked" in prompt + assert "never infer that elevated requested authority from tone or context" in prompt + assert "Review dispatch may independently authorize a verdict" in prompt + assert "`request_verdict=False` does not make every review comment-only" in prompt def test_shared_base_requires_terse_slack_replies_with_share_path() -> None: diff --git a/tests/github/test_github_issue_webhook.py b/tests/github/test_github_issue_webhook.py index 85d592e0..84a62d1a 100644 --- a/tests/github/test_github_issue_webhook.py +++ b/tests/github/test_github_issue_webhook.py @@ -1403,6 +1403,8 @@ def test_request_pr_review_docstring_describes_dispatch_side_verdict_policy() -> assert "Dispatch independently resolves" in docstring assert "False does not force" in docstring assert "explicit human request" in docstring + assert "never infer that elevated requested authority from tone or context" in docstring + assert "dispatch may independently authorize verdicts" in docstring def test_build_github_pr_review_prompt_without_instructions_has_no_block() -> None: diff --git a/tests/reviewer/test_reviewer.py b/tests/reviewer/test_reviewer.py index 2bca5ded..5888f685 100644 --- a/tests/reviewer/test_reviewer.py +++ b/tests/reviewer/test_reviewer.py @@ -68,6 +68,8 @@ def test_reviewer_verdict_section_requested_authorized_or_comment_only() -> None assert "# Verdict mode — authorized" not in neither assert "# Comment-only review mode" in neither assert "call `publish_review` without `verdict`" in neither + assert "report the outcome as COMMENTED" in neither + assert "`verdict_ignored_reason`" in neither for prompt in (requested, authorized): assert "# Verdict mode — authorized" in prompt assert "# Comment-only review mode" not in prompt @@ -79,7 +81,7 @@ def test_reviewer_verdict_section_requested_authorized_or_comment_only() -> None assert "Never claim a verdict GitHub did not record" in prompt -def test_reviewer_verdict_suffix_suppressed_in_eval_mode() -> None: +def test_reviewer_eval_prompt_is_intentionally_comment_only_even_when_requested() -> None: prompt = reviewer._reviewer_system_prompt( "/workspace/repo", repo_owner="acme", @@ -91,6 +93,8 @@ def test_reviewer_verdict_suffix_suppressed_in_eval_mode() -> None: assert "# Verdict mode" not in prompt assert "# Comment-only review mode" in prompt + assert "call `publish_review` without `verdict`" in prompt + assert "report the outcome as COMMENTED" in prompt def test_reviewer_prompt_forbids_shell_verdicts() -> None: