mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 23:13:15 +00:00
0f0f616cd4
Some checks failed
CI / Lint (push) Has been cancelled
CI / Format check (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Unit tests (push) Has been cancelled
CI / Playwright E2E (push) Has been cancelled
CI / Docker build smoke (push) Has been cancelled
CI / Triage ledger up to date (push) Has been cancelled
CI / ui bun.lock in sync (push) Has been cancelled
* feat(reviewer): explicit-request verdicts + shell verdict guard Mention-triggered reviews that explicitly ask for a verdict now submit a real APPROVE/REQUEST_CHANGES through publish_review; auto-reviews stay advisory (COMMENT). Authorization is enforced in code: publish_review honors a verdict only when the dispatching webhook set verdict_requested, which only the explicit-mention path does. - request_pr_review gains instructions (forwarded verbatim into an escaped requester_instructions data block) and request_verdict - self-review guard downgrades verdicts on Open SWE-authored PRs; stale APPROVEs are best-effort dismissed when later findings land - new PullRequestVerdictGuardMiddleware blocks gh pr review --approve/-a/--request-changes/-r, gh api, and curl verdict fallbacks on both the coding-agent and reviewer graphs - shared escape helper moved to agent/utils/prompt_data.py * fix(reviewer): harden verdict path against security-review findings Adversarial security review (detector fan-out + proof-or-kill verifier) of the verdict feature surfaced several verdict-integrity gaps; resolve the confirmed ones: - head-drift (high): a mid-run push moves the resolved head, so an APPROVE could anchor to an unreviewed commit. Downgrade any verdict to a comment when the resolved head differs from the reviewed head (verdict_ignored reason head_moved); the push's own re-review submits a fresh verdict. - self-review fail-open: downgrade to comment when the PR author cannot be confirmed (author_unknown), and compare bot logins case-insensitively. - verdict_submitted now reflects GitHub's returned review state, not just the event we asked for, so a coerced APPROVE isn't reported as submitted. - an authorized verdict whose findings all anchor outside the diff now posts as a bodied review with zero inline comments instead of failing. - add finding_reply to the shared data-block escape tag superset.
109 lines
4.2 KiB
Python
109 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from langchain_core.messages import ToolMessage
|
|
|
|
from agent.middleware.pr_verdict_guard import (
|
|
PullRequestVerdictGuardMiddleware,
|
|
is_pr_verdict_fallback_command,
|
|
)
|
|
|
|
|
|
class _Request:
|
|
def __init__(self, command: str, *, tool: str = "execute") -> None:
|
|
self.tool_call = {
|
|
"name": tool,
|
|
"args": {"command": command},
|
|
"id": "call-1",
|
|
}
|
|
|
|
|
|
async def _handler(_request: Any) -> ToolMessage:
|
|
return ToolMessage(content="allowed", tool_call_id="call-1")
|
|
|
|
|
|
def test_detects_gh_pr_review_verdict_flags() -> None:
|
|
assert is_pr_verdict_fallback_command("gh pr review 12 --approve")
|
|
assert is_pr_verdict_fallback_command("gh pr review -a")
|
|
assert is_pr_verdict_fallback_command("gh pr review 12 -r -b 'needs work'")
|
|
assert is_pr_verdict_fallback_command("gh pr review 12 --request-changes")
|
|
assert is_pr_verdict_fallback_command("gh pr review --approve=true")
|
|
assert is_pr_verdict_fallback_command("gh pr review --request-changes='fix it'")
|
|
assert is_pr_verdict_fallback_command("GH_TOKEN=dummy gh pr review 5 --approve")
|
|
assert is_pr_verdict_fallback_command("git fetch && gh pr review 3 -a")
|
|
|
|
|
|
def test_detects_gh_api_review_verdicts() -> None:
|
|
assert is_pr_verdict_fallback_command(
|
|
"gh api repos/langchain-ai/open-swe/pulls/5/reviews -f event=APPROVE"
|
|
)
|
|
assert is_pr_verdict_fallback_command(
|
|
"gh api repos/o/r/pulls/5/reviews -X POST -f event=REQUEST_CHANGES"
|
|
)
|
|
assert is_pr_verdict_fallback_command(
|
|
"gh api repos/o/r/pulls/5/reviews/9/events -f event=approve"
|
|
)
|
|
assert is_pr_verdict_fallback_command(
|
|
"gh api https://api.github.com/repos/o/r/pulls/5/reviews -f event=APPROVE"
|
|
)
|
|
|
|
|
|
def test_detects_curl_review_verdicts() -> None:
|
|
assert is_pr_verdict_fallback_command(
|
|
'curl -X POST https://api.github.com/repos/o/r/pulls/5/reviews -d \'{"event": "APPROVE"}\''
|
|
)
|
|
assert is_pr_verdict_fallback_command(
|
|
"curl https://api.github.com/repos/o/r/pulls/5/reviews/9/events "
|
|
'--json \'{"event":"REQUEST_CHANGES"}\''
|
|
)
|
|
|
|
|
|
def test_allows_safe_review_commands() -> None:
|
|
assert not is_pr_verdict_fallback_command("gh pr review 12 --comment -b 'looks good'")
|
|
assert not is_pr_verdict_fallback_command("gh pr review -c")
|
|
assert not is_pr_verdict_fallback_command("gh pr review")
|
|
assert not is_pr_verdict_fallback_command("gh pr diff 12")
|
|
assert not is_pr_verdict_fallback_command("gh pr view 12 --json reviews")
|
|
assert not is_pr_verdict_fallback_command("gh api repos/o/r/pulls/5/reviews")
|
|
assert not is_pr_verdict_fallback_command("gh pr create --draft")
|
|
assert not is_pr_verdict_fallback_command("curl https://api.github.com/repos/o/r/pulls/5")
|
|
|
|
|
|
async def test_middleware_blocks_execute_verdict_fallbacks() -> None:
|
|
for command in (
|
|
"gh pr review 12 --approve",
|
|
"gh api repos/o/r/pulls/5/reviews -f event=REQUEST_CHANGES",
|
|
'curl -X POST https://api.github.com/repos/o/r/pulls/5/reviews -d \'{"event": "APPROVE"}\'',
|
|
):
|
|
result = await PullRequestVerdictGuardMiddleware().awrap_tool_call(
|
|
_Request(command), _handler
|
|
)
|
|
|
|
assert isinstance(result, ToolMessage)
|
|
assert result.status == "error"
|
|
payload = json.loads(str(result.content))
|
|
assert payload["code"] == "pr_verdict_fallback_blocked"
|
|
assert payload["error_type"] == "PullRequestVerdictFallbackBlocked"
|
|
assert payload["recoverable_by_agent"] is False
|
|
assert "publish_review" in payload["error"]
|
|
assert payload["blocked_command"] == command
|
|
|
|
|
|
async def test_middleware_allows_comment_review() -> None:
|
|
result = await PullRequestVerdictGuardMiddleware().awrap_tool_call(
|
|
_Request("gh pr review 12 --comment -b 'nit: rename'"), _handler
|
|
)
|
|
|
|
assert isinstance(result, ToolMessage)
|
|
assert result.content == "allowed"
|
|
|
|
|
|
async def test_middleware_ignores_other_tools() -> None:
|
|
result = await PullRequestVerdictGuardMiddleware().awrap_tool_call(
|
|
_Request("gh pr review 12 --approve", tool="read_file"), _handler
|
|
)
|
|
|
|
assert isinstance(result, ToolMessage)
|
|
assert result.content == "allowed"
|