open-swe/tests/test_pr_ready_auto_review.py

340 lines
12 KiB
Python
Raw Normal View History

feat: auto-review PRs on opened / ready-for-review (#1325) * feat: auto-review PRs on opened / ready-for-review Trigger Open SWE Review on `pull_request` actions `opened` and `ready_for_review` against the canonical reviewer thread (no need to request open-swe[bot] as a reviewer). `converted_to_draft` now also flips watch=False on the existing reviewer thread. Draft PRs are gated by a tri-state user setting on the profile: inherit team default, always on, or always off. The team-wide `review_draft_prs` setting is the org-wide default; each user can override it in My Settings. External contributors with no Open SWE profile fall back to the team default. * fix: PR review comments — auth source + draft-aware watch toggle - `process_github_pr_ready` now dispatches with `source="github"` so the auth resolver finds the bot token persisted on the thread. The previous `source="github_auto"` fell through to the email-based path in non bot-token-only deployments and failed with a missing-user-email error. - `converted_to_draft` no longer unconditionally clears `watch`. When the PR author's effective `review_draft_prs` setting is on, watch stays on so subsequent pushes still trigger re-reviews while the PR is in draft. * feat(reviewer): skip "no issues found" comment on empty re-reviews A re-review run with no new findings to surface no longer posts another "Open SWE Review: No issues found" comment on the PR. The "no issues" summary now only appears on the first review of a PR — matching Devin's behavior, where subsequent reviews are silent unless there's something new to flag. Resolved-thread reconciliation and ``last_reviewed_sha`` persistence still happen on the skipped path, so findings the user just fixed still get their GitHub threads marked resolved, and the next push event sees an up-to-date dedup SHA. --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
2026-05-22 13:36:14 -07:00
"""Tests for the opened / ready_for_review auto-review webhook handlers."""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from agent import webapp
def _pr_payload(*, action: str, draft: bool, author: str = "alice") -> dict[str, Any]:
return {
"action": action,
"repository": {"owner": {"login": "lc"}, "name": "repo"},
"pull_request": {
"number": 7,
"html_url": "https://github.com/lc/repo/pull/7",
"title": "T",
"draft": draft,
"user": {"login": author},
"head": {"sha": "headsha", "ref": "feat-x"},
"base": {"sha": "basesha", "ref": "main"},
},
"sender": {"login": author, "id": 1},
}
def _patch_dispatch_deps(monkeypatch: pytest.MonkeyPatch, fake_client: Any) -> None:
monkeypatch.setattr(
webapp,
"get_github_app_installation_token_with_expiry",
AsyncMock(return_value=("token", None)),
)
monkeypatch.setattr(webapp, "_ensure_thread_exists_for_metadata", AsyncMock(return_value=True))
monkeypatch.setattr(webapp, "persist_encrypted_github_token", AsyncMock(return_value="enc"))
monkeypatch.setattr(webapp, "set_reviewer_thread_metadata", AsyncMock())
monkeypatch.setattr(webapp, "is_thread_active", AsyncMock(return_value=False))
monkeypatch.setattr(webapp, "get_client", lambda url: fake_client)
@pytest.mark.asyncio
async def test_pr_ready_non_draft_triggers_run(monkeypatch: pytest.MonkeyPatch) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
monkeypatch.setattr(webapp, "get_profile", AsyncMock(return_value=None))
monkeypatch.setattr(webapp, "get_team_settings", AsyncMock(return_value={}))
await webapp.process_github_pr_ready(_pr_payload(action="opened", draft=False))
fake_client.runs.create.assert_awaited_once()
_, kwargs = fake_client.runs.create.await_args
assert kwargs["config"]["configurable"]["source"] == "github"
assert kwargs["config"]["configurable"]["pr_number"] == 7
@pytest.mark.asyncio
async def test_pr_ready_for_review_triggers_run(monkeypatch: pytest.MonkeyPatch) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
monkeypatch.setattr(webapp, "_get_thread_metadata_safe", AsyncMock(return_value=None))
feat: auto-review PRs on opened / ready-for-review (#1325) * feat: auto-review PRs on opened / ready-for-review Trigger Open SWE Review on `pull_request` actions `opened` and `ready_for_review` against the canonical reviewer thread (no need to request open-swe[bot] as a reviewer). `converted_to_draft` now also flips watch=False on the existing reviewer thread. Draft PRs are gated by a tri-state user setting on the profile: inherit team default, always on, or always off. The team-wide `review_draft_prs` setting is the org-wide default; each user can override it in My Settings. External contributors with no Open SWE profile fall back to the team default. * fix: PR review comments — auth source + draft-aware watch toggle - `process_github_pr_ready` now dispatches with `source="github"` so the auth resolver finds the bot token persisted on the thread. The previous `source="github_auto"` fell through to the email-based path in non bot-token-only deployments and failed with a missing-user-email error. - `converted_to_draft` no longer unconditionally clears `watch`. When the PR author's effective `review_draft_prs` setting is on, watch stays on so subsequent pushes still trigger re-reviews while the PR is in draft. * feat(reviewer): skip "no issues found" comment on empty re-reviews A re-review run with no new findings to surface no longer posts another "Open SWE Review: No issues found" comment on the PR. The "no issues" summary now only appears on the first review of a PR — matching Devin's behavior, where subsequent reviews are silent unless there's something new to flag. Resolved-thread reconciliation and ``last_reviewed_sha`` persistence still happen on the skipped path, so findings the user just fixed still get their GitHub threads marked resolved, and the next push event sees an up-to-date dedup SHA. --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
2026-05-22 13:36:14 -07:00
monkeypatch.setattr(webapp, "get_profile", AsyncMock(return_value=None))
monkeypatch.setattr(webapp, "get_team_settings", AsyncMock(return_value={}))
await webapp.process_github_pr_ready(_pr_payload(action="ready_for_review", draft=False))
fake_client.runs.create.assert_awaited_once()
@pytest.mark.asyncio
async def test_pr_ready_for_review_skips_when_head_already_reviewed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
set_metadata = AsyncMock()
get_token = AsyncMock(return_value=("token", None))
monkeypatch.setattr(webapp, "get_github_app_installation_token_with_expiry", get_token)
monkeypatch.setattr(webapp, "set_reviewer_thread_metadata", set_metadata)
monkeypatch.setattr(
webapp,
"_get_thread_metadata_safe",
AsyncMock(
return_value={
"kind": "reviewer",
"watch": False,
"last_reviewed_sha": "headsha",
}
),
)
monkeypatch.setattr(webapp, "get_client", lambda url: fake_client)
monkeypatch.setattr(webapp, "get_profile", AsyncMock(return_value=None))
monkeypatch.setattr(webapp, "get_team_settings", AsyncMock(return_value={}))
await webapp.process_github_pr_ready(_pr_payload(action="ready_for_review", draft=False))
fake_client.runs.create.assert_not_called()
get_token.assert_not_awaited()
set_metadata.assert_awaited_once()
assert set_metadata.await_args.kwargs["watch"] is True
@pytest.mark.asyncio
async def test_pr_ready_for_review_uses_re_review_after_previous_review(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
monkeypatch.setattr(
webapp,
"_get_thread_metadata_safe",
AsyncMock(
return_value={
"kind": "reviewer",
"watch": False,
"last_reviewed_sha": "oldsha",
}
),
)
monkeypatch.setattr(webapp, "get_profile", AsyncMock(return_value=None))
monkeypatch.setattr(webapp, "get_team_settings", AsyncMock(return_value={}))
await webapp.process_github_pr_ready(_pr_payload(action="ready_for_review", draft=False))
fake_client.runs.create.assert_awaited_once()
_, kwargs = fake_client.runs.create.await_args
configurable = kwargs["config"]["configurable"]
assert configurable["re_review"] is True
assert configurable["last_reviewed_sha"] == "oldsha"
assert configurable["head_sha"] == "headsha"
assert "marked ready for review" in kwargs["input"]["messages"][0]["content"]
feat: auto-review PRs on opened / ready-for-review (#1325) * feat: auto-review PRs on opened / ready-for-review Trigger Open SWE Review on `pull_request` actions `opened` and `ready_for_review` against the canonical reviewer thread (no need to request open-swe[bot] as a reviewer). `converted_to_draft` now also flips watch=False on the existing reviewer thread. Draft PRs are gated by a tri-state user setting on the profile: inherit team default, always on, or always off. The team-wide `review_draft_prs` setting is the org-wide default; each user can override it in My Settings. External contributors with no Open SWE profile fall back to the team default. * fix: PR review comments — auth source + draft-aware watch toggle - `process_github_pr_ready` now dispatches with `source="github"` so the auth resolver finds the bot token persisted on the thread. The previous `source="github_auto"` fell through to the email-based path in non bot-token-only deployments and failed with a missing-user-email error. - `converted_to_draft` no longer unconditionally clears `watch`. When the PR author's effective `review_draft_prs` setting is on, watch stays on so subsequent pushes still trigger re-reviews while the PR is in draft. * feat(reviewer): skip "no issues found" comment on empty re-reviews A re-review run with no new findings to surface no longer posts another "Open SWE Review: No issues found" comment on the PR. The "no issues" summary now only appears on the first review of a PR — matching Devin's behavior, where subsequent reviews are silent unless there's something new to flag. Resolved-thread reconciliation and ``last_reviewed_sha`` persistence still happen on the skipped path, so findings the user just fixed still get their GitHub threads marked resolved, and the next push event sees an up-to-date dedup SHA. --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
2026-05-22 13:36:14 -07:00
@pytest.mark.asyncio
async def test_pr_ready_draft_user_override_off_wins_over_team_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
monkeypatch.setattr(
webapp,
"get_profile",
AsyncMock(return_value={"login": "alice", "review_draft_prs": False}),
)
monkeypatch.setattr(
webapp, "get_team_settings", AsyncMock(return_value={"review_draft_prs": True})
)
await webapp.process_github_pr_ready(_pr_payload(action="opened", draft=True))
fake_client.runs.create.assert_not_called()
@pytest.mark.asyncio
async def test_pr_ready_draft_user_override_on_wins_over_team_off(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
monkeypatch.setattr(
webapp,
"get_profile",
AsyncMock(return_value={"login": "alice", "review_draft_prs": True}),
)
monkeypatch.setattr(
webapp,
"get_team_settings",
AsyncMock(return_value={"review_draft_prs": False}),
)
await webapp.process_github_pr_ready(_pr_payload(action="opened", draft=True))
fake_client.runs.create.assert_awaited_once()
@pytest.mark.asyncio
async def test_pr_ready_draft_user_default_falls_back_to_team_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
# User profile exists but review_draft_prs is None — inherit team default.
monkeypatch.setattr(
webapp,
"get_profile",
AsyncMock(return_value={"login": "alice", "review_draft_prs": None}),
)
monkeypatch.setattr(
webapp, "get_team_settings", AsyncMock(return_value={"review_draft_prs": True})
)
await webapp.process_github_pr_ready(_pr_payload(action="opened", draft=True))
fake_client.runs.create.assert_awaited_once()
@pytest.mark.asyncio
async def test_pr_ready_draft_no_profile_falls_back_to_team_off(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
# External contributor — inherit team default (off).
monkeypatch.setattr(webapp, "get_profile", AsyncMock(return_value=None))
monkeypatch.setattr(
webapp,
"get_team_settings",
AsyncMock(return_value={"review_draft_prs": False}),
)
await webapp.process_github_pr_ready(_pr_payload(action="opened", draft=True))
fake_client.runs.create.assert_not_called()
@pytest.mark.asyncio
async def test_pr_ready_draft_no_profile_falls_back_to_team_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_client = MagicMock()
fake_client.runs.create = AsyncMock()
_patch_dispatch_deps(monkeypatch, fake_client)
monkeypatch.setattr(webapp, "get_profile", AsyncMock(return_value=None))
monkeypatch.setattr(
webapp, "get_team_settings", AsyncMock(return_value={"review_draft_prs": True})
)
await webapp.process_github_pr_ready(_pr_payload(action="opened", draft=True))
fake_client.runs.create.assert_awaited_once()
def _converted_to_draft_payload(author: str = "alice") -> dict[str, Any]:
return {
"action": "converted_to_draft",
"repository": {"owner": {"login": "lc"}, "name": "repo"},
"pull_request": {
"number": 7,
"head": {"ref": "feat-x"},
"user": {"login": author},
},
}
@pytest.mark.asyncio
async def test_converted_to_draft_disables_watch_when_drafts_off(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: list[Any] = []
async def fake_set(thread_id: str, **kwargs: Any) -> None:
captured.append((thread_id, kwargs))
with (
patch("agent.webapp._is_repo_allowed_for_reviewer", return_value=True),
patch(
"agent.webapp._get_thread_metadata_safe",
new_callable=AsyncMock,
return_value={"kind": "reviewer", "watch": True},
),
patch(
"agent.webapp.get_profile",
new_callable=AsyncMock,
return_value={"login": "alice", "review_draft_prs": False},
),
patch(
"agent.webapp.get_team_settings",
new_callable=AsyncMock,
return_value={"review_draft_prs": False},
),
patch("agent.webapp.set_reviewer_thread_metadata", side_effect=fake_set),
):
await webapp.process_github_pr_close(_converted_to_draft_payload())
assert captured and captured[0][1]["watch"] is False
@pytest.mark.asyncio
async def test_converted_to_draft_keeps_watch_when_author_drafts_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_set = AsyncMock()
with (
patch("agent.webapp._is_repo_allowed_for_reviewer", return_value=True),
patch(
"agent.webapp._get_thread_metadata_safe",
new_callable=AsyncMock,
return_value={"kind": "reviewer", "watch": True},
),
patch(
"agent.webapp.get_profile",
new_callable=AsyncMock,
return_value={"login": "alice", "review_draft_prs": True},
),
patch(
"agent.webapp.get_team_settings",
new_callable=AsyncMock,
return_value={"review_draft_prs": False},
),
patch("agent.webapp.set_reviewer_thread_metadata", new=fake_set),
):
await webapp.process_github_pr_close(_converted_to_draft_payload())
fake_set.assert_not_called()
@pytest.mark.asyncio
async def test_converted_to_draft_keeps_watch_when_team_default_drafts_on(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_set = AsyncMock()
with (
patch("agent.webapp._is_repo_allowed_for_reviewer", return_value=True),
patch(
"agent.webapp._get_thread_metadata_safe",
new_callable=AsyncMock,
return_value={"kind": "reviewer", "watch": True},
),
# Author inherits team default — team has drafts on.
patch(
"agent.webapp.get_profile",
new_callable=AsyncMock,
return_value={"login": "alice", "review_draft_prs": None},
),
patch(
"agent.webapp.get_team_settings",
new_callable=AsyncMock,
return_value={"review_draft_prs": True},
),
patch("agent.webapp.set_reviewer_thread_metadata", new=fake_set),
):
await webapp.process_github_pr_close(_converted_to_draft_payload())
fake_set.assert_not_called()