From e347aed851a08801233f1ce03671d6ae74d3a72c Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" <215916821+open-swe[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 13:30:18 -0700 Subject: [PATCH] feat: make PR creation policy opt-in (#1334) Co-authored-by: open-swe[bot] Co-authored-by: Johannes du Plessis <51395795+johannes117@users.noreply.github.com> --- AGENTS.md | 2 +- CLAUDE.md | 2 +- agent/dashboard/agent_overrides.py | 6 +- agent/dashboard/profiles.py | 2 +- agent/prompt.py | 31 ++++--- agent/server.py | 8 +- agent/utils/slack.py | 2 +- tests/test_github_comment_prompts.py | 24 +++++ ui/src/lib/profile.ts | 38 ++++---- ui/src/routes/cloud-agents.tsx | 126 +++++++++++++++------------ 10 files changed, 144 insertions(+), 97 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d21be2e2..4e0f6850 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Model + reasoning effort are resolved per run in this precedence (highest wins): 2. Per-user dashboard profile override (`agent/dashboard/agent_overrides.py:load_profile`), keyed by resolved GitHub login. 3. Team default model (`agent/dashboard/team_settings.py:get_team_default_model("agent")`). -Supported model IDs and per-model effort/reasoning rules live in `agent/dashboard/options.py`. Profile flags also drive run behavior — e.g. `profile_create_prs` disables PR creation for users who've opted out. Model construction goes through `agent/utils/model.py` (`make_model`, `provider_model_kwargs`, `fallback_model_id_for`). +Supported model IDs and per-model effort/reasoning rules live in `agent/dashboard/options.py`. Profile flags also drive run behavior — e.g. `profile_create_prs` enables the opt-in Always Create PRs policy. Model construction goes through `agent/utils/model.py` (`make_model`, `provider_model_kwargs`, `fallback_model_id_for`). ### Auth diff --git a/CLAUDE.md b/CLAUDE.md index 6fb6e017..6065a271 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,7 +93,7 @@ Model + reasoning effort are resolved per run in this precedence (highest wins): 2. Per-user dashboard profile override (`agent/dashboard/agent_overrides.py:load_profile`), keyed by resolved GitHub login. 3. Team default model (`agent/dashboard/team_settings.py:get_team_default_model("agent")`). -Supported model IDs and per-model effort/reasoning rules live in `agent/dashboard/options.py`. Profile flags also drive run behavior — e.g. `profile_create_prs` disables PR creation for users who've opted out. Model construction goes through `agent/utils/model.py` (`make_model`, `provider_model_kwargs`, `fallback_model_id_for`). +Supported model IDs and per-model effort/reasoning rules live in `agent/dashboard/options.py`. Profile flags also drive run behavior — e.g. `profile_create_prs` enables the opt-in Always Create PRs policy. Model construction goes through `agent/utils/model.py` (`make_model`, `provider_model_kwargs`, `fallback_model_id_for`). ### Auth diff --git a/agent/dashboard/agent_overrides.py b/agent/dashboard/agent_overrides.py index 526dd0fa..a6deeaa8 100644 --- a/agent/dashboard/agent_overrides.py +++ b/agent/dashboard/agent_overrides.py @@ -73,13 +73,13 @@ async def load_profile(login: str) -> dict[str, Any] | None: def profile_create_prs(profile: dict[str, Any] | None) -> bool: - """Return whether the agent should automatically open a PR. Defaults to True.""" + """Return whether the agent should always open a PR. Defaults to False.""" if not isinstance(profile, dict): - return True + return False value = profile.get("create_prs") if isinstance(value, bool): return value - return True + return False def normalize_profile_overrides(profile: dict[str, Any]) -> tuple[str | None, str | None]: diff --git a/agent/dashboard/profiles.py b/agent/dashboard/profiles.py index 21509dfc..b87ff3a2 100644 --- a/agent/dashboard/profiles.py +++ b/agent/dashboard/profiles.py @@ -39,7 +39,7 @@ class ProfileUpdate(BaseModel): base_branch: str | None = None branch_prefix: str | None = None auto_fix_ci: bool = True - create_prs: bool = True + create_prs: bool = False review_draft_prs: bool | None = None @field_validator("default_model") diff --git a/agent/prompt.py b/agent/prompt.py index f8e0148a..7ccce69b 100644 --- a/agent/prompt.py +++ b/agent/prompt.py @@ -128,12 +128,14 @@ If you make changes, communicate updates in the source channel: If a Slack-triggered request is asking you to review a GitHub pull request, do not clone the repo, edit files, commit, push, or open a PR. Call `request_pr_review` once with the GitHub PR URL, then use `slack_thread_reply` to say whether the review was started or why it could not be started, and stop. +First decide whether the user is asking for code/repository changes or for information only. Do not create commits, branches, or pull requests for questions, explanations, status checks, or other requests that can be fully answered without changing files. + For tasks that require code changes, follow this order: 1. **Understand** — Read the issue/task carefully. Explore relevant files before making any changes. 2. **Implement** — Make focused, minimal changes. Do not modify code outside the scope of the task. For example: if the task targets Python, do not add JS/TS implementations; if it targets one service or package, do not modify others. 3. **Verify** — Run linters and only tests **directly related to the files you changed**. Do NOT run the full test suite — CI handles that. If no related tests exist, skip this step. -4. **Submit** — Commit, push, and open or update a draft pull request with `GH_TOKEN=dummy gh`. +4. **Submit** — Commit and push your branch. Open or update a draft pull request with `GH_TOKEN=dummy gh` when the user asks for a PR, when a PR is necessary to deliver or review the changes, or when the Always Create PRs dashboard setting is enabled. 5. **Comment** — Call `linear_comment` or `slack_thread_reply` for Linear/Slack. For GitHub-triggered tasks, comment with `GH_TOKEN=dummy gh`. **Strict requirement:** Never claim "PR updated/opened" unless `gh` returned success and you have the PR URL from command output or `GH_TOKEN=dummy gh pr view --json url --jq .url`. If push or PR creation fails, state that explicitly. @@ -141,7 +143,8 @@ For tasks that require code changes, follow this order: For questions or status checks (no code changes needed): 1. **Answer** — Gather the information needed to respond. -2. **Comment** — Call `linear_comment` or `slack_thread_reply` for Linear/Slack. For GitHub-triggered tasks, use `GH_TOKEN=dummy gh issue comment` or `GH_TOKEN=dummy gh pr comment`. Never leave a question unanswered.""" +2. **Comment** — Call `linear_comment` or `slack_thread_reply` for Linear/Slack. For GitHub-triggered tasks, use `GH_TOKEN=dummy gh issue comment` or `GH_TOKEN=dummy gh pr comment`. Never leave a question unanswered. +3. **Do not submit changes** — Do not commit, push, or open/update a PR unless the user then asks for changes.""" TOOL_USAGE_SECTION = """--- @@ -216,7 +219,7 @@ CORE_BEHAVIOR_SECTION = """--- - **Persistence:** Keep working until the current task is completely resolved. Only terminate when you are certain the task is complete. - **Accuracy:** Never guess or make up information. Always use tools to gather accurate data about files and codebase structure. -- **Autonomy:** Never ask the user for permission mid-task. Run linters, fix errors, push commits, and open/update the draft PR without waiting for confirmation.""" +- **Autonomy:** Never ask the user for permission mid-task. For code-change tasks, run linters, fix errors, push commits, and open/update the draft PR without waiting for confirmation when the user asks for a PR, when a PR is necessary, or when the Always Create PRs dashboard setting is enabled. For information-only tasks, answer directly without creating commits or PRs.""" DEPENDENCY_SECTION = """--- @@ -275,6 +278,10 @@ COMMIT_PR_SECTION = """--- ### Committing Changes and Opening Pull Requests +This section applies only after you have made code or repository changes. For information-only requests, answer in the source channel and do not commit, push, or open/update a PR. + +By default, open or update a draft PR when the user asks for one or when a PR is necessary to deliver or review the changes. If a code-change task does not need a PR, still commit and push the branch so the work is preserved, then notify the source channel with the branch URL and summary. If the Always Create PRs dashboard setting is enabled, always open or update a draft PR for code-change tasks. + When you have completed your implementation, follow these steps in order: 1. **Run linters and formatters**: You MUST run the appropriate lint/format commands before submitting: @@ -292,7 +299,7 @@ When you have completed your implementation, follow these steps in order: 2. **Review your changes**: Review the diff to ensure correctness. Verify no regressions or unintended modifications. -3. **Submit via `gh`**: Commit locally, push with `git push origin `, then use `GH_TOKEN=dummy gh pr create --draft ...` or `GH_TOKEN=dummy gh pr edit ...`. +3. **Submit via `gh`**: Commit locally, push with `git push origin `, then use `GH_TOKEN=dummy gh pr create --draft ...` or `GH_TOKEN=dummy gh pr edit ...` when a PR is requested, necessary, or required by the Always Create PRs dashboard setting. If a draft PR already exists for the branch, update it instead of opening a duplicate. **PR Title** (under 70 characters): @@ -318,7 +325,7 @@ When you have completed your implementation, follow these steps in order: **Commit message**: Concise, focusing on the "why" rather than the "what". If not provided, the PR title is used. -**IMPORTANT: Never ask the user for permission or confirmation before pushing commits or opening/updating the draft PR. Do not say "if you want, I can proceed" or "shall I open the PR?". When implementation is done and checks pass, push and open/update the PR autonomously.** +**IMPORTANT: For code-change tasks, never ask the user for permission or confirmation before pushing commits or opening/updating a draft PR. Do not say "if you want, I can proceed" or "shall I open the PR?". When implementation is done and checks pass, push autonomously, and open/update a draft PR autonomously when requested, necessary, or required by the Always Create PRs dashboard setting.** **IMPORTANT: If you made commits directly via `git commit` or `git revert` in the sandbox, you MUST push those commits to GitHub. Never report the work as done without pushing.** @@ -328,7 +335,7 @@ When you have completed your implementation, follow these steps in order: **IMPORTANT: If `git push` or `gh` returns "403", "Permission denied", or another permanent authorization failure, do not retry. Report the error to the user immediately and stop.** -4. **Notify the source** immediately after PR creation/update succeeds. Include a brief summary and the PR link: +4. **Notify the source** immediately after pushing and, when applicable, PR creation/update succeeds. Include a brief summary plus the PR link or branch URL: - Linear-triggered: use `linear_comment` with an `@mention` of the user who triggered the task - Slack-triggered: use `slack_thread_reply` - GitHub-triggered: use `GH_TOKEN=dummy gh issue comment` or `GH_TOKEN=dummy gh pr comment` @@ -343,7 +350,7 @@ When you have completed your implementation, follow these steps in order: - ``` -Always push, open/update the draft PR with `gh`, and notify the appropriate source once implementation is complete and code quality checks pass.""" +For code-change tasks, push the branch and notify the appropriate source once implementation is complete and code quality checks pass. Include the PR link when you opened or updated a PR; otherwise include the branch URL.""" COLLABORATION_TEMPLATE = """--- @@ -377,11 +384,11 @@ def _render_collaboration_section(identity: CollaboratorIdentity | None) -> str: ) -NO_PR_OVERRIDE_SECTION = """--- +ALWAYS_CREATE_PR_SECTION = """--- -### Pull Request Policy Override +### Always Create PRs Policy Override -The user has disabled automatic PR creation. After implementation, **commit and push your branch** so the work is preserved, then notify the source channel with the branch URL (e.g. `https://github.com///tree/`) and a summary. **Do not** run `gh pr create` or `gh pr edit`. Ignore any instructions elsewhere in this prompt that tell you to open or update a draft pull request.""" +The user's dashboard setting **Always Create PRs** is enabled. For code-change tasks, always open or update a draft pull request after committing and pushing the branch. This does not apply to questions, explanations, status checks, or other information-only requests where no files are changed.""" SYSTEM_PROMPT_TEMPLATE = ( @@ -411,7 +418,7 @@ def construct_system_prompt( linear_project_id: str = "", linear_issue_number: str = "", triggering_user_identity: CollaboratorIdentity | None = None, - create_prs: bool = True, + create_prs: bool = False, ) -> str: default_prompt_section = _load_default_prompt() return SYSTEM_PROMPT_TEMPLATE.format( @@ -419,6 +426,6 @@ def construct_system_prompt( linear_project_id=linear_project_id or "", linear_issue_number=linear_issue_number or "", default_prompt_section=default_prompt_section, - pr_policy_override_section="" if create_prs else NO_PR_OVERRIDE_SECTION, + pr_policy_override_section=ALWAYS_CREATE_PR_SECTION if create_prs else "", collaboration_section=_render_collaboration_section(triggering_user_identity), ) diff --git a/agent/server.py b/agent/server.py index 639c2d60..d28bf09e 100644 --- a/agent/server.py +++ b/agent/server.py @@ -419,9 +419,9 @@ async def get_agent(config: RunnableConfig) -> Pregel: model_id = per_thread_model profile_effort = per_thread_effort - create_prs = profile_create_prs(profile) - if not create_prs: - logger.info("PR creation disabled by profile for %s", profile_login) + always_create_prs = profile_create_prs(profile) + if always_create_prs: + logger.info("Always Create PRs enabled by profile for %s", profile_login) model_kwargs = provider_model_kwargs( model_id, @@ -448,7 +448,7 @@ async def get_agent(config: RunnableConfig) -> Pregel: linear_project_id=linear_project_id, linear_issue_number=linear_issue_number, triggering_user_identity=triggering_user_identity, - create_prs=create_prs, + create_prs=always_create_prs, ), tools=[ http_request, diff --git a/agent/utils/slack.py b/agent/utils/slack.py index f284d0ef..c0ee4595 100644 --- a/agent/utils/slack.py +++ b/agent/utils/slack.py @@ -685,7 +685,7 @@ TRACE_REPLY_TIPS: tuple[str, ...] = ( "Kick off another task in parallel — each one runs in its own isolated sandbox, no queuing.", "Add `repo:owner/name` to your message to point me at a different repo for this task.", "Drop an `AGENTS.md` at your repo root and I'll read it on every run — it's the easiest way to teach me your conventions.", - "I'll open a draft PR automatically when I'm done and link it back here.", + "For code-change tasks, I'll open a draft PR when it's necessary or requested and link it back here.", "Tag me on a PR comment of an open-swe PR to have me address review feedback on the same branch.", "I can spawn subagents for independent subtasks — useful for parallel research or fan-out work.", "Click `View trace` above to watch every tool call and model response live in LangSmith.", diff --git a/tests/test_github_comment_prompts.py b/tests/test_github_comment_prompts.py index 133fb382..7a7d27c3 100644 --- a/tests/test_github_comment_prompts.py +++ b/tests/test_github_comment_prompts.py @@ -1,6 +1,7 @@ from __future__ import annotations from agent import webapp +from agent.dashboard.agent_overrides import profile_create_prs from agent.prompt import construct_system_prompt from agent.utils import github_comments from agent.utils.authorship import CollaboratorIdentity @@ -46,6 +47,29 @@ def test_construct_system_prompt_omits_collaboration_section_without_identity() assert "Co-authored-by:" not in prompt +def test_construct_system_prompt_does_not_require_pr_for_questions() -> None: + prompt = construct_system_prompt(working_dir="/workspace") + + assert "Do not create commits, branches, or pull requests for questions" in prompt + assert "For information-only requests" in prompt + assert "open or update a draft PR when the user asks for one" in prompt + assert "Always Create PRs Policy Override" not in prompt + assert "Always push, open/update the draft PR" not in prompt + + +def test_construct_system_prompt_includes_always_create_prs_override() -> None: + prompt = construct_system_prompt(working_dir="/workspace", create_prs=True) + + assert "Always Create PRs Policy Override" in prompt + assert "This does not apply to questions" in prompt + + +def test_profile_create_prs_defaults_to_normal_pr_policy() -> None: + assert profile_create_prs(None) is False + assert profile_create_prs({}) is False + assert profile_create_prs({"create_prs": True}) is True + + def test_construct_system_prompt_includes_coauthor_trailer_when_identity_present() -> None: identity = CollaboratorIdentity( display_name="octocat", diff --git a/ui/src/lib/profile.ts b/ui/src/lib/profile.ts index 0278dd14..5123a523 100644 --- a/ui/src/lib/profile.ts +++ b/ui/src/lib/profile.ts @@ -1,52 +1,52 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" -import { ApiError, api } from "./api"; -import { useSession } from "./session"; -import type {Profile, ProfileUpdate} from "./api"; +import { ApiError, api } from "./api" +import { useSession } from "./session" +import type { Profile, ProfileUpdate } from "./api" export function useProfile() { - const session = useSession(); + const session = useSession() return useQuery({ queryKey: ["profile"], queryFn: api.profile, enabled: !!session.data, - }); + }) } export function useOptions() { - const session = useSession(); + const session = useSession() return useQuery({ queryKey: ["options"], queryFn: api.options, enabled: !!session.data, - }); + }) } export function useRepos() { - const session = useSession(); + const session = useSession() return useQuery({ queryKey: ["repos"], queryFn: async () => { try { - return await api.repos(); + return await api.repos() } catch (e) { if (e instanceof ApiError && e.status === 401) - return { installations: [], repositories: [] }; - throw e; + return { installations: [], repositories: [] } + throw e } }, enabled: !!session.data, - }); + }) } export function useSaveProfile() { - const qc = useQueryClient(); + const qc = useQueryClient() return useMutation({ mutationFn: (body: ProfileUpdate) => api.saveProfile(body), onSuccess: (saved) => { - qc.setQueryData(["profile"], saved); + qc.setQueryData(["profile"], saved) }, - }); + }) } /** @@ -58,7 +58,7 @@ export function buildProfileUpdate( current: Profile | undefined, patch: Partial, fallbackModel: string, - fallbackEffort: string, + fallbackEffort: string ): ProfileUpdate { return { default_model: current?.default_model ?? fallbackModel, @@ -67,8 +67,8 @@ export function buildProfileUpdate( base_branch: current?.base_branch ?? null, branch_prefix: current?.branch_prefix ?? null, auto_fix_ci: current?.auto_fix_ci ?? true, - create_prs: current?.create_prs ?? true, + create_prs: current?.create_prs ?? false, review_draft_prs: current?.review_draft_prs ?? null, ...patch, - }; + } } diff --git a/ui/src/routes/cloud-agents.tsx b/ui/src/routes/cloud-agents.tsx index 7985506c..8fce3241 100644 --- a/ui/src/routes/cloud-agents.tsx +++ b/ui/src/routes/cloud-agents.tsx @@ -1,9 +1,9 @@ -import { Navigate, createFileRoute } from "@tanstack/react-router"; -import { useEffect, useRef, useState } from "react"; +import { Navigate, createFileRoute } from "@tanstack/react-router" +import { useEffect, useRef, useState } from "react" -import type { ModelOption } from "@/lib/api"; -import { AppShell, SettingsRow, SettingsSection } from "@/components/AppShell"; -import { Button } from "@/components/ui/button"; +import type { ModelOption } from "@/lib/api" +import { AppShell, SettingsRow, SettingsSection } from "@/components/AppShell" +import { Button } from "@/components/ui/button" import { Combobox, ComboboxContent, @@ -11,81 +11,89 @@ import { ComboboxInput, ComboboxItem, ComboboxList, -} from "@/components/ui/combobox"; -import { Input } from "@/components/ui/input"; +} from "@/components/ui/combobox" +import { Input } from "@/components/ui/input" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, -} from "@/components/ui/select"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Switch } from "@/components/ui/switch"; -import { buildProfileUpdate, useOptions, useProfile, useRepos, useSaveProfile } from "@/lib/profile"; -import { useSession } from "@/lib/session"; +} from "@/components/ui/select" +import { Skeleton } from "@/components/ui/skeleton" +import { Switch } from "@/components/ui/switch" +import { + buildProfileUpdate, + useOptions, + useProfile, + useRepos, + useSaveProfile, +} from "@/lib/profile" +import { useSession } from "@/lib/session" -export const Route = createFileRoute("/cloud-agents")({ component: CloudAgentsPage }); +export const Route = createFileRoute("/cloud-agents")({ + component: CloudAgentsPage, +}) function CloudAgentsPage() { - const session = useSession(); - const profile = useProfile(); - const options = useOptions(); - const repos = useRepos(); - const save = useSaveProfile(); + const session = useSession() + const profile = useProfile() + const options = useOptions() + const repos = useRepos() + const save = useSaveProfile() - const [modelId, setModelId] = useState(""); - const [effort, setEffort] = useState(""); - const [defaultRepo, setDefaultRepo] = useState(""); - const [baseBranch, setBaseBranch] = useState(""); - const [branchPrefix, setBranchPrefix] = useState(""); - const [error, setError] = useState(null); - const initialized = useRef(false); + const [modelId, setModelId] = useState("") + const [effort, setEffort] = useState("") + const [defaultRepo, setDefaultRepo] = useState("") + const [baseBranch, setBaseBranch] = useState("") + const [branchPrefix, setBranchPrefix] = useState("") + const [error, setError] = useState(null) + const initialized = useRef(false) - const firstModel: ModelOption | undefined = options.data?.models[0]; + const firstModel: ModelOption | undefined = options.data?.models[0] const currentModel: ModelOption | undefined = - options.data?.models.find((m) => m.id === modelId) ?? firstModel; + options.data?.models.find((m) => m.id === modelId) ?? firstModel useEffect(() => { - if (!profile.data || initialized.current) return; + if (!profile.data || initialized.current) return // For users with no saved profile, wait until the options API has loaded // so the model/effort selects can initialise to the first available option. - const hasModel = !!profile.data.default_model || !!firstModel; - if (!hasModel) return; - initialized.current = true; - setModelId(profile.data.default_model ?? firstModel?.id ?? ""); - setEffort(profile.data.reasoning_effort ?? firstModel?.default_effort ?? ""); - setDefaultRepo(profile.data.default_repo ?? ""); - setBaseBranch(profile.data.base_branch ?? ""); - setBranchPrefix(profile.data.branch_prefix ?? ""); - }, [profile.data, firstModel?.id, firstModel?.default_effort, firstModel]); + const hasModel = !!profile.data.default_model || !!firstModel + if (!hasModel) return + initialized.current = true + setModelId(profile.data.default_model ?? firstModel?.id ?? "") + setEffort(profile.data.reasoning_effort ?? firstModel?.default_effort ?? "") + setDefaultRepo(profile.data.default_repo ?? "") + setBaseBranch(profile.data.base_branch ?? "") + setBranchPrefix(profile.data.branch_prefix ?? "") + }, [profile.data, firstModel?.id, firstModel?.default_effort, firstModel]) useEffect(() => { if (currentModel && !currentModel.efforts.includes(effort)) { - setEffort(currentModel.default_effort); + setEffort(currentModel.default_effort) } - }, [currentModel, effort]); + }, [currentModel, effort]) if (session.isLoading) { return (
- ); + ) } - if (!session.data) return ; + if (!session.data) return - const fallbackModel = firstModel?.id ?? ""; - const fallbackEffort = firstModel?.default_effort ?? ""; + const fallbackModel = firstModel?.id ?? "" + const fallbackEffort = firstModel?.default_effort ?? "" const persist = (patch: Parameters[1]) => { - setError(null); + setError(null) save .mutateAsync( - buildProfileUpdate(profile.data, patch, fallbackModel, fallbackEffort), + buildProfileUpdate(profile.data, patch, fallbackModel, fallbackEffort) ) - .catch((e: Error) => setError(e.message)); - }; + .catch((e: Error) => setError(e.message)) + } const persistDefaults = () => { persist({ @@ -94,8 +102,8 @@ function CloudAgentsPage() { default_repo: defaultRepo || null, base_branch: baseBranch || null, branch_prefix: branchPrefix || null, - }); - }; + }) + } return ( - + No matches @@ -205,7 +217,11 @@ function CloudAgentsPage() { } />
-
@@ -227,11 +243,11 @@ function CloudAgentsPage() { } /> persist({ create_prs: v })} /> } @@ -241,5 +257,5 @@ function CloudAgentsPage() { {error &&

{error}

} - ); + ) }