From a8c7af419228f81e4b9dda5d57242abbc2861f14 Mon Sep 17 00:00:00 2001 From: Ramon Nogueira Date: Mon, 22 Jun 2026 15:54:46 -0400 Subject: [PATCH] =?UTF-8?q?test(open-swe):=20add=20Playwright=20E2E=20for?= =?UTF-8?q?=20the=20Slack=20=E2=86=92=20PR=20=E2=86=92=20web=20handoff=20(?= =?UTF-8?q?#1583)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(open-swe): add Playwright E2E for the Slack → PR → web handoff Local, secrets-free end-to-end suite that drives the full happy path through mock Slack/GitHub control panels and the real dashboard UI. Only the LLM and external SaaS HTTP boundaries (GitHub/Slack APIs, OAuth token mint) are faked — the real process_slack_mention, get_agent, deepagents loop, tools, middleware, and dashboard authorization all run under `langgraph dev` with a scripted fake chat model and a local temp-dir sandbox. - full_flow: a Slack mention runs the agent, which implements a change in the sandbox, opens a PR against a fake GitHub remote, and replies with the PR link in the same thread. - dashboard: clicking the bot's real "Open in Web" link loads the built ui/ app (served same-origin); the thread owner can continue the conversation, while a different user sees the same thread read-only (no composer). Wired into Agent CI as a `Playwright E2E` job that runs on pull requests. * fix(open-swe): serve E2E UI assets via explicit route; pin Playwright The dashboard E2E served the built ui/ SPA's /assets via app.mount(StaticFiles), but LangGraph's custom-app loader serves APIRoutes and drops sub-app Mounts, so /assets 404'd under `langgraph dev` in CI — the React app never booted and the composer/transcript never rendered. Serve assets via an explicit route instead. Also pin @playwright/test to the latest (1.61.0) for reproducible runs, and make the owner composer assertion tolerant of either hydration state. * test(open-swe): record Playwright trace + video on every E2E run Capture a replayable trace (DOM snapshots, network, console, source) and a screen recording for every test, not just retries, plus a screenshot on failure. The CI job already uploads playwright-report/ and test-results/, so each run now has a downloadable replay; documented how to open it. --- .github/workflows/ci.yml | 33 +++ tests/e2e/.gitignore | 8 + tests/e2e/README.md | 110 ++++++++ tests/e2e/agent_entrypoint.py | 21 ++ tests/e2e/e2e_env.py | 83 ++++++ tests/e2e/fake_llm.py | 151 +++++++++++ tests/e2e/fakes.py | 153 +++++++++++ tests/e2e/global-setup.ts | 24 ++ tests/e2e/harness.py | 426 ++++++++++++++++++++++++++++++ tests/e2e/langgraph.e2e.json | 11 + tests/e2e/package-lock.json | 76 ++++++ tests/e2e/package.json | 13 + tests/e2e/patches.py | 72 +++++ tests/e2e/playwright.config.ts | 44 +++ tests/e2e/static/github.html | 45 ++++ tests/e2e/static/slack.html | 69 +++++ tests/e2e/tests/dashboard.spec.ts | 70 +++++ tests/e2e/tests/full_flow.spec.ts | 53 ++++ 18 files changed, 1462 insertions(+) create mode 100644 tests/e2e/.gitignore create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/agent_entrypoint.py create mode 100644 tests/e2e/e2e_env.py create mode 100644 tests/e2e/fake_llm.py create mode 100644 tests/e2e/fakes.py create mode 100644 tests/e2e/global-setup.ts create mode 100644 tests/e2e/harness.py create mode 100644 tests/e2e/langgraph.e2e.json create mode 100644 tests/e2e/package-lock.json create mode 100644 tests/e2e/package.json create mode 100644 tests/e2e/patches.py create mode 100644 tests/e2e/playwright.config.ts create mode 100644 tests/e2e/static/github.html create mode 100644 tests/e2e/static/slack.html create mode 100644 tests/e2e/tests/dashboard.spec.ts create mode 100644 tests/e2e/tests/full_flow.spec.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2edb4414..f0d65a66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,3 +46,36 @@ jobs: run: uv sync --locked --extra dev - name: Run unit tests run: make test + + e2e: + name: Playwright E2E + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: oven-sh/setup-bun@v2 + - name: Install Python deps (langgraph dev runtime) + run: uv sync --locked + - name: Install Playwright + Chromium + working-directory: tests/e2e + run: | + npm ci + npx playwright install --with-deps chromium + # Playwright's webServer boots `langgraph dev`; globalSetup builds the real + # ui/ SPA. The fake LLM/GitHub/Slack boundaries need no secrets. + - name: Run E2E + working-directory: tests/e2e + run: npx playwright test + - name: Upload Playwright report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: | + tests/e2e/playwright-report + tests/e2e/test-results + retention-days: 7 diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore new file mode 100644 index 00000000..81cf1074 --- /dev/null +++ b/tests/e2e/.gitignore @@ -0,0 +1,8 @@ +node_modules/ +test-results/ +playwright-report/ +.playwright/ +blob-report/ +.e2e-tmp/ +.langgraph_api/ +__pycache__/ diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..3f581cb4 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,110 @@ +# Playwright E2E — the full Slack → implement → PR → reply flow + +This drives the **whole happy path** through two mock UIs: + +1. A user asks Open SWE to implement something in a **mock Slack** thread. +2. The **real agent** runs (via `langgraph dev`): it implements the change in a + **local temp-dir sandbox**, pushes a branch, and opens a PR on a **fake GitHub**. +3. It posts the PR link back to the **same Slack thread** — visible in the mock UI. + +## What is faked vs. real + +Only the **LLM** and the **external SaaS HTTP boundaries** are faked. All agent +code runs for real. + +| Piece | Real or fake | +|---|---| +| Slack webhook → `process_slack_mention` → run dispatch | **real** (`agent.webapp`) | +| `get_agent`, deepagents loop, tools, middleware, prompt | **real** | +| `open_pull_request`, `slack_thread_reply` tools | **real** | +| Sandbox | **real** `local` provider, rooted in a throwaway temp dir | +| Git remote ("GitHub") | **real git**, a local bare repo the agent clones/pushes | +| The LLM | **fake** — a scripted model (`fake_llm.py`) emitting a fixed tool sequence | +| `api.github.com` REST (PR create) | **fake** (`/fake-gh/...`), state rendered at `/mock/github` | +| `slack.com/api` (post message, etc.) | **fake** (`/fake-slack/...`), thread rendered at `/mock/slack` | +| GitHub App token mint, `api.github.com/user` identity | stubbed (offline) | + +The fake GitHub/Slack stores are the single source of truth the mock UIs render, +so what Playwright asserts on is exactly what the real agent produced. + +## Files + +- `e2e_env.py` — env + constants set before any `agent.*` import (sandbox=local, + fake API URLs, isolated `GIT_CONFIG_GLOBAL`, bot-token-only mode). +- `fake_llm.py` — the scripted `BaseChatModel` (the only faked agent piece). +- `patches.py` — monkeypatches the boundaries (LLM, GitHub/Slack URLs, token mint). +- `agent_entrypoint.py` — langgraph `agent` graph: applies patches, re-exports the + real `traced_agent`. +- `harness.py` — langgraph `http.app`: the real `agent.webapp` plus the fake + GitHub/Slack APIs, the mock UIs, and the control/compose endpoints. +- `fakes.py` — in-memory PR/Slack stores + git seeding of the bare remote. +- `langgraph.e2e.json` — dev-server config pointing at the two entrypoints above. +- `static/{slack,github}.html` — the mock Slack/GitHub UIs (external SaaS we can't + run locally). The dashboard is **not** mocked — it's the real `ui/` app. +- `global-setup.ts` — builds the real `ui/` SPA (once) so the harness can serve it. +- `tests/full_flow.spec.ts` — Slack → implement → PR → reply. +- `tests/dashboard.spec.ts` — the Slack → web handoff (below). + +## Slack → web handoff (dashboard.spec.ts) — the REAL ui/ app + +After the Slack run, the bot posts an "Open in Web" link +(`DASHBOARD_BASE_URL/agents/{thread_id}`). The test clicks that real link, which +loads the **actual built `ui/` React app** — served same-origin from the harness +so the session cookie and `/dashboard/api/*` calls work without CORS. The signed +session cookie is real (minted via `/control/login`), so the per-user +authorization is genuine: + +- **Same user** (session email = the Slack triggerer = thread owner): the real + `AgentThreadView` shows the transcript (incl. the PR link), the `AgentPromptBar` + composer is present, and submitting a follow-up streams a new agent reply into + the same thread. +- **Different user** (any other org login): the same transcript renders, but the + real UI shows **no composer** (`AgentThreadView` gates it on `thread.isOwner`). + +Ownership is by `github_login` / `triggering_user_email` on the thread metadata; +`GET /dashboard/api/threads/{id}` returns `isOwner`, which the real UI uses to +gate the composer. The only extra fake here is the OAuth-token store (an external +credential); the authorization logic itself is real. + +The UI is built by `global-setup.ts` with `VITE_DASHBOARD_API_BASE_URL` pointed at +the harness. It builds once; set `E2E_FORCE_UI_BUILD=1` to rebuild (e.g. after a +UI change or port change). Requires `bun`. + +## Run + +```bash +cd tests/e2e +npm install +npx playwright install chromium +npx playwright test # boots langgraph dev automatically, then runs +``` + +Watch it in human time: + +```bash +SLOW_MO=700 npx playwright test --headed +``` + +## Artifacts (replay a run) + +Every test records a **trace** (DOM-snapshot timeline + network + console + source) +and a **video**; failures also get a screenshot. Locally they land in +`test-results//` and are embedded in `playwright-report/`: + +```bash +npx playwright show-report # browse runs; each has a Trace tab +npx playwright show-trace test-results//trace.zip # open one trace directly +``` + +In CI the `Playwright E2E` job uploads both `playwright-report/` and +`test-results/` as the **playwright-report** artifact on the run. Download it, +then `npx playwright show-report ` (or drag a `trace.zip` onto +) to replay. + +Poke at it by hand (from the repo root): + +```bash +uv run langgraph dev --config tests/e2e/langgraph.e2e.json --port 2024 \ + --no-browser --allow-blocking --no-reload +# open http://127.0.0.1:2024/mock/slack and /mock/github +``` diff --git a/tests/e2e/agent_entrypoint.py b/tests/e2e/agent_entrypoint.py new file mode 100644 index 00000000..1bff7e54 --- /dev/null +++ b/tests/e2e/agent_entrypoint.py @@ -0,0 +1,21 @@ +"""LangGraph graph entrypoint for the E2E dev server. + +Applies the boundary patches (fake LLM + fake GitHub/Slack), then re-exports the +REAL traced agent factory. The langgraph dev config points the ``agent`` graph +here instead of ``agent.server`` so the patches are in effect in the worker. +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import patches # noqa: E402 + +patches.apply() + +from agent.server import traced_agent # noqa: E402 + +__all__ = ["traced_agent"] diff --git a/tests/e2e/e2e_env.py b/tests/e2e/e2e_env.py new file mode 100644 index 00000000..9c658844 --- /dev/null +++ b/tests/e2e/e2e_env.py @@ -0,0 +1,83 @@ +"""Shared environment + constants for the full-flow E2E. + +Imported FIRST by both the agent graph entrypoint and the HTTP harness, before +any ``agent.*`` module — several webapp/auth/slack constants are read into module +globals at import time, so the env must be set beforehand. + +Everything here only configures *boundaries* (which sandbox, which fake API +URLs, where git writes its global config). The agent code itself is unchanged. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +TMP = Path(os.environ.setdefault("E2E_TMP", str(Path(__file__).parent / ".e2e-tmp"))) + +# Demo repo the fake GitHub serves and the agent operates on. +OWNER = "fakeorg" +REPO = "demo" +BASE_BRANCH = "main" +FEATURE_BRANCH = "add-greet" +PR_TITLE = "Add greet() helper" +FEATURE_FILE = "greet.py" + +# Fixed Slack identifiers so the mock UI and assertions are deterministic. +BOT_USER_ID = "U0BOT" +BOT_USERNAME = "open-swe" +DEMO_CHANNEL = "C_DEMO" +HUMAN_USER = "U_HUMAN" + +PORT = os.environ.setdefault("E2E_PORT", "2024") +BASE_URL = os.environ.setdefault("E2E_BASE", f"http://127.0.0.1:{PORT}") + +_GH_DIR = TMP / "github" +_WORK_DIR = TMP / "work" +BARE_REMOTE = _GH_DIR / f"{OWNER}__{REPO}.git" + +_DEFAULTS = { + # Sandbox: real local provider, rooted in a throwaway temp dir. + "SANDBOX_TYPE": "local", + "LOCAL_SANDBOX_ROOT_DIR": str(_WORK_DIR), + # Keep git's --global writes (bot identity) out of the user's ~/.gitconfig. + "GIT_CONFIG_GLOBAL": str(TMP / "gitconfig-global"), + "GIT_CONFIG_SYSTEM": "/dev/null", + # Path the scripted agent clones from (a local bare repo = "fake GitHub"). + "E2E_REMOTE": str(BARE_REMOTE), + # Webhook signing + bot identity. + "GITHUB_WEBHOOK_SECRET": "test-github-secret", + "SLACK_SIGNING_SECRET": "test-slack-secret", + "SLACK_BOT_TOKEN": "xoxb-test-token", + "SLACK_BOT_USER_ID": BOT_USER_ID, + "SLACK_BOT_USERNAME": BOT_USERNAME, + # Slack runs resolve the repo from this when the channel/thread carry none. + "DEFAULT_REPO_OWNER": OWNER, + "DEFAULT_REPO_NAME": REPO, + # Bot-token-only mode: lets Slack runs proceed without a per-user OAuth token. + "LANGSMITH_API_KEY_PROD": "test-bot-mode", + # SDK client target (same dev server). + "LANGGRAPH_URL": BASE_URL, + # Dashboard: the "Open in Web" link target + session-cookie signing. Use + # 127.0.0.1 (not localhost) so the local-dev LLM-key check stays skipped. + "DASHBOARD_BASE_URL": BASE_URL, + "DASHBOARD_API_BASE_URL": BASE_URL, + "DASHBOARD_ALLOWED_ORIGINS": BASE_URL, + "DASHBOARD_JWT_SECRET": "test-dashboard-jwt-secret", +} + +# The Slack run's triggering user (fake users.info returns this email) — the +# thread owner. A session with this email may continue the thread on the web. +SAME_USER = {"login": "dev-user", "email": "dev@example.com"} +# Any other logged-in user: can view the thread read-only, cannot post. +OTHER_USER = {"login": "someone-else", "email": "someone@example.com"} + +for _k, _v in _DEFAULTS.items(): + os.environ.setdefault(_k, _v) + +for _d in (TMP, _GH_DIR, _WORK_DIR): + _d.mkdir(parents=True, exist_ok=True) + +FAKE_GITHUB_API = f"{BASE_URL}/fake-gh" +FAKE_SLACK_API = f"{BASE_URL}/fake-slack" diff --git a/tests/e2e/fake_llm.py b/tests/e2e/fake_llm.py new file mode 100644 index 00000000..2552a996 --- /dev/null +++ b/tests/e2e/fake_llm.py @@ -0,0 +1,151 @@ +"""A scripted fake chat model — the ONLY faked piece of the agent. + +It drives the real deepagents loop with a fixed sequence of tool calls that +implement a tiny feature, push a branch to the fake-GitHub remote, open a PR via +the real ``open_pull_request`` tool, and post the result back with the real +``slack_thread_reply`` tool. The final Slack step reads the actual PR URL out of +the preceding tool result, exactly as a real model would. +""" + +from __future__ import annotations + +import re +from typing import Any + +from e2e_env import ( + BASE_BRANCH, + FEATURE_BRANCH, + FEATURE_FILE, + OWNER, + PR_TITLE, + REPO, +) +from langchain_core.callbacks import CallbackManagerForLLMRun +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult + +# One shell command that does the whole git workflow. Each execute() runs in a +# fresh shell rooted at the sandbox dir, so the clone+commit+push is bundled. +_IMPLEMENT_SCRIPT = f""" +set -e +rm -rf repo +git clone "$E2E_REMOTE" repo +cd repo +git config user.email "dev@example.com" +git config user.name "Dev User" +git checkout -b {FEATURE_BRANCH} +cat > {FEATURE_FILE} <<'EOF' +def greet(name): + return f"Hello, {{name}}!" +EOF +git add -A +git commit -m "{PR_TITLE}" +git push origin {FEATURE_BRANCH} +echo PUSHED_OK +""".strip() + + +def _pr_url_from_messages(messages: list[BaseMessage]) -> str | None: + for msg in reversed(messages): + if isinstance(msg, ToolMessage): + text = msg.content if isinstance(msg.content, str) else str(msg.content) + match = re.search(r"https?://[^\s\"']+/pull/\d+", text) + if match: + return match.group(0) + return None + + +def _step_implement(_messages: list[BaseMessage]) -> AIMessage: + return AIMessage( + content="Setting up the repo and implementing the change.", + tool_calls=[{"name": "execute", "args": {"command": _IMPLEMENT_SCRIPT}, "id": "call-impl"}], + ) + + +def _step_open_pr(_messages: list[BaseMessage]) -> AIMessage: + return AIMessage( + content="Opening a pull request.", + tool_calls=[ + { + "name": "open_pull_request", + "args": { + "owner": OWNER, + "repo": REPO, + "head": FEATURE_BRANCH, + "base": BASE_BRANCH, + "title": PR_TITLE, + "body": "Adds a `greet()` helper as requested.", + "draft": True, + }, + "id": "call-pr", + } + ], + ) + + +def _step_reply(messages: list[BaseMessage]) -> AIMessage: + url = _pr_url_from_messages(messages) or "(PR url unavailable)" + text = ( + f"✅ Done! I implemented the change and opened a PR: <{url}|{PR_TITLE}>\n\n" + f"• Added `{FEATURE_FILE}` with a `greet()` helper.\n" + "Let me know if you'd like any changes." + ) + return AIMessage( + content="Replying in the Slack thread with the PR link.", + tool_calls=[{"name": "slack_thread_reply", "args": {"message": text}, "id": "call-reply"}], + ) + + +FOLLOW_UP_REPLY = "Thanks! The PR is ready for review — anything else you'd like changed?" + + +def _step_followup(_messages: list[BaseMessage]) -> AIMessage: + # A web/Slack follow-up after the PR exists: a plain reply, no new PR. Its + # content lands in the thread transcript the dashboard renders. + return AIMessage(content=FOLLOW_UP_REPLY) + + +def build_script() -> list[Any]: + return [_step_implement, _step_open_pr, _step_reply] + + +def build_followup_script() -> list[Any]: + return [_step_followup] + + +class FakeScriptedChatModel(BaseChatModel): + """Returns the next scripted AIMessage based on how far the loop has run.""" + + script: list[Any] = [] + + @property + def _llm_type(self) -> str: + return "fake-scripted" + + def bind_tools(self, tools: Any, **kwargs: Any) -> FakeScriptedChatModel: # noqa: ARG002 + return self + + def _generate( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, # noqa: ARG002 + run_manager: CallbackManagerForLLMRun | None = None, # noqa: ARG002 + **kwargs: Any, + ) -> ChatResult: + # First human turn implements + opens the PR; later turns (a web/Slack + # follow-up on the same thread) just reply. + human_turns = sum(1 for m in messages if isinstance(m, HumanMessage)) + script = build_script() if human_turns <= 1 else build_followup_script() + + # Step within the *current* turn: AIMessages since the last human turn. + # (Counting the whole thread would short-circuit reused/multi-turn threads.) + last_human = max( + (i for i, m in enumerate(messages) if isinstance(m, HumanMessage)), default=-1 + ) + step = sum(1 for m in messages[last_human + 1 :] if isinstance(m, AIMessage)) + if step < len(script): + message = script[step](messages) + else: + message = AIMessage(content="All set — the PR is open and linked in the thread.") + return ChatResult(generations=[ChatGeneration(message=message)]) diff --git a/tests/e2e/fakes.py b/tests/e2e/fakes.py new file mode 100644 index 00000000..d1e1a85b --- /dev/null +++ b/tests/e2e/fakes.py @@ -0,0 +1,153 @@ +"""In-memory state + git plumbing behind the fake GitHub and fake Slack. + +These stores are the single source of truth that both the real agent code +(via the faked HTTP endpoints) and the mock UIs read from — so what Playwright +sees in the UI is exactly what the agent produced. +""" + +from __future__ import annotations + +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any + +from e2e_env import BARE_REMOTE, BASE_BRANCH, OWNER, REPO + +# --- Slack ----------------------------------------------------------------- +# (channel, thread_ts) -> list of {user, text, ts, blocks, is_bot} +SLACK_MESSAGES: dict[tuple[str, str], list[dict[str, Any]]] = {} +_slack_seq = [1] + + +def next_slack_ts() -> str: + _slack_seq[0] += 1 + return f"1700000000.{_slack_seq[0]:06d}" + + +_thread_seq = [0] + + +def new_thread_ts() -> str: + """A globally-unique thread ts so every send maps to a fresh LangGraph thread + (the in-mem store persists across restarts, so reused ids would carry state). + Not reset by reset(), so back-to-back tests never collide.""" + _thread_seq[0] += 1 + return f"{int(time.time())}.{_thread_seq[0]:06d}" + + +def add_slack_message( + channel: str, thread_ts: str, *, user: str, text: str, blocks: Any = None, is_bot: bool = False +) -> str: + ts = next_slack_ts() + SLACK_MESSAGES.setdefault((channel, thread_ts), []).append( + { + "user": user, + "text": text, + "ts": ts, + "thread_ts": thread_ts, + "blocks": blocks, + "is_bot": is_bot, + } + ) + return ts + + +def slack_thread(channel: str, thread_ts: str) -> list[dict[str, Any]]: + return SLACK_MESSAGES.get((channel, thread_ts), []) + + +# --- GitHub ---------------------------------------------------------------- +PULLS: list[dict[str, Any]] = [] +_pr_seq = [0] + + +def _git(*args: str, cwd: Path | None = None) -> str: + result = subprocess.run( + ["git", *args], + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def seed_bare_remote() -> None: + """Create a fresh bare repo (the fake GitHub remote) with one commit on main.""" + if BARE_REMOTE.exists(): + shutil.rmtree(BARE_REMOTE) + seed_work = BARE_REMOTE.parent / f"seed-{OWNER}-{REPO}" + if seed_work.exists(): + shutil.rmtree(seed_work) + + seed_work.mkdir(parents=True) + ident = ["-c", "user.email=seed@example.com", "-c", "user.name=Seed"] + _git("init", "-b", BASE_BRANCH, str(seed_work)) + (seed_work / "README.md").write_text("# demo\n\nA tiny demo repo.\n") + _git("add", "-A", cwd=seed_work) + _git(*ident, "commit", "-m", "Initial commit", cwd=seed_work) + _git("init", "--bare", "-b", BASE_BRANCH, str(BARE_REMOTE)) + _git("remote", "add", "origin", str(BARE_REMOTE), cwd=seed_work) + _git("push", "origin", BASE_BRANCH, cwd=seed_work) + shutil.rmtree(seed_work) + + +def _diff_files(base: str, head: str) -> list[dict[str, Any]]: + """Compute changed files for a PR from the pushed branch in the bare remote.""" + try: + out = _git("--git-dir", str(BARE_REMOTE), "diff", "--numstat", base, head) + except subprocess.CalledProcessError: + return [] + files = [] + for line in out.splitlines(): + parts = line.split("\t") + if len(parts) == 3: + adds, dels, name = parts + files.append( + { + "filename": name, + "additions": int(adds) if adds.isdigit() else 0, + "deletions": int(dels) if dels.isdigit() else 0, + } + ) + return files + + +def create_pull( + owner: str, repo: str, *, head: str, base: str, title: str, body: str, draft: bool +) -> dict[str, Any]: + _pr_seq[0] += 1 + number = _pr_seq[0] + files = _diff_files(base, head) + pr = { + "number": number, + "owner": owner, + "repo": repo, + "head": head, + "base": base, + "title": title, + "body": body, + "draft": draft, + "state": "open", + "merged": False, + "author": "open-swe[bot]", + "files": files, + "additions": sum(f["additions"] for f in files), + "deletions": sum(f["deletions"] for f in files), + } + PULLS.append(pr) + return pr + + +def find_pull(number: int) -> dict[str, Any] | None: + return next((p for p in PULLS if p["number"] == number), None) + + +def reset() -> None: + SLACK_MESSAGES.clear() + PULLS.clear() + _pr_seq[0] = 0 + _slack_seq[0] = 1 + seed_bare_remote() diff --git a/tests/e2e/global-setup.ts b/tests/e2e/global-setup.ts new file mode 100644 index 00000000..bd4ed3d8 --- /dev/null +++ b/tests/e2e/global-setup.ts @@ -0,0 +1,24 @@ +import { execSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; + +// Build the real ui/ SPA once so the harness can serve it same-origin. The API +// base is baked in at build time, so it must match the harness port. Set +// E2E_FORCE_UI_BUILD=1 to rebuild (e.g. after changing the port or the UI). +export default function globalSetup() { + const repoRoot = resolve(__dirname, "..", ".."); + const ui = resolve(repoRoot, "ui"); + const shell = resolve(ui, ".output", "public", "_shell.html"); + const port = process.env.E2E_PORT ?? "2024"; + + if (existsSync(shell) && !process.env.E2E_FORCE_UI_BUILD) return; + + if (!existsSync(resolve(ui, "node_modules"))) { + execSync("bun install", { cwd: ui, stdio: "inherit" }); + } + execSync("bun run build", { + cwd: ui, + stdio: "inherit", + env: { ...process.env, VITE_DASHBOARD_API_BASE_URL: `http://127.0.0.1:${port}` }, + }); +} diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py new file mode 100644 index 00000000..e192c75d --- /dev/null +++ b/tests/e2e/harness.py @@ -0,0 +1,426 @@ +"""HTTP app for the full-flow E2E (served as langgraph dev's http.app). + +Mounts, on top of the REAL ``agent.webapp`` app: + - fake GitHub REST API (/fake-gh/...) the real open_pull_request hits this + - fake Slack API (/fake-slack/...) the real slack code hits this + - mock UIs (/mock/slack, /mock/github) what the user/Playwright sees + - control + compose (/control/*, /mock/slack/send) the test driver + +Nothing here touches agent logic — it only stands in for the SaaS boundaries +and renders their state back as a user-facing UI. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import e2e_env # noqa: E402 +import patches # noqa: E402 + +patches.apply() + +import fakes # noqa: E402 +import httpx # noqa: E402 +from e2e_env import ( # noqa: E402 + BASE_URL, + BOT_USER_ID, + DEMO_CHANNEL, + HUMAN_USER, + REPO_ROOT, +) +from fastapi import HTTPException, Request # noqa: E402 +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse # noqa: E402 + +from agent.dashboard.oauth import COOKIE_NAME, issue_session # noqa: E402 +from agent.webapp import app, generate_thread_id_from_slack_thread # noqa: E402 + +GITHUB_WEBHOOK_SECRET = os.environ["GITHUB_WEBHOOK_SECRET"] +SLACK_SIGNING_SECRET = os.environ["SLACK_SIGNING_SECRET"] +STATIC_DIR = Path(__file__).parent / "static" + +CURRENT_THREAD: dict[str, str | None] = {"channel": DEMO_CHANNEL, "thread_ts": None} + +fakes.seed_bare_remote() + + +# --- control + Slack compose (the test driver) ----------------------------- +@app.post("/control/reset") +async def control_reset() -> JSONResponse: + fakes.reset() + CURRENT_THREAD["thread_ts"] = None + return JSONResponse({"ok": True}) + + +@app.get("/control/state") +async def control_state() -> JSONResponse: + return JSONResponse( + {"channel": CURRENT_THREAD["channel"], "thread_ts": CURRENT_THREAD["thread_ts"]} + ) + + +@app.post("/mock/slack/send") +async def slack_send(request: Request) -> JSONResponse: + """Simulate a user posting in Slack: store the message, then deliver the + signed Events-API webhook to the real /webhooks/slack route.""" + form = await request.json() + text = str(form.get("text", "")) + mention_bot = bool(form.get("mention_bot", True)) + channel = DEMO_CHANNEL + + ts = fakes.new_thread_ts() + CURRENT_THREAD["thread_ts"] = ts + fakes.add_slack_message(channel, ts, user=HUMAN_USER, text=text, is_bot=False) + + payload = { + "type": "event_callback", + "event_id": f"Ev{ts}", + "authorizations": [{"user_id": BOT_USER_ID}], + "event": { + "type": "app_mention" if mention_bot else "message", + "channel": channel, + "user": HUMAN_USER, + "text": text, + "ts": ts, + "thread_ts": ts, + }, + } + raw = json.dumps(payload).encode() + req_ts = str(int(time.time())) + base = f"v0:{req_ts}:{raw.decode()}".encode() + sig = "v0=" + hmac.new(SLACK_SIGNING_SECRET.encode(), base, hashlib.sha256).hexdigest() + + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://harness") as client: + resp = await client.post( + "/webhooks/slack", + content=raw, + headers={ + "X-Slack-Signature": sig, + "X-Slack-Request-Timestamp": req_ts, + "Content-Type": "application/json", + }, + ) + return JSONResponse( + { + "thread_ts": ts, + "thread_id": generate_thread_id_from_slack_thread(channel, ts), + "webhook_status": resp.status_code, + "webhook": resp.json(), + } + ) + + +@app.post("/control/login") +async def control_login(request: Request) -> JSONResponse: + """Simulate a signed-in dashboard user by minting the real session cookie.""" + form = await request.json() + login = str(form.get("login", "dev-user")) + email = str(form.get("email", "dev@example.com")) + token = issue_session(login=login, email=email, avatar_url=None) + resp = JSONResponse({"ok": True, "login": login, "email": email}) + resp.set_cookie(COOKIE_NAME, token, httponly=True, samesite="lax", secure=False, path="/") + return resp + + +@app.post("/control/logout") +async def control_logout() -> JSONResponse: + resp = JSONResponse({"ok": True}) + resp.delete_cookie(COOKIE_NAME, path="/") + return resp + + +# --- serve the REAL built ui/ SPA, same-origin so the session cookie works ---- +# The "Open in Web" link (DASHBOARD_BASE_URL/agents/{id}) lands on the real app; +# it calls /dashboard/api/* (same origin) and streams via the dashboard proxy. +UI_PUBLIC = REPO_ROOT / "ui" / ".output" / "public" +_ASSETS_ROOT = (UI_PUBLIC / "assets").resolve() + + +def _ui_file(name: str) -> FileResponse: + path = UI_PUBLIC / name + if not path.is_file(): + raise HTTPException(404, f"{name} not built — run `bun run build` in ui/") + return FileResponse(path) + + +@app.get("/assets/{asset_path:path}") +async def ui_asset(asset_path: str) -> FileResponse: + # Explicit route, not app.mount(StaticFiles): LangGraph's custom-app loader + # serves APIRoutes but drops sub-app Mounts, so a mount 404s under it. + target = (_ASSETS_ROOT / asset_path).resolve() + if not str(target).startswith(str(_ASSETS_ROOT)) or not target.is_file(): + raise HTTPException(404, "asset not found") + return FileResponse(target) + + +@app.get("/_shell.html", response_class=HTMLResponse) +async def ui_shell() -> FileResponse: + return _ui_file("_shell.html") + + +@app.get("/manifest.webmanifest") +async def ui_manifest() -> FileResponse: + return _ui_file("manifest.webmanifest") + + +@app.get("/favicon.png") +async def ui_favicon() -> FileResponse: + return _ui_file("favicon.png") + + +@app.get("/apple-touch-icon.png") +async def ui_apple_icon() -> FileResponse: + return _ui_file("apple-touch-icon.png") + + +@app.get("/logo-mark.png") +async def ui_logo_mark() -> FileResponse: + return _ui_file("logo-mark.png") + + +# Client routes used by the handoff tests: serve the SPA shell; the client +# router boots at the current URL. Kept explicit (no catch-all) so LangGraph's +# own root routes — which the dashboard proxy calls server-side — are untouched. +@app.get("/agents/{thread_id}", response_class=HTMLResponse) +async def ui_agents_thread(thread_id: str) -> FileResponse: # noqa: ARG001 + return _ui_file("_shell.html") + + +@app.get("/login", response_class=HTMLResponse) +async def ui_login() -> FileResponse: + return _ui_file("_shell.html") + + +@app.get("/mock/slack/messages") +async def slack_messages() -> JSONResponse: + thread = CURRENT_THREAD["thread_ts"] + msgs = fakes.slack_thread(CURRENT_THREAD["channel"], thread) if thread else [] + return JSONResponse( + [{"user": m["user"], "text": m["text"], "is_bot": m["is_bot"], "ts": m["ts"]} for m in msgs] + ) + + +# --- mock UIs -------------------------------------------------------------- +@app.get("/mock/slack", response_class=HTMLResponse) +async def mock_slack_page() -> str: + return (STATIC_DIR / "slack.html").read_text() + + +@app.get("/mock/github", response_class=HTMLResponse) +async def mock_github_page() -> str: + return (STATIC_DIR / "github.html").read_text() + + +def _pr_html_url(pr: dict[str, Any]) -> str: + return f"{BASE_URL}/mock/github/{pr['owner']}/{pr['repo']}/pull/{pr['number']}" + + +@app.get("/mock/github/data") +async def mock_github_data() -> JSONResponse: + return JSONResponse( + [ + { + "number": p["number"], + "title": p["title"], + "head": p["head"], + "base": p["base"], + "state": p["state"], + "draft": p["draft"], + "author": p["author"], + "body": p["body"], + "files": p["files"], + "url": _pr_html_url(p), + } + for p in fakes.PULLS + ] + ) + + +@app.get("/mock/github/{owner}/{repo}/pull/{number}", response_class=HTMLResponse) +async def mock_github_pr(owner: str, repo: str, number: int) -> HTMLResponse: # noqa: ARG001 + pr = fakes.find_pull(number) + if pr is None: + return HTMLResponse(f"

PR #{number} not found

", status_code=404) + files = "".join( + f'
  • {f["filename"]} ' + f"+{f['additions']} −{f['deletions']}
    • " + for f in pr["files"] + ) + draft = " (draft)" if pr["draft"] else "" + return HTMLResponse( + f""" + PR #{pr["number"]} — {pr["owner"]}/{pr["repo"]} + +

    ← all pull requests

    +

    {pr["title"]}{draft}

    +

    #{pr["number"]} · {pr["state"]} · + {pr["head"]} → {pr["base"]} · + by {pr["author"]}

    +

    Description

    {pr["body"]}
    +

    Files changed ({len(pr["files"])})

    +
      {files}
    + """ + ) + + +# --- fake GitHub REST API (open_pull_request hits this) -------------------- +def _gh_pr_json(pr: dict[str, Any]) -> dict[str, Any]: + return { + "number": pr["number"], + "html_url": _pr_html_url(pr), + "state": pr["state"], + "draft": pr["draft"], + "merged": pr["merged"], + "title": pr["title"], + "body": pr["body"], + "user": {"login": pr["author"]}, + "head": {"ref": pr["head"]}, + "base": {"ref": pr["base"]}, + "additions": pr["additions"], + "deletions": pr["deletions"], + "changed_files": len(pr["files"]), + } + + +@app.get("/fake-gh/repos/{owner}/{repo}") +async def gh_get_repo(owner: str, repo: str) -> JSONResponse: + return JSONResponse({"full_name": f"{owner}/{repo}", "private": False}) + + +@app.get("/fake-gh/repos/{owner}/{repo}/pulls") +async def gh_list_pulls(owner: str, repo: str) -> JSONResponse: # noqa: ARG001 + return JSONResponse([]) + + +@app.post("/fake-gh/repos/{owner}/{repo}/pulls") +async def gh_create_pull(owner: str, repo: str, request: Request) -> JSONResponse: + body = await request.json() + pr = fakes.create_pull( + owner, + repo, + head=body.get("head", ""), + base=body.get("base", "main"), + title=body.get("title", ""), + body=body.get("body", ""), + draft=bool(body.get("draft", True)), + ) + return JSONResponse(_gh_pr_json(pr), status_code=201) + + +@app.get("/fake-gh/repos/{owner}/{repo}/pulls/{number}") +async def gh_get_pull(owner: str, repo: str, number: int) -> JSONResponse: # noqa: ARG001 + pr = fakes.find_pull(number) + if pr is None: + return JSONResponse({"message": "Not Found"}, status_code=404) + return JSONResponse(_gh_pr_json(pr)) + + +# --- fake Slack API (real slack code hits this) ---------------------------- +def _ok(extra: dict[str, Any] | None = None) -> JSONResponse: + return JSONResponse({"ok": True, **(extra or {})}) + + +@app.post("/fake-slack/chat.postMessage") +async def slack_post_message(request: Request) -> JSONResponse: + body = await request.json() + ts = fakes.add_slack_message( + body.get("channel", ""), + body.get("thread_ts", ""), + user=BOT_USER_ID, + text=body.get("text", ""), + blocks=body.get("blocks"), + is_bot=True, + ) + return _ok({"ts": ts, "message": {"ts": ts}}) + + +@app.post("/fake-slack/chat.postEphemeral") +async def slack_post_ephemeral(request: Request) -> JSONResponse: + await request.body() + return _ok({"message_ts": fakes.next_slack_ts()}) + + +@app.post("/fake-slack/assistant.threads.setStatus") +async def slack_set_status(request: Request) -> JSONResponse: + await request.body() + return _ok() + + +@app.post("/fake-slack/reactions.add") +async def slack_reactions_add(request: Request) -> JSONResponse: + await request.body() + return _ok() + + +@app.get("/fake-slack/users.info") +async def slack_users_info(user: str = "") -> JSONResponse: + return _ok( + { + "user": { + "id": user, + "name": "devuser", + "real_name": "Dev User", + "profile": { + "email": "dev@example.com", + "display_name": "Dev User", + "real_name": "Dev User", + }, + } + } + ) + + +@app.get("/fake-slack/conversations.info") +async def slack_conversations_info(channel: str = "") -> JSONResponse: + return _ok( + { + "channel": { + "id": channel, + "name": "demo", + "topic": {"value": ""}, + "purpose": {"value": ""}, + } + } + ) + + +@app.get("/fake-slack/conversations.replies") +async def slack_conversations_replies(channel: str = "", ts: str = "") -> JSONResponse: + msgs = fakes.slack_thread(channel, ts) + return _ok( + { + "messages": [ + { + "type": "message", + "user": m["user"], + "text": m["text"], + "ts": m["ts"], + "thread_ts": m["thread_ts"], + } + for m in msgs + ] + } + ) + + +@app.get("/fake-slack/conversations.history") +async def slack_conversations_history(channel: str = "") -> JSONResponse: # noqa: ARG001 + return _ok({"messages": []}) + + +@app.get("/fake-slack/chat.getPermalink") +async def slack_get_permalink(channel: str = "", message_ts: str = "") -> JSONResponse: # noqa: ARG001 + return _ok({"permalink": f"{BASE_URL}/mock/slack"}) + + +# Quietly reference imports used only for env side effects. +_ = (e2e_env, HUMAN_USER) diff --git a/tests/e2e/langgraph.e2e.json b/tests/e2e/langgraph.e2e.json new file mode 100644 index 00000000..91ea6e34 --- /dev/null +++ b/tests/e2e/langgraph.e2e.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://langgra.ph/schema.json", + "python_version": "3.12", + "dependencies": ["."], + "graphs": { + "agent": "./tests/e2e/agent_entrypoint.py:traced_agent" + }, + "http": { + "app": "./tests/e2e/harness.py:app" + } +} diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json new file mode 100644 index 00000000..24a2c9bf --- /dev/null +++ b/tests/e2e/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "open-swe-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "open-swe-e2e", + "devDependencies": { + "@playwright/test": "1.61.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.0.tgz", + "integrity": "sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", + "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/tests/e2e/package.json b/tests/e2e/package.json new file mode 100644 index 00000000..1cd31b38 --- /dev/null +++ b/tests/e2e/package.json @@ -0,0 +1,13 @@ +{ + "name": "open-swe-e2e", + "private": true, + "description": "Playwright E2E tests driving mock GitHub/Slack control panels against the real webhook routes.", + "scripts": { + "test": "playwright test", + "test:headed": "playwright test --headed", + "report": "playwright show-report" + }, + "devDependencies": { + "@playwright/test": "1.61.0" + } +} diff --git a/tests/e2e/patches.py b/tests/e2e/patches.py new file mode 100644 index 00000000..eb6332ad --- /dev/null +++ b/tests/e2e/patches.py @@ -0,0 +1,72 @@ +"""Boundary monkeypatches: fake the LLM and the external SaaS endpoints. + +Everything patched here is an *external boundary*, not agent logic: + - the LLM (model factory) -> scripted fake + - GitHub App token mint + GitHub REST base URL -> dummy token + fake GitHub + - Slack API base URL -> fake Slack + - the api.github.com/user identity lookup -> offline (falls back to config) + +Applied at import of both the graph entrypoint and the HTTP harness (same dev +process), so it runs before the first run regardless of import order. Idempotent. +""" + +from __future__ import annotations + +import e2e_env # noqa: F401 (sets env before any agent import) + +_applied = False + + +def apply() -> None: + global _applied + if _applied: + return + + import importlib + + from agent import server + from agent.utils import auth, authorship + from agent.utils import slack as slack_utils + + # NB: ``from agent.tools import open_pull_request`` returns the re-exported + # *function* (the tools package __init__ shadows the submodule), so patch the + # actual module object by name instead. + opr = importlib.import_module("agent.tools.open_pull_request") + + from e2e_env import FAKE_GITHUB_API, FAKE_SLACK_API + from fake_llm import FakeScriptedChatModel, build_script + + def _fake_make_model(model_id: str, **kwargs: object): # noqa: ARG001 + return FakeScriptedChatModel(script=build_script()) + + server.make_model = _fake_make_model + + async def _dummy_install_token_with_expiry() -> tuple[str, str | None]: + return "dummy-installation-token", None + + async def _dummy_install_token() -> str: + return "dummy-installation-token" + + auth.get_github_app_installation_token_with_expiry = _dummy_install_token_with_expiry + opr.get_github_app_installation_token = _dummy_install_token + + # Point the real PR/Slack code at the in-process fakes. + opr.GITHUB_API = FAKE_GITHUB_API + slack_utils.SLACK_API_BASE_URL = FAKE_SLACK_API + + # Keep the triggering-user identity lookup offline; the real fallback to + # config-derived identity (Slack name/email) still runs. + authorship._identity_from_github_token = lambda _token: None # noqa: SLF001 + + # OAuth-token store is an external credential boundary. Stub it so a web + # follow-up (dashboard run.start) and PR-as-user resolution have a token; + # the real ownership/authorization checks still run. + from agent.dashboard import profiles, thread_api + + async def _dummy_user_token(login: str, **_kwargs: object) -> str: # noqa: ARG001 + return "dummy-user-oauth-token" + + profiles.get_valid_access_token = _dummy_user_token + thread_api.get_valid_access_token = _dummy_user_token + + _applied = True diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts new file mode 100644 index 00000000..462c7d64 --- /dev/null +++ b/tests/e2e/playwright.config.ts @@ -0,0 +1,44 @@ +import { defineConfig, devices } from "@playwright/test"; +import { resolve } from "node:path"; + +const repoRoot = resolve(__dirname, "..", ".."); +const PORT = Number(process.env.E2E_PORT ?? 2024); +const baseURL = `http://127.0.0.1:${PORT}`; + +export default defineConfig({ + testDir: "./tests", + globalSetup: "./global-setup.ts", + fullyParallel: false, + workers: 1, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + timeout: 90_000, + expect: { timeout: 60_000 }, + reporter: [["list"], ["html", { open: "never" }]], + use: { + baseURL, + // Always capture the replayable artifacts: a trace (DOM snapshots, network, + // console, source — open with `npx playwright show-trace`) and a screen + // recording, plus a screenshot on failure. The CI job uploads them. + trace: "on", + video: "on", + screenshot: "only-on-failure", + // The built UI ships a PWA service worker; block it so tests never hit a + // stale cache and always see live API responses. + serviceWorkers: "block", + // SLOW_MO=700 npx playwright test --headed → watch it run in human time. + launchOptions: { slowMo: Number(process.env.SLOW_MO ?? 0) }, + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], + webServer: { + // Real langgraph dev: real agent graph + real webhook routes + the harness + // http app (fake GitHub/Slack + mock UIs). Only the LLM is faked. + command: + "uv run langgraph dev --config tests/e2e/langgraph.e2e.json " + + `--port ${PORT} --no-browser --allow-blocking --no-reload`, + cwd: repoRoot, + url: `${baseURL}/mock/github/data`, + reuseExistingServer: !process.env.CI, + timeout: 180_000, + }, +}); diff --git a/tests/e2e/static/github.html b/tests/e2e/static/github.html new file mode 100644 index 00000000..0bf9ed34 --- /dev/null +++ b/tests/e2e/static/github.html @@ -0,0 +1,45 @@ + + + + + + Mock GitHub — pull requests + + + +

    fakeorg/demo — Pull requests (mock)

    +
    + + + + diff --git a/tests/e2e/static/slack.html b/tests/e2e/static/slack.html new file mode 100644 index 00000000..5743a30a --- /dev/null +++ b/tests/e2e/static/slack.html @@ -0,0 +1,69 @@ + + + + + + Mock Slack — #demo + + + +

    #demo — Slack (mock)

    +
    + + + +
    + +
    + + + + diff --git a/tests/e2e/tests/dashboard.spec.ts b/tests/e2e/tests/dashboard.spec.ts new file mode 100644 index 00000000..b478458b --- /dev/null +++ b/tests/e2e/tests/dashboard.spec.ts @@ -0,0 +1,70 @@ +import { test, expect, type Page } from "@playwright/test"; + +// Drives the REAL built ui/ app (served same-origin from the harness) for the +// Slack → web handoff. Only the LLM/GitHub/Slack/token boundaries are faked. +const SAME_USER = { login: "dev-user", email: "dev@example.com" }; +const OTHER_USER = { login: "someone-else", email: "someone@example.com" }; + +async function loginAs(page: Page, user: { login: string; email: string }) { + const res = await page.request.post("/control/login", { data: user }); + expect(res.ok()).toBeTruthy(); +} + +// Run the Slack flow so a thread + PR exist, then click the bot's real +// "Open in Web" link, landing on the actual dashboard app. +async function openThreadViaSlackLink(page: Page) { + await page.goto("/mock/slack"); + await page.locator("#reset").click(); + await expect(page.locator("#thread")).toContainText("No messages yet"); + await page.locator("#text").fill("<@U0BOT> please add a greet() helper and open a PR"); + await page.locator("#send").click(); + await expect(page.locator(".msg.bot").filter({ hasText: "Add greet() helper" })).toBeVisible(); + + const webLink = page.locator('.msg.bot a[href*="/agents/"]').first(); + await expect(webLink).toBeVisible(); + await webLink.click(); + await expect(page).toHaveURL(/\/agents\//); +} + +// The SDK hydrates an idle thread's transcript from getState on load, which can +// briefly lag; a reload re-fetches it. Retry until the PR link renders. +async function expectTranscriptVisible(page: Page) { + await expect(async () => { + await page.reload(); + await expect( + page.getByRole("link", { name: "Add greet() helper" }).first(), + ).toBeVisible({ timeout: 8000 }); + }).toPass({ timeout: 60000 }); +} + +test.describe("Slack → web handoff (real dashboard UI)", () => { + test("the SAME user continues the conversation in the web app", async ({ page }) => { + await loginAs(page, SAME_USER); + await openThreadViaSlackLink(page); + + // The owner sees the composer (either the follow-up bar once the transcript + // hydrates, or the empty-state bar before it — both mean they can type). + const composer = page.getByPlaceholder(/Add a follow up|Send the first message/); + await expect(composer).toBeVisible(); + + // Continue from the web — a new agent reply streams into the same thread. + await composer.fill("Looks good — can you also add a docstring?"); + await composer.press("Enter"); + await expect(page.getByText(/anything else you'd like changed/)).toBeVisible(); + + // The transcript that started in Slack is here too (incl. the PR link). + await expect(page.getByRole("link", { name: "Add greet() helper" }).first()).toBeVisible(); + }); + + test("a DIFFERENT user sees the thread read-only (no composer)", async ({ page }) => { + await loginAs(page, OTHER_USER); + await openThreadViaSlackLink(page); + + // The same thread + transcript is visible… + await expectTranscriptVisible(page); + // …but a non-owner gets no composer. + await expect(page.getByPlaceholder("Add a follow up")).toHaveCount(0); + await expect(page.getByPlaceholder("Send the first message")).toHaveCount(0); + await expect(page.getByLabel("Send message")).toHaveCount(0); + }); +}); diff --git a/tests/e2e/tests/full_flow.spec.ts b/tests/e2e/tests/full_flow.spec.ts new file mode 100644 index 00000000..f3602eaf --- /dev/null +++ b/tests/e2e/tests/full_flow.spec.ts @@ -0,0 +1,53 @@ +import { test, expect } from "@playwright/test"; + +// Full happy path, driven entirely through the mock Slack + GitHub UIs: +// user asks in Slack -> real agent implements in a local sandbox -> opens a PR +// on the fake GitHub -> replies with the PR link in the SAME Slack thread. +// Only the LLM is faked; all agent code runs for real via langgraph dev. +test.describe("Open SWE full flow", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/mock/slack"); + await page.locator("#reset").click(); + await expect(page.locator("#thread")).toContainText("No messages yet"); + }); + + test("Slack request → implements → opens PR → links it back in the thread", async ({ page }) => { + await page.locator("#text").fill("<@U0BOT> please add a greet() helper and open a PR"); + await page.locator("#send").click(); + + // The user's message lands in the thread. + await expect(page.locator(".msg").filter({ hasText: "add a greet() helper" })).toBeVisible(); + + // The agent replies in the SAME thread with a link to the PR it opened. + const reply = page.locator(".msg.bot").filter({ hasText: "Add greet() helper" }); + await expect(reply).toBeVisible(); + const prLink = reply.locator('a[href*="/pull/"]'); + await expect(prLink).toBeVisible(); + + // Follow the link → the PR page shows what GitHub would show. + await prLink.click(); + await expect(page.locator("#pr-title")).toContainText("Add greet() helper"); + await expect(page.locator("#pr-state")).toHaveText("open"); + await expect(page.locator("#pr-head")).toHaveText("add-greet"); + await expect(page.locator('#pr-files li[data-file="greet.py"]')).toBeVisible(); + + // The PR list view shows it too. + await page.goto("/mock/github"); + await expect(page.locator('.pr[data-pr="1"]')).toContainText("Add greet() helper"); + await expect(page.locator('.pr[data-pr="1"]')).toContainText("greet.py"); + }); + + test("a message that does not mention the bot produces no run and no PR", async ({ page }) => { + await page.locator("#mention").uncheck(); + await page.locator("#text").fill("just chatting with the team, nothing for the bot"); + await page.locator("#send").click(); + + await expect(page.locator(".msg").filter({ hasText: "just chatting" })).toBeVisible(); + // No agent activity: give the (non-)run a moment, then assert nothing came back. + await page.waitForTimeout(3000); + await expect(page.locator(".msg.bot")).toHaveCount(0); + + const prs = await (await page.request.get("/mock/github/data")).json(); + expect(prs.length).toBe(0); + }); +});