open-swe/tests/e2e/tests/reviewer_verification.spec.ts

import { createHmac } from "node:crypto";

import { expect, test } from "@playwright/test";

const WEBHOOK_SECRET = "test-github-secret";

test.describe("Reviewer verification contracts", () => {
  test("direct @openswe review routes to the reviewer without user mapping", async ({
    page,
  }) => {
    const seeded = await page.request.post("/control/review-pr");
    expect(seeded.ok()).toBeTruthy();
    const pr = await seeded.json();
    const payload = {
      action: "created",
      repository: {
        id: 1,
        name: "demo",
        full_name: "fakeorg/demo",
        private: true,
        owner: { login: "fakeorg" },
      },
      issue: {
        number: pr.number,
        html_url: pr.html_url,
        pull_request: { html_url: pr.html_url },
      },
      comment: {
        id: 24601,
        node_id: "IC_kwDO_e2e",
        body: "@openswe review: focus on authorization boundaries",
        user: { login: "unmapped-reviewer" },
      },
      sender: { id: 9001, login: "unmapped-reviewer" },
    };
    const raw = JSON.stringify(payload);
    const signature = `sha256=${createHmac("sha256", WEBHOOK_SECRET)
      .update(raw)
      .digest("hex")}`;

    const webhook = await page.request.post("/webhooks/github", {
      data: raw,
      headers: {
        "Content-Type": "application/json",
        "X-GitHub-Event": "issue_comment",
        "X-Hub-Signature-256": signature,
      },
    });
    expect(webhook.ok()).toBeTruthy();
    expect(await webhook.json()).toEqual({
      status: "accepted",
      message: "Processing on-demand PR review",
    });

    await expect
      .poll(async () => {
        const response = await page.request.get("/control/review-dispatches");
        return (await response.json()).length;
      })
      .toBe(1);
    const dispatches = await (
      await page.request.get("/control/review-dispatches")
    ).json();
    expect(dispatches).toHaveLength(1);
    expect(dispatches[0].assistant_id).toBe("reviewer");
    expect(dispatches[0].source).toBe("github_comment");
    expect(dispatches[0].configurable).toMatchObject({
      github_login: "unmapped-reviewer",
      github_user_id: 9001,
      review_requested: true,
      verdict_requested: true,
      repo: { owner: "fakeorg", name: "demo" },
      pr_number: pr.number,
    });
    const emailMappingLookups = await (
      await page.request.get("/control/email-mapping-lookups")
    ).json();
    expect(emailMappingLookups).toEqual([]);
    expect(dispatches[0].prompt).toContain("focus on authorization boundaries");
  });

  test("review outcomes settle checks as success, failure, or neutral", async ({
    page,
  }) => {
    await page.request.post("/control/reset");
    const cases = [
      {
        outcome: {
          verdict_submitted: true,
          verdict_event: "APPROVE",
          blocking_finding_count: 0,
        },
        conclusion: "success",
        title: "Review approved",
      },
      {
        outcome: {
          verdict_submitted: true,
          verdict_event: "REQUEST_CHANGES",
          blocking_finding_count: 1,
        },
        conclusion: "failure",
        title: "Changes requested",
      },
      {
        outcome: {
          verdict_submitted: false,
          verdict_authorization: "consistent",
          verdict_ignored_reason: "approve_with_open_findings",
          blocking_finding_count: 2,
        },
        conclusion: "failure",
        title: "Found 2 blocking issues",
      },
      {
        outcome: {
          verdict_submitted: false,
          verdict_ignored_reason: "verdict_not_requested",
          blocking_finding_count: 0,
        },
        conclusion: "neutral",
        title: "Verdict withheld",
      },
    ];

    for (const [index, item] of cases.entries()) {
      const response = await page.request.post(
        "/control/review-check/evaluate",
        {
          data: {
            head_sha: `tri-state-${index + 1}`,
            outcome: item.outcome,
          },
        },
      );
      expect(response.ok()).toBeTruthy();
      expect(await response.json()).toMatchObject({
        conclusion: item.conclusion,
        title: item.title,
      });
    }

    const checks = await (await page.request.get("/control/check-runs")).json();
    expect(
      checks.map((check: { conclusion: string }) => check.conclusion),
    ).toEqual(["success", "failure", "failure", "neutral"]);
    expect(checks.map((check: { status: string }) => check.status)).toEqual([
      "completed",
      "completed",
      "completed",
      "completed",
    ]);
  });
});