feat: Evaluations to test open-swe's langGraph py ability (#385)

apps/open-swe/eval.tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "isolatedModules": true
+  },
+  "include": ["./evals/**/*.ts"]
+}

apps/open-swe/evals/evaluator.ts

@@ -0,0 +1,274 @@
+import "dotenv/config";
+import { OpenSWEInput, CodeTestDetails } from "./open-swe-types.js";
+import { Daytona, Sandbox } from "@daytonaio/sdk";
+import { createLogger, LogLevel } from "../src/utils/logger.js";
+import { TIMEOUT_SEC } from "@open-swe/shared/constants";
+import { DEFAULT_SANDBOX_CREATE_PARAMS } from "../src/constants.js";
+import { TargetRepository } from "@open-swe/shared/open-swe/types";
+import { cloneRepo } from "../src/utils/github/git.js";
+import { getRepoAbsolutePath } from "@open-swe/shared/git";
+import { SimpleEvaluationResult } from "langsmith/vitest";
+import { runRuffLint, runMyPyTypeCheck } from "./tests.js";
+
+const logger = createLogger(LogLevel.INFO, "Evaluator ");
+
+const VENV_PATH = ".venv";
+const RUN_PYTHON_IN_VENV = `${VENV_PATH}/bin/python`;
+const RUN_PIP_IN_VENV = `${VENV_PATH}/bin/pip`;
+
+/**
+ * Setup Python environment with requirements.txt + ruff + mypy
+ */
+async function setupEnv(
+  sandbox: Sandbox,
+  absoluteRepoDir: string,
+): Promise<boolean> {
+  logger.info("Setting up Python environment...");
+
+  const createVenvCommand = "python -m venv .venv";
+  const createVenvRes = await sandbox.process.executeCommand(
+    createVenvCommand,
+    absoluteRepoDir,
+    undefined,
+    TIMEOUT_SEC,
+  );
+  if (createVenvRes.exitCode !== 0) {
+    logger.error("Failed to create virtual environment", {
+      createVenvCommand,
+      createVenvRes,
+    });
+    return false;
+  }
+
+  const upgradePipRes = await sandbox.process.executeCommand(
+    `${RUN_PIP_IN_VENV} install --upgrade pip`,
+    absoluteRepoDir,
+    undefined,
+    TIMEOUT_SEC,
+  );
+  if (upgradePipRes.exitCode !== 0) {
+    logger.warn("Failed to upgrade pip, continuing anyway", { upgradePipRes });
+  }
+
+  const requirementsExistRes = await sandbox.process.executeCommand(
+    "test -f requirements.txt",
+    absoluteRepoDir,
+    undefined,
+    TIMEOUT_SEC,
+  );
+
+  if (requirementsExistRes.exitCode === 0) {
+    logger.info("Found requirements.txt, installing...");
+    const installReqRes = await sandbox.process.executeCommand(
+      `${RUN_PIP_IN_VENV} install -r requirements.txt`,
+      absoluteRepoDir,
+      undefined,
+      TIMEOUT_SEC * 3,
+    );
+    if (installReqRes.exitCode !== 0) {
+      logger.warn("Failed to install requirements.txt, continuing anyway", {
+        installReqRes,
+      });
+    }
+  } else {
+    logger.info("No requirements.txt found, skipping repository dependencies");
+  }
+
+  const installAnalysisToolsRes = await sandbox.process.executeCommand(
+    `${RUN_PIP_IN_VENV} install ruff mypy`,
+    absoluteRepoDir,
+    undefined,
+    TIMEOUT_SEC,
+  );
+  if (installAnalysisToolsRes.exitCode !== 0) {
+    logger.error("Failed to install ruff and mypy", {
+      installAnalysisToolsRes,
+    });
+    return false;
+  }
+
+  logger.info("Environment setup completed successfully");
+  return true;
+}
+
+/**
+ * Runs ruff and mypy analysis on all Python files in the repository
+ */
+async function runCodeTests(
+  sandbox: Sandbox,
+  absoluteRepoDir: string,
+): Promise<{ ruffScore: number; mypyScore: number; details: CodeTestDetails }> {
+  logger.info("Running code analysis on all Python files in repository");
+
+  const testResults: {
+    ruffScore: number;
+    mypyScore: number;
+    details: CodeTestDetails;
+  } = {
+    ruffScore: 0,
+    mypyScore: 0,
+    details: {
+      ruff: {
+        issues: [],
+        error: null,
+      },
+      mypy: {
+        issues: [],
+        error: null,
+      },
+    },
+  };
+
+  const [ruffLint, mypyCheck] = await Promise.all([
+    runRuffLint(sandbox, {
+      command: `${RUN_PYTHON_IN_VENV} -m ruff check . --output-format=json`,
+      workingDir: absoluteRepoDir,
+      env: undefined,
+      timeoutSec: TIMEOUT_SEC * 3,
+    }),
+    runMyPyTypeCheck(sandbox, {
+      command: `${RUN_PYTHON_IN_VENV} -m mypy . --no-error-summary --show-error-codes --no-color-output`,
+      workingDir: absoluteRepoDir,
+      env: undefined,
+      timeoutSec: TIMEOUT_SEC * 3,
+    }),
+  ]);
+
+  Object.assign(testResults, {
+    ruffScore: ruffLint.ruffScore,
+    mypyScore: mypyCheck.mypyScore,
+    details: {
+      ruff: {
+        issues: ruffLint.issues,
+        error: ruffLint.error,
+      },
+      mypy: {
+        issues: mypyCheck.issues,
+        error: mypyCheck.error,
+      },
+    },
+  });
+
+  logger.info("Code tests completed", {
+    ruffScore: testResults.ruffScore,
+    mypyScore: testResults.mypyScore,
+    ruffIssues: testResults.details.ruff.issues.length,
+    mypyIssues: testResults.details.mypy.issues.length,
+  });
+
+  return testResults;
+}
+
+/**
+ * Main evaluator function for OpenSWE code analysis
+ */
+export async function evaluator(inputs: {
+  openSWEInputs: OpenSWEInput;
+  output: {
+    branchName: string;
+    targetRepository: TargetRepository;
+  };
+}): Promise<SimpleEvaluationResult[]> {
+  const { openSWEInputs, output } = inputs;
+
+  const githubToken = process.env.GITHUB_PAT;
+  if (!githubToken) {
+    throw new Error("GITHUB_PAT environment variable is not set");
+  }
+
+  const daytonaInstance = new Daytona();
+  logger.info("Creating sandbox...", {
+    repo: openSWEInputs.repo,
+    originalBranch: openSWEInputs.branch,
+    solutionBranch: output.branchName,
+    user_input: openSWEInputs.user_input.substring(0, 100) + "...",
+  });
+
+  const sandbox = await daytonaInstance.create(DEFAULT_SANDBOX_CREATE_PARAMS);
+
+  try {
+    const res = await cloneRepo(sandbox, output.targetRepository, {
+      githubInstallationToken: githubToken,
+    });
+    if (res.exitCode !== 0) {
+      logger.error("Failed to clone repository", {
+        targetRepository: output.targetRepository,
+        cloneResult: res,
+      });
+      throw new Error("Failed to clone repository");
+    }
+
+    const absoluteRepoDir = getRepoAbsolutePath(output.targetRepository);
+
+    const solutionBranch = output.branchName;
+    logger.info(`Checking out agent's solution branch: ${solutionBranch}`);
+
+    const checkoutBranchRes = await sandbox.process.executeCommand(
+      `git checkout ${solutionBranch}`,
+      absoluteRepoDir,
+      undefined,
+      TIMEOUT_SEC,
+    );
+    if (checkoutBranchRes.exitCode !== 0) {
+      logger.error("Failed to checkout solution branch", {
+        solutionBranch,
+        checkoutResult: checkoutBranchRes,
+      });
+      throw new Error(`Failed to checkout solution branch: ${solutionBranch}`);
+    }
+
+    const envSetupSuccess = await setupEnv(sandbox, absoluteRepoDir);
+    if (!envSetupSuccess) {
+      logger.error("Failed to setup environment");
+      return [
+        {
+          key: "overall-score",
+          score: 0,
+        },
+      ];
+    }
+
+    const analysisResult = await runCodeTests(sandbox, absoluteRepoDir);
+
+    const overallScore = analysisResult.ruffScore + analysisResult.mypyScore;
+
+    logger.info("Evaluation completed", {
+      overallScore,
+      ruffScore: analysisResult.ruffScore,
+      mypyScore: analysisResult.mypyScore,
+      repo: openSWEInputs.repo,
+      originalBranch: openSWEInputs.branch,
+      solutionBranch: output.branchName,
+    });
+
+    return [
+      {
+        key: "overall-score",
+        score: overallScore,
+      },
+      {
+        key: "ruff-score",
+        score: analysisResult.ruffScore,
+      },
+      {
+        key: "mypy-score",
+        score: analysisResult.mypyScore,
+      },
+    ];
+  } catch (error) {
+    logger.error("Evaluation failed with error", { error });
+    return [
+      {
+        key: "overall-score",
+        score: 0,
+      },
+    ];
+  } finally {
+    try {
+      await sandbox.delete();
+      logger.info("Sandbox cleaned up successfully");
+    } catch (cleanupError) {
+      logger.error("Failed to cleanup sandbox", { cleanupError });
+    }
+  }
+}

apps/open-swe/evals/langgraph.eval.ts

@@ -0,0 +1,235 @@
+// Run evals over the development Open SWE dataset
+
+import { v4 as uuidv4 } from "uuid";
+import * as ls from "langsmith/vitest";
+import { formatInputs } from "./prompts.js";
+import { createLogger, LogLevel } from "../src/utils/logger.js";
+import { evaluator } from "./evaluator.js";
+import { MANAGER_GRAPH_ID, GITHUB_PAT } from "@open-swe/shared/constants";
+import { createLangGraphClient } from "../src/utils/langgraph-client.js";
+import { encryptSecret } from "@open-swe/shared/crypto";
+import { ManagerGraphState } from "@open-swe/shared/open-swe/manager/types";
+import { PlannerGraphState } from "@open-swe/shared/open-swe/planner/types";
+import { GraphState } from "@open-swe/shared/open-swe/types";
+import { withRetry } from "./utils/retry.js";
+
+const logger = createLogger(LogLevel.DEBUG, "Evaluator");
+
+const DATASET_NAME = process.env.DATASET_NAME || "";
+// const RUN_NAME = `${DATASET_NAME}-${new Date().toISOString().replace(/[:.]/g, '-')}`;
+
+// async function loadDataset(): Promise<Example[]> {
+//   const client = new LangSmithClient();
+//   const datasetStream = client.listExamples({ datasetName: DATASET_NAME });
+//   let examples: Example[] = [];
+//   for await (const example of datasetStream) {
+//     examples.push(example);
+//   }
+//   logger.info(
+//     `Loaded ${examples.length} examples from dataset "${DATASET_NAME}"`,
+//   );
+//   return examples;
+// }
+
+// const DATASET = await loadDataset().then((examples) =>
+//   examples.map(example => ({
+//     inputs: example.inputs as OpenSWEInput,
+//   })),
+// );
+
+const DATASET = [
+  {
+    inputs: {
+      repo: "mai-sandbox/open-swe_content_team_eval",
+      branch: "main",
+      user_input: `I have implemented a multi-agent content creation system using LangGraph that orchestrates collaboration between specialized agents. The system is experiencing multiple runtime errors and workflow failures that prevent proper execution.
+
+System Architecture
+The application implements a three-agent architecture:
+
+Research Agent: Utilizes web search tools to gather information on specified topics
+Writer Agent: Creates content based on research findings with creative temperature settings
+Reviewer Agent: Provides feedback using fact-checking tools and determines revision needs
+
+Expected Workflow
+User Request → Research Agent → Writer Agent → Reviewer Agent → [Revision Loop if needed] → Final Content
+
+Current Issues
+
+Runtime Errors: Application fails to start with import and graph compilation errors
+Agent Handoff Failures: Agents are not properly transferring control and context
+Tool Integration Problems: Tool calling mechanisms are not functioning correctly
+State Management Issues: Shared state is not being updated correctly across agent transitions
+Routing Logic Failures: Conditional edges and workflow routing are broken`,
+    },
+  },
+];
+
+logger.info(`Starting evals over ${DATASET.length} examples...`);
+
+//const LANGGRAPH_URL = process.env.LANGGRAPH_URL || "http://localhost:2024";
+
+ls.describe(DATASET_NAME, () => {
+  ls.test.each(DATASET)(
+    "Can resolve issue",
+    async ({ inputs }) => {
+      logger.info("Starting agent run", {
+        inputs,
+      });
+
+      const encryptionKey = process.env.SECRETS_ENCRYPTION_KEY;
+      const githubPat = process.env.GITHUB_PAT;
+
+      if (!encryptionKey || !githubPat) {
+        throw new Error(
+          "SECRETS_ENCRYPTION_KEY and GITHUB_PAT environment variables are required",
+        );
+      }
+
+      const encryptedGitHubToken = encryptSecret(githubPat, encryptionKey);
+
+      const lgClient = createLangGraphClient({
+        includeApiKey: true,
+        defaultHeaders: { [GITHUB_PAT]: encryptedGitHubToken },
+      });
+
+      const input = await formatInputs(inputs);
+
+      const threadId = uuidv4();
+      logger.info("Starting agent run", {
+        thread_id: threadId,
+        problem: inputs.user_input,
+        repo: inputs.repo,
+      });
+
+      // Run the agent with user input
+      let managerRun;
+      try {
+        managerRun = await withRetry(() =>
+          lgClient.runs.wait(threadId, MANAGER_GRAPH_ID, {
+            input,
+            config: {
+              recursion_limit: 250,
+            },
+            ifNotExists: "create",
+          }),
+        );
+      } catch (error) {
+        logger.error("Error in manager run", {
+          thread_id: threadId,
+          error:
+            error instanceof Error
+              ? {
+                  message: error.message,
+                  stack: error.stack,
+                  name: error.name,
+                  cause: error.cause,
+                }
+              : error,
+        });
+        return; // instead of skipping, we should award 0 points
+      }
+
+      const managerState = managerRun as unknown as ManagerGraphState;
+      const plannerSession = managerState?.plannerSession;
+
+      if (!plannerSession) {
+        logger.info("Agent did not create a planner session", {
+          thread_id: threadId,
+        });
+        return; // instead of skipping, we should award 0 points
+      }
+
+      let plannerRun;
+      try {
+        plannerRun = await withRetry(() =>
+          lgClient.runs.join(plannerSession.threadId, plannerSession.runId),
+        );
+      } catch (error) {
+        logger.error("Error joining planner run", {
+          thread_id: threadId,
+          plannerSession,
+          error:
+            error instanceof Error
+              ? {
+                  message: error.message,
+                  stack: error.stack,
+                  name: error.name,
+                  cause: error.cause,
+                }
+              : error,
+        });
+        return; // instead of skipping, we should award 0 points
+      }
+
+      // Type-safe access to planner run state
+      const plannerState = plannerRun as unknown as PlannerGraphState;
+      const programmerSession = plannerState?.programmerSession;
+
+      if (!programmerSession) {
+        logger.info("Agent did not create a programmer session", {
+          thread_id: threadId,
+        });
+        return; // instead of skipping, we should award 0 points
+      }
+
+      let programmerRun;
+      try {
+        programmerRun = await withRetry(() =>
+          lgClient.runs.join(
+            programmerSession.threadId,
+            programmerSession.runId,
+          ),
+        );
+      } catch (error) {
+        logger.error("Error joining programmer run", {
+          thread_id: threadId,
+          programmerSession,
+          error:
+            error instanceof Error
+              ? {
+                  message: error.message,
+                  stack: error.stack,
+                  name: error.name,
+                  cause: error.cause,
+                }
+              : error,
+        });
+        return; // instead of skipping, we should award 0 points
+      }
+
+      const programmerState = programmerRun as unknown as GraphState;
+      const branchName = programmerState?.branchName;
+
+      if (!branchName) {
+        logger.info("Agent did not create a branch", {
+          thread_id: threadId,
+        });
+        return; // instead of skipping, we should award 0 points
+      }
+
+      logger.info("Agent completed. Created branch:", {
+        branchName: branchName,
+      });
+
+      // Evaluation
+      const wrappedEvaluator = ls.wrapEvaluator(evaluator);
+      const evalResult = await wrappedEvaluator({
+        openSWEInputs: inputs,
+        output: {
+          branchName,
+          targetRepository: {
+            owner: inputs.repo.split("/")[0],
+            repo: inputs.repo.split("/")[1],
+          },
+        },
+      });
+
+      logger.info("Evaluation completed.", {
+        thread_id: threadId,
+        evalResult,
+      });
+    },
+    7200_000,
+  );
+});

apps/open-swe/evals/open-swe-types.ts

@@ -0,0 +1,104 @@
+/**
+ * Input structure for Open SWE evaluations
+ * This is much simpler than SWE-Bench since we only need
+ * problem statement + repo info for ruff/mypy analysis
+ */
+export interface OpenSWEInput {
+  /**
+   * The user request/problem statement that was given to Open SWE
+   * This is what gets passed to the agent to solve
+   */
+  user_input: string;
+
+  /**
+   * Repository information in "owner/repo" format
+   * e.g., "aliyanishfaq/my-project"
+   */
+  repo: string;
+
+  /**
+   * Optional: Branch name where the agent's solution is located
+   * If not provided, agent will create one (e.g., "open-swe/uuid")
+   */
+  branch: string;
+}
+
+/**
+ * Process execution options
+ */
+export interface ExecOptions {
+  command: string;
+  workingDir: string;
+  env: Record<string, string> | undefined;
+  timeoutSec: number;
+}
+
+/**
+ * Ruff issue location
+ */
+export interface RuffLocation {
+  column: number;
+  row: number;
+}
+
+/**
+ * Ruff fix edit
+ */
+export interface RuffEdit {
+  content: string;
+  end_location: RuffLocation;
+  location: RuffLocation;
+}
+
+/**
+ * Ruff fix suggestion
+ */
+export interface RuffFix {
+  applicability: "safe" | "unsafe" | "display";
+  edits: RuffEdit[];
+  message: string;
+}
+
+/**
+ * Individual Ruff issue
+ */
+export interface RuffIssue {
+  cell: string | null;
+  code: string;
+  end_location: RuffLocation;
+  filename: string;
+  fix: RuffFix | null;
+  location: RuffLocation;
+  message: string;
+  noqa_row: number;
+  url: string;
+}
+
+/**
+ * Return type for ruffPromise function
+ */
+export interface RuffResult {
+  ruffScore: number;
+  error: Error | null;
+  issues: RuffIssue[];
+}
+
+/**
+ * Return type for mypyPromise function
+ */
+export interface MyPyResult {
+  mypyScore: number;
+  error: Error | null;
+  issues: string[];
+}
+
+export interface CodeTestDetails {
+  ruff: {
+    issues: RuffIssue[];
+    error: Error | null;
+  };
+  mypy: {
+    issues: string[];
+    error: Error | null;
+  };
+}

apps/open-swe/evals/prompts.ts

@@ -0,0 +1,60 @@
+import { OpenSWEInput } from "./open-swe-types.js";
+import { TargetRepository } from "@open-swe/shared/open-swe/types";
+import { HumanMessage } from "@langchain/core/messages";
+import { Octokit } from "@octokit/rest";
+import { ManagerGraphUpdate } from "@open-swe/shared/open-swe/manager/types";
+
+async function getRepoReadmeContents(
+  targetRepository: TargetRepository,
+): Promise<string> {
+  if (!process.env.GITHUB_PAT) {
+    throw new Error("GITHUB_PAT environment variable missing.");
+  }
+  const octokit = new Octokit({
+    auth: process.env.GITHUB_PAT,
+  });
+
+  try {
+    const { data } = await octokit.repos.getReadme({
+      owner: targetRepository.owner,
+      repo: targetRepository.repo,
+    });
+    return Buffer.from(data.content, "base64").toString("utf-8");
+  } catch (_) {
+    return "";
+  }
+}
+
+export async function formatInputs(
+  inputs: OpenSWEInput,
+): Promise<ManagerGraphUpdate> {
+  const targetRepository: TargetRepository = {
+    owner: inputs.repo.split("/")[0],
+    repo: inputs.repo.split("/")[1],
+    branch: inputs.branch,
+  };
+
+  const readmeContents = await getRepoReadmeContents(targetRepository);
+
+  const SIMPLE_PROMPT_TEMPLATE = `<request>
+{USER_REQUEST}
+</request>
+
+<codebase-readme>
+{CODEBASE_README}
+</codebase-readme>`;
+
+  const userMessageContent = SIMPLE_PROMPT_TEMPLATE.replace(
+    "{REPO}",
+    inputs.repo,
+  )
+    .replace("{USER_REQUEST}", inputs.user_input)
+    .replace("{CODEBASE_README}", readmeContents);
+
+  const userMessage = new HumanMessage(userMessageContent);
+  return {
+    messages: [userMessage],
+    targetRepository,
+    autoAcceptPlan: true,
+  };
+}

apps/open-swe/evals/tests.ts

@@ -0,0 +1,132 @@
+// TODO: Add ruff promise and the mypy promise to the tests.
+import { Sandbox } from "@daytonaio/sdk";
+import { createLogger, LogLevel } from "../src/utils/logger.js";
+import {
+  ExecOptions,
+  RuffResult,
+  RuffIssue,
+  MyPyResult,
+} from "./open-swe-types.js";
+
+const logger = createLogger(LogLevel.DEBUG, " Evaluation Tests");
+
+/**
+ * Run ruff check and return score, error, and issues
+ */
+export const runRuffLint = async (
+  sandbox: Sandbox,
+  args: ExecOptions,
+): Promise<RuffResult> => {
+  logger.info("Running ruff check...");
+
+  try {
+    const execution = await sandbox.process.executeCommand(
+      args.command,
+      args.workingDir,
+      args.env,
+      args.timeoutSec,
+    );
+
+    if (execution.exitCode === 0) {
+      logger.info("Ruff analysis passed. No issues found.");
+      return {
+        ruffScore: 1,
+        error: null,
+        issues: [],
+      };
+    }
+
+    try {
+      const issues: RuffIssue[] = JSON.parse(execution.result);
+      const issueCount = Array.isArray(issues) ? issues.length : 0;
+      const ruffScore = issueCount === 0 ? 1 : 0;
+
+      logger.info(`Ruff found ${issueCount} issues`, {
+        score: ruffScore,
+        issues: issues.slice(0, 3), // Log first 3 issues
+      });
+
+      return {
+        ruffScore,
+        error: null,
+        issues,
+      };
+    } catch (parseError) {
+      logger.warn(
+        "Could not parse ruff JSON output. Setting Ruff score to 0.",
+        {
+          parseError,
+          output: execution.result?.substring(0, 200) + "...",
+        },
+      );
+
+      return {
+        ruffScore: 0,
+        error: parseError as Error,
+        issues: [],
+      };
+    }
+  } catch (error) {
+    logger.error("Failed to run ruff check", { error });
+    return {
+      ruffScore: 0,
+      error: error as Error,
+      issues: [],
+    };
+  }
+};
+
+/**
+ * Run mypy check and return score, error, and issues
+ */
+export const runMyPyTypeCheck = async (
+  sandbox: Sandbox,
+  args: ExecOptions,
+): Promise<MyPyResult> => {
+  logger.info("Running mypy check...");
+  try {
+    const execution = await sandbox.process.executeCommand(
+      args.command,
+      args.workingDir,
+      args.env,
+      args.timeoutSec,
+    );
+
+    if (execution.exitCode === 0) {
+      logger.info("Mypy analysis passed. No issues found.");
+      return {
+        mypyScore: 1,
+        error: null,
+        issues: [],
+      };
+    } else {
+      // Filter for actual type problems: errors and warnings
+      const errorLines = execution.result
+        .split("\n")
+        .filter(
+          (line) => line.includes(": error:") || line.includes(": warning:"),
+        );
+
+      const issueCount = errorLines.length;
+      const mypyScore = issueCount === 0 ? 1 : 0;
+
+      logger.info(`Mypy found ${issueCount} issues`, {
+        score: mypyScore,
+        issues: errorLines.slice(0, 3),
+      });
+
+      return {
+        mypyScore,
+        error: null,
+        issues: errorLines,
+      };
+    }
+  } catch (error) {
+    logger.error("Failed to run mypy check", { error });
+    return {
+      mypyScore: 0,
+      error: error as Error,
+      issues: [],
+    };
+  }
+};

apps/open-swe/evals/utils/retry.ts

@@ -0,0 +1,51 @@
+import { createLogger, LogLevel } from "../../src/utils/logger.js";
+
+const logger = createLogger(LogLevel.DEBUG, "Retry");
+
+const RETRY_CONFIG = {
+  maxRetries: 5,
+  baseDelay: 1000,
+  maxDelay: 30000,
+  backoffMultiplier: 2,
+  timeoutErrors: ["UND_ERR_HEADERS_TIMEOUT"],
+};
+
+/**
+ * Retry decorator with exponential backoff for LangGraph client
+ * operations.
+ */
+export async function withRetry<T>(operation: () => Promise<T>): Promise<T> {
+  let lastError: any;
+
+  for (let attempt = 0; attempt < RETRY_CONFIG.maxRetries; attempt++) {
+    try {
+      return await operation();
+    } catch (error: any) {
+      lastError = error;
+
+      const isRetryable = RETRY_CONFIG.timeoutErrors.includes(
+        error?.cause?.code,
+      );
+
+      if (isRetryable && attempt < RETRY_CONFIG.maxRetries - 1) {
+        const delay = Math.min(
+          RETRY_CONFIG.baseDelay *
+            Math.pow(RETRY_CONFIG.backoffMultiplier, attempt),
+          RETRY_CONFIG.maxDelay,
+        );
+        logger.info(
+          `Retrying operation in ${delay}ms. Attempt ${attempt + 1} of ${RETRY_CONFIG.maxRetries}`,
+          {
+            attempt,
+            lastError,
+          },
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      } else {
+        throw lastError;
+      }
+    }
+  }
+
+  throw lastError;
+}

apps/open-swe/ls.vitest.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["**/*.eval.?(c|m)[jt]s"],
+    reporters: ["langsmith/vitest/reporter"],
+    setupFiles: ["dotenv/config"],
+    typecheck: {
+      tsconfig: "./eval.tsconfig.json",
+    },
+    testTimeout: 7200_000, // 120 minutes
+  },
+});

apps/open-swe/package.json

@@ -19,6 +19,7 @@
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern=\\.test\\.ts$ --testPathIgnorePatterns=\\.int\\.test\\.ts$",
     "test:int": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern=\\.int\\.test\\.ts$",
     "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.js --testTimeout 100000",
+    "eval:single": "NODE_OPTIONS=--experimental-vm-modules yarn run vitest --config ls.vitest.config.ts --run",
     "postinstall": "turbo build"
   },
   "dependencies": {
@@ -63,7 +64,8 @@
     "ts-jest": "^29.1.0",
     "turbo": "^2.5.0",
     "typescript": "~5.7.2",
-    "typescript-eslint": "^8.22.0"
+    "typescript-eslint": "^8.22.0",
+    "vitest": "^3.2.3"
   },
   "packageManager": "yarn@3.5.1"
 }

apps/open-swe/src/constants.ts

@@ -10,3 +10,20 @@ export const DEFAULT_SANDBOX_CREATE_PARAMS: CreateSandboxParams = {
   user: "daytona",
   image: SNAPSHOT_NAME,
 };
+
+export const LANGGRAPH_USER_PERMISSIONS = [
+  "threads:create",
+  "threads:create_run",
+  "threads:read",
+  "threads:delete",
+  "threads:update",
+  "threads:search",
+  "assistants:create",
+  "assistants:read",
+  "assistants:delete",
+  "assistants:update",
+  "assistants:search",
+  "deployments:read",
+  "deployments:search",
+  "store:access",
+];

apps/open-swe/src/security/auth.ts

@@ -14,6 +14,8 @@ import {
 import { decryptSecret } from "@open-swe/shared/crypto";
 import { verifyGitHubWebhookOrThrow } from "./github.js";
 import { createWithOwnerMetadata, createOwnerFilter } from "./utils.js";
+import { LANGGRAPH_USER_PERMISSIONS } from "../constants.js";
+import { getGitHubPatFromRequest } from "../utils/github-pat.js";
 
 // TODO: Export from LangGraph SDK
 export interface BaseAuthReturn {
@@ -43,6 +45,8 @@ export const auth = new Auth()
       };
     }
 
+    const isProd = process.env.NODE_ENV === "production";
+
     const ghSecretHashHeader = request.headers.get("X-Hub-Signature-256");
     if (ghSecretHashHeader) {
       // This will either return a valid user, or throw an error
@@ -54,6 +58,28 @@ export const auth = new Auth()
       throw new Error("Missing SECRETS_ENCRYPTION_KEY environment variable.");
     }
 
+    // Check for GitHub PAT authentication (simpler mode for evals, etc.)
+    const githubPat = getGitHubPatFromRequest(request, encryptionKey);
+    if (githubPat && !isProd) {
+      const user = await verifyGithubUser(githubPat);
+      if (!user) {
+        throw new HTTPException(401, {
+          message: "Invalid GitHub PAT",
+        });
+      }
+
+      return {
+        identity: user.id.toString(),
+        is_authenticated: true,
+        display_name: user.login,
+        metadata: {
+          installation_name: "pat-auth",
+        },
+        permissions: LANGGRAPH_USER_PERMISSIONS,
+      };
+    }
+
+    // GitHub App authentication mode (existing logic)
     const installationNameHeader = request.headers.get(
       GITHUB_INSTALLATION_NAME,
     );
@@ -112,22 +138,7 @@ export const auth = new Auth()
       metadata: {
         installation_name: installationNameHeader,
       },
-      permissions: [
+      permissions: LANGGRAPH_USER_PERMISSIONS,
-        "threads:create",
-        "threads:create_run",
-        "threads:read",
-        "threads:delete",
-        "threads:update",
-        "threads:search",
-        "assistants:create",
-        "assistants:read",
-        "assistants:delete",
-        "assistants:update",
-        "assistants:search",
-        "deployments:read",
-        "deployments:search",
-        "store:access",
-      ],
     };
   })
 

apps/open-swe/src/security/github.ts

@@ -1,6 +1,7 @@
 import { HTTPException } from "@langchain/langgraph-sdk/auth";
 import { Webhooks } from "@octokit/webhooks";
 import { createLogger, LogLevel } from "../utils/logger.js";
+import { LANGGRAPH_USER_PERMISSIONS } from "../constants.js";
 
 const logger = createLogger(LogLevel.INFO, "GitHubWebhookAuth");
 
@@ -41,21 +42,6 @@ export async function verifyGitHubWebhookOrThrow(request: Request) {
     metadata: {
       installation_name: "n/a",
     },
-    permissions: [
+    permissions: LANGGRAPH_USER_PERMISSIONS,
-      "threads:create",
-      "threads:create_run",
-      "threads:read",
-      "threads:delete",
-      "threads:update",
-      "threads:search",
-      "assistants:create",
-      "assistants:read",
-      "assistants:delete",
-      "assistants:update",
-      "assistants:search",
-      "deployments:read",
-      "deployments:search",
-      "store:access",
-    ],
   };
 }

apps/open-swe/src/utils/default-headers.ts

@@ -5,9 +5,19 @@ import {
   GITHUB_USER_ID_HEADER,
   GITHUB_USER_LOGIN_HEADER,
   GITHUB_INSTALLATION_NAME,
+  GITHUB_PAT,
 } from "@open-swe/shared/constants";
 
-export function getDefaultHeaders(config: GraphConfig) {
+export function getDefaultHeaders(config: GraphConfig): Record<string, string> {
+  const githubPat = config.configurable?.[GITHUB_PAT];
+  const isProd = process.env.NODE_ENV === "production";
+  if (githubPat && !isProd) {
+    // PAT-only
+    return {
+      [GITHUB_PAT]: githubPat,
+    };
+  }
+
   const githubInstallationTokenCookie =
     config.configurable?.[GITHUB_INSTALLATION_TOKEN_COOKIE];
   const githubInstallationName =

apps/open-swe/src/utils/github-pat.ts

@@ -0,0 +1,33 @@
+import { GITHUB_PAT } from "@open-swe/shared/constants";
+import { decryptSecret } from "@open-swe/shared/crypto";
+
+/**
+ * Simple helper to check if request has GitHub PAT and return decrypted value
+ */
+export function getGitHubPatFromRequest(
+  request: Request,
+  encryptionKey: string,
+): string | null {
+  const encryptedGitHubPat = request.headers.get(GITHUB_PAT);
+  if (!encryptedGitHubPat) {
+    return null;
+  }
+  return decryptSecret(encryptedGitHubPat, encryptionKey);
+}
+
+/**
+ * Helper to check if configurable has GitHub PAT and return decrypted value
+ */
+export function getGitHubPatFromConfig(
+  configurable: Record<string, any> | undefined,
+  encryptionKey: string,
+): string | null {
+  if (!configurable) {
+    return null;
+  }
+  const encryptedGitHubPat = configurable[GITHUB_PAT];
+  if (!encryptedGitHubPat) {
+    return null;
+  }
+  return decryptSecret(encryptedGitHubPat, encryptionKey);
+}

apps/open-swe/src/utils/github-tokens.ts

@@ -4,6 +4,7 @@ import {
 } from "@open-swe/shared/constants";
 import { GraphConfig } from "@open-swe/shared/open-swe/types";
 import { decryptSecret } from "@open-swe/shared/crypto";
+import { getGitHubPatFromConfig } from "./github-pat.js";
 
 export function getGitHubTokensFromConfig(config: GraphConfig): {
   githubAccessToken: string;
@@ -12,6 +13,24 @@ export function getGitHubTokensFromConfig(config: GraphConfig): {
   if (!config.configurable) {
     throw new Error("No configurable object found in graph config.");
   }
+
+  // Get the encryption key from environment variables
+  const encryptionKey = process.env.SECRETS_ENCRYPTION_KEY;
+  if (!encryptionKey) {
+    throw new Error("Missing SECRETS_ENCRYPTION_KEY environment variable.");
+  }
+
+  const isProd = process.env.NODE_ENV === "production";
+
+  const githubPat = getGitHubPatFromConfig(config.configurable, encryptionKey);
+  if (githubPat && !isProd) {
+    // check for PAT-only mode
+    return {
+      githubAccessToken: githubPat,
+      githubInstallationToken: githubPat,
+    };
+  }
+
   const encryptedGitHubToken = config.configurable[GITHUB_TOKEN_COOKIE];
   const encryptedInstallationToken =
     config.configurable[GITHUB_INSTALLATION_TOKEN_COOKIE];
@@ -21,12 +40,6 @@ export function getGitHubTokensFromConfig(config: GraphConfig): {
     );
   }
 
-  // Get the encryption key from environment variables
-  const encryptionKey = process.env.SECRETS_ENCRYPTION_KEY;
-  if (!encryptionKey) {
-    throw new Error("Missing SECRETS_ENCRYPTION_KEY environment variable.");
-  }
-
   // Decrypt the GitHub token
   const githubAccessToken = encryptedGitHubToken
     ? decryptSecret(encryptedGitHubToken, encryptionKey)

packages/shared/src/constants.ts

@@ -8,6 +8,7 @@ export const PLAN_INTERRUPT_ACTION_TITLE = "Approve/Edit Plan";
 export const GITHUB_TOKEN_COOKIE = "x-github-access-token";
 export const GITHUB_INSTALLATION_TOKEN_COOKIE = "x-github-installation-token";
 export const GITHUB_INSTALLATION_NAME = "x-github-installation-name";
+export const GITHUB_PAT = "x-github-pat";
 
 export const DO_NOT_RENDER_ID_PREFIX = "do-not-render-";
 export const GITHUB_AUTH_STATE_COOKIE = "github_auth_state";

packages/shared/src/open-swe/types.ts

@@ -19,6 +19,7 @@ import {
   GITHUB_TOKEN_COOKIE,
   GITHUB_USER_ID_HEADER,
   GITHUB_USER_LOGIN_HEADER,
+  GITHUB_PAT,
   DEFAULT_MCP_SERVERS,
 } from "../constants.js";
 import { withLangGraph } from "@langchain/langgraph/zod";
@@ -414,6 +415,11 @@ export const GraphConfigurationMetadata: {
       type: "hidden",
     },
   },
+  [GITHUB_PAT]: {
+    x_open_swe_ui_config: {
+      type: "hidden",
+    },
+  },
   mcpServers: {
     x_open_swe_ui_config: {
       type: "json",
@@ -598,6 +604,14 @@ export const GraphConfiguration = z.object({
     .string()
     .optional()
     .langgraph.metadata(GraphConfigurationMetadata[GITHUB_INSTALLATION_NAME]),
+  /**
+   * GitHub Personal Access Token. Used for simpler authentication in environments like evals
+   * where GitHub App installation tokens are not available or needed.
+   */
+  [GITHUB_PAT]: z
+    .string()
+    .optional()
+    .langgraph.metadata(GraphConfigurationMetadata[GITHUB_PAT]),
   /**
    * Custom MCP servers configuration as JSON string. Merges with default servers.
    * @default Default LangGraph docs MCP server