feat: Add diagnose error node to planner (#387)

apps/open-swe/src/__tests__/diagnose-error.test.ts

@@ -0,0 +1,181 @@
+import { describe, it, expect } from "@jest/globals";
+import { AIMessage, ToolMessage, HumanMessage } from "@langchain/core/messages";
+import { getAllLastFailedActions } from "../utils/tool-message-error.js";
+
+describe("getAllLastFailedActions", () => {
+  it("should return empty string for empty messages array", () => {
+    const result = getAllLastFailedActions([]);
+    expect(result).toBe("");
+  });
+
+  it("should return AI and error tool message pairs until a non-error tool message is encountered", () => {
+    // Create test messages
+    const aiMessage1 = new AIMessage({
+      content: "I'll try to execute this command",
+      id: "ai-1",
+    });
+
+    const errorToolMessage1 = new ToolMessage({
+      content: "Command failed: Permission denied",
+      tool_call_id: "tool-1",
+      name: "shell",
+      status: "error",
+    });
+
+    const aiMessage2 = new AIMessage({
+      content: "Let me try a different approach",
+      id: "ai-2",
+    });
+
+    const errorToolMessage2 = new ToolMessage({
+      content: "Error: File not found",
+      tool_call_id: "tool-2",
+      name: "read_file",
+      status: "error",
+    });
+
+    const aiMessage3 = new AIMessage({
+      content: "Let me try something else",
+      id: "ai-3",
+    });
+
+    const successToolMessage = new ToolMessage({
+      content: "Command executed successfully",
+      tool_call_id: "tool-3",
+      name: "shell",
+      status: "success",
+    });
+
+    const aiMessage4 = new AIMessage({
+      content: "Let me try one more thing",
+      id: "ai-4",
+    });
+
+    const errorToolMessage3 = new ToolMessage({
+      content: "Error: Invalid syntax",
+      tool_call_id: "tool-4",
+      name: "shell",
+      status: "error",
+    });
+
+    const messages = [
+      aiMessage1,
+      errorToolMessage1,
+      aiMessage2,
+      errorToolMessage2,
+      aiMessage3,
+      successToolMessage,
+      aiMessage4,
+      errorToolMessage3,
+    ];
+
+    const result = getAllLastFailedActions(messages);
+
+    // Should include the first two AI+error pairs, but stop at the success message
+    expect(result).toContain("I'll try to execute this command");
+    expect(result).toContain("Command failed: Permission denied");
+    expect(result).toContain("Let me try a different approach");
+    expect(result).toContain("Error: File not found");
+
+    // Should not include messages after the success message
+    expect(result).not.toContain("Let me try one more thing");
+    expect(result).not.toContain("Error: Invalid syntax");
+  });
+
+  it("should handle non-sequential AI and tool messages", () => {
+    const aiMessage = new AIMessage({
+      content: "I'll try to execute this command",
+      id: "ai-1",
+    });
+
+    const humanMessage = new HumanMessage({
+      content: "Can you try something else?",
+      id: "human-1",
+    });
+
+    const errorToolMessage = new ToolMessage({
+      content: "Command failed: Permission denied",
+      tool_call_id: "tool-1",
+      name: "shell",
+      status: "error",
+    });
+
+    const messages = [aiMessage, humanMessage, errorToolMessage];
+
+    const result = getAllLastFailedActions(messages);
+
+    // Should not include any messages since there's no AI+error pair
+    expect(result).toBe("");
+  });
+
+  it("should handle a mix of error and non-error tool messages", () => {
+    const aiMessage1 = new AIMessage({
+      content: "First command",
+      id: "ai-1",
+    });
+
+    const successToolMessage1 = new ToolMessage({
+      content: "Success",
+      tool_call_id: "tool-1",
+      name: "shell",
+      status: "success",
+    });
+
+    const aiMessage2 = new AIMessage({
+      content: "Second command",
+      id: "ai-2",
+    });
+
+    const errorToolMessage = new ToolMessage({
+      content: "Error occurred",
+      tool_call_id: "tool-2",
+      name: "shell",
+      status: "error",
+    });
+
+    const messages = [
+      aiMessage1,
+      successToolMessage1,
+      aiMessage2,
+      errorToolMessage,
+    ];
+
+    const result = getAllLastFailedActions(messages);
+
+    // Should not include any messages since we encounter a success message first
+    expect(result).toBe("");
+  });
+
+  it("should handle multiple tool messages after an AI message", () => {
+    const aiMessage = new AIMessage({
+      content: "Let me try multiple commands",
+      id: "ai-1",
+    });
+
+    const errorToolMessage1 = new ToolMessage({
+      content: "First command failed",
+      tool_call_id: "tool-1",
+      name: "shell",
+      status: "error",
+    });
+
+    const errorToolMessage2 = new ToolMessage({
+      content: "Second command failed",
+      tool_call_id: "tool-2",
+      name: "read_file",
+      status: "error",
+    });
+
+    const messages = [aiMessage, errorToolMessage1, errorToolMessage2];
+
+    const result = getAllLastFailedActions(messages);
+
+    // Should include the AI message and the first error tool message
+    expect(result).toContain("Let me try multiple commands");
+    expect(result).toContain("First command failed");
+
+    // The second error tool message should not be paired with the AI message
+    // since we're looking for AI+tool pairs
+    expect(result).not.toContain("Second command failed");
+  });
+});

apps/open-swe/src/graphs/planner/index.ts

@@ -18,6 +18,7 @@ import {
 } from "./nodes/index.js";
 import { isAIMessage } from "@langchain/core/messages";
 import { initializeSandbox } from "../shared/initialize-sandbox.js";
+import { diagnoseError } from "./nodes/diagnose-error.js";
 
 function takeActionOrGeneratePlan(
   state: PlannerGraphState,
@@ -48,7 +49,9 @@ const workflow = new StateGraph(PlannerGraphStateObj, GraphConfiguration)
   })
   .addNode("initialize-sandbox", initializeSandbox)
   .addNode("generate-plan-context-action", generateAction)
-  .addNode("take-plan-actions", takeActions)
+  .addNode("take-plan-actions", takeActions, {
+    ends: ["generate-plan-context-action", "diagnose-error"],
+  })
   .addNode("generate-plan", generatePlan)
   .addNode("notetaker", notetaker)
   .addNode("interrupt-proposed-plan", interruptProposedPlan, {
@@ -57,6 +60,7 @@ const workflow = new StateGraph(PlannerGraphStateObj, GraphConfiguration)
   .addNode("determine-needs-context", determineNeedsContext, {
     ends: ["generate-plan-context-action", "generate-plan"],
   })
+  .addNode("diagnose-error", diagnoseError)
   .addEdge(START, "prepare-graph-state")
   .addEdge("initialize-sandbox", "generate-plan-context-action")
   .addConditionalEdges(
@@ -64,7 +68,7 @@ const workflow = new StateGraph(PlannerGraphStateObj, GraphConfiguration)
     takeActionOrGeneratePlan,
     ["take-plan-actions", "generate-plan"],
   )
-  .addEdge("take-plan-actions", "generate-plan-context-action")
+  .addEdge("diagnose-error", "generate-plan-context-action")
   .addEdge("generate-plan", "notetaker")
   .addEdge("notetaker", "interrupt-proposed-plan");
 

apps/open-swe/src/graphs/planner/nodes/diagnose-error.ts

@@ -0,0 +1,127 @@
+import {
+  BaseMessage,
+  isToolMessage,
+  ToolMessage,
+} from "@langchain/core/messages";
+import { createDiagnoseErrorToolFields } from "@open-swe/shared/open-swe/tools";
+
+import { getMessageString } from "../../../utils/message/content.js";
+import { loadModel, Task } from "../../../utils/load-model.js";
+import { z } from "zod";
+import { createLogger, LogLevel } from "../../../utils/logger.js";
+import { getAllLastFailedActions } from "../../../utils/tool-message-error.js";
+import {
+  PlannerGraphState,
+  PlannerGraphUpdate,
+} from "@open-swe/shared/open-swe/planner/types";
+import { GraphConfig } from "@open-swe/shared/open-swe/types";
+
+const logger = createLogger(LogLevel.INFO, "DiagnoseError");
+
+const systemPrompt = `You are operating as a terminal-based agentic coding assistant built by LangChain. It wraps LLM models to enable natural language interaction with a local codebase. You are expected to be precise, safe, and helpful.
+
+The last few commands you tried to execute failed with an error. Please carefully diagnose the error, and provide a helpful explanation of exactly what the issue is, and how you can fix it.
+
+Following these rules when diagnosing the error:
+  - You should provide a clear, concise, and helpful explanation of exactly what the issue is, and how you can fix it.
+  - You do not want to be overly verbose in your diagnosis. You should only include information which is directly relevant to diagnosing and fixing the error.
+  - NEVER make up reasons, or make a guess as to what the issue is. Your reasoning must ALWAYS be grounded in the information provided to you.
+    - Making up reasons, or making a guess can lead to more problems, so it's best to say you don't know rather than make up a reason.
+  - Reference specific lines of code, or context from the conversation history to support your diagnosis.
+  
+Here are the last actions you attempted which resulted in errors:
+{FAILED_ACTIONS_OUTPUTS}
+
+Below is an up to date tree of the codebase (going 3 levels deep). This is up to date, and is updated after every action you take. Always assume this is the most up to date context about the codebase.
+It was generated by using the \`tree\` command, passing in the gitignore file to ignore files and directories you should not have access to (\`git ls-files | tree --fromfile -L 3\`). It is always executed inside the repo directory: {REPO_DIRECTORY}
+{CODEBASE_TREE}
+
+Please carefully go over all of this information, and provide a helpful explanation of exactly what the issue is, and how you can fix it. When you are ready to provide your diagnosis, call the \`diagnose_error\` tool.
+`;
+
+const userPrompt = `Here is the full conversation history from the steps taken to complete the current task, along with the user's initial request:
+
+{CONVERSATION_HISTORY}
+
+Please carefully go over all of this information, and provide a helpful explanation of exactly what the issue is, and how you can fix it. When you are ready to provide your diagnosis, call the \`diagnose_error\` tool.`;
+
+const diagnoseErrorTool = createDiagnoseErrorToolFields();
+
+const formatSystemPrompt = (
+  messages: BaseMessage[],
+  codebaseTree: string,
+): string => {
+  const lastFailedActions = getAllLastFailedActions(messages);
+
+  return systemPrompt
+    .replace(
+      "{FAILED_ACTIONS_OUTPUTS}",
+      `<failed-action-output>${lastFailedActions}</failed-action-output>`,
+    )
+    .replace(
+      "{CODEBASE_TREE}",
+      `<codebase-tree>\n${codebaseTree || "No codebase tree generated yet."}\n</codebase-tree>`,
+    );
+};
+
+const formatUserPrompt = (messages: BaseMessage[]): string => {
+  return userPrompt.replace(
+    "{CONVERSATION_HISTORY}",
+    messages.map(getMessageString).join("\n"),
+  );
+};
+
+export async function diagnoseError(
+  state: PlannerGraphState,
+  config: GraphConfig,
+): Promise<PlannerGraphUpdate> {
+  const lastFailedAction = state.messages.findLast(
+    (m) => isToolMessage(m) && m.status === "error",
+  );
+  if (!lastFailedAction?.content) {
+    throw new Error("No failed action found in messages");
+  }
+
+  logger.info("The last two tool calls resulted in errors. Diagnosing error.");
+
+  const model = await loadModel(config, Task.SUMMARIZER);
+  const modelWithTools = model.bindTools([diagnoseErrorTool], {
+    tool_choice: diagnoseErrorTool.name,
+    parallel_tool_calls: false,
+  });
+
+  const response = await modelWithTools.invoke([
+    {
+      role: "system",
+      content: formatSystemPrompt(state.messages, state.codebaseTree),
+    },
+    {
+      role: "user",
+      content: formatUserPrompt(state.messages),
+    },
+  ]);
+
+  const toolCall = response.tool_calls?.[0];
+  if (!toolCall) {
+    throw new Error("Failed to generate a tool call when diagnosing error.");
+  }
+
+  logger.info("Diagnosed error successfully.", {
+    diagnosis: (toolCall.args as z.infer<typeof diagnoseErrorTool.schema>)
+      .diagnosis,
+  });
+
+  const toolMessage = new ToolMessage({
+    tool_call_id: toolCall.id ?? "",
+    content: `Successfully diagnosed error. Please use the diagnosis to continue with the next action.`,
+    name: toolCall.name,
+    status: "success",
+    additional_kwargs: {
+      is_diagnosis: true,
+    },
+  });
+
+  return {
+    messages: [response, toolMessage],
+  };
+}

apps/open-swe/src/graphs/planner/nodes/take-action.ts

@@ -24,13 +24,15 @@ import { getRepoAbsolutePath } from "@open-swe/shared/git";
 import { createPlannerNotesTool } from "../../../tools/planner-notes.js";
 import { getMcpTools } from "../../../utils/mcp-client.js";
 import { getSandboxWithErrorHandling } from "../../../utils/sandbox.js";
+import { shouldDiagnoseError } from "../../../utils/tool-message-error.js";
+import { Command } from "@langchain/langgraph";
 
 const logger = createLogger(LogLevel.INFO, "TakeAction");
 
 export async function takeActions(
   state: PlannerGraphState,
   config: GraphConfig,
-): Promise<PlannerGraphUpdate> {
+): Promise<Command> {
   const { messages } = state;
   const lastMessage = messages[messages.length - 1];
 
@@ -184,10 +186,22 @@ ${tc.content}`,
     })),
   });
 
-  return {
+  const shouldRouteDiagnoseNode = shouldDiagnoseError([
+    ...state.messages,
+    ...toolCallResults,
+  ]);
+
+  const commandUpdate: PlannerGraphUpdate = {
     messages: toolCallResults,
     sandboxSessionId: sandbox.id,
     ...(codebaseTree && { codebaseTree }),
     ...(dependenciesInstalled !== null && { dependenciesInstalled }),
   };
+
+  return new Command({
+    goto: shouldRouteDiagnoseNode
+      ? "diagnose-error"
+      : "generate-plan-context-action",
+    update: commandUpdate,
+  });
 }

apps/open-swe/src/graphs/programmer/nodes/take-action.ts

@@ -23,10 +23,10 @@ import { truncateOutput } from "../../../utils/truncate-outputs.js";
 import { getSandboxWithErrorHandling } from "../../../utils/sandbox.js";
 import { getCodebaseTree } from "../../../utils/tree.js";
 import { getRepoAbsolutePath } from "@open-swe/shared/git";
-import { shouldDiagnoseError } from "../utils/tool-message-error.js";
 import { createInstallDependenciesTool } from "../../../tools/install-dependencies.js";
 import { createRgTool } from "../../../tools/rg.js";
 import { getMcpTools } from "../../../utils/mcp-client.js";
+import { shouldDiagnoseError } from "../../../utils/tool-message-error.js";
 
 const logger = createLogger(LogLevel.INFO, "TakeAction");
 

apps/open-swe/src/tests/sandbox.int.test.ts

@@ -1,89 +0,0 @@
-/* eslint-disable no-console */
-import { test, expect } from "@jest/globals";
-import { daytonaClient } from "../utils/sandbox.js";
-import { SANDBOX_ROOT_DIR } from "@open-swe/shared/constants";
-import { DEFAULT_SANDBOX_CREATE_PARAMS } from "../constants.js";
-
-test.skip("Can execute rg commands", async () => {
-  const githubToken = process.env.GITHUB_PAT;
-  if (!githubToken) {
-    throw new Error("GITHUB_PAT environment variable is not set");
-  }
-
-  const client = daytonaClient();
-
-  console.log("Setting up sandbox...");
-  const sandbox = await client.create(DEFAULT_SANDBOX_CREATE_PARAMS);
-  console.log("Setup sandbox:", sandbox.id);
-
-  const repoUrlWithToken = `https://x-access-token:${githubToken}@github.com/langchain-ai/open-swe.git`;
-  const cloneCommand = `git clone ${repoUrlWithToken}`;
-
-  console.log("Cloning repo...");
-  const cloneRes = await sandbox.process.executeCommand(
-    cloneCommand,
-    "/home/daytona",
-  );
-  expect(cloneRes.exitCode).toBe(0);
-
-  const testRes = await sandbox.process.executeCommand(
-    `script --return --quiet -c "$(cat <<'OPEN_SWE_X'
-rg -i logger
-OPEN_SWE_X
-)" /dev/null`,
-    "/home/daytona/open-swe",
-  );
-  console.log(
-    `test res status: ${testRes.exitCode}\ntest res output: ${testRes.result}`,
-  );
-
-  expect(testRes.exitCode).toBe(0);
-});
-
-const DEFAULT_ENV = {
-  // Prevents corepack from showing a y/n download prompt which causes the command to hang
-  COREPACK_ENABLE_DOWNLOAD_PROMPT: "0",
-};
-
-test("Installing dependencies", async () => {
-  const githubToken = process.env.GITHUB_PAT;
-  if (!githubToken) {
-    throw new Error("GITHUB_PAT environment variable is not set");
-  }
-
-  const client = daytonaClient();
-
-  console.log("Setting up sandbox...");
-  const sandbox = await client.create(DEFAULT_SANDBOX_CREATE_PARAMS);
-  console.log("Setup sandbox:", sandbox.id);
-
-  try {
-    const repoUrlWithToken = `https://x-access-token:${githubToken}@github.com/langchain-ai/open-swe.git`;
-    const cloneCommand = `git clone ${repoUrlWithToken}`;
-
-    console.log("Cloning repo...");
-    const cloneRes = await sandbox.process.executeCommand(
-      cloneCommand,
-      SANDBOX_ROOT_DIR,
-    );
-    expect(cloneRes.exitCode).toBe(0);
-
-    const installCommand = "yarn install";
-    const installRes = await sandbox.process.executeCommand(
-      installCommand,
-      `${SANDBOX_ROOT_DIR}/open-swe`,
-      DEFAULT_ENV,
-      120, // 120 seconds timeout
-    );
-    console.log(
-      `install res status: ${installRes.exitCode}\ninstall res output: ${installRes.result}`,
-    );
-    console.log(
-      `Install res exit code: ${installRes.exitCode}\nInstall res output: ${installRes.result}`,
-    );
-    expect(installRes.exitCode).toBe(0);
-  } finally {
-    await sandbox.delete();
-    console.log("Deleted sandbox:", sandbox.id);
-  }
-}, 300_000); // 5 minutes timeout

apps/open-swe/src/utils/tool-message-error.ts

@@ -1,8 +1,10 @@
 import {
+  BaseMessage,
   isAIMessage,
   isToolMessage,
   ToolMessage,
 } from "@langchain/core/messages";
+import { getMessageString } from "./message/content.js";
 
 /**
  * Group tool messages by their parent AI message
@@ -86,3 +88,36 @@ export function shouldDiagnoseError(messages: Array<any>) {
     (group) => calculateErrorRate(group) >= ERROR_THRESHOLD,
   );
 }
+
+export const getAllLastFailedActions = (messages: BaseMessage[]): string => {
+  const result: string[] = [];
+  let i = 0;
+
+  // Find pairs of AI messages followed by error tool messages
+  while (i < messages.length - 1) {
+    const currentMessage = messages[i];
+    const nextMessage = messages[i + 1];
+
+    if (
+      isAIMessage(currentMessage) &&
+      isToolMessage(nextMessage) &&
+      nextMessage?.status === "error"
+    ) {
+      // Add the AI message and its corresponding error tool message
+      result.push(getMessageString(currentMessage));
+      result.push(getMessageString(nextMessage));
+      i += 2; // Move to the next potential pair
+    } else if (
+      isToolMessage(currentMessage) &&
+      currentMessage?.status !== "error"
+    ) {
+      // Stop when we encounter a non-error tool message
+      break;
+    } else {
+      // Move to the next message if current one doesn't match our pattern
+      i++;
+    }
+  }
+
+  return result.join("\n");
+};

apps/web/src/app/api/[..._path]/route.ts

@@ -78,7 +78,6 @@ const isNewRunRequest = (reqUrlStr: string, reqMethod: string) => {
       reqPathnameParts?.[2] === "threads" &&
       validate(reqPathnameParts?.[3]) &&
       reqPathnameParts?.[4] === "runs" &&
-      reqPathnameParts.length === 5 &&
       reqMethod.toLowerCase() === "post";
     const isStreamRunReq =
       reqPathnameParts?.[1] === "api" &&