diff --git a/apps/open-swe/src/__tests__/diagnose-error.test.ts b/apps/open-swe/src/__tests__/diagnose-error.test.ts new file mode 100644 index 00000000..e8df21a4 --- /dev/null +++ b/apps/open-swe/src/__tests__/diagnose-error.test.ts @@ -0,0 +1,181 @@ +import { describe, it, expect } from "@jest/globals"; +import { AIMessage, ToolMessage, HumanMessage } from "@langchain/core/messages"; +import { getAllLastFailedActions } from "../utils/tool-message-error.js"; + +describe("getAllLastFailedActions", () => { + it("should return empty string for empty messages array", () => { + const result = getAllLastFailedActions([]); + expect(result).toBe(""); + }); + + it("should return AI and error tool message pairs until a non-error tool message is encountered", () => { + // Create test messages + const aiMessage1 = new AIMessage({ + content: "I'll try to execute this command", + id: "ai-1", + }); + + const errorToolMessage1 = new ToolMessage({ + content: "Command failed: Permission denied", + tool_call_id: "tool-1", + name: "shell", + status: "error", + }); + + const aiMessage2 = new AIMessage({ + content: "Let me try a different approach", + id: "ai-2", + }); + + const errorToolMessage2 = new ToolMessage({ + content: "Error: File not found", + tool_call_id: "tool-2", + name: "read_file", + status: "error", + }); + + const aiMessage3 = new AIMessage({ + content: "Let me try something else", + id: "ai-3", + }); + + const successToolMessage = new ToolMessage({ + content: "Command executed successfully", + tool_call_id: "tool-3", + name: "shell", + status: "success", + }); + + const aiMessage4 = new AIMessage({ + content: "Let me try one more thing", + id: "ai-4", + }); + + const errorToolMessage3 = new ToolMessage({ + content: "Error: Invalid syntax", + tool_call_id: "tool-4", + name: "shell", + status: "error", + }); + + const messages = [ + aiMessage1, + errorToolMessage1, + aiMessage2, + errorToolMessage2, + aiMessage3, + successToolMessage, + aiMessage4, + errorToolMessage3, + ]; + + const result = getAllLastFailedActions(messages); + + // Should include the first two AI+error pairs, but stop at the success message + expect(result).toContain("I'll try to execute this command"); + expect(result).toContain("Command failed: Permission denied"); + expect(result).toContain("Let me try a different approach"); + expect(result).toContain("Error: File not found"); + + // Should not include messages after the success message + expect(result).not.toContain("Let me try one more thing"); + expect(result).not.toContain("Error: Invalid syntax"); + }); + + it("should handle non-sequential AI and tool messages", () => { + const aiMessage = new AIMessage({ + content: "I'll try to execute this command", + id: "ai-1", + }); + + const humanMessage = new HumanMessage({ + content: "Can you try something else?", + id: "human-1", + }); + + const errorToolMessage = new ToolMessage({ + content: "Command failed: Permission denied", + tool_call_id: "tool-1", + name: "shell", + status: "error", + }); + + const messages = [aiMessage, humanMessage, errorToolMessage]; + + const result = getAllLastFailedActions(messages); + + // Should not include any messages since there's no AI+error pair + expect(result).toBe(""); + }); + + it("should handle a mix of error and non-error tool messages", () => { + const aiMessage1 = new AIMessage({ + content: "First command", + id: "ai-1", + }); + + const successToolMessage1 = new ToolMessage({ + content: "Success", + tool_call_id: "tool-1", + name: "shell", + status: "success", + }); + + const aiMessage2 = new AIMessage({ + content: "Second command", + id: "ai-2", + }); + + const errorToolMessage = new ToolMessage({ + content: "Error occurred", + tool_call_id: "tool-2", + name: "shell", + status: "error", + }); + + const messages = [ + aiMessage1, + successToolMessage1, + aiMessage2, + errorToolMessage, + ]; + + const result = getAllLastFailedActions(messages); + + // Should not include any messages since we encounter a success message first + expect(result).toBe(""); + }); + + it("should handle multiple tool messages after an AI message", () => { + const aiMessage = new AIMessage({ + content: "Let me try multiple commands", + id: "ai-1", + }); + + const errorToolMessage1 = new ToolMessage({ + content: "First command failed", + tool_call_id: "tool-1", + name: "shell", + status: "error", + }); + + const errorToolMessage2 = new ToolMessage({ + content: "Second command failed", + tool_call_id: "tool-2", + name: "read_file", + status: "error", + }); + + const messages = [aiMessage, errorToolMessage1, errorToolMessage2]; + + const result = getAllLastFailedActions(messages); + + // Should include the AI message and the first error tool message + expect(result).toContain("Let me try multiple commands"); + expect(result).toContain("First command failed"); + + // The second error tool message should not be paired with the AI message + // since we're looking for AI+tool pairs + expect(result).not.toContain("Second command failed"); + }); +}); diff --git a/apps/open-swe/src/graphs/programmer/__tests__/take-action.test.ts b/apps/open-swe/src/__tests__/take-action.test.ts similarity index 100% rename from apps/open-swe/src/graphs/programmer/__tests__/take-action.test.ts rename to apps/open-swe/src/__tests__/take-action.test.ts diff --git a/apps/open-swe/src/graphs/planner/index.ts b/apps/open-swe/src/graphs/planner/index.ts index 0a2e4da8..040fbaee 100644 --- a/apps/open-swe/src/graphs/planner/index.ts +++ b/apps/open-swe/src/graphs/planner/index.ts @@ -18,6 +18,7 @@ import { } from "./nodes/index.js"; import { isAIMessage } from "@langchain/core/messages"; import { initializeSandbox } from "../shared/initialize-sandbox.js"; +import { diagnoseError } from "./nodes/diagnose-error.js"; function takeActionOrGeneratePlan( state: PlannerGraphState, @@ -48,7 +49,9 @@ const workflow = new StateGraph(PlannerGraphStateObj, GraphConfiguration) }) .addNode("initialize-sandbox", initializeSandbox) .addNode("generate-plan-context-action", generateAction) - .addNode("take-plan-actions", takeActions) + .addNode("take-plan-actions", takeActions, { + ends: ["generate-plan-context-action", "diagnose-error"], + }) .addNode("generate-plan", generatePlan) .addNode("notetaker", notetaker) .addNode("interrupt-proposed-plan", interruptProposedPlan, { @@ -57,6 +60,7 @@ const workflow = new StateGraph(PlannerGraphStateObj, GraphConfiguration) .addNode("determine-needs-context", determineNeedsContext, { ends: ["generate-plan-context-action", "generate-plan"], }) + .addNode("diagnose-error", diagnoseError) .addEdge(START, "prepare-graph-state") .addEdge("initialize-sandbox", "generate-plan-context-action") .addConditionalEdges( @@ -64,7 +68,7 @@ const workflow = new StateGraph(PlannerGraphStateObj, GraphConfiguration) takeActionOrGeneratePlan, ["take-plan-actions", "generate-plan"], ) - .addEdge("take-plan-actions", "generate-plan-context-action") + .addEdge("diagnose-error", "generate-plan-context-action") .addEdge("generate-plan", "notetaker") .addEdge("notetaker", "interrupt-proposed-plan"); diff --git a/apps/open-swe/src/graphs/planner/nodes/diagnose-error.ts b/apps/open-swe/src/graphs/planner/nodes/diagnose-error.ts new file mode 100644 index 00000000..a192aa0a --- /dev/null +++ b/apps/open-swe/src/graphs/planner/nodes/diagnose-error.ts @@ -0,0 +1,127 @@ +import { + BaseMessage, + isToolMessage, + ToolMessage, +} from "@langchain/core/messages"; +import { createDiagnoseErrorToolFields } from "@open-swe/shared/open-swe/tools"; + +import { getMessageString } from "../../../utils/message/content.js"; +import { loadModel, Task } from "../../../utils/load-model.js"; +import { z } from "zod"; +import { createLogger, LogLevel } from "../../../utils/logger.js"; +import { getAllLastFailedActions } from "../../../utils/tool-message-error.js"; +import { + PlannerGraphState, + PlannerGraphUpdate, +} from "@open-swe/shared/open-swe/planner/types"; +import { GraphConfig } from "@open-swe/shared/open-swe/types"; + +const logger = createLogger(LogLevel.INFO, "DiagnoseError"); + +const systemPrompt = `You are operating as a terminal-based agentic coding assistant built by LangChain. It wraps LLM models to enable natural language interaction with a local codebase. You are expected to be precise, safe, and helpful. + +The last few commands you tried to execute failed with an error. Please carefully diagnose the error, and provide a helpful explanation of exactly what the issue is, and how you can fix it. + +Following these rules when diagnosing the error: + - You should provide a clear, concise, and helpful explanation of exactly what the issue is, and how you can fix it. + - You do not want to be overly verbose in your diagnosis. You should only include information which is directly relevant to diagnosing and fixing the error. + - NEVER make up reasons, or make a guess as to what the issue is. Your reasoning must ALWAYS be grounded in the information provided to you. + - Making up reasons, or making a guess can lead to more problems, so it's best to say you don't know rather than make up a reason. + - Reference specific lines of code, or context from the conversation history to support your diagnosis. + +Here are the last actions you attempted which resulted in errors: +{FAILED_ACTIONS_OUTPUTS} + +Below is an up to date tree of the codebase (going 3 levels deep). This is up to date, and is updated after every action you take. Always assume this is the most up to date context about the codebase. +It was generated by using the \`tree\` command, passing in the gitignore file to ignore files and directories you should not have access to (\`git ls-files | tree --fromfile -L 3\`). It is always executed inside the repo directory: {REPO_DIRECTORY} +{CODEBASE_TREE} + +Please carefully go over all of this information, and provide a helpful explanation of exactly what the issue is, and how you can fix it. When you are ready to provide your diagnosis, call the \`diagnose_error\` tool. +`; + +const userPrompt = `Here is the full conversation history from the steps taken to complete the current task, along with the user's initial request: + +{CONVERSATION_HISTORY} + +Please carefully go over all of this information, and provide a helpful explanation of exactly what the issue is, and how you can fix it. When you are ready to provide your diagnosis, call the \`diagnose_error\` tool.`; + +const diagnoseErrorTool = createDiagnoseErrorToolFields(); + +const formatSystemPrompt = ( + messages: BaseMessage[], + codebaseTree: string, +): string => { + const lastFailedActions = getAllLastFailedActions(messages); + + return systemPrompt + .replace( + "{FAILED_ACTIONS_OUTPUTS}", + `${lastFailedActions}`, + ) + .replace( + "{CODEBASE_TREE}", + `\n${codebaseTree || "No codebase tree generated yet."}\n`, + ); +}; + +const formatUserPrompt = (messages: BaseMessage[]): string => { + return userPrompt.replace( + "{CONVERSATION_HISTORY}", + messages.map(getMessageString).join("\n"), + ); +}; + +export async function diagnoseError( + state: PlannerGraphState, + config: GraphConfig, +): Promise { + const lastFailedAction = state.messages.findLast( + (m) => isToolMessage(m) && m.status === "error", + ); + if (!lastFailedAction?.content) { + throw new Error("No failed action found in messages"); + } + + logger.info("The last two tool calls resulted in errors. Diagnosing error."); + + const model = await loadModel(config, Task.SUMMARIZER); + const modelWithTools = model.bindTools([diagnoseErrorTool], { + tool_choice: diagnoseErrorTool.name, + parallel_tool_calls: false, + }); + + const response = await modelWithTools.invoke([ + { + role: "system", + content: formatSystemPrompt(state.messages, state.codebaseTree), + }, + { + role: "user", + content: formatUserPrompt(state.messages), + }, + ]); + + const toolCall = response.tool_calls?.[0]; + if (!toolCall) { + throw new Error("Failed to generate a tool call when diagnosing error."); + } + + logger.info("Diagnosed error successfully.", { + diagnosis: (toolCall.args as z.infer) + .diagnosis, + }); + + const toolMessage = new ToolMessage({ + tool_call_id: toolCall.id ?? "", + content: `Successfully diagnosed error. Please use the diagnosis to continue with the next action.`, + name: toolCall.name, + status: "success", + additional_kwargs: { + is_diagnosis: true, + }, + }); + + return { + messages: [response, toolMessage], + }; +} diff --git a/apps/open-swe/src/graphs/planner/nodes/take-action.ts b/apps/open-swe/src/graphs/planner/nodes/take-action.ts index 06bbfd54..816c47dd 100644 --- a/apps/open-swe/src/graphs/planner/nodes/take-action.ts +++ b/apps/open-swe/src/graphs/planner/nodes/take-action.ts @@ -24,13 +24,15 @@ import { getRepoAbsolutePath } from "@open-swe/shared/git"; import { createPlannerNotesTool } from "../../../tools/planner-notes.js"; import { getMcpTools } from "../../../utils/mcp-client.js"; import { getSandboxWithErrorHandling } from "../../../utils/sandbox.js"; +import { shouldDiagnoseError } from "../../../utils/tool-message-error.js"; +import { Command } from "@langchain/langgraph"; const logger = createLogger(LogLevel.INFO, "TakeAction"); export async function takeActions( state: PlannerGraphState, config: GraphConfig, -): Promise { +): Promise { const { messages } = state; const lastMessage = messages[messages.length - 1]; @@ -184,10 +186,22 @@ ${tc.content}`, })), }); - return { + const shouldRouteDiagnoseNode = shouldDiagnoseError([ + ...state.messages, + ...toolCallResults, + ]); + + const commandUpdate: PlannerGraphUpdate = { messages: toolCallResults, sandboxSessionId: sandbox.id, ...(codebaseTree && { codebaseTree }), ...(dependenciesInstalled !== null && { dependenciesInstalled }), }; + + return new Command({ + goto: shouldRouteDiagnoseNode + ? "diagnose-error" + : "generate-plan-context-action", + update: commandUpdate, + }); } diff --git a/apps/open-swe/src/graphs/programmer/nodes/take-action.ts b/apps/open-swe/src/graphs/programmer/nodes/take-action.ts index 4033ea86..aa0c30d0 100644 --- a/apps/open-swe/src/graphs/programmer/nodes/take-action.ts +++ b/apps/open-swe/src/graphs/programmer/nodes/take-action.ts @@ -23,10 +23,10 @@ import { truncateOutput } from "../../../utils/truncate-outputs.js"; import { getSandboxWithErrorHandling } from "../../../utils/sandbox.js"; import { getCodebaseTree } from "../../../utils/tree.js"; import { getRepoAbsolutePath } from "@open-swe/shared/git"; -import { shouldDiagnoseError } from "../utils/tool-message-error.js"; import { createInstallDependenciesTool } from "../../../tools/install-dependencies.js"; import { createRgTool } from "../../../tools/rg.js"; import { getMcpTools } from "../../../utils/mcp-client.js"; +import { shouldDiagnoseError } from "../../../utils/tool-message-error.js"; const logger = createLogger(LogLevel.INFO, "TakeAction"); diff --git a/apps/open-swe/src/tests/sandbox.int.test.ts b/apps/open-swe/src/tests/sandbox.int.test.ts deleted file mode 100644 index 40ebd859..00000000 --- a/apps/open-swe/src/tests/sandbox.int.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* eslint-disable no-console */ -import { test, expect } from "@jest/globals"; -import { daytonaClient } from "../utils/sandbox.js"; -import { SANDBOX_ROOT_DIR } from "@open-swe/shared/constants"; -import { DEFAULT_SANDBOX_CREATE_PARAMS } from "../constants.js"; - -test.skip("Can execute rg commands", async () => { - const githubToken = process.env.GITHUB_PAT; - if (!githubToken) { - throw new Error("GITHUB_PAT environment variable is not set"); - } - - const client = daytonaClient(); - - console.log("Setting up sandbox..."); - const sandbox = await client.create(DEFAULT_SANDBOX_CREATE_PARAMS); - console.log("Setup sandbox:", sandbox.id); - - const repoUrlWithToken = `https://x-access-token:${githubToken}@github.com/langchain-ai/open-swe.git`; - const cloneCommand = `git clone ${repoUrlWithToken}`; - - console.log("Cloning repo..."); - const cloneRes = await sandbox.process.executeCommand( - cloneCommand, - "/home/daytona", - ); - expect(cloneRes.exitCode).toBe(0); - - const testRes = await sandbox.process.executeCommand( - `script --return --quiet -c "$(cat <<'OPEN_SWE_X' -rg -i logger -OPEN_SWE_X -)" /dev/null`, - "/home/daytona/open-swe", - ); - console.log( - `test res status: ${testRes.exitCode}\ntest res output: ${testRes.result}`, - ); - - expect(testRes.exitCode).toBe(0); -}); - -const DEFAULT_ENV = { - // Prevents corepack from showing a y/n download prompt which causes the command to hang - COREPACK_ENABLE_DOWNLOAD_PROMPT: "0", -}; - -test("Installing dependencies", async () => { - const githubToken = process.env.GITHUB_PAT; - if (!githubToken) { - throw new Error("GITHUB_PAT environment variable is not set"); - } - - const client = daytonaClient(); - - console.log("Setting up sandbox..."); - const sandbox = await client.create(DEFAULT_SANDBOX_CREATE_PARAMS); - console.log("Setup sandbox:", sandbox.id); - - try { - const repoUrlWithToken = `https://x-access-token:${githubToken}@github.com/langchain-ai/open-swe.git`; - const cloneCommand = `git clone ${repoUrlWithToken}`; - - console.log("Cloning repo..."); - const cloneRes = await sandbox.process.executeCommand( - cloneCommand, - SANDBOX_ROOT_DIR, - ); - expect(cloneRes.exitCode).toBe(0); - - const installCommand = "yarn install"; - const installRes = await sandbox.process.executeCommand( - installCommand, - `${SANDBOX_ROOT_DIR}/open-swe`, - DEFAULT_ENV, - 120, // 120 seconds timeout - ); - console.log( - `install res status: ${installRes.exitCode}\ninstall res output: ${installRes.result}`, - ); - console.log( - `Install res exit code: ${installRes.exitCode}\nInstall res output: ${installRes.result}`, - ); - expect(installRes.exitCode).toBe(0); - } finally { - await sandbox.delete(); - console.log("Deleted sandbox:", sandbox.id); - } -}, 300_000); // 5 minutes timeout diff --git a/apps/open-swe/src/graphs/programmer/utils/tool-message-error.ts b/apps/open-swe/src/utils/tool-message-error.ts similarity index 72% rename from apps/open-swe/src/graphs/programmer/utils/tool-message-error.ts rename to apps/open-swe/src/utils/tool-message-error.ts index 367a25ca..6e26690c 100644 --- a/apps/open-swe/src/graphs/programmer/utils/tool-message-error.ts +++ b/apps/open-swe/src/utils/tool-message-error.ts @@ -1,8 +1,10 @@ import { + BaseMessage, isAIMessage, isToolMessage, ToolMessage, } from "@langchain/core/messages"; +import { getMessageString } from "./message/content.js"; /** * Group tool messages by their parent AI message @@ -86,3 +88,36 @@ export function shouldDiagnoseError(messages: Array) { (group) => calculateErrorRate(group) >= ERROR_THRESHOLD, ); } + +export const getAllLastFailedActions = (messages: BaseMessage[]): string => { + const result: string[] = []; + let i = 0; + + // Find pairs of AI messages followed by error tool messages + while (i < messages.length - 1) { + const currentMessage = messages[i]; + const nextMessage = messages[i + 1]; + + if ( + isAIMessage(currentMessage) && + isToolMessage(nextMessage) && + nextMessage?.status === "error" + ) { + // Add the AI message and its corresponding error tool message + result.push(getMessageString(currentMessage)); + result.push(getMessageString(nextMessage)); + i += 2; // Move to the next potential pair + } else if ( + isToolMessage(currentMessage) && + currentMessage?.status !== "error" + ) { + // Stop when we encounter a non-error tool message + break; + } else { + // Move to the next message if current one doesn't match our pattern + i++; + } + } + + return result.join("\n"); +}; diff --git a/apps/web/src/app/api/[..._path]/route.ts b/apps/web/src/app/api/[..._path]/route.ts index bbbc6799..fb1d7fb1 100644 --- a/apps/web/src/app/api/[..._path]/route.ts +++ b/apps/web/src/app/api/[..._path]/route.ts @@ -78,7 +78,6 @@ const isNewRunRequest = (reqUrlStr: string, reqMethod: string) => { reqPathnameParts?.[2] === "threads" && validate(reqPathnameParts?.[3]) && reqPathnameParts?.[4] === "runs" && - reqPathnameParts.length === 5 && reqMethod.toLowerCase() === "post"; const isStreamRunReq = reqPathnameParts?.[1] === "api" &&