feat: Give the agent a tool to request help from the user (#47)

* feat: Give the agent a tool to request help from the user

* fix: open pr if no tool call generated by action node

apps/open-swe/src/index.ts

@@ -11,6 +11,7 @@ import {
   generateConclusion,
   openPullRequest,
   diagnoseError,
+  requestHelp,
 } from "./nodes/index.js";
 import { isAIMessage } from "@langchain/core/messages";
 import { plannerGraph } from "./subgraphs/index.js";
@@ -21,20 +22,26 @@ import { plannerGraph } from "./subgraphs/index.js";
  * Otherwise, it ends the process.
  *
  * @param {GraphState} state - The current graph state.
- * @returns {typeof END | "take-action"} The next node to execute, or END if the process should stop.
+ * @returns {"open-pr" | "take-action" | "request-help"} The next node to execute, or END if the process should stop.
  */
-async function takeActionOrEnd(
+async function routeGeneratedAction(
   state: GraphState,
-): Promise<typeof END | "take-action"> {
+): Promise<"open-pr" | "take-action" | "request-help"> {
   const { messages } = state;
   const lastMessage = messages[messages.length - 1];
 
   // If the message is an AI message, and it has tool calls, we should take action.
   if (isAIMessage(lastMessage) && lastMessage.tool_calls?.length) {
+    const toolCall = lastMessage.tool_calls[0];
+    if (toolCall.name === "request_human_help") {
+      return "request-help";
+    }
+
     return "take-action";
   }
 
-  return END;
+  // No tool calls, create PR then end.
+  return "open-pr";
 }
 
 const workflow = new StateGraph(GraphAnnotation, GraphConfiguration)
@@ -55,6 +62,9 @@ const workflow = new StateGraph(GraphAnnotation, GraphConfiguration)
     ends: ["generate-action", "generate-conclusion"],
   })
   .addNode("generate-conclusion", generateConclusion)
+  .addNode("request-help", requestHelp, {
+    ends: ["generate-action", END],
+  })
   .addNode("open-pr", openPullRequest)
   .addNode("diagnose-error", diagnoseError)
   .addEdge(START, "initialize")
@@ -62,7 +72,11 @@ const workflow = new StateGraph(GraphAnnotation, GraphConfiguration)
   .addEdge("generate-plan-subgraph", "interrupt-plan")
   // Always interrupt after rewriting the plan.
   .addEdge("rewrite-plan", "interrupt-plan")
-  .addConditionalEdges("generate-action", takeActionOrEnd, ["take-action", END])
+  .addConditionalEdges("generate-action", routeGeneratedAction, [
+    "take-action",
+    "request-help",
+    "open-pr",
+  ])
   .addEdge("generate-conclusion", "open-pr")
   .addEdge("diagnose-error", "generate-action")
   .addEdge("open-pr", END);

apps/open-swe/src/nodes/generate-message.ts

@@ -1,6 +1,10 @@
 import { GraphState, GraphConfig, GraphUpdate } from "../types.js";
 import { loadModel, Task } from "../utils/load-model.js";
-import { shellTool, applyPatchTool } from "../tools/index.js";
+import {
+  shellTool,
+  applyPatchTool,
+  requestHumanHelpTool,
+} from "../tools/index.js";
 import { getRepoAbsolutePath } from "../utils/git/index.js";
 import { formatPlanPrompt } from "../utils/plan-prompt.js";
 import { pauseSandbox } from "../utils/sandbox.js";
@@ -54,7 +58,8 @@ You MUST adhere to the following criteria when executing the task:
 - When using the \`shell\` tool, always take advantage of the \`workdir\` parameter to run commands inside the repo directory. You should not try to generate a command with \`cd <some path>\` as passing that path to \`workdir\` is much more efficient.
 - Always use the correct package manager to install dependencies. If the package manager is not already installed in the sandbox, use the \`shell\` tool to install it.
   - If the package manager fails to install, or you have issues installing dependencies, do not try to use a different package manager. Instead, skip installing dependencies.
-  - If installing dependencies fails, it can be useful to try again passing in a much longer timeout than the default.
+- If you are lacking enough context to complete the user's task, you may call the \`request_human_help\` tool to request help from the human.
+  - This tool should only be used if you have already tried to gather all the context you need, and are still unable to complete the user's task.
 - If completing the user's task requires writing or modifying files:
     - Your code and final answer should follow these *CODING GUIDELINES*:
         - Avoid writing to files which you have not already read.
@@ -117,7 +122,7 @@ export async function generateAction(
   config: GraphConfig,
 ): Promise<GraphUpdate> {
   const model = await loadModel(config, Task.ACTION_GENERATOR);
-  const tools = [shellTool, applyPatchTool];
+  const tools = [shellTool, applyPatchTool, requestHumanHelpTool];
   const modelWithTools = model.bindTools(tools, { tool_choice: "auto" });
 
   const response = await modelWithTools.invoke([

apps/open-swe/src/nodes/index.ts

@@ -8,3 +8,4 @@ export * from "./summarize-task-steps.js";
 export * from "./generate-conclusion.js";
 export * from "./open-pr.js";
 export * from "./diagnose-error.js";
+export * from "./request-help.js";

apps/open-swe/src/nodes/request-help.ts

@@ -0,0 +1,74 @@
+import { isAIMessage } from "@langchain/core/messages";
+import { GraphState } from "../types.js";
+import { HumanInterrupt, HumanResponse } from "@langchain/langgraph/prebuilt";
+import { END, interrupt, Command } from "@langchain/langgraph";
+import { pauseSandbox, resumeSandbox } from "../utils/sandbox.js";
+
+const constructDescription = (helpRequest: string): string => {
+  return `The agent has requested help. Here is the help request:
+  
+\`\`\`
+${helpRequest}
+\`\`\``;
+};
+
+export async function requestHelp(state: GraphState): Promise<Command> {
+  const lastMessage = state.messages[state.messages.length - 1];
+  if (!isAIMessage(lastMessage) || !lastMessage.tool_calls?.length) {
+    throw new Error("Last message is not an AI message with tool calls.");
+  }
+  const sandboxSessionId = state.sandboxSessionId;
+  if (!sandboxSessionId) {
+    throw new Error("Sandbox session ID not found.");
+  }
+  await pauseSandbox(sandboxSessionId);
+
+  const toolCall = lastMessage.tool_calls[0];
+
+  const interruptInput: HumanInterrupt = {
+    action_request: {
+      action: "Help Requested",
+      args: {},
+    },
+    config: {
+      allow_accept: false,
+      allow_edit: false,
+      allow_ignore: true,
+      allow_respond: true,
+    },
+    description: constructDescription(toolCall.args.help_request),
+  };
+  const interruptRes = interrupt<HumanInterrupt[], HumanResponse[]>([
+    interruptInput,
+  ])[0];
+
+  if (interruptRes.type === "ignore") {
+    return new Command({
+      goto: END,
+    });
+  }
+
+  if (interruptRes.type === "response") {
+    if (typeof interruptRes.args !== "string") {
+      throw new Error("Interrupt response expected to be a string.");
+    }
+    await resumeSandbox(sandboxSessionId);
+    return new Command({
+      goto: "generate-action",
+      update: {
+        messages: [
+          {
+            role: "tool",
+            tool_call_id: toolCall.id,
+            content: `Human response: ${interruptRes.args}`,
+            status: "success",
+          },
+        ],
+      },
+    });
+  }
+
+  throw new Error(
+    `Invalid interrupt response type. Must be one of 'ignore' or 'response'. Received: ${interruptRes.type}`,
+  );
+}

apps/open-swe/src/tools/index.ts

@@ -1,3 +1,4 @@
 export * from "./apply-patch.js";
 export * from "./shell.js";
 export * from "./session-plan.js";
+export * from "./request-human-help.js";

apps/open-swe/src/tools/request-human-help.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+
+const requestHumanHelpSchema = z.object({
+  help_request: z
+    .string()
+    .describe(
+      "The help request to send to the human. Should be concise, but descriptive.",
+    ),
+});
+
+export const requestHumanHelpTool = {
+  name: "request_human_help",
+  schema: requestHumanHelpSchema,
+  description:
+    "Use this tool to request help from the human. This should only be called if you are stuck, and you are unable to continue. This will pause your execution until the user responds. You will not be able to go back and fourth with the user, so ensure the help request contains all of the necessary information and context the user might need to respond to your request.",
+};