From 708a9f6b1a82d66a19084d8fa181923c807a35c9 Mon Sep 17 00:00:00 2001 From: Dylan Boudro <121908331+starmorph@users.noreply.github.com> Date: Tue, 17 Jun 2025 09:37:52 -0700 Subject: [PATCH] feat: hookup shell & apply patch tools to client side messages stream (#161) * add action-step tool component into UI for apply_patch and shell tool calls * improve types * update open-swe zod to match web/ zod * CR: improving action-step implementation * CR: generating state of action-step * cr --------- Co-authored-by: Brace Sproul --- apps/web/public/favicon.png | Bin 2610 -> 0 bytes apps/web/src/app/favicon.ico | Bin 4286 -> 4286 bytes .../web/src/components/gen-ui/action-step.tsx | 7 +- .../web/src/components/thread/messages/ai.tsx | 190 +++++++++++++++++- apps/web/src/lib/langchain-messages.ts | 14 ++ 5 files changed, 199 insertions(+), 12 deletions(-) delete mode 100644 apps/web/public/favicon.png create mode 100644 apps/web/src/lib/langchain-messages.ts diff --git a/apps/web/public/favicon.png b/apps/web/public/favicon.png deleted file mode 100644 index 38454444bdce43df7d5f16a3f90b70f8db095913..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2610 zcmV-23eEM2P)$-pqIduk2v32{b_MvZZWMZEzA&0!oSr5D22kfQm$ZBvMg?TU7ymBqpr@4cG|=yc-*P#xwoSwLLY?%y{0|?~#~KdLGZ4`=JDp8ZUXuzC9U8` zIyyg6qT=JDNx~udq6lEmr}1MQ9v@5~3I8Ptxhe;cfP9xhiB2xy3g9;rP;Rr?p0G;q z)A%$z0%&-o2yJL+D7G=BfyTe5luU z`;HyMf;Znp1c{PkjD`g|5lG?^Lmt%EBlCj~d>Su!>6vHn*SFrnclYkW4Cp2xkUpum z+cA=ck4a4QtNpweU&M10CZLYT84cY81cI8ndi-|rV!zwVsPOctQTp;gfG~xC24?&r z0|)Bzo{8hfBgks?{9_05VhRDxqm-WeE~}{UPFM%#MpFo|actjz^oXuwPMkTTa);3{ zg@8~Ff0>64!$}e3&(Ky|SBI=qr(v@ge|x475EdMai&w8=%|9|!V)I5pGdaIsvj*Q> zyZV1sXdtXC5UhkQXwaL5#Ht>CvOUul=Z)HXogQ{w4F}VK#ESQmszP)-OzHeV#Ehs<{=K>$?-HSqgiV74snTm_ zS!E@L4IYe>AAgL!8#Z9#xUtj(PUQ_@AtB00Npun(q0AFPNF62f@F7F6_5C$CvuhVZ zx^`9jjDlevAcjUqM&j`1&FJ5!kJ^`4ScvBqEP$N`c~f~sMMWsi$wANV-PQNYd3ktt z{;v^iwHQ(!FvJ7Ixayi3q%T?GHoo3okNL}%p@LrFOX9}H_QROeR23G2gh7L_Y|%p8 zVJPUj2S3}jp8&?L#B&u5uDX9;ZE~~ICQV|Zo8fM@jr08TmoBS%fNyBH&s_1Bty|rD zmMvO@7m|~4kLs|p=^C<0u8`c(o;gFnAzOx(GY36mV)(nK%QYL&^c&lhu}ZXm(h8>e<%VJMssY z-V_?^h0z=Za=-Y(XD^r1D`qTRif^u8M<^4kUJ?Qd0lAe(X6I9xnO=<-NVfkcGkFQ) z-e2PdVuFkiMI|NJyff3Mi+7`7I>K-n6d$jOE_x9V0r$$w@wXK#+KgurG7D$U#A0sc zBtb?7Ut7!GtEj$@xzncd_*rNRq8&D@T&alD>jD0EOB3V*VZkAoI&or~2}o3&v{^Fu zRi*aA=`}4Ohc@4@h@dO^`MAqom*X4_4$S=J6!?q8C!fUl5yNqodBUb6De5Mmk`>(L z9NzrRywYyOu%S4-WlNiFm#-|yS*__z>AF2Y>iC;??Z%>4<|ygv4`k6pbb!Ss z?L8aQH8d2LIAspur<^zcRv|83yT+O0X!rx6?fQ)y3?F@(KsNz0)a0;QqZTR)26w#!YgmQ zfvl4!)x2NV!Y_}NkQ0^Hh51K@xaQ+)xKHVl03@*c{D#y6cr!Ai%h|aTgX0>dvbgc+ z)EN{h@K_IgR4}F@#9otC9Z5{$9nw2e7uc|Jh4{58#ZG_GAq03zZ&K#kbMPSkl(ETO z3YSfd1SYKc`3u9V!V(HQ)~&_*kG65K+K-lY;GsG~Pn zEmmAPdKB@ou@AMAG5^fPOL!qIjdD8_ns~!Qxx$!!GG}#Ivnb=ojU0)*tYe7l*YA66 zMEHyy{xf`i;B!PrN2~T`z!U=P4m-Nh3o_n*+nobEg8O58_r{*}>*0`D;Scc{p+F$j z+tG;$=+m>O&vwnU{DkGLnUg06RN~SLlOB<&!ylHurt6re5)$2V+DQ0KKzl}RIW|b= z`d-4&A*$AAG)(aTiDYMkQ`a%MdHL?uAtT{80qsdg@_@wZO@G&Q%!!;+YHh%1nDdAX z3CKE~qe|ocAph1a95|k>jx!pj5Fiq_wzdXo>FI4|c3D)N_4@0Kf6EW?Il;jBk^;)V zS-^OSCG`_$&U$R~IoJ6TrcT98E|CX&>-`SqM2k&M&Nu6|I8{KQ2gn4d2ZhmV^X6d? z>jf8bbFn3JCl^F3Rpyt?zns(V`N4T@KvL3fo+1?{L$rS>SAgwy2SV7}F?t`7S7;{z zF4tZgd;d5~S*8;p+P?=EHrt2xNoB5F@3B~dg0_&yay({`7b)c>N=rdjRyn^d#N&rm z@<(Y+d4G=OahbGFz~g~cg(ps^_axn7ns*UQt)bF}N#=ALh{-!s(fv@X~5rq}9Bd$X(~A+OI{i$ScmMzZ diff --git a/apps/web/src/app/favicon.ico b/apps/web/src/app/favicon.ico index 2a7a6db085f6c91063bb097c2540cf34317e2b21..f900fc7500df69d48e3c30cf8dfcdbcdbc49d1e4 100644 GIT binary patch literal 4286 zcmc&&dq`Ai6u;3e;ZB5YB+%qT1S2i4X0aq3hq5)u|cqnV~i4&f@!HMTV;CllS> z?;$^K_b1%DcLS?e&sFP(fb`~AM@P>_O-+Ri?c;rYz1X`qMZ`j>KLg#*_;GRb(c1dV zhbzAf4h|si@SlY5&5J-~D&5CA6c?YBVSW7d>o%CpdNnx^knd$Z!^0;dDQOwX%df-X z=!VnjKwDc2va{2I8hggqXr^Mz76VdJG%y(EAR{9YyLOpiGA$2U+_l;evTK@%tI4zi z&tEj+!Gl|r>tSNeTl@ZX$1&nF%w@l+;atyAQL~YpyqstSN=u7`zYphHz8Pmevu4dg ztY7~JjK<#uz8XEWB*$#l;V#9wz5NZ*Tew_77j^YjsI9FORA2uP=g%J%xq0sF5!&M!b*7@ihG>eTpRc~t z^VycPt+D9rw2zWyx3}ZYof6Uex&Pn3{V$Fm&ldJstvPVJ-J{O%{#A5O1)hV`#eSQu z*iV+n724r1MWk^pRO)D_V3@M$p5S-*N@S-1eY&c@#1*{<8b2~mHosz=ycrk7l^yOx$a{Y@ z=RAm~yn6Mjc@#eP=7NIVA|HMCJ9j22uJbRl1FKQ4^Q^25qR)6d z-d@+=@5GfWg`#JT>&Ka2ab0wVP+DE`8%Kg KJ@Atr#{LZb)P7U| literal 4286 zcmeHL?NgLh6o2{;`qXD9UPLR+tQo7(v{Bd>Kz9LI-cc9<%}|hd3CEx$c?%ko*g%{d zCI~E{#>vEC0&+%MVFnGy*O5`Ffm#q;dD#Vi-E)@vv=746#E1N_&(3-7-FweH=XZYR z+Wy=(LD~-=8%5#dM1PBpB%0?kH&X1V$S>+b@fTFxC67Zv;P<)=?o0?qq(UD`T0pm*`-H9{6@sbz4f0ZIVliX2cl71 zQ;ciZdN4ihS`dfRIfWn3HX}JX2)ZDD1a9#`P@wOeP~S(4Jk;_4PV`NHJ|&jhN^+5oLKD`_dx>Ck8}Byf$aoSJjjH zdYvB(`sD~Vt`zwEH1qO%NpYqILs!>Xn9XYz)PHxc8C6wvyVROZFQehBDs*<7L0YPjIr+K@W3C7ZjfB#)QY_<;UeaFOklKe=U z)Y4eRC%Gsu&sI6{oR>Nl%Q{@X+{wP>i!jt3FV(Cw8Uh$kb@fM_AL504m_$B^KdK8) z`!tudb2GBCVxFiusYiAGY32KvnVFHUxuqP?o}~IFU8=WZ#|qHje-j-)ok3#4CPak4 z%DmZZo#^WP0dcYGHF#+q@t&DsS=2w-ZEO2h>VI@}MC2<+!=qeNSg@OG zl}Gx7oI*Z@0cLJ}lgW0a4d<&Qk44 zJB0u0RU4+JCe_$mTTgOK^XyWF9?rqz|)c4d~pVr={gf=0Mt} zyj!ae!{u`F{QdiPxRxZw`S#>@nomtJ@|i*lt=6Ii{ZnqsO0zIAG0yghW8zk?Th2TZ z4#M|^^<$hmRg3-mwrlbu{Vmi|?kX$u&>+qx_a4#iJGxggGotvOW9M_Pv9X6ZUR#3j zu(e#LgptI3hfwSHuZ4J!caL00e(6pqEZBp4_wIPQR{Kt;6W!fiC@nc4&W0_#hv__D ziiCx|g1ER1ICwCb_p-w=ruti~<8rygZ;sz_{d%uBqc1H<`ue(=YdQnGkCy>215XzN K@_&H;tNjc8Cxl`E diff --git a/apps/web/src/components/gen-ui/action-step.tsx b/apps/web/src/components/gen-ui/action-step.tsx index 15644544..24e3da18 100644 --- a/apps/web/src/components/gen-ui/action-step.tsx +++ b/apps/web/src/components/gen-ui/action-step.tsx @@ -1,6 +1,5 @@ "use client"; -import "../app/globals.css"; import { useState } from "react"; import { Terminal, @@ -25,7 +24,7 @@ type BaseActionProps = { // Shell command specific props type ShellActionProps = BaseActionProps & { actionType: "shell"; - command: string; + command: string[]; workdir?: string; output?: string; errorCode?: number; @@ -41,7 +40,7 @@ type PatchActionProps = BaseActionProps & { }; // Union type for all possible action props -type ActionStepProps = +export type ActionStepProps = | (BaseActionProps & { status: "loading" }) | ShellActionProps | PatchActionProps; @@ -136,7 +135,7 @@ export function ActionStep(props: ActionStepProps) { )} - {props.command} + {props.command.join(" ")} ); diff --git a/apps/web/src/components/thread/messages/ai.tsx b/apps/web/src/components/thread/messages/ai.tsx index 1b735b35..56c901c4 100644 --- a/apps/web/src/components/thread/messages/ai.tsx +++ b/apps/web/src/components/thread/messages/ai.tsx @@ -1,6 +1,11 @@ import { parsePartialJson } from "@langchain/core/output_parsers"; import { useStreamContext } from "@/providers/Stream"; -import { AIMessage, Checkpoint, Message } from "@langchain/langgraph-sdk"; +import { + AIMessage, + Checkpoint, + Message, + ToolMessage, +} from "@langchain/langgraph-sdk"; import { getContentString } from "../utils"; import { BranchSwitcher, CommandBar } from "./shared"; import { MarkdownText } from "../markdown-text"; @@ -12,6 +17,24 @@ import { Fragment } from "react/jsx-runtime"; import { useQueryState, parseAsBoolean } from "nuqs"; import { useArtifact } from "../artifact"; import { Interrupt } from "./interrupt"; +import { + ActionStep, + type ActionStepProps, +} from "@/components/gen-ui/action-step"; +import { ToolCall } from "@langchain/core/messages/tool"; +import { + createApplyPatchToolFields, + createShellToolFields, +} from "@open-swe/shared/open-swe/tools"; +import { z } from "zod"; +import { isAIMessageSDK, isToolMessageSDK } from "@/lib/langchain-messages"; + +// Used only for Zod type inference. +const dummyRepo = { owner: "dummy", repo: "dummy" }; +const shellTool = createShellToolFields(dummyRepo); +type ShellToolArgs = z.infer; +const applyPatchTool = createApplyPatchToolFields(dummyRepo); +type ApplyPatchToolArgs = z.infer; function CustomComponent({ message, @@ -65,6 +88,54 @@ function parseAnthropicStreamedToolCalls( }); } +export function mapToolMessageToActionStepProps( + message: ToolMessage, + thread: { messages: Message[] }, +): ActionStepProps { + const toolCall: ToolCall | undefined = thread.messages + .filter(isAIMessageSDK) + .flatMap((m) => m.tool_calls ?? []) + .find((tc) => tc.id === message.tool_call_id); + + const aiMessage = thread.messages + .filter(isAIMessageSDK) + .find((m) => m.tool_calls?.some((tc) => tc.id === message.tool_call_id)); + const reasoningText = aiMessage + ? getContentString(aiMessage.content) + : undefined; + + const status: ActionStepProps["status"] = "done"; + const success = message.status === "success"; + + if (toolCall?.name === shellTool.name) { + const args = toolCall.args as ShellToolArgs; + return { + actionType: shellTool.name as "shell", + status, + success, + command: args.command || [], + workdir: args.workdir, + output: getContentString(message.content), + reasoningText, + }; + } else if (toolCall?.name === applyPatchTool.name) { + const args = toolCall.args as ApplyPatchToolArgs; + return { + actionType: "apply-patch", + status, + success, + file: args.file_path || "", + diff: args.diff, + reasoningText, + errorMessage: !success ? getContentString(message.content) : undefined, + }; + } + return { + status: "loading", + summaryText: reasoningText, + }; +} + export function AssistantMessage({ message, isLoading, @@ -84,18 +155,121 @@ export function AssistantMessage({ ); const thread = useStreamContext(); + const messages = thread.messages; + const idx = message ? messages.findIndex((m) => m.id === message.id) : -1; + const nextMessage = idx >= 0 ? messages[idx + 1] : undefined; + + const meta = message ? thread.getMessagesMetadata(message) : undefined; + const threadInterrupt = thread.interrupt; + const parentCheckpoint = meta?.firstSeenState?.parent_checkpoint; + const anthropicStreamedToolCalls = Array.isArray(content) + ? parseAnthropicStreamedToolCalls(content) + : undefined; + + // Helper: get tool call name from AI message (OpenAI or Anthropic) + const aiToolCallName = (() => { + if (message && isAIMessageSDK(message)) { + return message.tool_calls?.[0]?.name; + } + if (anthropicStreamedToolCalls?.length) { + return anthropicStreamedToolCalls[anthropicStreamedToolCalls.length - 1] + .name; + } + return undefined; + })(); + + const aiToolCallArgs = (() => { + if (message && isAIMessageSDK(message)) { + return message.tool_calls?.[0]?.args; + } + if (anthropicStreamedToolCalls?.length) { + return anthropicStreamedToolCalls[anthropicStreamedToolCalls.length - 1] + .args; + } + return undefined; + })(); + + const toolResult = + nextMessage && + isToolMessageSDK(nextMessage) && + aiToolCallName && + nextMessage.tool_call_id === + (message && isAIMessageSDK(message) + ? message.tool_calls?.[0]?.id + : anthropicStreamedToolCalls?.length + ? anthropicStreamedToolCalls[anthropicStreamedToolCalls.length - 1].id + : undefined) + ? nextMessage + : undefined; + + if ( + message && + (aiToolCallName === shellTool.name || + aiToolCallName === applyPatchTool.name) + ) { + if (toolResult) { + return ( + + ); + } + return ( + + ); + } + + if ( + message?.type === "tool" && + (message.name === shellTool.name || message.name === applyPatchTool.name) && + idx > 0 && + messages[idx - 1] && + ((messages[idx - 1] && + isAIMessageSDK(messages[idx - 1]) && + (messages[idx - 1] as AIMessage).tool_calls?.some( + (tc) => + tc.id === (message as ToolMessage).tool_call_id && + (tc.name === shellTool.name || tc.name === applyPatchTool.name), + )) || + (Array.isArray(messages[idx - 1].content) && + parseAnthropicStreamedToolCalls( + messages[idx - 1].content as MessageContentComplex[], + )?.some( + (tc) => + tc.id === (message as ToolMessage).tool_call_id && + (tc.name === shellTool.name || tc.name === applyPatchTool.name), + ))) + ) { + return null; + } + const isLastMessage = thread.messages[thread.messages.length - 1].id === message?.id; const hasNoAIOrToolMessages = !thread.messages.find( (m) => m.type === "ai" || m.type === "tool", ); - const meta = message ? thread.getMessagesMetadata(message) : undefined; - const threadInterrupt = thread.interrupt; - - const parentCheckpoint = meta?.firstSeenState?.parent_checkpoint; - const anthropicStreamedToolCalls = Array.isArray(content) - ? parseAnthropicStreamedToolCalls(content) - : undefined; const hasToolCalls = message && diff --git a/apps/web/src/lib/langchain-messages.ts b/apps/web/src/lib/langchain-messages.ts new file mode 100644 index 00000000..96634242 --- /dev/null +++ b/apps/web/src/lib/langchain-messages.ts @@ -0,0 +1,14 @@ +import { + AIMessage, + HumanMessage, + Message, + ToolMessage, +} from "@langchain/langgraph-sdk"; + +export function isAIMessageSDK(m: Message): m is AIMessage { + return m.type === "ai"; +} + +export function isToolMessageSDK(m: Message): m is ToolMessage { + return m.type === "tool"; +}