mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 09:13:14 +00:00
feat: support dashboard chat images (#1435)
* feat: support dashboard chat images Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> * fix: keep pending image prompts visible Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com> --------- Co-authored-by: open-swe[bot] <open-swe@users.noreply.github.com>
This commit is contained in:
Johannes du Plessis
GitHub
parent
1f619eae0f
commit
072c0158ff
11 changed files with 503 additions and 52 deletions
|
|
@ -3,6 +3,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
|
@ -14,6 +15,7 @@ _EDIT_TOOLS = frozenset({"write_file", "edit_file", "str_replace", "write", "edi
|
|||
_EXECUTE_TOOLS = frozenset({"execute", "bash", "shell", "run_terminal_cmd"})
|
||||
_SEARCH_TOOLS = frozenset({"glob", "grep", "web_search", "fetch_url", "search"})
|
||||
_INTERNAL_TOOLS = frozenset({"confirming_completion", "no_op"})
|
||||
_DATA_IMAGE_RE = re.compile(r"^data:(image/[^;]+);base64,(.+)$", re.DOTALL)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
|
|
@ -78,6 +80,43 @@ def _parse_tool_args(raw: Any) -> dict[str, Any]:
|
|||
return {}
|
||||
|
||||
|
||||
def _image_chunks(content: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(content, list):
|
||||
return []
|
||||
|
||||
chunks: list[dict[str, Any]] = []
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
item_type = item.get("type")
|
||||
base64_data: str | None = None
|
||||
mime_type: str | None = None
|
||||
if item_type == "image":
|
||||
data = item.get("data") or item.get("base64")
|
||||
mime = item.get("mime_type") or item.get("mimeType")
|
||||
if isinstance(data, str) and isinstance(mime, str):
|
||||
base64_data = data
|
||||
mime_type = mime
|
||||
elif item_type == "image_url":
|
||||
image_url = item.get("image_url")
|
||||
url = image_url.get("url") if isinstance(image_url, dict) else None
|
||||
if isinstance(url, str):
|
||||
match = _DATA_IMAGE_RE.match(url)
|
||||
if match:
|
||||
mime_type, base64_data = match.groups()
|
||||
if base64_data and mime_type:
|
||||
chunk: dict[str, Any] = {
|
||||
"kind": "image",
|
||||
"base64": base64_data,
|
||||
"mimeType": mime_type,
|
||||
}
|
||||
file_name = item.get("fileName") or item.get("file_name")
|
||||
if isinstance(file_name, str) and file_name:
|
||||
chunk["fileName"] = file_name
|
||||
chunks.append(chunk)
|
||||
return chunks
|
||||
|
||||
|
||||
def _maybe_diff_from_args(name: str, args: dict[str, Any]) -> dict[str, Any] | None:
|
||||
path = args.get("path") or args.get("file_path") or args.get("target_file")
|
||||
if not isinstance(path, str) or not path.strip():
|
||||
|
|
@ -150,15 +189,19 @@ def state_messages_to_ui(messages: list[Any]) -> list[dict[str, Any]]:
|
|||
agent_turn["chunks"] = _merge_text_chunks(agent_turn["chunks"])
|
||||
ui_messages.append(agent_turn)
|
||||
agent_turn = None
|
||||
text = extract_text_content(raw.get("content", ""))
|
||||
if not text:
|
||||
content = raw.get("content", "")
|
||||
chunks = _image_chunks(content)
|
||||
text = extract_text_content(content)
|
||||
if text:
|
||||
chunks.append({"kind": "text", "text": text})
|
||||
if not chunks:
|
||||
continue
|
||||
ui_messages.append(
|
||||
{
|
||||
"id": msg_id,
|
||||
"author": "user",
|
||||
"timestamp": timestamp,
|
||||
"chunks": [{"kind": "text", "text": text}],
|
||||
"chunks": chunks,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -11,8 +13,9 @@ from datetime import UTC, datetime
|
|||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from langchain_core.messages.content import create_image_block
|
||||
from langgraph_sdk.errors import InternalServerError
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from ..utils.thread_ops import is_thread_active, langgraph_client, queue_message_for_thread
|
||||
from .message_adapter import state_messages_to_ui
|
||||
|
|
@ -25,6 +28,9 @@ logger = logging.getLogger(__name__)
|
|||
_ASSISTANT_ID = "agent"
|
||||
_DASHBOARD_SOURCE = "dashboard"
|
||||
_DASHBOARD_STREAM_MODES: tuple[str, ...] = ("values", "updates", "messages-tuple")
|
||||
_SUPPORTED_IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"})
|
||||
_MAX_DASHBOARD_IMAGES = 5
|
||||
_MAX_DASHBOARD_IMAGE_BYTES = 10 * 1024 * 1024
|
||||
# Sources whose threads should surface in the Agents UI (besides "dashboard").
|
||||
_SURFACED_SOURCES: tuple[str, ...] = ("dashboard", "github", "slack", "linear", "schedule")
|
||||
|
||||
|
|
@ -45,8 +51,18 @@ async def _resolve_run_email(login: str, profile: dict[str, Any]) -> str | None:
|
|||
return mapped or profile.get("email")
|
||||
|
||||
|
||||
class DashboardImageBody(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
kind: str | None = None
|
||||
base64: str = Field(min_length=1)
|
||||
mime_type: str = Field(alias="mimeType", min_length=1)
|
||||
file_name: str | None = Field(default=None, alias="fileName")
|
||||
|
||||
|
||||
class ThreadCreateBody(BaseModel):
|
||||
prompt: str = Field(min_length=1, max_length=20_000)
|
||||
prompt: str = Field(default="", max_length=20_000)
|
||||
images: list[DashboardImageBody] = Field(default_factory=list)
|
||||
repo: str | None = None
|
||||
repo_explicitly_none: bool = False
|
||||
model_id: str | None = None
|
||||
|
|
@ -54,7 +70,8 @@ class ThreadCreateBody(BaseModel):
|
|||
|
||||
|
||||
class ThreadMessageBody(BaseModel):
|
||||
content: str = Field(min_length=1, max_length=20_000)
|
||||
content: str = Field(default="", max_length=20_000)
|
||||
images: list[DashboardImageBody] = Field(default_factory=list)
|
||||
model_id: str | None = None
|
||||
effort: str | None = None
|
||||
|
||||
|
|
@ -85,6 +102,41 @@ def _parse_repo(full_name: str | None) -> dict[str, str] | None:
|
|||
return {"owner": owner, "name": name}
|
||||
|
||||
|
||||
def _decode_dashboard_image(image: DashboardImageBody) -> bytes:
|
||||
if image.mime_type not in _SUPPORTED_IMAGE_MIME_TYPES:
|
||||
raise HTTPException(422, f"unsupported image type: {image.mime_type}")
|
||||
try:
|
||||
data = base64.b64decode(image.base64, validate=True)
|
||||
except binascii.Error as exc:
|
||||
raise HTTPException(422, "invalid image data") from exc
|
||||
if len(data) > _MAX_DASHBOARD_IMAGE_BYTES:
|
||||
raise HTTPException(422, "image exceeds 10MB limit")
|
||||
return data
|
||||
|
||||
|
||||
def _image_blocks(images: list[DashboardImageBody]) -> list[dict[str, Any]]:
|
||||
if len(images) > _MAX_DASHBOARD_IMAGES:
|
||||
raise HTTPException(422, f"at most {_MAX_DASHBOARD_IMAGES} images are supported")
|
||||
return [
|
||||
create_image_block(
|
||||
base64=base64.b64encode(_decode_dashboard_image(image)).decode("ascii"),
|
||||
mime_type=image.mime_type,
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
|
||||
|
||||
def _user_message_content(
|
||||
prompt: str, images: list[DashboardImageBody]
|
||||
) -> str | list[dict[str, Any]]:
|
||||
text = prompt.strip()
|
||||
if not text and not images:
|
||||
raise HTTPException(422, "prompt or image required")
|
||||
if not images:
|
||||
return text
|
||||
return [*_image_blocks(images), *([{"type": "text", "text": text}] if text else [])]
|
||||
|
||||
|
||||
async def _ensure_dashboard_github_token(login: str) -> None:
|
||||
token = await get_valid_access_token(login)
|
||||
if not token:
|
||||
|
|
@ -283,12 +335,15 @@ async def _start_agent_run(
|
|||
repo_config: dict[str, str],
|
||||
repo_explicitly_none: bool = False,
|
||||
prompt: str,
|
||||
images: list[DashboardImageBody] | None = None,
|
||||
title: str | None = None,
|
||||
model_id: str | None = None,
|
||||
effort: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
profile = await get_profile(login) or {}
|
||||
now_ms = _now_ms()
|
||||
prompt = prompt.strip()
|
||||
content = _user_message_content(prompt, images or [])
|
||||
chosen_model, chosen_effort = _normalize_model_choice(model_id, effort)
|
||||
metadata_model = chosen_model or profile.get("default_model") or "Default"
|
||||
metadata_effort = chosen_effort or profile.get("reasoning_effort")
|
||||
|
|
@ -332,7 +387,7 @@ async def _start_agent_run(
|
|||
run = await client.runs.create(
|
||||
thread_id,
|
||||
_ASSISTANT_ID,
|
||||
input={"messages": [{"role": "user", "content": prompt}]},
|
||||
input={"messages": [{"role": "user", "content": content}]},
|
||||
config={"configurable": configurable, "metadata": _agent_version_metadata()},
|
||||
if_not_exists="create",
|
||||
stream_mode=list(_DASHBOARD_STREAM_MODES),
|
||||
|
|
@ -357,7 +412,8 @@ async def create_dashboard_thread(login: str, body: ThreadCreateBody) -> dict[st
|
|||
login=login,
|
||||
repo_config=repo_config,
|
||||
repo_explicitly_none=body.repo_explicitly_none,
|
||||
prompt=body.prompt.strip(),
|
||||
prompt=body.prompt,
|
||||
images=body.images,
|
||||
model_id=body.model_id,
|
||||
effort=body.effort,
|
||||
)
|
||||
|
|
@ -377,6 +433,7 @@ async def send_dashboard_message(
|
|||
owner, name, _ = _metadata_repo(metadata)
|
||||
|
||||
prompt = body.content.strip()
|
||||
content = _user_message_content(prompt, body.images)
|
||||
now_ms = _now_ms()
|
||||
chosen_model, chosen_effort = _normalize_model_choice(body.model_id, body.effort)
|
||||
metadata_update: dict[str, Any] = {"source": _DASHBOARD_SOURCE, "updated_at_ms": now_ms}
|
||||
|
|
@ -386,9 +443,16 @@ async def send_dashboard_message(
|
|||
await client.threads.update(thread_id=thread_id, metadata=metadata_update)
|
||||
|
||||
if await is_thread_active(thread_id):
|
||||
queue_payload: dict[str, Any] = {"text": prompt, "source": _DASHBOARD_SOURCE}
|
||||
if isinstance(content, list):
|
||||
queue_payload["images"] = [
|
||||
block
|
||||
for block in content
|
||||
if isinstance(block, dict) and block.get("type") != "text"
|
||||
]
|
||||
queued = await queue_message_for_thread(
|
||||
thread_id,
|
||||
{"text": prompt, "source": _DASHBOARD_SOURCE},
|
||||
queue_payload,
|
||||
)
|
||||
if not queued:
|
||||
raise HTTPException(502, "failed to queue follow-up message")
|
||||
|
|
@ -415,7 +479,7 @@ async def send_dashboard_message(
|
|||
run = await client.runs.create(
|
||||
thread_id,
|
||||
_ASSISTANT_ID,
|
||||
input={"messages": [{"role": "user", "content": prompt}]},
|
||||
input={"messages": [{"role": "user", "content": content}]},
|
||||
config={"configurable": configurable, "metadata": _agent_version_metadata()},
|
||||
stream_mode=list(_DASHBOARD_STREAM_MODES),
|
||||
stream_resumable=True,
|
||||
|
|
|
|||
|
|
@ -39,9 +39,12 @@ async def _build_blocks_from_payload(
|
|||
) -> list[dict[str, Any]]:
|
||||
text = payload.get("text", "")
|
||||
image_urls = payload.get("image_urls", []) or []
|
||||
images = payload.get("images", []) or []
|
||||
blocks: list[dict[str, Any]] = []
|
||||
if text:
|
||||
blocks.append({"type": "text", "text": text})
|
||||
if isinstance(images, list):
|
||||
blocks.extend(image for image in images if isinstance(image, dict))
|
||||
|
||||
if not image_urls:
|
||||
return blocks
|
||||
|
|
@ -119,7 +122,9 @@ async def check_message_queue_before_model( # noqa: PLR0911
|
|||
content = msg.get("content")
|
||||
if _is_dashboard_queued_message(content):
|
||||
content_blocks.append({"type": "text", "text": DASHBOARD_HANDOFF_INSTRUCTION})
|
||||
if isinstance(content, dict) and ("text" in content or "image_urls" in content):
|
||||
if isinstance(content, dict) and (
|
||||
"text" in content or "image_urls" in content or "images" in content
|
||||
):
|
||||
logger.debug("Queued message contains text + image URLs")
|
||||
blocks = await _build_blocks_from_payload(content)
|
||||
content_blocks.extend(blocks)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,38 @@ def test_state_messages_to_ui_maps_user_and_tool_calls() -> None:
|
|||
assert tool_chunk["output"] == "print('hi')"
|
||||
|
||||
|
||||
def test_state_messages_to_ui_maps_user_images() -> None:
|
||||
messages = [
|
||||
{
|
||||
"type": "human",
|
||||
"id": "u1",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"source_type": "base64",
|
||||
"data": "aW1hZ2U=",
|
||||
"mime_type": "image/png",
|
||||
},
|
||||
{"type": "text", "text": "What changed?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
ui = state_messages_to_ui(messages)
|
||||
|
||||
assert ui == [
|
||||
{
|
||||
"id": "u1",
|
||||
"author": "user",
|
||||
"timestamp": ui[0]["timestamp"],
|
||||
"chunks": [
|
||||
{"kind": "image", "base64": "aW1hZ2U=", "mimeType": "image/png"},
|
||||
{"kind": "text", "text": "What changed?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_state_messages_to_ui_tags_slack_and_linear_replies() -> None:
|
||||
messages = [
|
||||
{"type": "human", "id": "u1", "content": "ping"},
|
||||
|
|
|
|||
|
|
@ -91,6 +91,51 @@ async def test_dashboard_followup_on_slack_thread_uses_dashboard_source(
|
|||
assert run_config["repo"] == {"owner": "octo", "name": "repo"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_followup_sends_image_content_blocks(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
metadata = {
|
||||
"source": "dashboard",
|
||||
"github_login": "octocat",
|
||||
"repo_owner": "octo",
|
||||
"repo_name": "repo",
|
||||
}
|
||||
client = _FakeClient(metadata)
|
||||
|
||||
monkeypatch.setattr(thread_api, "langgraph_client", lambda: client)
|
||||
monkeypatch.setattr(thread_api, "is_thread_active", _inactive_thread)
|
||||
monkeypatch.setattr(thread_api, "_ensure_dashboard_github_token", _noop_token_check)
|
||||
monkeypatch.setattr(thread_api, "get_profile", _empty_profile)
|
||||
monkeypatch.setattr(thread_api, "_resolve_run_email", _run_email)
|
||||
monkeypatch.setattr(
|
||||
thread_api,
|
||||
"create_image_block",
|
||||
lambda *, base64, mime_type: {"type": "image", "data": base64, "mime_type": mime_type},
|
||||
)
|
||||
|
||||
await thread_api.send_dashboard_message(
|
||||
"thread-1",
|
||||
"octocat",
|
||||
thread_api.ThreadMessageBody(
|
||||
content="describe this",
|
||||
images=[
|
||||
thread_api.DashboardImageBody(
|
||||
base64="aW1hZ2U=",
|
||||
mimeType="image/png",
|
||||
fileName="screenshot.png",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
content = client.runs.created[0]["kwargs"]["input"]["messages"][0]["content"]
|
||||
assert content == [
|
||||
{"type": "image", "data": "aW1hZ2U=", "mime_type": "image/png"},
|
||||
{"type": "text", "text": "describe this"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_followup_on_busy_thread_queues_dashboard_handoff(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -122,6 +167,48 @@ async def test_dashboard_followup_on_busy_thread_queues_dashboard_handoff(
|
|||
assert queued_messages == [{"text": "continue in web", "source": "dashboard"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_followup_on_busy_thread_queues_images(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
metadata = {
|
||||
"source": "dashboard",
|
||||
"github_login": "octocat",
|
||||
}
|
||||
client = _FakeClient(metadata)
|
||||
queued_messages: list[object] = []
|
||||
|
||||
async def fake_queue_message_for_thread(thread_id: str, message_content: object) -> bool:
|
||||
queued_messages.append(message_content)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(thread_api, "langgraph_client", lambda: client)
|
||||
monkeypatch.setattr(thread_api, "is_thread_active", _active_thread)
|
||||
monkeypatch.setattr(thread_api, "queue_message_for_thread", fake_queue_message_for_thread)
|
||||
monkeypatch.setattr(
|
||||
thread_api,
|
||||
"create_image_block",
|
||||
lambda *, base64, mime_type: {"type": "image", "data": base64, "mime_type": mime_type},
|
||||
)
|
||||
|
||||
await thread_api.send_dashboard_message(
|
||||
"thread-1",
|
||||
"octocat",
|
||||
thread_api.ThreadMessageBody(
|
||||
content="continue in web",
|
||||
images=[thread_api.DashboardImageBody(base64="aW1hZ2U=", mimeType="image/png")],
|
||||
),
|
||||
)
|
||||
|
||||
assert queued_messages == [
|
||||
{
|
||||
"text": "continue in web",
|
||||
"source": "dashboard",
|
||||
"images": [{"type": "image", "data": "aW1hZ2U=", "mime_type": "image/png"}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_followup_preserves_explicit_repo_less_thread(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,33 @@ interface AgentThreadViewProps {
|
|||
thread: AgentThread;
|
||||
}
|
||||
|
||||
function messageText(message: Message): string {
|
||||
return message.chunks
|
||||
.filter((chunk) => chunk.kind === "text")
|
||||
.map((chunk) => chunk.text)
|
||||
.join("");
|
||||
}
|
||||
|
||||
function messageImageKey(message: Message): string {
|
||||
return message.chunks
|
||||
.filter((chunk) => chunk.kind === "image")
|
||||
.map((chunk) => `${chunk.mimeType}:${chunk.base64}`)
|
||||
.join("\u0000");
|
||||
}
|
||||
|
||||
function pendingImageKey(entry: PendingPrompt): string {
|
||||
return (entry.images ?? [])
|
||||
.map((image) => `${image.mimeType}:${image.base64}`)
|
||||
.join("\u0000");
|
||||
}
|
||||
|
||||
function isPendingPromptConfirmed(entry: PendingPrompt, messages: Array<Message>): boolean {
|
||||
return messages.slice(entry.insertAt).some((message) => {
|
||||
if (message.author !== "user") return false;
|
||||
return messageText(message) === entry.prompt && messageImageKey(message) === pendingImageKey(entry);
|
||||
});
|
||||
}
|
||||
|
||||
export function AgentThreadView({ user, thread }: AgentThreadViewProps) {
|
||||
const sendMessage = useSendAgentMessage(thread.id);
|
||||
const cancelThread = useCancelAgentThread(thread.id);
|
||||
|
|
@ -37,39 +64,28 @@ export function AgentThreadView({ user, thread }: AgentThreadViewProps) {
|
|||
const [selection, setSelection] = useState<ModelSelection | null>(null);
|
||||
const activeSelection = selection ?? threadSelection ?? defaultSelection;
|
||||
|
||||
const userMessageTexts = useMemo(() => {
|
||||
return new Set(
|
||||
thread.messages
|
||||
.filter((m) => m.author === "user")
|
||||
.map((m) =>
|
||||
m.chunks
|
||||
.filter((c) => c.kind === "text")
|
||||
.map((c) => c.text)
|
||||
.join(""),
|
||||
),
|
||||
);
|
||||
}, [thread.messages]);
|
||||
|
||||
useEffect(() => {
|
||||
setPendingPrompts((prev) => {
|
||||
if (prev.length === 0) return prev;
|
||||
const next = dropPendingPrompts(thread.id, (entry) =>
|
||||
userMessageTexts.has(entry.prompt),
|
||||
isPendingPromptConfirmed(entry, thread.messages),
|
||||
);
|
||||
return next.length === prev.length ? prev : next;
|
||||
});
|
||||
}, [thread.id, userMessageTexts]);
|
||||
}, [thread.id, thread.messages]);
|
||||
|
||||
const displayMessages = useMemo<Array<Message>>(() => {
|
||||
if (pendingPrompts.length === 0) return thread.messages;
|
||||
const baseTimestamp = new Date().toISOString();
|
||||
const result = thread.messages.slice();
|
||||
pendingPrompts.forEach((entry, i) => {
|
||||
const chunks: Message["chunks"] = [...(entry.images ?? [])];
|
||||
if (entry.prompt) chunks.push({ kind: "text", text: entry.prompt });
|
||||
const synth: Message = {
|
||||
id: `pending-user-${i}`,
|
||||
author: "user",
|
||||
timestamp: baseTimestamp,
|
||||
chunks: [{ kind: "text", text: entry.prompt }],
|
||||
chunks,
|
||||
};
|
||||
const at = Math.min(Math.max(entry.insertAt, 0), result.length);
|
||||
result.splice(at, 0, synth);
|
||||
|
|
@ -99,9 +115,10 @@ export function AgentThreadView({ user, thread }: AgentThreadViewProps) {
|
|||
compact
|
||||
busy={hasActiveRun}
|
||||
disabled={sendMessage.isPending}
|
||||
onSubmit={(content) =>
|
||||
onSubmit={(content, images) =>
|
||||
sendMessage.mutate({
|
||||
content,
|
||||
images,
|
||||
model_id: activeSelection?.modelId ?? null,
|
||||
effort: activeSelection?.effort ?? null,
|
||||
})
|
||||
|
|
@ -124,9 +141,10 @@ export function AgentThreadView({ user, thread }: AgentThreadViewProps) {
|
|||
compact
|
||||
busy={hasActiveRun}
|
||||
disabled={sendMessage.isPending}
|
||||
onSubmit={(content) =>
|
||||
onSubmit={(content, images) =>
|
||||
sendMessage.mutate({
|
||||
content,
|
||||
images,
|
||||
model_id: activeSelection?.modelId ?? null,
|
||||
effort: activeSelection?.effort ?? null,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -32,9 +32,10 @@ export function AgentsHome() {
|
|||
<div className="flex w-full flex-col items-center gap-6">
|
||||
<Logo />
|
||||
<AgentPromptBar
|
||||
onSubmit={(prompt) =>
|
||||
onSubmit={(prompt, images) =>
|
||||
createThread.mutate({
|
||||
prompt,
|
||||
images,
|
||||
repo,
|
||||
repo_explicitly_none: repoOverride === null,
|
||||
model_id: activeSelection?.modelId ?? null,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,11 @@
|
|||
import { ArrowUp, ChevronDown, LoaderCircle, Square } from "lucide-react"
|
||||
import {
|
||||
ArrowUp,
|
||||
ChevronDown,
|
||||
ImagePlus,
|
||||
LoaderCircle,
|
||||
Square,
|
||||
X,
|
||||
} from "lucide-react"
|
||||
import {
|
||||
memo,
|
||||
useCallback,
|
||||
|
|
@ -10,19 +17,28 @@ import {
|
|||
} from "react"
|
||||
|
||||
import type { ModelOption } from "@/lib/api"
|
||||
import type { ImageChunk } from "@/lib/agents/types"
|
||||
import type { ModelSelection } from "@/lib/agents/useModelOptions"
|
||||
import { RepoSelector } from "@/components/agents/RepoSelector"
|
||||
import { formatModelSelection } from "@/lib/agents/useModelOptions"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const PROMPT_TEXTAREA_MAX_HEIGHT = 200
|
||||
const MAX_IMAGE_COUNT = 5
|
||||
const MAX_IMAGE_BYTES = 10 * 1024 * 1024
|
||||
const SUPPORTED_IMAGE_TYPES = new Set([
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/webp",
|
||||
])
|
||||
|
||||
export interface CloudPromptBarProps {
|
||||
placeholder?: string
|
||||
compact?: boolean
|
||||
disabled?: boolean
|
||||
busy?: boolean
|
||||
onSubmit?: (value: string) => void
|
||||
onSubmit?: (value: string, images: Array<ImageChunk>) => void
|
||||
/** Called to stop the running agent. When set, the send button becomes a stop button while busy and the input is empty. */
|
||||
onStop?: () => void
|
||||
stopping?: boolean
|
||||
|
|
@ -35,6 +51,32 @@ export interface CloudPromptBarProps {
|
|||
onRepoChange?: (repo: string | null) => void
|
||||
}
|
||||
|
||||
function fileToImageChunk(file: File): Promise<ImageChunk | null> {
|
||||
if (!SUPPORTED_IMAGE_TYPES.has(file.type) || file.size > MAX_IMAGE_BYTES) {
|
||||
return Promise.resolve(null)
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const reader = new FileReader()
|
||||
reader.onload = () => {
|
||||
const dataUrl = typeof reader.result === "string" ? reader.result : ""
|
||||
const base64 = dataUrl.split(",")[1]
|
||||
resolve(
|
||||
base64
|
||||
? {
|
||||
kind: "image",
|
||||
base64,
|
||||
mimeType: file.type,
|
||||
fileName: file.name,
|
||||
}
|
||||
: null
|
||||
)
|
||||
}
|
||||
reader.onerror = () => resolve(null)
|
||||
reader.readAsDataURL(file)
|
||||
})
|
||||
}
|
||||
|
||||
/** Web-adapted PromptBar from open-swe-app — local state, no Electron/Zustand deps. */
|
||||
export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
||||
placeholder = "Ask Open SWE to build, fix bugs, explore",
|
||||
|
|
@ -52,8 +94,12 @@ export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
|||
onRepoChange,
|
||||
}: CloudPromptBarProps) {
|
||||
const [value, setValue] = useState("")
|
||||
const [pendingImages, setPendingImages] = useState<Array<ImageChunk>>([])
|
||||
const [isDragOver, setIsDragOver] = useState(false)
|
||||
const [modelDropdownOpen, setModelDropdownOpen] = useState(false)
|
||||
const inputRef = useRef<HTMLTextAreaElement>(null)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const dragDepthRef = useRef(0)
|
||||
const modelDropdownRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
const combos = useMemo<Array<ModelSelection>>(() => {
|
||||
|
|
@ -68,12 +114,16 @@ export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
|||
|
||||
const selectionLabel = formatModelSelection(models, selection)
|
||||
|
||||
const canSubmit =
|
||||
!disabled && (value.trim().length > 0 || pendingImages.length > 0)
|
||||
|
||||
const handleSubmit = useCallback(() => {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed || disabled) return
|
||||
onSubmit?.(trimmed)
|
||||
if (!canSubmit) return
|
||||
onSubmit?.(trimmed, pendingImages)
|
||||
setValue("")
|
||||
}, [disabled, onSubmit, value])
|
||||
setPendingImages([])
|
||||
}, [canSubmit, onSubmit, pendingImages, value])
|
||||
|
||||
useLayoutEffect(() => {
|
||||
const el = inputRef.current
|
||||
|
|
@ -100,15 +150,69 @@ export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
|||
return () => document.removeEventListener("mousedown", handleClickOutside)
|
||||
}, [])
|
||||
|
||||
const addFiles = useCallback(async (files: FileList | Array<File>) => {
|
||||
const nextImages = await Promise.all(
|
||||
Array.from(files).map(fileToImageChunk)
|
||||
)
|
||||
const validImages = nextImages.filter(
|
||||
(image): image is ImageChunk => image !== null
|
||||
)
|
||||
if (validImages.length === 0) return
|
||||
setPendingImages((prev) =>
|
||||
[...prev, ...validImages].slice(0, MAX_IMAGE_COUNT)
|
||||
)
|
||||
}, [])
|
||||
|
||||
const handleFileChange = useCallback(
|
||||
(e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = e.target.files
|
||||
if (files) void addFiles(files)
|
||||
e.target.value = ""
|
||||
},
|
||||
[addFiles]
|
||||
)
|
||||
|
||||
const handleDragEnter = useCallback((e: React.DragEvent<HTMLDivElement>) => {
|
||||
if (!e.dataTransfer.types.includes("Files")) return
|
||||
e.preventDefault()
|
||||
dragDepthRef.current += 1
|
||||
setIsDragOver(true)
|
||||
}, [])
|
||||
|
||||
const handleDragOver = useCallback((e: React.DragEvent<HTMLDivElement>) => {
|
||||
if (!e.dataTransfer.types.includes("Files")) return
|
||||
e.preventDefault()
|
||||
setIsDragOver(true)
|
||||
}, [])
|
||||
|
||||
const handleDragLeave = useCallback((e: React.DragEvent<HTMLDivElement>) => {
|
||||
if (!e.dataTransfer.types.includes("Files")) return
|
||||
e.preventDefault()
|
||||
dragDepthRef.current = Math.max(0, dragDepthRef.current - 1)
|
||||
if (dragDepthRef.current === 0) setIsDragOver(false)
|
||||
}, [])
|
||||
|
||||
const handleDrop = useCallback(
|
||||
(e: React.DragEvent<HTMLDivElement>) => {
|
||||
if (!e.dataTransfer.types.includes("Files")) return
|
||||
e.preventDefault()
|
||||
dragDepthRef.current = 0
|
||||
setIsDragOver(false)
|
||||
void addFiles(e.dataTransfer.files)
|
||||
},
|
||||
[addFiles]
|
||||
)
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
if (e.key === "Enter" && !e.shiftKey && value.trim()) {
|
||||
if (e.key === "Enter" && !e.shiftKey && canSubmit) {
|
||||
e.preventDefault()
|
||||
handleSubmit()
|
||||
}
|
||||
}
|
||||
|
||||
const pickerDisabled = combos.length === 0 || !onSelectionChange
|
||||
const showStop = busy && !disabled && !value.trim() && !!onStop
|
||||
const showStop =
|
||||
busy && !disabled && !value.trim() && pendingImages.length === 0 && !!onStop
|
||||
|
||||
return (
|
||||
<div
|
||||
|
|
@ -127,11 +231,62 @@ export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
|||
</div>
|
||||
)}
|
||||
<div
|
||||
onDragEnter={handleDragEnter}
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
className={cn(
|
||||
"relative flex min-h-[106px] flex-col rounded-2xl border border-[var(--ui-border)] bg-[var(--ui-surface)] px-3 py-2.5 shadow-sm",
|
||||
compact && "min-h-[88px]"
|
||||
compact && "min-h-[88px]",
|
||||
isDragOver && "border-[var(--ui-accent)]"
|
||||
)}
|
||||
>
|
||||
{isDragOver && (
|
||||
<div className="pointer-events-none absolute inset-0 z-20 flex items-center justify-center rounded-2xl bg-[var(--ui-surface)]/80 backdrop-blur-sm">
|
||||
<span className="rounded-md bg-[var(--ui-panel-2)] px-3 py-1.5 text-sm font-medium text-[color:var(--ui-accent)]">
|
||||
Drop images here
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept="image/png,image/jpeg,image/gif,image/webp"
|
||||
multiple
|
||||
className="hidden"
|
||||
onChange={handleFileChange}
|
||||
/>
|
||||
|
||||
{pendingImages.length > 0 && (
|
||||
<div className="mb-2 flex flex-wrap gap-2">
|
||||
{pendingImages.map((image, index) => (
|
||||
<div
|
||||
key={`${image.fileName ?? "image"}-${index}`}
|
||||
className="group relative"
|
||||
>
|
||||
<img
|
||||
src={`data:${image.mimeType};base64,${image.base64}`}
|
||||
alt={image.fileName || "pending image"}
|
||||
className="size-16 rounded-lg border border-[var(--ui-border)] object-cover"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Remove image"
|
||||
onClick={() =>
|
||||
setPendingImages((prev) =>
|
||||
prev.filter((_, i) => i !== index)
|
||||
)
|
||||
}
|
||||
className="absolute -top-1.5 -right-1.5 flex size-5 items-center justify-center rounded-full border border-[var(--ui-border)] bg-[var(--ui-panel-2)] text-[color:var(--ui-text-muted)] opacity-0 shadow-sm transition-opacity group-hover:opacity-100 hover:text-[color:var(--ui-text)]"
|
||||
>
|
||||
<X className="size-3" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<textarea
|
||||
ref={inputRef}
|
||||
rows={1}
|
||||
|
|
@ -195,13 +350,23 @@ export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
|||
)}
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
disabled={disabled || pendingImages.length >= MAX_IMAGE_COUNT}
|
||||
aria-label="Attach images"
|
||||
className="ml-auto flex size-7 shrink-0 items-center justify-center rounded-full text-[color:var(--ui-text-muted)] transition-colors hover:bg-[var(--ui-panel-2)] hover:text-[color:var(--ui-text)] disabled:cursor-default disabled:opacity-40"
|
||||
>
|
||||
<ImagePlus className="size-4" />
|
||||
</button>
|
||||
|
||||
{showStop ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onStop}
|
||||
disabled={stopping}
|
||||
aria-label="Stop run"
|
||||
className="ml-auto flex size-7 shrink-0 items-center justify-center rounded-full bg-[var(--ui-accent)] text-white transition-opacity hover:opacity-90 disabled:cursor-default disabled:opacity-40"
|
||||
className="flex size-7 shrink-0 items-center justify-center rounded-full bg-[var(--ui-accent)] text-white transition-opacity hover:opacity-90 disabled:cursor-default disabled:opacity-40"
|
||||
>
|
||||
{stopping ? (
|
||||
<LoaderCircle className="size-3.5 animate-spin" />
|
||||
|
|
@ -213,9 +378,9 @@ export const CloudPromptBar = memo(function CloudPromptBarComponent({
|
|||
<button
|
||||
type="button"
|
||||
onClick={handleSubmit}
|
||||
disabled={!value.trim() || disabled}
|
||||
disabled={!canSubmit}
|
||||
aria-label="Send message"
|
||||
className="ml-auto flex size-7 shrink-0 items-center justify-center rounded-full bg-[var(--ui-accent)] text-white transition-opacity hover:opacity-90 disabled:cursor-default disabled:opacity-40"
|
||||
className="flex size-7 shrink-0 items-center justify-center rounded-full bg-[var(--ui-accent)] text-white transition-opacity hover:opacity-90 disabled:cursor-default disabled:opacity-40"
|
||||
>
|
||||
{disabled ? (
|
||||
<LoaderCircle className="size-3.5 animate-spin" />
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import type { AgentSchedule, AgentThread, Message } from "./types"
|
||||
import type { AgentSchedule, AgentThread, ImageChunk, Message } from "./types"
|
||||
|
||||
export type { AgentSchedule, AgentThread, Message }
|
||||
|
||||
export interface ThreadCreateRequest {
|
||||
prompt: string
|
||||
images?: Array<ImageChunk>
|
||||
repo?: string | null
|
||||
repo_explicitly_none?: boolean
|
||||
model_id?: string | null
|
||||
|
|
@ -12,6 +13,7 @@ export interface ThreadCreateRequest {
|
|||
|
||||
export interface ThreadMessageRequest {
|
||||
content: string
|
||||
images?: Array<ImageChunk>
|
||||
model_id?: string | null
|
||||
effort?: string | null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,22 @@
|
|||
import type { ImageChunk } from "./types";
|
||||
|
||||
const STORAGE_KEY = (threadId: string) => `open-swe:pending-prompts:${threadId}`;
|
||||
|
||||
export interface PendingPrompt {
|
||||
prompt: string;
|
||||
insertAt: number;
|
||||
images?: Array<ImageChunk>;
|
||||
}
|
||||
|
||||
function isImageChunk(value: unknown): value is ImageChunk {
|
||||
const image = value as { base64?: unknown; kind?: unknown; mimeType?: unknown };
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
image.kind === "image" &&
|
||||
typeof image.base64 === "string" &&
|
||||
typeof image.mimeType === "string"
|
||||
);
|
||||
}
|
||||
|
||||
function isPendingPrompt(value: unknown): value is PendingPrompt {
|
||||
|
|
@ -10,11 +24,14 @@ function isPendingPrompt(value: unknown): value is PendingPrompt {
|
|||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
typeof (value as PendingPrompt).prompt === "string" &&
|
||||
typeof (value as PendingPrompt).insertAt === "number"
|
||||
typeof (value as PendingPrompt).insertAt === "number" &&
|
||||
((value as PendingPrompt).images === undefined ||
|
||||
(Array.isArray((value as PendingPrompt).images) &&
|
||||
(value as PendingPrompt).images!.every(isImageChunk)))
|
||||
);
|
||||
}
|
||||
|
||||
function safeRead(threadId: string): PendingPrompt[] {
|
||||
function safeRead(threadId: string): Array<PendingPrompt> {
|
||||
if (typeof window === "undefined") return [];
|
||||
try {
|
||||
const raw = window.sessionStorage.getItem(STORAGE_KEY(threadId));
|
||||
|
|
@ -26,27 +43,36 @@ function safeRead(threadId: string): PendingPrompt[] {
|
|||
}
|
||||
}
|
||||
|
||||
function safeWrite(threadId: string, prompts: PendingPrompt[]): void {
|
||||
function safeWrite(threadId: string, prompts: Array<PendingPrompt>): void {
|
||||
if (typeof window === "undefined") return;
|
||||
if (prompts.length === 0) {
|
||||
window.sessionStorage.removeItem(STORAGE_KEY(threadId));
|
||||
return;
|
||||
}
|
||||
window.sessionStorage.setItem(STORAGE_KEY(threadId), JSON.stringify(prompts));
|
||||
try {
|
||||
window.sessionStorage.setItem(STORAGE_KEY(threadId), JSON.stringify(prompts));
|
||||
} catch {
|
||||
window.sessionStorage.removeItem(STORAGE_KEY(threadId));
|
||||
}
|
||||
}
|
||||
|
||||
export function getPendingPrompts(threadId: string): PendingPrompt[] {
|
||||
export function getPendingPrompts(threadId: string): Array<PendingPrompt> {
|
||||
return safeRead(threadId);
|
||||
}
|
||||
|
||||
export function addPendingPrompt(threadId: string, prompt: string, insertAt: number): void {
|
||||
safeWrite(threadId, [...safeRead(threadId), { prompt, insertAt }]);
|
||||
export function addPendingPrompt(
|
||||
threadId: string,
|
||||
prompt: string,
|
||||
insertAt: number,
|
||||
images?: Array<ImageChunk>,
|
||||
): void {
|
||||
safeWrite(threadId, [...safeRead(threadId), { prompt, insertAt, images }]);
|
||||
}
|
||||
|
||||
export function dropPendingPrompts(
|
||||
threadId: string,
|
||||
predicate: (entry: PendingPrompt) => boolean,
|
||||
): PendingPrompt[] {
|
||||
): Array<PendingPrompt> {
|
||||
const next = safeRead(threadId).filter((p) => !predicate(p));
|
||||
safeWrite(threadId, next);
|
||||
return next;
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { useNavigate } from "@tanstack/react-router"
|
|||
import { agentsApi } from "./api"
|
||||
import { addPendingPrompt } from "./pendingPrompts"
|
||||
import type { ScheduleUpdateRequest } from "./api"
|
||||
import type { ImageChunk } from "./types"
|
||||
|
||||
export const agentThreadKeys = {
|
||||
all: ["agent-threads"] as const,
|
||||
|
|
@ -80,7 +81,12 @@ export function useCreateAgentThread() {
|
|||
return useMutation({
|
||||
mutationFn: agentsApi.createThread,
|
||||
onSuccess: (thread, variables) => {
|
||||
addPendingPrompt(thread.id, variables.prompt, thread.messages.length)
|
||||
addPendingPrompt(
|
||||
thread.id,
|
||||
variables.prompt,
|
||||
thread.messages.length,
|
||||
variables.images
|
||||
)
|
||||
queryClient.setQueryData(agentThreadKeys.detail(thread.id), {
|
||||
...thread,
|
||||
status: thread.status === "idle" ? "running" : thread.status,
|
||||
|
|
@ -93,6 +99,7 @@ export function useCreateAgentThread() {
|
|||
|
||||
export interface SendAgentMessageVariables {
|
||||
content: string
|
||||
images?: Array<ImageChunk>
|
||||
model_id?: string | null
|
||||
effort?: string | null
|
||||
}
|
||||
|
|
@ -104,6 +111,7 @@ export function useSendAgentMessage(threadId: string) {
|
|||
mutationFn: (vars: SendAgentMessageVariables) =>
|
||||
agentsApi.sendMessage(threadId, {
|
||||
content: vars.content,
|
||||
images: vars.images,
|
||||
model_id: vars.model_id,
|
||||
effort: vars.effort,
|
||||
}),
|
||||
|
|
@ -114,7 +122,7 @@ export function useSendAgentMessage(threadId: string) {
|
|||
const insertAt = Array.isArray(cached?.messages)
|
||||
? cached.messages.length
|
||||
: 0
|
||||
addPendingPrompt(threadId, vars.content, insertAt)
|
||||
addPendingPrompt(threadId, vars.content, insertAt, vars.images)
|
||||
},
|
||||
onSuccess: (thread) => {
|
||||
queryClient.setQueryData(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue