mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 09:13:14 +00:00
3f9dbb6597
* feat: add Slack reaction feedback to LangSmith Record Slack reaction feedback against explicitly mapped LangGraph runs so user ratings are idempotent and tied to the message they reacted to. * Address review feedback on Slack reaction → LangSmith feedback - langsmith.py: drop lru_cache on _build_langsmith_feedback_clients so rotated keys / late env hydration are picked up; dedupe by (key, url) tuple instead of key alone so the same key pointing at different endpoints (cloud + self-hosted) builds both clients. - langsmith.py: treat LangSmithNotFoundError on delete_feedback as success — out-of-order or redelivered reaction_removed events would otherwise loop forever on Slack's retry policy. - slack_feedback.py: include channel_id in _feedback_key so the same message_ts in two channels can't collide on the same feedback id. - slack_feedback.py: treat conflicting +/- reactions from one user as ambiguous (clear feedback) instead of averaging to a misleading 0.5. - slack_feedback.py + slack.py + webapp.py: gate reaction handling to the user who triggered the run (stored in the slack_run_map mapping alongside run_id). Prevents bystanders in shared channels from polluting eval feedback.
176 lines
6.3 KiB
Python
176 lines
6.3 KiB
Python
"""Tests for the Slack Assistants API integration (feature-flagged)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.utils import slack as slack_utils
|
|
|
|
|
|
def _ok_response() -> MagicMock:
|
|
response = MagicMock()
|
|
response.json.return_value = {"ok": True, "ts": "1.0"}
|
|
response.raise_for_status.return_value = None
|
|
return response
|
|
|
|
|
|
def _err_response(error: str = "channel_not_found") -> MagicMock:
|
|
response = MagicMock()
|
|
response.json.return_value = {"ok": False, "error": error}
|
|
response.raise_for_status.return_value = None
|
|
return response
|
|
|
|
|
|
def _async_client_cm(post_response: MagicMock) -> AsyncMock:
|
|
client_cm = AsyncMock()
|
|
client_cm.__aenter__.return_value = client_cm
|
|
client_cm.post = AsyncMock(return_value=post_response)
|
|
return client_cm
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_noop_when_flag_disabled(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.delenv("SLACK_ASSISTANTS_API_ENABLED", raising=False)
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
ok = await slack_utils.set_slack_assistant_status("C1", "1.0", "thinking…")
|
|
|
|
assert ok is False
|
|
client_cm.post.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_noop_when_no_token(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
ok = await slack_utils.set_slack_assistant_status("C1", "1.0", "thinking…")
|
|
|
|
assert ok is False
|
|
client_cm.post.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_calls_correct_endpoint(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
ok = await slack_utils.set_slack_assistant_status("C1", "1.0", "thinking…")
|
|
|
|
assert ok is True
|
|
client_cm.post.assert_awaited_once()
|
|
args, kwargs = client_cm.post.call_args
|
|
assert args[0].endswith("/assistant.threads.setStatus")
|
|
assert kwargs["json"] == {
|
|
"channel_id": "C1",
|
|
"thread_ts": "1.0",
|
|
"status": "thinking…",
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_passes_loading_messages(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
await slack_utils.set_slack_assistant_status(
|
|
"C1", "1.0", "thinking…", loading_messages=["a", "b", "c"]
|
|
)
|
|
|
|
_, kwargs = client_cm.post.call_args
|
|
assert kwargs["json"]["loading_messages"] == ["a", "b", "c"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_caps_loading_messages_at_10(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
await slack_utils.set_slack_assistant_status(
|
|
"C1", "1.0", loading_messages=[f"m{i}" for i in range(15)]
|
|
)
|
|
|
|
_, kwargs = client_cm.post.call_args
|
|
assert len(kwargs["json"]["loading_messages"]) == 10
|
|
assert kwargs["json"]["loading_messages"][0] == "m0"
|
|
assert kwargs["json"]["loading_messages"][-1] == "m9"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_omits_loading_messages_when_unset(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
await slack_utils.set_slack_assistant_status("C1", "1.0", "thinking…")
|
|
|
|
_, kwargs = client_cm.post.call_args
|
|
assert "loading_messages" not in kwargs["json"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_slack_assistant_status_returns_false_on_slack_error(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_err_response("invalid_thread"))
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
ok = await slack_utils.set_slack_assistant_status("C1", "1.0", "thinking…")
|
|
|
|
assert ok is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_slack_thread_reply_does_not_call_set_status(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Slack auto-clears the indicator on post; no extra setStatus call needed."""
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", "true")
|
|
monkeypatch.setattr(slack_utils, "SLACK_BOT_TOKEN", "xoxb-test")
|
|
|
|
client_cm = _async_client_cm(_ok_response())
|
|
with patch.object(slack_utils.httpx, "AsyncClient", return_value=client_cm):
|
|
ok = await slack_utils.post_slack_thread_reply("C1", "1.0", "hello")
|
|
|
|
assert ok is True
|
|
assert client_cm.post.await_count == 1
|
|
assert client_cm.post.call_args.args[0].endswith("/chat.postMessage")
|
|
|
|
|
|
def test_is_slack_assistants_api_enabled_truthy_values(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
for truthy in ("1", "true", "TRUE", "yes", "Yes"):
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", truthy)
|
|
assert slack_utils._is_slack_assistants_api_enabled() is True
|
|
|
|
for falsy in ("0", "false", "no", "", "off"):
|
|
monkeypatch.setenv("SLACK_ASSISTANTS_API_ENABLED", falsy)
|
|
assert slack_utils._is_slack_assistants_api_enabled() is False
|