mirror of
https://github.com/Sea-Haven-Industries/open-swe.git
synced 2026-09-30 15:03:16 +00:00
4060933ce4
* feat: add github CI check run tools for shepherding CI Add get_pr_check_runs and rerun_failed_check_runs tools that authenticate using the GitHub App installation token so the agent can query and retry CI status on private repos without relying on GH_TOKEN or unauthenticated http_request calls. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix: handle paginated GitHub CI results * refactor(github_ci): address review feedback - Rename rerun_failed_check_runs -> rerun_failed_workflow_runs and clarify in docstrings that the tool only retries GitHub Actions workflow runs (not third-party CI checks surfaced by get_pr_check_runs). - Skip action_required workflow runs when rerunning; those need manual approval, not a rerun. - Run rerun-failed-jobs requests concurrently via asyncio.gather instead of sequentially. - Fix latent pagination bug in _fetch_paginated_items where caller-supplied params could overwrite per_page/page and break the end-of-pagination check; reserved keys now always win and the threshold uses a PER_PAGE constant. - Set an explicit 30s httpx timeout so a hung GitHub call cannot stall the agent loop. - Restore alphabetical ordering of tools in agent/tools/__init__.py. - Add tests for: a 500 surfaced on a later pagination page, and action_required runs being filtered out of rerun candidates. --------- Co-authored-by: Claude Agent <agent@anthropic.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Johannes du Plessis <johannes@langchain.dev>
422 lines
15 KiB
Python
422 lines
15 KiB
Python
"""Tests for the github_ci tools."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
import agent.tools.github_ci as github_ci
|
|
|
|
|
|
def _make_response(status_code: int, json_data: Any) -> MagicMock:
|
|
resp = MagicMock()
|
|
resp.status_code = status_code
|
|
resp.json.return_value = json_data
|
|
resp.text = str(json_data)
|
|
return resp
|
|
|
|
|
|
def _check_run(
|
|
run_id: int, conclusion: str = "success", status: str = "completed"
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"id": run_id,
|
|
"name": f"job-{run_id}",
|
|
"status": status,
|
|
"conclusion": conclusion,
|
|
"html_url": f"https://github.com/checks/{run_id}",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# get_pr_check_runs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_get_pr_check_runs_no_repo_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(github_ci, "_get_repo_config", lambda: {})
|
|
result = github_ci.get_pr_check_runs(42)
|
|
assert result == {"success": False, "error": "No repo config found"}
|
|
|
|
|
|
def test_get_pr_check_runs_no_token(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value=None))
|
|
result = github_ci.get_pr_check_runs(42)
|
|
assert result == {"success": False, "error": "Failed to get GitHub App installation token"}
|
|
|
|
|
|
def test_get_pr_check_runs_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
pr_data = {"head": {"sha": "abc123"}}
|
|
check_runs_data = {
|
|
"total_count": 2,
|
|
"check_runs": [
|
|
{
|
|
"id": 1,
|
|
"name": "test-job",
|
|
"status": "completed",
|
|
"conclusion": "success",
|
|
"html_url": "https://github.com/checks/1",
|
|
},
|
|
{
|
|
"id": 2,
|
|
"name": "lint-job",
|
|
"status": "completed",
|
|
"conclusion": "success",
|
|
"html_url": "https://github.com/checks/2",
|
|
},
|
|
],
|
|
}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, pr_data),
|
|
_make_response(200, check_runs_data),
|
|
]
|
|
)
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.get_pr_check_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["head_sha"] == "abc123"
|
|
assert result["total_count"] == 2
|
|
assert len(result["check_runs"]) == 2
|
|
assert result["all_passed"] is True
|
|
assert result["any_failed"] is False
|
|
assert result["any_pending"] is False
|
|
|
|
|
|
def test_get_pr_check_runs_with_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
pr_data = {"head": {"sha": "def456"}}
|
|
check_runs_data = {
|
|
"total_count": 2,
|
|
"check_runs": [
|
|
{
|
|
"id": 10,
|
|
"name": "test-job",
|
|
"status": "completed",
|
|
"conclusion": "failure",
|
|
"html_url": "https://github.com/checks/10",
|
|
},
|
|
{
|
|
"id": 11,
|
|
"name": "lint-job",
|
|
"status": "in_progress",
|
|
"conclusion": None,
|
|
"html_url": "https://github.com/checks/11",
|
|
},
|
|
],
|
|
}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, pr_data),
|
|
_make_response(200, check_runs_data),
|
|
]
|
|
)
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.get_pr_check_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["any_failed"] is True
|
|
assert result["any_pending"] is True
|
|
assert result["all_passed"] is False
|
|
|
|
|
|
def test_get_pr_check_runs_empty_checks_not_all_passed(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, {"head": {"sha": "empty123"}}),
|
|
_make_response(200, {"total_count": 0, "check_runs": []}),
|
|
]
|
|
)
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.get_pr_check_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["total_count"] == 0
|
|
assert result["check_runs"] == []
|
|
assert result["all_passed"] is False
|
|
assert result["any_failed"] is False
|
|
assert result["any_pending"] is False
|
|
|
|
|
|
def test_get_pr_check_runs_paginates_before_summarizing(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
first_page_runs = [_check_run(run_id) for run_id in range(1, 101)]
|
|
second_page_runs = [_check_run(101, conclusion="failure")]
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, {"head": {"sha": "paged123"}}),
|
|
_make_response(200, {"total_count": 101, "check_runs": first_page_runs}),
|
|
_make_response(200, {"total_count": 101, "check_runs": second_page_runs}),
|
|
]
|
|
)
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.get_pr_check_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["total_count"] == 101
|
|
assert len(result["check_runs"]) == 101
|
|
assert result["any_failed"] is True
|
|
assert result["all_passed"] is False
|
|
|
|
|
|
def test_get_pr_check_runs_pr_fetch_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(return_value=_make_response(404, {"message": "Not Found"}))
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.get_pr_check_runs(42)
|
|
|
|
assert result["success"] is False
|
|
assert "404" in result["error"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# rerun_failed_workflow_runs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rerun_failed_workflow_runs_no_repo_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(github_ci, "_get_repo_config", lambda: {})
|
|
result = github_ci.rerun_failed_workflow_runs(42)
|
|
assert result == {"success": False, "error": "No repo config found"}
|
|
|
|
|
|
def test_rerun_failed_workflow_runs_no_token(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value=None))
|
|
result = github_ci.rerun_failed_workflow_runs(42)
|
|
assert result == {"success": False, "error": "Failed to get GitHub App installation token"}
|
|
|
|
|
|
def test_rerun_failed_workflow_runs_no_failures(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
pr_data = {"head": {"sha": "abc999"}}
|
|
workflow_runs_data = {
|
|
"workflow_runs": [
|
|
{"id": 100, "conclusion": "success"},
|
|
]
|
|
}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, pr_data),
|
|
_make_response(200, workflow_runs_data),
|
|
]
|
|
)
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.rerun_failed_workflow_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["rerun_run_ids"] == []
|
|
assert "No failed workflow runs" in result["message"]
|
|
|
|
|
|
def test_rerun_failed_workflow_runs_with_failures(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
pr_data = {"head": {"sha": "bbb111"}}
|
|
workflow_runs_data = {
|
|
"workflow_runs": [
|
|
{"id": 200, "conclusion": "failure"},
|
|
{"id": 201, "conclusion": "timed_out"},
|
|
]
|
|
}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, pr_data),
|
|
_make_response(200, workflow_runs_data),
|
|
]
|
|
)
|
|
mock_client.post = AsyncMock(return_value=_make_response(201, {}))
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.rerun_failed_workflow_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert set(result["rerun_run_ids"]) == {200, 201}
|
|
assert len(result["rerun_results"]) == 2
|
|
assert all(r["success"] for r in result["rerun_results"])
|
|
|
|
|
|
def test_rerun_failed_workflow_runs_paginates_workflow_runs(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
first_page_runs = [{"id": run_id, "conclusion": "success"} for run_id in range(1, 101)]
|
|
second_page_runs = [{"id": 999, "conclusion": "failure"}]
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, {"head": {"sha": "paged-rerun"}}),
|
|
_make_response(200, {"workflow_runs": first_page_runs}),
|
|
_make_response(200, {"workflow_runs": second_page_runs}),
|
|
]
|
|
)
|
|
mock_client.post = AsyncMock(return_value=_make_response(201, {}))
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.rerun_failed_workflow_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["rerun_run_ids"] == [999]
|
|
assert len(result["rerun_results"]) == 1
|
|
|
|
|
|
def test_get_pr_check_runs_paginated_fetch_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Surface a non-200 response that occurs on a later page of pagination."""
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
first_page_runs = [_check_run(run_id) for run_id in range(1, 101)]
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, {"head": {"sha": "paged-error"}}),
|
|
_make_response(200, {"total_count": 200, "check_runs": first_page_runs}),
|
|
_make_response(500, {"message": "server error"}),
|
|
]
|
|
)
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.get_pr_check_runs(42)
|
|
|
|
assert result["success"] is False
|
|
assert "500" in result["error"]
|
|
|
|
|
|
def test_rerun_failed_workflow_runs_skips_action_required(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""`action_required` runs need manual approval and must not be rerun."""
|
|
monkeypatch.setattr(
|
|
github_ci, "_get_repo_config", lambda: {"owner": "langchain-ai", "name": "open-swe"}
|
|
)
|
|
monkeypatch.setattr(github_ci, "_get_token", AsyncMock(return_value="test-token"))
|
|
|
|
pr_data = {"head": {"sha": "ccc222"}}
|
|
workflow_runs_data = {
|
|
"workflow_runs": [
|
|
{"id": 300, "conclusion": "failure"},
|
|
{"id": 301, "conclusion": "action_required"},
|
|
{"id": 302, "conclusion": "success"},
|
|
]
|
|
}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.get = AsyncMock(
|
|
side_effect=[
|
|
_make_response(200, pr_data),
|
|
_make_response(200, workflow_runs_data),
|
|
]
|
|
)
|
|
mock_client.post = AsyncMock(return_value=_make_response(201, {}))
|
|
|
|
mock_async_context = MagicMock()
|
|
mock_async_context.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_async_context.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
with patch("agent.tools.github_ci.httpx.AsyncClient", return_value=mock_async_context):
|
|
result = github_ci.rerun_failed_workflow_runs(42)
|
|
|
|
assert result["success"] is True
|
|
assert result["rerun_run_ids"] == [300]
|
|
assert mock_client.post.await_count == 1
|