Adam Moussa
035c6e57e5
clarifier_llm: sha1 -> sha256 for the non-security cache-discriminator (CWE-327 false positive). github_adapter + github_intake: inline nosemgrep on the urlopen lines (dynamic-urllib-use-detected) — the URL is built from a fixed https GitHub API base, dynamic part is the path only, no SSRF/file:// surface (extends the existing noqa:S310 trusted-host judgment to semgrep). Scanner now reports 0 mediums on the agent-team scope.
295 lines
13 KiB
Python
295 lines
13 KiB
Python
"""GitHub-issue INTAKE poller: a labeled issue becomes a pipeline task (§3.3.1).
|
|
|
|
This is the *inbound front door* for the GitHub transport. Where
|
|
:mod:`agent_team.transport.github_adapter` delivers clarifier question-sets
|
|
*outbound* (and parses answers back), this leaf runs the other direction: it
|
|
polls a repository for open issues carrying a configured label and turns each
|
|
not-yet-ingested issue into one pipeline task by calling the coordinator's
|
|
intake entry,
|
|
:meth:`agent_team.coordinator.Coordinator.start_task` (``task_text=<issue
|
|
title+body>``, ``transport_name="github"``).
|
|
|
|
The shape mirrors the slack_listener seam: everything network/SDK is
|
|
**injected** so the poller is fully unit-testable with no GitHub SDK and no
|
|
socket:
|
|
|
|
* ``client`` is a small :class:`GithubIssueClient` protocol:
|
|
``list_open_issues(label) -> iterable of issue mappings``. Production wires a
|
|
thin client over the GitHub REST API (deferred import, see
|
|
:func:`build_default_issue_client`); tests pass an in-memory fake.
|
|
* ``coordinator`` is anything exposing ``start_task(task_text=...,
|
|
transport_name=...)``: the live :class:`~agent_team.coordinator.Coordinator`
|
|
in production, a stub in tests. No model or transport is touched here.
|
|
|
|
De-duplication (P3 scope note):
|
|
The poller tracks already-ingested issue ids in an **in-memory** set, so a
|
|
re-poll over the same open issue does not start a second task. This is
|
|
deliberately simple for now: it does NOT survive a process restart. Durable
|
|
de-dup (a ledger table of ingested issue ids, mirroring the
|
|
``pending_questions`` discipline) is a FOLLOW-UP and is intentionally not
|
|
shipped here. After a restart an already-ingested-but-still-open issue would
|
|
be re-ingested; document that and treat the in-memory set as a best-effort
|
|
guard, not a durable contract.
|
|
|
|
Design constraints honoured here (pre-deployment scaffolding):
|
|
* **No live infrastructure.** Nothing is provisioned or called at import.
|
|
The GitHub client is dependency-injected; the default client's SDK/HTTP
|
|
import is DEFERRED (mirrors
|
|
:func:`agent_team.graph.build_sqlite_checkpointer` and the slack_listener
|
|
SDK discipline), so this module imports cleanly with no optional SDK
|
|
present and the unit tests stay fully hermetic.
|
|
* **P2 stays the production default; this is OPT-IN and INERT.** This module
|
|
does no CI, no OIDC, no git/patch apply, and no network to GitHub Actions.
|
|
It only reads issues and calls the coordinator's existing intake entry.
|
|
* **Secrets never committed.** The default client reads the GitHub token
|
|
from the environment at call time, never from source.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any, Iterable, Protocol
|
|
|
|
__all__ = [
|
|
"GithubIntake",
|
|
"GithubIssueClient",
|
|
"build_default_issue_client",
|
|
"issue_task_text",
|
|
]
|
|
|
|
_LOG = logging.getLogger(__name__)
|
|
|
|
# The transport name handed to the coordinator's intake entry so the resulting
|
|
# task's clarifier question-sets route over the GitHub adapter (§3.3.1 D10).
|
|
GITHUB_TRANSPORT_NAME = "github"
|
|
|
|
# Environment variable the default client reads the GitHub token from at call
|
|
# time (never stored in source/state). Mirrors the github_adapter default.
|
|
DEFAULT_TOKEN_ENV = "GITHUB_TOKEN"
|
|
|
|
|
|
class GithubIssueClient(Protocol):
|
|
"""Injected GitHub issue source: list open issues carrying a label.
|
|
|
|
A narrow read-only seam so the poller has no hard dependency on any GitHub
|
|
SDK and the tests pass a pure in-memory fake. Each returned issue is a
|
|
mapping with at least an ``id`` (or ``number``) and a ``title``; ``body`` is
|
|
optional. The mapping shape mirrors the GitHub REST issue object so the
|
|
production client can return the API JSON unchanged.
|
|
"""
|
|
|
|
def list_open_issues(self, *, label: str) -> Iterable[dict[str, Any]]:
|
|
"""Return the open issues carrying ``label`` (most-recent-first is fine)."""
|
|
...
|
|
|
|
|
|
def issue_task_text(issue: dict[str, Any]) -> str:
|
|
"""Render one issue's intake ``task_text`` from its title + body.
|
|
|
|
The pipeline's task description is the issue title followed by its body (a
|
|
blank line between them when both are present). A missing/empty body yields
|
|
just the title; a missing/empty title falls back to ``issue #<id>`` so the
|
|
task is never an empty string. Whitespace is stripped at the edges so a
|
|
trailing-newline body does not produce trailing blank lines.
|
|
"""
|
|
title = str(issue.get("title") or "").strip()
|
|
body = str(issue.get("body") or "").strip()
|
|
|
|
if not title:
|
|
title = f"issue #{_issue_id(issue)}"
|
|
|
|
if body:
|
|
return f"{title}\n\n{body}"
|
|
return title
|
|
|
|
|
|
def _issue_id(issue: dict[str, Any]) -> str:
|
|
"""Return the de-dup identity for ``issue`` as a string.
|
|
|
|
Prefers the GitHub global ``id`` (stable across renames); falls back to the
|
|
per-repo ``number`` when ``id`` is absent (some payload shapes / fakes carry
|
|
only ``number``). Stringified so heterogeneous int/str ids compare cleanly
|
|
in the ingested set.
|
|
"""
|
|
raw = issue.get("id")
|
|
if raw is None:
|
|
raw = issue.get("number")
|
|
return str(raw)
|
|
|
|
|
|
class GithubIntake:
|
|
"""Poll a repo for labeled issues and start one pipeline task per new issue.
|
|
|
|
Construct with an injected ``client`` (a :class:`GithubIssueClient`), an
|
|
injected ``coordinator`` (anything exposing
|
|
``start_task(task_text=..., transport_name=...)``), and the ``label`` that
|
|
flags an issue as pipeline intake. Call :meth:`poll_once` on a cadence (an
|
|
operator loop or a cron); each call lists the open labeled issues and starts
|
|
a task for every one not yet ingested.
|
|
|
|
De-dup is in-memory only (see the module docstring): the set of ingested
|
|
issue ids lives on the instance, so a re-poll within one process never
|
|
double-ingests, but a restart loses the set. Durable de-dup is a follow-up.
|
|
|
|
Nothing here touches the network or any SDK directly (the client does, and
|
|
it is injected), so the whole poller is unit-testable with a fake client and
|
|
a stub coordinator.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
client: GithubIssueClient,
|
|
coordinator: Any,
|
|
label: str,
|
|
) -> None:
|
|
"""Bind the poller to one client, coordinator, and intake label.
|
|
|
|
Args:
|
|
client: The injected issue source. Its ``list_open_issues`` is the
|
|
only GitHub call the poller makes.
|
|
coordinator: The intake target. Must expose
|
|
``start_task(task_text=..., transport_name=...)``: the live
|
|
:class:`~agent_team.coordinator.Coordinator` in production.
|
|
label: The issue label that marks an issue as pipeline intake. Only
|
|
issues the client returns for this label are considered; an
|
|
empty label is rejected so a misconfiguration cannot ingest
|
|
every open issue.
|
|
"""
|
|
if not label:
|
|
raise ValueError(
|
|
"GithubIntake requires a non-empty intake label; an empty label "
|
|
"would ingest every open issue"
|
|
)
|
|
self._client = client
|
|
self._coordinator = coordinator
|
|
self._label = label
|
|
# In-memory de-dup set (P3 scope: best-effort, NOT durable across a
|
|
# restart; see the module docstring). Tracks issue ids already turned
|
|
# into tasks so a re-poll does not double-ingest.
|
|
self._ingested: set[str] = set()
|
|
|
|
@property
|
|
def label(self) -> str:
|
|
"""The configured intake label (read-only)."""
|
|
return self._label
|
|
|
|
@property
|
|
def ingested_ids(self) -> frozenset[str]:
|
|
"""A snapshot of the issue ids already ingested this process (read-only)."""
|
|
return frozenset(self._ingested)
|
|
|
|
def poll_once(self) -> list[str]:
|
|
"""List the labeled open issues and start a task for each new one.
|
|
|
|
One maintenance pass:
|
|
|
|
1. Ask the injected client for the open issues carrying the configured
|
|
label (:meth:`GithubIssueClient.list_open_issues`).
|
|
2. For each issue NOT already in the in-memory ingested set, call
|
|
``coordinator.start_task(task_text=<title+body>,
|
|
transport_name="github")`` and record its id so a subsequent poll
|
|
does not re-ingest it.
|
|
|
|
Issues already ingested this process are skipped (the in-memory de-dup),
|
|
and any issue the client returns without the label is *not* expected
|
|
(the client filters by label) but is ignored defensively if present.
|
|
An issue id is recorded as ingested ONLY after ``start_task`` returns,
|
|
so a failing intake leaves the issue eligible for retry on the next poll
|
|
rather than silently dropping it.
|
|
|
|
Returns the list of issue ids ingested on THIS pass (empty when nothing
|
|
new), so an operator loop can log/meter intake volume.
|
|
"""
|
|
ingested_now: list[str] = []
|
|
for issue in self._client.list_open_issues(label=self._label):
|
|
issue_id = _issue_id(issue)
|
|
if issue_id in self._ingested:
|
|
_LOG.debug("github-intake: issue %s already ingested; skip", issue_id)
|
|
continue
|
|
|
|
task_text = issue_task_text(issue)
|
|
_LOG.info(
|
|
"github-intake: starting task for issue %s (label=%s)",
|
|
issue_id,
|
|
self._label,
|
|
)
|
|
# start_task is the committed coordinator intake entry; the resulting
|
|
# task's clarifier question-sets route over the GitHub adapter. Record
|
|
# the id only after the call returns so a raise leaves the issue
|
|
# eligible for retry on the next poll (no silent drop).
|
|
self._coordinator.start_task(
|
|
task_text=task_text,
|
|
transport_name=GITHUB_TRANSPORT_NAME,
|
|
)
|
|
self._ingested.add(issue_id)
|
|
ingested_now.append(issue_id)
|
|
|
|
return ingested_now
|
|
|
|
|
|
def build_default_issue_client(
|
|
*,
|
|
owner: str,
|
|
repo: str,
|
|
token_env: str = DEFAULT_TOKEN_ENV,
|
|
api_root: str = "https://api.github.com",
|
|
) -> GithubIssueClient:
|
|
"""Build the production read-only issue client (deferred SDK/HTTP import).
|
|
|
|
Returns a :class:`GithubIssueClient` that lists a repo's open issues by
|
|
label over the GitHub REST API. The HTTP machinery (``urllib``) and the
|
|
token read are deferred to call time (mirroring
|
|
:func:`agent_team.graph.build_sqlite_checkpointer` and the slack_listener
|
|
SDK discipline), so importing this module never touches the network and the
|
|
unit tests (which inject a fake client) never reach this path.
|
|
|
|
The token is read from ``token_env`` at call time and sent as a bearer
|
|
credential; it is never stored in source or logged. ``api_root`` is
|
|
overridable for GitHub Enterprise.
|
|
|
|
This is intentionally a thin, read-only lister: it issues a single GET to
|
|
the issues endpoint with ``state=open&labels=<label>`` and returns the
|
|
parsed JSON array unchanged (each element is a GitHub issue object, which
|
|
already carries ``id`` / ``number`` / ``title`` / ``body``). It performs no
|
|
CI, OIDC, write, or GitHub-Actions call; it only reads issues.
|
|
"""
|
|
|
|
class _RestIssueClient:
|
|
"""Stdlib-only GitHub REST issue lister (built lazily, no import-time HTTP)."""
|
|
|
|
def __init__(self) -> None:
|
|
self._owner = owner
|
|
self._repo = repo
|
|
self._token_env = token_env
|
|
self._api_root = api_root.rstrip("/")
|
|
|
|
def list_open_issues(self, *, label: str) -> Iterable[dict[str, Any]]:
|
|
import json
|
|
import os
|
|
from urllib import parse as _urlparse
|
|
from urllib import request as _urlrequest
|
|
|
|
token = os.environ.get(self._token_env)
|
|
if not token:
|
|
raise RuntimeError(
|
|
f"no GitHub token available (env {self._token_env!r} unset); "
|
|
"cannot list issues for intake"
|
|
)
|
|
|
|
query = _urlparse.urlencode({"state": "open", "labels": label})
|
|
url = f"{self._api_root}/repos/{self._owner}/{self._repo}/issues?{query}"
|
|
request = _urlrequest.Request(url, method="GET")
|
|
request.add_header("Authorization", f"Bearer {token}")
|
|
request.add_header("Accept", "application/vnd.github+json")
|
|
request.add_header("X-GitHub-Api-Version", "2022-11-28")
|
|
# url is built from a fixed https GitHub API base; the dynamic part is
|
|
# the path, never the scheme, so there is no SSRF/file:// surface.
|
|
with _urlrequest.urlopen(request) as response: # noqa: S310 (trusted api host); nosemgrep
|
|
raw = response.read().decode("utf-8")
|
|
data = json.loads(raw) if raw else []
|
|
# The issues endpoint can include pull requests (they share the
|
|
# endpoint); filter them out so a PR is never intaken as an issue.
|
|
return [item for item in data if "pull_request" not in item]
|
|
|
|
return _RestIssueClient()
|