feat(agent-team): P1/P2 graph wiring + coordinator daemon + run-team start/serve

build_graph gains injected live_plan_node/review_node/route_review: P1 = plan->END, P2 = clarify->plan->review->{build|loop-back|parked}. Coordinator composes clarifier->graph->ResumeWorker, wraps planner fail-safe, binds the GPT-4.1 review loop; run-team start/serve opt production into P2. Re-delivery uses a guarded CAS so a concurrently-answered row is never clobbered (closes RACE-REDELIVER).
This commit is contained in:
Adam Moussa 2026-06-18 12:56:42 -04:00
parent 253e31b0e8
commit 22edd4143a
6 changed files with 1757 additions and 7 deletions

View file

@ -0,0 +1,710 @@
"""COORDINATOR daemon — the live Plane-2 runtime keystone (design §3.3, §3.3.1, §7.1 P1).
This module ties the committed Plane-2 leaves into a single running process. It
owns no new durable contract of its own: it *composes* the foundation pieces
into the live loop the design's §3.3.1 lifecycle requires:
live clarifier (Claude) ─► graph (durable, interrupt/resume) ─► ResumeWorker
The :class:`Coordinator` is the object ``run-team.py serve`` runs and
``run-team.py start`` drives one task through. Every network/SDK touch and the
transport are **injected**, so the coordinator is unit-testable with no Claude
SDK and no Slack:
* ``transport`` is a :class:`~agent_team.transport.base.Transport` — a live
``SlackTransport`` in production, a fake in tests.
* ``build_clarify_node`` defaults to the real Claude clarifier factory
(:func:`agent_team.nodes.clarifier_llm.build_claude_clarifier_callables` →
:func:`agent_team.nodes.clarifier.make_clarifier_node`); tests inject a stub
node so no model is ever called.
* the resume queue is a plain :class:`queue.Queue` of
:class:`~agent_team.responder.ResumeJob`, shared with the slack_listener so an
inbound answer and the drainer meet on one durable seam.
Composition (the keystone wiring, §3.3):
1. :meth:`setup` builds the SQLite checkpointer + the graph
(:func:`agent_team.graph.build_graph` with the injected ``live_clarify_node``)
and constructs a :class:`agent_team.resume_worker.ResumeWorker` over the
compiled graph (the worker is satisfied directly by the LangGraph app via
``get_state`` / ``invoke``). ``serve`` additionally calls
:func:`agent_team.invoker.bind_subscription_invoker` BEFORE the clarifier node
is built so the Claude path is live; tests skip ``serve`` and call ``setup``
directly, so no invoker is bound.
2. :meth:`start_task` runs the graph to the first human-gate ``interrupt()``,
reads the pending question payload, and notifies it (ledger row OPEN, then
transport post) via :func:`agent_team.responder.notify_question`.
3. :meth:`submit_answer` runs the responder's first-answer-wins compare-and-set
and enqueues a resume job onto the shared queue.
4. :meth:`drain_resumes` pulls jobs off the queue and runs them through the
single-flight, turn-guarded :class:`ResumeWorker`.
5. :meth:`tick` is one maintenance pass: the deadline sweep parks each expired
task (ALARM hook) and then drains resumes.
6. :meth:`recover` is the §3.3.1 startup sweep: re-drive ``answered`` rows and
clear ``open`` rows lacking a ``channel_ref`` so a reboot converges.
No SQL and no transport rendering live here; everything goes through the
committed leaves (responder, resume_worker, db.schema, graph, transport).
"""
from __future__ import annotations
import logging
import queue
from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable
from agent_team import graph as graph_mod
from agent_team import responder as responder_mod
from agent_team.db.schema import connect, init_db
from agent_team.resume_worker import ResumeResult, ResumeWorker
from agent_team.transport.base import Transport
if TYPE_CHECKING: # pragma: no cover - typing only
from agent_team.task_model import PipelineState
__all__ = [
"Coordinator",
"default_clarify_node_factory",
]
_LOG = logging.getLogger("agent_team.coordinator")
# Default poll cadence for the serve() maintenance loop (the deadline-sweep
# tick rate, §3.3.1). The transport inbound feed is the slack_listener's job;
# this cadence governs only the deadline/recovery maintenance pass.
DEFAULT_POLL_INTERVAL = timedelta(seconds=30)
# How long a parked-task ALARM message is; kept here so the ALARM hook default
# is one obvious place (§6.6 "ALARM rather than spin").
_PARK_ALARM_TEMPLATE = (
"task parked: clarifier question %s expired with no answer in window "
"(§3.3.1 deadline policy); ALARM raised, not spinning"
)
# A clarify-node factory: build the live LangGraph clarifier node. Injected so
# tests substitute a stub (no Claude). Returns a node callable suitable for
# build_graph(..., live_clarify_node=...).
ClarifyNodeFactory = Callable[[], "Callable[[PipelineState], PipelineState]"]
# Builds the live planner node (P2). None -> the graph's P1 plan stub (terminus).
PlanNodeFactory = Callable[[], "Callable[[PipelineState], PipelineState]"]
# Binds the review invoker and yields (review_node, route_review) for P2. None ->
# no review loop (P1: plan is the terminus).
ReviewWiring = Callable[
[],
"tuple[Callable[[PipelineState], PipelineState], Callable[[PipelineState], str]]",
]
# A checkpointer factory over the db path: returns the BaseCheckpointSaver the
# graph is compiled with. Defaults to the production SQLite checkpointer
# (:func:`agent_team.graph.build_sqlite_checkpointer`); tests inject a factory
# returning an in-memory saver so setup() needs no SQLite checkpointer package
# and no contextmanager unwrap.
CheckpointerFactory = Callable[[Path], Any]
# A park-ALARM hook: called once per task whose clarifier question expired. The
# default logs; production can inject a CloudWatch/Slack ALARM emitter. Injected
# so the deadline policy stays I/O-free in tests (§6.6).
AlarmHook = Callable[[str], None]
def default_clarify_node_factory() -> Callable[[PipelineState], PipelineState]:
"""Build the live Claude-backed clarifier node (§3.3, §7.1 P1).
Composes the two committed leaves: the Claude clarifier callables
(:func:`agent_team.nodes.clarifier_llm.build_claude_clarifier_callables`,
which route every reasoning step through the ``billing.claude_invoke`` seam)
and the clarifier loop node
(:func:`agent_team.nodes.clarifier.make_clarifier_node`). The result is the
real multi-turn ``interrupt()`` node handed to
:func:`agent_team.graph.build_graph` as ``live_clarify_node``.
Imported here (not at module load) so the coordinator module imports cleanly
and tests that inject a stub node never pull the Claude wiring. The billing
seam must already be bound (``bind_subscription_invoker``) for the node to
actually call Claude at run time; ``serve`` binds it before this runs.
"""
from agent_team.nodes.clarifier import make_clarifier_node
from agent_team.nodes.clarifier_llm import build_claude_clarifier_callables
assess_confidence, generate_questions = build_claude_clarifier_callables()
return make_clarifier_node(
assess_confidence=assess_confidence,
generate_questions=generate_questions,
)
def default_plan_node_factory() -> Callable[[PipelineState], PipelineState]:
"""Build the live planner node wrapped fail-safe (§3.3 P2).
The canonical planner is :func:`agent_team.nodes.planner.plan_node` (it owns
the prompt, the loop-back review-feedback folding, and the revision-cap
escalation). It calls Claude through the billing seam and *raises*
:class:`~agent_team.nodes.planner.PlannerError` on an unparseable reply. A
raise inside a LangGraph node would crash the run, so this wrapper catches it
and instead returns a **park** delta (status/phase PARKED) so a bad planner
reply escalates to Adam rather than taking down the pipeline — the same
fail-SAFE discipline the clarifier and review stages use.
Imported lazily so the coordinator module stays import-clean and the SDK is
only pulled when the live node is actually built.
"""
from agent_team.nodes import planner
from agent_team.task_model import Phase, PipelineState, TaskStatus
def plan_node(state: PipelineState) -> PipelineState:
try:
return planner.plan_node(state)
except planner.PlannerError:
# Unparseable plan -> park + ALARM rather than crash the graph.
return PipelineState(
status=TaskStatus.PARKED.value,
current_phase=Phase.PARKED.value,
)
return plan_node
def default_review_wiring() -> tuple[
Callable[[PipelineState], PipelineState], Callable[[PipelineState], str]
]:
"""Bind the GPT-4.1 review invoker and return the P2 review node + router.
Binds :func:`agent_team.nodes.review_loop_llm.default_plan_reviewer` (the
cross-family reviewer that shells the local orchestrator ``run.py`` ->
``cross_reviewer`` GPT-4.1) onto the review-loop seam via
:func:`~agent_team.nodes.review_loop.set_review_invoker`, then returns
``(review_node, route_after_review)`` for
:func:`agent_team.graph.build_graph`. Lazy-imported for the same
import-hygiene reason as the clarifier/planner factories.
"""
from agent_team.nodes import review_loop
from agent_team.nodes.review_loop_llm import default_plan_reviewer
review_loop.set_review_invoker(default_plan_reviewer)
# bind_review_node yields a single-arg node so LangGraph does not inject its
# RunnableConfig over the review config (no warning, config threads through).
return review_loop.bind_review_node(), review_loop.route_after_review
class Coordinator:
"""Owns the live Plane-2 runtime: graph + resume worker + transport (§3.3).
Construct with the durable DB path, an injected ``transport``, and optional
injected seams (clarify-node factory, resume queue, deadline window, ALARM
hook). Call :meth:`setup` once to build the graph + resume worker, then drive
it with :meth:`start_task` / :meth:`submit_answer` / :meth:`drain_resumes` /
:meth:`tick` / :meth:`recover`. :meth:`serve` binds the live Claude invoker
and runs the maintenance loop.
Everything network/SDK is injected so the whole surface (minus the infinite
serve loop) is unit-testable with a fake transport and a stub clarify node.
"""
def __init__(
self,
*,
db_path: Path | str,
transport: Transport,
build_clarify_node: ClarifyNodeFactory | None = None,
build_plan_node: PlanNodeFactory | None = None,
review_wiring: ReviewWiring | None = None,
build_checkpointer: CheckpointerFactory | None = None,
resume_queue: "queue.Queue[Any] | None" = None,
deadline_window: timedelta | None = None,
alarm_hook: AlarmHook | None = None,
) -> None:
self._db_path = Path(db_path)
self._transport = transport
self._build_clarify_node = build_clarify_node or default_clarify_node_factory
# P2 wiring is OPT-IN: left None, the graph uses its P1 plan stub and no
# review loop (keeps the unit suite model-free). The production serve/start
# path injects default_plan_node_factory + default_review_wiring.
self._build_plan_node = build_plan_node
self._review_wiring = review_wiring
self._build_checkpointer = (
build_checkpointer or graph_mod.build_sqlite_checkpointer
)
self._resume_queue: "queue.Queue[Any]" = resume_queue or queue.Queue()
self._deadline_window = deadline_window or graph_mod.DEFAULT_CLARIFY_DEADLINE
self._alarm_hook = alarm_hook or self._default_alarm_hook
# Built by setup().
self._graph: Any = None
self._resume_worker: ResumeWorker | None = None
# Retains a context-manager checkpointer (production SQLite saver) so its
# __exit__ is not run early; held open for the daemon's lifetime.
self._checkpointer_cm: Any = None
# ------------------------------------------------------------------ #
# Accessors (the shared queue is the slack_listener handoff seam).
# ------------------------------------------------------------------ #
@property
def resume_queue(self) -> "queue.Queue[Any]":
"""The shared resume-job queue (slack_listener enqueues, drainer drains)."""
return self._resume_queue
@property
def graph(self) -> Any:
"""The compiled LangGraph app (``None`` until :meth:`setup`)."""
return self._graph
@property
def resume_worker(self) -> ResumeWorker | None:
"""The single-flight resume worker (``None`` until :meth:`setup`)."""
return self._resume_worker
# ------------------------------------------------------------------ #
# Lifecycle wiring.
# ------------------------------------------------------------------ #
def setup(self) -> None:
"""Build the durable graph + resume worker (§3.3 keystone composition).
Idempotent-friendly one-time wiring: ensures the ledger tables exist
(:func:`agent_team.db.init_db`), builds the SQLite checkpointer and the
graph with the injected live clarifier node
(:func:`agent_team.graph.build_graph`), and constructs the
:class:`ResumeWorker` over the compiled graph and a fresh durable
connection.
Does NOT bind the Claude invoker — that is ``serve``'s job, kept out of
``setup`` so tests can build a coordinator with a stub clarify node and
never touch the SDK. The clarify node is built here (from the injected
factory) and handed to ``build_graph`` as ``live_clarify_node``.
The production :func:`agent_team.graph.build_sqlite_checkpointer` returns
a context manager (``SqliteSaver.from_conn_string``), not a bare saver,
so we enter it here to obtain the live saver and hold the context open
for the coordinator's lifetime (the process runs the saver until exit).
An already-unwrapped saver (the test in-memory factory) passes through.
"""
init_db(self._db_path)
checkpointer = self._enter_checkpointer(self._build_checkpointer(self._db_path))
clarify_node = self._build_clarify_node()
# P2 (opt-in): a wrapped real planner + the bound GPT-4.1 review loop.
plan_node = self._build_plan_node() if self._build_plan_node else None
review_node: Any = None
route_review: Any = None
if self._review_wiring is not None:
review_node, route_review = self._review_wiring()
self._graph = graph_mod.build_graph(
checkpointer,
live_clarify_node=clarify_node,
live_plan_node=plan_node,
review_node=review_node,
route_review=route_review,
)
# The ResumeWorker is satisfied directly by the compiled LangGraph app
# (get_state / invoke); it gets its own durable connection so its ledger
# writes do not contend with notify/answer connections.
self._resume_worker = ResumeWorker(self._graph, connect(self._db_path))
def _enter_checkpointer(self, checkpointer: Any) -> Any:
"""Unwrap a context-manager checkpointer into a live saver, or pass through.
``SqliteSaver.from_conn_string`` (the production factory) is a context
manager yielding the saver; entering it gives the live saver and keeps
the underlying connection open for the daemon's lifetime (the saved
``__exit__`` runs at process teardown). A plain ``BaseCheckpointSaver``
(the injected in-memory test factory) has no ``__enter__`` and is
returned unchanged.
"""
enter = getattr(checkpointer, "__enter__", None)
if enter is None:
return checkpointer
saver = enter()
# Retain the CM so its __exit__ is not run early by the GC; the daemon
# holds the saver open until the process exits.
self._checkpointer_cm = checkpointer
return saver
# ------------------------------------------------------------------ #
# Intake.
# ------------------------------------------------------------------ #
def start_task(self, *, task_text: str, transport_name: str) -> str:
"""Start one task: run to the first human gate, then notify (§3.3, §3.3.1).
Runs :func:`agent_team.graph.start_task` to the first clarifier
``interrupt()``, reads the pending question payload
(:func:`agent_team.graph.pending_question`), and delivers it via
:func:`agent_team.responder.notify_question` (ledger row OPEN first, then
transport post). Returns the minted ``thread_id``.
**Intake-seed decision (P1).** ``agent_team.graph.start_task`` builds its
own INTAKE seed and accepts only ``thread_id`` / ``transport`` — it takes
no task-description argument, and a value pre-seeded onto the START
checkpoint via ``update_state`` is overwritten by its own seed invoke
(and a post-suspend ``update_state`` clears the pending interrupt, which
would break the human gate). So for P1 the ``task_text`` is intake
metadata held coordinator-side (logged) rather than written into
``PipelineState``: the deterministic P1 clarifier does not consume a task
description anyway, and threading it into the graph state is a later phase
that extends the committed ``start_task`` seed contract. We keep it
minimal rather than reach past that contract or disturb the gate.
"""
if self._graph is None:
raise RuntimeError("Coordinator.start_task called before setup()")
_LOG.info("start_task intake (transport=%s): %s", transport_name, task_text)
thread_id, _state = graph_mod.start_task(self._graph, transport=transport_name)
question = graph_mod.pending_question(self._graph, thread_id=thread_id)
if question is None:
# No human gate raised (e.g. the clarifier already cleared the bar
# from context alone, or a stub that does not interrupt). Nothing to
# notify; the task proceeds without a pending question.
_LOG.info("start_task %s produced no pending question", thread_id)
return thread_id
question_set = question["question_set"]
deadline = question.get("deadline") or self._default_deadline()
conn = connect(self._db_path)
try:
responder_mod.notify_question(
conn,
self._transport,
question_set,
deadline=deadline,
)
finally:
conn.close()
return thread_id
# ------------------------------------------------------------------ #
# Answer intake (the slack_listener calls this, or submit_answer directly).
# ------------------------------------------------------------------ #
def submit_answer(self, raw: Any) -> responder_mod.AnswerOutcome:
"""Run the first-answer-wins compare-and-set and enqueue a resume (§3.3.1).
Normalizes ``raw`` through the transport, runs
:func:`agent_team.responder.submit_answer` (the single atomic
``open → answered`` flip), and on the rowcount-1 win enqueues a
:class:`ResumeJob` onto the shared resume queue via
``self._resume_queue.put``. The slack_listener calls this in production
(or calls ``responder.submit_answer`` directly with the same queue); the
shared queue is the coordination seam either way. Returns the
:class:`AnswerOutcome` so the caller can reply "already closed" on a
duplicate / late answer (``accepted=False``).
"""
conn = connect(self._db_path)
try:
return responder_mod.submit_answer(
conn,
self._transport,
raw,
enqueue_resume=self._resume_queue.put,
)
finally:
conn.close()
# ------------------------------------------------------------------ #
# Resume drain (single-flight, turn-guarded via ResumeWorker).
# ------------------------------------------------------------------ #
def drain_resumes(self) -> list[ResumeResult]:
"""Drain queued resume jobs through the turn-guarded worker (§3.3.1).
Pulls every currently-queued :class:`ResumeJob` and runs it through
:meth:`agent_team.resume_worker.ResumeWorker.resume`. Single-flight (per
``thread_id``) and the turn guard live inside the worker, so a stale or
redelivered job supersedes-and-skips rather than double-applying. Drains
only what is already queued (a snapshot of ``qsize``) so a steady stream
of new answers cannot starve the caller; the next tick drains the rest.
Returns the per-job results for logging/ALARM.
"""
if self._resume_worker is None:
raise RuntimeError("Coordinator.drain_resumes called before setup()")
results: list[ResumeResult] = []
for _ in range(self._resume_queue.qsize()):
try:
job = self._resume_queue.get_nowait()
except queue.Empty:
break
try:
results.append(
self._resume_worker.resume(
thread_id=job.thread_id,
question_id=job.question_id,
turn=job.turn,
answer=job.answer,
)
)
finally:
self._resume_queue.task_done()
return results
# ------------------------------------------------------------------ #
# Maintenance tick (deadline policy + drain).
# ------------------------------------------------------------------ #
def tick(self) -> list[ResumeResult]:
"""One maintenance pass: deadline sweep + park policy, then drain (§3.3.1).
Runs :func:`agent_team.responder.deadline_sweep` to flip overdue ``open``
questions to ``expired`` (the deterministic answer-vs-expiry race), then
applies the park policy to each newly-expired id (raise the ALARM hook —
§6.6 "ALARM rather than spin"; the durable ledger row is already
``expired``, which is the task's parked state for P1). Finally drains any
resume jobs that landed. Returns the drain results.
"""
conn = connect(self._db_path)
try:
expired = responder_mod.deadline_sweep(conn)
finally:
conn.close()
for question_id in expired:
self._park(question_id)
return self.drain_resumes()
def _park(self, question_id: str) -> None:
"""Apply the park policy to one expired question (§6.6 ALARM, not spin).
The deadline sweep already flipped the ledger row to ``expired`` (the
parked state for P1); this raises the injected ALARM hook so the stall is
surfaced rather than silently spun on. Kept separate so the park policy
is one obvious, testable place.
"""
self._alarm_hook(question_id)
@staticmethod
def _default_alarm_hook(question_id: str) -> None:
"""Default park ALARM: log at WARNING (production injects a real emitter)."""
_LOG.warning(_PARK_ALARM_TEMPLATE, question_id)
# ------------------------------------------------------------------ #
# Startup recovery (§3.3.1 converge-after-reboot).
# ------------------------------------------------------------------ #
def recover(self) -> list[ResumeResult]:
"""Startup sweep so a reboot converges from the durable ledger (§3.3.1).
Two halves of the §3.3.1 recovery:
1. Re-drive every ``answered`` ledger row whose graph is still
interrupted on its turn
(:meth:`agent_team.resume_worker.ResumeWorker.recover_pending_resumes`).
The turn guard makes this idempotent — a thread that already advanced
supersedes-and-skips, so converging cannot double-apply.
2. Re-deliver ``open`` rows that lack a ``channel_ref`` (a post lost to a
crash between the durable ledger write and the transport post): clear
any stale ref and re-post over the transport so the human gate is
actually reachable again (§3.3.1 "delivery / lost-post").
Returns the resume results from half (1) for logging/ALARM.
"""
if self._resume_worker is None:
raise RuntimeError("Coordinator.recover called before setup()")
results = self._resume_worker.recover_pending_resumes()
self._redeliver_unposted_open_questions()
return results
def _redeliver_unposted_open_questions(self) -> None:
"""Re-post ``open`` rows that never recorded a ``channel_ref`` (§3.3.1).
A crash between :func:`notify_question`'s durable ledger write and its
transport post leaves an ``open`` row with no ``channel_ref``. On reboot
we re-derive the pending interrupt payload from the live checkpoint and
re-post it, then record the fresh ref. Idempotent: a row already carrying
a ref is skipped, and a thread no longer interrupted (already advanced)
is skipped. The re-post reuses the committed
:func:`agent_team.responder.notify_question`, so the ledger-then-post
ordering and lost-post tolerance are identical to first delivery.
**Lost-update guard (RACE-REDELIVER-NONATOMIC-DELETE, CWE-362).** The
SELECT and the clear-then-reinsert are NOT one transaction, and the
answer compare-and-set writes on a *separate* connection. An answer that
lands between the SELECT and the clear flips the row ``open`` →
``answered`` on that other connection; an unconditional DELETE would then
destroy the just-recorded human answer, dangle the enqueued ResumeJob,
and re-ask the human with a fresh ``open`` row. So the clear is a
*guarded* compare-and-set: a conditional DELETE re-checking
``status='open' AND channel_ref IS NULL`` at WRITE time, run under
``BEGIN IMMEDIATE`` on a private connection (mirroring schema.py's
``_compare_and_set``). If its rowcount is 0 the row is no longer an
unposted-open question (it was answered/expired/advanced concurrently),
so re-delivery is SKIPPED — an accepted answer is never clobbered and the
human is never re-asked. Only a row STILL open-and-unposted at write time
is re-INSERTed and re-posted.
"""
conn = connect(self._db_path)
try:
rows = conn.execute(
"SELECT question_id, thread_id, deadline_at FROM pending_questions "
"WHERE status='open' AND channel_ref IS NULL"
).fetchall()
for row in rows:
thread_id = row["thread_id"]
question = graph_mod.pending_question(self._graph, thread_id=thread_id)
if question is None:
# Graph already advanced past this gate; nothing to re-post.
continue
# Guarded clear: re-check the unposted-open predicate atomically
# at WRITE time. If 0 rows are deleted, the row was answered /
# expired / advanced concurrently between the SELECT above and
# this write — SKIP re-delivery so an accepted answer is never
# destroyed and the human is never re-asked.
if not self._guarded_clear_unposted_open(row["question_id"]):
continue
question_set = question["question_set"]
deadline = (
row["deadline_at"]
or question.get("deadline")
or self._default_deadline()
)
responder_mod.notify_question(
conn,
self._transport,
question_set,
deadline=deadline,
)
finally:
conn.close()
# Bounded retry mirroring schema.py's _compare_and_set: if the write lock is
# still contended after busy_timeout elapses, retry a few times rather than
# surfacing a transient "database is locked" to the recovery sweep.
_CLEAR_RETRY_ATTEMPTS = 3
_CLEAR_RETRY_BACKOFF_S = 0.05
def _guarded_clear_unposted_open(self, question_id: str) -> bool:
"""Conditionally DELETE an unposted-open row, atomically (CWE-362 guard).
The compare-and-set that closes the redeliver lost-update window. Runs the
predicate-guarded DELETE
DELETE FROM pending_questions
WHERE question_id=? AND status='open' AND channel_ref IS NULL
under ``BEGIN IMMEDIATE`` on a *private*, short-lived connection — exactly
the discipline schema.py's :func:`~agent_team.db.schema._compare_and_set`
uses for ``answer_question`` / ``expire_question``. Taking the write lock
up front and re-checking ``status='open' AND channel_ref IS NULL`` at
write time means a concurrent answer-CAS (on its own connection) either
ran first — in which case the row is no longer ``open`` and this DELETE
matches 0 rows — or runs after — in which case it sees the row already
gone and its own ``WHERE status='open'`` guard matches 0 rows. Either way
the answer is never lost.
Returns ``True`` iff exactly one row was deleted (the row was STILL
unposted-open, so re-delivery should proceed); ``False`` if 0 rows matched
(answered/expired/advanced concurrently → caller SKIPS re-delivery).
For an in-memory DB (no backing file to reopen) the write runs on the
passed-through connection; in-memory DBs are single-connection and not the
concurrent production path — the same fallback schema.py makes.
"""
sql = (
"DELETE FROM pending_questions "
"WHERE question_id=? AND status='open' AND channel_ref IS NULL"
)
params = (question_id,)
probe = connect(self._db_path)
try:
db_file = getattr(probe, "agent_team_db_path", "") or None
finally:
probe.close()
if db_file is None:
return self._clear_once(connect(self._db_path), sql, params, close=True)
import time
last_err: Any = None
for attempt in range(self._CLEAR_RETRY_ATTEMPTS):
write = connect(Path(db_file))
try:
return self._clear_once(write, sql, params, close=False)
except Exception as err: # noqa: BLE001 - re-raised below if not a lock
if "locked" not in str(err).lower():
raise
last_err = err
finally:
write.close()
time.sleep(self._CLEAR_RETRY_BACKOFF_S * (attempt + 1))
assert last_err is not None # loop only exits early via return or raise
raise last_err
@staticmethod
def _clear_once(
conn: Any, sql: str, params: tuple[Any, ...], *, close: bool
) -> bool:
"""Run one ``BEGIN IMMEDIATE`` conditional DELETE; rowcount==1 → True.
Mirrors schema.py's ``_cas_once``: ``BEGIN IMMEDIATE`` before the try so a
lock-acquisition error propagates to the retry loop with no transaction to
unwind; once open, any failure rolls back (best-effort) and re-raises.
"""
try:
conn.execute("BEGIN IMMEDIATE")
try:
cur = conn.execute(sql, params)
changed = cur.rowcount == 1
conn.execute("COMMIT")
return changed
except BaseException:
try:
conn.execute("ROLLBACK")
except Exception: # noqa: BLE001 - best-effort unwind
pass
raise
finally:
if close:
conn.close()
# ------------------------------------------------------------------ #
# Serve loop (production daemon entry).
# ------------------------------------------------------------------ #
def serve(self, *, poll_interval: timedelta | None = None) -> None:
"""Run the live daemon: bind invoker, setup, recover, then tick forever.
The production entry. Binds the real Claude invoker
(:func:`agent_team.invoker.bind_subscription_invoker`) BEFORE
:meth:`setup` builds the clarifier node (so the node's
``billing.claude_invoke`` calls hit the live subscription path), runs the
startup :meth:`recover` sweep, then loops calling :meth:`tick` on the
deadline cadence.
The actual Slack inbound feed is the slack_listener's job; the
coordinator exposes :meth:`submit_answer` and the shared
:attr:`resume_queue` for it. This loop owns only the deadline/recovery
maintenance cadence.
"""
from agent_team.invoker import bind_subscription_invoker
bind_subscription_invoker()
self.setup()
self.recover()
interval = (poll_interval or DEFAULT_POLL_INTERVAL).total_seconds()
import time
while True: # pragma: no cover - the infinite daemon loop
self.tick()
time.sleep(interval)
# ------------------------------------------------------------------ #
# Internals.
# ------------------------------------------------------------------ #
def _default_deadline(self) -> str:
"""Compute a fresh ISO deadline from the configured window (§3.3.1)."""
from datetime import datetime, timezone
return (datetime.now(timezone.utc) + self._deadline_window).isoformat()

View file

@ -39,6 +39,7 @@ shapes the interrupt payload that drives it.
from __future__ import annotations
import uuid
from collections.abc import Callable
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import TYPE_CHECKING, Any
@ -55,15 +56,20 @@ from agent_team.task_model import (
from agent_team.transport import QuestionSet
if TYPE_CHECKING: # pragma: no cover - typing only
from contextlib import AbstractContextManager
from langgraph.checkpoint.base import BaseCheckpointSaver
from langgraph.graph.state import CompiledStateGraph
__all__ = [
"BUILD_ROUTE",
"CLARIFY",
"DEFAULT_CLARIFY_DEADLINE",
"INTAKE",
"P1_PHASE_SEQUENCE",
"PARKED_ROUTE",
"PLAN",
"REVIEW",
"build_graph",
"build_sqlite_checkpointer",
"clarify_node",
@ -83,6 +89,15 @@ __all__ = [
INTAKE = "intake"
CLARIFY = "clarify"
PLAN = "plan"
# P2 (review loop) node ids. REVIEW is the adversarial-review vertex; BUILD_ROUTE
# and PARKED_ROUTE are the *route ids* the injected route function returns (they
# mirror agent_team.nodes.review_loop.BUILD_NODE / PARKED_NODE by value, so the
# conditional-edge map matches without graph.py importing review_loop). In P2
# both terminate the graph (no builders yet); P3 replaces BUILD_ROUTE's target
# with the real builders subgraph.
REVIEW = "review"
BUILD_ROUTE = "build"
PARKED_ROUTE = "parked"
# The P1 stage order (§7.1): intake -> clarify -> plan, then stop. Builders and
# verifiers (BUILD/VERIFY) are deliberately NOT wired here — P1 ends at an
@ -246,6 +261,11 @@ def plan_phase(state: PipelineState) -> dict[str, Any]:
def build_graph(
checkpointer: BaseCheckpointSaver | None = None,
*,
live_clarify_node: Callable[[PipelineState], PipelineState] | None = None,
live_plan_node: Callable[[PipelineState], PipelineState] | None = None,
review_node: Callable[[PipelineState], PipelineState] | None = None,
route_review: Callable[[PipelineState], str] | None = None,
) -> CompiledStateGraph:
"""Assemble + compile the P1 pipeline ``StateGraph`` (§3.3, §7.1).
@ -256,29 +276,95 @@ def build_graph(
The ``checkpointer`` is **injected**, never constructed here: the design's
durable store is the SQLite checkpointer (D9), but pre-deploy scaffolding
must not provision it, and tests inject an in-memory saver. Production wires
:func:`build_sqlite_checkpointer`. A checkpointer is required for the
the **entered** saver yielded by :func:`build_sqlite_checkpointer` (which
returns a context manager the caller must enter and hold, not a bare saver).
A checkpointer is required for the
``interrupt()``/``resume`` mechanic to work, so callers that pass ``None``
get an uncheckpointed graph that can run straight-through but cannot
suspend; the driver functions therefore require a checkpointed graph.
``live_clarify_node`` is the **injected real clarifier** (P1a): the live
coordinator passes the Claude-backed multi-turn node built from
:func:`agent_team.nodes.clarifier.make_clarifier_node`, while tests and the
pre-deploy scaffold fall back to the deterministic single-turn
:func:`clarify_node` stub. Either node honours the same ``interrupt()``
suspend/resume contract, so the durable human gate is identical; only the
question authoring differs. Defaulting to the stub keeps the graph wiring
model-free and the existing tests unchanged.
``live_plan_node`` / ``review_node`` / ``route_review`` wire **P2** (planner
+ adversarial review loop). All are injected so this module stays decoupled
from the model + review code (the coordinator passes the wrapped
:func:`agent_team.nodes.planner.plan_node`, the
:func:`agent_team.nodes.review_loop.review_node`, and its
:func:`~agent_team.nodes.review_loop.route_after_review`):
* **P1 (default):** ``review_node`` is ``None`` -> ``plan -> END``. The plan
stage is the terminus (no review, no build), exactly as before.
* **P2:** ``review_node`` is given -> ``plan -> review -> {build|plan|parked}``.
The injected ``route_review`` reads the latest verdict and returns a route
id; the conditional-edge map sends ``"plan"`` back to the planner
(loop-back), and ``"build"`` / ``"parked"`` to ``END`` (P2 stops at an
approved-or-escalated plan; P3 will repoint ``"build"`` at the real
builders subgraph). The plan<->review cycle is bounded by the planner's
revision cap and the review round cap, so the loop always terminates.
``review_node`` requires ``route_review`` (and a real ``live_plan_node`` that
advances to REVIEW); passing one without the other is a wiring error.
"""
clarify = live_clarify_node if live_clarify_node is not None else clarify_node
plan = live_plan_node if live_plan_node is not None else plan_node
if review_node is not None and route_review is None:
raise ValueError(
"build_graph: review_node requires route_review (the conditional-edge "
"function, e.g. review_loop.route_after_review)."
)
builder: StateGraph = StateGraph(PipelineState)
builder.add_node(INTAKE, intake_node)
builder.add_node(CLARIFY, clarify_node)
builder.add_node(PLAN, plan_node)
builder.add_node(CLARIFY, clarify)
builder.add_node(PLAN, plan)
builder.add_edge(START, INTAKE)
builder.add_edge(INTAKE, CLARIFY)
builder.add_edge(CLARIFY, PLAN)
builder.add_edge(PLAN, END)
if review_node is None:
# P1: the plan stage is the terminus.
builder.add_edge(PLAN, END)
else:
# P2: plan -> review -> {loop-back to plan | END}.
builder.add_node(REVIEW, review_node)
builder.add_edge(PLAN, REVIEW)
builder.add_conditional_edges(
REVIEW,
route_review,
{BUILD_ROUTE: END, PLAN: PLAN, PARKED_ROUTE: END},
)
if checkpointer is None:
return builder.compile()
return builder.compile(checkpointer=checkpointer)
def build_sqlite_checkpointer(db_path: Path | str) -> BaseCheckpointSaver:
def build_sqlite_checkpointer(
db_path: Path | str,
) -> AbstractContextManager[BaseCheckpointSaver]:
"""Construct the production SQLite checkpointer over ``db_path`` (D9, §3.3).
Returns a **context manager**, not an entered saver: in
``langgraph-checkpoint-sqlite`` ``SqliteSaver.from_conn_string`` is a
``@contextmanager`` classmethod, so the caller MUST enter it (``with`` it,
or ``__enter__`` and retain it for the graph's lifetime) before passing the
yielded saver to :func:`build_graph`. The live coordinator owns that
lifecycle (it enters the CM at setup and holds it for the daemon's life);
passing the raw return value straight into ``build_graph`` would compile a
graph whose checkpointer is an un-entered CM and break ``get_state`` /
``invoke`` at runtime. The earlier ``-> BaseCheckpointSaver`` annotation
mis-stated this contract (review FIX); the type now matches reality so a
direct caller cannot be silently misled.
The import of ``langgraph.checkpoint.sqlite`` is deferred to call time so
this module imports cleanly in environments where that optional package is
not installed (pre-deploy scaffolding). The checkpointer creates its own

View file

@ -72,6 +72,12 @@ from agent_team.db.schema import ( # noqa: E402 (path bootstrap must precede)
reopen_question,
supersede_question,
)
from agent_team.transport.base import Transport # noqa: E402 (path bootstrap)
# Transport choices the start/serve commands accept (§3.3.1 D10). Only ``slack``
# has a live adapter wired for the P1 CLI; the others are accepted for forward
# compatibility and gated in _build_transport.
_TRANSPORT_CHOICES: tuple[str, ...] = ("slack", "github", "claude_code")
__all__ = [
"build_parser",
@ -470,6 +476,121 @@ def _cmd_supersede(args: argparse.Namespace, *, out: Any) -> int:
return 0
def _build_coordinator(args: argparse.Namespace) -> Any:
"""Construct a :class:`Coordinator` for the ``start`` / ``serve`` commands.
The live transport is built LAZILY here (never at import) so ``run-team.py``
imports, ``--help``, and the ledger subcommands all work with no Slack token
present. ``--dry-run`` substitutes a non-posting transport so an operator can
drive intake without a token (the ledger row is still written; only the
transport post is a no-op).
The coordinator is imported inside this function for the same reason: pulling
in the live runtime (and its optional SDK-adjacent deps) must not happen just
to render ``--help`` or run a read-only ledger command.
"""
from agent_team.coordinator import (
Coordinator,
default_plan_node_factory,
default_review_wiring,
)
transport = _build_transport(args)
# Production runs the full P2 graph: the wrapped real planner + the bound
# GPT-4.1 review loop (Plane-2 depth-first). These factories are lazy and
# only build/bind the model seams when a task actually runs.
return Coordinator(
db_path=args.db,
transport=transport,
build_plan_node=default_plan_node_factory,
review_wiring=default_review_wiring,
)
def _build_transport(args: argparse.Namespace) -> Any:
"""Build the transport for a coordinator command (lazy; token-tolerant).
``--dry-run`` (or any transport in dry-run) yields a non-posting transport so
intake works without credentials. Otherwise the live Slack transport is
constructed lazily from ``SLACK_BOT_TOKEN`` / ``SLACK_CHANNEL``; GitHub and
Claude-Code live transports are not wired for the P1 CLI surface and raise a
clear error rather than pretending to post.
"""
if getattr(args, "dry_run", False):
return _DryRunTransport()
if args.transport == "slack":
from agent_team.transport.slack_live import build_live_slack_transport
channel = os.environ.get("SLACK_CHANNEL", "")
return build_live_slack_transport(channel)
raise SystemExit(
f"live transport '{args.transport}' is not wired for the run-team CLI; "
"use --transport slack, or --dry-run for a no-token dry run"
)
class _DryRunTransport(Transport):
"""A non-posting transport for ``--dry-run`` intake (no token, no Slack).
``post_question`` records nothing on a real channel — it returns a synthetic
``channel_ref`` so :func:`agent_team.responder.notify_question` still writes
and stamps the durable ledger row (the durable seam is exercised; only the
side-effecting post is skipped). ``parse_answer`` is unused by the CLI path
but implemented so the ABC is concrete.
"""
def post_question(
self,
*,
thread_id: str,
question_id: str,
turn: int,
question_set: Any,
deadline: str,
) -> str:
print(
f"[dry-run] would post question {question_id} (turn {turn}) "
f"for thread {thread_id}",
file=sys.stderr,
)
return f"dry-run:{question_id}"
def parse_answer(self, raw: Any) -> tuple[str, Any, str]:
raise NotImplementedError("dry-run transport does not parse answers")
def _cmd_start(args: argparse.Namespace, *, out: Any) -> int:
"""Intake: start one task and run it to the first human gate (§3.3, §3.3.1).
Builds a :class:`Coordinator` (transport from the lazy factory; ``--dry-run``
posts nowhere), runs ``setup`` + ``start_task``, and prints the minted
``thread_id``. The clarifier question-set is delivered over the chosen
transport (or no-op under ``--dry-run``); the durable ledger row is written
either way.
"""
coordinator = _build_coordinator(args)
coordinator.setup()
thread_id = coordinator.start_task(
task_text=args.task, transport_name=args.transport
)
print(thread_id, file=out)
return 0
def _cmd_serve(args: argparse.Namespace, *, out: Any) -> int:
"""Run the coordinator daemon loop (binds the live invoker; §7.1 P1).
Delegates to :meth:`agent_team.coordinator.Coordinator.serve`, which binds
the real Claude subscription invoker, runs the startup recovery sweep, then
loops on the deadline cadence. The Slack inbound feed is the slack_listener's
job; this command owns the maintenance loop. Runs until interrupted.
"""
coordinator = _build_coordinator(args)
print("agent-team coordinator starting (Ctrl-C to stop)", file=out)
coordinator.serve()
return 0 # pragma: no cover - serve() loops until interrupted
def _cmd_force_resume(args: argparse.Namespace, *, out: Any) -> int:
"""Force-resume a parked task's question (destructive; audit-logged).
@ -666,6 +787,47 @@ def build_parser() -> argparse.ArgumentParser:
)
p_resume.set_defaults(func=_cmd_force_resume)
p_start = sub.add_parser(
"start",
help="intake: start one task and run it to the first human gate",
)
p_start.add_argument(
"--task",
required=True,
help="the task description (intake text) to run through the pipeline",
)
p_start.add_argument(
"--transport",
choices=_TRANSPORT_CHOICES,
default="slack",
help="channel for delivering clarifier questions (default: slack)",
)
p_start.add_argument(
"--dry-run",
action="store_true",
dest="dry_run",
help="use a non-posting transport (no token needed; ledger still written)",
)
p_start.set_defaults(func=_cmd_start)
p_serve = sub.add_parser(
"serve",
help="run the coordinator daemon (binds invoker, runs the loop)",
)
p_serve.add_argument(
"--transport",
choices=_TRANSPORT_CHOICES,
default="slack",
help="channel for delivering clarifier questions (default: slack)",
)
p_serve.add_argument(
"--dry-run",
action="store_true",
dest="dry_run",
help="use a non-posting transport (no token needed)",
)
p_serve.set_defaults(func=_cmd_serve)
return parser

View file

@ -0,0 +1,604 @@
"""Unit tests for the COORDINATOR daemon (design §3.3, §3.3.1, §7.1 P1).
The :class:`agent_team.coordinator.Coordinator` is the live keystone that ties
the Plane-2 pipeline into one process. These tests exercise its full surface —
``setup`` / ``start_task`` / ``submit_answer`` / ``drain_resumes`` / ``tick`` /
``recover`` — with NO Claude SDK and NO Slack: the transport is a fake and the
clarify node is a stub, both injected, so the durable composition is proven
deterministically.
Every test injects:
* an in-memory LangGraph checkpointer (``build_checkpointer``) so ``setup`` needs
no SQLite checkpointer package and the suspend/resume mechanic is real;
* the deterministic single-turn :func:`agent_team.graph.clarify_node` stub as the
``build_clarify_node`` so the real Claude clarifier is never built or called.
"""
from __future__ import annotations
import queue
from datetime import timedelta
from pathlib import Path
from typing import Any
import pytest
try: # InMemorySaver is the modern name; fall back on older langgraph.
from langgraph.checkpoint.memory import InMemorySaver as _Saver
except ImportError: # pragma: no cover - environment-dependent
from langgraph.checkpoint.memory import MemorySaver as _Saver
from agent_team import graph as graph_mod
from agent_team.coordinator import Coordinator, default_clarify_node_factory
from agent_team.db.schema import connect, init_db
from agent_team.resume_worker import ResumeOutcome
from agent_team.transport.base import QuestionSet, Transport
# --------------------------------------------------------------------------- #
# Test doubles
# --------------------------------------------------------------------------- #
class FakeTransport(Transport):
"""A record-only transport: no Slack, no network (§3.3.1 injection seam).
``post_question`` appends to ``posted`` and returns a deterministic
``channel_ref`` embedding the ``question_id`` (mirroring the real contract).
``parse_answer`` reads a plain dict ``{"question_id", "answer", "via"}`` so
tests can submit answers without a Slack payload. ``fail_post`` makes the
post raise once, to exercise the lost-post path.
"""
def __init__(self, *, fail_post: bool = False) -> None:
self.posted: list[QuestionSet] = []
self.fail_post = fail_post
def post_question(
self,
*,
thread_id: str,
question_id: str,
turn: int,
question_set: QuestionSet,
deadline: str,
) -> str:
if self.fail_post:
raise RuntimeError("simulated transport post failure")
self.posted.append(question_set)
return f"fake:{question_id}"
def parse_answer(self, raw: Any) -> tuple[str, Any, str]:
return raw["question_id"], raw["answer"], raw.get("via", "fake")
def _make_coordinator(
db_path: Path,
*,
transport: Transport | None = None,
resume_queue: "queue.Queue[Any] | None" = None,
deadline_window: timedelta | None = None,
alarm_hook: Any = None,
) -> Coordinator:
"""Build a Coordinator wired with an in-memory saver + the stub clarify node."""
saver = _Saver()
return Coordinator(
db_path=db_path,
transport=transport or FakeTransport(),
build_clarify_node=lambda: graph_mod.clarify_node,
build_checkpointer=lambda _path: saver,
resume_queue=resume_queue,
deadline_window=deadline_window,
alarm_hook=alarm_hook,
)
@pytest.fixture()
def db_path(tmp_path: Path) -> Path:
path = tmp_path / "state" / "agent_team.sqlite"
init_db(path)
return path
def _row(db_path: Path, question_id: str) -> dict[str, Any] | None:
conn = connect(db_path)
try:
row = conn.execute(
"SELECT * FROM pending_questions WHERE question_id=?",
(question_id,),
).fetchone()
finally:
conn.close()
return dict(row) if row is not None else None
def _only_open_row(db_path: Path) -> dict[str, Any]:
conn = connect(db_path)
try:
rows = conn.execute(
"SELECT * FROM pending_questions WHERE status='open'"
).fetchall()
finally:
conn.close()
assert len(rows) == 1
return dict(rows[0])
def _all_rows(db_path: Path, *, status: str) -> list[dict[str, Any]]:
conn = connect(db_path)
try:
rows = conn.execute(
"SELECT * FROM pending_questions WHERE status=?",
(status,),
).fetchall()
finally:
conn.close()
return [dict(r) for r in rows]
# --------------------------------------------------------------------------- #
# setup
# --------------------------------------------------------------------------- #
def test_setup_builds_graph_and_resume_worker(db_path: Path) -> None:
coord = _make_coordinator(db_path)
assert coord.graph is None
assert coord.resume_worker is None
coord.setup()
assert coord.graph is not None
assert coord.resume_worker is not None
def test_setup_initializes_ledger_tables(tmp_path: Path) -> None:
# init_db is invoked by setup even if the DB did not pre-exist.
db = tmp_path / "fresh" / "agent_team.sqlite"
coord = _make_coordinator(db)
coord.setup()
conn = connect(db)
try:
names = {
r["name"]
for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
}
finally:
conn.close()
assert "pending_questions" in names
def test_start_task_before_setup_raises(db_path: Path) -> None:
coord = _make_coordinator(db_path)
with pytest.raises(RuntimeError):
coord.start_task(task_text="x", transport_name="slack")
# --------------------------------------------------------------------------- #
# start_task — run to the human gate + notify
# --------------------------------------------------------------------------- #
def test_start_task_suspends_and_writes_open_ledger_row(db_path: Path) -> None:
transport = FakeTransport()
coord = _make_coordinator(db_path, transport=transport)
coord.setup()
thread_id = coord.start_task(task_text="build a thing", transport_name="slack")
assert thread_id
# The clarifier suspended: a pending interrupt exists on the live checkpoint.
pending = graph_mod.pending_question(coord.graph, thread_id=thread_id)
assert pending is not None
assert pending["thread_id"] == thread_id
# notify_question wrote the ledger row OPEN and posted, recording a ref.
row = _only_open_row(db_path)
assert row["thread_id"] == thread_id
assert row["status"] == "open"
assert row["channel_ref"] == f"fake:{row['question_id']}"
assert len(transport.posted) == 1
def test_start_task_lost_post_leaves_open_row_without_ref(db_path: Path) -> None:
# A failed transport post is recoverable: the row stays open with no ref.
transport = FakeTransport(fail_post=True)
coord = _make_coordinator(db_path, transport=transport)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
row = _only_open_row(db_path)
assert row["status"] == "open"
assert row["channel_ref"] is None
# --------------------------------------------------------------------------- #
# submit_answer — first-answer-wins + enqueue resume
# --------------------------------------------------------------------------- #
def test_submit_answer_accepts_and_enqueues(db_path: Path) -> None:
q: "queue.Queue[Any]" = queue.Queue()
transport = FakeTransport()
coord = _make_coordinator(db_path, transport=transport, resume_queue=q)
coord.setup()
thread_id = coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
outcome = coord.submit_answer(
{"question_id": qid, "answer": "the answer", "via": "fake:U1"}
)
assert outcome.accepted is True
assert outcome.question_id == qid
assert q.qsize() == 1
job = q.get_nowait()
assert job.thread_id == thread_id
assert job.question_id == qid
assert _row(db_path, qid)["status"] == "answered"
def test_submit_duplicate_answer_is_rejected(db_path: Path) -> None:
coord = _make_coordinator(db_path)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
first = coord.submit_answer({"question_id": qid, "answer": "a", "via": "v"})
second = coord.submit_answer({"question_id": qid, "answer": "b", "via": "v"})
assert first.accepted is True
assert second.accepted is False # first-answer-wins; late answer ignored
# --------------------------------------------------------------------------- #
# drain_resumes — single-flight, turn-guarded via ResumeWorker
# --------------------------------------------------------------------------- #
def test_drain_resumes_advances_graph_to_done(db_path: Path) -> None:
coord = _make_coordinator(db_path)
coord.setup()
thread_id = coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
coord.submit_answer({"question_id": qid, "answer": "go", "via": "v"})
results = coord.drain_resumes()
assert len(results) == 1
assert results[0].outcome is ResumeOutcome.RESUMED
# The graph ran through PLAN to its P1 terminus (DONE).
state = graph_mod.get_pipeline_state(coord.graph, thread_id=thread_id)
assert state["status"] == "done"
def test_drain_resumes_before_setup_raises(db_path: Path) -> None:
coord = _make_coordinator(db_path)
with pytest.raises(RuntimeError):
coord.drain_resumes()
def test_drain_resumes_empty_queue_is_noop(db_path: Path) -> None:
coord = _make_coordinator(db_path)
coord.setup()
assert coord.drain_resumes() == []
# --------------------------------------------------------------------------- #
# tick — deadline sweep + park ALARM + drain
# --------------------------------------------------------------------------- #
def test_tick_parks_expired_question_and_alarms(db_path: Path) -> None:
alarmed: list[str] = []
coord = _make_coordinator(db_path, alarm_hook=alarmed.append)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
# Force the deadline into the past so the sweep marks it overdue.
conn = connect(db_path)
try:
conn.execute(
"UPDATE pending_questions SET deadline_at=? WHERE question_id=?",
("2000-01-01T00:00:00+00:00", qid),
)
finally:
conn.close()
coord.tick()
# Deadline policy flipped it to expired (parked state) and raised the ALARM.
assert _row(db_path, qid)["status"] == "expired"
assert alarmed == [qid]
def test_tick_drains_pending_resume(db_path: Path) -> None:
coord = _make_coordinator(db_path)
coord.setup()
thread_id = coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
coord.submit_answer({"question_id": qid, "answer": "go", "via": "v"})
results = coord.tick()
assert len(results) == 1
assert results[0].outcome is ResumeOutcome.RESUMED
state = graph_mod.get_pipeline_state(coord.graph, thread_id=thread_id)
assert state["status"] == "done"
# --------------------------------------------------------------------------- #
# recover — startup convergence (§3.3.1)
# --------------------------------------------------------------------------- #
def test_recover_redrives_answered_row(db_path: Path) -> None:
# An answer that landed before a "crash" (no drain) is re-driven on recover.
coord = _make_coordinator(db_path)
coord.setup()
thread_id = coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
coord.submit_answer({"question_id": qid, "answer": "go", "via": "v"})
# Simulate a crash: the resume job is lost from the queue without draining.
coord.resume_queue.get_nowait()
results = coord.recover()
assert any(r.outcome is ResumeOutcome.RESUMED for r in results)
state = graph_mod.get_pipeline_state(coord.graph, thread_id=thread_id)
assert state["status"] == "done"
def test_recover_reposts_open_row_lacking_channel_ref(db_path: Path) -> None:
# First delivery's post is lost (row open, no ref); recover re-posts it.
transport = FakeTransport(fail_post=True)
coord = _make_coordinator(db_path, transport=transport)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
row = _only_open_row(db_path)
assert row["channel_ref"] is None
# The transport now succeeds; recovery should re-post and record a ref.
transport.fail_post = False
coord.recover()
reposted = _only_open_row(db_path)
assert reposted["channel_ref"] is not None
assert len(transport.posted) == 1
def test_redeliver_race_guard_preserves_concurrently_answered_row(
db_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""RACE-REDELIVER-NONATOMIC-DELETE (CWE-362): an answer landing in the window
BETWEEN the redeliver SELECT and the clear-write must survive.
This is the exact lost-update window the finding describes. The row is
open-and-null-ref at SELECT time (so it is selected for re-delivery), but an
answer flips it open → answered on a *separate* connection before the clear
runs. The injection point is :func:`graph_mod.pending_question`, which the
redeliver loop calls per-row AFTER the SELECT and BEFORE the clear-write —
monkeypatching it to land the answer reproduces the concurrent CAS precisely
in that window.
With the old UNCONDITIONAL ``DELETE ... WHERE question_id=?`` the now-answered
row would be destroyed (answer lost, ResumeJob dangling, human re-asked). The
guarded conditional DELETE re-checks ``status='open' AND channel_ref IS NULL``
at write time, matches 0 rows, and SKIPS — so the answered row survives with
its answer intact and no fresh open question is re-posted.
"""
from agent_team.db import schema as schema_mod
from agent_team import coordinator as coordinator_mod
# Lost post: open row with no channel_ref (selected by the redeliver SELECT).
transport = FakeTransport(fail_post=True)
coord = _make_coordinator(db_path, transport=transport)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
row = _only_open_row(db_path)
qid = row["question_id"]
assert row["channel_ref"] is None
# The transport now succeeds — so any re-post would be observable as a fresh
# open row + a posted question_set (the clobber symptom we must NOT see).
transport.fail_post = False
real_pending = graph_mod.pending_question
landed: dict[str, bool] = {"done": False}
def _pending_then_answer(graph: Any, *, thread_id: str) -> Any:
"""Return the live pending payload, then land the answer in the window.
The redeliver loop calls this between the SELECT and the guarded clear,
so landing the open->answered CAS here (on its own connection) is exactly
the concurrent answer the finding races against.
"""
result = real_pending(graph, thread_id=thread_id)
if not landed["done"]:
landed["done"] = True
conn = connect(db_path)
try:
assert schema_mod.answer_question(
conn,
question_id=qid,
answer_json='{"answer": "human said go"}',
answered_via="fake:U1",
)
finally:
conn.close()
return result
monkeypatch.setattr(
coordinator_mod.graph_mod, "pending_question", _pending_then_answer
)
# Drive the redeliver sweep (recover's half-2) with the race injected.
coord._redeliver_unposted_open_questions()
# The concurrently-answered row was NOT deleted; its answer_json is intact.
after = _row(db_path, qid)
assert after is not None, (
"guarded clear must not delete the concurrently-answered row"
)
assert after["status"] == "answered"
assert after["answer_json"] == '{"answer": "human said go"}'
# NO fresh open question was created and NO re-post happened (skip, not re-ask).
assert _all_rows(db_path, status="open") == []
assert transport.posted == []
def test_redeliver_happy_path_reposts_genuinely_unposted_open_row(
db_path: Path,
) -> None:
"""The guard preserves the normal case: a genuinely lost post (row still
open, null ref) IS re-delivered — re-posted and a fresh channel_ref recorded.
"""
transport = FakeTransport(fail_post=True)
coord = _make_coordinator(db_path, transport=transport)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
assert _only_open_row(db_path)["channel_ref"] is None
# Post now succeeds; the genuinely unposted-open row is re-delivered.
transport.fail_post = False
coord._redeliver_unposted_open_questions()
reposted = _only_open_row(db_path)
assert reposted["channel_ref"] is not None
assert len(transport.posted) == 1
def test_guarded_clear_uses_status_and_ref_predicate_rowcount_skip(
db_path: Path,
) -> None:
"""The guarded write is conditioned on ``status='open' AND channel_ref IS
NULL``: it returns False (rowcount 0 → skip) for a row that is not an
unposted-open question, and True only for a row that still is.
"""
transport = FakeTransport(fail_post=True)
coord = _make_coordinator(db_path, transport=transport)
coord.setup()
coord.start_task(task_text="x", transport_name="slack")
qid = _only_open_row(db_path)["question_id"]
# A row that is open BUT already has a channel_ref must not be cleared.
conn = connect(db_path)
try:
conn.execute(
"UPDATE pending_questions SET channel_ref='fake:ref' WHERE question_id=?",
(qid,),
)
finally:
conn.close()
assert coord._guarded_clear_unposted_open(qid) is False
assert _row(db_path, qid) is not None # untouched
# Reset to the genuine unposted-open state: now the predicate matches.
conn = connect(db_path)
try:
conn.execute(
"UPDATE pending_questions SET channel_ref=NULL WHERE question_id=?",
(qid,),
)
finally:
conn.close()
assert coord._guarded_clear_unposted_open(qid) is True
assert _row(db_path, qid) is None # deleted, ready for re-insert
def test_recover_before_setup_raises(db_path: Path) -> None:
coord = _make_coordinator(db_path)
with pytest.raises(RuntimeError):
coord.recover()
# --------------------------------------------------------------------------- #
# default_clarify_node_factory — lazy leaves compose into a callable node
# --------------------------------------------------------------------------- #
def test_default_clarify_node_factory_returns_callable_node() -> None:
"""The factory's lazy imports resolve and the live clarifier leaves compose.
This is the real (Claude-backed) node the coordinator hands to ``build_graph``
when no stub is injected. We only assert it constructs into a callable — the
billing seam is unbound and the node is NOT invoked here, so no Claude call
is made; this proves the deferred imports + composition wire up cleanly.
"""
node = default_clarify_node_factory()
assert callable(node)
# --------------------------------------------------------------------------- #
# P2 wiring — wrapped planner (fail-safe) + bound review loop
# --------------------------------------------------------------------------- #
def test_default_plan_node_factory_passes_through_real_plan(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A clean planner reply is returned verbatim by the wrapper."""
from agent_team.coordinator import default_plan_node_factory
from agent_team.nodes import planner
delta = {"plan": {"phases": ["x"]}, "current_phase": "review", "status": "active"}
monkeypatch.setattr(planner, "plan_node", lambda state, config=None: delta)
node = default_plan_node_factory()
assert node({"thread_id": "t-1"}) == delta
def test_default_plan_node_factory_parks_on_planner_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A PlannerError is caught and turned into a PARKED delta, never raised."""
from agent_team.coordinator import default_plan_node_factory
from agent_team.nodes import planner
def _boom(state: Any, config: Any = None) -> Any:
raise planner.PlannerError("unparseable plan")
monkeypatch.setattr(planner, "plan_node", _boom)
node = default_plan_node_factory()
out = node({"thread_id": "t-1"})
assert out["status"] == "parked"
assert out["current_phase"] == "parked"
def test_default_review_wiring_binds_and_returns_node_and_router() -> None:
"""The wiring binds the GPT-4.1 invoker and returns a single-arg node + router."""
from agent_team.coordinator import default_review_wiring
from agent_team.nodes import review_loop
saved = review_loop._review_invoker
try:
node, router = default_review_wiring()
assert callable(node) and callable(router)
# The review node is single-arg (no LangGraph config injection).
import inspect
assert len(inspect.signature(node).parameters) == 1
assert review_loop._review_invoker is not saved # invoker was bound
finally:
review_loop._review_invoker = saved
def test_setup_with_p2_factories_builds_a_review_node(db_path: Path) -> None:
"""Injecting the P2 factories compiles a graph that includes the review vertex."""
from agent_team.nodes import review_loop
saved = review_loop._review_invoker
saver = _Saver()
try:
coord = Coordinator(
db_path=db_path,
transport=FakeTransport(),
build_clarify_node=lambda: graph_mod.clarify_node,
build_plan_node=lambda: lambda state: {"current_phase": "review"},
review_wiring=lambda: (
review_loop.bind_review_node(),
review_loop.route_after_review,
),
build_checkpointer=lambda _path: saver,
)
coord.setup()
assert graph_mod.REVIEW in coord.graph.get_graph().nodes
finally:
review_loop._review_invoker = saved

View file

@ -281,9 +281,89 @@ def test_build_sqlite_checkpointer_builds_when_dep_present(tmp_path) -> None:
# If the optional package IS installed, the factory returns a checkpointer
# over the DB path. Skip cleanly where it's absent (pre-deploy scaffolding).
pytest.importorskip("langgraph.checkpoint.sqlite")
saver = build_sqlite_checkpointer(tmp_path / "nested" / "state.db")
assert saver is not None
cm = build_sqlite_checkpointer(tmp_path / "nested" / "state.db")
assert cm is not None
assert (tmp_path / "nested").is_dir()
# Contract: the factory returns a CONTEXT MANAGER (SqliteSaver.from_conn_string
# is a @contextmanager), so callers must enter it. Pin that here so a future
# change that returns a bare/un-entered object is caught (review FIX).
assert hasattr(cm, "__enter__") and hasattr(cm, "__exit__")
with cm as saver:
# The entered object is the real saver the graph compiles against.
assert hasattr(saver, "get_next_version")
# --- P2 review-loop wiring. -------------------------------------------------
def _p2_plan_stub(state: PipelineState) -> PipelineState:
"""Stand-in for the real planner: emit a plan and advance to REVIEW.
Mirrors planner.plan_node's contract (sets ``plan`` + phase REVIEW) without a
model call, so the P2 graph topology + the review loop can be driven in a
unit test. The revision index tracks prior review rounds.
"""
revisions = len(state.get("review_verdicts") or [])
return PipelineState(
plan={"phases": ["P1"], "revision": revisions},
current_phase=Phase.REVIEW.value,
status=TaskStatus.ACTIVE.value,
)
def _p2_graph(review_text: str):
"""Compile a P2 graph whose review invoker returns ``review_text``."""
from agent_team.nodes import review_loop
review_loop.set_review_invoker(lambda prompt, **kw: review_text)
return build_graph(
checkpointer=_Saver(),
live_plan_node=_p2_plan_stub,
review_node=review_loop.bind_review_node(),
route_review=review_loop.route_after_review,
)
@pytest.fixture
def restore_review_invoker():
"""Save/restore the review-loop module-global invoker around a test."""
from agent_team.nodes import review_loop
saved = review_loop._review_invoker
yield
review_loop._review_invoker = saved
def test_build_graph_review_node_requires_route() -> None:
from agent_team.nodes import review_loop
with pytest.raises(ValueError, match="route_review"):
build_graph(review_node=review_loop.review_node)
def test_p2_graph_approve_terminates(restore_review_invoker) -> None:
# clarify(stub) -> plan(stub->REVIEW) -> review(APPROVE) -> END.
graph = _p2_graph("VERDICT: APPROVE\nlooks solid")
thread_id, _ = start_task(graph, transport="slack")
final = resume_task(graph, thread_id=thread_id, answer="scope is X")
# The review node advanced an APPROVED plan toward BUILD and the graph ended.
assert final["current_phase"] == Phase.BUILD.value
assert len(final["review_verdicts"]) == 1
def test_p2_graph_loops_then_escalates_on_persistent_changes(
restore_review_invoker,
) -> None:
# A reviewer that never approves loops plan<->review until the round cap,
# then escalates (parks) rather than spinning. Default cap is 3 rounds.
graph = _p2_graph("VERDICT: REQUEST CHANGES\nstill not ready")
thread_id, _ = start_task(graph, transport="slack")
final = resume_task(graph, thread_id=thread_id, answer="scope is X")
assert final["current_phase"] == Phase.PARKED.value
assert final["status"] == TaskStatus.PARKED.value
assert len(final["review_verdicts"]) == 3 # looped to the cap, then escalated
# --- Module import hygiene. -------------------------------------------------

View file

@ -13,11 +13,13 @@ compare-and-set, and read-only commands needing no confirmation.
from __future__ import annotations
import argparse
import importlib.util
import io
import json
from pathlib import Path
from types import ModuleType
from typing import Any
import pytest
@ -625,3 +627,109 @@ def test_unwritable_audit_path_aborts_before_mutation(
code, _ = _run(cli, db_path, bad_audit, "expire", "q1", "--confirm")
assert code == 1 # clean failure, not an uncaught traceback
assert _status_of(db_path, "q1") == "open" # NOT mutated — no trail, no change
# --------------------------------------------------------------------------- #
# start / serve coordinator commands + transport factory (lazy, token-tolerant)
# --------------------------------------------------------------------------- #
class _FakeCoordinator:
"""Records setup()/start_task() so the ``start`` CLI boundary is testable."""
instances: list[_FakeCoordinator] = []
def __init__(
self,
*,
db_path: Any,
transport: Any,
build_plan_node: Any = None,
review_wiring: Any = None,
) -> None:
self.db_path = db_path
self.transport = transport
# The production CLI opts the coordinator into the P2 graph by injecting
# these factories; record them so the wiring is asserted, not ignored.
self.build_plan_node = build_plan_node
self.review_wiring = review_wiring
self.setup_called = False
self.start_kwargs: dict[str, Any] | None = None
_FakeCoordinator.instances.append(self)
def setup(self) -> None:
self.setup_called = True
def start_task(self, *, task_text: str, transport_name: str) -> str:
self.start_kwargs = {"task_text": task_text, "transport_name": transport_name}
return "thread-minted-42"
def test_start_runs_setup_and_start_task_and_prints_thread_id(
cli: ModuleType,
db_path: Path,
audit_log: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""``start --dry-run`` builds a coordinator, runs setup + start_task, prints id.
``_build_coordinator`` imports ``Coordinator`` lazily from
``agent_team.coordinator``, so patching it there intercepts construction.
``--dry-run`` means no Slack token is required (the real
``_build_transport`` returns a ``_DryRunTransport``).
"""
_FakeCoordinator.instances.clear()
monkeypatch.setattr(
"agent_team.coordinator.Coordinator", _FakeCoordinator, raising=True
)
code, out = _run(
cli, db_path, audit_log, "start", "--dry-run", "--task", "do the thing"
)
assert code == 0
assert len(_FakeCoordinator.instances) == 1
coord = _FakeCoordinator.instances[0]
assert coord.setup_called is True
assert coord.start_kwargs == {
"task_text": "do the thing",
"transport_name": "slack",
}
# The minted thread_id is printed to the captured stdout.
assert out.strip() == "thread-minted-42"
# --dry-run substitutes the non-posting transport (no token needed).
assert isinstance(coord.transport, cli._DryRunTransport)
# The production CLI opts into the full P2 graph: planner + review factories
# are injected (callables), not left at the P1-stub default of None.
assert callable(coord.build_plan_node)
assert callable(coord.review_wiring)
def test_build_transport_live_github_raises_system_exit(cli: ModuleType) -> None:
"""A non-slack live transport is not wired and raises a clear SystemExit."""
args = argparse.Namespace(dry_run=False, transport="github")
with pytest.raises(SystemExit, match="is not wired for the run-team CLI"):
cli._build_transport(args)
def test_build_transport_live_claude_code_raises_system_exit(cli: ModuleType) -> None:
"""claude_code is likewise un-wired for the P1 CLI surface."""
args = argparse.Namespace(dry_run=False, transport="claude_code")
with pytest.raises(SystemExit, match="is not wired for the run-team CLI"):
cli._build_transport(args)
def test_build_transport_dry_run_returns_dry_run_transport(cli: ModuleType) -> None:
"""``dry_run=True`` yields a _DryRunTransport whose post returns a synthetic ref."""
args = argparse.Namespace(dry_run=True, transport="slack")
transport = cli._build_transport(args)
assert isinstance(transport, cli._DryRunTransport)
ref = transport.post_question(
thread_id="t1",
question_id="q1",
turn=0,
question_set=None,
deadline="2026-06-18T00:00:00+00:00",
)
assert ref == "dry-run:q1"