Adam Moussa
f0c5cfe57f
0c-binding: per-task expected_run_id bound from state (gate rejects substituted run_id; None -> BLOCK, never vacuous pass). 0e: fail-safe serve default (failsafe_production_p3_wiring) — inert on unprovisioned env (one WARNING + one #agent-team notice), never crash-loops. 0a: reorder P3 subgraph BUILD -> DISPATCH -> VERIFY (preserves _instrument). 0d: ci_watcher engine + VERIFY interrupt()-wait (async resume-on-CI-complete). KNOWN-OPEN (adversarial review BLOCKs, to remediate next): - CI-watcher not wired into run-team serve (ci_pending_provider/ci_poller None) -> a VERIFY-suspended task never resumes/parks. - no durable ci_pending_provider enumerating threads suspended at VERIFY. Branch only; not merged, not deployed.
281 lines
11 KiB
Python
281 lines
11 KiB
Python
"""Unit tests for agent_team.nodes.verifier — the VERIFY node (§3.3, §3.3.2)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from agent_team.ci_gate import GateDecision, GateResult
|
|
from agent_team.nodes import verifier as verifier_mod
|
|
from agent_team.nodes.verifier import (
|
|
DEFAULT_MAX_BUILD_LOOPS,
|
|
VerifierConfig,
|
|
set_fix_advisor,
|
|
verifier_node,
|
|
)
|
|
from agent_team.state_store import compute_content_hash
|
|
from agent_team.task_model import Phase, PipelineState, TaskStatus
|
|
|
|
|
|
def _diff_for(*paths: str) -> str:
|
|
chunks = []
|
|
for p in paths:
|
|
chunks.append(f"diff --git a/{p} b/{p}\n@@ -1 +1 @@\n-old\n+new\n")
|
|
return "".join(chunks)
|
|
|
|
|
|
def _hash(diff: str) -> str:
|
|
return compute_content_hash(diff.encode("utf-8"))
|
|
|
|
|
|
def _state(diff: str, ci: dict | None) -> PipelineState:
|
|
return {
|
|
"thread_id": "t1",
|
|
"status": TaskStatus.ACTIVE.value,
|
|
"current_phase": Phase.VERIFY.value,
|
|
"candidate_diff": diff,
|
|
"diff_hash": _hash(diff),
|
|
"ci_results": ci,
|
|
}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_advisor():
|
|
"""Restore the default null advisor after each test."""
|
|
yield
|
|
set_fix_advisor(verifier_mod._null_advisor)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# PASS path
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_pass_advances_to_done() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
out = verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r1"))
|
|
assert out["status"] == TaskStatus.DONE.value
|
|
assert out["current_phase"] == Phase.DONE.value
|
|
assert out["ci_results"]["gate_decision"] == "pass"
|
|
|
|
|
|
def test_pass_does_not_consult_advisor() -> None:
|
|
calls: list = []
|
|
|
|
def advisor(result, state): # pragma: no cover - asserted not called
|
|
calls.append(result)
|
|
return "hint"
|
|
|
|
set_fix_advisor(advisor)
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r1"))
|
|
assert calls == [] # the LLM is never asked whether it passed
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# FAIL path — loop back to BUILD
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_fail_loops_back_to_build() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "failure", "diff_hash": _hash(diff)}
|
|
out = verifier_node(
|
|
_state(diff, ci), VerifierConfig(expected_run_id="r1", build_loops=0)
|
|
)
|
|
assert out["status"] == TaskStatus.ACTIVE.value
|
|
assert out["current_phase"] == Phase.BUILD.value
|
|
|
|
|
|
def test_fail_consults_advisor_for_hint() -> None:
|
|
def advisor(result: GateResult, state) -> str:
|
|
assert result.decision is GateDecision.FAIL
|
|
return "bump the pinned version"
|
|
|
|
set_fix_advisor(advisor)
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "failure", "diff_hash": _hash(diff)}
|
|
out = verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r1"))
|
|
assert out["review_verdicts"][0]["fix_hint"] == "bump the pinned version"
|
|
|
|
|
|
def test_fail_parks_when_build_loops_exhausted() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "failure", "diff_hash": _hash(diff)}
|
|
cfg = VerifierConfig(
|
|
expected_run_id="r1",
|
|
build_loops=DEFAULT_MAX_BUILD_LOOPS - 1,
|
|
max_build_loops=DEFAULT_MAX_BUILD_LOOPS,
|
|
)
|
|
out = verifier_node(_state(diff, ci), cfg)
|
|
assert out["status"] == TaskStatus.PARKED.value
|
|
assert out["current_phase"] == Phase.PARKED.value
|
|
assert any("max build loops" in r for r in out["review_verdicts"][0]["reasons"])
|
|
|
|
|
|
def test_fail_increments_build_loops_in_verdict() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "failure", "diff_hash": _hash(diff)}
|
|
out = verifier_node(
|
|
_state(diff, ci), VerifierConfig(expected_run_id="r1", build_loops=1)
|
|
)
|
|
assert out["review_verdicts"][0]["build_loops"] == 2
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# BLOCK path — park for human + GPT cross-review
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_block_on_denylist_parks() -> None:
|
|
diff = _diff_for(".github/workflows/ci.yml")
|
|
ci = {"run_id": "r1", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
out = verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r1"))
|
|
assert out["status"] == TaskStatus.PARKED.value
|
|
assert out["current_phase"] == Phase.PARKED.value
|
|
assert out["ci_results"]["gate_decision"] == "block"
|
|
|
|
|
|
def test_block_on_hash_mismatch_parks() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
state = _state(diff, {"run_id": "r1", "conclusion": "success"})
|
|
state["diff_hash"] = "tampered"
|
|
out = verifier_node(state, VerifierConfig(expected_run_id="r1"))
|
|
assert out["status"] == TaskStatus.PARKED.value
|
|
|
|
|
|
def test_block_never_advances_to_done_even_with_advisor() -> None:
|
|
set_fix_advisor(lambda result, state: "irrelevant")
|
|
diff = _diff_for(".github/workflows/ci.yml")
|
|
ci = {"run_id": "r1", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
out = verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r1"))
|
|
assert out["status"] != TaskStatus.DONE.value
|
|
|
|
|
|
def test_missing_diff_parks() -> None:
|
|
state: PipelineState = {
|
|
"thread_id": "t1",
|
|
"candidate_diff": None,
|
|
"diff_hash": None,
|
|
"ci_results": None,
|
|
}
|
|
out = verifier_node(state, VerifierConfig(expected_run_id="r1"))
|
|
assert out["status"] == TaskStatus.PARKED.value
|
|
assert any("no candidate_diff" in r for r in out["review_verdicts"][0]["reasons"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Provenance / partial-update shape
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_verdict_records_provenance() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r9", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
out = verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r9"))
|
|
verdict = out["review_verdicts"][0]
|
|
assert verdict["stage"] == "verify"
|
|
assert verdict["run_id"] == "r9"
|
|
assert verdict["diff_hash"] == _hash(diff)
|
|
assert verdict["ci_conclusion"] == "success"
|
|
assert "at" in verdict
|
|
|
|
|
|
def test_returns_partial_update_only() -> None:
|
|
diff = _diff_for("src/foo.py")
|
|
ci = {"run_id": "r1", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
out = verifier_node(_state(diff, ci), VerifierConfig(expected_run_id="r1"))
|
|
# Node returns only the keys it writes (LangGraph reducer merges the rest).
|
|
assert set(out) == {
|
|
"status",
|
|
"current_phase",
|
|
"review_verdicts",
|
|
"ci_results",
|
|
"updated_at",
|
|
}
|
|
|
|
|
|
def test_allowed_scope_threaded_to_gate() -> None:
|
|
diff = _diff_for("src/foo.py", "elsewhere/bar.py")
|
|
ci = {"run_id": "r1", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
cfg = VerifierConfig(expected_run_id="r1", allowed_scope=["src/"])
|
|
out = verifier_node(_state(diff, ci), cfg)
|
|
assert out["status"] == TaskStatus.PARKED.value # out-of-scope -> BLOCK -> park
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Per-task run-id binding (design §4 Decision 4)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_state_run_id_overrides_static_config_run_id() -> None:
|
|
# The gate must bind to the run id THIS task dispatched (state["run_id"]),
|
|
# not the static config constant. A CI conclusion keyed to the per-task
|
|
# run id passes even though config carries a different (stale) run id.
|
|
diff = _diff_for("src/foo.py")
|
|
state = _state(
|
|
diff, {"run_id": "task-run", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
)
|
|
state["run_id"] = "task-run"
|
|
# config.expected_run_id is a DIFFERENT, stale value — state must win.
|
|
out = verifier_node(state, VerifierConfig(expected_run_id="stale-wiring-run"))
|
|
assert out["status"] == TaskStatus.DONE.value
|
|
assert out["review_verdicts"][0]["run_id"] == "task-run"
|
|
|
|
|
|
def test_substituted_run_id_is_rejected() -> None:
|
|
# Anti-substitution: a CI result whose run_id != state["run_id"] is a BLOCK,
|
|
# even with a success conclusion (someone tried to graft a passing run from
|
|
# another task onto this one).
|
|
diff = _diff_for("src/foo.py")
|
|
state = _state(
|
|
diff,
|
|
{"run_id": "other-task-run", "conclusion": "success", "diff_hash": _hash(diff)},
|
|
)
|
|
state["run_id"] = "my-task-run"
|
|
out = verifier_node(state, VerifierConfig(expected_run_id=None))
|
|
assert out["status"] == TaskStatus.PARKED.value
|
|
assert out["ci_results"]["gate_decision"] == "block"
|
|
assert any("run-id mismatch" in r for r in out["review_verdicts"][0]["reasons"])
|
|
|
|
|
|
def test_two_concurrent_tasks_each_gate_against_own_run_id() -> None:
|
|
# Two tasks share ONE VerifierConfig but each gates against its OWN
|
|
# state["run_id"]. Task A's CI matches A's run id (PASS); task B's CI is
|
|
# keyed to A's run id (substitution) so B BLOCKs.
|
|
shared_cfg = VerifierConfig(expected_run_id=None)
|
|
|
|
diff_a = _diff_for("src/a.py")
|
|
state_a = _state(
|
|
diff_a, {"run_id": "run-A", "conclusion": "success", "diff_hash": _hash(diff_a)}
|
|
)
|
|
state_a["run_id"] = "run-A"
|
|
|
|
diff_b = _diff_for("src/b.py")
|
|
# B's fetched CI is wrongly keyed to run-A (a leaked/substituted run).
|
|
state_b = _state(
|
|
diff_b, {"run_id": "run-A", "conclusion": "success", "diff_hash": _hash(diff_b)}
|
|
)
|
|
state_b["run_id"] = "run-B"
|
|
|
|
out_a = verifier_node(state_a, shared_cfg)
|
|
out_b = verifier_node(state_b, shared_cfg)
|
|
|
|
assert out_a["status"] == TaskStatus.DONE.value
|
|
assert out_a["review_verdicts"][0]["run_id"] == "run-A"
|
|
assert out_b["status"] == TaskStatus.PARKED.value
|
|
assert out_b["review_verdicts"][0]["run_id"] == "run-B"
|
|
|
|
|
|
def test_none_run_id_at_gate_time_blocks_never_passes() -> None:
|
|
# No per-task run_id in state AND no config fallback -> BLOCK (park), never a
|
|
# vacuous pass, even when CI reports success.
|
|
diff = _diff_for("src/foo.py")
|
|
state = _state(
|
|
diff, {"run_id": "", "conclusion": "success", "diff_hash": _hash(diff)}
|
|
)
|
|
# state has no "run_id" key; config fallback is None.
|
|
out = verifier_node(state, VerifierConfig(expected_run_id=None))
|
|
assert out["status"] == TaskStatus.PARKED.value
|
|
assert out["ci_results"]["gate_decision"] == "block"
|