"""Unit tests for agent_team.nodes.clarifier_llm (§3.3, §7.1 P1). The Claude-backed clarifier callables are exercised with a FAKE invoke that returns canned :class:`~agent_team.billing.ClaudeResult` text — no network. The load-bearing properties under test: * **One call per turn (memoization).** ``assess_confidence`` then ``generate_questions`` on the same turn must reuse a single Claude call. * **Fail SAFE (the human gate).** A garbled / non-JSON reply must yield confidence ``0.0`` (never >= the 0.98 bar) and a non-empty fallback question. * **Defensive parsing.** ```json fences and surrounding prose still parse, and out-of-range confidence is clamped to ``[0, 1]``. * **Integration smoke.** The callables wire into the real :func:`~agent_team.nodes.clarifier.make_clarifier_node` and clear the gate once confidence rises across turns. """ from __future__ import annotations import json from typing import Any from langgraph.checkpoint.memory import MemorySaver from langgraph.graph import END, START, StateGraph from langgraph.types import Command from agent_team.billing import BillingMode, ClaudeResult from agent_team.nodes.clarifier import ( DEFAULT_CONFIDENCE_THRESHOLD, make_clarifier_node, ) from agent_team.nodes.clarifier_llm import ( FALLBACK_QUESTION, ClaudeClarifier, build_claude_clarifier_callables, ) from agent_team.task_model import Phase, PipelineState, TaskStatus # --------------------------------------------------------------------------- # # Fakes / helpers # --------------------------------------------------------------------------- # class _FakeInvoke: """A fake billing.claude_invoke that returns canned text and counts calls. ``replies`` may be a single string (returned every call) or a list of strings (consumed one per call, last one repeating) so a test can simulate rising confidence across turns. """ def __init__(self, replies: str | list[str]) -> None: self._replies = [replies] if isinstance(replies, str) else list(replies) self.calls: list[dict[str, Any]] = [] def __call__(self, prompt: str, **kw: Any) -> ClaudeResult: idx = min(len(self.calls), len(self._replies) - 1) text = self._replies[idx] self.calls.append({"prompt": prompt, "kw": kw}) return ClaudeResult(text=text, mode=BillingMode.SUBSCRIPTION) def _state(**overrides: Any) -> PipelineState: base: PipelineState = PipelineState( thread_id="t-1", status=TaskStatus.ACTIVE.value, current_phase=Phase.CLARIFY.value, qa_history=[], transport="slack", ) base.update(overrides) # type: ignore[typeddict-item] return base def _json(confidence: Any, questions: Any, rationale: str = "because") -> str: return json.dumps( {"confidence": confidence, "questions": questions, "rationale": rationale} ) # --------------------------------------------------------------------------- # # High confidence: assess returns ~value AND the call is reused (memoization). # --------------------------------------------------------------------------- # def test_high_confidence_parsed() -> None: fake = _FakeInvoke(_json(0.99, [])) clar = ClaudeClarifier(invoke=fake) assert clar.assess_confidence([], _state()) == 0.99 def test_clarifier_passes_max_turns_headroom_to_invoke_seam() -> None: # The single-shot Claude default (1 turn) is flaky: it crashes the clarify # node with "Reached maximum number of turns (1)" and leaves the task wedged # with no question posted. The clarifier asks for headroom so the model can # FINISH its JSON (mirrors the planner fix, PR #58). fake = _FakeInvoke(_json(0.99, [])) clar = ClaudeClarifier(invoke=fake) clar.assess_confidence([], _state()) assert fake.calls[0]["kw"].get("max_turns") == 4 assert fake.calls[0]["kw"]["max_turns"] > 1 def test_single_call_per_turn_memoized() -> None: fake = _FakeInvoke(_json(0.99, [])) clar = ClaudeClarifier(invoke=fake) qa: list[object] = [] # Both methods called for the same turn -> exactly ONE Claude call. conf = clar.assess_confidence(qa, _state()) questions = clar.generate_questions(qa, _state()) assert conf == 0.99 assert questions == [] # confident, no questions needed assert len(fake.calls) == 1 def test_memo_recomputes_when_history_grows() -> None: fake = _FakeInvoke([_json(0.10, ["q1"]), _json(0.99, [])]) clar = ClaudeClarifier(invoke=fake) # Turn 0: one answer-less call. assert clar.assess_confidence([], _state()) == 0.10 assert clar.generate_questions([], _state()) == ["q1"] assert len(fake.calls) == 1 # Turn 1: history grew -> a fresh call, now confident. assert clar.assess_confidence(["a1"], _state()) == 0.99 assert clar.generate_questions(["a1"], _state()) == [] assert len(fake.calls) == 2 def test_memo_isolates_across_tasks_no_cross_gate_clear() -> None: """A second task at the same history length must NOT reuse task A's memo. Regression for the fail-OPEN collision: one ClaudeClarifier instance serves every task through the shared graph node, so keying the memo on history length alone would let task A's cached 0.99 clear task B's human gate with no model call. Keying on thread_id forces a fresh assessment for task B. """ fake = _FakeInvoke([_json(0.99, []), _json(0.10, ["need more from B"])]) clar = ClaudeClarifier(invoke=fake) # Task A (thread t-A), empty history -> confident, cached. assert clar.assess_confidence([], _state(thread_id="t-A")) == 0.99 assert len(fake.calls) == 1 # Task B (thread t-B), SAME empty history/length -> must re-assess, NOT # inherit A's cache, so its low confidence holds and the gate stays shut. assert clar.assess_confidence([], _state(thread_id="t-B")) == 0.10 assert clar.generate_questions([], _state(thread_id="t-B")) == ["need more from B"] assert len(fake.calls) == 2 # a real second call happened for task B # --------------------------------------------------------------------------- # # Low confidence: below the bar, questions are returned. # --------------------------------------------------------------------------- # def test_low_confidence_returns_questions() -> None: fake = _FakeInvoke(_json(0.40, ["What is the scope?", "Which repo?"])) clar = ClaudeClarifier(invoke=fake) assert clar.assess_confidence([], _state()) < DEFAULT_CONFIDENCE_THRESHOLD assert clar.generate_questions([], _state()) == [ "What is the scope?", "Which repo?", ] assert len(fake.calls) == 1 # --------------------------------------------------------------------------- # # Malformed output: fail SAFE (0.0 confidence, non-empty fallback questions). # --------------------------------------------------------------------------- # def test_malformed_output_fails_safe() -> None: fake = _FakeInvoke("I'm sorry, I cannot help with that. ") clar = ClaudeClarifier(invoke=fake) assert clar.assess_confidence([], _state()) == 0.0 questions = clar.generate_questions([], _state()) assert questions == [FALLBACK_QUESTION] assert questions # non-empty def test_garbage_never_clears_the_gate() -> None: # The critical safety property: garbage must never read >= 0.98. for garbage in ["", " ", "not json", "{broken", "[1,2,3]", "null", "42"]: clar = ClaudeClarifier(invoke=_FakeInvoke(garbage)) conf = clar.assess_confidence([], _state()) assert conf < DEFAULT_CONFIDENCE_THRESHOLD assert conf == 0.0 assert clar.generate_questions([], _state()) == [FALLBACK_QUESTION] def test_missing_confidence_key_defaults_zero() -> None: fake = _FakeInvoke(json.dumps({"questions": ["q?"], "rationale": "x"})) clar = ClaudeClarifier(invoke=fake) assert clar.assess_confidence([], _state()) == 0.0 # Questions present in the reply are kept as-is. assert clar.generate_questions([], _state()) == ["q?"] def test_low_confidence_empty_questions_gets_fallback() -> None: # Below the bar but model gave no questions -> generic fallback so the loop # always has something to ask. fake = _FakeInvoke(_json(0.20, [])) clar = ClaudeClarifier(invoke=fake) assert clar.generate_questions([], _state()) == [FALLBACK_QUESTION] # --------------------------------------------------------------------------- # # Defensive parsing: fences and surrounding prose still parse. # --------------------------------------------------------------------------- # def test_json_in_code_fence_is_parsed() -> None: fenced = "```json\n" + _json(0.97, ["q?"]) + "\n```" clar = ClaudeClarifier(invoke=_FakeInvoke(fenced)) assert clar.assess_confidence([], _state()) == 0.97 assert clar.generate_questions([], _state()) == ["q?"] def test_json_wrapped_in_prose_is_parsed() -> None: prose = ( "Sure! Here is my assessment:\n" + _json(0.55, ["Clarify the deadline?"]) + "\nLet me know if that helps." ) clar = ClaudeClarifier(invoke=_FakeInvoke(prose)) assert clar.assess_confidence([], _state()) == 0.55 assert clar.generate_questions([], _state()) == ["Clarify the deadline?"] def test_plain_json_fence_without_lang_is_parsed() -> None: fenced = "```\n" + _json(0.33, ["q?"]) + "\n```" clar = ClaudeClarifier(invoke=_FakeInvoke(fenced)) assert clar.assess_confidence([], _state()) == 0.33 # --------------------------------------------------------------------------- # # Confidence clamping into [0, 1]. # --------------------------------------------------------------------------- # def test_confidence_above_one_is_clamped() -> None: clar = ClaudeClarifier(invoke=_FakeInvoke(_json(1.5, []))) assert clar.assess_confidence([], _state()) == 1.0 def test_confidence_below_zero_is_clamped() -> None: clar = ClaudeClarifier(invoke=_FakeInvoke(_json(-0.2, ["q?"]))) assert clar.assess_confidence([], _state()) == 0.0 def test_confidence_as_string_is_coerced() -> None: clar = ClaudeClarifier(invoke=_FakeInvoke(_json("0.88", ["q?"]))) assert clar.assess_confidence([], _state()) == 0.88 # --------------------------------------------------------------------------- # # Prompt assembly pulls task/repo/context out of state. # --------------------------------------------------------------------------- # def test_prompt_includes_task_repo_and_qa() -> None: fake = _FakeInvoke(_json(0.99, [])) clar = ClaudeClarifier(invoke=fake) state = _state(task="Add a webhook verifier", repo="agent-team") clar.assess_confidence(["prior answer"], state) prompt = fake.calls[0]["prompt"] assert "Add a webhook verifier" in prompt assert "agent-team" in prompt assert "prior answer" in prompt # --------------------------------------------------------------------------- # # Factory returns the exact ConfidenceAssessor / QuestionGenerator pair. # --------------------------------------------------------------------------- # def test_factory_returns_shared_memoized_pair() -> None: fake = _FakeInvoke(_json(0.45, ["q?"])) assess, generate = build_claude_clarifier_callables(invoke=fake) # Both come from one shared instance -> one call serves both this turn. assert assess([], _state()) == 0.45 assert generate([], _state()) == ["q?"] assert len(fake.calls) == 1 # --------------------------------------------------------------------------- # # Integration smoke: wire into the real make_clarifier_node, gate clears. # --------------------------------------------------------------------------- # def _build_app(node): graph = StateGraph(PipelineState) graph.add_node("clarify", node) graph.add_edge(START, "clarify") graph.add_edge("clarify", END) return graph.compile(checkpointer=MemorySaver()) def test_node_clears_gate_when_confidence_rises() -> None: # Turn 0 (no answers): low confidence, asks. Turn 1 (one answer): confident. fake = _FakeInvoke([_json(0.20, ["What is the goal?"]), _json(0.99, [])]) assess, generate = build_claude_clarifier_callables(invoke=fake) node = make_clarifier_node(assess_confidence=assess, generate_questions=generate) app = _build_app(node) cfg = {"configurable": {"thread_id": "t-1"}} first = app.invoke(_state(thread_id="t-1"), cfg) assert "__interrupt__" in first # suspended on the question-set final = app.invoke(Command(resume="ship feature X"), cfg) assert "__interrupt__" not in final assert final["qa_history"] == ["ship feature X"] assert final["current_phase"] == Phase.PLAN.value assert final["status"] == TaskStatus.ACTIVE.value def test_node_parks_when_garbage_never_clears_gate() -> None: # A model that only ever emits garbage must NEVER open the gate; the loop # asks until the turn cap and parks (human gate stays shut). fake = _FakeInvoke("garbage, no json") assess, generate = build_claude_clarifier_callables(invoke=fake) from agent_team.nodes.clarifier import ClarifierConfig node = make_clarifier_node( assess_confidence=assess, generate_questions=generate, config=ClarifierConfig(max_turns=2), ) app = _build_app(node) cfg = {"configurable": {"thread_id": "t-1"}} assert "__interrupt__" in app.invoke(_state(thread_id="t-1"), cfg) assert "__interrupt__" in app.invoke(Command(resume="a1"), cfg) final = app.invoke(Command(resume="a2"), cfg) assert "__interrupt__" not in final assert final["current_phase"] == Phase.PARKED.value assert final["status"] == TaskStatus.PARKED.value assert final["current_phase"] != Phase.PLAN.value