orchestrator/agent-team/tests/test_deadline_timer.py

"""Unit tests for agent_team.deadline_timer (design §3.3.1).

Covers the deadline / no-answer timer loop: overdue selection, the
deterministic answer-vs-timeout race on the ``open`` -> ``expired`` flip, the
PARK and DEFAULT_ANSWER policies, restart idempotency, side-effect isolation,
and the concurrent responder-vs-timer race.
"""

from __future__ import annotations

import sqlite3
import threading
from datetime import datetime, timedelta, timezone
from pathlib import Path

import pytest

from agent_team.db.schema import answer_question, connect, init_db
from agent_team.deadline_timer import (
    DeadlinePolicy,
    ExpiryAction,
    OverdueQuestion,
    TimerLoopReport,
    overdue_open_questions,
    run_deadline_timer,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

_PAST = "2000-01-01T00:00:00+00:00"
_FUTURE = "2999-01-01T00:00:00+00:00"


def _iso(dt: datetime) -> str:
    return dt.isoformat()


def _insert_question(
    conn: sqlite3.Connection,
    qid: str,
    *,
    thread_id: str = "thread-1",
    turn: int = 0,
    status: str = "open",
    transport: str = "slack",
    deadline_at: str | None = _PAST,
) -> None:
    conn.execute(
        "INSERT INTO pending_questions "
        "(question_id, thread_id, turn, status, transport, posted_at, deadline_at) "
        "VALUES (?, ?, ?, ?, ?, ?, ?)",
        (qid, thread_id, turn, status, transport, _PAST, deadline_at),
    )


@pytest.fixture()
def conn(tmp_path: Path) -> sqlite3.Connection:
    db = tmp_path / "db.sqlite"
    init_db(db)
    c = connect(db)
    yield c
    c.close()


def _status(conn: sqlite3.Connection, qid: str) -> str:
    return conn.execute(
        "SELECT status FROM pending_questions WHERE question_id=?", (qid,)
    ).fetchone()["status"]


class _Collector:
    """Records the questions handed to an injected side-effect callback."""

    def __init__(self) -> None:
        self.calls: list[OverdueQuestion] = []

    def __call__(self, question: OverdueQuestion) -> None:
        self.calls.append(question)


# ---------------------------------------------------------------------------
# overdue_open_questions
# ---------------------------------------------------------------------------


def test_overdue_selects_only_past_open_with_deadline(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "past", deadline_at=_PAST)
    _insert_question(conn, "future", deadline_at=_FUTURE)
    _insert_question(conn, "no-deadline", deadline_at=None)
    _insert_question(conn, "answered", status="answered", deadline_at=_PAST)
    _insert_question(conn, "expired", status="expired", deadline_at=_PAST)

    overdue = overdue_open_questions(conn)
    assert [q.question_id for q in overdue] == ["past"]


def test_overdue_uses_now_cutoff(conn: sqlite3.Connection) -> None:
    now = datetime(2026, 6, 17, tzinfo=timezone.utc)
    just_past = _iso(now - timedelta(seconds=1))
    just_future = _iso(now + timedelta(seconds=1))
    _insert_question(conn, "before", deadline_at=just_past)
    _insert_question(conn, "after", deadline_at=just_future)

    overdue = overdue_open_questions(conn, now=_iso(now))
    assert [q.question_id for q in overdue] == ["before"]


def test_overdue_includes_deadline_equal_to_now(conn: sqlite3.Connection) -> None:
    now = "2026-06-17T00:00:00+00:00"
    _insert_question(conn, "exact", deadline_at=now)
    overdue = overdue_open_questions(conn, now=now)
    assert [q.question_id for q in overdue] == ["exact"]


def test_overdue_ordered_oldest_deadline_first(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "newer", deadline_at="2010-01-01T00:00:00+00:00")
    _insert_question(conn, "older", deadline_at="2001-01-01T00:00:00+00:00")
    overdue = overdue_open_questions(conn)
    assert [q.question_id for q in overdue] == ["older", "newer"]


def test_overdue_row_fields_mapped(conn: sqlite3.Connection) -> None:
    _insert_question(
        conn, "q", thread_id="t-42", turn=3, transport="github", deadline_at=_PAST
    )
    (q,) = overdue_open_questions(conn)
    assert q.question_id == "q"
    assert q.thread_id == "t-42"
    assert q.turn == 3
    assert q.transport == "github"
    assert q.deadline_at == _PAST


# ---------------------------------------------------------------------------
# run_deadline_timer — PARK policy (default)
# ---------------------------------------------------------------------------


def test_park_policy_expires_and_alarms(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "q1")
    on_park = _Collector()

    report = run_deadline_timer(conn, on_park=on_park)

    assert _status(conn, "q1") == "expired"
    assert [q.question_id for q in on_park.calls] == ["q1"]
    assert report.examined == 1
    assert report.parked == 1
    assert report.expired == 1
    assert report.lost_race == 0
    assert report.outcomes[0].action is ExpiryAction.PARKED
    assert report.outcomes[0].policy is DeadlinePolicy.PARK


def test_default_policy_is_park(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "q1")
    on_park = _Collector()
    # No policy_resolver supplied -> PARK for every overdue question.
    run_deadline_timer(conn, on_park=on_park)
    assert len(on_park.calls) == 1


def test_multiple_overdue_all_parked(conn: sqlite3.Connection) -> None:
    for i in range(5):
        _insert_question(conn, f"q{i}")
    on_park = _Collector()

    report = run_deadline_timer(conn, on_park=on_park)

    assert report.examined == 5
    assert report.parked == 5
    assert {c.question_id for c in on_park.calls} == {f"q{i}" for i in range(5)}
    for i in range(5):
        assert _status(conn, f"q{i}") == "expired"


def test_future_and_null_deadlines_untouched(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "future", deadline_at=_FUTURE)
    _insert_question(conn, "none", deadline_at=None)
    on_park = _Collector()

    report = run_deadline_timer(conn, on_park=on_park)

    assert report.examined == 0
    assert on_park.calls == []
    assert _status(conn, "future") == "open"
    assert _status(conn, "none") == "open"


# ---------------------------------------------------------------------------
# run_deadline_timer — DEFAULT_ANSWER policy
# ---------------------------------------------------------------------------


def test_default_answer_policy_resumes(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "q1")
    on_park = _Collector()
    resume = _Collector()

    report = run_deadline_timer(
        conn,
        on_park=on_park,
        resume_with_default=resume,
        policy_resolver=lambda _q: DeadlinePolicy.DEFAULT_ANSWER,
    )

    assert _status(conn, "q1") == "expired"
    assert on_park.calls == []
    assert [q.question_id for q in resume.calls] == ["q1"]
    assert report.defaulted == 1
    assert report.parked == 0
    assert report.outcomes[0].action is ExpiryAction.DEFAULTED


def test_default_answer_without_callback_raises(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "q1")
    on_park = _Collector()

    with pytest.raises(ValueError, match="DEFAULT_ANSWER"):
        run_deadline_timer(
            conn,
            on_park=on_park,
            policy_resolver=lambda _q: DeadlinePolicy.DEFAULT_ANSWER,
        )
    # The row was already flipped to expired before the config error surfaced.
    assert _status(conn, "q1") == "expired"


def test_mixed_policies_routed_per_question(conn: sqlite3.Connection) -> None:
    _insert_question(conn, "park-me")
    _insert_question(conn, "default-me")
    on_park = _Collector()
    resume = _Collector()

    def resolver(q: OverdueQuestion) -> DeadlinePolicy:
        return (
            DeadlinePolicy.DEFAULT_ANSWER
            if q.question_id == "default-me"
            else DeadlinePolicy.PARK
        )

    report = run_deadline_timer(
        conn,
        on_park=on_park,
        resume_with_default=resume,
        policy_resolver=resolver,
    )

    assert [c.question_id for c in on_park.calls] == ["park-me"]
    assert [c.question_id for c in resume.calls] == ["default-me"]
    assert report.parked == 1
    assert report.defaulted == 1


# ---------------------------------------------------------------------------
# Deterministic answer-vs-timeout race (§3.3.1)
# ---------------------------------------------------------------------------


def test_answered_first_loses_race_no_policy(conn: sqlite3.Connection) -> None:
    """A question answered before the timer runs must NOT be parked/defaulted."""
    _insert_question(conn, "q1")
    # Responder wins first.
    assert answer_question(
        conn, question_id="q1", answer_json="{}", answered_via="slack"
    )
    on_park = _Collector()
    resume = _Collector()

    report = run_deadline_timer(conn, on_park=on_park, resume_with_default=resume)

    # The row is no longer ``open`` so it is not even returned by the overdue
    # query — examined is zero, no policy applied.
    assert report.examined == 0
    assert on_park.calls == []
    assert resume.calls == []
    assert _status(conn, "q1") == "answered"


def test_lost_race_when_answered_between_select_and_flip(
    conn: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch
) -> None:
    """If a responder answers a row after it was selected as overdue but before
    the timer flips it, the timer's compare-and-set returns False -> LOST_RACE,
    and NO no-answer policy is applied (the question was actually answered)."""
    import agent_team.deadline_timer as dt

    _insert_question(conn, "q1")
    on_park = _Collector()

    real_expire = dt.expire_question

    def racing_expire(c: sqlite3.Connection, *, question_id: str) -> bool:
        # Simulate the responder winning the compare-and-set in the window
        # between overdue selection and this flip. Uses the SAME connection so
        # the ordering is fully deterministic (no thread scheduling needed).
        answer_question(
            c, question_id=question_id, answer_json="{}", answered_via="slack"
        )
        return real_expire(c, question_id=question_id)

    monkeypatch.setattr(dt, "expire_question", racing_expire)

    report = run_deadline_timer(conn, on_park=on_park)

    # The flip lost: no park, row is ``answered`` not ``expired``.
    assert on_park.calls == []
    assert report.examined == 1
    assert report.lost_race == 1
    assert report.parked == 0
    assert report.expired == 0
    assert report.outcomes[0].action is ExpiryAction.LOST_RACE
    assert _status(conn, "q1") == "answered"


# ---------------------------------------------------------------------------
# Restart idempotency
# ---------------------------------------------------------------------------


def test_rerun_is_idempotent(conn: sqlite3.Connection) -> None:
    """A second pass after a 'crash' processes no already-expired rows."""
    _insert_question(conn, "q1")
    on_park = _Collector()

    first = run_deadline_timer(conn, on_park=on_park)
    assert first.parked == 1

    second = run_deadline_timer(conn, on_park=on_park)
    # Already expired -> no longer ``open`` -> not selected -> no double park.
    assert second.examined == 0
    assert len(on_park.calls) == 1
    assert _status(conn, "q1") == "expired"


# ---------------------------------------------------------------------------
# Side-effect isolation
# ---------------------------------------------------------------------------


def test_side_effect_error_isolated_row_still_expired(
    conn: sqlite3.Connection,
) -> None:
    _insert_question(conn, "boom")
    _insert_question(conn, "ok")

    def flaky_park(q: OverdueQuestion) -> None:
        if q.question_id == "boom":
            raise RuntimeError("alarm transport down")

    report = run_deadline_timer(conn, on_park=flaky_park)

    # Both rows are durably expired (flip commits before the side effect).
    assert _status(conn, "boom") == "expired"
    assert _status(conn, "ok") == "expired"
    assert report.errored == 1
    assert report.parked == 1
    errored = next(o for o in report.outcomes if o.action is ExpiryAction.ERRORED)
    assert errored.question_id == "boom"
    assert "RuntimeError" in (errored.error or "")
    assert errored.policy is DeadlinePolicy.PARK


def test_error_in_one_row_does_not_abort_batch(conn: sqlite3.Connection) -> None:
    for i in range(4):
        _insert_question(conn, f"q{i}")

    def park(q: OverdueQuestion) -> None:
        if q.question_id == "q1":
            raise ValueError("nope")

    report = run_deadline_timer(conn, on_park=park)

    assert report.examined == 4
    assert report.errored == 1
    assert report.parked == 3
    for i in range(4):
        assert _status(conn, f"q{i}") == "expired"


# ---------------------------------------------------------------------------
# TimerLoopReport counters
# ---------------------------------------------------------------------------


def test_empty_report_counters() -> None:
    report = TimerLoopReport()
    assert report.examined == 0
    assert report.parked == 0
    assert report.defaulted == 0
    assert report.lost_race == 0
    assert report.errored == 0
    assert report.expired == 0


def test_expired_equals_examined_minus_lost_race(
    conn: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch
) -> None:
    import agent_team.deadline_timer as dt

    _insert_question(conn, "parkable")
    _insert_question(conn, "raced")
    on_park = _Collector()

    real_expire = dt.expire_question

    def racing_expire(c: sqlite3.Connection, *, question_id: str) -> bool:
        if question_id == "raced":
            answer_question(
                c, question_id="raced", answer_json="{}", answered_via="slack"
            )
        return real_expire(c, question_id=question_id)

    monkeypatch.setattr(dt, "expire_question", racing_expire)

    report = run_deadline_timer(conn, on_park=on_park)

    assert report.examined == 2
    assert report.lost_race == 1
    assert report.expired == report.examined - report.lost_race == 1


# ---------------------------------------------------------------------------
# Concurrency: responder thread vs timer thread on the same question
# ---------------------------------------------------------------------------


def test_concurrent_timer_and_responder_single_winner(tmp_path: Path) -> None:
    """A timer pass and a responder race the same open question: the
    BEGIN-IMMEDIATE compare-and-set guarantees exactly one of 'expire'/'answer'
    wins, and the timer parks IFF it actually flipped the row to ``expired``.

    Repeated across many rows so the threads genuinely interleave (the barrier
    aligns each pair at the start), catching any non-determinism in the race.
    """
    db = tmp_path / "race.sqlite"
    init_db(db)

    n = 40
    seed = connect(db)
    try:
        for i in range(n):
            _insert_question(seed, f"race{i}", deadline_at=_PAST)
    finally:
        seed.close()

    park_calls: list[str] = []
    park_lock = threading.Lock()

    def run_one(qid: str) -> None:
        barrier = threading.Barrier(2)

        def timer_worker() -> None:
            c = connect(db)
            try:

                def on_park(q: OverdueQuestion) -> None:
                    with park_lock:
                        park_calls.append(q.question_id)

                barrier.wait()
                run_deadline_timer(c, on_park=on_park)
            finally:
                c.close()

        def responder_worker() -> None:
            c = connect(db)
            try:
                barrier.wait()
                answer_question(
                    c, question_id=qid, answer_json="{}", answered_via="slack"
                )
            finally:
                c.close()

        t1 = threading.Thread(target=timer_worker)
        t2 = threading.Thread(target=responder_worker)
        t1.start()
        t2.start()
        t1.join()
        t2.join()

    for i in range(n):
        run_one(f"race{i}")

    check = connect(db)
    try:
        rows = {
            r["question_id"]: r["status"]
            for r in check.execute(
                "SELECT question_id, status FROM pending_questions"
            ).fetchall()
        }
    finally:
        check.close()

    # Every row ends in exactly one terminal state.
    for i in range(n):
        assert rows[f"race{i}"] in {"expired", "answered"}
    # Every parked question must be one the timer actually expired.
    for qid in park_calls:
        assert rows[qid] == "expired"