mirror of
https://github.com/Sea-Haven-Industries/apm-wo-analysis.git
synced 2026-09-30 17:03:15 +00:00
271 lines
10 KiB
Python
271 lines
10 KiB
Python
"""Tests for the Haiku fallback in classify_with_haiku().
|
|
|
|
No real network or AWS calls are made — _call_haiku and _fetch_api_key are
|
|
monkeypatched at the function level. boto3 shape tests stub the secretsmanager
|
|
client directly via monkeypatch on boto3.client.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from unittest.mock import MagicMock
|
|
|
|
import classify
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Comment that will always land as "Other" deterministically (free-text with
|
|
# no structured hold signal and no regex match).
|
|
# ---------------------------------------------------------------------------
|
|
_OTHER_COMMENT = "<html>Uplift request submitted pending management approval.</html>"
|
|
_TRIVIAL_COMMENT = "<html>Ok</html>" # strips to "Ok" — len == 2 < 4
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _haiku_never_called():
|
|
"""Return a _call_haiku stub that raises if invoked."""
|
|
|
|
def _stub(text, api_key):
|
|
raise AssertionError("_call_haiku should not have been called")
|
|
|
|
return _stub
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Gating: Haiku is NOT called when the deterministic result is non-Other
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_haiku_not_called_when_deterministic_result_is_non_other(monkeypatch):
|
|
"""A comment that resolves deterministically must skip the Haiku path."""
|
|
monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called())
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key")
|
|
|
|
# "WO schedule confirmed with vendor" → Schedule Confirmed deterministically
|
|
cat, mm = classify.classify_with_haiku(
|
|
"IP", "", "<html>WO schedule confirmed with vendor.</html>"
|
|
)
|
|
assert cat == "Schedule Confirmed"
|
|
assert mm is None
|
|
|
|
|
|
def test_haiku_not_called_when_hold_reason_present(monkeypatch):
|
|
"""Hold reason present → structured state handles it; Haiku must not fire."""
|
|
monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called())
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key")
|
|
|
|
# REPORT hold with unrecognised comment → hold decides, not Haiku
|
|
cat, _ = classify.classify_with_haiku(
|
|
"H", "REPORT", "<html>xyz nothing here</html>"
|
|
)
|
|
assert cat == "Report / Docs Needed"
|
|
|
|
|
|
def test_haiku_not_called_when_comment_is_trivial(monkeypatch):
|
|
"""Comment len < 4 after stripping → trivial, skip Haiku."""
|
|
monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called())
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _TRIVIAL_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
def test_haiku_not_called_when_disabled_via_env(monkeypatch):
|
|
"""APM_HAIKU_FALLBACK=off must suppress the Haiku call."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "off")
|
|
monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called())
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
def test_haiku_not_called_when_disabled_via_zero(monkeypatch):
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "0")
|
|
monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called())
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
def test_haiku_not_called_when_disabled_via_false(monkeypatch):
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "false")
|
|
monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called())
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Haiku IS invoked (enabled, Other, no hold, non-trivial)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_haiku_called_for_true_residual_other(monkeypatch):
|
|
"""When all gates pass, _call_haiku must be invoked."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "on")
|
|
haiku_calls: list[tuple] = []
|
|
|
|
def _stub_haiku(text, api_key):
|
|
haiku_calls.append((text, api_key))
|
|
return "Rescheduled"
|
|
|
|
monkeypatch.setattr(classify, "_call_haiku", _stub_haiku)
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Rescheduled"
|
|
assert len(haiku_calls) == 1
|
|
|
|
|
|
def test_haiku_valid_bucket_replaces_other(monkeypatch):
|
|
"""When Haiku returns a valid bucket, that bucket is used."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "1")
|
|
monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Rescheduled")
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Rescheduled"
|
|
|
|
|
|
def test_haiku_none_response_stays_other(monkeypatch):
|
|
"""When Haiku returns None, the result stays Other."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "1")
|
|
monkeypatch.setattr(classify, "_call_haiku", lambda text, key: None)
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
def test_haiku_other_response_stays_other(monkeypatch):
|
|
"""When Haiku returns 'Other', the result stays Other (guard against self-loop)."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "1")
|
|
monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Other")
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
def test_haiku_exception_in_call_haiku_stays_other(monkeypatch):
|
|
"""Exception raised by _call_haiku (e.g. network failure) is caught and
|
|
the result safely stays at Other."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "1")
|
|
|
|
def _raise(text, api_key):
|
|
raise OSError("network failure")
|
|
|
|
monkeypatch.setattr(classify, "_call_haiku", _raise)
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Other"
|
|
|
|
|
|
def test_mismatch_preserved_through_haiku_path(monkeypatch):
|
|
"""The deterministic mismatch reason must survive through the Haiku path."""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "1")
|
|
monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Rescheduled")
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
# A comment that fires "Completed / Pending Close" intent while on a REPORT hold
|
|
# produces a mismatch; but the deterministic result is NOT Other here. To test
|
|
# mismatch preservation we need the deterministic result to be Other while a
|
|
# mismatch exists. Mismatch is computed from intent vs structured state; if the
|
|
# intent is None (→ Other from deterministic), no mismatch is produced by
|
|
# _mismatch(). So the relevant test is: Haiku fires on an Other row; the
|
|
# mismatch (None in this case) is preserved correctly.
|
|
cat, mm = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
# Comment has no intent → no mismatch → mm is None; Haiku returns Rescheduled
|
|
assert cat == "Rescheduled"
|
|
assert mm is None
|
|
|
|
|
|
def test_mismatch_preserved_when_haiku_overrides(monkeypatch):
|
|
"""Verify mismatch from classify() passes through unchanged when Haiku overrides.
|
|
|
|
We use a real deterministic Other-producing row and manually inject a mismatch
|
|
by monkeypatching classify.classify to return (Other, "fake mismatch reason").
|
|
"""
|
|
monkeypatch.setenv("APM_HAIKU_FALLBACK", "1")
|
|
monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Rescheduled")
|
|
monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key")
|
|
|
|
original_classify = classify.classify
|
|
|
|
def _patched_classify(wo_status, hold_reason, last_comment):
|
|
cat, _ = original_classify(wo_status, hold_reason, last_comment)
|
|
if cat == "Other":
|
|
return "Other", "injected mismatch reason"
|
|
return cat, _
|
|
|
|
monkeypatch.setattr(classify, "classify", _patched_classify)
|
|
|
|
cat, mm = classify.classify_with_haiku("IP", "", _OTHER_COMMENT)
|
|
assert cat == "Rescheduled"
|
|
assert mm == "injected mismatch reason"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _fetch_api_key JSON shapes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_secretsmanager_client(secret_string: str) -> MagicMock:
|
|
"""Build a minimal secretsmanager client stub returning the given secret."""
|
|
client = MagicMock()
|
|
client.get_secret_value.return_value = {"SecretString": secret_string}
|
|
return client
|
|
|
|
|
|
def test_fetch_api_key_bare_string(monkeypatch):
|
|
"""Bare string secret → returned as-is."""
|
|
fake_client = _make_secretsmanager_client("sk-ant-barekey123")
|
|
|
|
import boto3
|
|
|
|
monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client)
|
|
key = classify._fetch_api_key()
|
|
assert key == "sk-ant-barekey123"
|
|
|
|
|
|
def test_fetch_api_key_json_anthropic_api_key(monkeypatch):
|
|
"""JSON with 'anthropic-api-key' field → that value extracted."""
|
|
secret = json.dumps({"anthropic-api-key": "sk-ant-jsonkey456"})
|
|
fake_client = _make_secretsmanager_client(secret)
|
|
|
|
import boto3
|
|
|
|
monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client)
|
|
key = classify._fetch_api_key()
|
|
assert key == "sk-ant-jsonkey456"
|
|
|
|
|
|
def test_fetch_api_key_single_value_json(monkeypatch):
|
|
"""Single-value JSON object with an arbitrary key → the only value extracted."""
|
|
secret = json.dumps({"my_custom_key": "sk-ant-singleval789"})
|
|
fake_client = _make_secretsmanager_client(secret)
|
|
|
|
import boto3
|
|
|
|
monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client)
|
|
key = classify._fetch_api_key()
|
|
assert key == "sk-ant-singleval789"
|
|
|
|
|
|
def test_fetch_api_key_json_api_key_field(monkeypatch):
|
|
"""JSON with 'api_key' field → that value extracted."""
|
|
secret = json.dumps({"api_key": "sk-ant-apikey111"})
|
|
fake_client = _make_secretsmanager_client(secret)
|
|
|
|
import boto3
|
|
|
|
monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client)
|
|
key = classify._fetch_api_key()
|
|
assert key == "sk-ant-apikey111"
|