procurement-ingest/tests/test_shoc_emitter_handler.py
Adam Moussa eb56d39b93
feat(webhook): SHOC WO webhook emitter — dark-ship streams, HMAC secret + rotation
Implements docs/shoc-webhook-plan.md Phases 1-5 (PR-2 of the SHOC
call-and-be-called effort). Everything ships DARK: both DynamoDB event
source mappings deploy enabled=False; activation is a deliberate
one-line follow-up PR gated on the SHOC receiver passing the shared
HMAC test vectors.

- Streams: NEW_AND_OLD_IMAGES on WorkOrders + WorkOrderComments
  (in-place update, RETAIN + logical IDs untouched; no existing
  consumers — verified live, neither table had a stream).
- workorder-shoc-emitter (Py3.12/ARM64): stream -> envelope ->
  HMAC-signed POST per docs/shoc-webhook-contract.md; strict per-shard
  ordering (parallelization 1, bisect off, retry until 24h age,
  ReportBatchItemFailures); 429/5xx/timeout block the shard in order,
  other 4xx park to workorder-shoc-emitter-rejected; ESM failures ->
  workorder-shoc-emitter-failures (metadata; replay rebuilds from
  DynamoDB). Echo guard skips write_origin=shoc-write-api.
- Secret workorder-ingest/shoc-webhook-hmac on a dedicated CMK
  (alias workorder-ingest-shoc-webhook-kms); cross-account
  GetSecretValue/DescribeSecret + kms:Decrypt granted to exactly
  arn:aws:iam::396287094661:role/shoc-backend-dev. RemovalPolicy
  DESTROY deliberately (machine-generated material; avoids the
  fixed-name RETAIN-orphan deadlock).
- workorder-shoc-hmac-rotator: 30-day rotation, dual-key overlap,
  64-hex keys, kid = UTC %Y-%m-%dT%H.
- Alarms (ALARM-only -> site-alerts): emitter errors/throttles/
  duration + iterator-age (>=10 min) + failures/rejected queue
  depth; rotator standard trio.
- scripts/replay_shoc_webhooks.py: dry-run-default operator replay
  (rebuilds from tables, replay:true envelopes).
- Tests: 742 passing, 85.56% aggregate; golden HMAC vectors shared
  with SHOC in docs/shoc-webhook-test-vectors.json (emitter + replay
  signing pinned to identical vectors); bundle-consistency AST pins
  for both new bundles.
- README: WO stack + webhook feed section, alarm table, runbooks;
  removed stale seahaven-slack-bot consumer references.
2026-07-24 14:54:07 -04:00

177 lines
5.9 KiB
Python

"""Batch-loop tests for the SHOC webhook emitter handler (plan Phase 5).
Pins the partial-batch ordering contract: on a retryable failure at record i
the loop STOPS -- record i's DynamoDB SequenceNumber is reported via
report_batch_item_failures (so the ESM retries from it, in order) and later
records are never attempted; earlier in-batch successes are not re-delivered.
Non-retryable rejections park the full envelope on the rejected queue and the
loop CONTINUES (a contract bug must never block the shard). Skip records
(REMOVE / echo guard) produce no delivery attempts at all.
delivery.deliver is monkeypatched on the handler's own sibling instance (the
handler imports it by bare name); envelope.build_event runs for real, so
these batches exercise the true record -> envelope -> deliver path. SQS is a
plain fake on the handler's public ``sqs`` module global.
"""
import json
import pytest
from tests.support import load_lambda_module
WO_STREAM_ARN = (
"arn:aws:dynamodb:us-east-1:011934824531:table/WorkOrders"
"/stream/2026-07-23T00:00:00.000"
)
REJECTED_QUEUE_URL = (
"https://sqs.us-east-1.amazonaws.com/011934824531/workorder-shoc-emitter-rejected"
)
@pytest.fixture(scope="module")
def handler_mod():
return load_lambda_module("wo", "shoc_emitter/handler")
class _FakeSQS:
def __init__(self):
self.sent = []
def send_message(self, QueueUrl, MessageBody): # noqa: N803 (boto3 kwargs)
self.sent.append({"QueueUrl": QueueUrl, "MessageBody": MessageBody})
def _wo_record(event_id, sequence_number, event_name="INSERT", **image_overrides):
image = {
"work_order_id": {"S": f"wo-{event_id}"},
"wo_status": {"S": "new"},
"record_type": {"S": "new_work_order"},
}
image.update(image_overrides)
stream = {
"ApproximateCreationDateTime": 1784642602.0,
"SequenceNumber": sequence_number,
"StreamViewType": "NEW_AND_OLD_IMAGES",
}
if event_name != "REMOVE":
stream["NewImage"] = image
return {
"eventID": event_id,
"eventName": event_name,
"eventSource": "aws:dynamodb",
"eventSourceARN": WO_STREAM_ARN,
"dynamodb": stream,
}
def _wire(monkeypatch, handler_mod, outcomes):
"""Patch delivery.deliver with a per-delivery_id outcome table.
``outcomes`` maps delivery_id (the record eventID) to either a
("delivered"|"rejected", status) tuple or the string "retryable" (raise).
Returns the ordered list of attempted delivery_ids and the fake SQS.
"""
attempted = []
def _fake_deliver(webhook_event):
attempted.append(webhook_event["delivery_id"])
outcome = outcomes[webhook_event["delivery_id"]]
if outcome == "retryable":
raise handler_mod.delivery.RetryableDeliveryError(
"receiver returned 503", status_code=503
)
return outcome
monkeypatch.setattr(handler_mod.delivery, "deliver", _fake_deliver)
fake_sqs = _FakeSQS()
monkeypatch.setattr(handler_mod, "sqs", fake_sqs)
monkeypatch.setattr(handler_mod, "REJECTED_QUEUE_URL", REJECTED_QUEUE_URL)
return attempted, fake_sqs
def test_retryable_middle_record_stops_batch_and_reports_its_sequence(
monkeypatch, handler_mod
):
event = {
"Records": [
_wo_record("evt-1", "101"),
_wo_record("evt-2", "102"),
_wo_record("evt-3", "103"),
]
}
attempted, fake_sqs = _wire(
monkeypatch,
handler_mod,
{"evt-1": ("delivered", 200), "evt-2": "retryable"},
)
result = handler_mod.handler(event, None)
# EXACTLY the failed record's SequenceNumber: the ESM retries from it in
# order, and evt-1 (already delivered) is not re-delivered.
assert result == {"batchItemFailures": [{"itemIdentifier": "102"}]}
# The loop stopped at the failure: record 3 was never attempted.
assert attempted == ["evt-1", "evt-2"]
assert fake_sqs.sent == []
def test_rejected_record_parks_to_sqs_and_processing_continues(
monkeypatch, handler_mod
):
event = {
"Records": [
_wo_record("evt-1", "201"),
_wo_record("evt-2", "202"),
_wo_record("evt-3", "203"),
]
}
attempted, fake_sqs = _wire(
monkeypatch,
handler_mod,
{
"evt-1": ("delivered", 200),
"evt-2": ("rejected", 422),
"evt-3": ("delivered", 204),
},
)
result = handler_mod.handler(event, None)
# A 4xx rejection must NOT block the shard: no batch item failures, and
# the records after the rejection were still attempted.
assert result == {"batchItemFailures": []}
assert attempted == ["evt-1", "evt-2", "evt-3"]
# The full envelope was parked for operator replay.
assert len(fake_sqs.sent) == 1
assert fake_sqs.sent[0]["QueueUrl"] == REJECTED_QUEUE_URL
parked = json.loads(fake_sqs.sent[0]["MessageBody"])
assert set(parked) == {"envelope", "response_status"}
assert parked["response_status"] == 422
assert parked["envelope"]["delivery_id"] == "evt-2"
assert parked["envelope"]["event_type"] == "work_order.created"
assert parked["envelope"]["data"]["work_order_id"] == "wo-evt-2"
def test_skip_records_produce_no_delivery_calls(monkeypatch, handler_mod):
event = {
"Records": [
_wo_record("evt-1", "301", event_name="REMOVE"),
_wo_record("evt-2", "302", write_origin={"S": "shoc-write-api"}),
]
}
attempted, fake_sqs = _wire(monkeypatch, handler_mod, {})
result = handler_mod.handler(event, None)
assert result == {"batchItemFailures": []}
assert attempted == []
assert fake_sqs.sent == []
def test_empty_batch_returns_no_failures(monkeypatch, handler_mod):
attempted, _ = _wire(monkeypatch, handler_mod, {})
assert handler_mod.handler({"Records": []}, None) == {"batchItemFailures": []}
assert attempted == []