mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-10-05 19:51:58 +00:00
High-recall detector fan-out (injection/authz/secrets-crypto/iac-iam/logic) + proof-or-kill verifier. Gate PASSES: 1 confirmed medium, 0 confirmed critical/high. Confirmed finding fixed; several unverified-but-cheap hardenings applied since the emitter ships dark and activation is weeks out. - CONFIRMED medium (confused deputy): the rotation Lambda's generated invoke permission for secretsmanager.amazonaws.com carried no SourceAccount/SourceArn, so any account's Secrets Manager could invoke the rotator. Patched the generated CfnPermission in place (a second permission would be additive, not restrictive) to pin account + this secret ARN. - delivery + replay: refuse to follow receiver 3xx redirects (no-redirect opener) so live X-SH-* auth headers can't be forwarded to a receiver-chosen Location and an http:// Location can't slip past the https guard. Fixed the "unfollowed 3xx" comment that was factually wrong. - delivery: classify 401/403 as retryable (invalidate key cache + retry in order) instead of parking -- transient auth failures (rotation outran the TTL cache, clock skew) are availability events, not contract bugs. - envelope: build_event now genuinely total (guarded eventID / ApproximateCreationDateTime subscripts) per its own never-raise contract. - handler: catch-all so an unexpected per-record error (e.g. SQS park failure) reports only that record instead of failing the whole batch (which would re-deliver every earlier success for 24h); per-invocation emit/skip batch summary so a systemic silent drop is queryable/alarmable. - rotator: narrow the AWSCURRENT-read except to ResourceNotFound/JSONDecode (transient SM/KMS errors re-raise so the overlap key isn't silently dropped); kid uniqueness checked against ALL retained kids with a random suffix on collision (never reissue a kid for a different secret). - contract: skeleton-upsert required on ANY unknown work_order_id (not just comment-before-create) + monotonicity guard (ignore older updated_at), so a parked created or an out-of-order replay can't corrupt receiver state. Unverified/refuted findings left as-is with rationale: the two "high" logic claims (whole-batch crash triggers, ordering violation) were refuted on reachability (real stream records carry required fields; persistence writes strings only; full-state idempotent upsert absorbs the ordering gap). Signed kid/version binding (AUTHZ-002) declined: coordinated contract change, not cheap, no exploit with one algorithm/key.
205 lines
8.3 KiB
Python
205 lines
8.3 KiB
Python
"""HMAC-signed webhook delivery to the SHOC receiver (contract sections 6-7).
|
|
|
|
Owns the signed POST and its response classification. The signing key comes
|
|
from Secrets Manager (``workorder-ingest/shoc-webhook-hmac``, dual-key shape
|
|
``{"keys": [{"kid", "secret"}, ...]}``), cached in the warm container for a
|
|
short TTL so 30-day rotation propagates without waiting for the execution
|
|
environment to recycle -- the producer always signs with ``keys[0]``.
|
|
|
|
Response contract (section 7): 2xx -> delivered; 429/5xx/timeout/connection
|
|
error -> RetryableDeliveryError (the handler surfaces it as a batch item
|
|
failure so the ESM blocks the shard and retries in order); any other status ->
|
|
("rejected", code) for the handler to park (a contract bug must not block the
|
|
shard for 24 hours). Secret material and signatures are never logged.
|
|
|
|
``sign_body`` is a module-level pure function on purpose: the golden-vector
|
|
test suite and the replay script both pin against it, and it is the shared
|
|
definition Luby's receiver verifies with.
|
|
"""
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from http import HTTPStatus
|
|
|
|
import boto3
|
|
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.INFO)
|
|
|
|
# Endpoint path is TBD by SHOC; the activation PR confirms the final URL.
|
|
SHOC_WEBHOOK_URL = os.environ.get(
|
|
"SHOC_WEBHOOK_URL", "https://api.dev.seahaven.com/api/webhooks/work-orders"
|
|
)
|
|
HMAC_SECRET_ARN = os.environ.get("HMAC_SECRET_ARN")
|
|
|
|
POST_TIMEOUT_SECONDS = 10
|
|
USER_AGENT = "workorder-shoc-emitter/1"
|
|
|
|
# 2xx = delivered; 429 and 5xx retry; everything else parks (contract sec. 7).
|
|
_HTTP_SUCCESS_RANGE = range(HTTPStatus.OK, HTTPStatus.MULTIPLE_CHOICES)
|
|
_HTTP_SERVER_ERROR_MIN = HTTPStatus.INTERNAL_SERVER_ERROR
|
|
# Transient auth failures: a stale cached key mid-rotation, a receiver
|
|
# secret-fetch blip, or clock skew past the +/-300s window. These are
|
|
# availability events, not payload contract bugs, so they retry (in order)
|
|
# after the key cache is dropped -- never park, which would strand the
|
|
# delivery out of the ordered feed until a manual replay.
|
|
_HTTP_AUTH_FAILURES = (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN)
|
|
|
|
|
|
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
"""Refuse to follow receiver redirects.
|
|
|
|
The default opener follows 3xx transparently, which would (a) forward the
|
|
live X-SH-* auth headers to a receiver-chosen Location and (b) let an
|
|
http:// Location slip past the https-only guard on the configured URL. We
|
|
only ever POST to the one configured endpoint; a redirect is a receiver
|
|
misconfiguration, so surface the 3xx as an HTTPError and let it park.
|
|
"""
|
|
|
|
def redirect_request(self, *args, **kwargs):
|
|
return None
|
|
|
|
|
|
_opener = urllib.request.build_opener(_NoRedirectHandler)
|
|
|
|
# Refresh the cached key material this often so a rotated secret propagates
|
|
# without waiting for the execution environment to recycle (contract requires
|
|
# receiver-side TTL <= 300s; the producer matches it).
|
|
_HMAC_KEYS_CACHE_TTL_SECONDS = 300
|
|
_hmac_keys_cache = None
|
|
_hmac_keys_cached_at = 0.0
|
|
|
|
|
|
class RetryableDeliveryError(Exception):
|
|
"""Delivery failed in a way the ESM should retry in order (shard-blocking).
|
|
|
|
``status_code`` carries the HTTP status when one exists (429/5xx); it is
|
|
None for timeouts, connection errors, and secret-fetch failures.
|
|
"""
|
|
|
|
def __init__(self, message: str, status_code: int | None = None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
|
|
|
|
def _get_hmac_keys() -> list[dict]:
|
|
"""Fetch the signing-key list from Secrets Manager, TTL-cached.
|
|
|
|
An empty/missing ``keys`` list is the bootstrap state before the first
|
|
rotation has run -- retryable, not a crash: the shard blocks until the
|
|
rotator populates the secret. Fetch failures (throttle, transient IAM/KMS
|
|
denial) are likewise retryable so a Secrets Manager blip blocks in order
|
|
instead of failing the whole batch. Key material is never logged.
|
|
"""
|
|
global _hmac_keys_cache, _hmac_keys_cached_at
|
|
now = time.monotonic()
|
|
if (
|
|
_hmac_keys_cache is not None
|
|
and now - _hmac_keys_cached_at < _HMAC_KEYS_CACHE_TTL_SECONDS
|
|
):
|
|
return _hmac_keys_cache
|
|
return _refresh_hmac_keys()
|
|
|
|
|
|
def _invalidate_hmac_keys() -> None:
|
|
"""Drop the cached key material so the next sign re-fetches (rotation)."""
|
|
global _hmac_keys_cache
|
|
_hmac_keys_cache = None
|
|
|
|
|
|
def _refresh_hmac_keys() -> list[dict]:
|
|
"""Force a fresh Secrets Manager fetch, bypassing the TTL cache."""
|
|
global _hmac_keys_cache, _hmac_keys_cached_at
|
|
now = time.monotonic()
|
|
secrets = boto3.client("secretsmanager")
|
|
try:
|
|
secret = secrets.get_secret_value(SecretId=HMAC_SECRET_ARN)
|
|
keys = json.loads(secret["SecretString"]).get("keys") or []
|
|
except Exception as exc:
|
|
raise RetryableDeliveryError(f"hmac secret fetch failed: {exc}") from exc
|
|
if not keys:
|
|
raise RetryableDeliveryError("hmac secret not yet rotated")
|
|
_hmac_keys_cache = keys
|
|
_hmac_keys_cached_at = now
|
|
return keys
|
|
|
|
|
|
def sign_body(secret_hex: str, timestamp: int, raw_body: bytes) -> str:
|
|
"""HMAC-SHA256 hex digest over ``f"{timestamp}.{raw_body}"`` (raw bytes).
|
|
|
|
The key is the UTF-8 bytes of the secret string exactly as stored in the
|
|
secret's ``secret`` field (the receiver reads the same JSON field -- no
|
|
hex-decoding on either side). Pure function; the golden-vector tests and
|
|
Luby's receiver both pin against this definition.
|
|
"""
|
|
string_to_sign = f"{timestamp}.".encode() + raw_body
|
|
return hmac.new(
|
|
secret_hex.encode("utf-8"), string_to_sign, hashlib.sha256
|
|
).hexdigest()
|
|
|
|
|
|
def deliver(envelope: dict) -> tuple[str, int]:
|
|
"""POST one envelope to SHOC. Returns ("delivered"|"rejected", status).
|
|
|
|
Raises RetryableDeliveryError for 429/5xx/timeout/connection failures so
|
|
the caller can block the shard (in-order retry, contract section 7).
|
|
"""
|
|
if not SHOC_WEBHOOK_URL.startswith("https://"):
|
|
# Fail closed on any non-HTTPS scheme (file://, http://, ...): the
|
|
# URL is operator-set env config, but urllib would happily follow
|
|
# other schemes and the HMAC only protects an HTTPS transport.
|
|
raise RetryableDeliveryError(
|
|
"SHOC_WEBHOOK_URL is not an https:// URL; refusing to deliver"
|
|
)
|
|
raw_body = json.dumps(envelope).encode("utf-8")
|
|
timestamp = int(time.time())
|
|
signing_key = _get_hmac_keys()[0]
|
|
signature = sign_body(signing_key["secret"], timestamp, raw_body)
|
|
request = urllib.request.Request(
|
|
SHOC_WEBHOOK_URL,
|
|
data=raw_body,
|
|
headers={
|
|
"Content-Type": "application/json; charset=utf-8",
|
|
"User-Agent": USER_AGENT,
|
|
"X-SH-Timestamp": str(timestamp),
|
|
"X-SH-Key-Id": signing_key["kid"],
|
|
"X-SH-Signature": f"v1={signature}",
|
|
},
|
|
method="POST",
|
|
)
|
|
try:
|
|
# _opener refuses redirects, so a 3xx surfaces here as an HTTPError
|
|
# rather than silently re-issuing the request (with its auth headers)
|
|
# to a receiver-chosen Location.
|
|
with _opener.open(request, timeout=POST_TIMEOUT_SECONDS) as response:
|
|
status_code = response.status
|
|
except urllib.error.HTTPError as exc:
|
|
status_code = exc.code
|
|
if (
|
|
status_code == HTTPStatus.TOO_MANY_REQUESTS
|
|
or status_code >= _HTTP_SERVER_ERROR_MIN
|
|
):
|
|
raise RetryableDeliveryError(
|
|
f"receiver returned {status_code}", status_code=status_code
|
|
) from exc
|
|
if status_code in _HTTP_AUTH_FAILURES:
|
|
# Drop the cached key so an in-order retry re-signs with the
|
|
# current secret (handles a rotation that outran the TTL cache).
|
|
_invalidate_hmac_keys()
|
|
raise RetryableDeliveryError(
|
|
f"receiver auth failure {status_code}", status_code=status_code
|
|
) from exc
|
|
return ("rejected", status_code)
|
|
except (TimeoutError, urllib.error.URLError, OSError) as exc:
|
|
raise RetryableDeliveryError(f"connection error: {exc}") from exc
|
|
if status_code in _HTTP_SUCCESS_RANGE:
|
|
return ("delivered", status_code)
|
|
# A non-2xx that the opener did not raise for -- contract bug on
|
|
# someone's side. Park it, never block the shard.
|
|
return ("rejected", status_code)
|