mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-10-07 16:08:54 +00:00
Adam Moussa
88805d906d
--all is a distinct mode (the demoted full-prefix sweep), but the args.all branch unconditionally set prefix=inbound/ and since=None, so passing it alongside a narrower selector silently discarded that selector. `--all --since 2026-07-01` swept the entire corpus instead of the bounded window, triggering every documented --all hazard (Bedrock re-bill, metric double- count, merged-field regression) on objects the operator never targeted -- contradicting the tool's safety goal. Add the missing mutual-exclusion guard alongside the existing --key one, and pin --all+--prefix, --all+--since, and all three together as argparse rejections.
124 lines
5.2 KiB
Python
124 lines
5.2 KiB
Python
"""Synthetic-event-shape contract test for scripts/reprocess.py (refactor §4.7).
|
|
|
|
Pins the exact S3 event shape reprocess emits and asserts the object key is
|
|
the RAW list_objects_v2 key — reprocess applies NO URL-encoding or decoding.
|
|
|
|
Why this matters: a real S3 event notification URL-encodes the object key, and
|
|
the handler is where any decode would live. reprocess builds its synthetic
|
|
event from the RAW list_objects_v2 obj["Key"] (or the raw --key arg), so it
|
|
must emit that key byte-for-byte. Replaying a pre-decoded (or pre-encoded) key
|
|
would not match what s3.get_object(Bucket, Key) expects downstream.
|
|
|
|
Recon note (honored narrowly): neither the PO nor the WO handler imports
|
|
urllib.parse or calls unquote/unquote_plus today — the raw event key goes
|
|
straight into s3.get_object. So the "handler decodes -> replaying a decoded
|
|
key double-decodes" failure mode is NOT reproducible against today's handlers
|
|
(they never decode once). This test therefore guards ONLY reprocess's own
|
|
contract: emit the raw list_objects_v2 key with no transformation. That keeps
|
|
replay single-decode-correct if a future handler ever adds unquote_plus.
|
|
"""
|
|
|
|
import importlib.util
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
# reprocess.py builds its boto3 clients inside main()/functions, so importing
|
|
# it is side-effect-free (no AWS calls, no clients at module scope). Set dummy
|
|
# region/creds defensively anyway so import stays offline-safe even if that
|
|
# ever changes. build_s3_event itself makes no AWS calls.
|
|
os.environ.setdefault("AWS_DEFAULT_REGION", "us-east-1")
|
|
os.environ.setdefault("AWS_ACCESS_KEY_ID", "testing")
|
|
os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "testing")
|
|
os.environ.setdefault("AWS_SESSION_TOKEN", "testing")
|
|
|
|
# scripts/ is not a package (no __init__.py) and is not on sys.path, and
|
|
# tests/conftest.py only wires up per-handler dirs — so load reprocess.py by
|
|
# file path with importlib rather than `import scripts.reprocess`.
|
|
_p = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "reprocess.py"
|
|
_spec = importlib.util.spec_from_file_location("reprocess", _p)
|
|
reprocess = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(reprocess)
|
|
|
|
|
|
def test_build_s3_event_shape_and_raw_key():
|
|
bucket = "po-ingest-emails-328440206208"
|
|
# Deliberately contains a SPACE, a literal '+', and a literal '%41'.
|
|
# A real S3 event notification would deliver this key encoded as
|
|
# "inbound/2026/AB+12%2B34+%2541.eml"
|
|
# (space->'+', '+'->'%2B', '%'->'%25'); the handler is where any decode
|
|
# would live. reprocess builds from the RAW list_objects_v2 key, so it
|
|
# must emit "inbound/2026/AB 12+34 %41.eml" unchanged.
|
|
raw_key = "inbound/2026/AB 12+34 %41.eml"
|
|
|
|
event = reprocess.build_s3_event(bucket, raw_key)
|
|
|
|
# 1. Exact top-level shape: a single Records entry.
|
|
assert list(event.keys()) == ["Records"]
|
|
assert len(event["Records"]) == 1
|
|
|
|
# 2. Bucket name is nested at Records[0].s3.bucket.name.
|
|
assert event["Records"][0]["s3"]["bucket"]["name"] == bucket
|
|
|
|
# 3. Object key is nested at Records[0].s3.object.key, byte-for-byte.
|
|
assert event["Records"][0]["s3"]["object"]["key"] == raw_key
|
|
|
|
# 4. RAW / no-URL-encoding, no-URL-decoding on the emitted key.
|
|
emitted = event["Records"][0]["s3"]["object"]["key"]
|
|
assert emitted == raw_key # reprocess applies NO transformation
|
|
# space NOT percent-encoded (a real S3 notification would send %20):
|
|
assert " " in emitted and "%20" not in emitted
|
|
# literal '+' preserved, not turned into a space:
|
|
assert "+" in emitted
|
|
# '%41' NOT decoded to 'A':
|
|
assert "%41" in emitted and "A .eml" not in emitted
|
|
|
|
|
|
# --- Selector mutual-exclusion: --all is a MODE, not a modifier ---------------
|
|
# reprocess.main() reaches its first AWS call (sts get_caller_identity) only
|
|
# AFTER the argparse mutual-exclusion gate, so an invalid selector combination
|
|
# raises SystemExit(2) with no AWS access needed. This pins that `--all` cannot
|
|
# be silently combined with a narrower --prefix/--since (which the args.all
|
|
# clobber branch would otherwise discard -> unintended whole-corpus sweep).
|
|
def _expect_argparse_reject(monkeypatch, capsys, argv):
|
|
monkeypatch.setattr(sys, "argv", ["reprocess.py", *argv])
|
|
with pytest.raises(SystemExit) as exc:
|
|
reprocess.main()
|
|
assert exc.value.code == 2 # argparse error exit code
|
|
return capsys.readouterr().err
|
|
|
|
|
|
def test_all_rejects_prefix(monkeypatch, capsys):
|
|
err = _expect_argparse_reject(
|
|
monkeypatch,
|
|
capsys,
|
|
["--pipeline", "po", "--all", "--prefix", "inbound/2026/07/"],
|
|
)
|
|
assert "--all cannot be combined with --prefix or --since" in err
|
|
|
|
|
|
def test_all_rejects_since(monkeypatch, capsys):
|
|
err = _expect_argparse_reject(
|
|
monkeypatch,
|
|
capsys,
|
|
["--pipeline", "po", "--all", "--since", "2026-07-01T00:00:00Z"],
|
|
)
|
|
assert "--all cannot be combined with --prefix or --since" in err
|
|
|
|
|
|
def test_all_rejects_prefix_and_since_together(monkeypatch, capsys):
|
|
_expect_argparse_reject(
|
|
monkeypatch,
|
|
capsys,
|
|
[
|
|
"--pipeline",
|
|
"po",
|
|
"--all",
|
|
"--prefix",
|
|
"inbound/2026/",
|
|
"--since",
|
|
"2026-07-01T00:00:00Z",
|
|
],
|
|
)
|