mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-09-30 18:53:14 +00:00
ec416079f5
Enable DynamoDB Streams on purchase-orders table and add a site-extractor Lambda that extracts Amazon facility codes and addresses from PO ship-to data, upserting them into a new verified-sites table. Includes a backfill script for existing POs and upgrades existing Lambdas to arm64 + 60-day log retention.
76 lines
1.9 KiB
Python
76 lines
1.9 KiB
Python
"""
|
|
One-time backfill script: scans purchase-orders and populates verified-sites.
|
|
|
|
Usage:
|
|
python scripts/backfill_sites.py
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lambdas", "site_extractor"))
|
|
|
|
import boto3
|
|
from handler import extract_site_code, parse_address, upsert_site
|
|
|
|
PO_TABLE = os.environ.get("PO_TABLE", "purchase-orders")
|
|
REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
|
|
|
|
dynamodb_resource = boto3.resource("dynamodb", region_name=REGION)
|
|
|
|
|
|
def scan_all_pos():
|
|
table = dynamodb_resource.Table(PO_TABLE)
|
|
records = []
|
|
last_key = None
|
|
|
|
while True:
|
|
kwargs = {}
|
|
if last_key:
|
|
kwargs["ExclusiveStartKey"] = last_key
|
|
response = table.scan(**kwargs)
|
|
records.extend(response.get("Items", []))
|
|
last_key = response.get("LastEvaluatedKey")
|
|
if not last_key:
|
|
break
|
|
print(f" Scanned {len(records)} POs so far...")
|
|
|
|
return records
|
|
|
|
|
|
def main():
|
|
print(f"Scanning {PO_TABLE} table...")
|
|
pos = scan_all_pos()
|
|
print(f"Found {len(pos)} purchase orders")
|
|
|
|
extracted = 0
|
|
skipped = 0
|
|
sites_seen = set()
|
|
|
|
for po in pos:
|
|
po_number = po.get("po_number", "unknown")
|
|
site_code = extract_site_code(po)
|
|
|
|
if not site_code:
|
|
skipped += 1
|
|
continue
|
|
|
|
address = parse_address(po)
|
|
location_code = (po.get("ship_to") or {}).get("location_code")
|
|
|
|
upsert_site(site_code, address, po_number, location_code)
|
|
extracted += 1
|
|
sites_seen.add(site_code)
|
|
|
|
if extracted % 100 == 0:
|
|
print(f" Processed {extracted} POs with site codes...")
|
|
|
|
print(f"\nBackfill complete:")
|
|
print(f" Total POs scanned: {len(pos)}")
|
|
print(f" POs with site code: {extracted}")
|
|
print(f" POs without site code: {skipped}")
|
|
print(f" Unique sites upserted: {len(sites_seen)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|