procurement-ingest/scripts/backfill_sites.py

"""
One-time backfill script: scans purchase-orders and populates verified-sites.

Usage:
    python scripts/backfill_sites.py
"""

import sys
import os

sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lambdas", "site_extractor"))

import boto3
from handler import extract_site_code, parse_address, upsert_site

PO_TABLE = os.environ.get("PO_TABLE", "purchase-orders")
REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")

dynamodb_resource = boto3.resource("dynamodb", region_name=REGION)


def scan_all_pos():
    table = dynamodb_resource.Table(PO_TABLE)
    records = []
    last_key = None

    while True:
        kwargs = {}
        if last_key:
            kwargs["ExclusiveStartKey"] = last_key
        response = table.scan(**kwargs)
        records.extend(response.get("Items", []))
        last_key = response.get("LastEvaluatedKey")
        if not last_key:
            break
        print(f"  Scanned {len(records)} POs so far...")

    return records


def main():
    print(f"Scanning {PO_TABLE} table...")
    pos = scan_all_pos()
    print(f"Found {len(pos)} purchase orders")

    extracted = 0
    skipped = 0
    sites_seen = set()

    for po in pos:
        po_number = po.get("po_number", "unknown")
        site_code = extract_site_code(po)

        if not site_code:
            skipped += 1
            continue

        address = parse_address(po)
        location_code = (po.get("ship_to") or {}).get("location_code")

        upsert_site(site_code, address, po_number, location_code)
        extracted += 1
        sites_seen.add(site_code)

        if extracted % 100 == 0:
            print(f"  Processed {extracted} POs with site codes...")

    print(f"\nBackfill complete:")
    print(f"  Total POs scanned: {len(pos)}")
    print(f"  POs with site code: {extracted}")
    print(f"  POs without site code: {skipped}")
    print(f"  Unique sites upserted: {len(sites_seen)}")


if __name__ == "__main__":
    main()