mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-09-30 23:33:13 +00:00
5112c1345b
* Merge workorder-ingest pipeline into unified repo Move PO lambdas under lambdas/po/, add WO pipeline under lambdas/wo/. Two independent CloudFormation stacks in one CDK app. Fix WO stack compliance: ARM64 architecture, 60-day log retention, aarch64 bundling, RETAIN on Anthropic secret. Remove stale CodePipeline buildspec. * Fix test_local.py import path and remove dead shared/models.py test_local.py referenced the old lambdas/email_processor path. Updated to lambdas/wo/email_processor. Removed shared/ directory entirely as nothing imports from it. * Escape HTML in both web UI dashboards to prevent XSS Both Function URLs are public (auth_type=NONE) and render email-derived content via f-strings. Attacker-crafted emails could inject scripts. Added html.escape() on all interpolated values in both PO and WO dashboards. * Add pagination to WO web UI scan get_work_orders() only fetched the first 1MB page from DynamoDB. Loop on LastEvaluatedKey to match the PO web UI pattern. * Fix esc(None) TypeError and javascript: scheme in PO web UI Coerce supplier name through `or ""` before escaping to handle nested None from DynamoDB. Add scheme allowlist on view_order_url to block javascript:/data: hrefs from LLM-extracted URLs. * Fix WO render_badge None guard, updated_at slice, and backfill path Add null guard to WO render_badge matching the PO version. Use `or ""` before slicing updated_at to handle explicit None values. Fix backfill_sites.py sys.path to use new lambdas/po/site_extractor. * Harden WO web UI and fix JS-context XSS in both dashboards - Use json.dumps for onclick URLs to prevent JS string breakout - Add .lower() to WO render_badge color lookup matching PO pattern - Add pagination to get_comments query - Cap get_work_orders to 500 results matching PO pattern * Apply ruff formatting to web UI handlers
76 lines
1.9 KiB
Python
76 lines
1.9 KiB
Python
"""
|
|
One-time backfill script: scans purchase-orders and populates verified-sites.
|
|
|
|
Usage:
|
|
python scripts/backfill_sites.py
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lambdas", "po", "site_extractor"))
|
|
|
|
import boto3
|
|
from handler import extract_site_code, parse_address, upsert_site
|
|
|
|
PO_TABLE = os.environ.get("PO_TABLE", "purchase-orders")
|
|
REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
|
|
|
|
dynamodb_resource = boto3.resource("dynamodb", region_name=REGION)
|
|
|
|
|
|
def scan_all_pos():
|
|
table = dynamodb_resource.Table(PO_TABLE)
|
|
records = []
|
|
last_key = None
|
|
|
|
while True:
|
|
kwargs = {}
|
|
if last_key:
|
|
kwargs["ExclusiveStartKey"] = last_key
|
|
response = table.scan(**kwargs)
|
|
records.extend(response.get("Items", []))
|
|
last_key = response.get("LastEvaluatedKey")
|
|
if not last_key:
|
|
break
|
|
print(f" Scanned {len(records)} POs so far...")
|
|
|
|
return records
|
|
|
|
|
|
def main():
|
|
print(f"Scanning {PO_TABLE} table...")
|
|
pos = scan_all_pos()
|
|
print(f"Found {len(pos)} purchase orders")
|
|
|
|
extracted = 0
|
|
skipped = 0
|
|
sites_seen = set()
|
|
|
|
for po in pos:
|
|
po_number = po.get("po_number", "unknown")
|
|
site_code = extract_site_code(po)
|
|
|
|
if not site_code:
|
|
skipped += 1
|
|
continue
|
|
|
|
address = parse_address(po)
|
|
location_code = (po.get("ship_to") or {}).get("location_code")
|
|
|
|
upsert_site(site_code, address, po_number, location_code)
|
|
extracted += 1
|
|
sites_seen.add(site_code)
|
|
|
|
if extracted % 100 == 0:
|
|
print(f" Processed {extracted} POs with site codes...")
|
|
|
|
print(f"\nBackfill complete:")
|
|
print(f" Total POs scanned: {len(pos)}")
|
|
print(f" POs with site code: {extracted}")
|
|
print(f" POs without site code: {skipped}")
|
|
print(f" Unique sites upserted: {len(sites_seen)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|