mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-10-03 20:03:13 +00:00
Add CI workflow (#18)
* Add CI workflow and apply ruff formatting * Disable cdk synth — email_processor uses pre-built package dir The email_processor Lambda bundles deps into a gitignored package/ directory. cdk synth fails in CI without a build step to recreate it. Disabling until packaging is standardized. * Use CDK BundlingOptions for email_processor Lambda packaging Replaces the pre-built gitignored package/ directory with CDK's built-in bundling. Deps are now installed inside a Docker container during cdk synth, so the build works identically locally and in CI. Re-enables run-cdk-synth in the CI workflow.
This commit is contained in:
parent
d1c1cb2f8b
commit
abdf2aa035
6 changed files with 84 additions and 32 deletions
12
.github/workflows/ci.yaml
vendored
Normal file
12
.github/workflows/ci.yaml
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
name: CI
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
ci:
|
||||||
|
uses: Sea-Haven-Industries/.github/.github/workflows/ci-python-sam.yaml@main
|
||||||
|
with:
|
||||||
|
source-dirs: "lambdas cdk"
|
||||||
|
run-cdk-synth: true
|
||||||
|
run-sam-validate: false
|
||||||
|
|
@ -3,7 +3,9 @@ import aws_cdk as cdk
|
||||||
from stack import PoIngestStack
|
from stack import PoIngestStack
|
||||||
|
|
||||||
app = cdk.App()
|
app = cdk.App()
|
||||||
PoIngestStack(app, "po-ingest",
|
PoIngestStack(
|
||||||
|
app,
|
||||||
|
"po-ingest",
|
||||||
stack_name="po-ingest",
|
stack_name="po-ingest",
|
||||||
env=cdk.Environment(region="us-east-1"),
|
env=cdk.Environment(region="us-east-1"),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
49
cdk/stack.py
49
cdk/stack.py
|
|
@ -14,7 +14,6 @@ from aws_cdk import (
|
||||||
aws_ses as ses,
|
aws_ses as ses,
|
||||||
aws_ses_actions as ses_actions,
|
aws_ses_actions as ses_actions,
|
||||||
aws_secretsmanager as secretsmanager,
|
aws_secretsmanager as secretsmanager,
|
||||||
aws_iam as iam,
|
|
||||||
)
|
)
|
||||||
from constructs import Construct
|
from constructs import Construct
|
||||||
|
|
||||||
|
|
@ -25,7 +24,8 @@ class PoIngestStack(Stack):
|
||||||
|
|
||||||
# --- S3 bucket for raw emails ---
|
# --- S3 bucket for raw emails ---
|
||||||
email_bucket = s3.Bucket(
|
email_bucket = s3.Bucket(
|
||||||
self, "EmailBucket",
|
self,
|
||||||
|
"EmailBucket",
|
||||||
bucket_name=f"po-ingest-emails-{self.account}",
|
bucket_name=f"po-ingest-emails-{self.account}",
|
||||||
removal_policy=RemovalPolicy.RETAIN,
|
removal_policy=RemovalPolicy.RETAIN,
|
||||||
lifecycle_rules=[
|
lifecycle_rules=[
|
||||||
|
|
@ -37,7 +37,8 @@ class PoIngestStack(Stack):
|
||||||
# Owned by this stack. Streams enabled for the site-extractor pipeline.
|
# Owned by this stack. Streams enabled for the site-extractor pipeline.
|
||||||
# Other stacks (seahaven-slack-bot) reference this table via fromTableName().
|
# Other stacks (seahaven-slack-bot) reference this table via fromTableName().
|
||||||
po_table = dynamodb.Table(
|
po_table = dynamodb.Table(
|
||||||
self, "PurchaseOrdersTable",
|
self,
|
||||||
|
"PurchaseOrdersTable",
|
||||||
table_name="purchase-orders",
|
table_name="purchase-orders",
|
||||||
partition_key=dynamodb.Attribute(
|
partition_key=dynamodb.Attribute(
|
||||||
name="po_number",
|
name="po_number",
|
||||||
|
|
@ -50,7 +51,8 @@ class PoIngestStack(Stack):
|
||||||
|
|
||||||
# --- Secrets Manager for Anthropic API key ---
|
# --- Secrets Manager for Anthropic API key ---
|
||||||
anthropic_secret = secretsmanager.Secret(
|
anthropic_secret = secretsmanager.Secret(
|
||||||
self, "AnthropicApiKey",
|
self,
|
||||||
|
"AnthropicApiKey",
|
||||||
secret_name="po-ingest/anthropic-api-key",
|
secret_name="po-ingest/anthropic-api-key",
|
||||||
description="Anthropic API key for Coupa PO email parsing",
|
description="Anthropic API key for Coupa PO email parsing",
|
||||||
removal_policy=RemovalPolicy.RETAIN,
|
removal_policy=RemovalPolicy.RETAIN,
|
||||||
|
|
@ -58,12 +60,23 @@ class PoIngestStack(Stack):
|
||||||
|
|
||||||
# --- Lambda function ---
|
# --- Lambda function ---
|
||||||
email_processor = lambda_.Function(
|
email_processor = lambda_.Function(
|
||||||
self, "EmailProcessor",
|
self,
|
||||||
|
"EmailProcessor",
|
||||||
function_name="po-email-processor",
|
function_name="po-email-processor",
|
||||||
runtime=lambda_.Runtime.PYTHON_3_12,
|
runtime=lambda_.Runtime.PYTHON_3_12,
|
||||||
architecture=lambda_.Architecture.ARM_64,
|
architecture=lambda_.Architecture.ARM_64,
|
||||||
handler="handler.handler",
|
handler="handler.handler",
|
||||||
code=lambda_.Code.from_asset("../lambdas/email_processor/package"),
|
code=lambda_.Code.from_asset(
|
||||||
|
"../lambdas/email_processor",
|
||||||
|
bundling=cdk.BundlingOptions(
|
||||||
|
image=lambda_.Runtime.PYTHON_3_12.bundling_image,
|
||||||
|
command=[
|
||||||
|
"bash",
|
||||||
|
"-c",
|
||||||
|
"pip install -r requirements.txt -t /asset-output && cp handler.py /asset-output/",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
timeout=Duration.seconds(60),
|
timeout=Duration.seconds(60),
|
||||||
memory_size=256,
|
memory_size=256,
|
||||||
log_retention=logs.RetentionDays.TWO_MONTHS,
|
log_retention=logs.RetentionDays.TWO_MONTHS,
|
||||||
|
|
@ -88,7 +101,9 @@ class PoIngestStack(Stack):
|
||||||
# --- SES Receipt Rule ---
|
# --- SES Receipt Rule ---
|
||||||
# Reuse the existing INBOUND_MAIL rule set (shared with workorder-ingest)
|
# Reuse the existing INBOUND_MAIL rule set (shared with workorder-ingest)
|
||||||
rule_set = ses.ReceiptRuleSet.from_receipt_rule_set_name(
|
rule_set = ses.ReceiptRuleSet.from_receipt_rule_set_name(
|
||||||
self, "ExistingRuleSet", "INBOUND_MAIL",
|
self,
|
||||||
|
"ExistingRuleSet",
|
||||||
|
"INBOUND_MAIL",
|
||||||
)
|
)
|
||||||
|
|
||||||
rule_set.add_rule(
|
rule_set.add_rule(
|
||||||
|
|
@ -104,7 +119,8 @@ class PoIngestStack(Stack):
|
||||||
|
|
||||||
# --- Web UI Lambda ---
|
# --- Web UI Lambda ---
|
||||||
web_ui = lambda_.Function(
|
web_ui = lambda_.Function(
|
||||||
self, "WebUI",
|
self,
|
||||||
|
"WebUI",
|
||||||
function_name="po-web-ui",
|
function_name="po-web-ui",
|
||||||
runtime=lambda_.Runtime.PYTHON_3_12,
|
runtime=lambda_.Runtime.PYTHON_3_12,
|
||||||
architecture=lambda_.Architecture.ARM_64,
|
architecture=lambda_.Architecture.ARM_64,
|
||||||
|
|
@ -125,11 +141,14 @@ class PoIngestStack(Stack):
|
||||||
auth_type=lambda_.FunctionUrlAuthType.NONE,
|
auth_type=lambda_.FunctionUrlAuthType.NONE,
|
||||||
)
|
)
|
||||||
|
|
||||||
cdk.CfnOutput(self, "WebUIUrl", value=web_url.url, description="PO Dashboard URL")
|
cdk.CfnOutput(
|
||||||
|
self, "WebUIUrl", value=web_url.url, description="PO Dashboard URL"
|
||||||
|
)
|
||||||
|
|
||||||
# --- Verified sites table (extracted from PO ship-to addresses) ---
|
# --- Verified sites table (extracted from PO ship-to addresses) ---
|
||||||
verified_sites_table = dynamodb.Table(
|
verified_sites_table = dynamodb.Table(
|
||||||
self, "VerifiedSitesTable",
|
self,
|
||||||
|
"VerifiedSitesTable",
|
||||||
table_name="verified-sites",
|
table_name="verified-sites",
|
||||||
partition_key=dynamodb.Attribute(
|
partition_key=dynamodb.Attribute(
|
||||||
name="siteCode",
|
name="siteCode",
|
||||||
|
|
@ -149,7 +168,8 @@ class PoIngestStack(Stack):
|
||||||
|
|
||||||
# --- Site extractor Lambda (DynamoDB Streams → verified-sites) ---
|
# --- Site extractor Lambda (DynamoDB Streams → verified-sites) ---
|
||||||
site_extractor = lambda_.Function(
|
site_extractor = lambda_.Function(
|
||||||
self, "SiteExtractor",
|
self,
|
||||||
|
"SiteExtractor",
|
||||||
function_name="po-ingest-site-extractor",
|
function_name="po-ingest-site-extractor",
|
||||||
runtime=lambda_.Runtime.PYTHON_3_12,
|
runtime=lambda_.Runtime.PYTHON_3_12,
|
||||||
architecture=lambda_.Architecture.ARM_64,
|
architecture=lambda_.Architecture.ARM_64,
|
||||||
|
|
@ -177,14 +197,17 @@ class PoIngestStack(Stack):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
cdk.CfnOutput(self, "VerifiedSitesTableName",
|
cdk.CfnOutput(
|
||||||
|
self,
|
||||||
|
"VerifiedSitesTableName",
|
||||||
value=verified_sites_table.table_name,
|
value=verified_sites_table.table_name,
|
||||||
description="Verified site addresses extracted from POs",
|
description="Verified site addresses extracted from POs",
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- Pending site review table (POs with no extractable site code) ---
|
# --- Pending site review table (POs with no extractable site code) ---
|
||||||
pending_review_table = dynamodb.Table(
|
pending_review_table = dynamodb.Table(
|
||||||
self, "PendingSiteReviewTable",
|
self,
|
||||||
|
"PendingSiteReviewTable",
|
||||||
table_name="pending-site-review",
|
table_name="pending-site-review",
|
||||||
partition_key=dynamodb.Attribute(
|
partition_key=dynamodb.Attribute(
|
||||||
name="po_number",
|
name="po_number",
|
||||||
|
|
|
||||||
|
|
@ -290,7 +290,7 @@ def pad_zip(zip_code: str | None) -> str | None:
|
||||||
return zip_code
|
return zip_code
|
||||||
clean = zip_code.strip().split("-")[0]
|
clean = zip_code.strip().split("-")[0]
|
||||||
if clean.isdigit() and len(clean) < 5:
|
if clean.isdigit() and len(clean) < 5:
|
||||||
return clean.zfill(5) + zip_code.strip()[len(clean):]
|
return clean.zfill(5) + zip_code.strip()[len(clean) :]
|
||||||
return zip_code
|
return zip_code
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -369,7 +369,9 @@ def handler(event, context):
|
||||||
logger.info(f"Subject: {email_data['subject']}")
|
logger.info(f"Subject: {email_data['subject']}")
|
||||||
|
|
||||||
parsed = extract_with_claude(email_data)
|
parsed = extract_with_claude(email_data)
|
||||||
logger.info(f"Parsed: type={parsed.get('email_type')}, po={parsed.get('po_number')}")
|
logger.info(
|
||||||
|
f"Parsed: type={parsed.get('email_type')}, po={parsed.get('po_number')}"
|
||||||
|
)
|
||||||
|
|
||||||
if not parsed.get("po_number"):
|
if not parsed.get("po_number"):
|
||||||
logger.warning(f"No PO number found in email, skipping: {key}")
|
logger.warning(f"No PO number found in email, skipping: {key}")
|
||||||
|
|
|
||||||
|
|
@ -49,9 +49,7 @@ def extract_site_code(record: dict) -> str | None:
|
||||||
if m and SITE_CODE_PATTERN.match(m.group(1)):
|
if m and SITE_CODE_PATTERN.match(m.group(1)):
|
||||||
return m.group(1)
|
return m.group(1)
|
||||||
|
|
||||||
m = re.search(
|
m = re.search(r"(?:Station|DS)\s*[-–]?\s*([A-Z0-9]{3,5})\b", ship_to_name)
|
||||||
r"(?:Station|DS)\s*[-–]?\s*([A-Z0-9]{3,5})\b", ship_to_name
|
|
||||||
)
|
|
||||||
if m and SITE_CODE_PATTERN.match(m.group(1)):
|
if m and SITE_CODE_PATTERN.match(m.group(1)):
|
||||||
return m.group(1)
|
return m.group(1)
|
||||||
|
|
||||||
|
|
@ -85,7 +83,11 @@ def parse_address(record: dict) -> dict:
|
||||||
result["city"] = ship_to.get("city")
|
result["city"] = ship_to.get("city")
|
||||||
result["state"] = ship_to["state"]
|
result["state"] = ship_to["state"]
|
||||||
result["zip"] = ship_to.get("zip")
|
result["zip"] = ship_to.get("zip")
|
||||||
parts = [p for p in [result["address"], result["city"], result["state"], result["zip"]] if p]
|
parts = [
|
||||||
|
p
|
||||||
|
for p in [result["address"], result["city"], result["state"], result["zip"]]
|
||||||
|
if p
|
||||||
|
]
|
||||||
result["fullAddress"] = ", ".join(parts)
|
result["fullAddress"] = ", ".join(parts)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
@ -104,11 +106,13 @@ def parse_address(record: dict) -> dict:
|
||||||
result["city"] = m.group(1).strip()
|
result["city"] = m.group(1).strip()
|
||||||
result["state"] = m.group(2)
|
result["state"] = m.group(2)
|
||||||
result["zip"] = m.group(3)
|
result["zip"] = m.group(3)
|
||||||
result["fullAddress"] = f"{result['address']}, {result['city']}, {result['state']} {result['zip']}"
|
result["fullAddress"] = (
|
||||||
|
f"{result['address']}, {result['city']}, {result['state']} {result['zip']}"
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Fallback: try newline-separated format
|
# Fallback: try newline-separated format
|
||||||
lines = [l.strip() for l in clean.split("\n") if l.strip()]
|
lines = [line.strip() for line in clean.split("\n") if line.strip()]
|
||||||
city_re = re.compile(r"^(.+?),\s+([A-Z]{2})\s+(\d{5}(?:-\d{4})?)")
|
city_re = re.compile(r"^(.+?),\s+([A-Z]{2})\s+(\d{5}(?:-\d{4})?)")
|
||||||
for i, line in enumerate(lines):
|
for i, line in enumerate(lines):
|
||||||
m = city_re.match(line)
|
m = city_re.match(line)
|
||||||
|
|
@ -118,7 +122,9 @@ def parse_address(record: dict) -> dict:
|
||||||
result["state"] = m.group(2)
|
result["state"] = m.group(2)
|
||||||
result["zip"] = m.group(3)
|
result["zip"] = m.group(3)
|
||||||
street = result["address"] or result["city"]
|
street = result["address"] or result["city"]
|
||||||
result["fullAddress"] = f"{street}, {result['city']}, {result['state']} {result['zip']}"
|
result["fullAddress"] = (
|
||||||
|
f"{street}, {result['city']}, {result['state']} {result['zip']}"
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
result["address"] = clean
|
result["address"] = clean
|
||||||
|
|
@ -278,7 +284,10 @@ def handler(event, context):
|
||||||
ship_to_name = (po.get("ship_to") or {}).get("name")
|
ship_to_name = (po.get("ship_to") or {}).get("name")
|
||||||
write_pending_review(po_number, address, ship_to_name)
|
write_pending_review(po_number, address, ship_to_name)
|
||||||
pending += 1
|
pending += 1
|
||||||
logger.info("PO %s added to pending review (no site code or address match)", po_number)
|
logger.info(
|
||||||
|
"PO %s added to pending review (no site code or address match)",
|
||||||
|
po_number,
|
||||||
|
)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Batch complete: %d processed, %d extracted (%d via address), %d pending review",
|
"Batch complete: %d processed, %d extracted (%d via address), %d pending review",
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ Serves a simple HTML dashboard for viewing purchase orders.
|
||||||
Accessed via Lambda Function URL.
|
Accessed via Lambda Function URL.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
|
||||||
|
|
@ -103,7 +102,12 @@ def render_po_detail(po):
|
||||||
|
|
||||||
view_url = po.get("view_order_url")
|
view_url = po.get("view_order_url")
|
||||||
if view_url:
|
if view_url:
|
||||||
fields.append(("Coupa Link", f'<a href="{view_url}" target="_blank" style="color:#3b82f6;">View in Coupa</a>'))
|
fields.append(
|
||||||
|
(
|
||||||
|
"Coupa Link",
|
||||||
|
f'<a href="{view_url}" target="_blank" style="color:#3b82f6;">View in Coupa</a>',
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
details_html = ""
|
details_html = ""
|
||||||
for label, value in fields:
|
for label, value in fields:
|
||||||
|
|
@ -120,17 +124,17 @@ def render_po_detail(po):
|
||||||
if line_items:
|
if line_items:
|
||||||
rows = ""
|
rows = ""
|
||||||
for item in line_items:
|
for item in line_items:
|
||||||
qty = item.get('quantity', '') or ''
|
qty = item.get("quantity", "") or ""
|
||||||
unit = item.get('unit', '') or ''
|
unit = item.get("unit", "") or ""
|
||||||
price = item.get('price', '') or ''
|
price = item.get("price", "") or ""
|
||||||
rows += f"""
|
rows += f"""
|
||||||
<tr style="border-bottom:1px solid #f1f5f9;">
|
<tr style="border-bottom:1px solid #f1f5f9;">
|
||||||
<td style="padding:10px;font-size:14px;">{item.get('description', '')}</td>
|
<td style="padding:10px;font-size:14px;">{item.get("description", "")}</td>
|
||||||
<td style="padding:10px;font-size:13px;text-align:right;">{qty}</td>
|
<td style="padding:10px;font-size:13px;text-align:right;">{qty}</td>
|
||||||
<td style="padding:10px;font-size:13px;">{unit}</td>
|
<td style="padding:10px;font-size:13px;">{unit}</td>
|
||||||
<td style="padding:10px;font-size:13px;text-align:right;">{price}</td>
|
<td style="padding:10px;font-size:13px;text-align:right;">{price}</td>
|
||||||
<td style="padding:10px;font-size:14px;text-align:right;">{fmt_currency(item.get('amount'))}</td>
|
<td style="padding:10px;font-size:14px;text-align:right;">{fmt_currency(item.get("amount"))}</td>
|
||||||
<td style="padding:10px;font-size:13px;color:#64748b;">{item.get('need_by', '') or ''}</td>
|
<td style="padding:10px;font-size:13px;color:#64748b;">{item.get("need_by", "") or ""}</td>
|
||||||
</tr>"""
|
</tr>"""
|
||||||
|
|
||||||
items_html = f"""
|
items_html = f"""
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue