mirror of
https://github.com/Sea-Haven-Industries/proposal-system.git
synced 2026-10-03 15:03:17 +00:00
fix(lambdas): LAM-M2, M3, M6, M9 — prompt injection, PDF size check, numeric validation, API key TTL
LAM-M2: Add sanitize_user_text() to suggestions Lambda that strips common prompt injection patterns (blocklist + delimiter neutralisation) before including user-supplied text in Bedrock prompts. LAM-M3: Add file size check in pdf-extract before downloading — rejects PDFs over 50 MB with a logged warning and ValueError. LAM-M6: Add validate_line_item_numerics() to suggestions Lambda that rejects Bedrock-generated line items with negative values, NaN/Inf, or amounts exceeding $10M ceiling. LAM-M9: Replace indefinite API key cache with 5-minute TTL in all four Lambdas (suggestions, pdf-extract, pdf-generate, library-ingest) so rotated Secrets Manager values take effect promptly.
This commit is contained in:
parent
8da87e9301
commit
669e9c0e43
4 changed files with 202 additions and 33 deletions
|
|
@ -32,16 +32,26 @@ bedrock_agent = boto3.client("bedrock-agent")
|
||||||
secrets_client = boto3.client("secretsmanager")
|
secrets_client = boto3.client("secretsmanager")
|
||||||
|
|
||||||
_cached_api_key: str | None = None
|
_cached_api_key: str | None = None
|
||||||
|
_cached_api_key_ts: float = 0.0
|
||||||
|
_API_KEY_TTL_SECONDS = 300 # Fix: LAM-M9 — re-fetch every 5 minutes
|
||||||
|
|
||||||
|
|
||||||
def _get_api_key() -> str:
|
def _get_api_key() -> str:
|
||||||
global _cached_api_key
|
"""Fetch internal API key from Secrets Manager with a 5-minute TTL cache.
|
||||||
if _cached_api_key is None:
|
|
||||||
if INTERNAL_API_KEY_SECRET_ARN:
|
Fix: LAM-M9 — the key was previously cached indefinitely. Now the cached
|
||||||
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
value expires after _API_KEY_TTL_SECONDS so rotated secrets take effect.
|
||||||
_cached_api_key = resp["SecretString"]
|
"""
|
||||||
else:
|
global _cached_api_key, _cached_api_key_ts
|
||||||
_cached_api_key = ""
|
now = time.monotonic()
|
||||||
|
if _cached_api_key is not None and (now - _cached_api_key_ts) < _API_KEY_TTL_SECONDS:
|
||||||
|
return _cached_api_key
|
||||||
|
if INTERNAL_API_KEY_SECRET_ARN:
|
||||||
|
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
||||||
|
_cached_api_key = resp["SecretString"]
|
||||||
|
else:
|
||||||
|
_cached_api_key = ""
|
||||||
|
_cached_api_key_ts = now
|
||||||
return _cached_api_key
|
return _cached_api_key
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,16 +33,29 @@ bedrock_runtime = boto3.client("bedrock-runtime")
|
||||||
secrets_client = boto3.client("secretsmanager")
|
secrets_client = boto3.client("secretsmanager")
|
||||||
|
|
||||||
_cached_api_key: str | None = None
|
_cached_api_key: str | None = None
|
||||||
|
_cached_api_key_ts: float = 0.0
|
||||||
|
_API_KEY_TTL_SECONDS = 300 # Fix: LAM-M9 — re-fetch every 5 minutes
|
||||||
|
|
||||||
|
# Fix: LAM-M3 — maximum PDF file size (50 MB)
|
||||||
|
_MAX_PDF_SIZE_BYTES = 50 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
def _get_api_key() -> str:
|
def _get_api_key() -> str:
|
||||||
global _cached_api_key
|
"""Fetch internal API key from Secrets Manager with a 5-minute TTL cache.
|
||||||
if _cached_api_key is None:
|
|
||||||
if INTERNAL_API_KEY_SECRET_ARN:
|
Fix: LAM-M9 — the key was previously cached indefinitely. Now the cached
|
||||||
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
value expires after _API_KEY_TTL_SECONDS so rotated secrets take effect.
|
||||||
_cached_api_key = resp["SecretString"]
|
"""
|
||||||
else:
|
global _cached_api_key, _cached_api_key_ts
|
||||||
_cached_api_key = ""
|
now = time.monotonic()
|
||||||
|
if _cached_api_key is not None and (now - _cached_api_key_ts) < _API_KEY_TTL_SECONDS:
|
||||||
|
return _cached_api_key
|
||||||
|
if INTERNAL_API_KEY_SECRET_ARN:
|
||||||
|
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
||||||
|
_cached_api_key = resp["SecretString"]
|
||||||
|
else:
|
||||||
|
_cached_api_key = ""
|
||||||
|
_cached_api_key_ts = now
|
||||||
return _cached_api_key
|
return _cached_api_key
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -139,6 +152,22 @@ def process_pdf(proposal_id: str, s3_key: str, vendor_proposal_id: str):
|
||||||
|
|
||||||
|
|
||||||
def download_pdf(s3_key: str) -> str:
|
def download_pdf(s3_key: str) -> str:
|
||||||
|
# Fix: LAM-M3 — check file size before downloading to avoid processing
|
||||||
|
# excessively large PDFs that could exhaust Lambda memory/tmp storage.
|
||||||
|
head = s3.head_object(Bucket=UPLOADS_BUCKET, Key=s3_key)
|
||||||
|
file_size = head.get("ContentLength", 0)
|
||||||
|
if file_size > _MAX_PDF_SIZE_BYTES:
|
||||||
|
logger.warning(
|
||||||
|
"Fix: LAM-M3 — PDF too large: key=%s size=%d bytes (max=%d)",
|
||||||
|
s3_key,
|
||||||
|
file_size,
|
||||||
|
_MAX_PDF_SIZE_BYTES,
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
f"PDF file size ({file_size} bytes) exceeds maximum "
|
||||||
|
f"allowed size ({_MAX_PDF_SIZE_BYTES} bytes) for key: {s3_key}"
|
||||||
|
)
|
||||||
|
|
||||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
|
||||||
s3.download_file(UPLOADS_BUCKET, s3_key, tmp.name)
|
s3.download_file(UPLOADS_BUCKET, s3_key, tmp.name)
|
||||||
tmp.close()
|
tmp.close()
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,8 @@ s3 = boto3.client("s3")
|
||||||
secrets_client = boto3.client("secretsmanager")
|
secrets_client = boto3.client("secretsmanager")
|
||||||
|
|
||||||
_cached_api_key: str | None = None
|
_cached_api_key: str | None = None
|
||||||
|
_cached_api_key_ts: float = 0.0
|
||||||
|
_API_KEY_TTL_SECONDS = 300 # Fix: LAM-M9 — re-fetch every 5 minutes
|
||||||
|
|
||||||
# Fix: LAM-M8 — pattern for allowed S3 key characters
|
# Fix: LAM-M8 — pattern for allowed S3 key characters
|
||||||
_SAFE_S3_KEY_RE = re.compile(r"^[a-zA-Z0-9\-_./\s]+$")
|
_SAFE_S3_KEY_RE = re.compile(r"^[a-zA-Z0-9\-_./\s]+$")
|
||||||
|
|
@ -59,13 +61,21 @@ TERMS_AND_CONDITIONS = """
|
||||||
|
|
||||||
|
|
||||||
def _get_api_key() -> str:
|
def _get_api_key() -> str:
|
||||||
global _cached_api_key
|
"""Fetch internal API key from Secrets Manager with a 5-minute TTL cache.
|
||||||
if _cached_api_key is None:
|
|
||||||
if INTERNAL_API_KEY_SECRET_ARN:
|
Fix: LAM-M9 — the key was previously cached indefinitely. Now the cached
|
||||||
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
value expires after _API_KEY_TTL_SECONDS so rotated secrets take effect.
|
||||||
_cached_api_key = resp["SecretString"]
|
"""
|
||||||
else:
|
global _cached_api_key, _cached_api_key_ts
|
||||||
_cached_api_key = ""
|
now = time.monotonic()
|
||||||
|
if _cached_api_key is not None and (now - _cached_api_key_ts) < _API_KEY_TTL_SECONDS:
|
||||||
|
return _cached_api_key
|
||||||
|
if INTERNAL_API_KEY_SECRET_ARN:
|
||||||
|
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
||||||
|
_cached_api_key = resp["SecretString"]
|
||||||
|
else:
|
||||||
|
_cached_api_key = ""
|
||||||
|
_cached_api_key_ts = now
|
||||||
return _cached_api_key
|
return _cached_api_key
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,9 @@ to generate line item suggestions for new proposals.
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
|
|
@ -25,19 +27,129 @@ bedrock_runtime = boto3.client("bedrock-runtime")
|
||||||
secrets_client = boto3.client("secretsmanager")
|
secrets_client = boto3.client("secretsmanager")
|
||||||
|
|
||||||
_cached_api_key: str | None = None
|
_cached_api_key: str | None = None
|
||||||
|
_cached_api_key_ts: float = 0.0
|
||||||
|
_API_KEY_TTL_SECONDS = 300 # Fix: LAM-M9 — re-fetch every 5 minutes
|
||||||
|
|
||||||
|
|
||||||
def _get_api_key() -> str:
|
def _get_api_key() -> str:
|
||||||
global _cached_api_key
|
"""Fetch internal API key from Secrets Manager with a 5-minute TTL cache.
|
||||||
if _cached_api_key is None:
|
|
||||||
if INTERNAL_API_KEY_SECRET_ARN:
|
Fix: LAM-M9 — the key was previously cached indefinitely. Now the cached
|
||||||
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
value expires after _API_KEY_TTL_SECONDS so rotated secrets take effect.
|
||||||
_cached_api_key = resp["SecretString"]
|
"""
|
||||||
else:
|
global _cached_api_key, _cached_api_key_ts
|
||||||
_cached_api_key = ""
|
now = time.monotonic()
|
||||||
|
if _cached_api_key is not None and (now - _cached_api_key_ts) < _API_KEY_TTL_SECONDS:
|
||||||
|
return _cached_api_key
|
||||||
|
if INTERNAL_API_KEY_SECRET_ARN:
|
||||||
|
resp = secrets_client.get_secret_value(SecretId=INTERNAL_API_KEY_SECRET_ARN)
|
||||||
|
_cached_api_key = resp["SecretString"]
|
||||||
|
else:
|
||||||
|
_cached_api_key = ""
|
||||||
|
_cached_api_key_ts = now
|
||||||
return _cached_api_key
|
return _cached_api_key
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fix: LAM-M2 — Prompt injection mitigation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Blocklist patterns that commonly appear in prompt injection attempts.
|
||||||
|
# This is a simple defense-in-depth measure, not a full NLP solution.
|
||||||
|
_INJECTION_PATTERNS = [
|
||||||
|
re.compile(r"ignore\s+(all\s+)?previous\s+instructions", re.IGNORECASE),
|
||||||
|
re.compile(r"ignore\s+(all\s+)?above\s+instructions", re.IGNORECASE),
|
||||||
|
re.compile(r"disregard\s+(all\s+)?previous", re.IGNORECASE),
|
||||||
|
re.compile(r"override\s+(all\s+)?instructions", re.IGNORECASE),
|
||||||
|
re.compile(r"you\s+are\s+now\s+(a|an)\s+", re.IGNORECASE),
|
||||||
|
re.compile(r"new\s+instructions?\s*:", re.IGNORECASE),
|
||||||
|
re.compile(r"system\s*:", re.IGNORECASE),
|
||||||
|
re.compile(r"<\|?\s*(system|im_start|endoftext)\s*\|?>", re.IGNORECASE),
|
||||||
|
re.compile(r"\[INST\]", re.IGNORECASE),
|
||||||
|
re.compile(r"```\s*(system|instruction)", re.IGNORECASE),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Delimiter sequences that could be used to break out of the user-text section
|
||||||
|
_INJECTION_DELIMITERS = ["```", "---\n", "===\n", "***\n"]
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize_user_text(text: str) -> str:
|
||||||
|
"""Strip common prompt injection patterns from user-supplied text.
|
||||||
|
|
||||||
|
Fix: LAM-M2 — basic prompt injection mitigation before including user
|
||||||
|
text in Bedrock prompts. Applies a blocklist of known injection patterns
|
||||||
|
and neutralises delimiter sequences.
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return text
|
||||||
|
|
||||||
|
sanitized = text
|
||||||
|
|
||||||
|
# Remove blocklisted patterns
|
||||||
|
for pattern in _INJECTION_PATTERNS:
|
||||||
|
sanitized = pattern.sub("[removed]", sanitized)
|
||||||
|
|
||||||
|
# Neutralise delimiter sequences by replacing them with a safe alternative
|
||||||
|
for delim in _INJECTION_DELIMITERS:
|
||||||
|
sanitized = sanitized.replace(delim, " ")
|
||||||
|
|
||||||
|
return sanitized
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fix: LAM-M6 — Numeric validation for suggestion amounts
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
_MAX_LINE_ITEM_VALUE = 10_000_000 # $10M ceiling for any single value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_line_item_numerics(items: list[dict]) -> list[dict]:
|
||||||
|
"""Validate and filter line items with unreasonable numeric values.
|
||||||
|
|
||||||
|
Fix: LAM-M6 — after getting suggestions from Bedrock, reject items with
|
||||||
|
negative values, NaN, or values exceeding the $10M ceiling.
|
||||||
|
"""
|
||||||
|
validated = []
|
||||||
|
for item in items:
|
||||||
|
rejected = False
|
||||||
|
for field in ("quantity", "unitPrice", "totalPrice"):
|
||||||
|
value = item.get(field)
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
num = float(value)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
logger.warning(
|
||||||
|
"Fix: LAM-M6 — rejected line item: %s is not a valid number (%r) "
|
||||||
|
"in item %r",
|
||||||
|
field, value, item.get("description", "unknown"),
|
||||||
|
)
|
||||||
|
rejected = True
|
||||||
|
break
|
||||||
|
if math.isnan(num) or math.isinf(num):
|
||||||
|
logger.warning(
|
||||||
|
"Fix: LAM-M6 — rejected line item: %s is NaN/Inf in item %r",
|
||||||
|
field, item.get("description", "unknown"),
|
||||||
|
)
|
||||||
|
rejected = True
|
||||||
|
break
|
||||||
|
if num < 0:
|
||||||
|
logger.warning(
|
||||||
|
"Fix: LAM-M6 — rejected line item: %s is negative (%.2f) in item %r",
|
||||||
|
field, num, item.get("description", "unknown"),
|
||||||
|
)
|
||||||
|
rejected = True
|
||||||
|
break
|
||||||
|
if num > _MAX_LINE_ITEM_VALUE:
|
||||||
|
logger.warning(
|
||||||
|
"Fix: LAM-M6 — rejected line item: %s exceeds max (%.2f > %d) in item %r",
|
||||||
|
field, num, _MAX_LINE_ITEM_VALUE, item.get("description", "unknown"),
|
||||||
|
)
|
||||||
|
rejected = True
|
||||||
|
break
|
||||||
|
if not rejected:
|
||||||
|
validated.append(item)
|
||||||
|
return validated
|
||||||
|
|
||||||
|
|
||||||
def handler(event, context):
|
def handler(event, context):
|
||||||
batch_item_failures = []
|
batch_item_failures = []
|
||||||
|
|
||||||
|
|
@ -99,7 +211,7 @@ def process_suggestion(proposal_id: str, trigger: str):
|
||||||
logger.info("Proposal %s is in status %s, skipping suggestions", proposal_id, status)
|
logger.info("Proposal %s is in status %s, skipping suggestions", proposal_id, status)
|
||||||
return
|
return
|
||||||
|
|
||||||
similar_proposals = retrieve_similar(scope, category)
|
similar_proposals = retrieve_similar(sanitize_user_text(scope), category)
|
||||||
|
|
||||||
suggested_items = generate_line_items(scope, category, priority, similar_proposals)
|
suggested_items = generate_line_items(scope, category, priority, similar_proposals)
|
||||||
|
|
||||||
|
|
@ -204,6 +316,11 @@ def generate_line_items(
|
||||||
priority: str,
|
priority: str,
|
||||||
similar_proposals: list[dict],
|
similar_proposals: list[dict],
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
|
# Fix: LAM-M2 — sanitize user-supplied text before including in prompt
|
||||||
|
safe_scope = sanitize_user_text(scope)
|
||||||
|
safe_category = sanitize_user_text(category)
|
||||||
|
safe_priority = sanitize_user_text(priority)
|
||||||
|
|
||||||
context_block = ""
|
context_block = ""
|
||||||
if similar_proposals:
|
if similar_proposals:
|
||||||
context_block = "Here are similar historical proposals and their line items for reference:\n\n"
|
context_block = "Here are similar historical proposals and their line items for reference:\n\n"
|
||||||
|
|
@ -217,11 +334,11 @@ def generate_line_items(
|
||||||
Based on the scope of work and similar historical proposals, generate a detailed list of line items
|
Based on the scope of work and similar historical proposals, generate a detailed list of line items
|
||||||
with quantities, units, and estimated pricing.
|
with quantities, units, and estimated pricing.
|
||||||
|
|
||||||
Service Category: {category}
|
Service Category: {safe_category}
|
||||||
Priority: {priority}
|
Priority: {safe_priority}
|
||||||
|
|
||||||
Scope of Work:
|
Scope of Work:
|
||||||
{scope}
|
{safe_scope}
|
||||||
|
|
||||||
{context_block}
|
{context_block}
|
||||||
|
|
||||||
|
|
@ -259,7 +376,10 @@ Respond ONLY with the JSON array, no additional text."""
|
||||||
content = content.rsplit("```", 1)[0]
|
content = content.rsplit("```", 1)[0]
|
||||||
|
|
||||||
line_items = json.loads(content)
|
line_items = json.loads(content)
|
||||||
return line_items if isinstance(line_items, list) else []
|
if not isinstance(line_items, list):
|
||||||
|
return []
|
||||||
|
# Fix: LAM-M6 — validate numeric fields before returning suggestions
|
||||||
|
return validate_line_item_numerics(line_items)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Fix: LAM-M5 — include stack trace in error logging
|
# Fix: LAM-M5 — include stack trace in error logging
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue