mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-09-30 07:13:13 +00:00
344 lines
12 KiB
HCL
344 lines
12 KiB
HCL
resource "aws_cloudwatch_log_metric_filter" "wo_sender_auth_rejected" {
|
|
name = local.wo_sender_auth_filter_name
|
|
log_group_name = aws_cloudwatch_log_group.wo_email_processor.name
|
|
pattern = "\"sender_auth_rejected\""
|
|
|
|
metric_transformation {
|
|
name = "workorder-email-processor-sender-auth-rejected"
|
|
namespace = "Seahaven/ProcurementIngest"
|
|
value = "1"
|
|
default_value = "0"
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_errors" {
|
|
alarm_name = "workorder-email-processor-errors"
|
|
alarm_description = "workorder-email-processor async invocation errors"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "Errors"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_email_processor.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_throttles" {
|
|
alarm_name = "workorder-email-processor-throttles"
|
|
alarm_description = "workorder-email-processor invocation throttles"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "Throttles"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_email_processor.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_dlq" {
|
|
alarm_name = "workorder-email-processor-dlq-messages"
|
|
alarm_description = "workorder-email-processor DLQ has visible messages (dropped emails)"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "ApproximateNumberOfMessagesVisible"
|
|
namespace = "AWS/SQS"
|
|
period = 300
|
|
statistic = "Maximum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
QueueName = aws_sqs_queue.wo_email_processor_dlq.name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_duration" {
|
|
alarm_name = "workorder-email-processor-duration"
|
|
alarm_description = "workorder-email-processor p95 duration approaching the 60s timeout"
|
|
comparison_operator = "GreaterThanOrEqualToThreshold"
|
|
evaluation_periods = 3
|
|
datapoints_to_alarm = 2
|
|
metric_name = "Duration"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
extended_statistic = "p95"
|
|
threshold = 45000
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_email_processor.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_sender_auth_rejected" {
|
|
alarm_name = "workorder-email-processor-sender-auth-rejected"
|
|
alarm_description = "workorder-email-processor rejected inbound mail on sender authentication (possible allowlist/DKIM-domain drift silently dropping real mail)"
|
|
comparison_operator = "GreaterThanOrEqualToThreshold"
|
|
evaluation_periods = 6
|
|
datapoints_to_alarm = 2
|
|
metric_name = "workorder-email-processor-sender-auth-rejected"
|
|
namespace = "Seahaven/ProcurementIngest"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 1
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_fallback_rate" {
|
|
alarm_name = "workorder-email-processor-template-fallback-rate"
|
|
alarm_description = "workorder-email-processor deterministic-template coverage collapse: >15% of parses fell back to the Bedrock AI extractor"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 3
|
|
datapoints_to_alarm = 2
|
|
threshold = 15
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
metric_query {
|
|
id = "expr_1"
|
|
expression = "IF((FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0))>=10, 100*(FILL(fb,0)+FILL(rej,0))/(FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0)), 0)"
|
|
label = "TemplateFallbackRatePct"
|
|
return_data = true
|
|
}
|
|
|
|
metric_query {
|
|
id = "fb"
|
|
metric {
|
|
metric_name = "ParseOutcome"
|
|
namespace = "Seahaven/WorkorderIngest"
|
|
period = 900
|
|
stat = "Sum"
|
|
dimensions = {
|
|
ParseMethod = "ai_fallback"
|
|
}
|
|
}
|
|
return_data = false
|
|
}
|
|
|
|
metric_query {
|
|
id = "rej"
|
|
metric {
|
|
metric_name = "ParseOutcome"
|
|
namespace = "Seahaven/WorkorderIngest"
|
|
period = 900
|
|
stat = "Sum"
|
|
dimensions = {
|
|
ParseMethod = "ai_fallback_rejected"
|
|
}
|
|
}
|
|
return_data = false
|
|
}
|
|
|
|
metric_query {
|
|
id = "tmpl"
|
|
metric {
|
|
metric_name = "ParseOutcome"
|
|
namespace = "Seahaven/WorkorderIngest"
|
|
period = 900
|
|
stat = "Sum"
|
|
dimensions = {
|
|
ParseMethod = "template"
|
|
}
|
|
}
|
|
return_data = false
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_email_processor_ai_fallback_rejected" {
|
|
alarm_name = "workorder-email-processor-ai-fallback-rejected"
|
|
alarm_description = "workorder-email-processor is rejecting Bedrock AI-fallback output at the validation gate (possible prompt-injection probing or template drift silently dropping real mail)"
|
|
comparison_operator = "GreaterThanOrEqualToThreshold"
|
|
evaluation_periods = 6
|
|
datapoints_to_alarm = 2
|
|
metric_name = "ParseOutcome"
|
|
namespace = "Seahaven/WorkorderIngest"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 1
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
ParseMethod = "ai_fallback_rejected"
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_hmac_rotator_errors" {
|
|
alarm_name = "workorder-shoc-hmac-rotator-errors"
|
|
alarm_description = "workorder-shoc-hmac-rotator invocation errors"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "Errors"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_hmac_rotator.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_hmac_rotator_throttles" {
|
|
alarm_name = "workorder-shoc-hmac-rotator-throttles"
|
|
alarm_description = "workorder-shoc-hmac-rotator invocation throttles"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "Throttles"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_hmac_rotator.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_hmac_rotator_duration" {
|
|
alarm_name = "workorder-shoc-hmac-rotator-duration"
|
|
alarm_description = "workorder-shoc-hmac-rotator p99 duration approaching the 60s timeout"
|
|
comparison_operator = "GreaterThanOrEqualToThreshold"
|
|
evaluation_periods = 3
|
|
datapoints_to_alarm = 2
|
|
metric_name = "Duration"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
extended_statistic = "p99"
|
|
threshold = 45000
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_hmac_rotator.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_errors" {
|
|
alarm_name = "workorder-shoc-emitter-errors"
|
|
alarm_description = "workorder-shoc-emitter invocation errors"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "Errors"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_throttles" {
|
|
alarm_name = "workorder-shoc-emitter-throttles"
|
|
alarm_description = "workorder-shoc-emitter invocation throttles"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "Throttles"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Sum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_duration" {
|
|
alarm_name = "workorder-shoc-emitter-duration"
|
|
alarm_description = "workorder-shoc-emitter p99 duration approaching the 60s timeout"
|
|
comparison_operator = "GreaterThanOrEqualToThreshold"
|
|
evaluation_periods = 3
|
|
datapoints_to_alarm = 2
|
|
metric_name = "Duration"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
extended_statistic = "p99"
|
|
threshold = 45000
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_iterator_age" {
|
|
alarm_name = "workorder-shoc-emitter-iterator-age"
|
|
alarm_description = "workorder-shoc-emitter stream lag >= 10 min (SHOC receiver likely down; shard blocking on retries)"
|
|
comparison_operator = "GreaterThanOrEqualToThreshold"
|
|
evaluation_periods = 3
|
|
datapoints_to_alarm = 2
|
|
metric_name = "IteratorAge"
|
|
namespace = "AWS/Lambda"
|
|
period = 300
|
|
statistic = "Maximum"
|
|
threshold = 600000
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_failures_messages" {
|
|
alarm_name = "workorder-shoc-emitter-failures-messages"
|
|
alarm_description = "workorder-shoc-emitter retry-exhausted stream records parked (ESM failure metadata; replay rebuilds from DynamoDB)"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "ApproximateNumberOfMessagesVisible"
|
|
namespace = "AWS/SQS"
|
|
period = 300
|
|
statistic = "Maximum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
QueueName = aws_sqs_queue.shoc_emitter_failures.name
|
|
}
|
|
}
|
|
|
|
resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_rejected_messages" {
|
|
alarm_name = "workorder-shoc-emitter-rejected-messages"
|
|
alarm_description = "workorder-shoc-emitter parked non-retryable 4xx deliveries (contract bug; inspect payloads and replay)"
|
|
comparison_operator = "GreaterThanThreshold"
|
|
evaluation_periods = 1
|
|
metric_name = "ApproximateNumberOfMessagesVisible"
|
|
namespace = "AWS/SQS"
|
|
period = 300
|
|
statistic = "Maximum"
|
|
threshold = 0
|
|
treat_missing_data = "notBreaching"
|
|
alarm_actions = [data.aws_sns_topic.site_alerts.arn]
|
|
|
|
dimensions = {
|
|
QueueName = aws_sqs_queue.shoc_emitter_rejected.name
|
|
}
|
|
}
|