procurement-ingest/terraform/wo_alarms.tf

resource "aws_cloudwatch_log_metric_filter" "wo_sender_auth_rejected" {
  name           = local.wo_sender_auth_filter_name
  log_group_name = aws_cloudwatch_log_group.wo_email_processor.name
  pattern        = "\"sender_auth_rejected\""

  metric_transformation {
    name          = "workorder-email-processor-sender-auth-rejected"
    namespace     = "Seahaven/ProcurementIngest"
    value         = "1"
    default_value = "0"
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_errors" {
  alarm_name          = "workorder-email-processor-errors"
  alarm_description   = "workorder-email-processor async invocation errors"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "Errors"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Sum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_email_processor.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_throttles" {
  alarm_name          = "workorder-email-processor-throttles"
  alarm_description   = "workorder-email-processor invocation throttles"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "Throttles"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Sum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_email_processor.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_dlq" {
  alarm_name          = "workorder-email-processor-dlq-messages"
  alarm_description   = "workorder-email-processor DLQ has visible messages (dropped emails)"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "ApproximateNumberOfMessagesVisible"
  namespace           = "AWS/SQS"
  period              = 300
  statistic           = "Maximum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    QueueName = aws_sqs_queue.wo_email_processor_dlq.name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_dlq_age" {
  alarm_name          = "workorder-email-processor-dlq-age"
  alarm_description   = "workorder-email-processor DLQ oldest message is >= 1 day old (drain before 14-day retention expiry)"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 1
  metric_name         = "ApproximateAgeOfOldestMessage"
  namespace           = "AWS/SQS"
  period              = 300
  statistic           = "Maximum"
  threshold           = 86400
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    QueueName = aws_sqs_queue.wo_email_processor_dlq.name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_duration" {
  alarm_name          = "workorder-email-processor-duration"
  alarm_description   = "workorder-email-processor p95 duration approaching the 60s timeout"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 3
  datapoints_to_alarm = 2
  metric_name         = "Duration"
  namespace           = "AWS/Lambda"
  period              = 300
  extended_statistic  = "p95"
  threshold           = 45000
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_email_processor.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_sender_auth_rejected" {
  alarm_name          = "workorder-email-processor-sender-auth-rejected"
  alarm_description   = "workorder-email-processor rejected inbound mail on sender authentication (possible allowlist/DKIM-domain drift silently dropping real mail)"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 6
  datapoints_to_alarm = 2
  metric_name         = "workorder-email-processor-sender-auth-rejected"
  namespace           = "Seahaven/ProcurementIngest"
  period              = 300
  statistic           = "Sum"
  threshold           = 1
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_fallback_rate" {
  alarm_name          = "workorder-email-processor-template-fallback-rate"
  alarm_description   = "workorder-email-processor deterministic-template coverage collapse: >15% of parses fell back to the Bedrock AI extractor"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 3
  datapoints_to_alarm = 2
  threshold           = 15
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  metric_query {
    id          = "expr_1"
    expression  = "IF((FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0))>=10, 100*(FILL(fb,0)+FILL(rej,0))/(FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0)), 0)"
    label       = "TemplateFallbackRatePct"
    return_data = true
  }

  metric_query {
    id = "fb"
    metric {
      metric_name = "ParseOutcome"
      namespace   = "Seahaven/WorkorderIngest"
      period      = 900
      stat        = "Sum"
      dimensions = {
        ParseMethod = "ai_fallback"
      }
    }
    return_data = false
  }

  metric_query {
    id = "rej"
    metric {
      metric_name = "ParseOutcome"
      namespace   = "Seahaven/WorkorderIngest"
      period      = 900
      stat        = "Sum"
      dimensions = {
        ParseMethod = "ai_fallback_rejected"
      }
    }
    return_data = false
  }

  metric_query {
    id = "tmpl"
    metric {
      metric_name = "ParseOutcome"
      namespace   = "Seahaven/WorkorderIngest"
      period      = 900
      stat        = "Sum"
      dimensions = {
        ParseMethod = "template"
      }
    }
    return_data = false
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_email_processor_ai_fallback_rejected" {
  alarm_name          = "workorder-email-processor-ai-fallback-rejected"
  alarm_description   = "workorder-email-processor is rejecting Bedrock AI-fallback output at the validation gate (possible prompt-injection probing or template drift silently dropping real mail)"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 6
  datapoints_to_alarm = 2
  metric_name         = "ParseOutcome"
  namespace           = "Seahaven/WorkorderIngest"
  period              = 300
  statistic           = "Sum"
  threshold           = 1
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    ParseMethod = "ai_fallback_rejected"
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_hmac_rotator_errors" {
  alarm_name          = "workorder-shoc-hmac-rotator-errors"
  alarm_description   = "workorder-shoc-hmac-rotator invocation errors"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "Errors"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Sum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_hmac_rotator.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_hmac_rotator_throttles" {
  alarm_name          = "workorder-shoc-hmac-rotator-throttles"
  alarm_description   = "workorder-shoc-hmac-rotator invocation throttles"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "Throttles"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Sum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_hmac_rotator.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_hmac_rotator_duration" {
  alarm_name          = "workorder-shoc-hmac-rotator-duration"
  alarm_description   = "workorder-shoc-hmac-rotator p99 duration approaching the 60s timeout"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 3
  datapoints_to_alarm = 2
  metric_name         = "Duration"
  namespace           = "AWS/Lambda"
  period              = 300
  extended_statistic  = "p99"
  threshold           = 45000
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_hmac_rotator.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_errors" {
  alarm_name          = "workorder-shoc-emitter-errors"
  alarm_description   = "workorder-shoc-emitter invocation errors"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "Errors"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Sum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_throttles" {
  alarm_name          = "workorder-shoc-emitter-throttles"
  alarm_description   = "workorder-shoc-emitter invocation throttles"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "Throttles"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Sum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_duration" {
  alarm_name          = "workorder-shoc-emitter-duration"
  alarm_description   = "workorder-shoc-emitter p99 duration approaching the 60s timeout"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 3
  datapoints_to_alarm = 2
  metric_name         = "Duration"
  namespace           = "AWS/Lambda"
  period              = 300
  extended_statistic  = "p99"
  threshold           = 45000
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_iterator_age" {
  alarm_name          = "workorder-shoc-emitter-iterator-age"
  alarm_description   = "workorder-shoc-emitter stream lag >= 10 min (SHOC receiver likely down; shard blocking on retries)"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = 3
  datapoints_to_alarm = 2
  metric_name         = "IteratorAge"
  namespace           = "AWS/Lambda"
  period              = 300
  statistic           = "Maximum"
  threshold           = 600000
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    FunctionName = aws_lambda_function.wo_shoc_emitter.function_name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_failures_messages" {
  alarm_name          = "workorder-shoc-emitter-failures-messages"
  alarm_description   = "workorder-shoc-emitter retry-exhausted stream records parked (ESM failure metadata; replay rebuilds from DynamoDB)"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "ApproximateNumberOfMessagesVisible"
  namespace           = "AWS/SQS"
  period              = 300
  statistic           = "Maximum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    QueueName = aws_sqs_queue.shoc_emitter_failures.name
  }
}

resource "aws_cloudwatch_metric_alarm" "wo_shoc_emitter_rejected_messages" {
  alarm_name          = "workorder-shoc-emitter-rejected-messages"
  alarm_description   = "workorder-shoc-emitter parked non-retryable 4xx deliveries (contract bug; inspect payloads and replay)"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "ApproximateNumberOfMessagesVisible"
  namespace           = "AWS/SQS"
  period              = 300
  statistic           = "Maximum"
  threshold           = 0
  treat_missing_data  = "notBreaching"
  alarm_actions       = [data.aws_sns_topic.site_alerts.arn]

  dimensions = {
    QueueName = aws_sqs_queue.shoc_emitter_rejected.name
  }
}