mirror of
https://github.com/Sea-Haven-Industries/syslog-server.git
synced 2026-09-30 08:53:16 +00:00
2b12aba50e
Replace the public rsyslog-to-CloudWatch collector with Vector over a prod 10.40 IPsec VGW, Firehose, 90-day S3, Glue, and Athena.
105 lines
2.4 KiB
HCL
105 lines
2.4 KiB
HCL
data "aws_iam_policy_document" "firehose_assume" {
|
|
statement {
|
|
sid = "FirehoseAssume"
|
|
effect = "Allow"
|
|
actions = ["sts:AssumeRole"]
|
|
|
|
principals {
|
|
type = "Service"
|
|
identifiers = ["firehose.amazonaws.com"]
|
|
}
|
|
}
|
|
}
|
|
|
|
data "aws_iam_policy_document" "firehose" {
|
|
statement {
|
|
sid = "S3Delivery"
|
|
effect = "Allow"
|
|
actions = [
|
|
"s3:AbortMultipartUpload",
|
|
"s3:GetBucketLocation",
|
|
"s3:GetObject",
|
|
"s3:ListBucket",
|
|
"s3:ListBucketMultipartUploads",
|
|
"s3:PutObject",
|
|
]
|
|
resources = [
|
|
aws_s3_bucket.unifi.arn,
|
|
"${aws_s3_bucket.unifi.arn}/*",
|
|
]
|
|
}
|
|
}
|
|
|
|
resource "aws_iam_role" "firehose" {
|
|
name = local.firehose_role_name
|
|
path = "/tf-managed/"
|
|
assume_role_policy = data.aws_iam_policy_document.firehose_assume.json
|
|
permissions_boundary = aws_iam_policy.instance_boundary.arn
|
|
|
|
tags = {
|
|
Name = local.firehose_role_name
|
|
}
|
|
}
|
|
|
|
resource "aws_iam_role_policy" "firehose" {
|
|
name = "s3-delivery"
|
|
role = aws_iam_role.firehose.id
|
|
policy = data.aws_iam_policy_document.firehose.json
|
|
}
|
|
|
|
resource "aws_kinesis_firehose_delivery_stream" "unifi" {
|
|
name = local.firehose_name
|
|
destination = "extended_s3"
|
|
|
|
extended_s3_configuration {
|
|
role_arn = aws_iam_role.firehose.arn
|
|
bucket_arn = aws_s3_bucket.unifi.arn
|
|
prefix = "format=!{partitionKeyFromQuery:format}/dt=!{timestamp:yyyy-MM-dd}/"
|
|
error_output_prefix = "errors/!{firehose:error-output-type}/dt=!{timestamp:yyyy-MM-dd}/"
|
|
buffering_size = 64
|
|
buffering_interval = 300
|
|
compression_format = "GZIP"
|
|
file_extension = ".json.gz"
|
|
|
|
processing_configuration {
|
|
enabled = true
|
|
|
|
processors {
|
|
type = "MetadataExtraction"
|
|
|
|
parameters {
|
|
parameter_name = "JsonParsingEngine"
|
|
parameter_value = "JQ-1.6"
|
|
}
|
|
|
|
parameters {
|
|
parameter_name = "MetadataExtractionQuery"
|
|
parameter_value = "{format:.format}"
|
|
}
|
|
}
|
|
|
|
processors {
|
|
type = "AppendDelimiterToRecord"
|
|
|
|
parameters {
|
|
parameter_name = "Delimiter"
|
|
parameter_value = "\\n"
|
|
}
|
|
}
|
|
}
|
|
|
|
dynamic_partitioning_configuration {
|
|
enabled = true
|
|
}
|
|
|
|
cloudwatch_logging_options {
|
|
enabled = false
|
|
}
|
|
}
|
|
|
|
tags = {
|
|
Name = local.firehose_name
|
|
}
|
|
|
|
depends_on = [aws_iam_role_policy.firehose]
|
|
}
|