data "aws_iam_policy_document" "firehose_assume" { statement { sid = "FirehoseAssume" effect = "Allow" actions = ["sts:AssumeRole"] principals { type = "Service" identifiers = ["firehose.amazonaws.com"] } } } data "aws_iam_policy_document" "firehose" { statement { sid = "S3Delivery" effect = "Allow" actions = [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ] resources = [ aws_s3_bucket.unifi.arn, "${aws_s3_bucket.unifi.arn}/*", ] } } resource "aws_iam_role" "firehose" { name = local.firehose_role_name path = "/tf-managed/" assume_role_policy = data.aws_iam_policy_document.firehose_assume.json permissions_boundary = aws_iam_policy.instance_boundary.arn tags = { Name = local.firehose_role_name } } resource "aws_iam_role_policy" "firehose" { name = "s3-delivery" role = aws_iam_role.firehose.id policy = data.aws_iam_policy_document.firehose.json } resource "aws_kinesis_firehose_delivery_stream" "unifi" { name = local.firehose_name destination = "extended_s3" extended_s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.unifi.arn prefix = "format=!{partitionKeyFromQuery:format}/dt=!{timestamp:yyyy-MM-dd}/" error_output_prefix = "errors/!{firehose:error-output-type}/dt=!{timestamp:yyyy-MM-dd}/" buffering_size = 64 buffering_interval = 300 compression_format = "GZIP" file_extension = ".json.gz" processing_configuration { enabled = true processors { type = "MetadataExtraction" parameters { parameter_name = "JsonParsingEngine" parameter_value = "JQ-1.6" } parameters { parameter_name = "MetadataExtractionQuery" parameter_value = "{format:.format}" } } processors { type = "AppendDelimiterToRecord" parameters { parameter_name = "Delimiter" parameter_value = "\\n" } } } dynamic_partitioning_configuration { enabled = true } cloudwatch_logging_options { enabled = false } } tags = { Name = local.firehose_name } depends_on = [aws_iam_role_policy.firehose] }