mirror of
https://github.com/Sea-Haven-Industries/syslog-server.git
synced 2026-09-30 03:03:14 +00:00
2b12aba50e
Replace the public rsyslog-to-CloudWatch collector with Vector over a prod 10.40 IPsec VGW, Firehose, 90-day S3, Glue, and Athena.
59 lines
1.7 KiB
HCL
59 lines
1.7 KiB
HCL
resource "aws_glue_catalog_database" "unifi" {
|
|
name = local.glue_database_name
|
|
}
|
|
|
|
locals {
|
|
glue_columns = [
|
|
{ name = "timestamp", type = "string" },
|
|
{ name = "site", type = "string" },
|
|
{ name = "format", type = "string" },
|
|
{ name = "hostname", type = "string" },
|
|
{ name = "src", type = "string" },
|
|
{ name = "dst", type = "string" },
|
|
{ name = "proto", type = "string" },
|
|
{ name = "action", type = "string" },
|
|
{ name = "raw", type = "string" },
|
|
]
|
|
}
|
|
|
|
resource "aws_glue_catalog_table" "formats" {
|
|
for_each = toset(["cef", "iptables", "netflow"])
|
|
|
|
name = each.value
|
|
database_name = aws_glue_catalog_database.unifi.name
|
|
table_type = "EXTERNAL_TABLE"
|
|
|
|
parameters = {
|
|
classification = "json"
|
|
compressionType = "gzip"
|
|
"projection.enabled" = "true"
|
|
"projection.dt.type" = "date"
|
|
"projection.dt.format" = "yyyy-MM-dd"
|
|
"projection.dt.range" = "2026-01-01,NOW"
|
|
"storage.location.template" = "s3://${aws_s3_bucket.unifi.bucket}/format=${each.value}/dt=$${dt}/"
|
|
}
|
|
|
|
partition_keys {
|
|
name = "dt"
|
|
type = "string"
|
|
}
|
|
|
|
storage_descriptor {
|
|
location = "s3://${aws_s3_bucket.unifi.bucket}/format=${each.value}/"
|
|
input_format = "org.apache.hadoop.mapred.TextInputFormat"
|
|
output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
|
|
|
|
ser_de_info {
|
|
name = "json"
|
|
serialization_library = "org.openx.data.jsonserde.JsonSerDe"
|
|
}
|
|
|
|
dynamic "columns" {
|
|
for_each = local.glue_columns
|
|
content {
|
|
name = columns.value.name
|
|
type = columns.value.type
|
|
}
|
|
}
|
|
}
|
|
}
|