From 2b12aba50e85d4808cda61e86301dfd5600f3c5a Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:48:25 +0000 Subject: [PATCH] feat(infra): archive UniFi All Traffic to S3 via Vector (PLAT-206) (#41) Replace the public rsyslog-to-CloudWatch collector with Vector over a prod 10.40 IPsec VGW, Firehose, 90-day S3, Glue, and Athena. --- README.md | 140 +++++++++++++------- terraform/alarms.tf | 34 +++-- terraform/athena.tf | 63 +++++++++ terraform/ec2.tf | 27 ++-- terraform/firehose.tf | 105 +++++++++++++++ terraform/glue.tf | 59 +++++++++ terraform/hcp_iam.tf | 250 ++++++++++++++++++++++++++++++++--- terraform/iam.tf | 62 ++++++--- terraform/locals.tf | 29 ++-- terraform/logs.tf | 4 - terraform/outputs.tf | 55 ++++++-- terraform/s3.tf | 106 +++++++++++++++ terraform/user_data.sh | 143 -------------------- terraform/user_data.sh.tftpl | 25 ++++ terraform/variables.tf | 2 +- terraform/vector.yaml.tftpl | 146 ++++++++++++++++++++ terraform/vpc.tf | 77 ++++++++--- 17 files changed, 1027 insertions(+), 300 deletions(-) create mode 100644 terraform/athena.tf create mode 100644 terraform/firehose.tf create mode 100644 terraform/glue.tf delete mode 100644 terraform/logs.tf create mode 100644 terraform/s3.tf delete mode 100644 terraform/user_data.sh create mode 100644 terraform/user_data.sh.tftpl create mode 100644 terraform/vector.yaml.tftpl diff --git a/README.md b/README.md index 69acfcb..083236c 100644 --- a/README.md +++ b/README.md @@ -4,23 +4,34 @@ ![AWS](https://img.shields.io/badge/AWS-FF9900?logo=amazonaws&logoColor=white) ![CI](https://github.com/Sea-Haven-Industries/syslog-server/actions/workflows/ci.yaml/badge.svg) -EC2 collector that receives remote syslog (UDP/TCP 514) from the office UniFi -fleet over an Elastic IP and ships it to the `unifi-syslog` CloudWatch Logs -group via the CloudWatch agent. +Vector collector that receives UniFi All Traffic syslog, CEF, and IPFIX over +office IPsec, parses to JSON, and writes to S3 through Kinesis Data Firehose +for 90-day Athena search. -Deploy path (PLAT-78): HCP Terraform in seahaven-prod (`011934824531`), -workspace `syslog-server-prod`. CDK CD in mgmt is retired. +Deploy path (PLAT-78 / PLAT-206): HCP Terraform in seahaven-prod +(`011934824531`), workspace `syslog-server-prod`. CDK CD in mgmt is retired. ## Architecture ``` -office UniFi devices ──syslog/514──▶ EIP (prod) ──▶ EC2 (rsyslog) - │ - /var/log/remote//*.log - │ - CloudWatch agent ──▶ unifi-syslog (90d) - │ - Syslog-NoIncomingLogs alarm ──▶ site-alerts +Locust UDM 10.30 ──SD-WAN mesh──▶ Ronkonkoma UDM 10.10 + │ + IPsec UDP 514 + IPFIX 2055/2056 + │ + ▼ + Vector t4g.small (10.40) + │ + ▼ + Kinesis Data Firehose + │ + ▼ + S3 syslog-server-unifi-logs-* (90d) + │ + ▼ + Glue unifi + Athena + │ + Syslog-NoIncomingRecords ──▶ site-alerts + Syslog-FirehoseDeliveryFailed ──▶ site-alerts ``` | Resource | Value | @@ -28,55 +39,94 @@ office UniFi devices ──syslog/514──▶ EIP (prod) ──▶ EC2 (rsyslog | Account / region | seahaven-prod `011934824531` / us-east-1 | | HCP workspace | `syslog-server-prod` (project `seahaven-prod`; VCS `main`; working dir `terraform`; trigger `terraform/**`) | | HCP plan/apply roles | `hcptf-syslog-server-plan` / `hcptf-syslog-server` | -| Instance | `syslog-server`, t4g.nano, Amazon Linux 2023 (arm64), 30 GiB encrypted gp3 | -| VPC | dedicated `10.40.0.0/16`, public subnet `10.40.10.0/24` | -| Elastic IP | `184.193.220.187` (`eipalloc-07d82c1f79a22716a`) — UniFi still points at mgmt until INFRA-11 | -| Security group | `syslog-server` — 514 tcp/udp + 22 from office IPs + VPC/VPN CIDRs; 2055/2056 udp from office | -| Instance IAM | `/tf-managed/syslog-server-role` with `syslog-server-instance-boundary`; `AmazonSSMManagedInstanceCore` + `CloudWatchAgentServerPolicy` | -| Log group | `unifi-syslog` (90-day retention) | -| Alarms | `Syslog-NoIncomingLogs`, `EC2-StatusCheck-syslog-server`, `EC2-StatusCheckSystem-syslog-server-recover` → `site-alerts` | +| Instance | `syslog-server`, t4g.small, Amazon Linux 2023 (arm64), 20 GiB encrypted gp3, SSM only | +| VPC | dedicated `10.40.0.0/16`, public subnet `10.40.10.0/24` (egress IP for Vector install / Firehose / SSM; not a syslog target) | +| IPsec | VGW + customer gateway on Ronkonkoma WAN `47.21.61.4`; static routes `10.10.0.0/16` and `10.30.0.0/16` | +| UniFi target | instance **private IP**:514 (syslog + CEF) and :2055/:2056 (IPFIX). No public 514. | +| Security group | `syslog-server` — UDP/TCP 514 and UDP 2055/2056 from `10.10.0.0/16` and `10.30.0.0/16` only | +| Instance IAM | `/tf-managed/syslog-server-role` with `syslog-server-instance-boundary`; `AmazonSSMManagedInstanceCore` + `firehose:PutRecordBatch` | +| Store | S3 `syslog-server-unifi-logs-011934824531`, prefixes `format=cef\|iptables\|netflow/dt=YYYY-MM-DD/`, 90-day expire | +| Query | Glue database `unifi` (cef, iptables, netflow) and Athena workgroup `syslog-server` | +| Alarms | `Syslog-NoIncomingRecords`, `Syslog-FirehoseDeliveryFailed`, `EC2-StatusCheck-syslog-server`, `EC2-StatusCheckSystem-syslog-server-recover` → `site-alerts` | + +The office IPsec tunnel that already reaches mgmt `10.20.0.0/16` does **not** +land in this VPC. UniFi needs a second site-to-site peer for `10.40.0.0/16`. +Do not re-home this workspace in mgmt. ## Access -SSM Session Manager (no key pair). SSH 22 is open from office/VPC for -break-glass only. +SSM Session Manager. SSH 22 is closed. There is no Elastic IP forwarding +target. -## HCP first apply +## IAM bootstrap window -First apply uses the hcptf-bootstrap window (exact `StringEquals` trust, never -`StringLike`): +Instance-boundary document changes and apply-role inline policy changes need +the hcptf-bootstrap window (`DenySelfMutation` plus deny on +`iam:CreatePolicyVersion`). Sequence: -1. Create the HCP workspace. Auto-apply off. No project-level variable set. - Working directory `terraform`. File trigger prefix `terraform/**` only. - Speculative plans on. VCS on `main`. -2. From `seahaven-org-baseline`: +1. From `seahaven-org-baseline`: `scripts/create-hcptf-bootstrap-roles.sh --account prod --allow-workspace syslog-server-prod` -3. Point workspace `TFC_AWS_APPLY_ROLE_ARN` / `TFC_AWS_PLAN_ROLE_ARN` at - `hcptf-bootstrap` / `hcptf-bootstrap-plan`. Set `TFC_AWS_PROVIDER_AUTH=true`. +2. Point workspace `TFC_AWS_APPLY_ROLE_ARN` / `TFC_AWS_PLAN_ROLE_ARN` at + `hcptf-bootstrap` / `hcptf-bootstrap-plan`. Keep `TFC_AWS_PROVIDER_AUTH=true`. Never `TFC_AWS_RUN_ROLE_ARN`. -4. One manual apply as `hcptf-bootstrap` creates the scoped `hcptf-*` roles, - boundary, and instance role. Bootstrap cannot `ec2:CreateVpc`; the rest of - the stack applies as `hcptf-syslog-server`. -5. Retarget `TFC_AWS_*` to `hcptf-syslog-server` / `hcptf-syslog-server-plan`. +3. One manual apply as bootstrap creates/updates the scoped `hcptf-*` inline + policies and the instance boundary. +4. Retarget `TFC_AWS_*` to `hcptf-syslog-server` / `hcptf-syslog-server-plan`. Re-run the create script with no `--allow-workspace`. -6. Manual apply as the scoped role. After live-path proof, seal auto-apply on. +5. Manual apply as the scoped role for the rest of the stack (instance, + Firehose, S3, Glue, Athena, VGW). Auto-apply stays off until soak. -`Syslog-NoIncomingLogs` defaults `treat_missing_data` to `notBreaching` so the -empty prod log group does not page `site-alerts` before UniFi is re-pointed. -After devices deliver to the new EIP, set `no_logs_treat_missing_data=breaching`. +HCP outputs to copy: `private_ip`, `vpn_connection_id`, +`vpn_tunnel1_address`, `vpn_tunnel2_address`, `bucket_name`, +`firehose_name`, `athena_workgroup`. Read PSKs with +`terraform output -raw vpn_tunnel1_preshared_key` after apply. Do not commit +them. -HCP outputs to copy: `public_ip`, `instance_id`, `hcptf_apply_role_arn`, -`hcptf_plan_role_arn`. +AMI is pinned in `var.ami_id`. An AMI or user-data change replaces the +instance. The box is stateless; archives live in S3. -AMI is pinned in `var.ami_id`. An AMI change forces instance replacement. -User-data changes also replace the instance (the box is stateless; logs live -in CloudWatch; the EIP re-associates). +`Syslog-NoIncomingRecords` defaults `treat_missing_data` to `notBreaching` +until UniFi delivers over IPsec. After Firehose `IncomingRecords` is +non-zero, set `no_logs_treat_missing_data=breaching`. + +## UniFi cutover + +Do this after the HCP apply, not before. Apply drops public 514 and the +CloudWatch `unifi-syslog` log group. Point UniFi immediately. + +1. **Ronkonkoma site-to-site VPN** to the AWS tunnel addresses from HCP + outputs. Remote network `10.40.0.0/16`. Local network `10.10.0.0/16`. + IKEv2, AES-256, SHA-256, DH14 matches typical AWS defaults. Use the + Terraform PSK outputs. This is a second child SA alongside the existing + mgmt `10.20` tunnel. Do not replace the mgmt tunnel. +2. **Locust SD-WAN mesh** must already route AWS VPC CIDRs via Ronkonkoma + (same as jumpbox SSH). Add `10.40.0.0/16` if it is missing. +3. Both controllers, **Settings → CyberSecure / System Log**: + - SIEM server = collector **private IP**, port **514**, UDP + - Flow Logging = **All Traffic** + - Activity Logging SIEM contents include firewall + - Control Plane **CEF** to the same IP:514 +4. Enable syslog on WAN and inter-VLAN firewall rules, or All Traffic stays + silent. +5. NetFlow/IPFIX: Ronkonkoma → UDP **2055**, Locust → UDP **2056**, same + private IP. +6. Prove the path: send a test syslog from Ronkonkoma; Athena `SELECT` on + `iptables` and `cef`; confirm `format=netflow` objects for 2055/2056; + confirm `site-alerts` does not fire while traffic is present. +7. Flip off any remaining public EIP / mgmt collector **only after** + Firehose `IncomingRecords` is non-zero. PLAT-78 still owns deleting the + mgmt `syslog-server` CloudFormation stack after soak. + +Vector treats payloads as untrusted text. It parses fields and does not +shell out. IPFIX datagrams are archived as base64 JSON with a site tag +(Vector has no released IPFIX decoder). ## Documentation The canonical map of Sea Haven's AWS infrastructure lives in Confluence. - **[AWS Architecture Map](https://seahaven.atlassian.net/wiki/spaces/IT/pages/1540098)** (Confluence, IT space, page 1540098) +- **[Syslog Server](https://seahaven.atlassian.net/wiki/spaces/IT/pages/67141633)** (page 67141633) -To widen device coverage of the forwarded syslog feed, see **INFRA-11** -(UniFi controller remote-logging config). +Tracked as **PLAT-206**. PLAT-78 remains the HCP move plus mgmt stack delete +after this soak. diff --git a/terraform/alarms.tf b/terraform/alarms.tf index 3659fc8..8d107be 100644 --- a/terraform/alarms.tf +++ b/terraform/alarms.tf @@ -1,10 +1,10 @@ -resource "aws_cloudwatch_metric_alarm" "no_incoming_logs" { - alarm_name = "Syslog-NoIncomingLogs" - alarm_description = "No log events delivered to unifi-syslog for 2 days — syslog pipeline may be down." +resource "aws_cloudwatch_metric_alarm" "no_incoming_records" { + alarm_name = "Syslog-NoIncomingRecords" + alarm_description = "No Firehose IncomingRecords for 2 days. UniFi pipeline may be down." comparison_operator = "LessThanThreshold" evaluation_periods = 2 - metric_name = "IncomingLogEvents" - namespace = "AWS/Logs" + metric_name = "IncomingRecords" + namespace = "AWS/Firehose" period = 86400 statistic = "Sum" threshold = 1 @@ -12,13 +12,31 @@ resource "aws_cloudwatch_metric_alarm" "no_incoming_logs" { alarm_actions = [local.site_alerts_arn] dimensions = { - LogGroupName = local.log_group_name + DeliveryStreamName = aws_kinesis_firehose_delivery_stream.unifi.name + } +} + +resource "aws_cloudwatch_metric_alarm" "firehose_delivery" { + alarm_name = "Syslog-FirehoseDeliveryFailed" + alarm_description = "Firehose DeliveryToS3.Success average below 1 for 10 min. S3 PUTs are failing." + comparison_operator = "LessThanThreshold" + evaluation_periods = 2 + metric_name = "DeliveryToS3.Success" + namespace = "AWS/Firehose" + period = 300 + statistic = "Average" + threshold = 1 + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] + + dimensions = { + DeliveryStreamName = aws_kinesis_firehose_delivery_stream.unifi.name } } resource "aws_cloudwatch_metric_alarm" "status_check" { alarm_name = "EC2-StatusCheck-syslog-server" - alarm_description = "syslog-server EC2 status check failed (instance and/or system) for 10 min — host may be hung or unreachable." + alarm_description = "syslog-server EC2 status check failed (instance and/or system) for 10 min. Host may be hung or unreachable." comparison_operator = "GreaterThanOrEqualToThreshold" evaluation_periods = 2 metric_name = "StatusCheckFailed" @@ -36,7 +54,7 @@ resource "aws_cloudwatch_metric_alarm" "status_check" { resource "aws_cloudwatch_metric_alarm" "system_recover" { alarm_name = "EC2-StatusCheckSystem-syslog-server-recover" - alarm_description = "syslog-server EC2 system status check failed — underlying host impaired; auto-recovering onto new hardware." + alarm_description = "syslog-server EC2 system status check failed. Underlying host impaired; auto-recovering onto new hardware." comparison_operator = "GreaterThanOrEqualToThreshold" evaluation_periods = 2 metric_name = "StatusCheckFailed_System" diff --git a/terraform/athena.tf b/terraform/athena.tf new file mode 100644 index 0000000..8e02527 --- /dev/null +++ b/terraform/athena.tf @@ -0,0 +1,63 @@ +resource "aws_athena_workgroup" "this" { + name = local.athena_workgroup + + configuration { + enforce_workgroup_configuration = true + publish_cloudwatch_metrics_enabled = false + + result_configuration { + output_location = "s3://${aws_s3_bucket.unifi.bucket}/${local.athena_results_prefix}" + + encryption_configuration { + encryption_option = "SSE_S3" + } + } + } +} + +resource "aws_athena_named_query" "recent_denies" { + name = "unifi-recent-denies" + workgroup = aws_athena_workgroup.this.id + database = aws_glue_catalog_database.unifi.name + query = <<-SQL + SELECT timestamp, site, hostname, src, dst, proto, action, raw + FROM iptables + WHERE dt >= date_format(current_date - interval '7' day, '%Y-%m-%d') + AND action = 'deny' + ORDER BY timestamp DESC + LIMIT 200 + SQL +} + +resource "aws_athena_named_query" "src_dst_lookup" { + name = "unifi-src-dst-lookup" + workgroup = aws_athena_workgroup.this.id + database = aws_glue_catalog_database.unifi.name + query = <<-SQL + SELECT timestamp, format, site, hostname, src, dst, proto, action, raw + FROM iptables + WHERE dt >= date_format(current_date - interval '1' day, '%Y-%m-%d') + AND (src = 'x.x.x.x' OR dst = 'x.x.x.x') + ORDER BY timestamp DESC + LIMIT 200 + SQL +} + +resource "aws_athena_named_query" "cef_security" { + name = "unifi-cef-security" + workgroup = aws_athena_workgroup.this.id + database = aws_glue_catalog_database.unifi.name + query = <<-SQL + SELECT timestamp, site, hostname, src, dst, proto, action, raw + FROM cef + WHERE dt >= date_format(current_date - interval '7' day, '%Y-%m-%d') + AND ( + lower(raw) LIKE '%security%' + OR lower(raw) LIKE '%intrusion%' + OR lower(raw) LIKE '%blocked%' + OR lower(raw) LIKE '%threat%' + ) + ORDER BY timestamp DESC + LIMIT 200 + SQL +} diff --git a/terraform/ec2.tf b/terraform/ec2.tf index 017c7ad..73b2304 100644 --- a/terraform/ec2.tf +++ b/terraform/ec2.tf @@ -1,14 +1,15 @@ resource "aws_instance" "this" { ami = var.ami_id - instance_type = "t4g.nano" + instance_type = "t4g.small" subnet_id = aws_subnet.public.id vpc_security_group_ids = [aws_security_group.this.id] iam_instance_profile = aws_iam_instance_profile.this.name - user_data = file("${path.module}/user_data.sh") + associate_public_ip_address = true + user_data = local.user_data user_data_replace_on_change = true root_block_device { - volume_size = 30 + volume_size = 20 volume_type = "gp3" encrypted = true } @@ -23,17 +24,11 @@ resource "aws_instance" "this" { } } -resource "aws_eip" "this" { - domain = "vpc" - - tags = { - Name = "syslog-server" - } - - depends_on = [aws_internet_gateway.this] -} - -resource "aws_eip_association" "this" { - instance_id = aws_instance.this.id - allocation_id = aws_eip.this.id +locals { + user_data = templatefile("${path.module}/user_data.sh.tftpl", { + vector_yaml = templatefile("${path.module}/vector.yaml.tftpl", { + aws_region = var.aws_region + firehose_stream = aws_kinesis_firehose_delivery_stream.unifi.name + }) + }) } diff --git a/terraform/firehose.tf b/terraform/firehose.tf new file mode 100644 index 0000000..e60bb3d --- /dev/null +++ b/terraform/firehose.tf @@ -0,0 +1,105 @@ +data "aws_iam_policy_document" "firehose_assume" { + statement { + sid = "FirehoseAssume" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "firehose" { + statement { + sid = "S3Delivery" + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + resources = [ + aws_s3_bucket.unifi.arn, + "${aws_s3_bucket.unifi.arn}/*", + ] + } +} + +resource "aws_iam_role" "firehose" { + name = local.firehose_role_name + path = "/tf-managed/" + assume_role_policy = data.aws_iam_policy_document.firehose_assume.json + permissions_boundary = aws_iam_policy.instance_boundary.arn + + tags = { + Name = local.firehose_role_name + } +} + +resource "aws_iam_role_policy" "firehose" { + name = "s3-delivery" + role = aws_iam_role.firehose.id + policy = data.aws_iam_policy_document.firehose.json +} + +resource "aws_kinesis_firehose_delivery_stream" "unifi" { + name = local.firehose_name + destination = "extended_s3" + + extended_s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.unifi.arn + prefix = "format=!{partitionKeyFromQuery:format}/dt=!{timestamp:yyyy-MM-dd}/" + error_output_prefix = "errors/!{firehose:error-output-type}/dt=!{timestamp:yyyy-MM-dd}/" + buffering_size = 64 + buffering_interval = 300 + compression_format = "GZIP" + file_extension = ".json.gz" + + processing_configuration { + enabled = true + + processors { + type = "MetadataExtraction" + + parameters { + parameter_name = "JsonParsingEngine" + parameter_value = "JQ-1.6" + } + + parameters { + parameter_name = "MetadataExtractionQuery" + parameter_value = "{format:.format}" + } + } + + processors { + type = "AppendDelimiterToRecord" + + parameters { + parameter_name = "Delimiter" + parameter_value = "\\n" + } + } + } + + dynamic_partitioning_configuration { + enabled = true + } + + cloudwatch_logging_options { + enabled = false + } + } + + tags = { + Name = local.firehose_name + } + + depends_on = [aws_iam_role_policy.firehose] +} diff --git a/terraform/glue.tf b/terraform/glue.tf new file mode 100644 index 0000000..22b1abe --- /dev/null +++ b/terraform/glue.tf @@ -0,0 +1,59 @@ +resource "aws_glue_catalog_database" "unifi" { + name = local.glue_database_name +} + +locals { + glue_columns = [ + { name = "timestamp", type = "string" }, + { name = "site", type = "string" }, + { name = "format", type = "string" }, + { name = "hostname", type = "string" }, + { name = "src", type = "string" }, + { name = "dst", type = "string" }, + { name = "proto", type = "string" }, + { name = "action", type = "string" }, + { name = "raw", type = "string" }, + ] +} + +resource "aws_glue_catalog_table" "formats" { + for_each = toset(["cef", "iptables", "netflow"]) + + name = each.value + database_name = aws_glue_catalog_database.unifi.name + table_type = "EXTERNAL_TABLE" + + parameters = { + classification = "json" + compressionType = "gzip" + "projection.enabled" = "true" + "projection.dt.type" = "date" + "projection.dt.format" = "yyyy-MM-dd" + "projection.dt.range" = "2026-01-01,NOW" + "storage.location.template" = "s3://${aws_s3_bucket.unifi.bucket}/format=${each.value}/dt=$${dt}/" + } + + partition_keys { + name = "dt" + type = "string" + } + + storage_descriptor { + location = "s3://${aws_s3_bucket.unifi.bucket}/format=${each.value}/" + input_format = "org.apache.hadoop.mapred.TextInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" + + ser_de_info { + name = "json" + serialization_library = "org.openx.data.jsonserde.JsonSerDe" + } + + dynamic "columns" { + for_each = local.glue_columns + content { + name = columns.value.name + type = columns.value.type + } + } + } +} diff --git a/terraform/hcp_iam.tf b/terraform/hcp_iam.tf index 322cc6a..57ebdc7 100644 --- a/terraform/hcp_iam.tf +++ b/terraform/hcp_iam.tf @@ -1,4 +1,4 @@ -# HCP plan/apply roles for syslog-server-prod (PLAT-78 / PLAT-144). +# HCP plan/apply roles for syslog-server-prod (PLAT-78 / PLAT-206). # Copy of seahaven-org-baseline/examples/hcptf-workspace-iam/hcp_iam.tf.example # with the syslog-server EC2 service set. Create, do not import. # @@ -157,6 +157,19 @@ data "aws_iam_policy_document" "hcptf_scoped_iam" { } } + statement { + sid = "PassFirehoseRole" + effect = "Allow" + actions = ["iam:PassRole"] + resources = ["arn:aws:iam::${local.account_id}:role/tf-managed/${local.firehose_role_name}"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["firehose.amazonaws.com"] + } + } + statement { sid = "InstanceProfiles" effect = "Allow" @@ -248,34 +261,151 @@ data "aws_iam_policy_document" "hcptf_scoped_iam" { } data "aws_iam_policy_document" "hcptf_apply_services" { - # checkov:skip=CKV_AWS_111: EC2 VPC/instance lifecycle and describe APIs require Resource=*. Log group, alarm, and SNS writes are ARN-prefixed. + # checkov:skip=CKV_AWS_111: EC2 describe APIs and Glue catalog ARNs require Resource=*. S3, Firehose, Athena, and SNS writes are ARN-prefixed. statement { - sid = "CloudWatchLogs" + sid = "DescribeLogGroups" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + + statement { + sid = "DeleteLegacyCloudWatchLogGroup" effect = "Allow" actions = [ - "logs:CreateLogGroup", "logs:DeleteLogGroup", - "logs:PutRetentionPolicy", - "logs:DeleteRetentionPolicy", - "logs:TagResource", - "logs:UntagResource", "logs:ListTagsForResource", - "logs:AssociateKmsKey", - "logs:DisassociateKmsKey", + "logs:DeleteRetentionPolicy", ] resources = [ - "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:${local.log_group_name}", - "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:${local.log_group_name}:*", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:unifi-syslog", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:unifi-syslog:*", ] } statement { - sid = "CloudWatchLogsDescribe" + sid = "S3ArchiveBucket" + effect = "Allow" + actions = [ + "s3:CreateBucket", + "s3:DeleteBucket", + "s3:DeleteBucketPolicy", + "s3:GetAccelerateConfiguration", + "s3:GetBucketAcl", + "s3:GetBucketCORS", + "s3:GetBucketLocation", + "s3:GetBucketLogging", + "s3:GetBucketNotification", + "s3:GetBucketObjectLockConfiguration", + "s3:GetBucketOwnershipControls", + "s3:GetBucketPolicy", + "s3:GetBucketPolicyStatus", + "s3:GetBucketPublicAccessBlock", + "s3:GetBucketRequestPayment", + "s3:GetBucketTagging", + "s3:GetBucketVersioning", + "s3:GetBucketWebsite", + "s3:GetEncryptionConfiguration", + "s3:GetLifecycleConfiguration", + "s3:GetReplicationConfiguration", + "s3:ListBucket", + "s3:PutBucketOwnershipControls", + "s3:PutBucketPolicy", + "s3:PutBucketPublicAccessBlock", + "s3:PutBucketTagging", + "s3:PutEncryptionConfiguration", + "s3:PutLifecycleConfiguration", + ] + resources = ["arn:aws:s3:::${local.bucket_name}"] + } + + statement { + sid = "S3ArchiveObjects" + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:DeleteObject", + "s3:GetObject", + "s3:PutObject", + ] + resources = ["arn:aws:s3:::${local.bucket_name}/*"] + } + + statement { + sid = "FirehoseList" effect = "Allow" - actions = ["logs:DescribeLogGroups"] + actions = ["firehose:ListDeliveryStreams"] resources = ["*"] } + statement { + sid = "FirehoseStream" + effect = "Allow" + actions = [ + "firehose:CreateDeliveryStream", + "firehose:DeleteDeliveryStream", + "firehose:DescribeDeliveryStream", + "firehose:ListTagsForDeliveryStream", + "firehose:StartDeliveryStreamEncryption", + "firehose:StopDeliveryStreamEncryption", + "firehose:TagDeliveryStream", + "firehose:UntagDeliveryStream", + "firehose:UpdateDestination", + ] + resources = [ + "arn:aws:firehose:${var.aws_region}:${local.account_id}:deliverystream/${local.firehose_name}", + ] + } + + statement { + sid = "GlueCatalog" + effect = "Allow" + actions = [ + "glue:CreateDatabase", + "glue:DeleteDatabase", + "glue:GetDatabase", + "glue:GetDatabases", + "glue:UpdateDatabase", + "glue:CreateTable", + "glue:DeleteTable", + "glue:GetTable", + "glue:GetTables", + "glue:UpdateTable", + "glue:GetPartition", + "glue:GetPartitions", + "glue:BatchCreatePartition", + "glue:TagResource", + "glue:UntagResource", + "glue:GetTags", + ] + resources = [ + "arn:aws:glue:${var.aws_region}:${local.account_id}:catalog", + "arn:aws:glue:${var.aws_region}:${local.account_id}:database/${local.glue_database_name}", + "arn:aws:glue:${var.aws_region}:${local.account_id}:table/${local.glue_database_name}/*", + ] + } + + statement { + sid = "AthenaWorkgroup" + effect = "Allow" + actions = [ + "athena:CreateWorkGroup", + "athena:DeleteWorkGroup", + "athena:GetWorkGroup", + "athena:UpdateWorkGroup", + "athena:CreateNamedQuery", + "athena:DeleteNamedQuery", + "athena:GetNamedQuery", + "athena:ListNamedQueries", + "athena:ListTagsForResource", + "athena:TagResource", + "athena:UntagResource", + ] + resources = [ + "arn:aws:athena:${var.aws_region}:${local.account_id}:workgroup/${local.athena_workgroup}", + ] + } + statement { sid = "CloudWatchAlarms" effect = "Allow" @@ -365,6 +495,22 @@ data "aws_iam_policy_document" "hcptf_apply_services" { "ec2:DescribeVpcAttribute", "ec2:DescribeVpcs", "ec2:DescribePrefixLists", + "ec2:DescribeVpnConnections", + "ec2:DescribeVpnGateways", + "ec2:DescribeCustomerGateways", + "ec2:CreateVpnGateway", + "ec2:DeleteVpnGateway", + "ec2:AttachVpnGateway", + "ec2:DetachVpnGateway", + "ec2:CreateCustomerGateway", + "ec2:DeleteCustomerGateway", + "ec2:CreateVpnConnection", + "ec2:DeleteVpnConnection", + "ec2:CreateVpnConnectionRoute", + "ec2:DeleteVpnConnectionRoute", + "ec2:ModifyVpnConnection", + "ec2:ModifyVpnConnectionOptions", + "ec2:ModifyVpnTunnelOptions", "ec2:DetachInternetGateway", "ec2:DisassociateAddress", "ec2:DisassociateRouteTable", @@ -466,15 +612,82 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { } statement { - sid = "RefreshLogs" + sid = "RefreshS3" effect = "Allow" actions = [ - "logs:DescribeLogGroups", - "logs:ListTagsForResource", + "s3:GetAccelerateConfiguration", + "s3:GetBucketAcl", + "s3:GetBucketCORS", + "s3:GetBucketLocation", + "s3:GetBucketLogging", + "s3:GetBucketNotification", + "s3:GetBucketObjectLockConfiguration", + "s3:GetBucketOwnershipControls", + "s3:GetBucketPolicy", + "s3:GetBucketPolicyStatus", + "s3:GetBucketPublicAccessBlock", + "s3:GetBucketRequestPayment", + "s3:GetBucketTagging", + "s3:GetBucketVersioning", + "s3:GetBucketWebsite", + "s3:GetEncryptionConfiguration", + "s3:GetLifecycleConfiguration", + "s3:GetReplicationConfiguration", + "s3:ListBucket", ] + resources = ["arn:aws:s3:::${local.bucket_name}"] + } + + statement { + sid = "RefreshFirehoseList" + effect = "Allow" + actions = ["firehose:ListDeliveryStreams"] resources = ["*"] } + statement { + sid = "RefreshFirehose" + effect = "Allow" + actions = [ + "firehose:DescribeDeliveryStream", + "firehose:ListTagsForDeliveryStream", + ] + resources = [ + "arn:aws:firehose:${var.aws_region}:${local.account_id}:deliverystream/${local.firehose_name}", + ] + } + + statement { + sid = "RefreshGlue" + effect = "Allow" + actions = [ + "glue:GetDatabase", + "glue:GetDatabases", + "glue:GetTable", + "glue:GetTables", + "glue:GetTags", + ] + resources = [ + "arn:aws:glue:${var.aws_region}:${local.account_id}:catalog", + "arn:aws:glue:${var.aws_region}:${local.account_id}:database/${local.glue_database_name}", + "arn:aws:glue:${var.aws_region}:${local.account_id}:table/${local.glue_database_name}/*", + ] + } + + statement { + sid = "RefreshAthena" + effect = "Allow" + actions = [ + "athena:GetWorkGroup", + "athena:GetNamedQuery", + "athena:ListNamedQueries", + "athena:ListTagsForResource", + ] + resources = [ + "arn:aws:athena:${var.aws_region}:${local.account_id}:workgroup/${local.athena_workgroup}", + ] + } + statement { sid = "RefreshAlarms" effect = "Allow" @@ -524,6 +737,9 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { "ec2:DescribeVolumes", "ec2:DescribeVpcAttribute", "ec2:DescribeVpcs", + "ec2:DescribeVpnConnections", + "ec2:DescribeVpnGateways", + "ec2:DescribeCustomerGateways", "ec2:GetConsoleOutput", ] resources = ["*"] diff --git a/terraform/iam.tf b/terraform/iam.tf index 88fe497..31fda65 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -1,31 +1,37 @@ -# Instance permissions boundary. Created on the first (bootstrap) apply. +# Instance (and Firehose delivery role) permissions boundary. # The scoped apply role denies iam:CreatePolicy / CreatePolicyVersion, # so later edits to this document need the hcptf-bootstrap window. data "aws_iam_policy_document" "instance_boundary" { - # checkov:skip=CKV_AWS_111: SSM and CloudWatch agent managed policies require Resource=* for ssmmessages, ec2messages, and describe APIs. Log writes are ARN-prefixed. + # checkov:skip=CKV_AWS_111: SSM managed policy requires Resource=* for ssmmessages and describe APIs. Firehose and S3 writes are ARN-prefixed. statement { - sid = "CloudWatchLogsWrite" + sid = "FirehosePut" effect = "Allow" actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:PutLogEvents", - "logs:PutRetentionPolicy", + "firehose:PutRecord", + "firehose:PutRecordBatch", ] resources = [ - "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:${local.log_group_name}", - "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:${local.log_group_name}:*", + "arn:aws:firehose:${var.aws_region}:${local.account_id}:deliverystream/${local.firehose_name}", ] } statement { - sid = "CloudWatchLogsDescribe" - effect = "Allow" - actions = ["logs:DescribeLogGroups"] - resources = ["*"] + sid = "S3Archive" + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + "s3:DeleteObject", + ] + resources = [ + "arn:aws:s3:::${local.bucket_name}", + "arn:aws:s3:::${local.bucket_name}/*", + ] } statement { @@ -94,7 +100,6 @@ data "aws_iam_policy_document" "instance_boundary" { resources = [ "arn:aws:ssm:${var.aws_region}::parameter/aws/service/*", "arn:aws:ssm:${var.aws_region}:${local.account_id}:parameter/aws/service/*", - "arn:aws:ssm:${var.aws_region}:${local.account_id}:parameter/AmazonCloudWatch-*", ] } @@ -126,10 +131,10 @@ data "aws_iam_policy_document" "instance_boundary" { } resource "aws_iam_policy" "instance_boundary" { - # checkov:skip=CKV_AWS_111: SSM and CloudWatch agent managed policies require Resource=* for ssmmessages, ec2messages, and describe APIs. Log writes are ARN-prefixed. + # checkov:skip=CKV_AWS_111: SSM managed policy requires Resource=* for ssmmessages and describe APIs. Firehose and S3 writes are ARN-prefixed. name = local.boundary_name path = "/tf-managed/" - description = "Per-workload EC2 permissions boundary for syslog-server (PLAT-78)." + description = "Per-workload permissions boundary for syslog-server EC2 and Firehose (PLAT-206)." policy = data.aws_iam_policy_document.instance_boundary.json } @@ -162,9 +167,24 @@ resource "aws_iam_role_policy_attachment" "ssm" { policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } -resource "aws_iam_role_policy_attachment" "cloudwatch_agent" { - role = aws_iam_role.instance.name - policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" +data "aws_iam_policy_document" "instance_firehose" { + statement { + sid = "FirehosePut" + effect = "Allow" + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + resources = [ + "arn:aws:firehose:${var.aws_region}:${local.account_id}:deliverystream/${local.firehose_name}", + ] + } +} + +resource "aws_iam_role_policy" "instance_firehose" { + name = "firehose-put" + role = aws_iam_role.instance.id + policy = data.aws_iam_policy_document.instance_firehose.json } resource "aws_iam_instance_profile" "this" { diff --git a/terraform/locals.tf b/terraform/locals.tf index adeeca6..8b3bdc6 100644 --- a/terraform/locals.tf +++ b/terraform/locals.tf @@ -12,22 +12,21 @@ locals { instance_role_name = "syslog-server-role" instance_profile_name = "syslog-server-profile" + firehose_role_name = "syslog-server-firehose-role" boundary_name = "syslog-server-instance-boundary" - log_group_name = "unifi-syslog" site_alerts_arn = "arn:aws:sns:${var.aws_region}:${local.account_id}:site-alerts" - vpc_cidr = "10.40.0.0/16" - public_subnet_cidr = "10.40.10.0/24" - office_cidrs = ["47.21.61.4/32", "96.250.164.146/32"] - office_vpn_cidr = "10.10.0.0/16" - vpn_pool_cidr = "10.30.0.0/16" - syslog_vpc_cidr = local.vpc_cidr - syslog_ingress_cidrs = concat( - local.office_cidrs, - [local.office_vpn_cidr, local.vpn_pool_cidr, local.syslog_vpc_cidr], - ) - ssh_ingress_cidrs = concat( - local.office_cidrs, - [local.office_vpn_cidr, local.syslog_vpc_cidr], - ) + vpc_cidr = "10.40.0.0/16" + public_subnet_cidr = "10.40.10.0/24" + office_lan_cidrs = ["10.10.0.0/16", "10.30.0.0/16"] + ronkonkoma_wan_ip = "47.21.61.4" + customer_gateway_bgp_asn = 65000 + + bucket_name = "syslog-server-unifi-logs-${local.account_id}" + firehose_name = "syslog-server-unifi" + glue_database_name = "unifi" + athena_workgroup = "syslog-server" + athena_results_prefix = "athena-results/" + logs_expire_days = 90 + athena_results_expire_days = 30 } diff --git a/terraform/logs.tf b/terraform/logs.tf deleted file mode 100644 index c6d2bed..0000000 --- a/terraform/logs.tf +++ /dev/null @@ -1,4 +0,0 @@ -resource "aws_cloudwatch_log_group" "unifi_syslog" { - name = local.log_group_name - retention_in_days = 90 -} diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 292fe3e..e902de0 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -3,19 +3,56 @@ output "instance_id" { value = aws_instance.this.id } -output "public_ip" { - description = "Elastic IP — UniFi remote-syslog forwarding target." - value = aws_eip.this.public_ip +output "private_ip" { + description = "Private IP — UniFi SIEM/IPFIX forwarding target over IPsec." + value = aws_instance.this.private_ip } -output "allocation_id" { - description = "Elastic IP allocation id." - value = aws_eip.this.allocation_id +output "vpn_connection_id" { + description = "Prod office IPsec connection. Configure a UniFi site-to-site VPN to these tunnels with remote network 10.40.0.0/16." + value = aws_vpn_connection.office.id } -output "log_group_name" { - description = "CloudWatch Logs group the agent ships remote syslog into." - value = aws_cloudwatch_log_group.unifi_syslog.name +output "vpn_tunnel1_address" { + description = "AWS tunnel 1 outside IP for the UniFi IPsec peer." + value = aws_vpn_connection.office.tunnel1_address +} + +output "vpn_tunnel2_address" { + description = "AWS tunnel 2 outside IP for the UniFi IPsec peer." + value = aws_vpn_connection.office.tunnel2_address +} + +output "vpn_tunnel1_preshared_key" { + description = "IPsec PSK for tunnel 1. Read with terraform output -raw after apply. Do not commit." + value = aws_vpn_connection.office.tunnel1_preshared_key + sensitive = true +} + +output "vpn_tunnel2_preshared_key" { + description = "IPsec PSK for tunnel 2. Read with terraform output -raw after apply. Do not commit." + value = aws_vpn_connection.office.tunnel2_preshared_key + sensitive = true +} + +output "bucket_name" { + description = "S3 bucket holding 90-day UniFi JSON archives." + value = aws_s3_bucket.unifi.bucket +} + +output "firehose_name" { + description = "Kinesis Data Firehose delivery stream name." + value = aws_kinesis_firehose_delivery_stream.unifi.name +} + +output "athena_workgroup" { + description = "Athena workgroup for UniFi log search." + value = aws_athena_workgroup.this.name +} + +output "glue_database" { + description = "Glue catalog database with cef, iptables, and netflow tables." + value = aws_glue_catalog_database.unifi.name } output "hcptf_apply_role_arn" { diff --git a/terraform/s3.tf b/terraform/s3.tf new file mode 100644 index 0000000..2a6f77c --- /dev/null +++ b/terraform/s3.tf @@ -0,0 +1,106 @@ +resource "aws_s3_bucket" "unifi" { + bucket = local.bucket_name + + tags = { + Name = local.bucket_name + } +} + +resource "aws_s3_bucket_public_access_block" "unifi" { + bucket = aws_s3_bucket.unifi.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_ownership_controls" "unifi" { + bucket = aws_s3_bucket.unifi.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "unifi" { + bucket = aws_s3_bucket.unifi.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "unifi" { + bucket = aws_s3_bucket.unifi.id + + rule { + id = "expire-logs" + status = "Enabled" + + filter { + prefix = "format=" + } + + expiration { + days = local.logs_expire_days + } + } + + rule { + id = "expire-athena-results" + status = "Enabled" + + filter { + prefix = local.athena_results_prefix + } + + expiration { + days = local.athena_results_expire_days + } + } + + rule { + id = "expire-errors" + status = "Enabled" + + filter { + prefix = "errors/" + } + + expiration { + days = 14 + } + } +} + +data "aws_iam_policy_document" "bucket" { + statement { + sid = "DenyInsecureTransport" + effect = "Deny" + actions = ["s3:*"] + + principals { + type = "*" + identifiers = ["*"] + } + + resources = [ + aws_s3_bucket.unifi.arn, + "${aws_s3_bucket.unifi.arn}/*", + ] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + +resource "aws_s3_bucket_policy" "unifi" { + bucket = aws_s3_bucket.unifi.id + policy = data.aws_iam_policy_document.bucket.json +} diff --git a/terraform/user_data.sh b/terraform/user_data.sh deleted file mode 100644 index da1d592..0000000 --- a/terraform/user_data.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -# ── 1 GiB swap (build headroom + stability on the 512 MiB t4g.nano) ── -if [ ! -f /swapfile ]; then - fallocate -l 1G /swapfile || dd if=/dev/zero of=/swapfile bs=1M count=1024 - chmod 600 /swapfile - mkswap /swapfile - echo '/swapfile none swap sw 0 0' >> /etc/fstab -fi -swapon -a || true - -# ── rsyslog: listen on UDP/TCP 514 ── -dnf install -y rsyslog -cat > /etc/rsyslog.d/10-listen.conf <<'EOF' -module(load="imudp") -input(type="imudp" port="514") -module(load="imtcp") -input(type="imtcp" port="514") -EOF - -# ── Write remote syslog to /var/log/remote//.log ── -cat > /etc/rsyslog.d/20-remote.conf <<'EOF' -template(name="RemoteHost" type="string" string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log") -if $fromhost-ip != '127.0.0.1' then { - action(type="omfile" dynaFile="RemoteHost" createDirs="on") - stop -} -EOF - -mkdir -p /var/log/remote -systemctl enable rsyslog -systemctl restart rsyslog - -# ── Rotate /var/log/remote so it can't grow unbounded ── -# CloudWatch (90d) is the system of record; these local files are just a -# spool for the CW agent, so keep only a short window. copytruncate keeps -# rsyslog's open dynaFile handles valid (truncate in place, same inode). -cat > /etc/logrotate.d/remote-syslog <<'EOF' -/var/log/remote/*/*.log { - daily - rotate 7 - compress - delaycompress - missingok - notifempty - copytruncate -} -EOF - -# ── CloudWatch agent: ship /var/log/remote/**/*.log to unifi-syslog ── -dnf install -y amazon-cloudwatch-agent -cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json <<'EOF' -{ - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/var/log/remote/**/*.log", - "log_group_name": "unifi-syslog", - "log_stream_name": "{hostname}/{file_name}", - "retention_in_days": 90 - } - ] - } - } - } -} -EOF - -/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s -systemctl enable amazon-cloudwatch-agent - -# ── NetFlow/IPFIX collectors (nfcapd) ── -# nfdump is not packaged for AL2023; build 1.6.23 from source (needs -# rrdtool-devel for librrd). Reconstructed under IaC for INFRA-12 — the -# original instance ran these as hand-installed systemd units. Captures -# are local-only (no consumer/shipping today); 30-day retention enforced. -dnf install -y gcc gcc-c++ make automake autoconf libtool flex bison libpcap-devel zlib-devel bzip2-devel rrdtool-devel tar -NFVER=1.6.23 -curl -sfL https://github.com/phaag/nfdump/archive/refs/tags/v${NFVER}.tar.gz | tar xz -C /tmp -( cd /tmp/nfdump-${NFVER} && ./autogen.sh && ./configure && make -j1 && make install ) -ldconfig - -mkdir -p /var/log/netflow/ronkonkoma /var/log/netflow/locust -chown -R ec2-user:ec2-user /var/log/netflow - -# Ronkonkoma gateway -> UDP 2055 -cat > /etc/systemd/system/nfcapd.service <<'EOF' -[Unit] -Description=nfcapd NetFlow collector (Ronkonkoma, udp/2055) -After=network.target -[Service] -Type=simple -User=ec2-user -ExecStart=/usr/local/bin/nfcapd -p 2055 -l /var/log/netflow/ronkonkoma -Restart=always -[Install] -WantedBy=multi-user.target -EOF - -# Locust Ave gateway -> UDP 2056 -cat > /etc/systemd/system/nfcapd-locust.service <<'EOF' -[Unit] -Description=nfcapd NetFlow collector (Locust Ave, udp/2056) -After=network.target -[Service] -Type=simple -User=ec2-user -ExecStart=/usr/local/bin/nfcapd -p 2056 -l /var/log/netflow/locust -Restart=always -[Install] -WantedBy=multi-user.target -EOF - -# 30-day retention sweep (daily 03:30 UTC) -cat > /usr/local/sbin/netflow-retention.sh <<'EOF' -#!/bin/bash -find /var/log/netflow -type f -name 'nfcapd.*' -mtime +30 -delete -EOF -chmod +x /usr/local/sbin/netflow-retention.sh -cat > /etc/systemd/system/netflow-retention.service <<'EOF' -[Unit] -Description=Delete NetFlow captures older than 30 days -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/netflow-retention.sh -EOF -cat > /etc/systemd/system/netflow-retention.timer <<'EOF' -[Unit] -Description=Daily NetFlow retention sweep -[Timer] -OnCalendar=*-*-* 03:30:00 UTC -Persistent=true -[Install] -WantedBy=timers.target -EOF - -systemctl daemon-reload -systemctl enable --now nfcapd.service nfcapd-locust.service netflow-retention.timer diff --git a/terraform/user_data.sh.tftpl b/terraform/user_data.sh.tftpl new file mode 100644 index 0000000..a2f8a6c --- /dev/null +++ b/terraform/user_data.sh.tftpl @@ -0,0 +1,25 @@ +#!/bin/bash +set -euxo pipefail + +# Vector: listen on UDP/TCP 514 and IPFIX 2055/2056, parse, ship to Firehose. +curl -sSL https://setup.vector.dev | bash +dnf install -y vector + +install -d -m 0755 /etc/vector /var/lib/vector +cat > /etc/vector/vector.yaml <<'VECTOREOF' +${vector_yaml} +VECTOREOF +chmod 0644 /etc/vector/vector.yaml +vector validate /etc/vector/vector.yaml + +install -d -m 0755 /etc/systemd/system/vector.service.d +cat > /etc/systemd/system/vector.service.d/override.conf <<'EOF' +[Service] +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID +NoNewPrivileges=false +EOF + +systemctl daemon-reload +systemctl enable --now vector +systemctl restart vector diff --git a/terraform/variables.tf b/terraform/variables.tf index aa0bcb4..1b42e8c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -11,7 +11,7 @@ variable "ami_id" { } variable "no_logs_treat_missing_data" { - description = "CloudWatch treat_missing_data for Syslog-NoIncomingLogs. Keep notBreaching until UniFi points at the new EIP, then set breaching." + description = "CloudWatch treat_missing_data for Syslog-NoIncomingRecords. Keep notBreaching until UniFi points at the private IP, then set breaching." type = string default = "notBreaching" diff --git a/terraform/vector.yaml.tftpl b/terraform/vector.yaml.tftpl new file mode 100644 index 0000000..9b82c63 --- /dev/null +++ b/terraform/vector.yaml.tftpl @@ -0,0 +1,146 @@ +data_dir: /var/lib/vector + +sources: + syslog_udp: + type: socket + address: 0.0.0.0:514 + mode: udp + max_length: 65507 + decoding: + codec: bytes + syslog_tcp: + type: socket + address: 0.0.0.0:514 + mode: tcp + decoding: + codec: bytes + framing: + method: newline_delimited + # Vector has no released IPFIX decoder. Archive datagrams with a site tag. + netflow_ronkonkoma: + type: socket + address: 0.0.0.0:2055 + mode: udp + max_length: 65507 + decoding: + codec: bytes + netflow_locust: + type: socket + address: 0.0.0.0:2056 + mode: udp + max_length: 65507 + decoding: + codec: bytes + +transforms: + parse_syslog: + type: remap + inputs: [syslog_udp, syslog_tcp] + source: |- + raw = to_string(.message) ?? encode_json(.) + src_ip = to_string(.host) ?? "" + site = "unknown" + if starts_with(src_ip, "10.10.") { + site = "ronkonkoma" + } + if starts_with(src_ip, "10.30.") { + site = "locust" + } + + format = "other" + if contains(raw, "CEF:") { + format = "cef" + } else if contains(raw, "SRC=") && contains(raw, "DST=") { + format = "iptables" + } + + src = null + dst = null + proto = null + action = null + hostname = to_string(.hostname) ?? "" + + if format == "iptables" { + src_m, err = parse_regex(raw, r'SRC=(?P[0-9.]+)') + if err == null { src = src_m.v } + dst_m, err = parse_regex(raw, r'DST=(?P[0-9.]+)') + if err == null { dst = dst_m.v } + proto_m, err = parse_regex(raw, r'PROTO=(?P[A-Za-z0-9]+)') + if err == null { proto = proto_m.v } + if contains(raw, "DROP") || contains(raw, "REJECT") { + action = "deny" + } else if contains(raw, "ACCEPT") { + action = "allow" + } + } + + if format == "cef" { + src_m, err = parse_regex(raw, r'(?:src|sourceAddress)=(?P[0-9.]+)') + if err == null { src = src_m.v } + dst_m, err = parse_regex(raw, r'(?:dst|destinationAddress)=(?P[0-9.]+)') + if err == null { dst = dst_m.v } + proto_m, err = parse_regex(raw, r'proto=(?P[A-Za-z0-9]+)') + if err == null { proto = proto_m.v } + if contains(upcase(raw), "BLOCK") || contains(upcase(raw), "DENY") || contains(upcase(raw), "DROP") { + action = "deny" + } + host_m, err = parse_regex(raw, r'UNIFIhost=(?P[^ ]+)') + if err == null { hostname = host_m.v } + } + + . = { + "timestamp": format_timestamp!(now(), "%Y-%m-%dT%H:%M:%SZ"), + "site": site, + "format": format, + "hostname": hostname, + "src": src, + "dst": dst, + "proto": proto, + "action": action, + "raw": raw + } + + parse_netflow_ronkonkoma: + type: remap + inputs: [netflow_ronkonkoma] + source: |- + payload = to_string(.message) ?? encode_json(.) + . = { + "timestamp": format_timestamp!(now(), "%Y-%m-%dT%H:%M:%SZ"), + "site": "ronkonkoma", + "format": "netflow", + "hostname": "", + "src": null, + "dst": null, + "proto": "ipfix", + "action": null, + "raw": encode_base64(payload) ?? payload + } + + parse_netflow_locust: + type: remap + inputs: [netflow_locust] + source: |- + payload = to_string(.message) ?? encode_json(.) + . = { + "timestamp": format_timestamp!(now(), "%Y-%m-%dT%H:%M:%SZ"), + "site": "locust", + "format": "netflow", + "hostname": "", + "src": null, + "dst": null, + "proto": "ipfix", + "action": null, + "raw": encode_base64(payload) ?? payload + } + +sinks: + firehose: + type: aws_kinesis_firehose + inputs: [parse_syslog, parse_netflow_ronkonkoma, parse_netflow_locust] + region: ${aws_region} + stream_name: ${firehose_stream} + encoding: + codec: json + request: + timeout_secs: 30 diff --git a/terraform/vpc.tf b/terraform/vpc.tf index 4471da1..3fbdf18 100644 --- a/terraform/vpc.tf +++ b/terraform/vpc.tf @@ -45,14 +45,60 @@ resource "aws_route" "public_default" { gateway_id = aws_internet_gateway.this.id } +resource "aws_route" "office_lans" { + for_each = toset(local.office_lan_cidrs) + + route_table_id = aws_route_table.public.id + destination_cidr_block = each.value + gateway_id = aws_vpn_gateway.office.id +} + resource "aws_route_table_association" "public" { subnet_id = aws_subnet.public.id route_table_id = aws_route_table.public.id } +# Prod has no existing IPsec. Mgmt still owns the 10.20 tunnel. +# This VGW is a second child SA so UniFi can reach 10.40.0.0/16 privately. +resource "aws_vpn_gateway" "office" { + vpc_id = aws_vpc.this.id + + tags = { + Name = "syslog-server-office" + } +} + +resource "aws_customer_gateway" "ronkonkoma" { + bgp_asn = local.customer_gateway_bgp_asn + ip_address = local.ronkonkoma_wan_ip + type = "ipsec.1" + + tags = { + Name = "syslog-server-ronkonkoma" + } +} + +resource "aws_vpn_connection" "office" { + customer_gateway_id = aws_customer_gateway.ronkonkoma.id + vpn_gateway_id = aws_vpn_gateway.office.id + type = "ipsec.1" + static_routes_only = true + + tags = { + Name = "syslog-server-office" + } +} + +resource "aws_vpn_connection_route" "office_lans" { + for_each = toset(local.office_lan_cidrs) + + destination_cidr_block = each.value + vpn_connection_id = aws_vpn_connection.office.id +} + resource "aws_security_group" "this" { name = "syslog-server" - description = "Syslog collector - rsyslog 514 from office + VPC" + description = "UniFi syslog/IPFIX collector over office IPsec" vpc_id = aws_vpc.this.id tags = { @@ -64,60 +110,49 @@ resource "aws_vpc_security_group_egress_rule" "all" { security_group_id = aws_security_group.this.id ip_protocol = "-1" cidr_ipv4 = "0.0.0.0/0" - description = "All outbound for package installs and CloudWatch" + description = "Outbound for Vector install, Firehose, and SSM" } resource "aws_vpc_security_group_ingress_rule" "syslog_tcp" { - for_each = toset(local.syslog_ingress_cidrs) + for_each = toset(local.office_lan_cidrs) security_group_id = aws_security_group.this.id ip_protocol = "tcp" from_port = 514 to_port = 514 cidr_ipv4 = each.value - description = "syslog TCP 514" + description = "syslog TCP 514 from office LAN" } resource "aws_vpc_security_group_ingress_rule" "syslog_udp" { - for_each = toset(local.syslog_ingress_cidrs) + for_each = toset(local.office_lan_cidrs) security_group_id = aws_security_group.this.id ip_protocol = "udp" from_port = 514 to_port = 514 cidr_ipv4 = each.value - description = "syslog UDP 514" -} - -resource "aws_vpc_security_group_ingress_rule" "ssh" { - for_each = toset(local.ssh_ingress_cidrs) - - security_group_id = aws_security_group.this.id - ip_protocol = "tcp" - from_port = 22 - to_port = 22 - cidr_ipv4 = each.value - description = "SSH break-glass" + description = "syslog UDP 514 from office LAN" } resource "aws_vpc_security_group_ingress_rule" "netflow_2055" { - for_each = toset(local.office_cidrs) + for_each = toset(local.office_lan_cidrs) security_group_id = aws_security_group.this.id ip_protocol = "udp" from_port = 2055 to_port = 2055 cidr_ipv4 = each.value - description = "netflow/sflow UDP 2055" + description = "NetFlow/IPFIX UDP 2055 Ronkonkoma" } resource "aws_vpc_security_group_ingress_rule" "netflow_2056" { - for_each = toset(local.office_cidrs) + for_each = toset(local.office_lan_cidrs) security_group_id = aws_security_group.this.id ip_protocol = "udp" from_port = 2056 to_port = 2056 cidr_ipv4 = each.value - description = "netflow/sflow UDP 2056" + description = "NetFlow/IPFIX UDP 2056 Locust" }