syslog-server/lib/syslog-server-stack.ts

import * as cdk from "aws-cdk-lib";
import * as ec2 from "aws-cdk-lib/aws-ec2";
import * as iam from "aws-cdk-lib/aws-iam";
import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
import * as cwactions from "aws-cdk-lib/aws-cloudwatch-actions";
import * as sns from "aws-cdk-lib/aws-sns";
import { Construct } from "constructs";

/**
 * syslog-server — EC2 collector that receives remote syslog (UDP/TCP 514) from
 * the office UniFi fleet over the EIP and ships it to the `unifi-syslog`
 * CloudWatch Logs group via the CloudWatch agent.
 *
 * Brought under IaC for INFRA-12 (AWS audit L-6). Recreated to mirror the
 * file-share/forgejo CDK pattern; the existing EIP (184.72.154.32) is imported
 * by allocation ID and re-associated so the forwarding target is unchanged.
 *
 * The `unifi-syslog` log group is intentionally NOT a CloudFormation resource:
 * it holds 90 days of history and is created/retained by the CloudWatch agent
 * per the user-data config below (log_group_name + retention_in_days). Managing
 * it as a CFN resource would either collide with the live group on create or
 * risk deleting the history on a future replacement. The agent owns it; this
 * stack owns the instance that runs the agent.
 */
export class SyslogServerStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    const vpc = ec2.Vpc.fromLookup(this, "SeaHavenVpc", {
      vpcId: "vpc-0d3d4b67bd0cf8a68",
    });

    // Public subnet (IGW route present) — the instance must be internet-facing
    // so the office gateways can forward syslog to the EIP.
    const publicSubnet = ec2.Subnet.fromSubnetAttributes(this, "PublicSubnet", {
      subnetId: "subnet-0eea820effe1b3ae5",
      availabilityZone: "us-east-1a",
    });

    // Office public IPs that forward syslog (see reference_office_ips).
    const OFFICE_1 = "47.21.61.4/32";
    const OFFICE_2 = "96.250.164.146/32";

    const sg = new ec2.SecurityGroup(this, "SecurityGroup", {
      vpc,
      securityGroupName: "syslog-server",
      description: "Syslog collector - rsyslog 514 from office + VPC",
      allowAllOutbound: true,
    });

    // Remote syslog (UDP + TCP 514) from the office public IPs and the internal
    // VPC / VPN CIDRs.
    for (const proto of [ec2.Port.tcp(514), ec2.Port.udp(514)]) {
      sg.addIngressRule(ec2.Peer.ipv4(OFFICE_1), proto, "syslog from office-1");
      sg.addIngressRule(ec2.Peer.ipv4(OFFICE_2), proto, "syslog from office-2");
      sg.addIngressRule(ec2.Peer.ipv4("10.10.0.0/16"), proto, "syslog from office VPN");
      sg.addIngressRule(ec2.Peer.ipv4("10.20.0.0/16"), proto, "syslog from VPC");
      sg.addIngressRule(ec2.Peer.ipv4("10.30.0.0/16"), proto, "syslog from VPN pool");
    }

    // SSH (SSM is the primary access path; 22 kept for break-glass from office/VPC).
    sg.addIngressRule(ec2.Peer.ipv4(OFFICE_1), ec2.Port.tcp(22), "SSH from office-1");
    sg.addIngressRule(ec2.Peer.ipv4(OFFICE_2), ec2.Port.tcp(22), "SSH from office-2");
    sg.addIngressRule(ec2.Peer.ipv4("10.10.0.0/16"), ec2.Port.tcp(22), "SSH from office VPN");
    sg.addIngressRule(ec2.Peer.ipv4("10.20.0.0/16"), ec2.Port.tcp(22), "SSH from VPC");

    // NetFlow / sFlow ingress reserved from the office IPs. No collector is
    // configured in user-data yet; kept to preserve the prior capability.
    for (const port of [ec2.Port.udp(2055), ec2.Port.udp(2056)]) {
      sg.addIngressRule(ec2.Peer.ipv4(OFFICE_1), port, "netflow/sflow from office-1");
      sg.addIngressRule(ec2.Peer.ipv4(OFFICE_2), port, "netflow/sflow from office-2");
    }

    const role = new iam.Role(this, "InstanceRole", {
      roleName: "syslog-server-role",
      assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"),
      managedPolicies: [
        iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMManagedInstanceCore"),
        iam.ManagedPolicy.fromAwsManagedPolicyName("CloudWatchAgentServerPolicy"),
      ],
    });

    const userData = ec2.UserData.forLinux();
    userData.addCommands(
      "set -euxo pipefail",
      "",
      "# ── 1 GiB swap (build headroom + stability on the 512 MiB t4g.nano) ──",
      "if [ ! -f /swapfile ]; then",
      "  fallocate -l 1G /swapfile || dd if=/dev/zero of=/swapfile bs=1M count=1024",
      "  chmod 600 /swapfile",
      "  mkswap /swapfile",
      "  echo '/swapfile none swap sw 0 0' >> /etc/fstab",
      "fi",
      "swapon -a || true",
      "",
      "# ── rsyslog: listen on UDP/TCP 514 ──",
      "dnf install -y rsyslog",
      "cat > /etc/rsyslog.d/10-listen.conf <<'EOF'",
      'module(load="imudp")',
      'input(type="imudp" port="514")',
      'module(load="imtcp")',
      'input(type="imtcp" port="514")',
      "EOF",
      "",
      "# ── Write remote syslog to /var/log/remote/<host>/<program>.log ──",
      "cat > /etc/rsyslog.d/20-remote.conf <<'EOF'",
      'template(name="RemoteHost" type="string" string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")',
      "if $fromhost-ip != '127.0.0.1' then {",
      '    action(type="omfile" dynaFile="RemoteHost" createDirs="on")',
      "    stop",
      "}",
      "EOF",
      "",
      "mkdir -p /var/log/remote",
      "systemctl enable rsyslog",
      "systemctl restart rsyslog",
      "",
      "# ── Rotate /var/log/remote so it can't grow unbounded ──",
      "# CloudWatch (90d) is the system of record; these local files are just a",
      "# spool for the CW agent, so keep only a short window. copytruncate keeps",
      "# rsyslog's open dynaFile handles valid (truncate in place, same inode).",
      "cat > /etc/logrotate.d/remote-syslog <<'EOF'",
      "/var/log/remote/*/*.log {",
      "    daily",
      "    rotate 7",
      "    compress",
      "    delaycompress",
      "    missingok",
      "    notifempty",
      "    copytruncate",
      "}",
      "EOF",
      "",
      "# ── CloudWatch agent: ship /var/log/remote/**/*.log to unifi-syslog ──",
      "dnf install -y amazon-cloudwatch-agent",
      "cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json <<'EOF'",
      "{",
      '  "logs": {',
      '    "logs_collected": {',
      '      "files": {',
      '        "collect_list": [',
      "          {",
      '            "file_path": "/var/log/remote/**/*.log",',
      '            "log_group_name": "unifi-syslog",',
      '            "log_stream_name": "{hostname}/{file_name}",',
      '            "retention_in_days": 90',
      "          }",
      "        ]",
      "      }",
      "    }",
      "  }",
      "}",
      "EOF",
      "",
      "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \\",
      "  -a fetch-config -m ec2 \\",
      "  -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s",
      "systemctl enable amazon-cloudwatch-agent",
      "",
      "# ── NetFlow/IPFIX collectors (nfcapd) ──",
      "# nfdump is not packaged for AL2023; build 1.6.23 from source (needs",
      "# rrdtool-devel for librrd). Reconstructed under IaC for INFRA-12 — the",
      "# original instance ran these as hand-installed systemd units. Captures",
      "# are local-only (no consumer/shipping today); 30-day retention enforced.",
      "dnf install -y gcc gcc-c++ make automake autoconf libtool flex bison libpcap-devel zlib-devel bzip2-devel rrdtool-devel tar",
      "NFVER=1.6.23",
      "curl -sfL https://github.com/phaag/nfdump/archive/refs/tags/v${NFVER}.tar.gz | tar xz -C /tmp",
      "( cd /tmp/nfdump-${NFVER} && ./autogen.sh && ./configure && make -j1 && make install )",
      "ldconfig",
      "",
      "mkdir -p /var/log/netflow/ronkonkoma /var/log/netflow/locust",
      "chown -R ec2-user:ec2-user /var/log/netflow",
      "",
      "# Ronkonkoma gateway -> UDP 2055",
      "cat > /etc/systemd/system/nfcapd.service <<'EOF'",
      "[Unit]",
      "Description=nfcapd NetFlow collector (Ronkonkoma, udp/2055)",
      "After=network.target",
      "[Service]",
      "Type=simple",
      "User=ec2-user",
      "ExecStart=/usr/local/bin/nfcapd -p 2055 -l /var/log/netflow/ronkonkoma",
      "Restart=always",
      "[Install]",
      "WantedBy=multi-user.target",
      "EOF",
      "",
      "# Locust Ave gateway -> UDP 2056",
      "cat > /etc/systemd/system/nfcapd-locust.service <<'EOF'",
      "[Unit]",
      "Description=nfcapd NetFlow collector (Locust Ave, udp/2056)",
      "After=network.target",
      "[Service]",
      "Type=simple",
      "User=ec2-user",
      "ExecStart=/usr/local/bin/nfcapd -p 2056 -l /var/log/netflow/locust",
      "Restart=always",
      "[Install]",
      "WantedBy=multi-user.target",
      "EOF",
      "",
      "# 30-day retention sweep (daily 03:30 UTC)",
      "cat > /usr/local/sbin/netflow-retention.sh <<'EOF'",
      "#!/bin/bash",
      "find /var/log/netflow -type f -name 'nfcapd.*' -mtime +30 -delete",
      "EOF",
      "chmod +x /usr/local/sbin/netflow-retention.sh",
      "cat > /etc/systemd/system/netflow-retention.service <<'EOF'",
      "[Unit]",
      "Description=Delete NetFlow captures older than 30 days",
      "[Service]",
      "Type=oneshot",
      "ExecStart=/usr/local/sbin/netflow-retention.sh",
      "EOF",
      "cat > /etc/systemd/system/netflow-retention.timer <<'EOF'",
      "[Unit]",
      "Description=Daily NetFlow retention sweep",
      "[Timer]",
      "OnCalendar=*-*-* 03:30:00 UTC",
      "Persistent=true",
      "[Install]",
      "WantedBy=timers.target",
      "EOF",
      "",
      "systemctl daemon-reload",
      "systemctl enable --now nfcapd.service nfcapd-locust.service netflow-retention.timer",
    );

    const instance = new ec2.Instance(this, "Instance", {
      instanceName: "syslog-server",
      vpc,
      vpcSubnets: { subnets: [publicSubnet] },
      instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.NANO),
      machineImage: ec2.MachineImage.latestAmazonLinux2023({
        cpuType: ec2.AmazonLinuxCpuType.ARM_64,
        // Cache the resolved AMI in cdk.context.json so deploys don't implicitly
        // pick up new AL2023 releases (AMI change forces instance replacement).
        // Refresh deliberately: cdk context --reset <ami key> && cdk synth
        cachedInContext: true,
      }),
      securityGroup: sg,
      role,
      userData,
      // A user-data change must actually re-run, so force instance replacement
      // (the box is stateless — logs live in CloudWatch, the EIP re-associates).
      userDataCausesReplacement: true,
      blockDevices: [
        {
          deviceName: "/dev/xvda",
          volume: ec2.BlockDeviceVolume.ebs(30, {
            volumeType: ec2.EbsDeviceVolumeType.GP3,
            encrypted: true,
          }),
        },
      ],
    });

    // Re-associate the existing Elastic IP (184.72.154.32) so the UniFi fleet's
    // forwarding target is unchanged. The allocation is UNMANAGED (referenced by
    // ID) — CloudFormation can associate it but never release it.
    new ec2.CfnEIPAssociation(this, "EipAssociation", {
      allocationId: "eipalloc-006bdefc9802f3285",
      instanceId: instance.instanceId,
    });

    // ALARM-only "no incoming logs" alarm to the shared site-alerts topic
    // (alias/seahaven-alarm-topics CMK). Mirrors the prior standalone alarm:
    // IncomingLogEvents (Sum) < 1 over two 1-day periods. 2-day window tolerates
    // quiet weekends; treatMissingData=breaching catches a dead pipeline.
    const alarmTopic = sns.Topic.fromTopicArn(
      this, "SiteAlerts", "arn:aws:sns:us-east-1:328440206208:site-alerts",
    );

    const noLogsAlarm = new cloudwatch.Alarm(this, "NoIncomingLogsAlarm", {
      alarmName: "Syslog-NoIncomingLogs",
      alarmDescription:
        "No log events delivered to unifi-syslog for 2 days — syslog pipeline may be down.",
      metric: new cloudwatch.Metric({
        namespace: "AWS/Logs",
        metricName: "IncomingLogEvents",
        dimensionsMap: { LogGroupName: "unifi-syslog" },
        statistic: "Sum",
        period: cdk.Duration.days(1),
      }),
      threshold: 1,
      comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
      evaluationPeriods: 2,
      treatMissingData: cloudwatch.TreatMissingData.BREACHING,
    });
    noLogsAlarm.addAlarmAction(new cwactions.SnsAction(alarmTopic));

    new cdk.CfnOutput(this, "InstanceId", { value: instance.instanceId });
    new cdk.CfnOutput(this, "PublicIp", {
      value: "184.72.154.32",
      description: "Elastic IP — UniFi remote-syslog forwarding target",
    });
  }
}