syslog-server/terraform/user_data.sh

#!/bin/bash
set -euxo pipefail

# ── 1 GiB swap (build headroom + stability on the 512 MiB t4g.nano) ──
if [ ! -f /swapfile ]; then
  fallocate -l 1G /swapfile || dd if=/dev/zero of=/swapfile bs=1M count=1024
  chmod 600 /swapfile
  mkswap /swapfile
  echo '/swapfile none swap sw 0 0' >> /etc/fstab
fi
swapon -a || true

# ── rsyslog: listen on UDP/TCP 514 ──
dnf install -y rsyslog
cat > /etc/rsyslog.d/10-listen.conf <<'EOF'
module(load="imudp")
input(type="imudp" port="514")
module(load="imtcp")
input(type="imtcp" port="514")
EOF

# ── Write remote syslog to /var/log/remote/<host>/<program>.log ──
cat > /etc/rsyslog.d/20-remote.conf <<'EOF'
template(name="RemoteHost" type="string" string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")
if $fromhost-ip != '127.0.0.1' then {
    action(type="omfile" dynaFile="RemoteHost" createDirs="on")
    stop
}
EOF

mkdir -p /var/log/remote
systemctl enable rsyslog
systemctl restart rsyslog

# ── Rotate /var/log/remote so it can't grow unbounded ──
# CloudWatch (90d) is the system of record; these local files are just a
# spool for the CW agent, so keep only a short window. copytruncate keeps
# rsyslog's open dynaFile handles valid (truncate in place, same inode).
cat > /etc/logrotate.d/remote-syslog <<'EOF'
/var/log/remote/*/*.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    copytruncate
}
EOF

# ── CloudWatch agent: ship /var/log/remote/**/*.log to unifi-syslog ──
dnf install -y amazon-cloudwatch-agent
cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json <<'EOF'
{
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/remote/**/*.log",
            "log_group_name": "unifi-syslog",
            "log_stream_name": "{hostname}/{file_name}",
            "retention_in_days": 90
          }
        ]
      }
    }
  }
}
EOF

/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a fetch-config -m ec2 \
  -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s
systemctl enable amazon-cloudwatch-agent

# ── NetFlow/IPFIX collectors (nfcapd) ──
# nfdump is not packaged for AL2023; build 1.6.23 from source (needs
# rrdtool-devel for librrd). Reconstructed under IaC for INFRA-12 — the
# original instance ran these as hand-installed systemd units. Captures
# are local-only (no consumer/shipping today); 30-day retention enforced.
dnf install -y gcc gcc-c++ make automake autoconf libtool flex bison libpcap-devel zlib-devel bzip2-devel rrdtool-devel tar
NFVER=1.6.23
curl -sfL https://github.com/phaag/nfdump/archive/refs/tags/v${NFVER}.tar.gz | tar xz -C /tmp
( cd /tmp/nfdump-${NFVER} && ./autogen.sh && ./configure && make -j1 && make install )
ldconfig

mkdir -p /var/log/netflow/ronkonkoma /var/log/netflow/locust
chown -R ec2-user:ec2-user /var/log/netflow

# Ronkonkoma gateway -> UDP 2055
cat > /etc/systemd/system/nfcapd.service <<'EOF'
[Unit]
Description=nfcapd NetFlow collector (Ronkonkoma, udp/2055)
After=network.target
[Service]
Type=simple
User=ec2-user
ExecStart=/usr/local/bin/nfcapd -p 2055 -l /var/log/netflow/ronkonkoma
Restart=always
[Install]
WantedBy=multi-user.target
EOF

# Locust Ave gateway -> UDP 2056
cat > /etc/systemd/system/nfcapd-locust.service <<'EOF'
[Unit]
Description=nfcapd NetFlow collector (Locust Ave, udp/2056)
After=network.target
[Service]
Type=simple
User=ec2-user
ExecStart=/usr/local/bin/nfcapd -p 2056 -l /var/log/netflow/locust
Restart=always
[Install]
WantedBy=multi-user.target
EOF

# 30-day retention sweep (daily 03:30 UTC)
cat > /usr/local/sbin/netflow-retention.sh <<'EOF'
#!/bin/bash
find /var/log/netflow -type f -name 'nfcapd.*' -mtime +30 -delete
EOF
chmod +x /usr/local/sbin/netflow-retention.sh
cat > /etc/systemd/system/netflow-retention.service <<'EOF'
[Unit]
Description=Delete NetFlow captures older than 30 days
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/netflow-retention.sh
EOF
cat > /etc/systemd/system/netflow-retention.timer <<'EOF'
[Unit]
Description=Daily NetFlow retention sweep
[Timer]
OnCalendar=*-*-* 03:30:00 UTC
Persistent=true
[Install]
WantedBy=timers.target
EOF

systemctl daemon-reload
systemctl enable --now nfcapd.service nfcapd-locust.service netflow-retention.timer