From 6d52b246b1fe84bb3d17c0fcf7a9b6c3e66fe179 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:14:28 -0400 Subject: [PATCH] fix(syslog-server): rotate /var/log/remote so the disk can't fill (INFRA-11) (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collector's remote-syslog spool had no rotation, so each gateway's /var/log/remote//.log grew unbounded. Low risk at the old ~109 events/day, but the gateways now forward ~60k/day. CloudWatch (90d) is the system of record; the local files are only a CW-agent spool, so keep a short 7-day compressed window. copytruncate keeps rsyslog's open dynaFile handles valid (truncate in place). Applied live already; this codifies it so an instance replacement keeps it (mirrors the existing netflow-retention timer). Deploying this user-data change forces an instance replacement (userDataCausesReplacement) — the EIP re-associates and the forwarding target is unchanged, so do it in a window. --- lib/syslog-server-stack.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/syslog-server-stack.ts b/lib/syslog-server-stack.ts index ae2528d..b6a23e8 100644 --- a/lib/syslog-server-stack.ts +++ b/lib/syslog-server-stack.ts @@ -115,6 +115,22 @@ export class SyslogServerStack extends cdk.Stack { "systemctl enable rsyslog", "systemctl restart rsyslog", "", + "# ── Rotate /var/log/remote so it can't grow unbounded ──", + "# CloudWatch (90d) is the system of record; these local files are just a", + "# spool for the CW agent, so keep only a short window. copytruncate keeps", + "# rsyslog's open dynaFile handles valid (truncate in place, same inode).", + "cat > /etc/logrotate.d/remote-syslog <<'EOF'", + "/var/log/remote/*/*.log {", + " daily", + " rotate 7", + " compress", + " delaycompress", + " missingok", + " notifempty", + " copytruncate", + "}", + "EOF", + "", "# ── CloudWatch agent: ship /var/log/remote/**/*.log to unifi-syslog ──", "dnf install -y amazon-cloudwatch-agent", "cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json <<'EOF'",