Bump aws-cdk-lib from 2.257.0 to 2.258.0

Bumps [aws-cdk-lib](https://github.com/aws/aws-cdk/tree/HEAD/packages/aws-cdk-lib) from 2.257.0 to 2.258.0. - [Release notes](https://github.com/aws/aws-cdk/releases) - [Changelog](https://github.com/aws/aws-cdk/blob/main/CHANGELOG.v2.alpha.md) - [Commits](https://github.com/aws/aws-cdk/commits/v2.258.0/packages/aws-cdk-lib) --- updated-dependencies: - dependency-name: aws-cdk-lib dependency-version: 2.257.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>
fix(monitoring): wire backup-verification alarms to site-alerts topic (#24 )
2026-10-03 08:03:21 +00:00 · 2026-06-05 19:01:30 +00:00 · 2026-06-05 14:47:09 -04:00 · 2026-06-05 14:34:45 -04:00 · 2026-06-05 13:47:51 -04:00
5 changed files with 125 additions and 42 deletions
--- a/cdk.context.json
+++ b/cdk.context.json
@ -43,5 +43,6 @@
        ]
      }
    ]
-  }
+  },
+  "ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64:region=us-east-1": "ami-0b183bb1259186479"
 }
--- a/lib/constructs/backup-verification.ts
+++ b/lib/constructs/backup-verification.ts
@ -1,5 +1,7 @@
 import * as cdk from "aws-cdk-lib";
 import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
+import * as cloudwatch_actions from "aws-cdk-lib/aws-cloudwatch-actions";
+import * as sns from "aws-cdk-lib/aws-sns";
 import * as events from "aws-cdk-lib/aws-events";
 import * as events_targets from "aws-cdk-lib/aws-events-targets";
 import * as iam from "aws-cdk-lib/aws-iam";
@ -99,15 +101,48 @@ export class BackupVerification extends Construct {
      ],
    });

-    new cloudwatch.Alarm(this, "ErrorAlarm", {
+    // Errors alarm: only fires when the function actually runs and errors.
+    // The function runs once daily, so for ~23h there is no data. Treating
+    // missing data as BREACHING flipped this alarm OK->ALARM every day around
+    // 11:01 UTC even though no error ever occurred. NOT_BREACHING means "no
+    // data = no errors = healthy"; the separate not-running alarm below covers
+    // the "verification never ran" case.
+    // Org operational alarm topic (site-alerts, CMK-encrypted — never
+    // alias/aws/sns, which CloudWatch cannot publish to). Alarm action only,
+    // no OK action, per org convention.
+    const alertTopic = sns.Topic.fromTopicArn(
+      this,
+      "AlertTopic",
+      "arn:aws:sns:us-east-1:328440206208:site-alerts"
+    );
+
+    const errorAlarm = new cloudwatch.Alarm(this, "ErrorAlarm", {
      alarmName: "forgejo-backup-verification-errors",
      alarmDescription: "Backup verification Lambda is failing — Slack notifications may not be firing",
      metric: fn.metricErrors({ period: cdk.Duration.hours(1) }),
      threshold: 1,
      evaluationPeriods: 1,
-      treatMissingData: cloudwatch.TreatMissingData.BREACHING,
+      treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
      comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
    });
+    errorAlarm.addAlarmAction(new cloudwatch_actions.SnsAction(alertTopic));
+
+    // Not-running alarm: fires if the daily verification did not invoke at all
+    // in a 24h window. This is the real "missing run" guard that the errors
+    // alarm's BREACHING setting was previously (and incorrectly) providing.
+    const notRunningAlarm = new cloudwatch.Alarm(this, "NotRunningAlarm", {
+      alarmName: "forgejo-backup-verification-not-running",
+      alarmDescription: "Backup verification Lambda has not run in the last 24h — daily verification may be broken",
+      metric: fn.metricInvocations({
+        period: cdk.Duration.hours(24),
+        statistic: cloudwatch.Stats.SUM,
+      }),
+      threshold: 1,
+      evaluationPeriods: 1,
+      treatMissingData: cloudwatch.TreatMissingData.BREACHING,
+      comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
+    });
+    notRunningAlarm.addAlarmAction(new cloudwatch_actions.SnsAction(alertTopic));

    this.functionArn = fn.functionArn;
  }
--- a/lib/forgejo-stack.ts
+++ b/lib/forgejo-stack.ts
@ -182,6 +182,19 @@ export class ForgejoStack extends cdk.Stack {
      "",
      "useradd --system --shell /bin/bash --home-dir /home/forgejo --create-home forgejo",
      "",
+      "# ── Persistent data volume (wait for CfnVolumeAttachment) ──",
+      "until lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | grep -q .; do",
+      "  echo 'Waiting for data volume...'",
+      "  sleep 5",
+      "done",
+      "DATA_DEVICE=/dev/$(lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | head -1)",
+      "if ! blkid \"$DATA_DEVICE\"; then",
+      "  mkfs.ext4 -L forgejo-data \"$DATA_DEVICE\"",
+      "fi",
+      "mkdir -p /var/lib/forgejo",
+      "echo \"LABEL=forgejo-data /var/lib/forgejo ext4 defaults,nofail 0 2\" >> /etc/fstab",
+      "mount -a",
+      "",
      "mkdir -p /var/lib/forgejo/{data,log}",
      "chown -R forgejo:forgejo /var/lib/forgejo",
      "chmod 750 /var/lib/forgejo",
@ -248,6 +261,27 @@ export class ForgejoStack extends cdk.Stack {
      "SVCEOF",
      "",
      "systemctl daemon-reload",
+      "",
+      "# ── Restore from latest S3 dump if the data volume is empty (first boot or volume loss) ──",
+      "if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then",
+      "  echo 'No database on data volume - restoring latest backup from S3'",
+      "  S3_PREFIX=$(aws ssm get-parameter --name /forgejo/backup-s3-prefix --query Parameter.Value --output text --region us-east-1 || echo 'archive')",
+      "  LATEST=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/ --region us-east-1 | awk '{print $2}' | sort | tail -1 | tr -d '/')",
+      "  if [ -n \"$LATEST\" ]; then",
+      "    FILE=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/ --region us-east-1 | awk '{print $4}' | tail -1)",
+      "    RESTORE_DIR=$(mktemp -d)",
+      "    aws s3 cp \"s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/${FILE}\" \"$RESTORE_DIR/dump.tar.gz\" --region us-east-1",
+      "    tar xzf \"$RESTORE_DIR/dump.tar.gz\" -C \"$RESTORE_DIR\"",
+      "    cp -a \"$RESTORE_DIR\"/data/. /var/lib/forgejo/data/",
+      "    mkdir -p /var/lib/forgejo/data/repositories",
+      "    cp -a \"$RESTORE_DIR\"/repos/. /var/lib/forgejo/data/repositories/",
+      "    chown -R forgejo:forgejo /var/lib/forgejo",
+      "    rm -rf \"$RESTORE_DIR\"",
+      "  else",
+      "    echo 'No backup found in S3 - starting fresh'",
+      "  fi",
+      "fi",
+      "",
      "systemctl enable --now forgejo",
      "",
      "cat > /usr/local/bin/forgejo-backup.sh << 'BAKEOF'",
@ -360,13 +394,17 @@ export class ForgejoStack extends cdk.Stack {
      instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.SMALL),
      machineImage: ec2.MachineImage.latestAmazonLinux2023({
        cpuType: ec2.AmazonLinuxCpuType.ARM_64,
+        // Cache the resolved AMI in cdk.context.json so deploys don't pick up
+        // new AL2023 releases implicitly (AMI change forces instance replacement).
+        // Refresh deliberately with: cdk context --reset <ami key> && cdk synth
+        cachedInContext: true,
      }),
      securityGroup: sg,
      role,
      userData,
      blockDevices: [{
        deviceName: "/dev/xvda",
-        volume: ec2.BlockDeviceVolume.ebs(50, {
+        volume: ec2.BlockDeviceVolume.ebs(20, {
          volumeType: ec2.EbsDeviceVolumeType.GP3,
          encrypted: true,
        }),
@ -375,6 +413,26 @@ export class ForgejoStack extends cdk.Stack {

    cdk.Tags.of(instance).add("forgejo-backup", "true");

+    // All Forgejo state (sqlite db, repositories, logs) lives on this volume,
+    // mounted at /var/lib/forgejo. RETAIN means it survives instance
+    // replacement and stack deletion; userdata mounts the existing filesystem
+    // (blkid guard prevents formatting) and restores from S3 only when empty.
+    const dataVolume = new ec2.Volume(this, "DataVolume", {
+      availabilityZone: "us-east-1a",
+      size: cdk.Size.gibibytes(50),
+      volumeType: ec2.EbsDeviceVolumeType.GP3,
+      encrypted: true,
+      removalPolicy: cdk.RemovalPolicy.RETAIN,
+    });
+    cdk.Tags.of(dataVolume).add("Name", "forgejo-data");
+    cdk.Tags.of(dataVolume).add("forgejo-backup", "true");
+
+    new ec2.CfnVolumeAttachment(this, "DataVolumeAttachment", {
+      instanceId: instance.instanceId,
+      volumeId: dataVolume.volumeId,
+      device: "/dev/xvdf",
+    });
+
    const dlmRole = new iam.Role(this, "DlmRole", {
      roleName: "forgejo-dlm",
      assumedBy: new iam.ServicePrincipal("dlm.amazonaws.com"),
--- a/package-lock.json
+++ b/package-lock.json
@ -9,7 +9,7 @@
      "version": "1.0.0",
      "dependencies": {
        "@aws-cdk/aws-lambda-python-alpha": "2.257.0-alpha.0",
-        "aws-cdk-lib": "2.257.0",
+        "aws-cdk-lib": "2.258.0",
        "constructs": "^10.0.0"
      },
      "bin": {
@ -23,9 +23,9 @@
      }
    },
    "node_modules/@aws-cdk/asset-awscli-v1": {
-      "version": "2.2.273",
-      "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.273.tgz",
-      "integrity": "sha512-X57HYUtHt9BQrlrzUNcMyRsDUCoakYNnY6qh5lNwRCHPtQoTfXmuISkfLk0AjLkcbS5lw1LLTQFiQhTDXfiTvg==",
+      "version": "2.2.282",
+      "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.282.tgz",
+      "integrity": "sha512-7hKMi5tTxDcKGIMIOq14PnY0GBcugW33Uh/2YHDZiEwSxLeFOCYBwhR+BFXONb/EJeVI3RETFgailNZbkcKF6g==",
      "license": "Apache-2.0"
    },
    "node_modules/@aws-cdk/asset-node-proxy-agent-v6": {
@ -48,9 +48,9 @@
      }
    },
    "node_modules/@aws-cdk/cloud-assembly-schema": {
-      "version": "53.28.0",
-      "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-53.28.0.tgz",
-      "integrity": "sha512-pZS+9bLGv2tCqcgxfA0WD3XjcqT3yE4ICvKeJEicw6aTdCxBl8FQ/AUsorY/6f2JrMS3kUQgvhXxA30MWcji0A==",
+      "version": "54.2.0",
+      "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-54.2.0.tgz",
+      "integrity": "sha512-u3lFXmiXSBozxGBmKTCVD/2mTDsaXzLZH3KYiIQKcB+zPldXOeE5TnooBgKV9ih2jVTo8ML0HpkhfAq2eiv0eQ==",
      "bundleDependencies": [
        "jsonschema",
        "semver"
@ -58,7 +58,7 @@
      "license": "Apache-2.0",
      "dependencies": {
        "jsonschema": "^1.5.0",
-        "semver": "^7.8.0"
+        "semver": "^7.8.1"
      },
      "engines": {
        "node": ">= 18.0.0"
@ -73,7 +73,7 @@
      }
    },
    "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": {
-      "version": "7.8.0",
+      "version": "7.8.1",
      "inBundle": true,
      "license": "ISC",
      "bin": {
@ -107,9 +107,9 @@
      }
    },
    "node_modules/aws-cdk-lib": {
-      "version": "2.257.0",
-      "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.257.0.tgz",
-      "integrity": "sha512-GoHfWklrBJcMwLtDlY64pvaT7cD2KyDXC8sik89DR6jHl6nQsBtYTKSJCM+C/k4jgXaecbv8myNX75FySejq0A==",
+      "version": "2.258.0",
+      "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.258.0.tgz",
+      "integrity": "sha512-OfFfg30ikBRJ3dimlzsWhPIrj6qug1p2XYsdB38CtMtcur7SufzoUczgI6kWjbQkOVcQgYzhzN29DErV0yZG7A==",
      "bundleDependencies": [
        "@balena/dockerignore",
        "@aws-cdk/cloud-assembly-api",
@ -126,19 +126,19 @@
      ],
      "license": "Apache-2.0",
      "dependencies": {
-        "@aws-cdk/asset-awscli-v1": "2.2.273",
-        "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.1",
-        "@aws-cdk/cloud-assembly-api": "^2.2.4",
-        "@aws-cdk/cloud-assembly-schema": "^53.25.0",
+        "@aws-cdk/asset-awscli-v1": "2.2.282",
+        "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.2",
+        "@aws-cdk/cloud-assembly-api": "^2.2.5",
+        "@aws-cdk/cloud-assembly-schema": "^54.0.0",
        "@balena/dockerignore": "^1.0.2",
        "case": "1.6.3",
-        "fs-extra": "^11.3.3",
+        "fs-extra": "^11.3.5",
        "ignore": "^5.3.2",
        "jsonschema": "^1.5.0",
        "mime-types": "^2.1.35",
-        "minimatch": "^10.2.3",
+        "minimatch": "^10.2.5",
        "punycode": "^2.3.1",
-        "semver": "^7.7.4",
+        "semver": "^7.8.1",
        "table": "^6.9.0",
        "yaml": "1.10.3"
      },
@ -150,7 +150,7 @@
      }
    },
    "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api": {
-      "version": "2.2.4",
+      "version": "2.2.5",
      "inBundle": true,
      "license": "Apache-2.0",
      "dependencies": {
@ -161,18 +161,7 @@
        "node": ">= 18.0.0"
      },
      "peerDependencies": {
-        "@aws-cdk/cloud-assembly-schema": ">=53.25.0"
-      }
-    },
-    "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api/node_modules/semver": {
-      "version": "7.8.0",
-      "inBundle": true,
-      "license": "ISC",
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
+        "@aws-cdk/cloud-assembly-schema": ">=53.28.0"
      }
    },
    "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": {
@ -181,7 +170,7 @@
      "license": "Apache-2.0"
    },
    "node_modules/aws-cdk-lib/node_modules/ajv": {
-      "version": "8.18.0",
+      "version": "8.20.0",
      "inBundle": true,
      "license": "MIT",
      "dependencies": {
@ -234,7 +223,7 @@
      }
    },
    "node_modules/aws-cdk-lib/node_modules/brace-expansion": {
-      "version": "5.0.5",
+      "version": "5.0.6",
      "inBundle": true,
      "license": "MIT",
      "dependencies": {
@ -294,7 +283,7 @@
      "license": "BSD-3-Clause"
    },
    "node_modules/aws-cdk-lib/node_modules/fs-extra": {
-      "version": "11.3.3",
+      "version": "11.3.5",
      "inBundle": true,
      "license": "MIT",
      "dependencies": {
@ -333,7 +322,7 @@
      "license": "MIT"
    },
    "node_modules/aws-cdk-lib/node_modules/jsonfile": {
-      "version": "6.2.0",
+      "version": "6.2.1",
      "inBundle": true,
      "license": "MIT",
      "dependencies": {
@ -406,7 +395,7 @@
      }
    },
    "node_modules/aws-cdk-lib/node_modules/semver": {
-      "version": "7.7.4",
+      "version": "7.8.1",
      "inBundle": true,
      "license": "ISC",
      "bin": {
--- a/package.json
+++ b/package.json
@ -19,7 +19,7 @@
  },
  "dependencies": {
    "@aws-cdk/aws-lambda-python-alpha": "2.257.0-alpha.0",
-    "aws-cdk-lib": "2.257.0",
+    "aws-cdk-lib": "2.258.0",
    "constructs": "^10.0.0"
  }
 }
Author	SHA1	Message	Date
dependabot[bot]	74868766a6	Bump aws-cdk-lib from 2.257.0 to 2.258.0 Bumps [aws-cdk-lib](https://github.com/aws/aws-cdk/tree/HEAD/packages/aws-cdk-lib) from 2.257.0 to 2.258.0. - [Release notes](https://github.com/aws/aws-cdk/releases) - [Changelog](https://github.com/aws/aws-cdk/blob/main/CHANGELOG.v2.alpha.md) - [Commits](https://github.com/aws/aws-cdk/commits/v2.258.0/packages/aws-cdk-lib) --- updated-dependencies: - dependency-name: aws-cdk-lib dependency-version: 2.257.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2026-06-05 19:01:30 +00:00
Adam Moussa	149178e1e2	fix(monitoring): wire backup-verification alarms to site-alerts topic (#24 ) Both alarms changed state but notified nobody (no AlarmActions). Wire each to the org operational alarm topic (site-alerts, CMK-encrypted) with an ALARM action only — no OK action per org convention.	2026-06-05 14:47:09 -04:00
Adam Moussa	a7536f0bd7	Fix daily flapping of backup-verification errors alarm (#23 ) The forgejo-backup-verification Lambda runs once per day, so its Errors metric has data for only one hour and is missing for the other ~23h. The errors alarm used TreatMissingData=BREACHING, which treated those 23h of missing data as a breach and flipped the alarm OK->ALARM every day around 11:01 UTC despite zero actual errors. Changes (alarm-only, no instance changes): - ErrorAlarm: TreatMissingData BREACHING -> NOT_BREACHING. No data now means "no errors = healthy" instead of a false breach. - Add forgejo-backup-verification-not-running: Invocations Sum over a 24h period, alarms when < 1 invocation. This is the real "the daily verification never ran" guard that the BREACHING setting was trying (incorrectly) to provide. Both alarms keep the existing action wiring (no SNS/OK actions), per the org convention of never notifying on recovery.	2026-06-05 14:34:45 -04:00
Adam Moussa	f6b105a9fd	fix(infra): persistent data volume + restore-on-boot + cached AMI (#22 ) Some checks are pending Deploy / deploy (push) Waiting to run Details Today's instance replacement (uncached AMI lookup resolved a new AL2023 release) destroyed the root volume holding all Forgejo state; restored manually from the 05:00 S3 dump. This makes replacement harmless: - New 50 GiB standalone volume (RemovalPolicy.RETAIN) mounted at /var/lib/forgejo — sqlite db, repositories, and logs all survive instance replacement and stack deletion. No app.ini path changes. - Restore-on-boot: if the data volume has no database (first boot or total volume loss), userdata restores the latest S3 dump automatically before starting the service. Volume loss self-heals to <=24h-old state. - blkid guard: an existing filesystem is mounted, never formatted. - cachedInContext: true + committed cdk.context.json — AMI changes (and the instance replacement they force) become deliberate. - Root volume 50 -> 20 GiB; state no longer lives there. - forgejo-backup tag on the data volume brings it under the existing DLM snapshot policy. Deploy replaces the instance once; restore-on-boot pulls the fresh 17:43 UTC dump.	2026-06-05 13:47:51 -04:00