fix(infra): persistent data volume + restore-on-boot + cached AMI (#22)

Today's instance replacement (uncached AMI lookup resolved a new AL2023
release) destroyed the root volume holding all Forgejo state; restored
manually from the 05:00 S3 dump. This makes replacement harmless:

- New 50 GiB standalone volume (RemovalPolicy.RETAIN) mounted at
  /var/lib/forgejo — sqlite db, repositories, and logs all survive
  instance replacement and stack deletion. No app.ini path changes.
- Restore-on-boot: if the data volume has no database (first boot or
  total volume loss), userdata restores the latest S3 dump
  automatically before starting the service. Volume loss self-heals
  to <=24h-old state.
- blkid guard: an existing filesystem is mounted, never formatted.
- cachedInContext: true + committed cdk.context.json — AMI changes
  (and the instance replacement they force) become deliberate.
- Root volume 50 -> 20 GiB; state no longer lives there.
- forgejo-backup tag on the data volume brings it under the existing
  DLM snapshot policy.

Deploy replaces the instance once; restore-on-boot pulls the fresh
17:43 UTC dump.

cdk.context.json

@@ -43,5 +43,6 @@
         ]
       }
     ]
-  }
+  },
+  "ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64:region=us-east-1": "ami-0b183bb1259186479"
 }

lib/forgejo-stack.ts

@@ -182,6 +182,19 @@ export class ForgejoStack extends cdk.Stack {
       "",
       "useradd --system --shell /bin/bash --home-dir /home/forgejo --create-home forgejo",
       "",
+      "# ── Persistent data volume (wait for CfnVolumeAttachment) ──",
+      "until lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | grep -q .; do",
+      "  echo 'Waiting for data volume...'",
+      "  sleep 5",
+      "done",
+      "DATA_DEVICE=/dev/$(lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | head -1)",
+      "if ! blkid \"$DATA_DEVICE\"; then",
+      "  mkfs.ext4 -L forgejo-data \"$DATA_DEVICE\"",
+      "fi",
+      "mkdir -p /var/lib/forgejo",
+      "echo \"LABEL=forgejo-data /var/lib/forgejo ext4 defaults,nofail 0 2\" >> /etc/fstab",
+      "mount -a",
+      "",
       "mkdir -p /var/lib/forgejo/{data,log}",
       "chown -R forgejo:forgejo /var/lib/forgejo",
       "chmod 750 /var/lib/forgejo",
@@ -248,6 +261,27 @@ export class ForgejoStack extends cdk.Stack {
       "SVCEOF",
       "",
       "systemctl daemon-reload",
+      "",
+      "# ── Restore from latest S3 dump if the data volume is empty (first boot or volume loss) ──",
+      "if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then",
+      "  echo 'No database on data volume - restoring latest backup from S3'",
+      "  S3_PREFIX=$(aws ssm get-parameter --name /forgejo/backup-s3-prefix --query Parameter.Value --output text --region us-east-1 || echo 'archive')",
+      "  LATEST=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/ --region us-east-1 | awk '{print $2}' | sort | tail -1 | tr -d '/')",
+      "  if [ -n \"$LATEST\" ]; then",
+      "    FILE=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/ --region us-east-1 | awk '{print $4}' | tail -1)",
+      "    RESTORE_DIR=$(mktemp -d)",
+      "    aws s3 cp \"s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/${FILE}\" \"$RESTORE_DIR/dump.tar.gz\" --region us-east-1",
+      "    tar xzf \"$RESTORE_DIR/dump.tar.gz\" -C \"$RESTORE_DIR\"",
+      "    cp -a \"$RESTORE_DIR\"/data/. /var/lib/forgejo/data/",
+      "    mkdir -p /var/lib/forgejo/data/repositories",
+      "    cp -a \"$RESTORE_DIR\"/repos/. /var/lib/forgejo/data/repositories/",
+      "    chown -R forgejo:forgejo /var/lib/forgejo",
+      "    rm -rf \"$RESTORE_DIR\"",
+      "  else",
+      "    echo 'No backup found in S3 - starting fresh'",
+      "  fi",
+      "fi",
+      "",
       "systemctl enable --now forgejo",
       "",
       "cat > /usr/local/bin/forgejo-backup.sh << 'BAKEOF'",
@@ -360,13 +394,17 @@ export class ForgejoStack extends cdk.Stack {
       instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.SMALL),
       machineImage: ec2.MachineImage.latestAmazonLinux2023({
         cpuType: ec2.AmazonLinuxCpuType.ARM_64,
+        // Cache the resolved AMI in cdk.context.json so deploys don't pick up
+        // new AL2023 releases implicitly (AMI change forces instance replacement).
+        // Refresh deliberately with: cdk context --reset <ami key> && cdk synth
+        cachedInContext: true,
       }),
       securityGroup: sg,
       role,
       userData,
       blockDevices: [{
         deviceName: "/dev/xvda",
-        volume: ec2.BlockDeviceVolume.ebs(50, {
+        volume: ec2.BlockDeviceVolume.ebs(20, {
           volumeType: ec2.EbsDeviceVolumeType.GP3,
           encrypted: true,
         }),
@@ -375,6 +413,26 @@ export class ForgejoStack extends cdk.Stack {
 
     cdk.Tags.of(instance).add("forgejo-backup", "true");
 
+    // All Forgejo state (sqlite db, repositories, logs) lives on this volume,
+    // mounted at /var/lib/forgejo. RETAIN means it survives instance
+    // replacement and stack deletion; userdata mounts the existing filesystem
+    // (blkid guard prevents formatting) and restores from S3 only when empty.
+    const dataVolume = new ec2.Volume(this, "DataVolume", {
+      availabilityZone: "us-east-1a",
+      size: cdk.Size.gibibytes(50),
+      volumeType: ec2.EbsDeviceVolumeType.GP3,
+      encrypted: true,
+      removalPolicy: cdk.RemovalPolicy.RETAIN,
+    });
+    cdk.Tags.of(dataVolume).add("Name", "forgejo-data");
+    cdk.Tags.of(dataVolume).add("forgejo-backup", "true");
+
+    new ec2.CfnVolumeAttachment(this, "DataVolumeAttachment", {
+      instanceId: instance.instanceId,
+      volumeId: dataVolume.volumeId,
+      device: "/dev/xvdf",
+    });
+
     const dlmRole = new iam.Role(this, "DlmRole", {
       roleName: "forgejo-dlm",
       assumedBy: new iam.ServicePrincipal("dlm.amazonaws.com"),