From f6b105a9fd9d5e42a535f046b0861b03c1c8eee1 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 5 Jun 2026 13:47:51 -0400 Subject: [PATCH] fix(infra): persistent data volume + restore-on-boot + cached AMI (#22) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today's instance replacement (uncached AMI lookup resolved a new AL2023 release) destroyed the root volume holding all Forgejo state; restored manually from the 05:00 S3 dump. This makes replacement harmless: - New 50 GiB standalone volume (RemovalPolicy.RETAIN) mounted at /var/lib/forgejo — sqlite db, repositories, and logs all survive instance replacement and stack deletion. No app.ini path changes. - Restore-on-boot: if the data volume has no database (first boot or total volume loss), userdata restores the latest S3 dump automatically before starting the service. Volume loss self-heals to <=24h-old state. - blkid guard: an existing filesystem is mounted, never formatted. - cachedInContext: true + committed cdk.context.json — AMI changes (and the instance replacement they force) become deliberate. - Root volume 50 -> 20 GiB; state no longer lives there. - forgejo-backup tag on the data volume brings it under the existing DLM snapshot policy. Deploy replaces the instance once; restore-on-boot pulls the fresh 17:43 UTC dump. --- cdk.context.json | 3 ++- lib/forgejo-stack.ts | 60 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/cdk.context.json b/cdk.context.json index 068ec0c..34db403 100644 --- a/cdk.context.json +++ b/cdk.context.json @@ -43,5 +43,6 @@ ] } ] - } + }, + "ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64:region=us-east-1": "ami-0b183bb1259186479" } diff --git a/lib/forgejo-stack.ts b/lib/forgejo-stack.ts index 9acda77..897ffd2 100644 --- a/lib/forgejo-stack.ts +++ b/lib/forgejo-stack.ts @@ -182,6 +182,19 @@ export class ForgejoStack extends cdk.Stack { "", "useradd --system --shell /bin/bash --home-dir /home/forgejo --create-home forgejo", "", + "# ── Persistent data volume (wait for CfnVolumeAttachment) ──", + "until lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | grep -q .; do", + " echo 'Waiting for data volume...'", + " sleep 5", + "done", + "DATA_DEVICE=/dev/$(lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | head -1)", + "if ! blkid \"$DATA_DEVICE\"; then", + " mkfs.ext4 -L forgejo-data \"$DATA_DEVICE\"", + "fi", + "mkdir -p /var/lib/forgejo", + "echo \"LABEL=forgejo-data /var/lib/forgejo ext4 defaults,nofail 0 2\" >> /etc/fstab", + "mount -a", + "", "mkdir -p /var/lib/forgejo/{data,log}", "chown -R forgejo:forgejo /var/lib/forgejo", "chmod 750 /var/lib/forgejo", @@ -248,6 +261,27 @@ export class ForgejoStack extends cdk.Stack { "SVCEOF", "", "systemctl daemon-reload", + "", + "# ── Restore from latest S3 dump if the data volume is empty (first boot or volume loss) ──", + "if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then", + " echo 'No database on data volume - restoring latest backup from S3'", + " S3_PREFIX=$(aws ssm get-parameter --name /forgejo/backup-s3-prefix --query Parameter.Value --output text --region us-east-1 || echo 'archive')", + " LATEST=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/ --region us-east-1 | awk '{print $2}' | sort | tail -1 | tr -d '/')", + " if [ -n \"$LATEST\" ]; then", + " FILE=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/ --region us-east-1 | awk '{print $4}' | tail -1)", + " RESTORE_DIR=$(mktemp -d)", + " aws s3 cp \"s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/${FILE}\" \"$RESTORE_DIR/dump.tar.gz\" --region us-east-1", + " tar xzf \"$RESTORE_DIR/dump.tar.gz\" -C \"$RESTORE_DIR\"", + " cp -a \"$RESTORE_DIR\"/data/. /var/lib/forgejo/data/", + " mkdir -p /var/lib/forgejo/data/repositories", + " cp -a \"$RESTORE_DIR\"/repos/. /var/lib/forgejo/data/repositories/", + " chown -R forgejo:forgejo /var/lib/forgejo", + " rm -rf \"$RESTORE_DIR\"", + " else", + " echo 'No backup found in S3 - starting fresh'", + " fi", + "fi", + "", "systemctl enable --now forgejo", "", "cat > /usr/local/bin/forgejo-backup.sh << 'BAKEOF'", @@ -360,13 +394,17 @@ export class ForgejoStack extends cdk.Stack { instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.SMALL), machineImage: ec2.MachineImage.latestAmazonLinux2023({ cpuType: ec2.AmazonLinuxCpuType.ARM_64, + // Cache the resolved AMI in cdk.context.json so deploys don't pick up + // new AL2023 releases implicitly (AMI change forces instance replacement). + // Refresh deliberately with: cdk context --reset && cdk synth + cachedInContext: true, }), securityGroup: sg, role, userData, blockDevices: [{ deviceName: "/dev/xvda", - volume: ec2.BlockDeviceVolume.ebs(50, { + volume: ec2.BlockDeviceVolume.ebs(20, { volumeType: ec2.EbsDeviceVolumeType.GP3, encrypted: true, }), @@ -375,6 +413,26 @@ export class ForgejoStack extends cdk.Stack { cdk.Tags.of(instance).add("forgejo-backup", "true"); + // All Forgejo state (sqlite db, repositories, logs) lives on this volume, + // mounted at /var/lib/forgejo. RETAIN means it survives instance + // replacement and stack deletion; userdata mounts the existing filesystem + // (blkid guard prevents formatting) and restores from S3 only when empty. + const dataVolume = new ec2.Volume(this, "DataVolume", { + availabilityZone: "us-east-1a", + size: cdk.Size.gibibytes(50), + volumeType: ec2.EbsDeviceVolumeType.GP3, + encrypted: true, + removalPolicy: cdk.RemovalPolicy.RETAIN, + }); + cdk.Tags.of(dataVolume).add("Name", "forgejo-data"); + cdk.Tags.of(dataVolume).add("forgejo-backup", "true"); + + new ec2.CfnVolumeAttachment(this, "DataVolumeAttachment", { + instanceId: instance.instanceId, + volumeId: dataVolume.volumeId, + device: "/dev/xvdf", + }); + const dlmRole = new iam.Role(this, "DlmRole", { roleName: "forgejo-dlm", assumedBy: new iam.ServicePrincipal("dlm.amazonaws.com"),