fix(infra): persistent data volume + restore-on-boot + cached AMI (#22 )

Today's instance replacement (uncached AMI lookup resolved a new AL2023 release) destroyed the root volume holding all Forgejo state; restored manually from the 05:00 S3 dump. This makes replacement harmless: - New 50 GiB standalone volume (RemovalPolicy.RETAIN) mounted at /var/lib/forgejo — sqlite db, repositories, and logs all survive instance replacement and stack deletion. No app.ini path changes. - Restore-on-boot: if the data volume has no database (first boot or total volume loss), userdata restores the latest S3 dump automatically before starting the service. Volume loss self-heals to <=24h-old state. - blkid guard: an existing filesystem is mounted, never formatted. - cachedInContext: true + committed cdk.context.json — AMI changes (and the instance replacement they force) become deliberate. - Root volume 50 -> 20 GiB; state no longer lives there. - forgejo-backup tag on the data volume brings it under the existing DLM snapshot policy. Deploy replaces the instance once; restore-on-boot pulls the fresh 17:43 UTC dump.
fix(deps): bump aws-cdk-lib pin to 2.257.0 (#21 )
2026-10-03 04:33:22 +00:00 · 2026-06-05 13:47:51 -04:00 · 2026-06-05 12:58:26 -04:00 · 2026-06-05 12:26:38 -04:00
5 changed files with 112 additions and 40 deletions
--- a/.github/workflows/dependency-review.yml
+++ b/.github/workflows/dependency-review.yml
@ -0,0 +1,6 @@
+name: Dependency Review
+on:
+  pull_request:
+jobs:
+  review:
+    uses: Sea-Haven-Industries/.github/.github/workflows/callable-dependency-review.yaml@main
--- a/cdk.context.json
+++ b/cdk.context.json
@ -43,5 +43,6 @@
        ]
      }
    ]
-  }
+  },
+  "ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64:region=us-east-1": "ami-0b183bb1259186479"
 }
--- a/lib/forgejo-stack.ts
+++ b/lib/forgejo-stack.ts
@ -182,6 +182,19 @@ export class ForgejoStack extends cdk.Stack {
      "",
      "useradd --system --shell /bin/bash --home-dir /home/forgejo --create-home forgejo",
      "",
+      "# ── Persistent data volume (wait for CfnVolumeAttachment) ──",
+      "until lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | grep -q .; do",
+      "  echo 'Waiting for data volume...'",
+      "  sleep 5",
+      "done",
+      "DATA_DEVICE=/dev/$(lsblk -dno NAME | grep -v $(lsblk -no PKNAME $(findmnt -n -o SOURCE /) 2>/dev/null || echo xvda) | head -1)",
+      "if ! blkid \"$DATA_DEVICE\"; then",
+      "  mkfs.ext4 -L forgejo-data \"$DATA_DEVICE\"",
+      "fi",
+      "mkdir -p /var/lib/forgejo",
+      "echo \"LABEL=forgejo-data /var/lib/forgejo ext4 defaults,nofail 0 2\" >> /etc/fstab",
+      "mount -a",
+      "",
      "mkdir -p /var/lib/forgejo/{data,log}",
      "chown -R forgejo:forgejo /var/lib/forgejo",
      "chmod 750 /var/lib/forgejo",
@ -248,6 +261,27 @@ export class ForgejoStack extends cdk.Stack {
      "SVCEOF",
      "",
      "systemctl daemon-reload",
+      "",
+      "# ── Restore from latest S3 dump if the data volume is empty (first boot or volume loss) ──",
+      "if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then",
+      "  echo 'No database on data volume - restoring latest backup from S3'",
+      "  S3_PREFIX=$(aws ssm get-parameter --name /forgejo/backup-s3-prefix --query Parameter.Value --output text --region us-east-1 || echo 'archive')",
+      "  LATEST=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/ --region us-east-1 | awk '{print $2}' | sort | tail -1 | tr -d '/')",
+      "  if [ -n \"$LATEST\" ]; then",
+      "    FILE=$(aws s3 ls s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/ --region us-east-1 | awk '{print $4}' | tail -1)",
+      "    RESTORE_DIR=$(mktemp -d)",
+      "    aws s3 cp \"s3://forgejo-backups-328440206208/${S3_PREFIX}/${LATEST}/${FILE}\" \"$RESTORE_DIR/dump.tar.gz\" --region us-east-1",
+      "    tar xzf \"$RESTORE_DIR/dump.tar.gz\" -C \"$RESTORE_DIR\"",
+      "    cp -a \"$RESTORE_DIR\"/data/. /var/lib/forgejo/data/",
+      "    mkdir -p /var/lib/forgejo/data/repositories",
+      "    cp -a \"$RESTORE_DIR\"/repos/. /var/lib/forgejo/data/repositories/",
+      "    chown -R forgejo:forgejo /var/lib/forgejo",
+      "    rm -rf \"$RESTORE_DIR\"",
+      "  else",
+      "    echo 'No backup found in S3 - starting fresh'",
+      "  fi",
+      "fi",
+      "",
      "systemctl enable --now forgejo",
      "",
      "cat > /usr/local/bin/forgejo-backup.sh << 'BAKEOF'",
@ -360,13 +394,17 @@ export class ForgejoStack extends cdk.Stack {
      instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.SMALL),
      machineImage: ec2.MachineImage.latestAmazonLinux2023({
        cpuType: ec2.AmazonLinuxCpuType.ARM_64,
+        // Cache the resolved AMI in cdk.context.json so deploys don't pick up
+        // new AL2023 releases implicitly (AMI change forces instance replacement).
+        // Refresh deliberately with: cdk context --reset <ami key> && cdk synth
+        cachedInContext: true,
      }),
      securityGroup: sg,
      role,
      userData,
      blockDevices: [{
        deviceName: "/dev/xvda",
-        volume: ec2.BlockDeviceVolume.ebs(50, {
+        volume: ec2.BlockDeviceVolume.ebs(20, {
          volumeType: ec2.EbsDeviceVolumeType.GP3,
          encrypted: true,
        }),
@ -375,6 +413,26 @@ export class ForgejoStack extends cdk.Stack {

    cdk.Tags.of(instance).add("forgejo-backup", "true");

+    // All Forgejo state (sqlite db, repositories, logs) lives on this volume,
+    // mounted at /var/lib/forgejo. RETAIN means it survives instance
+    // replacement and stack deletion; userdata mounts the existing filesystem
+    // (blkid guard prevents formatting) and restores from S3 only when empty.
+    const dataVolume = new ec2.Volume(this, "DataVolume", {
+      availabilityZone: "us-east-1a",
+      size: cdk.Size.gibibytes(50),
+      volumeType: ec2.EbsDeviceVolumeType.GP3,
+      encrypted: true,
+      removalPolicy: cdk.RemovalPolicy.RETAIN,
+    });
+    cdk.Tags.of(dataVolume).add("Name", "forgejo-data");
+    cdk.Tags.of(dataVolume).add("forgejo-backup", "true");
+
+    new ec2.CfnVolumeAttachment(this, "DataVolumeAttachment", {
+      instanceId: instance.instanceId,
+      volumeId: dataVolume.volumeId,
+      device: "/dev/xvdf",
+    });
+
    const dlmRole = new iam.Role(this, "DlmRole", {
      roleName: "forgejo-dlm",
      assumedBy: new iam.ServicePrincipal("dlm.amazonaws.com"),
--- a/package-lock.json
+++ b/package-lock.json
@ -8,8 +8,8 @@
      "name": "forgejo",
      "version": "1.0.0",
      "dependencies": {
-        "@aws-cdk/aws-lambda-python-alpha": "2.253.1-alpha.0",
-        "aws-cdk-lib": "2.253.1",
+        "@aws-cdk/aws-lambda-python-alpha": "2.257.0-alpha.0",
+        "aws-cdk-lib": "2.257.0",
        "constructs": "^10.0.0"
      },
      "bin": {
@ -29,43 +29,43 @@
      "license": "Apache-2.0"
    },
    "node_modules/@aws-cdk/asset-node-proxy-agent-v6": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.1.tgz",
-      "integrity": "sha512-We4bmHaowOPHr+IQR4/FyTGjRfjgBj4ICMjtqmJeBDWad3Q/6St12NT07leNtyuukv2qMhtSZJQorD8KpKTwRA==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.2.tgz",
+      "integrity": "sha512-pDiuqH+qY3zM9lhhLjbKJ1tnKOHzQ2V4Wr/3qsxyKeKAkuPMI/BVGvZG1PbrikUw949cGVTfVEt4ETKKYnrj0Q==",
      "license": "Apache-2.0"
    },
    "node_modules/@aws-cdk/aws-lambda-python-alpha": {
-      "version": "2.253.1-alpha.0",
-      "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda-python-alpha/-/aws-lambda-python-alpha-2.253.1-alpha.0.tgz",
-      "integrity": "sha512-k4yeoKExF7aGvmcY7pPFP6wjrt1L/mNtnddA72yIqcSDumAKI0hRqkqp7yG6GC45ubeuAD0pCQeSW9YGkEMI9A==",
+      "version": "2.257.0-alpha.0",
+      "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda-python-alpha/-/aws-lambda-python-alpha-2.257.0-alpha.0.tgz",
+      "integrity": "sha512-snBmZkyvwG0P7EkUCEcYGy5TBzNpLdxIDrSoS8sD5aYXFbX+SeDs3rcYdx6Y22W8cn5vhXiIPYxl4NCDimKBSA==",
      "license": "Apache-2.0",
      "engines": {
        "node": ">= 20.0.0"
      },
      "peerDependencies": {
-        "aws-cdk-lib": "^2.253.1",
+        "aws-cdk-lib": "^2.257.0",
        "constructs": "^10.5.0"
      }
    },
    "node_modules/@aws-cdk/cloud-assembly-schema": {
-      "version": "53.22.0",
-      "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-53.22.0.tgz",
-      "integrity": "sha512-2GLhjjf7Db697rRyYt6rjN06fwbBIiewPo/jxSCyUN259ejF7NQQRwvrdAQb9Aq2jPaIIp1cfHSoEdXyA6Bb7Q==",
+      "version": "53.28.0",
+      "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-53.28.0.tgz",
+      "integrity": "sha512-pZS+9bLGv2tCqcgxfA0WD3XjcqT3yE4ICvKeJEicw6aTdCxBl8FQ/AUsorY/6f2JrMS3kUQgvhXxA30MWcji0A==",
      "bundleDependencies": [
        "jsonschema",
        "semver"
      ],
      "license": "Apache-2.0",
      "dependencies": {
-        "jsonschema": "~1.4.1",
-        "semver": "^7.7.4"
+        "jsonschema": "^1.5.0",
+        "semver": "^7.8.0"
      },
      "engines": {
        "node": ">= 18.0.0"
      }
    },
    "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": {
-      "version": "1.4.1",
+      "version": "1.5.0",
      "inBundle": true,
      "license": "MIT",
      "engines": {
@ -73,7 +73,7 @@
      }
    },
    "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": {
-      "version": "7.7.4",
+      "version": "7.8.0",
      "inBundle": true,
      "license": "ISC",
      "bin": {
@ -84,9 +84,9 @@
      }
    },
    "node_modules/@types/node": {
-      "version": "25.9.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.0.tgz",
-      "integrity": "sha512-AOQwYUNolgy3VosiRqXrACUXTN8nJUtPl7FJXMqZVyxiiCLhQuG3jXKvCS1ALr+Y2OmZhzzLVlYPEqJaiqkaJQ==",
+      "version": "25.9.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz",
+      "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
@ -94,9 +94,9 @@
      }
    },
    "node_modules/aws-cdk": {
-      "version": "2.1122.0",
-      "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.1122.0.tgz",
-      "integrity": "sha512-AI2Ks9qioWLvBPD4IoEtTet3wUG/o/q6U3WR3VCQKH5sNYLLPALo8o9sermNpMnfd1OQkqhL20tp4cEyojrIZg==",
+      "version": "2.1126.0",
+      "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.1126.0.tgz",
+      "integrity": "sha512-uNoocb3vCPiAT3j9+SwL6pn/VVggHWBsgC2XpxyhNvYQYt6cE9BM/149GWwtdcwnLrPjnwW1+CV/5nSSh5dV+w==",
      "dev": true,
      "license": "Apache-2.0",
      "bin": {
@ -107,9 +107,9 @@
      }
    },
    "node_modules/aws-cdk-lib": {
-      "version": "2.253.1",
-      "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.253.1.tgz",
-      "integrity": "sha512-vy+hA15/ZfSQpivkNdlIn2ZDA2hesp3WJgmtIZJDFwu6xzwv7wH7glbAdu5xCHGcOjepOaTKZSvCPC6sN+0/Vw==",
+      "version": "2.257.0",
+      "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.257.0.tgz",
+      "integrity": "sha512-GoHfWklrBJcMwLtDlY64pvaT7cD2KyDXC8sik89DR6jHl6nQsBtYTKSJCM+C/k4jgXaecbv8myNX75FySejq0A==",
      "bundleDependencies": [
        "@balena/dockerignore",
        "@aws-cdk/cloud-assembly-api",
@ -128,8 +128,8 @@
      "dependencies": {
        "@aws-cdk/asset-awscli-v1": "2.2.273",
        "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.1",
-        "@aws-cdk/cloud-assembly-api": "^2.2.2",
-        "@aws-cdk/cloud-assembly-schema": "^53.18.0",
+        "@aws-cdk/cloud-assembly-api": "^2.2.4",
+        "@aws-cdk/cloud-assembly-schema": "^53.25.0",
        "@balena/dockerignore": "^1.0.2",
        "case": "1.6.3",
        "fs-extra": "^11.3.3",
@ -150,22 +150,29 @@
      }
    },
    "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api": {
-      "version": "2.2.2",
-      "bundleDependencies": [
-        "jsonschema",
-        "semver"
-      ],
+      "version": "2.2.4",
      "inBundle": true,
      "license": "Apache-2.0",
      "dependencies": {
-        "jsonschema": "~1.4.1",
-        "semver": "^7.7.4"
+        "jsonschema": "^1.5.0",
+        "semver": "^7.8.0"
      },
      "engines": {
        "node": ">= 18.0.0"
      },
      "peerDependencies": {
-        "@aws-cdk/cloud-assembly-schema": ">=53.15.0"
+        "@aws-cdk/cloud-assembly-schema": ">=53.25.0"
+      }
+    },
+    "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api/node_modules/semver": {
+      "version": "7.8.0",
+      "inBundle": true,
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
      }
    },
    "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": {
@ -272,7 +279,7 @@
      "license": "MIT"
    },
    "node_modules/aws-cdk-lib/node_modules/fast-uri": {
-      "version": "3.1.0",
+      "version": "3.1.2",
      "funding": [
        {
          "type": "github",
--- a/package.json
+++ b/package.json
@ -18,8 +18,8 @@
    "typescript": "~6.0.3"
  },
  "dependencies": {
-    "@aws-cdk/aws-lambda-python-alpha": "2.253.1-alpha.0",
-    "aws-cdk-lib": "2.253.1",
+    "@aws-cdk/aws-lambda-python-alpha": "2.257.0-alpha.0",
+    "aws-cdk-lib": "2.257.0",
    "constructs": "^10.0.0"
  }
 }