fix(terraform): restore the dump app.ini with the database

INTERNAL_TOKEN, JWT_SECRET, and LFS_JWT_SECRET live in that file. A restore that keeps the generated file cannot decrypt the dumped secrets.

README.md

@@ -107,37 +107,101 @@ sudo /usr/local/bin/forgejo-backup.sh
 For backups older than 30 days (Glacier), restore the object first:
 
 ```bash
-aws s3api restore-object --bucket forgejo-backups-328440206208 \
+aws s3api restore-object --bucket forgejo-backups-011934824531 \
   --key "archive/<date>/forgejo-<date>.tar.gz" \
   --restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}'
 # Wait ~3-5 hours for restore to complete, then:
 ```
 
-Download and restore:
+Download and restore. The copy does not start until the dump contains a database, so a failed download leaves the live data alone.
 
 ```bash
-aws s3 cp s3://forgejo-backups-328440206208/archive/<date>/forgejo-<date>.tar.gz /tmp/
+set -euo pipefail
+rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore
+aws s3 cp s3://forgejo-backups-011934824531/archive/<date>/forgejo-<date>.tar.gz - --no-progress \
+  | tar -xz -C /var/lib/forgejo/.restore
+if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then
+  echo "Dump has no database" >&2
+  exit 1
+fi
 systemctl stop forgejo
-mkdir -p /tmp/forgejo-restore && tar -xzf /tmp/forgejo-<date>.tar.gz -C /tmp/forgejo-restore
+if [ -d /var/lib/forgejo/.restore/data ]; then
-cd /tmp/forgejo-restore
+  cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/
-cp app.ini /etc/forgejo/app.ini
+fi
-cp gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
 rm -rf /var/lib/forgejo/data/repositories
-cp -a repos /var/lib/forgejo/data/repositories
+mkdir -p /var/lib/forgejo/data/repositories
-cp -a data/. /var/lib/forgejo/data/
+if [ -d /var/lib/forgejo/.restore/repos ]; then
-[ -d lfs ] && cp -a lfs/. /var/lib/forgejo/data/lfs/
+  cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/
-[ -d custom ] && cp -a custom/. /var/lib/forgejo/custom/
+fi
-chown -R forgejo:forgejo /var/lib/forgejo /etc/forgejo/app.ini
+if [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ]; then
+  cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
+fi
+if [ -d /var/lib/forgejo/.restore/lfs ]; then
+  mkdir -p /var/lib/forgejo/data/lfs
+  cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/
+fi
+if [ -d /var/lib/forgejo/.restore/custom ]; then
+  cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/
+fi
+if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then
+  echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2
+  exit 1
+fi
+if [ -f /var/lib/forgejo/.restore/app.ini ]; then
+  cp /var/lib/forgejo/.restore/app.ini /etc/forgejo/app.ini
+  chown root:forgejo /etc/forgejo/app.ini
+  chmod 660 /etc/forgejo/app.ini
+fi
+chown -R forgejo:forgejo /var/lib/forgejo
 systemctl start forgejo
-rm -rf /tmp/forgejo-restore /tmp/forgejo-<date>.tar.gz
+rm -rf /var/lib/forgejo/.restore
 ```
 
 ### Restore from GCS (disaster recovery)
 
+This path does not read S3. It unpacks the offsite object on the data volume and uses the same copy order as the S3 restore.
+
 ```bash
+set -euo pipefail
 gcloud config set project sea-haven-backups
-gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz /tmp/
+rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore
-# Then follow the same restore steps as S3 above
+gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz - \
+  | tar -xz -C /var/lib/forgejo/.restore
+if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then
+  echo "Dump has no database" >&2
+  exit 1
+fi
+systemctl stop forgejo
+if [ -d /var/lib/forgejo/.restore/data ]; then
+  cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/
+fi
+rm -rf /var/lib/forgejo/data/repositories
+mkdir -p /var/lib/forgejo/data/repositories
+if [ -d /var/lib/forgejo/.restore/repos ]; then
+  cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/
+fi
+if [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ]; then
+  cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
+fi
+if [ -d /var/lib/forgejo/.restore/lfs ]; then
+  mkdir -p /var/lib/forgejo/data/lfs
+  cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/
+fi
+if [ -d /var/lib/forgejo/.restore/custom ]; then
+  cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/
+fi
+if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then
+  echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2
+  exit 1
+fi
+if [ -f /var/lib/forgejo/.restore/app.ini ]; then
+  cp /var/lib/forgejo/.restore/app.ini /etc/forgejo/app.ini
+  chown root:forgejo /etc/forgejo/app.ini
+  chmod 660 /etc/forgejo/app.ini
+fi
+chown -R forgejo:forgejo /var/lib/forgejo
+systemctl start forgejo
+rm -rf /var/lib/forgejo/.restore
 ```
 
 ## Autodiscovery

terraform/hcp_iam.tf

@@ -13,9 +13,10 @@
 #   4. Point TFC_AWS_* at hcptf-forgejo / hcptf-forgejo-plan.
 #   5. Re-run the script without --allow-workspace.
 #   6. Second Manual apply creates the instance, buckets, ALB, and Lambda.
-# Later edits to these hcptf-* inline policies need the same window.
+# Later edits to hcptf-* inline policies and to policy/tf-managed/forgejo-services
-# DenySelfMutation blocks PutRolePolicy on hcptf-* from the scoped role.
+# need the org-account role. The scoped role cannot PutRolePolicy or
-# Do not add StringLike on bootstrap trust. CreatePolicy stays on hcptf-bootstrap.
+# CreatePolicyVersion. Do not add StringLike on bootstrap trust.
+# CreatePolicy stays on hcptf-bootstrap.
 
 data "aws_iam_policy_document" "hcptf_apply_trust" {
   statement {

terraform/user_data.sh

@@ -125,8 +125,10 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then
     if [ -d "$RESTORE_DIR/repos" ]; then
       cp -a "$RESTORE_DIR"/repos/. /var/lib/forgejo/data/repositories/
     fi
-    # Dump sqlite lives at the archive root, not under data/. Copy it last.
+    # Older dumps keep sqlite at the archive root. Current dumps already have data/forgejo.db.
-    cp "$RESTORE_DIR/gitea-db.sqlite3" /var/lib/forgejo/data/forgejo.db
+    if [ -f "$RESTORE_DIR/gitea-db.sqlite3" ]; then
+      cp "$RESTORE_DIR/gitea-db.sqlite3" /var/lib/forgejo/data/forgejo.db
+    fi
     if [ -d "$RESTORE_DIR/lfs" ]; then
       mkdir -p /var/lib/forgejo/data/lfs
       cp -a "$RESTORE_DIR"/lfs/. /var/lib/forgejo/data/lfs/
@@ -134,6 +136,12 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then
     if [ -d "$RESTORE_DIR/custom" ]; then
       cp -a "$RESTORE_DIR"/custom/. /var/lib/forgejo/custom/
     fi
+    # The dump's app.ini carries INTERNAL_TOKEN, JWT_SECRET, and LFS_JWT_SECRET.
+    if [ -f "$RESTORE_DIR/app.ini" ]; then
+      cp "$RESTORE_DIR/app.ini" /etc/forgejo/app.ini
+      chown root:forgejo /etc/forgejo/app.ini
+      chmod 660 /etc/forgejo/app.ini
+    fi
     chown -R forgejo:forgejo /var/lib/forgejo
     rm -rf "$RESTORE_DIR"
     if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then