From 2541cd934c378c3cd3b51c94c543642baac9df5e Mon Sep 17 00:00:00 2001
From: Adam Moussa <adam@seahavenind.com>
Date: Tue, 29 Sep 2026 19:48:39 -0400
Subject: [PATCH] docs(terraform): keep restore runbook on one bucket and fail
 closed

Glacier and the download use the prod bucket. GCS unpacks on the data volume. Neither path deletes live repos until the dump has a database.
---
 README.md | 51 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 41 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 9639699..28a3d32 100644
--- a/README.md
+++ b/README.md
@@ -107,27 +107,35 @@ sudo /usr/local/bin/forgejo-backup.sh
 For backups older than 30 days (Glacier), restore the object first:
 
 ```bash
-aws s3api restore-object --bucket forgejo-backups-328440206208 \
+aws s3api restore-object --bucket forgejo-backups-011934824531 \
   --key "archive/<date>/forgejo-<date>.tar.gz" \
   --restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}'
 # Wait ~3-5 hours for restore to complete, then:
 ```
 
-Download and restore:
+Download and restore. The copy does not start until the dump contains a database, so a failed download leaves the live data alone.
 
 ```bash
+set -euo pipefail
 rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore
 aws s3 cp s3://forgejo-backups-011934824531/archive/<date>/forgejo-<date>.tar.gz - --no-progress \
   | tar -xz -C /var/lib/forgejo/.restore
+if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then
+  echo "Dump has no database" >&2
+  exit 1
+fi
 systemctl stop forgejo
-cd /var/lib/forgejo/.restore
-[ -d data ] && cp -a data/. /var/lib/forgejo/data/
+[ -d /var/lib/forgejo/.restore/data ] && cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/
 rm -rf /var/lib/forgejo/data/repositories
 mkdir -p /var/lib/forgejo/data/repositories
-[ -d repos ] && cp -a repos/. /var/lib/forgejo/data/repositories/
-[ -f gitea-db.sqlite3 ] && cp gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
-[ -d lfs ] && cp -a lfs/. /var/lib/forgejo/data/lfs/
-[ -d custom ] && cp -a custom/. /var/lib/forgejo/custom/
+[ -d /var/lib/forgejo/.restore/repos ] && cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/
+[ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
+[ -d /var/lib/forgejo/.restore/lfs ] && mkdir -p /var/lib/forgejo/data/lfs && cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/
+[ -d /var/lib/forgejo/.restore/custom ] && cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/
+if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then
+  echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2
+  exit 1
+fi
 chown -R forgejo:forgejo /var/lib/forgejo
 systemctl start forgejo
 rm -rf /var/lib/forgejo/.restore
@@ -135,10 +143,33 @@ rm -rf /var/lib/forgejo/.restore
 
 ### Restore from GCS (disaster recovery)
 
+This path does not read S3. It unpacks the offsite object on the data volume and uses the same copy order as the S3 restore.
+
 ```bash
+set -euo pipefail
 gcloud config set project sea-haven-backups
-gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz /tmp/
-# Then follow the same restore steps as S3 above
+rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore
+gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz - \
+  | tar -xz -C /var/lib/forgejo/.restore
+if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then
+  echo "Dump has no database" >&2
+  exit 1
+fi
+systemctl stop forgejo
+[ -d /var/lib/forgejo/.restore/data ] && cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/
+rm -rf /var/lib/forgejo/data/repositories
+mkdir -p /var/lib/forgejo/data/repositories
+[ -d /var/lib/forgejo/.restore/repos ] && cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/
+[ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
+[ -d /var/lib/forgejo/.restore/lfs ] && mkdir -p /var/lib/forgejo/data/lfs && cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/
+[ -d /var/lib/forgejo/.restore/custom ] && cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/
+if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then
+  echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2
+  exit 1
+fi
+chown -R forgejo:forgejo /var/lib/forgejo
+systemctl start forgejo
+rm -rf /var/lib/forgejo/.restore
 ```
 
 ## Autodiscovery