adam/forgejo
1
0
Fork 0
mirror of https://github.com/Sea-Haven-Industries/forgejo.git synced 2026-09-30 05:23:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

docs(terraform): keep restore runbook on one bucket and fail closed

Browse source 
Glacier and the download use the prod bucket. GCS unpacks on the data volume. Neither path deletes live repos until the dump has a database.
This commit is contained in:
Adam Moussa 2026-09-29 19:48:39 -04:00
parent 2cc849024f
commit 2541cd934c
No known key found for this signature in database
1 changed files with 41 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
README.md
View file

@ -107,27 +107,35 @@ sudo /usr/local/bin/forgejo-backup.sh
For backups older than 30 days (Glacier), restore the object first: For backups older than 30 days (Glacier), restore the object first:
```bash ```bash
aws s3api restore-object --bucket forgejo-backups-328440206208 \ aws s3api restore-object --bucket forgejo-backups-011934824531 \
--key "archive/<date>/forgejo-<date>.tar.gz" \ --key "archive/<date>/forgejo-<date>.tar.gz" \
--restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}' --restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}'
# Wait ~3-5 hours for restore to complete, then: # Wait ~3-5 hours for restore to complete, then:
``` ```
Download and restore: Download and restore. The copy does not start until the dump contains a database, so a failed download leaves the live data alone.
```bash ```bash
set -euo pipefail
rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore
aws s3 cp s3://forgejo-backups-011934824531/archive/<date>/forgejo-<date>.tar.gz - --no-progress \ aws s3 cp s3://forgejo-backups-011934824531/archive/<date>/forgejo-<date>.tar.gz - --no-progress \
| tar -xz -C /var/lib/forgejo/.restore | tar -xz -C /var/lib/forgejo/.restore
if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then
echo "Dump has no database" >&2
exit 1
fi
systemctl stop forgejo systemctl stop forgejo
cd /var/lib/forgejo/.restore [ -d /var/lib/forgejo/.restore/data ] && cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/
[ -d data ] && cp -a data/. /var/lib/forgejo/data/
rm -rf /var/lib/forgejo/data/repositories rm -rf /var/lib/forgejo/data/repositories
mkdir -p /var/lib/forgejo/data/repositories mkdir -p /var/lib/forgejo/data/repositories
[ -d repos ] && cp -a repos/. /var/lib/forgejo/data/repositories/ [ -d /var/lib/forgejo/.restore/repos ] && cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/
[ -f gitea-db.sqlite3 ] && cp gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
[ -d lfs ] && cp -a lfs/. /var/lib/forgejo/data/lfs/ [ -d /var/lib/forgejo/.restore/lfs ] && mkdir -p /var/lib/forgejo/data/lfs && cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/
[ -d custom ] && cp -a custom/. /var/lib/forgejo/custom/ [ -d /var/lib/forgejo/.restore/custom ] && cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/
if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then
echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2
exit 1
fi
chown -R forgejo:forgejo /var/lib/forgejo chown -R forgejo:forgejo /var/lib/forgejo
systemctl start forgejo systemctl start forgejo
rm -rf /var/lib/forgejo/.restore rm -rf /var/lib/forgejo/.restore
@ -135,10 +143,33 @@ rm -rf /var/lib/forgejo/.restore
### Restore from GCS (disaster recovery) ### Restore from GCS (disaster recovery)
This path does not read S3. It unpacks the offsite object on the data volume and uses the same copy order as the S3 restore.
```bash ```bash
set -euo pipefail
gcloud config set project sea-haven-backups gcloud config set project sea-haven-backups
gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz /tmp/ rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore
# Then follow the same restore steps as S3 above gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz - \
| tar -xz -C /var/lib/forgejo/.restore
if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then
echo "Dump has no database" >&2
exit 1
fi
systemctl stop forgejo
[ -d /var/lib/forgejo/.restore/data ] && cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/
rm -rf /var/lib/forgejo/data/repositories
mkdir -p /var/lib/forgejo/data/repositories
[ -d /var/lib/forgejo/.restore/repos ] && cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/
[ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
[ -d /var/lib/forgejo/.restore/lfs ] && mkdir -p /var/lib/forgejo/data/lfs && cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/
[ -d /var/lib/forgejo/.restore/custom ] && cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/
if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then
echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2
exit 1
fi
chown -R forgejo:forgejo /var/lib/forgejo
systemctl start forgejo
rm -rf /var/lib/forgejo/.restore
``` ```
## Autodiscovery ## Autodiscovery