diff --git a/README.md b/README.md index 25bff69..fce3b30 100644 --- a/README.md +++ b/README.md @@ -107,37 +107,101 @@ sudo /usr/local/bin/forgejo-backup.sh For backups older than 30 days (Glacier), restore the object first: ```bash -aws s3api restore-object --bucket forgejo-backups-328440206208 \ +aws s3api restore-object --bucket forgejo-backups-011934824531 \ --key "archive//forgejo-.tar.gz" \ --restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}' # Wait ~3-5 hours for restore to complete, then: ``` -Download and restore: +Download and restore. The copy does not start until the dump contains a database, so a failed download leaves the live data alone. ```bash -aws s3 cp s3://forgejo-backups-328440206208/archive//forgejo-.tar.gz /tmp/ +set -euo pipefail +rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore +aws s3 cp s3://forgejo-backups-011934824531/archive//forgejo-.tar.gz - --no-progress \ + | tar -xz -C /var/lib/forgejo/.restore +if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then + echo "Dump has no database" >&2 + exit 1 +fi systemctl stop forgejo -mkdir -p /tmp/forgejo-restore && tar -xzf /tmp/forgejo-.tar.gz -C /tmp/forgejo-restore -cd /tmp/forgejo-restore -cp app.ini /etc/forgejo/app.ini -cp gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db +if [ -d /var/lib/forgejo/.restore/data ]; then + cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/ +fi rm -rf /var/lib/forgejo/data/repositories -cp -a repos /var/lib/forgejo/data/repositories -cp -a data/. /var/lib/forgejo/data/ -[ -d lfs ] && cp -a lfs/. /var/lib/forgejo/data/lfs/ -[ -d custom ] && cp -a custom/. /var/lib/forgejo/custom/ -chown -R forgejo:forgejo /var/lib/forgejo /etc/forgejo/app.ini +mkdir -p /var/lib/forgejo/data/repositories +if [ -d /var/lib/forgejo/.restore/repos ]; then + cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/ +fi +if [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ]; then + cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db +fi +if [ -d /var/lib/forgejo/.restore/lfs ]; then + mkdir -p /var/lib/forgejo/data/lfs + cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/ +fi +if [ -d /var/lib/forgejo/.restore/custom ]; then + cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/ +fi +if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then + echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2 + exit 1 +fi +if [ -f /var/lib/forgejo/.restore/app.ini ]; then + cp /var/lib/forgejo/.restore/app.ini /etc/forgejo/app.ini + chown root:forgejo /etc/forgejo/app.ini + chmod 660 /etc/forgejo/app.ini +fi +chown -R forgejo:forgejo /var/lib/forgejo systemctl start forgejo -rm -rf /tmp/forgejo-restore /tmp/forgejo-.tar.gz +rm -rf /var/lib/forgejo/.restore ``` ### Restore from GCS (disaster recovery) +This path does not read S3. It unpacks the offsite object on the data volume and uses the same copy order as the S3 restore. + ```bash +set -euo pipefail gcloud config set project sea-haven-backups -gsutil cp gs://forgejo-backups-offsite-seahaven/archive//forgejo-.tar.gz /tmp/ -# Then follow the same restore steps as S3 above +rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore +gsutil cp gs://forgejo-backups-offsite-seahaven/archive//forgejo-.tar.gz - \ + | tar -xz -C /var/lib/forgejo/.restore +if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then + echo "Dump has no database" >&2 + exit 1 +fi +systemctl stop forgejo +if [ -d /var/lib/forgejo/.restore/data ]; then + cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/ +fi +rm -rf /var/lib/forgejo/data/repositories +mkdir -p /var/lib/forgejo/data/repositories +if [ -d /var/lib/forgejo/.restore/repos ]; then + cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/ +fi +if [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ]; then + cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db +fi +if [ -d /var/lib/forgejo/.restore/lfs ]; then + mkdir -p /var/lib/forgejo/data/lfs + cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/ +fi +if [ -d /var/lib/forgejo/.restore/custom ]; then + cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/ +fi +if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then + echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2 + exit 1 +fi +if [ -f /var/lib/forgejo/.restore/app.ini ]; then + cp /var/lib/forgejo/.restore/app.ini /etc/forgejo/app.ini + chown root:forgejo /etc/forgejo/app.ini + chmod 660 /etc/forgejo/app.ini +fi +chown -R forgejo:forgejo /var/lib/forgejo +systemctl start forgejo +rm -rf /var/lib/forgejo/.restore ``` ## Autodiscovery diff --git a/terraform/hcp_iam.tf b/terraform/hcp_iam.tf index a64145a..11fb618 100644 --- a/terraform/hcp_iam.tf +++ b/terraform/hcp_iam.tf @@ -13,9 +13,10 @@ # 4. Point TFC_AWS_* at hcptf-forgejo / hcptf-forgejo-plan. # 5. Re-run the script without --allow-workspace. # 6. Second Manual apply creates the instance, buckets, ALB, and Lambda. -# Later edits to these hcptf-* inline policies need the same window. -# DenySelfMutation blocks PutRolePolicy on hcptf-* from the scoped role. -# Do not add StringLike on bootstrap trust. CreatePolicy stays on hcptf-bootstrap. +# Later edits to hcptf-* inline policies and to policy/tf-managed/forgejo-services +# need the org-account role. The scoped role cannot PutRolePolicy or +# CreatePolicyVersion. Do not add StringLike on bootstrap trust. +# CreatePolicy stays on hcptf-bootstrap. data "aws_iam_policy_document" "hcptf_apply_trust" { statement { @@ -437,6 +438,7 @@ data "aws_iam_policy_document" "hcptf_apply_services" { } # CreateLogGroup is not reliable on the log-group ARN before the group exists. + # DescribeLogGroups is a list API. Its resource is log-group::log-stream:, not the group ARN. statement { sid = "CreateVerificationLogGroup" effect = "Allow" @@ -444,6 +446,13 @@ data "aws_iam_policy_document" "hcptf_apply_services" { resources = ["*"] } + statement { + sid = "DescribeVerificationLogGroups" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + statement { sid = "VerificationLogs" effect = "Allow" @@ -501,6 +510,14 @@ data "aws_iam_policy_document" "hcptf_apply_services" { ] } + # DescribeParameters does not accept a parameter ARN. The provider calls it on *. + statement { + sid = "DescribeBackupParameters" + effect = "Allow" + actions = ["ssm:DescribeParameters"] + resources = ["*"] + } + statement { sid = "AlertTopicRead" effect = "Allow" @@ -586,6 +603,7 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { "s3:GetBucketRequestPayment", "s3:GetBucketTagging", "s3:GetBucketVersioning", + "s3:GetBucketWebsite", "s3:GetEncryptionConfiguration", "s3:GetLifecycleConfiguration", "s3:GetReplicationConfiguration", @@ -598,6 +616,22 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { ] } + # aws_s3_object refresh calls HeadObject and object metadata reads. Scope + # them to the verification package. Backup and replica objects stay unread. + statement { + sid = "RefreshLambdaPackage" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:GetObjectAcl", + "s3:GetObjectAttributes", + "s3:GetObjectTagging", + ] + resources = [ + "arn:aws:s3:::${local.artifacts_bucket_name}/functions/forgejo-backup-verification.zip", + ] + } + statement { sid = "RefreshEc2" effect = "Allow" @@ -687,7 +721,6 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { sid = "RefreshLogs" effect = "Allow" actions = [ - "logs:DescribeLogGroups", "logs:ListTagsForResource", ] resources = [ @@ -696,6 +729,14 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { ] } + # DescribeLogGroups is a list API. Its resource is log-group::log-stream:, not the group ARN. + statement { + sid = "DescribeVerificationLogGroups" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + statement { sid = "RefreshAlarms" effect = "Allow" @@ -718,6 +759,13 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { ] } + statement { + sid = "DescribeBackupParameters" + effect = "Allow" + actions = ["ssm:DescribeParameters"] + resources = ["*"] + } + statement { sid = "RefreshSnapshots" effect = "Allow" diff --git a/terraform/user_data.sh b/terraform/user_data.sh index c46f17e..ceafffc 100644 --- a/terraform/user_data.sh +++ b/terraform/user_data.sh @@ -111,9 +111,11 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then LATEST="$(aws s3 ls "s3://${BACKUP_BUCKET}/${S3_PREFIX}/" --region "${AWS_REGION}" | awk '{print $2}' | sort | tail -1 | tr -d '/')" if [ -n "$LATEST" ]; then FILE="$(aws s3 ls "s3://${BACKUP_BUCKET}/${S3_PREFIX}/${LATEST}/" --region "${AWS_REGION}" | awk '{print $4}' | tail -1)" - RESTORE_DIR="$(mktemp -d)" - aws s3 cp "s3://${BACKUP_BUCKET}/${S3_PREFIX}/${LATEST}/${FILE}" "$RESTORE_DIR/dump.tar.gz" --region "${AWS_REGION}" - tar xzf "$RESTORE_DIR/dump.tar.gz" -C "$RESTORE_DIR" + # The root volume is 20 GB. A dump expands well past that, so unpack on the data volume. + RESTORE_DIR=/var/lib/forgejo/.restore + rm -rf "$RESTORE_DIR" + mkdir -p "$RESTORE_DIR" + aws s3 cp "s3://${BACKUP_BUCKET}/${S3_PREFIX}/${LATEST}/${FILE}" - --region "${AWS_REGION}" --no-progress | tar -xz -C "$RESTORE_DIR" mkdir -p /var/lib/forgejo/data /var/lib/forgejo/custom if [ -d "$RESTORE_DIR/data" ]; then cp -a "$RESTORE_DIR"/data/. /var/lib/forgejo/data/ @@ -123,8 +125,10 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then if [ -d "$RESTORE_DIR/repos" ]; then cp -a "$RESTORE_DIR"/repos/. /var/lib/forgejo/data/repositories/ fi - # Dump sqlite lives at the archive root, not under data/. Copy it last. - cp "$RESTORE_DIR/gitea-db.sqlite3" /var/lib/forgejo/data/forgejo.db + # Older dumps keep sqlite at the archive root. Current dumps already have data/forgejo.db. + if [ -f "$RESTORE_DIR/gitea-db.sqlite3" ]; then + cp "$RESTORE_DIR/gitea-db.sqlite3" /var/lib/forgejo/data/forgejo.db + fi if [ -d "$RESTORE_DIR/lfs" ]; then mkdir -p /var/lib/forgejo/data/lfs cp -a "$RESTORE_DIR"/lfs/. /var/lib/forgejo/data/lfs/ @@ -132,6 +136,12 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then if [ -d "$RESTORE_DIR/custom" ]; then cp -a "$RESTORE_DIR"/custom/. /var/lib/forgejo/custom/ fi + # The dump's app.ini carries INTERNAL_TOKEN, JWT_SECRET, and LFS_JWT_SECRET. + if [ -f "$RESTORE_DIR/app.ini" ]; then + cp "$RESTORE_DIR/app.ini" /etc/forgejo/app.ini + chown root:forgejo /etc/forgejo/app.ini + chmod 660 /etc/forgejo/app.ini + fi chown -R forgejo:forgejo /var/lib/forgejo rm -rf "$RESTORE_DIR" if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then