From b2164d8c7edd12fdbd2617967166c05d804dfa31 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:10:09 +0000 Subject: [PATCH] fix(terraform): allow HCP refresh of the log group and parameter (PLAT-80) (#101) * fix(terraform): allow HCP refresh of the log group and parameter The scoped apply role can create those resources, but CloudWatch and SSM list them on a wildcard ARN. DescribeLogGroups and DescribeParameters need that resource. * fix(terraform): let the plan role read bucket website config The S3 provider refreshes GetBucketWebsite. The plan role was denied on the three Forgejo buckets. * fix(terraform): let the plan role read backup object metadata HeadObject on the Lambda zip is s3:GetObject. The plan role only had the bucket ARNs. * fix(terraform): let the plan role read object tags and retention The S3 provider refreshes tagging, ACL, attributes, and Object Lock on the Lambda zip. * fix(terraform): scope plan object reads and restore on the data volume The plan role only needs object reads for the verification zip. Unpacking a dump in /tmp fills the root volume. * fix(terraform): accept dumps that already contain data/forgejo.db Today's archive has no gitea-db.sqlite3 at the root. Copy that file only when it is present. * docs(terraform): keep restore runbook on one bucket and fail closed Glacier and the download use the prod bucket. GCS unpacks on the data volume. Neither path deletes live repos until the dump has a database. * docs(terraform): keep optional restore copies from aborting under set -e if/fi matches user_data.sh. The file comment now says forgejo-services versions also go through the org-account role. * fix(terraform): restore the dump app.ini with the database INTERNAL_TOKEN, JWT_SECRET, and LFS_JWT_SECRET live in that file. A restore that keeps the generated file cannot decrypt the dumped secrets. --- README.md | 94 +++++++++++++++++++++++++++++++++++------- terraform/hcp_iam.tf | 56 +++++++++++++++++++++++-- terraform/user_data.sh | 20 ++++++--- 3 files changed, 146 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 25bff69..fce3b30 100644 --- a/README.md +++ b/README.md @@ -107,37 +107,101 @@ sudo /usr/local/bin/forgejo-backup.sh For backups older than 30 days (Glacier), restore the object first: ```bash -aws s3api restore-object --bucket forgejo-backups-328440206208 \ +aws s3api restore-object --bucket forgejo-backups-011934824531 \ --key "archive//forgejo-.tar.gz" \ --restore-request '{"Days":7,"GlacierJobParameters":{"Tier":"Standard"}}' # Wait ~3-5 hours for restore to complete, then: ``` -Download and restore: +Download and restore. The copy does not start until the dump contains a database, so a failed download leaves the live data alone. ```bash -aws s3 cp s3://forgejo-backups-328440206208/archive//forgejo-.tar.gz /tmp/ +set -euo pipefail +rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore +aws s3 cp s3://forgejo-backups-011934824531/archive//forgejo-.tar.gz - --no-progress \ + | tar -xz -C /var/lib/forgejo/.restore +if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then + echo "Dump has no database" >&2 + exit 1 +fi systemctl stop forgejo -mkdir -p /tmp/forgejo-restore && tar -xzf /tmp/forgejo-.tar.gz -C /tmp/forgejo-restore -cd /tmp/forgejo-restore -cp app.ini /etc/forgejo/app.ini -cp gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db +if [ -d /var/lib/forgejo/.restore/data ]; then + cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/ +fi rm -rf /var/lib/forgejo/data/repositories -cp -a repos /var/lib/forgejo/data/repositories -cp -a data/. /var/lib/forgejo/data/ -[ -d lfs ] && cp -a lfs/. /var/lib/forgejo/data/lfs/ -[ -d custom ] && cp -a custom/. /var/lib/forgejo/custom/ -chown -R forgejo:forgejo /var/lib/forgejo /etc/forgejo/app.ini +mkdir -p /var/lib/forgejo/data/repositories +if [ -d /var/lib/forgejo/.restore/repos ]; then + cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/ +fi +if [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ]; then + cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db +fi +if [ -d /var/lib/forgejo/.restore/lfs ]; then + mkdir -p /var/lib/forgejo/data/lfs + cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/ +fi +if [ -d /var/lib/forgejo/.restore/custom ]; then + cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/ +fi +if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then + echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2 + exit 1 +fi +if [ -f /var/lib/forgejo/.restore/app.ini ]; then + cp /var/lib/forgejo/.restore/app.ini /etc/forgejo/app.ini + chown root:forgejo /etc/forgejo/app.ini + chmod 660 /etc/forgejo/app.ini +fi +chown -R forgejo:forgejo /var/lib/forgejo systemctl start forgejo -rm -rf /tmp/forgejo-restore /tmp/forgejo-.tar.gz +rm -rf /var/lib/forgejo/.restore ``` ### Restore from GCS (disaster recovery) +This path does not read S3. It unpacks the offsite object on the data volume and uses the same copy order as the S3 restore. + ```bash +set -euo pipefail gcloud config set project sea-haven-backups -gsutil cp gs://forgejo-backups-offsite-seahaven/archive//forgejo-.tar.gz /tmp/ -# Then follow the same restore steps as S3 above +rm -rf /var/lib/forgejo/.restore && mkdir -p /var/lib/forgejo/.restore +gsutil cp gs://forgejo-backups-offsite-seahaven/archive//forgejo-.tar.gz - \ + | tar -xz -C /var/lib/forgejo/.restore +if [ ! -s /var/lib/forgejo/.restore/gitea-db.sqlite3 ] && [ ! -s /var/lib/forgejo/.restore/data/forgejo.db ]; then + echo "Dump has no database" >&2 + exit 1 +fi +systemctl stop forgejo +if [ -d /var/lib/forgejo/.restore/data ]; then + cp -a /var/lib/forgejo/.restore/data/. /var/lib/forgejo/data/ +fi +rm -rf /var/lib/forgejo/data/repositories +mkdir -p /var/lib/forgejo/data/repositories +if [ -d /var/lib/forgejo/.restore/repos ]; then + cp -a /var/lib/forgejo/.restore/repos/. /var/lib/forgejo/data/repositories/ +fi +if [ -f /var/lib/forgejo/.restore/gitea-db.sqlite3 ]; then + cp /var/lib/forgejo/.restore/gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db +fi +if [ -d /var/lib/forgejo/.restore/lfs ]; then + mkdir -p /var/lib/forgejo/data/lfs + cp -a /var/lib/forgejo/.restore/lfs/. /var/lib/forgejo/data/lfs/ +fi +if [ -d /var/lib/forgejo/.restore/custom ]; then + cp -a /var/lib/forgejo/.restore/custom/. /var/lib/forgejo/custom/ +fi +if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then + echo "Restore did not produce /var/lib/forgejo/data/forgejo.db" >&2 + exit 1 +fi +if [ -f /var/lib/forgejo/.restore/app.ini ]; then + cp /var/lib/forgejo/.restore/app.ini /etc/forgejo/app.ini + chown root:forgejo /etc/forgejo/app.ini + chmod 660 /etc/forgejo/app.ini +fi +chown -R forgejo:forgejo /var/lib/forgejo +systemctl start forgejo +rm -rf /var/lib/forgejo/.restore ``` ## Autodiscovery diff --git a/terraform/hcp_iam.tf b/terraform/hcp_iam.tf index a64145a..11fb618 100644 --- a/terraform/hcp_iam.tf +++ b/terraform/hcp_iam.tf @@ -13,9 +13,10 @@ # 4. Point TFC_AWS_* at hcptf-forgejo / hcptf-forgejo-plan. # 5. Re-run the script without --allow-workspace. # 6. Second Manual apply creates the instance, buckets, ALB, and Lambda. -# Later edits to these hcptf-* inline policies need the same window. -# DenySelfMutation blocks PutRolePolicy on hcptf-* from the scoped role. -# Do not add StringLike on bootstrap trust. CreatePolicy stays on hcptf-bootstrap. +# Later edits to hcptf-* inline policies and to policy/tf-managed/forgejo-services +# need the org-account role. The scoped role cannot PutRolePolicy or +# CreatePolicyVersion. Do not add StringLike on bootstrap trust. +# CreatePolicy stays on hcptf-bootstrap. data "aws_iam_policy_document" "hcptf_apply_trust" { statement { @@ -437,6 +438,7 @@ data "aws_iam_policy_document" "hcptf_apply_services" { } # CreateLogGroup is not reliable on the log-group ARN before the group exists. + # DescribeLogGroups is a list API. Its resource is log-group::log-stream:, not the group ARN. statement { sid = "CreateVerificationLogGroup" effect = "Allow" @@ -444,6 +446,13 @@ data "aws_iam_policy_document" "hcptf_apply_services" { resources = ["*"] } + statement { + sid = "DescribeVerificationLogGroups" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + statement { sid = "VerificationLogs" effect = "Allow" @@ -501,6 +510,14 @@ data "aws_iam_policy_document" "hcptf_apply_services" { ] } + # DescribeParameters does not accept a parameter ARN. The provider calls it on *. + statement { + sid = "DescribeBackupParameters" + effect = "Allow" + actions = ["ssm:DescribeParameters"] + resources = ["*"] + } + statement { sid = "AlertTopicRead" effect = "Allow" @@ -586,6 +603,7 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { "s3:GetBucketRequestPayment", "s3:GetBucketTagging", "s3:GetBucketVersioning", + "s3:GetBucketWebsite", "s3:GetEncryptionConfiguration", "s3:GetLifecycleConfiguration", "s3:GetReplicationConfiguration", @@ -598,6 +616,22 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { ] } + # aws_s3_object refresh calls HeadObject and object metadata reads. Scope + # them to the verification package. Backup and replica objects stay unread. + statement { + sid = "RefreshLambdaPackage" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:GetObjectAcl", + "s3:GetObjectAttributes", + "s3:GetObjectTagging", + ] + resources = [ + "arn:aws:s3:::${local.artifacts_bucket_name}/functions/forgejo-backup-verification.zip", + ] + } + statement { sid = "RefreshEc2" effect = "Allow" @@ -687,7 +721,6 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { sid = "RefreshLogs" effect = "Allow" actions = [ - "logs:DescribeLogGroups", "logs:ListTagsForResource", ] resources = [ @@ -696,6 +729,14 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { ] } + # DescribeLogGroups is a list API. Its resource is log-group::log-stream:, not the group ARN. + statement { + sid = "DescribeVerificationLogGroups" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + statement { sid = "RefreshAlarms" effect = "Allow" @@ -718,6 +759,13 @@ data "aws_iam_policy_document" "hcptf_plan_refresh" { ] } + statement { + sid = "DescribeBackupParameters" + effect = "Allow" + actions = ["ssm:DescribeParameters"] + resources = ["*"] + } + statement { sid = "RefreshSnapshots" effect = "Allow" diff --git a/terraform/user_data.sh b/terraform/user_data.sh index c46f17e..ceafffc 100644 --- a/terraform/user_data.sh +++ b/terraform/user_data.sh @@ -111,9 +111,11 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then LATEST="$(aws s3 ls "s3://${BACKUP_BUCKET}/${S3_PREFIX}/" --region "${AWS_REGION}" | awk '{print $2}' | sort | tail -1 | tr -d '/')" if [ -n "$LATEST" ]; then FILE="$(aws s3 ls "s3://${BACKUP_BUCKET}/${S3_PREFIX}/${LATEST}/" --region "${AWS_REGION}" | awk '{print $4}' | tail -1)" - RESTORE_DIR="$(mktemp -d)" - aws s3 cp "s3://${BACKUP_BUCKET}/${S3_PREFIX}/${LATEST}/${FILE}" "$RESTORE_DIR/dump.tar.gz" --region "${AWS_REGION}" - tar xzf "$RESTORE_DIR/dump.tar.gz" -C "$RESTORE_DIR" + # The root volume is 20 GB. A dump expands well past that, so unpack on the data volume. + RESTORE_DIR=/var/lib/forgejo/.restore + rm -rf "$RESTORE_DIR" + mkdir -p "$RESTORE_DIR" + aws s3 cp "s3://${BACKUP_BUCKET}/${S3_PREFIX}/${LATEST}/${FILE}" - --region "${AWS_REGION}" --no-progress | tar -xz -C "$RESTORE_DIR" mkdir -p /var/lib/forgejo/data /var/lib/forgejo/custom if [ -d "$RESTORE_DIR/data" ]; then cp -a "$RESTORE_DIR"/data/. /var/lib/forgejo/data/ @@ -123,8 +125,10 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then if [ -d "$RESTORE_DIR/repos" ]; then cp -a "$RESTORE_DIR"/repos/. /var/lib/forgejo/data/repositories/ fi - # Dump sqlite lives at the archive root, not under data/. Copy it last. - cp "$RESTORE_DIR/gitea-db.sqlite3" /var/lib/forgejo/data/forgejo.db + # Older dumps keep sqlite at the archive root. Current dumps already have data/forgejo.db. + if [ -f "$RESTORE_DIR/gitea-db.sqlite3" ]; then + cp "$RESTORE_DIR/gitea-db.sqlite3" /var/lib/forgejo/data/forgejo.db + fi if [ -d "$RESTORE_DIR/lfs" ]; then mkdir -p /var/lib/forgejo/data/lfs cp -a "$RESTORE_DIR"/lfs/. /var/lib/forgejo/data/lfs/ @@ -132,6 +136,12 @@ if [ ! -f /var/lib/forgejo/data/forgejo.db ]; then if [ -d "$RESTORE_DIR/custom" ]; then cp -a "$RESTORE_DIR"/custom/. /var/lib/forgejo/custom/ fi + # The dump's app.ini carries INTERNAL_TOKEN, JWT_SECRET, and LFS_JWT_SECRET. + if [ -f "$RESTORE_DIR/app.ini" ]; then + cp "$RESTORE_DIR/app.ini" /etc/forgejo/app.ini + chown root:forgejo /etc/forgejo/app.ini + chmod 660 /etc/forgejo/app.ini + fi chown -R forgejo:forgejo /var/lib/forgejo rm -rf "$RESTORE_DIR" if [ ! -s /var/lib/forgejo/data/forgejo.db ]; then