From d57aea19f3cce1c88047d6e18f4b72e0a2f82fd2 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Wed, 13 May 2026 18:02:50 -0400 Subject: [PATCH] Add 3-2-1 backup strategy with cross-region replication and GCS offsite Implements a fully compliant 3-2-1 backup architecture: - Copy 1 (live): Harden existing EBS snapshots to 30-day retention - Copy 2 (near-site): S3 cross-region replication to us-west-2 with Object Lock (governance 90d) and versioning - Copy 3 (offsite): GCS bucket in dedicated seahaven-backups GCP project with 2-year irreversible retention lock Also adds a verification Lambda that checks all 3 locations daily and runs monthly restore tests with SQLite integrity checks. --- .github/workflows/deploy.yaml | 2 + .gitignore | 1 + README.md | 102 +++++++++- bin/app.ts | 7 + lambda/backup-verification/app.py | 198 ++++++++++++++++++++ lambda/backup-verification/requirements.txt | 1 + lib/constructs/backup-verification.ts | 94 ++++++++++ lib/forgejo-replica-stack.ts | 48 +++++ lib/forgejo-stack.ts | 103 +++++++++- package-lock.json | 14 ++ package.json | 1 + scripts/gcp-setup.sh | 151 +++++++++++++++ 12 files changed, 704 insertions(+), 18 deletions(-) create mode 100644 lambda/backup-verification/app.py create mode 100644 lambda/backup-verification/requirements.txt create mode 100644 lib/constructs/backup-verification.ts create mode 100644 lib/forgejo-replica-stack.ts create mode 100755 scripts/gcp-setup.sh diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index e5462cf..6acd7ee 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -14,5 +14,7 @@ concurrency: jobs: deploy: uses: Sea-Haven-Industries/.github/.github/workflows/cd-cdk.yaml@main + with: + enable-qemu: true secrets: deploy-role-arn: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} diff --git a/.gitignore b/.gitignore index a6b3201..ecfe493 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ cdk.out/ *.d.ts *.js.map cdk.context.json +docs/*.pdf diff --git a/README.md b/README.md index 0e82e68..ba8fe71 100644 --- a/README.md +++ b/README.md @@ -20,19 +20,67 @@ Self-hosted Forgejo git server for archiving GitHub repos and mirroring active o | 3000 | HTTP | ALB → instance | Internal traffic from ALB | | 2222 | SSH | VPC + VPN | Git SSH operations | -## S3 Backups +## 3-2-1 Backup Strategy -A nightly `forgejo dump` runs at 5:00 UTC and uploads the archive to `s3://forgejo-backups-328440206208`. +All backups follow a 3-2-1 strategy: 3 copies, 2 storage types, 1 offsite provider. -**S3 lifecycle policy:** +| Copy | Location | Type | Retention | +|------|----------|------|-----------| +| Live | EBS volume (us-east-1) | Block | N/A | +| Near-site | S3 replica (us-west-2) | Object | Archive: indefinite, noncurrent versions: 90d | +| Offsite | GCS `forgejo-backups-offsite-seahaven` (GCP us-central1) | Object | 2-year locked retention | -| Phase | Duration | -|-------|----------| -| Standard | First 30 days | -| Glacier | Days 31–365 | -| Expired | After 365 days | +**Daily data flow:** -EBS snapshots are managed separately by DLM and run nightly at 6:00 UTC with a 7-day retention window. +| Time (UTC) | Event | +|------------|-------| +| 05:00 | `forgejo dump` → `s3://forgejo-backups-328440206208/archive/{date}/` | +| ~05:01 | S3 CRR replicates to `forgejo-backups-replica-328440206208` (us-west-2) | +| 06:00 | DLM EBS snapshot (30-day retention) | +| 08:00 | Verification Lambda checks all 3 locations, posts to Slack | +| 10:00 | GCS Storage Transfer pulls from S3 to GCS offsite | + +**S3 source lifecycle:** Standard 30d → Glacier (no expiration). + +**Immutability layers:** +- S3 Versioning on both source and replica buckets +- S3 Object Lock (Governance, 90d) on the replica bucket +- GCS Bucket Lock (2yr, irreversible) on the offsite bucket + +### Verification + +The `forgejo-backup-verification` Lambda runs daily at 08:00 UTC and checks: +1. S3 source has a recent dump under `archive/` +2. S3 replica has replicated the latest dump +3. GCS offsite has received the latest transfer +4. EBS snapshots exist within the last 48 hours + +On the 1st of each month at 09:00 UTC, it runs a restore test: downloads the latest dump, extracts the archive, and runs SQLite integrity checks. + +### Manual backup + +```bash +sudo /usr/local/bin/forgejo-backup.sh +``` + +### Restore from S3 + +```bash +aws s3 cp s3://forgejo-backups-328440206208/archive//forgejo-.tar.gz /tmp/ +systemctl stop forgejo +cd /tmp && tar xzf forgejo-.tar.gz +forgejo restore --config /etc/forgejo/app.ini --from /tmp/forgejo-dump-* +chown -R forgejo:forgejo /var/lib/forgejo +systemctl start forgejo +``` + +### Restore from GCS (disaster recovery) + +```bash +gcloud config set project seahaven-backups +gsutil cp gs://forgejo-backups-offsite-seahaven/archive//forgejo-.tar.gz /tmp/ +# Then follow the same restore steps as S3 +``` To test the backup manually: @@ -78,6 +126,9 @@ sudo /usr/local/bin/forgejo-refresh-tokens.sh | `forgejo/admin-password` | Forgejo admin user password | | `forgejo/api-token` | Forgejo API token (used by autodiscovery and token refresh scripts) | | `forgejo/github-pat` | GitHub fine-grained PAT for mirroring | +| `forgejo/gcs-sa-key` | GCP service account key for offsite backup verification | +| `forgejo/gcs-transfer-credentials` | AWS IAM credentials for GCS Storage Transfer Service | +| `forgejo/slack-webhook` | Slack webhook URL for backup verification alerts | ## First-time setup @@ -108,15 +159,46 @@ In the Forgejo web UI: **New Migration → GitHub** → paste the GitHub repo UR Same migration flow, but check **This Repository Will Be A Mirror**. Forgejo polls GitHub hourly (`DEFAULT_INTERVAL = 1h` in app.ini) and keeps the mirror in sync. +## GCP Offsite Setup (one-time) + +Run the setup script to create the GCS offsite bucket, service account, and store credentials: + +```bash +./scripts/gcp-setup.sh +``` + +This creates the `seahaven-backups` GCP project with a locked-retention GCS bucket. After running, configure the Storage Transfer job in the GCP Console using the AWS credentials from `forgejo/gcs-transfer-credentials`. + ## Deployment ```bash npm install -npx cdk deploy +npx cdk deploy --all ``` +This deploys two stacks: +- `forgejo-replica` (us-west-2) — S3 replica bucket with Object Lock +- `forgejo` (us-east-1) — main stack with Forgejo instance, CRR, and verification Lambda + CI/CD is handled by GitHub Actions — PRs run CI, merges to `main` deploy via the reusable CDK workflow. +## Post-deploy: update running instance backup path + +After the first deploy with the 3-2-1 changes, the running instance's backup script still uses the old S3 path (without the `archive/` prefix). Update it via SSM: + +```bash +aws ssm start-session --target i-0d3005fb3c36124cd +sudo sed -i 's|s3://forgejo-backups-328440206208/${TIMESTAMP}/|s3://forgejo-backups-328440206208/archive/${TIMESTAMP}/|' /usr/local/bin/forgejo-backup.sh +``` + +Also store the Slack webhook URL for backup verification alerts: + +```bash +aws secretsmanager create-secret --name forgejo/slack-webhook \ + --secret-string "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" \ + --region us-east-1 +``` + ## Updating Forgejo Update the `FORGEJO_VERSION` constant in `lib/forgejo-stack.ts` and deploy. This replaces the instance, so ensure the latest EBS snapshot is available for data recovery if needed. Alternatively, update in-place via SSM: diff --git a/bin/app.ts b/bin/app.ts index 3d93d46..1721cd7 100644 --- a/bin/app.ts +++ b/bin/app.ts @@ -2,8 +2,15 @@ import "source-map-support/register"; import * as cdk from "aws-cdk-lib"; import { ForgejoStack } from "../lib/forgejo-stack"; +import { ForgejoReplicaStack } from "../lib/forgejo-replica-stack"; const app = new cdk.App(); + +new ForgejoReplicaStack(app, "forgejo-replica", { + stackName: "forgejo-replica", + env: { account: "328440206208", region: "us-west-2" }, +}); + new ForgejoStack(app, "forgejo", { stackName: "forgejo", env: { account: "328440206208", region: "us-east-1" }, diff --git a/lambda/backup-verification/app.py b/lambda/backup-verification/app.py new file mode 100644 index 0000000..bc7a0dc --- /dev/null +++ b/lambda/backup-verification/app.py @@ -0,0 +1,198 @@ +import json +import os +import tarfile +import tempfile +import urllib.request +from datetime import datetime, timedelta, timezone + +import boto3 +from google.cloud import storage as gcs +from google.oauth2 import service_account + + +s3 = boto3.client("s3") +s3_west = boto3.client("s3", region_name="us-west-2") +ec2 = boto3.client("ec2") +secrets = boto3.client("secretsmanager") + +SOURCE_BUCKET = os.environ["SOURCE_BUCKET"] +REPLICA_BUCKET = os.environ["REPLICA_BUCKET"] +GCS_BUCKET = os.environ["GCS_BUCKET"] +GCS_SA_SECRET_ARN = os.environ["GCS_SA_SECRET_ARN"] +SLACK_WEBHOOK_SECRET_ARN = os.environ["SLACK_WEBHOOK_SECRET_ARN"] + +_gcs_client = None + + +def _get_gcs_client(): + global _gcs_client + if _gcs_client is None: + raw = secrets.get_secret_value(SecretId=GCS_SA_SECRET_ARN)["SecretString"] + info = json.loads(raw) + creds = service_account.Credentials.from_service_account_info(info) + _gcs_client = gcs.Client(credentials=creds, project=info.get("project_id")) + return _gcs_client + + +def _check_s3_bucket(client, bucket, label): + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=48) + try: + resp = client.list_objects_v2(Bucket=bucket, Prefix="archive/", MaxKeys=1000) + contents = resp.get("Contents", []) + if not contents: + return False, f"{label}: No objects found under archive/" + latest = max(contents, key=lambda o: o["LastModified"]) + if latest["LastModified"] < cutoff: + age = (now - latest["LastModified"]).total_seconds() / 3600 + return False, f"{label}: Latest dump is {age:.0f}h old ({latest['Key']})" + if latest["Size"] < 1_000_000: + return False, f"{label}: Latest dump suspiciously small ({latest['Size']} bytes)" + return True, f"{label}: OK — {latest['Key']} ({latest['Size'] / 1_000_000:.1f} MB)" + except Exception as e: + return False, f"{label}: Error — {e}" + + +def _check_gcs(): + try: + client = _get_gcs_client() + bucket = client.bucket(GCS_BUCKET) + blobs = list(bucket.list_blobs(prefix="archive/", max_results=1000)) + if not blobs: + return False, "GCS Offsite: No objects found under archive/" + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=72) + latest = max(blobs, key=lambda b: b.updated) + if latest.updated < cutoff: + age = (now - latest.updated).total_seconds() / 3600 + return False, f"GCS Offsite: Latest object is {age:.0f}h old ({latest.name})" + return True, f"GCS Offsite: OK — {latest.name} ({latest.size / 1_000_000:.1f} MB)" + except Exception as e: + return False, f"GCS Offsite: Error — {e}" + + +def _check_ebs_snapshots(): + try: + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=48) + resp = ec2.describe_snapshots( + Filters=[{"Name": "tag:forgejo-backup", "Values": ["true"]}], + OwnerIds=["self"], + ) + snapshots = resp.get("Snapshots", []) + if not snapshots: + return False, "EBS Snapshots: No snapshots found with forgejo-backup tag" + recent = [s for s in snapshots if s["StartTime"] >= cutoff] + if not recent: + latest = max(snapshots, key=lambda s: s["StartTime"]) + age = (now - latest["StartTime"]).total_seconds() / 3600 + return False, f"EBS Snapshots: Latest is {age:.0f}h old ({latest['SnapshotId']})" + return True, f"EBS Snapshots: OK — {len(snapshots)} total, {len(recent)} in last 48h" + except Exception as e: + return False, f"EBS Snapshots: Error — {e}" + + +def _restore_test(): + results = [] + try: + resp = s3.list_objects_v2(Bucket=SOURCE_BUCKET, Prefix="archive/", MaxKeys=1000) + contents = resp.get("Contents", []) + if not contents: + return [{"pass": False, "msg": "Restore test: No dumps found in source bucket"}] + latest = max(contents, key=lambda o: o["LastModified"]) + results.append({"pass": True, "msg": f"Restore test: Using {latest['Key']} ({latest['Size'] / 1_000_000:.1f} MB)"}) + + with tempfile.TemporaryDirectory() as tmpdir: + local_path = os.path.join(tmpdir, "dump.tar.gz") + s3.download_file(SOURCE_BUCKET, latest["Key"], local_path) + results.append({"pass": True, "msg": "Restore test: Download OK"}) + + try: + with tarfile.open(local_path, "r:gz") as tf: + names = tf.getnames() + results.append({"pass": True, "msg": f"Restore test: Archive OK — {len(names)} entries"}) + + db_entries = [n for n in names if n.endswith(".db") or n.endswith("forgejo.db")] + if db_entries: + import sqlite3 as sqlite_mod + tf.extract(db_entries[0], path=tmpdir) + db_path = os.path.join(tmpdir, db_entries[0]) + conn = sqlite_mod.connect(db_path) + result = conn.execute("PRAGMA integrity_check").fetchone() + conn.close() + if result[0] == "ok": + results.append({"pass": True, "msg": "Restore test: SQLite integrity OK"}) + else: + results.append({"pass": False, "msg": f"Restore test: SQLite integrity FAILED — {result[0]}"}) + else: + results.append({"pass": True, "msg": "Restore test: No .db file found in archive (may use different format)"}) + except tarfile.TarError as e: + results.append({"pass": False, "msg": f"Restore test: Archive extraction FAILED — {e}"}) + except Exception as e: + results.append({"pass": False, "msg": f"Restore test: Error — {e}"}) + return results + + +def _post_slack(blocks): + raw = secrets.get_secret_value(SecretId=SLACK_WEBHOOK_SECRET_ARN)["SecretString"] + webhook_url = raw.strip() + payload = json.dumps({"blocks": blocks}).encode() + req = urllib.request.Request( + webhook_url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + urllib.request.urlopen(req) + + +def handler(event, context): + mode = event.get("mode", "daily") + results = [] + + if mode == "daily": + results.append(_check_s3_bucket(s3, SOURCE_BUCKET, "S3 Source (us-east-1)")) + results.append(_check_s3_bucket(s3_west, REPLICA_BUCKET, "S3 Replica (us-west-2)")) + results.append(_check_gcs()) + results.append(_check_ebs_snapshots()) + + all_pass = all(r[0] for r in results) + header = "Forgejo Backup Verification" + blocks = [ + {"type": "header", "text": {"type": "plain_text", "text": header}}, + {"type": "section", "text": {"type": "mrkdwn", "text": f"*Date:* {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"}}, + {"type": "divider"}, + ] + for passed, msg in results: + emoji = ":white_check_mark:" if passed else ":x:" + blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"{emoji} {msg}"}}) + blocks.append({"type": "divider"}) + overall = ":white_check_mark: All checks passed" if all_pass else ":rotating_light: One or more checks failed" + blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"*Overall:* {overall}"}}) + + elif mode == "restore-test": + test_results = _restore_test() + all_pass = all(r["pass"] for r in test_results) + header = "Forgejo Monthly Restore Test" + blocks = [ + {"type": "header", "text": {"type": "plain_text", "text": header}}, + {"type": "section", "text": {"type": "mrkdwn", "text": f"*Date:* {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"}}, + {"type": "divider"}, + ] + for r in test_results: + emoji = ":white_check_mark:" if r["pass"] else ":x:" + blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"{emoji} {r['msg']}"}}) + blocks.append({"type": "divider"}) + overall = ":white_check_mark: Restore test passed" if all_pass else ":rotating_light: Restore test failed" + blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"*Overall:* {overall}"}}) + + _post_slack(blocks) + + return { + "statusCode": 200, + "body": json.dumps({ + "mode": mode, + "all_pass": all_pass, + "results": [{"pass": r[0], "msg": r[1]} for r in results] if mode == "daily" else test_results, + }), + } diff --git a/lambda/backup-verification/requirements.txt b/lambda/backup-verification/requirements.txt new file mode 100644 index 0000000..4a52721 --- /dev/null +++ b/lambda/backup-verification/requirements.txt @@ -0,0 +1 @@ +google-cloud-storage>=2.18.0,<3.0.0 diff --git a/lib/constructs/backup-verification.ts b/lib/constructs/backup-verification.ts new file mode 100644 index 0000000..c7607da --- /dev/null +++ b/lib/constructs/backup-verification.ts @@ -0,0 +1,94 @@ +import * as cdk from "aws-cdk-lib"; +import * as events from "aws-cdk-lib/aws-events"; +import * as events_targets from "aws-cdk-lib/aws-events-targets"; +import * as iam from "aws-cdk-lib/aws-iam"; +import * as lambda from "aws-cdk-lib/aws-lambda"; +import * as logs from "aws-cdk-lib/aws-logs"; +import * as s3 from "aws-cdk-lib/aws-s3"; +import { PythonFunction } from "@aws-cdk/aws-lambda-python-alpha"; +import { Construct } from "constructs"; + +interface BackupVerificationProps { + sourceBucket: s3.IBucket; + replicaBucketName: string; + gcsBucket: string; + gcsSaSecretName: string; + slackWebhookSecretName: string; +} + +export class BackupVerification extends Construct { + constructor(scope: Construct, id: string, props: BackupVerificationProps) { + super(scope, id); + + const fn = new PythonFunction(this, "Function", { + functionName: "forgejo-backup-verification", + entry: "lambda/backup-verification", + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + handler: "handler", + index: "app.py", + memorySize: 512, + timeout: cdk.Duration.minutes(5), + environment: { + SOURCE_BUCKET: props.sourceBucket.bucketName, + REPLICA_BUCKET: props.replicaBucketName, + GCS_BUCKET: props.gcsBucket, + GCS_SA_SECRET_ARN: props.gcsSaSecretName, + SLACK_WEBHOOK_SECRET_ARN: props.slackWebhookSecretName, + }, + logRetention: logs.RetentionDays.TWO_MONTHS, + }); + + props.sourceBucket.grantRead(fn); + + fn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["s3:ListBucket", "s3:GetObject"], + resources: [ + `arn:aws:s3:::${props.replicaBucketName}`, + `arn:aws:s3:::${props.replicaBucketName}/*`, + ], + }) + ); + + const account = cdk.Stack.of(this).account; + const region = cdk.Stack.of(this).region; + + fn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["secretsmanager:GetSecretValue"], + resources: [ + `arn:aws:secretsmanager:${region}:${account}:secret:${props.gcsSaSecretName}-*`, + `arn:aws:secretsmanager:${region}:${account}:secret:${props.slackWebhookSecretName}-*`, + ], + }) + ); + + fn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["ec2:DescribeSnapshots"], + resources: ["*"], + }) + ); + + new events.Rule(this, "DailyCheck", { + ruleName: "forgejo-backup-daily-check", + schedule: events.Schedule.cron({ hour: "8", minute: "0" }), + targets: [new events_targets.LambdaFunction(fn)], + }); + + new events.Rule(this, "MonthlyRestoreTest", { + ruleName: "forgejo-backup-monthly-restore-test", + schedule: events.Schedule.cron({ + hour: "9", + minute: "0", + day: "1", + }), + targets: [ + new events_targets.LambdaFunction(fn, { + event: events.RuleTargetInput.fromObject({ mode: "restore-test" }), + }), + ], + }); + } +} diff --git a/lib/forgejo-replica-stack.ts b/lib/forgejo-replica-stack.ts new file mode 100644 index 0000000..1dd21fa --- /dev/null +++ b/lib/forgejo-replica-stack.ts @@ -0,0 +1,48 @@ +import * as cdk from "aws-cdk-lib"; +import * as s3 from "aws-cdk-lib/aws-s3"; +import { Construct } from "constructs"; + +export class ForgejoReplicaStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + new s3.Bucket(this, "ReplicaBucket", { + bucketName: "forgejo-backups-replica-328440206208", + encryption: s3.BucketEncryption.S3_MANAGED, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + versioned: true, + objectLockEnabled: true, + objectLockDefaultRetention: s3.ObjectLockRetention.governance( + cdk.Duration.days(90) + ), + lifecycleRules: [ + { + id: "archive-to-glacier", + prefix: "archive/", + transitions: [ + { + storageClass: s3.StorageClass.GLACIER, + transitionAfter: cdk.Duration.days(30), + }, + ], + }, + { + id: "mirror-to-glacier-then-expire", + prefix: "mirror/", + transitions: [ + { + storageClass: s3.StorageClass.GLACIER, + transitionAfter: cdk.Duration.days(30), + }, + ], + expiration: cdk.Duration.days(365), + }, + { + id: "cleanup-noncurrent-versions", + noncurrentVersionExpiration: cdk.Duration.days(90), + }, + ], + removalPolicy: cdk.RemovalPolicy.RETAIN, + }); + } +} diff --git a/lib/forgejo-stack.ts b/lib/forgejo-stack.ts index 3e6195c..ae29111 100644 --- a/lib/forgejo-stack.ts +++ b/lib/forgejo-stack.ts @@ -8,6 +8,7 @@ import * as route53 from "aws-cdk-lib/aws-route53"; import * as route53Targets from "aws-cdk-lib/aws-route53-targets"; import * as dlm from "aws-cdk-lib/aws-dlm"; import { Construct } from "constructs"; +import { BackupVerification } from "./constructs/backup-verification"; const FORGEJO_VERSION = "10.0.1"; @@ -63,17 +64,95 @@ export class ForgejoStack extends cdk.Stack { bucketName: "forgejo-backups-328440206208", encryption: s3.BucketEncryption.S3_MANAGED, blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, - lifecycleRules: [{ - transitions: [ - { storageClass: s3.StorageClass.GLACIER, transitionAfter: cdk.Duration.days(30) }, - ], - expiration: cdk.Duration.days(365), - }], + versioned: true, + lifecycleRules: [ + { + id: "archive-to-glacier", + prefix: "archive/", + transitions: [ + { storageClass: s3.StorageClass.GLACIER, transitionAfter: cdk.Duration.days(30) }, + ], + }, + { + id: "cleanup-noncurrent-versions", + noncurrentVersionExpiration: cdk.Duration.days(90), + }, + ], removalPolicy: cdk.RemovalPolicy.RETAIN, }); backupBucket.grantReadWrite(role); + const replicaBucketArn = "arn:aws:s3:::forgejo-backups-replica-328440206208"; + + const replicationRole = new iam.Role(this, "ReplicationRole", { + roleName: "forgejo-s3-replication", + assumedBy: new iam.ServicePrincipal("s3.amazonaws.com"), + }); + + replicationRole.addToPolicy(new iam.PolicyStatement({ + actions: [ + "s3:GetReplicationConfiguration", + "s3:ListBucket", + ], + resources: [backupBucket.bucketArn], + })); + + replicationRole.addToPolicy(new iam.PolicyStatement({ + actions: [ + "s3:GetObjectVersionForReplication", + "s3:GetObjectVersionAcl", + "s3:GetObjectVersionTagging", + ], + resources: [`${backupBucket.bucketArn}/*`], + })); + + replicationRole.addToPolicy(new iam.PolicyStatement({ + actions: [ + "s3:ReplicateObject", + "s3:ReplicateDelete", + "s3:ReplicateTags", + ], + resources: [`${replicaBucketArn}/*`], + })); + + const cfnBucket = backupBucket.node.defaultChild as s3.CfnBucket; + cfnBucket.replicationConfiguration = { + role: replicationRole.roleArn, + rules: [{ + id: "replicate-to-west", + status: "Enabled", + destination: { + bucket: replicaBucketArn, + storageClass: "STANDARD", + }, + }], + }; + + const gcsTransferUser = new iam.User(this, "GcsTransferUser", { + userName: "forgejo-gcs-transfer", + }); + + gcsTransferUser.addToPolicy(new iam.PolicyStatement({ + actions: ["s3:GetObject", "s3:ListBucket"], + resources: [backupBucket.bucketArn, `${backupBucket.bucketArn}/*`], + })); + + const gcsTransferKey = new iam.AccessKey(this, "GcsTransferAccessKey", { + user: gcsTransferUser, + }); + + new cdk.aws_secretsmanager.CfnSecret(this, "GcsTransferCredentials", { + name: "forgejo/gcs-transfer-credentials", + secretString: cdk.Fn.join("", [ + '{"accessKeyId":"', + gcsTransferKey.accessKeyId, + '","secretAccessKey":"', + gcsTransferKey.secretAccessKey.unsafeUnwrap(), + '"}', + ]), + }); + const userData = ec2.UserData.forLinux(); userData.addCommands( "set -euxo pipefail", @@ -161,7 +240,7 @@ export class ForgejoStack extends cdk.Stack { "chown forgejo:forgejo \"$DUMP_DIR\"", "cd \"$DUMP_DIR\"", "sudo -u forgejo /usr/local/bin/forgejo dump --config /etc/forgejo/app.ini --type tar.gz --file \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\"", - "aws s3 cp \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\" s3://forgejo-backups-328440206208/${TIMESTAMP}/forgejo-${TIMESTAMP}.tar.gz", + "aws s3 cp \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\" s3://forgejo-backups-328440206208/archive/${TIMESTAMP}/forgejo-${TIMESTAMP}.tar.gz", "rm -rf \"$DUMP_DIR\"", "BAKEOF", "chmod +x /usr/local/bin/forgejo-backup.sh", @@ -297,7 +376,7 @@ export class ForgejoStack extends cdk.Stack { schedules: [{ name: "forgejo-nightly", createRule: { interval: 24, intervalUnit: "HOURS", times: ["06:00"] }, - retainRule: { count: 7 }, + retainRule: { count: 30 }, copyTags: true, }], }, @@ -356,6 +435,14 @@ export class ForgejoStack extends cdk.Stack { ), }); + new BackupVerification(this, "BackupVerification", { + sourceBucket: backupBucket, + replicaBucketName: "forgejo-backups-replica-328440206208", + gcsBucket: "forgejo-backups-offsite-seahaven", + gcsSaSecretName: "forgejo/gcs-sa-key", + slackWebhookSecretName: "forgejo/slack-webhook", + }); + new cdk.CfnOutput(this, "ForgejoUrl", { value: "https://forgejo.seahaven.com", }); diff --git a/package-lock.json b/package-lock.json index 7871675..df1e7f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "forgejo", "version": "1.0.0", "dependencies": { + "@aws-cdk/aws-lambda-python-alpha": "^2.252.0-alpha.0", "aws-cdk-lib": "^2.252.0", "constructs": "^10.0.0" }, @@ -33,6 +34,19 @@ "integrity": "sha512-We4bmHaowOPHr+IQR4/FyTGjRfjgBj4ICMjtqmJeBDWad3Q/6St12NT07leNtyuukv2qMhtSZJQorD8KpKTwRA==", "license": "Apache-2.0" }, + "node_modules/@aws-cdk/aws-lambda-python-alpha": { + "version": "2.252.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda-python-alpha/-/aws-lambda-python-alpha-2.252.0-alpha.0.tgz", + "integrity": "sha512-hVcursqZQ6tjToN4AvzOTfoeiN9epJGbUVi5VCGbIT88ide4hUzKsOda29+vw1Ket8nLi5i/xNB9odWU5QWdkw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "aws-cdk-lib": "^2.252.0", + "constructs": "^10.5.0" + } + }, "node_modules/@aws-cdk/cloud-assembly-schema": { "version": "53.22.0", "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-53.22.0.tgz", diff --git a/package.json b/package.json index 2f12e6b..debffb0 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "typescript": "~5.7.0" }, "dependencies": { + "@aws-cdk/aws-lambda-python-alpha": "^2.252.0-alpha.0", "aws-cdk-lib": "^2.252.0", "constructs": "^10.0.0" } diff --git a/scripts/gcp-setup.sh b/scripts/gcp-setup.sh new file mode 100755 index 0000000..9daa14b --- /dev/null +++ b/scripts/gcp-setup.sh @@ -0,0 +1,151 @@ +#!/bin/bash +set -euo pipefail + +PROJECT_ID="seahaven-backups" +BUCKET_NAME="forgejo-backups-offsite-seahaven" +LOCATION="us-central1" +SA_NAME="forgejo-backup-writer" +SA_EMAIL="${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" +RETENTION_SECONDS=$((2 * 365 * 24 * 3600)) # 2 years +AWS_REGION="us-east-1" +AWS_SOURCE_BUCKET="forgejo-backups-328440206208" + +GCLOUD="${GCLOUD:-gcloud}" +GSUTIL="${GSUTIL:-gsutil}" + +echo "=== Forgejo 3-2-1 Offsite Backup — GCP Setup ===" + +# --- Project --- +echo "" +echo "--- Step 1: Create GCP project ---" +if $GCLOUD projects describe "$PROJECT_ID" &>/dev/null; then + echo "Project $PROJECT_ID already exists." +else + $GCLOUD projects create "$PROJECT_ID" --name="Sea Haven Backups" + echo "Created project $PROJECT_ID." +fi +$GCLOUD config set project "$PROJECT_ID" + +echo "" +echo "--- Step 2: Enable required APIs ---" +$GCLOUD services enable storage.googleapis.com storagetransfer.googleapis.com + +# --- Bucket --- +echo "" +echo "--- Step 3: Create GCS bucket ---" +if $GSUTIL ls -b "gs://$BUCKET_NAME" &>/dev/null; then + echo "Bucket gs://$BUCKET_NAME already exists." +else + $GSUTIL mb -p "$PROJECT_ID" -l "$LOCATION" -c NEARLINE -b on "gs://$BUCKET_NAME" + echo "Created bucket gs://$BUCKET_NAME." +fi + +echo "" +echo "--- Step 4: Set lifecycle rules ---" +LIFECYCLE_JSON=$(cat <<'LCEOF' +{ + "rule": [ + { + "action": {"type": "SetStorageClass", "storageClass": "COLDLINE"}, + "condition": {"age": 90} + }, + { + "action": {"type": "SetStorageClass", "storageClass": "ARCHIVE"}, + "condition": {"age": 180} + } + ] +} +LCEOF +) +echo "$LIFECYCLE_JSON" | $GSUTIL lifecycle set /dev/stdin "gs://$BUCKET_NAME" +echo "Lifecycle rules applied." + +echo "" +echo "--- Step 5: Enable object versioning ---" +$GSUTIL versioning set on "gs://$BUCKET_NAME" + +echo "" +echo "--- Step 6: Set retention policy (2 years) ---" +$GSUTIL retention set "${RETENTION_SECONDS}s" "gs://$BUCKET_NAME" +echo "Retention policy set to 2 years." + +echo "" +echo "!!! IMPORTANT: Locking the retention policy is IRREVERSIBLE. !!!" +echo "Once locked, objects cannot be deleted before the retention period expires." +echo "Even the project owner cannot shorten or remove the policy." +echo "" +read -p "Lock the retention policy now? (yes/no): " CONFIRM +if [ "$CONFIRM" = "yes" ]; then + $GSUTIL retention lock "gs://$BUCKET_NAME" + echo "Retention policy LOCKED." +else + echo "Retention policy set but NOT locked. Run 'gsutil retention lock gs://$BUCKET_NAME' when ready." +fi + +# --- Service Account --- +echo "" +echo "--- Step 7: Create service account ---" +if $GCLOUD iam service-accounts describe "$SA_EMAIL" &>/dev/null 2>&1; then + echo "Service account $SA_EMAIL already exists." +else + $GCLOUD iam service-accounts create "$SA_NAME" \ + --display-name="Forgejo Backup Writer" \ + --description="Write-only access to forgejo offsite backup bucket" + echo "Created service account $SA_EMAIL." +fi + +echo "" +echo "--- Step 8: Grant bucket permissions ---" +$GSUTIL iam ch "serviceAccount:${SA_EMAIL}:objectCreator" "gs://$BUCKET_NAME" +$GSUTIL iam ch "serviceAccount:${SA_EMAIL}:objectViewer" "gs://$BUCKET_NAME" +echo "Granted objectCreator + objectViewer to $SA_EMAIL." + +echo "" +echo "--- Step 9: Create and store service account key ---" +KEY_FILE=$(mktemp) +$GCLOUD iam service-accounts keys create "$KEY_FILE" --iam-account="$SA_EMAIL" +echo "Service account key created." + +if aws secretsmanager describe-secret --secret-id forgejo/gcs-sa-key --region "$AWS_REGION" &>/dev/null 2>&1; then + aws secretsmanager put-secret-value \ + --secret-id forgejo/gcs-sa-key \ + --secret-string "file://$KEY_FILE" \ + --region "$AWS_REGION" + echo "Updated existing secret forgejo/gcs-sa-key." +else + aws secretsmanager create-secret \ + --name forgejo/gcs-sa-key \ + --secret-string "file://$KEY_FILE" \ + --region "$AWS_REGION" + echo "Created secret forgejo/gcs-sa-key." +fi +rm -f "$KEY_FILE" +echo "Key stored in AWS Secrets Manager, local copy deleted." + +# --- Storage Transfer --- +echo "" +echo "--- Step 10: Configure Storage Transfer Service ---" +echo "" +echo "Storage Transfer Service requires AWS credentials to read from S3." +echo "Create a read-only IAM user 'forgejo-gcs-transfer' in AWS with access to:" +echo " s3://forgejo-backups-328440206208 (GetObject, ListBucket)" +echo "" +echo "Then configure the transfer job in the GCP Console:" +echo " 1. Go to: https://console.cloud.google.com/transfer/jobs" +echo " 2. Source: Amazon S3 — bucket '$AWS_SOURCE_BUCKET'" +echo " 3. Destination: GCS — bucket '$BUCKET_NAME'" +echo " 4. Schedule: Daily at 10:00 UTC" +echo " 5. Enter the AWS access key ID and secret for the read-only user" +echo "" +echo "The CDK stack creates the 'forgejo-gcs-transfer' IAM user automatically." +echo "Retrieve its credentials from Secrets Manager: forgejo/gcs-transfer-credentials" + +echo "" +echo "=== Setup complete ===" +echo "" +echo "Summary:" +echo " GCP Project: $PROJECT_ID" +echo " GCS Bucket: gs://$BUCKET_NAME" +echo " Service Account: $SA_EMAIL" +echo " SA Key Secret: forgejo/gcs-sa-key (AWS Secrets Manager)" +echo " Retention: 2 years (check lock status above)"