From d57aea19f3cce1c88047d6e18f4b72e0a2f82fd2 Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 13 May 2026 18:02:50 -0400
Subject: [PATCH 1/7] Add 3-2-1 backup strategy with cross-region replication
 and GCS offsite

Implements a fully compliant 3-2-1 backup architecture:
- Copy 1 (live): Harden existing EBS snapshots to 30-day retention
- Copy 2 (near-site): S3 cross-region replication to us-west-2 with
  Object Lock (governance 90d) and versioning
- Copy 3 (offsite): GCS bucket in dedicated seahaven-backups GCP project
  with 2-year irreversible retention lock

Also adds a verification Lambda that checks all 3 locations daily and
runs monthly restore tests with SQLite integrity checks.
---
 .github/workflows/deploy.yaml               |   2 +
 .gitignore                                  |   1 +
 README.md                                   | 102 +++++++++-
 bin/app.ts                                  |   7 +
 lambda/backup-verification/app.py           | 198 ++++++++++++++++++++
 lambda/backup-verification/requirements.txt |   1 +
 lib/constructs/backup-verification.ts       |  94 ++++++++++
 lib/forgejo-replica-stack.ts                |  48 +++++
 lib/forgejo-stack.ts                        | 103 +++++++++-
 package-lock.json                           |  14 ++
 package.json                                |   1 +
 scripts/gcp-setup.sh                        | 151 +++++++++++++++
 12 files changed, 704 insertions(+), 18 deletions(-)
 create mode 100644 lambda/backup-verification/app.py
 create mode 100644 lambda/backup-verification/requirements.txt
 create mode 100644 lib/constructs/backup-verification.ts
 create mode 100644 lib/forgejo-replica-stack.ts
 create mode 100755 scripts/gcp-setup.sh

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index e5462cf..6acd7ee 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -14,5 +14,7 @@ concurrency:
 jobs:
   deploy:
     uses: Sea-Haven-Industries/.github/.github/workflows/cd-cdk.yaml@main
+    with:
+      enable-qemu: true
     secrets:
       deploy-role-arn: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
diff --git a/.gitignore b/.gitignore
index a6b3201..ecfe493 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ cdk.out/
 *.d.ts
 *.js.map
 cdk.context.json
+docs/*.pdf
diff --git a/README.md b/README.md
index 0e82e68..ba8fe71 100644
--- a/README.md
+++ b/README.md
@@ -20,19 +20,67 @@ Self-hosted Forgejo git server for archiving GitHub repos and mirroring active o
 | 3000 | HTTP | ALB → instance | Internal traffic from ALB |
 | 2222 | SSH | VPC + VPN | Git SSH operations |
 
-## S3 Backups
+## 3-2-1 Backup Strategy
 
-A nightly `forgejo dump` runs at 5:00 UTC and uploads the archive to `s3://forgejo-backups-328440206208`.
+All backups follow a 3-2-1 strategy: 3 copies, 2 storage types, 1 offsite provider.
 
-**S3 lifecycle policy:**
+| Copy | Location | Type | Retention |
+|------|----------|------|-----------|
+| Live | EBS volume (us-east-1) | Block | N/A |
+| Near-site | S3 replica (us-west-2) | Object | Archive: indefinite, noncurrent versions: 90d |
+| Offsite | GCS `forgejo-backups-offsite-seahaven` (GCP us-central1) | Object | 2-year locked retention |
 
-| Phase | Duration |
-|-------|----------|
-| Standard | First 30 days |
-| Glacier | Days 31–365 |
-| Expired | After 365 days |
+**Daily data flow:**
 
-EBS snapshots are managed separately by DLM and run nightly at 6:00 UTC with a 7-day retention window.
+| Time (UTC) | Event |
+|------------|-------|
+| 05:00 | `forgejo dump` → `s3://forgejo-backups-328440206208/archive/{date}/` |
+| ~05:01 | S3 CRR replicates to `forgejo-backups-replica-328440206208` (us-west-2) |
+| 06:00 | DLM EBS snapshot (30-day retention) |
+| 08:00 | Verification Lambda checks all 3 locations, posts to Slack |
+| 10:00 | GCS Storage Transfer pulls from S3 to GCS offsite |
+
+**S3 source lifecycle:** Standard 30d → Glacier (no expiration).
+
+**Immutability layers:**
+- S3 Versioning on both source and replica buckets
+- S3 Object Lock (Governance, 90d) on the replica bucket
+- GCS Bucket Lock (2yr, irreversible) on the offsite bucket
+
+### Verification
+
+The `forgejo-backup-verification` Lambda runs daily at 08:00 UTC and checks:
+1. S3 source has a recent dump under `archive/`
+2. S3 replica has replicated the latest dump
+3. GCS offsite has received the latest transfer
+4. EBS snapshots exist within the last 48 hours
+
+On the 1st of each month at 09:00 UTC, it runs a restore test: downloads the latest dump, extracts the archive, and runs SQLite integrity checks.
+
+### Manual backup
+
+```bash
+sudo /usr/local/bin/forgejo-backup.sh
+```
+
+### Restore from S3
+
+```bash
+aws s3 cp s3://forgejo-backups-328440206208/archive/<date>/forgejo-<date>.tar.gz /tmp/
+systemctl stop forgejo
+cd /tmp && tar xzf forgejo-<date>.tar.gz
+forgejo restore --config /etc/forgejo/app.ini --from /tmp/forgejo-dump-*
+chown -R forgejo:forgejo /var/lib/forgejo
+systemctl start forgejo
+```
+
+### Restore from GCS (disaster recovery)
+
+```bash
+gcloud config set project seahaven-backups
+gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz /tmp/
+# Then follow the same restore steps as S3
+```
 
 To test the backup manually:
 
@@ -78,6 +126,9 @@ sudo /usr/local/bin/forgejo-refresh-tokens.sh
 | `forgejo/admin-password` | Forgejo admin user password |
 | `forgejo/api-token` | Forgejo API token (used by autodiscovery and token refresh scripts) |
 | `forgejo/github-pat` | GitHub fine-grained PAT for mirroring |
+| `forgejo/gcs-sa-key` | GCP service account key for offsite backup verification |
+| `forgejo/gcs-transfer-credentials` | AWS IAM credentials for GCS Storage Transfer Service |
+| `forgejo/slack-webhook` | Slack webhook URL for backup verification alerts |
 
 ## First-time setup
 
@@ -108,15 +159,46 @@ In the Forgejo web UI: **New Migration → GitHub** → paste the GitHub repo UR
 
 Same migration flow, but check **This Repository Will Be A Mirror**. Forgejo polls GitHub hourly (`DEFAULT_INTERVAL = 1h` in app.ini) and keeps the mirror in sync.
 
+## GCP Offsite Setup (one-time)
+
+Run the setup script to create the GCS offsite bucket, service account, and store credentials:
+
+```bash
+./scripts/gcp-setup.sh
+```
+
+This creates the `seahaven-backups` GCP project with a locked-retention GCS bucket. After running, configure the Storage Transfer job in the GCP Console using the AWS credentials from `forgejo/gcs-transfer-credentials`.
+
 ## Deployment
 
 ```bash
 npm install
-npx cdk deploy
+npx cdk deploy --all
 ```
 
+This deploys two stacks:
+- `forgejo-replica` (us-west-2) — S3 replica bucket with Object Lock
+- `forgejo` (us-east-1) — main stack with Forgejo instance, CRR, and verification Lambda
+
 CI/CD is handled by GitHub Actions — PRs run CI, merges to `main` deploy via the reusable CDK workflow.
 
+## Post-deploy: update running instance backup path
+
+After the first deploy with the 3-2-1 changes, the running instance's backup script still uses the old S3 path (without the `archive/` prefix). Update it via SSM:
+
+```bash
+aws ssm start-session --target i-0d3005fb3c36124cd
+sudo sed -i 's|s3://forgejo-backups-328440206208/${TIMESTAMP}/|s3://forgejo-backups-328440206208/archive/${TIMESTAMP}/|' /usr/local/bin/forgejo-backup.sh
+```
+
+Also store the Slack webhook URL for backup verification alerts:
+
+```bash
+aws secretsmanager create-secret --name forgejo/slack-webhook \
+  --secret-string "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" \
+  --region us-east-1
+```
+
 ## Updating Forgejo
 
 Update the `FORGEJO_VERSION` constant in `lib/forgejo-stack.ts` and deploy. This replaces the instance, so ensure the latest EBS snapshot is available for data recovery if needed. Alternatively, update in-place via SSM:
diff --git a/bin/app.ts b/bin/app.ts
index 3d93d46..1721cd7 100644
--- a/bin/app.ts
+++ b/bin/app.ts
@@ -2,8 +2,15 @@
 import "source-map-support/register";
 import * as cdk from "aws-cdk-lib";
 import { ForgejoStack } from "../lib/forgejo-stack";
+import { ForgejoReplicaStack } from "../lib/forgejo-replica-stack";
 
 const app = new cdk.App();
+
+new ForgejoReplicaStack(app, "forgejo-replica", {
+  stackName: "forgejo-replica",
+  env: { account: "328440206208", region: "us-west-2" },
+});
+
 new ForgejoStack(app, "forgejo", {
   stackName: "forgejo",
   env: { account: "328440206208", region: "us-east-1" },
diff --git a/lambda/backup-verification/app.py b/lambda/backup-verification/app.py
new file mode 100644
index 0000000..bc7a0dc
--- /dev/null
+++ b/lambda/backup-verification/app.py
@@ -0,0 +1,198 @@
+import json
+import os
+import tarfile
+import tempfile
+import urllib.request
+from datetime import datetime, timedelta, timezone
+
+import boto3
+from google.cloud import storage as gcs
+from google.oauth2 import service_account
+
+
+s3 = boto3.client("s3")
+s3_west = boto3.client("s3", region_name="us-west-2")
+ec2 = boto3.client("ec2")
+secrets = boto3.client("secretsmanager")
+
+SOURCE_BUCKET = os.environ["SOURCE_BUCKET"]
+REPLICA_BUCKET = os.environ["REPLICA_BUCKET"]
+GCS_BUCKET = os.environ["GCS_BUCKET"]
+GCS_SA_SECRET_ARN = os.environ["GCS_SA_SECRET_ARN"]
+SLACK_WEBHOOK_SECRET_ARN = os.environ["SLACK_WEBHOOK_SECRET_ARN"]
+
+_gcs_client = None
+
+
+def _get_gcs_client():
+    global _gcs_client
+    if _gcs_client is None:
+        raw = secrets.get_secret_value(SecretId=GCS_SA_SECRET_ARN)["SecretString"]
+        info = json.loads(raw)
+        creds = service_account.Credentials.from_service_account_info(info)
+        _gcs_client = gcs.Client(credentials=creds, project=info.get("project_id"))
+    return _gcs_client
+
+
+def _check_s3_bucket(client, bucket, label):
+    now = datetime.now(timezone.utc)
+    cutoff = now - timedelta(hours=48)
+    try:
+        resp = client.list_objects_v2(Bucket=bucket, Prefix="archive/", MaxKeys=1000)
+        contents = resp.get("Contents", [])
+        if not contents:
+            return False, f"{label}: No objects found under archive/"
+        latest = max(contents, key=lambda o: o["LastModified"])
+        if latest["LastModified"] < cutoff:
+            age = (now - latest["LastModified"]).total_seconds() / 3600
+            return False, f"{label}: Latest dump is {age:.0f}h old ({latest['Key']})"
+        if latest["Size"] < 1_000_000:
+            return False, f"{label}: Latest dump suspiciously small ({latest['Size']} bytes)"
+        return True, f"{label}: OK — {latest['Key']} ({latest['Size'] / 1_000_000:.1f} MB)"
+    except Exception as e:
+        return False, f"{label}: Error — {e}"
+
+
+def _check_gcs():
+    try:
+        client = _get_gcs_client()
+        bucket = client.bucket(GCS_BUCKET)
+        blobs = list(bucket.list_blobs(prefix="archive/", max_results=1000))
+        if not blobs:
+            return False, "GCS Offsite: No objects found under archive/"
+        now = datetime.now(timezone.utc)
+        cutoff = now - timedelta(hours=72)
+        latest = max(blobs, key=lambda b: b.updated)
+        if latest.updated < cutoff:
+            age = (now - latest.updated).total_seconds() / 3600
+            return False, f"GCS Offsite: Latest object is {age:.0f}h old ({latest.name})"
+        return True, f"GCS Offsite: OK — {latest.name} ({latest.size / 1_000_000:.1f} MB)"
+    except Exception as e:
+        return False, f"GCS Offsite: Error — {e}"
+
+
+def _check_ebs_snapshots():
+    try:
+        now = datetime.now(timezone.utc)
+        cutoff = now - timedelta(hours=48)
+        resp = ec2.describe_snapshots(
+            Filters=[{"Name": "tag:forgejo-backup", "Values": ["true"]}],
+            OwnerIds=["self"],
+        )
+        snapshots = resp.get("Snapshots", [])
+        if not snapshots:
+            return False, "EBS Snapshots: No snapshots found with forgejo-backup tag"
+        recent = [s for s in snapshots if s["StartTime"] >= cutoff]
+        if not recent:
+            latest = max(snapshots, key=lambda s: s["StartTime"])
+            age = (now - latest["StartTime"]).total_seconds() / 3600
+            return False, f"EBS Snapshots: Latest is {age:.0f}h old ({latest['SnapshotId']})"
+        return True, f"EBS Snapshots: OK — {len(snapshots)} total, {len(recent)} in last 48h"
+    except Exception as e:
+        return False, f"EBS Snapshots: Error — {e}"
+
+
+def _restore_test():
+    results = []
+    try:
+        resp = s3.list_objects_v2(Bucket=SOURCE_BUCKET, Prefix="archive/", MaxKeys=1000)
+        contents = resp.get("Contents", [])
+        if not contents:
+            return [{"pass": False, "msg": "Restore test: No dumps found in source bucket"}]
+        latest = max(contents, key=lambda o: o["LastModified"])
+        results.append({"pass": True, "msg": f"Restore test: Using {latest['Key']} ({latest['Size'] / 1_000_000:.1f} MB)"})
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_path = os.path.join(tmpdir, "dump.tar.gz")
+            s3.download_file(SOURCE_BUCKET, latest["Key"], local_path)
+            results.append({"pass": True, "msg": "Restore test: Download OK"})
+
+            try:
+                with tarfile.open(local_path, "r:gz") as tf:
+                    names = tf.getnames()
+                    results.append({"pass": True, "msg": f"Restore test: Archive OK — {len(names)} entries"})
+
+                    db_entries = [n for n in names if n.endswith(".db") or n.endswith("forgejo.db")]
+                    if db_entries:
+                        import sqlite3 as sqlite_mod
+                        tf.extract(db_entries[0], path=tmpdir)
+                        db_path = os.path.join(tmpdir, db_entries[0])
+                        conn = sqlite_mod.connect(db_path)
+                        result = conn.execute("PRAGMA integrity_check").fetchone()
+                        conn.close()
+                        if result[0] == "ok":
+                            results.append({"pass": True, "msg": "Restore test: SQLite integrity OK"})
+                        else:
+                            results.append({"pass": False, "msg": f"Restore test: SQLite integrity FAILED — {result[0]}"})
+                    else:
+                        results.append({"pass": True, "msg": "Restore test: No .db file found in archive (may use different format)"})
+            except tarfile.TarError as e:
+                results.append({"pass": False, "msg": f"Restore test: Archive extraction FAILED — {e}"})
+    except Exception as e:
+        results.append({"pass": False, "msg": f"Restore test: Error — {e}"})
+    return results
+
+
+def _post_slack(blocks):
+    raw = secrets.get_secret_value(SecretId=SLACK_WEBHOOK_SECRET_ARN)["SecretString"]
+    webhook_url = raw.strip()
+    payload = json.dumps({"blocks": blocks}).encode()
+    req = urllib.request.Request(
+        webhook_url,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    urllib.request.urlopen(req)
+
+
+def handler(event, context):
+    mode = event.get("mode", "daily")
+    results = []
+
+    if mode == "daily":
+        results.append(_check_s3_bucket(s3, SOURCE_BUCKET, "S3 Source (us-east-1)"))
+        results.append(_check_s3_bucket(s3_west, REPLICA_BUCKET, "S3 Replica (us-west-2)"))
+        results.append(_check_gcs())
+        results.append(_check_ebs_snapshots())
+
+        all_pass = all(r[0] for r in results)
+        header = "Forgejo Backup Verification"
+        blocks = [
+            {"type": "header", "text": {"type": "plain_text", "text": header}},
+            {"type": "section", "text": {"type": "mrkdwn", "text": f"*Date:* {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"}},
+            {"type": "divider"},
+        ]
+        for passed, msg in results:
+            emoji = ":white_check_mark:" if passed else ":x:"
+            blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"{emoji} {msg}"}})
+        blocks.append({"type": "divider"})
+        overall = ":white_check_mark: All checks passed" if all_pass else ":rotating_light: One or more checks failed"
+        blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"*Overall:* {overall}"}})
+
+    elif mode == "restore-test":
+        test_results = _restore_test()
+        all_pass = all(r["pass"] for r in test_results)
+        header = "Forgejo Monthly Restore Test"
+        blocks = [
+            {"type": "header", "text": {"type": "plain_text", "text": header}},
+            {"type": "section", "text": {"type": "mrkdwn", "text": f"*Date:* {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"}},
+            {"type": "divider"},
+        ]
+        for r in test_results:
+            emoji = ":white_check_mark:" if r["pass"] else ":x:"
+            blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"{emoji} {r['msg']}"}})
+        blocks.append({"type": "divider"})
+        overall = ":white_check_mark: Restore test passed" if all_pass else ":rotating_light: Restore test failed"
+        blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"*Overall:* {overall}"}})
+
+    _post_slack(blocks)
+
+    return {
+        "statusCode": 200,
+        "body": json.dumps({
+            "mode": mode,
+            "all_pass": all_pass,
+            "results": [{"pass": r[0], "msg": r[1]} for r in results] if mode == "daily" else test_results,
+        }),
+    }
diff --git a/lambda/backup-verification/requirements.txt b/lambda/backup-verification/requirements.txt
new file mode 100644
index 0000000..4a52721
--- /dev/null
+++ b/lambda/backup-verification/requirements.txt
@@ -0,0 +1 @@
+google-cloud-storage>=2.18.0,<3.0.0
diff --git a/lib/constructs/backup-verification.ts b/lib/constructs/backup-verification.ts
new file mode 100644
index 0000000..c7607da
--- /dev/null
+++ b/lib/constructs/backup-verification.ts
@@ -0,0 +1,94 @@
+import * as cdk from "aws-cdk-lib";
+import * as events from "aws-cdk-lib/aws-events";
+import * as events_targets from "aws-cdk-lib/aws-events-targets";
+import * as iam from "aws-cdk-lib/aws-iam";
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import * as logs from "aws-cdk-lib/aws-logs";
+import * as s3 from "aws-cdk-lib/aws-s3";
+import { PythonFunction } from "@aws-cdk/aws-lambda-python-alpha";
+import { Construct } from "constructs";
+
+interface BackupVerificationProps {
+  sourceBucket: s3.IBucket;
+  replicaBucketName: string;
+  gcsBucket: string;
+  gcsSaSecretName: string;
+  slackWebhookSecretName: string;
+}
+
+export class BackupVerification extends Construct {
+  constructor(scope: Construct, id: string, props: BackupVerificationProps) {
+    super(scope, id);
+
+    const fn = new PythonFunction(this, "Function", {
+      functionName: "forgejo-backup-verification",
+      entry: "lambda/backup-verification",
+      runtime: lambda.Runtime.PYTHON_3_12,
+      architecture: lambda.Architecture.ARM_64,
+      handler: "handler",
+      index: "app.py",
+      memorySize: 512,
+      timeout: cdk.Duration.minutes(5),
+      environment: {
+        SOURCE_BUCKET: props.sourceBucket.bucketName,
+        REPLICA_BUCKET: props.replicaBucketName,
+        GCS_BUCKET: props.gcsBucket,
+        GCS_SA_SECRET_ARN: props.gcsSaSecretName,
+        SLACK_WEBHOOK_SECRET_ARN: props.slackWebhookSecretName,
+      },
+      logRetention: logs.RetentionDays.TWO_MONTHS,
+    });
+
+    props.sourceBucket.grantRead(fn);
+
+    fn.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: ["s3:ListBucket", "s3:GetObject"],
+        resources: [
+          `arn:aws:s3:::${props.replicaBucketName}`,
+          `arn:aws:s3:::${props.replicaBucketName}/*`,
+        ],
+      })
+    );
+
+    const account = cdk.Stack.of(this).account;
+    const region = cdk.Stack.of(this).region;
+
+    fn.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: ["secretsmanager:GetSecretValue"],
+        resources: [
+          `arn:aws:secretsmanager:${region}:${account}:secret:${props.gcsSaSecretName}-*`,
+          `arn:aws:secretsmanager:${region}:${account}:secret:${props.slackWebhookSecretName}-*`,
+        ],
+      })
+    );
+
+    fn.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: ["ec2:DescribeSnapshots"],
+        resources: ["*"],
+      })
+    );
+
+    new events.Rule(this, "DailyCheck", {
+      ruleName: "forgejo-backup-daily-check",
+      schedule: events.Schedule.cron({ hour: "8", minute: "0" }),
+      targets: [new events_targets.LambdaFunction(fn)],
+    });
+
+    new events.Rule(this, "MonthlyRestoreTest", {
+      ruleName: "forgejo-backup-monthly-restore-test",
+      schedule: events.Schedule.cron({
+        hour: "9",
+        minute: "0",
+        day: "1",
+      }),
+      targets: [
+        new events_targets.LambdaFunction(fn, {
+          event: events.RuleTargetInput.fromObject({ mode: "restore-test" }),
+        }),
+      ],
+    });
+  }
+}
diff --git a/lib/forgejo-replica-stack.ts b/lib/forgejo-replica-stack.ts
new file mode 100644
index 0000000..1dd21fa
--- /dev/null
+++ b/lib/forgejo-replica-stack.ts
@@ -0,0 +1,48 @@
+import * as cdk from "aws-cdk-lib";
+import * as s3 from "aws-cdk-lib/aws-s3";
+import { Construct } from "constructs";
+
+export class ForgejoReplicaStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    new s3.Bucket(this, "ReplicaBucket", {
+      bucketName: "forgejo-backups-replica-328440206208",
+      encryption: s3.BucketEncryption.S3_MANAGED,
+      blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+      versioned: true,
+      objectLockEnabled: true,
+      objectLockDefaultRetention: s3.ObjectLockRetention.governance(
+        cdk.Duration.days(90)
+      ),
+      lifecycleRules: [
+        {
+          id: "archive-to-glacier",
+          prefix: "archive/",
+          transitions: [
+            {
+              storageClass: s3.StorageClass.GLACIER,
+              transitionAfter: cdk.Duration.days(30),
+            },
+          ],
+        },
+        {
+          id: "mirror-to-glacier-then-expire",
+          prefix: "mirror/",
+          transitions: [
+            {
+              storageClass: s3.StorageClass.GLACIER,
+              transitionAfter: cdk.Duration.days(30),
+            },
+          ],
+          expiration: cdk.Duration.days(365),
+        },
+        {
+          id: "cleanup-noncurrent-versions",
+          noncurrentVersionExpiration: cdk.Duration.days(90),
+        },
+      ],
+      removalPolicy: cdk.RemovalPolicy.RETAIN,
+    });
+  }
+}
diff --git a/lib/forgejo-stack.ts b/lib/forgejo-stack.ts
index 3e6195c..ae29111 100644
--- a/lib/forgejo-stack.ts
+++ b/lib/forgejo-stack.ts
@@ -8,6 +8,7 @@ import * as route53 from "aws-cdk-lib/aws-route53";
 import * as route53Targets from "aws-cdk-lib/aws-route53-targets";
 import * as dlm from "aws-cdk-lib/aws-dlm";
 import { Construct } from "constructs";
+import { BackupVerification } from "./constructs/backup-verification";
 
 const FORGEJO_VERSION = "10.0.1";
 
@@ -63,17 +64,95 @@ export class ForgejoStack extends cdk.Stack {
       bucketName: "forgejo-backups-328440206208",
       encryption: s3.BucketEncryption.S3_MANAGED,
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
-      lifecycleRules: [{
-        transitions: [
-          { storageClass: s3.StorageClass.GLACIER, transitionAfter: cdk.Duration.days(30) },
-        ],
-        expiration: cdk.Duration.days(365),
-      }],
+      versioned: true,
+      lifecycleRules: [
+        {
+          id: "archive-to-glacier",
+          prefix: "archive/",
+          transitions: [
+            { storageClass: s3.StorageClass.GLACIER, transitionAfter: cdk.Duration.days(30) },
+          ],
+        },
+        {
+          id: "cleanup-noncurrent-versions",
+          noncurrentVersionExpiration: cdk.Duration.days(90),
+        },
+      ],
       removalPolicy: cdk.RemovalPolicy.RETAIN,
     });
 
     backupBucket.grantReadWrite(role);
 
+    const replicaBucketArn = "arn:aws:s3:::forgejo-backups-replica-328440206208";
+
+    const replicationRole = new iam.Role(this, "ReplicationRole", {
+      roleName: "forgejo-s3-replication",
+      assumedBy: new iam.ServicePrincipal("s3.amazonaws.com"),
+    });
+
+    replicationRole.addToPolicy(new iam.PolicyStatement({
+      actions: [
+        "s3:GetReplicationConfiguration",
+        "s3:ListBucket",
+      ],
+      resources: [backupBucket.bucketArn],
+    }));
+
+    replicationRole.addToPolicy(new iam.PolicyStatement({
+      actions: [
+        "s3:GetObjectVersionForReplication",
+        "s3:GetObjectVersionAcl",
+        "s3:GetObjectVersionTagging",
+      ],
+      resources: [`${backupBucket.bucketArn}/*`],
+    }));
+
+    replicationRole.addToPolicy(new iam.PolicyStatement({
+      actions: [
+        "s3:ReplicateObject",
+        "s3:ReplicateDelete",
+        "s3:ReplicateTags",
+      ],
+      resources: [`${replicaBucketArn}/*`],
+    }));
+
+    const cfnBucket = backupBucket.node.defaultChild as s3.CfnBucket;
+    cfnBucket.replicationConfiguration = {
+      role: replicationRole.roleArn,
+      rules: [{
+        id: "replicate-to-west",
+        status: "Enabled",
+        destination: {
+          bucket: replicaBucketArn,
+          storageClass: "STANDARD",
+        },
+      }],
+    };
+
+    const gcsTransferUser = new iam.User(this, "GcsTransferUser", {
+      userName: "forgejo-gcs-transfer",
+    });
+
+    gcsTransferUser.addToPolicy(new iam.PolicyStatement({
+      actions: ["s3:GetObject", "s3:ListBucket"],
+      resources: [backupBucket.bucketArn, `${backupBucket.bucketArn}/*`],
+    }));
+
+    const gcsTransferKey = new iam.AccessKey(this, "GcsTransferAccessKey", {
+      user: gcsTransferUser,
+    });
+
+    new cdk.aws_secretsmanager.CfnSecret(this, "GcsTransferCredentials", {
+      name: "forgejo/gcs-transfer-credentials",
+      secretString: cdk.Fn.join("", [
+        '{"accessKeyId":"',
+        gcsTransferKey.accessKeyId,
+        '","secretAccessKey":"',
+        gcsTransferKey.secretAccessKey.unsafeUnwrap(),
+        '"}',
+      ]),
+    });
+
     const userData = ec2.UserData.forLinux();
     userData.addCommands(
       "set -euxo pipefail",
@@ -161,7 +240,7 @@ export class ForgejoStack extends cdk.Stack {
       "chown forgejo:forgejo \"$DUMP_DIR\"",
       "cd \"$DUMP_DIR\"",
       "sudo -u forgejo /usr/local/bin/forgejo dump --config /etc/forgejo/app.ini --type tar.gz --file \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\"",
-      "aws s3 cp \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\" s3://forgejo-backups-328440206208/${TIMESTAMP}/forgejo-${TIMESTAMP}.tar.gz",
+      "aws s3 cp \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\" s3://forgejo-backups-328440206208/archive/${TIMESTAMP}/forgejo-${TIMESTAMP}.tar.gz",
       "rm -rf \"$DUMP_DIR\"",
       "BAKEOF",
       "chmod +x /usr/local/bin/forgejo-backup.sh",
@@ -297,7 +376,7 @@ export class ForgejoStack extends cdk.Stack {
         schedules: [{
           name: "forgejo-nightly",
           createRule: { interval: 24, intervalUnit: "HOURS", times: ["06:00"] },
-          retainRule: { count: 7 },
+          retainRule: { count: 30 },
           copyTags: true,
         }],
       },
@@ -356,6 +435,14 @@ export class ForgejoStack extends cdk.Stack {
       ),
     });
 
+    new BackupVerification(this, "BackupVerification", {
+      sourceBucket: backupBucket,
+      replicaBucketName: "forgejo-backups-replica-328440206208",
+      gcsBucket: "forgejo-backups-offsite-seahaven",
+      gcsSaSecretName: "forgejo/gcs-sa-key",
+      slackWebhookSecretName: "forgejo/slack-webhook",
+    });
+
     new cdk.CfnOutput(this, "ForgejoUrl", {
       value: "https://forgejo.seahaven.com",
     });
diff --git a/package-lock.json b/package-lock.json
index 7871675..df1e7f9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,6 +8,7 @@
       "name": "forgejo",
       "version": "1.0.0",
       "dependencies": {
+        "@aws-cdk/aws-lambda-python-alpha": "^2.252.0-alpha.0",
         "aws-cdk-lib": "^2.252.0",
         "constructs": "^10.0.0"
       },
@@ -33,6 +34,19 @@
       "integrity": "sha512-We4bmHaowOPHr+IQR4/FyTGjRfjgBj4ICMjtqmJeBDWad3Q/6St12NT07leNtyuukv2qMhtSZJQorD8KpKTwRA==",
       "license": "Apache-2.0"
     },
+    "node_modules/@aws-cdk/aws-lambda-python-alpha": {
+      "version": "2.252.0-alpha.0",
+      "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda-python-alpha/-/aws-lambda-python-alpha-2.252.0-alpha.0.tgz",
+      "integrity": "sha512-hVcursqZQ6tjToN4AvzOTfoeiN9epJGbUVi5VCGbIT88ide4hUzKsOda29+vw1Ket8nLi5i/xNB9odWU5QWdkw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">= 20.0.0"
+      },
+      "peerDependencies": {
+        "aws-cdk-lib": "^2.252.0",
+        "constructs": "^10.5.0"
+      }
+    },
     "node_modules/@aws-cdk/cloud-assembly-schema": {
       "version": "53.22.0",
       "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-53.22.0.tgz",
diff --git a/package.json b/package.json
index 2f12e6b..debffb0 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
     "typescript": "~5.7.0"
   },
   "dependencies": {
+    "@aws-cdk/aws-lambda-python-alpha": "^2.252.0-alpha.0",
     "aws-cdk-lib": "^2.252.0",
     "constructs": "^10.0.0"
   }
diff --git a/scripts/gcp-setup.sh b/scripts/gcp-setup.sh
new file mode 100755
index 0000000..9daa14b
--- /dev/null
+++ b/scripts/gcp-setup.sh
@@ -0,0 +1,151 @@
+#!/bin/bash
+set -euo pipefail
+
+PROJECT_ID="seahaven-backups"
+BUCKET_NAME="forgejo-backups-offsite-seahaven"
+LOCATION="us-central1"
+SA_NAME="forgejo-backup-writer"
+SA_EMAIL="${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com"
+RETENTION_SECONDS=$((2 * 365 * 24 * 3600))  # 2 years
+AWS_REGION="us-east-1"
+AWS_SOURCE_BUCKET="forgejo-backups-328440206208"
+
+GCLOUD="${GCLOUD:-gcloud}"
+GSUTIL="${GSUTIL:-gsutil}"
+
+echo "=== Forgejo 3-2-1 Offsite Backup — GCP Setup ==="
+
+# --- Project ---
+echo ""
+echo "--- Step 1: Create GCP project ---"
+if $GCLOUD projects describe "$PROJECT_ID" &>/dev/null; then
+  echo "Project $PROJECT_ID already exists."
+else
+  $GCLOUD projects create "$PROJECT_ID" --name="Sea Haven Backups"
+  echo "Created project $PROJECT_ID."
+fi
+$GCLOUD config set project "$PROJECT_ID"
+
+echo ""
+echo "--- Step 2: Enable required APIs ---"
+$GCLOUD services enable storage.googleapis.com storagetransfer.googleapis.com
+
+# --- Bucket ---
+echo ""
+echo "--- Step 3: Create GCS bucket ---"
+if $GSUTIL ls -b "gs://$BUCKET_NAME" &>/dev/null; then
+  echo "Bucket gs://$BUCKET_NAME already exists."
+else
+  $GSUTIL mb -p "$PROJECT_ID" -l "$LOCATION" -c NEARLINE -b on "gs://$BUCKET_NAME"
+  echo "Created bucket gs://$BUCKET_NAME."
+fi
+
+echo ""
+echo "--- Step 4: Set lifecycle rules ---"
+LIFECYCLE_JSON=$(cat <<'LCEOF'
+{
+  "rule": [
+    {
+      "action": {"type": "SetStorageClass", "storageClass": "COLDLINE"},
+      "condition": {"age": 90}
+    },
+    {
+      "action": {"type": "SetStorageClass", "storageClass": "ARCHIVE"},
+      "condition": {"age": 180}
+    }
+  ]
+}
+LCEOF
+)
+echo "$LIFECYCLE_JSON" | $GSUTIL lifecycle set /dev/stdin "gs://$BUCKET_NAME"
+echo "Lifecycle rules applied."
+
+echo ""
+echo "--- Step 5: Enable object versioning ---"
+$GSUTIL versioning set on "gs://$BUCKET_NAME"
+
+echo ""
+echo "--- Step 6: Set retention policy (2 years) ---"
+$GSUTIL retention set "${RETENTION_SECONDS}s" "gs://$BUCKET_NAME"
+echo "Retention policy set to 2 years."
+
+echo ""
+echo "!!! IMPORTANT: Locking the retention policy is IRREVERSIBLE. !!!"
+echo "Once locked, objects cannot be deleted before the retention period expires."
+echo "Even the project owner cannot shorten or remove the policy."
+echo ""
+read -p "Lock the retention policy now? (yes/no): " CONFIRM
+if [ "$CONFIRM" = "yes" ]; then
+  $GSUTIL retention lock "gs://$BUCKET_NAME"
+  echo "Retention policy LOCKED."
+else
+  echo "Retention policy set but NOT locked. Run 'gsutil retention lock gs://$BUCKET_NAME' when ready."
+fi
+
+# --- Service Account ---
+echo ""
+echo "--- Step 7: Create service account ---"
+if $GCLOUD iam service-accounts describe "$SA_EMAIL" &>/dev/null 2>&1; then
+  echo "Service account $SA_EMAIL already exists."
+else
+  $GCLOUD iam service-accounts create "$SA_NAME" \
+    --display-name="Forgejo Backup Writer" \
+    --description="Write-only access to forgejo offsite backup bucket"
+  echo "Created service account $SA_EMAIL."
+fi
+
+echo ""
+echo "--- Step 8: Grant bucket permissions ---"
+$GSUTIL iam ch "serviceAccount:${SA_EMAIL}:objectCreator" "gs://$BUCKET_NAME"
+$GSUTIL iam ch "serviceAccount:${SA_EMAIL}:objectViewer" "gs://$BUCKET_NAME"
+echo "Granted objectCreator + objectViewer to $SA_EMAIL."
+
+echo ""
+echo "--- Step 9: Create and store service account key ---"
+KEY_FILE=$(mktemp)
+$GCLOUD iam service-accounts keys create "$KEY_FILE" --iam-account="$SA_EMAIL"
+echo "Service account key created."
+
+if aws secretsmanager describe-secret --secret-id forgejo/gcs-sa-key --region "$AWS_REGION" &>/dev/null 2>&1; then
+  aws secretsmanager put-secret-value \
+    --secret-id forgejo/gcs-sa-key \
+    --secret-string "file://$KEY_FILE" \
+    --region "$AWS_REGION"
+  echo "Updated existing secret forgejo/gcs-sa-key."
+else
+  aws secretsmanager create-secret \
+    --name forgejo/gcs-sa-key \
+    --secret-string "file://$KEY_FILE" \
+    --region "$AWS_REGION"
+  echo "Created secret forgejo/gcs-sa-key."
+fi
+rm -f "$KEY_FILE"
+echo "Key stored in AWS Secrets Manager, local copy deleted."
+
+# --- Storage Transfer ---
+echo ""
+echo "--- Step 10: Configure Storage Transfer Service ---"
+echo ""
+echo "Storage Transfer Service requires AWS credentials to read from S3."
+echo "Create a read-only IAM user 'forgejo-gcs-transfer' in AWS with access to:"
+echo "  s3://forgejo-backups-328440206208 (GetObject, ListBucket)"
+echo ""
+echo "Then configure the transfer job in the GCP Console:"
+echo "  1. Go to: https://console.cloud.google.com/transfer/jobs"
+echo "  2. Source: Amazon S3 — bucket '$AWS_SOURCE_BUCKET'"
+echo "  3. Destination: GCS — bucket '$BUCKET_NAME'"
+echo "  4. Schedule: Daily at 10:00 UTC"
+echo "  5. Enter the AWS access key ID and secret for the read-only user"
+echo ""
+echo "The CDK stack creates the 'forgejo-gcs-transfer' IAM user automatically."
+echo "Retrieve its credentials from Secrets Manager: forgejo/gcs-transfer-credentials"
+
+echo ""
+echo "=== Setup complete ==="
+echo ""
+echo "Summary:"
+echo "  GCP Project:     $PROJECT_ID"
+echo "  GCS Bucket:      gs://$BUCKET_NAME"
+echo "  Service Account: $SA_EMAIL"
+echo "  SA Key Secret:   forgejo/gcs-sa-key (AWS Secrets Manager)"
+echo "  Retention:       2 years (check lock status above)"

From 5904fcc4d7bbecfdc5e514a074591e55b9b571e7 Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 13 May 2026 18:09:21 -0400
Subject: [PATCH 2/7] Enable QEMU in CI for arm64 Lambda Docker builds

---
 .github/workflows/ci.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1846096..e88ef57 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -6,3 +6,5 @@ on:
 jobs:
   ci:
     uses: Sea-Haven-Industries/.github/.github/workflows/ci-typescript-cdk.yaml@main
+    with:
+      enable-qemu: true

From 0d3f9ad5a3c8e13b9d02d16d469c55435314870c Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 13 May 2026 18:15:37 -0400
Subject: [PATCH 3/7] Commit cdk.context.json for CI synth without AWS
 credentials

Vpc.fromLookup requires cached context to synthesize without
AWS credentials. Required for CI which runs cdk synth without
an OIDC role.
---
 .gitignore       |  1 -
 cdk.context.json | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 cdk.context.json

diff --git a/.gitignore b/.gitignore
index ecfe493..b5b2f33 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,5 +3,4 @@ cdk.out/
 *.js
 *.d.ts
 *.js.map
-cdk.context.json
 docs/*.pdf
diff --git a/cdk.context.json b/cdk.context.json
new file mode 100644
index 0000000..068ec0c
--- /dev/null
+++ b/cdk.context.json
@@ -0,0 +1,47 @@
+{
+  "vpc-provider:account=328440206208:filter.vpc-id=vpc-0d3d4b67bd0cf8a68:region=us-east-1:returnAsymmetricSubnets=true": {
+    "vpcId": "vpc-0d3d4b67bd0cf8a68",
+    "vpcCidrBlock": "10.20.0.0/16",
+    "ownerAccountId": "328440206208",
+    "availabilityZones": [],
+    "vpnGatewayId": "vgw-073737d44762dffc2",
+    "subnetGroups": [
+      {
+        "name": "Private",
+        "type": "Private",
+        "subnets": [
+          {
+            "subnetId": "subnet-04e38c507e96f1926",
+            "cidr": "10.20.30.0/24",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-06a2f56f492b9b4de"
+          },
+          {
+            "subnetId": "subnet-0a0b4fc6f296dfba5",
+            "cidr": "10.20.40.0/24",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-01e152fe5cabca7d6"
+          }
+        ]
+      },
+      {
+        "name": "Public",
+        "type": "Public",
+        "subnets": [
+          {
+            "subnetId": "subnet-0eea820effe1b3ae5",
+            "cidr": "10.20.10.0/24",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-0f2232493a5c43fe8"
+          },
+          {
+            "subnetId": "subnet-0012f5895182c1580",
+            "cidr": "10.20.20.0/24",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-0f2232493a5c43fe8"
+          }
+        ]
+      }
+    ]
+  }
+}

From 17179c84b21bb8b3c108eb2d57d13850affef014 Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 13 May 2026 18:15:56 -0400
Subject: [PATCH 4/7] Fix GCP project ID to sea-haven-backups

---
 scripts/gcp-setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/gcp-setup.sh b/scripts/gcp-setup.sh
index 9daa14b..274c2a1 100755
--- a/scripts/gcp-setup.sh
+++ b/scripts/gcp-setup.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-PROJECT_ID="seahaven-backups"
+PROJECT_ID="sea-haven-backups"
 BUCKET_NAME="forgejo-backups-offsite-seahaven"
 LOCATION="us-central1"
 SA_NAME="forgejo-backup-writer"

From fa51085578cd5234117ccd0b6d276d37f1c2218a Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 13 May 2026 18:29:28 -0400
Subject: [PATCH 5/7] Address code review findings for backup verification

Fix 4 critical issues:
- Add filter/priority/deleteMarkerReplication to S3 CRR rule (deploy would fail without)
- Add stack dependency so replica deploys before main stack
- Fix DB file extension matching (.sqlite3/.sql instead of .db)
- Replace nonexistent `forgejo restore` command with actual restore steps in README

Fix 4 moderate issues:
- Add timeout=10 to Slack webhook urlopen call
- Add filter='data' to tarfile.extract for PEP 706 compliance
- Add explicit ValueError for unknown handler mode
- Use date-scoped S3/GCS prefix instead of unbounded listing
---
 README.md                         | 17 ++++++++-----
 bin/app.ts                        |  6 +++--
 lambda/backup-verification/app.py | 41 +++++++++++++++++++++----------
 lib/forgejo-stack.ts              |  3 +++
 4 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index ba8fe71..93226b2 100644
--- a/README.md
+++ b/README.md
@@ -68,18 +68,23 @@ sudo /usr/local/bin/forgejo-backup.sh
 ```bash
 aws s3 cp s3://forgejo-backups-328440206208/archive/<date>/forgejo-<date>.tar.gz /tmp/
 systemctl stop forgejo
-cd /tmp && tar xzf forgejo-<date>.tar.gz
-forgejo restore --config /etc/forgejo/app.ini --from /tmp/forgejo-dump-*
-chown -R forgejo:forgejo /var/lib/forgejo
+mkdir -p /tmp/forgejo-restore && tar -xzf /tmp/forgejo-<date>.tar.gz -C /tmp/forgejo-restore
+cd /tmp/forgejo-restore
+cp app.ini /etc/forgejo/app.ini
+cp gitea-db.sqlite3 /var/lib/forgejo/data/forgejo.db
+rm -rf /var/lib/forgejo/data/repositories
+cp -a repos /var/lib/forgejo/data/repositories
+chown -R forgejo:forgejo /var/lib/forgejo /etc/forgejo/app.ini
 systemctl start forgejo
+rm -rf /tmp/forgejo-restore /tmp/forgejo-<date>.tar.gz
 ```
 
 ### Restore from GCS (disaster recovery)
 
 ```bash
-gcloud config set project seahaven-backups
+gcloud config set project sea-haven-backups
 gsutil cp gs://forgejo-backups-offsite-seahaven/archive/<date>/forgejo-<date>.tar.gz /tmp/
-# Then follow the same restore steps as S3
+# Then follow the same restore steps as S3 above
 ```
 
 To test the backup manually:
@@ -167,7 +172,7 @@ Run the setup script to create the GCS offsite bucket, service account, and stor
 ./scripts/gcp-setup.sh
 ```
 
-This creates the `seahaven-backups` GCP project with a locked-retention GCS bucket. After running, configure the Storage Transfer job in the GCP Console using the AWS credentials from `forgejo/gcs-transfer-credentials`.
+This creates the `sea-haven-backups` GCP project with a locked-retention GCS bucket. After running, configure the Storage Transfer job in the GCP Console using the AWS credentials from `forgejo/gcs-transfer-credentials`.
 
 ## Deployment
 
diff --git a/bin/app.ts b/bin/app.ts
index 1721cd7..518b551 100644
--- a/bin/app.ts
+++ b/bin/app.ts
@@ -6,12 +6,14 @@ import { ForgejoReplicaStack } from "../lib/forgejo-replica-stack";
 
 const app = new cdk.App();
 
-new ForgejoReplicaStack(app, "forgejo-replica", {
+const replicaStack = new ForgejoReplicaStack(app, "forgejo-replica", {
   stackName: "forgejo-replica",
   env: { account: "328440206208", region: "us-west-2" },
 });
 
-new ForgejoStack(app, "forgejo", {
+const forgejoStack = new ForgejoStack(app, "forgejo", {
   stackName: "forgejo",
   env: { account: "328440206208", region: "us-east-1" },
 });
+
+forgejoStack.addDependency(replicaStack);
diff --git a/lambda/backup-verification/app.py b/lambda/backup-verification/app.py
index bc7a0dc..46e4828 100644
--- a/lambda/backup-verification/app.py
+++ b/lambda/backup-verification/app.py
@@ -38,10 +38,14 @@ def _check_s3_bucket(client, bucket, label):
     now = datetime.now(timezone.utc)
     cutoff = now - timedelta(hours=48)
     try:
-        resp = client.list_objects_v2(Bucket=bucket, Prefix="archive/", MaxKeys=1000)
-        contents = resp.get("Contents", [])
+        today = now.strftime("%Y-%m-%d")
+        yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d")
+        contents = []
+        for date_prefix in [today, yesterday]:
+            resp = client.list_objects_v2(Bucket=bucket, Prefix=f"archive/{date_prefix}/")
+            contents.extend(resp.get("Contents", []))
         if not contents:
-            return False, f"{label}: No objects found under archive/"
+            return False, f"{label}: No objects found under archive/ for last 2 days"
         latest = max(contents, key=lambda o: o["LastModified"])
         if latest["LastModified"] < cutoff:
             age = (now - latest["LastModified"]).total_seconds() / 3600
@@ -57,10 +61,14 @@ def _check_gcs():
     try:
         client = _get_gcs_client()
         bucket = client.bucket(GCS_BUCKET)
-        blobs = list(bucket.list_blobs(prefix="archive/", max_results=1000))
-        if not blobs:
-            return False, "GCS Offsite: No objects found under archive/"
         now = datetime.now(timezone.utc)
+        today = now.strftime("%Y-%m-%d")
+        yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d")
+        blobs = []
+        for date_prefix in [today, yesterday]:
+            blobs.extend(list(bucket.list_blobs(prefix=f"archive/{date_prefix}/")))
+        if not blobs:
+            return False, "GCS Offsite: No objects found under archive/ for last 2 days"
         cutoff = now - timedelta(hours=72)
         latest = max(blobs, key=lambda b: b.updated)
         if latest.updated < cutoff:
@@ -95,10 +103,14 @@ def _check_ebs_snapshots():
 def _restore_test():
     results = []
     try:
-        resp = s3.list_objects_v2(Bucket=SOURCE_BUCKET, Prefix="archive/", MaxKeys=1000)
-        contents = resp.get("Contents", [])
+        now = datetime.now(timezone.utc)
+        contents = []
+        for days_ago in range(7):
+            date_prefix = (now - timedelta(days=days_ago)).strftime("%Y-%m-%d")
+            resp = s3.list_objects_v2(Bucket=SOURCE_BUCKET, Prefix=f"archive/{date_prefix}/")
+            contents.extend(resp.get("Contents", []))
         if not contents:
-            return [{"pass": False, "msg": "Restore test: No dumps found in source bucket"}]
+            return [{"pass": False, "msg": "Restore test: No dumps found in source bucket (last 7 days)"}]
         latest = max(contents, key=lambda o: o["LastModified"])
         results.append({"pass": True, "msg": f"Restore test: Using {latest['Key']} ({latest['Size'] / 1_000_000:.1f} MB)"})
 
@@ -112,10 +124,10 @@ def _restore_test():
                     names = tf.getnames()
                     results.append({"pass": True, "msg": f"Restore test: Archive OK — {len(names)} entries"})
 
-                    db_entries = [n for n in names if n.endswith(".db") or n.endswith("forgejo.db")]
+                    db_entries = [n for n in names if n.endswith(".sqlite3") or n.endswith(".sql")]
                     if db_entries:
                         import sqlite3 as sqlite_mod
-                        tf.extract(db_entries[0], path=tmpdir)
+                        tf.extract(db_entries[0], path=tmpdir, filter="data")
                         db_path = os.path.join(tmpdir, db_entries[0])
                         conn = sqlite_mod.connect(db_path)
                         result = conn.execute("PRAGMA integrity_check").fetchone()
@@ -125,7 +137,7 @@ def _restore_test():
                         else:
                             results.append({"pass": False, "msg": f"Restore test: SQLite integrity FAILED — {result[0]}"})
                     else:
-                        results.append({"pass": True, "msg": "Restore test: No .db file found in archive (may use different format)"})
+                        results.append({"pass": False, "msg": "Restore test: No SQLite DB file found in archive"})
             except tarfile.TarError as e:
                 results.append({"pass": False, "msg": f"Restore test: Archive extraction FAILED — {e}"})
     except Exception as e:
@@ -143,7 +155,7 @@ def _post_slack(blocks):
         headers={"Content-Type": "application/json"},
         method="POST",
     )
-    urllib.request.urlopen(req)
+    urllib.request.urlopen(req, timeout=10)
 
 
 def handler(event, context):
@@ -186,6 +198,9 @@ def handler(event, context):
         overall = ":white_check_mark: Restore test passed" if all_pass else ":rotating_light: Restore test failed"
         blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"*Overall:* {overall}"}})
 
+    else:
+        raise ValueError(f"Unknown mode: {mode!r} (expected 'daily' or 'restore-test')")
+
     _post_slack(blocks)
 
     return {
diff --git a/lib/forgejo-stack.ts b/lib/forgejo-stack.ts
index ae29111..be9b46f 100644
--- a/lib/forgejo-stack.ts
+++ b/lib/forgejo-stack.ts
@@ -122,6 +122,9 @@ export class ForgejoStack extends cdk.Stack {
       rules: [{
         id: "replicate-to-west",
         status: "Enabled",
+        priority: 1,
+        filter: { prefix: "" },
+        deleteMarkerReplication: { status: "Disabled" },
         destination: {
           bucket: replicaBucketArn,
           storageClass: "STANDARD",

From b9726e517633c145efae9403aa05004f2792ca45 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 13 May 2026 22:31:23 +0000
Subject: [PATCH 6/7] Fix backup strategy bug findings

---
 lambda/backup-verification/app.py |  8 +++++++-
 lib/forgejo-stack.ts              | 18 +++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/lambda/backup-verification/app.py b/lambda/backup-verification/app.py
index 46e4828..2dec9aa 100644
--- a/lambda/backup-verification/app.py
+++ b/lambda/backup-verification/app.py
@@ -199,7 +199,13 @@ def handler(event, context):
         blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": f"*Overall:* {overall}"}})
 
     else:
-        raise ValueError(f"Unknown mode: {mode!r} (expected 'daily' or 'restore-test')")
+        return {
+            "statusCode": 400,
+            "body": json.dumps({
+                "mode": mode,
+                "error": f"Unsupported backup verification mode: {mode}",
+            }),
+        }
 
     _post_slack(blocks)
 
diff --git a/lib/forgejo-stack.ts b/lib/forgejo-stack.ts
index be9b46f..732079c 100644
--- a/lib/forgejo-stack.ts
+++ b/lib/forgejo-stack.ts
@@ -2,6 +2,7 @@ import * as cdk from "aws-cdk-lib";
 import * as ec2 from "aws-cdk-lib/aws-ec2";
 import * as iam from "aws-cdk-lib/aws-iam";
 import * as s3 from "aws-cdk-lib/aws-s3";
+import * as secretsmanager from "aws-cdk-lib/aws-secretsmanager";
 import * as elbv2 from "aws-cdk-lib/aws-elasticloadbalancingv2";
 import * as elbv2_targets from "aws-cdk-lib/aws-elasticloadbalancingv2-targets";
 import * as route53 from "aws-cdk-lib/aws-route53";
@@ -145,16 +146,15 @@ export class ForgejoStack extends cdk.Stack {
       user: gcsTransferUser,
     });
 
-    new cdk.aws_secretsmanager.CfnSecret(this, "GcsTransferCredentials", {
-      name: "forgejo/gcs-transfer-credentials",
-      secretString: cdk.Fn.join("", [
-        '{"accessKeyId":"',
-        gcsTransferKey.accessKeyId,
-        '","secretAccessKey":"',
-        gcsTransferKey.secretAccessKey.unsafeUnwrap(),
-        '"}',
-      ]),
+    const gcsTransferCredentials = new secretsmanager.Secret(this, "GcsTransferCredentials", {
+      secretName: "forgejo/gcs-transfer-credentials",
+      secretObjectValue: {
+        accessKeyId: cdk.SecretValue.unsafePlainText(gcsTransferKey.accessKeyId),
+        secretAccessKey: gcsTransferKey.secretAccessKey,
+      },
     });
+    const gcsTransferCredentialsResource = gcsTransferCredentials.node.defaultChild as secretsmanager.CfnSecret;
+    gcsTransferCredentialsResource.overrideLogicalId("GcsTransferCredentials");
 
     const userData = ec2.UserData.forLinux();
     userData.addCommands(

From 0ab9cc9f3074d0c741bee4718b14681dc0725b11 Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 13 May 2026 18:53:56 -0400
Subject: [PATCH 7/7] Handle SQL text dumps separately from binary SQLite in
 restore test

Forgejo dump produces gitea-db.sql as a text SQL dump (XORM export),
not a binary SQLite file. Opening it directly with sqlite3.connect()
throws DatabaseError. Now imports the SQL dump into a temp DB first.
---
 lambda/backup-verification/app.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/lambda/backup-verification/app.py b/lambda/backup-verification/app.py
index 2dec9aa..4935e03 100644
--- a/lambda/backup-verification/app.py
+++ b/lambda/backup-verification/app.py
@@ -124,11 +124,12 @@ def _restore_test():
                     names = tf.getnames()
                     results.append({"pass": True, "msg": f"Restore test: Archive OK — {len(names)} entries"})
 
-                    db_entries = [n for n in names if n.endswith(".sqlite3") or n.endswith(".sql")]
-                    if db_entries:
+                    sqlite_entries = [n for n in names if n.endswith(".sqlite3")]
+                    sql_entries = [n for n in names if n.endswith(".sql")]
+                    if sqlite_entries:
                         import sqlite3 as sqlite_mod
-                        tf.extract(db_entries[0], path=tmpdir, filter="data")
-                        db_path = os.path.join(tmpdir, db_entries[0])
+                        tf.extract(sqlite_entries[0], path=tmpdir, filter="data")
+                        db_path = os.path.join(tmpdir, sqlite_entries[0])
                         conn = sqlite_mod.connect(db_path)
                         result = conn.execute("PRAGMA integrity_check").fetchone()
                         conn.close()
@@ -136,8 +137,26 @@ def _restore_test():
                             results.append({"pass": True, "msg": "Restore test: SQLite integrity OK"})
                         else:
                             results.append({"pass": False, "msg": f"Restore test: SQLite integrity FAILED — {result[0]}"})
+                    elif sql_entries:
+                        import sqlite3 as sqlite_mod
+                        tf.extract(sql_entries[0], path=tmpdir, filter="data")
+                        sql_path = os.path.join(tmpdir, sql_entries[0])
+                        with open(sql_path, "r") as f:
+                            sql_text = f.read()
+                        if len(sql_text) < 100:
+                            results.append({"pass": False, "msg": f"Restore test: SQL dump suspiciously small ({len(sql_text)} bytes)"})
+                        else:
+                            db_path = os.path.join(tmpdir, "restore-test.db")
+                            conn = sqlite_mod.connect(db_path)
+                            conn.executescript(sql_text)
+                            result = conn.execute("PRAGMA integrity_check").fetchone()
+                            conn.close()
+                            if result[0] == "ok":
+                                results.append({"pass": True, "msg": "Restore test: SQL dump import + integrity OK"})
+                            else:
+                                results.append({"pass": False, "msg": f"Restore test: Integrity FAILED after SQL import — {result[0]}"})
                     else:
-                        results.append({"pass": False, "msg": "Restore test: No SQLite DB file found in archive"})
+                        results.append({"pass": False, "msg": "Restore test: No database file found in archive"})
             except tarfile.TarError as e:
                 results.append({"pass": False, "msg": f"Restore test: Archive extraction FAILED — {e}"})
     except Exception as e: