diff --git a/.gitignore b/.gitignore index b5b2f33..1d1dbbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.env node_modules/ cdk.out/ *.js diff --git a/README.md b/README.md index 250e288..1db4f1a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,10 @@ Self-hosted Forgejo git server for archiving GitHub repos and mirroring active o ## Architecture -- **EC2**: t4g.small (arm64), Amazon Linux 2023, 50GB gp3 EBS — instance `i-0d3005fb3c36124cd` +- **EC2**: t4g.small (arm64), Amazon Linux 2023, 50GB gp3 EBS — look up instance ID with: + ``` + aws cloudformation describe-stacks --stack-name forgejo --query 'Stacks[0].Outputs[?OutputKey==`InstanceId`].OutputValue' --output text + ``` - **Network**: Private subnet (us-east-1a), behind `seahaven-com` ALB for SSL termination - **DNS**: `forgejo.seahaven.com` — Route53 alias record pointing to the `seahaven-com` ALB (not a direct A record) - **TLS**: Wildcard cert on ALB, HTTP internally on port 3000 @@ -148,7 +151,8 @@ sudo /usr/local/bin/forgejo-refresh-tokens.sh After the stack deploys, connect via SSM and create the admin user: ```bash -aws ssm start-session --target i-0d3005fb3c36124cd +INSTANCE_ID=$(aws cloudformation describe-stacks --stack-name forgejo --query 'Stacks[0].Outputs[?OutputKey==`InstanceId`].OutputValue' --output text) +aws ssm start-session --target "$INSTANCE_ID" sudo -u forgejo /usr/local/bin/forgejo admin user create \ --admin \ @@ -195,16 +199,9 @@ This deploys two stacks: CI/CD is handled by GitHub Actions — PRs run CI, merges to `main` deploy via the reusable CDK workflow. -## Post-deploy: update running instance backup path +## Post-deploy: store Slack webhook -After the first deploy with the 3-2-1 changes, the running instance's backup script still uses the old S3 path (without the `archive/` prefix). Update it via SSM: - -```bash -aws ssm start-session --target i-0d3005fb3c36124cd -sudo sed -i 's|s3://forgejo-backups-328440206208/${TIMESTAMP}/|s3://forgejo-backups-328440206208/archive/${TIMESTAMP}/|' /usr/local/bin/forgejo-backup.sh -``` - -Also store the Slack webhook URL for backup verification alerts: +Store the Slack webhook URL for backup verification alerts: ```bash aws secretsmanager create-secret --name forgejo/slack-webhook \ @@ -217,7 +214,8 @@ aws secretsmanager create-secret --name forgejo/slack-webhook \ Update the `FORGEJO_VERSION` constant in `lib/forgejo-stack.ts` and deploy. This replaces the instance, so ensure the latest EBS snapshot is available for data recovery if needed. Alternatively, update in-place via SSM: ```bash -aws ssm start-session --target i-0d3005fb3c36124cd +INSTANCE_ID=$(aws cloudformation describe-stacks --stack-name forgejo --query 'Stacks[0].Outputs[?OutputKey==`InstanceId`].OutputValue' --output text) +aws ssm start-session --target "$INSTANCE_ID" sudo systemctl stop forgejo sudo curl -Lo /usr/local/bin/forgejo "https://codeberg.org/forgejo/forgejo/releases/download/v/forgejo--linux-arm64" diff --git a/bin/app.ts b/bin/app.ts index 518b551..b79e622 100644 --- a/bin/app.ts +++ b/bin/app.ts @@ -14,6 +14,8 @@ const replicaStack = new ForgejoReplicaStack(app, "forgejo-replica", { const forgejoStack = new ForgejoStack(app, "forgejo", { stackName: "forgejo", env: { account: "328440206208", region: "us-east-1" }, + replicaBucketArn: replicaStack.replicaBucketArn, + replicaBucketName: replicaStack.replicaBucketName, }); forgejoStack.addDependency(replicaStack); diff --git a/lambda/backup-verification/app.py b/lambda/backup-verification/app.py index a471c0a..9da8adc 100644 --- a/lambda/backup-verification/app.py +++ b/lambda/backup-verification/app.py @@ -41,9 +41,10 @@ def _check_s3_bucket(client, bucket, label): today = now.strftime("%Y-%m-%d") yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d") contents = [] + paginator = client.get_paginator("list_objects_v2") for date_prefix in [today, yesterday]: - resp = client.list_objects_v2(Bucket=bucket, Prefix=f"archive/{date_prefix}/") - contents.extend(resp.get("Contents", [])) + for page in paginator.paginate(Bucket=bucket, Prefix=f"archive/{date_prefix}/"): + contents.extend(page.get("Contents", [])) if not contents: return False, f"{label}: No objects found under archive/ for last 2 days" latest = max(contents, key=lambda o: o["LastModified"]) @@ -113,13 +114,17 @@ def _restore_test(): try: now = datetime.now(timezone.utc) contents = [] + paginator = s3.get_paginator("list_objects_v2") for days_ago in range(7): date_prefix = (now - timedelta(days=days_ago)).strftime("%Y-%m-%d") - resp = s3.list_objects_v2(Bucket=SOURCE_BUCKET, Prefix=f"archive/{date_prefix}/") - contents.extend(resp.get("Contents", [])) + for page in paginator.paginate(Bucket=SOURCE_BUCKET, Prefix=f"archive/{date_prefix}/"): + contents.extend(page.get("Contents", [])) if not contents: return [{"pass": False, "msg": "Restore test: No dumps found in source bucket (last 7 days)"}] latest = max(contents, key=lambda o: o["LastModified"]) + max_bytes = 4 * 1024 * 1024 * 1024 - 512 * 1024 * 1024 + if latest["Size"] > max_bytes: + return [{"pass": False, "msg": f"Restore test: Dump too large for ephemeral storage ({latest['Size'] / 1_000_000_000:.1f} GB)"}] results.append({"pass": True, "msg": f"Restore test: Using {latest['Key']} ({latest['Size'] / 1_000_000:.1f} MB)"}) with tempfile.TemporaryDirectory() as tmpdir: diff --git a/lib/constructs/backup-verification.ts b/lib/constructs/backup-verification.ts index 0da7d9e..dc4a3de 100644 --- a/lib/constructs/backup-verification.ts +++ b/lib/constructs/backup-verification.ts @@ -1,4 +1,5 @@ import * as cdk from "aws-cdk-lib"; +import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch"; import * as events from "aws-cdk-lib/aws-events"; import * as events_targets from "aws-cdk-lib/aws-events-targets"; import * as iam from "aws-cdk-lib/aws-iam"; @@ -75,7 +76,11 @@ export class BackupVerification extends Construct { new events.Rule(this, "DailyCheck", { ruleName: "forgejo-backup-daily-check", schedule: events.Schedule.cron({ hour: "8", minute: "0" }), - targets: [new events_targets.LambdaFunction(fn)], + targets: [ + new events_targets.LambdaFunction(fn, { + event: events.RuleTargetInput.fromObject({ mode: "daily" }), + }), + ], }); new events.Rule(this, "MonthlyRestoreTest", { @@ -91,5 +96,15 @@ export class BackupVerification extends Construct { }), ], }); + + new cloudwatch.Alarm(this, "ErrorAlarm", { + alarmName: "forgejo-backup-verification-errors", + alarmDescription: "Backup verification Lambda is failing — Slack notifications may not be firing", + metric: fn.metricErrors({ period: cdk.Duration.hours(1) }), + threshold: 1, + evaluationPeriods: 1, + treatMissingData: cloudwatch.TreatMissingData.BREACHING, + comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + }); } } diff --git a/lib/forgejo-replica-stack.ts b/lib/forgejo-replica-stack.ts index ca8fa5a..fe848f6 100644 --- a/lib/forgejo-replica-stack.ts +++ b/lib/forgejo-replica-stack.ts @@ -3,10 +3,13 @@ import * as s3 from "aws-cdk-lib/aws-s3"; import { Construct } from "constructs"; export class ForgejoReplicaStack extends cdk.Stack { + public readonly replicaBucketArn: string; + public readonly replicaBucketName: string; + constructor(scope: Construct, id: string, props?: cdk.StackProps) { super(scope, id, props); - new s3.Bucket(this, "ReplicaBucket", { + const replicaBucket = new s3.Bucket(this, "ReplicaBucket", { bucketName: "forgejo-backups-replica-328440206208", encryption: s3.BucketEncryption.S3_MANAGED, blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, @@ -33,5 +36,8 @@ export class ForgejoReplicaStack extends cdk.Stack { ], removalPolicy: cdk.RemovalPolicy.RETAIN, }); + + this.replicaBucketArn = replicaBucket.bucketArn; + this.replicaBucketName = replicaBucket.bucketName; } } diff --git a/lib/forgejo-stack.ts b/lib/forgejo-stack.ts index 0d9a996..c4c20d3 100644 --- a/lib/forgejo-stack.ts +++ b/lib/forgejo-stack.ts @@ -7,14 +7,20 @@ import * as elbv2 from "aws-cdk-lib/aws-elasticloadbalancingv2"; import * as elbv2_targets from "aws-cdk-lib/aws-elasticloadbalancingv2-targets"; import * as route53 from "aws-cdk-lib/aws-route53"; import * as route53Targets from "aws-cdk-lib/aws-route53-targets"; +import * as ssm from "aws-cdk-lib/aws-ssm"; import * as dlm from "aws-cdk-lib/aws-dlm"; import { Construct } from "constructs"; import { BackupVerification } from "./constructs/backup-verification"; const FORGEJO_VERSION = "10.0.1"; +interface ForgejoStackProps extends cdk.StackProps { + replicaBucketArn: string; + replicaBucketName: string; +} + export class ForgejoStack extends cdk.Stack { - constructor(scope: Construct, id: string, props?: cdk.StackProps) { + constructor(scope: Construct, id: string, props: ForgejoStackProps) { super(scope, id, props); const vpc = ec2.Vpc.fromLookup(this, "SeaHavenVpc", { @@ -84,7 +90,15 @@ export class ForgejoStack extends cdk.Stack { backupBucket.grantReadWrite(role); - const replicaBucketArn = "arn:aws:s3:::forgejo-backups-replica-328440206208"; + const backupS3Prefix = new ssm.StringParameter(this, "BackupS3Prefix", { + parameterName: "/forgejo/backup-s3-prefix", + description: "S3 key prefix for Forgejo backup dumps", + stringValue: "archive", + }); + + backupS3Prefix.grantRead(role); + + const replicaBucketArn = props.replicaBucketArn; const replicationRole = new iam.Role(this, "ReplicationRole", { roleName: "forgejo-s3-replication", @@ -153,9 +167,6 @@ export class ForgejoStack extends cdk.Stack { secretAccessKey: gcsTransferKey.secretAccessKey, }, }); - const gcsTransferCredentialsResource = gcsTransferCredentials.node.defaultChild as secretsmanager.CfnSecret; - gcsTransferCredentialsResource.overrideLogicalId("GcsTransferCredentials"); - const userData = ec2.UserData.forLinux(); userData.addCommands( "set -euxo pipefail", @@ -239,11 +250,12 @@ export class ForgejoStack extends cdk.Stack { "#!/bin/bash", "set -euo pipefail", "TIMESTAMP=$(date +%Y-%m-%d)", + "S3_PREFIX=$(aws ssm get-parameter --name /forgejo/backup-s3-prefix --query Parameter.Value --output text --region us-east-1 || echo 'archive')", "DUMP_DIR=$(mktemp -d)", "chown forgejo:forgejo \"$DUMP_DIR\"", "cd \"$DUMP_DIR\"", "sudo -u forgejo /usr/local/bin/forgejo dump --config /etc/forgejo/app.ini --type tar.gz --file \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\"", - "aws s3 cp \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\" s3://forgejo-backups-328440206208/archive/${TIMESTAMP}/forgejo-${TIMESTAMP}.tar.gz", + "aws s3 cp \"$DUMP_DIR/forgejo-${TIMESTAMP}.tar.gz\" s3://forgejo-backups-328440206208/${S3_PREFIX}/${TIMESTAMP}/forgejo-${TIMESTAMP}.tar.gz", "rm -rf \"$DUMP_DIR\"", "BAKEOF", "chmod +x /usr/local/bin/forgejo-backup.sh", @@ -441,7 +453,7 @@ export class ForgejoStack extends cdk.Stack { new BackupVerification(this, "BackupVerification", { sourceBucket: backupBucket, - replicaBucketName: "forgejo-backups-replica-328440206208", + replicaBucketName: props.replicaBucketName, gcsBucket: "forgejo-backups-offsite-seahaven", gcsSaSecretName: "forgejo/gcs-sa-key", slackWebhookSecretName: "forgejo/slack-webhook",