mirror of
https://github.com/Sea-Haven-Industries/forgejo.git
synced 2026-09-30 10:03:11 +00:00
* Add 3-2-1 backup strategy with cross-region replication and GCS offsite Implements a fully compliant 3-2-1 backup architecture: - Copy 1 (live): Harden existing EBS snapshots to 30-day retention - Copy 2 (near-site): S3 cross-region replication to us-west-2 with Object Lock (governance 90d) and versioning - Copy 3 (offsite): GCS bucket in dedicated seahaven-backups GCP project with 2-year irreversible retention lock Also adds a verification Lambda that checks all 3 locations daily and runs monthly restore tests with SQLite integrity checks. * Enable QEMU in CI for arm64 Lambda Docker builds * Commit cdk.context.json for CI synth without AWS credentials Vpc.fromLookup requires cached context to synthesize without AWS credentials. Required for CI which runs cdk synth without an OIDC role. * Fix GCP project ID to sea-haven-backups * Address code review findings for backup verification Fix 4 critical issues: - Add filter/priority/deleteMarkerReplication to S3 CRR rule (deploy would fail without) - Add stack dependency so replica deploys before main stack - Fix DB file extension matching (.sqlite3/.sql instead of .db) - Replace nonexistent `forgejo restore` command with actual restore steps in README Fix 4 moderate issues: - Add timeout=10 to Slack webhook urlopen call - Add filter='data' to tarfile.extract for PEP 706 compliance - Add explicit ValueError for unknown handler mode - Use date-scoped S3/GCS prefix instead of unbounded listing * Fix backup strategy bug findings * Handle SQL text dumps separately from binary SQLite in restore test Forgejo dump produces gitea-db.sql as a text SQL dump (XORM export), not a binary SQLite file. Opening it directly with sqlite3.connect() throws DatabaseError. Now imports the SQL dump into a temp DB first. * Fix GCS backup check: align staleness cutoff and add size validation GCS check used a 72h cutoff but only listed 2 days of prefixes (~48h), making the staleness check unreachable. Also added 1MB minimum file size validation to match the S3 check. * Rename SECRET_ARN env vars to SECRET_NAME to match actual values * Fix EBS snapshot state check, drop unused GCS write grant and dead lifecycle rule * Fix restore runbook, DLM snapshot tagging, README cleanup, and gsutil prompt * Fix restore runbook: trailing-dot cp idiom and Glacier restore step * Rename GCS service account to match read-only permissions * Add 4 GiB ephemeral storage to verification Lambda Monthly restore-test downloads and extracts the full dump tarball in /tmp. As the dump grows with LFS data, the default 512 MB will eventually cause ENOSPC failures. --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
95 lines
2.9 KiB
TypeScript
95 lines
2.9 KiB
TypeScript
import * as cdk from "aws-cdk-lib";
|
|
import * as events from "aws-cdk-lib/aws-events";
|
|
import * as events_targets from "aws-cdk-lib/aws-events-targets";
|
|
import * as iam from "aws-cdk-lib/aws-iam";
|
|
import * as lambda from "aws-cdk-lib/aws-lambda";
|
|
import * as logs from "aws-cdk-lib/aws-logs";
|
|
import * as s3 from "aws-cdk-lib/aws-s3";
|
|
import { PythonFunction } from "@aws-cdk/aws-lambda-python-alpha";
|
|
import { Construct } from "constructs";
|
|
|
|
interface BackupVerificationProps {
|
|
sourceBucket: s3.IBucket;
|
|
replicaBucketName: string;
|
|
gcsBucket: string;
|
|
gcsSaSecretName: string;
|
|
slackWebhookSecretName: string;
|
|
}
|
|
|
|
export class BackupVerification extends Construct {
|
|
constructor(scope: Construct, id: string, props: BackupVerificationProps) {
|
|
super(scope, id);
|
|
|
|
const fn = new PythonFunction(this, "Function", {
|
|
functionName: "forgejo-backup-verification",
|
|
entry: "lambda/backup-verification",
|
|
runtime: lambda.Runtime.PYTHON_3_12,
|
|
architecture: lambda.Architecture.ARM_64,
|
|
handler: "handler",
|
|
index: "app.py",
|
|
memorySize: 512,
|
|
ephemeralStorageSize: cdk.Size.gibibytes(4),
|
|
timeout: cdk.Duration.minutes(5),
|
|
environment: {
|
|
SOURCE_BUCKET: props.sourceBucket.bucketName,
|
|
REPLICA_BUCKET: props.replicaBucketName,
|
|
GCS_BUCKET: props.gcsBucket,
|
|
GCS_SA_SECRET_NAME: props.gcsSaSecretName,
|
|
SLACK_WEBHOOK_SECRET_NAME: props.slackWebhookSecretName,
|
|
},
|
|
logRetention: logs.RetentionDays.TWO_MONTHS,
|
|
});
|
|
|
|
props.sourceBucket.grantRead(fn);
|
|
|
|
fn.addToRolePolicy(
|
|
new iam.PolicyStatement({
|
|
actions: ["s3:ListBucket", "s3:GetObject"],
|
|
resources: [
|
|
`arn:aws:s3:::${props.replicaBucketName}`,
|
|
`arn:aws:s3:::${props.replicaBucketName}/*`,
|
|
],
|
|
})
|
|
);
|
|
|
|
const account = cdk.Stack.of(this).account;
|
|
const region = cdk.Stack.of(this).region;
|
|
|
|
fn.addToRolePolicy(
|
|
new iam.PolicyStatement({
|
|
actions: ["secretsmanager:GetSecretValue"],
|
|
resources: [
|
|
`arn:aws:secretsmanager:${region}:${account}:secret:${props.gcsSaSecretName}-*`,
|
|
`arn:aws:secretsmanager:${region}:${account}:secret:${props.slackWebhookSecretName}-*`,
|
|
],
|
|
})
|
|
);
|
|
|
|
fn.addToRolePolicy(
|
|
new iam.PolicyStatement({
|
|
actions: ["ec2:DescribeSnapshots"],
|
|
resources: ["*"],
|
|
})
|
|
);
|
|
|
|
new events.Rule(this, "DailyCheck", {
|
|
ruleName: "forgejo-backup-daily-check",
|
|
schedule: events.Schedule.cron({ hour: "8", minute: "0" }),
|
|
targets: [new events_targets.LambdaFunction(fn)],
|
|
});
|
|
|
|
new events.Rule(this, "MonthlyRestoreTest", {
|
|
ruleName: "forgejo-backup-monthly-restore-test",
|
|
schedule: events.Schedule.cron({
|
|
hour: "9",
|
|
minute: "0",
|
|
day: "1",
|
|
}),
|
|
targets: [
|
|
new events_targets.LambdaFunction(fn, {
|
|
event: events.RuleTargetInput.fromObject({ mode: "restore-test" }),
|
|
}),
|
|
],
|
|
});
|
|
}
|
|
}
|