diff --git a/deploy/ami/open-swe-base.pkr.hcl b/deploy/ami/open-swe-base.pkr.hcl index d5c30737..af2c1982 100644 --- a/deploy/ami/open-swe-base.pkr.hcl +++ b/deploy/ami/open-swe-base.pkr.hcl @@ -113,10 +113,17 @@ build { name = "open-swe-base" sources = ["source.amazon-ebs.open-swe"] - # Stage the boot-time templates and helper scripts into the image. + # Stage the boot-time templates into the image. The destination dir must exist + # BEFORE a trailing-slash (contents-only) file upload — packer's file provisioner + # does not create it, and uploading the directory itself trips scp ("Is a + # directory"). So mkdir first, then upload the contents into it. + provisioner "shell" { + inline = ["mkdir -p /tmp/open-swe-templates"] + } + provisioner "file" { source = "${path.root}/templates/" - destination = "/tmp/open-swe-templates/" + destination = "/tmp/open-swe-templates" } provisioner "shell" { @@ -127,7 +134,9 @@ build { "CLOUDWATCH_AGENT_DEB_URL=${var.cloudwatch_agent_deb_url}", "AWSCLI_ZIP_URL=${var.awscli_zip_url}", ] - execute_command = "chmod +x {{ .Path }}; sudo -E bash '{{ .Path }}'" + # {{ .Vars }} MUST be included or the environment_vars above never reach the + # script (provision.sh runs under `set -u` and fails on the first reference). + execute_command = "chmod +x {{ .Path }}; {{ .Vars }} sudo -E bash '{{ .Path }}'" script = "${path.root}/scripts/provision.sh" } } diff --git a/infra/README.md b/infra/README.md index e1d4b0a6..256cb868 100644 --- a/infra/README.md +++ b/infra/README.md @@ -21,11 +21,11 @@ infra/ │ ├── instance-role.ts # open-swe--instance-role (least-privilege) │ ├── config-store.ts # Secrets Manager + SSM Parameter Store shells (T11) │ ├── app-service.ts # EC2 box + imported-ALB ingress + Route53 + logs (T12) -│ └── ami-cache.ts # cached ARM64 AL2023 helper + EBS/AMI discipline docs +│ └── ami-cache.ts # baked open-swe AMI pin (by id) + EBS/replacement docs ├── test/ │ └── kebab-naming-aspect.test.ts # jest: Aspect passes conforming names, flags bad ones ├── cdk.json -├── cdk.context.json # COMMITTED — pins the AMI (see AMI cache discipline) +├── cdk.context.json # COMMITTED — {} (AMI is a static id pin; no lookups) ├── package.json # aws-cdk-lib pinned EXACT (2.260.0) ├── tsconfig.json ├── jest.config.js @@ -197,17 +197,21 @@ hooks hostname is scoped to `/webhooks/*` only (OSWE-T12-02 hygiene). An X-Forwarded-For spoof candidate was **killed** — no code trusts the leftmost XFF. No confirmed critical/high; no block. -## AMI cache discipline (EBS-fix plumbing — consumed by T12 `AppService`) +## Baked AMI + EBS-replacement discipline -`cachedArm64AmazonLinux2023()` (in `lib/constructs/ami-cache.ts`) returns an -ARM64 Amazon Linux 2023 image with `cachedInContext: true`, so the resolved AMI -id is pinned in the committed `cdk.context.json`. Without the pin, every deploy -could pick up a newer AL2023 release → AMI change → **EC2 instance replacement** -(the file-share data-loss root cause — memory `feedback_inline_ebs_volumes`). +`bakedOpenSweArm64()` (in `lib/constructs/ami-cache.ts`) pins the custom +**open-swe-base-arm64** image by EXACT id (`BAKED_OPEN_SWE_AMI_ID`) via +`MachineImage.genericLinux({ "us-east-1": "" })` — no SSM lookup, so synth +and deploy are fully offline/deterministic. The image is built by +`deploy/ami/open-swe-base.pkr.hcl` (ARM64 Ubuntu 24.04 + uv/py3.12 + nginx + CW +agent + boot templates, **no secrets**); the box's `user-data.sh` assumes that +baked layout (`/opt/open-swe`, `openswe` user, nginx, CW agent). -Design intent (now consumed by `AppService`): +Pinning by exact id (vs a `most_recent` name filter) is what prevents a routine +deploy from silently swapping the AMI → **EC2 instance replacement** (the +file-share data-loss root cause — memory `feedback_inline_ebs_volumes`). -- `userDataCausesReplacement: true` is the **deliberate** choice — user-data is +- `userDataCausesReplacement: true` is **deliberate** — user-data is provisioning-only and carries no durable state. - **No durable state on the box → no RETAIN volume.** The in-memory langgraph store is rebuilt on every boot from S3 + Secrets Manager / SSM, so there is @@ -217,18 +221,16 @@ Design intent (now consumed by `AppService`): deploy snapshot the root volume and wait `state=completed`, and re-verify "no local-only durable state" first. -Refresh the AMI pin deliberately: +Refresh the AMI deliberately: ```bash -cdk context --reset 'ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64:region=us-east-1' -cdk synth # review the diff — it WILL show "requires replacement" +cd deploy/ami && packer build open-swe-base.pkr.hcl # prints the new ami-… id +# update BAKED_OPEN_SWE_AMI_ID in infra/lib/constructs/ami-cache.ts +cd infra && npx cdk diff OpenSweDevStack # WILL show "requires replacement" ``` -> The committed `cdk.context.json` ships a dummy-but-valid-shaped AMI id -> (`ami-00000000000000000`) so `cdk synth` resolves the cache locally without any -> live AWS call. **Before the first real deploy**, repoint `AppService` to the -> baked `open-swe-base-arm64` AMI and pin its real id — the placeholder is -> intentionally un-bootable on stock AL2023. +> `cdk.context.json` is `{}` — nothing is resolved via context anymore (the AMI is +> a static id pin), so synth makes no live AWS call. ## Commands diff --git a/infra/cdk.context.json b/infra/cdk.context.json index 5e1a4249..0967ef42 100644 --- a/infra/cdk.context.json +++ b/infra/cdk.context.json @@ -1,3 +1 @@ -{ - "ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64:region=us-east-1": "ami-00000000000000000" -} +{} diff --git a/infra/lib/constructs/ami-cache.ts b/infra/lib/constructs/ami-cache.ts index dea21aa0..279f0f81 100644 --- a/infra/lib/constructs/ami-cache.ts +++ b/infra/lib/constructs/ami-cache.ts @@ -1,62 +1,35 @@ import * as ec2 from "aws-cdk-lib/aws-ec2"; +import { REGION } from "../config"; /** - * cdk.context.json key for the cached AL2023 ARM64 AMI. `latestAmazonLinux2023` - * + ARM_64 in the pinned aws-cdk-lib resolves the public SSM parameter below - * (the default-kernel alias for this CDK version is the `kernel-6.1` line); with - * `cachedInContext: true` CDK stores the resolved id under this exact key. - * Exported so the env stack can check "is the AMI already pinned?" and skip the - * resolve at synth when it is not — guaranteeing T3 synth makes no live AWS call. + * The baked open-swe base AMI (ARM64 Ubuntu 24.04 + uv/py3.12 + nginx + CW agent + * + boot templates — NO secrets), produced by `deploy/ami/open-swe-base.pkr.hcl`. + * Pinned by EXACT id (not a name filter) so synth/deploy is fully offline and + * deterministic. * - * If an aws-cdk-lib bump changes the default kernel alias, `cdk synth` will write - * a new key into cdk.context.json — update this constant + the committed pin to - * match (the AMI cache naturally tracks the CDK version). + * Built 2026-06-26 from open-swe-base-arm64-20260626-201929. + * + * ── EBS / AMI replacement discipline (memory feedback_inline_ebs_volumes) ── + * + * Refresh DELIBERATELY: `cd deploy/ami && packer build open-swe-base.pkr.hcl`, + * then update this id. A new id → EC2 instance REPLACEMENT. Pinning by exact id + * (vs a `most_recent` name filter) is what prevents a routine deploy from silently + * swapping the AMI — the root cause of the file-share data-loss incidents + * (5/15, 5/27, 6/5). + * + * `userDataCausesReplacement: true` (AppService) is likewise DELIBERATE: user-data + * is provisioning-only and the box holds NO durable state (the langgraph store is + * in-memory, rebuilt every boot from S3 + Secrets Manager / SSM), so there is + * intentionally no standalone `ec2.Volume` + `removalPolicy.RETAIN`. The design + * goal is replacement-TOLERANCE, not avoidance. + * + * Operational guard before ANY replacing deploy (AMI / userData / instance-type): + * snapshot the root volume AND wait `state=completed`, re-verify "no local-only + * durable state", and review the `cdk diff` replacement at PR time. */ -export const AL2023_ARM64_SSM_CONTEXT_KEY = - "ssm:account=328440206208:parameterName=/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64:region=us-east-1"; +export const BAKED_OPEN_SWE_AMI_ID = "ami-0545363bb147229ff"; -/** - * Cached ARM64 Amazon Linux 2023 machine image. - * - * ── EBS / AMI cache discipline (see memory feedback_inline_ebs_volumes) ── - * - * `cachedInContext: true` PINS the resolved AMI id into the committed - * cdk.context.json. Without it, `latestAmazonLinux2023()` resolves the NEWEST - * AL2023 release on every synth/deploy, so a routine deploy can swap the AMI → - * EC2 instance REPLACEMENT whenever AWS ships a release. That was the root cause - * of the file-share data-loss incidents (5/15, 5/27, 6/5). Refresh the pin - * DELIBERATELY: - * - * cdk context --reset '' && cdk synth - * - * then review the `cdk diff` (it WILL report "requires replacement") before - * deploying. - * - * ── userDataCausesReplacement intent (consumed at T12) ── - * - * open-swe user-data is provisioning-only: install the langgraph runtime, pull - * config from Secrets Manager / SSM, pull the build artifact from S3, start the - * service. It holds NO durable state. T12 sets `userDataCausesReplacement: true` - * DELIBERATELY so a config/bootstrap change rolls a fresh, known-good box. - * - * ── "No durable state on box → no RETAIN volume" assertion ── - * - * The langgraph store is in-memory and is reconstructed on every boot from S3 - * (artifact) + Secrets Manager / SSM (config). Nothing of record lives only on - * the instance's disk. Therefore there is intentionally NO standalone - * `ec2.Volume` + `removalPolicy.RETAIN` here: the design goal is replacement- - * TOLERANCE, not replacement-avoidance. - * - * ── Operational guard still applies at T12 (feedback_inline_ebs_volumes) ── - * - * Before ANY replacing deploy (AMI / userData / instance-type change): snapshot - * the root volume AND wait for `state=completed`, re-verify "no local-only - * durable state" first, and ensure cdk-diff-on-PR surfaces the replacement at - * review time. - */ -export function cachedArm64AmazonLinux2023(): ec2.IMachineImage { - return ec2.MachineImage.latestAmazonLinux2023({ - cpuType: ec2.AmazonLinuxCpuType.ARM_64, - cachedInContext: true, - }); +/** The baked open-swe base image, pinned by id (offline, deterministic). */ +export function bakedOpenSweArm64(): ec2.IMachineImage { + return ec2.MachineImage.genericLinux({ [REGION]: BAKED_OPEN_SWE_AMI_ID }); } diff --git a/infra/lib/constructs/app-service.ts b/infra/lib/constructs/app-service.ts index ae6faf8d..a1d67fac 100644 --- a/infra/lib/constructs/app-service.ts +++ b/infra/lib/constructs/app-service.ts @@ -9,7 +9,7 @@ import * as route53 from "aws-cdk-lib/aws-route53"; import * as iam from "aws-cdk-lib/aws-iam"; import { Construct } from "constructs"; import { EnvName, prefix } from "../config"; -import { cachedArm64AmazonLinux2023 } from "./ami-cache"; +import { bakedOpenSweArm64 } from "./ami-cache"; /** * Shared seahaven-vpc + internet-facing ALB facts (read-only recon 2026-06-26; @@ -177,13 +177,11 @@ export class AppService extends Construct { ], }, instanceType: new ec2.InstanceType(net.instanceType), - // TODO(T12-deploy): repoint to the custom open-swe-base-arm64 AMI baked by - // packer (deploy/ami). The AL2023 ARM64 cache is a synth-time placeholder - // (pinned in cdk.context.json) so the box DEFINITION synths offline; the - // baked AMI id is pinned (cdk context) before the first real deploy. - // user-data.sh assumes the baked layout (/opt/open-swe, openswe user, nginx, - // CloudWatch agent) — it is NOT runnable on the stock AL2023 placeholder. - machineImage: cachedArm64AmazonLinux2023(), + // The baked open-swe base AMI (deploy/ami packer build) — ARM64 Ubuntu 24.04 + // with the /opt/open-swe layout, openswe user, nginx, and CW agent that + // user-data.sh assumes. Pinned by exact id (see ami-cache.ts); refresh by + // rebuilding and updating BAKED_OPEN_SWE_AMI_ID. + machineImage: bakedOpenSweArm64(), role: props.instanceRole, securityGroup: instanceSg, userData, diff --git a/infra/lib/open-swe-stack.ts b/infra/lib/open-swe-stack.ts index 4d4f7377..86ed39cb 100644 --- a/infra/lib/open-swe-stack.ts +++ b/infra/lib/open-swe-stack.ts @@ -4,10 +4,7 @@ import { EnvName, prefix } from "./config"; import { AppService } from "./constructs/app-service"; import { ConfigStore } from "./constructs/config-store"; import { InstanceRole } from "./constructs/instance-role"; -import { - AL2023_ARM64_SSM_CONTEXT_KEY, - cachedArm64AmazonLinux2023, -} from "./constructs/ami-cache"; +import { BAKED_OPEN_SWE_AMI_ID } from "./constructs/ami-cache"; export interface OpenSweStackProps extends cdk.StackProps { /** open-swe environment — drives the `open-swe--*` resource naming. */ @@ -48,18 +45,12 @@ export class OpenSweStack extends cdk.Stack { // The instance role already grants read on open-swe-/* + /open-swe-/*. this.configStore = new ConfigStore(this, "Config", { envName }); - // AMI cache discipline (see lib/constructs/ami-cache.ts). T3 is synth-only: - // resolve + surface the pinned AMI id ONLY when it is already cached in - // cdk.context.json, so synth never makes a live SSM call. T12 consumes - // `cachedArm64AmazonLinux2023()` for the actual ec2.Instance. - if (this.node.tryGetContext(AL2023_ARM64_SSM_CONTEXT_KEY) !== undefined) { - const amiId = cachedArm64AmazonLinux2023().getImage(this).imageId; - new cdk.CfnOutput(this, "PinnedAmiId", { - value: amiId, - description: - "Cached AL2023 ARM64 AMI id (pinned in cdk.context.json; consumed by the T12 EC2 instance).", - }); - } + // Surface the baked open-swe base AMI id the box runs on (pinned by id in + // ami-cache.ts; refreshed by a deliberate packer rebuild → replacement). + new cdk.CfnOutput(this, "BakedAmiId", { + value: BAKED_OPEN_SWE_AMI_ID, + description: "Baked open-swe-base-arm64 AMI id consumed by the EC2 instance.", + }); // T12: compute + ingress. Imports the shared seahaven-vpc + ALB and adds the // env's EC2 box, instance SG, target group, listener rules, DNS, log groups.