feat(ci): add docker and ECR image reusables (#164)

* feat(ci): add docker build and ECR image publish reusables

Image repos need a registry-free build check and an ECR publish that stops
at a mutable tag, without an ECS or Lambda update.

* fix(ci): retry ECR digest checks when describe-images fails

A tag that is not visible yet makes the AWS CLI exit non-zero, and set -e
was aborting the retry loop on that first error.

.github/actionlint.yaml

@@ -4,3 +4,6 @@ paths:
   .github/workflows/ci-terraform.yaml:
     ignore:
       - 'specifying action "\$/.github/actions/app-terraform-isolation" in invalid format because ref is missing'
+  .github/workflows/cd-ecr-image.yaml:
+    ignore:
+      - 'specifying action "\$/.github/actions/verify-ecr-promote-digest" in invalid format because ref is missing'

.github/actions/verify-ecr-promote-digest/action.yml

@@ -0,0 +1,25 @@
+name: Verify ECR promote digest
+description: Fail unless the promote tag and the sha tag have the same ECR image digest.
+
+inputs:
+  image-name:
+    description: ECR repository name.
+    required: true
+  sha:
+    description: Full commit SHA used in the sha-<commit> tag.
+    required: true
+  promote-tag:
+    description: Mutable tag that must match the commit tag.
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Compare tag digests
+      shell: bash
+      env:
+        IMAGE_NAME: ${{ inputs.image-name }}
+        SHA: ${{ inputs.sha }}
+        PROMOTE_TAG: ${{ inputs.promote-tag }}
+        VERIFIER: ${{ github.action_path }}/verify.sh
+      run: bash "${VERIFIER}"

.github/actions/verify-ecr-promote-digest/verify.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Confirm promote-tag and sha-<commit> point at the same ECR digest.
+# describe-images can error while a tag just pushed is not yet visible, so
+# a failed call is a retry, not an immediate abort.
+set -euo pipefail
+
+: "${IMAGE_NAME:?image-name is required}"
+: "${SHA:?sha is required}"
+: "${PROMOTE_TAG:?promote-tag is required}"
+
+retry_sleep="${DIGEST_RETRY_SLEEP:-5}"
+sha_tag="sha-${SHA}"
+
+query_digest() {
+  local tag="$1"
+  local err digest
+  err="$(mktemp)"
+  if digest="$(aws ecr describe-images \
+    --repository-name "${IMAGE_NAME}" \
+    --image-ids "imageTag=${tag}" \
+    --query 'imageDetails[0].imageDigest' \
+    --output text 2>"${err}")"; then
+    rm -f "${err}"
+    printf '%s\n' "${digest}"
+    return 0
+  fi
+  echo "describe-images ${tag}: $(tr '\n' ' ' < "${err}")" >&2
+  rm -f "${err}"
+  return 1
+}
+
+for _ in 1 2 3 4 5 6; do
+  sha_digest="$(query_digest "${sha_tag}")" || sha_digest=""
+  promote_digest="$(query_digest "${PROMOTE_TAG}")" || promote_digest=""
+  if [ "${sha_digest}" = "${promote_digest}" ] && [ -n "${sha_digest}" ] && [ "${sha_digest}" != "None" ]; then
+    echo "promote tag ${PROMOTE_TAG} digest matches ${sha_tag}: ${sha_digest}"
+    exit 0
+  fi
+  echo "digest mismatch (sha=${sha_digest:-empty} promote=${promote_digest:-empty}); retrying"
+  sleep "${retry_sleep}"
+done
+
+echo "promote tag ${PROMOTE_TAG} digest does not match ${sha_tag}" >&2
+exit 1

.github/workflows/cd-ecr-image.yaml

@@ -0,0 +1,299 @@
+name: CD — ECR image
+
+# Reusable ECR image publish. The caller owns triggers and passes
+# environment as a with: input. This job owns environment:, concurrency,
+# OIDC, and vars.DEPLOY_ROLE_ARN. GitHub rejects environment: beside uses:.
+#
+# Publishes sha-<commit>, optionally smokes that image, then moves
+# promote-tag (default current). A failed smoke does not move the promote
+# tag. Nothing here updates ECS, Lambda, or an HCP run.
+#
+# dockerfile is relative to context. After the promote push, DescribeImages
+# must show the same digest on promote-tag and sha-<commit>. That check
+# retries when the API errors or the digests differ, including while the
+# promote tag is not yet visible.
+#
+# Caller example:
+#   jobs:
+#     publish:
+#       uses: Sea-Haven-Industries/.github/.github/workflows/cd-ecr-image.yaml@<sha> # vX.Y.Z
+#       permissions: { contents: read, id-token: write }
+#       secrets: inherit
+#       with:
+#         environment: ci
+#         image-name: actions-runner
+#         context: runner
+#         platform: linux/amd64
+#         smoke-script: runner/smoke.sh
+#         ship-gate: true
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "GitHub Environment whose DEPLOY_ROLE_ARN publishes the image"
+        type: string
+        required: true
+      ref:
+        description: "Git ref to build. Empty means github.sha."
+        type: string
+        required: false
+        default: ""
+      image-name:
+        description: "ECR repository name"
+        type: string
+        required: true
+      context:
+        description: "Docker build context, relative to the repo root"
+        type: string
+        required: false
+        default: "."
+      dockerfile:
+        description: "Dockerfile path relative to context"
+        type: string
+        required: false
+        default: "Dockerfile"
+      platform:
+        description: "docker build --platform value"
+        type: string
+        required: false
+        default: "linux/amd64"
+      promote-tag:
+        description: "Mutable tag moved after a successful smoke"
+        type: string
+        required: false
+        default: "current"
+      smoke-script:
+        description: "Checkout path to run inside the image with bash before promotion. Empty skips the smoke."
+        type: string
+        required: false
+        default: ""
+      ship-gate:
+        description: "Require the ref to be on main or a legal hotfix/release tag"
+        type: boolean
+        required: false
+        default: false
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  deploy:
+    name: Publish ${{ inputs.image-name }} to ${{ inputs.environment }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    environment: ${{ inputs.environment }}
+    concurrency:
+      group: deploy-${{ inputs.image-name }}-${{ inputs.environment }}
+      cancel-in-progress: false
+    env:
+      AWS_REGION: us-east-1
+      DEPLOY_ROLE_ARN: ${{ vars.DEPLOY_ROLE_ARN }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.ref != '' && inputs.ref || github.sha }}
+          persist-credentials: false
+          fetch-tags: true
+
+      - name: Resolve commit
+        id: commit
+        run: |
+          set -euo pipefail
+          sha="$(git rev-parse HEAD)"
+          echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
+          echo "Building ${sha}"
+
+      - name: Ship-gate
+        if: ${{ inputs.ship-gate }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          INPUT_REF: ${{ inputs.ref != '' && inputs.ref || github.sha }}
+          ENVIRONMENT: ${{ inputs.environment }}
+          HEAD_SHA: ${{ steps.commit.outputs.sha }}
+        run: |
+          set -euo pipefail
+
+          status="$(gh api "repos/${REPO}/compare/main...${INPUT_REF}" --jq .status)"
+          if [ "${status}" = "behind" ] || [ "${status}" = "identical" ]; then
+            echo "ship-gate: ${INPUT_REF} is ${status} relative to main"
+            exit 0
+          fi
+
+          echo "ship-gate: ${INPUT_REF} is not on main (compare status: ${status}); checking hotfix/release path"
+
+          TAG="${INPUT_REF}"
+          if [[ ! "${TAG}" =~ ^v ]]; then
+            TAG="$(git describe --tags --exact-match "${HEAD_SHA}" 2>/dev/null || true)"
+          fi
+
+          if [ "${ENVIRONMENT}" = "staging" ]; then
+            PATTERN='^v[0-9]+\.[0-9]+\.[0-9]+-staging$'
+          else
+            PATTERN='^v[0-9]+\.[0-9]+\.[0-9]+$'
+          fi
+
+          if [[ -z "${TAG}" ]] || [[ ! "${TAG}" =~ ${PATTERN} ]]; then
+            echo "ship-gate: ref ${INPUT_REF} (tag ${TAG:-none}) does not match ${PATTERN} for environment ${ENVIRONMENT} and is not on main" >&2
+            exit 1
+          fi
+
+          export PATTERN TAG
+          PREV="$(
+            gh api "repos/${REPO}/releases" --paginate --jq '.[].tag_name' | python3 -c 'import os, re, sys
+          pattern = re.compile(os.environ["PATTERN"])
+          current = os.environ["TAG"]
+          tags = [
+              line.strip()
+              for line in sys.stdin
+              if pattern.fullmatch(line.strip()) and line.strip() != current
+          ]
+          def key(tag):
+              body = tag[1:]
+              core = body.split("-", 1)[0]
+              return tuple(int(part) for part in core.split("."))
+          tags.sort(key=key)
+          print(tags[-1] if tags else "")'
+          )"
+
+          if [ -z "${PREV}" ]; then
+            echo "ship-gate: no previous matching release tag; hotfix path requires a prior ${PATTERN} release" >&2
+            exit 1
+          fi
+
+          ff_status="$(gh api "repos/${REPO}/compare/${PREV}...${TAG}" --jq .status)"
+          if [ "${ff_status}" != "ahead" ]; then
+            echo "ship-gate: ${TAG} is not a fast-forward of ${PREV} (status: ${ff_status})" >&2
+            exit 1
+          fi
+
+          from_train=false
+          TARGET_COMMITISH="$(gh api "repos/${REPO}/releases/tags/${TAG}" --jq .target_commitish 2>/dev/null || true)"
+          if [[ "${TARGET_COMMITISH}" == hotfix/* || "${TARGET_COMMITISH}" == release/* ]]; then
+            from_train=true
+          fi
+
+          if [ "${from_train}" = false ]; then
+            git fetch origin '+refs/heads/hotfix/*:refs/remotes/origin/hotfix/*' '+refs/heads/release/*:refs/remotes/origin/release/*' || true
+            if git branch -r --contains "${HEAD_SHA}" | grep -Eq 'origin/(hotfix|release)/'; then
+              from_train=true
+            fi
+          fi
+
+          if [ "${from_train}" = false ]; then
+            echo "ship-gate: ${TAG} was not created from hotfix/* or release/* (target_commitish=${TARGET_COMMITISH:-none})" >&2
+            exit 1
+          fi
+
+          echo "ship-gate: hotfix/release path accepted (${PREV} -> ${TAG} from ${TARGET_COMMITISH:-branch})"
+
+      - name: Configure AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ env.DEPLOY_ROLE_ARN }}
+          aws-region: us-east-1
+          audience: sts.amazonaws.com
+
+      - name: Login to Amazon ECR
+        id: login
+        uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd # v2.1.7
+
+      - name: Build and push the commit tag
+        id: build
+        env:
+          REGISTRY: ${{ steps.login.outputs.registry }}
+          IMAGE_NAME: ${{ inputs.image-name }}
+          SHA: ${{ steps.commit.outputs.sha }}
+          CONTEXT: ${{ inputs.context }}
+          DOCKERFILE: ${{ inputs.dockerfile }}
+          PLATFORM: ${{ inputs.platform }}
+        run: |
+          set -euo pipefail
+          if [ -z "${REGISTRY}" ]; then
+            echo "ECR login did not return a registry" >&2
+            exit 1
+          fi
+          case "${IMAGE_NAME}" in
+            ""|*[[:space:]]*|*..*)
+              echo "image-name must be an ECR repository name" >&2
+              exit 1
+              ;;
+          esac
+          if [[ ! "${SHA}" =~ ^[0-9a-f]{40}$ ]]; then
+            echo "resolved commit is not a full SHA" >&2
+            exit 1
+          fi
+          case "${CONTEXT}" in
+            ""|/*|*..*)
+              echo "context must be a relative path without '..'" >&2
+              exit 1
+              ;;
+          esac
+          case "${DOCKERFILE}" in
+            ""|/*|*..*)
+              echo "dockerfile must be a relative path without '..'" >&2
+              exit 1
+              ;;
+          esac
+          if [ -z "${PLATFORM}" ]; then
+            echo "platform is required" >&2
+            exit 1
+          fi
+          dockerfile_path="${CONTEXT%/}/${DOCKERFILE}"
+          if [ ! -f "${dockerfile_path}" ]; then
+            echo "dockerfile not found: ${dockerfile_path}" >&2
+            exit 1
+          fi
+          image="${REGISTRY}/${IMAGE_NAME}:sha-${SHA}"
+          docker build --platform "${PLATFORM}" -f "${dockerfile_path}" -t "${image}" "${CONTEXT}"
+          docker push "${image}"
+          echo "image=${image}" >> "${GITHUB_OUTPUT}"
+
+      - name: Smoke the image
+        if: ${{ inputs.smoke-script != '' }}
+        env:
+          IMAGE: ${{ steps.build.outputs.image }}
+          SMOKE_SCRIPT: ${{ inputs.smoke-script }}
+        run: |
+          set -euo pipefail
+          case "${SMOKE_SCRIPT}" in
+            ""|/*|*..*)
+              echo "smoke-script must be a relative path without '..'" >&2
+              exit 1
+              ;;
+          esac
+          if [ ! -f "${SMOKE_SCRIPT}" ]; then
+            echo "smoke-script not found: ${SMOKE_SCRIPT}" >&2
+            exit 1
+          fi
+          docker run --rm --entrypoint bash \
+            -v "${GITHUB_WORKSPACE}/${SMOKE_SCRIPT}:/tmp/smoke.sh:ro" \
+            "${IMAGE}" \
+            /tmp/smoke.sh
+
+      - name: Promote tag
+        env:
+          IMAGE: ${{ steps.build.outputs.image }}
+          REGISTRY: ${{ steps.login.outputs.registry }}
+          IMAGE_NAME: ${{ inputs.image-name }}
+          PROMOTE_TAG: ${{ inputs.promote-tag }}
+        run: |
+          set -euo pipefail
+          case "${PROMOTE_TAG}" in
+            ""|*[[:space:]]*|*/*)
+              echo "promote-tag must be a single image tag" >&2
+              exit 1
+              ;;
+          esac
+          promoted="${REGISTRY}/${IMAGE_NAME}:${PROMOTE_TAG}"
+          docker tag "${IMAGE}" "${promoted}"
+          docker push "${promoted}"
+
+      - name: Verify promoted digest
+        uses: $/.github/actions/verify-ecr-promote-digest
+        with:
+          image-name: ${{ inputs.image-name }}
+          sha: ${{ steps.commit.outputs.sha }}
+          promote-tag: ${{ inputs.promote-tag }}

.github/workflows/ci-docker.yaml

@@ -0,0 +1,114 @@
+name: CI — Docker
+
+# Reusable Docker image CI. Builds from the checkout and, when smoke-script
+# is set, runs that file inside the image. No registry login and no push.
+# The caller owns the ci-complete aggregator.
+#
+# dockerfile is relative to context. context runner with the default
+# Dockerfile builds runner/Dockerfile.
+#
+# Caller example:
+#   jobs:
+#     image:
+#       uses: Sea-Haven-Industries/.github/.github/workflows/ci-docker.yaml@<sha> # vX.Y.Z
+#       with:
+#         context: runner
+#         smoke-script: runner/smoke.sh
+#
+# The trailing concurrency segment is the job id written literally, not
+# github.job. In a called workflow that expression is the caller's job id.
+
+on:
+  workflow_call:
+    inputs:
+      context:
+        description: "Docker build context, relative to the repo root"
+        type: string
+        required: false
+        default: "."
+      dockerfile:
+        description: "Dockerfile path relative to context"
+        type: string
+        required: false
+        default: "Dockerfile"
+      platform:
+        description: "docker build --platform value"
+        type: string
+        required: false
+        default: "linux/amd64"
+      smoke-script:
+        description: "Checkout path to run inside the image with bash. Empty skips the smoke."
+        type: string
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+jobs:
+  docker:
+    name: docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    concurrency:
+      group: ci-docker-${{ github.workflow }}-${{ github.ref }}-${{ inputs.context }}-${{ inputs.dockerfile }}-docker
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Build image
+        id: build
+        env:
+          CONTEXT: ${{ inputs.context }}
+          DOCKERFILE: ${{ inputs.dockerfile }}
+          PLATFORM: ${{ inputs.platform }}
+        run: |
+          set -euo pipefail
+          case "${CONTEXT}" in
+            ""|/*|*..*)
+              echo "context must be a relative path without '..'" >&2
+              exit 1
+              ;;
+          esac
+          case "${DOCKERFILE}" in
+            ""|/*|*..*)
+              echo "dockerfile must be a relative path without '..'" >&2
+              exit 1
+              ;;
+          esac
+          if [ -z "${PLATFORM}" ]; then
+            echo "platform is required" >&2
+            exit 1
+          fi
+          dockerfile_path="${CONTEXT%/}/${DOCKERFILE}"
+          if [ ! -f "${dockerfile_path}" ]; then
+            echo "dockerfile not found: ${dockerfile_path}" >&2
+            exit 1
+          fi
+          image="ci-docker:local"
+          docker build --platform "${PLATFORM}" -f "${dockerfile_path}" -t "${image}" "${CONTEXT}"
+          echo "image=${image}" >> "${GITHUB_OUTPUT}"
+
+      - name: Smoke the image
+        if: ${{ inputs.smoke-script != '' }}
+        env:
+          IMAGE: ${{ steps.build.outputs.image }}
+          SMOKE_SCRIPT: ${{ inputs.smoke-script }}
+        run: |
+          set -euo pipefail
+          case "${SMOKE_SCRIPT}" in
+            ""|/*|*..*)
+              echo "smoke-script must be a relative path without '..'" >&2
+              exit 1
+              ;;
+          esac
+          if [ ! -f "${SMOKE_SCRIPT}" ]; then
+            echo "smoke-script not found: ${SMOKE_SCRIPT}" >&2
+            exit 1
+          fi
+          docker run --rm --entrypoint bash \
+            -v "${GITHUB_WORKSPACE}/${SMOKE_SCRIPT}:/tmp/smoke.sh:ro" \
+            "${IMAGE}" \
+            /tmp/smoke.sh

.github/workflows/ci.yaml

@@ -11,8 +11,9 @@ name: ci
 # "Expected — Waiting for status to be reported" and could not merge.
 #
 # The portions are genuinely useful CI for a repo whose whole product is
-# GitHub Actions YAML: the isolation-checker unit tests and actionlint over
+# GitHub Actions YAML: isolation-tests (the isolation checker and the ECR
-# every workflow file. They run in parallel; `ci-complete` requires both.
+# promote digest verifier) and actionlint over every workflow file. They
+# run in parallel; `ci-complete` requires both.
 #
 # Naming is load-bearing: the ruleset matches the required status check against
 # the JOB's check-run name, NOT "workflow / job". For a normal (non-reusable) job
@@ -59,6 +60,10 @@ jobs:
         run: python3 scripts/test_check_app_terraform_isolation.py
         shell: bash
 
+      - name: ECR promote digest verifier tests
+        run: bash scripts/test_verify_ecr_promote_digest.sh
+        shell: bash
+
   actionlint:
     name: actionlint
     runs-on: ubuntu-latest

README.md

@@ -55,10 +55,14 @@ The formatter GitHub App is not on the main-branch bypass list.
 
 **`.github/workflows/ci-terraform.yaml`** — Terraform `fmt -check`, `init -backend=false`, and `validate`. Default version `1.16.0`.
 
+**`.github/workflows/ci-docker.yaml`** — Docker image CI with no registry login and no push. `docker build` from `context` (default `.`); `dockerfile` is relative to that context (default `Dockerfile`); `platform` defaults to `linux/amd64`. When `smoke-script` is set, that checkout path is mounted read-only and run inside the image with `bash`. The caller owns `ci-complete`.
+
 **`.github/workflows/ci-autofix.yaml`** — Pull-request-only formatter. Mints a GitHub App token (`AUTOFMT_APP_ID`, `AUTOFMT_APP_PRIVATE_KEY`), runs the requested presets and any optional write commands, and pushes `style: apply formatter` only when the tree is dirty. Presets are `prettier` (`npm run format`), `eslint` (`npx eslint . --fix`, opt-in), `ruff` (`ruff format .` and `ruff check --fix .`), and `terraform` (`terraform fmt -recursive` in `terraform-working-directory`, default `terraform`). Output `committed` lets the caller skip portions on SHA_old. Does not `--no-verify` and does not push to `main`.
 
 **`.github/workflows/cd-hcp-fargate.yaml`** — HCP Fargate image CD. Checkout at `ref` (empty means `github.sha`), OIDC, SSM cluster/service/family/ecr/container/api-url, docker build+push tagged `$sha` and `$environment`, patch `GIT_SHA`, RegisterTaskDefinition + UpdateService + services-stable, poll health SHA. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency, OIDC, and `vars.DEPLOY_ROLE_ARN`.
 
+**`.github/workflows/cd-ecr-image.yaml`** — ECR image publish with no ECS, Lambda, or HCP update. Checkout at `ref` (empty means `github.sha`), optional `ship-gate`, OIDC via `vars.DEPLOY_ROLE_ARN`, `docker build` and push `sha-<commit>`, optional `smoke-script` inside that image, then retag and push `promote-tag` (default `current`). A failed smoke does not move the promote tag. `ecr:DescribeImages` must show the same digest on both tags. That check retries when the call errors or the digests differ. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency `deploy-<image-name>-<environment>`, and OIDC.
+
 **`.github/workflows/cd-hcp-lambda.yaml`** — HCP Lambda zip CD for Node packagers. Checkout at `ref`, optional `ship-gate`, Node 24, `scripts/package_lambdas.mjs --git-sha --out-dir --only <key>`, verifies `src/buildInfo.js` carries the SHA, uploads `functions/<key>/<sha>.zip` to the SSM `artifacts-bucket`, `update-function-code` on each `<key>-function-name`, waits for `function-updated-v2`. Terraform owns the functions and ignores code attributes.
 
 **`.github/workflows/cd-hcp-lambda-python.yaml`** — HCP Lambda zip CD for Python packagers. Same contract as `cd-hcp-lambda.yaml` with `python-version` (default `3.12`), `scripts/package_lambdas.sh --git-sha --out-dir --only <key>`, and `build_info.py` as the SHA marker. After each `update-function-code` settles, `CodeSha256` must equal the base64 SHA-256 of the local zip and `LastUpdateStatus` must be `Successful`. That is the live-state check for functions with no health URL.

scripts/test_verify_ecr_promote_digest.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# The verifier must retry a failed describe-images call. set -e used to abort
+# the loop on the first ImageNotFoundException.
+set -euo pipefail
+
+root="$(cd "$(dirname "$0")/.." && pwd)"
+verifier="${root}/.github/actions/verify-ecr-promote-digest/verify.sh"
+failures=0
+
+assert_eq() {
+  local name="$1"
+  local got="$2"
+  local want="$3"
+  if [ "${got}" != "${want}" ]; then
+    echo "${name}: got ${got}, want ${want}" >&2
+    failures=$((failures + 1))
+  fi
+}
+
+run_case() {
+  local name="$1"
+  local expect_status="$2"
+  local stub_dir
+  stub_dir="$(mktemp -d)"
+  cat > "${stub_dir}/aws" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+tag=""
+for arg in "$@"; do
+  case "${arg}" in
+    imageTag=*) tag="${arg#imageTag=}" ;;
+  esac
+done
+if [ -z "${tag}" ]; then
+  echo "stub aws: missing image tag" >&2
+  exit 1
+fi
+dir="${AWS_STUB_DIR}"
+count_file="${dir}/count-${tag}"
+n=0
+if [ -f "${count_file}" ]; then
+  n="$(cat "${count_file}")"
+fi
+n=$((n + 1))
+printf '%s\n' "${n}" > "${count_file}"
+line="$(sed -n "${n}p" "${dir}/behavior-${tag}" || true)"
+if [ -z "${line}" ]; then
+  line="$(tail -n 1 "${dir}/behavior-${tag}")"
+fi
+case "${line}" in
+  ok\ *)
+    printf '%s\n' "${line#ok }"
+    ;;
+  fail)
+    echo "ImageNotFoundException: ${tag}" >&2
+    exit 254
+    ;;
+  *)
+    echo "stub aws: no behavior for ${tag} call ${n}" >&2
+    exit 1
+    ;;
+esac
+EOF
+  chmod +x "${stub_dir}/aws"
+  shift 2
+  while [ "$#" -gt 0 ]; do
+    printf '%s\n' "$2" > "${stub_dir}/behavior-$1"
+    shift 2
+  done
+  set +e
+  IMAGE_NAME=actions-runner \
+    SHA=0123456789abcdef0123456789abcdef01234567 \
+    PROMOTE_TAG=current \
+    DIGEST_RETRY_SLEEP=0 \
+    AWS_STUB_DIR="${stub_dir}" \
+    PATH="${stub_dir}:${PATH}" \
+    bash "${verifier}" >"${stub_dir}/out" 2>"${stub_dir}/err"
+  status=$?
+  set -e
+  if [ "${status}" -ne "${expect_status}" ]; then
+    echo "${name}: exit ${status}, want ${expect_status}" >&2
+    cat "${stub_dir}/err" >&2
+    failures=$((failures + 1))
+  fi
+  sha_calls=0
+  promote_calls=0
+  if [ -f "${stub_dir}/count-sha-0123456789abcdef0123456789abcdef01234567" ]; then
+    sha_calls="$(cat "${stub_dir}/count-sha-0123456789abcdef0123456789abcdef01234567")"
+  fi
+  if [ -f "${stub_dir}/count-current" ]; then
+    promote_calls="$(cat "${stub_dir}/count-current")"
+  fi
+  printf '%s\n' "${sha_calls}" > "${stub_dir}/sha_calls"
+  printf '%s\n' "${promote_calls}" > "${stub_dir}/promote_calls"
+  # shellcheck disable=SC2034
+  CASE_DIR="${stub_dir}"
+}
+
+run_case "match on first read" 0 \
+  "sha-0123456789abcdef0123456789abcdef01234567" "ok sha256:aaa" \
+  "current" "ok sha256:aaa"
+assert_eq "match on first read sha calls" "$(cat "${CASE_DIR}/sha_calls")" "1"
+assert_eq "match on first read promote calls" "$(cat "${CASE_DIR}/promote_calls")" "1"
+rm -rf "${CASE_DIR}"
+
+run_case "retry when promote tag is not visible yet" 0 \
+  "sha-0123456789abcdef0123456789abcdef01234567" "ok sha256:aaa" \
+  "current" "$(printf 'fail\nok sha256:aaa')"
+assert_eq "not visible yet sha calls" "$(cat "${CASE_DIR}/sha_calls")" "2"
+assert_eq "not visible yet promote calls" "$(cat "${CASE_DIR}/promote_calls")" "2"
+rm -rf "${CASE_DIR}"
+
+run_case "give up when describe-images keeps failing" 1 \
+  "sha-0123456789abcdef0123456789abcdef01234567" "fail" \
+  "current" "fail"
+assert_eq "keep failing sha calls" "$(cat "${CASE_DIR}/sha_calls")" "6"
+assert_eq "keep failing promote calls" "$(cat "${CASE_DIR}/promote_calls")" "6"
+rm -rf "${CASE_DIR}"
+
+if [ "${failures}" -ne 0 ]; then
+  echo "${failures} assertion(s) failed" >&2
+  exit 1
+fi
+echo "verify_ecr_promote_digest: ok"