fix(ci): retry ECR digest checks when describe-images fails

A tag that is not visible yet makes the AWS CLI exit non-zero, and set -e
was aborting the retry loop on that first error.

.github/actionlint.yaml

@@ -4,3 +4,6 @@ paths:
   .github/workflows/ci-terraform.yaml:
     ignore:
       - 'specifying action "\$/.github/actions/app-terraform-isolation" in invalid format because ref is missing'
+  .github/workflows/cd-ecr-image.yaml:
+    ignore:
+      - 'specifying action "\$/.github/actions/verify-ecr-promote-digest" in invalid format because ref is missing'

.github/actions/verify-ecr-promote-digest/action.yml

@@ -0,0 +1,25 @@
+name: Verify ECR promote digest
+description: Fail unless the promote tag and the sha tag have the same ECR image digest.
+
+inputs:
+  image-name:
+    description: ECR repository name.
+    required: true
+  sha:
+    description: Full commit SHA used in the sha-<commit> tag.
+    required: true
+  promote-tag:
+    description: Mutable tag that must match the commit tag.
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Compare tag digests
+      shell: bash
+      env:
+        IMAGE_NAME: ${{ inputs.image-name }}
+        SHA: ${{ inputs.sha }}
+        PROMOTE_TAG: ${{ inputs.promote-tag }}
+        VERIFIER: ${{ github.action_path }}/verify.sh
+      run: bash "${VERIFIER}"

.github/actions/verify-ecr-promote-digest/verify.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Confirm promote-tag and sha-<commit> point at the same ECR digest.
+# describe-images can error while a tag just pushed is not yet visible, so
+# a failed call is a retry, not an immediate abort.
+set -euo pipefail
+
+: "${IMAGE_NAME:?image-name is required}"
+: "${SHA:?sha is required}"
+: "${PROMOTE_TAG:?promote-tag is required}"
+
+retry_sleep="${DIGEST_RETRY_SLEEP:-5}"
+sha_tag="sha-${SHA}"
+
+query_digest() {
+  local tag="$1"
+  local err digest
+  err="$(mktemp)"
+  if digest="$(aws ecr describe-images \
+    --repository-name "${IMAGE_NAME}" \
+    --image-ids "imageTag=${tag}" \
+    --query 'imageDetails[0].imageDigest' \
+    --output text 2>"${err}")"; then
+    rm -f "${err}"
+    printf '%s\n' "${digest}"
+    return 0
+  fi
+  echo "describe-images ${tag}: $(tr '\n' ' ' < "${err}")" >&2
+  rm -f "${err}"
+  return 1
+}
+
+for _ in 1 2 3 4 5 6; do
+  sha_digest="$(query_digest "${sha_tag}")" || sha_digest=""
+  promote_digest="$(query_digest "${PROMOTE_TAG}")" || promote_digest=""
+  if [ "${sha_digest}" = "${promote_digest}" ] && [ -n "${sha_digest}" ] && [ "${sha_digest}" != "None" ]; then
+    echo "promote tag ${PROMOTE_TAG} digest matches ${sha_tag}: ${sha_digest}"
+    exit 0
+  fi
+  echo "digest mismatch (sha=${sha_digest:-empty} promote=${promote_digest:-empty}); retrying"
+  sleep "${retry_sleep}"
+done
+
+echo "promote tag ${PROMOTE_TAG} digest does not match ${sha_tag}" >&2
+exit 1

.github/workflows/cd-ecr-image.yaml

@@ -9,7 +9,9 @@ name: CD — ECR image
 # tag. Nothing here updates ECS, Lambda, or an HCP run.
 #
 # dockerfile is relative to context. After the promote push, DescribeImages
-# must show the same digest on promote-tag and sha-<commit>.
+# must show the same digest on promote-tag and sha-<commit>. That check
+# retries when the API errors or the digests differ, including while the
+# promote tag is not yet visible.
 #
 # Caller example:
 #   jobs:
@@ -290,30 +292,8 @@ jobs:
           docker push "${promoted}"
 
       - name: Verify promoted digest
-        env:
-          IMAGE_NAME: ${{ inputs.image-name }}
-          SHA: ${{ steps.commit.outputs.sha }}
-          PROMOTE_TAG: ${{ inputs.promote-tag }}
-        run: |
-          set -euo pipefail
-          sha_tag="sha-${SHA}"
-          for _ in 1 2 3 4 5 6; do
-            sha_digest="$(aws ecr describe-images \
-              --repository-name "${IMAGE_NAME}" \
-              --image-ids "imageTag=${sha_tag}" \
-              --query 'imageDetails[0].imageDigest' \
-              --output text)"
-            promote_digest="$(aws ecr describe-images \
-              --repository-name "${IMAGE_NAME}" \
-              --image-ids "imageTag=${PROMOTE_TAG}" \
-              --query 'imageDetails[0].imageDigest' \
-              --output text)"
-            if [ "${sha_digest}" = "${promote_digest}" ] && [ -n "${sha_digest}" ] && [ "${sha_digest}" != "None" ]; then
-              echo "promote tag ${PROMOTE_TAG} digest matches ${sha_tag}: ${sha_digest}"
-              exit 0
-            fi
-            echo "digest mismatch (sha=${sha_digest:-empty} promote=${promote_digest:-empty}); retrying"
-            sleep 5
-          done
-          echo "promote tag ${PROMOTE_TAG} digest does not match ${sha_tag}" >&2
-          exit 1
+        uses: $/.github/actions/verify-ecr-promote-digest
+        with:
+          image-name: ${{ inputs.image-name }}
+          sha: ${{ steps.commit.outputs.sha }}
+          promote-tag: ${{ inputs.promote-tag }}

.github/workflows/ci.yaml

@@ -11,8 +11,9 @@ name: ci
 # "Expected — Waiting for status to be reported" and could not merge.
 #
 # The portions are genuinely useful CI for a repo whose whole product is
-# GitHub Actions YAML: the isolation-checker unit tests and actionlint over
-# every workflow file. They run in parallel; `ci-complete` requires both.
+# GitHub Actions YAML: isolation-tests (the isolation checker and the ECR
+# promote digest verifier) and actionlint over every workflow file. They
+# run in parallel; `ci-complete` requires both.
 #
 # Naming is load-bearing: the ruleset matches the required status check against
 # the JOB's check-run name, NOT "workflow / job". For a normal (non-reusable) job
@@ -59,6 +60,10 @@ jobs:
         run: python3 scripts/test_check_app_terraform_isolation.py
         shell: bash
 
+      - name: ECR promote digest verifier tests
+        run: bash scripts/test_verify_ecr_promote_digest.sh
+        shell: bash
+
   actionlint:
     name: actionlint
     runs-on: ubuntu-latest

README.md

@@ -61,7 +61,7 @@ The formatter GitHub App is not on the main-branch bypass list.
 
 **`.github/workflows/cd-hcp-fargate.yaml`** — HCP Fargate image CD. Checkout at `ref` (empty means `github.sha`), OIDC, SSM cluster/service/family/ecr/container/api-url, docker build+push tagged `$sha` and `$environment`, patch `GIT_SHA`, RegisterTaskDefinition + UpdateService + services-stable, poll health SHA. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency, OIDC, and `vars.DEPLOY_ROLE_ARN`.
 
-**`.github/workflows/cd-ecr-image.yaml`** — ECR image publish with no ECS, Lambda, or HCP update. Checkout at `ref` (empty means `github.sha`), optional `ship-gate`, OIDC via `vars.DEPLOY_ROLE_ARN`, `docker build` and push `sha-<commit>`, optional `smoke-script` inside that image, then retag and push `promote-tag` (default `current`). A failed smoke does not move the promote tag. `ecr:DescribeImages` must show the same digest on both tags. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency `deploy-<image-name>-<environment>`, and OIDC.
+**`.github/workflows/cd-ecr-image.yaml`** — ECR image publish with no ECS, Lambda, or HCP update. Checkout at `ref` (empty means `github.sha`), optional `ship-gate`, OIDC via `vars.DEPLOY_ROLE_ARN`, `docker build` and push `sha-<commit>`, optional `smoke-script` inside that image, then retag and push `promote-tag` (default `current`). A failed smoke does not move the promote tag. `ecr:DescribeImages` must show the same digest on both tags. That check retries when the call errors or the digests differ. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency `deploy-<image-name>-<environment>`, and OIDC.
 
 **`.github/workflows/cd-hcp-lambda.yaml`** — HCP Lambda zip CD for Node packagers. Checkout at `ref`, optional `ship-gate`, Node 24, `scripts/package_lambdas.mjs --git-sha --out-dir --only <key>`, verifies `src/buildInfo.js` carries the SHA, uploads `functions/<key>/<sha>.zip` to the SSM `artifacts-bucket`, `update-function-code` on each `<key>-function-name`, waits for `function-updated-v2`. Terraform owns the functions and ignores code attributes.
 

scripts/test_verify_ecr_promote_digest.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# The verifier must retry a failed describe-images call. set -e used to abort
+# the loop on the first ImageNotFoundException.
+set -euo pipefail
+
+root="$(cd "$(dirname "$0")/.." && pwd)"
+verifier="${root}/.github/actions/verify-ecr-promote-digest/verify.sh"
+failures=0
+
+assert_eq() {
+  local name="$1"
+  local got="$2"
+  local want="$3"
+  if [ "${got}" != "${want}" ]; then
+    echo "${name}: got ${got}, want ${want}" >&2
+    failures=$((failures + 1))
+  fi
+}
+
+run_case() {
+  local name="$1"
+  local expect_status="$2"
+  local stub_dir
+  stub_dir="$(mktemp -d)"
+  cat > "${stub_dir}/aws" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+tag=""
+for arg in "$@"; do
+  case "${arg}" in
+    imageTag=*) tag="${arg#imageTag=}" ;;
+  esac
+done
+if [ -z "${tag}" ]; then
+  echo "stub aws: missing image tag" >&2
+  exit 1
+fi
+dir="${AWS_STUB_DIR}"
+count_file="${dir}/count-${tag}"
+n=0
+if [ -f "${count_file}" ]; then
+  n="$(cat "${count_file}")"
+fi
+n=$((n + 1))
+printf '%s\n' "${n}" > "${count_file}"
+line="$(sed -n "${n}p" "${dir}/behavior-${tag}" || true)"
+if [ -z "${line}" ]; then
+  line="$(tail -n 1 "${dir}/behavior-${tag}")"
+fi
+case "${line}" in
+  ok\ *)
+    printf '%s\n' "${line#ok }"
+    ;;
+  fail)
+    echo "ImageNotFoundException: ${tag}" >&2
+    exit 254
+    ;;
+  *)
+    echo "stub aws: no behavior for ${tag} call ${n}" >&2
+    exit 1
+    ;;
+esac
+EOF
+  chmod +x "${stub_dir}/aws"
+  shift 2
+  while [ "$#" -gt 0 ]; do
+    printf '%s\n' "$2" > "${stub_dir}/behavior-$1"
+    shift 2
+  done
+  set +e
+  IMAGE_NAME=actions-runner \
+    SHA=0123456789abcdef0123456789abcdef01234567 \
+    PROMOTE_TAG=current \
+    DIGEST_RETRY_SLEEP=0 \
+    AWS_STUB_DIR="${stub_dir}" \
+    PATH="${stub_dir}:${PATH}" \
+    bash "${verifier}" >"${stub_dir}/out" 2>"${stub_dir}/err"
+  status=$?
+  set -e
+  if [ "${status}" -ne "${expect_status}" ]; then
+    echo "${name}: exit ${status}, want ${expect_status}" >&2
+    cat "${stub_dir}/err" >&2
+    failures=$((failures + 1))
+  fi
+  sha_calls=0
+  promote_calls=0
+  if [ -f "${stub_dir}/count-sha-0123456789abcdef0123456789abcdef01234567" ]; then
+    sha_calls="$(cat "${stub_dir}/count-sha-0123456789abcdef0123456789abcdef01234567")"
+  fi
+  if [ -f "${stub_dir}/count-current" ]; then
+    promote_calls="$(cat "${stub_dir}/count-current")"
+  fi
+  printf '%s\n' "${sha_calls}" > "${stub_dir}/sha_calls"
+  printf '%s\n' "${promote_calls}" > "${stub_dir}/promote_calls"
+  # shellcheck disable=SC2034
+  CASE_DIR="${stub_dir}"
+}
+
+run_case "match on first read" 0 \
+  "sha-0123456789abcdef0123456789abcdef01234567" "ok sha256:aaa" \
+  "current" "ok sha256:aaa"
+assert_eq "match on first read sha calls" "$(cat "${CASE_DIR}/sha_calls")" "1"
+assert_eq "match on first read promote calls" "$(cat "${CASE_DIR}/promote_calls")" "1"
+rm -rf "${CASE_DIR}"
+
+run_case "retry when promote tag is not visible yet" 0 \
+  "sha-0123456789abcdef0123456789abcdef01234567" "ok sha256:aaa" \
+  "current" "$(printf 'fail\nok sha256:aaa')"
+assert_eq "not visible yet sha calls" "$(cat "${CASE_DIR}/sha_calls")" "2"
+assert_eq "not visible yet promote calls" "$(cat "${CASE_DIR}/promote_calls")" "2"
+rm -rf "${CASE_DIR}"
+
+run_case "give up when describe-images keeps failing" 1 \
+  "sha-0123456789abcdef0123456789abcdef01234567" "fail" \
+  "current" "fail"
+assert_eq "keep failing sha calls" "$(cat "${CASE_DIR}/sha_calls")" "6"
+assert_eq "keep failing promote calls" "$(cat "${CASE_DIR}/promote_calls")" "6"
+rm -rf "${CASE_DIR}"
+
+if [ "${failures}" -ne 0 ]; then
+  echo "${failures} assertion(s) failed" >&2
+  exit 1
+fi
+echo "verify_ecr_promote_digest: ok"