From 301ca7627c43b7fb216a0d6c7a1af7478b11fca5 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Tue, 6 Oct 2026 14:16:18 -0400 Subject: [PATCH] fix(ci): retry ECR digest checks when describe-images fails A tag that is not visible yet makes the AWS CLI exit non-zero, and set -e was aborting the retry loop on that first error. --- .github/actionlint.yaml | 3 + .../verify-ecr-promote-digest/action.yml | 25 ++++ .../verify-ecr-promote-digest/verify.sh | 44 +++++++ .github/workflows/cd-ecr-image.yaml | 36 ++--- .github/workflows/ci.yaml | 9 +- README.md | 2 +- scripts/test_verify_ecr_promote_digest.sh | 124 ++++++++++++++++++ 7 files changed, 212 insertions(+), 31 deletions(-) create mode 100644 .github/actions/verify-ecr-promote-digest/action.yml create mode 100755 .github/actions/verify-ecr-promote-digest/verify.sh create mode 100755 scripts/test_verify_ecr_promote_digest.sh diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index ce3828a..07cdb0d 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -4,3 +4,6 @@ paths: .github/workflows/ci-terraform.yaml: ignore: - 'specifying action "\$/.github/actions/app-terraform-isolation" in invalid format because ref is missing' + .github/workflows/cd-ecr-image.yaml: + ignore: + - 'specifying action "\$/.github/actions/verify-ecr-promote-digest" in invalid format because ref is missing' diff --git a/.github/actions/verify-ecr-promote-digest/action.yml b/.github/actions/verify-ecr-promote-digest/action.yml new file mode 100644 index 0000000..20bdea7 --- /dev/null +++ b/.github/actions/verify-ecr-promote-digest/action.yml @@ -0,0 +1,25 @@ +name: Verify ECR promote digest +description: Fail unless the promote tag and the sha tag have the same ECR image digest. + +inputs: + image-name: + description: ECR repository name. + required: true + sha: + description: Full commit SHA used in the sha- tag. + required: true + promote-tag: + description: Mutable tag that must match the commit tag. + required: true + +runs: + using: composite + steps: + - name: Compare tag digests + shell: bash + env: + IMAGE_NAME: ${{ inputs.image-name }} + SHA: ${{ inputs.sha }} + PROMOTE_TAG: ${{ inputs.promote-tag }} + VERIFIER: ${{ github.action_path }}/verify.sh + run: bash "${VERIFIER}" diff --git a/.github/actions/verify-ecr-promote-digest/verify.sh b/.github/actions/verify-ecr-promote-digest/verify.sh new file mode 100755 index 0000000..562f988 --- /dev/null +++ b/.github/actions/verify-ecr-promote-digest/verify.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Confirm promote-tag and sha- point at the same ECR digest. +# describe-images can error while a tag just pushed is not yet visible, so +# a failed call is a retry, not an immediate abort. +set -euo pipefail + +: "${IMAGE_NAME:?image-name is required}" +: "${SHA:?sha is required}" +: "${PROMOTE_TAG:?promote-tag is required}" + +retry_sleep="${DIGEST_RETRY_SLEEP:-5}" +sha_tag="sha-${SHA}" + +query_digest() { + local tag="$1" + local err digest + err="$(mktemp)" + if digest="$(aws ecr describe-images \ + --repository-name "${IMAGE_NAME}" \ + --image-ids "imageTag=${tag}" \ + --query 'imageDetails[0].imageDigest' \ + --output text 2>"${err}")"; then + rm -f "${err}" + printf '%s\n' "${digest}" + return 0 + fi + echo "describe-images ${tag}: $(tr '\n' ' ' < "${err}")" >&2 + rm -f "${err}" + return 1 +} + +for _ in 1 2 3 4 5 6; do + sha_digest="$(query_digest "${sha_tag}")" || sha_digest="" + promote_digest="$(query_digest "${PROMOTE_TAG}")" || promote_digest="" + if [ "${sha_digest}" = "${promote_digest}" ] && [ -n "${sha_digest}" ] && [ "${sha_digest}" != "None" ]; then + echo "promote tag ${PROMOTE_TAG} digest matches ${sha_tag}: ${sha_digest}" + exit 0 + fi + echo "digest mismatch (sha=${sha_digest:-empty} promote=${promote_digest:-empty}); retrying" + sleep "${retry_sleep}" +done + +echo "promote tag ${PROMOTE_TAG} digest does not match ${sha_tag}" >&2 +exit 1 diff --git a/.github/workflows/cd-ecr-image.yaml b/.github/workflows/cd-ecr-image.yaml index 89b29af..41b6159 100644 --- a/.github/workflows/cd-ecr-image.yaml +++ b/.github/workflows/cd-ecr-image.yaml @@ -9,7 +9,9 @@ name: CD — ECR image # tag. Nothing here updates ECS, Lambda, or an HCP run. # # dockerfile is relative to context. After the promote push, DescribeImages -# must show the same digest on promote-tag and sha-. +# must show the same digest on promote-tag and sha-. That check +# retries when the API errors or the digests differ, including while the +# promote tag is not yet visible. # # Caller example: # jobs: @@ -290,30 +292,8 @@ jobs: docker push "${promoted}" - name: Verify promoted digest - env: - IMAGE_NAME: ${{ inputs.image-name }} - SHA: ${{ steps.commit.outputs.sha }} - PROMOTE_TAG: ${{ inputs.promote-tag }} - run: | - set -euo pipefail - sha_tag="sha-${SHA}" - for _ in 1 2 3 4 5 6; do - sha_digest="$(aws ecr describe-images \ - --repository-name "${IMAGE_NAME}" \ - --image-ids "imageTag=${sha_tag}" \ - --query 'imageDetails[0].imageDigest' \ - --output text)" - promote_digest="$(aws ecr describe-images \ - --repository-name "${IMAGE_NAME}" \ - --image-ids "imageTag=${PROMOTE_TAG}" \ - --query 'imageDetails[0].imageDigest' \ - --output text)" - if [ "${sha_digest}" = "${promote_digest}" ] && [ -n "${sha_digest}" ] && [ "${sha_digest}" != "None" ]; then - echo "promote tag ${PROMOTE_TAG} digest matches ${sha_tag}: ${sha_digest}" - exit 0 - fi - echo "digest mismatch (sha=${sha_digest:-empty} promote=${promote_digest:-empty}); retrying" - sleep 5 - done - echo "promote tag ${PROMOTE_TAG} digest does not match ${sha_tag}" >&2 - exit 1 + uses: $/.github/actions/verify-ecr-promote-digest + with: + image-name: ${{ inputs.image-name }} + sha: ${{ steps.commit.outputs.sha }} + promote-tag: ${{ inputs.promote-tag }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 60c1150..7f9ec16 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,9 @@ name: ci # "Expected — Waiting for status to be reported" and could not merge. # # The portions are genuinely useful CI for a repo whose whole product is -# GitHub Actions YAML: the isolation-checker unit tests and actionlint over -# every workflow file. They run in parallel; `ci-complete` requires both. +# GitHub Actions YAML: isolation-tests (the isolation checker and the ECR +# promote digest verifier) and actionlint over every workflow file. They +# run in parallel; `ci-complete` requires both. # # Naming is load-bearing: the ruleset matches the required status check against # the JOB's check-run name, NOT "workflow / job". For a normal (non-reusable) job @@ -59,6 +60,10 @@ jobs: run: python3 scripts/test_check_app_terraform_isolation.py shell: bash + - name: ECR promote digest verifier tests + run: bash scripts/test_verify_ecr_promote_digest.sh + shell: bash + actionlint: name: actionlint runs-on: ubuntu-latest diff --git a/README.md b/README.md index 88b277c..0bb5c35 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The formatter GitHub App is not on the main-branch bypass list. **`.github/workflows/cd-hcp-fargate.yaml`** — HCP Fargate image CD. Checkout at `ref` (empty means `github.sha`), OIDC, SSM cluster/service/family/ecr/container/api-url, docker build+push tagged `$sha` and `$environment`, patch `GIT_SHA`, RegisterTaskDefinition + UpdateService + services-stable, poll health SHA. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency, OIDC, and `vars.DEPLOY_ROLE_ARN`. -**`.github/workflows/cd-ecr-image.yaml`** — ECR image publish with no ECS, Lambda, or HCP update. Checkout at `ref` (empty means `github.sha`), optional `ship-gate`, OIDC via `vars.DEPLOY_ROLE_ARN`, `docker build` and push `sha-`, optional `smoke-script` inside that image, then retag and push `promote-tag` (default `current`). A failed smoke does not move the promote tag. `ecr:DescribeImages` must show the same digest on both tags. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency `deploy--`, and OIDC. +**`.github/workflows/cd-ecr-image.yaml`** — ECR image publish with no ECS, Lambda, or HCP update. Checkout at `ref` (empty means `github.sha`), optional `ship-gate`, OIDC via `vars.DEPLOY_ROLE_ARN`, `docker build` and push `sha-`, optional `smoke-script` inside that image, then retag and push `promote-tag` (default `current`). A failed smoke does not move the promote tag. `ecr:DescribeImages` must show the same digest on both tags. That check retries when the call errors or the digests differ. `environment` is a `with:` input. The reusable job owns `environment:`, concurrency `deploy--`, and OIDC. **`.github/workflows/cd-hcp-lambda.yaml`** — HCP Lambda zip CD for Node packagers. Checkout at `ref`, optional `ship-gate`, Node 24, `scripts/package_lambdas.mjs --git-sha --out-dir --only `, verifies `src/buildInfo.js` carries the SHA, uploads `functions//.zip` to the SSM `artifacts-bucket`, `update-function-code` on each `-function-name`, waits for `function-updated-v2`. Terraform owns the functions and ignores code attributes. diff --git a/scripts/test_verify_ecr_promote_digest.sh b/scripts/test_verify_ecr_promote_digest.sh new file mode 100755 index 0000000..e1712d3 --- /dev/null +++ b/scripts/test_verify_ecr_promote_digest.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# The verifier must retry a failed describe-images call. set -e used to abort +# the loop on the first ImageNotFoundException. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd)" +verifier="${root}/.github/actions/verify-ecr-promote-digest/verify.sh" +failures=0 + +assert_eq() { + local name="$1" + local got="$2" + local want="$3" + if [ "${got}" != "${want}" ]; then + echo "${name}: got ${got}, want ${want}" >&2 + failures=$((failures + 1)) + fi +} + +run_case() { + local name="$1" + local expect_status="$2" + local stub_dir + stub_dir="$(mktemp -d)" + cat > "${stub_dir}/aws" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +tag="" +for arg in "$@"; do + case "${arg}" in + imageTag=*) tag="${arg#imageTag=}" ;; + esac +done +if [ -z "${tag}" ]; then + echo "stub aws: missing image tag" >&2 + exit 1 +fi +dir="${AWS_STUB_DIR}" +count_file="${dir}/count-${tag}" +n=0 +if [ -f "${count_file}" ]; then + n="$(cat "${count_file}")" +fi +n=$((n + 1)) +printf '%s\n' "${n}" > "${count_file}" +line="$(sed -n "${n}p" "${dir}/behavior-${tag}" || true)" +if [ -z "${line}" ]; then + line="$(tail -n 1 "${dir}/behavior-${tag}")" +fi +case "${line}" in + ok\ *) + printf '%s\n' "${line#ok }" + ;; + fail) + echo "ImageNotFoundException: ${tag}" >&2 + exit 254 + ;; + *) + echo "stub aws: no behavior for ${tag} call ${n}" >&2 + exit 1 + ;; +esac +EOF + chmod +x "${stub_dir}/aws" + shift 2 + while [ "$#" -gt 0 ]; do + printf '%s\n' "$2" > "${stub_dir}/behavior-$1" + shift 2 + done + set +e + IMAGE_NAME=actions-runner \ + SHA=0123456789abcdef0123456789abcdef01234567 \ + PROMOTE_TAG=current \ + DIGEST_RETRY_SLEEP=0 \ + AWS_STUB_DIR="${stub_dir}" \ + PATH="${stub_dir}:${PATH}" \ + bash "${verifier}" >"${stub_dir}/out" 2>"${stub_dir}/err" + status=$? + set -e + if [ "${status}" -ne "${expect_status}" ]; then + echo "${name}: exit ${status}, want ${expect_status}" >&2 + cat "${stub_dir}/err" >&2 + failures=$((failures + 1)) + fi + sha_calls=0 + promote_calls=0 + if [ -f "${stub_dir}/count-sha-0123456789abcdef0123456789abcdef01234567" ]; then + sha_calls="$(cat "${stub_dir}/count-sha-0123456789abcdef0123456789abcdef01234567")" + fi + if [ -f "${stub_dir}/count-current" ]; then + promote_calls="$(cat "${stub_dir}/count-current")" + fi + printf '%s\n' "${sha_calls}" > "${stub_dir}/sha_calls" + printf '%s\n' "${promote_calls}" > "${stub_dir}/promote_calls" + # shellcheck disable=SC2034 + CASE_DIR="${stub_dir}" +} + +run_case "match on first read" 0 \ + "sha-0123456789abcdef0123456789abcdef01234567" "ok sha256:aaa" \ + "current" "ok sha256:aaa" +assert_eq "match on first read sha calls" "$(cat "${CASE_DIR}/sha_calls")" "1" +assert_eq "match on first read promote calls" "$(cat "${CASE_DIR}/promote_calls")" "1" +rm -rf "${CASE_DIR}" + +run_case "retry when promote tag is not visible yet" 0 \ + "sha-0123456789abcdef0123456789abcdef01234567" "ok sha256:aaa" \ + "current" "$(printf 'fail\nok sha256:aaa')" +assert_eq "not visible yet sha calls" "$(cat "${CASE_DIR}/sha_calls")" "2" +assert_eq "not visible yet promote calls" "$(cat "${CASE_DIR}/promote_calls")" "2" +rm -rf "${CASE_DIR}" + +run_case "give up when describe-images keeps failing" 1 \ + "sha-0123456789abcdef0123456789abcdef01234567" "fail" \ + "current" "fail" +assert_eq "keep failing sha calls" "$(cat "${CASE_DIR}/sha_calls")" "6" +assert_eq "keep failing promote calls" "$(cat "${CASE_DIR}/promote_calls")" "6" +rm -rf "${CASE_DIR}" + +if [ "${failures}" -ne 0 ]; then + echo "${failures} assertion(s) failed" >&2 + exit 1 +fi +echo "verify_ecr_promote_digest: ok"