mirror of
https://github.com/Sea-Haven-Industries/shoc-backend.git
synced 2026-09-30 08:23:12 +00:00
d1e3fb03ce
Elastic Beanstalk reports Ready as soon as a rollout finishes, before enhanced health has converged. Both verification gates decided on the first Ready poll, so a release whose version had activated correctly was failed on a health value that had not settled yet — and then rolled back. The failure message compounded it: run 34293894914 printed 'Environment became Ready without activating expected version a0fdd199...' when the active version was exactly a0fdd199... The discriminator was health, not version, which sends whoever reads the log after the wrong problem. Separate the two conditions, keep polling while the correct version is active but health has not settled, and report the last observed state on timeout. An environment that stays unhealthy for the full window still fails; this does not widen what counts as a good deploy.
858 lines
35 KiB
YAML
858 lines
35 KiB
YAML
name: Validate and deploy
|
|
|
|
on:
|
|
pull_request:
|
|
branches: [dev, staging, main]
|
|
push:
|
|
branches: [dev]
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
validate:
|
|
name: Validate deployable source bundle
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Set up .NET
|
|
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
|
with:
|
|
dotnet-version: "8.0.x"
|
|
|
|
- name: Repository quality gate
|
|
env:
|
|
BASE_REF: ${{ github.event.pull_request.base.sha || 'origin/dev' }}
|
|
HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }}
|
|
run: bash scripts/governance-check.sh
|
|
|
|
- name: Build Elastic Beanstalk source bundle
|
|
run: bash scripts/package-elastic-beanstalk.sh
|
|
|
|
- name: Inspect source bundle contract
|
|
run: bash scripts/validate-elastic-beanstalk-bundle.sh
|
|
|
|
deploy-dev:
|
|
name: Deploy shoc-backend-dev through Terraform
|
|
if: >
|
|
(github.event_name == 'push' && github.ref == 'refs/heads/dev' &&
|
|
vars.TERRAFORM_APP_CD_ENABLED == 'true') ||
|
|
(github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/dev')
|
|
needs: validate
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 180
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
environment:
|
|
name: dev
|
|
concurrency:
|
|
group: deploy-dev
|
|
cancel-in-progress: false
|
|
env:
|
|
TF_CLOUD_ORGANIZATION: seahaven
|
|
TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}
|
|
EB_APPLICATION_NAME: shoc-backend
|
|
EB_ENVIRONMENT_NAME: shoc-backend-dev
|
|
SMOKE_URL: https://api.dev.seahaven.com
|
|
EB_BUCKET: elasticbeanstalk-us-east-1-396287094661
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
|
|
- name: Set up .NET
|
|
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
|
with:
|
|
dotnet-version: "8.0.x"
|
|
|
|
- name: Build Elastic Beanstalk source bundle
|
|
run: bash scripts/package-elastic-beanstalk.sh
|
|
|
|
- name: Validate exact release bundle
|
|
run: bash scripts/validate-elastic-beanstalk-bundle.sh
|
|
|
|
- name: Configure AWS credentials (OIDC)
|
|
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
|
|
with:
|
|
role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
|
|
aws-region: us-east-1
|
|
audience: sts.amazonaws.com
|
|
|
|
- name: Capture current environment version
|
|
run: |
|
|
set -euo pipefail
|
|
prev="$(aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].VersionLabel' \
|
|
--output text)"
|
|
echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt
|
|
echo "Previous version label: $prev"
|
|
|
|
- name: Assign immutable release identity
|
|
id: release
|
|
run: |
|
|
set -euo pipefail
|
|
version_label="${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
s3_key="shoc-backend/releases/dev/${GITHUB_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip"
|
|
{
|
|
echo "version_label=${version_label}"
|
|
echo "s3_key=${s3_key}"
|
|
} >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Upload immutable bundle
|
|
run: |
|
|
set -euo pipefail
|
|
aws s3 cp .artifacts/elastic-beanstalk/site.zip \
|
|
"s3://${EB_BUCKET}/${{ steps.release.outputs.s3_key }}" \
|
|
--region us-east-1
|
|
|
|
- name: Create Elastic Beanstalk application version
|
|
run: |
|
|
set -euo pipefail
|
|
aws elasticbeanstalk create-application-version \
|
|
--application-name "${EB_APPLICATION_NAME}" \
|
|
--version-label "${{ steps.release.outputs.version_label }}" \
|
|
--description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" \
|
|
--source-bundle "S3Bucket=${EB_BUCKET},S3Key=${{ steps.release.outputs.s3_key }}" \
|
|
--process \
|
|
--region us-east-1
|
|
|
|
status="UNPROCESSED"
|
|
for _ in $(seq 1 36); do
|
|
status="$(aws elasticbeanstalk describe-application-versions \
|
|
--application-name "${EB_APPLICATION_NAME}" \
|
|
--version-labels "${{ steps.release.outputs.version_label }}" \
|
|
--region us-east-1 \
|
|
--query 'ApplicationVersions[0].Status' \
|
|
--output text)"
|
|
echo "application version status: $status"
|
|
if [ "$status" = "PROCESSED" ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$status" = "FAILED" ]; then
|
|
echo "Elastic Beanstalk failed to process ${{ steps.release.outputs.version_label }}." >&2
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
echo "Application version did not become PROCESSED." >&2
|
|
exit 1
|
|
|
|
- name: Discard blocking VCS run before GitHub CD
|
|
run: |
|
|
set -euo pipefail
|
|
python3 << 'PY'
|
|
import json, os, urllib.error, urllib.request
|
|
|
|
token = os.environ["TF_API_TOKEN"]
|
|
workspace = "shoc-backend-dev"
|
|
headers = {
|
|
"Authorization": f"Bearer {token}",
|
|
"Content-Type": "application/vnd.api+json",
|
|
}
|
|
|
|
def get(url):
|
|
req = urllib.request.Request(url, headers=headers)
|
|
with urllib.request.urlopen(req) as resp:
|
|
return json.load(resp)
|
|
|
|
def post(url, payload):
|
|
data = json.dumps(payload).encode()
|
|
req = urllib.request.Request(
|
|
url, data=data, method="POST", headers=headers
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req) as resp:
|
|
return resp.status
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code in (409, 404):
|
|
body = exc.read().decode("utf-8", "replace")
|
|
print(f"discard returned HTTP {exc.code}: {body}")
|
|
return exc.code
|
|
raise
|
|
|
|
ws = get(
|
|
f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
|
|
)["data"]
|
|
attrs = ws["attributes"]
|
|
if attrs.get("auto-apply") is True:
|
|
raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue")
|
|
if not attrs.get("speculative-enabled"):
|
|
raise SystemExit("speculative plans are off; refuse to continue")
|
|
if (attrs.get("vcs-repo") or {}).get("tags-regex"):
|
|
raise SystemExit("tag-based VCS triggering is set; refuse to continue")
|
|
expected_patterns = [
|
|
"terraform/live/dev/**",
|
|
"terraform/live/modules/**",
|
|
]
|
|
if attrs.get("trigger-patterns") != expected_patterns:
|
|
raise SystemExit(
|
|
"trigger-patterns must be "
|
|
f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
|
|
)
|
|
if not attrs.get("locked"):
|
|
print("workspace is unlocked")
|
|
raise SystemExit(0)
|
|
|
|
current = (
|
|
ws.get("relationships", {})
|
|
.get("current-run", {})
|
|
.get("data")
|
|
)
|
|
if not current:
|
|
raise SystemExit("workspace is locked without a current run")
|
|
run_id = current["id"]
|
|
run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
|
|
run_attrs = run["attributes"]
|
|
status = run_attrs.get("status")
|
|
plan_only = run_attrs.get("plan-only")
|
|
print(f"current run {run_id} status={status} plan-only={plan_only}")
|
|
if plan_only:
|
|
print("speculative run does not block GitHub CD")
|
|
raise SystemExit(0)
|
|
if status in {"applying", "apply_queued"}:
|
|
raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
|
|
discardable = {
|
|
"pending", "planned", "cost_estimated", "policy_checked", "policy_override"
|
|
}
|
|
if status not in discardable:
|
|
raise SystemExit(f"{run_id} status {status} is not discardable")
|
|
code = post(
|
|
f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
|
|
{"comment": "Discarded so GitHub CD can create the version-only applyable run"},
|
|
)
|
|
print(f"discarded {run_id} http={code}")
|
|
PY
|
|
|
|
- name: Create Terraform release run
|
|
id: release-run
|
|
uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
env:
|
|
TF_VAR_release_version_label: '"${{ steps.release.outputs.version_label }}"'
|
|
with:
|
|
workspace: shoc-backend-dev
|
|
message: "Release ${{ steps.release.outputs.version_label }} from GitHub Actions"
|
|
|
|
- name: Read Terraform release plan counts
|
|
id: release-plan
|
|
uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
with:
|
|
plan: ${{ steps.release-run.outputs.plan_id }}
|
|
|
|
- name: Reject non-version-only resource counts
|
|
env:
|
|
PLAN_ADD: ${{ steps.release-plan.outputs.add }}
|
|
PLAN_CHANGE: ${{ steps.release-plan.outputs.change }}
|
|
PLAN_DESTROY: ${{ steps.release-plan.outputs.destroy }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
|
|
echo "HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Guard version-only Terraform plan
|
|
run: |
|
|
set -euo pipefail
|
|
python scripts/check-terraform-release-plan.py \
|
|
--plan-id "${{ steps.release-run.outputs.plan_id }}" \
|
|
--expected-version-label "${{ steps.release.outputs.version_label }}"
|
|
|
|
- name: Discard release run when the guard fails
|
|
if: failure() && steps.release-run.outcome == 'success'
|
|
uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
with:
|
|
run: ${{ steps.release-run.outputs.run_id }}
|
|
comment: Rejected by the version-only plan guard from GitHub Actions
|
|
|
|
- name: Apply Terraform release run
|
|
id: release-apply
|
|
continue-on-error: true
|
|
uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
with:
|
|
run: ${{ steps.release-run.outputs.run_id }}
|
|
comment: Apply version-only release from GitHub Actions ${{ github.sha }}
|
|
|
|
- name: Treat already-applied release run as success
|
|
env:
|
|
APPLY_OUTCOME: ${{ steps.release-apply.outcome }}
|
|
RUN_ID: ${{ steps.release-run.outputs.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$APPLY_OUTCOME" = "success" ]; then
|
|
echo "Apply succeeded."
|
|
exit 0
|
|
fi
|
|
python3 << 'PY'
|
|
import json, os, urllib.request
|
|
run_id = os.environ["RUN_ID"]
|
|
token = os.environ["TF_API_TOKEN"]
|
|
req = urllib.request.Request(
|
|
f"https://app.terraform.io/api/v2/runs/{run_id}",
|
|
headers={
|
|
"Authorization": f"Bearer {token}",
|
|
"Content-Type": "application/vnd.api+json",
|
|
},
|
|
)
|
|
with urllib.request.urlopen(req) as resp:
|
|
status = json.load(resp)["data"]["attributes"]["status"]
|
|
print(f"HCP run {run_id} status={status}")
|
|
if status == "applied":
|
|
raise SystemExit(0)
|
|
raise SystemExit(
|
|
f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
|
|
f"HCP status={status}"
|
|
)
|
|
PY
|
|
|
|
- name: Verify exact application version is active
|
|
run: |
|
|
set -euo pipefail
|
|
expected="${{ steps.release.outputs.version_label }}"
|
|
status="Unknown"
|
|
current="Unknown"
|
|
health="Unknown"
|
|
|
|
for _ in $(seq 1 80); do
|
|
read -r status current health < <(
|
|
aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].[Status,VersionLabel,Health]' \
|
|
--output text
|
|
)
|
|
echo "environment status: $status; version: $current; health: $health"
|
|
|
|
if [ "$status" = "Ready" ]; then
|
|
if [ "$current" != "$expected" ]; then
|
|
echo "Environment became Ready on version $current, not the expected $expected." >&2
|
|
exit 1
|
|
fi
|
|
if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
|
|
echo "Expected application version is Ready and healthy."
|
|
exit 0
|
|
fi
|
|
# The expected version IS active. Elastic Beanstalk reports Ready as
|
|
# soon as the rollout finishes, before enhanced health has converged,
|
|
# so deciding on the first Ready poll fails a good release on a health
|
|
# value that was always going to change. Keep polling; an environment
|
|
# that is genuinely unhealthy still fails when the window runs out.
|
|
echo "Expected version is active; waiting for health to leave $health."
|
|
fi
|
|
sleep 15
|
|
done
|
|
|
|
echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2
|
|
exit 1
|
|
|
|
- name: Post-deploy smoke
|
|
run: bash scripts/smoke-elastic-beanstalk.sh "${SMOKE_URL}"
|
|
|
|
- name: Verify webhook secret source is operational
|
|
run: |
|
|
set -euo pipefail
|
|
response_file="$(mktemp)"
|
|
trap 'rm -f "$response_file"' EXIT
|
|
status="$(curl --silent --show-error \
|
|
--output "$response_file" \
|
|
--write-out '%{http_code}' \
|
|
--request POST \
|
|
--header 'Content-Type: application/json' \
|
|
--header "X-SH-Timestamp: $(date +%s)" \
|
|
--header 'X-SH-Key-Id: deployment-smoke-invalid-key' \
|
|
--header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \
|
|
--data '{}' \
|
|
"${SMOKE_URL}/api/webhooks/work-orders")"
|
|
if [ "$status" != "401" ]; then
|
|
echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2
|
|
sed -n '1,20p' "$response_file" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Restore previous application version on failure (schema is not reverted)
|
|
if: failure()
|
|
run: |
|
|
set -euo pipefail
|
|
prev_file=".artifacts/elastic-beanstalk/previous-version.txt"
|
|
if [ ! -f "$prev_file" ]; then
|
|
echo "No previous version captured; nothing to roll back." >&2
|
|
exit 0
|
|
fi
|
|
prev="$(cat "$prev_file")"
|
|
if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then
|
|
echo "No previous version recorded; nothing to roll back." >&2
|
|
exit 0
|
|
fi
|
|
|
|
echo "Waiting for any in-flight environment update to settle..."
|
|
status="Unknown"
|
|
current="Unknown"
|
|
health="Unknown"
|
|
for _ in $(seq 1 80); do
|
|
read -r status current health < <(
|
|
aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].[Status,VersionLabel,Health]' \
|
|
--output text
|
|
)
|
|
echo "environment status: $status; version: $current; health: $health"
|
|
if [ "$status" = "Ready" ]; then
|
|
break
|
|
fi
|
|
sleep 15
|
|
done
|
|
|
|
if [ "$status" != "Ready" ]; then
|
|
echo "Environment did not settle before rollback." >&2
|
|
exit 1
|
|
fi
|
|
if [ "$current" = "$prev" ]; then
|
|
echo "Environment is already on previous version $prev."
|
|
exit 0
|
|
fi
|
|
if [[ ! "$prev" =~ ^[0-9a-f]{40}-[0-9]+-[0-9]+$ ]]; then
|
|
echo "Previous version $prev is not a Terraform-managed release label; cannot roll back through HCP." >&2
|
|
exit 1
|
|
fi
|
|
echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}"
|
|
id: rollback-prepare
|
|
|
|
- name: Discard blocking VCS run before GitHub rollback
|
|
id: rollback-discard-vcs
|
|
if: failure() && steps.rollback-prepare.outputs.rollback_label != ''
|
|
run: |
|
|
set -euo pipefail
|
|
python3 << 'PY'
|
|
import json, os, urllib.error, urllib.request
|
|
|
|
token = os.environ["TF_API_TOKEN"]
|
|
workspace = "shoc-backend-dev"
|
|
headers = {
|
|
"Authorization": f"Bearer {token}",
|
|
"Content-Type": "application/vnd.api+json",
|
|
}
|
|
|
|
def get(url):
|
|
req = urllib.request.Request(url, headers=headers)
|
|
with urllib.request.urlopen(req) as resp:
|
|
return json.load(resp)
|
|
|
|
def post(url, payload):
|
|
data = json.dumps(payload).encode()
|
|
req = urllib.request.Request(
|
|
url, data=data, method="POST", headers=headers
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req) as resp:
|
|
return resp.status
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code in (409, 404):
|
|
body = exc.read().decode("utf-8", "replace")
|
|
print(f"discard returned HTTP {exc.code}: {body}")
|
|
return exc.code
|
|
raise
|
|
|
|
ws = get(
|
|
f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
|
|
)["data"]
|
|
attrs = ws["attributes"]
|
|
if attrs.get("auto-apply") is True:
|
|
raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue")
|
|
if not attrs.get("speculative-enabled"):
|
|
raise SystemExit("speculative plans are off; refuse to continue")
|
|
if (attrs.get("vcs-repo") or {}).get("tags-regex"):
|
|
raise SystemExit("tag-based VCS triggering is set; refuse to continue")
|
|
expected_patterns = [
|
|
"terraform/live/dev/**",
|
|
"terraform/live/modules/**",
|
|
]
|
|
if attrs.get("trigger-patterns") != expected_patterns:
|
|
raise SystemExit(
|
|
"trigger-patterns must be "
|
|
f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
|
|
)
|
|
if not attrs.get("locked"):
|
|
print("workspace is unlocked")
|
|
raise SystemExit(0)
|
|
|
|
current = (
|
|
ws.get("relationships", {})
|
|
.get("current-run", {})
|
|
.get("data")
|
|
)
|
|
if not current:
|
|
raise SystemExit("workspace is locked without a current run")
|
|
run_id = current["id"]
|
|
run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
|
|
run_attrs = run["attributes"]
|
|
status = run_attrs.get("status")
|
|
plan_only = run_attrs.get("plan-only")
|
|
print(f"current run {run_id} status={status} plan-only={plan_only}")
|
|
if plan_only:
|
|
print("speculative run does not block GitHub CD")
|
|
raise SystemExit(0)
|
|
if status in {"applying", "apply_queued"}:
|
|
raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
|
|
discardable = {
|
|
"pending", "planned", "cost_estimated", "policy_checked", "policy_override"
|
|
}
|
|
if status not in discardable:
|
|
raise SystemExit(f"{run_id} status {status} is not discardable")
|
|
code = post(
|
|
f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
|
|
{"comment": "Discarded so GitHub CD can create the version-only applyable run"},
|
|
)
|
|
print(f"discarded {run_id} http={code}")
|
|
PY
|
|
|
|
- name: Create Terraform rollback run
|
|
id: rollback-run
|
|
if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success'
|
|
uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
env:
|
|
TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"'
|
|
with:
|
|
workspace: shoc-backend-dev
|
|
message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions"
|
|
- name: Read Terraform rollback plan counts
|
|
id: rollback-plan
|
|
if: failure() && steps.rollback-run.outcome == 'success'
|
|
uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
with:
|
|
plan: ${{ steps.rollback-run.outputs.plan_id }}
|
|
|
|
- name: Reject non-version-only rollback counts
|
|
id: rollback-count-guard
|
|
if: failure() && steps.rollback-plan.outcome == 'success'
|
|
env:
|
|
PLAN_ADD: ${{ steps.rollback-plan.outputs.add }}
|
|
PLAN_CHANGE: ${{ steps.rollback-plan.outputs.change }}
|
|
PLAN_DESTROY: ${{ steps.rollback-plan.outputs.destroy }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
|
|
echo "Rollback HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Guard version-only Terraform rollback plan
|
|
id: rollback-json-guard
|
|
if: failure() && steps.rollback-count-guard.outcome == 'success'
|
|
run: |
|
|
set -euo pipefail
|
|
python scripts/check-terraform-release-plan.py \
|
|
--plan-id "${{ steps.rollback-run.outputs.plan_id }}" \
|
|
--expected-version-label "${{ steps.rollback-prepare.outputs.rollback_label }}"
|
|
|
|
- name: Discard rollback run when the guard fails
|
|
if: failure() && steps.rollback-run.outcome == 'success' && steps.rollback-json-guard.outcome != 'success'
|
|
uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
with:
|
|
run: ${{ steps.rollback-run.outputs.run_id }}
|
|
comment: Rejected by the version-only rollback plan guard from GitHub Actions
|
|
|
|
- name: Apply Terraform rollback run
|
|
id: rollback-apply
|
|
if: failure() && steps.rollback-json-guard.outcome == 'success'
|
|
continue-on-error: true
|
|
uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
|
with:
|
|
run: ${{ steps.rollback-run.outputs.run_id }}
|
|
comment: Apply version-only rollback from GitHub Actions ${{ github.sha }}
|
|
|
|
- name: Treat already-applied rollback run as success
|
|
id: rollback-apply-result
|
|
if: failure() && steps.rollback-apply.outcome != 'skipped'
|
|
env:
|
|
APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }}
|
|
RUN_ID: ${{ steps.rollback-run.outputs.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$APPLY_OUTCOME" = "success" ]; then
|
|
echo "Apply succeeded."
|
|
exit 0
|
|
fi
|
|
python3 << 'PY'
|
|
import json, os, urllib.request
|
|
run_id = os.environ["RUN_ID"]
|
|
token = os.environ["TF_API_TOKEN"]
|
|
req = urllib.request.Request(
|
|
f"https://app.terraform.io/api/v2/runs/{run_id}",
|
|
headers={
|
|
"Authorization": f"Bearer {token}",
|
|
"Content-Type": "application/vnd.api+json",
|
|
},
|
|
)
|
|
with urllib.request.urlopen(req) as resp:
|
|
status = json.load(resp)["data"]["attributes"]["status"]
|
|
print(f"HCP run {run_id} status={status}")
|
|
if status == "applied":
|
|
raise SystemExit(0)
|
|
raise SystemExit(
|
|
f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
|
|
f"HCP status={status}"
|
|
)
|
|
PY
|
|
|
|
- name: Verify previous application version is active
|
|
if: failure() && steps.rollback-apply-result.outcome == 'success'
|
|
run: |
|
|
set -euo pipefail
|
|
prev="${{ steps.rollback-prepare.outputs.rollback_label }}"
|
|
echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy."
|
|
status="Unknown"
|
|
current="Unknown"
|
|
health="Unknown"
|
|
for _ in $(seq 1 80); do
|
|
read -r status current health < <(
|
|
aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].[Status,VersionLabel,Health]' \
|
|
--output text
|
|
)
|
|
echo "environment status: $status; version: $current; health: $health"
|
|
if [ "$status" = "Ready" ]; then
|
|
if [ "$current" = "$prev" ] && { [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; }; then
|
|
echo "Application version restore complete; previous code is Ready and healthy."
|
|
exit 0
|
|
fi
|
|
echo "Rollback reached Ready in an unexpected version/health state." >&2
|
|
exit 1
|
|
fi
|
|
sleep 15
|
|
done
|
|
echo "Environment did not return to Ready within rollback window." >&2
|
|
exit 1
|
|
|
|
deploy-staging:
|
|
name: Deploy shoc-backend-staging to Elastic Beanstalk
|
|
if: >
|
|
github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging'
|
|
needs: validate
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
environment:
|
|
name: staging
|
|
concurrency:
|
|
group: deploy-staging
|
|
cancel-in-progress: false
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
|
|
- name: Resolve deploy target
|
|
id: target
|
|
run: |
|
|
set -euo pipefail
|
|
application=shoc-backend
|
|
environment=shoc-backend-staging
|
|
smoke_url=https://api.staging.seahaven.com
|
|
{
|
|
echo "application=${application}"
|
|
echo "environment=${environment}"
|
|
echo "smoke_url=${smoke_url}"
|
|
} >> "${GITHUB_OUTPUT}"
|
|
{
|
|
echo "EB_APPLICATION_NAME=${application}"
|
|
echo "EB_ENVIRONMENT_NAME=${environment}"
|
|
echo "SMOKE_URL=${smoke_url}"
|
|
} >> "${GITHUB_ENV}"
|
|
|
|
- name: Set up .NET
|
|
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
|
with:
|
|
dotnet-version: "8.0.x"
|
|
|
|
- name: Build Elastic Beanstalk source bundle
|
|
run: bash scripts/package-elastic-beanstalk.sh
|
|
|
|
- name: Validate exact release bundle
|
|
run: bash scripts/validate-elastic-beanstalk-bundle.sh
|
|
|
|
- name: Configure AWS credentials (OIDC)
|
|
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
|
|
with:
|
|
role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
|
|
aws-region: us-east-1
|
|
audience: sts.amazonaws.com
|
|
|
|
- name: Capture current environment version
|
|
run: |
|
|
set -euo pipefail
|
|
prev="$(aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].VersionLabel' \
|
|
--output text)"
|
|
echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt
|
|
echo "Previous version label: $prev"
|
|
|
|
- name: Deploy prebuilt bundle to existing environment
|
|
uses: aws-actions/aws-elasticbeanstalk-deploy@7883cdd454c162051bf6fc13389536b045149b4c # v1.0.8
|
|
with:
|
|
aws-region: us-east-1
|
|
application-name: ${{ steps.target.outputs.application }}
|
|
environment-name: ${{ steps.target.outputs.environment }}
|
|
version-label: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
deployment-package-path: .artifacts/elastic-beanstalk/site.zip
|
|
s3-bucket-name: elasticbeanstalk-us-east-1-396287094661
|
|
create-application-if-not-exists: "false"
|
|
create-environment-if-not-exists: "false"
|
|
create-s3-bucket-if-not-exists: "false"
|
|
use-existing-application-version-if-available: "false"
|
|
wait-for-deployment: "true"
|
|
wait-for-environment-recovery: "true"
|
|
|
|
- name: Verify exact application version is active
|
|
run: |
|
|
set -euo pipefail
|
|
expected="${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}"
|
|
status="Unknown"
|
|
current="Unknown"
|
|
health="Unknown"
|
|
|
|
for _ in $(seq 1 80); do
|
|
read -r status current health < <(
|
|
aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].[Status,VersionLabel,Health]' \
|
|
--output text
|
|
)
|
|
echo "environment status: $status; version: $current; health: $health"
|
|
|
|
if [ "$status" = "Ready" ]; then
|
|
if [ "$current" != "$expected" ]; then
|
|
echo "Environment became Ready on version $current, not the expected $expected." >&2
|
|
exit 1
|
|
fi
|
|
if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
|
|
echo "Expected application version is Ready and healthy."
|
|
exit 0
|
|
fi
|
|
# The expected version IS active. Elastic Beanstalk reports Ready as
|
|
# soon as the rollout finishes, before enhanced health has converged,
|
|
# so deciding on the first Ready poll fails a good release on a health
|
|
# value that was always going to change. Keep polling; an environment
|
|
# that is genuinely unhealthy still fails when the window runs out.
|
|
echo "Expected version is active; waiting for health to leave $health."
|
|
fi
|
|
sleep 15
|
|
done
|
|
|
|
echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2
|
|
exit 1
|
|
|
|
- name: Post-deploy smoke
|
|
run: bash scripts/smoke-elastic-beanstalk.sh "${SMOKE_URL}"
|
|
|
|
- name: Verify webhook secret source is operational
|
|
run: |
|
|
set -euo pipefail
|
|
response_file="$(mktemp)"
|
|
trap 'rm -f "$response_file"' EXIT
|
|
status="$(curl --silent --show-error \
|
|
--output "$response_file" \
|
|
--write-out '%{http_code}' \
|
|
--request POST \
|
|
--header 'Content-Type: application/json' \
|
|
--header "X-SH-Timestamp: $(date +%s)" \
|
|
--header 'X-SH-Key-Id: deployment-smoke-invalid-key' \
|
|
--header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \
|
|
--data '{}' \
|
|
"${SMOKE_URL}/api/webhooks/work-orders")"
|
|
if [ "$status" != "401" ]; then
|
|
echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2
|
|
sed -n '1,20p' "$response_file" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Restore previous application version on failure (schema is not reverted)
|
|
if: failure()
|
|
run: |
|
|
set -euo pipefail
|
|
prev_file=".artifacts/elastic-beanstalk/previous-version.txt"
|
|
if [ ! -f "$prev_file" ]; then
|
|
echo "No previous version captured; nothing to roll back." >&2
|
|
exit 0
|
|
fi
|
|
prev="$(cat "$prev_file")"
|
|
if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then
|
|
echo "No previous version recorded; nothing to roll back." >&2
|
|
exit 0
|
|
fi
|
|
|
|
echo "Waiting for any in-flight environment update to settle..."
|
|
status="Unknown"
|
|
current="Unknown"
|
|
health="Unknown"
|
|
for _ in $(seq 1 80); do
|
|
read -r status current health < <(
|
|
aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].[Status,VersionLabel,Health]' \
|
|
--output text
|
|
)
|
|
echo "environment status: $status; version: $current; health: $health"
|
|
if [ "$status" = "Ready" ]; then
|
|
break
|
|
fi
|
|
sleep 15
|
|
done
|
|
|
|
if [ "$status" != "Ready" ]; then
|
|
echo "Environment did not settle before rollback." >&2
|
|
exit 1
|
|
fi
|
|
if [ "$current" = "$prev" ]; then
|
|
echo "Environment is already on previous version $prev."
|
|
exit 0
|
|
fi
|
|
|
|
echo "Restoring ${EB_ENVIRONMENT_NAME} application code to version label: $prev"
|
|
echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy."
|
|
aws elasticbeanstalk update-environment \
|
|
--environment-name "${EB_ENVIRONMENT_NAME}" \
|
|
--version-label "$prev" \
|
|
--region us-east-1
|
|
|
|
echo "Waiting for previous version to become healthy..."
|
|
for _ in $(seq 1 80); do
|
|
read -r status current health < <(
|
|
aws elasticbeanstalk describe-environments \
|
|
--environment-names "${EB_ENVIRONMENT_NAME}" \
|
|
--region us-east-1 \
|
|
--query 'Environments[0].[Status,VersionLabel,Health]' \
|
|
--output text
|
|
)
|
|
echo "environment status: $status; version: $current; health: $health"
|
|
if [ "$status" = "Ready" ]; then
|
|
if [ "$current" != "$prev" ]; then
|
|
echo "Rollback reached Ready on version $current, not the previous $prev." >&2
|
|
exit 1
|
|
fi
|
|
if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
|
|
echo "Application version restore complete; previous code is Ready and healthy."
|
|
exit 0
|
|
fi
|
|
# Same convergence gap as the release check above: the previous version
|
|
# is back, health has not settled yet, and reporting a failed rollback
|
|
# here hides the fact that the restore itself worked.
|
|
echo "Previous version is active; waiting for health to leave $health."
|
|
fi
|
|
sleep 15
|
|
done
|
|
|
|
echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2
|
|
exit 1
|