mirror of
https://github.com/Sea-Haven-Industries/shoc-backend.git
synced 2026-09-30 04:53:11 +00:00
Merge pull request #124 from Sea-Haven-Industries/fix/ab/sh-287-backend-staging-lane
feat(deploy): rebuild protected backend staging lane
This commit is contained in:
commit
d02aa11285
9 changed files with 645 additions and 70 deletions
464
.github/workflows/deploy.yml
vendored
464
.github/workflows/deploy.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
pull_request:
|
||||
branches: [dev, staging, main]
|
||||
push:
|
||||
branches: [dev]
|
||||
branches: [dev, staging]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
|
@ -143,7 +143,6 @@ jobs:
|
|||
done
|
||||
echo "Application version did not become PROCESSED." >&2
|
||||
exit 1
|
||||
|
||||
- name: Discard blocking VCS run before GitHub CD
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
|
@ -639,11 +638,13 @@ jobs:
|
|||
exit 1
|
||||
|
||||
deploy-staging:
|
||||
name: Deploy shoc-backend-staging to Elastic Beanstalk
|
||||
name: Deploy shoc-backend-staging through Terraform
|
||||
if: >
|
||||
github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging'
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/staging') ||
|
||||
(github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging')
|
||||
needs: validate
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
|
@ -652,28 +653,17 @@ jobs:
|
|||
concurrency:
|
||||
group: deploy-staging
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
TF_CLOUD_ORGANIZATION: seahaven
|
||||
TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}
|
||||
EB_APPLICATION_NAME: shoc-backend
|
||||
EB_ENVIRONMENT_NAME: shoc-backend-staging
|
||||
SMOKE_URL: https://api.staging.seahaven.com
|
||||
EB_BUCKET: elasticbeanstalk-us-east-1-396287094661
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Resolve deploy target
|
||||
id: target
|
||||
run: |
|
||||
set -euo pipefail
|
||||
application=shoc-backend
|
||||
environment=shoc-backend-staging
|
||||
smoke_url=https://api.staging.seahaven.com
|
||||
{
|
||||
echo "application=${application}"
|
||||
echo "environment=${environment}"
|
||||
echo "smoke_url=${smoke_url}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
{
|
||||
echo "EB_APPLICATION_NAME=${application}"
|
||||
echo "EB_ENVIRONMENT_NAME=${environment}"
|
||||
echo "SMOKE_URL=${smoke_url}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up .NET
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
|
|
@ -703,26 +693,227 @@ jobs:
|
|||
echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt
|
||||
echo "Previous version label: $prev"
|
||||
|
||||
- name: Deploy prebuilt bundle to existing environment
|
||||
uses: aws-actions/aws-elasticbeanstalk-deploy@7883cdd454c162051bf6fc13389536b045149b4c # v1.0.8
|
||||
- name: Assign immutable release identity
|
||||
id: release
|
||||
run: |
|
||||
set -euo pipefail
|
||||
version_label="${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
s3_key="shoc-backend/releases/staging/${GITHUB_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip"
|
||||
{
|
||||
echo "version_label=${version_label}"
|
||||
echo "s3_key=${s3_key}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Upload immutable bundle
|
||||
run: |
|
||||
set -euo pipefail
|
||||
aws s3 cp .artifacts/elastic-beanstalk/site.zip \
|
||||
"s3://${EB_BUCKET}/${{ steps.release.outputs.s3_key }}" \
|
||||
--region us-east-1
|
||||
|
||||
- name: Create Elastic Beanstalk application version
|
||||
run: |
|
||||
set -euo pipefail
|
||||
aws elasticbeanstalk create-application-version \
|
||||
--application-name "${EB_APPLICATION_NAME}" \
|
||||
--version-label "${{ steps.release.outputs.version_label }}" \
|
||||
--description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" \
|
||||
--source-bundle "S3Bucket=${EB_BUCKET},S3Key=${{ steps.release.outputs.s3_key }}" \
|
||||
--process \
|
||||
--region us-east-1
|
||||
|
||||
status="UNPROCESSED"
|
||||
for _ in $(seq 1 36); do
|
||||
status="$(aws elasticbeanstalk describe-application-versions \
|
||||
--application-name "${EB_APPLICATION_NAME}" \
|
||||
--version-labels "${{ steps.release.outputs.version_label }}" \
|
||||
--region us-east-1 \
|
||||
--query 'ApplicationVersions[0].Status' \
|
||||
--output text)"
|
||||
echo "application version status: $status"
|
||||
if [ "$status" = "PROCESSED" ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$status" = "FAILED" ]; then
|
||||
echo "Elastic Beanstalk failed to process ${{ steps.release.outputs.version_label }}." >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Application version did not become PROCESSED." >&2
|
||||
exit 1
|
||||
|
||||
- name: Discard blocking VCS run before GitHub CD
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 << 'PY'
|
||||
import json, os, urllib.error, urllib.request
|
||||
|
||||
token = os.environ["TF_API_TOKEN"]
|
||||
workspace = "shoc-backend-staging"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
}
|
||||
|
||||
def get(url):
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.load(resp)
|
||||
|
||||
def post(url, payload):
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST", headers=headers
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (409, 404):
|
||||
body = exc.read().decode("utf-8", "replace")
|
||||
print(f"discard returned HTTP {exc.code}: {body}")
|
||||
return exc.code
|
||||
raise
|
||||
|
||||
ws = get(
|
||||
f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
|
||||
)["data"]
|
||||
attrs = ws["attributes"]
|
||||
if attrs.get("auto-apply") is True:
|
||||
raise SystemExit("shoc-backend-staging auto-apply is on; refuse to continue")
|
||||
if not attrs.get("speculative-enabled"):
|
||||
raise SystemExit("speculative plans are off; refuse to continue")
|
||||
if (attrs.get("vcs-repo") or {}).get("tags-regex"):
|
||||
raise SystemExit("tag-based VCS triggering is set; refuse to continue")
|
||||
expected_patterns = [
|
||||
"terraform/live/staging/**",
|
||||
"terraform/live/modules/**",
|
||||
]
|
||||
if attrs.get("trigger-patterns") != expected_patterns:
|
||||
raise SystemExit(
|
||||
"trigger-patterns must be "
|
||||
f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
|
||||
)
|
||||
if not attrs.get("locked"):
|
||||
print("workspace is unlocked")
|
||||
raise SystemExit(0)
|
||||
|
||||
current = (
|
||||
ws.get("relationships", {})
|
||||
.get("current-run", {})
|
||||
.get("data")
|
||||
)
|
||||
if not current:
|
||||
raise SystemExit("workspace is locked without a current run")
|
||||
run_id = current["id"]
|
||||
run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
|
||||
run_attrs = run["attributes"]
|
||||
status = run_attrs.get("status")
|
||||
plan_only = run_attrs.get("plan-only")
|
||||
print(f"current run {run_id} status={status} plan-only={plan_only}")
|
||||
if plan_only:
|
||||
print("speculative run does not block GitHub CD")
|
||||
raise SystemExit(0)
|
||||
if status in {"applying", "apply_queued"}:
|
||||
raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
|
||||
discardable = {
|
||||
"pending", "planned", "cost_estimated", "policy_checked", "policy_override"
|
||||
}
|
||||
if status not in discardable:
|
||||
raise SystemExit(f"{run_id} status {status} is not discardable")
|
||||
code = post(
|
||||
f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
|
||||
{"comment": "Discarded so GitHub CD can create the version-only applyable run"},
|
||||
)
|
||||
print(f"discarded {run_id} http={code}")
|
||||
PY
|
||||
|
||||
- name: Create Terraform release run
|
||||
id: release-run
|
||||
uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
env:
|
||||
TF_VAR_release_version_label: '"${{ steps.release.outputs.version_label }}"'
|
||||
with:
|
||||
aws-region: us-east-1
|
||||
application-name: ${{ steps.target.outputs.application }}
|
||||
environment-name: ${{ steps.target.outputs.environment }}
|
||||
version-label: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
deployment-package-path: .artifacts/elastic-beanstalk/site.zip
|
||||
s3-bucket-name: elasticbeanstalk-us-east-1-396287094661
|
||||
create-application-if-not-exists: "false"
|
||||
create-environment-if-not-exists: "false"
|
||||
create-s3-bucket-if-not-exists: "false"
|
||||
use-existing-application-version-if-available: "false"
|
||||
wait-for-deployment: "true"
|
||||
wait-for-environment-recovery: "true"
|
||||
workspace: shoc-backend-staging
|
||||
message: "Release ${{ steps.release.outputs.version_label }} from GitHub Actions"
|
||||
|
||||
- name: Read Terraform release plan counts
|
||||
id: release-plan
|
||||
uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
with:
|
||||
plan: ${{ steps.release-run.outputs.plan_id }}
|
||||
|
||||
- name: Reject non-version-only resource counts
|
||||
env:
|
||||
PLAN_ADD: ${{ steps.release-plan.outputs.add }}
|
||||
PLAN_CHANGE: ${{ steps.release-plan.outputs.change }}
|
||||
PLAN_DESTROY: ${{ steps.release-plan.outputs.destroy }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
|
||||
echo "HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Guard version-only Terraform plan
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python scripts/check-terraform-release-plan.py \
|
||||
--plan-id "${{ steps.release-run.outputs.plan_id }}" \
|
||||
--expected-version-label "${{ steps.release.outputs.version_label }}"
|
||||
|
||||
- name: Discard release run when the guard fails
|
||||
if: failure() && steps.release-run.outcome == 'success'
|
||||
uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
with:
|
||||
run: ${{ steps.release-run.outputs.run_id }}
|
||||
comment: Rejected by the version-only plan guard from GitHub Actions
|
||||
|
||||
- name: Apply Terraform release run
|
||||
id: release-apply
|
||||
continue-on-error: true
|
||||
uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
with:
|
||||
run: ${{ steps.release-run.outputs.run_id }}
|
||||
comment: Apply version-only release from GitHub Actions ${{ github.sha }}
|
||||
|
||||
- name: Treat already-applied release run as success
|
||||
env:
|
||||
APPLY_OUTCOME: ${{ steps.release-apply.outcome }}
|
||||
RUN_ID: ${{ steps.release-run.outputs.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$APPLY_OUTCOME" = "success" ]; then
|
||||
echo "Apply succeeded."
|
||||
exit 0
|
||||
fi
|
||||
python3 << 'PY'
|
||||
import json, os, urllib.request
|
||||
run_id = os.environ["RUN_ID"]
|
||||
token = os.environ["TF_API_TOKEN"]
|
||||
req = urllib.request.Request(
|
||||
f"https://app.terraform.io/api/v2/runs/{run_id}",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
status = json.load(resp)["data"]["attributes"]["status"]
|
||||
print(f"HCP run {run_id} status={status}")
|
||||
if status == "applied":
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(
|
||||
f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
|
||||
f"HCP status={status}"
|
||||
)
|
||||
PY
|
||||
|
||||
- name: Verify exact application version is active
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
expected="${{ steps.release.outputs.version_label }}"
|
||||
status="Unknown"
|
||||
current="Unknown"
|
||||
health="Unknown"
|
||||
|
|
@ -825,15 +1016,199 @@ jobs:
|
|||
echo "Environment is already on previous version $prev."
|
||||
exit 0
|
||||
fi
|
||||
if [[ ! "$prev" =~ ^[0-9a-f]{40}-[0-9]+-[0-9]+$ ]]; then
|
||||
echo "Previous version $prev is not a Terraform-managed release label; cannot roll back through HCP." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}"
|
||||
id: rollback-prepare
|
||||
|
||||
echo "Restoring ${EB_ENVIRONMENT_NAME} application code to version label: $prev"
|
||||
- name: Discard blocking VCS run before GitHub rollback
|
||||
id: rollback-discard-vcs
|
||||
if: failure() && steps.rollback-prepare.outputs.rollback_label != ''
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 << 'PY'
|
||||
import json, os, urllib.error, urllib.request
|
||||
|
||||
token = os.environ["TF_API_TOKEN"]
|
||||
workspace = "shoc-backend-staging"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
}
|
||||
|
||||
def get(url):
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.load(resp)
|
||||
|
||||
def post(url, payload):
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST", headers=headers
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (409, 404):
|
||||
body = exc.read().decode("utf-8", "replace")
|
||||
print(f"discard returned HTTP {exc.code}: {body}")
|
||||
return exc.code
|
||||
raise
|
||||
|
||||
ws = get(
|
||||
f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
|
||||
)["data"]
|
||||
attrs = ws["attributes"]
|
||||
if attrs.get("auto-apply") is True:
|
||||
raise SystemExit("shoc-backend-staging auto-apply is on; refuse to continue")
|
||||
if not attrs.get("speculative-enabled"):
|
||||
raise SystemExit("speculative plans are off; refuse to continue")
|
||||
if (attrs.get("vcs-repo") or {}).get("tags-regex"):
|
||||
raise SystemExit("tag-based VCS triggering is set; refuse to continue")
|
||||
expected_patterns = [
|
||||
"terraform/live/staging/**",
|
||||
"terraform/live/modules/**",
|
||||
]
|
||||
if attrs.get("trigger-patterns") != expected_patterns:
|
||||
raise SystemExit(
|
||||
"trigger-patterns must be "
|
||||
f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
|
||||
)
|
||||
if not attrs.get("locked"):
|
||||
print("workspace is unlocked")
|
||||
raise SystemExit(0)
|
||||
|
||||
current = (
|
||||
ws.get("relationships", {})
|
||||
.get("current-run", {})
|
||||
.get("data")
|
||||
)
|
||||
if not current:
|
||||
raise SystemExit("workspace is locked without a current run")
|
||||
run_id = current["id"]
|
||||
run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
|
||||
run_attrs = run["attributes"]
|
||||
status = run_attrs.get("status")
|
||||
plan_only = run_attrs.get("plan-only")
|
||||
print(f"current run {run_id} status={status} plan-only={plan_only}")
|
||||
if plan_only:
|
||||
print("speculative run does not block GitHub CD")
|
||||
raise SystemExit(0)
|
||||
if status in {"applying", "apply_queued"}:
|
||||
raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
|
||||
discardable = {
|
||||
"pending", "planned", "cost_estimated", "policy_checked", "policy_override"
|
||||
}
|
||||
if status not in discardable:
|
||||
raise SystemExit(f"{run_id} status {status} is not discardable")
|
||||
code = post(
|
||||
f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
|
||||
{"comment": "Discarded so GitHub CD can create the version-only applyable run"},
|
||||
)
|
||||
print(f"discarded {run_id} http={code}")
|
||||
PY
|
||||
|
||||
- name: Create Terraform rollback run
|
||||
id: rollback-run
|
||||
if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success'
|
||||
uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
env:
|
||||
TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"'
|
||||
with:
|
||||
workspace: shoc-backend-staging
|
||||
message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions"
|
||||
- name: Read Terraform rollback plan counts
|
||||
id: rollback-plan
|
||||
if: failure() && steps.rollback-run.outcome == 'success'
|
||||
uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
with:
|
||||
plan: ${{ steps.rollback-run.outputs.plan_id }}
|
||||
|
||||
- name: Reject non-version-only rollback counts
|
||||
id: rollback-count-guard
|
||||
if: failure() && steps.rollback-plan.outcome == 'success'
|
||||
env:
|
||||
PLAN_ADD: ${{ steps.rollback-plan.outputs.add }}
|
||||
PLAN_CHANGE: ${{ steps.rollback-plan.outputs.change }}
|
||||
PLAN_DESTROY: ${{ steps.rollback-plan.outputs.destroy }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
|
||||
echo "Rollback HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Guard version-only Terraform rollback plan
|
||||
id: rollback-json-guard
|
||||
if: failure() && steps.rollback-count-guard.outcome == 'success'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python scripts/check-terraform-release-plan.py \
|
||||
--plan-id "${{ steps.rollback-run.outputs.plan_id }}" \
|
||||
--expected-version-label "${{ steps.rollback-prepare.outputs.rollback_label }}"
|
||||
|
||||
- name: Discard rollback run when the guard fails
|
||||
if: failure() && steps.rollback-run.outcome == 'success' && steps.rollback-json-guard.outcome != 'success'
|
||||
uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
with:
|
||||
run: ${{ steps.rollback-run.outputs.run_id }}
|
||||
comment: Rejected by the version-only rollback plan guard from GitHub Actions
|
||||
|
||||
- name: Apply Terraform rollback run
|
||||
id: rollback-apply
|
||||
if: failure() && steps.rollback-json-guard.outcome == 'success'
|
||||
continue-on-error: true
|
||||
uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
|
||||
with:
|
||||
run: ${{ steps.rollback-run.outputs.run_id }}
|
||||
comment: Apply version-only rollback from GitHub Actions ${{ github.sha }}
|
||||
|
||||
- name: Treat already-applied rollback run as success
|
||||
id: rollback-apply-result
|
||||
if: failure() && steps.rollback-apply.outcome != 'skipped'
|
||||
env:
|
||||
APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }}
|
||||
RUN_ID: ${{ steps.rollback-run.outputs.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$APPLY_OUTCOME" = "success" ]; then
|
||||
echo "Apply succeeded."
|
||||
exit 0
|
||||
fi
|
||||
python3 << 'PY'
|
||||
import json, os, urllib.request
|
||||
run_id = os.environ["RUN_ID"]
|
||||
token = os.environ["TF_API_TOKEN"]
|
||||
req = urllib.request.Request(
|
||||
f"https://app.terraform.io/api/v2/runs/{run_id}",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/vnd.api+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
status = json.load(resp)["data"]["attributes"]["status"]
|
||||
print(f"HCP run {run_id} status={status}")
|
||||
if status == "applied":
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(
|
||||
f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
|
||||
f"HCP status={status}"
|
||||
)
|
||||
PY
|
||||
|
||||
- name: Verify previous application version is active
|
||||
if: failure() && steps.rollback-apply-result.outcome == 'success'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
prev="${{ steps.rollback-prepare.outputs.rollback_label }}"
|
||||
echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy."
|
||||
aws elasticbeanstalk update-environment \
|
||||
--environment-name "${EB_ENVIRONMENT_NAME}" \
|
||||
--version-label "$prev" \
|
||||
--region us-east-1
|
||||
|
||||
echo "Waiting for previous version to become healthy..."
|
||||
status="Unknown"
|
||||
current="Unknown"
|
||||
health="Unknown"
|
||||
for _ in $(seq 1 80); do
|
||||
read -r status current health < <(
|
||||
aws elasticbeanstalk describe-environments \
|
||||
|
|
@ -859,6 +1234,5 @@ jobs:
|
|||
fi
|
||||
sleep 15
|
||||
done
|
||||
|
||||
echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -10,8 +10,10 @@ from pathlib import Path
|
|||
|
||||
from terraform_import_plan_resources import (
|
||||
DEV_IMPORT_BASELINE,
|
||||
DEV_IMPORT_IDS,
|
||||
IMPORT_BASELINES,
|
||||
IMPORT_IDS,
|
||||
REQUIRED_RESOURCES,
|
||||
STAGING_IMPORT_BASELINE,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -92,6 +94,51 @@ def validate_dev_import_baseline(
|
|||
)
|
||||
|
||||
|
||||
def validate_staging_import_baseline(
|
||||
resources_by_address: dict[str, dict], violations: list[str]
|
||||
) -> None:
|
||||
environment_address = (
|
||||
"module.environment.aws_elastic_beanstalk_environment.this"
|
||||
)
|
||||
route_address = "module.environment.aws_route53_record.api_cname[0]"
|
||||
expected_tags = STAGING_IMPORT_BASELINE["environment_tags"]
|
||||
expected_cname = STAGING_IMPORT_BASELINE["api_cname"]
|
||||
|
||||
for side in ("before", "after"):
|
||||
environment = (
|
||||
resources_by_address.get(environment_address, {})
|
||||
.get("change", {})
|
||||
.get(side)
|
||||
or {}
|
||||
)
|
||||
if environment.get("tags") != expected_tags:
|
||||
violations.append(
|
||||
f"{environment_address}: {side} environment tags do not match "
|
||||
f"the exact staging import baseline"
|
||||
)
|
||||
if environment.get("setting") != []:
|
||||
violations.append(
|
||||
f"{environment_address}: {side} contains managed EB settings "
|
||||
"during the import-only phase"
|
||||
)
|
||||
|
||||
route = (
|
||||
resources_by_address.get(route_address, {})
|
||||
.get("change", {})
|
||||
.get(side)
|
||||
or {}
|
||||
)
|
||||
actual_cname = {
|
||||
"records": route.get("records"),
|
||||
"ttl": route.get("ttl"),
|
||||
}
|
||||
if actual_cname != expected_cname:
|
||||
violations.append(
|
||||
f"{route_address}: {side} CNAME does not match the exact "
|
||||
"live ALB target and TTL"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
plan = json.loads(args.plan_json.read_text(encoding="utf-8"))
|
||||
|
|
@ -143,13 +190,14 @@ def main() -> int:
|
|||
f"{address}: update is not explicitly allowlisted"
|
||||
)
|
||||
|
||||
if initial_import and args.environment == "dev":
|
||||
if initial_import and args.environment in IMPORT_IDS:
|
||||
if actions != {"no-op"}:
|
||||
violations.append(
|
||||
f"{address}: initial dev import actions must be ['no-op'], "
|
||||
f"{address}: initial {args.environment} import actions "
|
||||
"must be ['no-op'], "
|
||||
f"got {sorted(actions)}"
|
||||
)
|
||||
expected_import_id = DEV_IMPORT_IDS.get(address)
|
||||
expected_import_id = IMPORT_IDS[args.environment].get(address)
|
||||
actual_import_id = (
|
||||
resource.get("change", {}).get("importing") or {}
|
||||
).get("id")
|
||||
|
|
@ -167,6 +215,8 @@ def main() -> int:
|
|||
|
||||
if initial_import and args.environment == "dev":
|
||||
validate_dev_import_baseline(resources_by_address, violations)
|
||||
elif initial_import and args.environment == "staging":
|
||||
validate_staging_import_baseline(resources_by_address, violations)
|
||||
|
||||
if violations:
|
||||
print("FAIL: live Terraform plan is not import-safe", file=sys.stderr)
|
||||
|
|
@ -191,8 +241,8 @@ def main() -> int:
|
|||
for address, resource in sorted(resources_by_address.items())
|
||||
},
|
||||
}
|
||||
if args.environment == "dev" and initial_import:
|
||||
evidence["asserted_live_baseline"] = DEV_IMPORT_BASELINE
|
||||
if args.environment in IMPORT_BASELINES and initial_import:
|
||||
evidence["asserted_live_baseline"] = IMPORT_BASELINES[args.environment]
|
||||
args.evidence_out.write_text(
|
||||
json.dumps(evidence, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
|
|
|
|||
|
|
@ -65,3 +65,54 @@ DEV_IMPORT_BASELINE = {
|
|||
"evaluate_target_health": True,
|
||||
},
|
||||
}
|
||||
|
||||
STAGING_IMPORT_IDS = {
|
||||
"module.environment.aws_elastic_beanstalk_environment.this": "e-6c9m4vb62z",
|
||||
"module.environment.aws_iam_instance_profile.runtime": "shoc-backend-staging",
|
||||
"module.environment.aws_iam_role.github_deploy": (
|
||||
"githubdeploy-shoc-backend-staging"
|
||||
),
|
||||
"module.environment.aws_iam_role.runtime": "shoc-backend-staging",
|
||||
"module.environment.aws_iam_role_policy.github_deploy": (
|
||||
"githubdeploy-shoc-backend-staging:GithubDeployRoleDefaultPolicyE8F540D1"
|
||||
),
|
||||
"module.environment.aws_iam_role_policy.runtime_app_config": (
|
||||
"shoc-backend-staging:shoc-staging-secrets-read"
|
||||
),
|
||||
"module.environment.aws_iam_role_policy.runtime_webhook[0]": (
|
||||
"shoc-backend-staging:shoc-backend-staging-webhook-secret-access"
|
||||
),
|
||||
"module.environment.aws_iam_role_policy_attachment.web_tier": (
|
||||
"shoc-backend-staging/arn:aws:iam::aws:policy/AWSElasticBeanstalkWebTier"
|
||||
),
|
||||
"module.environment.aws_secretsmanager_secret.app_config": (
|
||||
"arn:aws:secretsmanager:us-east-1:396287094661:secret:"
|
||||
"shoc/staging/app-config-CVV99L"
|
||||
),
|
||||
"module.environment.aws_route53_record.api_cname[0]": (
|
||||
"Z02602739VQWBWCAGXP4_api.staging.seahaven.com_CNAME"
|
||||
),
|
||||
}
|
||||
|
||||
STAGING_IMPORT_BASELINE = {
|
||||
"environment_tags": {
|
||||
"env": "staging",
|
||||
"project": "shoc",
|
||||
},
|
||||
"api_cname": {
|
||||
"records": [
|
||||
"awseb--AWSEB-pPXqiRgNnZe8-16996010.us-east-1.elb.amazonaws.com"
|
||||
],
|
||||
"ttl": 60,
|
||||
},
|
||||
}
|
||||
|
||||
IMPORT_IDS = {
|
||||
"dev": DEV_IMPORT_IDS,
|
||||
"staging": STAGING_IMPORT_IDS,
|
||||
}
|
||||
|
||||
IMPORT_BASELINES = {
|
||||
"dev": DEV_IMPORT_BASELINE,
|
||||
"staging": STAGING_IMPORT_BASELINE,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,9 @@ from pathlib import Path
|
|||
|
||||
from terraform_import_plan_resources import (
|
||||
DEV_IMPORT_BASELINE,
|
||||
DEV_IMPORT_IDS,
|
||||
IMPORT_IDS,
|
||||
REQUIRED_RESOURCES,
|
||||
STAGING_IMPORT_BASELINE,
|
||||
)
|
||||
|
||||
SCRIPT = Path(__file__).with_name("check-terraform-import-plan.py")
|
||||
|
|
@ -36,29 +37,40 @@ def run_case(
|
|||
continue
|
||||
actions = (actions_by_address or {}).get(address, ["no-op"])
|
||||
change: dict[str, object] = {"actions": actions}
|
||||
if environment == "dev":
|
||||
if environment in IMPORT_IDS:
|
||||
change["importing"] = {
|
||||
"id": (import_id_overrides or {}).get(
|
||||
address, DEV_IMPORT_IDS[address]
|
||||
address, IMPORT_IDS[environment][address]
|
||||
)
|
||||
}
|
||||
if (
|
||||
address
|
||||
== "module.environment.aws_elastic_beanstalk_environment.this"
|
||||
if address == (
|
||||
"module.environment.aws_elastic_beanstalk_environment.this"
|
||||
):
|
||||
baseline = (
|
||||
DEV_IMPORT_BASELINE
|
||||
if environment == "dev"
|
||||
else STAGING_IMPORT_BASELINE
|
||||
)
|
||||
state: dict[str, object] = {
|
||||
"tags": DEV_IMPORT_BASELINE["environment_tags"],
|
||||
"tags": baseline["environment_tags"],
|
||||
"setting": [],
|
||||
}
|
||||
change["before"] = state
|
||||
change["after"] = state
|
||||
elif (
|
||||
elif environment == "dev" and (
|
||||
address
|
||||
== "module.environment.aws_route53_record.api_alias[0]"
|
||||
):
|
||||
state = {"alias": [DEV_IMPORT_BASELINE["api_alias"]]}
|
||||
change["before"] = state
|
||||
change["after"] = state
|
||||
elif environment == "staging" and (
|
||||
address
|
||||
== "module.environment.aws_route53_record.api_cname[0]"
|
||||
):
|
||||
state = STAGING_IMPORT_BASELINE["api_cname"]
|
||||
change["before"] = state
|
||||
change["after"] = state
|
||||
if address in (state_overrides or {}):
|
||||
change["before"] = (state_overrides or {})[address]
|
||||
change["after"] = (state_overrides or {})[address]
|
||||
|
|
@ -177,6 +189,59 @@ def main() -> int:
|
|||
),
|
||||
1,
|
||||
),
|
||||
(
|
||||
"wrong staging import id",
|
||||
run_case(
|
||||
"staging",
|
||||
import_id_overrides={controlled_address: "wrong-role"},
|
||||
),
|
||||
1,
|
||||
),
|
||||
(
|
||||
"wrong staging environment tags",
|
||||
run_case(
|
||||
"staging",
|
||||
state_overrides={
|
||||
"module.environment.aws_elastic_beanstalk_environment.this": {
|
||||
"tags": {
|
||||
"Name": "shoc-backend-staging",
|
||||
"env": "staging",
|
||||
"project": "shoc",
|
||||
},
|
||||
"setting": [],
|
||||
}
|
||||
},
|
||||
),
|
||||
1,
|
||||
),
|
||||
(
|
||||
"staging managed settings during import",
|
||||
run_case(
|
||||
"staging",
|
||||
state_overrides={
|
||||
"module.environment.aws_elastic_beanstalk_environment.this": {
|
||||
"tags": STAGING_IMPORT_BASELINE["environment_tags"],
|
||||
"setting": [{"name": "ASPNETCORE_ENVIRONMENT"}],
|
||||
}
|
||||
},
|
||||
),
|
||||
1,
|
||||
),
|
||||
(
|
||||
"wrong staging cname",
|
||||
run_case(
|
||||
"staging",
|
||||
state_overrides={
|
||||
"module.environment.aws_route53_record.api_cname[0]": {
|
||||
"records": [
|
||||
"shoc-backend-staging.us-east-1.elasticbeanstalk.com"
|
||||
],
|
||||
"ttl": 60,
|
||||
}
|
||||
},
|
||||
),
|
||||
1,
|
||||
),
|
||||
(
|
||||
"controlled update",
|
||||
run_case(
|
||||
|
|
|
|||
|
|
@ -153,15 +153,23 @@ cannot lock the workspace out from under GitHub CD. GitHub applies only after
|
|||
`plan-output` counts are `0/1/0` and
|
||||
`scripts/check-terraform-release-plan.py` accepts a version-only plan JSON.
|
||||
|
||||
Staging keeps today's direct Elastic Beanstalk deploy path until staging
|
||||
adoption.
|
||||
Staging application CD uses the same guarded lane against workspace
|
||||
`shoc-backend-staging`. The workspace stays branch-based on `staging` with
|
||||
trigger patterns `terraform/live/staging/**` and `terraform/live/modules/**`,
|
||||
and GitHub deploys on pushes to `staging` and on manual `workflow_dispatch`.
|
||||
|
||||
### Credentials and enablement
|
||||
|
||||
Store a dedicated HCP team token only as the GitHub `dev` environment secret
|
||||
`TF_API_TOKEN`. Scope it to workspace `shoc-backend-dev`. Plan JSON download
|
||||
requires workspace admin on that one workspace. Do not grant project admin,
|
||||
workspace create/move/delete, or staging access. Rotate at least every 90 days.
|
||||
Store dedicated HCP team tokens as the GitHub environment secret `TF_API_TOKEN`:
|
||||
|
||||
- `dev`: use a token scoped only to workspace `shoc-backend-dev`.
|
||||
- `staging`: use a separate token scoped only to workspace
|
||||
`shoc-backend-staging`.
|
||||
|
||||
Plan JSON download requires workspace admin on the corresponding workspace. Do
|
||||
not grant project admin or workspace create/move/delete permissions. Do not rely
|
||||
on a repository-level token or reuse the dev-scoped token for staging. Rotate
|
||||
each token at least every 90 days.
|
||||
|
||||
Repository variable `TERRAFORM_APP_CD_ENABLED` starts unset/false so pushes to
|
||||
`dev` do not deploy. `workflow_dispatch` on `dev` still runs a release for the
|
||||
|
|
@ -187,8 +195,10 @@ identifiers make accidental cross-environment reuse fail review and planning.
|
|||
## Safety invariants
|
||||
|
||||
- Auto-apply remains off.
|
||||
- VCS stays branch-based on `dev` with speculative PR plans enabled and
|
||||
trigger patterns `terraform/live/dev/**` and `terraform/live/modules/**`.
|
||||
Do not switch Automatic Run Triggering to tag-based.
|
||||
- VCS stays branch-based on `dev` for `shoc-backend-dev` and on `staging` for
|
||||
`shoc-backend-staging`, with speculative PR plans enabled and trigger
|
||||
patterns `terraform/live/dev/**` (dev) and `terraform/live/staging/**`
|
||||
(staging), each alongside `terraform/live/modules/**`. Do not switch
|
||||
Automatic Run Triggering to tag-based.
|
||||
- Org baseline owns final HCP plan/apply permissions and manager tags.
|
||||
- Every imported Terraform resource has `prevent_destroy`.
|
||||
|
|
|
|||
|
|
@ -500,7 +500,9 @@ resource "aws_route53_record" "api_cname" {
|
|||
name = var.api_domain
|
||||
type = "CNAME"
|
||||
ttl = 60
|
||||
records = [aws_elastic_beanstalk_environment.this.endpoint_url]
|
||||
records = [
|
||||
var.api_cname_target == null ? aws_elastic_beanstalk_environment.this.endpoint_url : var.api_cname_target,
|
||||
]
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
|
|
|
|||
|
|
@ -242,6 +242,12 @@ variable "api_alias_target" {
|
|||
default = null
|
||||
}
|
||||
|
||||
variable "api_cname_target" {
|
||||
type = string
|
||||
description = "Exact existing Route 53 CNAME target preserved during import. Null resolves the target from Elastic Beanstalk."
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "metadata_before_adoption" {
|
||||
description = "Exact current metadata preserved while adoption_complete is false."
|
||||
type = object({
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ module "environment" {
|
|||
aws_region = local.aws_region
|
||||
environment = "staging"
|
||||
adoption_complete = false
|
||||
manage_eb_settings = false
|
||||
eb_application_name = local.eb_application_name
|
||||
eb_environment_name = local.eb_environment_name
|
||||
eb_environment_id = local.eb_environment_id
|
||||
|
|
@ -60,6 +61,8 @@ module "environment" {
|
|||
hosted_zone_id = "Z02602739VQWBWCAGXP4"
|
||||
api_domain = local.api_domain
|
||||
api_record_type = "CNAME"
|
||||
api_cname_target = "awseb--AWSEB-pPXqiRgNnZe8-16996010.us-east-1.elb.amazonaws.com"
|
||||
release_version_label = var.release_version_label
|
||||
metadata_before_adoption = {
|
||||
runtime_role_description = "SHOC backend staging compute role (EB instance profile)"
|
||||
runtime_role_tags = {
|
||||
|
|
@ -81,7 +84,6 @@ module "environment" {
|
|||
Project = "shoc-backend"
|
||||
}
|
||||
environment_tags = {
|
||||
Name = "shoc-backend-staging"
|
||||
env = "staging"
|
||||
project = "shoc"
|
||||
}
|
||||
|
|
|
|||
15
terraform/live/staging/variables.tf
Normal file
15
terraform/live/staging/variables.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
variable "release_version_label" {
|
||||
type = string
|
||||
default = null
|
||||
nullable = true
|
||||
|
||||
description = "Immutable Elastic Beanstalk application version. Null VCS plans leave the live version unchanged."
|
||||
|
||||
validation {
|
||||
condition = (
|
||||
var.release_version_label == null ||
|
||||
can(regex("^[0-9a-f]{40}-[0-9]+-[0-9]+$", var.release_version_label))
|
||||
)
|
||||
error_message = "release_version_label must be <full-sha>-<run-id>-<attempt>."
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue