From e6eb9de2e5ba3332ed8661152cc633c9b8f32379 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Mon, 5 Oct 2026 18:18:00 -0400 Subject: [PATCH] feat(ci): fall back when hosted jobs sit queued past max-queue-minutes --- .github/workflows/callable-select-runner.yaml | 95 ++++++++++++++++--- .github/workflows/ci.yaml | 2 + .github/workflows/labeler.yaml | 2 + README.md | 2 +- workflow-templates/ci-dotnet.yml | 2 + workflow-templates/ci-hcp.yml | 2 + workflow-templates/ci-mobile-ios.yml | 2 + workflow-templates/ci-node.yml | 2 + workflow-templates/ci-python-app.yml | 2 + workflow-templates/ci-python.yml | 2 + workflow-templates/ci-static.yml | 2 + workflow-templates/ci-terraform.yml | 2 + workflow-templates/ci-typescript-frontend.yml | 2 + workflow-templates/dependency-review.yml | 2 + workflow-templates/labeler.yml | 2 + 15 files changed, 110 insertions(+), 13 deletions(-) diff --git a/.github/workflows/callable-select-runner.yaml b/.github/workflows/callable-select-runner.yaml index c7a4282..c453603 100644 --- a/.github/workflows/callable-select-runner.yaml +++ b/.github/workflows/callable-select-runner.yaml @@ -11,14 +11,28 @@ name: Select runner # to exercise the fallback path on demand, or to `ubuntu-latest` to pin # hosted runners if the status page misreports. Delete it to return to # automatic selection. -# 3. Otherwise the public GitHub status page is consulted. `fallback` is -# returned only when the Actions component reports `partial_outage`, -# `major_outage`, or `under_maintenance`. Everything else, including -# `degraded_performance` and an unreadable status page, returns -# `primary`. Degraded means slow-but-working, and one self-hosted box -# serialising every run is slower than that; treating "unknown" as -# healthy means a network problem on the self-hosted box does not -# silently route every run onto it. +# 3. The public GitHub status page is consulted. `fallback` is returned +# when the Actions component reports `partial_outage`, `major_outage`, +# or `under_maintenance`. `degraded_performance` and an unreadable +# status page do not trigger fallback on their own. Degraded means +# slow-but-working, and one self-hosted box serialising every run is +# slower than that; treating "unknown" as healthy means a network +# problem on the self-hosted box does not silently route every run +# onto it. +# 4. The calling repo's own job queue is consulted. If any job targeting +# `primary` has sat in `queued` for more than `max-queue-minutes` +# (default 5), hosted runners are not picking up work regardless of +# what the status page says, and `fallback` is returned. This catches +# the common failure mode of Actions "operational" on paper but +# queueing in practice. It only sees the calling repo, so a quiet repo +# with nothing in flight gets no signal here and relies on step 3. The +# caller must grant `actions: read` to this job; without it the check +# logs a warning and is skipped. +# +# What this cannot do: move a job that is already queued on a hosted runner. +# `timeout-minutes` does not start until a runner picks the job up, and a +# job's `runs-on` is fixed once created. Fallback is decided up front, once +# per run, before any downstream job is queued. # # This job itself runs on `fallback`. That is the only runner that can be # expected to pick up work when hosted runners are down, so every workflow @@ -35,6 +49,8 @@ name: Select runner # jobs: # select-runner: # uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@ # vX.Y.Z +# permissions: +# actions: read # ci: # needs: select-runner # uses: Sea-Haven-Industries/.github/.github/workflows/ci-terraform.yaml@ # vX.Y.Z @@ -64,12 +80,21 @@ on: type: string required: false default: self-hosted + max-queue-minutes: + description: >- + Return `fallback` when any job in the calling repo targeting + `primary` has been queued longer than this many minutes. 0 disables + the queue check. Requires the caller to grant `actions: read`. + type: number + required: false + default: 5 outputs: runner: description: "Runner label for downstream `runs-on` and `runner` inputs." value: ${{ jobs.select.outputs.runner }} -permissions: {} +permissions: + actions: read jobs: select: @@ -83,8 +108,10 @@ jobs: env: PRIMARY: ${{ inputs.primary }} FALLBACK: ${{ inputs.fallback }} + MAX_QUEUE_MINUTES: ${{ inputs.max-queue-minutes }} OVERRIDE: ${{ vars.CI_RUNNER_OVERRIDE }} IS_FORK: ${{ github.event.pull_request.head.repo.fork }} + GH_TOKEN: ${{ github.token }} shell: bash run: | set -euo pipefail @@ -116,7 +143,51 @@ jobs: # partial_outage, major_outage, under_maintenance. case "$actions_status" in partial_outage|major_outage|under_maintenance) - pick "$FALLBACK" "status $actions_status" ;; - *) - pick "$PRIMARY" "status $actions_status" ;; + pick "$FALLBACK" "status $actions_status" + exit 0 ;; esac + + # Second signal: is this repo already waiting on hosted runners? + # Jobs blocked on `needs` are not listed by the jobs API until they + # are actually queued, so status == "queued" plus age is a clean + # "no runner has picked this up" measure. A token without + # actions:read makes every call fail; that degrades to stuck=0 with + # a warning rather than failing the job. + stuck=0 + if [ "$MAX_QUEUE_MINUTES" -gt 0 ]; then + api() { + curl -sf --max-time 10 \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/$1" + } + api_ok=true + runs="" + for status in queued in_progress; do + ids=$(api "actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { api_ok=false; break; } + runs="$runs $ids" + done + if [ "$api_ok" = true ]; then + for run in $(echo "$runs" | tr ' ' '\n' | sort -u | sed '/^$/d'); do + n=$(api "actions/runs/$run/jobs?per_page=100" | jq \ + --arg primary "$PRIMARY" --argjson max "$MAX_QUEUE_MINUTES" \ + '[.jobs[] | select(.status == "queued" + and (.labels | index($primary)) + and (.created_at | fromdateiso8601) < (now - $max * 60))] + | length') || { api_ok=false; break; } + stuck=$((stuck + n)) + done + fi + if [ "$api_ok" = true ]; then + echo "Hosted jobs queued longer than ${MAX_QUEUE_MINUTES}m in this repo: $stuck" + else + echo "::warning::Could not read this repo's job queue (does the caller grant actions: read to this job?). Skipping the queue check." + stuck=0 + fi + fi + + if [ "$stuck" -gt 0 ]; then + pick "$FALLBACK" "$stuck hosted job(s) queued over ${MAX_QUEUE_MINUTES}m" + else + pick "$PRIMARY" "status $actions_status" + fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 27299f7..aa9c2b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -55,6 +55,8 @@ permissions: jobs: select-runner: uses: ./.github/workflows/callable-select-runner.yaml + permissions: + actions: read isolation-tests: name: isolation-tests diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml index d7cf960..6ecae93 100644 --- a/.github/workflows/labeler.yaml +++ b/.github/workflows/labeler.yaml @@ -23,6 +23,8 @@ permissions: jobs: select-runner: uses: ./.github/workflows/callable-select-runner.yaml + permissions: + actions: read label: needs: select-runner diff --git a/README.md b/README.md index c47acc5..e218203 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ The formatter GitHub App is not on the main-branch bypass list. **`.github/workflows/callable-dependency-review.yaml`** — Dependency review on PRs, failing on high severity. Requires Dependency Graph. -**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`; otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours. +**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job in the calling repo targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check needs the caller to grant `permissions: actions: read` on the selector job; without it the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours. **`.github/workflows/release.yaml`** — Reusable release workflow: creates an annotated git tag at a commit and publishes a GitHub Release pointing at it. The version is an input (not read from a manifest). diff --git a/workflow-templates/ci-dotnet.yml b/workflow-templates/ci-dotnet.yml index b8d20ad..7e31030 100644 --- a/workflow-templates/ci-dotnet.yml +++ b/workflow-templates/ci-dotnet.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check diff --git a/workflow-templates/ci-hcp.yml b/workflow-templates/ci-hcp.yml index 24bc137..0763e61 100644 --- a/workflow-templates/ci-hcp.yml +++ b/workflow-templates/ci-hcp.yml @@ -14,6 +14,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read autofix: needs: select-runner diff --git a/workflow-templates/ci-mobile-ios.yml b/workflow-templates/ci-mobile-ios.yml index 17e4037..689fdd8 100644 --- a/workflow-templates/ci-mobile-ios.yml +++ b/workflow-templates/ci-mobile-ios.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the diff --git a/workflow-templates/ci-node.yml b/workflow-templates/ci-node.yml index 290ec21..7dae92a 100644 --- a/workflow-templates/ci-node.yml +++ b/workflow-templates/ci-node.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: needs: select-runner diff --git a/workflow-templates/ci-python-app.yml b/workflow-templates/ci-python-app.yml index 6c91c42..8096968 100644 --- a/workflow-templates/ci-python-app.yml +++ b/workflow-templates/ci-python-app.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the diff --git a/workflow-templates/ci-python.yml b/workflow-templates/ci-python.yml index 31a5e54..469a789 100644 --- a/workflow-templates/ci-python.yml +++ b/workflow-templates/ci-python.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: needs: select-runner diff --git a/workflow-templates/ci-static.yml b/workflow-templates/ci-static.yml index 43bebdf..0a5189a 100644 --- a/workflow-templates/ci-static.yml +++ b/workflow-templates/ci-static.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check diff --git a/workflow-templates/ci-terraform.yml b/workflow-templates/ci-terraform.yml index 49d3afc..690d1c5 100644 --- a/workflow-templates/ci-terraform.yml +++ b/workflow-templates/ci-terraform.yml @@ -14,6 +14,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read terraform: needs: select-runner diff --git a/workflow-templates/ci-typescript-frontend.yml b/workflow-templates/ci-typescript-frontend.yml index ccb5ef7..19a18e0 100644 --- a/workflow-templates/ci-typescript-frontend.yml +++ b/workflow-templates/ci-typescript-frontend.yml @@ -9,6 +9,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read ci: # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check diff --git a/workflow-templates/dependency-review.yml b/workflow-templates/dependency-review.yml index 8194918..8cee8a4 100644 --- a/workflow-templates/dependency-review.yml +++ b/workflow-templates/dependency-review.yml @@ -11,6 +11,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read dependency-review: needs: select-runner diff --git a/workflow-templates/labeler.yml b/workflow-templates/labeler.yml index a5e5784..531cb7b 100644 --- a/workflow-templates/labeler.yml +++ b/workflow-templates/labeler.yml @@ -16,6 +16,8 @@ jobs: # Picks GitHub-hosted while Actions is operational, the org self-hosted # runner otherwise. See callable-select-runner.yaml for the policy. uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z + permissions: + actions: read label: needs: select-runner