feat(ci): fall back when hosted jobs sit queued past max-queue-minutes

.github/workflows/callable-select-runner.yaml

@@ -11,14 +11,28 @@ name: Select runner
 #      to exercise the fallback path on demand, or to `ubuntu-latest` to pin
 #      hosted runners if the status page misreports. Delete it to return to
 #      automatic selection.
-#   3. Otherwise the public GitHub status page is consulted. `fallback` is
+#   3. The public GitHub status page is consulted. `fallback` is returned
-#      returned only when the Actions component reports `partial_outage`,
+#      when the Actions component reports `partial_outage`, `major_outage`,
-#      `major_outage`, or `under_maintenance`. Everything else, including
+#      or `under_maintenance`. `degraded_performance` and an unreadable
-#      `degraded_performance` and an unreadable status page, returns
+#      status page do not trigger fallback on their own. Degraded means
-#      `primary`. Degraded means slow-but-working, and one self-hosted box
+#      slow-but-working, and one self-hosted box serialising every run is
-#      serialising every run is slower than that; treating "unknown" as
+#      slower than that; treating "unknown" as healthy means a network
-#      healthy means a network problem on the self-hosted box does not
+#      problem on the self-hosted box does not silently route every run
-#      silently route every run onto it.
+#      onto it.
+#   4. The calling repo's own job queue is consulted. If any job targeting
+#      `primary` has sat in `queued` for more than `max-queue-minutes`
+#      (default 5), hosted runners are not picking up work regardless of
+#      what the status page says, and `fallback` is returned. This catches
+#      the common failure mode of Actions "operational" on paper but
+#      queueing in practice. It only sees the calling repo, so a quiet repo
+#      with nothing in flight gets no signal here and relies on step 3. The
+#      caller must grant `actions: read` to this job; without it the check
+#      logs a warning and is skipped.
+#
+# What this cannot do: move a job that is already queued on a hosted runner.
+# `timeout-minutes` does not start until a runner picks the job up, and a
+# job's `runs-on` is fixed once created. Fallback is decided up front, once
+# per run, before any downstream job is queued.
 #
 # This job itself runs on `fallback`. That is the only runner that can be
 # expected to pick up work when hosted runners are down, so every workflow
@@ -35,6 +49,8 @@ name: Select runner
 #   jobs:
 #     select-runner:
 #       uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@<sha> # vX.Y.Z
+#       permissions:
+#         actions: read
 #     ci:
 #       needs: select-runner
 #       uses: Sea-Haven-Industries/.github/.github/workflows/ci-terraform.yaml@<sha> # vX.Y.Z
@@ -64,12 +80,21 @@ on:
         type: string
         required: false
         default: self-hosted
+      max-queue-minutes:
+        description: >-
+          Return `fallback` when any job in the calling repo targeting
+          `primary` has been queued longer than this many minutes. 0 disables
+          the queue check. Requires the caller to grant `actions: read`.
+        type: number
+        required: false
+        default: 5
     outputs:
       runner:
         description: "Runner label for downstream `runs-on` and `runner` inputs."
         value: ${{ jobs.select.outputs.runner }}
 
-permissions: {}
+permissions:
+  actions: read
 
 jobs:
   select:
@@ -83,8 +108,10 @@ jobs:
         env:
           PRIMARY: ${{ inputs.primary }}
           FALLBACK: ${{ inputs.fallback }}
+          MAX_QUEUE_MINUTES: ${{ inputs.max-queue-minutes }}
           OVERRIDE: ${{ vars.CI_RUNNER_OVERRIDE }}
           IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
+          GH_TOKEN: ${{ github.token }}
         shell: bash
         run: |
           set -euo pipefail
@@ -116,7 +143,51 @@ jobs:
           # partial_outage, major_outage, under_maintenance.
           case "$actions_status" in
             partial_outage|major_outage|under_maintenance)
-              pick "$FALLBACK" "status $actions_status" ;;
+              pick "$FALLBACK" "status $actions_status"
-            *)
+              exit 0 ;;
-              pick "$PRIMARY" "status $actions_status" ;;
           esac
+
+          # Second signal: is this repo already waiting on hosted runners?
+          # Jobs blocked on `needs` are not listed by the jobs API until they
+          # are actually queued, so status == "queued" plus age is a clean
+          # "no runner has picked this up" measure. A token without
+          # actions:read makes every call fail; that degrades to stuck=0 with
+          # a warning rather than failing the job.
+          stuck=0
+          if [ "$MAX_QUEUE_MINUTES" -gt 0 ]; then
+            api() {
+              curl -sf --max-time 10 \
+                -H "Accept: application/vnd.github+json" \
+                -H "Authorization: Bearer $GH_TOKEN" \
+                "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/$1"
+            }
+            api_ok=true
+            runs=""
+            for status in queued in_progress; do
+              ids=$(api "actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { api_ok=false; break; }
+              runs="$runs $ids"
+            done
+            if [ "$api_ok" = true ]; then
+              for run in $(echo "$runs" | tr ' ' '\n' | sort -u | sed '/^$/d'); do
+                n=$(api "actions/runs/$run/jobs?per_page=100" | jq \
+                  --arg primary "$PRIMARY" --argjson max "$MAX_QUEUE_MINUTES" \
+                  '[.jobs[] | select(.status == "queued"
+                                     and (.labels | index($primary))
+                                     and (.created_at | fromdateiso8601) < (now - $max * 60))]
+                   | length') || { api_ok=false; break; }
+                stuck=$((stuck + n))
+              done
+            fi
+            if [ "$api_ok" = true ]; then
+              echo "Hosted jobs queued longer than ${MAX_QUEUE_MINUTES}m in this repo: $stuck"
+            else
+              echo "::warning::Could not read this repo's job queue (does the caller grant actions: read to this job?). Skipping the queue check."
+              stuck=0
+            fi
+          fi
+
+          if [ "$stuck" -gt 0 ]; then
+            pick "$FALLBACK" "$stuck hosted job(s) queued over ${MAX_QUEUE_MINUTES}m"
+          else
+            pick "$PRIMARY" "status $actions_status"
+          fi

.github/workflows/ci.yaml

@@ -55,6 +55,8 @@ permissions:
 jobs:
   select-runner:
     uses: ./.github/workflows/callable-select-runner.yaml
+    permissions:
+      actions: read
 
   isolation-tests:
     name: isolation-tests

.github/workflows/labeler.yaml

@@ -23,6 +23,8 @@ permissions:
 jobs:
   select-runner:
     uses: ./.github/workflows/callable-select-runner.yaml
+    permissions:
+      actions: read
 
   label:
     needs: select-runner

README.md

@@ -87,7 +87,7 @@ The formatter GitHub App is not on the main-branch bypass list.
 
 **`.github/workflows/callable-dependency-review.yaml`** — Dependency review on PRs, failing on high severity. Requires Dependency Graph.
 
-**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`; otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours.
+**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job in the calling repo targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check needs the caller to grant `permissions: actions: read` on the selector job; without it the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours.
 
 **`.github/workflows/release.yaml`** — Reusable release workflow: creates an annotated git tag at a commit and publishes a GitHub Release pointing at it. The version is an input (not read from a manifest).
 

workflow-templates/ci-dotnet.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check

workflow-templates/ci-hcp.yml

@@ -14,6 +14,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   autofix:
     needs: select-runner

workflow-templates/ci-mobile-ios.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the

workflow-templates/ci-node.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     needs: select-runner

workflow-templates/ci-python-app.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the

workflow-templates/ci-python.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     needs: select-runner

workflow-templates/ci-static.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check

workflow-templates/ci-terraform.yml

@@ -14,6 +14,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   terraform:
     needs: select-runner

workflow-templates/ci-typescript-frontend.yml

@@ -9,6 +9,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   ci:
     # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check

workflow-templates/dependency-review.yml

@@ -11,6 +11,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   dependency-review:
     needs: select-runner

workflow-templates/labeler.yml

@@ -16,6 +16,8 @@ jobs:
     # Picks GitHub-hosted while Actions is operational, the org self-hosted
     # runner otherwise. See callable-select-runner.yaml for the policy.
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
+    permissions:
+      actions: read
 
   label:
     needs: select-runner