feat(ci): scan the org job queue with the runner-selector GitHub App

.github/workflows/callable-select-runner.yaml

@@ -19,15 +19,21 @@ name: Select runner
 #      slower than that; treating "unknown" as healthy means a network
 #      problem on the self-hosted box does not silently route every run
 #      onto it.
-#   4. The calling repo's own job queue is consulted. If any job targeting
+#   4. The job queue is consulted. If any job targeting `primary` has sat in
-#      `primary` has sat in `queued` for more than `max-queue-minutes`
+#      `queued` for more than `max-queue-minutes` (default 5), hosted runners
-#      (default 5), hosted runners are not picking up work regardless of
+#      are not picking up work regardless of what the status page says, and
-#      what the status page says, and `fallback` is returned. This catches
+#      `fallback` is returned. This catches the common failure mode of
-#      the common failure mode of Actions "operational" on paper but
+#      Actions "operational" on paper but queueing in practice.
-#      queueing in practice. It only sees the calling repo, so a quiet repo
+#
-#      with nothing in flight gets no signal here and relies on step 3. The
+#      Scope depends on credentials. With the org `sea-haven-runner-selector`
-#      caller must grant `actions: read` to this job; without it the check
+#      GitHub App (secrets RUNNER_SELECTOR_APP_ID and
-#      logs a warning and is skipped.
+#      RUNNER_SELECTOR_APP_PRIVATE_KEY, passed via `secrets: inherit`) the
+#      scan covers every non-archived org repo pushed in the last 24h, up to
+#      `scan-repo-limit`. Without the app it covers only the calling repo
+#      using GITHUB_TOKEN, and the caller must grant `actions: read`. If
+#      nothing can be read the check logs a warning and is skipped. Either
+#      way it only observes jobs that are already queued somewhere; a quiet
+#      org gets no signal here and relies on step 3.
 #
 # What this cannot do: move a job that is already queued on a hosted runner.
 # `timeout-minutes` does not start until a runner picks the job up, and a
@@ -51,6 +57,7 @@ name: Select runner
 #       uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@<sha> # vX.Y.Z
 #       permissions:
 #         actions: read
+#       secrets: inherit
 #     ci:
 #       needs: select-runner
 #       uses: Sea-Haven-Industries/.github/.github/workflows/ci-terraform.yaml@<sha> # vX.Y.Z
@@ -82,12 +89,31 @@ on:
         default: self-hosted
       max-queue-minutes:
         description: >-
-          Return `fallback` when any job in the calling repo targeting
+          Return `fallback` when any job targeting `primary` has been queued
-          `primary` has been queued longer than this many minutes. 0 disables
+          longer than this many minutes. With the app secrets the whole org is
-          the queue check. Requires the caller to grant `actions: read`.
+          scanned; without them only the calling repo is, and the caller must
+          grant `actions: read`. 0 disables the queue check.
         type: number
         required: false
         default: 5
+      scan-repo-limit:
+        description: >-
+          Org-wide scan only. Maximum number of non-archived repos to inspect,
+          most recently pushed first, and only those pushed in the last 24h.
+        type: number
+        required: false
+        default: 30
+    secrets:
+      RUNNER_SELECTOR_APP_ID:
+        description: >-
+          App ID of the org `sea-haven-runner-selector` GitHub App (Actions
+          read, Metadata read, Self-hosted runners read). Pass with
+          `secrets: inherit`. Optional; without it the queue check is limited
+          to the calling repo.
+        required: false
+      RUNNER_SELECTOR_APP_PRIVATE_KEY:
+        description: "Private key for RUNNER_SELECTOR_APP_ID. Optional."
+        required: false
     outputs:
       runner:
         description: "Runner label for downstream `runs-on` and `runner` inputs."
@@ -102,16 +128,32 @@ jobs:
     timeout-minutes: 5
     outputs:
       runner: ${{ steps.pick.outputs.runner }}
+    env:
+      # `secrets` is not available in step-level `if`; surface presence here.
+      HAS_APP: ${{ secrets.RUNNER_SELECTOR_APP_ID != '' && secrets.RUNNER_SELECTOR_APP_PRIVATE_KEY != '' }}
     steps:
+      - name: Mint org-wide read token
+        id: app-token
+        if: env.HAS_APP == 'true' && inputs.max-queue-minutes > 0
+        continue-on-error: true
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.RUNNER_SELECTOR_APP_ID }}
+          private-key: ${{ secrets.RUNNER_SELECTOR_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
       - name: Pick runner
         id: pick
         env:
           PRIMARY: ${{ inputs.primary }}
           FALLBACK: ${{ inputs.fallback }}
           MAX_QUEUE_MINUTES: ${{ inputs.max-queue-minutes }}
+          SCAN_REPO_LIMIT: ${{ inputs.scan-repo-limit }}
           OVERRIDE: ${{ vars.CI_RUNNER_OVERRIDE }}
           IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
-          GH_TOKEN: ${{ github.token }}
+          # App token when minted, else the repo-scoped GITHUB_TOKEN.
+          APP_TOKEN: ${{ steps.app-token.outputs.token }}
+          REPO_TOKEN: ${{ github.token }}
         shell: bash
         run: |
           set -euo pipefail
@@ -147,41 +189,84 @@ jobs:
               exit 0 ;;
           esac
 
-          # Second signal: is this repo already waiting on hosted runners?
+          # Second signal: are hosted jobs already sitting in queue?
           # Jobs blocked on `needs` are not listed by the jobs API until they
           # are actually queued, so status == "queued" plus age is a clean
-          # "no runner has picked this up" measure. A token without
+          # "no runner has picked this up" measure. With the app token the
-          # actions:read makes every call fail; that degrades to stuck=0 with
+          # scan covers the org's recently active repos; with only the
-          # a warning rather than failing the job.
+          # repo-scoped GITHUB_TOKEN it covers the calling repo. A token that
+          # cannot read a repo's runs degrades to a warning, not a failure.
           stuck=0
           if [ "$MAX_QUEUE_MINUTES" -gt 0 ]; then
+            if [ -n "$APP_TOKEN" ]; then
+              GH_TOKEN="$APP_TOKEN"; scope="org"
+            else
+              GH_TOKEN="$REPO_TOKEN"; scope="repo"
+              if [ "$HAS_APP" = "true" ]; then
+                echo "::warning::App token could not be minted; falling back to a repo-scoped queue check."
+              fi
+            fi
+
             api() {
               curl -sf --max-time 10 \
                 -H "Accept: application/vnd.github+json" \
                 -H "Authorization: Bearer $GH_TOKEN" \
-                "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/$1"
+                "$GITHUB_API_URL/$1"
             }
-            api_ok=true
+
-            runs=""
+            if [ "$scope" = "org" ]; then
-            for status in queued in_progress; do
+              # Non-archived repos pushed in the last 24h, most recent first.
-              ids=$(api "actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { api_ok=false; break; }
+              repos=$(api "orgs/$GITHUB_REPOSITORY_OWNER/repos?type=all&sort=pushed&direction=desc&per_page=100" \
-              runs="$runs $ids"
+                | jq -r --argjson limit "$SCAN_REPO_LIMIT" \
-            done
+                  '[.[] | select((.archived | not) and (.pushed_at | fromdateiso8601) > (now - 86400)) | .full_name]
-            if [ "$api_ok" = true ]; then
+                   | .[:$limit] | .[]') || repos=""
+              if [ -z "$repos" ]; then
+                echo "::warning::Could not list org repos with the app token; falling back to a repo-scoped queue check."
+                GH_TOKEN="$REPO_TOKEN"; scope="repo"
+              fi
+            fi
+            if [ "$scope" = "repo" ]; then
+              repos="$GITHUB_REPOSITORY"
+            fi
+
+            # One repo per line of output: "ok <count>" or "fail". Repos are
+            # scanned in parallel; the whole scan is bounded by the slowest
+            # repo rather than the sum.
+            scan_repo() {
+              local repo=$1 runs="" ids run n total=0
+              for status in queued in_progress; do
+                ids=$(api "repos/$repo/actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { echo fail; return; }
+                runs="$runs $ids"
+              done
               for run in $(echo "$runs" | tr ' ' '\n' | sort -u | sed '/^$/d'); do
-                n=$(api "actions/runs/$run/jobs?per_page=100" | jq \
+                n=$(api "repos/$repo/actions/runs/$run/jobs?per_page=100" | jq \
                   --arg primary "$PRIMARY" --argjson max "$MAX_QUEUE_MINUTES" \
                   '[.jobs[] | select(.status == "queued"
                                      and (.labels | index($primary))
                                      and (.created_at | fromdateiso8601) < (now - $max * 60))]
-                   | length') || { api_ok=false; break; }
+                   | length') || { echo fail; return; }
-                stuck=$((stuck + n))
+                total=$((total + n))
               done
-            fi
+              echo "ok $total"
-            if [ "$api_ok" = true ]; then
+            }
-              echo "Hosted jobs queued longer than ${MAX_QUEUE_MINUTES}m in this repo: $stuck"
+            # Parallelism is bounded by scan-repo-limit. Each result is one
-            else
+            # short line, well under PIPE_BUF, so writes do not interleave.
-              echo "::warning::Could not read this repo's job queue (does the caller grant actions: read to this job?). Skipping the queue check."
+            results=$(
+              echo "$repos" | {
+                while read -r repo; do
+                  [ -n "$repo" ] && scan_repo "$repo" &
+                done
+                wait
+              }
+            )
+
+            scanned=$(echo "$results" | grep -c '^ok' || true)
+            unreadable=$(echo "$results" | grep -c '^fail' || true)
+            stuck=$(echo "$results" | awk '/^ok/ {s += $2} END {print s + 0}')
+
+            echo "Queue scan ($scope): $scanned repo(s) scanned, $unreadable unreadable, $stuck hosted job(s) queued longer than ${MAX_QUEUE_MINUTES}m"
+            if [ "$scanned" -eq 0 ]; then
+              echo "::warning::No repo's job queue could be read (does the caller grant actions: read to this job, or pass secrets: inherit?). Skipping the queue check."
               stuck=0
             fi
           fi

.github/workflows/ci.yaml

@@ -57,6 +57,7 @@ jobs:
     uses: ./.github/workflows/callable-select-runner.yaml
     permissions:
       actions: read
+    secrets: inherit
 
   isolation-tests:
     name: isolation-tests

.github/workflows/labeler.yaml

@@ -25,6 +25,7 @@ jobs:
     uses: ./.github/workflows/callable-select-runner.yaml
     permissions:
       actions: read
+    secrets: inherit
 
   label:
     needs: select-runner

README.md

@@ -87,7 +87,7 @@ The formatter GitHub App is not on the main-branch bypass list.
 
 **`.github/workflows/callable-dependency-review.yaml`** — Dependency review on PRs, failing on high severity. Requires Dependency Graph.
 
-**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job in the calling repo targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check needs the caller to grant `permissions: actions: read` on the selector job; without it the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours.
+**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check scans the whole org (non-archived repos pushed in the last 24h, up to `scan-repo-limit`, default 30) when the caller passes `secrets: inherit` so the selector can mint a token for the org `sea-haven-runner-selector` GitHub App (`RUNNER_SELECTOR_APP_ID`, `RUNNER_SELECTOR_APP_PRIVATE_KEY`; permissions Actions read, Metadata read, Self-hosted runners read). Without the app it scans only the calling repo with `GITHUB_TOKEN`, which needs `permissions: actions: read` on the selector job. If nothing can be read the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours.
 
 **`.github/workflows/release.yaml`** — Reusable release workflow: creates an annotated git tag at a commit and publishes a GitHub Release pointing at it. The version is an input (not read from a manifest).
 
@@ -191,6 +191,8 @@ Managed under **Organization Settings > Secrets and variables > Actions**. Each
 | `ANTHROPIC_API_KEY` | Anthropic API key | `reviewer-eval.yml` in `open-swe` |
 | `AUTOFMT_APP_ID` | Formatter GitHub App id | `ci-autofix.yaml` |
 | `AUTOFMT_APP_PRIVATE_KEY` | Formatter GitHub App private key | `ci-autofix.yaml` |
+| `RUNNER_SELECTOR_APP_ID` | Runner selector GitHub App id (Actions read, Metadata read, Self-hosted runners read; private-repo visibility) | `callable-select-runner.yaml` |
+| `RUNNER_SELECTOR_APP_PRIVATE_KEY` | Runner selector GitHub App private key | `callable-select-runner.yaml` |
 
 The remaining-lane CI and CD workflows below need no org secret — CD authenticates to AWS via OIDC using the per-repo `AWS_DEPLOY_ROLE_ARN` secret (see §3). HCP CD uses `vars.DEPLOY_ROLE_ARN` on the GitHub Environment after OIDC. Adam installs the formatter App (contents: write, metadata: read; not a main-branch ruleset bypass) and grants the two autofmt secrets before the first converted repo runs autofix.
 

workflow-templates/ci-dotnet.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check

workflow-templates/ci-hcp.yml

@@ -16,6 +16,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   autofix:
     needs: select-runner

workflow-templates/ci-mobile-ios.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the

workflow-templates/ci-node.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     needs: select-runner

workflow-templates/ci-python-app.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the

workflow-templates/ci-python.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     needs: select-runner

workflow-templates/ci-static.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check

workflow-templates/ci-terraform.yml

@@ -16,6 +16,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   terraform:
     needs: select-runner

workflow-templates/ci-typescript-frontend.yml

@@ -11,6 +11,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   ci:
     # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check

workflow-templates/dependency-review.yml

@@ -13,6 +13,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   dependency-review:
     needs: select-runner

workflow-templates/labeler.yml

@@ -18,6 +18,7 @@ jobs:
     uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
     permissions:
       actions: read
+    secrets: inherit
 
   label:
     needs: select-runner