diff --git a/.github/workflows/callable-select-runner.yaml b/.github/workflows/callable-select-runner.yaml index c453603..eac3275 100644 --- a/.github/workflows/callable-select-runner.yaml +++ b/.github/workflows/callable-select-runner.yaml @@ -19,15 +19,21 @@ name: Select runner # slower than that; treating "unknown" as healthy means a network # problem on the self-hosted box does not silently route every run # onto it. -# 4. The calling repo's own job queue is consulted. If any job targeting -# `primary` has sat in `queued` for more than `max-queue-minutes` -# (default 5), hosted runners are not picking up work regardless of -# what the status page says, and `fallback` is returned. This catches -# the common failure mode of Actions "operational" on paper but -# queueing in practice. It only sees the calling repo, so a quiet repo -# with nothing in flight gets no signal here and relies on step 3. The -# caller must grant `actions: read` to this job; without it the check -# logs a warning and is skipped. +# 4. The job queue is consulted. If any job targeting `primary` has sat in +# `queued` for more than `max-queue-minutes` (default 5), hosted runners +# are not picking up work regardless of what the status page says, and +# `fallback` is returned. This catches the common failure mode of +# Actions "operational" on paper but queueing in practice. +# +# Scope depends on credentials. With the org `sea-haven-runner-selector` +# GitHub App (secrets RUNNER_SELECTOR_APP_ID and +# RUNNER_SELECTOR_APP_PRIVATE_KEY, passed via `secrets: inherit`) the +# scan covers every non-archived org repo pushed in the last 24h, up to +# `scan-repo-limit`. Without the app it covers only the calling repo +# using GITHUB_TOKEN, and the caller must grant `actions: read`. If +# nothing can be read the check logs a warning and is skipped. Either +# way it only observes jobs that are already queued somewhere; a quiet +# org gets no signal here and relies on step 3. # # What this cannot do: move a job that is already queued on a hosted runner. # `timeout-minutes` does not start until a runner picks the job up, and a @@ -51,6 +57,7 @@ name: Select runner # uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@ # vX.Y.Z # permissions: # actions: read +# secrets: inherit # ci: # needs: select-runner # uses: Sea-Haven-Industries/.github/.github/workflows/ci-terraform.yaml@ # vX.Y.Z @@ -82,12 +89,31 @@ on: default: self-hosted max-queue-minutes: description: >- - Return `fallback` when any job in the calling repo targeting - `primary` has been queued longer than this many minutes. 0 disables - the queue check. Requires the caller to grant `actions: read`. + Return `fallback` when any job targeting `primary` has been queued + longer than this many minutes. With the app secrets the whole org is + scanned; without them only the calling repo is, and the caller must + grant `actions: read`. 0 disables the queue check. type: number required: false default: 5 + scan-repo-limit: + description: >- + Org-wide scan only. Maximum number of non-archived repos to inspect, + most recently pushed first, and only those pushed in the last 24h. + type: number + required: false + default: 30 + secrets: + RUNNER_SELECTOR_APP_ID: + description: >- + App ID of the org `sea-haven-runner-selector` GitHub App (Actions + read, Metadata read, Self-hosted runners read). Pass with + `secrets: inherit`. Optional; without it the queue check is limited + to the calling repo. + required: false + RUNNER_SELECTOR_APP_PRIVATE_KEY: + description: "Private key for RUNNER_SELECTOR_APP_ID. Optional." + required: false outputs: runner: description: "Runner label for downstream `runs-on` and `runner` inputs." @@ -102,16 +128,32 @@ jobs: timeout-minutes: 5 outputs: runner: ${{ steps.pick.outputs.runner }} + env: + # `secrets` is not available in step-level `if`; surface presence here. + HAS_APP: ${{ secrets.RUNNER_SELECTOR_APP_ID != '' && secrets.RUNNER_SELECTOR_APP_PRIVATE_KEY != '' }} steps: + - name: Mint org-wide read token + id: app-token + if: env.HAS_APP == 'true' && inputs.max-queue-minutes > 0 + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RUNNER_SELECTOR_APP_ID }} + private-key: ${{ secrets.RUNNER_SELECTOR_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - name: Pick runner id: pick env: PRIMARY: ${{ inputs.primary }} FALLBACK: ${{ inputs.fallback }} MAX_QUEUE_MINUTES: ${{ inputs.max-queue-minutes }} + SCAN_REPO_LIMIT: ${{ inputs.scan-repo-limit }} OVERRIDE: ${{ vars.CI_RUNNER_OVERRIDE }} IS_FORK: ${{ github.event.pull_request.head.repo.fork }} - GH_TOKEN: ${{ github.token }} + # App token when minted, else the repo-scoped GITHUB_TOKEN. + APP_TOKEN: ${{ steps.app-token.outputs.token }} + REPO_TOKEN: ${{ github.token }} shell: bash run: | set -euo pipefail @@ -147,41 +189,84 @@ jobs: exit 0 ;; esac - # Second signal: is this repo already waiting on hosted runners? + # Second signal: are hosted jobs already sitting in queue? # Jobs blocked on `needs` are not listed by the jobs API until they # are actually queued, so status == "queued" plus age is a clean - # "no runner has picked this up" measure. A token without - # actions:read makes every call fail; that degrades to stuck=0 with - # a warning rather than failing the job. + # "no runner has picked this up" measure. With the app token the + # scan covers the org's recently active repos; with only the + # repo-scoped GITHUB_TOKEN it covers the calling repo. A token that + # cannot read a repo's runs degrades to a warning, not a failure. stuck=0 if [ "$MAX_QUEUE_MINUTES" -gt 0 ]; then + if [ -n "$APP_TOKEN" ]; then + GH_TOKEN="$APP_TOKEN"; scope="org" + else + GH_TOKEN="$REPO_TOKEN"; scope="repo" + if [ "$HAS_APP" = "true" ]; then + echo "::warning::App token could not be minted; falling back to a repo-scoped queue check." + fi + fi + api() { curl -sf --max-time 10 \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GH_TOKEN" \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/$1" + "$GITHUB_API_URL/$1" } - api_ok=true - runs="" - for status in queued in_progress; do - ids=$(api "actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { api_ok=false; break; } - runs="$runs $ids" - done - if [ "$api_ok" = true ]; then + + if [ "$scope" = "org" ]; then + # Non-archived repos pushed in the last 24h, most recent first. + repos=$(api "orgs/$GITHUB_REPOSITORY_OWNER/repos?type=all&sort=pushed&direction=desc&per_page=100" \ + | jq -r --argjson limit "$SCAN_REPO_LIMIT" \ + '[.[] | select((.archived | not) and (.pushed_at | fromdateiso8601) > (now - 86400)) | .full_name] + | .[:$limit] | .[]') || repos="" + if [ -z "$repos" ]; then + echo "::warning::Could not list org repos with the app token; falling back to a repo-scoped queue check." + GH_TOKEN="$REPO_TOKEN"; scope="repo" + fi + fi + if [ "$scope" = "repo" ]; then + repos="$GITHUB_REPOSITORY" + fi + + # One repo per line of output: "ok " or "fail". Repos are + # scanned in parallel; the whole scan is bounded by the slowest + # repo rather than the sum. + scan_repo() { + local repo=$1 runs="" ids run n total=0 + for status in queued in_progress; do + ids=$(api "repos/$repo/actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { echo fail; return; } + runs="$runs $ids" + done for run in $(echo "$runs" | tr ' ' '\n' | sort -u | sed '/^$/d'); do - n=$(api "actions/runs/$run/jobs?per_page=100" | jq \ + n=$(api "repos/$repo/actions/runs/$run/jobs?per_page=100" | jq \ --arg primary "$PRIMARY" --argjson max "$MAX_QUEUE_MINUTES" \ '[.jobs[] | select(.status == "queued" and (.labels | index($primary)) and (.created_at | fromdateiso8601) < (now - $max * 60))] - | length') || { api_ok=false; break; } - stuck=$((stuck + n)) + | length') || { echo fail; return; } + total=$((total + n)) done - fi - if [ "$api_ok" = true ]; then - echo "Hosted jobs queued longer than ${MAX_QUEUE_MINUTES}m in this repo: $stuck" - else - echo "::warning::Could not read this repo's job queue (does the caller grant actions: read to this job?). Skipping the queue check." + echo "ok $total" + } + # Parallelism is bounded by scan-repo-limit. Each result is one + # short line, well under PIPE_BUF, so writes do not interleave. + results=$( + echo "$repos" | { + while read -r repo; do + [ -n "$repo" ] && scan_repo "$repo" & + done + wait + } + ) + + scanned=$(echo "$results" | grep -c '^ok' || true) + unreadable=$(echo "$results" | grep -c '^fail' || true) + stuck=$(echo "$results" | awk '/^ok/ {s += $2} END {print s + 0}') + + echo "Queue scan ($scope): $scanned repo(s) scanned, $unreadable unreadable, $stuck hosted job(s) queued longer than ${MAX_QUEUE_MINUTES}m" + if [ "$scanned" -eq 0 ]; then + echo "::warning::No repo's job queue could be read (does the caller grant actions: read to this job, or pass secrets: inherit?). Skipping the queue check." stuck=0 fi fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index aa9c2b7..3b1ab74 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,6 +57,7 @@ jobs: uses: ./.github/workflows/callable-select-runner.yaml permissions: actions: read + secrets: inherit isolation-tests: name: isolation-tests diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml index 6ecae93..ae7a83f 100644 --- a/.github/workflows/labeler.yaml +++ b/.github/workflows/labeler.yaml @@ -25,6 +25,7 @@ jobs: uses: ./.github/workflows/callable-select-runner.yaml permissions: actions: read + secrets: inherit label: needs: select-runner diff --git a/README.md b/README.md index e218203..e8ea324 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ The formatter GitHub App is not on the main-branch bypass list. **`.github/workflows/callable-dependency-review.yaml`** — Dependency review on PRs, failing on high severity. Requires Dependency Graph. -**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job in the calling repo targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check needs the caller to grant `permissions: actions: read` on the selector job; without it the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours. +**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check scans the whole org (non-archived repos pushed in the last 24h, up to `scan-repo-limit`, default 30) when the caller passes `secrets: inherit` so the selector can mint a token for the org `sea-haven-runner-selector` GitHub App (`RUNNER_SELECTOR_APP_ID`, `RUNNER_SELECTOR_APP_PRIVATE_KEY`; permissions Actions read, Metadata read, Self-hosted runners read). Without the app it scans only the calling repo with `GITHUB_TOKEN`, which needs `permissions: actions: read` on the selector job. If nothing can be read the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours. **`.github/workflows/release.yaml`** — Reusable release workflow: creates an annotated git tag at a commit and publishes a GitHub Release pointing at it. The version is an input (not read from a manifest). @@ -191,6 +191,8 @@ Managed under **Organization Settings > Secrets and variables > Actions**. Each | `ANTHROPIC_API_KEY` | Anthropic API key | `reviewer-eval.yml` in `open-swe` | | `AUTOFMT_APP_ID` | Formatter GitHub App id | `ci-autofix.yaml` | | `AUTOFMT_APP_PRIVATE_KEY` | Formatter GitHub App private key | `ci-autofix.yaml` | +| `RUNNER_SELECTOR_APP_ID` | Runner selector GitHub App id (Actions read, Metadata read, Self-hosted runners read; private-repo visibility) | `callable-select-runner.yaml` | +| `RUNNER_SELECTOR_APP_PRIVATE_KEY` | Runner selector GitHub App private key | `callable-select-runner.yaml` | The remaining-lane CI and CD workflows below need no org secret — CD authenticates to AWS via OIDC using the per-repo `AWS_DEPLOY_ROLE_ARN` secret (see §3). HCP CD uses `vars.DEPLOY_ROLE_ARN` on the GitHub Environment after OIDC. Adam installs the formatter App (contents: write, metadata: read; not a main-branch ruleset bypass) and grants the two autofmt secrets before the first converted repo runs autofix. diff --git a/workflow-templates/ci-dotnet.yml b/workflow-templates/ci-dotnet.yml index 7e31030..c09ca80 100644 --- a/workflow-templates/ci-dotnet.yml +++ b/workflow-templates/ci-dotnet.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check diff --git a/workflow-templates/ci-hcp.yml b/workflow-templates/ci-hcp.yml index 0763e61..6a39cbb 100644 --- a/workflow-templates/ci-hcp.yml +++ b/workflow-templates/ci-hcp.yml @@ -16,6 +16,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit autofix: needs: select-runner diff --git a/workflow-templates/ci-mobile-ios.yml b/workflow-templates/ci-mobile-ios.yml index 689fdd8..4c7adbf 100644 --- a/workflow-templates/ci-mobile-ios.yml +++ b/workflow-templates/ci-mobile-ios.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the diff --git a/workflow-templates/ci-node.yml b/workflow-templates/ci-node.yml index 7dae92a..b7f2c48 100644 --- a/workflow-templates/ci-node.yml +++ b/workflow-templates/ci-node.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: needs: select-runner diff --git a/workflow-templates/ci-python-app.yml b/workflow-templates/ci-python-app.yml index 8096968..f701acb 100644 --- a/workflow-templates/ci-python-app.yml +++ b/workflow-templates/ci-python-app.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: # Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the diff --git a/workflow-templates/ci-python.yml b/workflow-templates/ci-python.yml index 469a789..0176b27 100644 --- a/workflow-templates/ci-python.yml +++ b/workflow-templates/ci-python.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: needs: select-runner diff --git a/workflow-templates/ci-static.yml b/workflow-templates/ci-static.yml index 0a5189a..d7cb9a2 100644 --- a/workflow-templates/ci-static.yml +++ b/workflow-templates/ci-static.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check diff --git a/workflow-templates/ci-terraform.yml b/workflow-templates/ci-terraform.yml index 690d1c5..3e1aeb7 100644 --- a/workflow-templates/ci-terraform.yml +++ b/workflow-templates/ci-terraform.yml @@ -16,6 +16,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit terraform: needs: select-runner diff --git a/workflow-templates/ci-typescript-frontend.yml b/workflow-templates/ci-typescript-frontend.yml index 19a18e0..2b9cfc3 100644 --- a/workflow-templates/ci-typescript-frontend.yml +++ b/workflow-templates/ci-typescript-frontend.yml @@ -11,6 +11,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit ci: # Job id MUST stay `ci`: the reusable's job is also `ci`, so the check diff --git a/workflow-templates/dependency-review.yml b/workflow-templates/dependency-review.yml index 8cee8a4..2344f93 100644 --- a/workflow-templates/dependency-review.yml +++ b/workflow-templates/dependency-review.yml @@ -13,6 +13,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit dependency-review: needs: select-runner diff --git a/workflow-templates/labeler.yml b/workflow-templates/labeler.yml index 531cb7b..c1ec73f 100644 --- a/workflow-templates/labeler.yml +++ b/workflow-templates/labeler.yml @@ -18,6 +18,7 @@ jobs: uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z permissions: actions: read + secrets: inherit label: needs: select-runner