mirror of
https://github.com/Sea-Haven-Industries/.github.git
synced 2026-10-07 04:41:59 +00:00
feat(ci): scan the org job queue with the runner-selector GitHub App
This commit is contained in:
parent
e6eb9de2e5
commit
854703b316
15 changed files with 134 additions and 34 deletions
151
.github/workflows/callable-select-runner.yaml
vendored
151
.github/workflows/callable-select-runner.yaml
vendored
|
|
@ -19,15 +19,21 @@ name: Select runner
|
|||
# slower than that; treating "unknown" as healthy means a network
|
||||
# problem on the self-hosted box does not silently route every run
|
||||
# onto it.
|
||||
# 4. The calling repo's own job queue is consulted. If any job targeting
|
||||
# `primary` has sat in `queued` for more than `max-queue-minutes`
|
||||
# (default 5), hosted runners are not picking up work regardless of
|
||||
# what the status page says, and `fallback` is returned. This catches
|
||||
# the common failure mode of Actions "operational" on paper but
|
||||
# queueing in practice. It only sees the calling repo, so a quiet repo
|
||||
# with nothing in flight gets no signal here and relies on step 3. The
|
||||
# caller must grant `actions: read` to this job; without it the check
|
||||
# logs a warning and is skipped.
|
||||
# 4. The job queue is consulted. If any job targeting `primary` has sat in
|
||||
# `queued` for more than `max-queue-minutes` (default 5), hosted runners
|
||||
# are not picking up work regardless of what the status page says, and
|
||||
# `fallback` is returned. This catches the common failure mode of
|
||||
# Actions "operational" on paper but queueing in practice.
|
||||
#
|
||||
# Scope depends on credentials. With the org `sea-haven-runner-selector`
|
||||
# GitHub App (secrets RUNNER_SELECTOR_APP_ID and
|
||||
# RUNNER_SELECTOR_APP_PRIVATE_KEY, passed via `secrets: inherit`) the
|
||||
# scan covers every non-archived org repo pushed in the last 24h, up to
|
||||
# `scan-repo-limit`. Without the app it covers only the calling repo
|
||||
# using GITHUB_TOKEN, and the caller must grant `actions: read`. If
|
||||
# nothing can be read the check logs a warning and is skipped. Either
|
||||
# way it only observes jobs that are already queued somewhere; a quiet
|
||||
# org gets no signal here and relies on step 3.
|
||||
#
|
||||
# What this cannot do: move a job that is already queued on a hosted runner.
|
||||
# `timeout-minutes` does not start until a runner picks the job up, and a
|
||||
|
|
@ -51,6 +57,7 @@ name: Select runner
|
|||
# uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@<sha> # vX.Y.Z
|
||||
# permissions:
|
||||
# actions: read
|
||||
# secrets: inherit
|
||||
# ci:
|
||||
# needs: select-runner
|
||||
# uses: Sea-Haven-Industries/.github/.github/workflows/ci-terraform.yaml@<sha> # vX.Y.Z
|
||||
|
|
@ -82,12 +89,31 @@ on:
|
|||
default: self-hosted
|
||||
max-queue-minutes:
|
||||
description: >-
|
||||
Return `fallback` when any job in the calling repo targeting
|
||||
`primary` has been queued longer than this many minutes. 0 disables
|
||||
the queue check. Requires the caller to grant `actions: read`.
|
||||
Return `fallback` when any job targeting `primary` has been queued
|
||||
longer than this many minutes. With the app secrets the whole org is
|
||||
scanned; without them only the calling repo is, and the caller must
|
||||
grant `actions: read`. 0 disables the queue check.
|
||||
type: number
|
||||
required: false
|
||||
default: 5
|
||||
scan-repo-limit:
|
||||
description: >-
|
||||
Org-wide scan only. Maximum number of non-archived repos to inspect,
|
||||
most recently pushed first, and only those pushed in the last 24h.
|
||||
type: number
|
||||
required: false
|
||||
default: 30
|
||||
secrets:
|
||||
RUNNER_SELECTOR_APP_ID:
|
||||
description: >-
|
||||
App ID of the org `sea-haven-runner-selector` GitHub App (Actions
|
||||
read, Metadata read, Self-hosted runners read). Pass with
|
||||
`secrets: inherit`. Optional; without it the queue check is limited
|
||||
to the calling repo.
|
||||
required: false
|
||||
RUNNER_SELECTOR_APP_PRIVATE_KEY:
|
||||
description: "Private key for RUNNER_SELECTOR_APP_ID. Optional."
|
||||
required: false
|
||||
outputs:
|
||||
runner:
|
||||
description: "Runner label for downstream `runs-on` and `runner` inputs."
|
||||
|
|
@ -102,16 +128,32 @@ jobs:
|
|||
timeout-minutes: 5
|
||||
outputs:
|
||||
runner: ${{ steps.pick.outputs.runner }}
|
||||
env:
|
||||
# `secrets` is not available in step-level `if`; surface presence here.
|
||||
HAS_APP: ${{ secrets.RUNNER_SELECTOR_APP_ID != '' && secrets.RUNNER_SELECTOR_APP_PRIVATE_KEY != '' }}
|
||||
steps:
|
||||
- name: Mint org-wide read token
|
||||
id: app-token
|
||||
if: env.HAS_APP == 'true' && inputs.max-queue-minutes > 0
|
||||
continue-on-error: true
|
||||
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
||||
with:
|
||||
app-id: ${{ secrets.RUNNER_SELECTOR_APP_ID }}
|
||||
private-key: ${{ secrets.RUNNER_SELECTOR_APP_PRIVATE_KEY }}
|
||||
owner: ${{ github.repository_owner }}
|
||||
|
||||
- name: Pick runner
|
||||
id: pick
|
||||
env:
|
||||
PRIMARY: ${{ inputs.primary }}
|
||||
FALLBACK: ${{ inputs.fallback }}
|
||||
MAX_QUEUE_MINUTES: ${{ inputs.max-queue-minutes }}
|
||||
SCAN_REPO_LIMIT: ${{ inputs.scan-repo-limit }}
|
||||
OVERRIDE: ${{ vars.CI_RUNNER_OVERRIDE }}
|
||||
IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
# App token when minted, else the repo-scoped GITHUB_TOKEN.
|
||||
APP_TOKEN: ${{ steps.app-token.outputs.token }}
|
||||
REPO_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
|
@ -147,41 +189,84 @@ jobs:
|
|||
exit 0 ;;
|
||||
esac
|
||||
|
||||
# Second signal: is this repo already waiting on hosted runners?
|
||||
# Second signal: are hosted jobs already sitting in queue?
|
||||
# Jobs blocked on `needs` are not listed by the jobs API until they
|
||||
# are actually queued, so status == "queued" plus age is a clean
|
||||
# "no runner has picked this up" measure. A token without
|
||||
# actions:read makes every call fail; that degrades to stuck=0 with
|
||||
# a warning rather than failing the job.
|
||||
# "no runner has picked this up" measure. With the app token the
|
||||
# scan covers the org's recently active repos; with only the
|
||||
# repo-scoped GITHUB_TOKEN it covers the calling repo. A token that
|
||||
# cannot read a repo's runs degrades to a warning, not a failure.
|
||||
stuck=0
|
||||
if [ "$MAX_QUEUE_MINUTES" -gt 0 ]; then
|
||||
if [ -n "$APP_TOKEN" ]; then
|
||||
GH_TOKEN="$APP_TOKEN"; scope="org"
|
||||
else
|
||||
GH_TOKEN="$REPO_TOKEN"; scope="repo"
|
||||
if [ "$HAS_APP" = "true" ]; then
|
||||
echo "::warning::App token could not be minted; falling back to a repo-scoped queue check."
|
||||
fi
|
||||
fi
|
||||
|
||||
api() {
|
||||
curl -sf --max-time 10 \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "Authorization: Bearer $GH_TOKEN" \
|
||||
"$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/$1"
|
||||
"$GITHUB_API_URL/$1"
|
||||
}
|
||||
api_ok=true
|
||||
runs=""
|
||||
for status in queued in_progress; do
|
||||
ids=$(api "actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { api_ok=false; break; }
|
||||
runs="$runs $ids"
|
||||
done
|
||||
if [ "$api_ok" = true ]; then
|
||||
|
||||
if [ "$scope" = "org" ]; then
|
||||
# Non-archived repos pushed in the last 24h, most recent first.
|
||||
repos=$(api "orgs/$GITHUB_REPOSITORY_OWNER/repos?type=all&sort=pushed&direction=desc&per_page=100" \
|
||||
| jq -r --argjson limit "$SCAN_REPO_LIMIT" \
|
||||
'[.[] | select((.archived | not) and (.pushed_at | fromdateiso8601) > (now - 86400)) | .full_name]
|
||||
| .[:$limit] | .[]') || repos=""
|
||||
if [ -z "$repos" ]; then
|
||||
echo "::warning::Could not list org repos with the app token; falling back to a repo-scoped queue check."
|
||||
GH_TOKEN="$REPO_TOKEN"; scope="repo"
|
||||
fi
|
||||
fi
|
||||
if [ "$scope" = "repo" ]; then
|
||||
repos="$GITHUB_REPOSITORY"
|
||||
fi
|
||||
|
||||
# One repo per line of output: "ok <count>" or "fail". Repos are
|
||||
# scanned in parallel; the whole scan is bounded by the slowest
|
||||
# repo rather than the sum.
|
||||
scan_repo() {
|
||||
local repo=$1 runs="" ids run n total=0
|
||||
for status in queued in_progress; do
|
||||
ids=$(api "repos/$repo/actions/runs?status=$status&per_page=20" | jq -r '.workflow_runs[].id') || { echo fail; return; }
|
||||
runs="$runs $ids"
|
||||
done
|
||||
for run in $(echo "$runs" | tr ' ' '\n' | sort -u | sed '/^$/d'); do
|
||||
n=$(api "actions/runs/$run/jobs?per_page=100" | jq \
|
||||
n=$(api "repos/$repo/actions/runs/$run/jobs?per_page=100" | jq \
|
||||
--arg primary "$PRIMARY" --argjson max "$MAX_QUEUE_MINUTES" \
|
||||
'[.jobs[] | select(.status == "queued"
|
||||
and (.labels | index($primary))
|
||||
and (.created_at | fromdateiso8601) < (now - $max * 60))]
|
||||
| length') || { api_ok=false; break; }
|
||||
stuck=$((stuck + n))
|
||||
| length') || { echo fail; return; }
|
||||
total=$((total + n))
|
||||
done
|
||||
fi
|
||||
if [ "$api_ok" = true ]; then
|
||||
echo "Hosted jobs queued longer than ${MAX_QUEUE_MINUTES}m in this repo: $stuck"
|
||||
else
|
||||
echo "::warning::Could not read this repo's job queue (does the caller grant actions: read to this job?). Skipping the queue check."
|
||||
echo "ok $total"
|
||||
}
|
||||
# Parallelism is bounded by scan-repo-limit. Each result is one
|
||||
# short line, well under PIPE_BUF, so writes do not interleave.
|
||||
results=$(
|
||||
echo "$repos" | {
|
||||
while read -r repo; do
|
||||
[ -n "$repo" ] && scan_repo "$repo" &
|
||||
done
|
||||
wait
|
||||
}
|
||||
)
|
||||
|
||||
scanned=$(echo "$results" | grep -c '^ok' || true)
|
||||
unreadable=$(echo "$results" | grep -c '^fail' || true)
|
||||
stuck=$(echo "$results" | awk '/^ok/ {s += $2} END {print s + 0}')
|
||||
|
||||
echo "Queue scan ($scope): $scanned repo(s) scanned, $unreadable unreadable, $stuck hosted job(s) queued longer than ${MAX_QUEUE_MINUTES}m"
|
||||
if [ "$scanned" -eq 0 ]; then
|
||||
echo "::warning::No repo's job queue could be read (does the caller grant actions: read to this job, or pass secrets: inherit?). Skipping the queue check."
|
||||
stuck=0
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
1
.github/workflows/ci.yaml
vendored
1
.github/workflows/ci.yaml
vendored
|
|
@ -57,6 +57,7 @@ jobs:
|
|||
uses: ./.github/workflows/callable-select-runner.yaml
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
isolation-tests:
|
||||
name: isolation-tests
|
||||
|
|
|
|||
1
.github/workflows/labeler.yaml
vendored
1
.github/workflows/labeler.yaml
vendored
|
|
@ -25,6 +25,7 @@ jobs:
|
|||
uses: ./.github/workflows/callable-select-runner.yaml
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
label:
|
||||
needs: select-runner
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ The formatter GitHub App is not on the main-branch bypass list.
|
|||
|
||||
**`.github/workflows/callable-dependency-review.yaml`** — Dependency review on PRs, failing on high severity. Requires Dependency Graph.
|
||||
|
||||
**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job in the calling repo targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check needs the caller to grant `permissions: actions: read` on the selector job; without it the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours.
|
||||
**`.github/workflows/callable-select-runner.yaml`** — Runner selector. Outputs `runner`: `self-hosted` when the GitHub status page reports the Actions component as `partial_outage`, `major_outage`, or `under_maintenance`, or when any job targeting `ubuntu-latest` has been queued longer than `max-queue-minutes` (default 5); otherwise `ubuntu-latest`, including on `degraded_performance` and when the status page cannot be read. The queue check scans the whole org (non-archived repos pushed in the last 24h, up to `scan-repo-limit`, default 30) when the caller passes `secrets: inherit` so the selector can mint a token for the org `sea-haven-runner-selector` GitHub App (`RUNNER_SELECTOR_APP_ID`, `RUNNER_SELECTOR_APP_PRIVATE_KEY`; permissions Actions read, Metadata read, Self-hosted runners read). Without the app it scans only the calling repo with `GITHUB_TOKEN`, which needs `permissions: actions: read` on the selector job. If nothing can be read the check is skipped with a warning. Fork PRs always get `ubuntu-latest`. A caller-repo Actions variable `CI_RUNNER_OVERRIDE` forces a value; set it to `self-hosted` to exercise the fallback. Fallback is decided once per run before downstream jobs are queued; a job already waiting on a hosted runner cannot be moved, and `timeout-minutes` does not count queue time. The selector job itself runs on the fallback runner, so adopting it makes the org self-hosted runner a hard dependency of that workflow. Every Linux reusable above takes a `runner` input (default `ubuntu-latest`) to receive the output; `cd-mobile-ios.yaml` is macOS-only and does not. There is no native `runs-on` fallback in GitHub Actions; a label array is an AND match and an unmatched job queues for 24 hours.
|
||||
|
||||
**`.github/workflows/release.yaml`** — Reusable release workflow: creates an annotated git tag at a commit and publishes a GitHub Release pointing at it. The version is an input (not read from a manifest).
|
||||
|
||||
|
|
@ -191,6 +191,8 @@ Managed under **Organization Settings > Secrets and variables > Actions**. Each
|
|||
| `ANTHROPIC_API_KEY` | Anthropic API key | `reviewer-eval.yml` in `open-swe` |
|
||||
| `AUTOFMT_APP_ID` | Formatter GitHub App id | `ci-autofix.yaml` |
|
||||
| `AUTOFMT_APP_PRIVATE_KEY` | Formatter GitHub App private key | `ci-autofix.yaml` |
|
||||
| `RUNNER_SELECTOR_APP_ID` | Runner selector GitHub App id (Actions read, Metadata read, Self-hosted runners read; private-repo visibility) | `callable-select-runner.yaml` |
|
||||
| `RUNNER_SELECTOR_APP_PRIVATE_KEY` | Runner selector GitHub App private key | `callable-select-runner.yaml` |
|
||||
|
||||
The remaining-lane CI and CD workflows below need no org secret — CD authenticates to AWS via OIDC using the per-repo `AWS_DEPLOY_ROLE_ARN` secret (see §3). HCP CD uses `vars.DEPLOY_ROLE_ARN` on the GitHub Environment after OIDC. Adam installs the formatter App (contents: write, metadata: read; not a main-branch ruleset bypass) and grants the two autofmt secrets before the first converted repo runs autofix.
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
# Job id MUST stay `ci`: the reusable's job is also `ci`, so the check
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
autofix:
|
||||
needs: select-runner
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
# Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
needs: select-runner
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
# Job id MUST stay `ci`: the reusable's aggregator job is also `ci`, so the
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
needs: select-runner
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
# Job id MUST stay `ci`: the reusable's job is also `ci`, so the check
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
terraform:
|
||||
needs: select-runner
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
ci:
|
||||
# Job id MUST stay `ci`: the reusable's job is also `ci`, so the check
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
dependency-review:
|
||||
needs: select-runner
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ jobs:
|
|||
uses: Sea-Haven-Industries/.github/.github/workflows/callable-select-runner.yaml@REPLACE-ME # vX.Y.Z
|
||||
permissions:
|
||||
actions: read
|
||||
secrets: inherit
|
||||
|
||||
label:
|
||||
needs: select-runner
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue