refactor(secrev): factor shared sweep substrate out of nightly_sweep.sh (Plane-1 Phase 0)
Extract discovery/mirror/budget-ledger/rotation/Slack-ALARM(redaction)/canary into security-review/lib/sweep_substrate.sh (sourceable, bash, stdlib only); nightly_sweep.sh (415->362) now sources it. Zero behavior change proven: shellcheck -x clean, offline two-tier dry-run byte-identical before/after, token discipline (read-only PAT, REST-only, no gh CLI, origin scrubbed) preserved, review.sh untouched. Revert point: 73e35f3. Foundation both planes' scheduled side reuses.
This commit is contained in:
parent
73e35f3acd
commit
1a1facd945
2 changed files with 140 additions and 67 deletions
126
security-review/lib/sweep_substrate.sh
Normal file
126
security-review/lib/sweep_substrate.sh
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
#!/usr/bin/env bash
|
||||
# sweep_substrate.sh - shared, sourceable substrate for Sea Haven R720 sweeps.
|
||||
#
|
||||
# This module factors the reusable concerns out of nightly_sweep.sh (the LIVE sh-secrev
|
||||
# Path B nightly sweep) so both secrev and the R720 agent-team (a separate track) can
|
||||
# reuse one implementation. See docs/r720-agent-team-design.md section 7 Phase 0.
|
||||
#
|
||||
# Design contract (IMPORTANT - read before editing):
|
||||
# These functions are extracted VERBATIM from nightly_sweep.sh. They preserve secrev
|
||||
# behavior exactly. Bash uses dynamic scoping, so a function sourced here closes over
|
||||
# the CALLER'S variables by name. Each function below documents which caller globals
|
||||
# it reads or mutates. Callers MUST provide those globals (the names are part of the
|
||||
# contract); this keeps the extraction zero-behavior-change versus the old inline copy.
|
||||
#
|
||||
# bash, stdlib/coreutils only (jq, curl, git, sed, date). No new dependencies.
|
||||
#
|
||||
# Usage:
|
||||
# source "<dir>/lib/sweep_substrate.sh"
|
||||
# ... then call the functions exactly as the inline versions were called.
|
||||
#
|
||||
# Functions (each small + individually testable):
|
||||
# --- budget ledger ---
|
||||
# add_spend AMOUNT accumulate agentic spend into TOTAL_SPEND (jq exact-add)
|
||||
# over_budget true if TOTAL_SPEND >= TOTAL_BUDGET_USD (and ceiling > 0)
|
||||
# --- Slack ALARM-only reporting (with secret redaction) ---
|
||||
# redact stdin->stdout: mask AWS/GitHub/Slack/high-entropy secrets
|
||||
# post_slack_alarm TEXT POST the alarm to SLACK_WEBHOOK_URL, or log-only if unset
|
||||
# --- discovery (org enumeration via REST + GH_TOKEN, no gh CLI) ---
|
||||
# discover_repos emit "name<TAB>clone_url<TAB>default_branch" per non-archived repo
|
||||
# --- mirror (clean shallow clone; token never persisted to .git/config) ---
|
||||
# mirror_repo NAME URL BRANCH mirror one repo into MIRROR_DIR/NAME (0 ok / 1 fail)
|
||||
# --- round-robin rotation (persistent cycle pointer) ---
|
||||
# to_epoch DATE UTC date string -> epoch seconds (GNU or BSD date)
|
||||
# --- canary / testbed gate ---
|
||||
# canary_confirmed_count JSON count confirmed crit+high findings in a review.sh result JSON
|
||||
|
||||
# --- budget ledger ------------------------------------------------------------
|
||||
# Reads/mutates caller globals: TOTAL_SPEND. Reads: TOTAL_BUDGET_USD.
|
||||
add_spend() { TOTAL_SPEND="$(jq -n --argjson a "$TOTAL_SPEND" --argjson b "${1:-0}" '$a + $b')"; }
|
||||
over_budget() { jq -n --argjson s "$TOTAL_SPEND" --argjson c "$TOTAL_BUDGET_USD" -e '$c > 0 and $s >= $c' >/dev/null; }
|
||||
|
||||
# --- Secret redaction for the Slack string (defense-in-depth; reports stay on the VM) --
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/AKIA[0-9A-Z]{16}/AKIA****REDACTED****/g' \
|
||||
-e 's/gh[pousr]_[A-Za-z0-9]{20,}/gh*_****REDACTED****/g' \
|
||||
-e 's/(xox[baprs]-)[A-Za-z0-9-]{10,}/\1****REDACTED****/g' \
|
||||
-e 's/[A-Za-z0-9/+]{40,}/****REDACTED-HIENTROPY****/g'
|
||||
}
|
||||
|
||||
# --- Slack ALARM-only delivery -------------------------------------------------
|
||||
# Posts the (already-redacted, already-composed) alarm text. If SLACK_WEBHOOK_URL is
|
||||
# unset or curl is missing, logs only; the alarm text remains in the sweep log.
|
||||
# Reads caller globals: SLACK_WEBHOOK_URL, REPORT_DIR, SWEEP_LOG. Uses log() from caller.
|
||||
post_slack_alarm() { # alarm_text
|
||||
local slack_text="$1"
|
||||
if [ -n "${SLACK_WEBHOOK_URL:-}" ] && command -v curl >/dev/null; then
|
||||
local payload; payload="$(jq -n --arg t "$slack_text" '{text:$t}')"
|
||||
if curl -fsS -X POST -H 'Content-Type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" >/dev/null 2>>"$REPORT_DIR/slack.log"; then
|
||||
log "Slack alarm posted."
|
||||
else
|
||||
log "Slack POST FAILED — see $REPORT_DIR/slack.log. Alarm text is in $SWEEP_LOG."
|
||||
fi
|
||||
else
|
||||
log "SLACK_WEBHOOK_URL unset (or curl missing) — alarm logged to $SWEEP_LOG only."
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Discovery: enumerate non-archived org repos via the REST API -------------
|
||||
# Emits "name<TAB>clone_url<TAB>default_branch" per repo. Returns non-zero on failure.
|
||||
# Reads caller globals: GH_TOKEN, GH_ORG, REPORT_DIR.
|
||||
discover_repos() {
|
||||
[ -n "${GH_TOKEN:-}" ] || { log "GH_TOKEN unset — cannot enumerate org"; return 1; }
|
||||
local page=1 got body
|
||||
while :; do
|
||||
body="$(curl -fsS \
|
||||
-H "Authorization: Bearer $GH_TOKEN" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"https://api.github.com/orgs/$GH_ORG/repos?per_page=100&type=all&page=$page" 2>>"$REPORT_DIR/discover.log")" || return 1
|
||||
echo "$body" | jq -e 'type=="array"' >/dev/null 2>&1 || return 1
|
||||
got="$(echo "$body" | jq -r '[.[] | select(.archived==false)] | .[] | [.name, .clone_url, .default_branch] | @tsv')"
|
||||
[ -n "$got" ] && echo "$got"
|
||||
[ "$(echo "$body" | jq 'length')" -lt 100 ] && break
|
||||
page=$((page+1))
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Mirror one repo as a shallow clean clone -------------------------------------------
|
||||
# The token is NEVER persisted to .git/config: the fetch path passes the auth URL inline
|
||||
# (transient, command-args only), and the clone path scrubs origin immediately after. So a
|
||||
# failed fetch cannot leave GH_TOKEN at rest on disk. (Residual: the token is briefly visible
|
||||
# in process args to a local `ps`; acceptable on this single-user unattended box.)
|
||||
# Reads caller globals: MIRROR_DIR, GH_TOKEN.
|
||||
mirror_repo() { # name clone_url default_branch -> 0 ok / 1 fail
|
||||
local name="$1" url="$2" branch="$3"
|
||||
local dir="$MIRROR_DIR/$name"
|
||||
local auth_url="https://x-access-token:${GH_TOKEN}@${url#https://}"
|
||||
if [ -d "$dir/.git" ]; then
|
||||
git -C "$dir" fetch --depth=1 "$auth_url" "$branch" >/dev/null 2>&1 || return 1
|
||||
git -C "$dir" reset --hard FETCH_HEAD >/dev/null 2>&1 || return 1
|
||||
git -C "$dir" clean -fdq >/dev/null 2>&1 || true
|
||||
else
|
||||
git clone --depth=1 --branch "$branch" "$auth_url" "$dir" >/dev/null 2>&1 || return 1
|
||||
git -C "$dir" remote set-url origin "$url" >/dev/null 2>&1 || true # clone wrote auth URL → scrub it
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Rotation helper: portable UTC date-string -> epoch ------------------------
|
||||
# Used by the round-robin rotation cycle-age accounting. GNU date (Linux/VM) and BSD
|
||||
# date (macOS) both handled; unparseable -> 0.
|
||||
to_epoch() { date -u -d "$1" +%s 2>/dev/null || date -u -j -f '%Y-%m-%d' "$1" +%s 2>/dev/null || echo 0; }
|
||||
|
||||
# --- Canary / testbed gate helper ---------------------------------------------
|
||||
# Count confirmed crit+high findings in a review.sh result JSON (the anti-complacency
|
||||
# recall measure). Prints 0 if the file is missing/unreadable.
|
||||
canary_confirmed_count() { # result_json
|
||||
local result_json="$1"
|
||||
if [ -n "$result_json" ] && [ -f "$result_json" ]; then
|
||||
jq -r '[.findings[]? | select(.status=="confirmed" and (.severity|IN("critical","high")))] | length' "$result_json" 2>/dev/null || echo 0
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
|
@ -59,8 +59,15 @@ export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"
|
|||
log() { echo "[nightly_sweep] $*" >&2; }
|
||||
die() { echo "[nightly_sweep] FATAL: $*" >&2; exit 2; }
|
||||
|
||||
# --- Config + defaults --------------------------------------------------------
|
||||
# --- Shared substrate (discovery / mirror / budget / rotation / Slack / canary) -
|
||||
# Factored out so secrev and the R720 agent-team reuse one implementation, WITHOUT
|
||||
# changing any secrev behavior. The functions close over this script's globals by name
|
||||
# (bash dynamic scoping); see lib/sweep_substrate.sh for the read/mutate contract.
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=lib/sweep_substrate.sh
|
||||
. "$HERE/lib/sweep_substrate.sh"
|
||||
|
||||
# --- Config + defaults --------------------------------------------------------
|
||||
ORCHESTRATOR_DIR="${ORCHESTRATOR_DIR:-$HOME/orchestrator}"
|
||||
VENV_PY="${VENV_PY:-$ORCHESTRATOR_DIR/.venv/bin/python}"
|
||||
RUN_HEADLESS="$HERE/run_headless.py"
|
||||
|
|
@ -100,57 +107,10 @@ log "org=$GH_ORG mirror=$MIRROR_DIR report=$REPORT_DIR total-budget=\$$TOTAL_BUD
|
|||
# --- Aggregate state ----------------------------------------------------------
|
||||
TOTAL_SPEND="0"; BUDGET_HIT=0
|
||||
declare -a ALARM_LINES=(); declare -a XMODEL_LINES=(); declare -a MARKER_SKIPS=()
|
||||
add_spend() { TOTAL_SPEND="$(jq -n --argjson a "$TOTAL_SPEND" --argjson b "${1:-0}" '$a + $b')"; }
|
||||
over_budget() { jq -n --argjson s "$TOTAL_SPEND" --argjson c "$TOTAL_BUDGET_USD" -e '$c > 0 and $s >= $c' >/dev/null; }
|
||||
|
||||
# --- Secret redaction for the Slack string (defense-in-depth; reports stay on the VM) --
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/AKIA[0-9A-Z]{16}/AKIA****REDACTED****/g' \
|
||||
-e 's/gh[pousr]_[A-Za-z0-9]{20,}/gh*_****REDACTED****/g' \
|
||||
-e 's/(xox[baprs]-)[A-Za-z0-9-]{10,}/\1****REDACTED****/g' \
|
||||
-e 's/[A-Za-z0-9/+]{40,}/****REDACTED-HIENTROPY****/g'
|
||||
}
|
||||
|
||||
# --- Discovery: enumerate non-archived org repos via the REST API -------------
|
||||
# Emits "name<TAB>clone_url<TAB>default_branch" per repo. Returns non-zero on failure.
|
||||
discover_repos() {
|
||||
[ -n "${GH_TOKEN:-}" ] || { log "GH_TOKEN unset — cannot enumerate org"; return 1; }
|
||||
local page=1 got body
|
||||
while :; do
|
||||
body="$(curl -fsS \
|
||||
-H "Authorization: Bearer $GH_TOKEN" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"https://api.github.com/orgs/$GH_ORG/repos?per_page=100&type=all&page=$page" 2>>"$REPORT_DIR/discover.log")" || return 1
|
||||
echo "$body" | jq -e 'type=="array"' >/dev/null 2>&1 || return 1
|
||||
got="$(echo "$body" | jq -r '[.[] | select(.archived==false)] | .[] | [.name, .clone_url, .default_branch] | @tsv')"
|
||||
[ -n "$got" ] && echo "$got"
|
||||
[ "$(echo "$body" | jq 'length')" -lt 100 ] && break
|
||||
page=$((page+1))
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Mirror one repo as a shallow clean clone -------------------------------------------
|
||||
# The token is NEVER persisted to .git/config: the fetch path passes the auth URL inline
|
||||
# (transient, command-args only), and the clone path scrubs origin immediately after. So a
|
||||
# failed fetch cannot leave GH_TOKEN at rest on disk. (Residual: the token is briefly visible
|
||||
# in process args to a local `ps`; acceptable on this single-user unattended box.)
|
||||
mirror_repo() { # name clone_url default_branch -> 0 ok / 1 fail
|
||||
local name="$1" url="$2" branch="$3"
|
||||
local dir="$MIRROR_DIR/$name"
|
||||
local auth_url="https://x-access-token:${GH_TOKEN}@${url#https://}"
|
||||
if [ -d "$dir/.git" ]; then
|
||||
git -C "$dir" fetch --depth=1 "$auth_url" "$branch" >/dev/null 2>&1 || return 1
|
||||
git -C "$dir" reset --hard FETCH_HEAD >/dev/null 2>&1 || return 1
|
||||
git -C "$dir" clean -fdq >/dev/null 2>&1 || true
|
||||
else
|
||||
git clone --depth=1 --branch "$branch" "$auth_url" "$dir" >/dev/null 2>&1 || return 1
|
||||
git -C "$dir" remote set-url origin "$url" >/dev/null 2>&1 || true # clone wrote auth URL → scrub it
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
# add_spend / over_budget (budget ledger), redact (Slack secret redaction),
|
||||
# discover_repos (org enumeration), mirror_repo (clean shallow clone): provided by
|
||||
# lib/sweep_substrate.sh, sourced above. They close over the globals defined here
|
||||
# (TOTAL_SPEND, TOTAL_BUDGET_USD, GH_TOKEN, GH_ORG, MIRROR_DIR, REPORT_DIR).
|
||||
|
||||
# --- Skip resolution: "" = scan, else reason ("marker"|"central") --------------
|
||||
declare -a CENTRAL_SKIP=()
|
||||
|
|
@ -243,10 +203,7 @@ CANARY_OK=1
|
|||
if [ -d "$TESTBED" ]; then
|
||||
log "--- canary (anti-complacency): $TESTBED ---"
|
||||
scan_agentic "$TESTBED" "canary"
|
||||
CANARY_CONFIRMED=0
|
||||
if [ -n "$LAST_RESULT_JSON" ] && [ -f "$LAST_RESULT_JSON" ]; then
|
||||
CANARY_CONFIRMED="$(jq -r '[.findings[]? | select(.status=="confirmed" and (.severity|IN("critical","high")))] | length' "$LAST_RESULT_JSON" 2>/dev/null || echo 0)"
|
||||
fi
|
||||
CANARY_CONFIRMED="$(canary_confirmed_count "$LAST_RESULT_JSON")"
|
||||
log "canary: block=$LAST_BLOCK confirmed(crit+high)=$CANARY_CONFIRMED (floor=$CANARY_FLOOR)"
|
||||
if [ "$LAST_BLOCK" -ne 1 ]; then
|
||||
CANARY_OK=0; ALARM_LINES+=( "*COMPLACENCY ALARM*: canary testbed did NOT block. Result: \`$LAST_RESULT_JSON\`" )
|
||||
|
|
@ -362,7 +319,6 @@ fi
|
|||
jq -n --arg cs "$CYCLE_START" --argjson sc "$SCANNED_JSON" '{cycle_start:$cs, scanned:$sc}' > "$ROTATION_STATE"
|
||||
# Coverage accounting + lag alarm.
|
||||
REMAINING="$(jq -n --argjson all "$SCANNABLE_JSON" --argjson sc "$SCANNED_JSON" '($all - $sc) | length')"
|
||||
to_epoch() { date -u -d "$1" +%s 2>/dev/null || date -u -j -f '%Y-%m-%d' "$1" +%s 2>/dev/null || echo 0; }
|
||||
CYCLE_AGE=$(( ( $(to_epoch "$UTC_DATE") - $(to_epoch "$CYCLE_START") ) / 86400 ))
|
||||
log "agentic rotation: scanned $AGENTIC_DONE this night, $REMAINING still pending in cycle (started $CYCLE_START, age ${CYCLE_AGE}d)"
|
||||
if [ "$REMAINING" -gt 0 ] && [ "$CYCLE_AGE" -ge "$MAX_CYCLE_NIGHTS" ]; then
|
||||
|
|
@ -400,16 +356,7 @@ SLACK_TEXT="$(echo "$SLACK_TEXT" | redact)"
|
|||
log "ALARM conditions present — composing Slack post"
|
||||
echo "$SLACK_TEXT" >&2
|
||||
|
||||
if [ -n "${SLACK_WEBHOOK_URL:-}" ] && command -v curl >/dev/null; then
|
||||
PAYLOAD="$(jq -n --arg t "$SLACK_TEXT" '{text:$t}')"
|
||||
if curl -fsS -X POST -H 'Content-Type: application/json' --data "$PAYLOAD" "$SLACK_WEBHOOK_URL" >/dev/null 2>>"$REPORT_DIR/slack.log"; then
|
||||
log "Slack alarm posted."
|
||||
else
|
||||
log "Slack POST FAILED — see $REPORT_DIR/slack.log. Alarm text is in $SWEEP_LOG."
|
||||
fi
|
||||
else
|
||||
log "SLACK_WEBHOOK_URL unset (or curl missing) — alarm logged to $SWEEP_LOG only."
|
||||
fi
|
||||
post_slack_alarm "$SLACK_TEXT"
|
||||
|
||||
# An alarm is a reportable condition, not a script crash. Exit 0 so systemd shows success.
|
||||
exit 0
|
||||
|
|
|
|||
Reference in a new issue