diff --git a/security-review/lib/sweep_substrate.sh b/security-review/lib/sweep_substrate.sh new file mode 100644 index 0000000..f8883d8 --- /dev/null +++ b/security-review/lib/sweep_substrate.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# sweep_substrate.sh - shared, sourceable substrate for Sea Haven R720 sweeps. +# +# This module factors the reusable concerns out of nightly_sweep.sh (the LIVE sh-secrev +# Path B nightly sweep) so both secrev and the R720 agent-team (a separate track) can +# reuse one implementation. See docs/r720-agent-team-design.md section 7 Phase 0. +# +# Design contract (IMPORTANT - read before editing): +# These functions are extracted VERBATIM from nightly_sweep.sh. They preserve secrev +# behavior exactly. Bash uses dynamic scoping, so a function sourced here closes over +# the CALLER'S variables by name. Each function below documents which caller globals +# it reads or mutates. Callers MUST provide those globals (the names are part of the +# contract); this keeps the extraction zero-behavior-change versus the old inline copy. +# +# bash, stdlib/coreutils only (jq, curl, git, sed, date). No new dependencies. +# +# Usage: +# source "/lib/sweep_substrate.sh" +# ... then call the functions exactly as the inline versions were called. +# +# Functions (each small + individually testable): +# --- budget ledger --- +# add_spend AMOUNT accumulate agentic spend into TOTAL_SPEND (jq exact-add) +# over_budget true if TOTAL_SPEND >= TOTAL_BUDGET_USD (and ceiling > 0) +# --- Slack ALARM-only reporting (with secret redaction) --- +# redact stdin->stdout: mask AWS/GitHub/Slack/high-entropy secrets +# post_slack_alarm TEXT POST the alarm to SLACK_WEBHOOK_URL, or log-only if unset +# --- discovery (org enumeration via REST + GH_TOKEN, no gh CLI) --- +# discover_repos emit "nameclone_urldefault_branch" per non-archived repo +# --- mirror (clean shallow clone; token never persisted to .git/config) --- +# mirror_repo NAME URL BRANCH mirror one repo into MIRROR_DIR/NAME (0 ok / 1 fail) +# --- round-robin rotation (persistent cycle pointer) --- +# to_epoch DATE UTC date string -> epoch seconds (GNU or BSD date) +# --- canary / testbed gate --- +# canary_confirmed_count JSON count confirmed crit+high findings in a review.sh result JSON + +# --- budget ledger ------------------------------------------------------------ +# Reads/mutates caller globals: TOTAL_SPEND. Reads: TOTAL_BUDGET_USD. +add_spend() { TOTAL_SPEND="$(jq -n --argjson a "$TOTAL_SPEND" --argjson b "${1:-0}" '$a + $b')"; } +over_budget() { jq -n --argjson s "$TOTAL_SPEND" --argjson c "$TOTAL_BUDGET_USD" -e '$c > 0 and $s >= $c' >/dev/null; } + +# --- Secret redaction for the Slack string (defense-in-depth; reports stay on the VM) -- +redact() { + sed -E \ + -e 's/AKIA[0-9A-Z]{16}/AKIA****REDACTED****/g' \ + -e 's/gh[pousr]_[A-Za-z0-9]{20,}/gh*_****REDACTED****/g' \ + -e 's/(xox[baprs]-)[A-Za-z0-9-]{10,}/\1****REDACTED****/g' \ + -e 's/[A-Za-z0-9/+]{40,}/****REDACTED-HIENTROPY****/g' +} + +# --- Slack ALARM-only delivery ------------------------------------------------- +# Posts the (already-redacted, already-composed) alarm text. If SLACK_WEBHOOK_URL is +# unset or curl is missing, logs only; the alarm text remains in the sweep log. +# Reads caller globals: SLACK_WEBHOOK_URL, REPORT_DIR, SWEEP_LOG. Uses log() from caller. +post_slack_alarm() { # alarm_text + local slack_text="$1" + if [ -n "${SLACK_WEBHOOK_URL:-}" ] && command -v curl >/dev/null; then + local payload; payload="$(jq -n --arg t "$slack_text" '{text:$t}')" + if curl -fsS -X POST -H 'Content-Type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" >/dev/null 2>>"$REPORT_DIR/slack.log"; then + log "Slack alarm posted." + else + log "Slack POST FAILED — see $REPORT_DIR/slack.log. Alarm text is in $SWEEP_LOG." + fi + else + log "SLACK_WEBHOOK_URL unset (or curl missing) — alarm logged to $SWEEP_LOG only." + fi +} + +# --- Discovery: enumerate non-archived org repos via the REST API ------------- +# Emits "nameclone_urldefault_branch" per repo. Returns non-zero on failure. +# Reads caller globals: GH_TOKEN, GH_ORG, REPORT_DIR. +discover_repos() { + [ -n "${GH_TOKEN:-}" ] || { log "GH_TOKEN unset — cannot enumerate org"; return 1; } + local page=1 got body + while :; do + body="$(curl -fsS \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/$GH_ORG/repos?per_page=100&type=all&page=$page" 2>>"$REPORT_DIR/discover.log")" || return 1 + echo "$body" | jq -e 'type=="array"' >/dev/null 2>&1 || return 1 + got="$(echo "$body" | jq -r '[.[] | select(.archived==false)] | .[] | [.name, .clone_url, .default_branch] | @tsv')" + [ -n "$got" ] && echo "$got" + [ "$(echo "$body" | jq 'length')" -lt 100 ] && break + page=$((page+1)) + done + return 0 +} + +# --- Mirror one repo as a shallow clean clone ------------------------------------------- +# The token is NEVER persisted to .git/config: the fetch path passes the auth URL inline +# (transient, command-args only), and the clone path scrubs origin immediately after. So a +# failed fetch cannot leave GH_TOKEN at rest on disk. (Residual: the token is briefly visible +# in process args to a local `ps`; acceptable on this single-user unattended box.) +# Reads caller globals: MIRROR_DIR, GH_TOKEN. +mirror_repo() { # name clone_url default_branch -> 0 ok / 1 fail + local name="$1" url="$2" branch="$3" + local dir="$MIRROR_DIR/$name" + local auth_url="https://x-access-token:${GH_TOKEN}@${url#https://}" + if [ -d "$dir/.git" ]; then + git -C "$dir" fetch --depth=1 "$auth_url" "$branch" >/dev/null 2>&1 || return 1 + git -C "$dir" reset --hard FETCH_HEAD >/dev/null 2>&1 || return 1 + git -C "$dir" clean -fdq >/dev/null 2>&1 || true + else + git clone --depth=1 --branch "$branch" "$auth_url" "$dir" >/dev/null 2>&1 || return 1 + git -C "$dir" remote set-url origin "$url" >/dev/null 2>&1 || true # clone wrote auth URL → scrub it + fi + return 0 +} + +# --- Rotation helper: portable UTC date-string -> epoch ------------------------ +# Used by the round-robin rotation cycle-age accounting. GNU date (Linux/VM) and BSD +# date (macOS) both handled; unparseable -> 0. +to_epoch() { date -u -d "$1" +%s 2>/dev/null || date -u -j -f '%Y-%m-%d' "$1" +%s 2>/dev/null || echo 0; } + +# --- Canary / testbed gate helper --------------------------------------------- +# Count confirmed crit+high findings in a review.sh result JSON (the anti-complacency +# recall measure). Prints 0 if the file is missing/unreadable. +canary_confirmed_count() { # result_json + local result_json="$1" + if [ -n "$result_json" ] && [ -f "$result_json" ]; then + jq -r '[.findings[]? | select(.status=="confirmed" and (.severity|IN("critical","high")))] | length' "$result_json" 2>/dev/null || echo 0 + else + echo 0 + fi +} diff --git a/security-review/nightly_sweep.sh b/security-review/nightly_sweep.sh index 5a4634d..526efec 100755 --- a/security-review/nightly_sweep.sh +++ b/security-review/nightly_sweep.sh @@ -59,8 +59,15 @@ export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" log() { echo "[nightly_sweep] $*" >&2; } die() { echo "[nightly_sweep] FATAL: $*" >&2; exit 2; } -# --- Config + defaults -------------------------------------------------------- +# --- Shared substrate (discovery / mirror / budget / rotation / Slack / canary) - +# Factored out so secrev and the R720 agent-team reuse one implementation, WITHOUT +# changing any secrev behavior. The functions close over this script's globals by name +# (bash dynamic scoping); see lib/sweep_substrate.sh for the read/mutate contract. HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/sweep_substrate.sh +. "$HERE/lib/sweep_substrate.sh" + +# --- Config + defaults -------------------------------------------------------- ORCHESTRATOR_DIR="${ORCHESTRATOR_DIR:-$HOME/orchestrator}" VENV_PY="${VENV_PY:-$ORCHESTRATOR_DIR/.venv/bin/python}" RUN_HEADLESS="$HERE/run_headless.py" @@ -100,57 +107,10 @@ log "org=$GH_ORG mirror=$MIRROR_DIR report=$REPORT_DIR total-budget=\$$TOTAL_BUD # --- Aggregate state ---------------------------------------------------------- TOTAL_SPEND="0"; BUDGET_HIT=0 declare -a ALARM_LINES=(); declare -a XMODEL_LINES=(); declare -a MARKER_SKIPS=() -add_spend() { TOTAL_SPEND="$(jq -n --argjson a "$TOTAL_SPEND" --argjson b "${1:-0}" '$a + $b')"; } -over_budget() { jq -n --argjson s "$TOTAL_SPEND" --argjson c "$TOTAL_BUDGET_USD" -e '$c > 0 and $s >= $c' >/dev/null; } - -# --- Secret redaction for the Slack string (defense-in-depth; reports stay on the VM) -- -redact() { - sed -E \ - -e 's/AKIA[0-9A-Z]{16}/AKIA****REDACTED****/g' \ - -e 's/gh[pousr]_[A-Za-z0-9]{20,}/gh*_****REDACTED****/g' \ - -e 's/(xox[baprs]-)[A-Za-z0-9-]{10,}/\1****REDACTED****/g' \ - -e 's/[A-Za-z0-9/+]{40,}/****REDACTED-HIENTROPY****/g' -} - -# --- Discovery: enumerate non-archived org repos via the REST API ------------- -# Emits "nameclone_urldefault_branch" per repo. Returns non-zero on failure. -discover_repos() { - [ -n "${GH_TOKEN:-}" ] || { log "GH_TOKEN unset — cannot enumerate org"; return 1; } - local page=1 got body - while :; do - body="$(curl -fsS \ - -H "Authorization: Bearer $GH_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/orgs/$GH_ORG/repos?per_page=100&type=all&page=$page" 2>>"$REPORT_DIR/discover.log")" || return 1 - echo "$body" | jq -e 'type=="array"' >/dev/null 2>&1 || return 1 - got="$(echo "$body" | jq -r '[.[] | select(.archived==false)] | .[] | [.name, .clone_url, .default_branch] | @tsv')" - [ -n "$got" ] && echo "$got" - [ "$(echo "$body" | jq 'length')" -lt 100 ] && break - page=$((page+1)) - done - return 0 -} - -# --- Mirror one repo as a shallow clean clone ------------------------------------------- -# The token is NEVER persisted to .git/config: the fetch path passes the auth URL inline -# (transient, command-args only), and the clone path scrubs origin immediately after. So a -# failed fetch cannot leave GH_TOKEN at rest on disk. (Residual: the token is briefly visible -# in process args to a local `ps`; acceptable on this single-user unattended box.) -mirror_repo() { # name clone_url default_branch -> 0 ok / 1 fail - local name="$1" url="$2" branch="$3" - local dir="$MIRROR_DIR/$name" - local auth_url="https://x-access-token:${GH_TOKEN}@${url#https://}" - if [ -d "$dir/.git" ]; then - git -C "$dir" fetch --depth=1 "$auth_url" "$branch" >/dev/null 2>&1 || return 1 - git -C "$dir" reset --hard FETCH_HEAD >/dev/null 2>&1 || return 1 - git -C "$dir" clean -fdq >/dev/null 2>&1 || true - else - git clone --depth=1 --branch "$branch" "$auth_url" "$dir" >/dev/null 2>&1 || return 1 - git -C "$dir" remote set-url origin "$url" >/dev/null 2>&1 || true # clone wrote auth URL → scrub it - fi - return 0 -} +# add_spend / over_budget (budget ledger), redact (Slack secret redaction), +# discover_repos (org enumeration), mirror_repo (clean shallow clone): provided by +# lib/sweep_substrate.sh, sourced above. They close over the globals defined here +# (TOTAL_SPEND, TOTAL_BUDGET_USD, GH_TOKEN, GH_ORG, MIRROR_DIR, REPORT_DIR). # --- Skip resolution: "" = scan, else reason ("marker"|"central") -------------- declare -a CENTRAL_SKIP=() @@ -243,10 +203,7 @@ CANARY_OK=1 if [ -d "$TESTBED" ]; then log "--- canary (anti-complacency): $TESTBED ---" scan_agentic "$TESTBED" "canary" - CANARY_CONFIRMED=0 - if [ -n "$LAST_RESULT_JSON" ] && [ -f "$LAST_RESULT_JSON" ]; then - CANARY_CONFIRMED="$(jq -r '[.findings[]? | select(.status=="confirmed" and (.severity|IN("critical","high")))] | length' "$LAST_RESULT_JSON" 2>/dev/null || echo 0)" - fi + CANARY_CONFIRMED="$(canary_confirmed_count "$LAST_RESULT_JSON")" log "canary: block=$LAST_BLOCK confirmed(crit+high)=$CANARY_CONFIRMED (floor=$CANARY_FLOOR)" if [ "$LAST_BLOCK" -ne 1 ]; then CANARY_OK=0; ALARM_LINES+=( "*COMPLACENCY ALARM*: canary testbed did NOT block. Result: \`$LAST_RESULT_JSON\`" ) @@ -362,7 +319,6 @@ fi jq -n --arg cs "$CYCLE_START" --argjson sc "$SCANNED_JSON" '{cycle_start:$cs, scanned:$sc}' > "$ROTATION_STATE" # Coverage accounting + lag alarm. REMAINING="$(jq -n --argjson all "$SCANNABLE_JSON" --argjson sc "$SCANNED_JSON" '($all - $sc) | length')" -to_epoch() { date -u -d "$1" +%s 2>/dev/null || date -u -j -f '%Y-%m-%d' "$1" +%s 2>/dev/null || echo 0; } CYCLE_AGE=$(( ( $(to_epoch "$UTC_DATE") - $(to_epoch "$CYCLE_START") ) / 86400 )) log "agentic rotation: scanned $AGENTIC_DONE this night, $REMAINING still pending in cycle (started $CYCLE_START, age ${CYCLE_AGE}d)" if [ "$REMAINING" -gt 0 ] && [ "$CYCLE_AGE" -ge "$MAX_CYCLE_NIGHTS" ]; then @@ -400,16 +356,7 @@ SLACK_TEXT="$(echo "$SLACK_TEXT" | redact)" log "ALARM conditions present — composing Slack post" echo "$SLACK_TEXT" >&2 -if [ -n "${SLACK_WEBHOOK_URL:-}" ] && command -v curl >/dev/null; then - PAYLOAD="$(jq -n --arg t "$SLACK_TEXT" '{text:$t}')" - if curl -fsS -X POST -H 'Content-Type: application/json' --data "$PAYLOAD" "$SLACK_WEBHOOK_URL" >/dev/null 2>>"$REPORT_DIR/slack.log"; then - log "Slack alarm posted." - else - log "Slack POST FAILED — see $REPORT_DIR/slack.log. Alarm text is in $SWEEP_LOG." - fi -else - log "SLACK_WEBHOOK_URL unset (or curl missing) — alarm logged to $SWEEP_LOG only." -fi +post_slack_alarm "$SLACK_TEXT" # An alarm is a reportable condition, not a script crash. Exit 0 so systemd shows success. exit 0