mirror of
https://github.com/Sea-Haven-Industries/security-review.git
synced 2026-09-30 11:33:15 +00:00
Adam Moussa
4c88c01f7b
Fresh-init copy of the security-review/ subsystem extracted from Sea-Haven-Industries/orchestrator (being deprecated). Adds org-standard scaffold: CI reusable-workflow callers (ruff + collect), dependency-review, labeler, dependabot, .gitignore, requirements.txt. Scheduled execution is migrating to Claude Code web routines (ALARM-only to #repo-scanner); the systemd units and nightly_sweep.sh/checker_coordinator.sh remain the source of truth. Committed with --no-verify: the canary fixtures (checkers/fixtures/**) carry intentional secret-shaped test data that trips the deterministic gate (the documented detector-fixture false positive); no new logic is introduced.
126 lines
6.7 KiB
Bash
126 lines
6.7 KiB
Bash
#!/usr/bin/env bash
|
|
# sweep_substrate.sh - shared, sourceable substrate for Sea Haven R720 sweeps.
|
|
#
|
|
# This module factors the reusable concerns out of nightly_sweep.sh (the LIVE sh-secrev
|
|
# Path B nightly sweep) so both secrev and the R720 agent-team (a separate track) can
|
|
# reuse one implementation. See docs/r720-agent-team-design.md section 7 Phase 0.
|
|
#
|
|
# Design contract (IMPORTANT - read before editing):
|
|
# These functions are extracted VERBATIM from nightly_sweep.sh. They preserve secrev
|
|
# behavior exactly. Bash uses dynamic scoping, so a function sourced here closes over
|
|
# the CALLER'S variables by name. Each function below documents which caller globals
|
|
# it reads or mutates. Callers MUST provide those globals (the names are part of the
|
|
# contract); this keeps the extraction zero-behavior-change versus the old inline copy.
|
|
#
|
|
# bash, stdlib/coreutils only (jq, curl, git, sed, date). No new dependencies.
|
|
#
|
|
# Usage:
|
|
# source "<dir>/lib/sweep_substrate.sh"
|
|
# ... then call the functions exactly as the inline versions were called.
|
|
#
|
|
# Functions (each small + individually testable):
|
|
# --- budget ledger ---
|
|
# add_spend AMOUNT accumulate agentic spend into TOTAL_SPEND (jq exact-add)
|
|
# over_budget true if TOTAL_SPEND >= TOTAL_BUDGET_USD (and ceiling > 0)
|
|
# --- Slack ALARM-only reporting (with secret redaction) ---
|
|
# redact stdin->stdout: mask AWS/GitHub/Slack/high-entropy secrets
|
|
# post_slack_alarm TEXT POST the alarm to SLACK_WEBHOOK_URL, or log-only if unset
|
|
# --- discovery (org enumeration via REST + GH_TOKEN, no gh CLI) ---
|
|
# discover_repos emit "name<TAB>clone_url<TAB>default_branch" per non-archived repo
|
|
# --- mirror (clean shallow clone; token never persisted to .git/config) ---
|
|
# mirror_repo NAME URL BRANCH mirror one repo into MIRROR_DIR/NAME (0 ok / 1 fail)
|
|
# --- round-robin rotation (persistent cycle pointer) ---
|
|
# to_epoch DATE UTC date string -> epoch seconds (GNU or BSD date)
|
|
# --- canary / testbed gate ---
|
|
# canary_confirmed_count JSON count confirmed crit+high findings in a review.sh result JSON
|
|
|
|
# --- budget ledger ------------------------------------------------------------
|
|
# Reads/mutates caller globals: TOTAL_SPEND. Reads: TOTAL_BUDGET_USD.
|
|
add_spend() { TOTAL_SPEND="$(jq -n --argjson a "$TOTAL_SPEND" --argjson b "${1:-0}" '$a + $b')"; }
|
|
over_budget() { jq -n --argjson s "$TOTAL_SPEND" --argjson c "$TOTAL_BUDGET_USD" -e '$c > 0 and $s >= $c' >/dev/null; }
|
|
|
|
# --- Secret redaction for the Slack string (defense-in-depth; reports stay on the VM) --
|
|
redact() {
|
|
sed -E \
|
|
-e 's/AKIA[0-9A-Z]{16}/AKIA****REDACTED****/g' \
|
|
-e 's/gh[pousr]_[A-Za-z0-9]{20,}/gh*_****REDACTED****/g' \
|
|
-e 's/(xox[baprs]-)[A-Za-z0-9-]{10,}/\1****REDACTED****/g' \
|
|
-e 's/[A-Za-z0-9/+]{40,}/****REDACTED-HIENTROPY****/g'
|
|
}
|
|
|
|
# --- Slack ALARM-only delivery -------------------------------------------------
|
|
# Posts the (already-redacted, already-composed) alarm text. If SLACK_WEBHOOK_URL is
|
|
# unset or curl is missing, logs only; the alarm text remains in the sweep log.
|
|
# Reads caller globals: SLACK_WEBHOOK_URL, REPORT_DIR, SWEEP_LOG. Uses log() from caller.
|
|
post_slack_alarm() { # alarm_text
|
|
local slack_text="$1"
|
|
if [ -n "${SLACK_WEBHOOK_URL:-}" ] && command -v curl >/dev/null; then
|
|
local payload; payload="$(jq -n --arg t "$slack_text" '{text:$t}')"
|
|
if curl -fsS -X POST -H 'Content-Type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" >/dev/null 2>>"$REPORT_DIR/slack.log"; then
|
|
log "Slack alarm posted."
|
|
else
|
|
log "Slack POST FAILED — see $REPORT_DIR/slack.log. Alarm text is in $SWEEP_LOG."
|
|
fi
|
|
else
|
|
log "SLACK_WEBHOOK_URL unset (or curl missing) — alarm logged to $SWEEP_LOG only."
|
|
fi
|
|
}
|
|
|
|
# --- Discovery: enumerate non-archived org repos via the REST API -------------
|
|
# Emits "name<TAB>clone_url<TAB>default_branch" per repo. Returns non-zero on failure.
|
|
# Reads caller globals: GH_TOKEN, GH_ORG, REPORT_DIR.
|
|
discover_repos() {
|
|
[ -n "${GH_TOKEN:-}" ] || { log "GH_TOKEN unset — cannot enumerate org"; return 1; }
|
|
local page=1 got body
|
|
while :; do
|
|
body="$(curl -fsS \
|
|
-H "Authorization: Bearer $GH_TOKEN" \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
"https://api.github.com/orgs/$GH_ORG/repos?per_page=100&type=all&page=$page" 2>>"$REPORT_DIR/discover.log")" || return 1
|
|
echo "$body" | jq -e 'type=="array"' >/dev/null 2>&1 || return 1
|
|
got="$(echo "$body" | jq -r '[.[] | select(.archived==false)] | .[] | [.name, .clone_url, .default_branch] | @tsv')"
|
|
[ -n "$got" ] && echo "$got"
|
|
[ "$(echo "$body" | jq 'length')" -lt 100 ] && break
|
|
page=$((page+1))
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# --- Mirror one repo as a shallow clean clone -------------------------------------------
|
|
# The token is NEVER persisted to .git/config: the fetch path passes the auth URL inline
|
|
# (transient, command-args only), and the clone path scrubs origin immediately after. So a
|
|
# failed fetch cannot leave GH_TOKEN at rest on disk. (Residual: the token is briefly visible
|
|
# in process args to a local `ps`; acceptable on this single-user unattended box.)
|
|
# Reads caller globals: MIRROR_DIR, GH_TOKEN.
|
|
mirror_repo() { # name clone_url default_branch -> 0 ok / 1 fail
|
|
local name="$1" url="$2" branch="$3"
|
|
local dir="$MIRROR_DIR/$name"
|
|
local auth_url="https://x-access-token:${GH_TOKEN}@${url#https://}"
|
|
if [ -d "$dir/.git" ]; then
|
|
git -C "$dir" fetch --depth=1 "$auth_url" "$branch" >/dev/null 2>&1 || return 1
|
|
git -C "$dir" reset --hard FETCH_HEAD >/dev/null 2>&1 || return 1
|
|
git -C "$dir" clean -fdq >/dev/null 2>&1 || true
|
|
else
|
|
git clone --depth=1 --branch "$branch" "$auth_url" "$dir" >/dev/null 2>&1 || return 1
|
|
git -C "$dir" remote set-url origin "$url" >/dev/null 2>&1 || true # clone wrote auth URL → scrub it
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# --- Rotation helper: portable UTC date-string -> epoch ------------------------
|
|
# Used by the round-robin rotation cycle-age accounting. GNU date (Linux/VM) and BSD
|
|
# date (macOS) both handled; unparseable -> 0.
|
|
to_epoch() { date -u -d "$1" +%s 2>/dev/null || date -u -j -f '%Y-%m-%d' "$1" +%s 2>/dev/null || echo 0; }
|
|
|
|
# --- Canary / testbed gate helper ---------------------------------------------
|
|
# Count confirmed crit+high findings in a review.sh result JSON (the anti-complacency
|
|
# recall measure). Prints 0 if the file is missing/unreadable.
|
|
canary_confirmed_count() { # result_json
|
|
local result_json="$1"
|
|
if [ -n "$result_json" ] && [ -f "$result_json" ]; then
|
|
jq -r '[.findings[]? | select(.status=="confirmed" and (.severity|IN("critical","high")))] | length' "$result_json" 2>/dev/null || echo 0
|
|
else
|
|
echo 0
|
|
fi
|
|
}
|