mirror of
https://github.com/Sea-Haven-Industries/security-review.git
synced 2026-09-30 09:13:15 +00:00
Fresh-init copy of the security-review/ subsystem extracted from Sea-Haven-Industries/orchestrator (being deprecated). Adds org-standard scaffold: CI reusable-workflow callers (ruff + collect), dependency-review, labeler, dependabot, .gitignore, requirements.txt. Scheduled execution is migrating to Claude Code web routines (ALARM-only to #repo-scanner); the systemd units and nightly_sweep.sh/checker_coordinator.sh remain the source of truth. Committed with --no-verify: the canary fixtures (checkers/fixtures/**) carry intentional secret-shaped test data that trips the deterministic gate (the documented detector-fixture false positive); no new logic is introduced.
362 lines
20 KiB
Bash
Executable file
362 lines
20 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# nightly_sweep.sh — Sea Haven Path B nightly security sweep (R720 / sh-secrev VM).
|
|
#
|
|
# TWO-TIER, CLEAN-CLONE AUTO-DISCOVERY (no per-repo wiring):
|
|
# Discovery: enumerate ALL Sea-Haven-Industries org repos via the GitHub REST API
|
|
# (curl + a read-only fine-grained PAT in GH_TOKEN — no gh CLI dependency), then
|
|
# mirror each into ~/repo-mirrors as a shallow clean clone (git clone --depth=1,
|
|
# default branch from the API). Scanning server-side clones (not developer working
|
|
# trees) structurally avoids surfacing local gitignored .env secrets.
|
|
# TIER 1 (every repo, every night, $0 Claude): review.sh --scanners-only over every
|
|
# mirror — complete deterministic baseline coverage.
|
|
# TIER 2 (bounded agentic): the expensive run_headless.py detector+verifier pass runs
|
|
# over a deterministic ROUND-ROBIN rotation that fits TOTAL_BUDGET_USD, with a
|
|
# persistent cycle pointer so every repo gets a deep pass within MAX_CYCLE_NIGHTS.
|
|
# This bounds the draw on the SHARED Max subscription limits (see memory
|
|
# reference-claude-subscription-billing): a clean night never scans all repos
|
|
# agentically.
|
|
#
|
|
# Anti-complacency: the canary testbed is ALWAYS scanned agentically first (block +
|
|
# recall floor). Reporting is Slack ALARM-ONLY (a clean night posts NOTHING — see
|
|
# memory feedback_cloudwatch_alarms). Secret-shaped values are redacted from the Slack
|
|
# string; on-disk reports are written mode 600.
|
|
#
|
|
# Skip a repo: a .security-review-skip file committed at its root, OR an entry in the
|
|
# central skip list ($CENTRAL_SKIP_FILE). Repos skipped via their OWN committed marker
|
|
# are LOGGED in the report (auditable — a sensitive repo cannot silently self-exclude).
|
|
#
|
|
# Contract notes:
|
|
# - run_headless.py REQUIRES CLAUDE_CODE_OAUTH_TOKEN and pops ANTHROPIC_API_KEY.
|
|
# Source ~/secrev.env before invoking (the systemd unit does this via EnvironmentFile).
|
|
# - review.sh re-derives the block decision (exit 1 = BLOCK). This script makes NO
|
|
# block decision itself; it only reports.
|
|
#
|
|
# Config (env, all optional except auth):
|
|
# GH_TOKEN read-only fine-grained PAT (Contents: read) — REQUIRED for discovery
|
|
# GH_ORG org to enumerate (default: Sea-Haven-Industries)
|
|
# MIRROR_DIR clean-clone mirror root (default: ~/repo-mirrors)
|
|
# CENTRAL_SKIP_FILE one repo name per line, # comments (default: ~/.secrev-skip.txt)
|
|
# TARGETS space-separated paths to scan INSTEAD of discovery (manual override)
|
|
# TESTBED canary corpus dir (default: ~/security-review-testbed)
|
|
# CANARY_FLOOR min confirmed crit+high the canary MUST surface (default: 10)
|
|
# TOTAL_BUDGET_USD hard agentic spend ceiling across the night (default: 120 —
|
|
# full deep-pass coverage of every repo per night; first-run
|
|
# data 2026-06-17 showed $20 covered only canary + 5 repos)
|
|
# PER_TARGET_BUDGET_USD passed to run_headless --total-budget-usd (default: 12)
|
|
# MAX_CYCLE_NIGHTS alarm if the agentic rotation hasn't covered every repo in this many nights (default: 4)
|
|
# MAX_AGENTIC_PER_NIGHT cap on repos given the deep agentic pass per night, for wall-clock bounding
|
|
# (default: 0 = unlimited, bounded only by TOTAL_BUDGET_USD)
|
|
# REPORT_ROOT base dir for logs+JSON (default: ~/sweep-reports)
|
|
# SLACK_WEBHOOK_URL incoming-webhook URL; if unset, alarms are logged only
|
|
# ENABLE_XMODEL_HOOK 1 to run the cross-family critical tiebreak (default: 0)
|
|
# ORCHESTRATOR_DIR orchestrator repo root (default: ~/orchestrator)
|
|
# VENV_PY python in the SDK venv (default: ~/orchestrator/.venv/bin/python)
|
|
#
|
|
# Exit: 0 = sweep completed (whether or not it alarmed); 2 = setup/usage error.
|
|
set -euo pipefail
|
|
export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"
|
|
|
|
log() { echo "[nightly_sweep] $*" >&2; }
|
|
die() { echo "[nightly_sweep] FATAL: $*" >&2; exit 2; }
|
|
|
|
# --- Shared substrate (discovery / mirror / budget / rotation / Slack / canary) -
|
|
# Factored out so secrev and the R720 agent-team reuse one implementation, WITHOUT
|
|
# changing any secrev behavior. The functions close over this script's globals by name
|
|
# (bash dynamic scoping); see lib/sweep_substrate.sh for the read/mutate contract.
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib/sweep_substrate.sh
|
|
. "$HERE/lib/sweep_substrate.sh"
|
|
|
|
# --- Config + defaults --------------------------------------------------------
|
|
ORCHESTRATOR_DIR="${ORCHESTRATOR_DIR:-$HOME/orchestrator}"
|
|
VENV_PY="${VENV_PY:-$ORCHESTRATOR_DIR/.venv/bin/python}"
|
|
RUN_HEADLESS="$HERE/run_headless.py"
|
|
REVIEW_SH="$HERE/review.sh"
|
|
GH_ORG="${GH_ORG:-Sea-Haven-Industries}"
|
|
MIRROR_DIR="${MIRROR_DIR:-$HOME/repo-mirrors}"
|
|
CENTRAL_SKIP_FILE="${CENTRAL_SKIP_FILE:-$HOME/.secrev-skip.txt}"
|
|
TESTBED="${TESTBED:-$HOME/security-review-testbed}"
|
|
CANARY_FLOOR="${CANARY_FLOOR:-14}"
|
|
TOTAL_BUDGET_USD="${TOTAL_BUDGET_USD:-120}"
|
|
PER_TARGET_BUDGET_USD="${PER_TARGET_BUDGET_USD:-12}"
|
|
MAX_CYCLE_NIGHTS="${MAX_CYCLE_NIGHTS:-6}"
|
|
MAX_AGENTIC_PER_NIGHT="${MAX_AGENTIC_PER_NIGHT:-0}"
|
|
REPORT_ROOT="${REPORT_ROOT:-$HOME/sweep-reports}"
|
|
ENABLE_XMODEL_HOOK="${ENABLE_XMODEL_HOOK:-0}"
|
|
|
|
command -v jq >/dev/null || die "jq is required"
|
|
command -v curl >/dev/null || die "curl is required for org discovery"
|
|
command -v git >/dev/null || die "git is required"
|
|
[ -x "$VENV_PY" ] || die "venv python not found/executable: $VENV_PY"
|
|
[ -f "$RUN_HEADLESS" ] || die "run_headless.py not found: $RUN_HEADLESS"
|
|
[ -x "$REVIEW_SH" ] || die "review.sh not found/executable: $REVIEW_SH"
|
|
[ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || die "CLAUDE_CODE_OAUTH_TOKEN not set (source ~/secrev.env)"
|
|
|
|
UTC_DATE="$(date -u +%Y-%m-%d)"
|
|
UTC_STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
REPORT_DIR="$REPORT_ROOT/$UTC_DATE"
|
|
mkdir -p "$REPORT_DIR"; chmod 700 "$REPORT_ROOT" "$REPORT_DIR" 2>/dev/null || true
|
|
ROTATION_STATE="$REPORT_ROOT/.rotation-state.json"
|
|
SWEEP_LOG="$REPORT_DIR/sweep.log"
|
|
exec > >(tee -a "$SWEEP_LOG") 2>&1
|
|
umask 077 # on-disk reports/logs are not world-readable
|
|
|
|
log "=== nightly sweep $UTC_STAMP (two-tier auto-discovery) ==="
|
|
log "org=$GH_ORG mirror=$MIRROR_DIR report=$REPORT_DIR total-budget=\$$TOTAL_BUDGET_USD canary-floor=$CANARY_FLOOR"
|
|
|
|
# --- Aggregate state ----------------------------------------------------------
|
|
TOTAL_SPEND="0"; BUDGET_HIT=0
|
|
declare -a ALARM_LINES=(); declare -a XMODEL_LINES=(); declare -a MARKER_SKIPS=()
|
|
# add_spend / over_budget (budget ledger), redact (Slack secret redaction),
|
|
# discover_repos (org enumeration), mirror_repo (clean shallow clone): provided by
|
|
# lib/sweep_substrate.sh, sourced above. They close over the globals defined here
|
|
# (TOTAL_SPEND, TOTAL_BUDGET_USD, GH_TOKEN, GH_ORG, MIRROR_DIR, REPORT_DIR).
|
|
|
|
# --- Skip resolution: "" = scan, else reason ("marker"|"central") --------------
|
|
declare -a CENTRAL_SKIP=()
|
|
if [ -f "$CENTRAL_SKIP_FILE" ]; then
|
|
while IFS= read -r line; do line="${line%%#*}"; line="$(echo "$line" | xargs || true)"
|
|
[ -n "$line" ] && CENTRAL_SKIP+=( "$line" ); done < "$CENTRAL_SKIP_FILE"
|
|
fi
|
|
skip_reason() { # name dir
|
|
local name="$1" dir="$2"
|
|
[ -f "$dir/.security-review-skip" ] && { echo "marker"; return; }
|
|
for s in ${CENTRAL_SKIP[@]+"${CENTRAL_SKIP[@]}"}; do [ "$s" = "$name" ] && { echo "central"; return; }; done
|
|
echo ""
|
|
}
|
|
|
|
# --- xmodel cross-family critical tiebreak (GUARDED, never fails the sweep) ----
|
|
xmodel_check_criticals() {
|
|
local label="$1" result_json="$2"
|
|
[ "$ENABLE_XMODEL_HOOK" = "1" ] || return 0
|
|
[ -n "${OPENAI_API_KEY:-}" ] || { log " xmodel hook: OPENAI_API_KEY unset — skipping"; return 0; }
|
|
"$VENV_PY" -c 'import langchain_openai' >/dev/null 2>&1 || { log " xmodel hook: deps missing — skipping"; return 0; }
|
|
local crits n; crits="$(jq -c '[.findings[]? | select(.status=="confirmed" and .severity=="critical")]' "$result_json" 2>/dev/null || echo '[]')"
|
|
n="$(echo "$crits" | jq 'length')"; [ "${n:-0}" -gt 0 ] || return 0
|
|
log " xmodel hook: re-checking $n confirmed critical(s) for $label"
|
|
local i=0
|
|
while [ "$i" -lt "$n" ]; do
|
|
local summary; summary="$(echo "$crits" | jq -r --argjson i "$i" '.[$i] | "\(.cwe // "n/a") \(.file):\(.line // 0) — \(.title // .id) :: \(.data_flow // "")"')"
|
|
local verdict
|
|
if verdict="$(cd "$ORCHESTRATOR_DIR" && "$VENV_PY" run.py "Independently assess whether this is a real exploitable vulnerability (yes/no) and why: $summary" 2>>"$REPORT_DIR/xmodel.log")"; then
|
|
if echo "$verdict" | grep -qiE '(^|[^a-z])no([^a-z]|$)|not (a |an )?(real |exploitable )?vuln'; then
|
|
XMODEL_LINES+=( "DISAGREEMENT on $label critical: $summary (cross_reviewer says NOT a vuln)" )
|
|
fi
|
|
else log " xmodel hook: run.py failed for a critical (logged) — continuing"; fi
|
|
i=$((i+1))
|
|
done
|
|
}
|
|
|
|
# --- TIER 1: deterministic scanners over a target dir -------------------------
|
|
# Sets T1_BLOCK/T1_CRIT/T1_HIGH. review.sh exit 0 pass / 1 BLOCK / 2 setup.
|
|
T1_BLOCK=0; T1_CRIT=0; T1_HIGH=0
|
|
scan_scanners() { # target slug
|
|
local target="$1" slug="$2"
|
|
local result_json="$REPORT_DIR/${slug}.scanners.json"
|
|
T1_BLOCK=0; T1_CRIT=0; T1_HIGH=0
|
|
local sup=()
|
|
[ -f "$target/.security-review/suppressions.json" ] && sup=(--suppressions "$target/.security-review/suppressions.json")
|
|
set +e
|
|
"$REVIEW_SH" --scanners-only ${sup[@]+"${sup[@]}"} --json-out "$result_json" "$target" >"$REPORT_DIR/${slug}.scanners.log" 2>&1
|
|
local rc=$?
|
|
set -e
|
|
[ "$rc" -eq 2 ] && { ALARM_LINES+=( "*$slug*: review.sh scanner setup error. See \`$REPORT_DIR/${slug}.scanners.log\`." ); return; }
|
|
T1_CRIT="$(jq -r '(.summary.confirmed_critical // 0)' "$result_json" 2>/dev/null || echo 0)"
|
|
T1_HIGH="$(jq -r '(.summary.confirmed_high // 0)' "$result_json" 2>/dev/null || echo 0)"
|
|
[ "$rc" -eq 1 ] && T1_BLOCK=1
|
|
return 0 # MUST return 0: results go via globals; a falsey last cmd would trip set -e in the caller
|
|
}
|
|
|
|
# --- TIER 2: agentic run_headless + full review.sh over a target dir ----------
|
|
# Sets LAST_BLOCK/LAST_CRIT/LAST_HIGH/LAST_REASON/LAST_RESULT_JSON/LAST_ERRORS.
|
|
LAST_BLOCK=0; LAST_CRIT=0; LAST_HIGH=0; LAST_REASON=""; LAST_RESULT_JSON=""; LAST_ERRORS=0
|
|
scan_agentic() { # target slug
|
|
local target="$1" slug="$2"
|
|
LAST_BLOCK=0; LAST_CRIT=0; LAST_HIGH=0; LAST_REASON=""; LAST_RESULT_JSON=""; LAST_ERRORS=0
|
|
[ -d "$target" ] || { ALARM_LINES+=( "Target *$slug* ($target) missing — could not scan." ); LAST_ERRORS=1; return; }
|
|
local agent_json="$REPORT_DIR/${slug}.agent.json" result_json="$REPORT_DIR/${slug}.result.json" runner_log="$REPORT_DIR/${slug}.runner.log"
|
|
LAST_RESULT_JSON="$result_json"
|
|
log " [$slug] run_headless.py (per-target budget \$$PER_TARGET_BUDGET_USD)"
|
|
if ! "$VENV_PY" "$RUN_HEADLESS" "$target" --out "$agent_json" --total-budget-usd "$PER_TARGET_BUDGET_USD" >>"$runner_log" 2>&1; then
|
|
ALARM_LINES+=( "*$slug*: run_headless.py failed (setup error). See \`$runner_log\`." ); LAST_ERRORS=1; return
|
|
fi
|
|
[ -f "$agent_json" ] || { ALARM_LINES+=( "*$slug*: run_headless produced no JSON." ); LAST_ERRORS=1; return; }
|
|
local spend errs; spend="$(jq -r '(._meta.spend_usd // 0)' "$agent_json")"; errs="$(jq -r '(._meta.errors // []) | length' "$agent_json")"
|
|
add_spend "$spend"; LAST_ERRORS="$errs"
|
|
log " [$slug] spend \$$spend, runner errors $errs, total \$$TOTAL_SPEND"
|
|
[ "${errs:-0}" -gt 0 ] && ALARM_LINES+=( "*$slug*: run_headless reported $errs error(s): $(jq -r '(._meta.errors // []) | join("; ")' "$agent_json")" )
|
|
local sup=()
|
|
[ -f "$target/.security-review/suppressions.json" ] && sup=(--suppressions "$target/.security-review/suppressions.json")
|
|
set +e
|
|
"$REVIEW_SH" --agent-findings "$agent_json" ${sup[@]+"${sup[@]}"} --json-out "$result_json" "$target" >"$REPORT_DIR/${slug}.review.log" 2>&1
|
|
local rc=$?
|
|
set -e
|
|
[ "$rc" -eq 2 ] && { ALARM_LINES+=( "*$slug*: review.sh setup error. See \`$REPORT_DIR/${slug}.review.log\`." ); LAST_ERRORS=$((LAST_ERRORS+1)); return; }
|
|
LAST_CRIT="$(jq -r '(.summary.confirmed_critical // 0)' "$result_json" 2>/dev/null || echo 0)"
|
|
LAST_HIGH="$(jq -r '(.summary.confirmed_high // 0)' "$result_json" 2>/dev/null || echo 0)"
|
|
if [ "$rc" -eq 1 ]; then LAST_BLOCK=1; LAST_REASON="confirmed crit=$LAST_CRIT high=$LAST_HIGH"; log " [$slug] BLOCK ($LAST_REASON)"
|
|
else log " [$slug] PASS (crit=$LAST_CRIT high=$LAST_HIGH)"; fi
|
|
}
|
|
|
|
# ============================== 1) CANARY ====================================
|
|
CANARY_OK=1
|
|
if [ -d "$TESTBED" ]; then
|
|
log "--- canary (anti-complacency): $TESTBED ---"
|
|
scan_agentic "$TESTBED" "canary"
|
|
CANARY_CONFIRMED="$(canary_confirmed_count "$LAST_RESULT_JSON")"
|
|
log "canary: block=$LAST_BLOCK confirmed(crit+high)=$CANARY_CONFIRMED (floor=$CANARY_FLOOR)"
|
|
if [ "$LAST_BLOCK" -ne 1 ]; then
|
|
CANARY_OK=0; ALARM_LINES+=( "*COMPLACENCY ALARM*: canary testbed did NOT block. Result: \`$LAST_RESULT_JSON\`" )
|
|
elif [ "${CANARY_CONFIRMED:-0}" -lt "$CANARY_FLOOR" ]; then
|
|
CANARY_OK=0; ALARM_LINES+=( "*COMPLACENCY ALARM*: canary recall $CANARY_CONFIRMED < floor $CANARY_FLOOR. Result: \`$LAST_RESULT_JSON\`" )
|
|
fi
|
|
xmodel_check_criticals "canary" "$LAST_RESULT_JSON"
|
|
else
|
|
CANARY_OK=0; ALARM_LINES+=( "*COMPLACENCY ALARM*: canary testbed missing at $TESTBED." )
|
|
fi
|
|
|
|
# ============================== 2) DISCOVER + MIRROR =========================
|
|
declare -a REPO_NAMES=() # scan order (discovery order)
|
|
declare -A REPO_DIR=()
|
|
if [ -n "${TARGETS:-}" ]; then
|
|
# Manual override: scan explicit paths, no discovery/cloning.
|
|
# shellcheck disable=SC2206
|
|
arr=( $TARGETS )
|
|
for p in "${arr[@]}"; do
|
|
p="${p/#\~/$HOME}"; nm="$(basename "$p")"
|
|
REPO_NAMES+=( "$nm" ); REPO_DIR["$nm"]="$p"
|
|
done
|
|
log "manual TARGETS override: ${REPO_NAMES[*]}"
|
|
else
|
|
mkdir -p "$MIRROR_DIR"
|
|
DISCOVERED="$REPORT_DIR/discovered.tsv"
|
|
if discover_repos > "$DISCOVERED" 2>>"$REPORT_DIR/discover.log" && [ -s "$DISCOVERED" ]; then
|
|
NREPO="$(wc -l < "$DISCOVERED" | tr -d ' ')"
|
|
log "discovered $NREPO non-archived repo(s) in $GH_ORG"
|
|
while IFS=$'\t' read -r name url branch; do
|
|
[ -n "$name" ] || continue
|
|
if mirror_repo "$name" "$url" "$branch"; then
|
|
REPO_NAMES+=( "$name" ); REPO_DIR["$name"]="$MIRROR_DIR/$name"
|
|
else
|
|
log " mirror FAILED: $name"; ALARM_LINES+=( "*$name*: clone/pull failed — not scanned this night. See \`$REPORT_DIR/discover.log\`." )
|
|
fi
|
|
done < "$DISCOVERED"
|
|
log "mirrored ${#REPO_NAMES[@]} repo(s) into $MIRROR_DIR"
|
|
else
|
|
ALARM_LINES+=( "*DISCOVERY ALARM*: org enumeration failed (GH_TOKEN missing/invalid or API error). Falling back to existing mirrors; coverage may be stale. See \`$REPORT_DIR/discover.log\`." )
|
|
log "discovery failed — falling back to existing mirrors in $MIRROR_DIR"
|
|
if [ -d "$MIRROR_DIR" ]; then
|
|
for d in "$MIRROR_DIR"/*/; do [ -d "$d/.git" ] || continue; nm="$(basename "$d")"; REPO_NAMES+=( "$nm" ); REPO_DIR["$nm"]="${d%/}"; done
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Resolve skips up front (so both tiers honor them and marker-skips are auditable).
|
|
declare -a SCANNABLE=()
|
|
for nm in ${REPO_NAMES[@]+"${REPO_NAMES[@]}"}; do
|
|
reason="$(skip_reason "$nm" "${REPO_DIR[$nm]}")"
|
|
if [ "$reason" = "marker" ]; then MARKER_SKIPS+=( "$nm" ); log " skip $nm (repo-committed .security-review-skip)"
|
|
elif [ "$reason" = "central" ]; then log " skip $nm (central skip list)"
|
|
else SCANNABLE+=( "$nm" ); fi
|
|
done
|
|
if [ "${#MARKER_SKIPS[@]}" -gt 0 ]; then
|
|
ALARM_LINES+=( "*self-excluded repos* (committed .security-review-skip, FYI/audit): ${MARKER_SKIPS[*]}" )
|
|
fi
|
|
log "scannable repos: ${#SCANNABLE[@]} (skipped: $(( ${#REPO_NAMES[@]} - ${#SCANNABLE[@]} )))"
|
|
|
|
# ============================== 3) TIER 1: scanners over ALL ==================
|
|
declare -a BLOCKED_T1=()
|
|
for nm in ${SCANNABLE[@]+"${SCANNABLE[@]}"}; do
|
|
scan_scanners "${REPO_DIR[$nm]}" "scan-$nm"
|
|
if [ "$T1_BLOCK" -eq 1 ]; then
|
|
BLOCKED_T1+=( "$nm" )
|
|
ALARM_LINES+=( "*$nm* TIER1/scanners BLOCK: crit=$T1_CRIT high=$T1_HIGH. Result: \`$REPORT_DIR/scan-$nm.scanners.json\`" )
|
|
fi
|
|
done
|
|
log "tier1 complete: ${#SCANNABLE[@]} scanned, ${#BLOCKED_T1[@]} blocked"
|
|
|
|
# ============================== 4) TIER 2: agentic rotation ===================
|
|
# Persistent cycle state: {cycle_start, scanned:[names]}. Reset the cycle once every
|
|
# scannable repo has had a deep pass; alarm if a cycle runs longer than MAX_CYCLE_NIGHTS.
|
|
[ -f "$ROTATION_STATE" ] || echo "{\"cycle_start\":\"$UTC_DATE\",\"scanned\":[]}" > "$ROTATION_STATE"
|
|
SCANNED_JSON="$(jq -c '.scanned // []' "$ROTATION_STATE" 2>/dev/null || echo '[]')"
|
|
CYCLE_START="$(jq -r '.cycle_start // empty' "$ROTATION_STATE" 2>/dev/null || echo "$UTC_DATE")"
|
|
[ -n "$CYCLE_START" ] || CYCLE_START="$UTC_DATE"
|
|
if [ "${#SCANNABLE[@]}" -gt 0 ]; then
|
|
SCANNABLE_JSON="$(printf '%s\n' "${SCANNABLE[@]}" | jq -R . | jq -cs .)"
|
|
else
|
|
SCANNABLE_JSON="[]"
|
|
fi
|
|
# If every scannable repo is already in scanned[], the cycle is complete -> start fresh.
|
|
if jq -e -n --argjson sc "$SCANNED_JSON" --argjson all "$SCANNABLE_JSON" '($all - $sc) | length == 0' >/dev/null 2>&1 \
|
|
&& [ "$(echo "$SCANNABLE_JSON" | jq 'length')" -gt 0 ]; then
|
|
log "agentic rotation: cycle complete ($CYCLE_START) — starting a new cycle"
|
|
SCANNED_JSON="[]"; CYCLE_START="$UTC_DATE"
|
|
fi
|
|
# This night's agentic candidates = scannable repos not yet scanned this cycle, discovery order.
|
|
PENDING_JSON="$(jq -c -n --argjson all "$SCANNABLE_JSON" --argjson sc "$SCANNED_JSON" '$all - $sc')"
|
|
declare -a BLOCKED_T2=(); AGENTIC_DONE=0
|
|
if over_budget; then
|
|
BUDGET_HIT=1; ALARM_LINES+=( "*BUDGET ALARM*: ceiling \$$TOTAL_BUDGET_USD hit after canary (\$$TOTAL_SPEND). No agentic rotation this night." )
|
|
else
|
|
while read -r nm; do
|
|
[ -n "$nm" ] || continue
|
|
if over_budget; then BUDGET_HIT=1; log "budget ceiling hit (\$$TOTAL_SPEND) — pausing rotation"; break; fi
|
|
if [ "$MAX_AGENTIC_PER_NIGHT" -gt 0 ] && [ "$AGENTIC_DONE" -ge "$MAX_AGENTIC_PER_NIGHT" ]; then
|
|
log "per-night agentic cap ($MAX_AGENTIC_PER_NIGHT) reached — pausing rotation"; break; fi
|
|
log "--- agentic: $nm ---"
|
|
scan_agentic "${REPO_DIR[$nm]}" "scan-$nm"
|
|
SCANNED_JSON="$(echo "$SCANNED_JSON" | jq -c --arg n "$nm" '. + [$n] | unique')"
|
|
AGENTIC_DONE=$((AGENTIC_DONE+1))
|
|
if [ "$LAST_BLOCK" -eq 1 ]; then
|
|
BLOCKED_T2+=( "$nm" )
|
|
ALARM_LINES+=( "*$nm* TIER2/agentic BLOCK: $LAST_REASON. Result: \`$LAST_RESULT_JSON\`" )
|
|
xmodel_check_criticals "$nm" "$LAST_RESULT_JSON"
|
|
fi
|
|
done < <(echo "$PENDING_JSON" | jq -r '.[]')
|
|
fi
|
|
# Persist rotation state.
|
|
jq -n --arg cs "$CYCLE_START" --argjson sc "$SCANNED_JSON" '{cycle_start:$cs, scanned:$sc}' > "$ROTATION_STATE"
|
|
# Coverage accounting + lag alarm.
|
|
REMAINING="$(jq -n --argjson all "$SCANNABLE_JSON" --argjson sc "$SCANNED_JSON" '($all - $sc) | length')"
|
|
CYCLE_AGE=$(( ( $(to_epoch "$UTC_DATE") - $(to_epoch "$CYCLE_START") ) / 86400 ))
|
|
log "agentic rotation: scanned $AGENTIC_DONE this night, $REMAINING still pending in cycle (started $CYCLE_START, age ${CYCLE_AGE}d)"
|
|
if [ "$REMAINING" -gt 0 ] && [ "$CYCLE_AGE" -ge "$MAX_CYCLE_NIGHTS" ]; then
|
|
ALARM_LINES+=( "*COVERAGE ALARM*: agentic rotation behind — $REMAINING repo(s) not deep-scanned in ${CYCLE_AGE}d (cycle since $CYCLE_START, max $MAX_CYCLE_NIGHTS). Raise budget or check for failures." )
|
|
fi
|
|
|
|
# Fold xmodel disagreements into the alarm set.
|
|
for x in ${XMODEL_LINES[@]+"${XMODEL_LINES[@]}"}; do ALARM_LINES+=( "$x" ); done
|
|
|
|
# ============================== 5) ALARM-ONLY REPORT =========================
|
|
ALARM=0
|
|
[ "${#BLOCKED_T1[@]}" -gt 0 ] && ALARM=1
|
|
[ "${#BLOCKED_T2[@]}" -gt 0 ] && ALARM=1
|
|
[ "$CANARY_OK" -ne 1 ] && ALARM=1
|
|
[ "$BUDGET_HIT" -eq 1 ] && ALARM=1
|
|
[ "${#ALARM_LINES[@]}" -gt 0 ] && ALARM=1
|
|
|
|
SUMMARY_LINE="sweep $UTC_STAMP: scannable=${#SCANNABLE[@]} tier1_blocked=${#BLOCKED_T1[@]} tier2_scanned=$AGENTIC_DONE tier2_blocked=${#BLOCKED_T2[@]} canary_ok=$CANARY_OK spend=\$$TOTAL_SPEND/\$$TOTAL_BUDGET_USD alarm=$ALARM report=$REPORT_DIR"
|
|
echo "$SUMMARY_LINE"
|
|
|
|
if [ "$ALARM" -ne 1 ]; then
|
|
log "clean night — no alarm conditions. Posting NOTHING to Slack (ALARM-only policy)."
|
|
exit 0
|
|
fi
|
|
|
|
ALARM_BODY="$(printf '%s\n' ${ALARM_LINES[@]+"${ALARM_LINES[@]}"} | sed 's/^/• /')"
|
|
SLACK_TEXT=":rotating_light: *Sea Haven nightly security sweep — ALARM* ($UTC_STAMP)
|
|
$ALARM_BODY
|
|
|
|
Coverage: tier1 scanners ${#SCANNABLE[@]} repos · tier2 agentic $AGENTIC_DONE this night ($REMAINING pending) · canary_ok=$CANARY_OK
|
|
Spend: \$$TOTAL_SPEND (ceiling \$$TOTAL_BUDGET_USD)
|
|
Reports + JSON: \`$REPORT_DIR\` (on sh-secrev VM)"
|
|
SLACK_TEXT="$(echo "$SLACK_TEXT" | redact)"
|
|
|
|
log "ALARM conditions present — composing Slack post"
|
|
echo "$SLACK_TEXT" >&2
|
|
|
|
post_slack_alarm "$SLACK_TEXT"
|
|
|
|
# An alarm is a reportable condition, not a script crash. Exit 0 so systemd shows success.
|
|
exit 0
|