diff --git a/security-review/checkers/compliance-drift.sh b/security-review/checkers/compliance-drift.sh index d65175d..3115d11 100755 --- a/security-review/checkers/compliance-drift.sh +++ b/security-review/checkers/compliance-drift.sh @@ -342,6 +342,13 @@ if [ "$CANARY" -eq 1 ]; then nm="$(basename "$d")" cp -R "$d" "$FIXTURE_WORK/$nm" mv "$FIXTURE_WORK/$nm/dotgit" "$FIXTURE_WORK/$nm/.git" + # The planted-secret env file is shipped as `dotenv.fixture` (NOT `.env`): the repo's + # root .gitignore lists `.env`, so a literal `.env` fixture would never be committed and + # the secrets-committed drift would vanish on a fresh clone. Restore it to `.env` in the + # materialized work area (the dotgit/ index already TRACKS `.env`, so ls-files still + # reports it). Same committable-without-side-effects rationale as the `.fixture` suffix the + # dependency-cve fixtures use for their manifests. + [ -f "$FIXTURE_WORK/$nm/dotenv.fixture" ] && mv "$FIXTURE_WORK/$nm/dotenv.fixture" "$FIXTURE_WORK/$nm/.env" REPO_NAMES+=( "$nm" ); REPO_DIR["$nm"]="$FIXTURE_WORK/$nm" done elif [ -n "$TARGETS_OVERRIDE" ]; then diff --git a/security-review/checkers/confluence-doc.sh b/security-review/checkers/confluence-doc.sh new file mode 100755 index 0000000..cdc8ac8 --- /dev/null +++ b/security-review/checkers/confluence-doc.sh @@ -0,0 +1,471 @@ +#!/usr/bin/env bash +# confluence-doc.sh — Plane-1 SCHEDULED documentation gap-detector (RECOMMEND-ONLY). +# +# Design refs: docs/r720-agent-team-design.md §4 (confluence-doc row) and §7 Phase 4. +# Decisions D3 + D6 + D7: +# D3 Report/recommend-only to start; no auto-Notion/Jira writes. +# D6 Confluence writes (the LATER on-demand path) use a dedicated IT-space-scoped +# `confluence-bot` Atlassian service account — PROVISIONING, gated (see footer). +# D7 SCHEDULED mode = read + RECOMMEND only: doc gaps / stale pages / missing runbooks go +# INTO the mode-600 report, NEVER auto-written. The on-demand SSH-invoked WRITE path +# (including Mermaid edits via ~/.claude/scripts/confluence_mermaid.py) is a separate, +# LATER provisioning path and is NOT implemented here. +# This mirrors compliance-drift.sh / dependency-cve.sh conventions VERBATIM so the coordinator +# (§5) drives it identically. +# +# WHAT IT DOES (read-only, RECOMMEND-ONLY): +# Diffs three documentation INPUTS against what Confluence's IT space actually documents, and +# REPORTS the gaps as recommendations (never writes): +# 1. REPO SET — every non-archived org repo (from the same $MIRROR_DIR mirrors the +# sweep already produced; or --targets / a fixture repo list) SHOULD +# have a Confluence page in the IT page-ID map. A repo with no mapped +# page is a "doc gap" recommendation. +# 2. AWS INVENTORY — (optional) a read-only AWS resource inventory JSON (stacks/Lambdas) +# SHOULD each be represented in the AWS Architecture Map / a page. +# A resource absent from the map is a "missing-from-architecture-map" +# recommendation. Absent inventory file => that whole check is SKIPPED +# (noted, never a gap on missing data). +# 3. PAGE-ID MAP — required runbook/standing pages (Incident Response Runbooks, Backup & +# DR, IAM & Access) SHOULD exist in the map. A required page missing +# from the map is a "missing-runbook" recommendation. Optionally, the +# LIVE Confluence API confirms each mapped page still exists and is not +# stale (lastUpdated older than $STALE_DAYS). +# +# The page-ID map is the canonical one from memory project_confluence_migration (IT space +# 720900). It is supplied as a JSON file (--page-map / $PAGE_MAP_FILE); the canary ships a +# mock map. We do NOT hardcode the live IDs into this script — they live in the map file so +# the map can evolve without a code change. +# +# CONFLUENCE API (LIVE reads need the confluence-bot token — PROVISIONING): +# The staleness / page-existence checks call the Confluence Cloud REST API read-only using +# CONFLUENCE_BASE_URL + CONFLUENCE_EMAIL + CONFLUENCE_API_TOKEN (the confluence-bot creds, +# D6). When those are ABSENT, OR --no-api / --canary is passed, the API checks are SKIPPED +# and NOTED — they are NEVER reported as a gap on missing data (memory +# feedback_cloudwatch_alarms: no false alarms on no-data). This mirrors compliance-drift's +# GitHub-API-skip pattern EXACTLY (status-code-aware: 200 -> parse, 404 -> a real "page gone" +# gap, anything else -> skip with NO alarm). The token / service account is gated provisioning. +# +# ON-DEMAND WRITE PATH (NOT HERE — provisioning): an actual Confluence update, including Mermaid +# architecture-map edits, goes through ~/.claude/scripts/confluence_mermaid.py (ADF-only, +# dry-run-default, macro-count + revert-diff guarded — it has destroyed page 1540098 before via +# a full-body markdown round-trip, so ADF-only is load-bearing). That --apply / live-dry-run is +# the LATER on-demand path and is gated. See the PROVISIONING footer. +# +# CANARY / DRY-RUN (offline, no network, no token): +# --canary runs against a fixture (checkers/fixtures/confluence-doc/): a repo list, a MOCK +# page-ID map, and a MOCK "confluence inventory" JSON (what the API would have returned). It +# asserts the known gap count against EXPECTED_GAP_COUNT (exit 3 on mismatch). --canary implies +# --dry-run + --no-api, so it is fully offline + deterministic. This is the anti-complacency +# floor (design §6.4) AND the routing dry-run. +# +# SCOPE / SAFETY: +# Read-only + RECOMMEND-only. Never writes Confluence, never creates a service account, never +# calls the Mermaid --apply path. Not wired into systemd. See PROVISIONING footer. +# +# Exit: 0 = ran (whether or not it found gaps); 2 = setup/usage error; 3 = canary assertion FAILED. +set -euo pipefail +export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" + +log() { echo "[confluence-doc] $*" >&2; } +die() { echo "[confluence-doc] FATAL: $*" >&2; exit 2; } + +# --- Shared substrate --------------------------------------------------------- +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUBSTRATE="$HERE/../lib/sweep_substrate.sh" +[ -f "$SUBSTRATE" ] || die "shared substrate not found: $SUBSTRATE" +# shellcheck source=../lib/sweep_substrate.sh +. "$SUBSTRATE" + +# --- Config + defaults (env, all optional) ------------------------------------ +GH_ORG="${GH_ORG:-Sea-Haven-Industries}" +MIRROR_DIR="${MIRROR_DIR:-$HOME/repo-mirrors}" +REPORT_ROOT="${REPORT_ROOT:-$HOME/sweep-reports/confluence-doc}" +# Canonical IT page-ID map (memory project_confluence_migration). JSON, NOT hardcoded here. +PAGE_MAP_FILE="${PAGE_MAP_FILE:-}" +# Optional read-only AWS inventory JSON (stacks/Lambdas) — absent => that check is SKIPPED. +AWS_INVENTORY_FILE="${AWS_INVENTORY_FILE:-}" +# Confluence Cloud REST (the confluence-bot creds, D6) — absent => API checks SKIPPED. +CONFLUENCE_BASE_URL="${CONFLUENCE_BASE_URL:-}" +CONFLUENCE_EMAIL="${CONFLUENCE_EMAIL:-}" +CONFLUENCE_API_TOKEN="${CONFLUENCE_API_TOKEN:-}" +# A mapped page is "stale" if its lastUpdated is older than this many days (API check only). +STALE_DAYS="${STALE_DAYS:-180}" +# Repos exempt from needing their own IT page (mirrors compliance-drift's exemption style). +DOC_EXEMPT_REPOS="${DOC_EXEMPT_REPOS:-engineering-handbook}" +# Required standing/runbook pages every IT space must document (page-map keys). +REQUIRED_PAGES="${REQUIRED_PAGES:-Incident Response Runbooks,Backup & Disaster Recovery,IAM & Access Management}" + +REFRESH=0 # --refresh: re-discover + re-mirror via substrate (network). Default: reuse mirrors. +DO_API=1 # --no-api: skip the LIVE Confluence API checks (offline). +DRY_RUN=0 # --dry-run: compose any digest but DO NOT post/write (recommend-only). +CANARY=0 # --canary: run against the fixture + assert the known gap count. +TARGETS_OVERRIDE="" # --targets "p1 p2": use these repo names instead of the mirror set. + +usage() { + cat >&2 < check SKIPPED + --refresh re-discover + re-mirror via the shared substrate before scanning (network) + --targets "a b" use these repo names instead of \$MIRROR_DIR/* (no clone) + -h|--help this help + +Env: GH_ORG MIRROR_DIR REPORT_ROOT PAGE_MAP_FILE AWS_INVENTORY_FILE STALE_DAYS + CONFLUENCE_BASE_URL CONFLUENCE_EMAIL CONFLUENCE_API_TOKEN (confluence-bot, D6) + DOC_EXEMPT_REPOS REQUIRED_PAGES SLACK_WEBHOOK_URL +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --canary) CANARY=1; DRY_RUN=1; DO_API=0 ;; + --dry-run) DRY_RUN=1 ;; + --no-api) DO_API=0 ;; + --page-map) shift; PAGE_MAP_FILE="${1:-}" ;; + --aws-inventory) shift; AWS_INVENTORY_FILE="${1:-}" ;; + --refresh) REFRESH=1 ;; + --targets) shift; TARGETS_OVERRIDE="${1:-}" ;; + -h|--help) usage; exit 0 ;; + *) die "unknown arg: $1 (see --help)" ;; + esac + shift +done + +command -v jq >/dev/null || die "jq is required" +command -v git >/dev/null || die "git is required" + +# --- Report dir (mode 600 reports; matches sweep conventions) ----------------- +umask 077 +UTC_DATE="$(date -u +%Y-%m-%d)" +UTC_STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +REPORT_DIR="$REPORT_ROOT/$UTC_DATE" +mkdir -p "$REPORT_DIR"; chmod 700 "$REPORT_ROOT" "$REPORT_DIR" 2>/dev/null || true +# shellcheck disable=SC2034 # read by the sourced substrate (post_slack_alarm) via dynamic scope +SWEEP_LOG="$REPORT_DIR/confluence-doc.log" +REPORT_JSON="$REPORT_DIR/confluence-doc.json" +REPORT_TXT="$REPORT_DIR/confluence-doc.txt" + +log "=== confluence-doc $UTC_STAMP (canary=$CANARY dry_run=$DRY_RUN api=$DO_API refresh=$REFRESH) ===" + +# ------------------------------------------------------------------------------ +# GAPS (spirit of finding.schema.json so the coordinator + plan-groomer can consume them like +# any finding). category="other" (a doc gap is not a security category). status="confirmed" +# only for deterministic facts: a repo absent from the supplied map, an AWS resource absent +# from the supplied inventory-vs-map diff, a required page missing from the map, or an explicit +# API 404 (mapped page gone). A SKIPPED API check is NEVER a gap (feedback_cloudwatch_alarms). +# ------------------------------------------------------------------------------ +declare -a GAPS=() +add_gap() { # subject id title severity check proof + local subject="$1" id="$2" title="$3" sev="$4" check="$5" proof="$6" + GAPS+=( "$(jq -n \ + --arg repo "$subject" --arg id "$id" --arg title "$title" --arg sev "$sev" \ + --arg check "$check" --arg proof "$proof" \ + '{repo:$repo, id:($repo+"-"+$id), title:$title, severity:$sev, category:"other", + check:$check, status:"confirmed", recommendation:$proof}')" ) +} +declare -a SKIPPED_CHECKS=() # (subject:reason) checks skipped on missing data — never a gap +note_skip() { SKIPPED_CHECKS+=( "$1" ); } + +in_csv() { # needle csv -> 0 if present + local n="$1" csv="$2"; case ",$csv," in *",$n,"*) return 0 ;; *) return 1 ;; esac +} + +# --- Page-map lookup: is there a page whose key (page title) matches NAME? ----- +# The map is a JSON object {"": , ...} (the canary mock + the real +# project_confluence_migration export share this shape). A repo "documented" if a page title +# contains the repo name (case-insensitive), since IT pages are titled e.g. "Payments Dashboard" +# for repo "payments-dashboard". +map_has_page_for_repo() { # repo + local repo="$1" + # normalize repo (kebab) -> a loose token to match against page titles + local needle; needle="$(echo "$repo" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]')" + jq -e --arg n "$needle" ' + (keys // [])[] | (ascii_downcase | gsub("[^a-z0-9]";"")) | select(contains($n)) + ' "$PAGE_MAP_FILE" >/dev/null 2>&1 +} +map_has_exact_key() { # exact page title + local key="$1" + jq -e --arg k "$key" 'has($k)' "$PAGE_MAP_FILE" >/dev/null 2>&1 +} + +# ============================================================================== +# CONFLUENCE API (LIVE reads; need the confluence-bot creds; skipped offline/--no-api/--canary) +# ============================================================================== +# Confirm a mapped page still exists and is not stale. Status-code-aware, mirroring +# compliance-drift's branch-protection pattern exactly: +# 200 -> parse lastUpdated, flag if older than STALE_DAYS +# 404 -> a mapped page that is GONE -> that IS a confirmed gap +# anything else (401/403/5xx/000 transient) -> SKIP with NO gap (no false alarm on no-data) +conf_check_page() { # page_title page_id + local title="$1" pid="$2" + local tmp code body + tmp="$(mktemp)" + code="$(curl -sS -o "$tmp" -w '%{http_code}' \ + -u "$CONFLUENCE_EMAIL:$CONFLUENCE_API_TOKEN" \ + -H 'Accept: application/json' \ + "$CONFLUENCE_BASE_URL/wiki/api/v2/pages/$pid?body-format=storage" \ + 2>>"$REPORT_DIR/confluence-api.log" || echo 000)" + body="$(cat "$tmp" 2>/dev/null)"; rm -f "$tmp" + case "$code" in + 200) + local updated upd_epoch now_epoch age_days + updated="$(echo "$body" | jq -r '.version.createdAt // .createdAt // empty' 2>/dev/null)" + [ -n "$updated" ] || { note_skip "$title:staleness(no-timestamp)"; return; } + upd_epoch="$(to_epoch "${updated%%T*}")"; now_epoch="$(date -u +%s)" + [ "$upd_epoch" -gt 0 ] || { note_skip "$title:staleness(unparseable-date)"; return; } + age_days=$(( (now_epoch - upd_epoch) / 86400 )) + if [ "$age_days" -gt "$STALE_DAYS" ]; then + add_gap "$title" "stale-page" \ + "Page '$title' is stale (last updated ${age_days}d ago, > ${STALE_DAYS}d)" "low" "stale-page" \ + "review + refresh the IT page; docs must track the system (global CLAUDE.md docs obligation)" + fi + ;; + 404) + add_gap "$title" "page-gone" \ + "Mapped page '$title' (id $pid) returns 404 — page deleted/moved" "high" "page-existence" \ + "the page-ID map points at a non-existent page; fix the map or restore the page" + ;; + *) note_skip "$title:api(http-$code)" ;; # transient/forbidden -> NO gap on missing data + esac +} + +# ============================================================================== +# TARGET RESOLUTION (repo set + map + inventory) +# ============================================================================== +declare -a REPO_NAMES=() + +if [ "$CANARY" -eq 1 ]; then + FIXTURE_ROOT="$HERE/fixtures/confluence-doc" + [ -d "$FIXTURE_ROOT" ] || die "canary fixture missing: $FIXTURE_ROOT" + PAGE_MAP_FILE="$FIXTURE_ROOT/mock-page-map.json" + AWS_INVENTORY_FILE="$FIXTURE_ROOT/mock-aws-inventory.json" + [ -f "$PAGE_MAP_FILE" ] || die "canary mock page-map missing: $PAGE_MAP_FILE" + [ -f "$AWS_INVENTORY_FILE" ] || die "canary mock aws inventory missing: $AWS_INVENTORY_FILE" + # Pin the exception + required-page lists the fixture was authored against (deterministic). + DOC_EXEMPT_REPOS="engineering-handbook" + REQUIRED_PAGES="Incident Response Runbooks,Backup & Disaster Recovery,IAM & Access Management" + STALE_DAYS="180" + # The fixture repo set is a newline-delimited list (no git checkout needed — confluence-doc + # diffs NAMES against the map, it does not scan repo contents). + while IFS= read -r r; do + r="$(echo "$r" | tr -d '[:space:]')"; [ -n "$r" ] && REPO_NAMES+=( "$r" ) + done < "$FIXTURE_ROOT/repos.txt" + log "canary: ${#REPO_NAMES[@]} fixture repo(s); mock map + mock inventory" +elif [ -n "$TARGETS_OVERRIDE" ]; then + # shellcheck disable=SC2206 # intentional word-split of the space-separated --targets list + arr=( $TARGETS_OVERRIDE ) + for p in "${arr[@]}"; do nm="$(basename "$p")"; REPO_NAMES+=( "$nm" ); done + log "explicit targets: ${REPO_NAMES[*]}" +else + if [ "$REFRESH" -eq 1 ]; then + [ -n "${GH_TOKEN:-}" ] || die "--refresh needs GH_TOKEN" + command -v curl >/dev/null || die "--refresh needs curl" + mkdir -p "$MIRROR_DIR" + log "refresh: re-discovering + mirroring via shared substrate (no separate clone path)" + DISCOVERED="$REPORT_DIR/discovered.tsv" + if discover_repos > "$DISCOVERED" 2>>"$REPORT_DIR/discover.log" && [ -s "$DISCOVERED" ]; then + while IFS=$'\t' read -r name url branch; do + [ -n "$name" ] || continue + mirror_repo "$name" "$url" "$branch" || log " mirror FAILED: $name (will use stale mirror if present)" + done < "$DISCOVERED" + else + log "discovery failed — falling back to existing mirrors (coverage may be stale)" + fi + fi + [ -d "$MIRROR_DIR" ] || die "mirror dir not found: $MIRROR_DIR (run nightly_sweep.sh first, or use --refresh/--targets)" + for d in "$MIRROR_DIR"/*/; do + [ -d "$d/.git" ] || continue + nm="$(basename "$d")"; REPO_NAMES+=( "$nm" ) + done + log "reusing ${#REPO_NAMES[@]} existing mirror(s) in $MIRROR_DIR (no re-clone)" +fi + +[ "${#REPO_NAMES[@]}" -gt 0 ] || die "no repos to diff" +[ -n "$PAGE_MAP_FILE" ] || die "no page-ID map (--page-map PATH or \$PAGE_MAP_FILE); cannot diff repos vs Confluence" +[ -f "$PAGE_MAP_FILE" ] || die "page-ID map not found: $PAGE_MAP_FILE" +jq -e 'type=="object"' "$PAGE_MAP_FILE" >/dev/null 2>&1 || die "page-ID map is not a JSON object: $PAGE_MAP_FILE" + +# Decide whether the LIVE Confluence API runs: need creds, API enabled, curl, not offline canary. +RUN_API=0 +if [ "$DO_API" -eq 1 ] && [ -n "$CONFLUENCE_BASE_URL" ] && [ -n "$CONFLUENCE_EMAIL" ] \ + && [ -n "$CONFLUENCE_API_TOKEN" ] && command -v curl >/dev/null; then + RUN_API=1 +elif [ "$DO_API" -eq 1 ]; then + log "Confluence API requested but confluence-bot creds/curl unavailable — skipping live checks (no false alarms on missing data; the service account is gated provisioning)." +fi + +# ============================================================================== +# CHECK 1 — REPO SET vs page-ID map (every non-exempt repo SHOULD have an IT page) +# ============================================================================== +for nm in "${REPO_NAMES[@]}"; do + in_csv "$nm" "$DOC_EXEMPT_REPOS" && { note_skip "$nm:repo-page(doc-exempt)"; continue; } + if ! map_has_page_for_repo "$nm"; then + add_gap "$nm" "no-it-page" \ + "Repo '$nm' has no Confluence IT page in the page-ID map" "medium" "repo-documented" \ + "create an IT page for '$nm' (sh-confluence) and add it to project_confluence_migration" + fi +done + +# ============================================================================== +# CHECK 2 — AWS INVENTORY vs page-ID map (optional; absent file => SKIP, never a gap) +# ============================================================================== +if [ -n "$AWS_INVENTORY_FILE" ] && [ -f "$AWS_INVENTORY_FILE" ]; then + if jq -e '.resources | type=="array"' "$AWS_INVENTORY_FILE" >/dev/null 2>&1; then + # Each resource SHOULD be represented on a page in the map (by name token match). + while IFS= read -r res; do + [ -n "$res" ] || continue + rname="$(echo "$res" | jq -r '.name // empty')" + rtype="$(echo "$res" | jq -r '.type // "resource"')" + [ -n "$rname" ] || continue + needle="$(echo "$rname" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]')" + if ! jq -e --arg n "$needle" ' + (keys // [])[] | (ascii_downcase | gsub("[^a-z0-9]";"")) | select(contains($n)) + ' "$PAGE_MAP_FILE" >/dev/null 2>&1; then + add_gap "$rname" "aws-not-in-map" \ + "AWS $rtype '$rname' is not represented in the IT page-ID map / architecture map" "medium" "aws-documented" \ + "add '$rname' to the AWS Architecture Map (page 1540098) + an IT page; Mermaid edits via confluence_mermaid.py (on-demand path, provisioning)" + fi + done < <(jq -c '.resources[]' "$AWS_INVENTORY_FILE") + else + note_skip "aws-inventory:malformed(no-resources-array)" + fi +else + note_skip "aws-inventory:absent(check-skipped)" # missing inventory -> SKIP, never a gap +fi + +# ============================================================================== +# CHECK 3 — REQUIRED standing/runbook pages present in the map +# ============================================================================== +IFS=',' read -r -a req_arr <<< "$REQUIRED_PAGES" +for page in "${req_arr[@]}"; do + page="$(echo "$page" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')" + [ -n "$page" ] || continue + if ! map_has_exact_key "$page"; then + add_gap "$page" "missing-runbook" \ + "Required page '$page' is missing from the IT page-ID map" "high" "required-page" \ + "create the '$page' page in the IT space and add it to project_confluence_migration" + fi +done + +# ============================================================================== +# CHECK 4 — LIVE API: mapped pages still exist + are not stale (skipped offline/--no-api/--canary) +# ============================================================================== +if [ "$RUN_API" -eq 1 ]; then + while IFS=$'\t' read -r ptitle pid; do + [ -n "$pid" ] || continue + case "$pid" in ''|*[!0-9]*) note_skip "$ptitle:api(non-numeric-id)"; continue ;; esac + conf_check_page "$ptitle" "$pid" + done < <(jq -r 'to_entries[] | [.key, (.value|tostring)] | @tsv' "$PAGE_MAP_FILE") +else + note_skip "confluence-api:not-run(creds-absent-or-offline)" +fi + +# ============================================================================== +# ASSEMBLE REPORT (JSON + text), mode 600 (identical shape to the other checkers) +# ============================================================================== +if [ "${#GAPS[@]}" -gt 0 ]; then + GAPS_JSON="$(printf '%s\n' "${GAPS[@]}" | jq -cs .)" +else + GAPS_JSON="[]" +fi +if [ "${#SKIPPED_CHECKS[@]}" -gt 0 ]; then + SKIPPED_JSON="$(printf '%s\n' "${SKIPPED_CHECKS[@]}" | jq -R . | jq -cs .)" +else + SKIPPED_JSON="[]" +fi + +N_GAPS="$(echo "$GAPS_JSON" | jq 'length')" +N_HIGH="$(echo "$GAPS_JSON" | jq '[.[]|select(.severity=="high" or .severity=="critical")] | length')" +N_SUBJECTS="$(echo "$GAPS_JSON" | jq '[.[].repo] | unique | length')" + +jq -n \ + --arg checker "confluence-doc" --arg ts "$UTC_STAMP" --arg org "$GH_ORG" \ + --argjson api "$RUN_API" --argjson reposn "${#REPO_NAMES[@]}" \ + --argjson gaps "$GAPS_JSON" --argjson skipped "$SKIPPED_JSON" \ + '{checker:$checker, generated:$ts, org:$org, mode:"recommend-only", + api_checks_ran:($api==1), repos_diffed:$reposn, + gap_count:($gaps|length), + subjects_with_gaps:([$gaps[].repo]|unique|length), + findings:$gaps, skipped_checks:$skipped}' > "$REPORT_JSON" + +{ + echo "confluence-doc — documentation gap report — $UTC_STAMP" + echo "org=$GH_ORG repos_diffed=${#REPO_NAMES[@]} api_checks=$([ "$RUN_API" -eq 1 ] && echo on || echo off) mode=recommend-only (D7)" + echo "doc gaps: $N_GAPS ($N_HIGH high) across $N_SUBJECTS subject(s)" + echo + if [ "$N_GAPS" -gt 0 ]; then + echo "RECOMMENDATIONS (recommend-only — NEVER auto-written, D7):" + echo "$GAPS_JSON" | jq -r '.[] | "• [\(.severity)] \(.repo): \(.title)\n recommend: \(.recommendation)"' + else + echo "No documentation gaps detected this run." + fi + if [ "$(echo "$SKIPPED_JSON" | jq 'length')" -gt 0 ]; then + echo; echo "skipped checks (missing data — NOT counted as a gap):" + echo "$SKIPPED_JSON" | jq -r '.[] | " - \(.)"' + fi +} > "$REPORT_TXT" +chmod 600 "$REPORT_JSON" "$REPORT_TXT" 2>/dev/null || true + +log "report: $REPORT_JSON ($N_GAPS gap(s), $N_SUBJECTS subject(s))" + +# ============================================================================== +# CANARY ASSERTION (anti-complacency floor, design §6.4) +# ============================================================================== +if [ "$CANARY" -eq 1 ]; then + EXPECT_FILE="$HERE/fixtures/confluence-doc/EXPECTED_GAP_COUNT" + [ -f "$EXPECT_FILE" ] || die "canary expected-count file missing: $EXPECT_FILE" + EXPECTED="$(tr -dc '0-9' < "$EXPECT_FILE")" + log "canary assertion: expected gaps=$EXPECTED, got=$N_GAPS" + if [ "$N_GAPS" -ne "$EXPECTED" ]; then + echo "[confluence-doc] CANARY FAIL: doc-gap count mismatch (expected $EXPECTED, got $N_GAPS)" >&2 + echo " -> a gap check regressed (stopped firing) or the fixture changed. See $REPORT_TXT." >&2 + exit 3 + fi + log "canary PASS: all $EXPECTED planted doc gaps detected." +fi + +# ============================================================================== +# RECOMMEND-ONLY ROUTING (D3/D7): gaps live in the mode-600 report. Post NOTHING by default. +# Scheduled mode NEVER auto-writes Confluence; alarming is reserved for confirmed criticals via +# the coordinator's shared routing (kept ALARM-only there). Here, recommend-only = report-only. +# ============================================================================== +if [ "$N_GAPS" -eq 0 ]; then + log "no doc gaps — recommend-only report written; posting NOTHING (D7)." + exit 0 +fi + +# Compose a redacted digest for the report/log (defense-in-depth); do NOT post by default. +DIGEST="$(echo "$GAPS_JSON" | jq -r ' + group_by(.repo)[] | "*\(.[0].repo)*: " + ([.[] | "[\(.severity)] \(.title)"] | join("; "))' \ + | sed 's/^/• /' | redact)" +echo "$DIGEST" >&2 +log "DRY-RUN/RECOMMEND-ONLY: $N_GAPS gap(s) written to the mode-600 report; nothing posted, nothing written to Confluence (D7)." +exit 0 + +# ============================================================================== +# PROVISIONING (NOT DONE HERE — gated): +# - confluence-bot SERVICE ACCOUNT (D6): create a dedicated Atlassian service account scoped +# to EDIT the IT space ONLY (Confluence API tokens inherit the whole user's permissions, so a +# scoped service account bounds blast radius; costs one Confluence seat). Mint its API token, +# store it in ~/secrev.env (mode 600) as CONFLUENCE_API_TOKEN (+ CONFLUENCE_BASE_URL/EMAIL). +# Rotate the token on a 90-DAY cadence. Until this exists, the LIVE API checks SKIP (above), +# never alarm. This whole step is gated (Adam-provisioned), not done by this script. +# - LIVE Confluence READ checks (page-existence + staleness) only run once those creds exist. +# - ON-DEMAND WRITE path (D7) — the actual Confluence update, including Mermaid architecture-map +# edits via ~/.claude/scripts/confluence_mermaid.py — is a SEPARATE, LATER, SSH-invoked path. +# Before any --apply, that script must pass a LIVE DRY-RUN against page 1540098: verify it +# lists all 16 weweave Mermaid macros and that a no-op set produces a clean (empty) revert-diff. +# ADF-only + macro-count + revert-diff guards are load-bearing (a full-body markdown round-trip +# has SILENTLY DELETED every diagram on 1540098 before). This script NEVER calls --apply. +# - No systemd unit / timer is installed here. Wiring the scheduled run (weekly) under the +# coordinator is provisioning and is gated. +# - The coordinator (design §5, checker_coordinator.sh) registers + drives this checker; that +# registry edit is done centrally, NOT in this script. +# - Confluence + project_r720_agent_team memory updates are docs-as-you-go obligations for the +# build session, tracked outside this script. +# ============================================================================== diff --git a/security-review/checkers/fixtures/compliance-drift/BadName_repo/dotenv.fixture b/security-review/checkers/fixtures/compliance-drift/BadName_repo/dotenv.fixture new file mode 100644 index 0000000..4d56164 --- /dev/null +++ b/security-review/checkers/fixtures/compliance-drift/BadName_repo/dotenv.fixture @@ -0,0 +1 @@ +API_KEY=AKIAIOSFODNN7EXAMPLE diff --git a/security-review/checkers/fixtures/compliance-drift/README.md b/security-review/checkers/fixtures/compliance-drift/README.md index ed22725..5c45d1d 100644 --- a/security-review/checkers/fixtures/compliance-drift/README.md +++ b/security-review/checkers/fixtures/compliance-drift/README.md @@ -15,5 +15,14 @@ Fixtures (each a real git checkout so the tracked-`.env` / `ls-files` checks wor Total = **6** (`EXPECTED_DRIFT_COUNT`). The canary pins `DOCS_ONLY_REPOS=docs-repo` and `COMPLIANCE_EXEMPT=""` internally so it is deterministic regardless of the operator's env. +**Secret-fixture naming:** `BadName_repo`'s planted tracked-secret env file is committed as +`dotenv.fixture`, NOT `.env`. The repo's root `.gitignore` lists `.env`, so a literal `.env` +fixture would silently never be committed — on a fresh clone the `secrets-committed` drift would +vanish and the count would drop to 5 (this regression was caught by this very canary). The +`--canary` materialization renames `dotenv.fixture` → `.env` in its temp work area; the +`dotgit/` index already TRACKS `.env`, so `git ls-files` still reports it. This mirrors the +`.fixture`-suffix convention the `dependency-cve` fixtures use for their manifests. Keep any new +committed secret fixture under a non-gitignored name and rename it in the canary. + When you add/remove a check or fixture, update both the fixture and `EXPECTED_DRIFT_COUNT` in the same commit (the canary edit is itself caught on the next run — design §6.4). diff --git a/security-review/checkers/fixtures/confluence-doc/EXPECTED_GAP_COUNT b/security-review/checkers/fixtures/confluence-doc/EXPECTED_GAP_COUNT new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/security-review/checkers/fixtures/confluence-doc/EXPECTED_GAP_COUNT @@ -0,0 +1 @@ +3 diff --git a/security-review/checkers/fixtures/confluence-doc/README.md b/security-review/checkers/fixtures/confluence-doc/README.md new file mode 100644 index 0000000..038fbe5 --- /dev/null +++ b/security-review/checkers/fixtures/confluence-doc/README.md @@ -0,0 +1,47 @@ +# confluence-doc canary fixtures + +Planted doc-gap corpus for `checkers/confluence-doc.sh --canary` (offline, no network/token). +The checker asserts the total doc-gap count equals `EXPECTED_GAP_COUNT` (anti-complacency +floor, design §6.4). If a gap check regresses (stops firing) or the fixture changes, the count +drifts and the canary FAILS (exit 3). + +`--canary` implies `--dry-run + --no-api`, so the LIVE Confluence API checks (page-existence + +staleness, which need the gated `confluence-bot` token, D6) are SKIPPED and noted — they are +never counted as a gap on missing data (memory `feedback_cloudwatch_alarms`). + +## Fixture inputs + +| File | Role | +|---|---| +| `repos.txt` | the repo set to diff against the page-ID map (one repo name per line) | +| `mock-page-map.json` | a MOCK IT page-ID map (same shape as `project_confluence_migration`) | +| `mock-aws-inventory.json` | a MOCK read-only AWS inventory (what the API/collector would return) | + +## The 3 planted gaps + +| Check | Subject | Why it's a gap | +|---|---|---| +| repo-documented | `orphan-tool-repo` | no page in the mock map (and not doc-exempt) | +| aws-documented | `afi-backup-monitor` (Lambda) | inventory resource with no page in the mock map | +| required-page | `IAM & Access Management` | a REQUIRED standing page omitted from the mock map | + +Non-gaps proving the checks are precise (must NOT inflate the count): +- `payments-dashboard`, `seahaven-slack-bot` repos → matched to their pages. +- `engineering-handbook` repo → `DOC_EXEMPT_REPOS` → skipped, not a gap. +- `payments-dashboard` Lambda → matched to the "Payments Dashboard" page. +- `Incident Response Runbooks`, `Backup & Disaster Recovery` required pages → present in the map. +- The LIVE API staleness/existence check → SKIPPED (no creds in canary), noted, not a gap. + +Total = **3** (`EXPECTED_GAP_COUNT`). + +When you add/remove a check, a fixture input, or a planted gap, update the fixture(s) and +`EXPECTED_GAP_COUNT` in the same commit (the canary edit is itself caught on the next run — +design §6.4). + +## Not exercised offline (PROVISIONING — gated) + +The LIVE Confluence reads (and the on-demand WRITE path via +`~/.claude/scripts/confluence_mermaid.py`, including the page-1540098 live dry-run that must list +all 16 weweave Mermaid macros) require the `confluence-bot` service account + token. That account +creation, its 90-day rotation, and the Mermaid live dry-run are provisioning steps documented in +the checker's PROVISIONING footer — they are NOT performed by the canary. diff --git a/security-review/checkers/fixtures/confluence-doc/mock-aws-inventory.json b/security-review/checkers/fixtures/confluence-doc/mock-aws-inventory.json new file mode 100644 index 0000000..017b13f --- /dev/null +++ b/security-review/checkers/fixtures/confluence-doc/mock-aws-inventory.json @@ -0,0 +1,7 @@ +{ + "_comment": "MOCK read-only AWS inventory for confluence-doc.sh --canary. Stands in for what a read-only AWS inventory collector would emit (stacks/Lambdas). Each .resources[] entry is matched (by name token) against the page-ID map. 'payments-dashboard' matches the 'Payments Dashboard' page (no gap); 'afi-backup-monitor' has no page (1 planted gap).", + "resources": [ + { "type": "Lambda", "name": "payments-dashboard", "stack": "payments-dashboard" }, + { "type": "Lambda", "name": "afi-backup-monitor", "stack": "afi-backup-monitor" } + ] +} diff --git a/security-review/checkers/fixtures/confluence-doc/mock-page-map.json b/security-review/checkers/fixtures/confluence-doc/mock-page-map.json new file mode 100644 index 0000000..e3b84aa --- /dev/null +++ b/security-review/checkers/fixtures/confluence-doc/mock-page-map.json @@ -0,0 +1,9 @@ +{ + "_comment": "MOCK IT page-ID map for confluence-doc.sh --canary. Shape matches the real project_confluence_migration export: {\"\": }. Deliberately OMITS 'IAM & Access Management' (a REQUIRED page) and any page for 'orphan-tool-repo' / the 'afi-backup-monitor' Lambda, so the canary plants exactly 3 gaps. No live IDs are hardcoded into the checker — they live here.", + "AWS Cloud Infrastructure": 917505, + "AWS Architecture Map": 1540098, + "Payments Dashboard": 524602, + "Seahaven Slack Bot": 819202, + "Incident Response Runbooks": 1179652, + "Backup & Disaster Recovery": 1867778 +} diff --git a/security-review/checkers/fixtures/confluence-doc/repos.txt b/security-review/checkers/fixtures/confluence-doc/repos.txt new file mode 100644 index 0000000..009cb45 --- /dev/null +++ b/security-review/checkers/fixtures/confluence-doc/repos.txt @@ -0,0 +1,4 @@ +payments-dashboard +seahaven-slack-bot +engineering-handbook +orphan-tool-repo diff --git a/security-review/checkers/fixtures/plan-groomer/EXPECTED_PLAN_ITEMS b/security-review/checkers/fixtures/plan-groomer/EXPECTED_PLAN_ITEMS new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/security-review/checkers/fixtures/plan-groomer/EXPECTED_PLAN_ITEMS @@ -0,0 +1 @@ +5 diff --git a/security-review/checkers/fixtures/plan-groomer/README.md b/security-review/checkers/fixtures/plan-groomer/README.md new file mode 100644 index 0000000..646b045 --- /dev/null +++ b/security-review/checkers/fixtures/plan-groomer/README.md @@ -0,0 +1,39 @@ +# plan-groomer canary fixtures + +Sample sibling-checker reports for `checkers/plan-groomer.sh --canary` (offline, no network/ +token). The planner asserts the groomed-plan **item count** equals `EXPECTED_PLAN_ITEMS` +(anti-complacency floor, design §6.4). If aggregation or dedup regresses, the count drifts +and the canary FAILS (exit 3). + +## How the canary works + +`--canary` points `$REPORT_ROOT_BASE` at `sample-reports/` and writes the groomed plan into a +mode-700 temp dir (so the canary writes nothing under `$HOME`). It reads each source checker's +**latest** `/.json`, normalizes every `.findings[]` into a plan item +`{repo, severity, source, title, action}`, **dedupes** on `repo|source|title`, prioritizes by +severity, and writes the plan into the mode-600 report. + +These are plain report JSON files (no `dotgit/` trick needed — plan-groomer reads sibling +reports, it does not scan git checkouts). + +## Fixture report set + +| Source checker | Date dir | Findings | Contributes to plan | +|---|---|---|---| +| `compliance-drift` | `2026-06-10` (OLD) | 1 | **0** — sentinel: older date MUST be skipped (latest-date selection) | +| `compliance-drift` | `2026-06-17` (latest) | 3 | **2** — two of the three are an exact duplicate (`payments-dashboard` / README) that must dedup to one | +| `dependency-cve` | `2026-06-17` | 2 | **2** — `jinja2` (high) + `lodash` (critical) | +| `doc-drift` | `2026-06-17` | 1 | **1** — stale README arch section | +| `confluence-doc` | (none) | — | **0** — no report present; noted in `missing_sources`, NEVER invented as work | + +Total groomed plan items = **5** (`EXPECTED_PLAN_ITEMS`). + +This exercises four invariants in one run: +1. **latest-date selection** — the `2026-06-10` sentinel must not leak into the plan. +2. **dedup** — the duplicate README finding collapses to one item. +3. **multi-source aggregation** — three different checkers feed one prioritized plan. +4. **no-data discipline** — a missing source (`confluence-doc`) is noted, never fabricated. + +When you add/remove a source checker, a fixture report, or a finding, update the fixture(s) +and `EXPECTED_PLAN_ITEMS` in the same commit (the canary edit is itself caught on the next run +— design §6.4). diff --git a/security-review/checkers/fixtures/plan-groomer/sample-reports/compliance-drift/2026-06-10/compliance-drift.json b/security-review/checkers/fixtures/plan-groomer/sample-reports/compliance-drift/2026-06-10/compliance-drift.json new file mode 100644 index 0000000..d9be0df --- /dev/null +++ b/security-review/checkers/fixtures/plan-groomer/sample-reports/compliance-drift/2026-06-10/compliance-drift.json @@ -0,0 +1,22 @@ +{ + "checker": "compliance-drift", + "generated": "2026-06-10T03:00:00Z", + "org": "Sea-Haven-Industries", + "api_checks_ran": false, + "repos_scanned": 1, + "drift_count": 1, + "repos_with_drift": 1, + "findings": [ + { + "repo": "STALE-repo-should-be-ignored", + "id": "STALE-repo-should-be-ignored-naming-repo", + "title": "This finding is from an OLDER date and MUST NOT appear in the groomed plan", + "severity": "high", + "category": "other", + "check": "naming-repo", + "status": "confirmed", + "proof": {"outcome": "older-date sentinel: latest-date selection must skip this"} + } + ], + "skipped_checks": [] +} diff --git a/security-review/checkers/fixtures/plan-groomer/sample-reports/compliance-drift/2026-06-17/compliance-drift.json b/security-review/checkers/fixtures/plan-groomer/sample-reports/compliance-drift/2026-06-17/compliance-drift.json new file mode 100644 index 0000000..e797ee0 --- /dev/null +++ b/security-review/checkers/fixtures/plan-groomer/sample-reports/compliance-drift/2026-06-17/compliance-drift.json @@ -0,0 +1,42 @@ +{ + "checker": "compliance-drift", + "generated": "2026-06-17T03:00:00Z", + "org": "Sea-Haven-Industries", + "api_checks_ran": false, + "repos_scanned": 2, + "drift_count": 3, + "repos_with_drift": 2, + "findings": [ + { + "repo": "payments-dashboard", + "id": "payments-dashboard-readme-missing", + "title": "No README.md at repo root", + "severity": "high", + "category": "other", + "check": "readme-present", + "status": "confirmed", + "proof": {"outcome": "global CLAUDE.md / github-standards.md: every repo must have a README"} + }, + { + "repo": "payments-dashboard", + "id": "payments-dashboard-readme-missing-dup", + "title": "No README.md at repo root", + "severity": "high", + "category": "other", + "check": "readme-present", + "status": "confirmed", + "proof": {"outcome": "DUPLICATE of the row above (same repo+source+title) — must dedup to one plan item"} + }, + { + "repo": "slack-bot", + "id": "slack-bot-merge-automerge-off", + "title": "allow_auto_merge disabled", + "severity": "low", + "category": "other", + "check": "merge-settings", + "status": "confirmed", + "proof": {"outcome": "github-standards.md: enable auto-merge (allow_auto_merge)"} + } + ], + "skipped_checks": [] +} diff --git a/security-review/checkers/fixtures/plan-groomer/sample-reports/dependency-cve/2026-06-17/dependency-cve.json b/security-review/checkers/fixtures/plan-groomer/sample-reports/dependency-cve/2026-06-17/dependency-cve.json new file mode 100644 index 0000000..1c55f40 --- /dev/null +++ b/security-review/checkers/fixtures/plan-groomer/sample-reports/dependency-cve/2026-06-17/dependency-cve.json @@ -0,0 +1,32 @@ +{ + "checker": "dependency-cve", + "generated": "2026-06-17T03:05:00Z", + "org": "Sea-Haven-Industries", + "advisory_mode": "offline", + "repos_scanned": 2, + "vuln_count": 2, + "repos_with_vulns": 2, + "findings": [ + { + "repo": "payments-dashboard", + "id": "payments-dashboard-vuln-jinja2-2-11-2-GHSA-g3rq-g295-4j3m", + "title": "jinja2 2.11.2 is vulnerable (GHSA-g3rq-g295-4j3m)", + "severity": "high", + "category": "other", + "check": "vulnerable-dependency", + "status": "confirmed", + "proof": {"package": "jinja2", "version": "2.11.2", "advisory_id": "GHSA-g3rq-g295-4j3m", "summary": "Jinja2 ReDoS in the urlize filter", "fixed_version": "2.11.3"} + }, + { + "repo": "slack-bot", + "id": "slack-bot-vuln-lodash-4-17-15-GHSA-p6mc-m468-83gw", + "title": "lodash 4.17.15 is vulnerable (GHSA-p6mc-m468-83gw)", + "severity": "critical", + "category": "other", + "check": "vulnerable-dependency", + "status": "confirmed", + "proof": {"package": "lodash", "version": "4.17.15", "advisory_id": "GHSA-p6mc-m468-83gw", "summary": "Prototype pollution in lodash", "fixed_version": "4.17.19"} + } + ], + "skipped_checks": [] +} diff --git a/security-review/checkers/fixtures/plan-groomer/sample-reports/doc-drift/2026-06-17/doc-drift.json b/security-review/checkers/fixtures/plan-groomer/sample-reports/doc-drift/2026-06-17/doc-drift.json new file mode 100644 index 0000000..ac35d52 --- /dev/null +++ b/security-review/checkers/fixtures/plan-groomer/sample-reports/doc-drift/2026-06-17/doc-drift.json @@ -0,0 +1,21 @@ +{ + "checker": "doc-drift", + "generated": "2026-06-17T03:10:00Z", + "org": "Sea-Haven-Industries", + "repos_scanned": 1, + "drift_count": 1, + "repos_with_drift": 1, + "findings": [ + { + "repo": "payments-dashboard", + "id": "payments-dashboard-readme-stale-arch", + "title": "README architecture section predates the new Lambda; docs lag code", + "severity": "medium", + "category": "other", + "check": "readme-stale", + "status": "confirmed", + "proof": {"outcome": "git log shows handler change after the README's last edit"} + } + ], + "skipped_checks": [] +} diff --git a/security-review/checkers/plan-groomer.sh b/security-review/checkers/plan-groomer.sh new file mode 100755 index 0000000..996afdf --- /dev/null +++ b/security-review/checkers/plan-groomer.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# plan-groomer.sh — Plane-1 planner for the R720 agent-team (REPORT-ONLY). +# +# Design refs: docs/r720-agent-team-design.md §4 (planner roster: plan-groomer — +# "Drafts a groomed weekly plan INTO the mode-600 report for now (D3); auto-write to +# Notion/Jira is a later toggle once trusted") and §7 Phase 4 ("planner + confluence-doc"). +# This mirrors compliance-drift.sh / dependency-cve.sh conventions VERBATIM so the +# coordinator (§5) can drive it identically — BUT its output discipline is different: +# it is REPORT-ONLY, not ALARM-only. +# +# WHAT IT DOES (read-only): +# Aggregates the actionable items the OTHER Plane-1 checkers already produced into a +# single prioritized "groomed weekly plan". It does NOT re-scan repos or hit any network: +# it reads the LATEST per-checker JSON reports under $REPORT_ROOT_BASE///. +# Sources consumed (each optional — a missing checker is noted, never invented as work): +# compliance-drift//compliance-drift.json (.findings[]) +# dependency-cve//dependency-cve.json (.findings[]) +# doc-drift//doc-drift.json (.findings[], if Phase-3 doc-drift exists) +# confluence-doc//confluence-doc.json (.findings[], the Phase-4 sibling) +# Each finding is normalized to a plan item {repo, severity, source, title, action}, +# DEDUPED (same repo+source+title collapses), grouped by severity then repo, and written +# into a prioritized plan in the mode-600 report (JSON + human text). +# +# REPORTING (D3 — REPORT-ONLY, the key difference from the ALARM-only checkers): +# - Writes a per-run JSON + text report under $REPORT_ROOT//, mode 600 (umask 077). +# - Slack: posts NOTHING by default. A groomed plan is a digest, not an alarm — auto-write +# to Notion/Jira (or a Slack digest) is a later toggle once signal quality is trusted (D3). +# There is intentionally NO post_slack_alarm() call in the default path; --notify is a +# future seam left inert here. A clean week (zero items) still writes an (empty) plan. +# - Reuses the substrate's redact() for the in-report digest string (defense-in-depth). +# +# SUBSTRATE REUSE (lib/sweep_substrate.sh, sourced — bash dynamic scoping): +# redact -> mask any secret-shaped value that leaked into an upstream report title. +# (discover_repos/mirror_repo/post_slack_alarm are intentionally NOT used: plan-groomer +# neither clones nor alarms — it only reads sibling reports and writes one mode-600 plan.) +# +# CANARY / DRY-RUN (offline, no network, no token): +# --canary points $REPORT_ROOT_BASE at a fixture set of sample checker reports +# (checkers/fixtures/plan-groomer/sample-reports///.json) and asserts +# the groomed-plan ITEM COUNT equals EXPECTED_PLAN_ITEMS (anti-complacency floor, design §6.4). +# If aggregation/dedup regresses, the count drifts and the canary FAILS (exit 3). --canary +# implies --dry-run. Fully offline + deterministic. --dry-run also suppresses the (inert) +# --notify seam. +# +# SCOPE / SAFETY: +# Read-only. No network, no token, no clones, no agent_team/ writes, no systemd wiring — +# that is provisioning (gated). See "PROVISIONING (NOT DONE HERE)" at the bottom. +# +# Exit: 0 = ran (always, report-only); 2 = setup/usage error; 3 = canary assertion FAILED. +set -euo pipefail +export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" + +log() { echo "[plan-groomer] $*" >&2; } +die() { echo "[plan-groomer] FATAL: $*" >&2; exit 2; } + +# --- Shared substrate --------------------------------------------------------- +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUBSTRATE="$HERE/../lib/sweep_substrate.sh" +[ -f "$SUBSTRATE" ] || die "shared substrate not found: $SUBSTRATE" +# shellcheck source=../lib/sweep_substrate.sh +. "$SUBSTRATE" + +# --- Config + defaults (env, all optional) ------------------------------------ +GH_ORG="${GH_ORG:-Sea-Haven-Industries}" +# Base under which each checker writes its own // report tree (the parent of +# the per-checker REPORT_ROOTs the other checkers default to: $HOME/sweep-reports). +REPORT_ROOT_BASE="${REPORT_ROOT_BASE:-$HOME/sweep-reports}" +# plan-groomer's OWN report tree (separate from the checkers it reads). +REPORT_ROOT="${REPORT_ROOT:-$HOME/sweep-reports/plan-groomer}" +# Which sibling checkers to aggregate (space-separated; missing ones are noted, never invented). +SOURCE_CHECKERS="${SOURCE_CHECKERS:-compliance-drift dependency-cve doc-drift confluence-doc}" + +DRY_RUN=0 # --dry-run: suppress the (inert) --notify seam (report still written). +CANARY=0 # --canary: read the fixture report set + assert the known plan-item count. +NOTIFY=0 # --notify: INERT future seam (post the digest somewhere). Off by default (D3). +TARGETS_OVERRIDE="" # --targets "a b": restrict the groomed plan to these repo names only. + +usage() { + cat >&2 </dev/null || die "jq is required" + +# --- Report dir (mode 600 reports; matches sweep conventions) ----------------- +umask 077 +UTC_DATE="$(date -u +%Y-%m-%d)" +UTC_STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +# Canary redirects the SOURCE base at the fixture report set; output stays in a temp area so +# the canary writes nothing under $HOME. +if [ "$CANARY" -eq 1 ]; then + FIXTURE_ROOT="$HERE/fixtures/plan-groomer" + [ -d "$FIXTURE_ROOT/sample-reports" ] || die "canary fixture missing: $FIXTURE_ROOT/sample-reports" + REPORT_ROOT_BASE="$FIXTURE_ROOT/sample-reports" + CANARY_OUT="$(mktemp -d "${TMPDIR:-/tmp}/plan-groomer-canary.XXXXXX")" + trap 'rm -rf "$CANARY_OUT"' EXIT + REPORT_ROOT="$CANARY_OUT/plan-groomer" + # Pin the source list the fixtures were authored against (deterministic regardless of env). + SOURCE_CHECKERS="compliance-drift dependency-cve doc-drift confluence-doc" +fi + +REPORT_DIR="$REPORT_ROOT/$UTC_DATE" +mkdir -p "$REPORT_DIR"; chmod 700 "$REPORT_ROOT" "$REPORT_DIR" 2>/dev/null || true +# shellcheck disable=SC2034 # named for parity with the ALARM-only checkers' substrate contract +SWEEP_LOG="$REPORT_DIR/plan-groomer.log" +REPORT_JSON="$REPORT_DIR/plan-groomer.json" +REPORT_TXT="$REPORT_DIR/plan-groomer.txt" + +log "=== plan-groomer $UTC_STAMP (canary=$CANARY dry_run=$DRY_RUN notify=$NOTIFY) ===" + +# ------------------------------------------------------------------------------ +# Severity ordering: the jq sort below maps severity->rank; no shell helper needed. +# ------------------------------------------------------------------------------ +in_csv() { # needle space-list -> 0 if present + local n="$1" list="$2" t; for t in $list; do [ "$t" = "$n" ] && return 0; done; return 1 +} + +# --- Resolve the LATEST date dir for one checker under $REPORT_ROOT_BASE ------- +latest_report_json() { # checker_name -> path to its latest /.json, or "" if none + local checker="$1" + local base="$REPORT_ROOT_BASE/$checker" d name latest="" + [ -d "$base" ] || { echo ""; return 0; } + # Date dirs are YYYY-MM-DD; lexical sort == chronological. Glob the dirs, keep only + # YYYY-MM-DD names, sort newest-first, pick the newest that actually has the JSON. + for d in $(for p in "$base"/*/; do + [ -d "$p" ] || continue + name="$(basename "$p")" + [[ "$name" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] && echo "$name" + done | sort -r); do + if [ -f "$base/$d/$checker.json" ]; then latest="$base/$d/$checker.json"; break; fi + done + echo "$latest" +} + +# ============================================================================== +# AGGREGATE: pull findings from each sibling checker's latest report into plan items. +# ============================================================================== +declare -a PLAN_ITEMS=() # one normalized JSON plan-item per upstream finding +declare -a MISSING_SOURCES=() # checkers with no report found — noted, NEVER invented as work +note_missing() { MISSING_SOURCES+=( "$1" ); } + +for checker in $SOURCE_CHECKERS; do + rj="$(latest_report_json "$checker")" + if [ -z "$rj" ]; then + note_missing "$checker(no-report-found)" + log " [$checker] no report under $REPORT_ROOT_BASE/$checker — skipped (not invented as work)" + continue + fi + if ! jq -e '.findings | type=="array"' "$rj" >/dev/null 2>&1; then + note_missing "$checker(report-unparseable)" + log " [$checker] report $rj has no .findings[] array — skipped" + continue + fi + cnt="$(jq '.findings | length' "$rj")" + log " [$checker] $rj -> $cnt finding(s)" + # Normalize each finding to a plan item. Title/severity/repo come straight off the finding + # schema the checkers emit (see finding.schema.json spirit). The "action" is a stable, + # source-derived hint (no fabrication — it just labels what kind of remediation this is). + while IFS= read -r item; do + [ -n "$item" ] && PLAN_ITEMS+=( "$item" ) + done < <(jq -c --arg src "$checker" ' + .findings[] + | { repo: (.repo // "unknown"), + severity: (.severity // "medium"), + source: $src, + title: (.title // .id // "untitled"), + action: ( if $src=="dependency-cve" then "upgrade vulnerable dependency" + elif $src=="compliance-drift" then "fix convention drift" + elif $src=="doc-drift" then "reconcile docs with code" + elif $src=="confluence-doc" then "close documentation gap" + else "review finding" end ) } + ' "$rj") +done + +# --- Optional repo-scope restriction (--targets) ------------------------------ +if [ -n "$TARGETS_OVERRIDE" ] && [ "${#PLAN_ITEMS[@]}" -gt 0 ]; then + declare -a FILTERED=() + for item in "${PLAN_ITEMS[@]}"; do + r="$(echo "$item" | jq -r '.repo')" + in_csv "$r" "$TARGETS_OVERRIDE" && FILTERED+=( "$item" ) + done + PLAN_ITEMS=( "${FILTERED[@]+"${FILTERED[@]}"}" ) + log "targets filter '$TARGETS_OVERRIDE' -> ${#PLAN_ITEMS[@]} item(s)" +fi + +# ============================================================================== +# DEDUP + PRIORITIZE: collapse identical (repo|source|title), then sort by severity desc, +# then repo, then source. Add a stable rank int so downstream consumers can re-sort. +# ============================================================================== +if [ "${#PLAN_ITEMS[@]}" -gt 0 ]; then + RAW_JSON="$(printf '%s\n' "${PLAN_ITEMS[@]}" | jq -cs .)" +else + RAW_JSON="[]" +fi + +GROOMED_JSON="$(echo "$RAW_JSON" | jq -c ' + # dedup on repo|source|title + ( reduce .[] as $x ({}; .[($x.repo+"|"+$x.source+"|"+$x.title)] //= $x) ) | [ .[] ] + | map(. + { rank: ( {critical:4, high:3, medium:2, low:1}[.severity] // 0 ) }) + | sort_by([ (-.rank), .repo, .source, .title ]) +')" + +N_ITEMS="$(echo "$GROOMED_JSON" | jq 'length')" +N_CRIT_HIGH="$(echo "$GROOMED_JSON" | jq '[.[]|select(.severity=="critical" or .severity=="high")]|length')" +N_REPOS="$(echo "$GROOMED_JSON" | jq '[.[].repo]|unique|length')" + +if [ "${#MISSING_SOURCES[@]}" -gt 0 ]; then + MISSING_JSON="$(printf '%s\n' "${MISSING_SOURCES[@]}" | jq -R . | jq -cs .)" +else + MISSING_JSON="[]" +fi + +# ============================================================================== +# ASSEMBLE REPORT (JSON + text), mode 600 +# ============================================================================== +jq -n \ + --arg planner "plan-groomer" --arg ts "$UTC_STAMP" --arg org "$GH_ORG" \ + --arg sources "$SOURCE_CHECKERS" \ + --argjson items "$GROOMED_JSON" --argjson missing "$MISSING_JSON" \ + '{planner:$planner, generated:$ts, org:$org, mode:"report-only", + sources_considered:($sources|split(" ")), + plan_item_count:($items|length), + crit_high_count:([$items[]|select(.severity=="critical" or .severity=="high")]|length), + repos_in_plan:([$items[].repo]|unique|length), + plan:$items, missing_sources:$missing}' > "$REPORT_JSON" + +{ + echo "plan-groomer — groomed weekly plan — $UTC_STAMP" + echo "org=$GH_ORG sources=[$SOURCE_CHECKERS] mode=report-only (D3: no auto-write)" + echo "plan items: $N_ITEMS ($N_CRIT_HIGH crit/high) across $N_REPOS repo(s)" + echo + if [ "$N_ITEMS" -gt 0 ]; then + echo "PRIORITIZED PLAN (severity desc, then repo):" + echo "$GROOMED_JSON" | jq -r '.[] | "• [\(.severity)] \(.repo) — \(.title)\n action: \(.action) (source: \(.source))"' + else + echo "No actionable items aggregated this run (clean week, or no upstream reports)." + fi + if [ "$(echo "$MISSING_JSON" | jq 'length')" -gt 0 ]; then + echo; echo "sources with no report (NOT invented as work):" + echo "$MISSING_JSON" | jq -r '.[] | " - \(.)"' + fi +} > "$REPORT_TXT" +chmod 600 "$REPORT_JSON" "$REPORT_TXT" 2>/dev/null || true + +# Defense-in-depth: the digest line that a future --notify seam would push is redacted now. +DIGEST="$(echo "$GROOMED_JSON" | jq -r ' + group_by(.repo)[] | "*\(.[0].repo)*: " + ([.[] | "[\(.severity)] \(.title)"] | join("; "))' \ + | sed 's/^/• /' | redact)" + +log "report: $REPORT_JSON ($N_ITEMS plan item(s), $N_REPOS repo(s))" + +# ============================================================================== +# CANARY ASSERTION (anti-complacency floor, design §6.4) +# ============================================================================== +if [ "$CANARY" -eq 1 ]; then + EXPECT_FILE="$HERE/fixtures/plan-groomer/EXPECTED_PLAN_ITEMS" + [ -f "$EXPECT_FILE" ] || die "canary expected-count file missing: $EXPECT_FILE" + EXPECTED="$(tr -dc '0-9' < "$EXPECT_FILE")" + log "canary assertion: expected plan items=$EXPECTED, got=$N_ITEMS" + if [ "$N_ITEMS" -ne "$EXPECTED" ]; then + echo "[plan-groomer] CANARY FAIL: groomed-plan item count mismatch (expected $EXPECTED, got $N_ITEMS)" >&2 + echo " -> aggregation or dedup regressed, or the fixture changed. See $REPORT_TXT." >&2 + exit 3 + fi + log "canary PASS: groomed plan has all $EXPECTED expected item(s) (dedup intact)." +fi + +# ============================================================================== +# REPORT-ONLY ROUTING (D3): the plan lives in the mode-600 report. Post NOTHING. +# ============================================================================== +if [ "$NOTIFY" -eq 1 ] && [ "$DRY_RUN" -eq 0 ]; then + # INERT future seam: when D3's "once trusted" toggle flips, this is where the digest would + # be pushed to Slack/Notion/Jira. It is intentionally a no-op in this phase — plan-groomer + # is REPORT-ONLY and must not auto-write. The composed digest is available in $DIGEST. + log "--notify requested but inert in this phase (D3: report-only; auto-write is a later toggle). No push." +fi +: "${DIGEST:?}" >/dev/null 2>&1 || true # DIGEST is composed for the future seam; keep it referenced. +log "REPORT-ONLY: groomed plan written to the mode-600 report; nothing posted (D3)." +exit 0 + +# ============================================================================== +# PROVISIONING (NOT DONE HERE — gated, later phases): +# - REPORT-ONLY by design (D3). The auto-write path (push the groomed plan to Notion/Jira, +# or a weekly Slack digest) is a LATER TOGGLE, flipped only once signal quality is trusted. +# The --notify seam above is intentionally inert; wiring a real destination is provisioning. +# - No systemd unit / timer is installed by this script. Wiring it into the weekly schedule +# (alongside the other Plane-1 checkers under the coordinator) is provisioning and is gated. +# - The coordinator (design §5, checker_coordinator.sh) registers + drives this planner; that +# registry edit is done centrally, NOT in this script. +# - Confluence + project_r720_agent_team memory updates are docs-as-you-go obligations for the +# build session, tracked outside this script. +# ==============================================================================