open-swe/deploy/seahaven/fetch-config.sh

#!/usr/bin/env bash
# fetch-config.sh — AWS-sourced boot hook that materializes the app's .env.
#
# The stock `langgraph dev` runtime + the Open SWE app read a plain `.env` from
# the app working directory (python-dotenv). On the AWS lift-and-shift we do NOT
# commit a .env; instead every non-sensitive value lives in SSM Parameter Store
# (`/open-swe-<env>/*`) and every secret lives in AWS Secrets Manager
# (`open-swe-<env>/*`). This hook is run by systemd BEFORE the service starts; it
# pulls both sources via the EC2 instance role (no static keys), assembles a
# single .env on a tmpfs, and writes it owned by the unprivileged service user
# `chmod 600` (T5 SC-01: the privileged pre-hook materializes the secret; the app
# itself then runs as that NON-root service user, not root).
#
# It is intentionally FAIL-FAST: if any required secret/param is missing or empty
# it prints the offending variable NAMES (never values) and exits 1, so the
# service never starts with a partial .env.
#
# ---------------------------------------------------------------------------
# Naming contract (source of truth: T9 env/secret/config inventory)
#   SSM      /open-swe-<env>/<ENV_VAR_NAME>      -> exported as ENV_VAR_NAME
#   Secrets  open-swe-<env>/<ENV_VAR_NAME>       -> exported as ENV_VAR_NAME
# i.e. the last path segment IS the literal environment-variable name. This is a
# deliberate (documented) deviation from the handbook's kebab-case value-name
# example (`my-stack/slack-signing`): a .env materializer needs a lossless,
# unambiguous round-trip from store key -> env var, and the env var name is the
# only key that guarantees that. The `open-swe-<env>` stack prefix still follows
# kebab-case per naming-conventions.md.
# ---------------------------------------------------------------------------
#
# Wiring into systemd (AWS EC2 variant):
#   The unit runs as the unprivileged service user (User=openswe). ONLY the
#   ExecStartPre pre-hook runs as root (the `+` prefix) so it can pull from AWS,
#   write the tmpfs .env, and chown it to the service user. The app (ExecStart)
#   and the seeder (ExecStartPost) then run as openswe and read the openswe-owned
#   0600 .env — the agent never runs as root (T5 SC-01). Pass the env as the
#   positional arg (T5 BOOT-01):
#
#     [Service]
#     User=openswe
#     Group=openswe
#     Environment=ENV_DIR=/run/open-swe SERVICE_USER=openswe
#     # ExecStartPre runs as root (+) so it can chown the .env to the service user.
#     ExecStartPre=+/opt/open-swe/deploy/seahaven/fetch-config.sh prod
#     ExecStart=/opt/open-swe/.venv/bin/langgraph dev --host 127.0.0.1 --port 2024 \
#               --no-browser --no-reload
#     ExecStartPost=/opt/open-swe/deploy/seahaven/seed_store.sh prod
#
#   tmpfs: /run is already a tmpfs on systemd hosts, so ENV_DIR=/run/open-swe is
#   tmpfs-backed by default (the .env never touches disk). Set RUN_DEDICATED_TMPFS=1
#   to mount a private tmpfs at ENV_DIR instead. The app's CWD `.env` is a symlink
#   into ENV_DIR (created idempotently below), so python-dotenv finds it unchanged.
#
# Idempotent, re-runnable on every (re)start. No secret is ever echoed.

set -euo pipefail
umask 077

# --- Inputs ------------------------------------------------------------------
ENV="${1:-${OPENSWE_ENV:-}}"
case "$ENV" in
  dev | prod) ;;
  *)
    echo "fetch-config: ENV must be 'dev' or 'prod' (got '${ENV:-<empty>}')" >&2
    echo "usage: fetch-config.sh <dev|prod>   (or set OPENSWE_ENV)" >&2
    exit 2
    ;;
esac

REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}"
SSM_PREFIX="/open-swe-${ENV}/"
SECRET_PREFIX="open-swe-${ENV}/"

ENV_DIR="${ENV_DIR:-/run/open-swe}"          # tmpfs-backed (/run) by default
ENV_FILE="${ENV_DIR}/.env"
APP_DIR="${APP_DIR:-/opt/open-swe}"           # where the app + its CWD .env live
APP_ENV_LINK="${APP_DIR}/.env"                # symlink -> ENV_FILE

# The unprivileged service user that runs the app and OWNS the .env (T5 SC-01).
# fetch-config runs as root (ExecStartPre=+) only to chown the secret to it.
SERVICE_USER="${SERVICE_USER:-openswe}"
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"

# Sea Haven override: upstream defaults DEFAULT_REPO_OWNER to "langchain-ai".
# For the Sea Haven deployment it MUST be the org. fetch-config forces this so a
# stale/blank SSM value can never point the agent at the upstream org.
SH_REPO_OWNER="${OPENSWE_REPO_OWNER:-Sea-Haven-Industries}"

for bin in aws jq; do
  command -v "$bin" >/dev/null 2>&1 || { echo "fetch-config: '$bin' not found on PATH" >&2; exit 3; }
done

log() { echo "fetch-config[$ENV]: $*"; }   # NAMES/counts only — never values
b64d() { base64 --decode; }                # GNU coreutils on the EC2 host

# Accept a store key into VARS iff it is a valid env-var identifier and not a
# duplicate. Rejects non-identifier names (T5 SH-INJ-002 / set -e DoS hardening)
# and flat-namespace collisions (T5 SSM-05). $3 = source label for logs.
accept_var() {
  local key="$1" value="$2" src="$3"
  if ! [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
    log "WARNING: skipping ${src} key with non-identifier name (rejected)"
    return 0
  fi
  if [ -n "${VARS[$key]+set}" ]; then
    echo "fetch-config[$ENV]: FAIL-FAST — duplicate key '${key}' from ${src} (flat-namespace collision)" >&2
    exit 1
  fi
  VARS["$key"]="$value"
}

# --- tmpfs ------------------------------------------------------------------
mkdir -p "$ENV_DIR"
# Owned by the service user so the unprivileged app can traverse it (T5 SC-01).
chown "${SERVICE_USER}:${SERVICE_GROUP}" "$ENV_DIR" 2>/dev/null || true
chmod 700 "$ENV_DIR"
if [ "${RUN_DEDICATED_TMPFS:-0}" = "1" ] && ! mountpoint -q "$ENV_DIR"; then
  mount -t tmpfs -o nosuid,nodev,noexec,mode=0700,size=4m tmpfs "$ENV_DIR"
  log "mounted dedicated tmpfs at $ENV_DIR"
fi

# --- Collect values into an associative array --------------------------------
declare -A VARS=()

# 1) SSM Parameter Store (non-sensitive config). NOT --recursive: the contract is
#    a FLAT namespace /open-swe-<env>/<VAR>, so a non-recursive list returns exactly
#    those keys and cannot collapse two nested paths onto one name (T5 SSM-05). aws
#    CLI v2 auto-paginates NextToken.
log "reading SSM params under ${SSM_PREFIX} ..."
ssm_json="$(
  aws ssm get-parameters-by-path \
    --path "$SSM_PREFIX" \
    --with-decryption \
    --region "$REGION" \
    --no-cli-pager \
    --output json
)"
# Records are base64-encoded (name<TAB>value) so values with spaces/newlines/tabs
# survive the line-based read intact.
ssm_count=0
while IFS=$'\t' read -r nb vb; do
  [ -n "$nb" ] || continue
  name="$(printf '%s' "$nb" | b64d)"
  value="$(printf '%s' "$vb" | b64d; printf 'x')"; value="${value%x}"
  key="${name##*/}"                        # strip /open-swe-<env>/ prefix
  [ -n "$key" ] || continue
  accept_var "$key" "$value" "SSM"
  ssm_count=$((ssm_count + 1))
done < <(jq -r '.Parameters[] | (.Name|@base64) + "\t" + (.Value|@base64)' <<<"$ssm_json")
log "loaded ${ssm_count} config param(s) from SSM"

# 2) Secrets Manager (sensitive values). batch-get-secret-value filters by name
#    prefix and auto-paginates; one secret per env var, SecretString = the value.
log "reading secrets under ${SECRET_PREFIX} ..."
secret_count=0
while IFS=$'\t' read -r nb vb; do
  [ -n "$nb" ] || continue
  name="$(printf '%s' "$nb" | b64d)"
  case "$name" in
    "${SECRET_PREFIX}"*) ;;                # defensive: exact-prefix only
    *) continue ;;
  esac
  value="$(printf '%s' "$vb" | b64d; printf 'x')"; value="${value%x}"
  key="${name##*/}"
  [ -n "$key" ] || continue
  accept_var "$key" "$value" "Secrets"
  secret_count=$((secret_count + 1))
done < <(
  aws secretsmanager batch-get-secret-value \
    --filters "Key=name,Values=${SECRET_PREFIX}" \
    --region "$REGION" \
    --no-cli-pager \
    --output json \
  | jq -r '.SecretValues[] | select(.SecretString != null) | (.Name|@base64) + "\t" + (.SecretString|@base64)'
)
log "loaded ${secret_count} secret(s) from Secrets Manager"

# --- Sea Haven DEFAULT_REPO_OWNER hard pin -----------------------------------
# T5 OSWE-OWNER-04: a HARD pin, not a deny-list. Whatever the store holds (blank,
# upstream 'langchain-ai', a case/space variant, or any other org), the owner is
# unconditionally forced to the Sea Haven org so the agent can never target the
# wrong owner.
cur_owner="${VARS[DEFAULT_REPO_OWNER]:-}"
if [ -n "$cur_owner" ] && [ "$cur_owner" != "$SH_REPO_OWNER" ]; then
  log "WARNING: SSM DEFAULT_REPO_OWNER differs from the pinned org -> hard-pinning to '${SH_REPO_OWNER}'"
fi
VARS[DEFAULT_REPO_OWNER]="$SH_REPO_OWNER"

# --- FAIL-FAST: required vars -------------------------------------------------
# Hard-required regardless of mode:
required=(
  DASHBOARD_JWT_SECRET            # RuntimeError on startup if missing (oauth.py)
  TOKEN_ENCRYPTION_KEY           # Fernet key(s); decrypts per-user GitHub tokens
  GITHUB_APP_ID                  # GitHub App trio (installation-token minting) ...
  GITHUB_APP_PRIVATE_KEY         # ... multiline PEM ...
  GITHUB_APP_INSTALLATION_ID     # ... used by utils/github_app.py
  GITHUB_APP_CLIENT_ID           # dashboard OAuth login (prod parity)
  GITHUB_APP_CLIENT_SECRET       # dashboard OAuth login (prod parity)
)

# Active model-provider key(s): model selection is store-driven (team_settings),
# so fetch-config cannot infer it from .env. Default to the seeded cross-family
# pair (anthropic builder + openai reviewer). Override with a comma list.
IFS=',' read -r -a provider_keys <<<"${REQUIRED_PROVIDER_KEYS:-ANTHROPIC_API_KEY,OPENAI_API_KEY}"
for k in "${provider_keys[@]}"; do
  k="${k//[[:space:]]/}"
  [ -n "$k" ] && required+=("$k")
done

# Sandbox provider key(s) — depends on SANDBOX_TYPE (default langsmith).
sandbox_type="${VARS[SANDBOX_TYPE]:-langsmith}"
case "$sandbox_type" in
  langsmith) required+=(LANGSMITH_API_KEY_PROD DEFAULT_SANDBOX_SNAPSHOT_ID) ;;
  daytona)   required+=(DAYTONA_API_KEY) ;;
  runloop)   required+=(RUNLOOP_API_KEY) ;;
  modal | local) ;;                         # no key required
  *) log "WARNING: unknown SANDBOX_TYPE='${sandbox_type}' — not enforcing a sandbox key" ;;
esac

# Prod parity: webhook-signing secrets default empty in code but are required in
# prod. Linear's is required only when the Linear integration is wired.
if [ "$ENV" = "prod" ]; then
  required+=(GITHUB_WEBHOOK_SECRET SLACK_SIGNING_SECRET)
  if [ -n "${VARS[LINEAR_API_KEY]:-}" ] && [ "${OPENSWE_REQUIRE_LINEAR:-1}" = "1" ]; then
    required+=(LINEAR_WEBHOOK_SECRET)
  fi
fi

missing=()
for k in "${required[@]}"; do
  [ -n "${VARS[$k]:-}" ] || missing+=("$k")
done
# de-dup the names for a clean report
if [ "${#missing[@]}" -gt 0 ]; then
  mapfile -t missing < <(printf '%s\n' "${missing[@]}" | sort -u)
  echo "fetch-config[$ENV]: FAIL-FAST — ${#missing[@]} required var(s) missing/empty:" >&2
  printf '  - %s\n' "${missing[@]}" >&2
  echo "fetch-config[$ENV]: refusing to write a partial .env; service will not start." >&2
  exit 1
fi

# --- Write the .env atomically (root-only on tmpfs) --------------------------
# python-dotenv reads double-quoted values (incl. multiline PEMs). Its decoder
# unescapes ONLY backslash and double-quote (\\ -> \, \" -> "); it does NOT honor
# \$ or \` escapes, so escaping those would leave a spurious backslash. Escape
# exactly backslash then double-quote — real newlines stay literal (multiline OK).
# (Caveat: python-dotenv interpolates a literal `${VAR}` substring; the secret
# domain here — base64/hex/PEM keys — never contains one, so no extra guard.)
emit_var() {
  local name="$1" value="$2" esc
  esc="${value//\\/\\\\}"
  esc="${esc//\"/\\\"}"
  printf '%s="%s"\n' "$name" "$esc"
}

tmp="$(mktemp "${ENV_DIR}/.env.XXXXXX")"
chmod 600 "$tmp"
{
  printf '# Generated by fetch-config.sh for env=%s at %s — DO NOT EDIT.\n' \
    "$ENV" "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  printf '# Source: SSM /open-swe-%s/* + Secrets Manager open-swe-%s/*\n\n' "$ENV" "$ENV"
  for k in $(printf '%s\n' "${!VARS[@]}" | sort); do
    emit_var "$k" "${VARS[$k]}"
  done
} >"$tmp"

mv -f "$tmp" "$ENV_FILE"
# Owned by the unprivileged service user (T5 SC-01) so the app reads it without
# running as root. fetch-config itself runs as root (ExecStartPre=+) to chown.
chown "${SERVICE_USER}:${SERVICE_GROUP}" "$ENV_FILE"
chmod 600 "$ENV_FILE"

# Point the app's CWD .env at the tmpfs file (idempotent).
if [ "$APP_ENV_LINK" != "$ENV_FILE" ]; then
  if [ -L "$APP_ENV_LINK" ] || [ ! -e "$APP_ENV_LINK" ]; then
    ln -sfn "$ENV_FILE" "$APP_ENV_LINK"
  elif [ "$(readlink -f "$APP_ENV_LINK" 2>/dev/null || true)" != "$(readlink -f "$ENV_FILE")" ]; then
    log "WARNING: ${APP_ENV_LINK} exists and is not a symlink to ${ENV_FILE} — leaving it untouched"
  fi
fi

total=$((ssm_count + secret_count))
log "wrote ${ENV_FILE} (${total} vars, sandbox=${sandbox_type}) — ${SERVICE_USER}:${SERVICE_GROUP} 0600"