diff --git a/README.md b/README.md index 4be7625..f8e0788 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,27 @@ No secrets in Lambda environment variables. The export reaches S3 by **direct upload or a local drop-folder**, never SES/email. - **Direct:** `aws s3 cp ./export.xlsx s3://apm-wo-analysis-exports-328440206208/raw/` -- **Drop-folder (optional):** the launchd agent in `scripts/`. Both the script and - the watched folder must live **outside `~/Documents`** (macOS TCC sandbox). +- **Drop-folder (optional zero-touch):** a launchd agent (`scripts/apm-wo-uploader.sh` + + `scripts/com.seahaven.apm-wo-uploader.plist`) that watches `~/apm-wo-drop/`, + uploads new `.xlsx`/`.csv` files to `raw/`, and archives them to `uploaded/`. + It uploads with the scoped `apm-wo-drop` AWS profile (IAM user + `apm-wo-drop-uploader` — `s3:PutObject` on `raw/*` only). -The classifier Lambda is S3-triggered on the `raw/` prefix regardless of path. + Install (the runnable copy **must** live outside `~/Documents` — macOS TCC + sandbox; a repo-path script fails silently with `LastExitStatus=32256`): + ```bash + install -d "$HOME/.local/bin" "$HOME/apm-wo-drop" + cp scripts/apm-wo-uploader.sh "$HOME/.local/bin/apm-wo-uploader.sh" + chmod +x "$HOME/.local/bin/apm-wo-uploader.sh" + cp scripts/com.seahaven.apm-wo-uploader.plist "$HOME/Library/LaunchAgents/" + launchctl load -w "$HOME/Library/LaunchAgents/com.seahaven.apm-wo-uploader.plist" + ``` + Re-copy the script to `~/.local/bin` after editing the repo source. Configure + the profile once with the uploader's access key: + `aws configure --profile apm-wo-drop`. + +The classifier Lambda is S3-triggered on the `raw/` prefix regardless of path +(added in Phase 2 — uploads currently land in `raw/` and wait). ## Deployment diff --git a/cdk/stacks/pipeline_stack.py b/cdk/stacks/pipeline_stack.py index 014d413..77f7bd2 100644 --- a/cdk/stacks/pipeline_stack.py +++ b/cdk/stacks/pipeline_stack.py @@ -14,6 +14,9 @@ from aws_cdk import ( RemovalPolicy, Stack, ) +from aws_cdk import ( + aws_iam as iam, +) from aws_cdk import ( aws_s3 as s3, ) @@ -43,6 +46,21 @@ class PipelineStack(Stack): ], ) + # Phase 1 — least-privilege identity for the local drop-folder uploader. + # Scoped to s3:PutObject on raw/* only. The access key is created + # out-of-band (aws iam create-access-key) and stored in the local + # ~/.aws/credentials profile `apm-wo-drop` — never in CloudFormation. + self.drop_uploader = iam.User( + self, "DropUploader", user_name="apm-wo-drop-uploader" + ) + self.drop_uploader.add_to_policy( + iam.PolicyStatement( + sid="PutRawExportsOnly", + actions=["s3:PutObject"], + resources=[self.exports_bucket.arn_for_objects("raw/*")], + ) + ) + # Phase 2 — classifier Lambda, S3-triggered on the raw/ prefix. TODO # Phase 3 — Glue database `apm_wo_analysis` + Athena workgroup # (partition projection on dt; no crawler). TODO diff --git a/scripts/apm-wo-uploader.sh b/scripts/apm-wo-uploader.sh new file mode 100755 index 0000000..e009786 --- /dev/null +++ b/scripts/apm-wo-uploader.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# apm-wo-analysis local drop-folder uploader (optional zero-touch ingestion). +# +# Mirrors the proven stampli-drop-folder pattern. launchd invokes the INSTALLED +# copy at ~/.local/bin/apm-wo-uploader.sh, which must live OUTSIDE ~/Documents: +# macOS TCC denies launchd read access to ~/Documents, ~/Desktop, ~/Downloads, +# and a repo-path script fails silently with LastExitStatus=32256. Re-copy this +# source to ~/.local/bin after editing it. +# +# Uploads new .xlsx/.csv exports to the raw/ prefix using the scoped `apm-wo-drop` +# profile (IAM user apm-wo-drop-uploader — s3:PutObject on raw/ only), then +# archives them locally. The classifier Lambda is S3-triggered from raw/. +set -euo pipefail + +DROP_DIR="$HOME/apm-wo-drop" +UPLOADED_DIR="$DROP_DIR/uploaded" +LOG_FILE="$DROP_DIR/.upload.log" +BUCKET="apm-wo-analysis-exports-328440206208" +PROFILE="${APM_WO_AWS_PROFILE:-apm-wo-drop}" + +mkdir -p "$UPLOADED_DIR" +exec >> "$LOG_FILE" 2>&1 + +# Single-flight: WatchPaths can fire several times for one save. +LOCK_DIR="$DROP_DIR/.upload.lock" +if ! mkdir "$LOCK_DIR" 2>/dev/null; then + echo "$(date '+%Y-%m-%dT%H:%M:%S') skipping — another run holds the lock" + exit 0 +fi +trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT + +# Let the file finish writing before uploading. +sleep 2 + +shopt -s nullglob +uploaded_count=0 +failed_count=0 + +for f in "$DROP_DIR"/*.xlsx "$DROP_DIR"/*.csv; do + [ -f "$f" ] || continue + name=$(basename "$f") + ts=$(date '+%Y%m%d-%H%M%S') + + echo "$(date '+%Y-%m-%dT%H:%M:%S') uploading $name" + if aws --profile "$PROFILE" s3 cp "$f" "s3://$BUCKET/raw/$name"; then + mv "$f" "$UPLOADED_DIR/$ts-$name" + echo "$(date '+%Y-%m-%dT%H:%M:%S') OK -> uploaded/$ts-$name" + uploaded_count=$((uploaded_count + 1)) + osascript -e "display notification \"Uploaded $name\" with title \"APM WO Uploader\"" 2>/dev/null || true + else + echo "$(date '+%Y-%m-%dT%H:%M:%S') FAIL $name" + failed_count=$((failed_count + 1)) + osascript -e "display notification \"Failed to upload $name — see .upload.log\" with title \"APM WO Uploader\" sound name \"Basso\"" 2>/dev/null || true + fi +done + +if [ $uploaded_count -eq 0 ] && [ $failed_count -eq 0 ]; then + echo "$(date '+%Y-%m-%dT%H:%M:%S') folder change triggered but no .xlsx/.csv files found" +fi diff --git a/scripts/com.seahaven.apm-wo-drop.plist b/scripts/com.seahaven.apm-wo-drop.plist deleted file mode 100644 index 778c224..0000000 --- a/scripts/com.seahaven.apm-wo-drop.plist +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - Label - com.seahaven.apm-wo-drop - ProgramArguments - - /bin/bash - /Users//Library/Application Support/seahaven/apm-wo-drop/upload.sh - - WatchPaths - - /Users//APM-WO-Drop - - StandardOutPath - /tmp/apm-wo-drop.out.log - StandardErrorPath - /tmp/apm-wo-drop.err.log - - diff --git a/scripts/com.seahaven.apm-wo-uploader.plist b/scripts/com.seahaven.apm-wo-uploader.plist new file mode 100644 index 0000000..1751591 --- /dev/null +++ b/scripts/com.seahaven.apm-wo-uploader.plist @@ -0,0 +1,44 @@ + + + + + + Label + com.seahaven.apm-wo-uploader + ProgramArguments + + /bin/bash + /Users/adammoussa/.local/bin/apm-wo-uploader.sh + + WatchPaths + + /Users/adammoussa/apm-wo-drop + + EnvironmentVariables + + PATH + /usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + HOME + /Users/adammoussa + + StandardOutPath + /tmp/apm-wo-uploader.out.log + StandardErrorPath + /tmp/apm-wo-uploader.err.log + + diff --git a/scripts/drop_folder_upload.sh b/scripts/drop_folder_upload.sh deleted file mode 100755 index 851d58d..0000000 --- a/scripts/drop_folder_upload.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# apm-wo-analysis local drop-folder uploader (optional zero-touch ingestion). -# -# Mirrors the stampli-drop-folder pattern. When deployed, BOTH this script and -# the watched folder must live OUTSIDE ~/Documents (macOS TCC sandbox — see the -# macos-tcc-launchd memory). Suggested install locations: -# script: ~/Library/Application Support/seahaven/apm-wo-drop/upload.sh -# folder: ~/APM-WO-Drop -# -# Triggered by the launchd agent (scripts/com.seahaven.apm-wo-drop.plist) on a -# WatchPaths change. Uploads each new export to the raw/ prefix, then moves it to -# a local processed/ subfolder. Uses a least-privilege local AWS profile scoped -# to s3:PutObject on raw/ only. The classifier Lambda is S3-triggered from there. -set -euo pipefail - -DROP_DIR="${APM_WO_DROP_DIR:-$HOME/APM-WO-Drop}" -PROCESSED_DIR="$DROP_DIR/processed" -BUCKET="apm-wo-analysis-exports-328440206208" -PROFILE="${APM_WO_AWS_PROFILE:-apm-wo-drop}" - -mkdir -p "$PROCESSED_DIR" - -shopt -s nullglob -for f in "$DROP_DIR"/*.xlsx "$DROP_DIR"/*.csv; do - [ -e "$f" ] || continue - name="$(basename "$f")" - aws --profile "$PROFILE" s3 cp "$f" "s3://${BUCKET}/raw/${name}" - mv "$f" "$PROCESSED_DIR/$name" -done