mirror of
https://github.com/Sea-Haven-Industries/apm-wo-analysis.git
synced 2026-09-30 03:03:14 +00:00
Add drop-folder ingestion and scoped uploader IAM user
Complete Phase 1 ingestion. Add a least-privilege IAM user (apm-wo-drop-uploader) to the pipeline stack, scoped to s3:PutObject on the raw/ prefix only — the local launchd uploader authenticates as this user via a dedicated profile, so a laptop credential leak cannot read, list, or touch the analytics data. Replace the scaffold uploader stub with the hardened stampli-pattern script (lockfile, logging, timestamped archive, notifications, settle delay) and align names to the convention (~/apm-wo-drop, ~/.local/bin, com.seahaven.apm-wo-uploader). The plist sets PATH/HOME because launchd runs with a stripped environment and otherwise cannot find aws. The exports bucket already shipped in the Phase 0 scaffold, so the code delta here is the uploader identity and tooling.
This commit is contained in:
Adam Moussa
parent
58b91bda70
commit
7befc8f0d7
6 changed files with 141 additions and 63 deletions
23
README.md
23
README.md
|
|
@ -89,10 +89,27 @@ No secrets in Lambda environment variables.
|
|||
The export reaches S3 by **direct upload or a local drop-folder**, never SES/email.
|
||||
|
||||
- **Direct:** `aws s3 cp ./export.xlsx s3://apm-wo-analysis-exports-328440206208/raw/`
|
||||
- **Drop-folder (optional):** the launchd agent in `scripts/`. Both the script and
|
||||
the watched folder must live **outside `~/Documents`** (macOS TCC sandbox).
|
||||
- **Drop-folder (optional zero-touch):** a launchd agent (`scripts/apm-wo-uploader.sh`
|
||||
+ `scripts/com.seahaven.apm-wo-uploader.plist`) that watches `~/apm-wo-drop/`,
|
||||
uploads new `.xlsx`/`.csv` files to `raw/`, and archives them to `uploaded/`.
|
||||
It uploads with the scoped `apm-wo-drop` AWS profile (IAM user
|
||||
`apm-wo-drop-uploader` — `s3:PutObject` on `raw/*` only).
|
||||
|
||||
The classifier Lambda is S3-triggered on the `raw/` prefix regardless of path.
|
||||
Install (the runnable copy **must** live outside `~/Documents` — macOS TCC
|
||||
sandbox; a repo-path script fails silently with `LastExitStatus=32256`):
|
||||
```bash
|
||||
install -d "$HOME/.local/bin" "$HOME/apm-wo-drop"
|
||||
cp scripts/apm-wo-uploader.sh "$HOME/.local/bin/apm-wo-uploader.sh"
|
||||
chmod +x "$HOME/.local/bin/apm-wo-uploader.sh"
|
||||
cp scripts/com.seahaven.apm-wo-uploader.plist "$HOME/Library/LaunchAgents/"
|
||||
launchctl load -w "$HOME/Library/LaunchAgents/com.seahaven.apm-wo-uploader.plist"
|
||||
```
|
||||
Re-copy the script to `~/.local/bin` after editing the repo source. Configure
|
||||
the profile once with the uploader's access key:
|
||||
`aws configure --profile apm-wo-drop`.
|
||||
|
||||
The classifier Lambda is S3-triggered on the `raw/` prefix regardless of path
|
||||
(added in Phase 2 — uploads currently land in `raw/` and wait).
|
||||
|
||||
## Deployment
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ from aws_cdk import (
|
|||
RemovalPolicy,
|
||||
Stack,
|
||||
)
|
||||
from aws_cdk import (
|
||||
aws_iam as iam,
|
||||
)
|
||||
from aws_cdk import (
|
||||
aws_s3 as s3,
|
||||
)
|
||||
|
|
@ -43,6 +46,21 @@ class PipelineStack(Stack):
|
|||
],
|
||||
)
|
||||
|
||||
# Phase 1 — least-privilege identity for the local drop-folder uploader.
|
||||
# Scoped to s3:PutObject on raw/* only. The access key is created
|
||||
# out-of-band (aws iam create-access-key) and stored in the local
|
||||
# ~/.aws/credentials profile `apm-wo-drop` — never in CloudFormation.
|
||||
self.drop_uploader = iam.User(
|
||||
self, "DropUploader", user_name="apm-wo-drop-uploader"
|
||||
)
|
||||
self.drop_uploader.add_to_policy(
|
||||
iam.PolicyStatement(
|
||||
sid="PutRawExportsOnly",
|
||||
actions=["s3:PutObject"],
|
||||
resources=[self.exports_bucket.arn_for_objects("raw/*")],
|
||||
)
|
||||
)
|
||||
|
||||
# Phase 2 — classifier Lambda, S3-triggered on the raw/ prefix. TODO
|
||||
# Phase 3 — Glue database `apm_wo_analysis` + Athena workgroup
|
||||
# (partition projection on dt; no crawler). TODO
|
||||
|
|
|
|||
59
scripts/apm-wo-uploader.sh
Executable file
59
scripts/apm-wo-uploader.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/bin/bash
|
||||
# apm-wo-analysis local drop-folder uploader (optional zero-touch ingestion).
|
||||
#
|
||||
# Mirrors the proven stampli-drop-folder pattern. launchd invokes the INSTALLED
|
||||
# copy at ~/.local/bin/apm-wo-uploader.sh, which must live OUTSIDE ~/Documents:
|
||||
# macOS TCC denies launchd read access to ~/Documents, ~/Desktop, ~/Downloads,
|
||||
# and a repo-path script fails silently with LastExitStatus=32256. Re-copy this
|
||||
# source to ~/.local/bin after editing it.
|
||||
#
|
||||
# Uploads new .xlsx/.csv exports to the raw/ prefix using the scoped `apm-wo-drop`
|
||||
# profile (IAM user apm-wo-drop-uploader — s3:PutObject on raw/ only), then
|
||||
# archives them locally. The classifier Lambda is S3-triggered from raw/.
|
||||
set -euo pipefail
|
||||
|
||||
DROP_DIR="$HOME/apm-wo-drop"
|
||||
UPLOADED_DIR="$DROP_DIR/uploaded"
|
||||
LOG_FILE="$DROP_DIR/.upload.log"
|
||||
BUCKET="apm-wo-analysis-exports-328440206208"
|
||||
PROFILE="${APM_WO_AWS_PROFILE:-apm-wo-drop}"
|
||||
|
||||
mkdir -p "$UPLOADED_DIR"
|
||||
exec >> "$LOG_FILE" 2>&1
|
||||
|
||||
# Single-flight: WatchPaths can fire several times for one save.
|
||||
LOCK_DIR="$DROP_DIR/.upload.lock"
|
||||
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||
echo "$(date '+%Y-%m-%dT%H:%M:%S') skipping — another run holds the lock"
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
|
||||
|
||||
# Let the file finish writing before uploading.
|
||||
sleep 2
|
||||
|
||||
shopt -s nullglob
|
||||
uploaded_count=0
|
||||
failed_count=0
|
||||
|
||||
for f in "$DROP_DIR"/*.xlsx "$DROP_DIR"/*.csv; do
|
||||
[ -f "$f" ] || continue
|
||||
name=$(basename "$f")
|
||||
ts=$(date '+%Y%m%d-%H%M%S')
|
||||
|
||||
echo "$(date '+%Y-%m-%dT%H:%M:%S') uploading $name"
|
||||
if aws --profile "$PROFILE" s3 cp "$f" "s3://$BUCKET/raw/$name"; then
|
||||
mv "$f" "$UPLOADED_DIR/$ts-$name"
|
||||
echo "$(date '+%Y-%m-%dT%H:%M:%S') OK -> uploaded/$ts-$name"
|
||||
uploaded_count=$((uploaded_count + 1))
|
||||
osascript -e "display notification \"Uploaded $name\" with title \"APM WO Uploader\"" 2>/dev/null || true
|
||||
else
|
||||
echo "$(date '+%Y-%m-%dT%H:%M:%S') FAIL $name"
|
||||
failed_count=$((failed_count + 1))
|
||||
osascript -e "display notification \"Failed to upload $name — see .upload.log\" with title \"APM WO Uploader\" sound name \"Basso\"" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $uploaded_count -eq 0 ] && [ $failed_count -eq 0 ]; then
|
||||
echo "$(date '+%Y-%m-%dT%H:%M:%S') folder change triggered but no .xlsx/.csv files found"
|
||||
fi
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<!--
|
||||
launchd agent for the apm-wo-analysis drop-folder uploader.
|
||||
|
||||
Install (paths must be OUTSIDE ~/Documents — macOS TCC sandbox):
|
||||
cp scripts/drop_folder_upload.sh "$HOME/Library/Application Support/seahaven/apm-wo-drop/upload.sh"
|
||||
cp scripts/com.seahaven.apm-wo-drop.plist "$HOME/Library/LaunchAgents/"
|
||||
launchctl load "$HOME/Library/LaunchAgents/com.seahaven.apm-wo-drop.plist"
|
||||
|
||||
Replace <USER> below with the deploying account's home before loading.
|
||||
-->
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.seahaven.apm-wo-drop</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/bin/bash</string>
|
||||
<string>/Users/<USER>/Library/Application Support/seahaven/apm-wo-drop/upload.sh</string>
|
||||
</array>
|
||||
<key>WatchPaths</key>
|
||||
<array>
|
||||
<string>/Users/<USER>/APM-WO-Drop</string>
|
||||
</array>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/apm-wo-drop.out.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/apm-wo-drop.err.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
44
scripts/com.seahaven.apm-wo-uploader.plist
Normal file
44
scripts/com.seahaven.apm-wo-uploader.plist
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<!--
|
||||
launchd agent for the apm-wo-analysis drop-folder uploader.
|
||||
|
||||
Install (the runnable copy MUST live outside ~/Documents — macOS TCC sandbox):
|
||||
install -d "$HOME/.local/bin" "$HOME/apm-wo-drop"
|
||||
cp scripts/apm-wo-uploader.sh "$HOME/.local/bin/apm-wo-uploader.sh"
|
||||
chmod +x "$HOME/.local/bin/apm-wo-uploader.sh"
|
||||
cp scripts/com.seahaven.apm-wo-uploader.plist "$HOME/Library/LaunchAgents/"
|
||||
launchctl load "$HOME/Library/LaunchAgents/com.seahaven.apm-wo-uploader.plist"
|
||||
|
||||
Verify it is healthy (LastExitStatus 0, not 32256 = TCC-blocked):
|
||||
launchctl list | grep apm-wo-uploader
|
||||
|
||||
Paths are absolute for the deploying account (adammoussa). Re-copy the script
|
||||
to ~/.local/bin after any edit to the repo source.
|
||||
-->
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.seahaven.apm-wo-uploader</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/bin/bash</string>
|
||||
<string>/Users/adammoussa/.local/bin/apm-wo-uploader.sh</string>
|
||||
</array>
|
||||
<key>WatchPaths</key>
|
||||
<array>
|
||||
<string>/Users/adammoussa/apm-wo-drop</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>PATH</key>
|
||||
<string>/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>
|
||||
<key>HOME</key>
|
||||
<string>/Users/adammoussa</string>
|
||||
</dict>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/apm-wo-uploader.out.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/apm-wo-uploader.err.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# apm-wo-analysis local drop-folder uploader (optional zero-touch ingestion).
|
||||
#
|
||||
# Mirrors the stampli-drop-folder pattern. When deployed, BOTH this script and
|
||||
# the watched folder must live OUTSIDE ~/Documents (macOS TCC sandbox — see the
|
||||
# macos-tcc-launchd memory). Suggested install locations:
|
||||
# script: ~/Library/Application Support/seahaven/apm-wo-drop/upload.sh
|
||||
# folder: ~/APM-WO-Drop
|
||||
#
|
||||
# Triggered by the launchd agent (scripts/com.seahaven.apm-wo-drop.plist) on a
|
||||
# WatchPaths change. Uploads each new export to the raw/ prefix, then moves it to
|
||||
# a local processed/ subfolder. Uses a least-privilege local AWS profile scoped
|
||||
# to s3:PutObject on raw/ only. The classifier Lambda is S3-triggered from there.
|
||||
set -euo pipefail
|
||||
|
||||
DROP_DIR="${APM_WO_DROP_DIR:-$HOME/APM-WO-Drop}"
|
||||
PROCESSED_DIR="$DROP_DIR/processed"
|
||||
BUCKET="apm-wo-analysis-exports-328440206208"
|
||||
PROFILE="${APM_WO_AWS_PROFILE:-apm-wo-drop}"
|
||||
|
||||
mkdir -p "$PROCESSED_DIR"
|
||||
|
||||
shopt -s nullglob
|
||||
for f in "$DROP_DIR"/*.xlsx "$DROP_DIR"/*.csv; do
|
||||
[ -e "$f" ] || continue
|
||||
name="$(basename "$f")"
|
||||
aws --profile "$PROFILE" s3 cp "$f" "s3://${BUCKET}/raw/${name}"
|
||||
mv "$f" "$PROCESSED_DIR/$name"
|
||||
done
|
||||
Loading…
Add table
Reference in a new issue