mirror of
https://github.com/Sea-Haven-Industries/sh-openswe-traces.git
synced 2026-10-03 11:33:30 +00:00
feat(infra): convert sh-openswe-traces to HCP Terraform (#8)
Freeze SAM CD and add terraform/ for seahaven-prod with account-suffixed buckets so HCP owns deploy before mgmt cutover. Suppress CKV_AWS_40 for the LangSmith export IAM user with documented mitigations.
This commit is contained in:
parent
50ac3de884
commit
5f430d9dfd
16 changed files with 533 additions and 53 deletions
12
.github/workflows/deploy.yaml
vendored
12
.github/workflows/deploy.yaml
vendored
|
|
@ -1,7 +1,14 @@
|
||||||
|
# FROZEN for PLAT-73: HCP Terraform is the sole deploy path.
|
||||||
|
# Previous SAM CD preserved as deploy.yaml.frozen until mgmt cutover.
|
||||||
|
# Do not re-enable until the migration is rolled back intentionally.
|
||||||
name: Deploy
|
name: Deploy
|
||||||
on:
|
on:
|
||||||
push:
|
workflow_dispatch:
|
||||||
branches: [main]
|
inputs:
|
||||||
|
confirm_unfreeze:
|
||||||
|
description: "Type UNFREEZE to acknowledge this bypasses PLAT-73 freeze"
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
id-token: write
|
id-token: write
|
||||||
|
|
@ -13,6 +20,7 @@ concurrency:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
deploy:
|
||||||
|
if: ${{ inputs.confirm_unfreeze == 'UNFREEZE' }}
|
||||||
uses: Sea-Haven-Industries/.github/.github/workflows/cd-sam.yaml@3f746774229d41770727e2e4fd63ed5f5555a8b3 # v1.0.3
|
uses: Sea-Haven-Industries/.github/.github/workflows/cd-sam.yaml@3f746774229d41770727e2e4fd63ed5f5555a8b3 # v1.0.3
|
||||||
with:
|
with:
|
||||||
stack-name: sh-openswe-traces
|
stack-name: sh-openswe-traces
|
||||||
|
|
|
||||||
22
.github/workflows/deploy.yaml.frozen
vendored
Normal file
22
.github/workflows/deploy.yaml.frozen
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
name: Deploy
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: deploy
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
uses: Sea-Haven-Industries/.github/.github/workflows/cd-sam.yaml@3f746774229d41770727e2e4fd63ed5f5555a8b3 # v1.0.3
|
||||||
|
with:
|
||||||
|
stack-name: sh-openswe-traces
|
||||||
|
cfn-role-arn: arn:aws:iam::328440206208:role/sh-openswe-traces-cfn-exec-role
|
||||||
|
secrets:
|
||||||
|
deploy-role-arn: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
|
||||||
|
parameter-overrides: ${{ secrets.SAM_PARAMETER_OVERRIDES }}
|
||||||
11
.gitignore
vendored
11
.gitignore
vendored
|
|
@ -5,3 +5,14 @@ samconfig.toml
|
||||||
*.pyc
|
*.pyc
|
||||||
__pycache__/
|
__pycache__/
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
# Local Terraform
|
||||||
|
terraform/.terraform/
|
||||||
|
*.tfstate
|
||||||
|
*.tfstate.*
|
||||||
|
crash.log
|
||||||
|
override.tf
|
||||||
|
override.tf.json
|
||||||
|
*_override.tf
|
||||||
|
*_override.tf.json
|
||||||
|
.terraformrc
|
||||||
|
terraform.rc
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,11 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "checkov-CKV_AWS_111-13",
|
"id": "checkov-CKV_AWS_111-13",
|
||||||
"justification": "False positive. Same ExecRole. CKV_AWS_111 (write without constraint on Resource:*) fires on the KMS block (kms:CreateKey/CreateAlias), which is inherent to key creation — not-yet-existent keys cannot be ARN-scoped — with blast radius bounded to this stack's deploys. The other Resource:* statement is a Deny (flagging a Deny as over-permissive is nonsensical). CFN-only assumable, SourceAccount-conditioned. Verified proof-or-kill 2026-07-13."
|
"justification": "False positive. Same ExecRole. CKV_AWS_111 (write without constraint on Resource:*) fires on the KMS block (kms:CreateKey/CreateAlias), which is inherent to key creation \u2014 not-yet-existent keys cannot be ARN-scoped \u2014 with blast radius bounded to this stack's deploys. The other Resource:* statement is a Deny (flagging a Deny as over-permissive is nonsensical). CFN-only assumable, SourceAccount-conditioned. Verified proof-or-kill 2026-07-13."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "checkov-CKV_AWS_40-10",
|
||||||
|
"justification": "Accepted risk for LangSmith Bulk Export (PLAT-73). CKV_AWS_40 prefers groups/roles over inline user policies; LangSmith's S3 destination requires long-lived IAM user access keys, so an IAM user with a single inline write-only policy is required. Mitigations: PutObject/AbortMultipartUpload only (no GetObject/DeleteObject); KMS Encrypt/GenerateDataKey/Decrypt gated by kms:ViaService=s3; access key minted out-of-band and never stored in Terraform state; secret on aws/secretsmanager key separate from the data CMK. Same pattern as the prior SAM LangSmithExportUser. REVISIT if LangSmith supports role assumption for Bulk Export destinations."
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
107
README.md
107
README.md
|
|
@ -1,10 +1,13 @@
|
||||||
# sh-openswe-traces
|
# sh-openswe-traces
|
||||||
|
|
||||||
LangSmith Bulk Export destination for the Open SWE deployment. A **storage-only**
|
LangSmith Bulk Export destination for the Open SWE deployment. A **storage-only**
|
||||||
SAM stack — no compute. LangSmith runs the export on its own schedule and writes
|
stack — no compute. LangSmith runs the export on its own schedule and writes
|
||||||
Parquet run/trace data into this bucket; we retain it for periodic auditing and
|
Parquet run/trace data into this bucket; we retain it for periodic auditing and
|
||||||
prompt/instruction improvement (query with Athena).
|
prompt/instruction improvement (query with Athena).
|
||||||
|
|
||||||
|
Owned by **HCP Terraform** in seahaven-prod (`sh-openswe-traces-prod`). Migrated
|
||||||
|
from mgmt SAM under [PLAT-73](https://seahaven.atlassian.net/browse/PLAT-73).
|
||||||
|
|
||||||
## Why this exists
|
## Why this exists
|
||||||
|
|
||||||
LangSmith retains traces for ~14 days. To audit agent behavior over longer
|
LangSmith retains traces for ~14 days. To audit agent behavior over longer
|
||||||
|
|
@ -18,7 +21,8 @@ this repo is just the destination and its access controls.
|
||||||
LangSmith (Bulk Export, scheduled LangSmith-side)
|
LangSmith (Bulk Export, scheduled LangSmith-side)
|
||||||
│ s3:PutObject (IAM user access key, least-privilege)
|
│ s3:PutObject (IAM user access key, least-privilege)
|
||||||
▼
|
▼
|
||||||
s3://sh-openswe-traces/langsmith/… SSE-KMS (alias/sh-openswe-traces)
|
s3://sh-openswe-traces-<account-id>/langsmith/…
|
||||||
|
│ SSE-KMS (alias/sh-openswe-traces)
|
||||||
│ lifecycle: → DEEP_ARCHIVE @ 90d
|
│ lifecycle: → DEEP_ARCHIVE @ 90d
|
||||||
▼
|
▼
|
||||||
Athena / manual audit
|
Athena / manual audit
|
||||||
|
|
@ -26,64 +30,64 @@ LangSmith (Bulk Export, scheduled LangSmith-side)
|
||||||
|
|
||||||
| Resource | Name | Notes |
|
| Resource | Name | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| S3 bucket | `sh-openswe-traces` | BPA all-on, SSE-KMS (default), versioned, TLS-only, `Retain` on delete |
|
| S3 bucket | `sh-openswe-traces-<account-id>` | BPA all-on, SSE-KMS (default), versioned, TLS-only, `prevent_destroy` |
|
||||||
| Log bucket | `sh-openswe-traces-logs` | S3 server access logs (SSE-S3) for read attribution; logs expire 365d |
|
| Log bucket | `sh-openswe-traces-logs-<account-id>` | S3 server access logs (SSE-S3); logs expire 365d |
|
||||||
| KMS CMK | `alias/sh-openswe-traces` | Rotation on; bucket default + writer encrypt through it |
|
| KMS CMK | `alias/sh-openswe-traces` | Rotation on; bucket default + writer encrypt through it |
|
||||||
| IAM user | auto-named (tag `sh-openswe-langsmith-export`) | Write-only LangSmith writer; `PutObject` bucket-wide, no read/delete |
|
| IAM user | `sh-openswe-langsmith-export` | Write-only LangSmith writer; `PutObject` bucket-wide, no read/delete |
|
||||||
| Secret | `sh-openswe/langsmith-export-s3` | Writer's access key (aws/secretsmanager key); populated post-deploy |
|
| Secret | `sh-openswe/langsmith-export-s3` | Writer's access key (aws/secretsmanager key); populated post-apply |
|
||||||
|
|
||||||
The IAM user is **not** given an explicit name so the app stack deploys under
|
Account-suffixed bucket names avoid global S3 name collision with the retired mgmt
|
||||||
`CAPABILITY_IAM`. It is referenced by ARN.
|
buckets during cutover.
|
||||||
|
|
||||||
## Deploy
|
## Deploy (HCP Terraform)
|
||||||
|
|
||||||
Two stacks:
|
Workspace: `sh-openswe-traces-prod` (org `seahaven`, project `seahaven-prod`).
|
||||||
|
Working directory: `terraform/`. Apply method: **Manual** until sealed.
|
||||||
- **`bootstrap.yaml`** → `sh-openswe-traces-bootstrap` — the CI IAM roles (OIDC deploy
|
Roles: `hcptf-sh-openswe-traces-plan` / `hcptf-sh-openswe-traces` (org-baseline
|
||||||
role + a least-privilege CFN exec role). Deployed **once, manually, under admin**
|
terraform-substrate).
|
||||||
(`CAPABILITY_NAMED_IAM`); rarely changes. Kept separate so CI never touches the shared
|
|
||||||
`github-cfn-execution-role`, which is roles-only and can't create this stack's KMS key,
|
|
||||||
secret, or IAM user.
|
|
||||||
- **`template.yaml`** → `sh-openswe-traces` — the app (buckets, CMK, writer, secret). CI
|
|
||||||
(`cd-sam.yaml`) deploys it on merge to `main`, assuming the OIDC deploy role and passing
|
|
||||||
`sh-openswe-traces-cfn-exec-role` as `--role-arn`.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# One-time bootstrap (admin):
|
cd terraform
|
||||||
aws cloudformation deploy --template-file bootstrap.yaml \
|
terraform init
|
||||||
--stack-name sh-openswe-traces-bootstrap --capabilities CAPABILITY_NAMED_IAM \
|
terraform plan
|
||||||
--region us-east-1
|
# First apply is Manual from the HCP UI (or terraform apply after plan confirm).
|
||||||
|
|
||||||
# App stack — CI does this on merge to main; for a local/admin deploy:
|
|
||||||
cp samconfig.toml.example samconfig.toml
|
|
||||||
sam validate --lint && sam build
|
|
||||||
sam deploy --capabilities CAPABILITY_IAM --resolve-s3
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Wire CI (after bootstrap + repo exist)
|
If the secret shell was created out-of-band (checklist step 1), import before the
|
||||||
|
first apply that manages it:
|
||||||
|
|
||||||
- Set repo secret **`AWS_DEPLOY_ROLE_ARN`** to the bootstrap `DeployRoleArn` output
|
```bash
|
||||||
(`arn:aws:iam::328440206208:role/githubdeploy-sh-openswe-traces`).
|
terraform import aws_secretsmanager_secret.export_key \
|
||||||
- `deploy.yaml` already passes `cfn-role-arn = sh-openswe-traces-cfn-exec-role`; the OIDC
|
'arn:aws:secretsmanager:us-east-1:011934824531:secret:sh-openswe/langsmith-export-s3-OQoqqU'
|
||||||
trust is pinned to this repo's `main` ref.
|
```
|
||||||
|
|
||||||
### Post-deploy: mint and store the writer access key
|
### Workspace env (once)
|
||||||
|
|
||||||
|
Workspace-level only (never project-scoped variable sets):
|
||||||
|
|
||||||
|
- `TFC_AWS_PROVIDER_AUTH=true`
|
||||||
|
- `TFC_AWS_PLAN_ROLE_ARN=arn:aws:iam::011934824531:role/hcptf-sh-openswe-traces-plan`
|
||||||
|
- `TFC_AWS_APPLY_ROLE_ARN=arn:aws:iam::011934824531:role/hcptf-sh-openswe-traces`
|
||||||
|
|
||||||
|
### Post-apply: mint and store the writer access key
|
||||||
|
|
||||||
The stack creates the IAM user and an empty secret; the access key is minted
|
The stack creates the IAM user and an empty secret; the access key is minted
|
||||||
out-of-band so it never lands in CloudFormation state. **Run this only on a trusted
|
out-of-band so it never lands in Terraform state. **Run this only on a trusted
|
||||||
single-user workstation, never in CI** — it handles a live credential.
|
single-user workstation, never in CI** — it handles a live credential.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
USER=$(aws cloudformation describe-stacks --stack-name sh-openswe-traces \
|
USER=$(terraform -chdir=terraform output -raw export_user_name)
|
||||||
--query "Stacks[0].Outputs[?OutputKey=='ExportUserName'].OutputValue" --output text)
|
|
||||||
|
|
||||||
KEY_JSON=$(aws iam create-access-key --user-name "$USER" \
|
KEY_JSON=$(aws iam create-access-key --user-name "$USER" --profile seahaven-prod \
|
||||||
| jq '{AccessKeyId: .AccessKey.AccessKeyId, SecretAccessKey: .AccessKey.SecretAccessKey}')
|
| jq '{AccessKeyId: .AccessKey.AccessKeyId, SecretAccessKey: .AccessKey.SecretAccessKey}')
|
||||||
|
|
||||||
# Pass the secret via stdin, not argv, so it never appears in the process table.
|
# Pass the secret via stdin, not argv, so it never appears in the process table.
|
||||||
aws secretsmanager put-secret-value \
|
# Strip trailing newlines — a trailing \n breaks HTTP headers at runtime.
|
||||||
|
printf '%s' "$KEY_JSON" | aws secretsmanager put-secret-value \
|
||||||
--secret-id sh-openswe/langsmith-export-s3 \
|
--secret-id sh-openswe/langsmith-export-s3 \
|
||||||
--secret-string file:///dev/stdin <<<"$KEY_JSON"
|
--secret-string file:///dev/stdin \
|
||||||
|
--profile seahaven-prod \
|
||||||
|
--region us-east-1
|
||||||
|
|
||||||
unset KEY_JSON
|
unset KEY_JSON
|
||||||
```
|
```
|
||||||
|
|
@ -97,7 +101,9 @@ destination call validates by test-writing to the bucket.
|
||||||
**1. Create the destination** (creds pulled from Secrets Manager, never pasted):
|
**1. Create the destination** (creds pulled from Secrets Manager, never pasted):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
BUCKET=$(terraform -chdir=terraform output -raw bucket_name)
|
||||||
CREDS=$(aws secretsmanager get-secret-value --secret-id sh-openswe/langsmith-export-s3 \
|
CREDS=$(aws secretsmanager get-secret-value --secret-id sh-openswe/langsmith-export-s3 \
|
||||||
|
--profile seahaven-prod --region us-east-1 \
|
||||||
--query SecretString --output text)
|
--query SecretString --output text)
|
||||||
AKID=$(jq -r .AccessKeyId <<<"$CREDS"); SAK=$(jq -r .SecretAccessKey <<<"$CREDS")
|
AKID=$(jq -r .AccessKeyId <<<"$CREDS"); SAK=$(jq -r .SecretAccessKey <<<"$CREDS")
|
||||||
|
|
||||||
|
|
@ -105,7 +111,7 @@ curl -sS -X POST 'https://api.smith.langchain.com/api/v1/bulk-exports/destinatio
|
||||||
-H 'Content-Type: application/json' -H "X-API-Key: $LS_API_KEY" -H "X-Tenant-Id: $LS_TENANT" \
|
-H 'Content-Type: application/json' -H "X-API-Key: $LS_API_KEY" -H "X-Tenant-Id: $LS_TENANT" \
|
||||||
--data @- <<JSON | jq .
|
--data @- <<JSON | jq .
|
||||||
{ "destination_type": "s3", "display_name": "sh-openswe-traces us-east-1",
|
{ "destination_type": "s3", "display_name": "sh-openswe-traces us-east-1",
|
||||||
"config": { "bucket_name": "sh-openswe-traces", "prefix": "langsmith", "region": "us-east-1" },
|
"config": { "bucket_name": "$BUCKET", "prefix": "langsmith", "region": "us-east-1" },
|
||||||
"credentials": { "access_key_id": "$AKID", "secret_access_key": "$SAK" } }
|
"credentials": { "access_key_id": "$AKID", "secret_access_key": "$SAK" } }
|
||||||
JSON
|
JSON
|
||||||
```
|
```
|
||||||
|
|
@ -162,10 +168,10 @@ curl -sS 'https://api.smith.langchain.com/api/v1/bulk-exports' \
|
||||||
secret + LangSmith destination (`PATCH`/recreate), then delete the old key. Key age is
|
secret + LangSmith destination (`PATCH`/recreate), then delete the old key. Key age is
|
||||||
monitored account-wide by the Security Hub `ACCESS_KEYS_ROTATED` Config rule (flags at
|
monitored account-wide by the Security Hub `ACCESS_KEYS_ROTATED` Config rule (flags at
|
||||||
90d) — rotation keeps it compliant.
|
90d) — rotation keeps it compliant.
|
||||||
- **Audit**: point Athena at `s3://sh-openswe-traces/langsmith/` (Parquet). Objects older
|
- **Audit**: point Athena at `s3://sh-openswe-traces-<account-id>/langsmith/` (Parquet).
|
||||||
than 90 days are in Deep Archive — restore before querying.
|
Objects older than 90 days are in Deep Archive — restore before querying.
|
||||||
- **Read attribution**: object access is logged to `s3://sh-openswe-traces-logs/s3-access/`
|
- **Read attribution**: object access is logged to
|
||||||
(S3 server access logging).
|
`s3://sh-openswe-traces-logs-<account-id>/s3-access/` (S3 server access logging).
|
||||||
- **Cost**: Deep Archive ≈ $1/TB/mo; expect the archive to dominate storage cost.
|
- **Cost**: Deep Archive ≈ $1/TB/mo; expect the archive to dominate storage cost.
|
||||||
|
|
||||||
## Gotchas (from IAM cross-review)
|
## Gotchas (from IAM cross-review)
|
||||||
|
|
@ -186,9 +192,10 @@ Traces can contain source code and secrets surfaced in tool I/O. Controls: SSE-K
|
||||||
rest (customer-managed CMK, bucket default), versioning (overwrite recovery), Block Public
|
rest (customer-managed CMK, bucket default), versioning (overwrite recovery), Block Public
|
||||||
Access, TLS-only bucket policy, a write-only least-privilege writer (bucket-wide `PutObject`
|
Access, TLS-only bucket policy, a write-only least-privilege writer (bucket-wide `PutObject`
|
||||||
+ `kms:Decrypt` gated to `kms:ViaService=s3`, no read/delete), the credential secret on a
|
+ `kms:Decrypt` gated to `kms:ViaService=s3`, no read/delete), the credential secret on a
|
||||||
separate managed key, and S3 access logging. CI deploys via a dedicated least-privilege
|
separate managed key, and S3 access logging.
|
||||||
exec role (see `bootstrap.yaml`), not the shared execution role.
|
|
||||||
|
|
||||||
The IAM surface passed GPT-4.1 cross-review and a `/sh-security-review` fan-out +
|
SAM `template.yaml` / `bootstrap.yaml` and the frozen GitHub `deploy.yaml` are retained
|
||||||
proof-or-kill verifier (no blocking findings). Deferred, non-blocking: CloudTrail S3
|
only until mgmt cutover completes; do not re-enable SAM CD.
|
||||||
data-events (org trail carries none; access logging covers attribution for now).
|
|
||||||
|
Deferred, non-blocking: CloudTrail S3 data-events (org trail carries none; access logging
|
||||||
|
covers attribution for now).
|
||||||
|
|
|
||||||
29
terraform/.terraform.lock.hcl
generated
Normal file
29
terraform/.terraform.lock.hcl
generated
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
# This file is maintained automatically by "terraform init".
|
||||||
|
# Manual edits may be lost in future updates.
|
||||||
|
|
||||||
|
provider "registry.terraform.io/hashicorp/aws" {
|
||||||
|
version = "6.58.0"
|
||||||
|
constraints = "~> 6.57"
|
||||||
|
hashes = [
|
||||||
|
"h1:1im6ypdeXLXq3sHElserTE62qmIqNiHLAyC3R5EnFFw=",
|
||||||
|
"h1:2kpake4zZKRX5437QVIRU3qFYH6Bjw/QE1fgVCPOrUg=",
|
||||||
|
"h1:OWl47Bo8Vzlf5srTUCmA6v4kvQGfah/P1joRtIYUUMc=",
|
||||||
|
"h1:UFot9S97tuAPvjKvoxm08sDG/gKYdDK+lMwsZKtLieY=",
|
||||||
|
"zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250",
|
||||||
|
"zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f",
|
||||||
|
"zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48",
|
||||||
|
"zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba",
|
||||||
|
"zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7",
|
||||||
|
"zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56",
|
||||||
|
"zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6",
|
||||||
|
"zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80",
|
||||||
|
"zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75",
|
||||||
|
"zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a",
|
||||||
|
"zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af",
|
||||||
|
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||||
|
"zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c",
|
||||||
|
"zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae",
|
||||||
|
"zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca",
|
||||||
|
"zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056",
|
||||||
|
]
|
||||||
|
}
|
||||||
48
terraform/iam.tf
Normal file
48
terraform/iam.tf
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
resource "aws_iam_user" "langsmith_export" {
|
||||||
|
name = local.export_user_name
|
||||||
|
|
||||||
|
tags = {
|
||||||
|
Name = "sh-openswe-langsmith-export"
|
||||||
|
purpose = "langsmith-bulk-export-writer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_user_policy" "langsmith_export_put" {
|
||||||
|
name = "langsmith-export-put"
|
||||||
|
user = aws_iam_user.langsmith_export.name
|
||||||
|
|
||||||
|
# Bucket-wide PutObject (not prefix-scoped): LangSmith destination validation
|
||||||
|
# writes a test object whose key is not guaranteed under export_prefix.
|
||||||
|
# Deliberately NO s3:GetObject / s3:DeleteObject — write-only.
|
||||||
|
policy = jsonencode({
|
||||||
|
Version = "2012-10-17"
|
||||||
|
Statement = [
|
||||||
|
{
|
||||||
|
Sid = "PutExportObjects"
|
||||||
|
Effect = "Allow"
|
||||||
|
Action = [
|
||||||
|
"s3:PutObject",
|
||||||
|
"s3:AbortMultipartUpload",
|
||||||
|
]
|
||||||
|
Resource = "${aws_s3_bucket.traces.arn}/*"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid = "EncryptWithBucketKey"
|
||||||
|
Effect = "Allow"
|
||||||
|
Action = [
|
||||||
|
"kms:GenerateDataKey",
|
||||||
|
"kms:Encrypt",
|
||||||
|
# Required by S3 at CompleteMultipartUpload for SSE-KMS. Safe: the
|
||||||
|
# writer has no s3:GetObject, so there is no object to decrypt/exfil.
|
||||||
|
"kms:Decrypt",
|
||||||
|
]
|
||||||
|
Resource = aws_kms_key.traces.arn
|
||||||
|
Condition = {
|
||||||
|
StringEquals = {
|
||||||
|
"kms:ViaService" = "s3.${var.aws_region}.amazonaws.com"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
25
terraform/kms.tf
Normal file
25
terraform/kms.tf
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
resource "aws_kms_key" "traces" {
|
||||||
|
description = "SSE-KMS CMK for sh-openswe-traces (LangSmith export archive)."
|
||||||
|
enable_key_rotation = true
|
||||||
|
deletion_window_in_days = 30
|
||||||
|
|
||||||
|
policy = jsonencode({
|
||||||
|
Version = "2012-10-17"
|
||||||
|
Statement = [
|
||||||
|
{
|
||||||
|
Sid = "EnableIAMPolicies"
|
||||||
|
Effect = "Allow"
|
||||||
|
Principal = {
|
||||||
|
AWS = "arn:aws:iam::${local.account_id}:root"
|
||||||
|
}
|
||||||
|
Action = "kms:*"
|
||||||
|
Resource = "*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_kms_alias" "traces" {
|
||||||
|
name = local.kms_alias_name
|
||||||
|
target_key_id = aws_kms_key.traces.key_id
|
||||||
|
}
|
||||||
14
terraform/locals.tf
Normal file
14
terraform/locals.tf
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
locals {
|
||||||
|
account_id = data.aws_caller_identity.current.account_id
|
||||||
|
|
||||||
|
# Account-suffixed names so seahaven-prod can apply in parallel with mgmt
|
||||||
|
# teardown (S3 bucket names are globally unique; PLAT-73 locked decision).
|
||||||
|
traces_bucket_name = "sh-openswe-traces-${local.account_id}"
|
||||||
|
logs_bucket_name = "sh-openswe-traces-logs-${local.account_id}"
|
||||||
|
|
||||||
|
export_user_name = "sh-openswe-langsmith-export"
|
||||||
|
secret_name = "sh-openswe/langsmith-export-s3"
|
||||||
|
kms_alias_name = "alias/sh-openswe-traces"
|
||||||
|
}
|
||||||
|
|
||||||
|
data "aws_caller_identity" "current" {}
|
||||||
36
terraform/outputs.tf
Normal file
36
terraform/outputs.tf
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
output "bucket_name" {
|
||||||
|
value = aws_s3_bucket.traces.id
|
||||||
|
description = "LangSmith Bulk Export destination bucket (account-suffixed)"
|
||||||
|
}
|
||||||
|
|
||||||
|
output "bucket_arn" {
|
||||||
|
value = aws_s3_bucket.traces.arn
|
||||||
|
}
|
||||||
|
|
||||||
|
output "logs_bucket_name" {
|
||||||
|
value = aws_s3_bucket.logs.id
|
||||||
|
}
|
||||||
|
|
||||||
|
output "kms_key_arn" {
|
||||||
|
value = aws_kms_key.traces.arn
|
||||||
|
}
|
||||||
|
|
||||||
|
output "kms_alias" {
|
||||||
|
value = aws_kms_alias.traces.name
|
||||||
|
}
|
||||||
|
|
||||||
|
output "export_user_name" {
|
||||||
|
value = aws_iam_user.langsmith_export.name
|
||||||
|
}
|
||||||
|
|
||||||
|
output "export_user_arn" {
|
||||||
|
value = aws_iam_user.langsmith_export.arn
|
||||||
|
}
|
||||||
|
|
||||||
|
output "export_key_secret_name" {
|
||||||
|
value = aws_secretsmanager_secret.export_key.name
|
||||||
|
}
|
||||||
|
|
||||||
|
output "export_key_secret_arn" {
|
||||||
|
value = aws_secretsmanager_secret.export_key.arn
|
||||||
|
}
|
||||||
11
terraform/providers.tf
Normal file
11
terraform/providers.tf
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
provider "aws" {
|
||||||
|
region = var.aws_region
|
||||||
|
|
||||||
|
default_tags {
|
||||||
|
tags = {
|
||||||
|
Project = "sh-openswe-traces"
|
||||||
|
ManagedBy = "terraform"
|
||||||
|
Workspace = "sh-openswe-traces-prod"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
219
terraform/s3.tf
Normal file
219
terraform/s3.tf
Normal file
|
|
@ -0,0 +1,219 @@
|
||||||
|
resource "aws_s3_bucket" "logs" {
|
||||||
|
bucket = local.logs_bucket_name
|
||||||
|
|
||||||
|
tags = {
|
||||||
|
role = "s3-access-logs"
|
||||||
|
}
|
||||||
|
|
||||||
|
lifecycle {
|
||||||
|
prevent_destroy = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_public_access_block" "logs" {
|
||||||
|
bucket = aws_s3_bucket.logs.id
|
||||||
|
|
||||||
|
block_public_acls = true
|
||||||
|
block_public_policy = true
|
||||||
|
ignore_public_acls = true
|
||||||
|
restrict_public_buckets = true
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_ownership_controls" "logs" {
|
||||||
|
bucket = aws_s3_bucket.logs.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
object_ownership = "BucketOwnerEnforced"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_server_side_encryption_configuration" "logs" {
|
||||||
|
bucket = aws_s3_bucket.logs.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
apply_server_side_encryption_by_default {
|
||||||
|
# SSE-S3 only: S3 log delivery cannot write to an SSE-KMS bucket.
|
||||||
|
sse_algorithm = "AES256"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_lifecycle_configuration" "logs" {
|
||||||
|
bucket = aws_s3_bucket.logs.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
id = "expire-access-logs"
|
||||||
|
status = "Enabled"
|
||||||
|
|
||||||
|
filter {}
|
||||||
|
|
||||||
|
expiration {
|
||||||
|
days = 365
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_policy" "logs" {
|
||||||
|
bucket = aws_s3_bucket.logs.id
|
||||||
|
|
||||||
|
policy = jsonencode({
|
||||||
|
Version = "2012-10-17"
|
||||||
|
Statement = [
|
||||||
|
{
|
||||||
|
Sid = "S3ServerAccessLogsWrite"
|
||||||
|
Effect = "Allow"
|
||||||
|
Principal = {
|
||||||
|
Service = "logging.s3.amazonaws.com"
|
||||||
|
}
|
||||||
|
Action = "s3:PutObject"
|
||||||
|
Resource = "${aws_s3_bucket.logs.arn}/s3-access/*"
|
||||||
|
Condition = {
|
||||||
|
ArnLike = {
|
||||||
|
"aws:SourceArn" = aws_s3_bucket.traces.arn
|
||||||
|
}
|
||||||
|
StringEquals = {
|
||||||
|
"aws:SourceAccount" = local.account_id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid = "DenyInsecureTransport"
|
||||||
|
Effect = "Deny"
|
||||||
|
Principal = "*"
|
||||||
|
Action = "s3:*"
|
||||||
|
Resource = [
|
||||||
|
aws_s3_bucket.logs.arn,
|
||||||
|
"${aws_s3_bucket.logs.arn}/*",
|
||||||
|
]
|
||||||
|
Condition = {
|
||||||
|
Bool = {
|
||||||
|
"aws:SecureTransport" = "false"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket" "traces" {
|
||||||
|
bucket = local.traces_bucket_name
|
||||||
|
|
||||||
|
lifecycle {
|
||||||
|
prevent_destroy = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_public_access_block" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
block_public_acls = true
|
||||||
|
block_public_policy = true
|
||||||
|
ignore_public_acls = true
|
||||||
|
restrict_public_buckets = true
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_ownership_controls" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
object_ownership = "BucketOwnerEnforced"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_versioning" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
versioning_configuration {
|
||||||
|
status = "Enabled"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_server_side_encryption_configuration" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
apply_server_side_encryption_by_default {
|
||||||
|
sse_algorithm = "aws:kms"
|
||||||
|
kms_master_key_id = aws_kms_key.traces.arn
|
||||||
|
}
|
||||||
|
bucket_key_enabled = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_lifecycle_configuration" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
id = "archive-exports-to-deep-archive"
|
||||||
|
status = "Enabled"
|
||||||
|
|
||||||
|
filter {
|
||||||
|
prefix = var.export_prefix
|
||||||
|
}
|
||||||
|
|
||||||
|
transition {
|
||||||
|
days = 90
|
||||||
|
storage_class = "DEEP_ARCHIVE"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
id = "expire-noncurrent-versions"
|
||||||
|
status = "Enabled"
|
||||||
|
|
||||||
|
filter {}
|
||||||
|
|
||||||
|
noncurrent_version_expiration {
|
||||||
|
noncurrent_days = 90
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
id = "abort-incomplete-multipart"
|
||||||
|
status = "Enabled"
|
||||||
|
|
||||||
|
filter {}
|
||||||
|
|
||||||
|
abort_incomplete_multipart_upload {
|
||||||
|
days_after_initiation = 7
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_logging" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
target_bucket = aws_s3_bucket.logs.id
|
||||||
|
target_prefix = "s3-access/"
|
||||||
|
|
||||||
|
depends_on = [aws_s3_bucket_policy.logs]
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_policy" "traces" {
|
||||||
|
bucket = aws_s3_bucket.traces.id
|
||||||
|
|
||||||
|
# No SSE-header enforcement Deny. LangSmith's exporter does not send an
|
||||||
|
# "aws:kms" SSE header, so a "must be aws:kms" Deny blocks its writes — and
|
||||||
|
# StringNotEqualsIfExists on a Deny also blocks header-less puts. Encryption
|
||||||
|
# is guaranteed by the bucket DEFAULT (SSE-KMS with our CMK).
|
||||||
|
policy = jsonencode({
|
||||||
|
Version = "2012-10-17"
|
||||||
|
Statement = [
|
||||||
|
{
|
||||||
|
Sid = "DenyInsecureTransport"
|
||||||
|
Effect = "Deny"
|
||||||
|
Principal = "*"
|
||||||
|
Action = "s3:*"
|
||||||
|
Resource = [
|
||||||
|
aws_s3_bucket.traces.arn,
|
||||||
|
"${aws_s3_bucket.traces.arn}/*",
|
||||||
|
]
|
||||||
|
Condition = {
|
||||||
|
Bool = {
|
||||||
|
"aws:SecureTransport" = "false"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
15
terraform/secrets.tf
Normal file
15
terraform/secrets.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
# Holder only — the access key is minted out-of-band and written with
|
||||||
|
# `aws secretsmanager put-secret-value` (see README). Never manage secret
|
||||||
|
# values in Terraform so they never enter HCP state.
|
||||||
|
#
|
||||||
|
# Pre-created in seahaven-prod for PLAT-73 step 1. Import on first apply:
|
||||||
|
# terraform import aws_secretsmanager_secret.export_key \
|
||||||
|
# arn:aws:secretsmanager:us-east-1:011934824531:secret:sh-openswe/langsmith-export-s3-OQoqqU
|
||||||
|
resource "aws_secretsmanager_secret" "export_key" {
|
||||||
|
name = local.secret_name
|
||||||
|
description = "Access key for the sh-openswe-langsmith-export IAM user, consumed by LangSmith Bulk Export. Populated out-of-band post-deploy; rotate quarterly."
|
||||||
|
|
||||||
|
# Encrypted with the aws/secretsmanager managed key — deliberately NOT the
|
||||||
|
# trace CMK, so the credential and the data it protects never share a key
|
||||||
|
# the writer principal holds any KMS grant on.
|
||||||
|
}
|
||||||
2
terraform/terraform.tfvars.example
Normal file
2
terraform/terraform.tfvars.example
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
aws_region = "us-east-1"
|
||||||
|
export_prefix = "langsmith/"
|
||||||
11
terraform/variables.tf
Normal file
11
terraform/variables.tf
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
variable "aws_region" {
|
||||||
|
type = string
|
||||||
|
description = "AWS region for all resources"
|
||||||
|
default = "us-east-1"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "export_prefix" {
|
||||||
|
type = string
|
||||||
|
description = "S3 key prefix LangSmith writes exports under (also the lifecycle scope)"
|
||||||
|
default = "langsmith/"
|
||||||
|
}
|
||||||
18
terraform/versions.tf
Normal file
18
terraform/versions.tf
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
terraform {
|
||||||
|
required_version = ">= 1.7.0"
|
||||||
|
|
||||||
|
required_providers {
|
||||||
|
aws = {
|
||||||
|
source = "hashicorp/aws"
|
||||||
|
version = "~> 6.57"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cloud {
|
||||||
|
organization = "seahaven"
|
||||||
|
|
||||||
|
workspaces {
|
||||||
|
name = "sh-openswe-traces-prod"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Reference in a new issue