diff --git a/cdk/assets/grafana_userdata.sh b/cdk/assets/grafana_userdata.sh new file mode 100644 index 0000000..bd7c5bd --- /dev/null +++ b/cdk/assets/grafana_userdata.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Grafana OSS bootstrap for the apm-wo-analysis dashboard host (Amazon Linux 2023, +# ARM64). Idempotent enough to re-run. Config + dashboards are pulled from S3 +# (the repo is the source of truth); a systemd timer re-syncs dashboards so panel +# updates ship by re-uploading to S3 — no instance rebuild. +# +# Templated by CDK: __CONFIG_BUCKET__ / __CONFIG_PREFIX__ / __PLUGIN_VERSION__. +set -euxo pipefail + +CONFIG_BUCKET="__CONFIG_BUCKET__" +CONFIG_PREFIX="__CONFIG_PREFIX__" +PLUGIN_VERSION="__PLUGIN_VERSION__" + +# --- Grafana OSS repo + install --- +cat >/etc/yum.repos.d/grafana.repo <<'REPO' +[grafana] +name=grafana +baseurl=https://rpm.grafana.com +repo_gpgcheck=1 +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.grafana.com/gpg.key +sslverify=1 +REPO +dnf install -y grafana + +# --- Athena datasource plugin (pinned for reproducibility) --- +grafana-cli --pluginsDir /var/lib/grafana/plugins plugins install grafana-athena-datasource "${PLUGIN_VERSION}" + +# --- grafana.ini: behind the ALB at grafana.seahaven.com, kiosk-friendly --- +cat >/etc/grafana/grafana.ini <<'INI' +[server] +protocol = http +http_port = 3000 +root_url = https://grafana.seahaven.com/ +enforce_domain = false + +[security] +# Behind an office-IP-restricted ALB; allow embedding for the kiosk wall display. +allow_embedding = true +cookie_secure = true + +[users] +default_theme = dark + +[analytics] +reporting_enabled = false +check_for_updates = false +INI + +# --- sync provisioning + dashboards from S3 (repo is source of truth) --- +sync_config() { + aws s3 sync "s3://${CONFIG_BUCKET}/${CONFIG_PREFIX}/provisioning/" /etc/grafana/provisioning/ --delete + aws s3 sync "s3://${CONFIG_BUCKET}/${CONFIG_PREFIX}/dashboards/" /var/lib/grafana/dashboards/ --delete + chown -R grafana:grafana /etc/grafana/provisioning /var/lib/grafana/dashboards +} +mkdir -p /var/lib/grafana/dashboards +sync_config + +systemctl daemon-reload +systemctl enable --now grafana-server + +# --- systemd timer: re-sync dashboards every 15 min so repo edits land without a rebuild --- +cat >/usr/local/bin/grafana-config-sync.sh </etc/systemd/system/grafana-config-sync.service <<'SVC' +[Unit] +Description=Sync apm-wo Grafana config/dashboards from S3 +[Service] +Type=oneshot +ExecStart=/usr/local/bin/grafana-config-sync.sh +SVC + +cat >/etc/systemd/system/grafana-config-sync.timer <<'TIMER' +[Unit] +Description=Periodic apm-wo Grafana config sync +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min +[Install] +WantedBy=timers.target +TIMER + +systemctl daemon-reload +systemctl enable --now grafana-config-sync.timer diff --git a/cdk/cdk.json b/cdk/cdk.json index 164a462..5d9f993 100644 --- a/cdk/cdk.json +++ b/cdk/cdk.json @@ -5,6 +5,13 @@ "wildcardCertArn": "arn:aws:acm:us-east-1:328440206208:certificate/a66c0994-90d4-410a-a1d9-5595c2a3fae3", "hostedZoneId": "Z06652411XKH89KTZD3XA", "hostedZoneName": "seahaven.com", - "slackInteractionsDomain": "apm-wo.seahaven.com" + "slackInteractionsDomain": "apm-wo.seahaven.com", + "grafanaVpcId": "vpc-0d3d4b67bd0cf8a68", + "grafanaAzs": ["us-east-1a", "us-east-1b"], + "grafanaPublicSubnetIds": ["subnet-0eea820effe1b3ae5", "subnet-0012f5895182c1580"], + "grafanaPrivateSubnetIds": ["subnet-04e38c507e96f1926", "subnet-0a0b4fc6f296dfba5"], + "grafanaDomain": "grafana.seahaven.com", + "officeCidrs": ["47.21.61.4/32", "96.250.164.146/32"], + "athenaPluginVersion": "2.18.2" } } diff --git a/cdk/stacks/grafana_stack.py b/cdk/stacks/grafana_stack.py index 368bbb1..ee1a713 100644 --- a/cdk/stacks/grafana_stack.py +++ b/cdk/stacks/grafana_stack.py @@ -1,22 +1,290 @@ -"""Grafana stack: VPC import, EC2, ALB, SG, Route53, Athena datasource role. +"""Grafana stack: imported VPC, EC2 (Grafana OSS), ALB, SG, Route53, IAM, backup. -Scaffold — resources are added in Phase 5 of docs/BUILD.md. The running box is -the only non-serverless piece here (self-hosted Grafana OSS on a t4g.small, -ARM64, VPN-only) and carries an OS/Grafana patching + config-backup obligation. -Dashboards are provisioned as code from grafana/ — the running instance is never -the source of truth. +The only non-serverless piece in this repo (self-hosted Grafana OSS on a +t4g.small, ARM64, AL2023). Reachability: an internet-facing ALB whose security +group only admits the office CIDRs — there is no Client VPN in the account, so +"VPN-only" is realized as office-IP restriction (the same pattern syslog-server +uses). The instance sits in private subnets, reachable only from the ALB SG and +administered via SSM Session Manager (no SSH, no key pair). + +Dashboards/datasources are provisioned as code: CDK uploads ``grafana/`` to an S3 +config prefix; instance user-data syncs it on boot and a systemd timer re-syncs, +so the running box is never the source of truth. The gp3 root volume is RETAINed +and snapshotted daily by DLM; ``grafana.db`` lives there. + +Config (cdk.json context): grafanaVpcId, grafanaAzs, grafana{Public,Private}SubnetIds, +grafanaDomain, officeCidrs, wildcardCertArn, hostedZoneId/Name, athenaPluginVersion. """ -from aws_cdk import Stack +import os + +from aws_cdk import ( + CfnTag, + Stack, + Tags, +) +from aws_cdk import ( + aws_certificatemanager as acm, +) +from aws_cdk import ( + aws_dlm as dlm, +) +from aws_cdk import ( + aws_ec2 as ec2, +) +from aws_cdk import ( + aws_elasticloadbalancingv2 as elbv2, +) +from aws_cdk import ( + aws_elasticloadbalancingv2_targets as elbv2_targets, +) +from aws_cdk import ( + aws_iam as iam, +) +from aws_cdk import ( + aws_route53 as route53, +) +from aws_cdk import ( + aws_route53_targets as route53_targets, +) +from aws_cdk import ( + aws_s3 as s3, +) +from aws_cdk import ( + aws_s3_deployment as s3deploy, +) from constructs import Construct +EXPORTS_BUCKET = "apm-wo-analysis-exports-328440206208" +CONFIG_PREFIX = "grafana-config" +BACKUP_TAG = "apm-grafana-backup" +GRAFANA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "grafana") +USERDATA_PATH = os.path.join( + os.path.dirname(__file__), "..", "assets", "grafana_userdata.sh" +) + class GrafanaStack(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) - # Phase 5 — EC2 (Amazon Linux 2023, Grafana OSS via user-data), - # internal ALB (HTTPS, *.seahaven.com ACM cert), - # SG ingress from VPN/office CIDRs only, - # Route53 alias grafana.seahaven.com, - # instance role: Athena + Glue + S3 read (no static keys). TODO + ctx = self.node.try_get_context + domain = ctx("grafanaDomain") + + # Import the shared seahaven-vpc by explicit attributes (no context + # lookup, so the offline synth test needs no AWS credentials). + vpc = ec2.Vpc.from_vpc_attributes( + self, + "SeahavenVpc", + vpc_id=ctx("grafanaVpcId"), + availability_zones=ctx("grafanaAzs"), + public_subnet_ids=ctx("grafanaPublicSubnetIds"), + private_subnet_ids=ctx("grafanaPrivateSubnetIds"), + ) + + # ----- Security groups ----- + alb_sg = ec2.SecurityGroup( + self, + "AlbSg", + vpc=vpc, + description="apm-wo grafana ALB", + allow_all_outbound=True, + ) + for cidr in ctx("officeCidrs"): + alb_sg.add_ingress_rule( + ec2.Peer.ipv4(cidr), ec2.Port.tcp(443), f"HTTPS from office {cidr}" + ) + + instance_sg = ec2.SecurityGroup( + self, + "InstanceSg", + vpc=vpc, + description="apm-wo grafana instance", + allow_all_outbound=True, + ) + instance_sg.add_ingress_rule( + alb_sg, ec2.Port.tcp(3000), "Grafana HTTP from the ALB only" + ) + + # ----- Instance role: Athena query + Glue read + S3 (no static keys) ----- + role = iam.Role( + self, + "GrafanaInstanceRole", + assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), + managed_policies=[ + # Session Manager admin access; no SSH / bastion / key pair. + iam.ManagedPolicy.from_aws_managed_policy_name( + "AmazonSSMManagedInstanceCore" + ) + ], + ) + role.add_to_policy( + iam.PolicyStatement( + sid="AthenaQuery", + actions=[ + "athena:StartQueryExecution", + "athena:StopQueryExecution", + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:GetWorkGroup", + "athena:ListWorkGroups", + ], + resources=[ + f"arn:aws:athena:{self.region}:{self.account}:workgroup/apm-wo-analysis" + ], + ) + ) + role.add_to_policy( + iam.PolicyStatement( + sid="GlueReadOnly", + actions=[ + "glue:GetDatabase", + "glue:GetDatabases", + "glue:GetTable", + "glue:GetTables", + "glue:GetPartition", + "glue:GetPartitions", + ], + resources=[ + f"arn:aws:glue:{self.region}:{self.account}:catalog", + f"arn:aws:glue:{self.region}:{self.account}:database/apm_wo_analysis", + f"arn:aws:glue:{self.region}:{self.account}:table/apm_wo_analysis/*", + ], + ) + ) + bucket = s3.Bucket.from_bucket_name(self, "Exports", EXPORTS_BUCKET) + # Read the analytics snapshots; read+write athena-results (query output). + bucket.grant_read(role, "analytics/*") + bucket.grant_read_write(role, "athena-results/*") + # Config sync reads the grafana-config prefix. + bucket.grant_read(role, f"{CONFIG_PREFIX}/*") + + # ----- Instance (Grafana OSS via user-data) ----- + with open(USERDATA_PATH, encoding="utf-8") as fh: + userdata_script = fh.read() + userdata_script = ( + userdata_script.replace("__CONFIG_BUCKET__", EXPORTS_BUCKET) + .replace("__CONFIG_PREFIX__", CONFIG_PREFIX) + .replace("__PLUGIN_VERSION__", ctx("athenaPluginVersion")) + ) + user_data = ec2.UserData.for_linux() + user_data.add_commands(userdata_script) + + instance = ec2.Instance( + self, + "Grafana", + vpc=vpc, + vpc_subnets=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS + ), + instance_type=ec2.InstanceType("t4g.small"), + machine_image=ec2.MachineImage.latest_amazon_linux2023( + cpu_type=ec2.AmazonLinuxCpuType.ARM_64 + ), + security_group=instance_sg, + role=role, + user_data=user_data, + require_imdsv2=True, + block_devices=[ + ec2.BlockDevice( + device_name="/dev/xvda", + # RETAIN the gp3 root volume (grafana.db lives here). + volume=ec2.BlockDeviceVolume.ebs( + 20, + volume_type=ec2.EbsDeviceVolumeType.GP3, + delete_on_termination=False, + ), + ) + ], + ) + Tags.of(instance).add(BACKUP_TAG, "true") + + # ----- ALB (internet-facing, office-IP-restricted, HTTPS) ----- + cert = acm.Certificate.from_certificate_arn( + self, "WildcardCert", ctx("wildcardCertArn") + ) + alb = elbv2.ApplicationLoadBalancer( + self, + "Alb", + vpc=vpc, + internet_facing=True, + security_group=alb_sg, + vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), + ) + listener = alb.add_listener( + "Https", + port=443, + protocol=elbv2.ApplicationProtocol.HTTPS, + certificates=[cert], + # Do NOT auto-open 0.0.0.0/0 on 443 — the ALB SG already scopes + # ingress to the office CIDRs. open=True would undo that. + open=False, + ) + listener.add_targets( + "GrafanaTarget", + port=3000, + protocol=elbv2.ApplicationProtocol.HTTP, + targets=[elbv2_targets.InstanceTarget(instance, 3000)], + health_check=elbv2.HealthCheck( + path="/api/health", healthy_http_codes="200" + ), + ) + + # ----- Route53 alias grafana.seahaven.com -> ALB ----- + zone = route53.HostedZone.from_hosted_zone_attributes( + self, + "SeahavenZone", + hosted_zone_id=ctx("hostedZoneId"), + zone_name=ctx("hostedZoneName"), + ) + route53.ARecord( + self, + "GrafanaAlias", + zone=zone, + record_name=domain.split(".")[0], + target=route53.RecordTarget.from_alias( + route53_targets.LoadBalancerTarget(alb) + ), + ) + + # ----- Dashboards-as-code: upload grafana/ to the S3 config prefix ----- + s3deploy.BucketDeployment( + self, + "GrafanaConfig", + sources=[s3deploy.Source.asset(GRAFANA_DIR)], + destination_bucket=bucket, + destination_key_prefix=CONFIG_PREFIX, + prune=True, + ) + + # ----- Daily DLM snapshot of the (tagged) instance's volume ----- + dlm_role = iam.Role( + self, + "DlmRole", + assumed_by=iam.ServicePrincipal("dlm.amazonaws.com"), + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name( + "service-role/AWSDataLifecycleManagerServiceRole" + ) + ], + ) + dlm.CfnLifecyclePolicy( + self, + "GrafanaBackup", + description="Daily snapshot of the apm-wo grafana volume", + state="ENABLED", + execution_role_arn=dlm_role.role_arn, + policy_details=dlm.CfnLifecyclePolicy.PolicyDetailsProperty( + resource_types=["INSTANCE"], + target_tags=[CfnTag(key=BACKUP_TAG, value="true")], + schedules=[ + dlm.CfnLifecyclePolicy.ScheduleProperty( + name="daily", + create_rule=dlm.CfnLifecyclePolicy.CreateRuleProperty( + interval=24, interval_unit="HOURS", times=["07:00"] + ), + retain_rule=dlm.CfnLifecyclePolicy.RetainRuleProperty(count=7), + ) + ], + ), + ) diff --git a/grafana/dashboards/apm-work-orders.json b/grafana/dashboards/apm-work-orders.json index 5fdc3c8..ffecd14 100644 --- a/grafana/dashboards/apm-work-orders.json +++ b/grafana/dashboards/apm-work-orders.json @@ -1,7 +1,7 @@ { "uid": "apm-wo", "title": "APM Work Orders", - "description": "Daily APM work-order analysis — breakdown, escalations, trend, filterable WO table, mismatches. Built in Phase 5 (docs/BUILD.md) by the grafana-author agent.", + "description": "Daily APM work-order analysis — breakdown, escalations, trend, filterable WO table, mismatches. Built in Phase 5 by the grafana-author agent. Athena datasource uid='athena'; data grain is one row per WO per daily snapshot partitioned by dt.", "tags": ["apm", "work-orders"], "timezone": "browser", "schemaVersion": 39, @@ -12,7 +12,799 @@ "to": "now" }, "templating": { - "list": [] + "list": [ + { + "name": "dt", + "label": "Snapshot Date", + "description": "Single daily snapshot partition. Defaults to the latest available dt. Most panels filter WHERE dt = '$dt'.", + "type": "query", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "query": { + "rawSql": "SELECT DISTINCT dt FROM apm_wo_analysis.apm_wo_snapshots ORDER BY dt DESC", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + }, + "refresh": 2, + "sort": 0, + "multi": false, + "includeAll": false, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "site", + "label": "Site", + "description": "Multi-select. WO table applies: AND ('${site:raw}' = 'All' OR site IN (${site:singlequote}))", + "type": "query", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "query": { + "rawSql": "SELECT DISTINCT site FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND site<>'' ORDER BY 1", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": "All", + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "department", + "label": "Department", + "description": "Multi-select. WO table applies: AND ('${department:raw}' = 'All' OR department IN (${department:singlequote}))", + "type": "query", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "query": { + "rawSql": "SELECT DISTINCT department FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND department<>'' ORDER BY 1", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": "All", + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "category", + "label": "Category", + "description": "Multi-select. WO table applies: AND ('${category:raw}' = 'All' OR category IN (${category:singlequote}))", + "type": "query", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "query": { + "rawSql": "SELECT DISTINCT category FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND category<>'' ORDER BY 1", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": "All", + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "wo_status", + "label": "WO Status", + "description": "Multi-select. WO table applies: AND ('${wo_status:raw}' = 'All' OR wo_status IN (${wo_status:singlequote}))", + "type": "query", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "query": { + "rawSql": "SELECT DISTINCT wo_status FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND wo_status<>'' ORDER BY 1", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": "All", + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "hold_reason", + "label": "Hold Reason", + "description": "Multi-select. WO table applies: AND ('${hold_reason:raw}' = 'All' OR hold_reason IN (${hold_reason:singlequote}))", + "type": "query", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "query": { + "rawSql": "SELECT DISTINCT hold_reason FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND hold_reason<>'' ORDER BY 1", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": "All", + "current": {}, + "options": [], + "hide": 0 + } + ] }, - "panels": [] + "panels": [ + { + "id": 1, + "type": "bar-chart", + "title": "Category Distribution", + "description": "Count of work orders by classification category for the selected snapshot date. Sorted descending by volume. Filter using the template variables above.", + "gridPos": { "x": 0, "y": 0, "w": 14, "h": 9 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT category, COUNT(*) AS wos FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' GROUP BY 1 ORDER BY 2 DESC", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "orientation": "horizontal", + "barRadius": 0, + "groupWidth": 0.7, + "showValue": "always", + "stacking": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "showLegend": false, "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 80 } + ] + } + }, + "overrides": [] + } + }, + { + "id": 2, + "type": "piechart", + "title": "Escalation Summary", + "description": "Distribution across escalation categories (1st, 2nd, 3rd Escalation, SIM Ticket, Other Escalation) for the selected snapshot date. 3rd Escalation is highlighted red.", + "gridPos": { "x": 14, "y": 0, "w": 10, "h": 9 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT category, COUNT(*) AS escalations FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND is_escalation=true GROUP BY 1 ORDER BY 2 DESC", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "pieType": "pie", + "displayLabels": ["name", "value"], + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "showLegend": true, "displayMode": "table", "placement": "right", "values": ["value", "percent"] } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "hideFrom": { "legend": false, "tooltip": false, "viz": false } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "3rd Escalation" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "2nd Escalation" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "1st Escalation" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "SIM Ticket" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } + ] + } + ] + } + }, + { + "id": 3, + "type": "piechart", + "title": "Action-Needed vs Routine", + "description": "Action-needed WOs require attention (escalations, scheduling, vendor/report waits, status inquiries, vendor no-shows). Routine WOs are on track. Counts are for the selected snapshot date.", + "gridPos": { "x": 0, "y": 9, "w": 8, "h": 8 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT CASE WHEN is_action=true THEN 'Action Needed' ELSE 'Routine' END AS action_type, COUNT(*) AS wos FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' GROUP BY 1", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "pieType": "donut", + "displayLabels": ["name", "percent"], + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "values": ["value", "percent"] } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "hideFrom": { "legend": false, "tooltip": false, "viz": false } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Action Needed" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "semi-dark-orange", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Routine" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] + } + }, + { + "id": 4, + "type": "bar-chart", + "title": "Escalations by Site", + "description": "Total escalations per site for the selected snapshot date. Includes all escalation categories.", + "gridPos": { "x": 8, "y": 9, "w": 16, "h": 8 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT site, COUNT(*) AS escalations FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND is_escalation=true AND site<>'' GROUP BY 1 ORDER BY 2 DESC", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "orientation": "vertical", + "barRadius": 0, + "groupWidth": 0.7, + "showValue": "always", + "stacking": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "showLegend": false, "displayMode": "list", "placement": "bottom" }, + "xTickLabelRotation": -45, + "xTickLabelMaxLength": 12 + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "semi-dark-red" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Escalations", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + } + }, + "overrides": [] + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Trend Over Time — Escalations & Action-Needed per Day", + "description": "Runs across all partitions (no $dt filter) to show daily escalation and action-needed volume over time. Use the dashboard time range picker to zoom in. This is the capability the legacy Sheet never had.", + "gridPos": { "x": 0, "y": 17, "w": 24, "h": 9 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT date_parse(dt, '%Y-%m-%d') AS time, SUM(CAST(is_escalation AS INTEGER)) AS escalations, SUM(CAST(is_action AS INTEGER)) AS action_needed FROM apm_wo_analysis.apm_wo_snapshots GROUP BY 1 ORDER BY 1", + "format": "timeSeries", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Work Orders", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "escalations" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } }, + { "id": "displayName", "value": "Escalations" } + ] + }, + { + "matcher": { "id": "byName", "options": "action_needed" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "semi-dark-orange", "mode": "fixed" } }, + { "id": "displayName", "value": "Action Needed" } + ] + } + ] + } + }, + { + "id": 6, + "type": "table", + "title": "Work Order Detail Table", + "description": "Filterable, exportable table of all work orders for the selected snapshot date. Multi-value template variables are applied via: AND ('${variable:raw}' = 'All' OR column IN (${variable:singlequote})). Escalation rows are color-coded: 3rd Escalation = red, 2nd Escalation = orange, 1st Escalation = yellow. WO numbers are plain text (no APM deep-link per project decision). Use the Download CSV button (table header menu) to export. last_comment is wrapped for readability.", + "gridPos": { "x": 0, "y": 26, "w": 24, "h": 14 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT wo_number, site, department, category, wo_status, hold_reason, wo_description, last_comment FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND ('${site:raw}' = 'All' OR site IN (${site:singlequote})) AND ('${department:raw}' = 'All' OR department IN (${department:singlequote})) AND ('${category:raw}' = 'All' OR category IN (${category:singlequote})) AND ('${wo_status:raw}' = 'All' OR wo_status IN (${wo_status:singlequote})) AND ('${hold_reason:raw}' = 'All' OR hold_reason IN (${hold_reason:singlequote})) ORDER BY category, site", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "sortBy": [], + "footer": { + "show": false, + "reducer": ["sum"], + "fields": "", + "enablePagination": false + } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "align": "left", + "cellOptions": { "type": "auto" }, + "inspect": false, + "filterable": true, + "minWidth": 80, + "width": 0 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "text", "value": null } + ] + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "wo_number" }, + "properties": [ + { "id": "displayName", "value": "WO #" }, + { "id": "custom.width", "value": 100 } + ] + }, + { + "matcher": { "id": "byName", "options": "site" }, + "properties": [ + { "id": "displayName", "value": "Site" }, + { "id": "custom.width", "value": 80 } + ] + }, + { + "matcher": { "id": "byName", "options": "department" }, + "properties": [ + { "id": "displayName", "value": "Dept" }, + { "id": "custom.width", "value": 80 } + ] + }, + { + "matcher": { "id": "byName", "options": "category" }, + "properties": [ + { "id": "displayName", "value": "Category" }, + { "id": "custom.width", "value": 160 }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "3rd Escalation": { + "color": "dark-red", + "index": 0 + }, + "2nd Escalation": { + "color": "dark-orange", + "index": 1 + }, + "1st Escalation": { + "color": "dark-yellow", + "index": 2 + }, + "SIM Ticket": { + "color": "dark-purple", + "index": 3 + }, + "Other Escalation": { + "color": "orange", + "index": 4 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { "type": "color-text" } + } + ] + }, + { + "matcher": { "id": "byName", "options": "wo_status" }, + "properties": [ + { "id": "displayName", "value": "Status" }, + { "id": "custom.width", "value": 80 } + ] + }, + { + "matcher": { "id": "byName", "options": "hold_reason" }, + "properties": [ + { "id": "displayName", "value": "Hold Reason" }, + { "id": "custom.width", "value": 120 } + ] + }, + { + "matcher": { "id": "byName", "options": "wo_description" }, + "properties": [ + { "id": "displayName", "value": "Description" }, + { "id": "custom.width", "value": 220 } + ] + }, + { + "matcher": { "id": "byName", "options": "last_comment" }, + "properties": [ + { "id": "displayName", "value": "Last Comment" }, + { + "id": "custom.cellOptions", + "value": { "type": "auto", "wrapText": true } + }, + { "id": "custom.width", "value": 420 }, + { "id": "custom.minWidth", "value": 200 } + ] + } + ] + } + }, + { + "id": 7, + "type": "table", + "title": "Mismatch Panel — Comment vs Structured-State Contradictions", + "description": "Work orders where the classified comment intent contradicts the structured WO state (e.g. comment says 'completed' but WO is on a REPORT/VENDOR/SCHEDULING hold, or comment says 'scheduled' while status is IP). Non-empty mismatch column only. Use these rows for manual review before the next export.", + "gridPos": { "x": 0, "y": 40, "w": 24, "h": 8 }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-athena-datasource", + "uid": "athena" + }, + "rawSql": "SELECT wo_number, site, category, wo_status, hold_reason, mismatch FROM apm_wo_analysis.apm_wo_snapshots WHERE dt='$dt' AND mismatch<>'' ORDER BY category, site", + "format": "table", + "connectionArgs": { + "catalog": "AwsDataCatalog", + "database": "apm_wo_analysis", + "region": "us-east-1" + } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "sortBy": [], + "footer": { + "show": false, + "reducer": ["sum"], + "fields": "", + "enablePagination": false + } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "align": "left", + "cellOptions": { "type": "auto" }, + "inspect": false, + "filterable": true, + "minWidth": 80, + "width": 0 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "text", "value": null } + ] + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "wo_number" }, + "properties": [ + { "id": "displayName", "value": "WO #" }, + { "id": "custom.width", "value": 100 } + ] + }, + { + "matcher": { "id": "byName", "options": "site" }, + "properties": [ + { "id": "displayName", "value": "Site" }, + { "id": "custom.width", "value": 80 } + ] + }, + { + "matcher": { "id": "byName", "options": "category" }, + "properties": [ + { "id": "displayName", "value": "Category" }, + { "id": "custom.width", "value": 160 }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "3rd Escalation": { + "color": "dark-red", + "index": 0 + }, + "2nd Escalation": { + "color": "dark-orange", + "index": 1 + }, + "1st Escalation": { + "color": "dark-yellow", + "index": 2 + }, + "SIM Ticket": { + "color": "dark-purple", + "index": 3 + }, + "Other Escalation": { + "color": "orange", + "index": 4 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { "type": "color-text" } + } + ] + }, + { + "matcher": { "id": "byName", "options": "wo_status" }, + "properties": [ + { "id": "displayName", "value": "WO Status" }, + { "id": "custom.width", "value": 90 } + ] + }, + { + "matcher": { "id": "byName", "options": "hold_reason" }, + "properties": [ + { "id": "displayName", "value": "Hold Reason" }, + { "id": "custom.width", "value": 120 } + ] + }, + { + "matcher": { "id": "byName", "options": "mismatch" }, + "properties": [ + { "id": "displayName", "value": "Mismatch Description" }, + { + "id": "custom.cellOptions", + "value": { "type": "auto", "wrapText": true } + }, + { "id": "custom.width", "value": 500 }, + { "id": "custom.minWidth", "value": 200 }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "semi-dark-orange" } } + ] + } + ] + } + } + ] } diff --git a/grafana/provisioning/datasources/athena.yaml b/grafana/provisioning/datasources/athena.yaml index 5831d8b..58072bf 100644 --- a/grafana/provisioning/datasources/athena.yaml +++ b/grafana/provisioning/datasources/athena.yaml @@ -4,6 +4,7 @@ apiVersion: 1 datasources: - name: Athena type: grafana-athena-datasource + uid: athena isDefault: true jsonData: authType: ec2_iam_role diff --git a/tests/test_grafana_synth.py b/tests/test_grafana_synth.py new file mode 100644 index 0000000..42f29b1 --- /dev/null +++ b/tests/test_grafana_synth.py @@ -0,0 +1,177 @@ +"""Synth-level assertions for the Grafana stack (Phase 5). + +Synthesizes ``apm-wo-analysis-grafana`` and asserts the security posture that +can't be eyeballed: the ALB only admits the office CIDRs on 443 (never +0.0.0.0/0), the instance only takes traffic from the ALB SG, the instance role +carries no static keys and only scoped Athena/Glue-read/S3 access, the root +volume is gp3 + retained, a daily DLM backup exists, and grafana.seahaven.com +aliases the ALB. No AWS, no Docker (bundling skipped). + +Run with the repo venv: + + python -m pytest tests/test_grafana_synth.py -q +""" + +import json +import sys +from pathlib import Path + +import aws_cdk as cdk +from aws_cdk.assertions import Match, Template + +CDK_DIR = Path(__file__).resolve().parents[1] / "cdk" +sys.path.insert(0, str(CDK_DIR)) + +from stacks.grafana_stack import GrafanaStack # noqa: E402 + +OFFICE_CIDRS = {"47.21.61.4/32", "96.250.164.146/32"} + + +def _cdk_context() -> dict: + ctx = json.loads((CDK_DIR / "cdk.json").read_text())["context"] + ctx["aws:cdk:bundling-stacks"] = [] + return ctx + + +def _template() -> Template: + app = cdk.App(context=_cdk_context()) + stack = GrafanaStack( + app, + "apm-wo-analysis-grafana", + env=cdk.Environment(account="328440206208", region="us-east-1"), + ) + return Template.from_stack(stack) + + +def _all_cidr_ingress(t: Template): + """Every CIDR-based ingress rule, inline on SGs and standalone, as + (cidr, from_port, to_port) tuples.""" + rules = [] + for sg in t.find_resources("AWS::EC2::SecurityGroup").values(): + for r in sg["Properties"].get("SecurityGroupIngress", []): + if "CidrIp" in r: + rules.append((r["CidrIp"], r.get("FromPort"), r.get("ToPort"))) + for ing in t.find_resources("AWS::EC2::SecurityGroupIngress").values(): + p = ing["Properties"] + if "CidrIp" in p: + rules.append((p["CidrIp"], p.get("FromPort"), p.get("ToPort"))) + return rules + + +def test_alb_only_admits_office_cidrs_on_443(): + rules = _all_cidr_ingress(_template()) + cidrs_443 = {c for c, fp, tp in rules if fp == 443 and tp == 443} + assert cidrs_443 == OFFICE_CIDRS, ( + f"443 ingress should be office-only, got {cidrs_443}" + ) + # Nothing anywhere may be open to the world. + assert all(c != "0.0.0.0/0" for c, _, _ in rules), "found a 0.0.0.0/0 ingress" + + +def test_instance_only_reachable_from_alb_on_3000(): + # The instance SG ingress on 3000 is a SourceSecurityGroup rule, not a CIDR. + _template().has_resource_properties( + "AWS::EC2::SecurityGroupIngress", + Match.object_like( + { + "FromPort": 3000, + "ToPort": 3000, + "SourceSecurityGroupId": Match.any_value(), + } + ), + ) + + +def test_alb_internet_facing_https_listener(): + t = _template() + t.has_resource_properties( + "AWS::ElasticLoadBalancingV2::LoadBalancer", {"Scheme": "internet-facing"} + ) + t.has_resource_properties( + "AWS::ElasticLoadBalancingV2::Listener", + Match.object_like( + {"Port": 443, "Protocol": "HTTPS", "Certificates": Match.any_value()} + ), + ) + + +def test_no_static_keys_in_stack(): + t = _template() + t.resource_count_is("AWS::IAM::User", 0) + t.resource_count_is("AWS::IAM::AccessKey", 0) + + +def test_instance_role_scoped_and_uses_ssm(): + t = _template() + # Session Manager (no SSH) — the SSM managed policy is attached. + t.has_resource_properties( + "AWS::IAM::Role", + Match.object_like( + { + "ManagedPolicyArns": Match.array_with( + [ + { + "Fn::Join": [ + "", + Match.array_with( + [":iam::aws:policy/AmazonSSMManagedInstanceCore"] + ), + ] + } + ] + ) + } + ), + ) + # The instance role must not be able to write the catalog or run wide Athena. + for policy in t.find_resources("AWS::IAM::Policy").values(): + for stmt in policy["Properties"]["PolicyDocument"]["Statement"]: + actions = stmt.get("Action", []) + actions = [actions] if isinstance(actions, str) else actions + for a in actions: + if isinstance(a, str): + assert a not in ("glue:*", "athena:*", "s3:*", "*"), ( + f"too broad: {a}" + ) + assert not a.startswith("glue:Create"), f"no Glue writes: {a}" + assert not a.startswith("glue:Update"), f"no Glue writes: {a}" + + +def test_root_volume_gp3_and_retained(): + _template().has_resource_properties( + "AWS::EC2::Instance", + Match.object_like( + { + "BlockDeviceMappings": Match.array_with( + [ + Match.object_like( + { + "Ebs": Match.object_like( + {"VolumeType": "gp3", "DeleteOnTermination": False} + ) + } + ) + ] + ) + } + ), + ) + + +def test_daily_dlm_backup_enabled(): + _template().has_resource_properties( + "AWS::DLM::LifecyclePolicy", + Match.object_like( + { + "State": "ENABLED", + "PolicyDetails": Match.object_like({"ResourceTypes": ["INSTANCE"]}), + } + ), + ) + + +def test_route53_alias_for_grafana(): + _template().has_resource_properties( + "AWS::Route53::RecordSet", + Match.object_like({"Type": "A", "Name": "grafana.seahaven.com."}), + )