Fix dashboard rendering: barchart panel type + metadata off the table prefix

Two issues found loading the deployed dashboard:

1. Panels used type "bar-chart" (hyphenated); Grafana's core panel is "barchart"
   — hence "plugin bar-chart required". Fixed both panels.

2. ALL panels showed "no data" because Athena failed with HIVE_BAD_DATA:
   the classifier wrote summary.json/details.json INTO analytics/dt=*/ — the
   same prefix the Glue table scans — so Athena tried to read the JSON as
   Parquet and every query failed. Move the metadata to a separate meta/dt=*/
   prefix: classifier writes there (grant_read_write meta/*), the Slack Lambdas
   read there (read_meta_json, grant_read meta/*), and analytics/ holds only
   Parquet. Verified: the category GROUP BY query now succeeds against Athena.
This commit is contained in:
Adam Moussa 2026-05-28 18:49:41 -04:00
parent ea54cb1e60
commit 68f3acded8
6 changed files with 21 additions and 13 deletions

View file

@ -274,6 +274,9 @@ class PipelineStack(Stack):
# Parquet to S3 — it never touches the catalog (Phase 3).
self.exports_bucket.grant_read(self.classifier_fn, "raw/*")
self.exports_bucket.grant_read_write(self.classifier_fn, "analytics/*")
# summary.json/details.json live under meta/ (kept out of the Athena
# table's analytics/ prefix so queries don't read JSON as Parquet).
self.exports_bucket.grant_read_write(self.classifier_fn, "meta/*")
secretsmanager.Secret.from_secret_name_v2(
self, "AnthropicKey", ANTHROPIC_SECRET
).grant_read(self.classifier_fn)
@ -335,7 +338,8 @@ class PipelineStack(Stack):
),
),
)
self.exports_bucket.grant_read(fn, "analytics/*")
# Slack Lambdas read only the daily JSON under meta/ (not the Parquet).
self.exports_bucket.grant_read(fn, "meta/*")
slack_secret.grant_read(fn)
dashboard_param.grant_read(fn)
return fn

View file

@ -179,7 +179,7 @@
"panels": [
{
"id": 1,
"type": "bar-chart",
"type": "barchart",
"title": "Category Distribution",
"description": "Count of work orders by classification category for the selected snapshot date. Sorted descending by volume. Filter using the template variables above.",
"gridPos": { "x": 0, "y": 0, "w": 14, "h": 9 },
@ -366,7 +366,7 @@
},
{
"id": 4,
"type": "bar-chart",
"type": "barchart",
"title": "Escalations by Site",
"description": "Total escalations per site for the selected snapshot date. Includes all escalation categories.",
"gridPos": { "x": 8, "y": 9, "w": 16, "h": 8 },

View file

@ -30,7 +30,8 @@ import pandas as pd
import classify as clf
ANALYTICS_PREFIX = "analytics"
ANALYTICS_PREFIX = "analytics" # Parquet snapshots — the Glue/Athena table reads this
META_PREFIX = "meta" # summary.json/details.json — kept OUT of the table's prefix
# Map snapshot field -> substring matched (case-insensitively) against the export
# header, tolerating minor header drift in the 13-column APM export.
@ -237,17 +238,19 @@ def handler(event, context):
mode="overwrite_partitions",
)
# summary.json/details.json go under meta/ — NOT analytics/. Athena reads
# every object in the table's prefix as Parquet, so JSON there breaks queries.
summary = _build_summary(df, dt, key, blank)
_s3.put_object(
Bucket=bucket,
Key=f"{ANALYTICS_PREFIX}/dt={dt}/summary.json",
Key=f"{META_PREFIX}/dt={dt}/summary.json",
Body=json.dumps(summary, indent=2).encode("utf-8"),
ContentType="application/json",
)
# details.json — per-WO index the slack-post Lambda reads for drill-down modals.
_s3.put_object(
Bucket=bucket,
Key=f"{ANALYTICS_PREFIX}/dt={dt}/details.json",
Key=f"{META_PREFIX}/dt={dt}/details.json",
Body=json.dumps(_build_details(df)).encode("utf-8"),
ContentType="application/json",
)

View file

@ -25,11 +25,11 @@ def handler(event, context):
"""Post the daily summary and (conditionally) the 3rd-escalation alert."""
dt = (event or {}).get("dt") or datetime.now(timezone.utc).strftime("%Y-%m-%d")
today = slackio.read_analytics_json(dt, "summary.json")
today = slackio.read_meta_json(dt, "summary.json")
if today is None:
print(f"No summary.json for dt={dt}; nothing to post.")
return {"posted": False, "reason": "no summary", "dt": dt}
yesterday = slackio.read_analytics_json(_yesterday(dt), "summary.json")
yesterday = slackio.read_meta_json(_yesterday(dt), "summary.json")
client = slackio.web_client()
channel = slackio.channel_id()
@ -44,7 +44,7 @@ def handler(event, context):
# Standalone batched alert — only when there are 3rd escalations. Pull the
# rows from details.json so the alert can name the WOs.
if today.get("third_escalation_count", 0) > 0:
details = slackio.read_analytics_json(dt, "details.json") or []
details = slackio.read_meta_json(dt, "details.json") or []
thirds = [d for d in details if d.get("category") == "3rd Escalation"]
alert_blocks = blockkit.build_escalation_alert(thirds)
if alert_blocks:

View file

@ -60,7 +60,7 @@ def handler(event, context):
# The daily post is same-day; default the drill to today's snapshot.
dt = datetime.now(timezone.utc).strftime("%Y-%m-%d")
details = slackio.read_analytics_json(dt, "details.json") or []
details = slackio.read_meta_json(dt, "details.json") or []
wos = [d for d in details if d.get(field) == value]
view = blockkit.build_wo_modal(

View file

@ -60,9 +60,10 @@ def verify_signature(body: str, timestamp: str, signature: str) -> bool:
return verifier.is_valid(body=body, timestamp=timestamp, signature=signature)
def read_analytics_json(dt: str, name: str):
"""Read ``analytics/dt=<dt>/<name>`` as JSON, or None if absent."""
key = f"analytics/dt={dt}/{name}"
def read_meta_json(dt: str, name: str):
"""Read ``meta/dt=<dt>/<name>`` as JSON, or None if absent. (summary.json /
details.json live under meta/, kept out of the Athena table's analytics/ prefix.)"""
key = f"meta/dt={dt}/{name}"
try:
obj = _s3.get_object(Bucket=_BUCKET, Key=key)
except _s3.exceptions.NoSuchKey: