mirror of
https://github.com/Sea-Haven-Industries/apm-wo-analysis.git
synced 2026-10-02 02:33:14 +00:00
Fix dashboard rendering: barchart panel type + metadata off the table prefix
Two issues found loading the deployed dashboard: 1. Panels used type "bar-chart" (hyphenated); Grafana's core panel is "barchart" — hence "plugin bar-chart required". Fixed both panels. 2. ALL panels showed "no data" because Athena failed with HIVE_BAD_DATA: the classifier wrote summary.json/details.json INTO analytics/dt=*/ — the same prefix the Glue table scans — so Athena tried to read the JSON as Parquet and every query failed. Move the metadata to a separate meta/dt=*/ prefix: classifier writes there (grant_read_write meta/*), the Slack Lambdas read there (read_meta_json, grant_read meta/*), and analytics/ holds only Parquet. Verified: the category GROUP BY query now succeeds against Athena.
This commit is contained in:
parent
ea54cb1e60
commit
68f3acded8
6 changed files with 21 additions and 13 deletions
|
|
@ -274,6 +274,9 @@ class PipelineStack(Stack):
|
|||
# Parquet to S3 — it never touches the catalog (Phase 3).
|
||||
self.exports_bucket.grant_read(self.classifier_fn, "raw/*")
|
||||
self.exports_bucket.grant_read_write(self.classifier_fn, "analytics/*")
|
||||
# summary.json/details.json live under meta/ (kept out of the Athena
|
||||
# table's analytics/ prefix so queries don't read JSON as Parquet).
|
||||
self.exports_bucket.grant_read_write(self.classifier_fn, "meta/*")
|
||||
secretsmanager.Secret.from_secret_name_v2(
|
||||
self, "AnthropicKey", ANTHROPIC_SECRET
|
||||
).grant_read(self.classifier_fn)
|
||||
|
|
@ -335,7 +338,8 @@ class PipelineStack(Stack):
|
|||
),
|
||||
),
|
||||
)
|
||||
self.exports_bucket.grant_read(fn, "analytics/*")
|
||||
# Slack Lambdas read only the daily JSON under meta/ (not the Parquet).
|
||||
self.exports_bucket.grant_read(fn, "meta/*")
|
||||
slack_secret.grant_read(fn)
|
||||
dashboard_param.grant_read(fn)
|
||||
return fn
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@
|
|||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "bar-chart",
|
||||
"type": "barchart",
|
||||
"title": "Category Distribution",
|
||||
"description": "Count of work orders by classification category for the selected snapshot date. Sorted descending by volume. Filter using the template variables above.",
|
||||
"gridPos": { "x": 0, "y": 0, "w": 14, "h": 9 },
|
||||
|
|
@ -366,7 +366,7 @@
|
|||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "bar-chart",
|
||||
"type": "barchart",
|
||||
"title": "Escalations by Site",
|
||||
"description": "Total escalations per site for the selected snapshot date. Includes all escalation categories.",
|
||||
"gridPos": { "x": 8, "y": 9, "w": 16, "h": 8 },
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@ import pandas as pd
|
|||
|
||||
import classify as clf
|
||||
|
||||
ANALYTICS_PREFIX = "analytics"
|
||||
ANALYTICS_PREFIX = "analytics" # Parquet snapshots — the Glue/Athena table reads this
|
||||
META_PREFIX = "meta" # summary.json/details.json — kept OUT of the table's prefix
|
||||
|
||||
# Map snapshot field -> substring matched (case-insensitively) against the export
|
||||
# header, tolerating minor header drift in the 13-column APM export.
|
||||
|
|
@ -237,17 +238,19 @@ def handler(event, context):
|
|||
mode="overwrite_partitions",
|
||||
)
|
||||
|
||||
# summary.json/details.json go under meta/ — NOT analytics/. Athena reads
|
||||
# every object in the table's prefix as Parquet, so JSON there breaks queries.
|
||||
summary = _build_summary(df, dt, key, blank)
|
||||
_s3.put_object(
|
||||
Bucket=bucket,
|
||||
Key=f"{ANALYTICS_PREFIX}/dt={dt}/summary.json",
|
||||
Key=f"{META_PREFIX}/dt={dt}/summary.json",
|
||||
Body=json.dumps(summary, indent=2).encode("utf-8"),
|
||||
ContentType="application/json",
|
||||
)
|
||||
# details.json — per-WO index the slack-post Lambda reads for drill-down modals.
|
||||
_s3.put_object(
|
||||
Bucket=bucket,
|
||||
Key=f"{ANALYTICS_PREFIX}/dt={dt}/details.json",
|
||||
Key=f"{META_PREFIX}/dt={dt}/details.json",
|
||||
Body=json.dumps(_build_details(df)).encode("utf-8"),
|
||||
ContentType="application/json",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ def handler(event, context):
|
|||
"""Post the daily summary and (conditionally) the 3rd-escalation alert."""
|
||||
dt = (event or {}).get("dt") or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
today = slackio.read_analytics_json(dt, "summary.json")
|
||||
today = slackio.read_meta_json(dt, "summary.json")
|
||||
if today is None:
|
||||
print(f"No summary.json for dt={dt}; nothing to post.")
|
||||
return {"posted": False, "reason": "no summary", "dt": dt}
|
||||
yesterday = slackio.read_analytics_json(_yesterday(dt), "summary.json")
|
||||
yesterday = slackio.read_meta_json(_yesterday(dt), "summary.json")
|
||||
|
||||
client = slackio.web_client()
|
||||
channel = slackio.channel_id()
|
||||
|
|
@ -44,7 +44,7 @@ def handler(event, context):
|
|||
# Standalone batched alert — only when there are 3rd escalations. Pull the
|
||||
# rows from details.json so the alert can name the WOs.
|
||||
if today.get("third_escalation_count", 0) > 0:
|
||||
details = slackio.read_analytics_json(dt, "details.json") or []
|
||||
details = slackio.read_meta_json(dt, "details.json") or []
|
||||
thirds = [d for d in details if d.get("category") == "3rd Escalation"]
|
||||
alert_blocks = blockkit.build_escalation_alert(thirds)
|
||||
if alert_blocks:
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ def handler(event, context):
|
|||
|
||||
# The daily post is same-day; default the drill to today's snapshot.
|
||||
dt = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
details = slackio.read_analytics_json(dt, "details.json") or []
|
||||
details = slackio.read_meta_json(dt, "details.json") or []
|
||||
wos = [d for d in details if d.get(field) == value]
|
||||
|
||||
view = blockkit.build_wo_modal(
|
||||
|
|
|
|||
|
|
@ -60,9 +60,10 @@ def verify_signature(body: str, timestamp: str, signature: str) -> bool:
|
|||
return verifier.is_valid(body=body, timestamp=timestamp, signature=signature)
|
||||
|
||||
|
||||
def read_analytics_json(dt: str, name: str):
|
||||
"""Read ``analytics/dt=<dt>/<name>`` as JSON, or None if absent."""
|
||||
key = f"analytics/dt={dt}/{name}"
|
||||
def read_meta_json(dt: str, name: str):
|
||||
"""Read ``meta/dt=<dt>/<name>`` as JSON, or None if absent. (summary.json /
|
||||
details.json live under meta/, kept out of the Athena table's analytics/ prefix.)"""
|
||||
key = f"meta/dt={dt}/{name}"
|
||||
try:
|
||||
obj = _s3.get_object(Bucket=_BUCKET, Key=key)
|
||||
except _s3.exceptions.NoSuchKey:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue