mirror of
https://github.com/Sea-Haven-Industries/apm-wo-analysis.git
synced 2026-10-07 15:08:59 +00:00
Fix dashboard rendering: barchart panel type + metadata off the table prefix
Two issues found loading the deployed dashboard: 1. Panels used type "bar-chart" (hyphenated); Grafana's core panel is "barchart" — hence "plugin bar-chart required". Fixed both panels. 2. ALL panels showed "no data" because Athena failed with HIVE_BAD_DATA: the classifier wrote summary.json/details.json INTO analytics/dt=*/ — the same prefix the Glue table scans — so Athena tried to read the JSON as Parquet and every query failed. Move the metadata to a separate meta/dt=*/ prefix: classifier writes there (grant_read_write meta/*), the Slack Lambdas read there (read_meta_json, grant_read meta/*), and analytics/ holds only Parquet. Verified: the category GROUP BY query now succeeds against Athena.
This commit is contained in:
parent
ea54cb1e60
commit
68f3acded8
6 changed files with 21 additions and 13 deletions
|
|
@ -274,6 +274,9 @@ class PipelineStack(Stack):
|
||||||
# Parquet to S3 — it never touches the catalog (Phase 3).
|
# Parquet to S3 — it never touches the catalog (Phase 3).
|
||||||
self.exports_bucket.grant_read(self.classifier_fn, "raw/*")
|
self.exports_bucket.grant_read(self.classifier_fn, "raw/*")
|
||||||
self.exports_bucket.grant_read_write(self.classifier_fn, "analytics/*")
|
self.exports_bucket.grant_read_write(self.classifier_fn, "analytics/*")
|
||||||
|
# summary.json/details.json live under meta/ (kept out of the Athena
|
||||||
|
# table's analytics/ prefix so queries don't read JSON as Parquet).
|
||||||
|
self.exports_bucket.grant_read_write(self.classifier_fn, "meta/*")
|
||||||
secretsmanager.Secret.from_secret_name_v2(
|
secretsmanager.Secret.from_secret_name_v2(
|
||||||
self, "AnthropicKey", ANTHROPIC_SECRET
|
self, "AnthropicKey", ANTHROPIC_SECRET
|
||||||
).grant_read(self.classifier_fn)
|
).grant_read(self.classifier_fn)
|
||||||
|
|
@ -335,7 +338,8 @@ class PipelineStack(Stack):
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
self.exports_bucket.grant_read(fn, "analytics/*")
|
# Slack Lambdas read only the daily JSON under meta/ (not the Parquet).
|
||||||
|
self.exports_bucket.grant_read(fn, "meta/*")
|
||||||
slack_secret.grant_read(fn)
|
slack_secret.grant_read(fn)
|
||||||
dashboard_param.grant_read(fn)
|
dashboard_param.grant_read(fn)
|
||||||
return fn
|
return fn
|
||||||
|
|
|
||||||
|
|
@ -179,7 +179,7 @@
|
||||||
"panels": [
|
"panels": [
|
||||||
{
|
{
|
||||||
"id": 1,
|
"id": 1,
|
||||||
"type": "bar-chart",
|
"type": "barchart",
|
||||||
"title": "Category Distribution",
|
"title": "Category Distribution",
|
||||||
"description": "Count of work orders by classification category for the selected snapshot date. Sorted descending by volume. Filter using the template variables above.",
|
"description": "Count of work orders by classification category for the selected snapshot date. Sorted descending by volume. Filter using the template variables above.",
|
||||||
"gridPos": { "x": 0, "y": 0, "w": 14, "h": 9 },
|
"gridPos": { "x": 0, "y": 0, "w": 14, "h": 9 },
|
||||||
|
|
@ -366,7 +366,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": 4,
|
"id": 4,
|
||||||
"type": "bar-chart",
|
"type": "barchart",
|
||||||
"title": "Escalations by Site",
|
"title": "Escalations by Site",
|
||||||
"description": "Total escalations per site for the selected snapshot date. Includes all escalation categories.",
|
"description": "Total escalations per site for the selected snapshot date. Includes all escalation categories.",
|
||||||
"gridPos": { "x": 8, "y": 9, "w": 16, "h": 8 },
|
"gridPos": { "x": 8, "y": 9, "w": 16, "h": 8 },
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,8 @@ import pandas as pd
|
||||||
|
|
||||||
import classify as clf
|
import classify as clf
|
||||||
|
|
||||||
ANALYTICS_PREFIX = "analytics"
|
ANALYTICS_PREFIX = "analytics" # Parquet snapshots — the Glue/Athena table reads this
|
||||||
|
META_PREFIX = "meta" # summary.json/details.json — kept OUT of the table's prefix
|
||||||
|
|
||||||
# Map snapshot field -> substring matched (case-insensitively) against the export
|
# Map snapshot field -> substring matched (case-insensitively) against the export
|
||||||
# header, tolerating minor header drift in the 13-column APM export.
|
# header, tolerating minor header drift in the 13-column APM export.
|
||||||
|
|
@ -237,17 +238,19 @@ def handler(event, context):
|
||||||
mode="overwrite_partitions",
|
mode="overwrite_partitions",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# summary.json/details.json go under meta/ — NOT analytics/. Athena reads
|
||||||
|
# every object in the table's prefix as Parquet, so JSON there breaks queries.
|
||||||
summary = _build_summary(df, dt, key, blank)
|
summary = _build_summary(df, dt, key, blank)
|
||||||
_s3.put_object(
|
_s3.put_object(
|
||||||
Bucket=bucket,
|
Bucket=bucket,
|
||||||
Key=f"{ANALYTICS_PREFIX}/dt={dt}/summary.json",
|
Key=f"{META_PREFIX}/dt={dt}/summary.json",
|
||||||
Body=json.dumps(summary, indent=2).encode("utf-8"),
|
Body=json.dumps(summary, indent=2).encode("utf-8"),
|
||||||
ContentType="application/json",
|
ContentType="application/json",
|
||||||
)
|
)
|
||||||
# details.json — per-WO index the slack-post Lambda reads for drill-down modals.
|
# details.json — per-WO index the slack-post Lambda reads for drill-down modals.
|
||||||
_s3.put_object(
|
_s3.put_object(
|
||||||
Bucket=bucket,
|
Bucket=bucket,
|
||||||
Key=f"{ANALYTICS_PREFIX}/dt={dt}/details.json",
|
Key=f"{META_PREFIX}/dt={dt}/details.json",
|
||||||
Body=json.dumps(_build_details(df)).encode("utf-8"),
|
Body=json.dumps(_build_details(df)).encode("utf-8"),
|
||||||
ContentType="application/json",
|
ContentType="application/json",
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -25,11 +25,11 @@ def handler(event, context):
|
||||||
"""Post the daily summary and (conditionally) the 3rd-escalation alert."""
|
"""Post the daily summary and (conditionally) the 3rd-escalation alert."""
|
||||||
dt = (event or {}).get("dt") or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
dt = (event or {}).get("dt") or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
|
||||||
today = slackio.read_analytics_json(dt, "summary.json")
|
today = slackio.read_meta_json(dt, "summary.json")
|
||||||
if today is None:
|
if today is None:
|
||||||
print(f"No summary.json for dt={dt}; nothing to post.")
|
print(f"No summary.json for dt={dt}; nothing to post.")
|
||||||
return {"posted": False, "reason": "no summary", "dt": dt}
|
return {"posted": False, "reason": "no summary", "dt": dt}
|
||||||
yesterday = slackio.read_analytics_json(_yesterday(dt), "summary.json")
|
yesterday = slackio.read_meta_json(_yesterday(dt), "summary.json")
|
||||||
|
|
||||||
client = slackio.web_client()
|
client = slackio.web_client()
|
||||||
channel = slackio.channel_id()
|
channel = slackio.channel_id()
|
||||||
|
|
@ -44,7 +44,7 @@ def handler(event, context):
|
||||||
# Standalone batched alert — only when there are 3rd escalations. Pull the
|
# Standalone batched alert — only when there are 3rd escalations. Pull the
|
||||||
# rows from details.json so the alert can name the WOs.
|
# rows from details.json so the alert can name the WOs.
|
||||||
if today.get("third_escalation_count", 0) > 0:
|
if today.get("third_escalation_count", 0) > 0:
|
||||||
details = slackio.read_analytics_json(dt, "details.json") or []
|
details = slackio.read_meta_json(dt, "details.json") or []
|
||||||
thirds = [d for d in details if d.get("category") == "3rd Escalation"]
|
thirds = [d for d in details if d.get("category") == "3rd Escalation"]
|
||||||
alert_blocks = blockkit.build_escalation_alert(thirds)
|
alert_blocks = blockkit.build_escalation_alert(thirds)
|
||||||
if alert_blocks:
|
if alert_blocks:
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ def handler(event, context):
|
||||||
|
|
||||||
# The daily post is same-day; default the drill to today's snapshot.
|
# The daily post is same-day; default the drill to today's snapshot.
|
||||||
dt = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
dt = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
details = slackio.read_analytics_json(dt, "details.json") or []
|
details = slackio.read_meta_json(dt, "details.json") or []
|
||||||
wos = [d for d in details if d.get(field) == value]
|
wos = [d for d in details if d.get(field) == value]
|
||||||
|
|
||||||
view = blockkit.build_wo_modal(
|
view = blockkit.build_wo_modal(
|
||||||
|
|
|
||||||
|
|
@ -60,9 +60,10 @@ def verify_signature(body: str, timestamp: str, signature: str) -> bool:
|
||||||
return verifier.is_valid(body=body, timestamp=timestamp, signature=signature)
|
return verifier.is_valid(body=body, timestamp=timestamp, signature=signature)
|
||||||
|
|
||||||
|
|
||||||
def read_analytics_json(dt: str, name: str):
|
def read_meta_json(dt: str, name: str):
|
||||||
"""Read ``analytics/dt=<dt>/<name>`` as JSON, or None if absent."""
|
"""Read ``meta/dt=<dt>/<name>`` as JSON, or None if absent. (summary.json /
|
||||||
key = f"analytics/dt={dt}/{name}"
|
details.json live under meta/, kept out of the Athena table's analytics/ prefix.)"""
|
||||||
|
key = f"meta/dt={dt}/{name}"
|
||||||
try:
|
try:
|
||||||
obj = _s3.get_object(Bucket=_BUCKET, Key=key)
|
obj = _s3.get_object(Bucket=_BUCKET, Key=key)
|
||||||
except _s3.exceptions.NoSuchKey:
|
except _s3.exceptions.NoSuchKey:
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue