front-integrations/src/sla_monitor/app.py

422 lines
14 KiB
Python
Raw Normal View History

import json
import logging
import os
import time
from datetime import datetime, timedelta, timezone
from urllib import error, parse, request
from zoneinfo import ZoneInfo
import boto3
logger = logging.getLogger()
logger.setLevel(logging.INFO)
EASTERN = ZoneInfo("America/New_York")
FRONT_BASE = "https://api2.frontapp.com"
SLACK_BASE = "https://slack.com/api"
RATE_LIMIT_DELAY = 0.6
BH_START = 8
BH_END = 17
_sm = boto3.client("secretsmanager")
_ddb = boto3.resource("dynamodb")
_table = None
_front_token = None
_slack_token = None
_slack_user_cache = {}
def _get_table():
global _table
if _table is None:
_table = _ddb.Table(os.environ["TABLE_NAME"])
return _table
def _get_front_token():
global _front_token
if _front_token is None:
resp = _sm.get_secret_value(SecretId=os.environ["FRONT_SECRET_NAME"])
_front_token = resp["SecretString"]
return _front_token
def _get_slack_token():
global _slack_token
if _slack_token is None:
resp = _sm.get_secret_value(SecretId=os.environ["SLACK_SECRET_NAME"])
_slack_token = resp["SecretString"]
return _slack_token
# ---------------------------------------------------------------------------
# Front API
# ---------------------------------------------------------------------------
def _front_get(url_or_path, params=None):
if url_or_path.startswith("http"):
url = url_or_path
else:
url = f"{FRONT_BASE}{url_or_path}"
if params:
url += "?" + parse.urlencode(params, doseq=True)
req = request.Request(url, headers={
"Authorization": f"Bearer {_get_front_token()}",
"Accept": "application/json",
})
time.sleep(RATE_LIMIT_DELAY)
try:
with request.urlopen(req) as resp:
return json.loads(resp.read().decode())
except error.HTTPError as e:
body = e.read().decode() if e.fp else ""
raise RuntimeError(f"Front GET {url_or_path} failed ({e.code}): {body}")
MAX_PAGES = 10
def _front_paginate(path, params=None):
results = []
data = _front_get(path, params)
results.extend(data.get("_results", []))
pages = 1
while data.get("_pagination", {}).get("next") and pages < MAX_PAGES:
data = _front_get(data["_pagination"]["next"])
results.extend(data.get("_results", []))
pages += 1
return results
# ---------------------------------------------------------------------------
# Slack API
# ---------------------------------------------------------------------------
def _slack_post(method, payload):
data = json.dumps(payload).encode()
req = request.Request(f"{SLACK_BASE}/{method}", data=data, headers={
"Content-Type": "application/json; charset=utf-8",
"Authorization": f"Bearer {_get_slack_token()}",
})
try:
with request.urlopen(req) as resp:
result = json.loads(resp.read().decode())
if not result.get("ok"):
raise RuntimeError(f"Slack {method}: {result.get('error')}")
return result
except error.HTTPError as e:
body = e.read().decode() if e.fp else ""
raise RuntimeError(f"Slack {method} HTTP {e.code}: {body}")
def _slack_get(method, params):
url = f"{SLACK_BASE}/{method}?" + parse.urlencode(params)
req = request.Request(url, headers={
"Authorization": f"Bearer {_get_slack_token()}",
})
try:
with request.urlopen(req) as resp:
result = json.loads(resp.read().decode())
if not result.get("ok"):
raise RuntimeError(f"Slack {method}: {result.get('error')}")
return result
except error.HTTPError as e:
body = e.read().decode() if e.fp else ""
raise RuntimeError(f"Slack {method} HTTP {e.code}: {body}")
def _resolve_slack_user(email):
if email in _slack_user_cache:
return _slack_user_cache[email]
try:
result = _slack_get("users.lookupByEmail", {"email": email})
uid = result["user"]["id"]
_slack_user_cache[email] = uid
return uid
except RuntimeError:
logger.warning("Could not resolve Slack user for %s", email)
return None
def _send_slack(channel, blocks, text):
_slack_post("chat.postMessage", {
"channel": channel,
"blocks": blocks,
"text": text,
})
# ---------------------------------------------------------------------------
# Business time calculation (weekdays only, Eastern time)
# ---------------------------------------------------------------------------
def _business_minutes_elapsed(since_utc, now_utc):
since = since_utc.astimezone(EASTERN)
now = now_utc.astimezone(EASTERN)
if since >= now:
return 0
total = 0.0
current = since
while current < now:
if current.weekday() < 5:
end_of_day = (current + timedelta(days=1)).replace(
hour=0, minute=0, second=0, microsecond=0
)
day_end = min(end_of_day, now)
total += (day_end - current).total_seconds() / 60
current = (current + timedelta(days=1)).replace(
hour=0, minute=0, second=0, microsecond=0
)
return total
# ---------------------------------------------------------------------------
# DynamoDB dedup
# ---------------------------------------------------------------------------
def _already_alerted(conv_id, tier):
resp = _get_table().get_item(Key={"conversationId": conv_id})
item = resp.get("Item")
if not item:
return False
return f"tier{tier}AlertedAt" in item
def _record_alert(conv_id, tier):
now_iso = datetime.now(timezone.utc).isoformat()
ttl_epoch = int(time.time()) + 7 * 86400
_get_table().update_item(
Key={"conversationId": conv_id},
UpdateExpression="SET #ttl = :ttl, #alert = :ts",
ExpressionAttributeNames={
"#ttl": "ttl",
"#alert": f"tier{tier}AlertedAt",
},
ExpressionAttributeValues={
":ttl": ttl_epoch,
":ts": now_iso,
},
)
def _is_first_run_today(today_str):
resp = _get_table().get_item(Key={"conversationId": "_monitor_state"})
return resp.get("Item", {}).get("lastRunDate", "") != today_str
def _record_run(today_str):
_get_table().update_item(
Key={"conversationId": "_monitor_state"},
UpdateExpression="SET lastRunDate = :d",
ExpressionAttributeValues={":d": today_str},
)
# ---------------------------------------------------------------------------
# Slack message blocks
# ---------------------------------------------------------------------------
def _tier1_blocks(conv, assignee_email=None):
subject = conv.get("subject", "No subject")
conv_id = conv.get("id", "")
link = f"https://app.frontapp.com/open/{conv_id}"
msg_at = conv.get("last_message", {}).get("created_at", 0)
ts = datetime.fromtimestamp(msg_at, tz=EASTERN).strftime("%b %d, %I:%M %p ET")
status = f"Assigned to {assignee_email}" if assignee_email else "Unassigned"
return [
{"type": "header", "text": {"type": "plain_text", "text": ":warning: SLA Breach: 1-Hour Acknowledgment"}},
{"type": "section", "text": {"type": "mrkdwn", "text": (
f"*<{link}|{subject}>*\n"
f"Last inbound: {ts}\n"
f"Status: {status}\n"
f"_No reply for over 1 business hour._"
)}},
]
def _tier2_blocks(conv):
subject = conv.get("subject", "No subject")
conv_id = conv.get("id", "")
link = f"https://app.frontapp.com/open/{conv_id}"
msg_at = conv.get("last_message", {}).get("created_at", 0)
ts = datetime.fromtimestamp(msg_at, tz=EASTERN).strftime("%b %d, %I:%M %p ET")
assignee = conv.get("assignee")
status = f"Assigned to {assignee['email']}" if assignee else "Unassigned"
return [
{"type": "header", "text": {"type": "plain_text", "text": ":rotating_light: SLA Breach: 1-Day Action Required"}},
{"type": "section", "text": {"type": "mrkdwn", "text": (
f"*<{link}|{subject}>*\n"
f"Last inbound: {ts}\n"
f"Status: {status}\n"
f"_No reply for over 1 business day. Immediate attention required._"
)}},
]
def _summary_blocks(tier1_breaches, tier2_breaches):
lines = []
if tier2_breaches:
lines.append("*:rotating_light: Over 1 Business Day Without Reply*")
for conv in tier2_breaches:
subject = conv.get("subject", "No subject")
link = f"https://app.frontapp.com/open/{conv.get('id', '')}"
assignee = conv.get("assignee")
who = assignee["email"] if assignee else "Unassigned"
lines.append(f"• <{link}|{subject}> — {who}")
lines.append("")
if tier1_breaches:
lines.append("*:warning: Over 1 Business Hour Without Reply*")
for conv, email in tier1_breaches:
subject = conv.get("subject", "No subject")
link = f"https://app.frontapp.com/open/{conv.get('id', '')}"
who = email or "Unassigned"
lines.append(f"• <{link}|{subject}> — {who}")
total = len(tier1_breaches) + len(tier2_breaches)
return [
{"type": "header", "text": {"type": "plain_text", "text": f":sunrise: Morning SLA Summary — {total} breach{'es' if total != 1 else ''}"}},
{"type": "section", "text": {"type": "mrkdwn", "text": "\n".join(lines)}},
]
# ---------------------------------------------------------------------------
# Handler
# ---------------------------------------------------------------------------
def handler(event, context):
ack_threshold = int(os.environ["ACK_SLA_MINUTES"])
action_threshold = int(os.environ["ACTION_SLA_MINUTES"])
alert_channel = os.environ["SLACK_ALERT_CHANNEL"]
adam_email = os.environ["ADAM_EMAIL"]
start_date = os.environ.get("START_DATE", "")
now = datetime.now(timezone.utc)
now_et = now.astimezone(EASTERN)
if start_date:
start = datetime.strptime(start_date, "%Y-%m-%d").date()
if now_et.date() < start:
logger.info("Before start date %s, skipping", start_date)
return {"skipped": True, "reason": "before_start_date"}
if now_et.hour < BH_START or now_et.hour >= BH_END:
logger.info("Outside business hours (%s ET), skipping", now_et.strftime("%H:%M"))
return {"skipped": True, "reason": "outside_business_hours"}
today_str = now_et.strftime("%Y-%m-%d")
first_run = _is_first_run_today(today_str)
_record_run(today_str)
monitor_names = os.environ.get("MONITOR_INBOXES", "").strip()
if monitor_names:
monitor_set = {n.strip().lower() for n in monitor_names.split(",")}
else:
monitor_set = None
inboxes = _front_paginate("/inboxes")
shared = [i for i in inboxes if not i.get("is_private", False)]
if monitor_set:
shared = [i for i in shared if i.get("name", "").lower() in monitor_set]
logger.info("Monitoring %d inboxes: %s", len(shared),
", ".join(i.get("name", "?") for i in shared))
tier1_breaches = []
tier2_breaches = []
seen = set()
for inbox in shared:
inbox_id = inbox["id"]
inbox_name = inbox.get("name", inbox_id)
try:
conversations = _front_paginate(
f"/inboxes/{inbox_id}/conversations",
params={"q[statuses][]": ["open", "unassigned"]},
)
except RuntimeError as e:
logger.error("Failed to fetch conversations for %s: %s", inbox_name, e)
continue
logger.info("Inbox '%s': %d recent open conversations", inbox_name, len(conversations))
for conv in conversations:
if conv.get("id") in seen:
continue
seen.add(conv.get("id"))
try:
last_msg = conv.get("last_message")
if not last_msg or last_msg.get("type") != "inbound":
continue
msg_dt = datetime.fromtimestamp(last_msg["created_at"], tz=timezone.utc)
elapsed = _business_minutes_elapsed(msg_dt, now)
conv_id = conv["id"]
assignee = conv.get("assignee")
if elapsed >= action_threshold and not _already_alerted(conv_id, 2):
tier2_breaches.append(conv)
logger.info("Tier 2 breach: %s", conv_id)
elif elapsed >= ack_threshold and not _already_alerted(conv_id, 1):
email = assignee["email"] if assignee and assignee.get("email") else None
tier1_breaches.append((conv, email))
logger.info("Tier 1 breach: %s", conv_id)
except Exception:
logger.exception("Error processing conversation %s", conv.get("id", "?"))
total = len(tier1_breaches) + len(tier2_breaches)
if first_run and total > 1:
logger.info("Morning summary: %d breaches accumulated overnight", total)
blocks = _summary_blocks(tier1_breaches, tier2_breaches)
_send_slack(alert_channel, blocks, f"Morning SLA Summary: {total} breaches")
if tier2_breaches:
adam_uid = _resolve_slack_user(adam_email)
if adam_uid:
_send_slack(adam_uid, blocks, f"Morning SLA Summary: {total} breaches")
else:
for conv in tier2_breaches:
adam_uid = _resolve_slack_user(adam_email)
target = adam_uid or alert_channel
_send_slack(target, _tier2_blocks(conv),
f"SLA Breach: {conv.get('subject', '')} - 1 day without reply")
for conv, email in tier1_breaches:
if email:
uid = _resolve_slack_user(email)
target = uid or alert_channel
_send_slack(target, _tier1_blocks(conv, email),
f"SLA Breach: {conv.get('subject', '')} - 1 hour without reply")
else:
_send_slack(alert_channel, _tier1_blocks(conv),
f"SLA Breach: {conv.get('subject', '')} - unassigned, 1 hour without reply")
for conv in tier2_breaches:
_record_alert(conv["id"], 2)
for conv, _ in tier1_breaches:
_record_alert(conv["id"], 1)
result = {"tier1_alerts": len(tier1_breaches), "tier2_alerts": len(tier2_breaches)}
logger.info("Run complete: %s", result)
return result