leesander1 · February 24, 2022 18:04 · Apr 10, 2019 · Nov 14, 2018 · Nov 14, 2018
diff --git a/monzo-alertmanager-config.yaml b/monzo-alertmanager-config.yaml
@@ -0,0 +1,38 @@
+###################################################
+##
+## Alertmanager YAML configuration for routing.
+##
+## Will route alerts with a code_owner label to the slack-code-owners receiver 
+## configured above, but will continue processing them to send to both a
+## central Slack channel (slack-monitoring) and PagerDuty receivers
+## (pd-warning and pd-critical)
+##
+
+routes:
+  ###################################################
+  ## Duplicate code_owner routes to teams
+  ## These will send alerts to team channels but continue
+  ## processing through the rest of the tree to handled by on-call
+  - match_re:
+      code_owner: '.+'
+    routes:
+      - match: {severity: info|warning|critical}
+        continue: true
+        receiver: slack-code-owners
+
+  ###################################################
+  ## Standard on-call routes
+  - match_re:
+      severity: info|warning|critical
+    receiver: slack-monitoring
+    continue: true
+
+
+  ###################################################
+  ## Pagerduty routes
+  - match: {severity: warning}
+    routes:
+    - receiver: pd-warning
+  - match: {severity: critical}
+    routes:
+    - receiver: pd-critical
diff --git a/monzo-alertmanager-receiver.yaml b/monzo-alertmanager-receiver.yaml
@@ -0,0 +1,27 @@
+receivers:
+###################################################
+## Slack Receivers
+- name: slack-code-owners
+  slack_configs:
+  - channel: '#{{- template "slack.monzo.code_owner_channel" . -}}'
+    send_resolved: true
+    title: '{{ template "slack.monzo.title" . }}'
+    icon_emoji: '{{ template "slack.monzo.icon_emoji" . }}'
+    color: '{{ template "slack.monzo.color" . }}'
+    text: '{{ template "slack.monzo.text" . }}'
+    actions:
+    - type: button
+      text: 'Runbook :green_book:'
+      url: '{{ (index .Alerts 0).Annotations.runbook }}'
+    - type: button
+      text: 'Query :mag:'
+      url: '{{ (index .Alerts 0).GeneratorURL }}'
+    - type: button
+      text: 'Dashboard :grafana:'
+      url: '{{ (index .Alerts 0).Annotations.dashboard }}'
+    - type: button
+      text: 'Silence :no_bell:'
+      url: '{{ template "__alert_silence_link" . }}'
+    - type: button
+      text: '{{ template "slack.monzo.link_button_text" . }}'
+      url: '{{ .CommonAnnotations.link_url }}'
diff --git a/monzo-slack-templates.tmpl b/monzo-slack-templates.tmpl
@@ -0,0 +1,114 @@
+# This builds the silence URL.  We exclude the alertname in the range
+# to avoid the issue of having trailing comma separator (%2C) at the end
+# of the generated URL
+{{ define "__alert_silence_link" -}}
+    {{ .ExternalURL }}/#/silences/new?filter=%7B
+    {{- range .CommonLabels.SortedPairs -}}
+        {{- if ne .Name "alertname" -}}
+            {{- .Name }}%3D"{{- .Value -}}"%2C%20
+        {{- end -}}
+    {{- end -}}
+    alertname%3D"{{ .CommonLabels.alertname }}"%7D
+{{- end }}
+
+
+
+{{ define "__alert_severity_prefix" -}}
+    {{ if ne .Status "firing" -}}
+    :lgtm:
+    {{- else if eq .Labels.severity "critical" -}}
+    :fire:
+    {{- else if eq .Labels.severity "warning" -}}
+    :warning:
+    {{- else -}}
+    :question:
+    {{- end }}
+{{- end }}
+
+{{ define "__alert_severity_prefix_title" -}}
+    {{ if ne .Status "firing" -}}
+    :lgtm:
+    {{- else if eq .CommonLabels.severity "critical" -}}
+    :fire:
+    {{- else if eq .CommonLabels.severity "warning" -}}
+    :warning:
+    {{- else if eq .CommonLabels.severity "info" -}}
+    :information_source:
+    {{- else -}}
+    :question:
+    {{- end }}
+{{- end }}
+
+
+{{/* First line of Slack alerts */}}
+{{ define "slack.monzo.title" -}}
+    [{{ .Status | toUpper -}}
+    {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}}
+    ] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }}
+{{- end }}
+
+
+{{/* Color of Slack attachment (appears as line next to alert )*/}}
+{{ define "slack.monzo.color" -}}
+    {{ if eq .Status "firing" -}}
+        {{ if eq .CommonLabels.severity "warning" -}}
+            warning
+        {{- else if eq .CommonLabels.severity "critical" -}}
+            danger
+        {{- else -}}
+            #439FE0
+        {{- end -}}
+    {{ else -}}
+    good
+    {{- end }}
+{{- end }}
+
+
+{{/* Emoji to display as user icon (custom emoji supported!) */}}
+{{ define "slack.monzo.icon_emoji" }}:prometheus:{{ end }}
+
+{{/* The test to display in the alert */}}
+{{ define "slack.monzo.text" -}}
+    {{ range .Alerts }}
+        {{- if .Annotations.message }}
+            {{ .Annotations.message }}
+        {{- end }}
+        {{- if .Annotations.description }}
+            {{ .Annotations.description }}
+        {{- end }}
+    {{- end }}
+{{- end }}
+
+
+
+{{- /* If none of the below matches, send to #monitoring-no-owner, and we 
+can then assign the expected code_owner to the alert or map the code_owner
+to the correct channel */ -}}
+{{ define "__get_channel_for_code_owner" -}}
+    {{- if eq . "platform-team" -}}
+        platform-alerts
+    {{- else if eq . "security-team" -}}
+        security-alerts
+    {{- else -}}
+        monitoring-no-owner
+    {{- end -}}
+{{- end }}
+
+{{- /* Select the channel based on the code_owner. We only expect to get
+into this template function if the code_owners label is present on an alert.
+This is to defend against us accidentally breaking the routing logic. */ -}}
+{{ define "slack.monzo.code_owner_channel" -}}
+    {{- if .CommonLabels.code_owner }}
+        {{ template "__get_channel_for_code_owner" .CommonLabels.code_owner }}
+    {{- else -}}
+        monitoring
+    {{- end }}
+{{- end }}
+
+{{ define "slack.monzo.link_button_text" -}}
+    {{- if .CommonAnnotations.link_text -}}
+        {{- .CommonAnnotations.link_text -}}
+    {{- else -}}
+        Link
+    {{- end }} :link:
+{{- end }}
No results found