alarm notifications support different severity per recipient
Costa Tsaousis (ktsaou) committed
Sep 16, 2016 at 21:35 UTC
929212dc85b59a487c5ef04fbf050a90e6a3b353
2 files changed
+81
-9
conf.d/health_alarm_notify.conf
+23
@@ -13,6 +13,10 @@
13
#
14
# This file is a BASH script itself.
15
16
+
17
+###############################################################################
18
+# proxy configuration
19
+
20
# if you need to send curl based notifications (pushover, slack) via a proxy
21
# set these:
22
#export http_proxy="http://10.0.0.1:3128/"
@@ -33,6 +37,24 @@ sendmail=""
37
curl=""
38
39
40
+###############################################################################
41
+# RECIPIENT ATTRIBUTES
42
+
43
+# When you define recipients (all types):
44
+#
45
+# - emails addresses
46
+# - pushover user tokens
47
+# - slack channels
48
+#
49
+# You can append |critical to limit the notifications to be sent.
50
+#
51
+# In these examples, the first recipient receives all the alarms
52
+# while the second one receives only the critical ones:
53
+# email : "user1@example.com user2@example.com|critical"
54
+# pushover: "2987343...9437837 8756278...2362736|critical"
55
+# slack : "alarms disasters|critical"
56
+
57
+
58
###############################################################################
59
# sending emails
60
@@ -44,6 +66,7 @@ SEND_EMAIL="YES"
66
67
# if a role recipient is not configured, an email will be send to:
68
DEFAULT_RECIPIENT_EMAIL="root"
69
+# to recieve only critical alarms, set it to "root|critical"
70
71
72
###############################################################################
plugins.d/alarm-notify.sh
+58
-9
@@ -94,6 +94,46 @@ if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
94
source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
95
fi
96
97
+# -----------------------------------------------------------------------------
98
+# filter recipients based on the criticality of each
99
+
100
+filter_recipient_by_criticality() {
101
+ local method="${1}" x="${2}" r s
102
+ shift
103
+
104
+ r="${x/|*/}"
105
+ s="${x/*|/}"
106
+
107
+ # no severity filtering for this person
108
+ [ "${r}" = "${s}" ] && return 0
109
+
110
+ # the severity is invalid
111
+ s="${s^^}"
112
+ [ "${s}" != "CRITICAL" ] && return 0
113
+
114
+ # the new or the old status matches the severity
115
+ if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
116
+ then
117
+ [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
118
+ mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
119
+
120
+ # we need to keep track of the notifications we sent
121
+ # so that the same user will receive the recovery
122
+ # even if old_status does not match the required severity
123
+ touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
124
+ return 0
125
+ fi
126
+
127
+ # it is a cleared alarm we have sent notification for
128
+ if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
129
+ then
130
+ rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
131
+ return 0
132
+ fi
133
+
134
+ return 1
135
+}
136
+
137
# -----------------------------------------------------------------------------
138
# find the recipient's addresses per method
139
@@ -108,17 +148,26 @@ do
148
# email
149
a="${role_recipients_email[${recipient}]}"
150
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
111
- for r in ${a//,/ }; do arr_email[${r}]="1"; done
151
+ for r in ${a//,/ }
152
+ do
153
+ filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
154
+ done
155
156
# pushover
157
a="${role_recipients_pushover[${recipient}]}"
158
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
116
- for r in ${a//,/ }; do arr_pushover[${r}]="1"; done
159
+ for r in ${a//,/ }
160
+ do
161
+ filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
162
+ done
163
164
# slack
165
a="${role_recipients_slack[${recipient}]}"
166
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
121
- for r in ${a//,/ }; do arr_slack[${r}]="1"; done
167
+ for r in ${a//,/ }
168
+ do
169
+ filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
170
+ done
171
done
172
173
# build the list of slack recipients (channels)
@@ -136,7 +185,7 @@ do
185
[ ! -z "${to_email}" ] && to_email="${to_email}, "
186
to_email="${to_email}${x}"
187
done
139
-[ -z "${to_email}" ] && to_email="root"
188
+[ -z "${to_email}" ] && SEND_EMAIL="NO"
189
190
191
# -----------------------------------------------------------------------------
@@ -339,9 +388,9 @@ send_pushover() {
388
# slack sender
389
390
send_slack() {
342
- local webhook="${1}" channels="${2}" when="${3}" username="${4}" image="${5}" author="${6}" httpcode sent=0 channel color
391
+ local webhook="${1}" channels="${2}" when="${3}" username="${4}" image="${5}" httpcode sent=0 channel color
392
344
- if [ "${SEND_SLACK}" = "YES" -a ! -z "${webhook}" -a ! -z "${channels}" -a ! -z "${username}" -a ! -z "${image}" -a ! -z "${author}" ]
393
+ if [ "${SEND_SLACK}" = "YES" -a ! -z "${webhook}" -a ! -z "${channels}" -a ! -z "${username}" -a ! -z "${image}" ]
394
then
395
396
case "${status}" in
@@ -354,7 +403,7 @@ send_slack() {
403
for channel in ${channels}
404
do
405
httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode \
357
- "payload={\"channel\": \"#${channel}\", \"username\": \"${username}\", \"text\": \"${host} ${status_message} - ${author} ${raised_for} - click <${goto_url}|here> to view the netdata dashboard.\", \"icon_url\": \"${image}\", \"attachments\": [{\"fallback\": \"${alarm} - ${info}\", \"color\": \"${color}\", \"title\": \"${alarm}\", \"title_link\": \"${goto_url}\", \"text\": \"${info}\", \"footer\": \"netdata\", \"footer_icon\": \"${images_base_url}/images/seo-performance-128.png\", \"ts\": ${when}}]}" \
406
+ "payload={\"channel\": \"#${channel}\", \"username\": \"${username}\", \"text\": \"${host} ${status_message} - ${chart} (${family}) ${alarm} - click <${goto_url}|here> to view the netdata dashboard.\", \"icon_url\": \"${image}\", \"attachments\": [{\"fallback\": \"${alarm} - ${info}\", \"color\": \"${color}\", \"title\": \"${alarm}\", \"title_link\": \"${goto_url}\", \"text\": \"${chart} (${family}): ${info}\", \"footer\": \"netdata\", \"footer_icon\": \"${images_base_url}/images/seo-performance-128.png\", \"ts\": ${when}}]}" \
407
"${webhook}")
408
409
if [ "${httpcode}" == "200" ]
@@ -424,7 +473,7 @@ case "${status}" in
473
474
# don't show the value when the status is CLEAR
475
# for certain alarms, this value might not have any meaning
427
- alarm="${name//_/ }"
476
+ alarm="${name//_/ } ${raised_for}"
477
;;
478
esac
479
@@ -469,7 +518,7 @@ raised_for_html=
518
# slack aggregates posts from the same username
519
# so we use "${host} ${status}" as the bot username, to make them diff
520
472
-send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}" "${when}" "${host} ${status}" "${image}" "${chart} (${family})"
521
+send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}" "${when}" "${host} ${status}" "${image}"
522
SENT_SLACK=$?
523
524
# -----------------------------------------------------------------------------