allow custom alarm notifications
Costa Tsaousis (ktsaou) committed
Jun 15, 2017 at 00:59 UTC
888bd4fbc215e9ca6560b3472657dbdbfeb609cc
2 files changed
+131
-9
conf.d/health_alarm_notify.conf
+72
@@ -303,6 +303,68 @@ SEND_PD="YES"
303
DEFAULT_RECIPIENT_PD=""
304
305
306
+#------------------------------------------------------------------------------
307
+# custom notifications
308
+#
309
+
310
+# enable/disable sending pagerduty notifications
311
+SEND_CUSTOM="YES"
312
+
313
+# if a role's recipients are not configured, use the following.
314
+# (empty = do not send a notification for unconfigured roles)
315
+DEFAULT_RECIPIENT_CUSTOM=""
316
+
317
+# The custom_sender() is a custom function to do whatever you need to do
318
+custom_sender() {
319
+ # variables you can use:
320
+ # ${host} the host generated this event
321
+ # ${url_host} same as ${host} but URL encoded
322
+ # ${unique_id} the unique id of this event
323
+ # ${alarm_id} the unique id of the alarm that generated this event
324
+ # ${event_id} the incremental id of the event, for this alarm id
325
+ # ${when} the timestamp this event occurred
326
+ # ${name} the name of the alarm, as given in netdata health.d entries
327
+ # ${url_name} same as ${name} but URL encoded
328
+ # ${chart} the name of the chart (type.id)
329
+ # ${url_chart} same as ${chart} but URL encoded
330
+ # ${family} the family of the chart
331
+ # ${url_family} same as ${family} but URL encoded
332
+ # ${status} the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
333
+ # ${old_status} the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
334
+ # ${value} the current value of the alarm
335
+ # ${old_value} the previous value of the alarm
336
+ # ${src} the line number and file the alarm has been configured
337
+ # ${duration} the duration in seconds of the previous alarm state
338
+ # ${duration_txt} same as ${duration} for humans
339
+ # ${non_clear_duration} the total duration in seconds this is/was non-clear
340
+ # ${non_clear_duration_txt} same as ${non_clear_duration} for humans
341
+ # ${units} the units of the value
342
+ # ${info} a short description of the alarm
343
+ # ${value_string} friendly value (with units)
344
+ # ${old_value_string} friendly old value (with units)
345
+ # ${image} the URL of an image to represent the status of the alarm
346
+ # ${color} a color in #AABBCC format for the alarm
347
+ # ${goto_url} the URL the user can click to see the netdata dashboard
348
+
349
+ # these are more human friendly:
350
+ # ${alarm} like "name = value units"
351
+ # ${status_message} like "needs attention", "recovered", "is critical"
352
+ # ${severity} like "Escalated to CRITICAL", "Recovered from WARNING"
353
+ # ${raised_for} like "(alarm was raised for 10 minutes)"
354
+
355
+ # example human readable SMS
356
+ local msg="${host} ${status_message}: ${alarm} ${raised_for}"
357
+
358
+ # limit it to 160 characters and encode it for use in a URL
359
+ urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}"
360
+
361
+ # a space separated list of the recipients to send alarms to
362
+ to="${1}"
363
+
364
+ info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured."
365
+}
366
+
367
+
368
###############################################################################
369
# RECIPIENTS PER ROLE
370
@@ -330,6 +392,8 @@ role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
392
393
role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"
394
395
+role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
396
+
397
# -----------------------------------------------------------------------------
398
# DNS related alarms
399
@@ -353,6 +417,8 @@ role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
417
418
role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
419
420
+role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
421
+
422
# -----------------------------------------------------------------------------
423
# database servers alarms
424
# mysql, redis, memcached, postgres, etc
@@ -377,6 +443,8 @@ role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
443
444
role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
445
446
+role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"
447
+
448
# -----------------------------------------------------------------------------
449
# web servers alarms
450
# apache, nginx, lighttpd, etc
@@ -401,6 +469,8 @@ role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
469
470
role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
471
472
+role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"
473
+
474
# -----------------------------------------------------------------------------
475
# proxy servers alarms
476
# squid, etc
@@ -424,3 +494,5 @@ role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"
494
role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
495
496
role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"
497
+
498
+role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
plugins.d/alarm-notify.sh
+59
-9
@@ -2,7 +2,7 @@
2
3
# netdata
4
# real-time performance and health monitoring, done right!
5
-# (C) 2016 Costa Tsaousis <costa@tsaousis.gr>
5
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
6
# GPL v3+
7
#
8
# Script to send alarm notifications for netdata
@@ -18,13 +18,14 @@
18
# - slack.com notifications by @ktsaou
19
# - discordapp.com notifications by @lowfive
20
# - pushover.net notifications by @ktsaou
21
-# - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
22
-# - telegram.org notifications by @hashworks PR #1002
23
-# - twilio.com notifications by Levi Blaney @shadycuz PR #1211
21
+# - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
22
+# - telegram.org notifications by @hashworks #1002
23
+# - twilio.com notifications by Levi Blaney @shadycuz #1211
24
# - kafka notifications by @ktsaou #1342
25
-# - pagerduty.com notifications by Jim Cooley @jimcooley PR #1373
25
+# - pagerduty.com notifications by Jim Cooley @jimcooley #1373
26
# - messagebird.com notifications by @tech_no_logical #1453
27
# - hipchat notifications by @ktsaou #1561
28
+# - custom notifications by @ktsaou
29
30
# -----------------------------------------------------------------------------
31
# testing notifications
@@ -103,6 +104,15 @@ debug() {
104
[ ${debug} -eq 1 ] && log DEBUG "${@}"
105
}
106
107
+
108
+# -----------------------------------------------------------------------------
109
+# this is to be overwritten by the config file
110
+
111
+custom_sender() {
112
+ info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
113
+}
114
+
115
+
116
# -----------------------------------------------------------------------------
117
118
# check for BASH v4+ (required for associative arrays)
@@ -128,8 +138,8 @@ when="${6}" # the timestamp this event occurred
138
name="${7}" # the name of the alarm, as given in netdata health.d entries
139
chart="${8}" # the name of the chart (type.id)
140
family="${9}" # the family of the chart
131
-status="${10}" # the current status : REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
132
-old_status="${11}" # the previous status: REMOVED, UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
141
+status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
142
+old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
143
value="${12}" # the current value of the alarm
144
old_value="${13}" # the previous value of the alarm
145
src="${14}" # the line number and file the alarm has been configured
@@ -189,6 +199,7 @@ SEND_EMAIL="YES"
199
SEND_PUSHBULLET="YES"
200
SEND_KAFKA="YES"
201
SEND_PD="YES"
202
+SEND_CUSTOM="YES"
203
204
# slack configs
205
SLACK_WEBHOOK_URL=
@@ -242,6 +253,10 @@ KAFKA_SENDER_IP=
253
PD_SERVICE_KEY=
254
declare -A role_recipients_pd=()
255
256
+# custom configs
257
+DEFAULT_RECIPIENT_CUSTOM=
258
+declare -A role_recipients_custom=()
259
+
260
# email configs
261
DEFAULT_RECIPIENT_EMAIL="root"
262
declare -A role_recipients_email=()
@@ -305,6 +320,7 @@ declare -A arr_hipchat=()
320
declare -A arr_telegram=()
321
declare -A arr_pd=()
322
declare -A arr_email=()
323
+declare -A arr_custom=()
324
325
# netdata may call us with multiple roles, and roles may have multiple but
326
# overlapping recipients - so, here we find the unique recipients.
@@ -393,6 +409,15 @@ do
409
do
410
[ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1"
411
done
412
+
413
+ # custom
414
+ a="${role_recipients_custom[${x}]}"
415
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}"
416
+ for r in ${a//,/ }
417
+ do
418
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality custom "${r}" && arr_custom[${r/|*/}]="1"
419
+ done
420
+
421
done
422
423
# build the list of slack recipients (channels)
@@ -431,6 +456,10 @@ to_telegram="${!arr_telegram[*]}"
456
to_pd="${!arr_pd[*]}"
457
[ -z "${to_pd}" ] && SEND_PD="NO"
458
459
+# build the list of custom recipients
460
+to_custom="${!arr_custom[*]}"
461
+[ -z "${to_custom}" ] && SEND_CUSTOM="NO"
462
+
463
# build the list of email recipients (email addresses)
464
to_email=
465
for x in "${!arr_email[@]}"
@@ -489,7 +518,7 @@ fi
518
if [ \( \
519
"${SEND_PUSHOVER}" = "YES" \
520
-o "${SEND_SLACK}" = "YES" \
492
- -o "${SEND_DISCORD}" = "YES" \
521
+ -o "${SEND_DISCORD}" = "YES" \
522
-o "${SEND_HIPCHAT}" = "YES" \
523
-o "${SEND_TWILIO}" = "YES" \
524
-o "${SEND_MESSAGEBIRD}" = "YES" \
@@ -527,13 +556,14 @@ if [ "${SEND_EMAIL}" != "YES" \
556
-a "${SEND_PUSHOVER}" != "YES" \
557
-a "${SEND_TELEGRAM}" != "YES" \
558
-a "${SEND_SLACK}" != "YES" \
530
- -a "${SEND_DISCORD}" != "YES" \
559
+ -a "${SEND_DISCORD}" != "YES" \
560
-a "${SEND_TWILIO}" != "YES" \
561
-a "${SEND_HIPCHAT}" != "YES" \
562
-a "${SEND_MESSAGEBIRD}" != "YES" \
563
-a "${SEND_PUSHBULLET}" != "YES" \
564
-a "${SEND_KAFKA}" != "YES" \
565
-a "${SEND_PD}" != "YES" \
566
+ -a "${SEND_CUSTOM}" != "YES" \
567
]
568
then
569
fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
@@ -1285,6 +1315,24 @@ send_pd "${to_pd}"
1315
SENT_PD=$?
1316
1317
1318
+# -----------------------------------------------------------------------------
1319
+# send the custom message
1320
+
1321
+send_custom() {
1322
+ # is it enabled?
1323
+ [ "${SEND_CUSTOM}" != "YES" ] && return 1
1324
+
1325
+ # do we have any sender?
1326
+ [ -z "${1}" ] && return 1
1327
+
1328
+ # call the custom_sender function
1329
+ custom_sender "${@}"
1330
+}
1331
+
1332
+send_custom "${to_custom}"
1333
+SENT_CUSTOM=$?
1334
+
1335
+
1336
# -----------------------------------------------------------------------------
1337
# send hipchat message
1338
@@ -1299,6 +1347,7 @@ ${host} ${status_message}<br/> \
1347
1348
SENT_HIPCHAT=$?
1349
1350
+
1351
# -----------------------------------------------------------------------------
1352
# send the email
1353
@@ -1430,6 +1479,7 @@ if [ ${SENT_EMAIL} -eq 0 \
1479
-o ${SENT_PUSHBULLET} -eq 0 \
1480
-o ${SENT_KAFKA} -eq 0 \
1481
-o ${SENT_PD} -eq 0 \
1482
+ -o ${SENT_CUSTOM} -eq 0 \
1483
]
1484
then
1485
# we did send something