added hipchat notifications; fixes #1526
Costa Tsaousis (ktsaou) committed
Jan 13, 2017 at 01:52 UTC
153aa5faac69706b361d7ffd4d431916a30247ee
2 files changed
+149
-12
conf.d/health_alarm_notify.conf
+38
-6
@@ -49,8 +49,7 @@ sendmail=""
49
50
# The full path of the curl command.
51
# If empty, the system $PATH will be searched for it.
52
-# If not found, pushover, telegram and slack notifications will be
53
-# silently disabled.
52
+# If not found, most notifications will be silently disabled.
53
curl=""
54
55
@@ -63,6 +62,7 @@ curl=""
62
# - pushover user tokens
63
# - telegram chat ids
64
# - slack channels
65
+# - hipchat rooms
66
# - sms phone numbers
67
# - pagerduty.com (pd) services
68
#
@@ -80,7 +80,7 @@ curl=""
80
# pd : "<pd_service_key_1> <pd_service_key_2>|critical"
81
#
82
# If a recipient is set to empty string, the default recipient of the given
83
-# notification method (email, pushover, telegram, slack, pd) will be used.
83
+# notification method (email, pushover, telegram, slack, etc) will be used.
84
# To disable a notification, use the recipient called: disabled
85
# This works for all notification methods (including the default recipients).
86
@@ -141,6 +141,7 @@ SEND_PUSHBULLET="YES"
141
PUSHBULLET_ACCESS_TOKEN=""
142
DEFAULT_RECIPIENT_PUSHBULLET=""
143
144
+
145
#------------------------------------------------------------------------------
146
# Twilio (twilio.com) SMS options
147
@@ -162,6 +163,7 @@ TWILIO_ACCOUNT_TOKEN=""
163
TWILIO_NUMBER=""
164
DEFAULT_RECIPIENT_TWILIO=""
165
166
+
167
#------------------------------------------------------------------------------
168
# Messagebird (messagebird.com) SMS options
169
@@ -182,6 +184,7 @@ MESSAGEBIRD_ACCESS_KEY=""
184
MESSAGEBIRD_NUMBER=""
185
DEFAULT_RECIPIENT_MESSAGEBIRD=""
186
187
+
188
#------------------------------------------------------------------------------
189
# telegram (telegram.org) global notification options
190
@@ -224,6 +227,25 @@ SLACK_WEBHOOK_URL=""
227
DEFAULT_RECIPIENT_SLACK=""
228
229
230
+#------------------------------------------------------------------------------
231
+# hipchat global notification options
232
+
233
+# multiple recipients can be given like this:
234
+# "ROOM1 ROOM2 ..."
235
+
236
+# enable/disable sending hipchat notifications
237
+SEND_HIPCHAT="YES"
238
+
239
+# api.hipchat.com authorization token
240
+# Without this, netdata cannot send hipchat notifications.
241
+HIPCHAT_AUTH_TOKEN=""
242
+
243
+# if a role's recipients are not configured, a notification will be send to
244
+# this hipchat room (empty = do not send a notification for unconfigured
245
+# roles):
246
+DEFAULT_RECIPIENT_HIPCHAT=""
247
+
248
+
249
#------------------------------------------------------------------------------
250
# kafka notification options
251
@@ -273,6 +295,8 @@ role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
295
296
role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
297
298
+role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
299
+
300
role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"
301
302
role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -292,6 +316,8 @@ role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
316
317
role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"
318
319
+role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
320
+
321
role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"
322
323
role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -300,7 +326,7 @@ role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
326
327
# -----------------------------------------------------------------------------
328
# database servers alarms
303
-# mysql, redis, memcached, etc
329
+# mysql, redis, memcached, postgres, etc
330
331
role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"
332
@@ -312,6 +338,8 @@ role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"
338
339
role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"
340
341
+role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}"
342
+
343
role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"
344
345
role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -320,7 +348,7 @@ role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
348
349
# -----------------------------------------------------------------------------
350
# web servers alarms
323
-# apache, nginx, etc
351
+# apache, nginx, lighttpd, etc
352
353
role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"
354
@@ -332,6 +360,8 @@ role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"
360
361
role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"
362
363
+role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}"
364
+
365
role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"
366
367
role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -340,7 +370,7 @@ role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
370
371
# -----------------------------------------------------------------------------
372
# proxy servers alarms
343
-# apache, nginx, etc
373
+# squid, etc
374
375
role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"
376
@@ -352,6 +382,8 @@ role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
382
383
role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"
384
385
+role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
386
+
387
role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"
388
389
role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
plugins.d/alarm-notify.sh
+111
-6
@@ -14,15 +14,16 @@
14
# - severity filtering per recipient
15
#
16
# Supported notification methods:
17
-# - emails
18
-# - slack.com notifications
19
-# - pushover.net notifications
17
+# - emails by @ktsaou
18
+# - slack.com notifications by @ktsaou
19
+# - pushover.net notifications by @ktsaou
20
# - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
21
# - telegram.org notifications by @hashworks PR #1002
22
# - twilio.com notifications by Levi Blaney @shadycuz PR #1211
23
-# - kafka notifications
23
+# - kafka notifications by @ktsaou #1342
24
# - pagerduty.com notifications by Jim Cooley @jimcooley PR #1373
25
-# - messagebird.com notifications by @tech_no_logical
25
+# - messagebird.com notifications by @tech_no_logical #1453
26
+# - hipchart notifications by @ktsaou
27
28
# -----------------------------------------------------------------------------
29
# testing notifications
@@ -173,6 +174,7 @@ sendmail=
174
SEND_SLACK="YES"
175
SEND_PUSHOVER="YES"
176
SEND_TWILIO="YES"
177
+SEND_HIPCHAT="YES"
178
SEND_MESSAGEBIRD="YES"
179
SEND_TELEGRAM="YES"
180
SEND_EMAIL="YES"
@@ -202,6 +204,11 @@ TWILIO_NUMBER=
204
DEFAULT_RECIPIENT_TWILIO=
205
declare -A role_recipients_twilio=()
206
207
+# hipchat configs
208
+HIPCHAT_AUTH_TOKEN=
209
+DEFAULT_RECIPIENT_HIPCHAT=
210
+declare -A role_recipients_hipchat=()
211
+
212
# messagebird configs
213
MESSAGEBIRD_ACCESS_KEY=
214
MESSAGEBIRD_NUMBER=
@@ -279,6 +286,7 @@ declare -A arr_slack=()
286
declare -A arr_pushover=()
287
declare -A arr_pushbullet=()
288
declare -A arr_twilio=()
289
+declare -A arr_hipchat=()
290
declare -A arr_telegram=()
291
declare -A arr_pd=()
292
declare -A arr_email=()
@@ -323,6 +331,14 @@ do
331
[ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1"
332
done
333
334
+ # hipchat
335
+ a="${role_recipients_hipchat[${x}]}"
336
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}"
337
+ for r in ${a//,/ }
338
+ do
339
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1"
340
+ done
341
+
342
# messagebird
343
a="${role_recipients_messagebird[${x}]}"
344
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
@@ -372,6 +388,10 @@ to_pushbullet="${!arr_pushbullet[*]}"
388
to_twilio="${!arr_twilio[*]}"
389
[ -z "${to_twilio}" ] && SEND_TWILIO="NO"
390
391
+# build the list of hipchat recipients (rooms)
392
+to_hipchat="${!arr_hipchat[*]}"
393
+[ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO"
394
+
395
# build the list of messagebird recipients (phone numbers)
396
to_messagebird="${!arr_messagebird[*]}"
397
[ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO"
@@ -409,6 +429,9 @@ done
429
# check twilio
430
[ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
431
432
+# check hipchat
433
+[ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
434
+
435
# check messagebird
436
[ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO"
437
@@ -433,7 +456,16 @@ if [ "${SEND_PD}" = "YES" ]
456
fi
457
458
# if we need curl, check for the curl command
436
-if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}" = "YES" -o "${SEND_MESSAGEBIRD}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" -o "${SEND_KAFKA}" = "YES" \) -a -z "${curl}" ]
459
+if [ \( \
460
+ "${SEND_PUSHOVER}" = "YES" \
461
+ -o "${SEND_SLACK}" = "YES" \
462
+ -o "${SEND_HIPCHAT}" = "YES" \
463
+ -o "${SEND_TWILIO}" = "YES" \
464
+ -o "${SEND_MESSAGEBIRD}" = "YES" \
465
+ -o "${SEND_TELEGRAM}" = "YES" \
466
+ -o "${SEND_PUSHBULLET}" = "YES" \
467
+ -o "${SEND_KAFKA}" = "YES" \
468
+ \) -a -z "${curl}" ]
469
then
470
curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
471
if [ -z "${curl}" ]
@@ -445,6 +477,7 @@ if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}
477
SEND_TELEGRAM="NO"
478
SEND_SLACK="NO"
479
SEND_TWILIO="NO"
480
+ SEND_HIPCHAT="NO"
481
SEND_MESSAGEBIRD="NO"
482
SEND_KAFKA="NO"
483
fi
@@ -463,6 +496,7 @@ if [ "${SEND_EMAIL}" != "YES" \
496
-a "${SEND_TELEGRAM}" != "YES" \
497
-a "${SEND_SLACK}" != "YES" \
498
-a "${SEND_TWILIO}" != "YES" \
499
+ -a "${SEND_HIPCHAT}" != "YES" \
500
-a "${SEND_MESSAGEBIRD}" != "YES" \
501
-a "${SEND_PUSHBULLET}" != "YES" \
502
-a "${SEND_KAFKA}" != "YES" \
@@ -785,6 +819,61 @@ send_twilio() {
819
return 1
820
}
821
822
+
823
+# -----------------------------------------------------------------------------
824
+# hipchat sender
825
+
826
+send_hipchat() {
827
+ local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify
828
+
829
+ if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
830
+ then
831
+
832
+ # A label to be shown in addition to the sender's name
833
+ # Valid length range: 0 - 64.
834
+ sender="netdata"
835
+
836
+ # Valid values: html, text.
837
+ # Defaults to 'html'.
838
+ msg_format="text"
839
+
840
+ # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
841
+ case "${status}" in
842
+ WARNING) color="yellow" ;;
843
+ CRITICAL) color="red" ;;
844
+ CLEAR) color="green" ;;
845
+ *) color="gray" ;;
846
+ esac
847
+
848
+ # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
849
+ # Each recipient's notification preferences are taken into account.
850
+ # Defaults to false.
851
+ notify="true"
852
+
853
+ for room in ${recipients}
854
+ do
855
+ httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
856
+ -H "Content-type: application/json" \
857
+ -H "Authorization: Bearer ${authtoken}" \
858
+ -d "{\"color\": \"${color}\", \"from\": \"${netdata}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
859
+ "https://api.hipchat.com/v2/room/${room}/notification")
860
+
861
+ if [ "${httpcode}" == "200" ]
862
+ then
863
+ info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
864
+ sent=$((sent + 1))
865
+ else
866
+ error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
867
+ fi
868
+ done
869
+
870
+ [ ${sent} -gt 0 ] && return 0
871
+ fi
872
+
873
+ return 1
874
+}
875
+
876
+
877
# -----------------------------------------------------------------------------
878
# messagebird sender
879
@@ -1095,6 +1184,21 @@ send_pd "${to_pd}"
1184
SENT_PD=$?
1185
1186
1187
+# -----------------------------------------------------------------------------
1188
+# send hipchat message
1189
+
1190
+send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" "
1191
+<b>${alarm}</b> ${info_html}<br/>
1192
+<small><b>${chart}</b><br/>Chart<br/> </small>
1193
+<small><b>${family}</b><br/>Family<br/> </small>
1194
+<small><b>${severity}</b><br/>Severity<br/> </small>
1195
+<small><b>${date}${raised_for_html}</b><br/>Time<br/> </small>
1196
+<a href=\"${goto_url}\">View Netdata</a><br/>
1197
+<small><small>The source of this alarm is line ${src}</small></small>
1198
+"
1199
+
1200
+SENT_HIPCHAT=$?
1201
+
1202
# -----------------------------------------------------------------------------
1203
# send the email
1204
@@ -1198,6 +1302,7 @@ if [ ${SENT_EMAIL} -eq 0 \
1302
-o ${SENT_TELEGRAM} -eq 0 \
1303
-o ${SENT_SLACK} -eq 0 \
1304
-o ${SENT_TWILIO} -eq 0 \
1305
+ -o ${SENT_HIPCHAT} -eq 0 \
1306
-o ${SENT_MESSAGEBIRD} -eq 0 \
1307
-o ${SENT_PUSHBULLET} -eq 0 \
1308
-o ${SENT_KAFKA} -eq 0 \