added kafka notifications; https://github.com/firehol/netdata/issues/174#issuecomment-266003431
Costa Tsaousis (ktsaou) committed
Dec 10, 2016 at 02:07 UTC
b51dca9dabcbe536d1e4fba6d6f7ad3cb6b7afad
2 files changed
+97
-20
conf.d/health_alarm_notify.conf
+23
-10
@@ -16,7 +16,7 @@
16
# This file is a BASH script itself.
17
#
18
#
19
-###############################################################################
19
+#------------------------------------------------------------------------------
20
# proxy configuration
21
#
22
# If you need to send curl based notifications (pushover, pushbullet, slack,
@@ -25,7 +25,7 @@
25
#export https_proxy="http://10.0.0.1:3128/"
26
27
28
-###############################################################################
28
+#------------------------------------------------------------------------------
29
# notifications images
30
#
31
# Images in notifications need to be downloaded from an Internet facing site.
@@ -37,7 +37,7 @@
37
#images_base_url="http://my.public.netdata.server:19999"
38
39
40
-###############################################################################
40
+#------------------------------------------------------------------------------
41
# external commands
42
43
# The full path to the sendmail command.
@@ -52,7 +52,7 @@ sendmail=""
52
curl=""
53
54
55
-###############################################################################
55
+#------------------------------------------------------------------------------
56
# NOTE ABOUT RECIPIENTS
57
#
58
# When you define recipients (all types):
@@ -80,7 +80,7 @@ curl=""
80
# This works for all notification methods (including the default recipients).
81
82
83
-###############################################################################
83
+#------------------------------------------------------------------------------
84
# email global notification options
85
86
# multiple recipients can be given like this:
@@ -94,7 +94,7 @@ DEFAULT_RECIPIENT_EMAIL="root"
94
# to receive only critical alarms, set it to "root|critical"
95
96
97
-###############################################################################
97
+#------------------------------------------------------------------------------
98
# pushover (pushover.net) global notification options
99
100
# multiple recipients can be given like this:
@@ -115,7 +115,7 @@ PUSHOVER_APP_TOKEN=""
115
DEFAULT_RECIPIENT_PUSHOVER=""
116
117
118
-###############################################################################
118
+#------------------------------------------------------------------------------
119
# pushbullet (pushbullet.com) push notification options
120
121
# multiple recipients can be given like this:
@@ -136,7 +136,7 @@ SEND_PUSHBULLET="YES"
136
PUSHBULLET_ACCESS_TOKEN=""
137
DEFAULT_RECIPIENT_PUSHBULLET=""
138
139
-###############################################################################
139
+#------------------------------------------------------------------------------
140
# Twilio (twilio.com) SMS options
141
142
# multiple recipients can be given like this:
@@ -157,7 +157,7 @@ TWILIO_ACCOUNT_TOKEN=""
157
TWILIO_NUMBER=""
158
DEFAULT_RECIPIENT_TWILIO=""
159
160
-###############################################################################
160
+#------------------------------------------------------------------------------
161
# telegram (telegram.org) global notification options
162
163
# To get your chat ID send the command /my_id to telegram bot @get_id.
@@ -178,7 +178,7 @@ TELEGRAM_BOT_TOKEN=""
178
DEFAULT_RECIPIENT_TELEGRAM=""
179
180
181
-###############################################################################
181
+#------------------------------------------------------------------------------
182
# slack (slack.com) global notification options
183
184
# multiple recipients can be given like this:
@@ -199,6 +199,19 @@ SLACK_WEBHOOK_URL=""
199
DEFAULT_RECIPIENT_SLACK=""
200
201
202
+#------------------------------------------------------------------------------
203
+# kafka notification options
204
+
205
+# enable/disable sending kafka notifications
206
+SEND_KAFKA="YES"
207
+
208
+# The URL to POST kafka alarm data to. It should be the full URL.
209
+KAFKA_URL=""
210
+
211
+# The IP to be used in the kafka message as the sender.
212
+KAFKA_SENDER_IP=""
213
+
214
+
215
###############################################################################
216
# RECIPIENTS PER ROLE
217
plugins.d/alarm-notify.sh
+74
-10
@@ -19,7 +19,8 @@
19
# - pushover.net notifications
20
# - pushbullet.com push notifications by Tiago Peralta @tperalta82 PR #1070
21
# - telegram.org notifications by @hashworks PR #1002
22
-#
22
+# - twilio.com notifications by Levi Blaney @shadycuz PR #1211
23
+# - kafka notifications
24
25
# -----------------------------------------------------------------------------
26
# testing notifications
@@ -173,6 +174,7 @@ SEND_TWILIO="YES"
174
SEND_TELEGRAM="YES"
175
SEND_EMAIL="YES"
176
SEND_PUSHBULLET="YES"
177
+SEND_KAFKA="YES"
178
179
# slack configs
180
SLACK_WEBHOOK_URL=
@@ -201,6 +203,10 @@ TELEGRAM_BOT_TOKEN=
203
DEFAULT_RECIPIENT_TELEGRAM=
204
declare -A role_recipients_telegram=()
205
206
+# kafka configs
207
+KAFKA_URL=
208
+KAFKA_SENDER_IP=
209
+
210
# email configs
211
DEFAULT_RECIPIENT_EMAIL="root"
212
declare -A role_recipients_email=()
@@ -359,27 +365,35 @@ done
365
[ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
366
367
# check pushbullet
362
-[ -z "${DEFAULT_RECIPIENT_PUSHBULLET}" ] && SEND_PUSHBULLET="NO"
368
+[ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
369
370
# check twilio
365
-[ -z "${DEFAULT_RECIPIENT_TWILIO}" ] && SEND_TWILIO="NO"
371
+[ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
372
373
# check telegram
374
[ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
375
370
-if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" \) -a -z "${curl}" ]
376
+# check kafka
377
+[ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
378
+
379
+# if we need curl, check for the curl command
380
+if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" -o "${SEND_KAFKA}" = "YES" \) -a -z "${curl}" ]
381
then
382
curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
383
if [ -z "${curl}" ]
384
then
385
+ # no curl available
386
+ # disable all curl based methods
387
SEND_PUSHOVER="NO"
388
SEND_PUSHBULLET="NO"
389
SEND_TELEGRAM="NO"
390
SEND_SLACK="NO"
391
SEND_TWILIO="NO"
392
+ SEND_KAFKA="NO"
393
fi
394
fi
395
396
+# if we need sendmail, check for the sendmail command
397
if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
398
then
399
sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
@@ -387,13 +401,20 @@ if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
401
fi
402
403
# check that we have at least a method enabled
390
-if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_TELEGRAM}" != "YES" -a "${SEND_SLACK}" != "YES" -a "${SEND_TWILIO}" != "YES" -a "${SEND_PUSHBULLET}" != "YES" ]
404
+if [ "${SEND_EMAIL}" != "YES" \
405
+ -a "${SEND_PUSHOVER}" != "YES" \
406
+ -a "${SEND_TELEGRAM}" != "YES" \
407
+ -a "${SEND_SLACK}" != "YES" \
408
+ -a "${SEND_TWILIO}" != "YES" \
409
+ -a "${SEND_PUSHBULLET}" != "YES" \
410
+ -a "${SEND_KAFKA}" != "YES" \
411
+ ]
412
then
413
fatal "All notification methods are disabled. Not sending notification to '${role}' for '${name}' = '${value}' of chart '${chart}' for status '${status}'."
414
fi
415
416
# -----------------------------------------------------------------------------
396
-# get the system hostname
417
+# find a suitable hostname to use, if netdata did not supply a hostname
418
419
[ -z "${host}" ] && host="${NETDATA_HOSTNAME}"
420
[ -z "${host}" ] && host="${NETDATA_REGISTRY_HOSTNAME}"
@@ -406,7 +427,7 @@ date="$(date --date=@${when} 2>/dev/null)"
427
[ -z "${date}" ] && date="$(date 2>/dev/null)"
428
429
# -----------------------------------------------------------------------------
409
-# URL encode a string
430
+# function to URL encode a string
431
432
urlencode() {
433
local string="${1}" strlen encoded pos c o
@@ -432,7 +453,7 @@ urlencode() {
453
}
454
455
# -----------------------------------------------------------------------------
435
-# convert a duration in seconds, to a human readable duration
456
+# function to convert a duration in seconds, to a human readable duration
457
# using DAYS, MINUTES, SECONDS
458
459
duration4human() {
@@ -592,6 +613,31 @@ EOF
613
return 1
614
}
615
616
+# -----------------------------------------------------------------------------
617
+# kafka sender
618
+
619
+send_kafka() {
620
+ local httpcode sent=0
621
+ if [ "${SEND_KAFKA}" = "YES" ]
622
+ then
623
+ httpcode=$(${curl} -X POST --write-out %{http_code} --silent --output /dev/null \
624
+ --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
625
+ "${KAFKA_URL}")
626
+
627
+ if [ "${httpcode}" == "200" ]
628
+ then
629
+ info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
630
+ sent=$((sent + 1))
631
+ else
632
+ error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
633
+ fi
634
+
635
+ [ ${sent} -gt 0 ] && return 0
636
+ fi
637
+
638
+ return 1
639
+}
640
+
641
# -----------------------------------------------------------------------------
642
# twilio sender
643
@@ -874,6 +920,14 @@ ${chart} (${family})
920
921
SENT_TELEGRAM=$?
922
923
+
924
+# -----------------------------------------------------------------------------
925
+# send the kafka message
926
+
927
+send_kafka
928
+SENT_KAFKA=$?
929
+
930
+
931
# -----------------------------------------------------------------------------
932
# send the email
933
@@ -972,8 +1026,18 @@ SENT_EMAIL=$?
1026
# -----------------------------------------------------------------------------
1027
# let netdata know
1028
975
-# we did send something
976
-[ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_TELEGRAM} -eq 0 -o ${SENT_SLACK} -eq 0 -o ${SENT_TWILIO} -eq 0 -o ${SENT_PUSHBULLET} -eq 0 ] && exit 0
1029
+if [ ${SENT_EMAIL} -eq 0 \
1030
+ -o ${SENT_PUSHOVER} -eq 0 \
1031
+ -o ${SENT_TELEGRAM} -eq 0 \
1032
+ -o ${SENT_SLACK} -eq 0 \
1033
+ -o ${SENT_TWILIO} -eq 0 \
1034
+ -o ${SENT_PUSHBULLET} -eq 0 \
1035
+ -o ${SENT_KAFKA} -eq 0 \
1036
+ ]
1037
+ then
1038
+ # we did send something
1039
+ exit 0
1040
+fi
1041
1042
# we did not send anything
1043
exit 1