Added pagerduty.com to alarm-notify.sh
Jim Cooley committed
Dec 13, 2016 at 18:31 UTC
fa694e38a8505c836851c25c848d2d3036da3551
1 file changed
+63
plugins.d/alarm-notify.sh
+63
@@ -21,6 +21,7 @@
21
# - telegram.org notifications by @hashworks PR #1002
22
# - twilio.com notifications by Levi Blaney @shadycuz PR #1211
23
# - kafka notifications
24
+# - pagerduty.com notifications by Jim Cooley @jimcooley PR#????
25
26
# -----------------------------------------------------------------------------
27
# testing notifications
@@ -175,6 +176,7 @@ SEND_TELEGRAM="YES"
176
SEND_EMAIL="YES"
177
SEND_PUSHBULLET="YES"
178
SEND_KAFKA="YES"
179
+SEND_PD="YES"
180
181
# slack configs
182
SLACK_WEBHOOK_URL=
@@ -207,6 +209,9 @@ declare -A role_recipients_telegram=()
209
KAFKA_URL=
210
KAFKA_SENDER_IP=
211
212
+# pagerduty.com configs
213
+PD_SERVICE_KEY=
214
+
215
# email configs
216
DEFAULT_RECIPIENT_EMAIL="root"
217
declare -A role_recipients_email=()
@@ -376,6 +381,21 @@ done
381
# check kafka
382
[ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
383
384
+# check pagerduty.com
385
+[ -z "${PD_SERVICE_KEY}" ] && SEND_PD="NO"
386
+
387
+# if we need pd-send, check for the pd-send command
388
+if [ "${SEND_PD}" = "YES" ]
389
+ then
390
+ pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)"
391
+ if [ -z "${pd_send}" ]
392
+ then
393
+ # no pd-send available
394
+ # disable pagerduty.com
395
+ SEND_PD="NO"
396
+ fi
397
+fi
398
+
399
# if we need curl, check for the curl command
400
if [ \( "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" -o "${SEND_TWILIO}" = "YES" -o "${SEND_TELEGRAM}" = "YES" -o "${SEND_PUSHBULLET}" = "YES" -o "${SEND_KAFKA}" = "YES" \) -a -z "${curl}" ]
401
then
@@ -408,6 +428,7 @@ if [ "${SEND_EMAIL}" != "YES" \
428
-a "${SEND_TWILIO}" != "YES" \
429
-a "${SEND_PUSHBULLET}" != "YES" \
430
-a "${SEND_KAFKA}" != "YES" \
431
+ -a "${SEND_PD}" != "YES" \
432
]
433
then
434
fatal "All notification methods are disabled. Not sending notification to '${role}' for '${name}' = '${value}' of chart '${chart}' for status '${status}'."
@@ -638,6 +659,40 @@ send_kafka() {
659
return 1
660
}
661
662
+# -----------------------------------------------------------------------------
663
+# pagerduty.com sender
664
+
665
+send_pd() {
666
+ unset t
667
+ case ${status} in
668
+ CLEAR) t='resolve';;
669
+ WARNING) t='trigger';;
670
+ CRITICAL) t='trigger';;
671
+ esac
672
+
673
+ if [ ${SEND_PD} = "YES" -a ! -z "${t}" ]
674
+ then
675
+ ${pd_send} -k ${PD_SERVICE_KEY} \
676
+ -t ${t} \
677
+ -d "${host} ${chart}.${name} is ${status}" \
678
+ -i ${alarm_id} \
679
+ -f 'info'="${info}" \
680
+ -f 'value'="${value} ${units}" \
681
+ -f 'when'="${when}" \
682
+ -f 'duration'="${duration}"
683
+ retval=$?
684
+ if [ ${retval} -eq 0 ]
685
+ then
686
+ info "sent pagerduty.com notification for: ${host} ${chart}.${name} is ${status}"
687
+ return 0
688
+ else
689
+ error "failed to send pagerduty.com notification for: ${host} ${chart}.${name} is ${status} with error code ${retval}."
690
+ fi
691
+ fi
692
+
693
+ return 1
694
+}
695
+
696
# -----------------------------------------------------------------------------
697
# twilio sender
698
@@ -928,6 +983,13 @@ send_kafka
983
SENT_KAFKA=$?
984
985
986
+# -----------------------------------------------------------------------------
987
+# send the pagerduty.com message
988
+
989
+send_pd
990
+SENT_PD=$?
991
+
992
+
993
# -----------------------------------------------------------------------------
994
# send the email
995
@@ -1033,6 +1095,7 @@ if [ ${SENT_EMAIL} -eq 0 \
1095
-o ${SENT_TWILIO} -eq 0 \
1096
-o ${SENT_PUSHBULLET} -eq 0 \
1097
-o ${SENT_KAFKA} -eq 0 \
1098
+ -o ${SENT_PD} -eq 0 \
1099
]
1100
then
1101
# we did send something