Mapping roles to service keys and allowing multiple service keys for pagerduty.com notifications
Jim Cooley committed
Dec 14, 2016 at 17:10 UTC
fcf9823df565109c0ec010f8963c0053e4c7b579
2 files changed
+75
-38
conf.d/health_alarm_notify.conf
+21
-3
@@ -9,6 +9,7 @@
9
# - messages to your slack team (slack.com),
10
# - messages to your telegram chat / group chat (telegram.org)
11
# - sms messages to your cell phone or any sms enabled device (twilio.com)
12
+# - notifications to users on pagerduty.com
13
#
14
# The 'to' line given at netdata alarms defines a *role*, so that many
15
# people can be notified for each role.
@@ -62,6 +63,7 @@ curl=""
63
# - telegram chat ids
64
# - slack channels
65
# - sms phone numbers
66
+# - pagerduty.com (pd) services
67
#
68
# You can append |critical to limit the notifications to be sent.
69
#
@@ -73,9 +75,10 @@ curl=""
75
# telegram: "111827421 112746832|critical"
76
# slack : "alarms disasters|critical"
77
# twilio : "+15555555555 +17777777777|critical"
78
+# pd : "<pd_service_key_1> <pd_service_key_2>|critical"
79
#
80
# If a recipient is set to empty string, the default recipient of the given
78
-# notification method (email, pushover, telegram, slack) will be used.
81
+# notification method (email, pushover, telegram, slack, pd) will be used.
82
# To disable a notification, use the recipient called: disabled
83
# This works for all notification methods (including the default recipients).
84
@@ -219,11 +222,16 @@ KAFKA_SENDER_IP=""
222
# a "Generic API" pagerduty service.
223
# https://www.pagerduty.com/docs/guides/agent-install-guide/
224
225
+# multiple recipients can be given like this:
226
+# "<pd_service_key_1> <pd_service_key_2> ..."
227
+
228
# enable/disable sending pagerduty notifications
229
SEND_PD="YES"
230
225
-# The service key for your "General API" pagerduty service.
226
-PD_SERVICE_KEY=''
231
+# if a role's recipients are not configured, a notification will be sent to
232
+# the "General API" pagerduty.com service that uses this service key.
233
+# (empty = do not send a notification for unconfigured roles):
234
+DEFAULT_RECIPIENT_PD=""
235
236
237
###############################################################################
@@ -245,6 +253,8 @@ role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
253
254
role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"
255
256
+role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"
257
+
258
# -----------------------------------------------------------------------------
259
# DNS related alarms
260
@@ -260,6 +270,8 @@ role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"
270
271
role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"
272
273
+role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
274
+
275
# -----------------------------------------------------------------------------
276
# database servers alarms
277
# mysql, redis, memcached, etc
@@ -276,6 +288,8 @@ role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"
288
289
role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"
290
291
+role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
292
+
293
# -----------------------------------------------------------------------------
294
# web servers alarms
295
# apache, nginx, etc
@@ -292,6 +306,8 @@ role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"
306
307
role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"
308
309
+role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
310
+
311
# -----------------------------------------------------------------------------
312
# proxy servers alarms
313
# apache, nginx, etc
@@ -307,3 +323,5 @@ role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
323
role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"
324
325
role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"
326
+
327
+role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"
plugins.d/alarm-notify.sh
+54
-35
@@ -211,6 +211,7 @@ KAFKA_SENDER_IP=
211
212
# pagerduty.com configs
213
PD_SERVICE_KEY=
214
+declare -A role_recipients_pd=()
215
216
# email configs
217
DEFAULT_RECIPIENT_EMAIL="root"
@@ -271,6 +272,7 @@ declare -A arr_pushover=()
272
declare -A arr_pushbullet=()
273
declare -A arr_twilio=()
274
declare -A arr_telegram=()
275
+declare -A arr_pd=()
276
declare -A arr_email=()
277
278
# netdata may call us with multiple roles, and roles may have multiple but
@@ -328,6 +330,14 @@ do
330
do
331
[ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
332
done
333
+
334
+ # pagerduty.com
335
+ a="${role_recipients_pd[${x}]}"
336
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PD}"
337
+ for r in ${a//,/ }
338
+ do
339
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1"
340
+ done
341
done
342
343
# build the list of slack recipients (channels)
@@ -350,6 +360,10 @@ to_twilio="${!arr_twilio[*]}"
360
to_telegram="${!arr_telegram[*]}"
361
[ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
362
363
+# build the list of pagerduty recipients (service keys)
364
+to_pd="${!arr_pd[*]}"
365
+[ -z "${to_pd}" ] && SEND_PD="NO"
366
+
367
# build the list of email recipients (email addresses)
368
to_email=
369
for x in "${!arr_email[@]}"
@@ -382,8 +396,6 @@ done
396
[ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
397
398
# check pagerduty.com
385
-[ -z "${PD_SERVICE_KEY}" ] && SEND_PD="NO"
386
-
399
# if we need pd-send, check for the pd-send command
400
# https://www.pagerduty.com/docs/guides/agent-install-guide/
401
if [ "${SEND_PD}" = "YES" ]
@@ -432,7 +444,7 @@ if [ "${SEND_EMAIL}" != "YES" \
444
-a "${SEND_PD}" != "YES" \
445
]
446
then
435
- fatal "All notification methods are disabled. Not sending notification to '${role}' for '${name}' = '${value}' of chart '${chart}' for status '${status}'."
447
+ fatal "All notification methods are disabled. Not sending notification to '${roles}' for '${name}' = '${value}' of chart '${chart}' for status '${status}'."
448
fi
449
450
# -----------------------------------------------------------------------------
@@ -664,6 +676,7 @@ send_kafka() {
676
# pagerduty.com sender
677
678
send_pd() {
679
+ local recipients="${1}" sent=0
680
unset t
681
case ${status} in
682
CLEAR) t='resolve';;
@@ -673,37 +686,43 @@ send_pd() {
686
687
if [ ${SEND_PD} = "YES" -a ! -z "${t}" ]
688
then
676
- ${pd_send} -k ${PD_SERVICE_KEY} \
677
- -t ${t} \
678
- -d "${status} ${name}=${value} ${units} - ${host}, ${family}" \
679
- -i ${alarm_id} \
680
- -f 'info'="${info}" \
681
- -f 'value_w_units'="${value} ${units}" \
682
- -f 'when'="${when}" \
683
- -f 'duration'="${duration}" \
684
- -f 'roles'="${roles}" \
685
- -f 'host'="${host}" \
686
- -f 'unique_id'="${unique_id}" \
687
- -f 'alarm_id'="${alarm_id}" \
688
- -f 'event_id'="${event_id}" \
689
- -f 'name'="${name}" \
690
- -f 'chart'="${chart}" \
691
- -f 'family'="${family}" \
692
- -f 'status'="${status}" \
693
- -f 'old_status'="${old_status}" \
694
- -f 'value'="${value}" \
695
- -f 'old_value'="${old_value}" \
696
- -f 'src'="${src}" \
697
- -f 'non_clear_duration'="${non_clear_duration}" \
698
- -f 'units'="${units}"
699
- retval=$?
700
- if [ ${retval} -eq 0 ]
701
- then
702
- info "sent pagerduty.com notification for: ${host} ${chart}.${name} is ${status}"
703
- return 0
704
- else
705
- error "failed to send pagerduty.com notification for: ${host} ${chart}.${name} is ${status} with error code ${retval}."
706
- fi
689
+ for PD_SERVICE_KEY in ${recipients}
690
+ do
691
+ d="${status} ${name}=${value} ${units} - ${host}, ${family}"
692
+ ${pd_send} -k ${PD_SERVICE_KEY} \
693
+ -t ${t} \
694
+ -d "${d}" \
695
+ -i ${alarm_id} \
696
+ -f 'info'="${info}" \
697
+ -f 'value_w_units'="${value} ${units}" \
698
+ -f 'when'="${when}" \
699
+ -f 'duration'="${duration}" \
700
+ -f 'roles'="${roles}" \
701
+ -f 'host'="${host}" \
702
+ -f 'unique_id'="${unique_id}" \
703
+ -f 'alarm_id'="${alarm_id}" \
704
+ -f 'event_id'="${event_id}" \
705
+ -f 'name'="${name}" \
706
+ -f 'chart'="${chart}" \
707
+ -f 'family'="${family}" \
708
+ -f 'status'="${status}" \
709
+ -f 'old_status'="${old_status}" \
710
+ -f 'value'="${value}" \
711
+ -f 'old_value'="${old_value}" \
712
+ -f 'src'="${src}" \
713
+ -f 'non_clear_duration'="${non_clear_duration}" \
714
+ -f 'units'="${units}"
715
+ retval=$?
716
+ if [ ${retval} -eq 0 ]
717
+ then
718
+ info "sent pagerduty.com notification using service key ${PD_SERVICE_KEY::-26}....: ${d}"
719
+ sent=$((sent + 1))
720
+ else
721
+ error "failed to send pagerduty.com notification using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}"
722
+ fi
723
+ done
724
+
725
+ [ ${sent} -gt 0 ] && return 0
726
fi
727
728
return 1
@@ -1002,7 +1021,7 @@ SENT_KAFKA=$?
1021
# -----------------------------------------------------------------------------
1022
# send the pagerduty.com message
1023
1005
-send_pd
1024
+send_pd "${to_pd}"
1025
SENT_PD=$?
1026
1027