fix(pagerduty): Use cURL instead of PagerDuty agent to send alarms. (#4694)
Elisiário Couto committed
Dec 3, 2018 at 12:39 UTC
f1e0ff8eba3a97b4dec7cc8db0ff39394935d1ab
2 files changed
+58
-65
health/notifications/alarm-notify.sh.in
+51
-58
@@ -479,25 +479,13 @@ filter_recipient_by_criticality() {
479
# check fleep
480
[ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO"
481
482
-# check pagerduty.com
483
-# if we need pd-send, check for the pd-send command
484
-# https://www.pagerduty.com/docs/guides/agent-install-guide/
485
-if [ "${SEND_PD}" = "YES" ]
486
- then
487
- pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)"
488
- if [ -z "${pd_send}" ]
489
- then
490
- error "Cannot find pd-send command in the system path. Disabling pagerduty.com notifications."
491
- SEND_PD="NO"
492
- fi
493
-fi
494
-
482
# if we need curl, check for the curl command
483
if [ \( \
484
"${SEND_PUSHOVER}" = "YES" \
485
-o "${SEND_SLACK}" = "YES" \
486
-o "${SEND_ROCKETCHAT}" = "YES" \
487
-o "${SEND_ALERTA}" = "YES" \
488
+ -o "${SEND_PD}" = "YES" \
489
-o "${SEND_FLOCK}" = "YES" \
490
-o "${SEND_DISCORD}" = "YES" \
491
-o "${SEND_HIPCHAT}" = "YES" \
@@ -523,6 +511,7 @@ if [ \( \
511
SEND_MSTEAM="NO"
512
SEND_ROCKETCHAT="NO"
513
SEND_ALERTA="NO"
514
+ SEND_PD="NO"
515
SEND_FLOCK="NO"
516
SEND_DISCORD="NO"
517
SEND_TWILIO="NO"
@@ -898,7 +887,7 @@ EOF
887
# kafka sender
888
889
send_kafka() {
901
- local httpcode sent=0
890
+ local httpcode sent=0
891
if [ "${SEND_KAFKA}" = "YES" ]
892
then
893
httpcode=$(docurl -X POST \
@@ -935,38 +924,42 @@ send_pd() {
924
then
925
for PD_SERVICE_KEY in ${recipients}
926
do
938
- d="${status} ${name} = ${value_string} - ${host}, ${family}"
939
- ${pd_send} -k ${PD_SERVICE_KEY} \
940
- -t ${t} \
941
- -d "${d}" \
942
- -i ${host}:${chart}:${name} \
943
- -f 'info'="${info}" \
944
- -f 'value_w_units'="${value_string}" \
945
- -f 'when'="${when}" \
946
- -f 'duration'="${duration}" \
947
- -f 'roles'="${roles}" \
948
- -f 'host'="${host}" \
949
- -f 'unique_id'="${unique_id}" \
950
- -f 'alarm_id'="${alarm_id}" \
951
- -f 'event_id'="${event_id}" \
952
- -f 'name'="${name}" \
953
- -f 'chart'="${chart}" \
954
- -f 'family'="${family}" \
955
- -f 'status'="${status}" \
956
- -f 'old_status'="${old_status}" \
957
- -f 'value'="${value}" \
958
- -f 'old_value'="${old_value}" \
959
- -f 'src'="${src}" \
960
- -f 'non_clear_duration'="${non_clear_duration}" \
961
- -f 'units'="${units}"
962
- retval=$?
963
- if [ ${retval} -eq 0 ]
964
- then
965
- info "sent pagerduty.com notification for host ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}....: ${d}"
966
- sent=$((sent + 1))
967
- else
968
- error "failed to send pagerduty.com notification for ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}"
969
- fi
927
+ d="${status} ${name} = ${value_string} - ${host}, ${family}"
928
+ payload="$(cat << EOF
929
+ {
930
+ "service_key": "${PD_SERVICE_KEY}",
931
+ "event_type": "${t}",
932
+ "incident_key" : "${alarm_id}",
933
+ "description": "${d}",
934
+ "details": {
935
+ "value_w_units": "${value_string}",
936
+ "when": "${when}",
937
+ "duration" : "${duration}",
938
+ "roles": "${roles}",
939
+ "alarm_id" : "${alarm_id}",
940
+ "name" : "${name}",
941
+ "chart" : "${chart}",
942
+ "family" : "${family}",
943
+ "status" : "${status}",
944
+ "old_status" : "${old_status}",
945
+ "value" : "${value}",
946
+ "old_value" : "${old_value}",
947
+ "src" : "${src}",
948
+ "non_clear_duration" : "${non_clear_duration}",
949
+ "units" : "${units}",
950
+ "info" : "${info}"
951
+ }
952
+ }
953
+EOF
954
+ )"
955
+ httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json")
956
+ if [ "${httpcode}" = "200" ]
957
+ then
958
+ info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
959
+ sent=$((sent + 1))
960
+ else
961
+ error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
962
+ fi
963
done
964
965
[ ${sent} -gt 0 ] && return 0
@@ -1021,7 +1014,7 @@ send_hipchat() {
1014
if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
1015
then
1016
# A label to be shown in addition to the sender's name
1024
- # Valid length range: 0 - 64.
1017
+ # Valid length range: 0 - 64.
1018
sender="netdata"
1019
1020
# Valid values: html, text.
@@ -1048,7 +1041,7 @@ send_hipchat() {
1041
-H "Authorization: Bearer ${authtoken}" \
1042
-d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
1043
"https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
1051
-
1044
+
1045
if [ "${httpcode}" = "204" ]
1046
then
1047
info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
@@ -1135,7 +1128,7 @@ send_telegram() {
1128
local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
1129
1130
if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
1138
-
1131
+
1132
case "${status}" in
1133
WARNING) emoji="⚠️" ;;
1134
CRITICAL) emoji="🔴" ;;
@@ -1601,7 +1594,7 @@ send_fleep() {
1594
1595
send_irc() {
1596
local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
1604
-
1597
+
1598
if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ]
1599
then
1600
case "${status}" in
@@ -1626,13 +1619,13 @@ send_irc() {
1619
info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
1620
sent=$((sent + 1))
1621
else
1629
- error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
1622
+ error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
1623
fi
1624
done
1625
fi
1633
-
1626
+
1627
[ ${sent} -gt 0 ] && return 0
1635
-
1628
+
1629
return 1
1630
}
1631
@@ -1655,7 +1648,7 @@ send_awssns() {
1648
info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
1649
sent=$((sent + 1))
1650
else
1658
- error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
1651
+ error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
1652
fi
1653
done
1654
@@ -1911,7 +1904,7 @@ SENT_PUSHBULLET=$?
1904
# -----------------------------------------------------------------------------
1905
# send the twilio SMS
1906
1914
-send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1907
+send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1908
Severity: ${severity}
1909
Chart: ${chart}
1910
Family: ${family}
@@ -1922,7 +1915,7 @@ SENT_TWILIO=$?
1915
# -----------------------------------------------------------------------------
1916
# send the messagebird SMS
1917
1925
-send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1918
+send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1919
Severity: ${severity}
1920
Chart: ${chart}
1921
Family: ${family}
@@ -1934,7 +1927,7 @@ SENT_MESSAGEBIRD=$?
1927
# -----------------------------------------------------------------------------
1928
# send the kavenegar SMS
1929
1937
-send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1930
+send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
1931
Severity: ${severity}
1932
Chart: ${chart}
1933
Family: ${family}
@@ -1977,7 +1970,7 @@ SENT_FLEEP=$?
1970
# -----------------------------------------------------------------------------
1971
# send the irc message
1972
1980
-send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
1973
+send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
1974
Severity: ${severity}
1975
Chart: ${chart}
1976
Family: ${family}
health/notifications/health_alarm_notify.conf
+7
-7
@@ -268,7 +268,7 @@ SEND_MESSAGEBIRD="YES"
268
# to get an access key, create a free account at https://www.messagebird.com
269
# verify and activate the account (no CC info needed)
270
# login to your account and enter your phonenumber to get some free credits
271
-# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)'
271
+# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)'
272
# click 'Add access key' and fill in data (you want a live key to send SMS)
273
274
# Without an access key, netdata cannot send Messagebird text messages.
@@ -407,7 +407,7 @@ DEFAULT_RECIPIENT_ROCKETCHAT=""
407
SEND_ALERTA="YES"
408
409
# here set your alerta server API url
410
-# this is the API url you defined when installed Alerta server,
410
+# this is the API url you defined when installed Alerta server,
411
# it is the same for all users. Do not include last slash.
412
# ALERTA_WEBHOOK_URL="https://<server>/alerta/api"
413
ALERTA_WEBHOOK_URL=""
@@ -495,9 +495,9 @@ KAFKA_SENDER_IP=""
495
#------------------------------------------------------------------------------
496
# pagerduty.com notification options
497
#
498
-# pagerduty.com notifications require the pagerduty agent to be installed and
499
-# a "Generic API" pagerduty service.
500
-# https://www.pagerduty.com/docs/guides/agent-install-guide/
498
+# pagerduty.com notifications require a "Generic API" (Events v1)
499
+# pagerduty service.
500
+# https://support.pagerduty.com/docs/services-and-integrations
501
502
# multiple recipients can be given like this:
503
# "<pd_service_key_1> <pd_service_key_2> ..."
@@ -536,7 +536,7 @@ FLEEP_SENDER=""
536
#------------------------------------------------------------------------------
537
# irc notification options
538
#
539
-# irc notifications require only the nc utility to be installed.
539
+# irc notifications require only the nc utility to be installed.
540
541
# multiple recipients can be given like this:
542
# "<irc_channel_1> <irc_channel_2> ..."
@@ -552,7 +552,7 @@ DEFAULT_RECIPIENT_IRC=""
552
# e.g. "irc.freenode.net"
553
IRC_NETWORK=""
554
555
-# The irc nickname which is required to send the notification. It must not be
555
+# The irc nickname which is required to send the notification. It must not be
556
# an already registered name as the connection's MODE is defined as a 'guest'.
557
IRC_NICKNAME=""
558