Add Amazon SNS notification support.
The Simple Notification Service (SNS) is a reasonably simple message broker service provided by Amazon as part of it's AWS offerings. SNS utilizes the concept of 'topics' (similar to Netdata's concept of 'roles' for notifications) to control message routing. Any given topic may have any number of subscribers of any of the following types: * Email addresses. * Phone numbers for SMS. * HTTP or HTTPS web hooks. * AWS Lambda endpoints. * AWS SQS endpoints. * Mobile applications (via various native push notification services). Topics are rpersented by Amazon Resource Names (ARN), which are a special type of URI. Unfortunately, properly signing requests to AWS endpoints is a serious pain in the arse, so we pretty much have to use the CLI interface. Because of the inflexibility of this tool, setup is somewhat painful. THis uses topic ARN's directly as recipients. SNS does not support delivery to multiple topics in bulk, so a separate call is required for each topic ARN. Users are provided with the ability to customize the message format used for the notifications.
Austin S. Hemmelgarn committed
Jul 13, 2018 at 11:22 UTC
a9b4db28b849e4ad10b9c68e0f723d326d30d5bd
2 files changed
+109
-1
conf.d/health_alarm_notify.conf
+41
-1
@@ -14,6 +14,7 @@
14
# - sms messages to your cell phone or any sms enabled device (twilio.com)
15
# - sms messages to your cell phone or any sms enabled device (messagebird.com)
16
# - notifications to users on pagerduty.com
17
+# - notifications to Amazon SNS topics (aws.amazon.com)
18
# - messages to your irc channel on your selected network
19
# - messages to a local or remote syslog daemon
20
# - message to Microsoft Team (thru webhook)
@@ -85,6 +86,11 @@ nc=""
86
# If not found, syslog notifications will be silently disabled.
87
logger=""
88
89
+# The full path of the aws command.
90
+# If empty, the system $PATH will be searched for it.
91
+# If not found, Amazon SNS notifications will be silently disabled.
92
+aws=""
93
+
94
#------------------------------------------------------------------------------
95
# extra options for external commands
96
#
@@ -618,6 +624,28 @@ SYSLOG_FACILITY=''
624
#
625
DEFAULT_RECIPIENT_SYSLOG="netdata"
626
627
+#------------------------------------------------------------------------------
628
+# Amazon SNS notifications
629
+#
630
+# This method requires potentially complex manual configuration. See the
631
+# netdata wiki for information on what is needed.
632
+
633
+# enable/disable sending Amazon SNS notifications
634
+SEND_AWSSNS="YES"
635
+
636
+# Specify a template for the Amazon SNS notifications. This supports
637
+# the same set of variables that are usable in the `custom_sender()`
638
+# function in the custom notification configuration below.
639
+#
640
+AWSSNS_MESSAGE_FORMAT="${status} on ${host} at ${date}: ${chart} ${value_string}"
641
+
642
+# If a role's recipients are not configured, use the following.
643
+# (empty = do not send a notification for unconfigured roles)
644
+#
645
+# Recipients for AWS SNS notifications are specified as topic ARN's.
646
+#
647
+DEFAULT_RECIPIENT_AWSSNS=""
648
+
649
#------------------------------------------------------------------------------
650
# custom notifications
651
#
@@ -719,6 +747,8 @@ role_recipients_irc[sysadmin]="${DEFAULT_RECIPIENT_IRC}"
747
748
role_recipients_syslog[sysadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
749
750
+role_recipients_awssns[sysadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
751
+
752
role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
753
754
role_recipients_msteam[sysadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
@@ -758,6 +788,8 @@ role_recipients_irc[domainadmin]="${DEFAULT_RECIPIENT_IRC}"
788
789
role_recipients_syslog[domainadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
790
791
+role_recipients_awssns[domainadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
792
+
793
role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
794
795
role_recipients_msteam[domainadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
@@ -798,6 +830,8 @@ role_recipients_irc[dba]="${DEFAULT_RECIPIENT_IRC}"
830
831
role_recipients_syslog[dba]="${DEFAULT_RECIPIENT_SYSLOG}"
832
833
+role_recipients_awssns[dba]="${DEFAULT_RECIPIENT_AWSSNS}"
834
+
835
role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"
836
837
role_recipients_msteam[dba]="${DEFAULT_RECIPIENT_MSTEAM}"
@@ -838,6 +872,8 @@ role_recipients_irc[webmaster]="${DEFAULT_RECIPIENT_IRC}"
872
873
role_recipients_syslog[webmaster]="${DEFAULT_RECIPIENT_SYSLOG}"
874
875
+role_recipients_awssns[webmaster]="${DEFAULT_RECIPIENT_AWSSNS}"
876
+
877
role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"
878
879
role_recipients_msteam[webmaster]="${DEFAULT_RECIPIENT_MSTEAM}"
@@ -878,6 +914,8 @@ role_recipients_irc[proxyadmin]="${DEFAULT_RECIPIENT_IRC}"
914
915
role_recipients_syslog[proxyadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
916
917
+role_recipients_awssns[porxyadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
918
+
919
role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
920
921
role_recipients_msteam[proxyadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
@@ -916,6 +954,8 @@ role_recipients_fleep[sitemgr]="${DEFAULT_RECIPIENT_FLEEP}"
954
955
role_recipients_syslog[sitemgr]="${DEFAULT_RECIPIENT_SYSLOG}"
956
957
+role_recipients_awssns[sitemgr]="${DEFAULT_RECIPIENT_AWSSNS}"
958
+
959
role_recipients_custom[sitemgr]="${DEFAULT_RECIPIENT_CUSTOM}"
960
921
-role_recipients_msteam[sitemgr]="${DEFAULT_RECIPIENT_MSTEAM}"
\ No newline at end of file
961
+role_recipients_msteam[sitemgr]="${DEFAULT_RECIPIENT_MSTEAM}"
plugins.d/alarm-notify.sh
+68
@@ -245,6 +245,7 @@ SEND_KAFKA="YES"
245
SEND_PD="YES"
246
SEND_FLEEP="YES"
247
SEND_IRC="YES"
248
+SEND_AWSSNS="YES"
249
SEND_SYSLOG="NO"
250
SEND_CUSTOM="YES"
251
@@ -334,6 +335,11 @@ FLEEP_SENDER="${host}"
335
DEFAULT_RECIPIENT_FLEEP=
336
declare -A role_recipients_fleep=()
337
338
+# Amazon SNS configs
339
+DEFAULT_RECIPIENT_AWSSNS=
340
+AWSSNS_MESSAGE_FORMAT=
341
+declare -A role_recipients_awssns=()
342
+
343
# syslog configs
344
SYSLOG_FACILITY=
345
declare -A role_recipients_syslog=()
@@ -452,6 +458,7 @@ declare -A arr_kavenegar=()
458
declare -A arr_fleep=()
459
declare -A arr_irc=()
460
declare -A arr_syslog=()
461
+declare -A arr_awssns=()
462
463
# netdata may call us with multiple roles, and roles may have multiple but
464
# overlapping recipients - so, here we find the unique recipients.
@@ -597,6 +604,14 @@ do
604
[ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1"
605
done
606
607
+ # awssns
608
+ a="${role_recipients_awssns[${x}]}"
609
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_awssns}"
610
+ for r in ${a//,/ }
611
+ do
612
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality awssns "${r}" && arr_awssns[${r/|*/}]="1"
613
+ done
614
+
615
# syslog
616
a="${role_recipients_syslog[${x}]}"
617
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}"
@@ -692,6 +707,10 @@ done
707
to_irc="${!arr_irc[*]}"
708
[ -z "${to_irc}" ] && SEND_IRC="NO"
709
710
+# build the list of awssns recipients (facilities, servers, and prefixes)
711
+to_awssns="${!arr_awssns[*]}"
712
+[ -z "${to_awssns}" ] && SEND_AWSSNS="NO"
713
+
714
# build the list of syslog recipients (facilities, servers, and prefixes)
715
to_syslog="${!arr_syslog[*]}"
716
[ -z "${to_syslog}" ] && SEND_SYSLOG="NO"
@@ -822,6 +841,17 @@ if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ]
841
fi
842
fi
843
844
+# if we need aws, check for the aws command
845
+if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ]
846
+ then
847
+ aws="$(which aws 2>/dev/null || command -v aws 2>/dev/null)"
848
+ if [ -z "${aws}" ]
849
+ then
850
+ debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
851
+ SEND_AWSSNS="NO"
852
+ fi
853
+fi
854
+
855
# check that we have at least a method enabled
856
if [ "${SEND_EMAIL}" != "YES" \
857
-a "${SEND_PUSHOVER}" != "YES" \
@@ -841,6 +871,7 @@ if [ "${SEND_EMAIL}" != "YES" \
871
-a "${SEND_FLEEP}" != "YES" \
872
-a "${SEND_CUSTOM}" != "YES" \
873
-a "${SEND_IRC}" != "YES" \
874
+ -a "${SEND_AWSSNS}" != "YES" \
875
-a "${SEND_SYSLOG}" != "YES" \
876
-a "${SEND_MSTEAM}" != "YES" \
877
]
@@ -1785,6 +1816,34 @@ send_irc() {
1816
return 1
1817
}
1818
1819
+# -----------------------------------------------------------------------------
1820
+# Amazon SNS sender
1821
+
1822
+send_awssns() {
1823
+ local targets="${1}" message='' sent=0 region=''
1824
+ local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
1825
+
1826
+ [ "${SEND_AWSSNS}" = "YES" ] || return 1
1827
+
1828
+ message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
1829
+
1830
+ for target in ${targets} ; do
1831
+ # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
1832
+ region="$(echo ${target} | cut -f 4 -d ':')"
1833
+ ${aws} --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null
1834
+ if [ $? = 0 ]; then
1835
+ info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
1836
+ sent=$((${sent} + 1))
1837
+ else
1838
+ error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
1839
+ fi
1840
+ done
1841
+
1842
+ [ ${sent} -gt 0 ] && return 0
1843
+
1844
+ return 1
1845
+}
1846
+
1847
# -----------------------------------------------------------------------------
1848
# syslog sender
1849
@@ -2139,6 +2198,14 @@ ${host} ${status_message}<br/> \
2198
SENT_HIPCHAT=$?
2199
2200
2201
+# -----------------------------------------------------------------------------
2202
+# send the Amazon SNS message
2203
+
2204
+send_awssns ${to_awssns}
2205
+
2206
+SENT_AWSSNS=$?
2207
+
2208
+
2209
# -----------------------------------------------------------------------------
2210
# send the syslog message
2211
@@ -2291,6 +2358,7 @@ if [ ${SENT_EMAIL} -eq 0 \
2358
-o ${SENT_PD} -eq 0 \
2359
-o ${SENT_FLEEP} -eq 0 \
2360
-o ${SENT_IRC} -eq 0 \
2361
+ -o ${SENT_AWSSNS} -eq 0 \
2362
-o ${SENT_CUSTOM} -eq 0 \
2363
-o ${SENT_SYSLOG} -eq 0 \
2364
]