Add syslog support to alarm-notify.sh.
This uses the logger command (found on all modern Linux and BSD systems) to allow alarm-notify.sh to send netdata alarm messages to a local or remote syslog server. By default, logging is done to the local6 facility at a warning log level, with critical alarms being logged at the err level. This provides a mechanism of easily generating a running log of all netdata alarm events on most standard UNIX systems, and when used with full-featured logging daemons like rsyslog or syslong-ng can be used to archive such messages in a wide varity of alternative formats. There are a few limitations currently: * It can only log to one location. This can be easily worked around in most syslog daemon's configurations, as almost all of them support some form of output multiplexing. * The default settings will usually not get the results most people would want, but because of the sometimes very significant differences in default syslog configurations from distribution to distribution, there's not really any way to make the defaults any saner. * While it is possible to log directly to a remote syslog server, there is no quick and easy wet up for this, it requires the user to manually compose the extra options that need to be passed to the logger command. Syslog messages generated by this will look something like this: Apr 3 09:00:30 localhost netdata: Netdata notification on wild-karde at Tue Apr 3 09:00:00 EDT 2018: WARNING, out of disk space time = 5h
Austin S. Hemmelgarn committed
Apr 3, 2018 at 09:55 UTC
0f389fd615c4b35ea6c61551605b42e7a4914c0b
2 files changed
+111
conf.d/health_alarm_notify.conf
+66
@@ -15,6 +15,7 @@
15
# - sms messages to your cell phone or any sms enabled device (messagebird.com)
16
# - notifications to users on pagerduty.com
17
# - messages to your irc channel on your selected network
18
+# - messages to a local or remote syslog daemon
19
#
20
# The 'to' line given at netdata alarms defines a *role*, so that many
21
# people can be notified for each role.
@@ -61,6 +62,11 @@ curl=""
62
# If not found, irc notifications will be silently disabled.
63
nc=""
64
65
+# The full path of the logger command.
66
+# If empty, the system $PATH will be searched for it.
67
+# If not found, syslog notifications will be silently disabled.
68
+logger=""
69
+
70
#------------------------------------------------------------------------------
71
# extra options for external commands
72
#
@@ -74,6 +80,13 @@ nc=""
80
# of potentially sensitive information.
81
#curl_options="--insecure"
82
83
+# Extra options to pass to logger. If you just want to send to your
84
+# local system logs, you can leave this empty and everything should work.
85
+# If you want to send to a remote syslog server, you need to specify the
86
+# remote server here with the `--host` option, and may need additional
87
+# options to select the protocol. See `man logger` for more information.
88
+#logger_options=""
89
+
90
#------------------------------------------------------------------------------
91
# NOTE ABOUT RECIPIENTS
92
#
@@ -440,6 +453,59 @@ IRC_NICKNAME=""
453
IRC_REALNAME=""
454
455
456
+#------------------------------------------------------------------------------
457
+# syslog notifications
458
+#
459
+# syslog notifications only need you to have a working logger command, which
460
+# should be the case on pretty much any Linux system.
461
+
462
+# enable/disable sending syslog notifications
463
+# NOTE: make sure you have everything else configured the way you want
464
+# it _before_ turning this on.
465
+SEND_SYSLOG="NO"
466
+
467
+# A note on log levels and facilities:
468
+#
469
+# The traditional UNIX syslog mechanism has the concept of both log
470
+# levels and facilities. A log level indicates the relaitve severity of
471
+# the message, while a facility specifies a generic source for the message
472
+# (for example, the `mail` facility is where sendmail and postfix log
473
+# their messages). All major syslog daemons have the ability to filter
474
+# messages based on both log level and facility, and can often also make
475
+# routing decisions for messages based on both factors.
476
+#
477
+# On Linux, the eight log levels in decreasing order of severity are:
478
+# emerg, alert, crit, err, warning, notice, info, debug
479
+#
480
+# By default, netdata alerts will be logged at the warning level
481
+# for warnings and clear notifications, and the err level for critical
482
+# notifications.
483
+#
484
+# And the 19 facilities you can log to are:
485
+# auth, authpriv, cron, daemon, ftp, lpr, mail, news, syslog, user,
486
+# uucp, local0, local1, local2, local3, local4, local5, local6, and local7
487
+#
488
+# By default, netdata alerts will be logged to the local6 facility.
489
+#
490
+# Depending on your distribution, this means that either all your
491
+# netdata alerts will by default end up in the main system log (usually
492
+# /var/log/messages), or they won't be logged to a file at all.
493
+# Neither of these are likely to be what you actually want, but any
494
+# configuration to change that needs to happen in the syslog daemon
495
+# configuration, not here.
496
+
497
+# This specifies the log level that warnings and clear notifications
498
+# will be sent at. If unset, it defaults to 'warning'.
499
+SYSLOG_LEVEL=''
500
+
501
+# This specifies the log level that critical alerts will be sent at.
502
+# If unset, it defaults to 'err'
503
+SYSLOG_CRITICAL_LEVEL=''
504
+
505
+# This controls which facility is used for logging. Defaults to local6.
506
+SYSLOG_FACILITY=''
507
+
508
+
509
#------------------------------------------------------------------------------
510
# custom notifications
511
#
plugins.d/alarm-notify.sh
+45
@@ -27,6 +27,7 @@
27
# - messagebird.com notifications by @tech_no_logical #1453
28
# - hipchat notifications by @ktsaou #1561
29
# - custom notifications by @ktsaou
30
+# - syslog messages by @Ferroin
31
32
# -----------------------------------------------------------------------------
33
# testing notifications
@@ -237,6 +238,7 @@ SEND_PUSHBULLET="YES"
238
SEND_KAFKA="YES"
239
SEND_PD="YES"
240
SEND_IRC="YES"
241
+SEND_SYSLOG="NO"
242
SEND_CUSTOM="YES"
243
244
# slack configs
@@ -310,6 +312,11 @@ PD_SERVICE_KEY=
312
DEFAULT_RECIPIENT_PD=
313
declare -A role_recipients_pd=()
314
315
+# syslog configs
316
+SYSLOG_LEVEL=
317
+SYSLOG_LEVEL_CRIT=
318
+SYSLOG_FACILITY=
319
+
320
# custom configs
321
DEFAULT_RECIPIENT_CUSTOM=
322
declare -A role_recipients_custom=()
@@ -718,6 +725,17 @@ if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
725
fi
726
fi
727
728
+# if we need logger, check for the logger command
729
+if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ]
730
+ then
731
+ logger="$(which logger 2>/dev/null || command -v logger 2>/dev/null)"
732
+ if [ -z "${logger}" ]
733
+ then
734
+ debug "Cannot find logger command in the system path. Disabling syslog notifications."
735
+ SEND_SYSLOG="NO"
736
+ fi
737
+fi
738
+
739
# check that we have at least a method enabled
740
if [ "${SEND_EMAIL}" != "YES" \
741
-a "${SEND_PUSHOVER}" != "YES" \
@@ -735,6 +753,7 @@ if [ "${SEND_EMAIL}" != "YES" \
753
-a "${SEND_PD}" != "YES" \
754
-a "${SEND_CUSTOM}" != "YES" \
755
-a "${SEND_IRC}" != "YES" \
756
+ -a "${SEND_SYSLOG}" != "YES" \
757
]
758
then
759
fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
@@ -1510,6 +1529,23 @@ send_irc() {
1529
return 1
1530
}
1531
1532
+# -----------------------------------------------------------------------------
1533
+# syslog sender
1534
+
1535
+send_syslog() {
1536
+ local message="${1}" level=${SYSLOG_LEVEL:-"warning"} crit_level=${SYSLOG_CRITICAL_LEVEL:-"err"} facility=${SYSLOG_FACILITY:-"local6"}
1537
+
1538
+ [ "${SEND_SYSLOG}" -eq "YES" ] || return 1
1539
+
1540
+ if [ "${status}" -eq "CRITICAL" ] ; then
1541
+ level=${crit_level}
1542
+ fi
1543
+
1544
+ ${logger} -p ${facility}.${level} ${logger_options} "${message}"
1545
+
1546
+ return $?
1547
+}
1548
+
1549
1550
# -----------------------------------------------------------------------------
1551
# prepare the content of the notification
@@ -1768,6 +1804,14 @@ ${host} ${status_message}<br/> \
1804
SENT_HIPCHAT=$?
1805
1806
1807
+# -----------------------------------------------------------------------------
1808
+# send the syslog message
1809
+
1810
+send_syslog "Netdata notification on ${host} at ${when}: ${status}, ${alarm}"
1811
+
1812
+SENT_SYSLOG=$?
1813
+
1814
+
1815
# -----------------------------------------------------------------------------
1816
# send the email
1817
@@ -1909,6 +1953,7 @@ if [ ${SENT_EMAIL} -eq 0 \
1953
-o ${SENT_PD} -eq 0 \
1954
-o ${SENT_IRC} -eq 0 \
1955
-o ${SENT_CUSTOM} -eq 0 \
1956
+ -o ${SENT_SYSLOG} -eq 0 \
1957
]
1958
then
1959
# we did send something