Update syslog support to allow per-role configuration.
This adds support to syslog notifications to have per-role configuration. This includes specifying facilty and log-level per-role, target servers per-role, and a message prefix per-role, possibly multiple times. This is done by specifying each combination of configuration as a 'recipient' in the role recipients variables using the following format: [[facility.level][@host[:port]]/]prefix As part of this, the default log level has been changed to depend solely on the type of alert, crit for CRITICAL, warning for WARNING, and info for everything else (most notably CLEAR), and the end-user configuration and the user configuration of these default levels has been removed. In addition, the message format has been changed to look like the following: Apr 3 09:00:30 localhost netdata: prefix WARNING on wild-karde at Tue Apr 3 09:00:00 EDT 2018: out of disk space time = 5h
Austin S. Hemmelgarn committed
Apr 5, 2018 at 09:59 UTC
79582316c1681a2242c9e0cf97f7a873c036aa1d
2 files changed
+113
-25
conf.d/health_alarm_notify.conf
+39
-18
@@ -80,11 +80,8 @@ logger=""
80
# of potentially sensitive information.
81
#curl_options="--insecure"
82
83
-# Extra options to pass to logger. If you just want to send to your
84
-# local system logs, you can leave this empty and everything should work.
85
-# If you want to send to a remote syslog server, you need to specify the
86
-# remote server here with the `--host` option, and may need additional
87
-# options to select the protocol. See `man logger` for more information.
83
+# Extra options to pass to logger. You shouldn't have to specify anything
84
+# here in most cases.
85
#logger_options=""
86
87
#------------------------------------------------------------------------------
@@ -477,9 +474,8 @@ SEND_SYSLOG="NO"
474
# On Linux, the eight log levels in decreasing order of severity are:
475
# emerg, alert, crit, err, warning, notice, info, debug
476
#
480
-# By default, netdata alerts will be logged at the warning level
481
-# for warnings and clear notifications, and the err level for critical
482
-# notifications.
477
+# By default, warnings will be logged at the warning level, critical
478
+# alerts at the crit level, and clear notifications at the invo level.
479
#
480
# And the 19 facilities you can log to are:
481
# auth, authpriv, cron, daemon, ftp, lpr, mail, news, syslog, user,
@@ -494,17 +490,31 @@ SEND_SYSLOG="NO"
490
# configuration to change that needs to happen in the syslog daemon
491
# configuration, not here.
492
497
-# This specifies the log level that warnings and clear notifications
498
-# will be sent at. If unset, it defaults to 'warning'.
499
-SYSLOG_LEVEL=''
500
-
501
-# This specifies the log level that critical alerts will be sent at.
502
-# If unset, it defaults to 'err'
503
-SYSLOG_CRITICAL_LEVEL=''
504
-
505
-# This controls which facility is used for logging. Defaults to local6.
493
+# This controls which facility is used by defalt for logging. Defaults
494
+# to local6.
495
SYSLOG_FACILITY=''
496
497
+# If a role's recipients are not configured, use the following.
498
+# (empty = do not send a notification for unconfigured roles)
499
+#
500
+# The recipient format for syslog uses the following format:
501
+# [[facility.level][@host[:port]]/]prefix
502
+#
503
+# `prefix` gets appended to the front of all log messages generated for
504
+# that recipient. The prefix is mandatory.
505
+# 'host' and 'port' can be used to specify a remote syslog server to
506
+# send messages to. Leave these out if you want messages to be delivered
507
+# locally. 'host' can be either a hostname or an IP address.
508
+# IPv6 addresses must have square around them.
509
+# 'facility' and 'level' are used to override the default logging facility
510
+# set above and the log level. If one is specified, both must be present.
511
+#
512
+# For example, to send messages with a 'netdata' prefix to a syslog
513
+# daemon listening on port 514 on 'loghost' using the daemon facility and
514
+# notice log level:
515
+# DEFAULT_RECIPIENT_SYSLOG='daemon.notice@loghost:514/netdata'
516
+#
517
+DEFAULT_RECIPIENT_SYSLOG="netdata"
518
519
#------------------------------------------------------------------------------
520
# custom notifications
@@ -603,6 +613,8 @@ role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"
613
614
role_recipients_irc[sysadmin]="${DEFAULT_RECIPIENT_IRC}"
615
616
+role_recipients_syslog[sysadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
617
+
618
role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
619
620
# -----------------------------------------------------------------------------
@@ -636,6 +648,8 @@ role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
648
649
role_recipients_irc[domainadmin]="${DEFAULT_RECIPIENT_IRC}"
650
651
+role_recipients_syslog[domainadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
652
+
653
role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
654
655
# -----------------------------------------------------------------------------
@@ -670,6 +684,8 @@ role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
684
685
role_recipients_irc[dba]="${DEFAULT_RECIPIENT_IRC}"
686
687
+role_recipients_syslog[dba]="${DEFAULT_RECIPIENT_SYSLOG}"
688
+
689
role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"
690
691
# -----------------------------------------------------------------------------
@@ -704,6 +720,8 @@ role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
720
721
role_recipients_irc[webmaster]="${DEFAULT_RECIPIENT_IRC}"
722
723
+role_recipients_syslog[webmaster]="${DEFAULT_RECIPIENT_SYSLOG}"
724
+
725
role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"
726
727
# -----------------------------------------------------------------------------
@@ -738,6 +756,8 @@ role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"
756
757
role_recipients_irc[proxyadmin]="${DEFAULT_RECIPIENT_IRC}"
758
759
+role_recipients_syslog[proxyadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
760
+
761
role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
762
763
# -----------------------------------------------------------------------------
@@ -770,5 +790,6 @@ role_recipients_kavenegar[sitemgr]="${DEFAULT_RECIPIENT_KAVENEGAR}"
790
791
role_recipients_pd[sitemgr]="${DEFAULT_RECIPIENT_PD}"
792
773
-role_recipients_custom[sitemgr]="${DEFAULT_RECIPIENT_CUSTOM}"
793
+role_recipients_syslog[sitemgr]="${DEFAULT_RECIPIENT_SYSLOG}"
794
795
+role_recipients_custom[sitemgr]="${DEFAULT_RECIPIENT_CUSTOM}"
plugins.d/alarm-notify.sh
+74
-7
@@ -313,9 +313,8 @@ DEFAULT_RECIPIENT_PD=
313
declare -A role_recipients_pd=()
314
315
# syslog configs
316
-SYSLOG_LEVEL=
317
-SYSLOG_CRITICAL_LEVEL=
316
SYSLOG_FACILITY=
317
+declare -A role_recipients_syslog=()
318
319
# custom configs
320
DEFAULT_RECIPIENT_CUSTOM=
@@ -426,6 +425,7 @@ declare -A arr_custom=()
425
declare -A arr_messagebird=()
426
declare -A arr_kavenegar=()
427
declare -A arr_irc=()
428
+declare -A arr_syslog=()
429
430
# netdata may call us with multiple roles, and roles may have multiple but
431
# overlapping recipients - so, here we find the unique recipients.
@@ -538,7 +538,7 @@ do
538
do
539
[ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1"
540
done
541
-
541
+
542
# irc
543
a="${role_recipients_irc[${x}]}"
544
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_IRC}"
@@ -547,6 +547,14 @@ do
547
[ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1"
548
done
549
550
+ # syslog
551
+ a="${role_recipients_syslog[${x}]}"
552
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}"
553
+ for r in ${a//,/ }
554
+ do
555
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality syslog "${r}" && arr_syslog[${r/|*/}]="1"
556
+ done
557
+
558
# custom
559
a="${role_recipients_custom[${x}]}"
560
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}"
@@ -622,6 +630,10 @@ done
630
to_irc="${!arr_irc[*]}"
631
[ -z "${to_irc}" ] && SEND_IRC="NO"
632
633
+# build the list of syslog recipients (facilities, servers, and prefixes)
634
+to_syslog="${!arr_syslog[*]}"
635
+[ -z "${to_syslog}" ] && SEND_SYSLOG="NO"
636
+
637
# -----------------------------------------------------------------------------
638
# verify the delivery methods supported
639
@@ -1533,15 +1545,70 @@ send_irc() {
1545
# syslog sender
1546
1547
send_syslog() {
1536
- local message="${1}" level=${SYSLOG_LEVEL:-"warning"} crit_level=${SYSLOG_CRITICAL_LEVEL:-"err"} facility=${SYSLOG_FACILITY:-"local6"}
1548
+ local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
1549
+ local prioiryt='' message='' host='' port='' prefix=''
1550
+ local temp1='' temp2=''
1551
1552
[ "${SEND_SYSLOG}" -eq "YES" ] || return 1
1553
1554
if [ "${status}" -eq "CRITICAL" ] ; then
1541
- level=${crit_level}
1555
+ level='crit'
1556
+ elif [ "${status}" -eq "WARNING" ] ; then
1557
+ level='warning'
1558
fi
1559
1544
- ${logger} -p ${facility}.${level} ${logger_options} "${message}"
1560
+ for target in ${targets} ; do
1561
+ priority="${facility}.${level}"
1562
+ message=''
1563
+ host=''
1564
+ port=''
1565
+ prefix=''
1566
+ temp1=''
1567
+ temp2=''
1568
+
1569
+ prefix=$(echo ${target} | cut -d '/' -f 2)
1570
+ temp1=$(echo ${target} | cut -d '/' -f 1)
1571
+
1572
+ if [ ${prefix} != ${temp1} ] ; then
1573
+ if (echo ${temp1} | grep -q '@' ) ; then
1574
+ temp2=$(echo ${temp1} | cut -d '@' -f 1)
1575
+ host=$(echo ${temp1} | cut -d '@' -f 2)
1576
+
1577
+ if [ ${temp2} != ${host} ] ; then
1578
+ priority=${temp2}
1579
+ fi
1580
+
1581
+ port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
1582
+
1583
+ if ( echo ${host} | grep -E -q '\[.*\]' ) ; then
1584
+ if ( echo ${port} | grep -q ']' ) ; then
1585
+ port=''
1586
+ else
1587
+ host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
1588
+ fi
1589
+ else
1590
+ if [ ${port} = ${host} ] ; then
1591
+ port=''
1592
+ else
1593
+ host=$(echo ${host} | cut -d ':' -f 1)
1594
+ fi
1595
+ fi
1596
+ else
1597
+ priority=${temp1}
1598
+ fi
1599
+ fi
1600
+
1601
+ message="${prefix} ${status} on ${host} at ${date}: ${value_string}"
1602
+
1603
+ if [ ${host} ] ; then
1604
+ logger_options="${logger_options} -n ${host}"
1605
+ if [ ${port} ] ; then
1606
+ logger_options="${logger_options} -P ${port}"
1607
+ fi
1608
+ fi
1609
+
1610
+ ${logger} -p ${priority} ${logger_options} "${message}"
1611
+ done
1612
1613
return $?
1614
}
@@ -1807,7 +1874,7 @@ SENT_HIPCHAT=$?
1874
# -----------------------------------------------------------------------------
1875
# send the syslog message
1876
1810
-send_syslog "Netdata notification on ${host} at ${when}: ${status}, ${alarm}"
1877
+send_syslog ${to_syslog}
1878
1879
SENT_SYSLOG=$?
1880