Add |nowarn and |noclear notification modifiers (#14330)
Martin Vobruba committed
Jan 26, 2023 at 21:18 UTC
c4f5524ea8279be492eb527a67242b408543382e
2 files changed
+99
-38
health/notifications/alarm-notify.sh.in
+89
-37
@@ -484,53 +484,105 @@ msteams_migration
484
# filter a recipient based on alarm event severity
485
486
filter_recipient_by_criticality() {
487
- local method="${1}" x="${2}" r s
488
- shift
489
-
490
- r="${x/|*/}" # the recipient
491
- s="${x/*|/}" # the severity required for notifying this recipient
487
+ local method="${1}" recipient_arg="${2}"
488
+ local tracking_dir tracking_file modifier modifiers recipient="${recipient_arg/|*/}"
489
+ local mod_critical=0 mod_noclear=0 mod_nowarn=0
490
491
# no severity filtering for this person
494
- [ "${r}" = "${s}" ] && return 0
492
+ [ "${recipient}" = "${recipient_arg}" ] && return 0
493
+
494
+ # find out which modifiers are set
495
+ modifiers="${recipient_arg#*|}"
496
+ modifiers="${modifiers//|/ }" # replace pipes with spaces
497
+ modifiers="${modifiers,,}" # lowercase
498
+ for modifier in ${modifiers}; do
499
+ case "${modifier}" in
500
+ critical) mod_critical=1 ;;
501
+ noclear) mod_noclear=1 ;;
502
+ nowarn) mod_nowarn=1 ;;
503
+
504
+ *)
505
+ error "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: invalid modifier '${modifier}'."
506
+ # invalid modifier, always send notification
507
+ return 0
508
+ ;;
509
+ esac
510
+ done
511
496
- # the severity is invalid
497
- s="${s^^}"
498
- if [ "${s}" != "CRITICAL" ]; then
499
- error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
500
- return 0
501
- fi
512
+ # set status tracking directory/file var
513
+ tracking_dir="${NETDATA_CACHE_DIR}/alarm-notify/${method}/${recipient}"
514
+ tracking_file="${tracking_dir}/${alarm_id}"
515
503
- # create the status tracking directory for this user
504
- [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
505
- mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
516
+ # create the status tracking directory for this user if "critical" modifier is set
517
+ [ "${mod_critical}" == "1" ] && [ ! -d "${tracking_dir}" ] && mkdir -p "${tracking_dir}"
518
519
case "${status}" in
508
- CRITICAL)
509
- # make sure he will get future notifications for this alarm too
510
- touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
511
- debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
512
- return 0
513
- ;;
514
-
515
- WARNING)
516
- if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
517
- # we do not remove the file, so that he will get future notifications of this alarm
518
- debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
519
- return 0
520
- fi
521
- ;;
520
+ CRITICAL)
521
+ # "critical" modifier set, create tracking file for future status changes
522
+ if [ "${mod_critical}" == "1" ]; then
523
+ touch "${tracking_file}"
524
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
525
+ return 0
526
+ fi
527
523
- *)
524
- if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
525
- # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
526
- rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
527
- debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
528
+ # always send CRITICAL notification
529
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is CRITICAL"
530
return 0
529
- fi
530
- ;;
531
+ ;;
532
+
533
+ WARNING)
534
+ # "nowarn" modifier set, block notification
535
+ if [ "${mod_nowarn}" == "1" ]; then
536
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: BLOCK: recipient should not receive this notification (nowarn modifier set)"
537
+ return 1
538
+ fi
539
+
540
+ # "critical" modifier not set, send notification
541
+ if [ "${mod_critical}" == "0" ]; then
542
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is WARNING"
543
+ return 0
544
+ fi
545
+
546
+ # "critical" modifier set, send notification if tracking file exists
547
+ if [ "${mod_critical}" == "1" ] && [ -f "${tracking_file}" ]; then
548
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
549
+ return 0
550
+ fi
551
+ ;;
552
+
553
+ CLEAR)
554
+ # remove tracking file
555
+ [ -f "${tracking_file}" ] && rm "${tracking_file}"
556
+
557
+ # "noclear" modifier set, block notification
558
+ if [ "${mod_noclear}" == "1" ]; then
559
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: BLOCK: recipient should not receive this notification (noclear modifier set)"
560
+ return 1
561
+ fi
562
+
563
+ # "critical" modifier not set, send notification
564
+ if [ "${mod_critical}" == "0" ]; then
565
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is CLEAR"
566
+ return 0
567
+ fi
568
+
569
+ # "critical" modifier set, send notification if tracking file exists
570
+ if [ "${mod_critical}" == "1" ] && [ -f "${tracking_file}" ]; then
571
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (no status change will be sent from now)"
572
+ return 0
573
+ fi
574
+ ;;
575
+
576
+ *)
577
+ # "critical" modifier set, send notification if tracking file exists
578
+ if [ "${mod_critical}" == "1" ] && [ -f "${tracking_file}" ]; then
579
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
580
+ return 0
581
+ fi
582
+ ;;
583
esac
584
533
- debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
585
+ debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: BLOCK: recipient should not receive this notification"
586
return 1
587
}
588
health/notifications/health_alarm_notify.conf
+10
-1
@@ -160,7 +160,11 @@ sendsms=""
160
# - pagerduty.com (pd) services
161
# - irc channels
162
#
163
-# You can append |critical to limit the notifications to be sent.
163
+# You can append modifiers to limit the notifications to be sent:
164
+# |critical - Send critical notifications and following status changes until
165
+# the alarm is cleared.
166
+# |nowarn - Do not send warning notifications.
167
+# |noclear - Do not send clear notifications.
168
#
169
# In these examples, the first recipient receives all the alarms
170
# while the second one receives only notifications for alarms that
@@ -182,6 +186,11 @@ sendsms=""
186
# irc : "<irc_channel_1> <irc_channel_2>|critical"
187
# hangouts : "alarms disasters|critical"
188
#
189
+# You can append multiple modifiers. In this example, recipient receives
190
+# notifications for critical alarms and following status changes except clear
191
+# notifications.
192
+# email : "user1@example.com|critical|noclear"
193
+#
194
# If a recipient is set to empty string, the default recipient of the given
195
# notification method (email, pushover, telegram, slack, alerta, etc) will be used.
196
# To disable a notification, use the recipient called: disabled