properly cleanup alarms severity filtering statuses; fixes #2618
Costa Tsaousis (ktsaou) committed
Aug 22, 2017 at 19:09 UTC
277441e40d05a28d6cd74ff81e91e5c24b86268a
1 file changed
+59
-40
plugins.d/alarm-notify.sh
+59
-40
@@ -43,7 +43,7 @@ then
43
44
id=1
45
last="CLEAR"
46
- for x in "CRITICAL" "WARNING" "CLEAR"
46
+ for x in "WARNING" "CRITICAL" "CLEAR"
47
do
48
echo >&2
49
echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
@@ -152,33 +152,33 @@ custom_sender() {
152
# -----------------------------------------------------------------------------
153
# defaults to allow running this script by hand
154
155
-[ -z "${NETDATA_CONFIG_DIR}" ] && NETDATA_CONFIG_DIR="$(dirname "${0}")/../../../../etc/netdata"
156
-[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="$(dirname "${0}")/../../../../var/cache/netdata"
155
+[ -z "${NETDATA_CONFIG_DIR}" ] && NETDATA_CONFIG_DIR="$(dirname "${0}")/../../../../etc/netdata"
156
+[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="$(dirname "${0}")/../../../../var/cache/netdata"
157
[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
158
159
# -----------------------------------------------------------------------------
160
# parse command line parameters
161
162
-roles="${1}" # the roles that should be notified for this event
163
-host="${2}" # the host generated this event
164
-unique_id="${3}" # the unique id of this event
165
-alarm_id="${4}" # the unique id of the alarm that generated this event
166
-event_id="${5}" # the incremental id of the event, for this alarm id
167
-when="${6}" # the timestamp this event occurred
168
-name="${7}" # the name of the alarm, as given in netdata health.d entries
169
-chart="${8}" # the name of the chart (type.id)
170
-family="${9}" # the family of the chart
171
-status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
172
-old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
173
-value="${12}" # the current value of the alarm
174
-old_value="${13}" # the previous value of the alarm
175
-src="${14}" # the line number and file the alarm has been configured
176
-duration="${15}" # the duration in seconds of the previous alarm state
162
+roles="${1}" # the roles that should be notified for this event
163
+host="${2}" # the host generated this event
164
+unique_id="${3}" # the unique id of this event
165
+alarm_id="${4}" # the unique id of the alarm that generated this event
166
+event_id="${5}" # the incremental id of the event, for this alarm id
167
+when="${6}" # the timestamp this event occurred
168
+name="${7}" # the name of the alarm, as given in netdata health.d entries
169
+chart="${8}" # the name of the chart (type.id)
170
+family="${9}" # the family of the chart
171
+status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
172
+old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
173
+value="${12}" # the current value of the alarm
174
+old_value="${13}" # the previous value of the alarm
175
+src="${14}" # the line number and file the alarm has been configured
176
+duration="${15}" # the duration in seconds of the previous alarm state
177
non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
178
-units="${17}" # the units of the value
179
-info="${18}" # a short description of the alarm
180
-value_string="${19}" # friendly value (with units)
181
-old_value_string="${20}" # friendly old value (with units)
178
+units="${17}" # the units of the value
179
+info="${18}" # a short description of the alarm
180
+value_string="${19}" # friendly value (with units)
181
+old_value_string="${20}" # friendly old value (with units)
182
183
# -----------------------------------------------------------------------------
184
# find a suitable hostname to use, if netdata did not supply a hostname
@@ -281,6 +281,7 @@ KAFKA_SENDER_IP=
281
282
# pagerduty.com configs
283
PD_SERVICE_KEY=
284
+DEFAULT_RECIPIENT_PD=
285
declare -A role_recipients_pd=()
286
287
# custom configs
@@ -325,28 +326,45 @@ filter_recipient_by_criticality() {
326
327
# the severity is invalid
328
s="${s^^}"
328
- [ "${s}" != "CRITICAL" ] && return 0
329
-
330
- # the new or the old status matches the severity
331
- if [ "${s}" = "${status}" -o "${s}" = "${old_status}" ]
332
- then
333
- [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
334
- mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
335
-
336
- # we need to keep track of the notifications we sent
337
- # so that the same user will receive the recovery
338
- # even if old_status does not match the required severity
339
- touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
329
+ if [ "${s}" != "CRITICAL" ]
330
+ then
331
+ error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
332
return 0
333
fi
334
343
- # it is a cleared alarm we have sent notification for
344
- if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
345
- then
346
- rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
347
- return 0
348
- fi
335
+ # create the status tracking directory for this user
336
+ [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
337
+ mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
338
+
339
+ case "${status}" in
340
+ CRITICAL)
341
+ # make sure he will get future notifications for this alarm too
342
+ touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
343
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
344
+ return 0
345
+ ;;
346
+
347
+ WARNING)
348
+ if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
349
+ then
350
+ # we do not remove the file, so that he will get future notifications of this alarm
351
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
352
+ return 0
353
+ fi
354
+ ;;
355
+
356
+ *)
357
+ if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
358
+ then
359
+ # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
360
+ rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
361
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
362
+ return 0
363
+ fi
364
+ ;;
365
+ esac
366
367
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
368
return 1
369
}
370
@@ -363,6 +381,7 @@ declare -A arr_telegram=()
381
declare -A arr_pd=()
382
declare -A arr_email=()
383
declare -A arr_custom=()
384
+declare -A arr_messagebird=()
385
386
# netdata may call us with multiple roles, and roles may have multiple but
387
# overlapping recipients - so, here we find the unique recipients.