Fix clear notification missing (#6638)
* alarm_clear: Mapping In this PR I mapped all the necessary steps to discover the solution for the ISSUE 6581 * alarm_clear: Documentation and fixes This commit fixes the problem that were present in Netdata and it also updates the documentation of the functions and Netdata. * alarm_clear: shell script The original implementation did not have a shell script, here I begin to fix this * alarm_clear: shell script It is necessay to verify why make is not producing the same binary than cmake and finish the changes in the script * alarm_clear: adjust in health.c I rewrote the health.c to be more readable, but I discovered the problem I had in the last few hours were due kernel update * alarm_clear: script changes In this commit I am bringing the final version of the script that test the alarm repetition * alarm_clear: script fix and remove comments IN this commit I am fixing the shellcheck errors and removing some debug messages that were present in the code while I was developing * alarm_clear: Format The health.c had wrong tabulation, this PR brings back the pattern of space as tab for this file * alarm_clear: Script The script was using killlall that is not more present in all Linux distribution this commit removes this and bring the new way to stop Netdata * alarm_clear: return to previous tabulation I am bringing back the old tabulation here and I will create a new PR exclusively for this * alarm_clear: Remove comments I am removing comments from this PR to keep the focus in the major problem * alarm_clear: Remove comments 2 I forgot one comment * alarm_clear: New variable I am appending a new variable in the check before the rebase, because the health.c changed in other file has a direct relationship with what I did here until now * alarm_clear: Fix clear repetition With this last commit, I am bringing a new way to raise the clear alarm, but it is not repeating more with this fix, it displayed one time when it is cleaned and it will display the message again, if and only if, the alarm was raised.
thiagoftsm committed
Sep 3, 2019 at 18:30 UTC
0798426c7dbfc290b1f6f6e397063a09ee8a3447
12 files changed
+361
-6
.gitignore
+1
@@ -166,6 +166,7 @@ gmon.txt
166
sitespeed-result/
167
tests/acls/acl.sh
168
tests/urls/request.sh
169
+tests/alarm_repetition/alarm.sh
170
171
# tests and temp files
172
python.d/python-modules-installer.sh
database/rrdcalc.h
+1
@@ -27,6 +27,7 @@
27
#define RRDCALC_FLAG_RUNNABLE 0x00000040
28
#define RRDCALC_FLAG_DISABLED 0x00000080
29
#define RRDCALC_FLAG_SILENCED 0x00000100
30
+#define RRDCALC_FLAG_RUN_ONCE 0x00000200
31
#define RRDCALC_FLAG_NO_CLEAR_NOTIFICATION 0x80000000
32
33
health/README.md
+2
-1
@@ -347,7 +347,8 @@ delay: [[[up U] [down D] multiplier M] max X]
347
348
#### Alarm line `repeat`
349
350
-Defines the interval between repeating notifications for the alarms in CRITICAL or WARNING mode. This will override the default interval settings inherited from health settings in `netdata.conf`. The default settings for repeating notifications are `default repeat warning = DURATION` and `default repeat critical = DURATION` which can be found in health stock configuration.
350
+Defines the interval between repeating notifications for the alarms in CRITICAL or WARNING mode. This will override the default interval settings inherited from health settings in `netdata.conf`. The default settings for repeating notifications are `default repeat warning = DURATION` and `default repeat critical = DURATION` which can be found in health stock configuration, when one of these interval is bigger than 0, Netdata will activate the repeat notification for `CRITICAL`, `CLEAR` and `WARNING` messages.
351
+`
352
353
Format:
354
health/health.c
+22
-5
@@ -216,9 +216,11 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
216
// we have not executed this alarm notification in the past
217
// so, don't send CLEAR notifications
218
if(unlikely(ae->new_status == RRDCALC_STATUS_CLEAR)) {
219
- debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s"
220
- , ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
221
- goto done;
219
+ if((!(ae->flags & HEALTH_ENTRY_RUN_ONCE)) || (ae->flags & HEALTH_ENTRY_RUN_ONCE && ae->old_status < RRDCALC_STATUS_RAISED) ) {
220
+ debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s"
221
+ , ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
222
+ goto done;
223
+ }
224
}
225
}
226
}
@@ -872,10 +874,21 @@ void *health_main(void *ptr) {
874
for(rc = host->alarms; rc ; rc = rc->next) {
875
int repeat_every = 0;
876
if(unlikely(rrdcalc_isrepeating(rc))) {
875
- if(unlikely(rc->status == RRDCALC_STATUS_WARNING))
877
+ if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
878
+ rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUN_ONCE;
879
repeat_every = rc->warn_repeat_every;
877
- else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL))
880
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
881
+ rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUN_ONCE;
882
repeat_every = rc->crit_repeat_every;
883
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
884
+ if(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUN_ONCE)) {
885
+ if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
886
+ repeat_every = rc->crit_repeat_every;
887
+ } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
888
+ repeat_every = rc->warn_repeat_every;
889
+ }
890
+ }
891
+ }
892
}
893
if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
894
rc->last_repeat = now;
@@ -890,6 +903,10 @@ void *health_main(void *ptr) {
903
)
904
);
905
ae->last_repeat = rc->last_repeat;
906
+ if (!(rc->rrdcalc_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
907
+ ae->flags |= HEALTH_ENTRY_RUN_ONCE;
908
+ }
909
+ rc->rrdcalc_flags |= RRDCALC_FLAG_RUN_ONCE;
910
health_process_notifications(host, ae);
911
debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
912
health_alarm_log_free_one_nochecks_nounlink(ae);
health/health.h
+1
@@ -23,6 +23,7 @@ extern unsigned int default_health_enabled;
23
#define HEALTH_ENTRY_FLAG_EXEC_RUN 0x00000004
24
#define HEALTH_ENTRY_FLAG_EXEC_FAILED 0x00000008
25
#define HEALTH_ENTRY_FLAG_SILENCED 0x00000010
26
+#define HEALTH_ENTRY_RUN_ONCE 0x00000020
27
28
#define HEALTH_ENTRY_FLAG_SAVED 0x10000000
29
#define HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION 0x80000000
health/health_json.c
+4
@@ -43,6 +43,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R
43
"\t\t\"updates_id\": %u,\n"
44
"\t\t\"value_string\": \"%s\",\n"
45
"\t\t\"old_value_string\": \"%s\",\n"
46
+ "\t\t\"last_repeat\": \"%lu\",\n"
47
"\t\t\"silenced\": \"%s\",\n"
48
, host->hostname
49
, ae->unique_id
@@ -71,6 +72,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R
72
, ae->updates_id
73
, ae->new_value_string
74
, ae->old_value_string
75
+ , (unsigned long)ae->last_repeat
76
, (ae->flags & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false"
77
);
78
@@ -143,6 +145,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
145
"\t\t\t\"warn_repeat_every\": \"%u\",\n"
146
"\t\t\t\"crit_repeat_every\": \"%u\",\n"
147
"\t\t\t\"value_string\": \"%s\",\n"
148
+ "\t\t\t\"last_repeat\": \"%lu\",\n"
149
, rc->chart, rc->name
150
, (unsigned long)rc->id
151
, rc->name
@@ -170,6 +173,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
173
, rc->warn_repeat_every
174
, rc->crit_repeat_every
175
, value_string
176
+ , (unsigned long)rc->last_repeat
177
);
178
179
if(unlikely(rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)) {
tests/Makefile.am
+3
@@ -7,6 +7,7 @@ CLEANFILES = \
7
health_mgmtapi/health-cmdapi-test.sh \
8
acls/acl.sh \
9
urls/request.sh \
10
+ alarm_repetition/alarm.sh \
11
$(NULL)
12
13
include $(top_srcdir)/build/subst.inc
@@ -26,12 +27,14 @@ dist_noinst_DATA = \
27
health_mgmtapi/health-cmdapi-test.sh.in \
28
acls/acl.sh.in \
29
urls/request.sh.in \
30
+ alarm_repetition/alarm.sh.in \
31
$(NULL)
32
33
dist_plugins_SCRIPTS = \
34
health_mgmtapi/health-cmdapi-test.sh \
35
acls/acl.sh \
36
urls/request.sh \
37
+ alarm_repetition/alarm.sh \
38
$(NULL)
39
40
dist_noinst_SCRIPTS = \
tests/alarm_repetition/alarm.sh.in
new
+86
@@ -0,0 +1,86 @@
1
+#!/bin/bash
2
+
3
+#The health directory to put the alarms
4
+HEALTHDIR="@configdir_POST@/health.d/"
5
+
6
+#output directory
7
+OUTDIR="workdir/"
8
+
9
+#url to do download
10
+MURL="http://localhost:19999/api/v1/alarms?active"
11
+
12
+#error messages
13
+RED='\033[0;31m'
14
+GREEN='\033[0;32m'
15
+NOCOLOR='\033[0m'
16
+
17
+MYCDIR="$(pwd)"
18
+CONFFILE="$MYCDIR/netdata.conf"
19
+
20
+change_alarm_file() {
21
+ if [ -f "$1" ]; then
22
+ rm "$1"
23
+ fi
24
+
25
+ #copy keeping the permissions
26
+ cp -a "$2" "$3"
27
+}
28
+
29
+netdata_test_download() {
30
+ OPT="-e"
31
+ if [ "$3" == "I" ]; then
32
+ OPT="-v"
33
+ fi
34
+
35
+ grep "HTTP/1.1 200 OK" "$1" 2>/dev/null 1>/dev/null
36
+ TEST="$?"
37
+ if [ "$TEST" -ne "0" ]; then
38
+ echo -e "${RED} Error to get the alarms"
39
+ killall netdata
40
+ exit 1
41
+ fi
42
+
43
+ COUNT=$(grep -w "\"last_repeat\":" "$2" | grep -c "$OPT" "\"0\"")
44
+ if [ "$COUNT" -eq "0" ]; then
45
+ echo -e "${RED} Netdata gave an unexpected result when alarm repetition is $4"
46
+ killall netdata
47
+ exit 1
48
+ fi
49
+
50
+ echo -e "${GREEN} I got the expected result"
51
+}
52
+
53
+get_the_logs() {
54
+ curl -v -k --create-dirs -o "$OUTDIR/$1.out" "$MURL" 2> "$OUTDIR/$1.err"
55
+ netdata_test_download "$OUTDIR/$1.err" "$OUTDIR/$1.out" "$2" "$3"
56
+}
57
+
58
+process_data() {
59
+ SEC=120
60
+ netdata -c "$CONFFILE" -D &
61
+ NETDATAPID=$!
62
+ echo -e "${NOCOLOR}Sleeping during $SEC seconds to create alarm entries"
63
+ sleep $SEC
64
+ get_the_logs "$1" "$2" "$3"
65
+ kill $NETDATAPID
66
+}
67
+
68
+mkdir "$OUTDIR"
69
+CREATEDIR="$?"
70
+if [ "$CREATEDIR" -ne "0" ]; then
71
+ echo -e "${RED}Cannot create the output directory, it already exists. The test will overwrite previous results."
72
+fi
73
+
74
+change_alarm_file "./0" "ram_without_repetition.conf" "$HEALTHDIR/ram.conf"
75
+cp -a netdata.conf_without_repetition netdata.conf
76
+process_data "ram_without" "K" "not activated."
77
+rm netdata.conf
78
+
79
+change_alarm_file "$HEALTHDIR/ram.conf" "ram_with_repetition.conf" "$HEALTHDIR/ram.conf"
80
+cp -a netdata.conf_with_repetition netdata.conf
81
+process_data "ram_with" "I" "activated."
82
+rm netdata.conf
83
+
84
+echo -e "${GREEN} all the tests were sucessful"
85
+rm "$HEALTHDIR/ram.conf"
86
+rm -rf $OUTDIR
tests/alarm_repetition/netdata.conf_with_repetition
new
+57
@@ -0,0 +1,57 @@
1
+# netdata configuration
2
+#
3
+# You can download the latest version of this file, using:
4
+#
5
+# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
6
+# or
7
+# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
8
+#
9
+# You can uncomment and change any of the options below.
10
+# The value shown in the commented settings, is the default value.
11
+#
12
+
13
+# global netdata configuration
14
+
15
+[global]
16
+ #run as user = netdata
17
+
18
+[web]
19
+ #ssl key = /etc/netdata/ssl/key2048.pem
20
+ #ssl certificate = /etc/netdata/ssl/cert2048.pem
21
+ mode = static-threaded
22
+ # listen backlog = 4096
23
+ default port = 19999
24
+ #bind to = *=dashboard|registry|streaming|netdata.conf|badges|management *:20000=dashboard|registry|streaming|netdata.conf|badges|management^SSL=optional *:20001=dashboard|registry|streaming|netdata.conf|badges|management^SSL=force unix:/tmp/netdata/netdata.sock
25
+ # web files owner = netdata
26
+ # web files group = netdata
27
+ #accept a streaming request every seconds = 2
28
+
29
+[plugins]
30
+ proc = yes
31
+ diskspace = no
32
+ cgroups = no
33
+ tc = no
34
+ idlejitter = no
35
+ enable running new plugins = no
36
+ check for new plugins every = 60
37
+ go.d = no
38
+ node.d = no
39
+ charts.d = no
40
+ nfacct = no
41
+ python.d = no
42
+ apps = no
43
+ fping = no
44
+ cups = no
45
+
46
+[health]
47
+ enabled = yes
48
+ in memory max health log entries = 1000
49
+ default repeat warning = 4s
50
+ default repeat critical = 2s
51
+
52
+[registry]
53
+ enabled = yes
54
+ allow from = *
55
+
56
+[cloud]
57
+ cloud base url = https://netdata.cloud
tests/alarm_repetition/netdata.conf_without_repetition
new
+57
@@ -0,0 +1,57 @@
1
+# netdata configuration
2
+#
3
+# You can download the latest version of this file, using:
4
+#
5
+# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
6
+# or
7
+# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
8
+#
9
+# You can uncomment and change any of the options below.
10
+# The value shown in the commented settings, is the default value.
11
+#
12
+
13
+# global netdata configuration
14
+
15
+[global]
16
+ #run as user = netdata
17
+
18
+[web]
19
+ #ssl key = /etc/netdata/ssl/key2048.pem
20
+ #ssl certificate = /etc/netdata/ssl/cert2048.pem
21
+ mode = static-threaded
22
+ # listen backlog = 4096
23
+ default port = 19999
24
+ #bind to = *=dashboard|registry|streaming|netdata.conf|badges|management *:20000=dashboard|registry|streaming|netdata.conf|badges|management^SSL=optional *:20001=dashboard|registry|streaming|netdata.conf|badges|management^SSL=force unix:/tmp/netdata/netdata.sock
25
+ # web files owner = netdata
26
+ # web files group = netdata
27
+ #accept a streaming request every seconds = 2
28
+
29
+[plugins]
30
+ proc = yes
31
+ diskspace = no
32
+ cgroups = no
33
+ tc = no
34
+ idlejitter = no
35
+ enable running new plugins = no
36
+ check for new plugins every = 60
37
+ go.d = no
38
+ node.d = no
39
+ charts.d = no
40
+ nfacct = no
41
+ python.d = no
42
+ apps = no
43
+ fping = no
44
+ cups = no
45
+
46
+[health]
47
+ enabled = yes
48
+ in memory max health log entries = 1000
49
+ #default repeat warning = 4s
50
+ #default repeat critical = 2s
51
+
52
+[registry]
53
+ enabled = yes
54
+ allow from = *
55
+
56
+[cloud]
57
+ cloud base url = https://netdata.cloud
tests/alarm_repetition/ram_with_repetition.conf
new
+64
@@ -0,0 +1,64 @@
1
+# you can disable an alarm notification by setting the 'to' line to: silent
2
+
3
+ alarm: used_ram_to_ignore
4
+ on: system.ram
5
+ os: linux freebsd
6
+ hosts: *
7
+ calc: ($zfs.arc_size.arcsz = nan)?(0):($zfs.arc_size.arcsz)
8
+ every: 10s
9
+ info: the amount of memory that is reported as used, but it is actually capable for resizing itself based on the system needs (eg. ZFS ARC)
10
+
11
+ alarm: ram_in_use
12
+ on: system.ram
13
+ os: linux
14
+ hosts: *
15
+# calc: $used * 100 / ($used + $cached + $free)
16
+ calc: ($used - $used_ram_to_ignore) * 100 / ($used - $used_ram_to_ignore + $cached + $free)
17
+ units: %
18
+ every: 1s
19
+ warn: $this > 1
20
+ crit: $this > 5
21
+ delay: down 15m multiplier 1.5 max 1h
22
+ info: system RAM used
23
+ to: sysadmin #alarms
24
+ repeat: warning 30s critical 60s
25
+
26
+ alarm: ram_available
27
+ on: mem.available
28
+ os: linux
29
+ hosts: *
30
+ calc: ($avail + $used_ram_to_ignore) * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers)
31
+ units: %
32
+ every: 10s
33
+ warn: $this < (($status >= $WARNING) ? (15) : (10))
34
+ crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
35
+ delay: down 15m multiplier 1.5 max 1h
36
+ info: estimated amount of RAM available for userspace processes, without causing swapping
37
+ to: sysadmin #alarms
38
+
39
+## FreeBSD
40
+alarm: ram_in_use
41
+ on: system.ram
42
+ os: freebsd
43
+hosts: *
44
+ calc: ($active + $wired + $laundry + $buffers - $used_ram_to_ignore) * 100 / ($active + $wired + $laundry + $buffers - $used_ram_to_ignore + $cache + $free + $inactive)
45
+units: %
46
+every: 10s
47
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
48
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
49
+delay: down 15m multiplier 1.5 max 1h
50
+ info: system RAM usage
51
+ to: sysadmin #alarms
52
+
53
+ alarm: ram_available
54
+ on: system.ram
55
+ os: freebsd
56
+ hosts: *
57
+ calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $laundry + $buffers)
58
+ units: %
59
+ every: 10s
60
+ warn: $this < (($status >= $WARNING) ? (15) : (10))
61
+ crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
62
+ delay: down 15m multiplier 1.5 max 1h
63
+ info: estimated amount of RAM available for userspace processes, without causing swapping
64
+ to: sysadmin #alarms
tests/alarm_repetition/ram_without_repetition.conf
new
+63
@@ -0,0 +1,63 @@
1
+# you can disable an alarm notification by setting the 'to' line to: silent
2
+
3
+ alarm: used_ram_to_ignore
4
+ on: system.ram
5
+ os: linux freebsd
6
+ hosts: *
7
+ calc: ($zfs.arc_size.arcsz = nan)?(0):($zfs.arc_size.arcsz)
8
+ every: 10s
9
+ info: the amount of memory that is reported as used, but it is actually capable for resizing itself based on the system needs (eg. ZFS ARC)
10
+
11
+ alarm: ram_in_use
12
+ on: system.ram
13
+ os: linux
14
+ hosts: *
15
+# calc: $used * 100 / ($used + $cached + $free)
16
+ calc: ($used - $used_ram_to_ignore) * 100 / ($used - $used_ram_to_ignore + $cached + $free)
17
+ units: %
18
+ every: 1s
19
+ warn: $this > 1
20
+ crit: $this > 5
21
+ delay: down 15m multiplier 1.5 max 1h
22
+ info: system RAM used
23
+ to: sysadmin #alarms
24
+
25
+ alarm: ram_available
26
+ on: mem.available
27
+ os: linux
28
+ hosts: *
29
+ calc: ($avail + $used_ram_to_ignore) * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers)
30
+ units: %
31
+ every: 10s
32
+ warn: $this < (($status >= $WARNING) ? (15) : (10))
33
+ crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
34
+ delay: down 15m multiplier 1.5 max 1h
35
+ info: estimated amount of RAM available for userspace processes, without causing swapping
36
+ to: sysadmin #alarms
37
+
38
+## FreeBSD
39
+alarm: ram_in_use
40
+ on: system.ram
41
+ os: freebsd
42
+hosts: *
43
+ calc: ($active + $wired + $laundry + $buffers - $used_ram_to_ignore) * 100 / ($active + $wired + $laundry + $buffers - $used_ram_to_ignore + $cache + $free + $inactive)
44
+units: %
45
+every: 10s
46
+ warn: $this > (($status >= $WARNING) ? (80) : (90))
47
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
48
+delay: down 15m multiplier 1.5 max 1h
49
+ info: system RAM usage
50
+ to: sysadmin #alarms
51
+
52
+ alarm: ram_available
53
+ on: system.ram
54
+ os: freebsd
55
+ hosts: *
56
+ calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $laundry + $buffers)
57
+ units: %
58
+ every: 10s
59
+ warn: $this < (($status >= $WARNING) ? (15) : (10))
60
+ crit: $this < (($status == $CRITICAL) ? (10) : ( 5))
61
+ delay: down 15m multiplier 1.5 max 1h
62
+ info: estimated amount of RAM available for userspace processes, without causing swapping
63
+ to: sysadmin #alarms