health: make alarms less sensitive (#10688)
Ilya Mashchenko committed
Mar 9, 2021 at 16:11 UTC
dc365288d11a010650a6bd3c8de14222cc25368d
13 files changed
+83
-160
health/Makefile.am
-1
@@ -29,7 +29,6 @@ dist_healthconfig_DATA = \
29
health.d/anomalies.conf \
30
health.d/apache.conf \
31
health.d/apcupsd.conf \
32
- health.d/apps_plugin.conf \
32
health.d/backend.conf \
33
health.d/bcache.conf \
34
health.d/beanstalkd.conf \
health/health.d/apps_plugin.conf
deleted
-15
@@ -1,15 +0,0 @@
1
-# you can disable an alarm notification by setting the 'to' line to: silent
2
-
3
-# disabled due to https://github.com/netdata/netdata/issues/10327
4
-#
5
-# alarm: used_file_descriptors
6
-# on: apps.files
7
-# hosts: *
8
-# calc: $fdperc
9
-# units: %
10
-# every: 5s
11
-# warn: $this > (($status >= $WARNING) ? (75) : (80))
12
-# crit: $this > (($status == $CRITICAL) ? (85) : (90))
13
-# delay: down 5m multiplier 1.5 max 1h
14
-# info: Peak percentage of file descriptors used
15
-# to: sysadmin
health/health.d/bcache.conf
+3
-4
@@ -1,13 +1,12 @@
1
2
template: bcache_cache_errors
3
on: disk.bcache_cache_read_races
4
- lookup: sum -10m unaligned absolute
4
+ lookup: sum -1m unaligned absolute
5
units: errors
6
every: 1m
7
warn: $this > 0
8
- crit: $this > ( ($status >= $CRITICAL) ? (0) : (10) )
9
- delay: down 1h multiplier 1.5 max 2h
10
- info: the number of times bcache had issues using the cache, during the last 10 mins (this usually means your SSD cache is failing)
8
+ delay: up 2m down 1h multiplier 1.5 max 2h
9
+ info: the number of times the data was being read from the cache, the bucket was reused and invalidated, during the last 10 mins (when this occurs the data is reread from the backing device)
10
to: sysadmin
11
12
template: bcache_cache_dirty
health/health.d/ceph.conf
+5
-6
@@ -2,12 +2,11 @@
2
3
template: cluster_space_usage
4
on: ceph.general_usage
5
- calc: $avail * 100 / ($avail + $used)
5
+ calc: $used * 100 / ($used + $avail)
6
units: %
7
- every: 10s
8
- warn: $this < 10
9
- crit: $this < 1
7
+ every: 1m
8
+ warn: $this > (($status >= $WARNING ) ? (85) : (90))
9
+ crit: $this > (($status == $CRITICAL) ? (90) : (98))
10
delay: down 5m multiplier 1.2 max 1h
11
- info: ceph disk usage is almost full
11
+ info: current ceph disk usage
12
to: sysadmin
13
-
health/health.d/disks.conf
+48
-48
@@ -49,35 +49,35 @@ families: !/dev !/dev/* !/run !/run/* *
49
# we will use it in the next template to find
50
# the hours remaining
51
52
-template: disk_fill_rate
53
- on: disk.space
54
- os: linux freebsd
55
- hosts: *
56
-families: *
57
- lookup: min -10m at -50m unaligned of avail
58
- calc: ($this - $avail) / (($now - $after) / 3600)
59
- every: 1m
60
- units: GB/hour
61
- info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
52
+# template: disk_fill_rate
53
+# on: disk.space
54
+# os: linux freebsd
55
+# hosts: *
56
+# families: *
57
+# lookup: min -10m at -50m unaligned of avail
58
+# calc: ($this - $avail) / (($now - $after) / 3600)
59
+# every: 1m
60
+# units: GB/hour
61
+# info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
62
63
64
# calculate the hours remaining
65
# if the disk continues to fill
66
# in this rate
67
68
-template: out_of_disk_space_time
69
- on: disk.space
70
- os: linux freebsd
71
- hosts: *
72
-families: *
73
- calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
74
- units: hours
75
- every: 10s
76
- warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
77
- crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
78
- delay: down 15m multiplier 1.2 max 1h
79
- info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
80
- to: sysadmin
68
+# template: out_of_disk_space_time
69
+# on: disk.space
70
+# os: linux freebsd
71
+# hosts: *
72
+# families: *
73
+# calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
74
+# units: hours
75
+# every: 10s
76
+# warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
77
+# crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
78
+# delay: down 15m multiplier 1.2 max 1h
79
+# info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
80
+# to: sysadmin
81
82
83
# -----------------------------------------------------------------------------
@@ -91,34 +91,34 @@ families: *
91
# we will use it in the next template to find
92
# the hours remaining
93
94
-template: disk_inode_rate
95
- on: disk.inodes
96
- os: linux freebsd
97
- hosts: *
98
-families: *
99
- lookup: min -10m at -50m unaligned of avail
100
- calc: ($this - $avail) / (($now - $after) / 3600)
101
- every: 1m
102
- units: inodes/hour
103
- info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour
94
+# template: disk_inode_rate
95
+# on: disk.inodes
96
+# os: linux freebsd
97
+# hosts: *
98
+# families: *
99
+# lookup: min -10m at -50m unaligned of avail
100
+# calc: ($this - $avail) / (($now - $after) / 3600)
101
+# every: 1m
102
+# units: inodes/hour
103
+# info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour
104
105
# calculate the hours remaining
106
# if the disk inodes are allocated
107
# in this rate
108
109
-template: out_of_disk_inodes_time
110
- on: disk.inodes
111
- os: linux freebsd
112
- hosts: *
113
-families: *
114
- calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf)
115
- units: hours
116
- every: 10s
117
- warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
118
- crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
119
- delay: down 15m multiplier 1.2 max 1h
120
- info: estimated time the disk will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
121
- to: sysadmin
109
+# template: out_of_disk_inodes_time
110
+# on: disk.inodes
111
+# os: linux freebsd
112
+# hosts: *
113
+# families: *
114
+# calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf)
115
+# units: hours
116
+# every: 10s
117
+# warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
118
+# crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
119
+# delay: down 15m multiplier 1.2 max 1h
120
+# info: estimated time the disk will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
121
+# to: sysadmin
122
123
124
# -----------------------------------------------------------------------------
@@ -142,7 +142,7 @@ families: *
142
crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
143
delay: down 15m multiplier 1.2 max 1h
144
info: the percentage of time the disk was busy, during the last 10 minutes
145
- to: sysadmin
145
+ to: silent
146
147
148
# raise an alarm if the disk backlog
@@ -164,4 +164,4 @@ families: *
164
crit: $this > $red * (($status == $CRITICAL) ? (0.7) : (1))
165
delay: down 15m multiplier 1.2 max 1h
166
info: average of the kernel estimated disk backlog, for the last 10 minutes
167
- to: sysadmin
167
+ to: silent
health/health.d/entropy.conf
+1
-1
@@ -7,7 +7,7 @@
7
on: system.entropy
8
os: linux
9
hosts: *
10
- lookup: min -10m unaligned
10
+ lookup: min -5m unaligned
11
units: entries
12
every: 5m
13
warn: $this < (($status >= $WARNING) ? (200) : (100))
health/health.d/load.conf
+5
-7
@@ -4,7 +4,7 @@
4
# Calculate the base trigger point for the load average alarms.
5
# This is the maximum number of CPU's in the system over the past 1
6
# minute, with a special case for a single CPU of setting the trigger at 2.
7
- alarm: load_trigger
7
+ alarm: load_cpu_number
8
on: system.load
9
os: linux
10
hosts: *
@@ -16,6 +16,7 @@
16
# Send alarms if the load average is unusually high.
17
# These intentionally _do not_ calculate the average over the sampled
18
# time period because the values being checked already are averages.
19
+
20
alarm: load_average_15
21
on: system.load
22
os: linux
@@ -23,8 +24,7 @@
24
lookup: max -1m unaligned of load15
25
units: load
26
every: 1m
26
- warn: $this > (($status >= $WARNING) ? (1.75 * $load_trigger) : (2 * $load_trigger))
27
- crit: $this > (($status == $CRITICAL) ? (3.5 * $load_trigger) : (4 * $load_trigger))
27
+ warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
28
delay: down 15m multiplier 1.5 max 1h
29
info: fifteen-minute load average
30
to: sysadmin
@@ -36,8 +36,7 @@
36
lookup: max -1m unaligned of load5
37
units: load
38
every: 1m
39
- warn: $this > (($status >= $WARNING) ? (3.5 * $load_trigger) : (4 * $load_trigger))
40
- crit: $this > (($status == $CRITICAL) ? (7 * $load_trigger) : (8 * $load_trigger))
39
+ warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
40
delay: down 15m multiplier 1.5 max 1h
41
info: five-minute load average
42
to: sysadmin
@@ -49,8 +48,7 @@
48
lookup: max -1m unaligned of load1
49
units: load
50
every: 1m
52
- warn: $this > (($status >= $WARNING) ? (7 * $load_trigger) : (8 * $load_trigger))
53
- crit: $this > (($status == $CRITICAL) ? (14 * $load_trigger) : (16 * $load_trigger))
51
+ warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
52
delay: down 15m multiplier 1.5 max 1h
53
info: one-minute load average
54
to: sysadmin
health/health.d/net.conf
+10
-20
@@ -23,9 +23,8 @@
23
calc: ($interface_speed > 0) ? ($this * 100 / ($interface_speed * 1000)) : ( nan )
24
units: %
25
every: 10s
26
- warn: $this > (($status >= $WARNING) ? (80) : (85))
27
- crit: $this > (($status == $CRITICAL) ? (85) : (90))
28
- delay: down 1m multiplier 1.5 max 1h
26
+ warn: $this > (($status >= $WARNING) ? (85) : (90))
27
+ delay: up 1m down 1m multiplier 1.5 max 1h
28
info: interface received bandwidth usage over net device speed max
29
to: sysadmin
30
@@ -38,9 +37,8 @@
37
calc: ($interface_speed > 0) ? ($this * 100 / ($interface_speed * 1000)) : ( nan )
38
units: %
39
every: 10s
41
- warn: $this > (($status >= $WARNING) ? (80) : (85))
42
- crit: $this > (($status == $CRITICAL) ? (85) : (90))
43
- delay: down 1m multiplier 1.5 max 1h
40
+ warn: $this > (($status >= $WARNING) ? (85) : (90))
41
+ delay: up 1m down 1m multiplier 1.5 max 1h
42
info: interface sent bandwidth usage over net device speed max
43
to: sysadmin
44
@@ -62,10 +60,7 @@ families: *
60
lookup: sum -10m unaligned absolute of inbound
61
units: packets
62
every: 1m
65
- warn: $this >= 5
66
- delay: down 1h multiplier 1.5 max 2h
63
info: interface inbound dropped packets in the last 10 minutes
68
- to: sysadmin
64
65
template: outbound_packets_dropped
66
on: net.drops
@@ -75,10 +70,7 @@ families: *
70
lookup: sum -10m unaligned absolute of outbound
71
units: packets
72
every: 1m
78
- warn: $this >= 5
79
- delay: down 1h multiplier 1.5 max 2h
73
info: interface outbound dropped packets in the last 10 minutes
81
- to: sysadmin
74
75
template: inbound_packets_dropped_ratio
76
on: net.packets
@@ -86,12 +78,11 @@ template: inbound_packets_dropped_ratio
78
hosts: *
79
families: *
80
lookup: sum -10m unaligned absolute of received
89
- calc: (($inbound_packets_dropped != nan AND $this > 0) ? ($inbound_packets_dropped * 100 / $this) : (0))
81
+ calc: (($inbound_packets_dropped != nan AND $this > 1000) ? ($inbound_packets_dropped * 100 / $this) : (0))
82
units: %
83
every: 1m
92
- warn: $this >= 0.1
93
- crit: $this >= 2
94
- delay: down 1h multiplier 1.5 max 2h
84
+ warn: $this >= 2
85
+ delay: up 1m down 1h multiplier 1.5 max 2h
86
info: the ratio of inbound dropped packets vs the total number of received packets of the network interface, during the last 10 minutes
87
to: sysadmin
88
@@ -101,12 +92,11 @@ template: outbound_packets_dropped_ratio
92
hosts: *
93
families: *
94
lookup: sum -10m unaligned absolute of sent
104
- calc: (($outbound_packets_dropped != nan AND $this > 0) ? ($outbound_packets_dropped * 100 / $this) : (0))
95
+ calc: (($outbound_packets_dropped != nan AND $this > 1000) ? ($outbound_packets_dropped * 100 / $this) : (0))
96
units: %
97
every: 1m
107
- warn: $this >= 0.1
108
- crit: $this >= 2
109
- delay: down 1h multiplier 1.5 max 2h
98
+ warn: $this >= 2
99
+ delay: up 1m down 1h multiplier 1.5 max 2h
100
info: the ratio of outbound dropped packets vs the total number of sent packets of the network interface, during the last 10 minutes
101
to: sysadmin
102
health/health.d/netfilter.conf
+2
-15
@@ -1,19 +1,6 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
- alarm: netfilter_last_collected_secs
5
- on: netfilter.conntrack_sockets
6
- os: linux
7
- hosts: *
8
- calc: $now - $last_collected_t
9
- units: seconds ago
10
- every: 10s
11
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
12
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
13
- delay: down 5m multiplier 1.5 max 1h
14
- info: number of seconds since the last successful data collection
15
- to: sysadmin
16
-
4
alarm: netfilter_conntrack_full
5
on: netfilter.conntrack_sockets
6
os: linux
@@ -22,8 +9,8 @@
9
calc: $this * 100 / $netfilter_conntrack_max
10
units: %
11
every: 10s
25
- warn: $this > (($status >= $WARNING) ? (70) : (80))
26
- crit: $this > (($status == $CRITICAL) ? (80) : (90))
12
+ warn: $this > (($status >= $WARNING) ? (85) : (90))
13
+ crit: $this > (($status == $CRITICAL) ? (90) : (95))
14
delay: down 5m multiplier 1.5 max 1h
15
info: the number of connections tracked by the netfilter connection tracker, as a percentage of the connection tracker table size
16
to: sysadmin
health/health.d/processes.conf
+2
-2
@@ -6,8 +6,8 @@
6
calc: $active * 100 / $pidmax
7
units: %
8
every: 5s
9
- warn: $this > (($status >= $WARNING) ? (75) : (80))
10
- crit: $this > (($status == $CRITICAL) ? (85) : (90))
9
+ warn: $this > (($status >= $WARNING) ? (85) : (90))
10
+ crit: $this > (($status == $CRITICAL) ? (90) : (95))
11
delay: down 5m multiplier 1.5 max 1h
12
info: the percentage of active processes
13
to: sysadmin
health/health.d/swap.conf
+3
-5
@@ -10,9 +10,8 @@
10
calc: $this / 1024 * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
11
units: % of RAM
12
every: 1m
13
- warn: $this > (($status >= $WARNING) ? (10) : (20))
14
- crit: $this > (($status == $CRITICAL) ? (20) : (30))
15
- delay: up 0 down 15m multiplier 1.5 max 1h
13
+ warn: $this > (($status >= $WARNING) ? (20) : (30))
14
+ delay: down 15m multiplier 1.5 max 1h
15
info: the amount of memory swapped in the last 30 minutes, as a percentage of the system RAM
16
to: sysadmin
17
@@ -23,8 +22,7 @@
22
calc: $used * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
23
units: % of RAM
24
every: 10s
26
- warn: $this > (($status >= $WARNING) ? (15) : (20))
27
- crit: $this > (($status == $CRITICAL) ? (40) : (50))
25
+ warn: $this > (($status >= $WARNING) ? (40) : (50))
26
delay: up 30s down 15m multiplier 1.5 max 1h
27
info: the swap memory used, as a percentage of the system RAM
28
to: sysadmin
health/health.d/tcp_resets.conf
-15
@@ -1,21 +1,6 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
-# -----------------------------------------------------------------------------
5
-
6
- alarm: ipv4_tcphandshake_last_collected_secs
7
- on: ipv4.tcphandshake
8
- os: linux freebsd
9
- hosts: *
10
- calc: $now - $last_collected_t
11
- units: seconds ago
12
- every: 10s
13
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
14
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
15
- delay: up 0 down 5m multiplier 1.5 max 1h
16
- info: number of seconds since the last successful data collection
17
- to: sysadmin
18
-
4
# -----------------------------------------------------------------------------
5
# tcp resets this host sends
6
health/health.d/udp_errors.conf
+4
-21
@@ -1,21 +1,6 @@
1
2
# you can disable an alarm notification by setting the 'to' line to: silent
3
4
-# -----------------------------------------------------------------------------
5
-
6
- alarm: ipv4_udperrors_last_collected_secs
7
- on: ipv4.udperrors
8
- os: linux freebsd
9
- hosts: *
10
- calc: $now - $last_collected_t
11
- units: seconds ago
12
- every: 10s
13
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
14
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
15
- delay: up 0 down 5m multiplier 1.5 max 1h
16
- info: number of seconds since the last successful data collection
17
- to: sysadmin
18
-
4
# -----------------------------------------------------------------------------
5
# UDP receive buffer errors
6
@@ -26,10 +11,9 @@
11
lookup: average -1m unaligned absolute of RcvbufErrors
12
units: errors
13
every: 10s
29
- warn: $this > 1
30
- crit: $this > (($status == $CRITICAL) ? (0) : (10))
14
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
15
info: average number of UDP receive buffer errors during the last minute
32
- delay: up 0 down 60m multiplier 1.2 max 2h
16
+ delay: up 1m down 60m multiplier 1.2 max 2h
17
to: sysadmin
18
19
# -----------------------------------------------------------------------------
@@ -42,8 +26,7 @@
26
lookup: average -1m unaligned absolute of SndbufErrors
27
units: errors
28
every: 10s
45
- warn: $this > 1
46
- crit: $this > (($status == $CRITICAL) ? (0) : (10))
29
+ warn: $this > (($status >= $WARNING) ? (0) : (10))
30
info: number of UDP send buffer errors during the last minute
48
- delay: up 0 down 60m multiplier 1.2 max 2h
31
+ delay: up 1m down 60m multiplier 1.2 max 2h
32
to: sysadmin