remove the noise by silencing alerts that dont need to wake up people (#15590)
Costa Tsaousis committed
Jul 27, 2023 at 21:15 UTC
233d5e4cd9341d8adaa76f8cb1c3dca031eaa9cf
24 files changed
+189
-202
health/health.d/bcache.conf
+2
-2
@@ -12,7 +12,7 @@ component: Disk
12
info: number of times data was read from the cache, \
13
the bucket was reused and invalidated in the last 10 minutes \
14
(when this occurs the data is reread from the backing device)
15
- to: sysadmin
15
+ to: silent
16
17
template: bcache_cache_dirty
18
on: disk.bcache_cache_alloc
@@ -26,4 +26,4 @@ component: Disk
26
delay: up 1m down 1h multiplier 1.5 max 2h
27
info: percentage of cache space used for dirty data and metadata \
28
(this usually means your SSD cache is too small)
29
- to: sysadmin
29
+ to: silent
health/health.d/btrfs.conf
+4
-5
@@ -9,11 +9,10 @@ component: File system
9
calc: 100 - ($unallocated * 100 / ($unallocated + $data_used + $data_free + $meta_used + $meta_free + $sys_used + $sys_free))
10
units: %
11
every: 10s
12
- warn: $this > (($status >= $WARNING) ? (90) : (95))
13
- crit: $this > (($status == $CRITICAL) ? (95) : (98))
12
+ warn: $this > (($status == $CRITICAL) ? (95) : (98))
13
delay: up 1m down 15m multiplier 1.5 max 1h
14
info: percentage of allocated BTRFS physical disk space
16
- to: sysadmin
15
+ to: silent
16
17
template: btrfs_data
18
on: btrfs.data
@@ -86,7 +85,7 @@ component: File system
85
hosts: *
86
units: errors
87
lookup: max -10m every 1m of write_errs
89
- warn: $this > 0
88
+ crit: $this > 0
89
delay: up 1m down 15m multiplier 1.5 max 1h
90
info: number of encountered BTRFS write errors
91
to: sysadmin
@@ -100,7 +99,7 @@ component: File system
99
hosts: *
100
units: errors
101
lookup: max -10m every 1m of flush_errs
103
- warn: $this > 0
102
+ crit: $this > 0
103
delay: up 1m down 15m multiplier 1.5 max 1h
104
info: number of encountered BTRFS flush errors
105
to: sysadmin
health/health.d/cgroups.conf
+79
-76
@@ -11,11 +11,10 @@ component: CPU
11
lookup: average -10m unaligned
12
units: %
13
every: 1m
14
- warn: $this > (($status >= $WARNING) ? (75) : (85))
15
- crit: $this > (($status == $CRITICAL) ? (85) : (95))
14
+ warn: $this > (($status == $CRITICAL) ? (85) : (95))
15
delay: down 15m multiplier 1.5 max 1h
16
info: average cgroup CPU utilization over the last 10 minutes
18
- to: sysadmin
17
+ to: silent
18
19
template: cgroup_ram_in_use
20
on: cgroup.mem_usage
@@ -31,44 +30,45 @@ component: Memory
30
crit: $this > (($status == $CRITICAL) ? (90) : (98))
31
delay: down 15m multiplier 1.5 max 1h
32
info: cgroup memory utilization
34
- to: sysadmin
35
-
36
-# -----------------------------------------------------------------------------
37
-# check for packet storms
38
-
39
-# 1. calculate the rate packets are received in 1m: 1m_received_packets_rate
40
-# 2. do the same for the last 10s
41
-# 3. raise an alarm if the later is 10x or 20x the first
42
-# we assume the minimum packet storm should at least have
43
-# 10000 packets/s, average of the last 10 seconds
44
-
45
- template: cgroup_1m_received_packets_rate
46
- on: cgroup.net_packets
47
- class: Workload
48
- type: Cgroups
49
-component: Network
50
- hosts: *
51
- lookup: average -1m unaligned of received
52
- units: packets
53
- every: 10s
54
- info: average number of packets received by the network interface ${label:device} over the last minute
55
-
56
- template: cgroup_10s_received_packets_storm
57
- on: cgroup.net_packets
58
- class: Workload
59
- type: Cgroups
60
-component: Network
61
- hosts: *
62
- lookup: average -10s unaligned of received
63
- calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
64
- every: 10s
65
- units: %
66
- warn: $this > (($status >= $WARNING)?(200):(5000))
67
- options: no-clear-notification
68
- info: ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
69
- compared to the rate over the last minute
70
- to: sysadmin
33
+ to: silent
34
35
+# FIXME COMMENTED DUE TO A BUG IN NETDATA
36
+## -----------------------------------------------------------------------------
37
+## check for packet storms
38
+#
39
+## 1. calculate the rate packets are received in 1m: 1m_received_packets_rate
40
+## 2. do the same for the last 10s
41
+## 3. raise an alarm if the later is 10x or 20x the first
42
+## we assume the minimum packet storm should at least have
43
+## 10000 packets/s, average of the last 10 seconds
44
+#
45
+# template: cgroup_1m_received_packets_rate
46
+# on: cgroup.net_packets
47
+# class: Workload
48
+# type: Cgroups
49
+#component: Network
50
+# hosts: *
51
+# lookup: average -1m unaligned of received
52
+# units: packets
53
+# every: 10s
54
+# info: average number of packets received by the network interface ${label:device} over the last minute
55
+#
56
+# template: cgroup_10s_received_packets_storm
57
+# on: cgroup.net_packets
58
+# class: Workload
59
+# type: Cgroups
60
+#component: Network
61
+# hosts: *
62
+# lookup: average -10s unaligned of received
63
+# calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
64
+# every: 10s
65
+# units: %
66
+# warn: $this > (($status >= $WARNING)?(200):(5000))
67
+# options: no-clear-notification
68
+# info: ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
69
+# compared to the rate over the last minute
70
+# to: sysadmin
71
+#
72
# ---------------------------------K8s containers--------------------------------------------
73
74
template: k8s_cgroup_10min_cpu_usage
@@ -83,8 +83,9 @@ component: CPU
83
every: 1m
84
warn: $this > (($status >= $WARNING) ? (75) : (85))
85
delay: down 15m multiplier 1.5 max 1h
86
- info: average cgroup CPU utilization over the last 10 minutes
87
- to: sysadmin
86
+ info: container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
87
+ average CPU utilization over the last 10 minutes
88
+ to: silent
89
90
template: k8s_cgroup_ram_in_use
91
on: k8s.cgroup.mem_usage
@@ -99,40 +100,42 @@ component: Memory
100
warn: $this > (($status >= $WARNING) ? (80) : (90))
101
crit: $this > (($status == $CRITICAL) ? (90) : (98))
102
delay: down 15m multiplier 1.5 max 1h
102
- info: cgroup memory utilization
103
- to: sysadmin
103
+ info: container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
104
+ memory utilization
105
+ to: silent
106
107
# check for packet storms
108
107
-# 1. calculate the rate packets are received in 1m: 1m_received_packets_rate
108
-# 2. do the same for the last 10s
109
-# 3. raise an alarm if the later is 10x or 20x the first
110
-# we assume the minimum packet storm should at least have
111
-# 10000 packets/s, average of the last 10 seconds
112
-
113
- template: k8s_cgroup_1m_received_packets_rate
114
- on: k8s.cgroup.net_packets
115
- class: Workload
116
- type: Cgroups
117
-component: Network
118
- hosts: *
119
- lookup: average -1m unaligned of received
120
- units: packets
121
- every: 10s
122
- info: average number of packets received by the network interface ${label:device} over the last minute
123
-
124
- template: k8s_cgroup_10s_received_packets_storm
125
- on: k8s.cgroup.net_packets
126
- class: Workload
127
- type: Cgroups
128
-component: Network
129
- hosts: *
130
- lookup: average -10s unaligned of received
131
- calc: $this * 100 / (($k8s_cgroup_10s_received_packets_storm < 1000)?(1000):($k8s_cgroup_10s_received_packets_storm))
132
- every: 10s
133
- units: %
134
- warn: $this > (($status >= $WARNING)?(200):(5000))
135
- options: no-clear-notification
136
- info: ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
137
- compared to the rate over the last minute
138
- to: sysadmin
109
+# FIXME COMMENTED DUE TO A BUG IN NETDATA
110
+## 1. calculate the rate packets are received in 1m: 1m_received_packets_rate
111
+## 2. do the same for the last 10s
112
+## 3. raise an alarm if the later is 10x or 20x the first
113
+## we assume the minimum packet storm should at least have
114
+## 10000 packets/s, average of the last 10 seconds
115
+#
116
+# template: k8s_cgroup_1m_received_packets_rate
117
+# on: k8s.cgroup.net_packets
118
+# class: Workload
119
+# type: Cgroups
120
+#component: Network
121
+# hosts: *
122
+# lookup: average -1m unaligned of received
123
+# units: packets
124
+# every: 10s
125
+# info: average number of packets received by the network interface ${label:device} over the last minute
126
+#
127
+# template: k8s_cgroup_10s_received_packets_storm
128
+# on: k8s.cgroup.net_packets
129
+# class: Workload
130
+# type: Cgroups
131
+#component: Network
132
+# hosts: *
133
+# lookup: average -10s unaligned of received
134
+# calc: $this * 100 / (($k8s_cgroup_10s_received_packets_storm < 1000)?(1000):($k8s_cgroup_10s_received_packets_storm))
135
+# every: 10s
136
+# units: %
137
+# warn: $this > (($status >= $WARNING)?(200):(5000))
138
+# options: no-clear-notification
139
+# info: ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
140
+# compared to the rate over the last minute
141
+# to: sysadmin
health/health.d/cpu.conf
+5
-5
@@ -15,7 +15,7 @@ component: CPU
15
crit: $this > (($status == $CRITICAL) ? (85) : (95))
16
delay: down 15m multiplier 1.5 max 1h
17
info: average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
18
- to: sysadmin
18
+ to: silent
19
20
template: 10min_cpu_iowait
21
on: system.cpu
@@ -28,9 +28,9 @@ component: CPU
28
units: %
29
every: 1m
30
warn: $this > (($status >= $WARNING) ? (20) : (40))
31
- delay: down 15m multiplier 1.5 max 1h
31
+ delay: up 30m down 30m multiplier 1.5 max 2h
32
info: average CPU iowait time over the last 10 minutes
33
- to: sysadmin
33
+ to: silent
34
35
template: 20min_steal_cpu
36
on: system.cpu
@@ -45,7 +45,7 @@ component: CPU
45
warn: $this > (($status >= $WARNING) ? (5) : (10))
46
delay: down 1h multiplier 1.5 max 2h
47
info: average CPU steal time over the last 20 minutes
48
- to: sysadmin
48
+ to: silent
49
50
## FreeBSD
51
template: 10min_cpu_usage
@@ -62,4 +62,4 @@ component: CPU
62
crit: $this > (($status == $CRITICAL) ? (85) : (95))
63
delay: down 15m multiplier 1.5 max 1h
64
info: average CPU utilization over the last 10 minutes (excluding nice)
65
- to: sysadmin
65
+ to: silent
health/health.d/disks.conf
+43
-44
@@ -21,7 +21,7 @@ chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
21
units: %
22
every: 1m
23
warn: $this > (($status >= $WARNING ) ? (80) : (90))
24
- crit: $this > (($status == $CRITICAL) ? (90) : (98))
24
+ crit: ($this > (($status == $CRITICAL) ? (90) : (98))) && $avail < 5
25
delay: up 1m down 15m multiplier 1.5 max 1h
26
info: disk ${label:mount_point} space utilization
27
to: sysadmin
@@ -55,33 +55,32 @@ chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
55
# we will use it in the next template to find
56
# the hours remaining
57
58
-# template: disk_fill_rate
59
-# on: disk.space
60
-# os: linux freebsd
61
-# hosts: *
62
-# lookup: min -10m at -50m unaligned of avail
63
-# calc: ($this - $avail) / (($now - $after) / 3600)
64
-# every: 1m
65
-# units: GB/hour
66
-# info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
67
-
58
+template: disk_fill_rate
59
+ on: disk.space
60
+ os: linux freebsd
61
+ hosts: *
62
+ lookup: min -10m at -50m unaligned of avail
63
+ calc: ($this - $avail) / (($now - $after) / 3600)
64
+ every: 1m
65
+ units: GB/hour
66
+ info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
67
68
# calculate the hours remaining
69
# if the disk continues to fill
70
# in this rate
71
73
-# template: out_of_disk_space_time
74
-# on: disk.space
75
-# os: linux freebsd
76
-# hosts: *
77
-# calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
78
-# units: hours
79
-# every: 10s
80
-# warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
81
-# crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
82
-# delay: down 15m multiplier 1.2 max 1h
83
-# info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
84
-# to: sysadmin
72
+template: out_of_disk_space_time
73
+ on: disk.space
74
+ os: linux freebsd
75
+ hosts: *
76
+ calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf)
77
+ units: hours
78
+ every: 10s
79
+ warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
80
+ crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
81
+ delay: down 15m multiplier 1.2 max 1h
82
+ info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
83
+ to: silent
84
85
86
# -----------------------------------------------------------------------------
@@ -95,32 +94,32 @@ chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
94
# we will use it in the next template to find
95
# the hours remaining
96
98
-# template: disk_inode_rate
99
-# on: disk.inodes
100
-# os: linux freebsd
101
-# hosts: *
102
-# lookup: min -10m at -50m unaligned of avail
103
-# calc: ($this - $avail) / (($now - $after) / 3600)
104
-# every: 1m
105
-# units: inodes/hour
106
-# info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour
97
+template: disk_inode_rate
98
+ on: disk.inodes
99
+ os: linux freebsd
100
+ hosts: *
101
+ lookup: min -10m at -50m unaligned of avail
102
+ calc: ($this - $avail) / (($now - $after) / 3600)
103
+ every: 1m
104
+ units: inodes/hour
105
+ info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour
106
107
# calculate the hours remaining
108
# if the disk inodes are allocated
109
# in this rate
110
112
-# template: out_of_disk_inodes_time
113
-# on: disk.inodes
114
-# os: linux freebsd
115
-# hosts: *
116
-# calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf)
117
-# units: hours
118
-# every: 10s
119
-# warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
120
-# crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
121
-# delay: down 15m multiplier 1.2 max 1h
122
-# info: estimated time the disk will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
123
-# to: sysadmin
111
+template: out_of_disk_inodes_time
112
+ on: disk.inodes
113
+ os: linux freebsd
114
+ hosts: *
115
+ calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf)
116
+ units: hours
117
+ every: 10s
118
+ warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
119
+ crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
120
+ delay: down 15m multiplier 1.2 max 1h
121
+ info: estimated time the disk will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
122
+ to: silent
123
124
125
# -----------------------------------------------------------------------------
health/health.d/file_descriptors.conf
+5
-5
@@ -22,10 +22,10 @@ component: Process
22
os: linux
23
module: *
24
hosts: *
25
- lookup: max -1m unaligned foreach *
25
+ lookup: max -10s unaligned foreach *
26
units: %
27
- every: 1m
28
- warn: $this > (($status >= $WARNING) ? (85) : (90))
29
- delay: up 5m down 15m multiplier 1.5 max 1h
30
- info: maximum utilization of open files among all application group PIDs
27
+ every: 10s
28
+ warn: $this > (($status >= $WARNING) ? (85) : (95))
29
+ delay: down 15m multiplier 1.5 max 1h
30
+ info: open files percentage against the processes limits, among all PIDs in application group
31
to: sysadmin
health/health.d/ioping.conf
+1
-1
@@ -10,4 +10,4 @@ component: Disk
10
warn: $this > $green
11
delay: down 30m multiplier 1.5 max 2h
12
info: average I/O latency over the last 10 seconds
13
- to: sysadmin
13
+ to: silent
health/health.d/ipmi.conf
+1
-1
@@ -23,4 +23,4 @@ component: IPMI
23
warn: $this > 0
24
delay: up 5m down 15m multiplier 1.5 max 1h
25
info: number of events in the IPMI System Event Log (SEL)
26
- to: sysadmin
26
+ to: silent
health/health.d/linux_power_supply.conf
+1
-1
@@ -11,4 +11,4 @@ component: Battery
11
warn: $this < 10
12
delay: up 30s down 5m multiplier 1.2 max 1h
13
info: percentage of remaining power supply capacity
14
- to: sysadmin
14
+ to: silent
health/health.d/load.conf
+3
-3
@@ -34,7 +34,7 @@ component: Load
34
warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
35
delay: down 15m multiplier 1.5 max 1h
36
info: system fifteen-minute load average
37
- to: sysadmin
37
+ to: silent
38
39
alarm: load_average_5
40
on: system.load
@@ -50,7 +50,7 @@ component: Load
50
warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
51
delay: down 15m multiplier 1.5 max 1h
52
info: system five-minute load average
53
- to: sysadmin
53
+ to: silent
54
55
alarm: load_average_1
56
on: system.load
@@ -66,4 +66,4 @@ component: Load
66
warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
67
delay: down 15m multiplier 1.5 max 1h
68
info: system one-minute load average
69
- to: sysadmin
69
+ to: silent
health/health.d/mdstat.conf
+2
-14
@@ -1,15 +1,3 @@
1
- template: mdstat_last_collected
2
- on: md.disks
3
- class: Latency
4
- type: System
5
-component: RAID
6
- calc: $now - $last_collected_t
7
- units: seconds ago
8
- every: 10s
9
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
- info: number of seconds since the last successful data collection
12
- to: sysadmin
1
2
template: mdstat_disks
3
on: md.disks
@@ -19,7 +7,7 @@ component: RAID
7
units: failed devices
8
every: 10s
9
calc: $down
22
- crit: $this > 0
10
+ warn: $this > 0
11
info: number of devices in the down state for the ${label:device} ${label:raid_level} array. \
12
Any number > 0 indicates that the array is degraded.
13
to: sysadmin
@@ -36,7 +24,7 @@ chart labels: raid_level=!raid1 !raid10 *
24
warn: $this > 1024
25
delay: up 30m
26
info: number of unsynchronized blocks for the ${label:device} ${label:raid_level} array
39
- to: sysadmin
27
+ to: silent
28
29
template: mdstat_nonredundant_last_collected
30
on: md.nonredundant
health/health.d/net.conf
+10
-10
@@ -30,7 +30,7 @@ component: Network
30
warn: $this > (($status >= $WARNING) ? (85) : (90))
31
delay: up 1m down 1m multiplier 1.5 max 1h
32
info: average inbound utilization for the network interface ${label:device} over the last minute
33
- to: sysadmin
33
+ to: silent
34
35
template: 1m_sent_traffic_overflow
36
on: net.net
@@ -46,7 +46,7 @@ component: Network
46
warn: $this > (($status >= $WARNING) ? (85) : (90))
47
delay: up 1m down 1m multiplier 1.5 max 1h
48
info: average outbound utilization for the network interface ${label:device} over the last minute
49
- to: sysadmin
49
+ to: silent
50
51
# -----------------------------------------------------------------------------
52
# dropped packets
@@ -97,7 +97,7 @@ chart labels: device=!wl* *
97
warn: $this >= 2
98
delay: up 1m down 1h multiplier 1.5 max 2h
99
info: ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
100
- to: sysadmin
100
+ to: silent
101
102
template: outbound_packets_dropped_ratio
103
on: net.packets
@@ -114,7 +114,7 @@ chart labels: device=!wl* *
114
warn: $this >= 2
115
delay: up 1m down 1h multiplier 1.5 max 2h
116
info: ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
117
- to: sysadmin
117
+ to: silent
118
119
template: wifi_inbound_packets_dropped_ratio
120
on: net.packets
@@ -131,7 +131,7 @@ chart labels: device=wl*
131
warn: $this >= 10
132
delay: up 1m down 1h multiplier 1.5 max 2h
133
info: ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
134
- to: sysadmin
134
+ to: silent
135
136
template: wifi_outbound_packets_dropped_ratio
137
on: net.packets
@@ -148,7 +148,7 @@ chart labels: device=wl*
148
warn: $this >= 10
149
delay: up 1m down 1h multiplier 1.5 max 2h
150
info: ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
151
- to: sysadmin
151
+ to: silent
152
153
# -----------------------------------------------------------------------------
154
# interface errors
@@ -166,7 +166,7 @@ component: Network
166
warn: $this >= 5
167
delay: down 1h multiplier 1.5 max 2h
168
info: number of inbound errors for the network interface ${label:device} in the last 10 minutes
169
- to: sysadmin
169
+ to: silent
170
171
template: interface_outbound_errors
172
on: net.errors
@@ -181,7 +181,7 @@ component: Network
181
warn: $this >= 5
182
delay: down 1h multiplier 1.5 max 2h
183
info: number of outbound errors for the network interface ${label:device} in the last 10 minutes
184
- to: sysadmin
184
+ to: silent
185
186
# -----------------------------------------------------------------------------
187
# FIFO errors
@@ -204,7 +204,7 @@ component: Network
204
warn: $this > 0
205
delay: down 1h multiplier 1.5 max 2h
206
info: number of FIFO errors for the network interface ${label:device} in the last 10 minutes
207
- to: sysadmin
207
+ to: silent
208
209
# -----------------------------------------------------------------------------
210
# check for packet storms
@@ -243,4 +243,4 @@ component: Network
243
options: no-clear-notification
244
info: ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
245
compared to the rate over the last minute
246
- to: sysadmin
246
+ to: silent
health/health.d/qos.conf
+10
-11
@@ -5,14 +5,13 @@
5
# the alarm is checked every 10 seconds
6
# and examines the last minute of data
7
8
-#template: 10min_qos_packet_drops
9
-# on: tc.qos_dropped
10
-# os: linux
11
-# hosts: *
12
-# lookup: sum -10m unaligned absolute
13
-# every: 30s
14
-# warn: $this > 0
15
-# delay: up 0 down 30m multiplier 1.5 max 1h
16
-# units: packets
17
-# info: dropped packets in the last 30 minutes
18
-# to: sysadmin
8
+template: 10min_qos_packet_drops
9
+ on: tc.qos_dropped
10
+ os: linux
11
+ hosts: *
12
+ lookup: sum -5m unaligned absolute
13
+ every: 30s
14
+ warn: $this > 0
15
+ units: packets
16
+ info: dropped packets in the last 5 minutes
17
+ to: silent
health/health.d/ram.conf
+3
-4
@@ -30,7 +30,7 @@ component: Memory
30
warn: $this < (($status >= $WARNING) ? (15) : (10))
31
delay: down 15m multiplier 1.5 max 1h
32
info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
33
- to: sysadmin
33
+ to: silent
34
35
alarm: oom_kill
36
on: mem.oom_kill
@@ -41,9 +41,8 @@ component: Memory
41
every: 5m
42
warn: $this > 0
43
delay: down 10m
44
-host labels: _is_k8s_node = false
44
info: number of out of memory kills in the last 30 minutes
46
- to: sysadmin
45
+ to: silent
46
47
## FreeBSD
48
alarm: ram_in_use
@@ -75,4 +74,4 @@ component: Memory
74
warn: $this < (($status >= $WARNING) ? (15) : (10))
75
delay: down 15m multiplier 1.5 max 1h
76
info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
78
- to: sysadmin
77
+ to: silent
health/health.d/softnet.conf
+2
-2
@@ -17,7 +17,7 @@ component: Network
17
delay: down 1h multiplier 1.5 max 2h
18
info: average number of dropped packets in the last minute \
19
due to exceeded net.core.netdev_max_backlog
20
- to: sysadmin
20
+ to: silent
21
22
alarm: 1min_netdev_budget_ran_outs
23
on: system.softnet_stat
@@ -51,4 +51,4 @@ component: Network
51
info: average number of drops in the last minute \
52
due to exceeded sysctl net.route.netisr_maxqlen \
53
(this can be a cause for dropped packets)
54
- to: sysadmin
54
+ to: silent
health/health.d/swap.conf
+1
-1
@@ -16,7 +16,7 @@ component: Memory
16
warn: $this > (($status >= $WARNING) ? (20) : (30))
17
delay: down 15m multiplier 1.5 max 1h
18
info: percentage of the system RAM swapped in the last 30 minutes
19
- to: sysadmin
19
+ to: silent
20
21
alarm: used_swap
22
on: mem.swap
health/health.d/synchronization.conf
+1
-1
@@ -9,4 +9,4 @@
9
info: number of sync() system calls. \
10
Every call causes all pending modifications to filesystem metadata and \
11
cached file data to be written to the underlying filesystems.
12
- to: sysadmin
12
+ to: silent
health/health.d/tcp_listen.conf
+4
-4
@@ -32,7 +32,7 @@ component: Network
32
crit: $this > (($status == $CRITICAL) ? (1) : (5))
33
delay: up 0 down 5m multiplier 1.5 max 1h
34
info: average number of overflows in the TCP accept queue over the last minute
35
- to: sysadmin
35
+ to: silent
36
37
# THIS IS TOO GENERIC
38
# CHECK: https://github.com/netdata/netdata/issues/3234#issuecomment-423935842
@@ -50,7 +50,7 @@ component: Network
50
crit: $this > (($status == $CRITICAL) ? (1) : (5))
51
delay: up 0 down 5m multiplier 1.5 max 1h
52
info: average number of dropped packets in the TCP accept queue over the last minute
53
- to: sysadmin
53
+ to: silent
54
55
56
# -----------------------------------------------------------------------------
@@ -76,7 +76,7 @@ component: Network
76
delay: up 10 down 5m multiplier 1.5 max 1h
77
info: average number of SYN requests was dropped due to the full TCP SYN queue over the last minute \
78
(SYN cookies were not enabled)
79
- to: sysadmin
79
+ to: silent
80
81
alarm: 1m_tcp_syn_queue_cookies
82
on: ip.tcp_syn_queue
@@ -92,5 +92,5 @@ component: Network
92
crit: $this > (($status == $CRITICAL) ? (0) : (5))
93
delay: up 10 down 5m multiplier 1.5 max 1h
94
info: average number of sent SYN cookies due to the full TCP SYN queue over the last minute
95
- to: sysadmin
95
+ to: silent
96
health/health.d/tcp_mem.conf
+1
-1
@@ -20,4 +20,4 @@ component: Network
20
crit: ${mem} > (($status == $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
21
delay: up 0 down 5m multiplier 1.5 max 1h
22
info: TCP memory utilization
23
- to: sysadmin
23
+ to: silent
health/health.d/tcp_orphans.conf
+1
-1
@@ -21,4 +21,4 @@ component: Network
21
crit: $this > (($status == $CRITICAL) ? ( 25 ) : ( 50 ))
22
delay: up 0 down 5m multiplier 1.5 max 1h
23
info: orphan IPv4 TCP sockets utilization
24
- to: sysadmin
24
+ to: silent
health/health.d/tcp_resets.conf
+2
-2
@@ -33,7 +33,7 @@ component: Network
33
This can indicate a port scan, \
34
or that a service running on this host has crashed. \
35
Netdata will not send a clear notification for this alarm.
36
- to: sysadmin
36
+ to: silent
37
38
# -----------------------------------------------------------------------------
39
# tcp resets this host receives
@@ -66,4 +66,4 @@ component: Network
66
info: average number of received TCP RESETS over the last 10 seconds. \
67
This can be an indication that a service this host needs has crashed. \
68
Netdata will not send a clear notification for this alarm.
69
- to: sysadmin
69
+ to: silent
health/health.d/udp_errors.conf
+2
-2
@@ -17,7 +17,7 @@ component: Network
17
warn: $this > (($status >= $WARNING) ? (0) : (10))
18
info: average number of UDP receive buffer errors over the last minute
19
delay: up 1m down 60m multiplier 1.2 max 2h
20
- to: sysadmin
20
+ to: silent
21
22
# -----------------------------------------------------------------------------
23
# UDP send buffer errors
@@ -35,4 +35,4 @@ component: Network
35
warn: $this > (($status >= $WARNING) ? (0) : (10))
36
info: average number of UDP send buffer errors over the last minute
37
delay: up 1m down 60m multiplier 1.2 max 2h
38
- to: sysadmin
38
+ to: silent
health/health.d/windows.conf
+5
-5
@@ -15,7 +15,7 @@ component: CPU
15
crit: $this > (($status == $CRITICAL) ? (85) : (95))
16
delay: down 15m multiplier 1.5 max 1h
17
info: average CPU utilization over the last 10 minutes
18
- to: sysadmin
18
+ to: silent
19
20
21
## Memory
@@ -52,7 +52,7 @@ component: Network
52
warn: $this >= 5
53
delay: down 1h multiplier 1.5 max 2h
54
info: number of inbound discarded packets for the network interface in the last 10 minutes
55
- to: sysadmin
55
+ to: silent
56
57
template: windows_outbound_packets_discarded
58
on: windows.net_nic_discarded
@@ -67,7 +67,7 @@ component: Network
67
warn: $this >= 5
68
delay: down 1h multiplier 1.5 max 2h
69
info: number of outbound discarded packets for the network interface in the last 10 minutes
70
- to: sysadmin
70
+ to: silent
71
72
template: windows_inbound_packets_errors
73
on: windows.net_nic_errors
@@ -82,7 +82,7 @@ component: Network
82
warn: $this >= 5
83
delay: down 1h multiplier 1.5 max 2h
84
info: number of inbound errors for the network interface in the last 10 minutes
85
- to: sysadmin
85
+ to: silent
86
87
template: windows_outbound_packets_errors
88
on: windows.net_nic_errors
@@ -97,7 +97,7 @@ component: Network
97
warn: $this >= 5
98
delay: down 1h multiplier 1.5 max 2h
99
info: number of outbound errors for the network interface in the last 10 minutes
100
- to: sysadmin
100
+ to: silent
101
102
103
## Disk
health/health.d/zfs.conf
+1
-1
@@ -10,7 +10,7 @@ component: File system
10
warn: $this > 0
11
delay: down 1h multiplier 1.5 max 2h
12
info: number of times ZFS had to limit the ARC growth in the last 10 minutes
13
- to: sysadmin
13
+ to: silent
14
15
# ZFS pool state
16