alarms reworked to lower CPU pressure on large systems
Costa Tsaousis (ktsaou) committed
Jan 22, 2017 at 02:01 UTC
35fd21781a71ad173ce3ddc757f49c57b8ad448d
4 files changed
+37
-37
conf.d/health.d/cpu.conf
+1
-1
@@ -28,6 +28,6 @@ template: 20min_steal_cpu
28
every: 5m
29
warn: $this > (($status >= $WARNING) ? (5) : (10))
30
crit: $this > (($status == $CRITICAL) ? (20) : (30))
31
- delay: down 15m multiplier 1.5 max 1h
31
+ delay: down 1h multiplier 1.5 max 2h
32
info: average CPU steal time for the last 20 minutes
33
to: sysadmin
conf.d/health.d/entropy.conf
+4
-4
@@ -3,12 +3,12 @@
3
# the alarm is checked every 1 minute
4
# and examines the last hour of data
5
6
- alarm: 1hour_lowest_entropy
6
+ alarm: lowest_entropy
7
on: system.entropy
8
- lookup: min -1h unaligned
8
+ lookup: min -10m unaligned
9
units: entries
10
every: 5m
11
warn: $this < (($status >= $WARNING) ? (200) : (100))
12
- delay: down 1h multiplier 1.5 max 1h
13
- info: minimum entries in the random numbers pool in the last 30 minutes
12
+ delay: down 1h multiplier 1.5 max 2h
13
+ info: minimum entries in the random numbers pool in the last 10 minutes
14
to: silent
conf.d/health.d/net.conf
+24
-24
@@ -19,54 +19,54 @@ families: *
19
20
# check if an interface is dropping packets
21
# the alarm is checked every 1 minute
22
-# and examines the last hour of data
22
+# and examines the last 10 minutes of data
23
24
-template: 1hour_packet_drops_inbound
24
+template: inbound_packets_dropped
25
on: net.drops
26
families: *
27
- lookup: sum -1h unaligned absolute of inbound
27
+ lookup: sum -10m unaligned absolute of inbound
28
units: packets
29
every: 1m
30
warn: $this > 0
31
- delay: down 30m multiplier 1.5 max 1h
32
- info: interface inbound dropped packets in the last hour
31
+ delay: down 1h multiplier 1.5 max 2h
32
+ info: interface inbound dropped packets in the last 10 minutes
33
to: sysadmin
34
35
-template: 1hour_packet_drops_outbound
35
+template: outbound_packets_dropped
36
on: net.drops
37
families: *
38
- lookup: sum -1h unaligned absolute of outbound
38
+ lookup: sum -10m unaligned absolute of outbound
39
units: packets
40
every: 1m
41
warn: $this > 0
42
- delay: down 30m multiplier 1.5 max 1h
43
- info: interface outbound dropped packets in the last hour
42
+ delay: down 1h multiplier 1.5 max 2h
43
+ info: interface outbound dropped packets in the last 10 minutes
44
to: sysadmin
45
46
-template: 1hour_packet_drops_ratio_inbound
46
+template: inbound_packets_dropped_ratio
47
on: net.packets
48
families: *
49
- lookup: sum -1h unaligned absolute of received
50
- calc: (($1hour_packet_drops_inbound != nan AND $this > 0) ? ($1hour_packet_drops_inbound * 100 / $this) : (0))
49
+ lookup: sum -10m unaligned absolute of received
50
+ calc: (($inbound_packets_dropped != nan AND $this > 0) ? ($inbound_packets_dropped * 100 / $this) : (0))
51
units: %
52
every: 1m
53
warn: $this > 0.5
54
crit: $this > 3
55
- delay: down 30m multiplier 1.5 max 1h
56
- info: the ratio of inbound dropped packets vs the total number of received packets of the network interface, during the last hour
55
+ delay: down 1h multiplier 1.5 max 2h
56
+ info: the ratio of inbound dropped packets vs the total number of received packets of the network interface, during the last 10 minutes
57
to: sysadmin
58
59
-template: 1hour_packet_drops_ratio_outbound
59
+template: outbound_packets_dropped_ratio
60
on: net.packets
61
families: *
62
- lookup: sum -1h unaligned absolute of sent
63
- calc: (($1hour_packet_drops_outbound != nan AND $this > 0) ? ($1hour_packet_drops_outbound * 100 / $this) : (0))
62
+ lookup: sum -10m unaligned absolute of sent
63
+ calc: (($outbound_packets_dropped != nan AND $this > 0) ? ($outbound_packets_dropped * 100 / $this) : (0))
64
units: %
65
every: 1m
66
warn: $this > 0.5
67
crit: $this > 3
68
- delay: down 30m multiplier 1.5 max 1h
69
- info: the ratio of outbound dropped packets vs the total number of sent packets of the network interface, during the last hour
68
+ delay: down 1h multiplier 1.5 max 2h
69
+ info: the ratio of outbound dropped packets vs the total number of sent packets of the network interface, during the last 10 minutes
70
to: sysadmin
71
72
@@ -76,17 +76,17 @@ families: *
76
# check if an interface is having FIFO
77
# buffer errors
78
# the alarm is checked every 1 minute
79
-# and examines the last hour of data
79
+# and examines the last 10 minutes of data
80
81
-template: 1hour_fifo_errors
81
+template: 10min_fifo_errors
82
on: net.fifo
83
families: *
84
- lookup: sum -1h unaligned absolute
84
+ lookup: sum -10m unaligned absolute
85
units: errors
86
every: 1m
87
warn: $this > 0
88
- delay: down 30m multiplier 1.5 max 1h
89
- info: interface fifo errors in the last hour
88
+ delay: down 1h multiplier 1.5 max 2h
89
+ info: interface fifo errors in the last 10 minutes
90
to: sysadmin
91
92
conf.d/health.d/softnet.conf
+8
-8
@@ -1,21 +1,21 @@
1
# check for common /proc/net/softnet_stat errors
2
3
- alarm: 1hour_netdev_backlog_exceeded
3
+ alarm: 10min_netdev_backlog_exceeded
4
on: system.softnet_stat
5
- lookup: sum -1h unaligned absolute of dropped
5
+ lookup: sum -10m unaligned absolute of dropped
6
units: packets
7
every: 1m
8
warn: $this > 0
9
- delay: down 30m multiplier 1.5 max 1h
10
- info: number of packets dropped because sysctl net.core.netdev_max_backlog was exceeded (this can be a cause for dropped packets)
9
+ delay: down 1h multiplier 1.5 max 2h
10
+ info: number of packets dropped in the last 10min, because sysctl net.core.netdev_max_backlog was exceeded (this can be a cause for dropped packets)
11
to: sysadmin
12
13
- alarm: 1hour_netdev_budget_ran_outs
13
+ alarm: 10min_netdev_budget_ran_outs
14
on: system.softnet_stat
15
- lookup: sum -1h unaligned absolute of squeezed
15
+ lookup: sum -10m unaligned absolute of squeezed
16
units: events
17
every: 1m
18
warn: $this > (($status >= $WARNING) ? (0) : (10))
19
- delay: down 30m multiplier 1.5 max 1h
20
- info: number of times ksoftirq ran out of sysctl net.core.netdev_budget or time slice, with work remaining (this can be a cause for dropped packets)
19
+ delay: down 1h multiplier 1.5 max 2h
20
+ info: number of times, during the last 10min, ksoftirq ran out of sysctl net.core.netdev_budget or time slice, with work remaining (this can be a cause for dropped packets)
21
to: silent