more elastic alarms; better names; properly invalidate all alarm log on alarms reload SIGUSR2
Costa Tsaousis (ktsaou) committed
Sep 7, 2016 at 03:28 UTC
3bb396eea416e51727d521054b919524f6701294
7 files changed
+46
-34
conf.d/health.d/cpu.conf
+12
-9
@@ -1,27 +1,30 @@
1
2
-template: 5min_cpu_pcent
2
+template: 10min_cpu_usage
3
on: system.cpu
4
- lookup: average -5m unaligned of user,system,nice,softirq,irq,guest,guest_nice
4
+ lookup: average -10m unaligned of user,system,nice,softirq,irq,guest,guest_nice
5
every: 1m
6
- warn: $this > 90
6
+ warn: $this > 80
7
+ crit: $this > 90
8
units: %
8
- info: average cpu utilization for the last 5 minutes
9
+ info: average cpu utilization for the last 10 minutes
10
to: sysadmin
11
11
-template: 5min_iowait_cpu_pcent
12
+template: 10min_cpu_iowait
13
on: system.cpu
13
- lookup: average -5m unaligned of iowait
14
+ lookup: average -10m unaligned of iowait
15
every: 1m
16
warn: $this > 10
17
+ crit: $this > 30
18
units: %
17
- info: average wait I/O for the last 5 minutes
19
+ info: average CPU wait I/O for the last 10 minutes
20
to: sysadmin
21
20
-template: 20min_steal_cpu_pcent
22
+template: 20min_steal_cpu
23
on: system.cpu
24
lookup: average -20m unaligned of steal
25
every: 5m
26
warn: $this > 10
27
+ crit: $this > 30
28
units: %
26
- info: average stolen CPU time for the last 20 minutes
29
+ info: average CPU steal time for the last 20 minutes
30
to: sysadmin
conf.d/health.d/disks.conf
+4
-4
@@ -5,7 +5,7 @@
5
# raise an alarm if the disk is low on
6
# available disk space
7
8
-template: disk_full_percent
8
+template: disk_space_usage
9
on: disk.space
10
calc: $used * 100 / ($avail + $used)
11
every: 1m
@@ -29,9 +29,9 @@ template: disk_full_percent
29
30
template: disk_fill_rate
31
on: disk.space
32
- lookup: max -2m at -30m unaligned of avail
32
+ lookup: max -5m at -30m unaligned of avail
33
calc: ($this - $avail) / (($now - $after) / 60)
34
- every: 15s
34
+ every: 1m
35
units: GB/min
36
info: average rate the disk fills up (positive), or frees up (negative) space, for the last 30 minutes
37
@@ -40,7 +40,7 @@ template: disk_fill_rate
40
# if the disk continues to fill
41
# in this rate
42
43
-template: disk_full_after_hours
43
+template: out_of_disk_space_in
44
on: disk.space
45
calc: $avail / ($disk_fill_rate * 60)
46
every: 10s
conf.d/health.d/entropy.conf
+3
-4
@@ -3,12 +3,11 @@
3
# the alarm is checked every 1 minute
4
# and examines the last 30 minutes of data
5
6
- alarm: min_30min_entropy
6
+ alarm: 30min_lowest_entropy
7
on: system.entropy
8
lookup: min -30m unaligned
9
every: 1m
10
- warn: $this < 200
11
- crit: $this < 100
10
+ warn: $this < 100
11
units: entries
13
- info: minimum entries in the random numbers pool (entropy), for the last 30 minutes
12
+ info: minimum entries in the random numbers pool in the last 30 minutes
13
to: sysadmin
conf.d/health.d/memcached.conf
+8
-8
@@ -14,7 +14,7 @@ template: memcached_last_collected_secs
14
15
# detect if memcached cache is full
16
17
-template: cache_full_pcent
17
+template: memcached_cache_memory_usage
18
on: memcached.cache
19
calc: $used * 100 / ($used + $available)
20
every: 10s
@@ -27,20 +27,20 @@ template: cache_full_pcent
27
28
# find the rate memcached cache is filling
29
30
-template: cache_fill_rate
30
+template: memcached_cache_fill_rate
31
on: memcached.cache
32
- lookup: max -1s at -30m unaligned of available
33
- calc: ($this - $available) / ($now - $after)
34
- every: 15s
35
- units: KB/s
32
+ lookup: max -5m at -30m unaligned of available
33
+ calc: ($this - $available) / (($now - $after) / 60)
34
+ every: 1m
35
+ units: KB/min
36
info: average rate the cache fills up (positive), or frees up (negative) space, for the last 30 minutes
37
38
39
# find the hours remaining until memcached cache is full
40
41
-template: cache_full_after_hours
41
+template: memcached_out_of_cache_in
42
on: memcached.cache
43
- calc: $available / $cache_fill_rate / 3600
43
+ calc: $available / ($memcached_cache_fill_rate * 60)
44
every: 10s
45
warn: $this > 0 and $this < 48
46
crit: $this > 0 and $this < 24
conf.d/health.d/net.conf
+8
-8
@@ -1,29 +1,29 @@
1
2
# check if an interface is dropping packets
3
# the alarm is checked every 10 seconds
4
-# and examines the last 30 minutes of data
4
+# and examines the last hour of data
5
6
-template: 30min_packet_drops
6
+template: 1hour_packet_drops
7
on: net.drops
8
- lookup: sum -30m unaligned absolute
8
+ lookup: sum -1h unaligned absolute
9
every: 1m
10
warn: $this > 0
11
units: packets
12
- info: dropped packets in the last 30 minutes
12
+ info: interface dropped packets in the last hour
13
to: sysadmin
14
15
16
# check if an interface is having FIFO
17
# buffer errors
18
# the alarm is checked every 10 seconds
19
-# and examines the last 30 minutes of data
19
+# and examines the last hour of data
20
21
-template: 30min_fifo_errors
21
+template: 1hour_fifo_errors
22
on: net.fifo
23
- lookup: sum -30m unaligned absolute
23
+ lookup: sum -1h unaligned absolute
24
every: 1m
25
warn: $this > 0
26
units: errors
27
- info: network interface fifo errors in the last 30 minutes
27
+ info: interface fifo errors in the last hour
28
to: sysadmin
29
conf.d/health.d/swap.conf
+1
-1
@@ -15,7 +15,7 @@
15
on: system.swap
16
calc: $used * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
17
every: 10s
18
- warn: $this > 10
18
+ warn: $this > 20
19
crit: $this > 50
20
units: % of RAM
21
info: the swap memory used, as a percentage of the system RAM
src/health.c
+10
@@ -1742,20 +1742,30 @@ void health_reload(void) {
1742
1743
char *path = health_config_dir();
1744
1745
+ // free all running alarms
1746
rrdhost_rwlock(&localhost);
1747
health_free_all_nolock(&localhost);
1748
rrdhost_unlock(&localhost);
1749
1750
+ // invalidate all previous entries in the alarm log
1751
+ ALARM_ENTRY *t;
1752
+ for(t = localhost.health_log.alarms ; t ; t = t->next) {
1753
+ t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1754
+ }
1755
+
1756
+ // reset all thresholds to all charts
1757
RRDSET *st;
1758
for(st = localhost.rrdset_root; st ; st = st->next) {
1759
st->green = NAN;
1760
st->red = NAN;
1761
}
1762
1763
+ // load the new alarms
1764
rrdhost_rwlock(&localhost);
1765
health_readdir(path);
1766
rrdhost_unlock(&localhost);
1767
1768
+ // link the loaded alarms to their charts
1769
for(st = localhost.rrdset_root; st ; st = st->next) {
1770
rrdhost_rwlock(&localhost);
1771