lowered the warning and critical levels of the prediction for out of space and increased the evaluation timeframe to 1 hour
Costa Tsaousis (ktsaou) committed
Sep 10, 2016 at 13:26 UTC
1b797018a1b68e56b354544bacb0db3c9a7bf1f6
4 files changed
+35
-28
conf.d/health.d/disks.conf
+10
-10
@@ -21,7 +21,7 @@ template: disk_space_usage
21
22
# calculate the rate the disk fills
23
# use as base, the available space change
24
-# during the last 30 minutes
24
+# during the last hour
25
26
# this is just a calculation - it has no alarm
27
# we will use it in the next template to find
@@ -29,25 +29,25 @@ template: disk_space_usage
29
30
template: disk_fill_rate
31
on: disk.space
32
- lookup: min -5m at -30m unaligned of avail
33
- calc: ($this - $avail) / (($now - $after) / 60)
32
+ lookup: min -10m at -50m unaligned of avail
33
+ calc: ($this - $avail) / (($now - $after) / 3600)
34
every: 1m
35
- units: GB/min
36
- info: average rate the disk fills up (positive), or frees up (negative) space, for the last 30 minutes
35
+ units: GB/hour
36
+ info: average rate the disk fills up (positive), or frees up (negative) space, for the last hour
37
38
39
# calculate the hours remaining
40
# if the disk continues to fill
41
# in this rate
42
43
-template: out_of_disk_space
43
+template: out_of_disk_space_time
44
on: disk.space
45
- calc: $avail / ($disk_fill_rate * 60)
45
+ calc: $avail / $disk_fill_rate
46
every: 10s
47
- warn: $this > 0 and $this < 48
48
- crit: $this > 0 and $this < 24
47
+ warn: $this > 0 and $this < 8
48
+ crit: $this > 0 and $this < 2
49
units: hours
50
- info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last 30 minutes
50
+ info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
51
to: sysadmin
52
53
conf.d/health.d/memcached.conf
+10
-10
@@ -27,23 +27,23 @@ template: memcached_cache_memory_usage
27
28
# find the rate memcached cache is filling
29
30
-template: memcached_cache_fill_rate
30
+template: cache_fill_rate
31
on: memcached.cache
32
- lookup: min -5m at -30m unaligned of available
33
- calc: ($this - $available) / (($now - $after) / 60)
32
+ lookup: min -10m at -50m unaligned of available
33
+ calc: ($this - $available) / (($now - $after) / 3600)
34
every: 1m
35
- units: KB/min
36
- info: average rate the cache fills up (positive), or frees up (negative) space, for the last 30 minutes
35
+ units: KB/hour
36
+ info: average rate the cache fills up (positive), or frees up (negative) space, for the last hour
37
38
39
# find the hours remaining until memcached cache is full
40
41
-template: memcached_out_of_cache_space
41
+template: out_of_cache_space_time
42
on: memcached.cache
43
- calc: $available / ($memcached_cache_fill_rate * 60)
43
+ calc: $available / $cache_fill_rate
44
every: 10s
45
- warn: $this > 0 and $this < 48
46
- crit: $this > 0 and $this < 24
45
+ warn: $this > 0 and $this < 8
46
+ crit: $this > 0 and $this < 2
47
units: hours
48
- info: estimated time the cache will run out of space, if the system continues to add data with the rate of the last 30 minutes
48
+ info: estimated time the cache will run out of space, if the system continues to add data with the rate of the last hour
49
to: dba
conf.d/health_alarm_notify.conf
+14
-7
@@ -1,9 +1,16 @@
1
-# Configuration for alarms recipients
2
-
3
-# netdata alarms have been categorized to allow different roles to receive
4
-# alarms related to their work.
1
+# Configuration for alarm notifications
2
+#
3
+# this configuration is used by: alarm-notify.sh
4
+# changes take effect immendiately (the next alarm will use them).
5
+#
6
+# alarm-notify.sh can send:
7
+# - e-mails,
8
+# - push notifications to your mobile phone,
9
+# - messages to your slack team
10
+#
11
+# the recipient given in netdata alarms defines a role, so that different
12
+# people can be notified for each role.
13
#
6
-# This file defines the addresses for each role.
14
# This file is a BASH script itself.
15
16
###############################################################################
@@ -48,7 +55,7 @@ SEND_PUSHOVER="YES"
55
# Without it, netdata cannot send pushover notifications.
56
PUSHOVER_APP_TOKEN=""
57
51
-# if a role recipient is not configured, a notification will be send to
58
+# if a role's recipients are not configured, a notification will be send to
59
# this pushover user token:
60
DEFAULT_RECIPIENT_PUSHOVER=""
61
@@ -67,7 +74,7 @@ SEND_SLACK="YES"
74
# Without it, netdata cannot send slack notifications.
75
SLACK_WEBHOOK_URL=""
76
70
-# if a role recipient is not configured, a notification will be send to
77
+# if a role's recipients are not configured, a notification will be send to
78
# this slack channel:
79
DEFAULT_RECIPIENT_SLACK=""
80
web/index.html
+1
-1
@@ -2593,7 +2593,7 @@ function alarmsUpdateModal() {
2593
var active = '<h3>Raised Alarms</h3><table class="table">';
2594
var all = '<h3>All Running Alarms</h3><div class="panel-group" id="alarms_all_accordion" role="tablist" aria-multiselectable="true">';
2595
var log = '<h3>Alarm Log</h3><table class="table"><tr><th>When</th><th>Chart</th><th>Alarm</th><th>Status</th>';
2596
- var footer = '<hr/><a href="https://github.com/firehol/netdata/wiki/Generating-Badges" target="_blank">netdata badges</a> refresh automatically. Their color indicates the state of the alarm: <span style="color: #e05d44"><b> red </b></span> is critical, <span style="color:#fe7d37"><b> orange </b></span> is warning, <span style="color: #4c1"><b> bright green </b></span> is ok, <span style="color: #9f9f9f"><b> light grey </b></span> is undefined (i.e. no data or no status), <span style="color: #000"><b> black </b></span> is not initialized. You can copy and paste their URLs to embed them in any web page.';
2596
+ var footer = '<hr/><a href="https://github.com/firehol/netdata/wiki/Generating-Badges" target="_blank">netdata badges</a> refresh automatically. Their color indicates the state of the alarm: <span style="color: #e05d44"><b> red </b></span> is critical, <span style="color:#fe7d37"><b> orange </b></span> is warning, <span style="color: #4c1"><b> bright green </b></span> is ok, <span style="color: #9f9f9f"><b> light grey </b></span> is undefined (i.e. no data or no status), <span style="color: #000"><b> black </b></span> is not initialized. You can copy and paste their URLs to embed them in any web page.<br/>netdata can send notifications for these alarms. Check <a href="https://github.com/firehol/netdata/blob/master/conf.d/health_alarm_notify.conf">this configuration file</a> for more information.';
2597
2598
NETDATA.alarms.get('all', function(data) {
2599
options.alarm_families = new Array();