Better alarms for httpcheck
Chris committed
Feb 26, 2018 at 16:04 UTC
4fcf6cb852c4484e4f725e561454e028a5c1fa77
1 file changed
+36
-24
conf.d/health.d/httpcheck.conf
+36
-24
@@ -8,11 +8,10 @@ families: *
8
crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
9
delay: down 5m multiplier 1.5 max 1h
10
info: number of seconds since the last successful data collection
11
- options: no-clear-notification
11
to: sysadmin
12
13
# This is a fast-reacting no-notification alarm ideal for custom dashboards or badges
15
-template: web_service_reachable
14
+template: web_service_up
15
families: *
16
on: httpcheck.status
17
lookup: average -1m unaligned percentage of success
@@ -22,7 +21,7 @@ families: *
21
info: at least 75% verified responses during last 60 seconds, ideal for badges
22
to: silent
23
25
-template: web_service_bad_content_received
24
+template: web_service_bad_content
25
families: *
26
on: httpcheck.status
27
lookup: average -5m unaligned percentage of bad_content
@@ -44,7 +43,7 @@ families: *
43
warn: $this >= 10 AND $this < 40
44
crit: $this >= 40
45
delay: down 5m multiplier 1.5 max 1h
47
- info: average of bad http status during the last 5 minutes
46
+ info: average of unexpected http status during the last 5 minutes
47
options: no-clear-notification
48
to: webmaster
49
@@ -54,34 +53,47 @@ families: *
53
lookup: average -5m unaligned percentage of timeout
54
every: 10s
55
units: %
57
- warn: $this >= 10 AND $this < 40
58
- crit: $this >= 40
59
- delay: down 5m multiplier 1.5 max 1h
56
info: average of timeouts during the last 5 minutes
61
- options: no-clear-notification
62
- to: webmaster
57
64
-template: web_service_fails
58
+template: no_web_service_connections
59
families: *
60
on: httpcheck.status
61
lookup: average -5m unaligned percentage of no_connection
62
every: 10s
63
units: %
70
- warn: $this >= 10 AND $this < 40
71
- crit: $this >= 40
72
- delay: down 5m multiplier 1.5 max 1h
64
info: average of failed requests during the last 5 minutes
65
+
66
+# combined timeout & no connection alarm
67
+template: web_service_unreachable
68
+families: *
69
+ on: httpcheck.status
70
+ calc: ($no_web_service_connections >= $web_service_timeouts) ? ($no_web_service_connections) : ($web_service_timeouts)
71
+ units: %
72
+ every: 10s
73
+ warn: ($no_web_service_connections >= 10 OR $web_service_timeouts >= 10) AND ($no_web_service_connections < 40 OR $web_service_timeouts < 40)
74
+ crit: $no_web_service_connections >= 40 OR $web_service_timeouts >= 40
75
+ delay: down 5m multiplier 1.5 max 1h
76
+ info: average of failed requests either due to timeouts or no connection during the last 5 minutes
77
options: no-clear-notification
78
to: webmaster
79
77
-#template: web_service_latency
78
-#families: *
79
-# on: httpcheck.responsetime
80
-# lookup: average -2m unaligned of response_time
81
-# every: 5s
82
-# units: %
83
-# warn: $this >= 5 AND $this < 40
84
-# crit: $this >= 40
85
-# delay: down 5m multiplier 1.5 max 1h
86
-# info:
87
-# to: sysadmin
80
+template: 1h_web_service_response_time
81
+families: *
82
+ on: httpcheck.responsetime
83
+ lookup: average -1h unaligned of time
84
+ every: 30s
85
+ units: ms
86
+ info: average response time over the last hour
87
+
88
+template: web_service_slow
89
+families: *
90
+ on: httpcheck.responsetime
91
+ lookup: average -3m unaligned of time
92
+ units: ms
93
+ every: 10s
94
+ warn: ($this > ($1h_web_service_response_time * 2) )
95
+ crit: ($this > ($1h_web_service_response_time * 3) )
96
+ info: average response time over the last 3 minutes, compared to the average over the last hour
97
+ delay: down 5m multiplier 1.5 max 1h
98
+ options: no-clear-notification
99
+ to: webmaster