make web_log alarms work above 30 reqs/minute; fix web_log plugin descriptions
Costa Tsaousis (ktsaou) committed
Feb 11, 2017 at 15:36 UTC
ce599880d6d8f74e95fb682a2d7a0d17439f15b8
2 files changed
+25
-13
conf.d/health.d/web_log.conf
+23
-11
@@ -16,6 +16,15 @@ families: *
16
# -----------------------------------------------------------------------------
17
# high level response code alarms
18
19
+template: 1m_requests
20
+ on: web_log.response_codes
21
+families: *
22
+ lookup: sum -1m unaligned
23
+ calc: ($this == 0)?(1):($this)
24
+ units: requests
25
+ every: 10s
26
+ info: the sum of all HTTP requests over the last minute
27
+
28
template: 1m_2xx
29
on: web_log.response_codes
30
families: *
@@ -23,7 +32,11 @@ families: *
32
calc: ($this == 0)?(1):($this)
33
units: requests
34
every: 10s
35
+ warn: ($1m_requests > 30) ? ($this > ($status >= $WARNING ) ? ( 1 ) : ( 2 )) : ( 0 )
36
+ crit: ($1m_requests > 30) ? ($this > ($status == $CRITICAL) ? ( 2 ) : ( 5 )) : ( 0 )
37
+ delay: down 15m multiplier 1.5 max 1h
38
info: the sum of successful HTTP requests over the last minute
39
+ to: webmaster
40
41
template: 1m_redirects
42
on: web_log.response_codes
@@ -32,8 +45,8 @@ families: *
45
calc: $this * 100 / ( $1m_2xx + $this )
46
units: %
47
every: 10s
35
- warn: $this > (($status >= $WARNING) ? ( 1 ) : ( 2 ))
36
- crit: $this > (($status == $CRITICAL) ? ( 2 ) : ( 5 ))
48
+ warn: ($1m_requests > 30) ? ($this > (($status >= $WARNING ) ? ( 1 ) : ( 2 )) ) : ( 0 )
49
+ crit: ($1m_requests > 30) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
50
delay: down 15m multiplier 1.5 max 1h
51
info: the ratio of HTTP redirects (3xx) vs the successful requests, \
52
over the last minute
@@ -46,8 +59,8 @@ families: *
59
calc: $this * 100 / ( $1m_2xx + $this )
60
units: %
61
every: 10s
49
- warn: $this > (($status >= $WARNING) ? ( 1 ) : ( 5 ))
50
- crit: $this > (($status == $CRITICAL) ? ( 5 ) : ( 10 ))
62
+ warn: ($1m_requests > 30) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 5 )) ) : ( 0 )
63
+ crit: ($1m_requests > 30) ? ($this > (($status == $CRITICAL) ? ( 5 ) : ( 10 )) ) : ( 0 )
64
delay: down 15m multiplier 1.5 max 1h
65
info: the ratio of HTTP bad requests (4xx) vs the successful requests, \
66
over the last minute
@@ -60,8 +73,8 @@ families: *
73
calc: $this * 100 / ( $1m_2xx + $this )
74
units: %
75
every: 10s
63
- warn: $this > (($status >= $WARNING) ? ( 1 ) : ( 2 ))
64
- crit: $this > (($status == $CRITICAL) ? ( 2 ) : ( 5 ))
76
+ warn: ($1m_requests > 30) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 2 )) ) : ( 0 )
77
+ crit: ($1m_requests > 30) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
78
delay: down 15m multiplier 1.5 max 1h
79
info: the ratio of HTTP internal server errors (5xx) vs the successful \
80
requests, over the last minute
@@ -78,7 +91,6 @@ families: *
91
every: 30s
92
info: the average time to respond to HTTP requests, over the last 10 minutes
93
81
-
94
template: web_slow
95
on: web_log.response_time
96
families: *
@@ -87,8 +99,8 @@ families: *
99
every: 10s
100
green: 500
101
red: 1000
90
- warn: $this > $green && $this > ($10m_response_time * 2)
91
- crit: $this > $red && $this > ($10m_response_time * 4)
102
+ warn: ($1m_requests > 30) ? ($this > $green && $this > ($10m_response_time * 2) ) : ( 0 )
103
+ crit: ($1m_requests > 30) ? ($this > $red && $this > ($10m_response_time * 4) ) : ( 0 )
104
delay: down 15m multiplier 1.5 max 1h
105
info: the average time to respond to HTTP requests, over the last 1 minute
106
to: webmaster
@@ -118,8 +130,8 @@ families: *
130
calc: ($5m_2xx_last > 0)?($5m_2xx_now * 100 / $5m_2xx_last):(100)
131
units: %
132
every: 30s
121
- warn: ($5m_2xx_last > 30)?($this > 200 OR $this < 50):(0)
122
- crit: ($5m_2xx_last > 30)?($this > 400 OR $this < 25):(0)
133
+ warn: ($1m_requests > 30) ? (($5m_2xx_last > 30) ? ($this > 200 OR $this < 50) : (0) ) : ( 0 )
134
+ crit: ($1m_requests > 30) ? (($5m_2xx_last > 30) ? ($this > 400 OR $this < 25) : (0) ) : ( 0 )
135
delay: down 15m multiplier 1.5 max 1h
136
options: no-clear-notification
137
info: the percentage of web requests over the last 5 minutes, \
python.d/web_log.chart.py
+2
-2
@@ -208,10 +208,10 @@ class Service(LogService):
208
209
job_name = find_job_name(self.override_name, self.name)
210
self.detailed_chart = 'CHART %s.detailed_response_codes ""' \
211
- ' "Response Codes" requests/s responses' \
211
+ ' "Detailed Response Codes" requests/s responses' \
212
' web_log.detailed_response_codes stacked 1 %s\n' % (job_name, self.update_every)
213
self.http_method_chart = 'CHART %s.http_method' \
214
- ' "" "Requests Per HTTP Method" requests/s requests' \
214
+ ' "" "Requests Per HTTP Method" requests/s "http methods"' \
215
' web_log.http_method stacked 2 %s\n' % (job_name, self.update_every)
216
217
if regex_name == 'access_apache_ext':