health: make vernemq alarms less sensitive (#10770)
Ilya Mashchenko committed
Mar 16, 2021 at 18:29 UTC
d9354c86b51bdeb9b6341660ad05a10b70a2640d
1 file changed
+84
-183
health/health.d/vernemq.conf
+84
-183
@@ -18,9 +18,9 @@ template: vernemq_socket_errors
18
on: vernemq.socket_errors
19
lookup: sum -1m unaligned absolute of socket_error
20
units: errors
21
- every: 10s
22
- warn: $this > (($status == $WARNING) ? (0) : (5))
23
- delay: down 5m multiplier 1.5 max 2h
21
+ every: 1m
22
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
23
+ delay: up 2m down 5m multiplier 1.5 max 2h
24
info: socket errors in the last minute
25
to: sysadmin
26
@@ -30,9 +30,9 @@ template: vernemq_queue_message_drop
30
on: vernemq.queue_undelivered_messages
31
lookup: sum -1m unaligned absolute of queue_message_drop
32
units: dropped messages
33
- every: 10s
34
- warn: $this > (($status == $WARNING) ? (0) : (5))
35
- delay: down 5m multiplier 1.5 max 2h
33
+ every: 1m
34
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
35
+ delay: up 5m down 5m multiplier 1.5 max 2h
36
info: dropped messaged due to full queues in the last minute
37
to: sysadmin
38
@@ -40,9 +40,9 @@ template: vernemq_queue_message_expired
40
on: vernemq.queue_undelivered_messages
41
lookup: sum -1m unaligned absolute of queue_message_expired
42
units: expired messages
43
- every: 10s
44
- warn: $this > (($status == $WARNING) ? (0) : (15))
45
- delay: down 5m multiplier 1.5 max 2h
43
+ every: 1m
44
+ warn: $this > (($status >= $WARNING) ? (0) : (15))
45
+ delay: up 5m down 5m multiplier 1.5 max 2h
46
info: messages which expired before delivery in the last minute
47
to: sysadmin
48
@@ -50,9 +50,9 @@ template: vernemq_queue_message_unhandled
50
on: vernemq.queue_undelivered_messages
51
lookup: sum -1m unaligned absolute of queue_message_unhandled
52
units: unhandled messages
53
- every: 10s
54
- warn: $this > (($status == $WARNING) ? (0) : (5))
55
- delay: down 5m multiplier 1.5 max 2h
53
+ every: 1m
54
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
55
+ delay: up 5m down 5m multiplier 1.5 max 2h
56
info: unhandled messages (connections with clean session=true) in the last minute
57
to: sysadmin
58
@@ -73,11 +73,11 @@ template: vernemq_average_scheduler_utilization
73
74
template: vernemq_cluster_dropped
75
on: vernemq.cluster_dropped
76
- lookup: average -1m unaligned
77
- units: KiB/s
78
- every: 10s
76
+ lookup: sum -1m unaligned
77
+ units: KiB
78
+ every: 1m
79
warn: $this > 0
80
- delay: down 5m multiplier 1.5 max 1h
80
+ delay: up 5m down 5m multiplier 1.5 max 1h
81
info: the amount of traffic dropped during communication with the cluster nodes in the last minute
82
to: sysadmin
83
@@ -93,62 +93,35 @@ template: vernemq_netsplits
93
94
# Unsuccessful CONNACK
95
96
-template: vernemq_mqtt_connack_sent_reason_success
97
- on: vernemq.mqtt_connack_sent_reason
98
- lookup: sum -1m unaligned absolute match-names of success
99
- units: packets
100
- every: 10s
101
- info: successful v3/v5 CONNACK sent in the last minute
102
- to: sysadmin
103
-
96
template: vernemq_mqtt_connack_sent_reason_unsuccessful
97
on: vernemq.mqtt_connack_sent_reason
106
- lookup: sum -1m unaligned absolute
107
- calc: $this - $vernemq_mqtt_connack_sent_reason_success
98
+ lookup: sum -1m unaligned absolute match-names of !success,*
99
units: packets
109
- every: 10s
110
- warn: $this > 0
111
- delay: down 5m multiplier 1.5 max 2h
100
+ every: 1m
101
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
102
+ delay: up 5m down 5m multiplier 1.5 max 2h
103
info: unsuccessful v3/v5 CONNACK sent in the last minute
104
to: sysadmin
105
106
# Not normal DISCONNECT
107
117
-template: vernemq_mqtt_disconnect_received_reason_normal_disconnect
118
- on: vernemq.mqtt_disconnect_received_reason
119
- lookup: sum -1m unaligned absolute match-names of normal_disconnect
120
- units: packets
121
- every: 10s
122
- info: normal v5 DISCONNECT received in the last minute
123
- to: sysadmin
124
-
125
-template: vernemq_mqtt_disconnect_sent_reason_normal_disconnect
126
- on: vernemq.mqtt_disconnect_sent_reason
127
- lookup: sum -1m unaligned absolute match-names of normal_disconnect
128
- units: packets
129
- every: 10s
130
- info: normal v5 DISCONNECT sent in the last minute
131
- to: sysadmin
132
-
108
template: vernemq_mqtt_disconnect_received_reason_not_normal
109
on: vernemq.mqtt_disconnect_received_reason
135
- lookup: sum -1m unaligned absolute
136
- calc: $this - $vernemq_mqtt_disconnect_received_reason_normal_disconnect
110
+ lookup: sum -1m unaligned absolute match-names of !normal_disconnect,*
111
units: packets
138
- every: 10s
139
- warn: $this > 0
140
- delay: down 5m multiplier 1.5 max 2h
112
+ every: 1m
113
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
114
+ delay: up 5m down 5m multiplier 1.5 max 2h
115
info: not normal v5 DISCONNECT received in the last minute
116
to: sysadmin
117
118
template: vernemq_mqtt_disconnect_sent_reason_not_normal
119
on: vernemq.mqtt_disconnect_sent_reason
146
- lookup: sum -1m unaligned absolute
147
- calc: $this - $vernemq_mqtt_disconnect_sent_reason_normal_disconnect
120
+ lookup: sum -1m unaligned absolute match-names of !normal_disconnect,*
121
units: packets
149
- every: 10s
150
- warn: $this > 0
151
- delay: down 5m multiplier 1.5 max 2h
122
+ every: 1m
123
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
124
+ delay: up 5m down 5m multiplier 1.5 max 2h
125
info: not normal v5 DISCONNECT sent in the last minute
126
to: sysadmin
127
@@ -158,9 +131,9 @@ template: vernemq_mqtt_subscribe_error
131
on: vernemq.mqtt_subscribe_error
132
lookup: sum -1m unaligned absolute
133
units: failed ops
161
- every: 10s
162
- warn: $this > 0
163
- delay: down 5m multiplier 1.5 max 2h
134
+ every: 1m
135
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
136
+ delay: up 5m down 5m multiplier 1.5 max 2h
137
info: failed v3/v5 SUBSCRIBE operations in the last minute
138
to: sysadmin
139
@@ -168,9 +141,9 @@ template: vernemq_mqtt_subscribe_auth_error
141
on: vernemq.mqtt_subscribe_auth_error
142
lookup: sum -1m unaligned absolute
143
units: attempts
171
- every: 10s
172
- warn: $this > 0
173
- delay: down 5m multiplier 1.5 max 2h
144
+ every: 1m
145
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
146
+ delay: up 5m down 5m multiplier 1.5 max 2h
147
info: unauthorized v3/v5 SUBSCRIBE attempts in the last minute
148
to: sysadmin
149
@@ -180,9 +153,9 @@ template: vernemq_mqtt_unsubscribe_error
153
on: vernemq.mqtt_unsubscribe_error
154
lookup: sum -1m unaligned absolute
155
units: failed ops
183
- every: 10s
184
- warn: $this > 0
185
- delay: down 5m multiplier 1.5 max 2h
156
+ every: 1m
157
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
158
+ delay: up 5m down 5m multiplier 1.5 max 2h
159
info: failed v3/v5 UNSUBSCRIBE operations in the last minute
160
to: sysadmin
161
@@ -192,9 +165,9 @@ template: vernemq_mqtt_publish_errors
165
on: vernemq.mqtt_publish_errors
166
lookup: sum -1m unaligned absolute
167
units: failed ops
195
- every: 10s
196
- warn: $this > 0
197
- delay: down 5m multiplier 1.5 max 2h
168
+ every: 1m
169
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
170
+ delay: up 5m down 5m multiplier 1.5 max 2h
171
info: failed v3/v5 PUBLISH operations in the last minute
172
to: sysadmin
173
@@ -202,49 +175,31 @@ template: vernemq_mqtt_publish_auth_errors
175
on: vernemq.mqtt_publish_auth_errors
176
lookup: sum -1m unaligned absolute
177
units: attempts
205
- every: 10s
206
- warn: $this > 0
207
- delay: down 5m multiplier 1.5 max 2h
178
+ every: 1m
179
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
180
+ delay: up 5m down 5m multiplier 1.5 max 2h
181
info: unauthorized v3/v5 PUBLISH attempts in the last minute
182
to: sysadmin
183
184
# Unsuccessful and unexpected PUBACK
185
213
-template: vernemq_mqtt_puback_received_reason_success
214
- on: vernemq.mqtt_puback_received_reason
215
- lookup: sum -1m unaligned absolute match-names of success
216
- units: packets
217
- every: 10s
218
- info: successful v5 PUBACK received in the last minute
219
- to: sysadmin
220
-
221
-template: vernemq_mqtt_puback_sent_reason_success
222
- on: vernemq.mqtt_puback_sent_reason
223
- lookup: sum -1m unaligned absolute match-names of success
224
- units: packets
225
- every: 10s
226
- info: successful v5 PUBACK sent in the last minute
227
- to: sysadmin
228
-
186
template: vernemq_mqtt_puback_received_reason_unsuccessful
187
on: vernemq.mqtt_puback_received_reason
231
- lookup: sum -1m unaligned absolute
232
- calc: $this - $vernemq_mqtt_puback_received_reason_success
188
+ lookup: sum -1m unaligned absolute match-names of !success,*
189
units: packets
234
- every: 10s
235
- warn: $this > 0
236
- delay: down 5m multiplier 1.5 max 2h
190
+ every: 1m
191
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
192
+ delay: up 5m down 5m multiplier 1.5 max 2h
193
info: unsuccessful v5 PUBACK received in the last minute
194
to: sysadmin
195
196
template: vernemq_mqtt_puback_sent_reason_unsuccessful
197
on: vernemq.mqtt_puback_sent_reason
242
- lookup: sum -1m unaligned absolute
243
- calc: $this - $vernemq_mqtt_puback_sent_reason_success
198
+ lookup: sum -1m unaligned absolute match-names of !success,*
199
units: packets
245
- every: 10s
246
- warn: $this > 0
247
- delay: down 5m multiplier 1.5 max 2h
200
+ every: 1m
201
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
202
+ delay: up 5m down 5m multiplier 1.5 max 2h
203
info: unsuccessful v5 PUBACK sent in the last minute
204
to: sysadmin
205
@@ -252,49 +207,31 @@ template: vernemq_mqtt_puback_unexpected
207
on: vernemq.mqtt_puback_invalid_error
208
lookup: sum -1m unaligned absolute
209
units: messages
255
- every: 10s
256
- warn: $this > 0
257
- delay: down 5m multiplier 1.5 max 2h
210
+ every: 1m
211
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
212
+ delay: up 5m down 5m multiplier 1.5 max 2h
213
info: unexpected v3/v5 PUBACK received in the last minute
214
to: sysadmin
215
216
# Unsuccessful and unexpected PUBREC
217
263
-template: vernemq_mqtt_pubrec_received_reason_success
264
- on: vernemq.mqtt_pubrec_received_reason
265
- lookup: sum -1m unaligned absolute match-names of success
266
- units: packets
267
- every: 10s
268
- info: successful v5 PUBREC received in the last minute
269
- to: sysadmin
270
-
271
-template: vernemq_mqtt_pubrec_sent_reason_success
272
- on: vernemq.mqtt_pubrec_sent_reason
273
- lookup: sum -1m unaligned absolute match-names of success
274
- units: packets
275
- every: 10s
276
- info: successful v5 PUBREC sent in the last minute
277
- to: sysadmin
278
-
218
template: vernemq_mqtt_pubrec_received_reason_unsuccessful
219
on: vernemq.mqtt_pubrec_received_reason
281
- lookup: sum -1m unaligned absolute
282
- calc: $this - $vernemq_mqtt_pubrec_received_reason_success
220
+ lookup: sum -1m unaligned absolute match-names of !success,*
221
units: packets
284
- every: 10s
285
- warn: $this > 0
286
- delay: down 5m multiplier 1.5 max 2h
222
+ every: 1m
223
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
224
+ delay: up 5m down 5m multiplier 1.5 max 2h
225
info: unsuccessful v5 PUBREC received in the last minute
226
to: sysadmin
227
228
template: vernemq_mqtt_pubrec_sent_reason_unsuccessful
229
on: vernemq.mqtt_pubrec_sent_reason
292
- lookup: sum -1m unaligned absolute
293
- calc: $this - $vernemq_mqtt_pubrec_sent_reason_success
230
+ lookup: sum -1m unaligned absolute match-names of !success,*
231
units: packets
295
- every: 10s
296
- warn: $this > 0
297
- delay: down 5m multiplier 1.5 max 2h
232
+ every: 1m
233
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
234
+ delay: up 5m down 5m multiplier 1.5 max 2h
235
info: unsuccessful v5 PUBREC sent in the last minute
236
to: sysadmin
237
@@ -302,89 +239,53 @@ template: vernemq_mqtt_pubrec_invalid_error
239
on: vernemq.mqtt_pubrec_invalid_error
240
lookup: sum -1m unaligned absolute
241
units: messages
305
- every: 10s
306
- warn: $this > 0
307
- delay: down 5m multiplier 1.5 max 2h
242
+ every: 1m
243
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
244
+ delay: up 5m down 5m multiplier 1.5 max 2h
245
info: unexpected v3 PUBREC received in the last minute
246
to: sysadmin
247
248
# Unsuccessful PUBREL
249
313
-template: vernemq_mqtt_pubrel_received_reason_success
314
- on: vernemq.mqtt_pubrel_received_reason
315
- lookup: sum -1m unaligned absolute match-names of success
316
- units: packets
317
- every: 10s
318
- info: successful v5 PUBREL received in the last minute
319
- to: sysadmin
320
-
321
-template: vernemq_mqtt_pubrel_sent_reason_success
322
- on: vernemq.mqtt_pubrel_sent_reason
323
- lookup: sum -1m unaligned absolute match-names of success
324
- units: packets
325
- every: 10s
326
- info: successful v5 PUBREL sent in the last minute
327
- to: sysadmin
328
-
250
template: vernemq_mqtt_pubrel_received_reason_unsuccessful
251
on: vernemq.mqtt_pubrel_received_reason
331
- lookup: sum -1m unaligned absolute
332
- calc: $this - $vernemq_mqtt_pubrel_received_reason_success
252
+ lookup: sum -1m unaligned absolute match-names of !success,*
253
units: packets
334
- every: 10s
335
- warn: $this > 0
336
- delay: down 5m multiplier 1.5 max 2h
254
+ every: 1m
255
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
256
+ delay: up 5m down 5m multiplier 1.5 max 2h
257
info: unsuccessful v5 PUBREL received in the last minute
258
to: sysadmin
259
260
template: vernemq_mqtt_pubrel_sent_reason_unsuccessful
261
on: vernemq.mqtt_pubrel_sent_reason
342
- lookup: sum -1m unaligned absolute
343
- calc: $this - $vernemq_mqtt_pubrel_sent_reason_success
262
+ lookup: sum -1m unaligned absolute match-names of !success,*
263
units: packets
345
- every: 10s
346
- warn: $this > 0
347
- delay: down 5m multiplier 1.5 max 2h
264
+ every: 1m
265
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
266
+ delay: up 5m down 5m multiplier 1.5 max 2h
267
info: unsuccessful v5 PUBREL sent in the last minute
268
to: sysadmin
269
270
# Unsuccessful and unexpected PUBCOMP
271
353
-template: vernemq_mqtt_pubcomp_received_reason_success
354
- on: vernemq.mqtt_pubcomp_received_reason
355
- lookup: sum -1m unaligned absolute match-names of success
356
- units: packets
357
- every: 10s
358
- info: successful v5 PUBCOMP received in the last minute
359
- to: sysadmin
360
-
361
-template: vernemq_mqtt_pubcomp_sent_reason_success
362
- on: vernemq.mqtt_pubcomp_sent_reason
363
- lookup: sum -1m unaligned absolute match-names of success
364
- units: packets
365
- every: 10s
366
- info: successful v5 PUBCOMP sent in the last minute
367
- to: sysadmin
368
-
272
template: vernemq_mqtt_pubcomp_received_reason_unsuccessful
273
on: vernemq.mqtt_pubcomp_received_reason
371
- lookup: sum -1m unaligned absolute
372
- calc: $this - $vernemq_mqtt_pubcomp_received_reason_success
274
+ lookup: sum -1m unaligned absolute match-names of !success,*
275
units: packets
374
- every: 10s
375
- warn: $this > 0
376
- delay: down 5m multiplier 1.5 max 2h
276
+ every: 1m
277
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
278
+ delay: up 5m down 5m multiplier 1.5 max 2h
279
info: unsuccessful v5 PUBCOMP received in the last minute
280
to: sysadmin
281
282
template: vernemq_mqtt_pubcomp_sent_reason_unsuccessful
283
on: vernemq.mqtt_pubcomp_sent_reason
382
- lookup: sum -1m unaligned absolute
383
- calc: $this - $vernemq_mqtt_pubcomp_sent_reason_success
284
+ lookup: sum -1m unaligned absolute match-names of !success,*
285
units: packets
385
- every: 10s
386
- warn: $this > 0
387
- delay: down 5m multiplier 1.5 max 2h
286
+ every: 1m
287
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
288
+ delay: up 5m down 5m multiplier 1.5 max 2h
289
info: unsuccessful v5 PUBCOMP sent in the last minute
290
to: sysadmin
291
@@ -392,8 +293,8 @@ template: vernemq_mqtt_pubcomp_unexpected
293
on: vernemq.mqtt_pubcomp_invalid_error
294
lookup: sum -1m unaligned absolute
295
units: messages
395
- every: 10s
396
- warn: $this > 0
397
- delay: down 5m multiplier 1.5 max 2h
296
+ every: 1m
297
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
298
+ delay: up 5m down 5m multiplier 1.5 max 2h
299
info: unexpected v3/v5 PUBCOMP received in the last minute
300
to: sysadmin