Add few alarms for elasticsearch (#14197)
Fixes https://github.com/netdata/netdata/issues/14184
Ilya Mashchenko committed
Jan 6, 2023 at 14:49 UTC
bc977526fcc612937f56a0c4e86432a454df3310
2 files changed
+74
health/Makefile.am
+1
@@ -41,6 +41,7 @@ dist_healthconfig_DATA = \
41
health.d/dnsmasq_dhcp.conf \
42
health.d/dns_query.conf \
43
health.d/dockerd.conf \
44
+ health.d/elasticsearch.conf \
45
health.d/entropy.conf \
46
health.d/exporting.conf \
47
health.d/geth.conf \
health/health.d/elasticsearch.conf
new
+73
@@ -0,0 +1,73 @@
1
+# you can disable an alarm notification by setting the 'to' line to: silent
2
+
3
+# 'red' is a threshold, can't lookup the 'red' dimension - using simple pattern is a workaround.
4
+
5
+ template: elasticsearch_cluster_health_status_red
6
+ on: elasticsearch.cluster_health_status
7
+ class: Errors
8
+ type: SearchEngine
9
+component: Elasticsearch
10
+ lookup: average -5s unaligned of *ed
11
+ every: 10s
12
+ units: status
13
+ warn: $this == 1
14
+ delay: down 5m multiplier 1.5 max 1h
15
+ info: cluster health status is red.
16
+ to: sysadmin
17
+
18
+# the idea of '-10m' is to handle yellow status after node restart,
19
+# (usually) no action is required because Elasticsearch will automatically restore the green status.
20
+ template: elasticsearch_cluster_health_status_yellow
21
+ on: elasticsearch.cluster_health_status
22
+ class: Errors
23
+ type: SearchEngine
24
+component: Elasticsearch
25
+ lookup: average -10m unaligned of yellow
26
+ every: 1m
27
+ units: status
28
+ warn: $this == 1
29
+ delay: down 5m multiplier 1.5 max 1h
30
+ info: cluster health status is yellow.
31
+ to: sysadmin
32
+
33
+ template: elasticsearch_node_index_health_red
34
+ on: elasticsearch.node_index_health
35
+ class: Errors
36
+ type: SearchEngine
37
+component: Elasticsearch
38
+ lookup: average -5s unaligned of *ed
39
+ every: 10s
40
+ units: status
41
+ warn: $this == 1
42
+ delay: down 5m multiplier 1.5 max 1h
43
+ info: node index $label:index health status is red.
44
+ to: sysadmin
45
+
46
+# don't convert 'lookup' value to seconds in 'calc' due to UI showing seconds as hh:mm:ss (0 as now).
47
+
48
+ template: elasticsearch_node_indices_search_time_query
49
+ on: elasticsearch.node_indices_search_time
50
+ class: Workload
51
+ type: SearchEngine
52
+component: Elasticsearch
53
+ lookup: average -10m unaligned of query
54
+ every: 10s
55
+ units: milliseconds
56
+ warn: $this > (($status >= $WARNING) ? (20 * 1000) : (30 * 1000))
57
+ delay: down 5m multiplier 1.5 max 1h
58
+ info: search performance is degraded, queries run slowly.
59
+ to: sysadmin
60
+
61
+ template: elasticsearch_node_indices_search_time_fetch
62
+ on: elasticsearch.node_indices_search_time
63
+ class: Workload
64
+ type: SearchEngine
65
+component: Elasticsearch
66
+ lookup: average -10m unaligned of fetch
67
+ every: 10s
68
+ units: milliseconds
69
+ warn: $this > (($status >= $WARNING) ? (3 * 1000) : (5 * 1000))
70
+ crit: $this > (($status == $CRITICAL) ? (5 * 1000) : (30 * 1000))
71
+ delay: down 5m multiplier 1.5 max 1h
72
+ info: search performance is degraded, fetches run slowly.
73
+ to: sysadmin