add alarms and dashboard info for Consul (#14163)
Ilya Mashchenko committed
Dec 23, 2022 at 15:10 UTC
2bca08132a436409b59803e0026034dd4ef823f2
3 files changed
+258
health/Makefile.am
+1
@@ -36,6 +36,7 @@ dist_healthconfig_DATA = \
36
health.d/cgroups.conf \
37
health.d/cpu.conf \
38
health.d/cockroachdb.conf \
39
+ health.d/consul.conf \
40
health.d/disks.conf \
41
health.d/dnsmasq_dhcp.conf \
42
health.d/dns_query.conf \
health/health.d/consul.conf
new
+146
@@ -0,0 +1,146 @@
1
+# you can disable an alarm notification by setting the 'to' line to: silent
2
+
3
+ template: consul_autopilot_health_status
4
+ on: consul.autopilot_health_status
5
+ class: Errors
6
+ type: ServiceMesh
7
+component: Consul
8
+ calc: $unhealthy
9
+ every: 10s
10
+ units: status
11
+ warn: $this == 1
12
+ delay: down 5m multiplier 1.5 max 1h
13
+ info: datacenter $label:datacenter cluster is unhealthy as reported by server $label:node_name
14
+ to: sysadmin
15
+
16
+ template: consul_autopilot_server_health_status
17
+ on: consul.autopilot_server_health_status
18
+ class: Errors
19
+ type: ServiceMesh
20
+component: Consul
21
+ calc: $unhealthy
22
+ every: 10s
23
+ units: status
24
+ warn: $this == 1
25
+ delay: down 5m multiplier 1.5 max 1h
26
+ info: server $label:node_name from datacenter $label:datacenter is unhealthy
27
+ to: sysadmin
28
+
29
+ template: consul_raft_leader_last_contact_time
30
+ on: consul.raft_leader_last_contact_time
31
+ class: Errors
32
+ type: ServiceMesh
33
+component: Consul
34
+ lookup: average -1m unaligned of quantile_0.5
35
+ every: 10s
36
+ units: milliseconds
37
+ warn: $this > (($status >= $WARNING) ? (150) : (200))
38
+ crit: $this > (($status == $CRITICAL) ? (200) : (500))
39
+ delay: down 5m multiplier 1.5 max 1h
40
+ info: median time elapsed since leader server $label:node_name datacenter $label:datacenter was last able to contact the follower nodes
41
+ to: sysadmin
42
+
43
+ template: consul_raft_leadership_transitions
44
+ on: consul.raft_leadership_transitions_rate
45
+ class: Errors
46
+ type: ServiceMesh
47
+component: Consul
48
+ lookup: sum -1m unaligned
49
+ every: 10s
50
+ units: percentage
51
+ warn: $this > 0
52
+ delay: down 5m multiplier 1.5 max 1h
53
+ info: there has been a leadership change and server $label:node_name datacenter $label:datacenter has become the leader
54
+ to: sysadmin
55
+
56
+ template: consul_raft_thread_main_saturation
57
+ on: consul.raft_thread_main_saturation_perc
58
+ class: Utilization
59
+ type: ServiceMesh
60
+component: Consul
61
+ lookup: average -1m unaligned of quantile_0.9
62
+ every: 10s
63
+ units: percentage
64
+ warn: $this > (($status >= $WARNING) ? (40) : (50))
65
+ delay: down 5m multiplier 1.5 max 1h
66
+ info: average saturation of the main Raft goroutine on server $label:node_name datacenter $label:datacenter
67
+ to: sysadmin
68
+
69
+ template: consul_raft_thread_fsm_saturation
70
+ on: consul.raft_thread_fsm_saturation_perc
71
+ class: Utilization
72
+ type: ServiceMesh
73
+component: Consul
74
+ lookup: average -1m unaligned of quantile_0.9
75
+ every: 10s
76
+ units: milliseconds
77
+ warn: $this > (($status >= $WARNING) ? (40) : (50))
78
+ delay: down 5m multiplier 1.5 max 1h
79
+ info: average saturation of the FSM Raft goroutine on server $label:node_name datacenter $label:datacenter
80
+ to: sysadmin
81
+
82
+ template: consul_client_rpc_requests_exceeded
83
+ on: consul.client_rpc_requests_exceeded_rate
84
+ class: Errors
85
+ type: ServiceMesh
86
+component: Consul
87
+ lookup: sum -1m unaligned
88
+ every: 10s
89
+ units: requests
90
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
91
+ delay: down 5m multiplier 1.5 max 1h
92
+ info: number of rate-limited RPC requests made by server $label:node_name datacenter $label:datacenter
93
+ to: sysadmin
94
+
95
+ template: consul_client_rpc_requests_failed
96
+ on: consul.client_rpc_requests_failed_rate
97
+ class: Errors
98
+ type: ServiceMesh
99
+component: Consul
100
+ lookup: sum -1m unaligned
101
+ every: 10s
102
+ units: requests
103
+ warn: $this > (($status >= $WARNING) ? (0) : (5))
104
+ delay: down 5m multiplier 1.5 max 1h
105
+ info: number of failed RPC requests made by server $label:node_name datacenter $label:datacenter
106
+ to: sysadmin
107
+
108
+ template: consul_node_health_check_status
109
+ on: consul.node_health_check_status
110
+ class: Errors
111
+ type: ServiceMesh
112
+component: Consul
113
+ calc: $warning + $critical
114
+ every: 10s
115
+ units: status
116
+ warn: $this != nan AND $this != 0
117
+ delay: down 5m multiplier 1.5 max 1h
118
+ info: node health check $label:check_name has failed on server $label:node_name datacenter $label:datacenter
119
+ to: sysadmin
120
+
121
+ template: consul_service_health_check_status
122
+ on: consul.service_health_check_status
123
+ class: Errors
124
+ type: ServiceMesh
125
+component: Consul
126
+ calc: $warning + $critical
127
+ every: 10s
128
+ units: status
129
+ warn: $this == 1
130
+ delay: down 5m multiplier 1.5 max 1h
131
+ info: service health check $label:check_name for service $label:service_name has failed on server $label:node_name datacenter $label:datacenter
132
+ to: sysadmin
133
+
134
+ template: consul_gc_pause_time
135
+ on: consul.gc_pause_time
136
+ class: Errors
137
+ type: ServiceMesh
138
+component: Consul
139
+ lookup: sum -1m unaligned
140
+ every: 10s
141
+ units: seconds
142
+ warn: $this > (($status >= $WARNING) ? (1) : (2))
143
+ crit: $this > (($status >= $WARNING) ? (2) : (5))
144
+ delay: down 5m multiplier 1.5 max 1h
145
+ info: time spent in stop-the-world garbage collection pauses on server $label:node_name datacenter $label:datacenter
146
+ to: sysadmin
web/gui/dashboard_info.js
+111
@@ -732,6 +732,12 @@ netdataDashboard.menu = {
732
title: 'Cassandra',
733
icon: '<i class="fas fa-database"></i>',
734
info: 'Performance metrics for Cassandra, the open source distributed NoSQL database management system'
735
+ },
736
+
737
+ 'consul': {
738
+ title: 'Consul',
739
+ icon: '<i class="fas fa-circle-notch"></i>',
740
+ info: 'Consul performance and health metrics. For details, see <a href="https://developer.hashicorp.com/consul/docs/agent/telemetry#key-metrics" target="_blank">Key Metrics</a>.'
741
}
742
};
743
@@ -4218,6 +4224,111 @@ netdataDashboard.context = {
4224
info: 'Requests for which a storage exception was encountered.'
4225
},
4226
4227
+ // ------------------------------------------------------------------------
4228
+ // Consul
4229
+ 'consul.node_health_check_status': {
4230
+ info: 'The current status of the <a href="https://developer.hashicorp.com/consul/tutorials/developer-discovery/service-registration-health-checks#monitor-a-node" target="_blank">node health check</a>. A node health check monitors the health of the entire node. If the node health check fails, Consul marks the node as unhealthy.'
4231
+ },
4232
+ 'consul.service_health_check_status': {
4233
+ info: 'The current status of the <a href="https://developer.hashicorp.com/consul/tutorials/developer-discovery/service-registration-health-checks#monitor-a-service" target="_blank">service health check</a>. A service check only affects the health of the service it is associated with. If the service health check fails, the DNS interface stops returning that service.'
4234
+ },
4235
+ 'consul.client_rpc_requests_rate': {
4236
+ info: 'The number of RPC requests to a Consul server.'
4237
+ },
4238
+ 'consul.client_rpc_requests_exceeded_rate': {
4239
+ info: 'The number of rate-limited RPC requests to a Consul server. An Increase of this metric either indicates the load is getting high enough to limit the rate or a <a href="https://developer.hashicorp.com/consul/docs/agent/config/config-files#limits" target="_blank">incorrectly configured</a> Consul agent.'
4240
+ },
4241
+ 'consul.client_rpc_requests_failed_rate': {
4242
+ info: 'The number of failed RPC requests to a Consul server.'
4243
+ },
4244
+ 'consul.memory_allocated': {
4245
+ info: 'The amount of memory allocated by the Consul process.'
4246
+ },
4247
+ 'consul.memory_sys': {
4248
+ info: 'The amount of memory obtained from the OS.'
4249
+ },
4250
+ 'consul.gc_pause_time': {
4251
+ info: 'The amount of time spent in garbage collection (GC) pauses. GC pause is a "stop-the-world" event, meaning that all runtime threads are blocked until GC completes. If memory usage is high, the Go runtime may GC so frequently that it starts to slow down Consul.'
4252
+ },
4253
+ 'consul.kvs_apply_time': {
4254
+ info: 'The time it takes to complete an update to the KV store.'
4255
+ },
4256
+ 'consul.kvs_apply_operations_rate': {
4257
+ info: 'The number of KV store updates.'
4258
+ },
4259
+ 'consul.txn_apply_time': {
4260
+ info: 'The time spent applying a transaction operation.'
4261
+ },
4262
+ 'consul.txn_apply_operations_rate': {
4263
+ info: 'The number of applied transaction operations.'
4264
+ },
4265
+ 'consul.raft_commit_time': {
4266
+ info: 'The time it takes to commit a new entry to the Raft log on the leader.'
4267
+ },
4268
+ 'consul.raft_commits_rate': {
4269
+ info: 'The number of applied Raft transactions.'
4270
+ },
4271
+ 'consul.autopilot_health_status': {
4272
+ info: 'The overall health of the local server cluster. The status is healthy if <b>all servers</b> are considered healthy by Autopilot.'
4273
+ },
4274
+ 'consul.autopilot_server_health_status': {
4275
+ info: 'Whether the server is healthy according to the current <a href="https://developer.hashicorp.com/consul/tutorials/datacenter-operations/autopilot-datacenter-operations#server-health-checking", target="_blank">Autopilot configuration</a>.'
4276
+ },
4277
+ 'consul.autopilot_server_stable_time': {
4278
+ info: 'The time this server has been in its current state.'
4279
+ },
4280
+ 'consul.autopilot_server_serf_status': {
4281
+ info: 'The SerfHealth check status for the server.'
4282
+ },
4283
+ 'consul.autopilot_server_voter_status': {
4284
+ info: 'Whether the server is a voting member of the Raft cluster.'
4285
+ },
4286
+ 'consul.autopilot_failure_tolerance': {
4287
+ info: 'The number of voting servers that the cluster can lose while continuing to function.'
4288
+ },
4289
+ 'consul.network_lan_rtt': {
4290
+ info: '<a href="https://developer.hashicorp.com/consul/docs/architecture/coordinates#working-with-coordinates" target="_blank">Estimated</a> network round-trip time between this node and other nodes of the cluster.'
4291
+ },
4292
+ 'consul.raft_leader_last_contact_time': {
4293
+ info: 'The time since the leader was last able to contact the follower nodes when checking its leader lease.'
4294
+ },
4295
+ 'consul.raft_follower_last_contact_leader_time': {
4296
+ info: 'The time elapsed since this server last contacted the leader.'
4297
+ },
4298
+ 'consul.raft_leader_elections_rate': {
4299
+ info: 'The number of leadership elections. Increments whenever a Consul server starts an election.'
4300
+ },
4301
+ 'consul.raft_leadership_transitions_rate': {
4302
+ info: 'The number of leadership elections. Increments whenever a Consul server becomes a leader.'
4303
+ },
4304
+ 'consul.server_leadership_status': {
4305
+ info: 'The Consul server leadership status.'
4306
+ },
4307
+ 'consul.raft_thread_main_saturation_perc': {
4308
+ info: 'An approximate measurement of the proportion of time the main Raft goroutine is busy and unavailable to accept new work.'
4309
+ },
4310
+ 'consul.raft_thread_fsm_saturation_perc': {
4311
+ info: 'An approximate measurement of the proportion of time the Raft FSM goroutine is busy and unavailable to accept new work.'
4312
+ },
4313
+ 'consul.raft_fsm_last_restore_duration': {
4314
+ info: 'The time taken to restore the FSM from a snapshot on an agent restart or from the leader calling <i>installSnapshot</i>.'
4315
+ },
4316
+ 'consul.raft_leader_oldest_log_age': {
4317
+ info: 'The time elapsed since the oldest journal was written to the leader\'s journal storage. This can be important for the health of replication when the write rate is high and the snapshot is large, because followers may not be able to recover from a restart if recovery takes longer than the minimum for the current leader.'
4318
+ },
4319
+ 'consul.raft_rpc_install_snapshot_time': {
4320
+ info: 'The time it takes to process the <i>installSnapshot</i> RPC call.'
4321
+ },
4322
+ 'consul.raft_boltdb_freelist_bytes': {
4323
+ info: 'The number of bytes necessary to encode the freelist metadata. When <a href="https://developer.hashicorp.com/consul/docs/agent/config/config-files#NoFreelistSync" target="_blank">raft_boltdb.NoFreelistSync</a> is set to <i>false</i> these metadata bytes must also be written to disk for each committed log.'
4324
+ },
4325
+ 'consul.raft_boltdb_logs_per_batch_rate': {
4326
+ info: 'The number of logs written per batch to the database.'
4327
+ },
4328
+ 'consul.raft_boltdb_store_logs_time': {
4329
+ info: 'The amount of time spent writing logs to the database.'
4330
+ },
4331
+
4332
// ------------------------------------------------------------------------
4333
// WMI (Process)
4334