improvement(go.d/rabbitmq): add queue status and net partitions (#18976)
Ilya Mashchenko committed
Nov 8, 2024 at 23:19 UTC
aa0326977886cb8d8f77326ccb1a3feda1265ff6
7 files changed
+208
-49
src/go/plugin/go.d/modules/rabbitmq/charts.go
+55
-19
@@ -19,6 +19,7 @@ const (
19
prioQueueChurnRate
20
21
prioNodeAvailStatus
22
+ prioNodeNetworkPartitioningStatus
23
prioNodeMemAlarmStatus
24
prioNodeDiskFreeAlarmStatus
25
prioNodeFileDescriptorsUsage
@@ -30,10 +31,11 @@ const (
31
prioNodeClusterLinkPeerTraffic
32
prioNodeUptime
33
34
+ prioVhostStatus
35
prioVhostMessagesCount
36
prioVhostMessagesRate
35
- prioVhostStatus
37
38
+ prioQueueStatus
39
prioQueueMessagesCount
40
prioQueueMessagesRate
41
)
@@ -141,6 +143,7 @@ var (
143
144
var nodeChartsTmpl = module.Charts{
145
nodeAvailStatusChartTmpl.Copy(),
146
+ nodeNetworkPartitionStatusChartTmpl.Copy(),
147
nodeMemAlarmStatusChartTmpl.Copy(),
148
nodeDiskFreeAlarmStatusChartTmpl.Copy(),
149
nodeFileDescriptorsUsageChartTmpl.Copy(),
@@ -166,6 +169,19 @@ var (
169
{ID: "node_%s_avail_status_down", Name: "down"},
170
},
171
}
172
+ nodeNetworkPartitionStatusChartTmpl = module.Chart{
173
+ ID: "node_%s_network_partition_status",
174
+ Title: "Node Network Partitioning Status",
175
+ Units: "status",
176
+ Fam: "node status",
177
+ Ctx: "rabbitmq.node_network_partition_status",
178
+ Type: module.Line,
179
+ Priority: prioNodeNetworkPartitioningStatus,
180
+ Dims: module.Dims{
181
+ {ID: "node_%s_network_partition_status_clear", Name: "clear"},
182
+ {ID: "node_%s_network_partition_status_detected", Name: "detected"},
183
+ },
184
+ }
185
nodeMemAlarmStatusChartTmpl = module.Chart{
186
ID: "node_%s_mem_alarm_status",
187
Title: "Node Memory Alarm Status",
@@ -198,7 +214,7 @@ var (
214
Units: "fd",
215
Fam: "node fds",
216
Ctx: "rabbitmq.node_file_descriptors_usage",
201
- Type: module.Stacked,
217
+ Type: module.Line,
218
Priority: prioNodeFileDescriptorsUsage,
219
Dims: module.Dims{
220
{ID: "node_%s_fds_used", Name: "used"},
@@ -210,7 +226,7 @@ var (
226
Units: "sockets",
227
Fam: "node sockets",
228
Ctx: "rabbitmq.node_sockets_usage",
213
- Type: module.Stacked,
229
+ Type: module.Line,
230
Priority: prioNodeSocketsUsage,
231
Dims: module.Dims{
232
{ID: "node_%s_sockets_used", Name: "used"},
@@ -222,7 +238,7 @@ var (
238
Units: "processes",
239
Fam: "node erlang",
240
Ctx: "rabbitmq.node_erlang_processes_usage",
225
- Type: module.Stacked,
241
+ Type: module.Line,
242
Priority: prioNodeErlangProcessesUsage,
243
Dims: module.Dims{
244
{ID: "node_%s_procs_used", Name: "used"},
@@ -234,6 +250,7 @@ var (
250
Units: "processes",
251
Fam: "node erlang",
252
Ctx: "rabbitmq.node_erlang_run_queue_processes_count",
253
+ Type: module.Line,
254
Priority: prioNodeErlangRunQueueProcessesCount,
255
Dims: module.Dims{
256
{ID: "node_%s_run_queue", Name: "length"},
@@ -298,12 +315,26 @@ var (
315
)
316
317
var vhostChartsTmpl = module.Charts{
318
+ vhostStatusChartTmpl.Copy(),
319
vhostMessageCountChartTmpl.Copy(),
320
vhostMessagesRateChartTmpl.Copy(),
303
- vhostStatusChartTmpl.Copy(),
321
}
322
323
var (
324
+ vhostStatusChartTmpl = module.Chart{
325
+ ID: "vhost_%s_status",
326
+ Title: "Vhost Status",
327
+ Units: "status",
328
+ Fam: "vhost status",
329
+ Ctx: "rabbitmq.vhost_status",
330
+ Type: module.Line,
331
+ Priority: prioVhostStatus,
332
+ Dims: module.Dims{
333
+ {ID: "vhost_%s_status_running", Name: "running"},
334
+ {ID: "vhost_%s_status_stopped", Name: "stopped"},
335
+ {ID: "vhost_%s_status_partial", Name: "partial"},
336
+ },
337
+ }
338
vhostMessageCountChartTmpl = module.Chart{
339
ID: "vhost_%s_message_count",
340
Title: "Vhost messages",
@@ -336,28 +367,33 @@ var (
367
{ID: "vhost_%s_message_stats_return_unroutable", Name: "return_unroutable", Algo: module.Incremental},
368
},
369
}
339
- vhostStatusChartTmpl = module.Chart{
340
- ID: "vhost_%s_status",
341
- Title: "Vhost Status",
342
- Units: "status",
343
- Fam: "vhost status",
344
- Ctx: "rabbitmq.vhost_status",
345
- Type: module.Line,
346
- Priority: prioVhostStatus,
347
- Dims: module.Dims{
348
- {ID: "vhost_%s_status_running", Name: "running"},
349
- {ID: "vhost_%s_status_stopped", Name: "stopped"},
350
- {ID: "vhost_%s_status_partial", Name: "partial"},
351
- },
352
- }
370
)
371
372
var queueChartsTmpl = module.Charts{
373
+ queueStatusChartTmpl.Copy(),
374
queueMessagesCountChartTmpl.Copy(),
375
queueMessagesRateChartTmpl.Copy(),
376
}
377
378
var (
379
+ queueStatusChartTmpl = module.Chart{
380
+ ID: "queue_%s_vhost_%s_node_%s_status",
381
+ Title: "Queue status",
382
+ Units: "status",
383
+ Fam: "queue status",
384
+ Ctx: "rabbitmq.queue_status",
385
+ Type: module.Line,
386
+ Priority: prioQueueStatus,
387
+ Dims: module.Dims{
388
+ {ID: "queue_%s_vhost_%s_node_%s_status_running", Name: "running"},
389
+ {ID: "queue_%s_vhost_%s_node_%s_status_down", Name: "down"},
390
+ {ID: "queue_%s_vhost_%s_node_%s_status_idle", Name: "idle"},
391
+ {ID: "queue_%s_vhost_%s_node_%s_status_crashed", Name: "crashed"},
392
+ {ID: "queue_%s_vhost_%s_node_%s_status_stopped", Name: "stopped"},
393
+ {ID: "queue_%s_vhost_%s_node_%s_status_minority", Name: "minority"},
394
+ {ID: "queue_%s_vhost_%s_node_%s_status_terminated", Name: "terminated"},
395
+ },
396
+ }
397
queueMessagesCountChartTmpl = module.Chart{
398
ID: "queue_%s_vhost_%s_node_%s_message_count",
399
Title: "Queue messages",
src/go/plugin/go.d/modules/rabbitmq/collect_nodes.go
+7
-1
@@ -28,10 +28,16 @@ func (r *RabbitMQ) collectNodes(mx map[string]int64) error {
28
mx[px+"avail_status_running"] = boolToInt(node.Running)
29
mx[px+"avail_status_down"] = boolToInt(!node.Running)
30
31
- if !node.Running || node.OsPid == "" {
31
+ if node.OsPid == "" {
32
continue
33
}
34
35
+ for _, v := range []string{"clear", "detected"} {
36
+ mx[px+"network_partition_status_"+v] = 0
37
+ }
38
+ mx[px+"network_partition_status_clear"] = boolToInt(len(node.Partitions) == 0)
39
+ mx[px+"network_partition_status_detected"] = boolToInt(len(node.Partitions) > 0)
40
+
41
mx[px+"mem_alarm_status_clear"] = boolToInt(!node.MemAlarm)
42
mx[px+"mem_alarm_status_triggered"] = boolToInt(node.MemAlarm)
43
mx[px+"disk_free_alarm_status_clear"] = boolToInt(!node.DiskFreeAlarm)
src/go/plugin/go.d/modules/rabbitmq/collect_queues.go
+10
@@ -29,6 +29,16 @@ func (r *RabbitMQ) collectQueues(mx map[string]int64) error {
29
for k, v := range stm.ToMap(q) {
30
mx[px+k] = v
31
}
32
+
33
+ // https://github.com/rabbitmq/rabbitmq-server/blob/8b554474a65857aa60b72b2dda4b6fa9b78f349b/deps/rabbitmq_management/priv/www/js/formatters.js#L552
34
+ s := q.State
35
+ if q.IdleSince != nil {
36
+ s = "idle"
37
+ }
38
+ for _, v := range []string{"running", "idle", "terminated", "down", "crashed", "stopped", "minority"} {
39
+ mx[px+"status_"+v] = 0
40
+ }
41
+ mx[px+"status_"+s] = 1
42
}
43
44
return nil
src/go/plugin/go.d/modules/rabbitmq/metadata.yaml
+39
-8
@@ -174,6 +174,10 @@ modules:
174
metric: rabbitmq.node_avail_status
175
info: RabbitMQ node is down (node ${label:node} cluster ${label:cluster_id})
176
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/rabbitmq.conf
177
+ - name: rabbitmq_node_network_partition_status
178
+ metric: rabbitmq.node_network_partition_status
179
+ info: RabbitMQ network partition detected (node ${label:node} cluster ${label:cluster_id})
180
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/rabbitmq.conf
181
- name: rabbitmq_node_mem_alarm_status_triggered
182
metric: rabbitmq.node_mem_alarm_status
183
info: RabbitMQ mem alarm triggered (node ${label:node} cluster ${label:cluster_id})
@@ -186,6 +190,14 @@ modules:
190
metric: rabbitmq.vhost_status
191
info: RabbitMQ vhost is not healthy (vhost ${label:vhost} cluster ${label:cluster_id})
192
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/rabbitmq.conf
193
+ - name: rabbitmq_queue_status_minority
194
+ metric: rabbitmq.queue_status
195
+ info: RabbitMQ queue insufficient online members (queue ${label:queue} node ${label:node} cluster ${label:cluster_id})
196
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/rabbitmq.conf
197
+ - name: rabbitmq_queue_status_unhealthy
198
+ metric: rabbitmq.queue_status
199
+ info: RabbitMQ queue is unhealthy (queue ${label:queue} node ${label:node} cluster ${label:cluster_id})
200
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/rabbitmq.conf
201
metrics:
202
folding:
203
title: Metrics
@@ -275,6 +287,13 @@ modules:
287
dimensions:
288
- name: running
289
- name: down
290
+ - name: rabbitmq.node_network_partition_status
291
+ description: Node Network Partitioning Status
292
+ unit: status
293
+ chart_type: line
294
+ dimensions:
295
+ - name: clear
296
+ - name: detected
297
- name: rabbitmq.node_mem_alarm_status
298
description: Node Memory Alarm Status
299
unit: status
@@ -360,6 +379,14 @@ modules:
379
- name: vhost
380
description: Name of the virtual host.
381
metrics:
382
+ - name: rabbitmq.vhost_status
383
+ description: Vhost Status
384
+ unit: status
385
+ chart_type: line
386
+ dimensions:
387
+ - name: running
388
+ - name: stopped
389
+ - name: partial
390
- name: rabbitmq.vhost_messages_count
391
description: Vhost messages
392
unit: messages
@@ -384,14 +411,6 @@ modules:
411
- name: deliver_get
412
- name: redeliver
413
- name: return_unroutable
387
- - name: rabbitmq.vhost_status
388
- description: Vhost Status
389
- unit: status
390
- chart_type: line
391
- dimensions:
392
- - name: running
393
- - name: stopped
394
- - name: partial
414
- name: queue
415
description: These metrics refer to the virtual host queue.
416
labels:
@@ -406,6 +425,18 @@ modules:
425
- name: queue
426
description: Name of the queue.
427
metrics:
428
+ - name: rabbitmq.queue_status
429
+ description: Queue status
430
+ unit: status
431
+ chart_type: line
432
+ dimensions:
433
+ - name: running
434
+ - name: down
435
+ - name: idle
436
+ - name: crashed
437
+ - name: stopped
438
+ - name: minority
439
+ - name: terminated
440
- name: rabbitmq.queue_messages_count
441
description: Queue messages
442
unit: messages
src/go/plugin/go.d/modules/rabbitmq/rabbitmq_test.go
+50
-18
@@ -150,8 +150,8 @@ func TestRabbitMQ_Collect(t *testing.T) {
150
"message_stats_publish_out": 0,
151
"message_stats_redeliver": 3,
152
"message_stats_return_unroutable": 0,
153
- "node_rabbit@ilyam-deb11-play_avail_status_running": 1,
153
"node_rabbit@ilyam-deb11-play_avail_status_down": 0,
154
+ "node_rabbit@ilyam-deb11-play_avail_status_running": 1,
155
"node_rabbit@ilyam-deb11-play_disk_free_alarm_status_clear": 1,
156
"node_rabbit@ilyam-deb11-play_disk_free_alarm_status_triggered": 0,
157
"node_rabbit@ilyam-deb11-play_disk_free_bytes": 46901432320,
@@ -161,6 +161,8 @@ func TestRabbitMQ_Collect(t *testing.T) {
161
"node_rabbit@ilyam-deb11-play_mem_alarm_status_triggered": 0,
162
"node_rabbit@ilyam-deb11-play_mem_available": 9935852339,
163
"node_rabbit@ilyam-deb11-play_mem_used": 142905344,
164
+ "node_rabbit@ilyam-deb11-play_network_partition_status_clear": 1,
165
+ "node_rabbit@ilyam-deb11-play_network_partition_status_detected": 0,
166
"node_rabbit@ilyam-deb11-play_peer_rabbit@pve-deb-work_cluster_link_recv_bytes": 2374358706,
167
"node_rabbit@ilyam-deb11-play_peer_rabbit@pve-deb-work_cluster_link_send_bytes": 2297379728,
168
"node_rabbit@ilyam-deb11-play_procs_available": 1048138,
@@ -169,8 +171,8 @@ func TestRabbitMQ_Collect(t *testing.T) {
171
"node_rabbit@ilyam-deb11-play_sockets_available": 0,
172
"node_rabbit@ilyam-deb11-play_sockets_used": 0,
173
"node_rabbit@ilyam-deb11-play_uptime": 241932,
172
- "node_rabbit@pve-deb-work_avail_status_running": 1,
174
"node_rabbit@pve-deb-work_avail_status_down": 0,
175
+ "node_rabbit@pve-deb-work_avail_status_running": 1,
176
"node_rabbit@pve-deb-work_disk_free_alarm_status_clear": 1,
177
"node_rabbit@pve-deb-work_disk_free_alarm_status_triggered": 0,
178
"node_rabbit@pve-deb-work_disk_free_bytes": 103827365888,
@@ -180,6 +182,8 @@ func TestRabbitMQ_Collect(t *testing.T) {
182
"node_rabbit@pve-deb-work_mem_alarm_status_triggered": 0,
183
"node_rabbit@pve-deb-work_mem_available": 14958259404,
184
"node_rabbit@pve-deb-work_mem_used": 160018432,
185
+ "node_rabbit@pve-deb-work_network_partition_status_clear": 1,
186
+ "node_rabbit@pve-deb-work_network_partition_status_detected": 0,
187
"node_rabbit@pve-deb-work_peer_rabbit@ilyam-deb11-play_cluster_link_recv_bytes": 2297460158,
188
"node_rabbit@pve-deb-work_peer_rabbit@ilyam-deb11-play_cluster_link_send_bytes": 2374459095,
189
"node_rabbit@pve-deb-work_procs_available": 1048109,
@@ -211,6 +215,13 @@ func TestRabbitMQ_Collect(t *testing.T) {
215
"queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_messages_persistent": 0,
216
"queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_messages_ready": 0,
217
"queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_messages_unacknowledged": 0,
218
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_crashed": 0,
219
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_down": 0,
220
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_idle": 0,
221
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_minority": 0,
222
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_running": 1,
223
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_stopped": 0,
224
+ "queue_MyFirstQueue_vhost_/_node_rabbit@pve-deb-work_status_terminated": 0,
225
"queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_message_stats_ack": 0,
226
"queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_message_stats_confirm": 0,
227
"queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_message_stats_deliver": 0,
@@ -229,6 +240,13 @@ func TestRabbitMQ_Collect(t *testing.T) {
240
"queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_messages_persistent": 0,
241
"queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_messages_ready": 0,
242
"queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_messages_unacknowledged": 0,
243
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_crashed": 0,
244
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_down": 0,
245
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_idle": 0,
246
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_minority": 0,
247
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_running": 1,
248
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_stopped": 0,
249
+ "queue_MyFirstQueue_vhost_myFirstVhost_node_rabbit@ilyam-deb11-play_status_terminated": 0,
250
"queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_message_stats_ack": 0,
251
"queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_message_stats_confirm": 0,
252
"queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_message_stats_deliver": 0,
@@ -247,6 +265,13 @@ func TestRabbitMQ_Collect(t *testing.T) {
265
"queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_messages_persistent": 0,
266
"queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_messages_ready": 0,
267
"queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_messages_unacknowledged": 0,
268
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_crashed": 0,
269
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_down": 0,
270
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_idle": 0,
271
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_minority": 0,
272
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_running": 1,
273
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_stopped": 0,
274
+ "queue_MySecondQueue_vhost_/_node_rabbit@pve-deb-work_status_terminated": 0,
275
"queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_message_stats_ack": 0,
276
"queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_message_stats_confirm": 0,
277
"queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_message_stats_deliver": 0,
@@ -265,22 +290,29 @@ func TestRabbitMQ_Collect(t *testing.T) {
290
"queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_messages_persistent": 0,
291
"queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_messages_ready": 0,
292
"queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_messages_unacknowledged": 0,
268
- "queue_totals_messages": 0,
269
- "queue_totals_messages_ready": 0,
270
- "queue_totals_messages_unacknowledged": 0,
271
- "vhost_/_message_stats_ack": 0,
272
- "vhost_/_message_stats_confirm": 1,
273
- "vhost_/_message_stats_deliver": 0,
274
- "vhost_/_message_stats_deliver_get": 4,
275
- "vhost_/_message_stats_deliver_no_ack": 0,
276
- "vhost_/_message_stats_get": 3,
277
- "vhost_/_message_stats_get_empty": 2,
278
- "vhost_/_message_stats_get_no_ack": 1,
279
- "vhost_/_message_stats_publish": 1,
280
- "vhost_/_message_stats_publish_in": 0,
281
- "vhost_/_message_stats_publish_out": 0,
282
- "vhost_/_message_stats_redeliver": 3,
283
- "vhost_/_message_stats_return_unroutable": 0,
293
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_crashed": 0,
294
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_down": 0,
295
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_idle": 0,
296
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_minority": 0,
297
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_running": 1,
298
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_stopped": 0,
299
+ "queue_myFirstQueue_vhost_myFirstVhost_node_rabbit@pve-deb-work_status_terminated": 0,
300
+ "queue_totals_messages": 0,
301
+ "queue_totals_messages_ready": 0,
302
+ "queue_totals_messages_unacknowledged": 0,
303
+ "vhost_/_message_stats_ack": 0,
304
+ "vhost_/_message_stats_confirm": 1,
305
+ "vhost_/_message_stats_deliver": 0,
306
+ "vhost_/_message_stats_deliver_get": 4,
307
+ "vhost_/_message_stats_deliver_no_ack": 0,
308
+ "vhost_/_message_stats_get": 3,
309
+ "vhost_/_message_stats_get_empty": 2,
310
+ "vhost_/_message_stats_get_no_ack": 1,
311
+ "vhost_/_message_stats_publish": 1,
312
+ "vhost_/_message_stats_publish_in": 0,
313
+ "vhost_/_message_stats_publish_out": 0,
314
+ "vhost_/_message_stats_redeliver": 3,
315
+ "vhost_/_message_stats_return_unroutable": 0,
316
"vhost_/_messages": 0,
317
"vhost_/_messages_ready": 0,
318
"vhost_/_messages_unacknowledged": 0,
src/go/plugin/go.d/modules/rabbitmq/restapi.go
+2
@@ -49,6 +49,7 @@ type (
49
apiNodeResp struct {
50
Name string `json:"name"`
51
OsPid string `json:"os_pid"`
52
+ Partitions []string `json:"partitions"` // network partitions https://www.rabbitmq.com/docs/partitions#detecting
53
FDTotal int64 `json:"fd_total"`
54
FDUsed int64 `json:"fd_used"`
55
MemLimit int64 `json:"mem_limit"`
@@ -89,6 +90,7 @@ type apiQueueResp struct {
90
Vhost string `json:"vhost"`
91
Type string `json:"type"`
92
State string `json:"state"`
93
+ IdleSince *any `json:"idle_since"`
94
Messages int64 `json:"messages" stm:"messages"`
95
MessagesReady int64 `json:"messages_ready" stm:"messages_ready"`
96
MessagesUnacknowledged int64 `json:"messages_unacknowledged" stm:"messages_unacknowledged"`
src/health/health.d/rabbitmq.conf
+45
-3
@@ -8,12 +8,26 @@ component: RabbitMQ
8
calc: $down
9
every: 10s
10
units: status
11
- warn: $this > 0
11
+ crit: $this > 0
12
summary: RabbitMQ node is down (node ${label:node} cluster ${label:cluster_id})
13
info: RabbitMQ node is down (node ${label:node} cluster ${label:cluster_id})
14
delay: down 1m
15
to: sysadmin
16
17
+ template: rabbitmq_node_network_partition_status
18
+ on: rabbitmq.node_network_partition_status
19
+ class: Errors
20
+ type: Messaging
21
+component: RabbitMQ
22
+ calc: $detected
23
+ every: 10s
24
+ units: status
25
+ crit: $this > 0
26
+ summary: RabbitMQ network partition detected (node ${label:node} cluster ${label:cluster_id})
27
+ info: RabbitMQ network partition detected (node ${label:node} cluster ${label:cluster_id})
28
+ delay: down 1m
29
+ to: sysadmin
30
+
31
template: rabbitmq_node_mem_alarm_status_triggered
32
on: rabbitmq.node_mem_alarm_status
33
class: Errors
@@ -51,7 +65,35 @@ component: RabbitMQ
65
every: 10s
66
units: status
67
warn: $this > 0
54
- summary: RabbitMQ vhost is not healthy (vhost ${label:vhost} cluster ${label:cluster_id})
55
- info: RabbitMQ vhost is not healthy (vhost ${label:vhost} cluster ${label:cluster_id})
68
+ summary: RabbitMQ vhost is unhealthy (vhost ${label:vhost} cluster ${label:cluster_id})
69
+ info: RabbitMQ vhost is unhealthy (vhost ${label:vhost} cluster ${label:cluster_id})
70
+ delay: down 1m
71
+ to: sysadmin
72
+
73
+ template: rabbitmq_queue_status_minority
74
+ on: rabbitmq.queue_status
75
+ class: Errors
76
+ type: Messaging
77
+component: RabbitMQ
78
+ calc: $minority
79
+ every: 10s
80
+ units: status
81
+ warn: $this > 0
82
+ summary: RabbitMQ queue insufficient online members (queue ${label:queue} node ${label:node} cluster ${label:cluster_id})
83
+ info: RabbitMQ queue insufficient online members (queue ${label:queue} node ${label:node} cluster ${label:cluster_id})
84
+ delay: down 1m
85
+ to: sysadmin
86
+
87
+ template: rabbitmq_queue_status_unhealthy
88
+ on: rabbitmq.queue_status
89
+ class: Errors
90
+ type: Messaging
91
+component: RabbitMQ
92
+ calc: $stopped + $crashed + $down
93
+ every: 10s
94
+ units: status
95
+ crit: $this > 0
96
+ summary: RabbitMQ queue is unhealthy (queue ${label:queue} node ${label:node} cluster ${label:cluster_id})
97
+ info: RabbitMQ queue is unhealthy (queue ${label:queue} node ${label:node} cluster ${label:cluster_id})
98
delay: down 1m
99
to: sysadmin