master
go 587 lines 15.4 KB
Raw
1 package mq
2
3 import (
4 "fmt"
5 "sort"
6 "strings"
7 "time"
8
9 "github.com/netdata/netdata/go/plugins/plugin/ibm.d/modules/mq/contexts"
10 "github.com/netdata/netdata/go/plugins/plugin/ibm.d/protocols/pcf"
11 )
12
13 type queueGroupAggregate struct {
14 DepthCurrent int64
15 DepthMax int64
16
17 MessagesEnqueued int64
18 MessagesDequeued int64
19 HasMessages bool
20
21 ConnectionsInput int64
22 ConnectionsOutput int64
23 HasConnections bool
24
25 Uncommitted int64
26 HasUncommitted bool
27
28 FileSizeCurrent int64
29 FileSizeMax int64
30 HasFileSize bool
31
32 OldestMessageAge int64
33 HasOldest bool
34 }
35
36 func (a *queueGroupAggregate) add(queue *pcf.QueueMetrics) {
37 a.DepthCurrent += queue.CurrentDepth
38 a.DepthMax += queue.MaxDepth
39
40 if queue.HasResetStats {
41 a.MessagesEnqueued += queue.EnqueueCount
42 a.MessagesDequeued += queue.DequeueCount
43 a.HasMessages = true
44 }
45
46 if queue.HasStatusMetrics {
47 if queue.OpenInputCount.IsCollected() {
48 a.ConnectionsInput += queue.OpenInputCount.Int64()
49 a.HasConnections = true
50 }
51 if queue.OpenOutputCount.IsCollected() {
52 a.ConnectionsOutput += queue.OpenOutputCount.Int64()
53 a.HasConnections = true
54 }
55 if queue.UncommittedMsgs.IsCollected() {
56 a.Uncommitted += queue.UncommittedMsgs.Int64()
57 a.HasUncommitted = true
58 }
59 if queue.CurrentFileSize.IsCollected() {
60 a.FileSizeCurrent += queue.CurrentFileSize.Int64()
61 a.HasFileSize = true
62 }
63 if queue.CurrentMaxFileSize.IsCollected() {
64 a.FileSizeMax += queue.CurrentMaxFileSize.Int64()
65 a.HasFileSize = true
66 }
67 if queue.OldestMsgAge.IsCollected() {
68 age := queue.OldestMsgAge.Int64()
69 if age >= 0 {
70 if !a.HasOldest || age > a.OldestMessageAge {
71 a.OldestMessageAge = age
72 }
73 a.HasOldest = true
74 }
75 }
76 }
77 }
78
79 func queueGroupKey(name string) string {
80 if name == "" {
81 return "__unknown__"
82 }
83 if strings.HasPrefix(name, "SYSTEM.") {
84 return "SYSTEM"
85 }
86 parts := strings.Split(name, ".")
87 switch len(parts) {
88 case 0:
89 return "__unknown__"
90 case 1:
91 return parts[0]
92 default:
93 return strings.Join(parts[:2], ".")
94 }
95 }
96
97 // convertMQDateTimeToSecondsSince converts MQ date (YYYYMMDD) and time (HHMMSSSS) to seconds since that time
98 // Returns -1 if the date/time is invalid or not collected
99 func convertMQDateTimeToSecondsSince(date, timeVal pcf.AttributeValue) int64 {
100 if !date.IsCollected() || !timeVal.IsCollected() {
101 return -1
102 }
103
104 dateInt := date.Int64()
105 timeInt := timeVal.Int64()
106
107 // Extract date components from YYYYMMDD
108 year := dateInt / 10000
109 month := (dateInt % 10000) / 100
110 day := dateInt % 100
111
112 // Extract time components from HHMMSSSS
113 hour := timeInt / 1000000
114 minute := (timeInt % 1000000) / 10000
115 second := (timeInt % 10000) / 100
116 centisecond := timeInt % 100
117
118 // Create time.Time object
119 t := time.Date(int(year), time.Month(month), int(day),
120 int(hour), int(minute), int(second), int(centisecond)*10000000,
121 time.UTC)
122
123 // Calculate seconds since that time
124 secondsSince := int64(time.Since(t).Seconds())
125
126 // Return -1 if the time is in the future or too far in the past (invalid)
127 if secondsSince < 0 || secondsSince > 365*24*60*60 { // More than a year
128 return -1
129 }
130
131 return secondsSince
132 }
133
134 func (c *Collector) shouldCollectQueue(name string) bool {
135 included := true
136 if len(c.Config.IncludeQueues) > 0 {
137 if c.queueIncludeMatcher == nil {
138 return false
139 }
140 included = c.queueIncludeMatcher.MatchString(name)
141 if !included {
142 return false
143 }
144 }
145
146 if c.queueExcludeMatcher != nil && c.queueExcludeMatcher.MatchString(name) {
147 if len(c.Config.IncludeQueues) > 0 {
148 return included
149 }
150 return false
151 }
152
153 return true
154 }
155
156 func (c *Collector) collectQueueMetrics() error {
157 c.Debugf("Collecting queues include=%v exclude=%v config=%v reset_stats=%v system=%v",
158 c.Config.IncludeQueues, c.Config.ExcludeQueues, c.Config.CollectQueueConfig, c.Config.CollectResetQueueStats, c.Config.CollectSystemQueues)
159
160 result, err := c.client.GetQueues(
161 c.Config.CollectQueueConfig,
162 true,
163 c.Config.CollectResetQueueStats,
164 0, // fetch everything; we enforce limits locally
165 "*", // selector - we perform filtering ourselves
166 c.Config.CollectSystemQueues,
167 )
168 if err != nil {
169 return fmt.Errorf("failed to collect queue metrics: %w", err)
170 }
171
172 if !result.Stats.Discovery.Success {
173 c.Errorf("Queue discovery failed completely")
174 return fmt.Errorf("queue discovery failed")
175 }
176
177 failed := result.Stats.Discovery.UnparsedItems
178 if result.Stats.Metrics != nil {
179 failed += result.Stats.Metrics.FailedItems
180 }
181
182 filtered := make([]*pcf.QueueMetrics, 0, len(result.Queues))
183 for i := range result.Queues {
184 queue := result.Queues[i]
185 if c.shouldCollectQueue(queue.Name) {
186 queueCopy := queue
187 filtered = append(filtered, &queueCopy)
188 }
189 }
190
191 excludedByFilter := int64(len(result.Queues) - len(filtered))
192 monitored := int64(len(filtered))
193
194 c.setQueueOverviewMetrics(
195 monitored,
196 result.Stats.Discovery.ExcludedItems+excludedByFilter,
197 result.Stats.Discovery.InvisibleItems,
198 failed,
199 )
200
201 if len(filtered) == 0 {
202 c.Debugf("No queues matched the include/exclude patterns")
203 c.clearWarnOnce("queue_overflow")
204 return nil
205 }
206
207 sort.Slice(filtered, func(i, j int) bool {
208 return filtered[i].Name < filtered[j].Name
209 })
210
211 limit := max(c.Config.MaxQueues, 0)
212
213 aggregated := make(map[string]*queueGroupAggregate)
214 overflowTotals := &queueGroupAggregate{}
215 overflowCount := 0
216 overflowGroups := make(map[string]int)
217 overflowExamples := make(map[string]string)
218
219 for idx, queue := range filtered {
220 groupKey := queueGroupKey(queue.Name)
221 agg := aggregated[groupKey]
222 if agg == nil {
223 agg = &queueGroupAggregate{}
224 aggregated[groupKey] = agg
225 }
226 agg.add(queue)
227
228 if limit == 0 || idx < limit {
229 c.emitPerQueueMetrics(queue)
230 continue
231 }
232
233 overflowTotals.add(queue)
234 overflowCount++
235 overflowGroups[groupKey]++
236 if _, exists := overflowExamples[groupKey]; !exists {
237 overflowExamples[groupKey] = queue.Name
238 }
239 }
240
241 c.emitQueueGroupMetrics(aggregated)
242
243 if overflowCount > 0 {
244 c.emitPerQueueOverflowMetrics(overflowTotals)
245
246 parts := make([]string, 0, len(overflowGroups))
247 for group, count := range overflowGroups {
248 sample := overflowExamples[group]
249 parts = append(parts, fmt.Sprintf("%s:%d (e.g. %s)", group, count, sample))
250 }
251 sort.Strings(parts)
252 c.warnOnce("queue_overflow", "too many queues for per-queue charts (MaxQueues=%d). Aggregated %d additional queues: %s", limit, overflowCount, strings.Join(parts, ", "))
253 } else {
254 c.clearWarnOnce("queue_overflow")
255 }
256
257 c.Debugf("queue collection complete - discovered:%d matched:%d overflow:%d groups:%d",
258 len(result.Queues), len(filtered), overflowCount, len(aggregated))
259
260 return nil
261 }
262
263 func (c *Collector) emitPerQueueMetrics(queue *pcf.QueueMetrics) {
264 labels := contexts.QueueLabels{
265 Queue: queue.Name,
266 Type: pcf.QueueTypeString(int32(queue.Type)),
267 }
268
269 contexts.Queue.Depth.Set(c.State, labels, contexts.QueueDepthValues{
270 Current: queue.CurrentDepth,
271 Max: queue.MaxDepth,
272 })
273
274 if queue.MaxDepth > 0 {
275 percentage := float64(queue.CurrentDepth) / float64(queue.MaxDepth) * 100.0
276 contexts.Queue.DepthPercentage.Set(c.State, labels, contexts.QueueDepthPercentageValues{
277 Percentage: int64(percentage * 1000),
278 })
279 }
280
281 if queue.HasStatusMetrics {
282 if queue.OpenInputCount.IsCollected() && queue.OpenOutputCount.IsCollected() {
283 contexts.Queue.Connections.Set(c.State, labels, contexts.QueueConnectionsValues{
284 Input: queue.OpenInputCount.Int64(),
285 Output: queue.OpenOutputCount.Int64(),
286 })
287 }
288
289 if queue.OldestMsgAge.IsCollected() && queue.OldestMsgAge.Int64() != -1 {
290 contexts.Queue.OldestMessageAge.Set(c.State, labels, contexts.QueueOldestMessageAgeValues{
291 Oldest_msg_age: queue.OldestMsgAge.Int64(),
292 })
293 }
294
295 if queue.UncommittedMsgs.IsCollected() {
296 contexts.Queue.UncommittedMessages.Set(c.State, labels, contexts.QueueUncommittedMessagesValues{
297 Uncommitted: queue.UncommittedMsgs.Int64(),
298 })
299 }
300
301 if queue.CurrentFileSize.IsCollected() || queue.CurrentMaxFileSize.IsCollected() {
302 fileSizeValues := contexts.QueueFileSizeValues{}
303 hasAny := false
304 if queue.CurrentFileSize.IsCollected() {
305 fileSizeValues.Current = queue.CurrentFileSize.Int64()
306 hasAny = true
307 }
308 if queue.CurrentMaxFileSize.IsCollected() {
309 fileSizeValues.Max = queue.CurrentMaxFileSize.Int64()
310 hasAny = true
311 }
312 if hasAny {
313 contexts.Queue.FileSize.Set(c.State, labels, fileSizeValues)
314 }
315 }
316
317 if queue.QTimeShort.IsCollected() && queue.QTimeLong.IsCollected() &&
318 queue.QTimeShort.Int64() != -1 && queue.QTimeLong.Int64() != -1 {
319 contexts.Queue.QueueTimeIndicators.Set(c.State, labels, contexts.QueueQueueTimeIndicatorsValues{
320 Short_period: queue.QTimeShort.Int64(),
321 Long_period: queue.QTimeLong.Int64(),
322 })
323 }
324
325 sinceLastGet := convertMQDateTimeToSecondsSince(queue.LastGetDate, queue.LastGetTime)
326 sinceLastPut := convertMQDateTimeToSecondsSince(queue.LastPutDate, queue.LastPutTime)
327 if sinceLastGet >= 0 || sinceLastPut >= 0 {
328 if sinceLastGet < 0 {
329 sinceLastGet = -1
330 }
331 if sinceLastPut < 0 {
332 sinceLastPut = -1
333 }
334 contexts.Queue.LastActivity.Set(c.State, labels, contexts.QueueLastActivityValues{
335 Since_last_get: sinceLastGet,
336 Since_last_put: sinceLastPut,
337 })
338 }
339 }
340
341 if queue.HasResetStats {
342 contexts.Queue.Messages.Set(c.State, labels, contexts.QueueMessagesValues{
343 Enqueued: queue.EnqueueCount,
344 Dequeued: queue.DequeueCount,
345 })
346 }
347
348 contexts.Queue.HighDepth.Set(c.State, labels, contexts.QueueHighDepthValues{
349 High_depth: queue.HighDepth,
350 })
351
352 if queue.InhibitGet.IsCollected() && queue.InhibitPut.IsCollected() {
353 contexts.Queue.InhibitStatus.Set(c.State, labels, contexts.QueueInhibitStatusValues{
354 Inhibit_get: queue.InhibitGet.Int64(),
355 Inhibit_put: queue.InhibitPut.Int64(),
356 })
357 }
358
359 if queue.MaxMsgLength.IsCollected() {
360 contexts.Queue.MaxMessageLength.Set(c.State, labels, contexts.QueueMaxMessageLengthValues{
361 Max_msg_length: queue.MaxMsgLength.Int64(),
362 })
363 }
364
365 if !c.Config.CollectQueueConfig {
366 return
367 }
368
369 if queue.DefPriority.IsCollected() {
370 contexts.Queue.Priority.Set(c.State, labels, contexts.QueuePriorityValues{
371 Def_priority: queue.DefPriority.Int64(),
372 })
373 }
374
375 if queue.TriggerDepth.IsCollected() && queue.TriggerType.IsCollected() {
376 contexts.Queue.Triggers.Set(c.State, labels, contexts.QueueTriggersValues{
377 Trigger_depth: queue.TriggerDepth.Int64(),
378 Trigger_type: queue.TriggerType.Int64(),
379 })
380 }
381
382 if queue.BackoutThreshold.IsCollected() {
383 contexts.Queue.BackoutThreshold.Set(c.State, labels, contexts.QueueBackoutThresholdValues{
384 Backout_threshold: queue.BackoutThreshold.Int64(),
385 })
386 }
387
388 if queue.ServiceInterval.IsCollected() {
389 contexts.Queue.ServiceInterval.Set(c.State, labels, contexts.QueueServiceIntervalValues{
390 Service_interval: queue.ServiceInterval.Int64(),
391 })
392 }
393
394 if queue.RetentionInterval.IsCollected() {
395 contexts.Queue.RetentionInterval.Set(c.State, labels, contexts.QueueRetentionIntervalValues{
396 Retention_interval: queue.RetentionInterval.Int64(),
397 })
398 }
399
400 if queue.DefPersistence.IsCollected() {
401 persistent := int64(0)
402 nonPersistent := int64(0)
403 if queue.DefPersistence.Int64() == 1 {
404 persistent = 1
405 } else {
406 nonPersistent = 1
407 }
408 contexts.Queue.MessagePersistence.Set(c.State, labels, contexts.QueueMessagePersistenceValues{
409 Persistent: persistent,
410 Non_persistent: nonPersistent,
411 })
412 }
413
414 if queue.Scope.IsCollected() {
415 queueManager := int64(0)
416 cell := int64(0)
417 if queue.Scope.Int64() == 0 {
418 queueManager = 1
419 } else {
420 cell = 1
421 }
422 contexts.Queue.QueueScope.Set(c.State, labels, contexts.QueueQueueScopeValues{
423 Queue_manager: queueManager,
424 Cell: cell,
425 })
426 }
427
428 if queue.Usage.IsCollected() {
429 normal := int64(0)
430 transmission := int64(0)
431 if queue.Usage.Int64() == 0 {
432 normal = 1
433 } else {
434 transmission = 1
435 }
436 contexts.Queue.QueueUsage.Set(c.State, labels, contexts.QueueQueueUsageValues{
437 Normal: normal,
438 Transmission: transmission,
439 })
440 }
441
442 if queue.MsgDeliverySequence.IsCollected() {
443 priority := int64(0)
444 fifo := int64(0)
445 if queue.MsgDeliverySequence.Int64() == 0 {
446 priority = 1
447 } else {
448 fifo = 1
449 }
450 contexts.Queue.MessageDeliverySequence.Set(c.State, labels, contexts.QueueMessageDeliverySequenceValues{
451 Priority: priority,
452 Fifo: fifo,
453 })
454 }
455
456 if queue.HardenGetBackout.IsCollected() {
457 enabled := int64(0)
458 disabled := int64(0)
459 if queue.HardenGetBackout.Int64() == 1 {
460 enabled = 1
461 } else {
462 disabled = 1
463 }
464 contexts.Queue.HardenGetBackout.Set(c.State, labels, contexts.QueueHardenGetBackoutValues{
465 Enabled: enabled,
466 Disabled: disabled,
467 })
468 }
469 }
470
471 func (c *Collector) emitPerQueueOverflowMetrics(total *queueGroupAggregate) {
472 if total == nil {
473 return
474 }
475
476 labels := contexts.QueueLabels{
477 Queue: "__other__",
478 Type: "aggregated",
479 }
480
481 contexts.Queue.Depth.Set(c.State, labels, contexts.QueueDepthValues{
482 Current: total.DepthCurrent,
483 Max: total.DepthMax,
484 })
485
486 if total.DepthMax > 0 {
487 percentage := float64(total.DepthCurrent) / float64(total.DepthMax) * 100.0
488 contexts.Queue.DepthPercentage.Set(c.State, labels, contexts.QueueDepthPercentageValues{
489 Percentage: int64(percentage * 1000),
490 })
491 }
492
493 if total.HasMessages {
494 contexts.Queue.Messages.Set(c.State, labels, contexts.QueueMessagesValues{
495 Enqueued: total.MessagesEnqueued,
496 Dequeued: total.MessagesDequeued,
497 })
498 }
499
500 if total.HasConnections {
501 contexts.Queue.Connections.Set(c.State, labels, contexts.QueueConnectionsValues{
502 Input: total.ConnectionsInput,
503 Output: total.ConnectionsOutput,
504 })
505 }
506
507 if total.HasUncommitted {
508 contexts.Queue.UncommittedMessages.Set(c.State, labels, contexts.QueueUncommittedMessagesValues{
509 Uncommitted: total.Uncommitted,
510 })
511 }
512
513 if total.HasFileSize {
514 contexts.Queue.FileSize.Set(c.State, labels, contexts.QueueFileSizeValues{
515 Current: total.FileSizeCurrent,
516 Max: total.FileSizeMax,
517 })
518 }
519
520 if total.HasOldest {
521 contexts.Queue.OldestMessageAge.Set(c.State, labels, contexts.QueueOldestMessageAgeValues{
522 Oldest_msg_age: total.OldestMessageAge,
523 })
524 }
525 }
526
527 func (c *Collector) emitQueueGroupMetrics(groups map[string]*queueGroupAggregate) {
528 if len(groups) == 0 {
529 return
530 }
531
532 keys := make([]string, 0, len(groups))
533 for k := range groups {
534 keys = append(keys, k)
535 }
536 sort.Strings(keys)
537
538 for _, key := range keys {
539 agg := groups[key]
540 labels := contexts.QueueGroupLabels{Group: key}
541
542 contexts.QueueGroup.Depth.Set(c.State, labels, contexts.QueueGroupDepthValues{
543 Current: agg.DepthCurrent,
544 Max: agg.DepthMax,
545 })
546
547 if agg.DepthMax > 0 {
548 percentage := float64(agg.DepthCurrent) / float64(agg.DepthMax) * 100.0
549 contexts.QueueGroup.DepthPercentage.Set(c.State, labels, contexts.QueueGroupDepthPercentageValues{
550 Percentage: int64(percentage * 1000),
551 })
552 }
553
554 if agg.HasMessages {
555 contexts.QueueGroup.Messages.Set(c.State, labels, contexts.QueueGroupMessagesValues{
556 Enqueued: agg.MessagesEnqueued,
557 Dequeued: agg.MessagesDequeued,
558 })
559 }
560
561 if agg.HasConnections {
562 contexts.QueueGroup.Connections.Set(c.State, labels, contexts.QueueGroupConnectionsValues{
563 Input: agg.ConnectionsInput,
564 Output: agg.ConnectionsOutput,
565 })
566 }
567
568 if agg.HasUncommitted {
569 contexts.QueueGroup.UncommittedMessages.Set(c.State, labels, contexts.QueueGroupUncommittedMessagesValues{
570 Uncommitted: agg.Uncommitted,
571 })
572 }
573
574 if agg.HasFileSize {
575 contexts.QueueGroup.FileSize.Set(c.State, labels, contexts.QueueGroupFileSizeValues{
576 Current: agg.FileSizeCurrent,
577 Max: agg.FileSizeMax,
578 })
579 }
580
581 if agg.HasOldest {
582 contexts.QueueGroup.OldestMessageAge.Set(c.State, labels, contexts.QueueGroupOldestMessageAgeValues{
583 Oldest_msg_age: agg.OldestMessageAge,
584 })
585 }
586 }
587 }