@cryptotaxi247 / netdata / commits / c5016560d

improve(go.d/ddsnmp): add composite virtual metrics (#20962)

Ilya Mashchenko committed Sep 12, 2025 at 07:26 UTC c5016560d0918b06d82403c111581f350617d2b3
3 files changed +239 -40
src/go/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition/virtual_metrics.go
+1
@@ -21,4 +21,5 @@ func (vm VirtualMetricConfig) Clone() VirtualMetricConfig {
21 type VirtualMetricSourceConfig struct {
22 Metric string `yaml:"metric"`
23 Table string `yaml:"table"` // Required for now
24 + As string `yaml:"as"` // dimension name for composite charts
25 }
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector_vmetrics.go
+94 -40
@@ -3,6 +3,8 @@
3 package ddsnmpcollector
4
5 import (
6 + "slices"
7 +
8 "github.com/netdata/netdata/go/plugins/logger"
9 "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp"
10 "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition"
@@ -18,20 +20,27 @@ func newVirtualMetricsCollector(log *logger.Logger) *vmetricsCollector {
20 }
21 }
22
21 -// vmetricsSourceKey identifies a metric source
22 -type vmetricsSourceKey struct {
23 - metricName string
24 - tableName string
25 -}
26 -
27 -// vmetricsAggregator holds accumulation state for a virtual metric
28 -type vmetricsAggregator struct {
29 - config ddprofiledefinition.VirtualMetricConfig
30 - sum int64
31 - multiSum map[string]int64 // For aggregating MultiValue metrics
32 - sourceCount int
33 - metricType ddprofiledefinition.ProfileMetricType
34 -}
23 +type (
24 + // vmetricsSourceKey identifies a metric source
25 + vmetricsSourceKey struct {
26 + metricName string
27 + tableName string
28 + }
29 + // which aggregator to feed and under which dimension name
30 + vmetricsSink struct {
31 + agg *vmetricsAggregator
32 + dim string // empty == non-composite (single-source behavior)
33 + }
34 + // vmetricsAggregator holds accumulation state for a virtual metric
35 + vmetricsAggregator struct {
36 + config ddprofiledefinition.VirtualMetricConfig
37 + sum int64
38 + multiSum map[string]int64 // for aggregating MultiValue metrics
39 + perDim map[string]int64 // per-source accumulation when composite
40 + sourceCount int
41 + metricType ddprofiledefinition.ProfileMetricType
42 + }
43 +)
44
45 func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefinition, collectedMetrics []ddsnmp.Metric) []ddsnmp.Metric {
46 if len(profDef.VirtualMetrics) == 0 {
@@ -41,18 +50,42 @@ func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefiniti
50 sourceToAggregators, aggregators := p.buildAggregators(profDef)
51
52 for _, metric := range collectedMetrics {
44 - if !metric.IsTable || metric.Table == "" {
45 - continue
46 - }
47 -
53 key := vmetricsSourceKey{
54 metricName: metric.Name,
55 tableName: metric.Table,
56 }
57
53 - // Find all aggregators that need this metric
54 - if aggrs, found := sourceToAggregators[key]; found {
55 - for _, agg := range aggrs {
58 + sinks, found := sourceToAggregators[key]
59 + if !found {
60 + continue
61 + }
62 +
63 + for _, sink := range sinks {
64 + if sink.agg == nil {
65 + continue
66 + }
67 +
68 + agg := sink.agg
69 +
70 + // If sink.dim != "" => composite path (this VM has multiple sources)
71 + if sink.dim != "" {
72 + // We need a single number per source (dimension).
73 + // If the incoming base metric is MultiValue, collapse it to a total; else use Value.
74 + var v int64
75 + if len(metric.MultiValue) > 0 {
76 + for _, mv := range metric.MultiValue {
77 + v += mv
78 + }
79 + } else {
80 + v = metric.Value
81 + }
82 +
83 + if agg.perDim == nil {
84 + agg.perDim = make(map[string]int64)
85 + }
86 + agg.perDim[sink.dim] += v
87 + } else {
88 + // Non-composite (single-source) => keep existing behavior:
89 if len(metric.MultiValue) > 0 {
90 if agg.multiSum == nil {
91 agg.multiSum = make(map[string]int64)
@@ -63,10 +96,14 @@ func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefiniti
96 } else {
97 agg.sum += metric.Value
98 }
66 - agg.sourceCount++
67 - if agg.metricType == "" {
68 - agg.metricType = metric.MetricType
69 - }
99 + }
100 +
101 + agg.sourceCount++
102 + if agg.metricType == "" {
103 + agg.metricType = metric.MetricType
104 + } else if agg.metricType != metric.MetricType {
105 + p.log.Debugf("virtual metric %q mixes MetricType (%s vs %s); using %s",
106 + agg.config.Name, agg.metricType, metric.MetricType, agg.metricType)
107 }
108 }
109 }
@@ -79,22 +116,33 @@ func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefiniti
116 continue
117 }
118
82 - virtualMetrics = append(virtualMetrics, ddsnmp.Metric{
119 + vm := ddsnmp.Metric{
120 Name: agg.config.Name,
84 - Value: agg.sum,
85 - MultiValue: agg.multiSum,
121 Description: agg.config.ChartMeta.Description,
122 Family: agg.config.ChartMeta.Family,
123 Unit: agg.config.ChartMeta.Unit,
124 MetricType: agg.metricType,
90 - })
125 + }
126 +
127 + switch {
128 + case len(agg.perDim) > 0:
129 + // Composite output: one metric with MultiValue where keys are dimension names (sources)
130 + vm.MultiValue = agg.perDim
131 + case len(agg.multiSum) > 0:
132 + // Single-source whose base metric was MultiValue
133 + vm.MultiValue = agg.multiSum
134 + default:
135 + // Single-source
136 + vm.Value = agg.sum
137 + }
138 + virtualMetrics = append(virtualMetrics, vm)
139 }
140
141 return virtualMetrics
142 }
143
96 -func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.ProfileDefinition) (map[vmetricsSourceKey][]*vmetricsAggregator, []*vmetricsAggregator) {
97 - sourceToAggregators := make(map[vmetricsSourceKey][]*vmetricsAggregator)
144 +func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.ProfileDefinition) (map[vmetricsSourceKey][]vmetricsSink, []*vmetricsAggregator) {
145 + sourceToAggregators := make(map[vmetricsSourceKey][]vmetricsSink)
146 aggregators := make([]*vmetricsAggregator, 0, len(profDef.VirtualMetrics))
147
148 existingNames := p.getDefinedMetricNames(profDef.Metrics)
@@ -105,24 +153,30 @@ func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.Profil
153 continue
154 }
155
108 - agg := &vmetricsAggregator{
109 - config: config,
110 - }
156 + agg := &vmetricsAggregator{config: config}
157 aggregators = append(aggregators, agg)
158
159 + isComposite := len(config.Sources) > 1 &&
160 + slices.ContainsFunc(config.Sources, func(s ddprofiledefinition.VirtualMetricSourceConfig) bool {
161 + return s.As != ""
162 + })
163 +
164 // Register this aggregator for each source it needs
165 for _, source := range config.Sources {
115 - if source.Table == "" {
116 - p.log.Warningf("virtual metric '%s' source '%s' missing table, skipping source", config.Name, source.Metric)
117 - continue
118 - }
119 -
166 key := vmetricsSourceKey{
167 metricName: source.Metric,
168 tableName: source.Table,
169 }
170
125 - sourceToAggregators[key] = append(sourceToAggregators[key], agg)
171 + var dim string
172 + if isComposite {
173 + dim = ternary(source.As != "", source.As, source.Metric)
174 + }
175 +
176 + sourceToAggregators[key] = append(sourceToAggregators[key], vmetricsSink{
177 + agg: agg,
178 + dim: dim,
179 + })
180 }
181 }
182
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector_vmetrics_test.go
+144
@@ -632,6 +632,150 @@ func TestVirtualMetricsCollector_Collect(t *testing.T) {
632 },
633 },
634 },
635 +
636 + "composite with as (two table sources)": {
637 + profileDef: &ddprofiledefinition.ProfileDefinition{
638 + Metrics: []ddprofiledefinition.MetricsConfig{
639 + {
640 + Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.31.1.1", Name: "ifXTable"},
641 + Symbols: []ddprofiledefinition.SymbolConfig{
642 + {OID: "1.3.6.1.2.1.31.1.1.1.6", Name: "ifHCInOctets"},
643 + {OID: "1.3.6.1.2.1.31.1.1.1.10", Name: "ifHCOutOctets"},
644 + },
645 + },
646 + },
647 + VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
648 + {
649 + Name: "ifTotalTraffic",
650 + Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
651 + {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
652 + {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
653 + },
654 + ChartMeta: ddprofiledefinition.ChartMeta{
655 + Description: "Total traffic by direction",
656 + Family: "Network/Total/Traffic",
657 + Unit: "bit/s",
658 + },
659 + },
660 + },
661 + },
662 + collectedMetrics: []ddsnmp.Metric{
663 + {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable"},
664 + {Name: "ifHCInOctets", Value: 2000, IsTable: true, Table: "ifXTable"},
665 + {Name: "ifHCOutOctets", Value: 500, IsTable: true, Table: "ifXTable"},
666 + {Name: "ifHCOutOctets", Value: 1500, IsTable: true, Table: "ifXTable"},
667 + },
668 + expected: []ddsnmp.Metric{
669 + {
670 + Name: "ifTotalTraffic",
671 + MultiValue: map[string]int64{"in": 3000, "out": 2000},
672 + Description: "Total traffic by direction",
673 + Family: "Network/Total/Traffic",
674 + Unit: "bit/s",
675 + },
676 + },
677 + },
678 +
679 + "composite with as (missing one source)": {
680 + profileDef: &ddprofiledefinition.ProfileDefinition{
681 + VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
682 + {
683 + Name: "ifTotalTraffic",
684 + Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
685 + {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
686 + {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
687 + },
688 + },
689 + },
690 + },
691 + collectedMetrics: []ddsnmp.Metric{
692 + // Only IN is present
693 + {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable"},
694 + {Name: "ifHCInOctets", Value: 2000, IsTable: true, Table: "ifXTable"},
695 + },
696 + expected: []ddsnmp.Metric{
697 + {
698 + Name: "ifTotalTraffic",
699 + MultiValue: map[string]int64{"in": 3000}, // "out" omitted
700 + },
701 + },
702 + },
703 +
704 + "composite with duplicate 'as' merges values": {
705 + profileDef: &ddprofiledefinition.ProfileDefinition{
706 + Metrics: []ddprofiledefinition.MetricsConfig{
707 + {
708 + Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.2.2", Name: "ifTable"},
709 + Symbols: []ddprofiledefinition.SymbolConfig{
710 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
711 + },
712 + },
713 + {
714 + Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.31.1.1", Name: "ifXTable"},
715 + Symbols: []ddprofiledefinition.SymbolConfig{
716 + {OID: "1.3.6.1.2.1.31.1.1.1.6", Name: "ifHCInOctets"},
717 + },
718 + },
719 + },
720 + VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
721 + {
722 + Name: "ifTotalInbound",
723 + Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
724 + {Metric: "ifInOctets", Table: "ifTable", As: "in"},
725 + {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"}, // same 'as'
726 + },
727 + ChartMeta: ddprofiledefinition.ChartMeta{
728 + Description: "Inbound traffic from two tables merged",
729 + },
730 + },
731 + },
732 + },
733 + collectedMetrics: []ddsnmp.Metric{
734 + {Name: "ifInOctets", Value: 100, IsTable: true, Table: "ifTable"},
735 + {Name: "ifInOctets", Value: 200, IsTable: true, Table: "ifTable"},
736 + {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable"},
737 + {Name: "ifHCInOctets", Value: 2000, IsTable: true, Table: "ifXTable"},
738 + },
739 + expected: []ddsnmp.Metric{
740 + {
741 + Name: "ifTotalInbound",
742 + MultiValue: map[string]int64{"in": 3300}, // 100+200+1000+2000
743 + Description: "Inbound traffic from two tables merged",
744 + },
745 + },
746 + },
747 +
748 + "composite with scalar sources (CPU)": {
749 + profileDef: &ddprofiledefinition.ProfileDefinition{
750 + VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
751 + {
752 + Name: "cpu_usage",
753 + Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
754 + {Metric: "ucd.ssCpuUser", Table: "", As: "user"},
755 + {Metric: "ucd.ssCpuSystem", Table: "", As: "system"},
756 + },
757 + ChartMeta: ddprofiledefinition.ChartMeta{
758 + Description: "CPU usage breakdown",
759 + Family: "System/CPU/Usage",
760 + Unit: "%",
761 + },
762 + },
763 + },
764 + },
765 + collectedMetrics: []ddsnmp.Metric{
766 + {Name: "ucd.ssCpuUser", Value: 12, IsTable: false, Table: ""},
767 + {Name: "ucd.ssCpuSystem", Value: 5, IsTable: false, Table: ""},
768 + },
769 + expected: []ddsnmp.Metric{
770 + {
771 + Name: "cpu_usage",
772 + MultiValue: map[string]int64{"user": 12, "system": 5},
773 + Description: "CPU usage breakdown",
774 + Family: "System/CPU/Usage",
775 + Unit: "%",
776 + },
777 + },
778 + },
779 }
780
781 for name, tc := range tests {