improve(go.d/ddsnmp): add composite virtual metrics (#20962)
Ilya Mashchenko committed
Sep 12, 2025 at 07:26 UTC
c5016560d0918b06d82403c111581f350617d2b3
3 files changed
+239
-40
src/go/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition/virtual_metrics.go
+1
@@ -21,4 +21,5 @@ func (vm VirtualMetricConfig) Clone() VirtualMetricConfig {
21
type VirtualMetricSourceConfig struct {
22
Metric string `yaml:"metric"`
23
Table string `yaml:"table"` // Required for now
24
+ As string `yaml:"as"` // dimension name for composite charts
25
}
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector_vmetrics.go
+94
-40
@@ -3,6 +3,8 @@
3
package ddsnmpcollector
4
5
import (
6
+ "slices"
7
+
8
"github.com/netdata/netdata/go/plugins/logger"
9
"github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp"
10
"github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition"
@@ -18,20 +20,27 @@ func newVirtualMetricsCollector(log *logger.Logger) *vmetricsCollector {
20
}
21
}
22
21
-// vmetricsSourceKey identifies a metric source
22
-type vmetricsSourceKey struct {
23
- metricName string
24
- tableName string
25
-}
26
-
27
-// vmetricsAggregator holds accumulation state for a virtual metric
28
-type vmetricsAggregator struct {
29
- config ddprofiledefinition.VirtualMetricConfig
30
- sum int64
31
- multiSum map[string]int64 // For aggregating MultiValue metrics
32
- sourceCount int
33
- metricType ddprofiledefinition.ProfileMetricType
34
-}
23
+type (
24
+ // vmetricsSourceKey identifies a metric source
25
+ vmetricsSourceKey struct {
26
+ metricName string
27
+ tableName string
28
+ }
29
+ // which aggregator to feed and under which dimension name
30
+ vmetricsSink struct {
31
+ agg *vmetricsAggregator
32
+ dim string // empty == non-composite (single-source behavior)
33
+ }
34
+ // vmetricsAggregator holds accumulation state for a virtual metric
35
+ vmetricsAggregator struct {
36
+ config ddprofiledefinition.VirtualMetricConfig
37
+ sum int64
38
+ multiSum map[string]int64 // for aggregating MultiValue metrics
39
+ perDim map[string]int64 // per-source accumulation when composite
40
+ sourceCount int
41
+ metricType ddprofiledefinition.ProfileMetricType
42
+ }
43
+)
44
45
func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefinition, collectedMetrics []ddsnmp.Metric) []ddsnmp.Metric {
46
if len(profDef.VirtualMetrics) == 0 {
@@ -41,18 +50,42 @@ func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefiniti
50
sourceToAggregators, aggregators := p.buildAggregators(profDef)
51
52
for _, metric := range collectedMetrics {
44
- if !metric.IsTable || metric.Table == "" {
45
- continue
46
- }
47
-
53
key := vmetricsSourceKey{
54
metricName: metric.Name,
55
tableName: metric.Table,
56
}
57
53
- // Find all aggregators that need this metric
54
- if aggrs, found := sourceToAggregators[key]; found {
55
- for _, agg := range aggrs {
58
+ sinks, found := sourceToAggregators[key]
59
+ if !found {
60
+ continue
61
+ }
62
+
63
+ for _, sink := range sinks {
64
+ if sink.agg == nil {
65
+ continue
66
+ }
67
+
68
+ agg := sink.agg
69
+
70
+ // If sink.dim != "" => composite path (this VM has multiple sources)
71
+ if sink.dim != "" {
72
+ // We need a single number per source (dimension).
73
+ // If the incoming base metric is MultiValue, collapse it to a total; else use Value.
74
+ var v int64
75
+ if len(metric.MultiValue) > 0 {
76
+ for _, mv := range metric.MultiValue {
77
+ v += mv
78
+ }
79
+ } else {
80
+ v = metric.Value
81
+ }
82
+
83
+ if agg.perDim == nil {
84
+ agg.perDim = make(map[string]int64)
85
+ }
86
+ agg.perDim[sink.dim] += v
87
+ } else {
88
+ // Non-composite (single-source) => keep existing behavior:
89
if len(metric.MultiValue) > 0 {
90
if agg.multiSum == nil {
91
agg.multiSum = make(map[string]int64)
@@ -63,10 +96,14 @@ func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefiniti
96
} else {
97
agg.sum += metric.Value
98
}
66
- agg.sourceCount++
67
- if agg.metricType == "" {
68
- agg.metricType = metric.MetricType
69
- }
99
+ }
100
+
101
+ agg.sourceCount++
102
+ if agg.metricType == "" {
103
+ agg.metricType = metric.MetricType
104
+ } else if agg.metricType != metric.MetricType {
105
+ p.log.Debugf("virtual metric %q mixes MetricType (%s vs %s); using %s",
106
+ agg.config.Name, agg.metricType, metric.MetricType, agg.metricType)
107
}
108
}
109
}
@@ -79,22 +116,33 @@ func (p *vmetricsCollector) Collect(profDef *ddprofiledefinition.ProfileDefiniti
116
continue
117
}
118
82
- virtualMetrics = append(virtualMetrics, ddsnmp.Metric{
119
+ vm := ddsnmp.Metric{
120
Name: agg.config.Name,
84
- Value: agg.sum,
85
- MultiValue: agg.multiSum,
121
Description: agg.config.ChartMeta.Description,
122
Family: agg.config.ChartMeta.Family,
123
Unit: agg.config.ChartMeta.Unit,
124
MetricType: agg.metricType,
90
- })
125
+ }
126
+
127
+ switch {
128
+ case len(agg.perDim) > 0:
129
+ // Composite output: one metric with MultiValue where keys are dimension names (sources)
130
+ vm.MultiValue = agg.perDim
131
+ case len(agg.multiSum) > 0:
132
+ // Single-source whose base metric was MultiValue
133
+ vm.MultiValue = agg.multiSum
134
+ default:
135
+ // Single-source
136
+ vm.Value = agg.sum
137
+ }
138
+ virtualMetrics = append(virtualMetrics, vm)
139
}
140
141
return virtualMetrics
142
}
143
96
-func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.ProfileDefinition) (map[vmetricsSourceKey][]*vmetricsAggregator, []*vmetricsAggregator) {
97
- sourceToAggregators := make(map[vmetricsSourceKey][]*vmetricsAggregator)
144
+func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.ProfileDefinition) (map[vmetricsSourceKey][]vmetricsSink, []*vmetricsAggregator) {
145
+ sourceToAggregators := make(map[vmetricsSourceKey][]vmetricsSink)
146
aggregators := make([]*vmetricsAggregator, 0, len(profDef.VirtualMetrics))
147
148
existingNames := p.getDefinedMetricNames(profDef.Metrics)
@@ -105,24 +153,30 @@ func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.Profil
153
continue
154
}
155
108
- agg := &vmetricsAggregator{
109
- config: config,
110
- }
156
+ agg := &vmetricsAggregator{config: config}
157
aggregators = append(aggregators, agg)
158
159
+ isComposite := len(config.Sources) > 1 &&
160
+ slices.ContainsFunc(config.Sources, func(s ddprofiledefinition.VirtualMetricSourceConfig) bool {
161
+ return s.As != ""
162
+ })
163
+
164
// Register this aggregator for each source it needs
165
for _, source := range config.Sources {
115
- if source.Table == "" {
116
- p.log.Warningf("virtual metric '%s' source '%s' missing table, skipping source", config.Name, source.Metric)
117
- continue
118
- }
119
-
166
key := vmetricsSourceKey{
167
metricName: source.Metric,
168
tableName: source.Table,
169
}
170
125
- sourceToAggregators[key] = append(sourceToAggregators[key], agg)
171
+ var dim string
172
+ if isComposite {
173
+ dim = ternary(source.As != "", source.As, source.Metric)
174
+ }
175
+
176
+ sourceToAggregators[key] = append(sourceToAggregators[key], vmetricsSink{
177
+ agg: agg,
178
+ dim: dim,
179
+ })
180
}
181
}
182
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector_vmetrics_test.go
+144
@@ -632,6 +632,150 @@ func TestVirtualMetricsCollector_Collect(t *testing.T) {
632
},
633
},
634
},
635
+
636
+ "composite with as (two table sources)": {
637
+ profileDef: &ddprofiledefinition.ProfileDefinition{
638
+ Metrics: []ddprofiledefinition.MetricsConfig{
639
+ {
640
+ Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.31.1.1", Name: "ifXTable"},
641
+ Symbols: []ddprofiledefinition.SymbolConfig{
642
+ {OID: "1.3.6.1.2.1.31.1.1.1.6", Name: "ifHCInOctets"},
643
+ {OID: "1.3.6.1.2.1.31.1.1.1.10", Name: "ifHCOutOctets"},
644
+ },
645
+ },
646
+ },
647
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
648
+ {
649
+ Name: "ifTotalTraffic",
650
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
651
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
652
+ {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
653
+ },
654
+ ChartMeta: ddprofiledefinition.ChartMeta{
655
+ Description: "Total traffic by direction",
656
+ Family: "Network/Total/Traffic",
657
+ Unit: "bit/s",
658
+ },
659
+ },
660
+ },
661
+ },
662
+ collectedMetrics: []ddsnmp.Metric{
663
+ {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable"},
664
+ {Name: "ifHCInOctets", Value: 2000, IsTable: true, Table: "ifXTable"},
665
+ {Name: "ifHCOutOctets", Value: 500, IsTable: true, Table: "ifXTable"},
666
+ {Name: "ifHCOutOctets", Value: 1500, IsTable: true, Table: "ifXTable"},
667
+ },
668
+ expected: []ddsnmp.Metric{
669
+ {
670
+ Name: "ifTotalTraffic",
671
+ MultiValue: map[string]int64{"in": 3000, "out": 2000},
672
+ Description: "Total traffic by direction",
673
+ Family: "Network/Total/Traffic",
674
+ Unit: "bit/s",
675
+ },
676
+ },
677
+ },
678
+
679
+ "composite with as (missing one source)": {
680
+ profileDef: &ddprofiledefinition.ProfileDefinition{
681
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
682
+ {
683
+ Name: "ifTotalTraffic",
684
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
685
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
686
+ {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
687
+ },
688
+ },
689
+ },
690
+ },
691
+ collectedMetrics: []ddsnmp.Metric{
692
+ // Only IN is present
693
+ {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable"},
694
+ {Name: "ifHCInOctets", Value: 2000, IsTable: true, Table: "ifXTable"},
695
+ },
696
+ expected: []ddsnmp.Metric{
697
+ {
698
+ Name: "ifTotalTraffic",
699
+ MultiValue: map[string]int64{"in": 3000}, // "out" omitted
700
+ },
701
+ },
702
+ },
703
+
704
+ "composite with duplicate 'as' merges values": {
705
+ profileDef: &ddprofiledefinition.ProfileDefinition{
706
+ Metrics: []ddprofiledefinition.MetricsConfig{
707
+ {
708
+ Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.2.2", Name: "ifTable"},
709
+ Symbols: []ddprofiledefinition.SymbolConfig{
710
+ {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
711
+ },
712
+ },
713
+ {
714
+ Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.31.1.1", Name: "ifXTable"},
715
+ Symbols: []ddprofiledefinition.SymbolConfig{
716
+ {OID: "1.3.6.1.2.1.31.1.1.1.6", Name: "ifHCInOctets"},
717
+ },
718
+ },
719
+ },
720
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
721
+ {
722
+ Name: "ifTotalInbound",
723
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
724
+ {Metric: "ifInOctets", Table: "ifTable", As: "in"},
725
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"}, // same 'as'
726
+ },
727
+ ChartMeta: ddprofiledefinition.ChartMeta{
728
+ Description: "Inbound traffic from two tables merged",
729
+ },
730
+ },
731
+ },
732
+ },
733
+ collectedMetrics: []ddsnmp.Metric{
734
+ {Name: "ifInOctets", Value: 100, IsTable: true, Table: "ifTable"},
735
+ {Name: "ifInOctets", Value: 200, IsTable: true, Table: "ifTable"},
736
+ {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable"},
737
+ {Name: "ifHCInOctets", Value: 2000, IsTable: true, Table: "ifXTable"},
738
+ },
739
+ expected: []ddsnmp.Metric{
740
+ {
741
+ Name: "ifTotalInbound",
742
+ MultiValue: map[string]int64{"in": 3300}, // 100+200+1000+2000
743
+ Description: "Inbound traffic from two tables merged",
744
+ },
745
+ },
746
+ },
747
+
748
+ "composite with scalar sources (CPU)": {
749
+ profileDef: &ddprofiledefinition.ProfileDefinition{
750
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
751
+ {
752
+ Name: "cpu_usage",
753
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
754
+ {Metric: "ucd.ssCpuUser", Table: "", As: "user"},
755
+ {Metric: "ucd.ssCpuSystem", Table: "", As: "system"},
756
+ },
757
+ ChartMeta: ddprofiledefinition.ChartMeta{
758
+ Description: "CPU usage breakdown",
759
+ Family: "System/CPU/Usage",
760
+ Unit: "%",
761
+ },
762
+ },
763
+ },
764
+ },
765
+ collectedMetrics: []ddsnmp.Metric{
766
+ {Name: "ucd.ssCpuUser", Value: 12, IsTable: false, Table: ""},
767
+ {Name: "ucd.ssCpuSystem", Value: 5, IsTable: false, Table: ""},
768
+ },
769
+ expected: []ddsnmp.Metric{
770
+ {
771
+ Name: "cpu_usage",
772
+ MultiValue: map[string]int64{"user": 12, "system": 5},
773
+ Description: "CPU usage breakdown",
774
+ Family: "System/CPU/Usage",
775
+ Unit: "%",
776
+ },
777
+ },
778
+ },
779
}
780
781
for name, tc := range tests {