improve(go.d/ddsnmp): add per_row mode for virtual metrics (#20971)
* improve(go.d/ddsnmp): add per_row mode for virtual metrics * copilot please
Ilya Mashchenko committed
Sep 13, 2025 at 20:17 UTC
46fc931810d4d9bd15c358386f2b458bb8eae7aa
3 files changed
+243
-68
src/go/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition/virtual_metrics.go
+2
@@ -6,6 +6,7 @@ import (
6
7
type VirtualMetricConfig struct {
8
Name string `yaml:"name"`
9
+ PerRow bool `yaml:"per_row"`
10
GroupBy []string `yaml:"group_by"`
11
Sources []VirtualMetricSourceConfig `yaml:"sources"`
12
ChartMeta ChartMeta `yaml:"chart_meta"`
@@ -14,6 +15,7 @@ type VirtualMetricConfig struct {
15
func (vm VirtualMetricConfig) Clone() VirtualMetricConfig {
16
return VirtualMetricConfig{
17
Name: vm.Name,
18
+ PerRow: vm.PerRow,
19
GroupBy: slices.Clone(vm.GroupBy),
20
Sources: slices.Clone(vm.Sources),
21
ChartMeta: vm.ChartMeta,
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector_vmetrics.go
+28
-10
@@ -76,9 +76,9 @@ type (
76
77
// --- grouping controls ---
78
grouped bool // len(GroupBy) > 0
79
- perRow bool // GroupBy == ["*"]
80
- groupBy []string // explicit labels (nil for perRow/none)
81
- groupTable string // v1: sources must share same table
79
+ perRow bool // from cfg.PerRow
80
+ groupBy []string // when perRow==true, groupBy are used as row-key hints
81
+ groupTable string // sources must share the same table
82
perGroup map[string]*vmetricsGroupBucket
83
84
// --- dimensions (composite) ---
@@ -244,14 +244,9 @@ func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.Profil
244
245
agg := &vmetricsAggregator{config: cfg}
246
247
- // --- grouping detection / validation ---
248
- if len(cfg.GroupBy) > 0 {
249
- agg.grouped = true
250
- agg.perRow = slices.Contains(cfg.GroupBy, "*")
251
- if !agg.perRow {
252
- agg.groupBy = cfg.GroupBy
253
- }
247
+ agg.grouped = cfg.PerRow || len(cfg.GroupBy) > 0
248
249
+ if agg.grouped {
250
// require all sources from the same table
251
var table string
252
same := true
@@ -267,6 +262,9 @@ func (p *vmetricsCollector) buildAggregators(profDef *ddprofiledefinition.Profil
262
p.log.Warningf("virtual metric '%s' uses group_by but sources span tables or have no table; skipping (no joins yet)", cfg.Name)
263
continue
264
}
265
+
266
+ agg.perRow = cfg.PerRow
267
+ agg.groupBy = cfg.GroupBy
268
agg.groupTable = table
269
agg.perGroup = make(map[string]*vmetricsGroupBucket, 64)
270
}
@@ -344,6 +342,26 @@ func vmBuildGroupKey(tags map[string]string, agg *vmetricsAggregator) (string, b
342
if len(tags) == 0 {
343
return "", false
344
}
345
+
346
+ if len(agg.groupBy) > 0 {
347
+ agg.keyBuf.Reset()
348
+ for i, l := range agg.groupBy {
349
+ v := tags[l]
350
+ if v == "" {
351
+ // missing hint
352
+ agg.keyBuf.Reset()
353
+ goto perRowFallback
354
+ }
355
+ if i > 0 {
356
+ agg.keyBuf.WriteByte(groupKeySep)
357
+ }
358
+ agg.keyBuf.WriteString(v)
359
+ }
360
+ return agg.keyBuf.String(), true
361
+ }
362
+
363
+ perRowFallback:
364
+ // Fallback: stable key from all tags (sorted k=v)
365
keys := make([]string, 0, len(tags))
366
for k := range tags {
367
keys = append(keys, k)
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector_vmetrics_test.go
+213
-58
@@ -777,60 +777,6 @@ func TestVirtualMetricsCollector_Collect(t *testing.T) {
777
},
778
},
779
780
- "group_by per-row composite (in/out)": {
781
- profileDef: &ddprofiledefinition.ProfileDefinition{
782
- VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
783
- {
784
- Name: "ifTrafficPerRow",
785
- GroupBy: []string{"*"},
786
- Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
787
- {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
788
- {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
789
- },
790
- ChartMeta: ddprofiledefinition.ChartMeta{
791
- Description: "Per-row traffic (in/out)",
792
- Family: "Network/Interface/Traffic",
793
- Unit: "bit/s",
794
- },
795
- },
796
- },
797
- },
798
- collectedMetrics: []ddsnmp.Metric{
799
- // eth0 row
800
- {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable",
801
- Tags: map[string]string{"interface": "eth0", "ifType": "ethernetCsmacd", "ifIndex": "1"}},
802
- {Name: "ifHCOutOctets", Value: 2000, IsTable: true, Table: "ifXTable",
803
- Tags: map[string]string{"interface": "eth0", "ifType": "ethernetCsmacd", "ifIndex": "1"}},
804
- // lo row
805
- {Name: "ifHCInOctets", Value: 10, IsTable: true, Table: "ifXTable",
806
- Tags: map[string]string{"interface": "lo", "ifType": "softwareLoopback", "ifIndex": "2"}},
807
- {Name: "ifHCOutOctets", Value: 15, IsTable: true, Table: "ifXTable",
808
- Tags: map[string]string{"interface": "lo", "ifType": "softwareLoopback", "ifIndex": "2"}},
809
- },
810
- expected: []ddsnmp.Metric{
811
- {
812
- Name: "ifTrafficPerRow",
813
- IsTable: true,
814
- Table: "ifXTable",
815
- Tags: map[string]string{"interface": "eth0", "ifType": "ethernetCsmacd", "ifIndex": "1"},
816
- MultiValue: map[string]int64{"in": 1000, "out": 2000},
817
- Description: "Per-row traffic (in/out)",
818
- Family: "Network/Interface/Traffic",
819
- Unit: "bit/s",
820
- },
821
- {
822
- Name: "ifTrafficPerRow",
823
- IsTable: true,
824
- Table: "ifXTable",
825
- Tags: map[string]string{"interface": "lo", "ifType": "softwareLoopback", "ifIndex": "2"},
826
- MultiValue: map[string]int64{"in": 10, "out": 15},
827
- Description: "Per-row traffic (in/out)",
828
- Family: "Network/Interface/Traffic",
829
- Unit: "bit/s",
830
- },
831
- },
832
- },
833
-
780
"group_by explicit labels (interface,ifType) merges duplicates": {
781
profileDef: &ddprofiledefinition.ProfileDefinition{
782
VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
@@ -919,8 +865,8 @@ func TestVirtualMetricsCollector_Collect(t *testing.T) {
865
profileDef: &ddprofiledefinition.ProfileDefinition{
866
VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
867
{
922
- Name: "ifTrafficPerRow",
923
- GroupBy: []string{"*"},
868
+ Name: "ifTrafficPerRow",
869
+ PerRow: true,
870
Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
871
{Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
872
{Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
@@ -980,8 +926,8 @@ func TestVirtualMetricsCollector_Collect(t *testing.T) {
926
profileDef: &ddprofiledefinition.ProfileDefinition{
927
VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
928
{
983
- Name: "ifTrafficPerRow",
984
- GroupBy: []string{"*"},
929
+ Name: "ifTrafficPerRow",
930
+ PerRow: true,
931
Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
932
{Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
933
{Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
@@ -1018,6 +964,215 @@ func TestVirtualMetricsCollector_Collect(t *testing.T) {
964
},
965
},
966
},
967
+
968
+ "per_row composite (in/out)": {
969
+ profileDef: &ddprofiledefinition.ProfileDefinition{
970
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
971
+ {
972
+ Name: "ifTrafficPerRow",
973
+ PerRow: true,
974
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
975
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
976
+ {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
977
+ },
978
+ ChartMeta: ddprofiledefinition.ChartMeta{
979
+ Description: "Per-row traffic (in/out)",
980
+ Family: "Network/Interface/Traffic",
981
+ Unit: "bit/s",
982
+ },
983
+ },
984
+ },
985
+ },
986
+ collectedMetrics: []ddsnmp.Metric{
987
+ // eth0
988
+ {Name: "ifHCInOctets", Value: 1000, IsTable: true, Table: "ifXTable",
989
+ Tags: map[string]string{"interface": "eth0", "ifType": "ethernetCsmacd", "ifIndex": "1"}},
990
+ {Name: "ifHCOutOctets", Value: 2000, IsTable: true, Table: "ifXTable",
991
+ Tags: map[string]string{"interface": "eth0", "ifType": "ethernetCsmacd", "ifIndex": "1"}},
992
+ // lo
993
+ {Name: "ifHCInOctets", Value: 10, IsTable: true, Table: "ifXTable",
994
+ Tags: map[string]string{"interface": "lo", "ifType": "softwareLoopback", "ifIndex": "2"}},
995
+ {Name: "ifHCOutOctets", Value: 15, IsTable: true, Table: "ifXTable",
996
+ Tags: map[string]string{"interface": "lo", "ifType": "softwareLoopback", "ifIndex": "2"}},
997
+ },
998
+ expected: []ddsnmp.Metric{
999
+ {
1000
+ Name: "ifTrafficPerRow",
1001
+ IsTable: true,
1002
+ Table: "ifXTable",
1003
+ Tags: map[string]string{"interface": "eth0", "ifType": "ethernetCsmacd", "ifIndex": "1"},
1004
+ MultiValue: map[string]int64{"in": 1000, "out": 2000},
1005
+ Description: "Per-row traffic (in/out)",
1006
+ Family: "Network/Interface/Traffic",
1007
+ Unit: "bit/s",
1008
+ },
1009
+ {
1010
+ Name: "ifTrafficPerRow",
1011
+ IsTable: true,
1012
+ Table: "ifXTable",
1013
+ Tags: map[string]string{"interface": "lo", "ifType": "softwareLoopback", "ifIndex": "2"},
1014
+ MultiValue: map[string]int64{"in": 10, "out": 15},
1015
+ Description: "Per-row traffic (in/out)",
1016
+ Family: "Network/Interface/Traffic",
1017
+ Unit: "bit/s",
1018
+ },
1019
+ },
1020
+ },
1021
+
1022
+ "per_row with key hints (group_by used as row key)": {
1023
+ profileDef: &ddprofiledefinition.ProfileDefinition{
1024
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
1025
+ {
1026
+ Name: "ifTrafficPerRowHint",
1027
+ PerRow: true,
1028
+ GroupBy: []string{"interface"}, // used as row-key hint
1029
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
1030
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
1031
+ {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
1032
+ },
1033
+ },
1034
+ },
1035
+ },
1036
+ collectedMetrics: []ddsnmp.Metric{
1037
+ {Name: "ifHCInOctets", Value: 5, IsTable: true, Table: "ifXTable",
1038
+ Tags: map[string]string{"interface": "ethA", "ifType": "ethernetCsmacd", "ifIndex": "11"}},
1039
+ {Name: "ifHCOutOctets", Value: 7, IsTable: true, Table: "ifXTable",
1040
+ Tags: map[string]string{"interface": "ethA", "ifType": "ethernetCsmacd", "ifIndex": "11"}},
1041
+ {Name: "ifHCInOctets", Value: 1, IsTable: true, Table: "ifXTable",
1042
+ Tags: map[string]string{"interface": "ethB", "ifType": "ethernetCsmacd", "ifIndex": "12"}},
1043
+ {Name: "ifHCOutOctets", Value: 2, IsTable: true, Table: "ifXTable",
1044
+ Tags: map[string]string{"interface": "ethB", "ifType": "ethernetCsmacd", "ifIndex": "12"}},
1045
+ },
1046
+ expected: []ddsnmp.Metric{
1047
+ {
1048
+ Name: "ifTrafficPerRowHint",
1049
+ IsTable: true,
1050
+ Table: "ifXTable",
1051
+ Tags: map[string]string{"interface": "ethA", "ifType": "ethernetCsmacd", "ifIndex": "11"},
1052
+ MultiValue: map[string]int64{"in": 5, "out": 7},
1053
+ },
1054
+ {
1055
+ Name: "ifTrafficPerRowHint",
1056
+ IsTable: true,
1057
+ Table: "ifXTable",
1058
+ Tags: map[string]string{"interface": "ethB", "ifType": "ethernetCsmacd", "ifIndex": "12"},
1059
+ MultiValue: map[string]int64{"in": 1, "out": 2},
1060
+ },
1061
+ },
1062
+ },
1063
+
1064
+ "per_row hint missing for a row (fallback to full-tag key)": {
1065
+ profileDef: &ddprofiledefinition.ProfileDefinition{
1066
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
1067
+ {
1068
+ Name: "ifTrafficPerRowFallback",
1069
+ PerRow: true,
1070
+ GroupBy: []string{"interface"}, // hint missing on one row
1071
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
1072
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
1073
+ {Metric: "ifHCOutOctets", Table: "ifXTable", As: "out"},
1074
+ },
1075
+ },
1076
+ },
1077
+ },
1078
+ collectedMetrics: []ddsnmp.Metric{
1079
+ // has 'interface'
1080
+ {Name: "ifHCInOctets", Value: 10, IsTable: true, Table: "ifXTable",
1081
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"}},
1082
+ {Name: "ifHCOutOctets", Value: 20, IsTable: true, Table: "ifXTable",
1083
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"}},
1084
+ // missing 'interface' -> falls back to full-tag key
1085
+ {Name: "ifHCInOctets", Value: 30, IsTable: true, Table: "ifXTable",
1086
+ Tags: map[string]string{"name": "weird0", "ifIndex": "9"}},
1087
+ {Name: "ifHCOutOctets", Value: 40, IsTable: true, Table: "ifXTable",
1088
+ Tags: map[string]string{"name": "weird0", "ifIndex": "9"}},
1089
+ },
1090
+ expected: []ddsnmp.Metric{
1091
+ {
1092
+ Name: "ifTrafficPerRowFallback",
1093
+ IsTable: true,
1094
+ Table: "ifXTable",
1095
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"},
1096
+ MultiValue: map[string]int64{"in": 10, "out": 20},
1097
+ },
1098
+ {
1099
+ Name: "ifTrafficPerRowFallback",
1100
+ IsTable: true,
1101
+ Table: "ifXTable",
1102
+ Tags: map[string]string{"name": "weird0", "ifIndex": "9"},
1103
+ MultiValue: map[string]int64{"in": 30, "out": 40},
1104
+ },
1105
+ },
1106
+ },
1107
+
1108
+ "per_row single-source (value path)": {
1109
+ profileDef: &ddprofiledefinition.ProfileDefinition{
1110
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
1111
+ {
1112
+ Name: "ifInErrorsPerRow",
1113
+ PerRow: true,
1114
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
1115
+ {Metric: "ifInErrors", Table: "ifTable"},
1116
+ },
1117
+ ChartMeta: ddprofiledefinition.ChartMeta{
1118
+ Description: "Per-row inbound errors",
1119
+ Family: "Network/Interface/Errors",
1120
+ Unit: "{error}/s",
1121
+ },
1122
+ },
1123
+ },
1124
+ },
1125
+ collectedMetrics: []ddsnmp.Metric{
1126
+ {Name: "ifInErrors", Value: 5, IsTable: true, Table: "ifTable",
1127
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"}},
1128
+ {Name: "ifInErrors", Value: 7, IsTable: true, Table: "ifTable",
1129
+ Tags: map[string]string{"interface": "eth1", "ifIndex": "2"}},
1130
+ },
1131
+ expected: []ddsnmp.Metric{
1132
+ {
1133
+ Name: "ifInErrorsPerRow",
1134
+ IsTable: true,
1135
+ Table: "ifTable",
1136
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"},
1137
+ Value: 5,
1138
+ Description: "Per-row inbound errors",
1139
+ Family: "Network/Interface/Errors",
1140
+ Unit: "{error}/s",
1141
+ },
1142
+ {
1143
+ Name: "ifInErrorsPerRow",
1144
+ IsTable: true,
1145
+ Table: "ifTable",
1146
+ Tags: map[string]string{"interface": "eth1", "ifIndex": "2"},
1147
+ Value: 7,
1148
+ Description: "Per-row inbound errors",
1149
+ Family: "Network/Interface/Errors",
1150
+ Unit: "{error}/s",
1151
+ },
1152
+ },
1153
+ },
1154
+
1155
+ "per_row sources in different tables (skipped VM)": {
1156
+ profileDef: &ddprofiledefinition.ProfileDefinition{
1157
+ VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{
1158
+ {
1159
+ Name: "invalidPerRow",
1160
+ PerRow: true,
1161
+ Sources: []ddprofiledefinition.VirtualMetricSourceConfig{
1162
+ {Metric: "ifHCInOctets", Table: "ifXTable", As: "in"},
1163
+ {Metric: "ifInOctets", Table: "ifTable", As: "in2"},
1164
+ },
1165
+ },
1166
+ },
1167
+ },
1168
+ collectedMetrics: []ddsnmp.Metric{
1169
+ {Name: "ifHCInOctets", Value: 1, IsTable: true, Table: "ifXTable",
1170
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"}},
1171
+ {Name: "ifInOctets", Value: 2, IsTable: true, Table: "ifTable",
1172
+ Tags: map[string]string{"interface": "eth0", "ifIndex": "1"}},
1173
+ },
1174
+ expected: []ddsnmp.Metric{}, // VM skipped in build phase
1175
+ },
1176
}
1177
1178
for name, tc := range tests {