improve(go.d/snmp-profiles): allow users to add custom SNMP profiles (#20526)
Ilya Mashchenko committed
Jun 19, 2025 at 16:11 UTC
829248181700f9bf15d9efe3adc3b02805dd1238
4 files changed
+438
-130
src/go/pkg/multipath/multipath.go
+1
-1
@@ -35,7 +35,7 @@ func New(paths ...string) MultiPath {
35
if dir == "" {
36
continue
37
}
38
- if d, err := homedir.Expand(dir); err != nil {
38
+ if d, err := homedir.Expand(dir); err == nil {
39
dir = d
40
}
41
if !set[dir] {
src/go/plugin/go.d/collector/snmp/ddsnmp/load.go
new
+222
@@ -0,0 +1,222 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+package ddsnmp
4
+
5
+import (
6
+ "errors"
7
+ "fmt"
8
+ "io/fs"
9
+ "os"
10
+ "path/filepath"
11
+ "slices"
12
+ "strings"
13
+ "sync"
14
+
15
+ "gopkg.in/yaml.v2"
16
+
17
+ "github.com/netdata/netdata/go/plugins/logger"
18
+ "github.com/netdata/netdata/go/plugins/pkg/executable"
19
+ "github.com/netdata/netdata/go/plugins/pkg/multipath"
20
+)
21
+
22
+var log = logger.New().With("component", "snmp/ddsnmp")
23
+
24
+var (
25
+ // Profile loading is intentionally global and cached to avoid reloading
26
+ // profiles for each SNMP job instance. This is a performance optimization
27
+ // as there can be many concurrent SNMP collection jobs.
28
+ ddProfiles []*Profile
29
+ loadOnce sync.Once
30
+)
31
+
32
+func loadProfiles() {
33
+ loadOnce.Do(func() {
34
+ userDirs, stockDirs := getProfilesDirs()
35
+ extendsPaths := multipath.New(userDirs, stockDirs)
36
+
37
+ seen := make(map[string]bool)
38
+
39
+ for _, dir := range extendsPaths {
40
+ if dir == "" {
41
+ continue
42
+ }
43
+ profiles, err := loadProfilesFromDir(dir, extendsPaths)
44
+ if err != nil {
45
+ log.Errorf("failed to load dd snmp profiles from '%s': %v", dir, err)
46
+ continue
47
+ }
48
+
49
+ if len(profiles) == 0 {
50
+ log.Infof("no dd snmp profiles found in '%s'", dir)
51
+ continue
52
+ }
53
+
54
+ log.Infof("found %d profiles in '%s'", len(profiles), dir)
55
+ profiles = slices.DeleteFunc(profiles, func(p *Profile) bool {
56
+ name := filepath.Base(p.SourceFile)
57
+ if seen[name] {
58
+ log.Infof("duplicate profile '%s' found in '%s', not adding it", name, dir)
59
+ return true
60
+ }
61
+ seen[name] = true
62
+ return false
63
+ })
64
+ ddProfiles = append(ddProfiles, profiles...)
65
+ }
66
+
67
+ if len(ddProfiles) == 0 {
68
+ log.Warningf("no dd snmp profiles found in any of the searched directories: %v", extendsPaths)
69
+ } else {
70
+ log.Infof("loaded %d dd snmp profiles total", len(ddProfiles))
71
+ }
72
+ })
73
+}
74
+
75
+func loadProfilesFromDir(dirpath string, extendsPaths multipath.MultiPath) ([]*Profile, error) {
76
+ var profiles []*Profile
77
+
78
+ if err := filepath.WalkDir(dirpath, func(path string, d fs.DirEntry, err error) error {
79
+ if err != nil {
80
+ return err
81
+ }
82
+ if !(strings.HasSuffix(d.Name(), ".yaml") || strings.HasSuffix(d.Name(), ".yml")) {
83
+ return nil
84
+ }
85
+
86
+ profile, err := loadProfile(path, extendsPaths)
87
+ if err != nil {
88
+ log.Warningf("invalid profile '%s': %v", path, err)
89
+ return nil
90
+ }
91
+
92
+ if err := profile.validate(); err != nil {
93
+ log.Warningf("invalid profile '%s': %v", path, err)
94
+ return nil
95
+ }
96
+
97
+ profile.removeConstantMetrics()
98
+
99
+ profiles = append(profiles, profile)
100
+ return nil
101
+ }); err != nil {
102
+ return nil, err
103
+ }
104
+
105
+ return profiles, nil
106
+}
107
+
108
+func loadProfile(filename string, extendsPaths multipath.MultiPath) (*Profile, error) {
109
+ return loadProfileWithExtendsMap(filename, extendsPaths, []string{})
110
+}
111
+
112
+func loadProfileWithExtendsMap(filename string, extendsPaths multipath.MultiPath, stack []string) (*Profile, error) {
113
+ content, err := os.ReadFile(filename)
114
+ if err != nil {
115
+ return nil, err
116
+ }
117
+
118
+ var prof Profile
119
+ if err := yaml.Unmarshal(content, &prof.Definition); err != nil {
120
+ return nil, err
121
+ }
122
+
123
+ if prof.SourceFile == "" {
124
+ prof.SourceFile, _ = filepath.Abs(filename)
125
+ }
126
+
127
+ // Merge extended profiles here
128
+ for _, name := range prof.Definition.Extends {
129
+ if slices.Contains(stack, name) {
130
+ return nil, fmt.Errorf("circular extends detected: '%s' already included (in file: %s)", name, prof.SourceFile)
131
+ }
132
+
133
+ extPath, err := extendsPaths.Find(name)
134
+ if err != nil {
135
+ return nil, fmt.Errorf("cannot find extension '%s': %w", name, err)
136
+ }
137
+
138
+ mergedBase, err := loadProfileWithExtendsMap(extPath, extendsPaths, append(stack, name))
139
+ if err != nil {
140
+ return nil, err
141
+ }
142
+
143
+ prof.merge(mergedBase)
144
+ }
145
+
146
+ return &prof, nil
147
+}
148
+
149
+func getProfilesDirs() (userDir, stockDir string) {
150
+ if executable.Name == "test" {
151
+ dir, _ := filepath.Abs("../../../config/go.d/snmp.profiles/default")
152
+ return "", dir
153
+ }
154
+
155
+ if userDir = handleDirOnWin(os.Getenv("NETDATA_USER_CONFIG_DIR")); userDir != "" {
156
+ if dir := filepath.Join(userDir, "go.d/snmp.profiles"); isDirExists(dir) {
157
+ userDir = dir
158
+ }
159
+ }
160
+ if stockDir = handleDirOnWin(os.Getenv("NETDATA_STOCK_CONFIG_DIR")); stockDir != "" {
161
+ if dir := filepath.Join(stockDir, "go.d/snmp.profiles/default"); isDirExists(dir) {
162
+ stockDir = dir
163
+ }
164
+ }
165
+
166
+ if userDir != "" || stockDir != "" {
167
+ return userDir, stockDir
168
+ }
169
+
170
+ // Development: When running from source (netdata/src/go/plugin/go.d/bin)
171
+ // Looks for profiles in the local git repository
172
+ if dir := filepath.Join(executable.Directory, "../config/go.d/snmp.profiles/default"); isDirExists(dir) {
173
+ return "", dir
174
+ }
175
+
176
+ possibleDirs := []string{
177
+ filepath.Join(executable.Directory, "../../../../etc/netdata/go.d/snmp.profiles"),
178
+ // User Standard installation paths
179
+ handleDirOnWin("/etc/netdata/go.d/snmp.profiles"),
180
+ handleDirOnWin("/opt/netdata/etc/netdata/go.d/snmp.profiles"),
181
+
182
+ filepath.Join(executable.Directory, "../../../lib/netdata/conf.d/go.d/snmp.profiles/default"),
183
+ // Stock standard installation paths
184
+ handleDirOnWin("/usr/lib/netdata/conf.d/go.d/snmp.profiles/default"),
185
+ handleDirOnWin("/opt/netdata/usr/lib/netdata/conf.d/go.d/snmp.profiles/default"),
186
+ }
187
+
188
+ for _, dir := range possibleDirs {
189
+ isStock := strings.HasSuffix(filepath.Base(dir), "default")
190
+ switch {
191
+ case userDir == "" && !isStock && isDirExists(dir):
192
+ userDir = dir
193
+ case stockDir == "" && isStock && isDirExists(dir):
194
+ stockDir = dir
195
+ }
196
+ }
197
+
198
+ return userDir, stockDir
199
+}
200
+
201
+func isDirExists(dir string) bool {
202
+ fi, err := os.Stat(dir)
203
+ if err != nil {
204
+ return !errors.Is(err, fs.ErrNotExist)
205
+ }
206
+ return fi.Mode().IsDir()
207
+}
208
+
209
+func handleDirOnWin(path string) string {
210
+ base := os.Getenv("NETDATA_CYGWIN_BASE_PATH")
211
+
212
+ // TODO: temp workaround for debug mode
213
+ if base == "" && strings.HasPrefix(executable.Directory, "C:\\msys64") {
214
+ base = "C:\\msys64"
215
+ }
216
+
217
+ if base == "" || !strings.HasPrefix(path, "/") {
218
+ return path
219
+ }
220
+
221
+ return filepath.Join(base, path)
222
+}
src/go/plugin/go.d/collector/snmp/ddsnmp/profile.go
+3
-127
@@ -5,51 +5,18 @@ package ddsnmp
5
import (
6
"errors"
7
"fmt"
8
- "io/fs"
9
- "os"
10
- "path/filepath"
8
"slices"
9
"sort"
10
"strings"
14
- "sync"
11
16
- "gopkg.in/yaml.v2"
17
-
18
- "github.com/netdata/netdata/go/plugins/logger"
19
- "github.com/netdata/netdata/go/plugins/pkg/executable"
12
"github.com/netdata/netdata/go/plugins/pkg/matcher"
13
"github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition"
14
)
15
24
-var log = logger.New().With("component", "snmp/ddsnmp")
25
-
26
-var (
27
- ddProfiles []*Profile
28
- loadOnce sync.Once
29
-)
30
-
31
-func load() {
32
- loadOnce.Do(func() {
33
- dir := getProfilesDir()
34
-
35
- profiles, err := loadProfiles(dir)
36
- if err != nil {
37
- log.Errorf("failed to loadProfiles dd snmp profiles: %v", err)
38
- return
39
- }
40
-
41
- if len(profiles) == 0 {
42
- log.Warningf("no dd snmp profiles found in '%s'", dir)
43
- return
44
- }
45
-
46
- log.Infof("found %d profiles in '%s'", len(profiles), dir)
47
- ddProfiles = profiles
48
- })
49
-}
50
-
16
+// FindProfiles returns profiles matching the given sysObjectID.
17
+// Profiles are loaded once on the first call and cached globally.
18
func FindProfiles(sysObjId string) []*Profile {
52
- load()
19
+ loadProfiles()
20
21
var profiles []*Profile
22
@@ -194,97 +161,6 @@ func (p *Profile) removeConstantMetrics() {
161
})
162
}
163
197
-func loadProfiles(dirpath string) ([]*Profile, error) {
198
- var profiles []*Profile
199
-
200
- if err := filepath.WalkDir(dirpath, func(path string, d fs.DirEntry, err error) error {
201
- if err != nil {
202
- return err
203
- }
204
- if !(strings.HasSuffix(d.Name(), ".yaml") || strings.HasSuffix(d.Name(), ".yml")) {
205
- return nil
206
- }
207
-
208
- profile, err := loadProfile(path)
209
- if err != nil {
210
- log.Warningf("invalid profile '%s': %v", path, err)
211
- return nil
212
- }
213
-
214
- if err := profile.validate(); err != nil {
215
- log.Warningf("invalid profile '%s': %v", path, err)
216
- return nil
217
- }
218
-
219
- profile.removeConstantMetrics()
220
-
221
- profiles = append(profiles, profile)
222
- return nil
223
- }); err != nil {
224
- return nil, err
225
- }
226
-
227
- return profiles, nil
228
-}
229
-
230
-func loadProfile(filename string) (*Profile, error) {
231
- content, err := os.ReadFile(filename)
232
- if err != nil {
233
- return nil, err
234
- }
235
-
236
- var prof Profile
237
- if err := yaml.Unmarshal(content, &prof.Definition); err != nil {
238
- return nil, err
239
- }
240
-
241
- if prof.SourceFile == "" {
242
- prof.SourceFile, _ = filepath.Abs(filename)
243
- }
244
-
245
- dir := filepath.Dir(filename)
246
-
247
- processedExtends := make(map[string]bool)
248
- if err := loadProfileExtensions(&prof, dir, processedExtends); err != nil {
249
- return nil, err
250
- }
251
-
252
- return &prof, nil
253
-}
254
-
255
-func loadProfileExtensions(profile *Profile, dir string, processedExtends map[string]bool) error {
256
- for _, name := range profile.Definition.Extends {
257
- if processedExtends[name] {
258
- continue
259
- }
260
- processedExtends[name] = true
261
-
262
- baseProf, err := loadProfile(filepath.Join(dir, name))
263
- if err != nil {
264
- return err
265
- }
266
-
267
- if err := loadProfileExtensions(baseProf, dir, processedExtends); err != nil {
268
- return err
269
- }
270
-
271
- profile.merge(baseProf)
272
- }
273
-
274
- return nil
275
-}
276
-
277
-func getProfilesDir() string {
278
- if executable.Name == "test" {
279
- dir, _ := filepath.Abs("../../../config/go.d/snmp.profiles/default")
280
- return dir
281
- }
282
- if dir := os.Getenv("NETDATA_STOCK_CONFIG_DIR"); dir != "" {
283
- return filepath.Join(dir, "go.d/snmp.profiles/default")
284
- }
285
- return filepath.Join(executable.Directory, "../../../config/go.d/snmp.profiles/default")
286
-}
287
-
164
func enrichProfiles(profiles []*Profile) {
165
for _, prof := range profiles {
166
if prof.Definition == nil {
src/go/plugin/go.d/collector/snmp/ddsnmp/profile_test.go
+212
-2
@@ -11,7 +11,9 @@ import (
11
12
"github.com/stretchr/testify/assert"
13
"github.com/stretchr/testify/require"
14
+ "gopkg.in/yaml.v2"
15
16
+ "github.com/netdata/netdata/go/plugins/pkg/multipath"
17
"github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition"
18
)
19
@@ -20,9 +22,9 @@ func Test_loadDDSnmpProfiles(t *testing.T) {
22
23
f, err := os.Open(dir)
24
require.NoError(t, err)
23
- defer f.Close()
25
+ defer func() { _ = f.Close() }()
26
25
- profiles, err := loadProfiles(dir)
27
+ profiles, err := loadProfilesFromDir(dir, multipath.New(dir))
28
require.NoError(t, err)
29
30
require.NotEmpty(t, profiles)
@@ -843,3 +845,211 @@ func TestDeduplicateMetricsAcrossProfiles(t *testing.T) {
845
})
846
}
847
}
848
+
849
+func Test_ProfileExtends_CircularReference(t *testing.T) {
850
+ tmp := t.TempDir()
851
+
852
+ a := filepath.Join(tmp, "a.yaml")
853
+ b := filepath.Join(tmp, "b.yaml")
854
+
855
+ writeYAML(t, a, map[string]any{
856
+ "extends": []string{"b.yaml"},
857
+ })
858
+ writeYAML(t, b, map[string]any{
859
+ "extends": []string{"a.yaml"},
860
+ })
861
+
862
+ paths := multipath.New(tmp)
863
+ _, err := loadProfile(a, paths)
864
+ require.Error(t, err)
865
+ require.Contains(t, err.Error(), "circular extends")
866
+}
867
+
868
+func Test_ProfileExtends_RecursiveChain(t *testing.T) {
869
+ tmp := t.TempDir()
870
+
871
+ base := filepath.Join(tmp, "base.yaml")
872
+ mid := filepath.Join(tmp, "mid.yaml")
873
+ top := filepath.Join(tmp, "top.yaml")
874
+
875
+ writeYAML(t, base, ddprofiledefinition.ProfileDefinition{
876
+ Metrics: []ddprofiledefinition.MetricsConfig{
877
+ {
878
+ Symbol: ddprofiledefinition.SymbolConfig{
879
+ OID: "1.3.6.1.2.1.1.3.0",
880
+ Name: "sysUpTime",
881
+ },
882
+ },
883
+ },
884
+ })
885
+ writeYAML(t, mid, map[string]any{
886
+ "extends": []string{"base.yaml"},
887
+ })
888
+ writeYAML(t, top, map[string]any{
889
+ "extends": []string{"mid.yaml"},
890
+ })
891
+
892
+ paths := multipath.New(tmp)
893
+ prof, err := loadProfile(top, paths)
894
+ require.NoError(t, err)
895
+ require.Len(t, prof.Definition.Metrics, 1)
896
+ require.Equal(t, "sysUpTime", prof.Definition.Metrics[0].Symbol.Name)
897
+}
898
+
899
+func Test_ProfileExtends_MultipleBases(t *testing.T) {
900
+ tmp := t.TempDir()
901
+
902
+ base1 := filepath.Join(tmp, "base1.yaml")
903
+ base2 := filepath.Join(tmp, "base2.yaml")
904
+ main := filepath.Join(tmp, "main.yaml")
905
+
906
+ writeYAML(t, base1, ddprofiledefinition.ProfileDefinition{
907
+ Metrics: []ddprofiledefinition.MetricsConfig{
908
+ {Symbol: ddprofiledefinition.SymbolConfig{
909
+ OID: "1.3.6.1.2.1.1.1.0", Name: "sysDescr",
910
+ }},
911
+ },
912
+ })
913
+ writeYAML(t, base2, ddprofiledefinition.ProfileDefinition{
914
+ Metrics: []ddprofiledefinition.MetricsConfig{
915
+ {Symbol: ddprofiledefinition.SymbolConfig{
916
+ OID: "1.3.6.1.2.1.1.5.0", Name: "sysName",
917
+ }},
918
+ },
919
+ })
920
+ writeYAML(t, main, map[string]any{
921
+ "extends": []string{"base1.yaml", "base2.yaml"},
922
+ })
923
+
924
+ paths := multipath.New(tmp)
925
+ prof, err := loadProfile(main, paths)
926
+ require.NoError(t, err)
927
+ require.Len(t, prof.Definition.Metrics, 2)
928
+}
929
+
930
+func Test_ProfileExtends_NonexistentFile(t *testing.T) {
931
+ tmp := t.TempDir()
932
+ main := filepath.Join(tmp, "main.yaml")
933
+
934
+ writeYAML(t, main, map[string]any{
935
+ "extends": []string{"missing.yaml"},
936
+ })
937
+
938
+ paths := multipath.New(tmp)
939
+ _, err := loadProfile(main, paths)
940
+ require.Error(t, err)
941
+ require.Contains(t, err.Error(), "missing.yaml")
942
+}
943
+
944
+func Test_ProfileExtends_SharedBase(t *testing.T) {
945
+ tmp := t.TempDir()
946
+
947
+ base := filepath.Join(tmp, "base.yaml")
948
+ a := filepath.Join(tmp, "a.yaml")
949
+ b := filepath.Join(tmp, "b.yaml")
950
+
951
+ writeYAML(t, base, ddprofiledefinition.ProfileDefinition{
952
+ Metrics: []ddprofiledefinition.MetricsConfig{
953
+ {Symbol: ddprofiledefinition.SymbolConfig{
954
+ OID: "1.3.6.1.2.1.1.1.0", Name: "sysDescr",
955
+ }},
956
+ },
957
+ })
958
+ writeYAML(t, a, map[string]any{"extends": []string{"base.yaml"}})
959
+ writeYAML(t, b, map[string]any{"extends": []string{"base.yaml"}})
960
+
961
+ paths := multipath.New(tmp)
962
+
963
+ profA, err := loadProfile(a, paths)
964
+ require.NoError(t, err)
965
+ require.Len(t, profA.Definition.Metrics, 1)
966
+
967
+ profB, err := loadProfile(b, paths)
968
+ require.NoError(t, err)
969
+ require.Len(t, profB.Definition.Metrics, 1)
970
+}
971
+
972
+func Test_ProfileExtends_OverrideIgnored(t *testing.T) {
973
+ tmp := t.TempDir()
974
+
975
+ base := filepath.Join(tmp, "base.yaml")
976
+ main := filepath.Join(tmp, "main.yaml")
977
+
978
+ writeYAML(t, base, ddprofiledefinition.ProfileDefinition{
979
+ Metrics: []ddprofiledefinition.MetricsConfig{
980
+ {Symbol: ddprofiledefinition.SymbolConfig{
981
+ OID: "1.3.6.1.2.1.1.3.0", Name: "sysUpTime",
982
+ }},
983
+ },
984
+ })
985
+ writeYAML(t, main, map[string]any{
986
+ "extends": []string{"base.yaml"},
987
+ "metrics": []map[string]any{
988
+ {
989
+ "symbol": map[string]string{
990
+ "OID": "1.3.6.1.2.1.1.3.0",
991
+ "name": "sysUpTime",
992
+ },
993
+ },
994
+ },
995
+ })
996
+
997
+ paths := multipath.New(tmp)
998
+ prof, err := loadProfile(main, paths)
999
+ require.NoError(t, err)
1000
+
1001
+ deduplicateMetricsAcrossProfiles([]*Profile{prof})
1002
+
1003
+ // Should not duplicate
1004
+ require.Len(t, prof.Definition.Metrics, 1)
1005
+}
1006
+
1007
+func Test_ProfileExtends_UserOverride(t *testing.T) {
1008
+ stockDir := filepath.Join(t.TempDir(), "stock")
1009
+ userDir := filepath.Join(t.TempDir(), "user")
1010
+
1011
+ require.NoError(t, os.MkdirAll(stockDir, 0755))
1012
+ require.NoError(t, os.MkdirAll(userDir, 0755))
1013
+
1014
+ // Stock base profile
1015
+ writeYAML(t, filepath.Join(stockDir, "_base.yaml"), ddprofiledefinition.ProfileDefinition{
1016
+ Metrics: []ddprofiledefinition.MetricsConfig{
1017
+ {Symbol: ddprofiledefinition.SymbolConfig{
1018
+ OID: "1.3.6.1.2.1.1.3.0", Name: "sysUpTime",
1019
+ }},
1020
+ },
1021
+ })
1022
+
1023
+ // User override of base profile
1024
+ writeYAML(t, filepath.Join(userDir, "_base.yaml"), ddprofiledefinition.ProfileDefinition{
1025
+ Metrics: []ddprofiledefinition.MetricsConfig{
1026
+ {Symbol: ddprofiledefinition.SymbolConfig{
1027
+ OID: "1.3.6.1.2.1.1.3.0", Name: "sysUpTime",
1028
+ }},
1029
+ {Symbol: ddprofiledefinition.SymbolConfig{
1030
+ OID: "1.3.6.1.2.1.1.5.0", Name: "sysName",
1031
+ }},
1032
+ },
1033
+ })
1034
+
1035
+ // Main profile in stock
1036
+ writeYAML(t, filepath.Join(stockDir, "device.yaml"), map[string]any{
1037
+ "extends": []string{"_base.yaml"},
1038
+ })
1039
+
1040
+ paths := multipath.New(userDir, stockDir)
1041
+ prof, err := loadProfile(filepath.Join(stockDir, "device.yaml"), paths)
1042
+ require.NoError(t, err)
1043
+
1044
+ // Should use user's _base.yaml, so should have 2 metrics
1045
+ require.Len(t, prof.Definition.Metrics, 2)
1046
+ assert.Equal(t, "sysName", prof.Definition.Metrics[1].Symbol.Name)
1047
+}
1048
+
1049
+func writeYAML(t *testing.T, path string, data any) {
1050
+ t.Helper()
1051
+
1052
+ content, err := yaml.Marshal(data)
1053
+ require.NoError(t, err)
1054
+ require.NoError(t, os.WriteFile(path, content, 0600))
1055
+}