@cryptotaxi247 / netdata-1 / commits / ebe6daecc

SNMP Collector, use custom YAML files for auto single metrics (#20020)

* work so far, units and description * tgo mod tidy * revert custom units and description * remove custom package go mod entries * tidy up go.mod * fix dimension initialization * fix tests and include new code due to new snmpGet run * disable new funct * change priority back * add enableProfile field --------- Co-authored-by: ilyam8 <ilya@netdata.cloud>

Fotis Voutsas committed Apr 9, 2025 at 13:27 UTC ebe6daecc78319f47d2721d175ce02e1b3b4becf

9 files changed +186 -721

src/go/go.mod

+1 -1

@@ -58,7 +58,6 @@ require (

58 gopkg.in/ini.v1 v1.67.0

59 gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2

60 gopkg.in/yaml.v2 v2.4.0

61 - gopkg.in/yaml.v3 v3.0.1

61 k8s.io/api v0.32.3

62 k8s.io/apimachinery v0.32.3

63 k8s.io/client-go v0.32.3

@@ -165,6 +164,7 @@ require (

164 gopkg.in/cenkalti/backoff.v2 v2.2.1 // indirect

165 gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect

166 gopkg.in/inf.v0 v0.9.1 // indirect

167 + gopkg.in/yaml.v3 v3.0.1 // indirect

168 k8s.io/klog/v2 v2.130.1 // indirect

169 k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect

170 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect

src/go/plugin/go.d/collector/snmp/charts.go

+46

@@ -19,6 +19,8 @@ const (

19 prioNetIfaceAdminStatus

20 prioNetIfaceOperStatus

21 prioSysUptime

22 +

23 + priosnmp

24 )

26 var netIfaceChartsTmpl = module.Charts{

@@ -32,6 +34,20 @@ var netIfaceChartsTmpl = module.Charts{

34 netIfaceOperStatusChartTmpl.Copy(),

35 }

37 +var (

38 + snmpChartTemplate = module.Chart{

39 + ID: "%s",

40 + Title: "%s",

41 + Units: "%s",

42 + Fam: "%s",

43 + Ctx: "snmp.%s",

44 + Priority: priosnmp,

45 + Dims: module.Dims{

46 + {ID: "%s", Name: "%s"},

47 + },

48 + }

49 +)

50 +

51 var (

52 netIfaceTrafficChartTmpl = module.Chart{

53 ID: "snmp_device_net_iface_%s_traffic",

@@ -178,6 +194,36 @@ func (c *Collector) addNetIfaceCharts(iface *netInterface) {

194 }

195 }

196

197 +func (c *Collector) addSNMPChart(processedMetric processedMetric) {

198 + if processedMetric.tableName == "" {

199 + chart := snmpChartTemplate.Copy()

200 +

201 + chart.ID = fmt.Sprintf(chart.ID, processedMetric.name)

202 + chart.Title = fmt.Sprintf(chart.Title, processedMetric.name)

203 + chart.Units = fmt.Sprintf(chart.Units, "TBD unit")

204 + chart.Ctx = fmt.Sprintf(chart.Ctx, processedMetric.name)

205 + chart.Fam = fmt.Sprintf(chart.Fam, processedMetric.name)

206 +

207 + for _, dim := range chart.Dims {

208 + dim.ID = fmt.Sprintf(dim.ID, processedMetric.name)

209 + dim.Name = fmt.Sprintf(dim.Name, processedMetric.name)

210 + }

211 +

212 + if err := c.Charts().Add(chart); err != nil {

213 + c.Warning(err)

214 + }

215 + }

216 +}

217 +

218 +func (c *Collector) removeSNMPChart(name string) {

219 + for _, chart := range *c.Charts() {

220 + if chart.ID == name {

221 + chart.MarkRemove()

222 + chart.MarkNotCreated()

223 + }

224 + }

225 +}

226 +

227 func (c *Collector) removeNetIfaceCharts(iface *netInterface) {

228 px := fmt.Sprintf("snmp_device_net_iface_%s_", cleanIfaceName(iface.ifName))

229 for _, chart := range *c.Charts() {

src/go/plugin/go.d/collector/snmp/collect.go

+43 -9

@@ -5,17 +5,36 @@ package snmp

5 import (

6 "errors"

7 "fmt"

8 + "log"

9 "slices"

10 "strings"

11 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/discovery/sd/discoverer/snmpsd"

12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/vnodes"

13 -

12 "github.com/google/uuid"

13 "github.com/gosnmp/gosnmp"

14 +

15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/discovery/sd/discoverer/snmpsd"

16 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/vnodes"

17 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp"

18 )

20 func (c *Collector) collect() (map[string]int64, error) {

21 + if c.enableProfiles {

22 + sysObjectID, err := c.getSysObjectID(snmpsd.OidSysObject)

23 + if err != nil {

24 + return nil, err

25 + }

26 +

27 + matchingProfiles := ddsnmp.Find(sysObjectID)

28 +

29 + metricMap, err := c.parseMetricsFromProfiles(matchingProfiles)

30 + if err != nil {

31 + return nil, err

32 + }

33 + seen := make(map[string]bool)

34 + mx := make(map[string]int64)

35 + c.makeChartsFromMetricMap(mx, metricMap, seen)

36 + }

37 +

38 if c.sysInfo == nil {

39 si, err := snmpsd.GetSysInfo(c.snmpClient)

40 if err != nil {

@@ -56,24 +75,39 @@ func (c *Collector) getSysObjectID(oid string) (string, error) {

75 if err != nil {

76 return "", err

77 }

59 -

78 return strings.Replace(resp.Variables[0].Value.(string), ".", "", 1), nil

79 }

63 -func (c *Collector) makeChartsFromMetricMap(mx map[string]int64, metricMap map[string]processedMetric) error {

64 -

81 +func (c *Collector) makeChartsFromMetricMap(mx map[string]int64, metricMap map[string]processedMetric, seen map[string]bool) error {

82 for _, metric := range metricMap {

83 if metric.tableName == "" {

84 switch s := metric.value.(type) {

85 case int:

70 - // log.Println(metric)

87 + log.Println(metric)

72 - // c.addSNMPChart(metric)

73 - mx[metric.name] = int64(s)

89 + name := metric.name

90 + if name == "" {

91 + continue

92 + }

94 + seen[name] = true

95 +

96 + if !c.seenMetrics[name] {

97 + c.seenMetrics[name] = true

98 + c.addSNMPChart(metric)

99 + }

100 +

101 + mx[metric.name] = int64(s)

102 }

103 }

104 +

105 + }

106 + for name := range c.seenMetrics {

107 + if !seen[name] {

108 + delete(c.seenMetrics, name)

109 + c.removeSNMPChart(name)

110 + }

111 }

112 return nil

113 }

src/go/plugin/go.d/collector/snmp/collector.go

+8 -1

@@ -50,6 +50,10 @@ func New() *Collector {

50 },

51 },

53 + charts: &module.Charts{},

54 +

55 + seenMetrics: make(map[string]bool),

56 +

57 newSnmpClient: gosnmp.NewHandler,

59 checkMaxReps: true,

@@ -62,9 +66,12 @@ type Collector struct {

66 module.Base

67 Config `yaml:",inline" json:""`

69 + enableProfiles bool

70 +

71 vnode *vnodes.VirtualNode

67 - charts *module.Charts

73 + charts *module.Charts

74 + seenMetrics map[string]bool

76 newSnmpClient func() gosnmp.Handler

77 snmpClient gosnmp.Handler

src/go/plugin/go.d/collector/snmp/collector_test.go

+52 -1

@@ -156,6 +156,9 @@ func TestCollector_Charts(t *testing.T) {

156 prepareSNMP: func(t *testing.T, m *snmpmock.MockHandler) *Collector {

157 collr := New()

158 collr.Config = prepareV2Config()

159 + if collr.enableProfiles {

160 + setMockClientSysObjectidExpect(m)

161 + }

162 setMockClientSysExpect(m)

163 setMockClientIfMibExpect(m)

164

@@ -205,6 +208,9 @@ func TestCollector_Check(t *testing.T) {

208 prepareSNMP: func(m *snmpmock.MockHandler) *Collector {

209 collr := New()

210 collr.Config = prepareV2Config()

211 + if collr.enableProfiles {

212 + setMockClientSysObjectidExpect(m)

213 + }

214 setMockClientIfMibExpect(m)

215

216 return collr

@@ -217,6 +223,10 @@ func TestCollector_Check(t *testing.T) {

223 collr.Config = prepareConfigWithUserCharts(prepareV2Config(), 0, 3)

224 collr.collectIfMib = false

225

226 + if collr.enableProfiles {

227 + setMockClientSysObjectidExpect(m)

228 + }

229 +

230 m.EXPECT().Get(gomock.Any()).Return(&gosnmp.SnmpPacket{

231 Variables: []gosnmp.SnmpPDU{

232 {Value: 10, Type: gosnmp.Counter32},

@@ -239,7 +249,9 @@ func TestCollector_Check(t *testing.T) {

249 collr := New()

250 collr.Config = prepareConfigWithUserCharts(prepareV2Config(), 0, 3)

251 collr.collectIfMib = false

242 -

252 + if collr.enableProfiles {

253 + setMockClientSysObjectidExpect(m)

254 + }

255 m.EXPECT().Get(gomock.Any()).Return(nil, errors.New("mock Get() error")).Times(1)

256

257 return collr

@@ -279,11 +291,15 @@ func TestCollector_Collect(t *testing.T) {

291 collr := New()

292 collr.Config = prepareV2Config()

293

294 + if collr.enableProfiles {

295 + setMockClientSysObjectidExpect(m)

296 + }

297 setMockClientIfMibExpect(m)

298

299 return collr

300 },

301 wantCollected: map[string]int64{

302 + //"TestMetric": 1,

303 "net_iface_ether1_admin_status_down": 0,

304 "net_iface_ether1_admin_status_testing": 0,

305 "net_iface_ether1_admin_status_up": 1,

@@ -381,6 +397,10 @@ func TestCollector_Collect(t *testing.T) {

397 collr.Config = prepareConfigWithUserCharts(prepareV2Config(), 0, 3)

398 collr.collectIfMib = false

399

400 + if collr.enableProfiles {

401 + setMockClientSysObjectidExpect(m)

402 + }

403 +

404 m.EXPECT().Get(gomock.Any()).Return(&gosnmp.SnmpPacket{

405 Variables: []gosnmp.SnmpPDU{

406 {Value: 10, Type: gosnmp.Counter32},

@@ -397,6 +417,7 @@ func TestCollector_Collect(t *testing.T) {

417 return collr

418 },

419 wantCollected: map[string]int64{

420 + //"TestMetric": 1,

421 "1.3.6.1.2.1.2.2.1.10.0": 10,

422 "1.3.6.1.2.1.2.2.1.16.0": 20,

423 "1.3.6.1.2.1.2.2.1.10.1": 30,

@@ -414,6 +435,10 @@ func TestCollector_Collect(t *testing.T) {

435 collr.Config = prepareConfigWithUserCharts(prepareV2Config(), 0, 2)

436 collr.collectIfMib = false

437

438 + if collr.enableProfiles {

439 + setMockClientSysObjectidExpect(m)

440 + }

441 +

442 m.EXPECT().Get(gomock.Any()).Return(&gosnmp.SnmpPacket{

443 Variables: []gosnmp.SnmpPDU{

444 {Value: 10, Type: gosnmp.Counter32},

@@ -428,6 +453,7 @@ func TestCollector_Collect(t *testing.T) {

453 return collr

454 },

455 wantCollected: map[string]int64{

456 + //"TestMetric": 1,

457 "1.3.6.1.2.1.2.2.1.10.0": 10,

458 "1.3.6.1.2.1.2.2.1.16.0": 20,

459 "1.3.6.1.2.1.2.2.1.10.1": 30,

@@ -440,6 +466,10 @@ func TestCollector_Collect(t *testing.T) {

466 collr.Config = prepareConfigWithUserCharts(prepareV2Config(), 0, 2)

467 collr.collectIfMib = false

468

469 + if collr.enableProfiles {

470 + setMockClientSysObjectidExpect(m)

471 + }

472 +

473 m.EXPECT().Get(gomock.Any()).Return(&gosnmp.SnmpPacket{

474 Variables: []gosnmp.SnmpPDU{

475 {Value: nil, Type: gosnmp.NoSuchInstance},

@@ -454,6 +484,7 @@ func TestCollector_Collect(t *testing.T) {

484 return collr

485 },

486 wantCollected: map[string]int64{

487 + //"TestMetric": 1,

488 "uptime": 60,

489 },

490 },

@@ -474,6 +505,10 @@ func TestCollector_Collect(t *testing.T) {

505

506 mx := collr.Collect(context.Background())

507

508 + if collr.enableProfiles {

509 + mx["TestMetric"] = 1

510 + }

511 +

512 assert.Equal(t, test.wantCollected, mx)

513 })

514 }

@@ -581,6 +616,22 @@ func setMockClientInitExpect(m *snmpmock.MockHandler) {

616 m.EXPECT().Connect().Return(nil).AnyTimes()

617 }

618

619 +func setMockClientSysObjectidExpect(m *snmpmock.MockHandler) {

620 + m.EXPECT().Get([]string{snmpsd.OidSysObject}).Return(&gosnmp.SnmpPacket{

621 + Variables: []gosnmp.SnmpPDU{

622 + {Value: ".1.1.1",

623 + Name: ".1.3.6.1.2.1.1.2.0",

624 + Type: gosnmp.ObjectIdentifier},

625 + },

626 + }, nil).MinTimes(1)

627 + m.EXPECT().Get([]string{"1.1.1.0"}).Return(&gosnmp.SnmpPacket{

628 + Variables: []gosnmp.SnmpPDU{

629 + {Name: "1.1.1.0", Value: 1, Type: gosnmp.Integer},

630 + },

631 + }, nil).MinTimes(1)

632 +

633 +}

634 +

635 func setMockClientSysExpect(m *snmpmock.MockHandler) {

636 m.EXPECT().WalkAll(snmpsd.RootOidMibSystem).Return([]gosnmp.SnmpPDU{

637 {Name: snmpsd.OidSysDescr, Value: []uint8("mock sysDescr"), Type: gosnmp.OctetString},

src/go/plugin/go.d/collector/snmp/helpers.go

+12 -12

@@ -13,18 +13,18 @@ func sliceToStrings(items []interface{}) []string {

13 return strs

14 }

16 -func sliceToTableMetricTags(items []interface{}) []TableMetricTag {

17 - var metricTag []TableMetricTag

18 - for _, v := range items {

19 - s, ok := v.(TableMetricTag)

20 - if !ok {

21 - // Handle error if an element is not a string.

22 - continue

23 - }

24 - metricTag = append(metricTag, s)

25 - }

26 - return metricTag

27 -}

16 +// func sliceToTableMetricTags(items []interface{}) []TableMetricTag {

17 +// var metricTag []TableMetricTag

18 +// for _, v := range items {

19 +// s, ok := v.(TableMetricTag)

20 +// if !ok {

21 +// // Handle error if an element is not a string.

22 +// continue

23 +// }

24 +// metricTag = append(metricTag, s)

25 +// }

26 +// return metricTag

27 +// }

29 func mergeTableBatches(target tableBatches, source tableBatches) tableBatches {

30 merged := tableBatches{}

src/go/plugin/go.d/collector/snmp/parsing.go

+19 -509

@@ -3,12 +3,12 @@ package snmp

3 import (

4 "errors"

5 "fmt"

6 - "log"

7 - "reflect"

6 "regexp"

7 +

8 + "github.com/DataDog/datadog-agent/pkg/networkdevice/profile/profiledefinition"

9 )

11 -func parseMetrics(metrics []Metric) (parsedResult, error) {

11 +func parseMetrics(metrics []profiledefinition.MetricsConfig) (parsedResult, error) {

12 oids := []string{}

13 next_oids := []string{}

14 bulk_oids := []string{}

@@ -49,7 +49,7 @@ func parseMetrics(metrics []Metric) (parsedResult, error) {

49 next_oids: next_oids, bulk_oids: bulk_oids, parsed_metrics: parsed_metrics}, nil

50 }

52 -func parseMetric(metric Metric) (metricParseResult, error) {

52 +func parseMetric(metric profiledefinition.MetricsConfig) (metricParseResult, error) {

53 /*Can either be:

55 * An OID metric:

@@ -83,177 +83,26 @@ func parseMetric(metric Metric) (metricParseResult, error) {

83 name: ifInErrors

84 ```*/

86 - // Cast the tags to either array

87 - castedStringMetricTags := []string{}

88 - castedTableMetricTags := []TableMetricTag{}

89 -

90 - if len(metric.MetricTags) > 0 {

91 - /*//TODO investigate if there are metric tags and not only table metric tags

92 - fmt.Println("parseMetric MetricTags switch, have:")

93 - spew.Dump(metric.MetricTags)

94 - switch metric.MetricTags[0].(type) {

95 - case string:

96 - os.Exit(-40)

97 - castedStringMetricTags = sliceToStrings(metric.MetricTags)

98 -

99 - case MetricTag:

100 - os.Exit(-40)

101 - castedTableMetricTags = sliceToTableMetricTags(metric.MetricTags)

102 - }*/

103 - }

86 + // Can't support tags at the moment

88 if len(metric.OID) > 0 {

89 // TODO investigate if this exists in the yamls

107 - return (parseOIDMetric(oidMetric{name: metric.Name, oid: metric.OID, metricTags: castedStringMetricTags, forcedType: metric.MetricType, options: metric.Options})), nil

90 + // return (parseOIDMetric(oidMetric{name: metric.Name, oid: metric.OID, metricTags: castedStringMetricTags, forcedType: string(metric.MetricType), options: metric.Options})), nil

91 + return metricParseResult{}, nil

92 +

93 } else if len(metric.MIB) == 0 {

94 return metricParseResult{}, fmt.Errorf("unsupported metric {%v}", metric)

110 - } else if metric.Symbol != (Symbol{}) {

95 + } else if metric.Symbol != (profiledefinition.SymbolConfig{}) {

96 // Single Metric

112 - return (parseSymbolMetric(symbolMetric{mib: metric.MIB, symbol: metric.Symbol, forcedType: metric.MetricType, metricTags: castedStringMetricTags, options: metric.Options}))

113 - } else if metric.Table != nil {

114 - // Table

115 - if len(metric.MetricTags) > 0 {

116 -

117 - /*//This will be cleared out when tags are supported

118 - // for _, rawItem := range metric.MetricTags {

119 - // item, ok := rawItem.(map[string]interface{})

120 - // if !ok {

121 - // continue

122 - // }

123 - // fmt.Println("ITEM", item["symbol"])

124 -

125 - // var index int

126 - // if val, exists := item["Index"]; exists {

127 - // if i, ok := val.(int); ok {

128 - // index = i

129 - // } else {

130 - // index = -1

131 - // }

132 - // }

133 -

134 - // var mapping map[int]string

135 - // if val, exists := item["mapping"]; exists {

136 - // if m, ok := val.(map[int]string); ok {

137 - // fmt.Print("IN")

138 - // os.Exit(-90)

139 - // mapping = m

140 - // }

141 - // }

142 -

143 - // var tag string

144 - // if val, exists := item["Tag"]; exists {

145 - // if s, ok := val.(string); ok {

146 - // tag = s

147 - // }

148 - // }

149 -

150 - // var symbol Symbol

151 - // if rawSymbol, exists := item["symbol"]; exists {

152 - // if symMap, ok := rawSymbol.(map[string]interface{}); ok {

153 - // var oid, name string

154 -

155 - // if v, exists := symMap["OID"]; exists {

156 - // if oidStr, ok := v.(string); ok {

157 - // oid = oidStr

158 - // }

159 - // }

160 -

161 - // if v, exists := symMap["name"]; exists {

162 - // if nameStr, ok := v.(string); ok {

163 - // name = nameStr

164 - // }

165 - // }

166 -

167 - // symbol = Symbol{OID: oid, Name: name}

168 - // } else {

169 - // fmt.Println("symbol is not a map[string]interface{}")

170 - // }

171 - // }

172 -

173 - // var table string

174 - // if val, exists := item["Table"]; exists {

175 - // if s, ok := val.(string); ok {

176 - // table = s

177 - // } else {

178 - // table = ""

179 - // }

180 - // }

181 -

182 - // var mib string

183 - // if val, exists := item["MIB"]; exists {

184 - // if s, ok := val.(string); ok {

185 - // mib = s

186 - // } else {

187 - // mib = ""

188 - // }

189 - // }

190 -

191 - // var indexTransform []IndexSlice

192 - // if val, exists := item["IndexTransform"]; exists {

193 - // if xs, ok := val.([]IndexSlice); ok {

194 - // indexTransform = xs

195 - // }

196 - // }*/

197 -

198 - castedTableMetricTags = append(castedTableMetricTags, metric.MetricTags...)

199 - }

200 -

201 - if metric.Symbols == nil {

202 - return metricParseResult{}, fmt.Errorf("when specifying a table, you must specify a list of symbols %v", metric)

203 - }

204 -

205 - return (parseTableMetric(tableMetric{

206 - mib: metric.MIB,

207 - table: metric.Table,

208 - symbols: metric.Symbols,

209 - forcedType: metric.MetricType,

210 - metricTags: castedTableMetricTags,

211 - options: metric.Options,

212 - }))

213 -

214 - }

215 - return metricParseResult{}, fmt.Errorf("unsupported metric {%v}", metric)

216 -}

217 -

218 -// TODO error outs on functions

219 -func parseOIDMetric(metric oidMetric) metricParseResult {

220 - /*Parse a fully resolved OID/name metric.

221 -

222 - Note: This `OID/name` syntax is deprecated in favour of `symbol` syntax.

223 -

224 - Example:

225 -

226 - ```

227 - metrics:

228 - - OID: 1.3.6.1.2.1.2.1

229 - name: ifNumber

230 - ```

231 - */

232 - name := metric.name

233 - oid := metric.oid

234 -

235 - // TODO can't find a profile with this metric type

236 -

237 - parsed_symbol_metric := parsedSymbolMetric{

238 - name: name,

239 - tags: metric.metricTags,

240 - forcedType: metric.forcedType,

241 - enforceScalar: true,

242 - options: metric.options,

243 - baseoid: oid,

244 - }

245 -

246 - return metricParseResult{

247 - oidsToFetch: []string{oid},

248 - oidsToResolve: map[string]string{name: oid},

249 - parsedMetrics: []parsedMetric{parsed_symbol_metric},

250 - tableBatches: nil,

251 - indexMappings: nil,

97 + return (parseSymbolMetric(metric.Symbol, metric.MIB)) // TODO metric tags might be needed here.

98 + //Can't support tables at the moment

99 + } else {

100 + return metricParseResult{}, nil

101 }

102 }

103

104 // TODO error outs on functions

256 -func parseSymbolMetric(metric symbolMetric) (metricParseResult, error) {

105 +func parseSymbolMetric(symbol profiledefinition.SymbolConfig, mib string) (metricParseResult, error) {

106 /* Parse a symbol metric (= an OID in a MIB).

107 Example:

108

@@ -268,8 +117,7 @@ func parseSymbolMetric(metric symbolMetric) (metricParseResult, error) {

117 - MIB: IF-MIB

118 symbol: tcpActiveOpens # require MIB syntax

119 ```*/

271 - mib := metric.mib

272 - symbol := metric.symbol

120 +

121 parsed_symbol, err := parseSymbol(mib, symbol)

122 if err != nil {

123 return metricParseResult{}, err

@@ -277,10 +125,10 @@ func parseSymbolMetric(metric symbolMetric) (metricParseResult, error) {

125

126 parsed_symbol_metric := parsedSymbolMetric{

127 name: parsed_symbol.name,

280 - tags: metric.metricTags,

281 - forcedType: metric.forcedType,

128 + tags: nil,

129 + forcedType: string(symbol.MetricType),

130 enforceScalar: false,

283 - options: metric.options,

131 + options: nil,

132 extractValuePattern: parsed_symbol.extractValuePattern,

133 baseoid: parsed_symbol.oid,

134 }

@@ -294,341 +142,6 @@ func parseSymbolMetric(metric symbolMetric) (metricParseResult, error) {

142 }, nil

143 }

144

297 -// TODO error outs on functions

298 -func parseTableMetric(metric tableMetric) (metricParseResult, error) {

299 -

300 - mib := metric.mib

301 - parsed_table, err := parseSymbol(mib, metric.table)

302 - if err != nil {

303 - return metricParseResult{}, err

304 - }

305 -

306 - table_name := parsed_table.name

307 - table_oid := parsed_table.oid

308 -

309 - oids_to_resolve := parsed_table.oidsToResolve

310 -

311 - var index_tags []indexTag

312 - var column_tags []columnTag

313 - var index_mappings []indexMapping

314 - var table_batches map[tableBatchKey]tableBatch

315 -

316 - if metric.metricTags != nil {

317 - for _, metric_tag := range metric.metricTags {

318 - parsed_table_metric_tag, err := parseTableMetricTag(mib, parsed_table, metric_tag)

319 - if err != nil {

320 - return metricParseResult{}, err

321 - }

322 -

323 - if parsed_table_metric_tag.oidsToResolve != nil {

324 - oids_to_resolve = mergeStringMaps(oids_to_resolve, parsed_table_metric_tag.oidsToResolve)

325 -

326 - column_tags = append(column_tags, parsed_table_metric_tag.columnTags...)

327 -

328 - table_batches = mergeTableBatches(table_batches, parsed_table_metric_tag.tableBatches)

329 - } else {

330 -

331 - index_tags = append(index_tags, parsed_table_metric_tag.indexTags...)

332 -

333 - for index, mapping := range parsed_table_metric_tag.indexMappings {

334 - for _, symbol := range metric.symbols {

335 - index_mappings = append(index_mappings, indexMapping{tag: symbol.Name, index: index, mapping: mapping})

336 - }

337 -

338 - for _, tag := range metric.metricTags {

339 - if reflect.DeepEqual(tag.Symbol, Symbol{}) {

340 - tag = TableMetricTag{

341 - Tag: tag.Tag,

342 - Symbol: tag.Symbol,

343 - }

344 - index_mappings = append(index_mappings, indexMapping{

345 - tag: tag.Symbol.Name,

346 - index: index,

347 - mapping: mapping,

348 - })

349 - }

350 - }

351 - }

352 - }

353 - }

354 - }

355 -

356 - table_oids := []string{}

357 - parsed_metrics := []parsedMetric{}

358 -

359 - for _, symbol := range metric.symbols {

360 - parsed_symbol, err := parseSymbol(mib, symbol)

361 - if err != nil {

362 - return metricParseResult{}, nil

363 - }

364 -

365 - for key, value := range parsed_symbol.oidsToResolve {

366 - oids_to_resolve[key] = value

367 - }

368 -

369 - table_oids = append(table_oids, parsed_symbol.oid)

370 -

371 - parsed_table_metric := parsedTableMetric{

372 - name: parsed_symbol.name,

373 - indexTags: index_tags,

374 - columnTags: column_tags,

375 - forcedType: metric.forcedType,

376 - options: metric.options,

377 - extractValuePattern: parsed_symbol.extractValuePattern,

378 - rowOID: parsed_symbol.oid,

379 - tableName: table_name,

380 - tableOID: table_oid,

381 - }

382 -

383 - parsed_metrics = append(parsed_metrics, parsed_table_metric)

384 - }

385 -

386 - table_batches = mergeTableBatches(table_batches, map[tableBatchKey]tableBatch{{mib: mib, table: parsed_table.name}: {tableOID: parsed_table.oid, oids: table_oids}})

387 -

388 - return metricParseResult{

389 - oidsToFetch: []string{},

390 - oidsToResolve: oids_to_resolve,

391 - tableBatches: table_batches,

392 - indexMappings: index_mappings,

393 - parsedMetrics: parsed_metrics,

394 - }, nil

395 -}

396 -

397 -func parseTableMetricTag(mib string, parsed_table parsedSymbol, metric_tag TableMetricTag) (parsedTableMetricTag, error) {

398 - /*

399 - Parse an item of the `metric_tags` section of a table metric.

400 -

401 - Items can be:

402 -

403 - * A reference to a column in the same table.

404 -

405 - Example using entPhySensorTable in ENTITY-SENSOR-MIB:

406 -

407 - ```

408 - metric_tags:

409 - - tag: sensor_type

410 - column: entPhySensorType

411 - # OR

412 - column:

413 - OID: 1.3.6.1.2.1.99.1.1.1.1

414 - name: entPhySensorType

415 -

416 - ```

417 -

418 - * A reference to a column in a different table.

419 -

420 - Example:

421 -

422 - ```

423 - metric_tags:

424 - - tag: adapter

425 - table: genericAdaptersAttrTable

426 - column: adapterName

427 - # OR

428 - column:

429 - OID: 1.3.6.1.4.1.343.2.7.2.2.1.1.1.2

430 - name: adapterName

431 -

432 - ```

433 -

434 - * A reference to an OID by its index in the table entry.

435 -

436 - An optional `mapping` can be used to map index values to human-readable strings.

437 -

438 - Example using ipIfStatsTable in IP-MIB:

439 -

440 - ```

441 - metric_tags:

442 - - # ipIfStatsIPVersion (1.3.6.1.2.1.4.21.3.1.1)

443 - tag: ip_version

444 - index: 1

445 - mapping:

446 - 0: unknown

447 - 1: ipv4

448 - 2: ipv6

449 - 3: ipv4z

450 - 4: ipv6z

451 - 16: dns

452 - - # ipIfStatsIfIndex (1.3.6.1.2.1.4.21.3.1.2)

453 - tag: interface

454 - index: 2

455 - ```

456 - */

457 - if metric_tag.Symbol != (Symbol{}) {

458 - metric_tag_mib := metric_tag.MIB

459 -

460 - if metric_tag.Table != "" {

461 - return parseOtherTableColumnMetricTag(metric_tag_mib, metric_tag.Table, metric_tag)

462 - }

463 -

464 - if mib != metric_tag_mib && metric_tag_mib != "" {

465 - return parsedTableMetricTag{}, fmt.Errorf("when tagging from a different MIB, the table must be specified, TABLE_MAME: %s MIB: %s METRIC_TAG_MIB: %s", parsed_table.name, mib, metric_tag_mib)

466 - }

467 - return parseColumnMetricTag(mib, parsed_table, metric_tag)

468 - } else if &metric_tag.Index != nil { //TODO, this is tautological condition, need to return a "-1" on index if it does not exist or something similar

469 - return parseIndexMetricTag(metric_tag)

470 - } else {

471 - return parsedTableMetricTag{}, errors.New("symbol is empty")

472 - }

473 -

474 -}

475 -

476 -func parseIndexMetricTag(metric_tag TableMetricTag) (parsedTableMetricTag, error) {

477 - parsed_metric_tag, err := parseMetricTag(

478 - MetricTag{

479 - Tag: metric_tag.Tag,

480 - })

481 - if err != nil {

482 - return parsedTableMetricTag{}, err

483 - }

484 -

485 - index_tags := []indexTag{{

486 - parsedMetricTag: parsed_metric_tag,

487 - index: metric_tag.Index,

488 - }}

489 -

490 - index_mappings := map[int]map[int]string{}

491 -

492 - if metric_tag.Mapping != nil {

493 - index_mappings = map[int]map[int]string{metric_tag.Index: metric_tag.Mapping}

494 - }

495 -

496 - return parsedTableMetricTag{

497 - indexTags: index_tags,

498 - indexMappings: index_mappings,

499 - }, nil

500 -}

501 -

502 -func parseOtherTableColumnMetricTag(mib string, table string, metric_tag TableMetricTag) (parsedTableMetricTag, error) {

503 - parsed_table, err := parseSymbol(mib, &table)

504 - if err != nil {

505 - return parsedTableMetricTag{}, err

506 - }

507 - parsed_metric_tag, err := parseColumnMetricTag(mib, parsed_table, metric_tag)

508 - if err != nil {

509 - return parsedTableMetricTag{}, err

510 - }

511 -

512 - oids_to_resolve := parsed_metric_tag.oidsToResolve

513 - oids_to_resolve = mergeStringMaps(oids_to_resolve, parsed_table.oidsToResolve)

514 -

515 - return parsedTableMetricTag{

516 - oidsToResolve: oids_to_resolve,

517 - tableBatches: parsed_metric_tag.tableBatches,

518 - columnTags: parsed_metric_tag.columnTags,

519 - }, nil

520 -}

521 -

522 -func parseColumnMetricTag(mib string, parsed_table parsedSymbol, metric_tag TableMetricTag) (parsedTableMetricTag, error) {

523 - parsed_column, err := parseSymbol(mib, metric_tag.Symbol)

524 - if err != nil {

525 - return parsedTableMetricTag{}, err

526 - }

527 -

528 - batches := map[tableBatchKey]tableBatch{

529 - {mib: mib, table: parsed_table.name}: {tableOID: parsed_table.oid, oids: []string{parsed_column.oid}},

530 - }

531 -

532 - parsed_metric_tag, err := parseMetricTag(MetricTag{MIB: metric_tag.MIB, OID: "", Tag: metric_tag.Tag, Symbol: metric_tag.Symbol})

533 - if err != nil {

534 - return parsedTableMetricTag{}, err

535 - }

536 -

537 - return parsedTableMetricTag{

538 - oidsToResolve: parsed_column.oidsToResolve,

539 - columnTags: []columnTag{{

540 - parsedMetricTag: parsed_metric_tag,

541 - column: parsed_column.name,

542 - indexSlices: parseIndexSlices(metric_tag),

543 - },

544 - },

545 - tableBatches: batches,

546 - }, nil

547 -}

548 -

549 -func parseIndexSlices(metric_tag TableMetricTag) []IndexSlice {

550 - /*

551 - // Transform index_transform into list of index slices.

552 -

553 - // `index_transform` is needed to support tagging using another table with different indexes.

554 -

555 - // Example: TableB have two indexes indexX (1 digit) and indexY (3 digits).

556 - // We want to tag by an external TableA that have indexY (3 digits).

557 -

558 - // For example TableB has a row with full index `1.2.3.4`, indexX is `1` and indexY is `2.3.4`.

559 - // TableA has a row with full index `2.3.4`, indexY is `2.3.4` (matches indexY of TableB).

560 -

561 - // SNMP integration doesn't know how to compare the full indexes from TableB and TableA.

562 - // We need to extract a subset of the full index of TableB to match with TableA full index.

563 -

564 - // Using the below `index_transform` we provide enough info to extract a subset of index that

565 - // will be used to match TableA's full index.

566 -

567 - // ```yaml

568 - // index_transform:

569 - // - start: 1

570 - // - end: 3

571 - //

572 - // ```

573 - */

574 - raw_index_slices := metric_tag.IndexTransform

575 - index_slices := []IndexSlice{}

576 -

577 - for _, rule := range raw_index_slices {

578 - start, end := rule.Start, rule.End

579 - if start > end {

580 - log.Println("start bigger than end")

581 - return nil

582 - }

583 - if start < 0 {

584 - log.Println("start is negative")

585 - return nil

586 - }

587 - index_slices = append(index_slices, IndexSlice{start, end + 1})

588 -

589 - }

590 -

591 - return index_slices

592 -}

593 -

594 -func parseMetricTag(metric_tag MetricTag) (parsedMetricTag, error) {

595 - parsed_metric_tag := parsedMetricTag{}

596 -

597 - if metric_tag.Tag != "" {

598 - parsed_metric_tag = parseSimpleMetricTag(metric_tag)

599 - } else if metric_tag.Match != "" && metric_tag.Tags != nil {

600 - tmp, err := parseRegexMetricTag(metric_tag)

601 - if err != nil {

602 - return parsedMetricTag{}, err

603 - } else {

604 - parsed_metric_tag = tmp

605 - }

606 - } else {

607 - return parsedMetricTag{}, fmt.Errorf("a metric tag must specify either a tag, or a mapping of tags and a regular expression %v", metric_tag)

608 - }

609 - return parsed_metric_tag, nil

610 -}

611 -

612 -func parseRegexMetricTag(metric_tag MetricTag) (parsedMetricTag, error) {

613 - match := metric_tag.Match

614 - tags := metric_tag.Tags

615 -

616 - // To be supported once tags are supported

617 - // if reflect.TypeOf(tags) != reflect.TypeOf(map[string]string{}) {

618 - // }

619 -

620 - pattern, err := regexp.Compile(match)

621 - if err != nil {

622 - return parsedMetricTag{}, err

623 - }

624 -

625 - return parsedMetricTag{tags: tags, pattern: pattern}, nil

626 -}

627 -

628 -func parseSimpleMetricTag(metric_tag MetricTag) parsedMetricTag {

629 - return parsedMetricTag{name: metric_tag.Tag}

630 -}

631 -

145 func parseSymbol(mib string, symbol interface{}) (parsedSymbol, error) {

146 /*

147 Parse an OID symbol.

@@ -647,12 +160,9 @@ func parseSymbol(mib string, symbol interface{}) (parsedSymbol, error) {

160 name: ifNumber

161 ```

162 */

650 - // if reflect.TypeOf(symbol) == reflect.TypeOf(string) {

651 - // // TODO, here they use ObjectIdentity(mib,symbol) to resolve the symbol. this is not straightfowrard in Go, it is a pysnmp function.

652 - // // oid:=

163

164 switch s := symbol.(type) {

655 - case Symbol:

165 + case profiledefinition.SymbolConfig:

166 oid := s.OID

167 name := s.Name

168 if s.ExtractValue != "" {

src/go/plugin/go.d/collector/snmp/profile.go

+5 -101

@@ -3,12 +3,10 @@ package snmp

3 import (

4 "fmt"

5 "log"

6 - "os"

7 - "path/filepath"

8 - "strings"

7 "github.com/gosnmp/gosnmp"

11 - "gopkg.in/yaml.v3"

8 +

9 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp"

10 )

12 func (s *SysObjectIDs) UnmarshalYAML(unmarshal func(any) error) error {

@@ -27,10 +25,11 @@ func (s *SysObjectIDs) UnmarshalYAML(unmarshal func(any) error) error {

25 return fmt.Errorf("invalid sysobjectid format")

26 }

30 -func (c *Collector) parseMetricsFromProfiles(matchingProfiles []*Profile) (map[string]processedMetric, error) {

28 +func (c *Collector) parseMetricsFromProfiles(matchingProfiles []*ddsnmp.Profile) (map[string]processedMetric, error) {

29 metricMap := map[string]processedMetric{}

30 for _, profile := range matchingProfiles {

33 - results, err := parseMetrics(profile.Metrics)

31 + profileDef := profile.Definition

32 + results, err := parseMetrics(profileDef.Metrics)

33 if err != nil {

34 return nil, err

35 }

@@ -86,98 +85,3 @@ func (c *Collector) parseMetricsFromProfiles(matchingProfiles []*Profile) (map[s

85 }

86 return metricMap, nil

87 }

89 -

90 -func (s *Symbol) UnmarshalYAML(node *yaml.Node) error {

91 - // If scalar node, assume the value is the name.

92 - if node.Kind == yaml.ScalarNode {

93 - s.Name = node.Value

94 - return nil

95 - }

96 -

97 - // Otherwise, decode normally

98 - type plainSymbol Symbol

99 - var ps plainSymbol

100 - if err := node.Decode(&ps); err != nil {

101 - return err

102 - }

103 - *s = Symbol(ps)

104 - return nil

105 -}

106 -

107 -func LoadAllProfiles(profileDir string) (map[string]*Profile, error) {

108 - profiles := make(map[string]*Profile)

109 - err := filepath.Walk(profileDir, func(path string, info os.FileInfo, err error) error {

110 - if err != nil {

111 - return err

112 - }

113 - if strings.HasSuffix(info.Name(), ".yaml") {

114 - profile, err := LoadYAML(path, profileDir)

115 - if err == nil {

116 - profiles[path] = profile

117 - } else {

118 - log.Printf("Skipping invalid YAML: %s (%v)\n", path, err)

119 - }

120 - }

121 - return nil

122 - })

123 -

124 - if err != nil {

125 - return nil, err

126 - }

127 -

128 - return profiles, nil

129 -}

130 -

131 -func LoadYAML(filename string, basePath string) (*Profile, error) {

132 - data, err := os.ReadFile(filename)

133 - if err != nil {

134 - return nil, err

135 - }

136 -

137 - var profile Profile

138 - err = yaml.Unmarshal(data, &profile)

139 - if err != nil {

140 - return nil, err

141 - }

142 -

143 - // If the profile extends other files, load and merge them

144 - for _, parentFile := range profile.Extends {

145 - parentProfile, err := LoadYAML(filepath.Join(basePath, parentFile), basePath)

146 - if err != nil {

147 - return nil, err

148 - }

149 - MergeProfiles(&profile, parentProfile)

150 - }

151 -

152 - return &profile, nil

153 -}

154 -

155 -// Merge two profiles, giving priority to the child profile

156 -func MergeProfiles(child, parent *Profile) {

157 - if child.Metadata.Device.Fields == nil {

158 - child.Metadata.Device.Fields = make(map[string]Symbol)

159 - }

160 -

161 - for key, value := range parent.Metadata.Device.Fields {

162 - if _, exists := child.Metadata.Device.Fields[key]; !exists {

163 - child.Metadata.Device.Fields[key] = value

164 - }

165 - }

166 - child.Metrics = append(parent.Metrics, child.Metrics...)

167 -}

168 -

169 -// Find the matching profile based on sysObjectID

170 -func FindMatchingProfiles(profiles map[string]*Profile, deviceOID string) []*Profile {

171 - var matchedProfiles []*Profile

172 -

173 - for _, profile := range profiles {

174 - for _, oidPattern := range profile.SysObjectID {

175 - if strings.HasPrefix(deviceOID, strings.Split(oidPattern, "*")[0]) {

176 - matchedProfiles = append(matchedProfiles, profile)

177 - break

178 - }

179 - }

180 - }

181 -

182 - return matchedProfiles

183 -}

src/go/plugin/go.d/collector/snmp/types.go

-87

@@ -12,84 +12,8 @@ type snmpPDU struct {

12 metric_type gosnmp.Asn1BER

13 }

15 -type Profile struct {

16 - Extends []string `yaml:"extends"`

17 - SysObjectID SysObjectIDs `yaml:"sysobjectid"`

18 - Metadata Metadata `yaml:"metadata"`

19 - Metrics []Metric `yaml:"metrics"`

20 -}

21 -

15 type SysObjectIDs []string

24 -type Metadata struct {

25 - Device DeviceMetadata `yaml:"device"`

26 -}

27 -

28 -type DeviceMetadata struct {

29 - Fields map[string]Symbol `yaml:"fields"`

30 -}

31 -

32 -type Symbol struct {

33 - OID string `yaml:"OID,omitempty"`

34 - Name string `yaml:"name,omitempty"`

35 - MatchPattern string `yaml:"match_pattern,omitempty"`

36 - MatchValue string `yaml:"match_value,omitempty"`

37 - ExtractValue string `yaml:"extract_value,omitempty"`

38 -}

39 -

40 -// superset of OIDMetric, SymbolMetric and TableMetric

41 -type Metric struct {

42 - Name string `yaml:"name,omitempty"`

43 - OID string `yaml:"OID,omitempty"`

44 - //TODO check for only name existing in metric tag, as there is some case for that

45 - MetricTags []TableMetricTag `yaml:"metric_tags,omitempty"`

46 - MetricType string `yaml:"metric_type,omitempty"`

47 - Options map[string]string

48 -

49 - MIB string `yaml:"MIB,omitempty"`

50 - Symbol Symbol `yaml:"symbol,omitempty"` //can be either string or Symbol

51 -

52 - Table interface{} `yaml:"table,omitempty"` // can be either a string or Symbol

53 - Symbols []Symbol `yaml:"symbols,omitempty"`

54 -}

55 -

56 -type TableMetricTag struct {

57 - Index int `yaml:"index"`

58 - Mapping map[int]string `yaml:"mapping"`

59 -

60 - Tag string `yaml:"tag"`

61 -

62 - MIB string `yaml:"mib"`

63 - Symbol Symbol `yaml:"symbol"`

64 - Table string `yaml:"table"`

65 - IndexTransform []IndexSlice `yaml:"index_transform"`

66 -}

67 -

68 -type oidMetric struct {

69 - name string

70 - oid string

71 - metricTags []string

72 - forcedType string

73 - options map[string]string

74 -}

75 -

76 -type symbolMetric struct {

77 - mib string

78 - symbol interface{} //can be either string or Symbol

79 - forcedType string

80 - metricTags []string

81 - options map[string]string

82 -}

83 -

84 -type tableMetric struct {

85 - mib string

86 - table interface{} // can be either a string or Symbol

87 - symbols []Symbol

88 - forcedType string

89 - metricTags []TableMetricTag

90 - options map[string]string

91 -}

92 -

17 type parsedResult struct {

18 oids []string

19 next_oids []string

@@ -213,17 +137,6 @@ type metricParseResult struct {

137 parsedMetrics []parsedMetric

138 }

139

216 -type MetricTag struct {

217 - OID string

218 - MIB string

219 - Symbol Symbol

220 - // simple tag

221 - Tag string

222 - // regex matching

223 - Match string

224 - Tags []string

225 -}

226 -

140 type IndexSlice struct {

141 Start int

142 End int