@cryptotaxi247 / netdata / commits / e0a41d8fb

chore(go.d/ddsnmp): fix table metrics again (#20497)

Ilya Mashchenko committed Jun 16, 2025 at 21:59 UTC e0a41d8fbc8c216b0506ebcad5a15a00511f6aad
4 files changed +874 -223
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collect_table.go
+123 -58
@@ -6,7 +6,6 @@ import (
6 "errors"
7 "fmt"
8 "maps"
9 - "sort"
9 "strings"
10
11 "github.com/gosnmp/gosnmp"
@@ -141,8 +140,8 @@ func (c *Collector) processTableWalkResults(walkResults []tableWalkResult) ([]Me
140 // Process a single table's data
141 func (c *Collector) processTableData(cfg ddprofiledefinition.MetricsConfig, pdus map[string]gosnmp.SnmpPDU, allWalkedData map[string]map[string]gosnmp.SnmpPDU, tableNameToOID map[string]string) ([]Metric, error) {
142 // Try to use cache if available
144 - if cachedIndexes, ok := c.tableCache.getCachedIndexes(cfg.Table.OID); ok {
145 - metrics, err := c.collectTableWithCache(cfg, cachedIndexes, allWalkedData, tableNameToOID)
143 + if cachedOIDs, cachedTags, ok := c.tableCache.getCachedData(cfg); ok {
144 + metrics, err := c.collectTableWithCache(cfg, cachedOIDs, cachedTags, buildColumnOIDs(cfg))
145 if err == nil {
146 c.log.Debugf("Successfully collected table %s using cache", cfg.Table.Name)
147 return metrics, nil
@@ -162,37 +161,28 @@ func (c *Collector) processTableData(cfg ddprofiledefinition.MetricsConfig, pdus
161 allColumnOIDs = append(allColumnOIDs, oid)
162 }
163
165 - // Extract unique indexes from walked data
166 - indexSet := make(map[string]bool)
167 - for oid := range pdus {
164 + // Group PDUs by row index and build cache structure
165 + rows := make(map[string]map[string]gosnmp.SnmpPDU)
166 + oidCache := make(map[string]map[string]string) // For caching: index -> column OID -> full OID
167 + tagCache := make(map[string]map[string]string) // For caching: index -> tag name -> value
168 +
169 + for oid, pdu := range pdus {
170 for _, columnOID := range allColumnOIDs {
171 if strings.HasPrefix(oid, columnOID+".") {
172 index := strings.TrimPrefix(oid, columnOID+".")
171 - indexSet[index] = true
173 +
174 + if rows[index] == nil {
175 + rows[index] = make(map[string]gosnmp.SnmpPDU)
176 + oidCache[index] = make(map[string]string)
177 + tagCache[index] = make(map[string]string)
178 + }
179 + rows[index][columnOID] = pdu
180 + oidCache[index][columnOID] = oid
181 break
182 }
183 }
184 }
185
177 - // Convert to sorted slice of indexes
178 - indexes := make([]string, 0, len(indexSet))
179 - for index := range indexSet {
180 - indexes = append(indexes, index)
181 - }
182 - sort.Strings(indexes)
183 -
184 - // Cache the table structure (indexes only)
185 - c.tableCache.cacheIndexes(cfg.Table.OID, indexes)
186 - c.log.Debugf("Cached table %s structure with %d rows", cfg.Table.Name, len(indexes))
187 -
188 - // Now process the walked data to create metrics
189 - return c.processTableRows(cfg, indexes, pdus, allWalkedData, tableNameToOID)
190 -}
191 -
192 -func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, indexes []string, pdus map[string]gosnmp.SnmpPDU, allWalkedData map[string]map[string]gosnmp.SnmpPDU, tableNameToOID map[string]string) ([]Metric, error) {
193 - columnOIDs := buildColumnOIDs(cfg)
194 - tagColumnOIDs := buildTagColumnOIDs(cfg)
195 -
186 rowStaticTags := make(map[string]string)
187 for _, tag := range cfg.StaticTags {
188 if n, v, _ := strings.Cut(tag, ":"); n != "" && v != "" {
@@ -202,13 +192,12 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
192
193 var metrics []Metric
194
205 - for _, index := range indexes {
195 + for index, rowPDUs := range rows {
196 rowTags := make(map[string]string)
197
208 - // Process same-table tags
198 + // Process tags for this row
199 for columnOID, tagCfg := range tagColumnOIDs {
210 - fullOID := columnOID + "." + index
211 - pdu, ok := pdus[fullOID]
200 + pdu, ok := rowPDUs[columnOID]
201 if !ok {
202 continue
203 }
@@ -221,6 +210,7 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
210
211 for k, v := range tags {
212 rowTags[k] = v
213 + tagCache[index][k] = v
214 }
215 }
216
@@ -231,7 +221,7 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
221 continue
222 }
223
234 - // Skip if it's an index-based tag
224 + // Skip if it's an index-based tag (handled separately)
225 if tagCfg.Index != 0 {
226 continue
227 }
@@ -250,7 +240,6 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
240 continue
241 }
242
253 - // Determine the index to use for lookup
243 lookupIndex := index
244 if len(tagCfg.IndexTransform) > 0 {
245 lookupIndex = applyIndexTransform(index, tagCfg.IndexTransform)
@@ -260,7 +249,7 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
249 }
250 }
251
263 - // Look up the value from the referenced table
252 + // Look up the value from the referenced table using the same index
253 refColumnOID := trimOID(tagCfg.Symbol.OID)
254 refFullOID := refColumnOID + "." + lookupIndex
255
@@ -279,16 +268,18 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
268
269 for k, v := range tags {
270 rowTags[k] = v
271 + tagCache[index][k] = v
272 }
273 }
274
275 // Process index-based tags
276 for _, tagCfg := range cfg.MetricTags {
277 + // Skip if not an index-based tag
278 if tagCfg.Index == 0 {
279 continue
280 }
281
291 - indexValue, ok := getIndexPosition(index, uint(tagCfg.Index))
282 + indexValue, ok := getIndexPosition(index, tagCfg.Index)
283 if !ok {
284 c.log.Debugf("Cannot extract position %d from index %s", tagCfg.Index, index)
285 continue
@@ -296,19 +287,17 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
287
288 tagName := ternary(tagCfg.Tag != "", tagCfg.Tag, fmt.Sprintf("index%d", tagCfg.Index))
289
299 - if len(tagCfg.Mapping) > 0 {
300 - if mappedValue, ok := tagCfg.Mapping[indexValue]; ok {
301 - indexValue = mappedValue
302 - }
290 + if v, ok := tagCfg.Mapping[indexValue]; ok {
291 + indexValue = v
292 }
293
294 rowTags[tagName] = indexValue
295 + tagCache[index][tagName] = indexValue
296 }
297
298 // Process metrics for this row
299 for columnOID, sym := range columnOIDs {
310 - fullOID := columnOID + "." + index
311 - pdu, ok := pdus[fullOID]
300 + pdu, ok := rowPDUs[columnOID]
301 if !ok {
302 continue
303 }
@@ -336,6 +325,12 @@ func (c *Collector) processTableRows(cfg ddprofiledefinition.MetricsConfig, inde
325 }
326 }
327
328 + deps := extractTableDependencies(cfg, tableNameToOID)
329 +
330 + // Cache the processed data
331 + c.tableCache.cacheData(cfg, oidCache, tagCache, deps)
332 + c.log.Debugf("Cached table %s structure with %d rows", cfg.Table.Name, len(oidCache))
333 +
334 return metrics, nil
335 }
336
@@ -359,23 +354,18 @@ func buildTagColumnOIDs(cfg ddprofiledefinition.MetricsConfig) map[string]ddprof
354
355 func (c *Collector) collectTableWithCache(
356 cfg ddprofiledefinition.MetricsConfig,
362 - cachedIndexes []string,
363 - allWalkedData map[string]map[string]gosnmp.SnmpPDU,
364 - tableNameToOID map[string]string,
357 + cachedOIDs map[string]map[string]string,
358 + cachedTags map[string]map[string]string,
359 + columnOIDs map[string]ddprofiledefinition.SymbolConfig,
360 ) ([]Metric, error) {
366 - // Build list of OIDs to GET based on cached indexes
367 - columnOIDs := buildColumnOIDs(cfg)
368 - tagColumnOIDs := buildTagColumnOIDs(cfg)
369 -
361 var oidsToGet []string
371 - for _, index := range cachedIndexes {
372 - // Get metric columns
373 - for columnOID := range columnOIDs {
374 - oidsToGet = append(oidsToGet, columnOID+"."+index)
375 - }
376 - // Get tag columns (same table only)
377 - for columnOID := range tagColumnOIDs {
378 - oidsToGet = append(oidsToGet, columnOID+"."+index)
362 +
363 + for _, columns := range cachedOIDs {
364 + for columnOID, fullOID := range columns {
365 + // Only GET metric columns, tags are cached
366 + if _, isMetric := columnOIDs[columnOID]; isMetric {
367 + oidsToGet = append(oidsToGet, fullOID)
368 + }
369 }
370 }
371
@@ -388,12 +378,65 @@ func (c *Collector) collectTableWithCache(
378 return nil, fmt.Errorf("failed to get cached OIDs: %w", err)
379 }
380
391 - if len(pdus) < len(oidsToGet)/2 { // If we got less than half, table structure probably changed
381 + if len(pdus) < len(oidsToGet)/2 { // If we got less than half, probably table structure changed
382 return nil, fmt.Errorf("table structure may have changed, got %d/%d PDUs", len(pdus), len(oidsToGet))
383 }
384
395 - // Process the rows using the same logic
396 - return c.processTableRows(cfg, cachedIndexes, pdus, allWalkedData, tableNameToOID)
385 + rowStaticTags := make(map[string]string)
386 +
387 + for _, tag := range cfg.StaticTags {
388 + if n, v, _ := strings.Cut(tag, ":"); n != "" && v != "" {
389 + rowStaticTags[n] = v
390 + }
391 + }
392 +
393 + var metrics []Metric
394 +
395 + for index, columns := range cachedOIDs {
396 + rowTags := make(map[string]string)
397 +
398 + if tags, ok := cachedTags[index]; ok {
399 + for k, v := range tags {
400 + rowTags[k] = v
401 + }
402 + }
403 +
404 + for columnOID, fullOID := range columns {
405 + sym, isMetric := columnOIDs[columnOID]
406 + if !isMetric {
407 + continue
408 + }
409 +
410 + pdu, ok := pdus[trimOID(fullOID)]
411 + if !ok {
412 + c.log.Debugf("Missing PDU for cached OID %s", fullOID)
413 + continue
414 + }
415 +
416 + value, err := processSymbolValue(sym, pdu)
417 + if err != nil {
418 + c.log.Debugf("Error processing value for %s: %v", sym.Name, err)
419 + continue
420 + }
421 +
422 + metric := Metric{
423 + Name: sym.Name,
424 + Value: value,
425 + StaticTags: ternary(len(rowStaticTags) > 0, rowStaticTags, nil),
426 + Tags: ternary(len(rowTags) > 0, rowTags, nil),
427 + Unit: sym.Unit,
428 + Description: sym.Description,
429 + MetricType: getMetricType(sym, pdu),
430 + Family: sym.Family,
431 + Mappings: convSymMappingToNumeric(sym),
432 + IsTable: true,
433 + }
434 +
435 + metrics = append(metrics, metric)
436 + }
437 + }
438 +
439 + return metrics, nil
440 }
441
442 func processTableMetricTagValue(cfg ddprofiledefinition.MetricTagConfig, pdu gosnmp.SnmpPDU) (map[string]string, error) {
@@ -508,3 +551,25 @@ func applyIndexTransform(index string, transforms []ddprofiledefinition.MetricIn
551
552 return strings.Join(result, ".")
553 }
554 +
555 +func extractTableDependencies(cfg ddprofiledefinition.MetricsConfig, tableNameToOID map[string]string) []string {
556 + deps := make(map[string]bool)
557 +
558 + for _, tagCfg := range cfg.MetricTags {
559 + // Skip if not a cross-table tag
560 + if tagCfg.Table == "" || tagCfg.Table == cfg.Table.Name {
561 + continue
562 + }
563 +
564 + if tableOID, ok := tableNameToOID[tagCfg.Table]; ok {
565 + deps[tableOID] = true
566 + }
567 + }
568 +
569 + result := make([]string, 0, len(deps))
570 + for oid := range deps {
571 + result = append(result, oid)
572 + }
573 +
574 + return result
575 +}
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector.go
+1 -1
@@ -46,7 +46,7 @@ func New(snmpClient gosnmp.Handler, profiles []*ddsnmp.Profile, log *logger.Logg
46 snmpClient: snmpClient,
47 profiles: make(map[string]*profileState),
48 missingOIDs: make(map[string]bool),
49 - tableCache: newTableCache(5*time.Minute, 1), // 5 min TTL with 100% jitter
49 + tableCache: newTableCache(10*time.Minute, 1), // 5 min TTL with 100% jitter
50 //doTableMetrics: true,
51 }
52
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/table_cache.go
+202 -32
@@ -4,27 +4,33 @@ package ddsnmpcollector
4
5 import (
6 "math/rand"
7 + "sort"
8 + "strings"
9 "sync"
10 "time"
11 +
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition"
13 )
14
15 // Table Cache Overview:
12 -// The table cache stores table structure (which rows exist) to convert repeated
13 -// SNMP walks into efficient GET operations.
14 -// - First collection: Full walk, cache row indexes
15 -// - Subsequent collections: GET only the columns needed for each MetricsConfig
16 +// The table cache converts repeated SNMP walks into efficient GET operations.
17 +// - First collection: Full walk, cache structure and tags
18 +// - Subsequent collections: GET metrics only, use cached tags
19 +// - Tables with dependencies expire together to maintain consistency
20 +// - Supports multiple metric configurations per table
21
22 type tableCache struct {
18 - // Table OID -> list of indexes (rows) that exist
19 - tableIndexes map[string][]string
23 + // Table OID -> config ID -> cached entry
24 + tables map[string]map[string]tableCacheEntry
25
21 - // Table OID -> when cached
26 + // Table OID -> when cached (table level, not config level)
27 timestamps map[string]time.Time
28
29 // Table OID -> specific TTL for this table (with jitter applied)
30 tableTTLs map[string]time.Duration
31
32 // Table OID -> list of dependent table OIDs (bidirectional)
33 + // If table A depends on table B, both A->B and B->A are stored
34 tableDeps map[string]map[string]bool
35
36 baseTTL time.Duration
@@ -33,15 +39,23 @@ type tableCache struct {
39 rng *rand.Rand
40 }
41
42 +type tableCacheEntry struct {
43 + // Index -> column OID -> full OID
44 + oidMap map[string]map[string]string
45 +
46 + // Index -> tag name -> value
47 + tagValues map[string]map[string]string
48 +}
49 +
50 func newTableCache(baseTTL time.Duration, jitterPct float64) *tableCache {
51 return &tableCache{
38 - tableIndexes: make(map[string][]string),
39 - timestamps: make(map[string]time.Time),
40 - tableTTLs: make(map[string]time.Duration),
41 - tableDeps: make(map[string]map[string]bool),
42 - baseTTL: baseTTL,
43 - jitterPct: jitterPct,
44 - rng: rand.New(rand.NewSource(time.Now().UnixNano())),
52 + tables: make(map[string]map[string]tableCacheEntry),
53 + timestamps: make(map[string]time.Time),
54 + tableTTLs: make(map[string]time.Duration),
55 + tableDeps: make(map[string]map[string]bool),
56 + baseTTL: baseTTL,
57 + jitterPct: jitterPct,
58 + rng: rand.New(rand.NewSource(time.Now().UnixNano())),
59 }
60 }
61
@@ -56,33 +70,41 @@ func (tc *tableCache) calculateTableTTL() time.Duration {
70 return time.Duration(base * multiplier)
71 }
72
59 -func (tc *tableCache) getCachedIndexes(tableOID string) ([]string, bool) {
73 +func (tc *tableCache) getCachedData(cfg ddprofiledefinition.MetricsConfig) (oids map[string]map[string]string, tags map[string]map[string]string, found bool) {
74 tc.mu.RLock()
75 defer tc.mu.RUnlock()
76
77 if tc.baseTTL == 0 {
64 - return nil, false
78 + return nil, nil, false
79 }
80
81 + tableOID := cfg.Table.OID
82 + configID := tc.generateConfigID(cfg)
83 +
84 timestamp, ok := tc.timestamps[tableOID]
85 if !ok {
69 - return nil, false
86 + return nil, nil, false
87 }
88
89 ttl, ok := tc.tableTTLs[tableOID]
90 if !ok || time.Since(timestamp) > ttl {
74 - return nil, false
91 + return nil, nil, false
92 }
93
77 - indexes := tc.tableIndexes[tableOID]
78 - return indexes, true
79 -}
94 + configEntries, ok := tc.tables[tableOID]
95 + if !ok {
96 + return nil, nil, false
97 + }
98 +
99 + entry, ok := configEntries[configID]
100 + if !ok {
101 + return nil, nil, false
102 + }
103
81 -func (tc *tableCache) cacheIndexes(tableOID string, indexes []string) {
82 - tc.cacheIndexesWithDeps(tableOID, indexes, nil)
104 + return entry.oidMap, entry.tagValues, true
105 }
106
85 -func (tc *tableCache) cacheIndexesWithDeps(tableOID string, indexes []string, dependencies []string) {
107 +func (tc *tableCache) cacheData(cfg ddprofiledefinition.MetricsConfig, oidMap map[string]map[string]string, tagValues map[string]map[string]string, dependencies []string) {
108 tc.mu.Lock()
109 defer tc.mu.Unlock()
110
@@ -90,13 +112,44 @@ func (tc *tableCache) cacheIndexesWithDeps(tableOID string, indexes []string, de
112 return
113 }
114
93 - // Deep copy the indexes
94 - indexesCopy := make([]string, len(indexes))
95 - copy(indexesCopy, indexes)
115 + tableOID := cfg.Table.OID
116 + configID := tc.generateConfigID(cfg)
117
97 - tc.tableIndexes[tableOID] = indexesCopy
98 - tc.timestamps[tableOID] = time.Now()
99 - tc.tableTTLs[tableOID] = tc.calculateTableTTL()
118 + // Deep copy the maps to avoid reference issues
119 + oidsCopy := make(map[string]map[string]string, len(oidMap))
120 + for index, columns := range oidMap {
121 + columnsCopy := make(map[string]string, len(columns))
122 + for colOID, fullOID := range columns {
123 + columnsCopy[colOID] = fullOID
124 + }
125 + oidsCopy[index] = columnsCopy
126 + }
127 +
128 + tagsCopy := make(map[string]map[string]string, len(tagValues))
129 + for index, tags := range tagValues {
130 + tagCopy := make(map[string]string, len(tags))
131 + for name, value := range tags {
132 + tagCopy[name] = value
133 + }
134 + tagsCopy[index] = tagCopy
135 + }
136 +
137 + // Create config entries map if it doesn't exist
138 + if tc.tables[tableOID] == nil {
139 + tc.tables[tableOID] = make(map[string]tableCacheEntry)
140 + }
141 +
142 + // Store the entry
143 + tc.tables[tableOID][configID] = tableCacheEntry{
144 + oidMap: oidsCopy,
145 + tagValues: tagsCopy,
146 + }
147 +
148 + // Update table-level metadata only if this is the first config for this table
149 + if _, exists := tc.timestamps[tableOID]; !exists {
150 + tc.timestamps[tableOID] = time.Now()
151 + tc.tableTTLs[tableOID] = tc.calculateTableTTL()
152 + }
153
154 // Set up bidirectional dependencies
155 if len(dependencies) > 0 {
@@ -142,7 +195,7 @@ func (tc *tableCache) clearExpired() []string {
195
196 // Clear all expired tables
197 for tableOID := range expiredTables {
145 - delete(tc.tableIndexes, tableOID)
198 + delete(tc.tables, tableOID)
199 delete(tc.timestamps, tableOID)
200 delete(tc.tableTTLs, tableOID)
201
@@ -175,9 +228,126 @@ func (tc *tableCache) setTTL(baseTTL time.Duration, jitterPct float64) {
228
229 if baseTTL == 0 {
230 // Clear cache if caching is disabled
178 - tc.tableIndexes = make(map[string][]string)
231 + tc.tables = make(map[string]map[string]tableCacheEntry)
232 tc.timestamps = make(map[string]time.Time)
233 tc.tableTTLs = make(map[string]time.Duration)
234 tc.tableDeps = make(map[string]map[string]bool)
235 }
236 }
237 +
238 +// Helper method to check if a group of tables is cached
239 +// All tables must be cached and not expired
240 +func (tc *tableCache) areTablesCached(tableOIDs []string) bool {
241 + tc.mu.RLock()
242 + defer tc.mu.RUnlock()
243 +
244 + if tc.baseTTL == 0 {
245 + return false
246 + }
247 +
248 + now := time.Now()
249 + for _, tableOID := range tableOIDs {
250 + timestamp, ok := tc.timestamps[tableOID]
251 + if !ok {
252 + return false
253 + }
254 +
255 + ttl, ok := tc.tableTTLs[tableOID]
256 + if !ok || now.Sub(timestamp) > ttl {
257 + return false
258 + }
259 + }
260 +
261 + return true
262 +}
263 +
264 +// Check if a specific table config is cached
265 +func (tc *tableCache) isConfigCached(cfg ddprofiledefinition.MetricsConfig) bool {
266 + tc.mu.RLock()
267 + defer tc.mu.RUnlock()
268 +
269 + if tc.baseTTL == 0 {
270 + return false
271 + }
272 +
273 + tableOID := cfg.Table.OID
274 + configID := tc.generateConfigID(cfg)
275 +
276 + timestamp, ok := tc.timestamps[tableOID]
277 + if !ok {
278 + return false
279 + }
280 +
281 + ttl, ok := tc.tableTTLs[tableOID]
282 + if !ok || time.Since(timestamp) > ttl {
283 + return false
284 + }
285 +
286 + configEntries, ok := tc.tables[tableOID]
287 + if !ok {
288 + return false
289 + }
290 +
291 + _, ok = configEntries[configID]
292 + return ok
293 +}
294 +
295 +// generateConfigID creates a unique identifier for a MetricsConfig based on its symbols
296 +func (tc *tableCache) generateConfigID(cfg ddprofiledefinition.MetricsConfig) string {
297 + var sb strings.Builder
298 +
299 + // Collect all symbol names
300 + names := make([]string, 0, len(cfg.Symbols))
301 + for _, sym := range cfg.Symbols {
302 + names = append(names, sym.Name)
303 + }
304 +
305 + // Sort to ensure consistent ordering
306 + sort.Strings(names)
307 +
308 + // Build the ID
309 + for i, name := range names {
310 + if i > 0 {
311 + sb.WriteByte(',')
312 + }
313 + sb.WriteString(name)
314 + }
315 +
316 + return sb.String()
317 +}
318 +
319 +func (tc *tableCache) stats() (tables int, configs int, withDeps int, totalDeps int) {
320 + tc.mu.RLock()
321 + defer tc.mu.RUnlock()
322 +
323 + tables = len(tc.tables)
324 +
325 + for _, configMap := range tc.tables {
326 + configs += len(configMap)
327 + }
328 +
329 + for _, deps := range tc.tableDeps {
330 + if len(deps) > 0 {
331 + withDeps++
332 + totalDeps += len(deps)
333 + }
334 + }
335 +
336 + return tables, configs, withDeps, totalDeps
337 +}
338 +
339 +func (tc *tableCache) getDependencies(tableOID string) []string {
340 + tc.mu.RLock()
341 + defer tc.mu.RUnlock()
342 +
343 + deps, ok := tc.tableDeps[tableOID]
344 + if !ok {
345 + return nil
346 + }
347 +
348 + result := make([]string, 0, len(deps))
349 + for dep := range deps {
350 + result = append(result, dep)
351 + }
352 + return result
353 +}
src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/table_cache_test.go
+548 -132
@@ -8,6 +8,8 @@ import (
8
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/require"
11 +
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition"
13 )
14
15 func TestTableCache(t *testing.T) {
@@ -22,73 +24,252 @@ func TestTableCache(t *testing.T) {
24 cache := newTableCache(100*time.Millisecond, 0.2)
25
26 // Test data
25 - tableOID := "1.3.6.1.2.1.2.2"
26 - indexes := []string{"1", "2", "3"}
27 + cfg := ddprofiledefinition.MetricsConfig{
28 + Table: ddprofiledefinition.SymbolConfig{
29 + OID: "1.3.6.1.2.1.2.2",
30 + Name: "ifTable",
31 + },
32 + Symbols: []ddprofiledefinition.SymbolConfig{
33 + {
34 + OID: "1.3.6.1.2.1.2.2.1.10",
35 + Name: "ifInOctets",
36 + },
37 + {
38 + OID: "1.3.6.1.2.1.2.2.1.16",
39 + Name: "ifOutOctets",
40 + },
41 + },
42 + }
43 +
44 + oidMap := map[string]map[string]string{
45 + "1": {
46 + "1.3.6.1.2.1.2.2.1.10": "1.3.6.1.2.1.2.2.1.10.1",
47 + "1.3.6.1.2.1.2.2.1.16": "1.3.6.1.2.1.2.2.1.16.1",
48 + },
49 + "2": {
50 + "1.3.6.1.2.1.2.2.1.10": "1.3.6.1.2.1.2.2.1.10.2",
51 + "1.3.6.1.2.1.2.2.1.16": "1.3.6.1.2.1.2.2.1.16.2",
52 + },
53 + }
54 + tagValues := map[string]map[string]string{
55 + "1": {"interface": "eth0"},
56 + "2": {"interface": "eth1"},
57 + }
58
59 // Cache data
29 - cache.cacheIndexes(tableOID, indexes)
60 + cache.cacheData(cfg, oidMap, tagValues, nil)
61
62 // Retrieve cached data - should work
32 - cachedIndexes, found := cache.getCachedIndexes(tableOID)
63 + cachedOIDs, cachedTags, found := cache.getCachedData(cfg)
64 assert.True(t, found)
34 - assert.Equal(t, indexes, cachedIndexes)
65 + assert.Equal(t, oidMap, cachedOIDs)
66 + assert.Equal(t, tagValues, cachedTags)
67 +
68 + // Try to get with different config (different symbols)
69 + cfg2 := cfg
70 + cfg2.Symbols = []ddprofiledefinition.SymbolConfig{
71 + {
72 + OID: "1.3.6.1.2.1.2.2.1.7",
73 + Name: "ifAdminStatus",
74 + },
75 + }
76 + _, _, found = cache.getCachedData(cfg2)
77 + assert.False(t, found)
78
79 // Wait for expiration (considering jitter)
80 time.Sleep(150 * time.Millisecond)
81
82 // Should be expired now
40 - _, found = cache.getCachedIndexes(tableOID)
83 + _, _, found = cache.getCachedData(cfg)
84 assert.False(t, found)
85
86 // Clean expired entries
87 expired := cache.clearExpired()
45 - assert.Contains(t, expired, tableOID)
88 + assert.Contains(t, expired, cfg.Table.OID)
89
90 // Cache should be empty now
48 - assert.Empty(t, cache.tableIndexes)
91 + assert.Empty(t, cache.tables)
92 assert.Empty(t, cache.timestamps)
93 assert.Empty(t, cache.tableTTLs)
94 })
95 }
96 }
97
98 +func TestTableCacheMultipleConfigs(t *testing.T) {
99 + cache := newTableCache(1*time.Hour, 0)
100 +
101 + tableOID := "1.3.6.1.2.1.2.2"
102 +
103 + // Config 1 - packet counters
104 + cfg1 := ddprofiledefinition.MetricsConfig{
105 + Table: ddprofiledefinition.SymbolConfig{
106 + OID: tableOID,
107 + Name: "ifTable",
108 + },
109 + Symbols: []ddprofiledefinition.SymbolConfig{
110 + {
111 + OID: "1.3.6.1.2.1.2.2.1.10",
112 + Name: "ifInOctets",
113 + },
114 + {
115 + OID: "1.3.6.1.2.1.2.2.1.16",
116 + Name: "ifOutOctets",
117 + },
118 + },
119 + }
120 + oidMap1 := map[string]map[string]string{
121 + "1": {
122 + "1.3.6.1.2.1.2.2.1.10": "1.3.6.1.2.1.2.2.1.10.1",
123 + "1.3.6.1.2.1.2.2.1.16": "1.3.6.1.2.1.2.2.1.16.1",
124 + },
125 + }
126 + tagValues1 := map[string]map[string]string{
127 + "1": {"interface": "eth0"},
128 + }
129 +
130 + // Config 2 - status metrics (same table, different metrics)
131 + cfg2 := ddprofiledefinition.MetricsConfig{
132 + Table: ddprofiledefinition.SymbolConfig{
133 + OID: tableOID,
134 + Name: "ifTable",
135 + },
136 + Symbols: []ddprofiledefinition.SymbolConfig{
137 + {
138 + OID: "1.3.6.1.2.1.2.2.1.7",
139 + Name: "ifAdminStatus",
140 + },
141 + {
142 + OID: "1.3.6.1.2.1.2.2.1.8",
143 + Name: "ifOperStatus",
144 + },
145 + },
146 + }
147 + oidMap2 := map[string]map[string]string{
148 + "1": {
149 + "1.3.6.1.2.1.2.2.1.7": "1.3.6.1.2.1.2.2.1.7.1",
150 + "1.3.6.1.2.1.2.2.1.8": "1.3.6.1.2.1.2.2.1.8.1",
151 + },
152 + }
153 + tagValues2 := map[string]map[string]string{
154 + "1": {"interface": "eth0", "type": "ethernet"},
155 + }
156 +
157 + // Cache both configs
158 + cache.cacheData(cfg1, oidMap1, tagValues1, nil)
159 + cache.cacheData(cfg2, oidMap2, tagValues2, nil)
160 +
161 + // Both should be retrievable
162 + cachedOIDs1, cachedTags1, found1 := cache.getCachedData(cfg1)
163 + assert.True(t, found1)
164 + assert.Equal(t, oidMap1, cachedOIDs1)
165 + assert.Equal(t, tagValues1, cachedTags1)
166 +
167 + cachedOIDs2, cachedTags2, found2 := cache.getCachedData(cfg2)
168 + assert.True(t, found2)
169 + assert.Equal(t, oidMap2, cachedOIDs2)
170 + assert.Equal(t, tagValues2, cachedTags2)
171 +
172 + // Stats should show 1 table, 2 configs
173 + tables, configs, _, _ := cache.stats()
174 + assert.Equal(t, 1, tables)
175 + assert.Equal(t, 2, configs)
176 +
177 + // Both configs should report as cached
178 + assert.True(t, cache.isConfigCached(cfg1))
179 + assert.True(t, cache.isConfigCached(cfg2))
180 +
181 + // Test that config ID generation is consistent
182 + // Create same config with symbols in different order
183 + cfg1Reordered := ddprofiledefinition.MetricsConfig{
184 + Table: cfg1.Table,
185 + Symbols: []ddprofiledefinition.SymbolConfig{
186 + cfg1.Symbols[1], // ifOutOctets first
187 + cfg1.Symbols[0], // ifInOctets second
188 + },
189 + }
190 +
191 + // Should find the same cached data (because symbols are sorted in configID)
192 + cachedOIDs1Reordered, _, found1Reordered := cache.getCachedData(cfg1Reordered)
193 + assert.True(t, found1Reordered)
194 + assert.Equal(t, cachedOIDs1, cachedOIDs1Reordered)
195 +}
196 +
197 func TestTableCacheDependencies(t *testing.T) {
198 cache := newTableCache(200*time.Millisecond, 0)
199
200 // Test data for three related tables
59 - table1OID := "1.3.6.1.2.1.2.2" // ifTable
60 - table2OID := "1.3.6.1.2.1.31.1.1" // ifXTable
61 - table3OID := "1.3.6.1.4.1.9.9.276" // cieIfInterfaceTable
201 + cfg1 := ddprofiledefinition.MetricsConfig{
202 + Table: ddprofiledefinition.SymbolConfig{
203 + OID: "1.3.6.1.2.1.2.2",
204 + Name: "ifTable",
205 + },
206 + Symbols: []ddprofiledefinition.SymbolConfig{
207 + {
208 + OID: "1.3.6.1.2.1.2.2.1.10",
209 + Name: "ifInOctets",
210 + },
211 + },
212 + }
213 +
214 + cfg2 := ddprofiledefinition.MetricsConfig{
215 + Table: ddprofiledefinition.SymbolConfig{
216 + OID: "1.3.6.1.2.1.31.1.1",
217 + Name: "ifXTable",
218 + },
219 + Symbols: []ddprofiledefinition.SymbolConfig{
220 + {
221 + OID: "1.3.6.1.2.1.31.1.1.1.6",
222 + Name: "ifHCInOctets",
223 + },
224 + },
225 + }
226
63 - indexes1 := []string{"1", "2"}
64 - indexes2 := []string{"1", "2"}
65 - indexes3 := []string{"1", "2"}
227 + cfg3 := ddprofiledefinition.MetricsConfig{
228 + Table: ddprofiledefinition.SymbolConfig{
229 + OID: "1.3.6.1.4.1.9.9.276",
230 + Name: "cieIfInterfaceTable",
231 + },
232 + Symbols: []ddprofiledefinition.SymbolConfig{
233 + {
234 + OID: "1.3.6.1.4.1.9.9.276.1.1",
235 + Name: "cieIfResetCount",
236 + },
237 + },
238 + }
239 +
240 + oidMap := map[string]map[string]string{
241 + "1": {"dummy": "dummy.1"},
242 + }
243 + tagValues := map[string]map[string]string{
244 + "1": {"interface": "eth0"},
245 + }
246
247 // Cache tables with dependencies
248 // table1 and table2 depend on each other
69 - cache.cacheIndexesWithDeps(table1OID, indexes1, []string{table2OID})
70 - cache.cacheIndexesWithDeps(table2OID, indexes2, []string{table1OID})
249 + cache.cacheData(cfg1, oidMap, tagValues, []string{cfg2.Table.OID})
250 + cache.cacheData(cfg2, oidMap, tagValues, []string{cfg1.Table.OID})
251
252 // table3 depends on table2
73 - cache.cacheIndexesWithDeps(table3OID, indexes3, []string{table2OID})
253 + cache.cacheData(cfg3, oidMap, nil, []string{cfg2.Table.OID})
254
255 // All tables should be cached
76 - assert.True(t, cache.areTablesCached([]string{table1OID, table2OID, table3OID}))
256 + assert.True(t, cache.areTablesCached([]string{cfg1.Table.OID, cfg2.Table.OID, cfg3.Table.OID}))
257
258 // Check dependencies
79 - deps1 := cache.getDependencies(table1OID)
80 - assert.Contains(t, deps1, table2OID)
259 + deps1 := cache.getDependencies(cfg1.Table.OID)
260 + assert.Contains(t, deps1, cfg2.Table.OID)
261
82 - deps2 := cache.getDependencies(table2OID)
83 - assert.Contains(t, deps2, table1OID)
84 - assert.Contains(t, deps2, table3OID) // Bidirectional
262 + deps2 := cache.getDependencies(cfg2.Table.OID)
263 + assert.Contains(t, deps2, cfg1.Table.OID)
264 + assert.Contains(t, deps2, cfg3.Table.OID) // Bidirectional
265
86 - deps3 := cache.getDependencies(table3OID)
87 - assert.Contains(t, deps3, table2OID)
266 + deps3 := cache.getDependencies(cfg3.Table.OID)
267 + assert.Contains(t, deps3, cfg2.Table.OID)
268
269 // Get cache stats
90 - tables, withDeps, totalDeps := cache.stats()
270 + tables, configs, withDeps, totalDeps := cache.stats()
271 assert.Equal(t, 3, tables)
272 + assert.Equal(t, 3, configs)
273 assert.Equal(t, 3, withDeps)
274 assert.Equal(t, 4, totalDeps) // 1->2, 2->1, 2->3, 3->2
275
@@ -100,12 +281,12 @@ func TestTableCacheDependencies(t *testing.T) {
281
282 // All three tables should be expired due to dependencies
283 assert.Len(t, expired, 3)
103 - assert.Contains(t, expired, table1OID)
104 - assert.Contains(t, expired, table2OID)
105 - assert.Contains(t, expired, table3OID)
284 + assert.Contains(t, expired, cfg1.Table.OID)
285 + assert.Contains(t, expired, cfg2.Table.OID)
286 + assert.Contains(t, expired, cfg3.Table.OID)
287
288 // Cache should be empty
108 - assert.Empty(t, cache.tableIndexes)
289 + assert.Empty(t, cache.tables)
290 assert.Empty(t, cache.tableDeps)
291 }
292
@@ -113,21 +294,56 @@ func TestTableCacheDependenciesCascade(t *testing.T) {
294 cache := newTableCache(100*time.Millisecond, 0)
295
296 // Create a chain: A -> B -> C -> D
116 - tableA := "1.3.6.1.2.1.1"
117 - tableB := "1.3.6.1.2.1.2"
118 - tableC := "1.3.6.1.2.1.3"
119 - tableD := "1.3.6.1.2.1.4"
297 + cfgA := ddprofiledefinition.MetricsConfig{
298 + Table: ddprofiledefinition.SymbolConfig{
299 + OID: "1.3.6.1.2.1.1",
300 + Name: "tableA",
301 + },
302 + Symbols: []ddprofiledefinition.SymbolConfig{
303 + {OID: "1.3.6.1.2.1.1.1", Name: "metricA"},
304 + },
305 + }
306 +
307 + cfgB := ddprofiledefinition.MetricsConfig{
308 + Table: ddprofiledefinition.SymbolConfig{
309 + OID: "1.3.6.1.2.1.2",
310 + Name: "tableB",
311 + },
312 + Symbols: []ddprofiledefinition.SymbolConfig{
313 + {OID: "1.3.6.1.2.1.2.1", Name: "metricB"},
314 + },
315 + }
316 +
317 + cfgC := ddprofiledefinition.MetricsConfig{
318 + Table: ddprofiledefinition.SymbolConfig{
319 + OID: "1.3.6.1.2.1.3",
320 + Name: "tableC",
321 + },
322 + Symbols: []ddprofiledefinition.SymbolConfig{
323 + {OID: "1.3.6.1.2.1.3.1", Name: "metricC"},
324 + },
325 + }
326 +
327 + cfgD := ddprofiledefinition.MetricsConfig{
328 + Table: ddprofiledefinition.SymbolConfig{
329 + OID: "1.3.6.1.2.1.4",
330 + Name: "tableD",
331 + },
332 + Symbols: []ddprofiledefinition.SymbolConfig{
333 + {OID: "1.3.6.1.2.1.4.1", Name: "metricD"},
334 + },
335 + }
336
121 - indexes := []string{"1"}
337 + data := map[string]map[string]string{"1": {"col": "val"}}
338
339 // Cache with chain dependencies
124 - cache.cacheIndexesWithDeps(tableA, indexes, []string{tableB})
125 - cache.cacheIndexesWithDeps(tableB, indexes, []string{tableA, tableC})
126 - cache.cacheIndexesWithDeps(tableC, indexes, []string{tableB, tableD})
127 - cache.cacheIndexesWithDeps(tableD, indexes, []string{tableC})
340 + cache.cacheData(cfgA, data, nil, []string{cfgB.Table.OID})
341 + cache.cacheData(cfgB, data, nil, []string{cfgA.Table.OID, cfgC.Table.OID})
342 + cache.cacheData(cfgC, data, nil, []string{cfgB.Table.OID, cfgD.Table.OID})
343 + cache.cacheData(cfgD, data, nil, []string{cfgC.Table.OID})
344
345 // All should be cached
130 - assert.True(t, cache.areTablesCached([]string{tableA, tableB, tableC, tableD}))
346 + assert.True(t, cache.areTablesCached([]string{cfgA.Table.OID, cfgB.Table.OID, cfgC.Table.OID, cfgD.Table.OID}))
347
348 // Wait for A to expire
349 time.Sleep(120 * time.Millisecond)
@@ -137,39 +353,61 @@ func TestTableCacheDependenciesCascade(t *testing.T) {
353
354 // All tables should expire due to cascade
355 assert.Len(t, expired, 4)
140 - assert.Contains(t, expired, tableA)
141 - assert.Contains(t, expired, tableB)
142 - assert.Contains(t, expired, tableC)
143 - assert.Contains(t, expired, tableD)
356 + assert.Contains(t, expired, cfgA.Table.OID)
357 + assert.Contains(t, expired, cfgB.Table.OID)
358 + assert.Contains(t, expired, cfgC.Table.OID)
359 + assert.Contains(t, expired, cfgD.Table.OID)
360 }
361
362 func TestTableCacheMixedDependencies(t *testing.T) {
363 cache := newTableCache(100*time.Millisecond, 0)
364
365 // Tables with deps
150 - table1 := "1.3.6.1.2.1.1"
151 - table2 := "1.3.6.1.2.1.2"
366 + cfg1 := ddprofiledefinition.MetricsConfig{
367 + Table: ddprofiledefinition.SymbolConfig{
368 + OID: "1.3.6.1.2.1.1",
369 + Name: "table1",
370 + },
371 + Symbols: []ddprofiledefinition.SymbolConfig{
372 + {OID: "1.3.6.1.2.1.1.1", Name: "metric1"},
373 + },
374 + }
375 +
376 + cfg2 := ddprofiledefinition.MetricsConfig{
377 + Table: ddprofiledefinition.SymbolConfig{
378 + OID: "1.3.6.1.2.1.2",
379 + Name: "table2",
380 + },
381 + Symbols: []ddprofiledefinition.SymbolConfig{
382 + {OID: "1.3.6.1.2.1.2.1", Name: "metric2"},
383 + },
384 + }
385
386 // Table without deps
154 - table3 := "1.3.6.1.2.1.3"
387 + cfg3 := ddprofiledefinition.MetricsConfig{
388 + Table: ddprofiledefinition.SymbolConfig{
389 + OID: "1.3.6.1.2.1.3",
390 + Name: "table3",
391 + },
392 + Symbols: []ddprofiledefinition.SymbolConfig{
393 + {OID: "1.3.6.1.2.1.3.1", Name: "metric3"},
394 + },
395 + }
396
156 - indexes := []string{"1", "2", "3"}
397 + data := map[string]map[string]string{"1": {"col": "val"}}
398
399 // Cache tables
159 - cache.cacheIndexesWithDeps(table1, indexes, []string{table2})
160 - cache.cacheIndexesWithDeps(table2, indexes, []string{table1})
161 - cache.cacheIndexes(table3, indexes) // No dependencies
400 + cache.cacheData(cfg1, data, nil, []string{cfg2.Table.OID})
401 + cache.cacheData(cfg2, data, nil, []string{cfg1.Table.OID})
402 + cache.cacheData(cfg3, data, nil, nil) // No dependencies
403
404 // All should be cached
164 - indexes1, found1 := cache.getCachedIndexes(table1)
165 - indexes2, found2 := cache.getCachedIndexes(table2)
166 - indexes3, found3 := cache.getCachedIndexes(table3)
405 + _, _, found1 := cache.getCachedData(cfg1)
406 + _, _, found2 := cache.getCachedData(cfg2)
407 + _, _, found3 := cache.getCachedData(cfg3)
408 assert.True(t, found1)
409 assert.True(t, found2)
410 assert.True(t, found3)
170 - assert.Equal(t, indexes, indexes1)
171 - assert.Equal(t, indexes, indexes2)
172 - assert.Equal(t, indexes, indexes3)
411
412 // Wait for expiration
413 time.Sleep(120 * time.Millisecond)
@@ -208,67 +446,113 @@ func TestTableCacheJitter(t *testing.T) {
446 func TestTableCacheDisabled(t *testing.T) {
447 cache := newTableCache(0, 0) // Disabled cache
448
211 - tableOID := "1.3.6.1.2.1.2.2"
212 - indexes := []string{"1", "2", "3"}
449 + cfg := ddprofiledefinition.MetricsConfig{
450 + Table: ddprofiledefinition.SymbolConfig{
451 + OID: "1.3.6.1.2.1.2.2",
452 + Name: "ifTable",
453 + },
454 + Symbols: []ddprofiledefinition.SymbolConfig{
455 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
456 + },
457 + }
458 +
459 + oidMap := map[string]map[string]string{
460 + "1": {"1.3.6.1.2.1.2.2.1.10": "1.3.6.1.2.1.2.2.1.10.1"},
461 + }
462 + tagValues := map[string]map[string]string{
463 + "1": {"interface": "eth0"},
464 + }
465
466 // Try to cache data
215 - cache.cacheIndexes(tableOID, indexes)
467 + cache.cacheData(cfg, oidMap, tagValues, nil)
468
469 // Should not find anything
218 - _, found := cache.getCachedIndexes(tableOID)
470 + _, _, found := cache.getCachedData(cfg)
471 assert.False(t, found)
472
473 // Try to cache with dependencies
222 - cache.cacheIndexesWithDeps(tableOID, indexes, []string{"other.table"})
474 + cache.cacheData(cfg, oidMap, tagValues, []string{"other.table"})
475
476 // Should not find anything
225 - _, found = cache.getCachedIndexes(tableOID)
477 + _, _, found = cache.getCachedData(cfg)
478 assert.False(t, found)
227 - assert.False(t, cache.areTablesCached([]string{tableOID}))
479 + assert.False(t, cache.areTablesCached([]string{cfg.Table.OID}))
480
481 // Cache should remain empty
230 - assert.Empty(t, cache.tableIndexes)
482 + assert.Empty(t, cache.tables)
483 }
484
485 func TestTableCacheDeepCopy(t *testing.T) {
486 cache := newTableCache(1*time.Hour, 0)
487
488 + cfg := ddprofiledefinition.MetricsConfig{
489 + Table: ddprofiledefinition.SymbolConfig{
490 + OID: "1.3.6.1.2.1.1",
491 + Name: "table1",
492 + },
493 + Symbols: []ddprofiledefinition.SymbolConfig{
494 + {OID: "1.3.6.1.2.1.1.1", Name: "metric1"},
495 + },
496 + }
497 +
498 // Original data
237 - indexes := []string{"1", "2", "3"}
499 + oidMap := map[string]map[string]string{
500 + "1": {"col1": "1.2.3.4.1"},
501 + }
502 + tagValues := map[string]map[string]string{
503 + "1": {"tag1": "value1"},
504 + }
505
506 // Cache the data
240 - cache.cacheIndexes("table1", indexes)
507 + cache.cacheData(cfg, oidMap, tagValues, nil)
508
242 - // Modify original slice
243 - indexes[0] = "999"
244 - indexes = append(indexes, "4")
509 + // Modify original maps
510 + oidMap["1"]["col2"] = "should not appear"
511 + tagValues["1"]["tag2"] = "should not appear"
512
513 // Retrieve cached data
247 - cachedIndexes, found := cache.getCachedIndexes("table1")
514 + cachedOIDs, cachedTags, found := cache.getCachedData(cfg)
515 require.True(t, found)
516
517 // Cached data should not have the modifications
251 - assert.Equal(t, []string{"1", "2", "3"}, cachedIndexes)
252 - assert.NotContains(t, cachedIndexes, "999")
253 - assert.NotContains(t, cachedIndexes, "4")
518 + assert.NotContains(t, cachedOIDs["1"], "col2")
519 + assert.NotContains(t, cachedTags["1"], "tag2")
520 }
521
522 func TestTableCacheDependencyCleanup(t *testing.T) {
523 cache := newTableCache(100*time.Millisecond, 0)
524
525 // Create circular dependencies
260 - table1 := "1.3.6.1.2.1.1"
261 - table2 := "1.3.6.1.2.1.2"
526 + cfg1 := ddprofiledefinition.MetricsConfig{
527 + Table: ddprofiledefinition.SymbolConfig{
528 + OID: "1.3.6.1.2.1.1",
529 + Name: "table1",
530 + },
531 + Symbols: []ddprofiledefinition.SymbolConfig{
532 + {OID: "1.3.6.1.2.1.1.1", Name: "metric1"},
533 + },
534 + }
535
263 - indexes := []string{"1"}
536 + cfg2 := ddprofiledefinition.MetricsConfig{
537 + Table: ddprofiledefinition.SymbolConfig{
538 + OID: "1.3.6.1.2.1.2",
539 + Name: "table2",
540 + },
541 + Symbols: []ddprofiledefinition.SymbolConfig{
542 + {OID: "1.3.6.1.2.1.2.1", Name: "metric2"},
543 + },
544 + }
545 +
546 + data := map[string]map[string]string{"1": {"col": "val"}}
547
548 // Cache with circular deps
266 - cache.cacheIndexesWithDeps(table1, indexes, []string{table2})
267 - cache.cacheIndexesWithDeps(table2, indexes, []string{table1})
549 + cache.cacheData(cfg1, data, nil, []string{cfg2.Table.OID})
550 + cache.cacheData(cfg2, data, nil, []string{cfg1.Table.OID})
551
552 // Check initial state
270 - tables, withDeps, totalDeps := cache.stats()
553 + tables, configs, withDeps, totalDeps := cache.stats()
554 assert.Equal(t, 2, tables)
555 + assert.Equal(t, 2, configs)
556 assert.Equal(t, 2, withDeps)
557 assert.Equal(t, 2, totalDeps)
558
@@ -279,8 +563,9 @@ func TestTableCacheDependencyCleanup(t *testing.T) {
563 cache.clearExpired()
564
565 // Check cleanup
282 - tables, withDeps, totalDeps = cache.stats()
566 + tables, configs, withDeps, totalDeps = cache.stats()
567 assert.Equal(t, 0, tables)
568 + assert.Equal(t, 0, configs)
569 assert.Equal(t, 0, withDeps)
570 assert.Equal(t, 0, totalDeps)
571
@@ -291,31 +576,52 @@ func TestTableCacheDependencyCleanup(t *testing.T) {
576 func TestTableCacheNonExistentDependency(t *testing.T) {
577 cache := newTableCache(100*time.Millisecond, 0)
578
579 + cfgA := ddprofiledefinition.MetricsConfig{
580 + Table: ddprofiledefinition.SymbolConfig{
581 + OID: "1.3.6.1.2.1.1",
582 + Name: "tableA",
583 + },
584 + Symbols: []ddprofiledefinition.SymbolConfig{
585 + {OID: "1.3.6.1.2.1.1.1", Name: "metricA"},
586 + },
587 + }
588 +
589 + cfgB := ddprofiledefinition.MetricsConfig{
590 + Table: ddprofiledefinition.SymbolConfig{
591 + OID: "1.3.6.1.2.1.2",
592 + Name: "tableB",
593 + },
594 + Symbols: []ddprofiledefinition.SymbolConfig{
595 + {OID: "1.3.6.1.2.1.2.1", Name: "metricB"},
596 + },
597 + }
598 +
599 // Cache tableA with dependency on non-existent tableB
295 - cache.cacheIndexesWithDeps("tableA",
296 - []string{"1", "2"},
297 - []string{"tableB"})
600 + cache.cacheData(cfgA,
601 + map[string]map[string]string{"1": {"col": "val"}},
602 + nil,
603 + []string{cfgB.Table.OID})
604
605 // tableA should be cached
300 - indexes, found := cache.getCachedIndexes("tableA")
606 + _, _, found := cache.getCachedData(cfgA)
607 assert.True(t, found)
302 - assert.Equal(t, []string{"1", "2"}, indexes)
608
609 // tableB should not be cached
305 - _, found = cache.getCachedIndexes("tableB")
610 + _, _, found = cache.getCachedData(cfgB)
611 assert.False(t, found)
612
613 // Dependencies should exist
309 - assert.Contains(t, cache.getDependencies("tableA"), "tableB")
310 - assert.Contains(t, cache.getDependencies("tableB"), "tableA")
614 + assert.Contains(t, cache.getDependencies(cfgA.Table.OID), cfgB.Table.OID)
615 + assert.Contains(t, cache.getDependencies(cfgB.Table.OID), cfgA.Table.OID)
616
617 // Now cache tableB
313 - cache.cacheIndexesWithDeps("tableB",
314 - []string{"1", "2"},
315 - []string{"tableA"})
618 + cache.cacheData(cfgB,
619 + map[string]map[string]string{"1": {"col": "val"}},
620 + nil,
621 + []string{cfgA.Table.OID})
622
623 // Both should be cached
318 - assert.True(t, cache.areTablesCached([]string{"tableA", "tableB"}))
624 + assert.True(t, cache.areTablesCached([]string{cfgA.Table.OID, cfgB.Table.OID}))
625
626 // Wait for expiration
627 time.Sleep(120 * time.Millisecond)
@@ -323,63 +629,173 @@ func TestTableCacheNonExistentDependency(t *testing.T) {
629 // Clear expired - both should expire together
630 expired := cache.clearExpired()
631 assert.Len(t, expired, 2)
326 - assert.Contains(t, expired, "tableA")
327 - assert.Contains(t, expired, "tableB")
632 + assert.Contains(t, expired, cfgA.Table.OID)
633 + assert.Contains(t, expired, cfgB.Table.OID)
634 }
635
330 -// Helper methods that need to be added to tableCache for tests
331 -func (tc *tableCache) areTablesCached(tableOIDs []string) bool {
332 - tc.mu.RLock()
333 - defer tc.mu.RUnlock()
636 +func TestTableCacheMultipleConfigsSameTableExpiration(t *testing.T) {
637 + cache := newTableCache(100*time.Millisecond, 0)
638 +
639 + tableOID := "1.3.6.1.2.1.2.2"
640
335 - if tc.baseTTL == 0 {
336 - return false
641 + cfg1 := ddprofiledefinition.MetricsConfig{
642 + Table: ddprofiledefinition.SymbolConfig{
643 + OID: tableOID,
644 + Name: "ifTable",
645 + },
646 + Symbols: []ddprofiledefinition.SymbolConfig{
647 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
648 + },
649 }
650
339 - now := time.Now()
340 - for _, tableOID := range tableOIDs {
341 - timestamp, ok := tc.timestamps[tableOID]
342 - if !ok {
343 - return false
344 - }
651 + cfg2 := ddprofiledefinition.MetricsConfig{
652 + Table: ddprofiledefinition.SymbolConfig{
653 + OID: tableOID,
654 + Name: "ifTable",
655 + },
656 + Symbols: []ddprofiledefinition.SymbolConfig{
657 + {OID: "1.3.6.1.2.1.2.2.1.16", Name: "ifOutOctets"},
658 + },
659 + }
660
346 - ttl, ok := tc.tableTTLs[tableOID]
347 - if !ok || now.Sub(timestamp) > ttl {
348 - return false
349 - }
661 + cfg3 := ddprofiledefinition.MetricsConfig{
662 + Table: ddprofiledefinition.SymbolConfig{
663 + OID: tableOID,
664 + Name: "ifTable",
665 + },
666 + Symbols: []ddprofiledefinition.SymbolConfig{
667 + {OID: "1.3.6.1.2.1.2.2.1.7", Name: "ifAdminStatus"},
668 + {OID: "1.3.6.1.2.1.2.2.1.8", Name: "ifOperStatus"},
669 + },
670 }
671
352 - return true
672 + // Cache multiple configs for the same table
673 + cache.cacheData(cfg1, map[string]map[string]string{"1": {"col1": "val1"}}, nil, nil)
674 + cache.cacheData(cfg2, map[string]map[string]string{"1": {"col2": "val2"}}, nil, nil)
675 + cache.cacheData(cfg3, map[string]map[string]string{"1": {"col3": "val3"}}, nil, nil)
676 +
677 + // All configs should be cached
678 + assert.True(t, cache.isConfigCached(cfg1))
679 + assert.True(t, cache.isConfigCached(cfg2))
680 + assert.True(t, cache.isConfigCached(cfg3))
681 +
682 + // Wait for expiration
683 + time.Sleep(120 * time.Millisecond)
684 +
685 + // Clear expired
686 + expired := cache.clearExpired()
687 +
688 + // Should expire the table once (not per config)
689 + assert.Len(t, expired, 1)
690 + assert.Contains(t, expired, tableOID)
691 +
692 + // All configs should be gone
693 + assert.False(t, cache.isConfigCached(cfg1))
694 + assert.False(t, cache.isConfigCached(cfg2))
695 + assert.False(t, cache.isConfigCached(cfg3))
696 }
697
355 -func (tc *tableCache) getDependencies(tableOID string) []string {
356 - tc.mu.RLock()
357 - defer tc.mu.RUnlock()
698 +func TestTableCacheConfigIsolation(t *testing.T) {
699 + cache := newTableCache(1*time.Hour, 0)
700 +
701 + tableOID := "1.3.6.1.2.1.2.2"
702
359 - deps, ok := tc.tableDeps[tableOID]
360 - if !ok {
361 - return nil
703 + cfg1 := ddprofiledefinition.MetricsConfig{
704 + Table: ddprofiledefinition.SymbolConfig{
705 + OID: tableOID,
706 + Name: "ifTable",
707 + },
708 + Symbols: []ddprofiledefinition.SymbolConfig{
709 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
710 + },
711 }
712
364 - result := make([]string, 0, len(deps))
365 - for dep := range deps {
366 - result = append(result, dep)
713 + cfg2 := ddprofiledefinition.MetricsConfig{
714 + Table: ddprofiledefinition.SymbolConfig{
715 + OID: tableOID,
716 + Name: "ifTable",
717 + },
718 + Symbols: []ddprofiledefinition.SymbolConfig{
719 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
720 + {OID: "1.3.6.1.2.1.2.2.1.16", Name: "ifOutOctets"},
721 + },
722 }
368 - return result
723 +
724 + // Different tag values for same table, different configs
725 + config1Tags := map[string]map[string]string{
726 + "1": {"interface": "eth0"},
727 + }
728 + config2Tags := map[string]map[string]string{
729 + "1": {"interface": "eth0", "type": "ethernet"},
730 + }
731 +
732 + // Cache configs with different tags
733 + cache.cacheData(cfg1, nil, config1Tags, nil)
734 + cache.cacheData(cfg2, nil, config2Tags, nil)
735 +
736 + // Retrieve and verify isolation
737 + _, tags1, found1 := cache.getCachedData(cfg1)
738 + assert.True(t, found1)
739 + assert.Equal(t, config1Tags, tags1)
740 + assert.NotContains(t, tags1["1"], "type") // Should not have config2's tag
741 +
742 + _, tags2, found2 := cache.getCachedData(cfg2)
743 + assert.True(t, found2)
744 + assert.Equal(t, config2Tags, tags2)
745 + assert.Contains(t, tags2["1"], "type") // Should have its own tag
746 }
747
371 -func (tc *tableCache) stats() (tables int, withDeps int, totalDeps int) {
372 - tc.mu.RLock()
373 - defer tc.mu.RUnlock()
748 +func TestTableCacheConfigIDGeneration(t *testing.T) {
749 + cache := newTableCache(1*time.Hour, 0)
750 +
751 + // Test that config ID is deterministic and based on symbol names
752 + cfg1 := ddprofiledefinition.MetricsConfig{
753 + Table: ddprofiledefinition.SymbolConfig{
754 + OID: "1.3.6.1.2.1.2.2",
755 + Name: "ifTable",
756 + },
757 + Symbols: []ddprofiledefinition.SymbolConfig{
758 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
759 + {OID: "1.3.6.1.2.1.2.2.1.16", Name: "ifOutOctets"},
760 + },
761 + }
762
375 - tables = len(tc.tableIndexes)
763 + // Same symbols but in different order
764 + cfg2 := ddprofiledefinition.MetricsConfig{
765 + Table: ddprofiledefinition.SymbolConfig{
766 + OID: "1.3.6.1.2.1.2.2",
767 + Name: "ifTable",
768 + },
769 + Symbols: []ddprofiledefinition.SymbolConfig{
770 + {OID: "1.3.6.1.2.1.2.2.1.16", Name: "ifOutOctets"}, // Swapped order
771 + {OID: "1.3.6.1.2.1.2.2.1.10", Name: "ifInOctets"},
772 + },
773 + }
774
377 - for _, deps := range tc.tableDeps {
378 - if len(deps) > 0 {
379 - withDeps++
380 - totalDeps += len(deps)
381 - }
775 + // Different symbols
776 + cfg3 := ddprofiledefinition.MetricsConfig{
777 + Table: ddprofiledefinition.SymbolConfig{
778 + OID: "1.3.6.1.2.1.2.2",
779 + Name: "ifTable",
780 + },
781 + Symbols: []ddprofiledefinition.SymbolConfig{
782 + {OID: "1.3.6.1.2.1.2.2.1.7", Name: "ifAdminStatus"},
783 + {OID: "1.3.6.1.2.1.2.2.1.8", Name: "ifOperStatus"},
784 + },
785 }
786
384 - return tables, withDeps, totalDeps
787 + // Generate config IDs
788 + id1 := cache.generateConfigID(cfg1)
789 + id2 := cache.generateConfigID(cfg2)
790 + id3 := cache.generateConfigID(cfg3)
791 +
792 + // Same symbols (even in different order) should generate same ID
793 + assert.Equal(t, id1, id2)
794 +
795 + // Different symbols should generate different ID
796 + assert.NotEqual(t, id1, id3)
797 +
798 + // IDs should be human-readable
799 + assert.Equal(t, "ifInOctets,ifOutOctets", id1)
800 + assert.Equal(t, "ifAdminStatus,ifOperStatus", id3)
801 }