master
go 515 lines 17.1 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 package chartengine
4
5 import (
6 "testing"
7 "time"
8
9 "github.com/stretchr/testify/assert"
10 "github.com/stretchr/testify/require"
11
12 "github.com/netdata/netdata/go/plugins/pkg/metrix"
13 metrixselector "github.com/netdata/netdata/go/plugins/pkg/metrix/selector"
14 )
15
16 func TestEngineRuntimeObservabilityScenarios(t *testing.T) {
17 tests := map[string]struct {
18 run func(t *testing.T)
19 }{
20 "build plan emits chartengine internal runtime metrics": {
21 run: func(t *testing.T) {
22 e, err := New()
23 require.NoError(t, err)
24 require.NoError(t, e.LoadYAML([]byte(runtimeObservabilityTemplateYAML()), 1))
25
26 store := metrix.NewCollectorStore()
27 cc := mustCycleController(t, store)
28 c := store.Write().SnapshotMeter("mysql").Counter("queries_total")
29
30 cc.BeginCycle()
31 c.ObserveTotal(10)
32 cc.CommitCycleSuccess()
33 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
34 require.NoError(t, err)
35
36 cc.BeginCycle()
37 c.ObserveTotal(20)
38 cc.CommitCycleSuccess()
39 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
40 require.NoError(t, err)
41
42 rs := e.RuntimeStore()
43 require.NotNil(t, rs)
44 r := rs.Read(metrix.ReadRaw())
45
46 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_success_total", nil, 2)
47 assertSummaryCountAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_duration_seconds", nil, 2)
48 assertSummaryCountAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_phase_duration_seconds", metrix.Labels{"phase": "scan"}, 2)
49 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_misses_total", nil, 1)
50 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_hits_total", nil, 1)
51 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_entries", nil, 1)
52 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_retained_total", nil, 1)
53 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.series_scanned_total", nil, 2)
54 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.planner_actions_total", metrix.Labels{"kind": "update_chart"}, 2)
55 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.plan_chart_instances", nil, 1)
56 assertMetricMeta(
57 t,
58 r,
59 "netdata.go.plugin.framework.chartengine.build_success_total",
60 metrix.MetricMeta{
61 Description: "Successful BuildPlan calls",
62 ChartFamily: "ChartEngine/Build",
63 Unit: "builds",
64 },
65 )
66 assertMetricMeta(
67 t,
68 r,
69 "netdata.go.plugin.framework.chartengine.planner_actions_total",
70 metrix.MetricMeta{
71 Description: "Planner actions by kind",
72 ChartFamily: "ChartEngine/Actions",
73 Unit: "actions",
74 },
75 )
76 },
77 },
78 "plan-size gauges keep last successful values on skipped build": {
79 run: func(t *testing.T) {
80 e, err := New()
81 require.NoError(t, err)
82 require.NoError(t, e.LoadYAML([]byte(runtimeObservabilityTemplateYAML()), 1))
83
84 store := metrix.NewCollectorStore()
85 cc := mustCycleController(t, store)
86 c := store.Write().SnapshotMeter("mysql").Counter("queries_total")
87
88 cc.BeginCycle()
89 c.ObserveTotal(10)
90 cc.CommitCycleSuccess()
91 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
92 require.NoError(t, err)
93
94 before := e.RuntimeStore().Read(metrix.ReadRaw())
95 beforeCharts, ok := before.Value("netdata.go.plugin.framework.chartengine.plan_chart_instances", nil)
96 require.True(t, ok)
97 require.GreaterOrEqual(t, beforeCharts, float64(1))
98
99 cc.BeginCycle()
100 cc.AbortCycle()
101 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
102 require.NoError(t, err)
103
104 after := e.RuntimeStore().Read(metrix.ReadRaw())
105 assertMetricValueAtLeast(t, after, "netdata.go.plugin.framework.chartengine.build_skipped_failed_collect_total", nil, 1)
106 assertMetricValueAtLeast(t, after, "netdata.go.plugin.framework.chartengine.build_success_total", nil, 1)
107
108 afterCharts, ok := after.Value("netdata.go.plugin.framework.chartengine.plan_chart_instances", nil)
109 require.True(t, ok)
110 assert.Equal(t, beforeCharts, afterCharts)
111 },
112 },
113 "selector-filtered series are counted by reason": {
114 run: func(t *testing.T) {
115 selectorExpr := metrixselector.Expr{
116 Allow: []string{`svc.errors_total`},
117 }
118 e, err := New(WithEnginePolicy(EnginePolicy{Selector: &selectorExpr}))
119 require.NoError(t, err)
120 require.NoError(t, e.LoadYAML([]byte(runtimeObservabilityTemplateYAML()), 1))
121
122 store := metrix.NewCollectorStore()
123 cc := mustCycleController(t, store)
124 c := store.Write().SnapshotMeter("mysql").Counter("queries_total")
125
126 cc.BeginCycle()
127 c.ObserveTotal(10)
128 cc.CommitCycleSuccess()
129 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
130 require.NoError(t, err)
131
132 r := e.RuntimeStore().Read(metrix.ReadRaw())
133 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.series_filtered_total", metrix.Labels{"reason": "by_selector"}, 1)
134 },
135 },
136 "expiry removals are counted by scope and reason": {
137 run: func(t *testing.T) {
138 e, err := New()
139 require.NoError(t, err)
140 require.NoError(t, e.LoadYAML([]byte(runtimeExpiryTemplateYAML()), 1))
141
142 store := metrix.NewCollectorStore()
143 cc := mustCycleController(t, store)
144 sm := store.Write().SnapshotMeter("svc")
145 total := sm.Gauge("total")
146 modeMetric := sm.Gauge("mode_metric")
147 modeOK := sm.LabelSet(metrix.Label{Key: "mode", Value: "ok"})
148
149 cc.BeginCycle()
150 total.Observe(100)
151 modeMetric.Observe(1, modeOK)
152 cc.CommitCycleSuccess()
153 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
154 require.NoError(t, err)
155
156 cc.BeginCycle()
157 total.Observe(101)
158 cc.CommitCycleSuccess()
159 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
160 require.NoError(t, err)
161
162 r := e.RuntimeStore().Read(metrix.ReadRaw())
163 assertMetricValueAtLeast(
164 t,
165 r,
166 "netdata.go.plugin.framework.chartengine.lifecycle_removed_total",
167 metrix.Labels{"scope": "dimension", "reason": "expiry"},
168 1,
169 )
170 },
171 },
172 "component runtime store metrics can be planned into charts": {
173 run: func(t *testing.T) {
174 producer, err := New()
175 require.NoError(t, err)
176 rs := producer.RuntimeStore()
177 require.NotNil(t, rs)
178
179 component := rs.Write().StatefulMeter("component").Counter("jobs_total")
180 component.Add(7)
181
182 observer, err := New(
183 WithRuntimeStore(nil),
184 WithSeriesSelectionAllVisible(),
185 WithRuntimePlannerMode(),
186 )
187 require.NoError(t, err)
188 require.NoError(t, observer.LoadYAML([]byte(runtimeComponentTemplateYAML()), 1))
189
190 plan, err := buildPlan(observer, rs.Read(metrix.ReadFlatten()))
191 require.NoError(t, err)
192 assert.Equal(t, []ActionKind{ActionCreateChart, ActionCreateDimension, ActionUpdateChart}, actionKinds(plan.Actions))
193
194 update := findUpdateAction(plan)
195 require.NotNil(t, update)
196 require.Equal(t, "netdata_go_plugin_component_component_jobs", update.ChartID)
197 require.Len(t, update.Values, 1)
198 assert.Equal(t, "total", update.Values[0].Name)
199 assert.False(t, update.Values[0].IsFloat)
200 assert.Equal(t, int64(7), update.Values[0].Int64)
201 },
202 },
203 "autogen planning uses runtime metric metadata": {
204 run: func(t *testing.T) {
205 producer, err := New()
206 require.NoError(t, err)
207 require.NoError(t, producer.LoadYAML([]byte(runtimeObservabilityTemplateYAML()), 1))
208
209 store := metrix.NewCollectorStore()
210 cc := mustCycleController(t, store)
211 c := store.Write().SnapshotMeter("mysql").Counter("queries_total")
212 cc.BeginCycle()
213 c.ObserveTotal(10)
214 cc.CommitCycleSuccess()
215 _, err = buildPlan(producer, store.Read(metrix.ReadFlatten()))
216 require.NoError(t, err)
217
218 observer, err := New(
219 WithRuntimeStore(nil),
220 WithEnginePolicy(EnginePolicy{Autogen: &AutogenPolicy{Enabled: true}}),
221 WithSeriesSelectionAllVisible(),
222 WithRuntimePlannerMode(),
223 )
224 require.NoError(t, err)
225 require.NoError(t, observer.LoadYAML([]byte(runtimeDummyTemplateYAML()), 1))
226
227 plan, err := buildPlan(observer, producer.RuntimeStore().Read(metrix.ReadFlatten()))
228 require.NoError(t, err)
229 create := findCreateChartByTitle(plan.Actions, "Successful BuildPlan calls")
230 require.NotNil(t, create)
231 assert.Equal(t, "Successful BuildPlan calls", create.Meta.Title)
232 assert.Equal(t, "ChartEngine/Build", create.Meta.Family)
233 assert.Equal(t, "builds/s", create.Meta.Units)
234
235 duration := findCreateChartByID(plan.Actions, "netdata.go.plugin.framework.chartengine.build_duration_seconds")
236 require.NotNil(t, duration)
237 assert.Equal(t, "BuildPlan duration in seconds", duration.Meta.Title)
238 assert.Equal(t, "ChartEngine/Build", duration.Meta.Family)
239 assert.Equal(t, "seconds", duration.Meta.Units)
240
241 durationSum := findCreateChartByID(plan.Actions, "netdata.go.plugin.framework.chartengine.build_duration_seconds_sum")
242 require.NotNil(t, durationSum)
243 assert.Equal(t, "BuildPlan duration in seconds", durationSum.Meta.Title)
244 assert.Equal(t, "ChartEngine/Build", durationSum.Meta.Family)
245 assert.Equal(t, "seconds", durationSum.Meta.Units)
246 },
247 },
248 "runtime aggregator rolls up multiple engine samples": {
249 run: func(t *testing.T) {
250 store := metrix.NewRuntimeStore()
251 agg := NewRuntimeAggregator(store)
252 now := testNow()
253 agg.Observe(PlanRuntimeSample{
254 startedAt: now,
255 buildSuccess: true,
256 planRouteStats: planRouteStats{routeCacheHits: 2, seriesScanned: 3},
257 routeCacheEntries: 4,
258 planChartInstances: 5,
259 planInferredDimensions: 6,
260 buildSeqObserved: true,
261 buildSeqViolation: true,
262 })
263 agg.Observe(PlanRuntimeSample{
264 startedAt: now,
265 buildSuccess: true,
266 planRouteStats: planRouteStats{routeCacheHits: 7, seriesScanned: 11},
267 routeCacheEntries: 13,
268 planChartInstances: 17,
269 planInferredDimensions: 19,
270 buildSeqObserved: true,
271 })
272
273 agg.Flush()
274
275 r := store.Read(metrix.ReadRaw())
276 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_success_total", nil, 2)
277 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.route_cache_hits_total", nil, 9)
278 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.series_scanned_total", nil, 14)
279 routeCacheEntries, ok := r.Value("netdata.go.plugin.framework.chartengine.route_cache_entries", nil)
280 require.True(t, ok)
281 assert.Equal(t, float64(17), routeCacheEntries)
282 planChartInstances, ok := r.Value("netdata.go.plugin.framework.chartengine.plan_chart_instances", nil)
283 require.True(t, ok)
284 assert.Equal(t, float64(22), planChartInstances)
285 planInferredDimensions, ok := r.Value("netdata.go.plugin.framework.chartengine.plan_inferred_dimensions", nil)
286 require.True(t, ok)
287 assert.Equal(t, float64(25), planInferredDimensions)
288 seqViolation, ok := r.Value("netdata.go.plugin.framework.chartengine.build_seq_violation_active", nil)
289 require.True(t, ok)
290 assert.Equal(t, float64(1), seqViolation)
291 assertSummaryCountAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_duration_seconds", nil, 2)
292 },
293 },
294 "runtime sample observer fires without direct runtime store": {
295 run: func(t *testing.T) {
296 observed := 0
297 e, err := New(
298 WithRuntimeStore(nil),
299 WithRuntimeSampleObserver(func(PlanRuntimeSample) {
300 observed++
301 }),
302 )
303 require.NoError(t, err)
304 require.NoError(t, e.LoadYAML([]byte(runtimeObservabilityTemplateYAML()), 1))
305
306 store := metrix.NewCollectorStore()
307 cc := mustCycleController(t, store)
308 c := store.Write().SnapshotMeter("mysql").Counter("queries_total")
309 cc.BeginCycle()
310 c.ObserveTotal(10)
311 cc.CommitCycleSuccess()
312
313 _, err = buildPlan(e, store.Read(metrix.ReadFlatten()))
314 require.NoError(t, err)
315 assert.Equal(t, 1, observed)
316 assert.Nil(t, e.RuntimeStore())
317 },
318 },
319 "outstanding attempt does not fire runtime sample observer": {
320 run: func(t *testing.T) {
321 observed := 0
322 e, err := New(
323 WithRuntimeStore(nil),
324 WithRuntimeSampleObserver(func(PlanRuntimeSample) {
325 observed++
326 }),
327 )
328 require.NoError(t, err)
329 require.NoError(t, e.LoadYAML([]byte(runtimeObservabilityTemplateYAML()), 1))
330
331 store := metrix.NewCollectorStore()
332 cc := mustCycleController(t, store)
333 c := store.Write().SnapshotMeter("mysql").Counter("queries_total")
334 cc.BeginCycle()
335 c.ObserveTotal(10)
336 cc.CommitCycleSuccess()
337 reader := store.Read(metrix.ReadFlatten())
338
339 attempt, err := e.PreparePlan(reader)
340 require.NoError(t, err)
341 defer attempt.Abort()
342 assert.Equal(t, 1, observed)
343
344 _, err = e.PreparePlan(reader)
345 require.ErrorIs(t, err, ErrOutstandingPlanAttempt)
346 assert.Equal(t, 1, observed)
347 },
348 },
349 "runtime aggregator nil store and empty flush are no-ops": {
350 run: func(t *testing.T) {
351 agg := NewRuntimeAggregator(nil)
352 require.NotPanics(t, func() {
353 agg.Flush()
354 agg.Observe(PlanRuntimeSample{startedAt: testNow(), buildSuccess: true})
355 agg.Reset()
356 agg.Flush()
357 })
358 },
359 },
360 "runtime aggregator clears build sequence violation on skipped-only rollup": {
361 run: func(t *testing.T) {
362 store := metrix.NewRuntimeStore()
363 agg := NewRuntimeAggregator(store)
364
365 agg.Observe(PlanRuntimeSample{
366 startedAt: testNow(),
367 buildSuccess: true,
368 buildSeqObserved: true,
369 buildSeqViolation: true,
370 })
371 agg.Flush()
372 r := store.Read(metrix.ReadRaw())
373 seqViolation, ok := r.Value("netdata.go.plugin.framework.chartengine.build_seq_violation_active", nil)
374 require.True(t, ok)
375 assert.Equal(t, float64(1), seqViolation)
376
377 agg.Observe(PlanRuntimeSample{startedAt: testNow(), skippedFailed: true})
378 agg.Flush()
379 r = store.Read(metrix.ReadRaw())
380 seqViolation, ok = r.Value("netdata.go.plugin.framework.chartengine.build_seq_violation_active", nil)
381 require.True(t, ok)
382 assert.Equal(t, float64(0), seqViolation)
383 assertMetricValueAtLeast(t, r, "netdata.go.plugin.framework.chartengine.build_skipped_failed_collect_total", nil, 1)
384 },
385 },
386 }
387
388 for name, tc := range tests {
389 t.Run(name, tc.run)
390 }
391 }
392
393 func testNow() time.Time {
394 return time.Now().Add(-time.Second)
395 }
396
397 func assertMetricValueAtLeast(t *testing.T, reader metrix.Reader, name string, labels metrix.Labels, min float64) {
398 t.Helper()
399 value, ok := reader.Value(name, labels)
400 require.Truef(t, ok, "expected metric %s with labels %v", name, labels)
401 assert.GreaterOrEqualf(t, value, min, "unexpected metric value for %s labels %v", name, labels)
402 }
403
404 func assertSummaryCountAtLeast(t *testing.T, reader metrix.Reader, name string, labels metrix.Labels, min float64) {
405 t.Helper()
406 point, ok := reader.Summary(name, labels)
407 require.Truef(t, ok, "expected summary %s with labels %v", name, labels)
408 assert.GreaterOrEqualf(t, point.Count, min, "unexpected summary count for %s labels %v", name, labels)
409 }
410
411 func assertMetricMeta(t *testing.T, reader metrix.Reader, name string, want metrix.MetricMeta) {
412 t.Helper()
413 got, ok := reader.MetricMeta(name)
414 require.Truef(t, ok, "expected metric metadata for %s", name)
415 assert.Equal(t, want, got)
416 }
417
418 func findCreateChartByID(actions []EngineAction, chartID string) *CreateChartAction {
419 for _, action := range actions {
420 create, ok := action.(CreateChartAction)
421 if !ok || create.ChartID != chartID {
422 continue
423 }
424 c := create
425 return &c
426 }
427 return nil
428 }
429
430 func findCreateChartByTitle(actions []EngineAction, title string) *CreateChartAction {
431 for _, action := range actions {
432 create, ok := action.(CreateChartAction)
433 if !ok || create.Meta.Title != title {
434 continue
435 }
436 c := create
437 return &c
438 }
439 return nil
440 }
441
442 func runtimeComponentTemplateYAML() string {
443 return `
444 version: v1
445 groups:
446 - family: Runtime
447 metrics:
448 - component.jobs_total
449 charts:
450 - title: Component jobs
451 context: netdata.go.plugin.component.component_jobs
452 units: jobs/s
453 dimensions:
454 - selector: component.jobs_total
455 name: total
456 `
457 }
458
459 func runtimeObservabilityTemplateYAML() string {
460 return `
461 version: v1
462 groups:
463 - family: Database
464 metrics:
465 - mysql.queries_total
466 charts:
467 - title: Queries
468 context: queries_total
469 units: queries/s
470 dimensions:
471 - selector: mysql.queries_total
472 name: total
473 `
474 }
475
476 func runtimeExpiryTemplateYAML() string {
477 return `
478 version: v1
479 groups:
480 - family: Service
481 metrics:
482 - svc.total
483 - svc.mode_metric
484 charts:
485 - title: Service status
486 context: service_status
487 units: state
488 lifecycle:
489 dimensions:
490 expire_after_cycles: 1
491 dimensions:
492 - selector: svc.total
493 name: total
494 - selector: svc.mode_metric
495 name_from_label: mode
496 `
497 }
498
499 func runtimeDummyTemplateYAML() string {
500 return `
501 version: v1
502 groups:
503 - family: Runtime
504 metrics:
505 - __runtime_dummy_metric
506 charts:
507 - id: runtime_dummy
508 title: Runtime dummy
509 context: netdata.go.plugin.runtime_dummy
510 units: "1"
511 dimensions:
512 - selector: __runtime_dummy_metric
513 name: value
514 `
515 }