| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package chartengine |
| 4 | |
| 5 | import ( |
| 6 | "sync" |
| 7 | "time" |
| 8 | |
| 9 | "github.com/netdata/netdata/go/plugins/pkg/metrix" |
| 10 | ) |
| 11 | |
| 12 | type runtimeMetrics struct { |
| 13 | buildSuccessTotal metrix.StatefulCounter |
| 14 | buildErrorTotal metrix.StatefulCounter |
| 15 | buildSkippedFailedTotal metrix.StatefulCounter |
| 16 | buildDurationSeconds metrix.StatefulSummary |
| 17 | buildPhasePrepareSec metrix.StatefulSummary |
| 18 | buildPhaseValidateSec metrix.StatefulSummary |
| 19 | buildPhaseScanSec metrix.StatefulSummary |
| 20 | buildPhaseRetainSec metrix.StatefulSummary |
| 21 | buildPhaseCapsSec metrix.StatefulSummary |
| 22 | buildPhaseMaterializeSec metrix.StatefulSummary |
| 23 | buildPhaseExpirySec metrix.StatefulSummary |
| 24 | buildPhaseSortSec metrix.StatefulSummary |
| 25 | |
| 26 | buildSeqBrokenTotal metrix.StatefulCounter |
| 27 | buildSeqRecoveredTotal metrix.StatefulCounter |
| 28 | buildSeqViolation metrix.StatefulGauge |
| 29 | |
| 30 | routeCacheHitsTotal metrix.StatefulCounter |
| 31 | routeCacheMissesTotal metrix.StatefulCounter |
| 32 | routeCacheEntries metrix.StatefulGauge |
| 33 | routeCacheRetainedTotal metrix.StatefulCounter |
| 34 | routeCachePrunedTotal metrix.StatefulCounter |
| 35 | routeCacheFullDropsTotal metrix.StatefulCounter |
| 36 | |
| 37 | seriesScannedTotal metrix.StatefulCounter |
| 38 | seriesMatchedTotal metrix.StatefulCounter |
| 39 | seriesUnmatchedTotal metrix.StatefulCounter |
| 40 | seriesAutogenMatchedTotal metrix.StatefulCounter |
| 41 | seriesFilteredBySeq metrix.StatefulCounter |
| 42 | seriesFilteredBySelector metrix.StatefulCounter |
| 43 | |
| 44 | planChartInstances metrix.StatefulGauge |
| 45 | planInferredDimensions metrix.StatefulGauge |
| 46 | |
| 47 | actionCreateChart metrix.StatefulCounter |
| 48 | actionCreateDimension metrix.StatefulCounter |
| 49 | actionUpdateChart metrix.StatefulCounter |
| 50 | actionRemoveDimension metrix.StatefulCounter |
| 51 | actionRemoveChart metrix.StatefulCounter |
| 52 | |
| 53 | lifecycleRemovedChartByCap metrix.StatefulCounter |
| 54 | lifecycleRemovedChartByExpiry metrix.StatefulCounter |
| 55 | lifecycleRemovedDimensionByCap metrix.StatefulCounter |
| 56 | lifecycleRemovedDimensionByExpiry metrix.StatefulCounter |
| 57 | } |
| 58 | |
| 59 | // PlanRuntimeSample is an opaque snapshot of one planner build used by |
| 60 | // chartengine runtime observers and RuntimeAggregator. Its fields are |
| 61 | // intentionally package-private so chartengine remains the owner of runtime |
| 62 | // metric semantics while callers can pass samples between chartengine APIs. |
| 63 | type PlanRuntimeSample struct { |
| 64 | startedAt time.Time |
| 65 | |
| 66 | buildErr bool |
| 67 | skippedFailed bool |
| 68 | buildSuccess bool |
| 69 | |
| 70 | planRouteStats |
| 71 | |
| 72 | planChartInstances int |
| 73 | planInferredDimensions int |
| 74 | |
| 75 | actionCreateChart int |
| 76 | actionCreateDimension int |
| 77 | actionUpdateChart int |
| 78 | actionRemoveDimension int |
| 79 | actionRemoveChart int |
| 80 | |
| 81 | lifecycleRemovedChartByCap int |
| 82 | lifecycleRemovedChartByExpiry int |
| 83 | lifecycleRemovedDimensionByCap int |
| 84 | lifecycleRemovedDimensionByExpiry int |
| 85 | |
| 86 | routeCacheEntries int |
| 87 | routeCacheRetained int |
| 88 | routeCachePruned int |
| 89 | routeCacheFullDrop bool |
| 90 | |
| 91 | buildSeqBroken bool |
| 92 | buildSeqRecovered bool |
| 93 | buildSeqViolation bool |
| 94 | buildSeqObserved bool |
| 95 | |
| 96 | phasePrepareSeconds float64 |
| 97 | phaseValidateSeconds float64 |
| 98 | phaseScanSeconds float64 |
| 99 | phaseRetainSeconds float64 |
| 100 | phaseLifecycleCapsSec float64 |
| 101 | phaseMaterializeSeconds float64 |
| 102 | phaseExpirySeconds float64 |
| 103 | phaseSortSeconds float64 |
| 104 | } |
| 105 | |
| 106 | func newRuntimeMetrics(store metrix.RuntimeStore) *runtimeMetrics { |
| 107 | if store == nil { |
| 108 | return nil |
| 109 | } |
| 110 | meter := store.Write().StatefulMeter("netdata.go.plugin.framework.chartengine") |
| 111 | phaseDuration := meter.Vec("phase").Summary( |
| 112 | "build_phase_duration_seconds", |
| 113 | metrix.WithSummaryQuantiles(0.5, 0.9, 0.99), |
| 114 | metrix.WithDescription("Build phase duration in seconds"), |
| 115 | metrix.WithChartFamily("ChartEngine/Build"), |
| 116 | metrix.WithUnit("seconds"), |
| 117 | ) |
| 118 | actions := meter.Vec("kind").Counter( |
| 119 | "planner_actions_total", |
| 120 | metrix.WithDescription("Planner actions by kind"), |
| 121 | metrix.WithChartFamily("ChartEngine/Actions"), |
| 122 | metrix.WithUnit("actions"), |
| 123 | ) |
| 124 | lifecycleRemoved := meter.Vec("scope", "reason").Counter( |
| 125 | "lifecycle_removed_total", |
| 126 | metrix.WithDescription("Lifecycle removals by scope and reason"), |
| 127 | metrix.WithChartFamily("ChartEngine/Lifecycle"), |
| 128 | metrix.WithUnit("removals"), |
| 129 | ) |
| 130 | seriesFiltered := meter.Vec("reason").Counter( |
| 131 | "series_filtered_total", |
| 132 | metrix.WithDescription("Series filtered before routing by reason"), |
| 133 | metrix.WithChartFamily("ChartEngine/Series"), |
| 134 | metrix.WithUnit("series"), |
| 135 | ) |
| 136 | buildSeqTransitions := meter.Vec("transition").Counter( |
| 137 | "build_seq_transitions_total", |
| 138 | metrix.WithDescription("Build sequence monotonicity transitions"), |
| 139 | metrix.WithChartFamily("ChartEngine/Build"), |
| 140 | metrix.WithUnit("transitions"), |
| 141 | ) |
| 142 | return &runtimeMetrics{ |
| 143 | buildSuccessTotal: metrix.SeededCounter(meter, |
| 144 | "build_success_total", |
| 145 | metrix.WithDescription("Successful BuildPlan calls"), |
| 146 | metrix.WithChartFamily("ChartEngine/Build"), |
| 147 | metrix.WithUnit("builds"), |
| 148 | ), |
| 149 | buildErrorTotal: metrix.SeededCounter(meter, |
| 150 | "build_error_total", |
| 151 | metrix.WithDescription("Failed BuildPlan calls"), |
| 152 | metrix.WithChartFamily("ChartEngine/Build"), |
| 153 | metrix.WithUnit("builds"), |
| 154 | ), |
| 155 | buildSkippedFailedTotal: metrix.SeededCounter(meter, |
| 156 | "build_skipped_failed_collect_total", |
| 157 | metrix.WithDescription("BuildPlan calls skipped due failed collect cycle"), |
| 158 | metrix.WithChartFamily("ChartEngine/Build"), |
| 159 | metrix.WithUnit("builds"), |
| 160 | ), |
| 161 | buildDurationSeconds: meter.Summary( |
| 162 | "build_duration_seconds", |
| 163 | metrix.WithSummaryQuantiles(0.5, 0.9, 0.99), |
| 164 | metrix.WithDescription("BuildPlan duration in seconds"), |
| 165 | metrix.WithChartFamily("ChartEngine/Build"), |
| 166 | metrix.WithUnit("seconds"), |
| 167 | ), |
| 168 | buildPhasePrepareSec: phaseDuration.WithLabelValues("prepare"), |
| 169 | buildPhaseValidateSec: phaseDuration.WithLabelValues("validate_reader"), |
| 170 | buildPhaseScanSec: phaseDuration.WithLabelValues("scan"), |
| 171 | buildPhaseRetainSec: phaseDuration.WithLabelValues("cache_retain"), |
| 172 | buildPhaseCapsSec: phaseDuration.WithLabelValues("lifecycle_caps"), |
| 173 | buildPhaseMaterializeSec: phaseDuration.WithLabelValues("materialize"), |
| 174 | buildPhaseExpirySec: phaseDuration.WithLabelValues("expiry"), |
| 175 | buildPhaseSortSec: phaseDuration.WithLabelValues("sort_inferred"), |
| 176 | |
| 177 | buildSeqBrokenTotal: buildSeqTransitions.WithLabelValues("broken"), |
| 178 | buildSeqRecoveredTotal: buildSeqTransitions.WithLabelValues("recovered"), |
| 179 | buildSeqViolation: metrix.SeededGauge(meter, |
| 180 | "build_seq_violation_active", |
| 181 | metrix.WithDescription("1 when build sequence monotonicity is currently violated"), |
| 182 | metrix.WithChartFamily("ChartEngine/Build"), |
| 183 | metrix.WithUnit("state"), |
| 184 | ), |
| 185 | |
| 186 | routeCacheHitsTotal: metrix.SeededCounter(meter, |
| 187 | "route_cache_hits_total", |
| 188 | metrix.WithDescription("Route cache lookup hits"), |
| 189 | metrix.WithChartFamily("ChartEngine/Route Cache"), |
| 190 | metrix.WithUnit("hits"), |
| 191 | ), |
| 192 | routeCacheMissesTotal: metrix.SeededCounter(meter, |
| 193 | "route_cache_misses_total", |
| 194 | metrix.WithDescription("Route cache lookup misses"), |
| 195 | metrix.WithChartFamily("ChartEngine/Route Cache"), |
| 196 | metrix.WithUnit("misses"), |
| 197 | ), |
| 198 | routeCacheEntries: metrix.SeededGauge(meter, |
| 199 | "route_cache_entries", |
| 200 | metrix.WithDescription("Route cache entries in the latest successful build rollup"), |
| 201 | metrix.WithChartFamily("ChartEngine/Route Cache"), |
| 202 | metrix.WithUnit("entries"), |
| 203 | ), |
| 204 | routeCacheRetainedTotal: metrix.SeededCounter(meter, |
| 205 | "route_cache_retained_total", |
| 206 | metrix.WithDescription("Route cache entries retained after prune"), |
| 207 | metrix.WithChartFamily("ChartEngine/Route Cache"), |
| 208 | metrix.WithUnit("entries"), |
| 209 | ), |
| 210 | routeCachePrunedTotal: metrix.SeededCounter(meter, |
| 211 | "route_cache_pruned_total", |
| 212 | metrix.WithDescription("Route cache entries pruned"), |
| 213 | metrix.WithChartFamily("ChartEngine/Route Cache"), |
| 214 | metrix.WithUnit("entries"), |
| 215 | ), |
| 216 | routeCacheFullDropsTotal: metrix.SeededCounter(meter, |
| 217 | "route_cache_full_drops_total", |
| 218 | metrix.WithDescription("Route cache full-drop prune events"), |
| 219 | metrix.WithChartFamily("ChartEngine/Route Cache"), |
| 220 | metrix.WithUnit("events"), |
| 221 | ), |
| 222 | |
| 223 | seriesScannedTotal: metrix.SeededCounter(meter, |
| 224 | "series_scanned_total", |
| 225 | metrix.WithDescription("Series scanned by planner"), |
| 226 | metrix.WithChartFamily("ChartEngine/Series"), |
| 227 | metrix.WithUnit("series"), |
| 228 | ), |
| 229 | seriesMatchedTotal: metrix.SeededCounter(meter, |
| 230 | "series_matched_total", |
| 231 | metrix.WithDescription("Series matched by template or autogen"), |
| 232 | metrix.WithChartFamily("ChartEngine/Series"), |
| 233 | metrix.WithUnit("series"), |
| 234 | ), |
| 235 | seriesUnmatchedTotal: metrix.SeededCounter(meter, |
| 236 | "series_unmatched_total", |
| 237 | metrix.WithDescription("Series left unmatched after routing"), |
| 238 | metrix.WithChartFamily("ChartEngine/Series"), |
| 239 | metrix.WithUnit("series"), |
| 240 | ), |
| 241 | seriesAutogenMatchedTotal: metrix.SeededCounter(meter, |
| 242 | "series_autogen_matched_total", |
| 243 | metrix.WithDescription("Series matched by autogen fallback"), |
| 244 | metrix.WithChartFamily("ChartEngine/Series"), |
| 245 | metrix.WithUnit("series"), |
| 246 | ), |
| 247 | seriesFilteredBySeq: seriesFiltered.WithLabelValues("by_seq"), |
| 248 | seriesFilteredBySelector: seriesFiltered.WithLabelValues("by_selector"), |
| 249 | |
| 250 | planChartInstances: metrix.SeededGauge(meter, |
| 251 | "plan_chart_instances", |
| 252 | metrix.WithDescription("Chart instances in the latest successful build rollup"), |
| 253 | metrix.WithChartFamily("ChartEngine/Plan"), |
| 254 | metrix.WithUnit("charts"), |
| 255 | ), |
| 256 | planInferredDimensions: metrix.SeededGauge(meter, |
| 257 | "plan_inferred_dimensions", |
| 258 | metrix.WithDescription("Inferred dimensions in the latest successful build rollup"), |
| 259 | metrix.WithChartFamily("ChartEngine/Plan"), |
| 260 | metrix.WithUnit("dimensions"), |
| 261 | ), |
| 262 | |
| 263 | actionCreateChart: actions.WithLabelValues("create_chart"), |
| 264 | actionCreateDimension: actions.WithLabelValues("create_dimension"), |
| 265 | actionUpdateChart: actions.WithLabelValues("update_chart"), |
| 266 | actionRemoveDimension: actions.WithLabelValues("remove_dimension"), |
| 267 | actionRemoveChart: actions.WithLabelValues("remove_chart"), |
| 268 | |
| 269 | lifecycleRemovedChartByCap: lifecycleRemoved.WithLabelValues("chart", "cap"), |
| 270 | lifecycleRemovedChartByExpiry: lifecycleRemoved.WithLabelValues("chart", "expiry"), |
| 271 | lifecycleRemovedDimensionByCap: lifecycleRemoved.WithLabelValues("dimension", "cap"), |
| 272 | lifecycleRemovedDimensionByExpiry: lifecycleRemoved.WithLabelValues("dimension", "expiry"), |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | func (m *runtimeMetrics) observeBuild(sample PlanRuntimeSample) { |
| 277 | if m == nil { |
| 278 | return |
| 279 | } |
| 280 | if sample.startedAt.IsZero() { |
| 281 | sample.startedAt = time.Now() |
| 282 | } |
| 283 | |
| 284 | switch { |
| 285 | case sample.buildSuccess: |
| 286 | m.buildSuccessTotal.Add(1) |
| 287 | case sample.skippedFailed: |
| 288 | m.buildSkippedFailedTotal.Add(1) |
| 289 | case sample.buildErr: |
| 290 | m.buildErrorTotal.Add(1) |
| 291 | } |
| 292 | |
| 293 | m.buildDurationSeconds.Observe(time.Since(sample.startedAt).Seconds()) |
| 294 | observeDurationSeconds(m.buildPhasePrepareSec, sample.phasePrepareSeconds) |
| 295 | observeDurationSeconds(m.buildPhaseValidateSec, sample.phaseValidateSeconds) |
| 296 | observeDurationSeconds(m.buildPhaseScanSec, sample.phaseScanSeconds) |
| 297 | observeDurationSeconds(m.buildPhaseRetainSec, sample.phaseRetainSeconds) |
| 298 | observeDurationSeconds(m.buildPhaseCapsSec, sample.phaseLifecycleCapsSec) |
| 299 | observeDurationSeconds(m.buildPhaseMaterializeSec, sample.phaseMaterializeSeconds) |
| 300 | observeDurationSeconds(m.buildPhaseExpirySec, sample.phaseExpirySeconds) |
| 301 | observeDurationSeconds(m.buildPhaseSortSec, sample.phaseSortSeconds) |
| 302 | |
| 303 | if sample.buildSeqObserved { |
| 304 | if sample.buildSeqBroken { |
| 305 | m.buildSeqBrokenTotal.Add(1) |
| 306 | } |
| 307 | if sample.buildSeqRecovered { |
| 308 | m.buildSeqRecoveredTotal.Add(1) |
| 309 | } |
| 310 | if sample.buildSeqViolation { |
| 311 | m.buildSeqViolation.Set(1) |
| 312 | } else { |
| 313 | m.buildSeqViolation.Set(0) |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | m.routeCacheHitsTotal.Add(float64(sample.routeCacheHits)) |
| 318 | m.routeCacheMissesTotal.Add(float64(sample.routeCacheMisses)) |
| 319 | if sample.routeCacheRetained > 0 { |
| 320 | m.routeCacheRetainedTotal.Add(float64(sample.routeCacheRetained)) |
| 321 | } |
| 322 | if sample.routeCachePruned > 0 { |
| 323 | m.routeCachePrunedTotal.Add(float64(sample.routeCachePruned)) |
| 324 | } |
| 325 | if sample.routeCacheFullDrop { |
| 326 | m.routeCacheFullDropsTotal.Add(1) |
| 327 | } |
| 328 | if sample.buildSuccess { |
| 329 | m.routeCacheEntries.Set(float64(sample.routeCacheEntries)) |
| 330 | } |
| 331 | |
| 332 | m.seriesScannedTotal.Add(float64(sample.seriesScanned)) |
| 333 | m.seriesMatchedTotal.Add(float64(sample.seriesMatched)) |
| 334 | m.seriesUnmatchedTotal.Add(float64(sample.seriesUnmatched)) |
| 335 | m.seriesAutogenMatchedTotal.Add(float64(sample.seriesAutogenMatched)) |
| 336 | if sample.seriesFilteredBySeq > 0 { |
| 337 | m.seriesFilteredBySeq.Add(float64(sample.seriesFilteredBySeq)) |
| 338 | } |
| 339 | if sample.seriesFilteredBySel > 0 { |
| 340 | m.seriesFilteredBySelector.Add(float64(sample.seriesFilteredBySel)) |
| 341 | } |
| 342 | |
| 343 | if sample.actionCreateChart > 0 { |
| 344 | m.actionCreateChart.Add(float64(sample.actionCreateChart)) |
| 345 | } |
| 346 | if sample.actionCreateDimension > 0 { |
| 347 | m.actionCreateDimension.Add(float64(sample.actionCreateDimension)) |
| 348 | } |
| 349 | if sample.actionUpdateChart > 0 { |
| 350 | m.actionUpdateChart.Add(float64(sample.actionUpdateChart)) |
| 351 | } |
| 352 | if sample.actionRemoveDimension > 0 { |
| 353 | m.actionRemoveDimension.Add(float64(sample.actionRemoveDimension)) |
| 354 | } |
| 355 | if sample.actionRemoveChart > 0 { |
| 356 | m.actionRemoveChart.Add(float64(sample.actionRemoveChart)) |
| 357 | } |
| 358 | |
| 359 | if sample.lifecycleRemovedChartByCap > 0 { |
| 360 | m.lifecycleRemovedChartByCap.Add(float64(sample.lifecycleRemovedChartByCap)) |
| 361 | } |
| 362 | if sample.lifecycleRemovedChartByExpiry > 0 { |
| 363 | m.lifecycleRemovedChartByExpiry.Add(float64(sample.lifecycleRemovedChartByExpiry)) |
| 364 | } |
| 365 | if sample.lifecycleRemovedDimensionByCap > 0 { |
| 366 | m.lifecycleRemovedDimensionByCap.Add(float64(sample.lifecycleRemovedDimensionByCap)) |
| 367 | } |
| 368 | if sample.lifecycleRemovedDimensionByExpiry > 0 { |
| 369 | m.lifecycleRemovedDimensionByExpiry.Add(float64(sample.lifecycleRemovedDimensionByExpiry)) |
| 370 | } |
| 371 | |
| 372 | if sample.buildSuccess { |
| 373 | m.planChartInstances.Set(float64(sample.planChartInstances)) |
| 374 | m.planInferredDimensions.Set(float64(sample.planInferredDimensions)) |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | func observeDurationSeconds(metric metrix.StatefulSummary, value float64) { |
| 379 | if value <= 0 { |
| 380 | return |
| 381 | } |
| 382 | metric.Observe(value) |
| 383 | } |
| 384 | |
| 385 | func (e *Engine) RuntimeStore() metrix.RuntimeStore { |
| 386 | if e == nil { |
| 387 | return nil |
| 388 | } |
| 389 | return e.state.runtimeStore |
| 390 | } |
| 391 | |
| 392 | func (e *Engine) observeBuildSample(sample PlanRuntimeSample) { |
| 393 | if e == nil { |
| 394 | return |
| 395 | } |
| 396 | if e.state.cfg.runtimeObserver != nil { |
| 397 | e.state.cfg.runtimeObserver(sample) |
| 398 | } |
| 399 | if e.state.runtimeStats == nil { |
| 400 | return |
| 401 | } |
| 402 | e.state.runtimeStats.observeBuild(sample) |
| 403 | } |
| 404 | |
| 405 | // RuntimeAggregator records runtime samples from multiple engines and emits one |
| 406 | // rolled-up chartengine runtime stream. |
| 407 | type RuntimeAggregator struct { |
| 408 | mu sync.Mutex |
| 409 | metrics *runtimeMetrics |
| 410 | samples []PlanRuntimeSample |
| 411 | } |
| 412 | |
| 413 | // NewRuntimeAggregator creates a runtime sample aggregator backed by store. |
| 414 | func NewRuntimeAggregator(store metrix.RuntimeStore) *RuntimeAggregator { |
| 415 | return &RuntimeAggregator{metrics: newRuntimeMetrics(store)} |
| 416 | } |
| 417 | |
| 418 | // Observe records one engine build sample. |
| 419 | func (a *RuntimeAggregator) Observe(sample PlanRuntimeSample) { |
| 420 | if a == nil { |
| 421 | return |
| 422 | } |
| 423 | a.mu.Lock() |
| 424 | a.samples = append(a.samples, sample) |
| 425 | a.mu.Unlock() |
| 426 | } |
| 427 | |
| 428 | // Reset drops accumulated samples without writing them. |
| 429 | func (a *RuntimeAggregator) Reset() { |
| 430 | if a == nil { |
| 431 | return |
| 432 | } |
| 433 | a.mu.Lock() |
| 434 | a.samples = nil |
| 435 | a.mu.Unlock() |
| 436 | } |
| 437 | |
| 438 | // Flush emits accumulated samples into the aggregate runtime store. |
| 439 | func (a *RuntimeAggregator) Flush() { |
| 440 | if a == nil { |
| 441 | return |
| 442 | } |
| 443 | a.mu.Lock() |
| 444 | samples := a.samples |
| 445 | a.samples = nil |
| 446 | a.mu.Unlock() |
| 447 | if len(samples) == 0 || a.metrics == nil { |
| 448 | return |
| 449 | } |
| 450 | a.metrics.observeBuildRollup(samples) |
| 451 | } |
| 452 | |
| 453 | func (m *runtimeMetrics) observeBuildRollup(samples []PlanRuntimeSample) { |
| 454 | if m == nil || len(samples) == 0 { |
| 455 | return |
| 456 | } |
| 457 | |
| 458 | var buildSuccess, skippedFailed, buildErr int |
| 459 | var routeCacheHits, routeCacheMisses uint64 |
| 460 | var routeCacheRetained, routeCachePruned, routeCacheFullDrops int |
| 461 | var seriesScanned, seriesMatched, seriesUnmatched, seriesAutogenMatched uint64 |
| 462 | var seriesFilteredBySeq, seriesFilteredBySel uint64 |
| 463 | var actionCreateChart, actionCreateDimension, actionUpdateChart, actionRemoveDimension, actionRemoveChart int |
| 464 | var lifecycleRemovedChartByCap, lifecycleRemovedChartByExpiry int |
| 465 | var lifecycleRemovedDimensionByCap, lifecycleRemovedDimensionByExpiry int |
| 466 | var routeCacheEntries, planChartInstances, planInferredDimensions int |
| 467 | var buildSeqBroken, buildSeqRecovered int |
| 468 | var buildSeqViolation, buildSeqObserved bool |
| 469 | |
| 470 | for _, sample := range samples { |
| 471 | if sample.startedAt.IsZero() { |
| 472 | sample.startedAt = time.Now() |
| 473 | } |
| 474 | switch { |
| 475 | case sample.buildSuccess: |
| 476 | buildSuccess++ |
| 477 | case sample.skippedFailed: |
| 478 | skippedFailed++ |
| 479 | case sample.buildErr: |
| 480 | buildErr++ |
| 481 | } |
| 482 | |
| 483 | m.buildDurationSeconds.Observe(time.Since(sample.startedAt).Seconds()) |
| 484 | observeDurationSeconds(m.buildPhasePrepareSec, sample.phasePrepareSeconds) |
| 485 | observeDurationSeconds(m.buildPhaseValidateSec, sample.phaseValidateSeconds) |
| 486 | observeDurationSeconds(m.buildPhaseScanSec, sample.phaseScanSeconds) |
| 487 | observeDurationSeconds(m.buildPhaseRetainSec, sample.phaseRetainSeconds) |
| 488 | observeDurationSeconds(m.buildPhaseCapsSec, sample.phaseLifecycleCapsSec) |
| 489 | observeDurationSeconds(m.buildPhaseMaterializeSec, sample.phaseMaterializeSeconds) |
| 490 | observeDurationSeconds(m.buildPhaseExpirySec, sample.phaseExpirySeconds) |
| 491 | observeDurationSeconds(m.buildPhaseSortSec, sample.phaseSortSeconds) |
| 492 | |
| 493 | if sample.buildSeqObserved { |
| 494 | buildSeqObserved = true |
| 495 | if sample.buildSeqBroken { |
| 496 | buildSeqBroken++ |
| 497 | } |
| 498 | if sample.buildSeqRecovered { |
| 499 | buildSeqRecovered++ |
| 500 | } |
| 501 | if sample.buildSeqViolation { |
| 502 | buildSeqViolation = true |
| 503 | } |
| 504 | } |
| 505 | |
| 506 | routeCacheHits += sample.routeCacheHits |
| 507 | routeCacheMisses += sample.routeCacheMisses |
| 508 | routeCacheRetained += sample.routeCacheRetained |
| 509 | routeCachePruned += sample.routeCachePruned |
| 510 | if sample.routeCacheFullDrop { |
| 511 | routeCacheFullDrops++ |
| 512 | } |
| 513 | if sample.buildSuccess { |
| 514 | routeCacheEntries += sample.routeCacheEntries |
| 515 | planChartInstances += sample.planChartInstances |
| 516 | planInferredDimensions += sample.planInferredDimensions |
| 517 | } |
| 518 | |
| 519 | seriesScanned += sample.seriesScanned |
| 520 | seriesMatched += sample.seriesMatched |
| 521 | seriesUnmatched += sample.seriesUnmatched |
| 522 | seriesAutogenMatched += sample.seriesAutogenMatched |
| 523 | seriesFilteredBySeq += sample.seriesFilteredBySeq |
| 524 | seriesFilteredBySel += sample.seriesFilteredBySel |
| 525 | |
| 526 | actionCreateChart += sample.actionCreateChart |
| 527 | actionCreateDimension += sample.actionCreateDimension |
| 528 | actionUpdateChart += sample.actionUpdateChart |
| 529 | actionRemoveDimension += sample.actionRemoveDimension |
| 530 | actionRemoveChart += sample.actionRemoveChart |
| 531 | |
| 532 | lifecycleRemovedChartByCap += sample.lifecycleRemovedChartByCap |
| 533 | lifecycleRemovedChartByExpiry += sample.lifecycleRemovedChartByExpiry |
| 534 | lifecycleRemovedDimensionByCap += sample.lifecycleRemovedDimensionByCap |
| 535 | lifecycleRemovedDimensionByExpiry += sample.lifecycleRemovedDimensionByExpiry |
| 536 | } |
| 537 | |
| 538 | addIfPositive(m.buildSuccessTotal, float64(buildSuccess)) |
| 539 | addIfPositive(m.buildSkippedFailedTotal, float64(skippedFailed)) |
| 540 | addIfPositive(m.buildErrorTotal, float64(buildErr)) |
| 541 | addIfPositive(m.buildSeqBrokenTotal, float64(buildSeqBroken)) |
| 542 | addIfPositive(m.buildSeqRecoveredTotal, float64(buildSeqRecovered)) |
| 543 | if buildSeqObserved { |
| 544 | if buildSeqViolation { |
| 545 | m.buildSeqViolation.Set(1) |
| 546 | } else { |
| 547 | m.buildSeqViolation.Set(0) |
| 548 | } |
| 549 | } else { |
| 550 | m.buildSeqViolation.Set(0) |
| 551 | } |
| 552 | |
| 553 | addIfPositive(m.routeCacheHitsTotal, float64(routeCacheHits)) |
| 554 | addIfPositive(m.routeCacheMissesTotal, float64(routeCacheMisses)) |
| 555 | addIfPositive(m.routeCacheRetainedTotal, float64(routeCacheRetained)) |
| 556 | addIfPositive(m.routeCachePrunedTotal, float64(routeCachePruned)) |
| 557 | addIfPositive(m.routeCacheFullDropsTotal, float64(routeCacheFullDrops)) |
| 558 | if buildSuccess > 0 { |
| 559 | m.routeCacheEntries.Set(float64(routeCacheEntries)) |
| 560 | } |
| 561 | |
| 562 | addIfPositive(m.seriesScannedTotal, float64(seriesScanned)) |
| 563 | addIfPositive(m.seriesMatchedTotal, float64(seriesMatched)) |
| 564 | addIfPositive(m.seriesUnmatchedTotal, float64(seriesUnmatched)) |
| 565 | addIfPositive(m.seriesAutogenMatchedTotal, float64(seriesAutogenMatched)) |
| 566 | addIfPositive(m.seriesFilteredBySeq, float64(seriesFilteredBySeq)) |
| 567 | addIfPositive(m.seriesFilteredBySelector, float64(seriesFilteredBySel)) |
| 568 | |
| 569 | addIfPositive(m.actionCreateChart, float64(actionCreateChart)) |
| 570 | addIfPositive(m.actionCreateDimension, float64(actionCreateDimension)) |
| 571 | addIfPositive(m.actionUpdateChart, float64(actionUpdateChart)) |
| 572 | addIfPositive(m.actionRemoveDimension, float64(actionRemoveDimension)) |
| 573 | addIfPositive(m.actionRemoveChart, float64(actionRemoveChart)) |
| 574 | |
| 575 | addIfPositive(m.lifecycleRemovedChartByCap, float64(lifecycleRemovedChartByCap)) |
| 576 | addIfPositive(m.lifecycleRemovedChartByExpiry, float64(lifecycleRemovedChartByExpiry)) |
| 577 | addIfPositive(m.lifecycleRemovedDimensionByCap, float64(lifecycleRemovedDimensionByCap)) |
| 578 | addIfPositive(m.lifecycleRemovedDimensionByExpiry, float64(lifecycleRemovedDimensionByExpiry)) |
| 579 | |
| 580 | if buildSuccess > 0 { |
| 581 | m.planChartInstances.Set(float64(planChartInstances)) |
| 582 | m.planInferredDimensions.Set(float64(planInferredDimensions)) |
| 583 | } |
| 584 | } |
| 585 | |
| 586 | func addIfPositive(metric metrix.StatefulCounter, value float64) { |
| 587 | if value > 0 { |
| 588 | metric.Add(value) |
| 589 | } |
| 590 | } |
| 591 | |
| 592 | func actionKindCounts(actions []EngineAction) PlanRuntimeSample { |
| 593 | out := PlanRuntimeSample{} |
| 594 | for _, action := range actions { |
| 595 | switch action.Kind() { |
| 596 | case ActionCreateChart: |
| 597 | out.actionCreateChart++ |
| 598 | case ActionCreateDimension: |
| 599 | out.actionCreateDimension++ |
| 600 | case ActionUpdateChart: |
| 601 | out.actionUpdateChart++ |
| 602 | case ActionRemoveDimension: |
| 603 | out.actionRemoveDimension++ |
| 604 | case ActionRemoveChart: |
| 605 | out.actionRemoveChart++ |
| 606 | } |
| 607 | } |
| 608 | return out |
| 609 | } |