@cryptotaxi247 / netdata-1 / commits / bbfdae6a6

feat(go/scripts.d): add Nagios V2 alerts and alertable state (#22008)

Ilya Mashchenko committed Mar 23, 2026 at 01:39 UTC bbfdae6a67c3af197ff2f8f3d14b17fa5709d0ac
24 files changed +873 -265
src/go/pkg/metrix/collector_store.go
+8 -4
@@ -542,10 +542,11 @@ func (c *storeCore) registerInstrument(name string, kind metricKind, mode metric
542 }
543
544 metricMeta := MetricMeta{
545 - Description: strings.TrimSpace(cfg.description),
546 - ChartFamily: strings.TrimSpace(cfg.chartFamily),
547 - Unit: strings.TrimSpace(cfg.unit),
548 - Float: cfg.float,
545 + Description: strings.TrimSpace(cfg.description),
546 + ChartFamily: strings.TrimSpace(cfg.chartFamily),
547 + ChartPriority: cfg.chartPriority,
548 + Unit: strings.TrimSpace(cfg.unit),
549 + Float: cfg.float,
550 }
551
552 var histogram *histogramSchema
@@ -620,6 +621,9 @@ func (c *storeCore) registerInstrument(name string, kind metricKind, mode metric
621 if cfg.chartFamilySet && d.meta.ChartFamily != metricMeta.ChartFamily {
622 return nil, fmt.Errorf("metrix: metric chart family mismatch for %s", name)
623 }
624 + if cfg.chartPrioritySet && d.meta.ChartPriority != metricMeta.ChartPriority {
625 + return nil, fmt.Errorf("metrix: metric chart priority mismatch for %s", name)
626 + }
627 if cfg.unitSet && d.meta.Unit != metricMeta.Unit {
628 return nil, fmt.Errorf("metrix: metric unit mismatch for %s", name)
629 }
src/go/pkg/metrix/metric_meta_store_test.go
+17
@@ -21,6 +21,7 @@ func TestMetricMetaScenarios(t *testing.T) {
21 "workers_busy",
22 WithDescription("Busy workers"),
23 WithChartFamily("Workers"),
24 + WithChartPriority(70000),
25 WithUnit("workers"),
26 WithFloat(true),
27 )
@@ -33,6 +34,7 @@ func TestMetricMetaScenarios(t *testing.T) {
34 require.True(t, ok)
35 assert.Equal(t, "Busy workers", meta.Description)
36 assert.Equal(t, "Workers", meta.ChartFamily)
37 + assert.Equal(t, 70000, meta.ChartPriority)
38 assert.Equal(t, "workers", meta.Unit)
39 assert.True(t, meta.Float)
40 },
@@ -78,6 +80,7 @@ func TestMetricMetaScenarios(t *testing.T) {
80 require.True(t, ok, "expected flattened histogram bucket metric metadata")
81 assert.Equal(t, "Latency", meta.Description)
82 assert.Equal(t, "Service", meta.ChartFamily)
83 + assert.Zero(t, meta.ChartPriority)
84 assert.Equal(t, "ms", meta.Unit)
85 assert.True(t, meta.Float)
86
@@ -85,6 +88,7 @@ func TestMetricMetaScenarios(t *testing.T) {
88 require.True(t, ok, "expected flattened histogram count metric metadata")
89 assert.Equal(t, "Latency", meta.Description)
90 assert.Equal(t, "Service", meta.ChartFamily)
91 + assert.Zero(t, meta.ChartPriority)
92 assert.Equal(t, "ms", meta.Unit)
93 assert.True(t, meta.Float)
94
@@ -92,6 +96,7 @@ func TestMetricMetaScenarios(t *testing.T) {
96 require.True(t, ok, "expected flattened histogram sum metric metadata")
97 assert.Equal(t, "Latency", meta.Description)
98 assert.Equal(t, "Service", meta.ChartFamily)
99 + assert.Zero(t, meta.ChartPriority)
100 assert.Equal(t, "ms", meta.Unit)
101 assert.True(t, meta.Float)
102 },
@@ -106,6 +111,16 @@ func TestMetricMetaScenarios(t *testing.T) {
111 })
112 },
113 },
114 + "chart priority redeclaration conflict panics": {
115 + run: func(t *testing.T) {
116 + s := NewCollectorStore()
117 + w := s.Write().SnapshotMeter("apache")
118 + _ = w.Gauge("workers_busy", WithChartPriority(70000))
119 + expectPanic(t, func() {
120 + _ = w.Gauge("workers_busy", WithChartPriority(70001))
121 + })
122 + },
123 + },
124 "float metadata redeclaration conflict panics": {
125 run: func(t *testing.T) {
126 s := NewCollectorStore()
@@ -124,6 +139,7 @@ func TestMetricMetaScenarios(t *testing.T) {
139 "workers_busy",
140 WithDescription("Busy workers"),
141 WithChartFamily("Workers"),
142 + WithChartPriority(70000),
143 WithUnit("workers"),
144 WithFloat(true),
145 )
@@ -138,6 +154,7 @@ func TestMetricMetaScenarios(t *testing.T) {
154 require.True(t, ok)
155 assert.Equal(t, "Busy workers", meta.Description)
156 assert.Equal(t, "Workers", meta.ChartFamily)
157 + assert.Equal(t, 70000, meta.ChartPriority)
158 assert.Equal(t, "workers", meta.Unit)
159 assert.True(t, meta.Float)
160 },
src/go/pkg/metrix/options.go
+20 -8
@@ -26,14 +26,16 @@ type instrumentConfig struct {
26 measureSetFields []MeasureFieldSpec
27 measureSetSemantics *MeasureSetSemantics
28
29 - descriptionSet bool
30 - description string
31 - chartFamilySet bool
32 - chartFamily string
33 - unitSet bool
34 - unit string
35 - floatSet bool
36 - float bool
29 + descriptionSet bool
30 + description string
31 + chartFamilySet bool
32 + chartFamily string
33 + chartPrioritySet bool
34 + chartPriority int
35 + unitSet bool
36 + unit string
37 + floatSet bool
38 + float bool
39 }
40
41 func WithFreshness(policy FreshnessPolicy) InstrumentOption {
@@ -113,6 +115,16 @@ func WithChartFamily(chartFamily string) InstrumentOption {
115 })
116 }
117
118 +// WithChartPriority sets optional metric-family chart priority metadata.
119 +// It is currently consumed only by chartengine autogen.
120 +// TODO: Revisit whether chart-template charts should also honor metrix priority hints.
121 +func WithChartPriority(chartPriority int) InstrumentOption {
122 + return optionFunc(func(cfg *instrumentConfig) {
123 + cfg.chartPrioritySet = true
124 + cfg.chartPriority = chartPriority
125 + })
126 +}
127 +
128 // WithUnit sets optional metric-family unit metadata.
129 func WithUnit(unit string) InstrumentOption {
130 return optionFunc(func(cfg *instrumentConfig) {
src/go/pkg/metrix/types.go
+5 -2
@@ -53,8 +53,11 @@ type SeriesMeta struct {
53 type MetricMeta struct {
54 Description string
55 ChartFamily string
56 - Unit string
57 - Float bool
56 + // ChartPriority is currently consumed only by chartengine autogen.
57 + // TODO: Revisit whether chart-template charts should also honor metrix priority hints.
58 + ChartPriority int
59 + Unit string
60 + Float bool
61 }
62
63 // MetricKind identifies the logical metric family type.
src/go/plugin/framework/chartengine/autogen.go
+6 -1
@@ -25,6 +25,7 @@ type autogenRoute struct {
25 units string
26 chartType program.ChartType
27 family string
28 + priority int
29 contextName string
30 staticDimension bool
31 float bool
@@ -88,6 +89,7 @@ func (e *Engine) resolveAutogenRoute(
89 if metricMeta, ok := autogenMetricMeta(reader, metricName, meta); ok {
90 route = applyAutogenMetricMeta(route, metricMeta, meta)
91 }
92 + route.priority = effectiveChartPriority(route.priority)
93 title := route.title
94 if title == "" {
95 title = getAutogenChartTitle(route.chartName)
@@ -114,7 +116,7 @@ func (e *Engine) resolveAutogenRoute(
116 Units: route.units,
117 Algorithm: route.algorithm,
118 Type: route.chartType,
117 - Priority: 0,
119 + Priority: route.priority,
120 },
121 Lifecycle: autogenLifecyclePolicy(policy),
122 },
@@ -190,6 +192,9 @@ func applyAutogenMetricMeta(route autogenRoute, meta metrix.MetricMeta, seriesMe
192 if family := strings.TrimSpace(meta.ChartFamily); family != "" {
193 route.family = family
194 }
195 + if meta.ChartPriority > 0 {
196 + route.priority = meta.ChartPriority
197 + }
198 if unit := strings.TrimSpace(meta.Unit); unit != "" && allowAutogenUnitOverride(seriesMeta) {
199 route.units = normalizeAutogenUnitByAlgorithm(unit, route.algorithm)
200 route.chartType = chartTypeFromUnits(route.units)
src/go/plugin/framework/chartengine/compiler.go
+8 -1
@@ -159,7 +159,7 @@ func (c *compiler) compileChart(chart charttpl.Chart, scope compileScope, templa
159 Units: strings.TrimSpace(chart.Units),
160 Algorithm: algorithm,
161 Type: chartType,
162 - Priority: chart.Priority,
162 + Priority: effectiveChartPriority(chart.Priority),
163 },
164 Identity: identity,
165 Labels: program.LabelPolicy{
@@ -536,3 +536,10 @@ func (v selectorLabelView) CloneMap() map[string]string {
536 })
537 return out
538 }
539 +
540 +func effectiveChartPriority(priority int) int {
541 + if priority > 0 {
542 + return priority
543 + }
544 + return Priority
545 +}
src/go/plugin/framework/chartengine/compiler_test.go
+55
@@ -112,6 +112,61 @@ func TestCompileScenarios(t *testing.T) {
112 assert.Equal(t, 0, charts[0].Lifecycle.Dimensions.ExpireAfterCycles)
113 },
114 },
115 + "defaults chart priority when omitted": {
116 + spec: charttpl.Spec{
117 + Version: charttpl.VersionV1,
118 + Groups: []charttpl.Group{
119 + {
120 + Family: "Service",
121 + Metrics: []string{"svc_requests_total"},
122 + Charts: []charttpl.Chart{
123 + {
124 + Title: "Requests",
125 + Context: "requests",
126 + Units: "requests/s",
127 + Dimensions: []charttpl.Dimension{
128 + {Selector: "svc_requests_total", Name: "total"},
129 + },
130 + },
131 + },
132 + },
133 + },
134 + },
135 + assert: func(t *testing.T, p *program.Program) {
136 + t.Helper()
137 + charts := p.Charts()
138 + require.Len(t, charts, 1)
139 + assert.Equal(t, Priority, charts[0].Meta.Priority)
140 + },
141 + },
142 + "preserves explicit chart priority": {
143 + spec: charttpl.Spec{
144 + Version: charttpl.VersionV1,
145 + Groups: []charttpl.Group{
146 + {
147 + Family: "Service",
148 + Metrics: []string{"svc_requests_total"},
149 + Charts: []charttpl.Chart{
150 + {
151 + Title: "Requests",
152 + Context: "requests",
153 + Units: "requests/s",
154 + Priority: Priority + 321,
155 + Dimensions: []charttpl.Dimension{
156 + {Selector: "svc_requests_total", Name: "total"},
157 + },
158 + },
159 + },
160 + },
161 + },
162 + },
163 + assert: func(t *testing.T, p *program.Program) {
164 + t.Helper()
165 + charts := p.Charts()
166 + require.Len(t, charts, 1)
167 + assert.Equal(t, Priority+321, charts[0].Meta.Priority)
168 + },
169 + },
170 "keeps default chart expiry when lifecycle is present without expire_after_cycles": {
171 spec: charttpl.Spec{
172 Version: charttpl.VersionV1,
src/go/plugin/framework/chartengine/planner_test.go
+44
@@ -217,6 +217,7 @@ func TestBuildPlanLegacySingleScenarioCases(t *testing.T) {
217 "BuildPlanAutogenOptionKeepsTemplateSelector": {run: runTestBuildPlanAutogenOptionKeepsTemplateSelector},
218 "BuildPlanAutogenCreatesChartForUnmatchedScalar": {run: runTestBuildPlanAutogenCreatesChartForUnmatchedScalar},
219 "BuildPlanAutogenUsesMetricMetadataForScalar": {run: runTestBuildPlanAutogenUsesMetricMetadataForScalar},
220 + "BuildPlanAutogenUsesMetricPriorityMetadataForScalar": {run: runTestBuildPlanAutogenUsesMetricPriorityMetadataForScalar},
221 "BuildPlanAutogenUsesMetricMetadataForHistogram": {run: runTestBuildPlanAutogenUsesMetricMetadataForHistogram},
222 "BuildPlanAutogenUsesMetricFloatMetadataForScalar": {run: runTestBuildPlanAutogenUsesMetricFloatMetadataForScalar},
223 "BuildPlanAutogenUsesMetricMetadataForSummaryWithoutQuantiles": {run: runTestBuildPlanAutogenUsesMetricMetadataForSummaryWithoutQuantiles},
@@ -1039,6 +1040,49 @@ groups:
1040 assert.Equal(t, "HTTP traffic", create.Meta.Title)
1041 assert.Equal(t, "Traffic", create.Meta.Family)
1042 assert.Equal(t, "bytes/s", create.Meta.Units)
1043 + assert.Equal(t, Priority, create.Meta.Priority)
1044 +}
1045 +
1046 +func runTestBuildPlanAutogenUsesMetricPriorityMetadataForScalar(t *testing.T) {
1047 + e, err := New(WithEnginePolicy(EnginePolicy{Autogen: &AutogenPolicy{Enabled: true}}))
1048 + require.NoError(t, err)
1049 +
1050 + yaml := `
1051 +version: v1
1052 +groups:
1053 + - family: Service
1054 + metrics:
1055 + - svc.requests_total
1056 + charts:
1057 + - title: Requests
1058 + context: requests
1059 + units: requests/s
1060 + dimensions:
1061 + - selector: svc.requests_total
1062 + name: total
1063 +`
1064 + require.NoError(t, e.LoadYAML([]byte(yaml), 1))
1065 +
1066 + store := metrix.NewCollectorStore()
1067 + cc := mustCycleController(t, store)
1068 + unmatched := store.Write().SnapshotMeter("svc").Counter(
1069 + "bytes_total",
1070 + metrix.WithDescription("HTTP traffic"),
1071 + metrix.WithChartFamily("Traffic"),
1072 + metrix.WithChartPriority(Priority+321),
1073 + metrix.WithUnit("bytes"),
1074 + )
1075 +
1076 + cc.BeginCycle()
1077 + unmatched.ObserveTotal(10)
1078 + cc.CommitCycleSuccess()
1079 +
1080 + plan, err := buildPlan(e, store.Read(metrix.ReadFlatten()))
1081 + require.NoError(t, err)
1082 +
1083 + create := findCreateChartAction(plan)
1084 + require.NotNil(t, create)
1085 + assert.Equal(t, Priority+321, create.Meta.Priority)
1086 }
1087
1088 func runTestBuildPlanAutogenUsesMetricMetadataForHistogram(t *testing.T) {
src/go/plugin/framework/chartengine/public_types.go
+6
@@ -14,6 +14,12 @@ type (
14 ChartType = program.ChartType
15 )
16
17 +const (
18 + // Priority is the default chart priority used by chartengine when templates
19 + // and autogen routes do not specify one explicitly.
20 + Priority = 70000
21 +)
22 +
23 const (
24 AlgorithmAbsolute = program.AlgorithmAbsolute
25 AlgorithmIncremental = program.AlgorithmIncremental
src/go/plugin/framework/collectorapi/registry.go
+2 -1
@@ -6,12 +6,13 @@ import (
6 "fmt"
7
8 "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/netdata/netdata/go/plugins/plugin/framework/chartengine"
10 )
11
12 const (
13 UpdateEvery = 1
14 AutoDetectionRetry = 0
14 - Priority = 70000
15 + Priority = chartengine.Priority
16 )
17
18 // Defaults is a set of module default parameters.
src/go/plugin/framework/jobruntime/job_v2_test.go
+11 -10
@@ -6,6 +6,7 @@ import (
6 "bytes"
7 "context"
8 "errors"
9 + "fmt"
10 "testing"
11 "time"
12
@@ -218,16 +219,16 @@ func TestJobV2Scenarios(t *testing.T) {
219 job.runOnce()
220
221 wire := out.String()
221 - assert.Contains(t, wire, `HOST ''
222 + assert.Contains(t, wire, fmt.Sprintf(`HOST ''
223
223 -CHART 'module_job.workers_busy' '' 'Workers Busy' 'workers' 'Workers' 'workers_busy' 'line' '0' '1' '' 'plugin' 'module'
224 +CHART 'module_job.workers_busy' '' 'Workers Busy' 'workers' 'Workers' 'workers_busy' 'line' '%d' '1' '' 'plugin' 'module'
225 CLABEL 'instance' 'localhost' '2'
226 CLABEL '_collect_job' 'job' '1'
227 CLABEL_COMMIT
228 DIMENSION 'busy' 'busy' 'absolute' '1' '1' ''
229 BEGIN 'module_job.workers_busy'
230 SET 'busy' = 7
230 -END`)
231 +END`, chartengine.Priority))
232 assert.False(t, job.Panicked())
233 },
234 },
@@ -311,14 +312,14 @@ END`)
312 job.runOnce()
313
314 wire := out.String()
314 - assert.Contains(t, wire, `CHART 'module_job.win_nic_traffic_eth0' '' 'NIC traffic' 'bytes/s' 'Net' 'nic_traffic' 'line' '0' '1' '' 'plugin' 'module'
315 + assert.Contains(t, wire, fmt.Sprintf(`CHART 'module_job.win_nic_traffic_eth0' '' 'NIC traffic' 'bytes/s' 'Net' 'nic_traffic' 'line' '%d' '1' '' 'plugin' 'module'
316 CLABEL 'instance' 'localhost' '2'
317 CLABEL 'nic' 'eth0' '1'
318 CLABEL '_collect_job' 'job' '1'
319 CLABEL_COMMIT
320 DIMENSION 'received' 'received' 'incremental' '1' '1' ''
321 DIMENSION 'sent' 'sent' 'incremental' '1' '1' ''
321 -CHART 'module_job.win_nic_traffic_eth1' '' 'NIC traffic' 'bytes/s' 'Net' 'nic_traffic' 'line' '0' '1' '' 'plugin' 'module'
322 +CHART 'module_job.win_nic_traffic_eth1' '' 'NIC traffic' 'bytes/s' 'Net' 'nic_traffic' 'line' '%d' '1' '' 'plugin' 'module'
323 CLABEL 'instance' 'localhost' '2'
324 CLABEL 'nic' 'eth1' '1'
325 CLABEL '_collect_job' 'job' '1'
@@ -333,7 +334,7 @@ END
334 BEGIN 'module_job.win_nic_traffic_eth1'
335 SET 'received' = 50
336 SET 'sent' = 40
336 -END`)
337 +END`, chartengine.Priority, chartengine.Priority))
338 },
339 },
340 "runtime component registers on successful autodetection": {
@@ -1013,9 +1014,9 @@ func TestJobV2CleanupUsesLastSuccessfulHostAfterFailedHostSwitch(t *testing.T) {
1014 job.Cleanup()
1015
1016 wire := out.String()
1016 - assert.Contains(t, wire, `HOST 'node-guid-a'
1017 + assert.Contains(t, wire, fmt.Sprintf(`HOST 'node-guid-a'
1018
1018 -CHART 'module_job.workers_busy' '' 'Workers Busy' 'workers' 'Workers' 'workers_busy' 'line' '0' '1' 'obsolete' 'plugin' 'module'`)
1019 +CHART 'module_job.workers_busy' '' 'Workers Busy' 'workers' 'Workers' 'workers_busy' 'line' '%d' '1' 'obsolete' 'plugin' 'module'`, chartengine.Priority))
1020 assert.NotContains(t, wire, "HOST 'node-guid-b'")
1021 assert.Empty(t, job.hostState.cleanupCharts)
1022 assert.False(t, job.hostState.cleanupOwner.isSet())
@@ -1104,9 +1105,9 @@ func TestJobV2CleanupDoesNotSuppressGlobalCleanupForDifferentStaleVnode(t *testi
1105 job.Cleanup()
1106
1107 wire := out.String()
1107 - assert.Contains(t, wire, `HOST ''
1108 + assert.Contains(t, wire, fmt.Sprintf(`HOST ''
1109
1109 -CHART 'module_job.workers_busy' '' 'Workers Busy' 'workers' 'Workers' 'workers_busy' 'line' '0' '1' 'obsolete' 'plugin' 'module'`)
1110 +CHART 'module_job.workers_busy' '' 'Workers Busy' 'workers' 'Workers' 'workers_busy' 'line' '%d' '1' 'obsolete' 'plugin' 'module'`, chartengine.Priority))
1111 assert.NotContains(t, wire, "HOST 'node-guid-b'")
1112 }
1113
src/go/plugin/scripts.d/README.md
+23 -8
@@ -113,19 +113,24 @@ Defaults:
113
114 Static template charts:
115
116 -- `nagios.job.state`
116 +- `nagios.job.execution_state`
117 +- `nagios.job.perfdata_threshold_state`
118 - `nagios.job.execution_duration`
118 -- `nagios.job.execution_cpu_total`
119 -- `nagios.job.execution_max_rss`
119 +- `nagios.job.execution_cpu`
120 +- `nagios.job.execution_memory`
121
121 -Perfdata is routed plugin-side and materialized via autogen (bounded lifecycle):
122 +Perfdata is routed plugin-side and materialized via autogen:
123
124 - Unit classes: `time`, `bytes`, `bits`, `percent`, `counter`, `generic`
125 - Metric identity: sanitized perfdata key (from Nagios perfdata label)
126 - Unit-class changes create a new metric identity
127 - Collision policy: deterministic keep-first, drop conflicting label
128 +- Per-job metric count is capped by the collector budget before emission
129 - Each perfdata metric creates one value chart.
128 -- Non-counter perfdata also creates one derived threshold-state chart with:
130 +- Non-counter perfdata also creates:
131 + - one plugin-scoped derived threshold-state chart for visualization
132 + - one static `nagios.job.perfdata.threshold_state` duplicate for alerting, labeled by `perfdata_value=<class>_<metricKey>`
133 +- Threshold-state values are:
134 - `no_threshold`
135 - `ok`
136 - `warning`
@@ -135,9 +140,19 @@ Perfdata is routed plugin-side and materialized via autogen (bounded lifecycle):
140
141 ## Alerts
142
138 -- This preview collector does not currently ship built-in Netdata health alerts.
139 -- Use `nagios.job.state` and the derived non-counter perfdata threshold-state
140 - charts as the inputs for your own alert rules.
143 +- Built-in Netdata health alerts are shipped for:
144 + - `nagios.job.execution_state`
145 + - `nagios.job.perfdata_threshold_state`
146 +- `nagios.job.execution_state` is a bitset chart. It always exposes the current
147 + primary state and also exposes `retry=1` while a non-OK result is still
148 + retrying.
149 +- `nagios.job.perfdata_threshold_state` is also a bitset chart. It exposes the
150 + current non-counter perfdata threshold state and also exposes `retry=1` while
151 + that threshold result comes from a retrying soft run.
152 +- Stock alerts cover only the `warning` and `critical` states and suppress
153 + retrying soft states on both built-in alert contexts.
154 +- If you want alerts for `unknown`, `timeout`, `paused`, or custom perfdata
155 + alerting rules, use these contexts as the base for your own rules.
156
157 ## Logging
158
src/go/plugin/scripts.d/collector/nagios/charts.yaml
+64 -52
@@ -3,56 +3,68 @@ context_namespace: nagios
3 engine:
4 autogen:
5 enabled: true
6 - max_type_id_len: 200
7 - expire_after_success_cycles: 3
6 + expire_after_success_cycles: 50
7 groups:
9 - - family: Job/Status
10 - metrics:
11 - - nagios.job.state
12 - charts:
13 - - id: job_state
14 - title: Job State
15 - context: job_state
16 - units: state
17 - instances:
18 - by_labels: [nagios_job]
19 - dimensions:
20 - - selector: nagios.job.state
21 -
22 - - family: Job/Execution
23 - metrics:
24 - - nagios.job.execution_duration
25 - - nagios.job.execution_cpu_total
26 - - nagios.job.execution_max_rss
27 - charts:
28 - - id: job_execution_duration
29 - title: Execution Duration
30 - context: job_execution_duration
31 - units: seconds
32 - instances:
33 - by_labels: [nagios_job]
34 - dimensions:
35 - - selector: nagios.job.execution_duration
36 - name: duration
37 - options:
38 - float: true
39 - - id: job_execution_cpu
40 - title: Execution CPU Time
41 - context: job_execution_cpu
42 - units: seconds
43 - instances:
44 - by_labels: [nagios_job]
45 - dimensions:
46 - - selector: nagios.job.execution_cpu_total
47 - name: total
48 - options:
49 - float: true
50 - - id: job_execution_memory
51 - title: Execution Peak RSS
52 - context: job_execution_memory
53 - units: bytes
54 - instances:
55 - by_labels: [nagios_job]
56 - dimensions:
57 - - selector: nagios.job.execution_max_rss
58 - name: rss
8 + - family: Job
9 + context_namespace: job
10 + groups:
11 + - family: Execution
12 + metrics:
13 + - nagios.job.execution_state
14 + - nagios.job.perfdata.threshold_state
15 + - nagios.job.execution_duration
16 + - nagios.job.execution_cpu_total
17 + - nagios.job.execution_max_rss
18 + charts:
19 + - id: job_execution_state
20 + title: Job Execution State
21 + context: execution_state
22 + units: state
23 + priority: 90000
24 + instances:
25 + by_labels: [nagios_job]
26 + dimensions:
27 + - selector: nagios.job.execution_state
28 + - id: job_perfdata_threshold_state
29 + title: Job Perfdata Threshold State
30 + context: perfdata_threshold_state
31 + units: state
32 + priority: 90001
33 + instances:
34 + by_labels: [nagios_job, perfdata_value]
35 + dimensions:
36 + - selector: nagios.job.perfdata.threshold_state
37 + - id: job_execution_duration
38 + title: Execution Duration
39 + context: execution_duration
40 + units: seconds
41 + priority: 90002
42 + instances:
43 + by_labels: [nagios_job]
44 + dimensions:
45 + - selector: nagios.job.execution_duration
46 + name: duration
47 + options:
48 + float: true
49 + - id: job_execution_cpu
50 + title: Execution CPU Time
51 + context: execution_cpu
52 + units: seconds
53 + priority: 90003
54 + instances:
55 + by_labels: [nagios_job]
56 + dimensions:
57 + - selector: nagios.job.execution_cpu_total
58 + name: total
59 + options:
60 + float: true
61 + - id: job_execution_memory
62 + title: Execution Peak RSS
63 + context: execution_memory
64 + units: bytes
65 + priority: 90004
66 + instances:
67 + by_labels: [nagios_job]
68 + dimensions:
69 + - selector: nagios.job.execution_max_rss
70 + name: rss
src/go/plugin/scripts.d/collector/nagios/collect.go
+31 -23
@@ -5,10 +5,10 @@ package nagios
5 import (
6 "context"
7 "runtime"
8 - "strings"
8 "time"
9
10 "github.com/netdata/netdata/go/plugins/pkg/metrix"
11 + "github.com/netdata/netdata/go/plugins/plugin/framework/chartengine"
12 )
13
14 func (c *Collector) collect(ctx context.Context) error {
@@ -78,13 +78,24 @@ func (c *Collector) emitMetrics(execMetrics executionMetrics) {
78
79 jobLbl := sm.LabelSet(metrix.Label{Key: "nagios_job", Value: jobName})
80 jobMeter := sm.WithLabelSet(jobLbl)
81 + jobStatePoint := projectJobExecutionState(c.state.currentJobState(), c.state.isRetrying())
82
83 jobMeter.StateSet(
83 - "job.state",
84 - metrix.WithStateSetMode(metrix.ModeEnum),
85 - metrix.WithStateSetStates("ok", "warning", "critical", "unknown", "timeout", "paused"),
84 + "job.execution_state",
85 + metrix.WithStateSetMode(metrix.ModeBitSet),
86 + metrix.WithStateSetStates(jobExecutionStateNames...),
87 metrix.WithUnit("state"),
87 - ).Enable(normalizeJobStateForMetric(c.state.currentJobState()))
88 + ).ObserveStateSet(jobStatePoint)
89 +
90 + scriptName := perfSourceFromPlugin(c.job.config.Plugin)
91 + jobMeter.StateSet(
92 + "perfdata."+scriptName+".job.execution_state",
93 + metrix.WithStateSetMode(metrix.ModeBitSet),
94 + metrix.WithStateSetStates(jobExecutionStateNames...),
95 + metrix.WithChartFamily(perfdataFamily(scriptName)),
96 + metrix.WithChartPriority(chartengine.Priority-10),
97 + metrix.WithUnit("state"),
98 + ).ObserveStateSet(jobStatePoint)
99
100 jobMeter.Gauge(
101 "job.execution_duration",
@@ -111,14 +122,14 @@ func (c *Collector) emitMetrics(execMetrics executionMetrics) {
122 jobMeter.MeasureSetCounter(
123 measureSet.name,
124 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
114 - metrix.WithChartFamily(measureSet.scriptName),
125 + metrix.WithChartFamily(perfdataFamily(measureSet.scriptName)),
126 metrix.WithUnit(measureSet.unit),
127 ).ObserveTotalFields(fields)
128 } else {
129 jobMeter.MeasureSetGauge(
130 measureSet.name,
131 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
121 - metrix.WithChartFamily(measureSet.scriptName),
132 + metrix.WithChartFamily(perfdataFamily(measureSet.scriptName)),
133 metrix.WithUnit(measureSet.unit),
134 ).ObserveFields(fields)
135 }
@@ -129,7 +140,7 @@ func (c *Collector) emitMetrics(execMetrics executionMetrics) {
140 thresholdState.name,
141 metrix.WithStateSetMode(metrix.ModeBitSet),
142 metrix.WithStateSetStates(perfThresholdStateNames...),
132 - metrix.WithChartFamily(thresholdState.scriptName),
143 + metrix.WithChartFamily(perfdataFamily(thresholdState.scriptName)),
144 metrix.WithUnit("state"),
145 )
146 if thresholdState.state == "" {
@@ -137,22 +148,19 @@ func (c *Collector) emitMetrics(execMetrics executionMetrics) {
148 } else {
149 inst.Enable(thresholdState.state)
150 }
151 +
152 + jobMeter.WithLabels(metrix.Label{
153 + Key: perfdataValueLabelKey,
154 + Value: thresholdState.perfdataValue,
155 + }).StateSet(
156 + jobPerfdataThresholdMetricName,
157 + metrix.WithStateSetMode(metrix.ModeBitSet),
158 + metrix.WithStateSetStates(perfThresholdAlertStateNames...),
159 + metrix.WithUnit("state"),
160 + ).ObserveStateSet(projectPerfThresholdAlertState(thresholdState.state, c.state.isRetrying()))
161 }
162 }
163
143 -func normalizeJobStateForMetric(state string) string {
144 - switch strings.ToUpper(strings.TrimSpace(state)) {
145 - case nagiosStateOK:
146 - return "ok"
147 - case nagiosStateWarning:
148 - return "warning"
149 - case nagiosStateCritical:
150 - return "critical"
151 - case jobStateTimeout:
152 - return "timeout"
153 - case jobStatePaused:
154 - return "paused"
155 - default:
156 - return "unknown"
157 - }
164 +func perfdataFamily(scriptName string) string {
165 + return "Perfdata/" + scriptName
166 }
src/go/plugin/scripts.d/collector/nagios/collector_test.go
+214 -18
@@ -42,17 +42,17 @@ func TestCollector_ChartTemplateYAML(t *testing.T) {
42 wantFloat bool
43 }{
44 "execution duration dimension is float": {
45 - context: "job_execution_duration",
45 + context: "execution_duration",
46 selector: "nagios.job.execution_duration",
47 wantFloat: true,
48 },
49 "execution cpu dimension is float": {
50 - context: "job_execution_cpu",
50 + context: "execution_cpu",
51 selector: "nagios.job.execution_cpu_total",
52 wantFloat: true,
53 },
54 "execution memory dimension is integer": {
55 - context: "job_execution_memory",
55 + context: "execution_memory",
56 selector: "nagios.job.execution_max_rss",
57 wantFloat: false,
58 },
@@ -322,7 +322,9 @@ func TestCollector_Collect(t *testing.T) {
322
323 read := coll.MetricStore().Read(metrix.ReadRaw())
324 flat := coll.MetricStore().Read(metrix.ReadFlatten())
325 - assertMetricValue(t, flat, "nagios.job.state", metrix.Labels{"nagios_job": "check_disk", "nagios.job.state": "ok"}, 1)
325 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "check_disk", "nagios.job.execution_state": "ok"}, 1)
326 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "check_disk", "nagios.perfdata.true.job.execution_state": "ok"}, 1)
327 + assertMetricChartFamily(t, flat, "nagios.perfdata.true.job.execution_state", "Perfdata/true")
328 assertMetricValue(t, flat, "nagios.job.execution_duration", metrix.Labels{"nagios_job": "check_disk"}, 2.5)
329 assertMetricMeta(t, flat, "nagios.job.execution_duration", "seconds", true)
330 if runtime.GOOS != "windows" {
@@ -334,8 +336,8 @@ func TestCollector_Collect(t *testing.T) {
336 assertMetricMissing(t, flat, "nagios.job.execution_cpu_total", metrix.Labels{"nagios_job": "check_disk"})
337 assertMetricMissing(t, flat, "nagios.job.execution_max_rss", metrix.Labels{"nagios_job": "check_disk"})
338 }
337 - assertMetricValue(t, flat, "nagios.true.bytes_used_value", metrix.Labels{"nagios_job": "check_disk", metrix.MeasureSetFieldLabel: "value"}, 30000)
338 - point, ok := read.MeasureSet("nagios.true.bytes_used", metrix.Labels{"nagios_job": "check_disk"})
339 + assertMetricValue(t, flat, "nagios.perfdata.true.bytes_used_value", metrix.Labels{"nagios_job": "check_disk", metrix.MeasureSetFieldLabel: "value"}, 30000)
340 + point, ok := read.MeasureSet("nagios.perfdata.true.bytes_used", metrix.Labels{"nagios_job": "check_disk"})
341 require.True(t, ok)
342 assert.Equal(t, 30000.0, point.Values[0])
343
@@ -352,7 +354,7 @@ func TestCollector_Collect(t *testing.T) {
354 assertMetricMissing(t, flat, "nagios.job.execution_cpu_total", metrix.Labels{"nagios_job": "check_disk"})
355 assertMetricMissing(t, flat, "nagios.job.execution_max_rss", metrix.Labels{"nagios_job": "check_disk"})
356 }
355 - assertMetricValue(t, flat, "nagios.true.bytes_used_value", metrix.Labels{"nagios_job": "check_disk", metrix.MeasureSetFieldLabel: "value"}, 30000)
357 + assertMetricValue(t, flat, "nagios.perfdata.true.bytes_used_value", metrix.Labels{"nagios_job": "check_disk", metrix.MeasureSetFieldLabel: "value"}, 30000)
358 },
359 },
360 "check period blocked cycles pause job state and zero threshold states": {
@@ -426,27 +428,46 @@ func TestCollector_Collect(t *testing.T) {
428 assert.Equal(t, 1, runner.calls)
429
430 flat := coll.MetricStore().Read(metrix.ReadFlatten())
429 - assertMetricValue(t, flat, "nagios.job.state", metrix.Labels{"nagios_job": "period_job", "nagios.job.state": "ok"}, 1)
430 - assertMetricValue(t, flat, "nagios.true.bytes_used_value", metrix.Labels{"nagios_job": "period_job", metrix.MeasureSetFieldLabel: "value"}, 30000)
431 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.job.execution_state": "ok"}, 1)
432 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.perfdata.true.job.execution_state": "ok"}, 1)
433 + assertMetricValue(t, flat, "nagios.perfdata.true.bytes_used_value", metrix.Labels{"nagios_job": "period_job", metrix.MeasureSetFieldLabel: "value"}, 30000)
434 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
435 + "nagios_job": "period_job",
436 + perfdataValueLabelKey: "bytes_used",
437 + "nagios.job.perfdata.threshold_state": perfThresholdStateWarning,
438 + }, 1)
439
440 raw := coll.MetricStore().Read()
433 - thresholdMetric := "nagios.true.bytes_used_threshold_state"
441 + thresholdMetric := "nagios.perfdata.true.bytes_used_threshold_state"
442 thresholdLabels := metrix.Labels{"nagios_job": "period_job"}
443 point, ok := raw.StateSet(thresholdMetric, thresholdLabels)
444 require.True(t, ok)
445 assert.True(t, point.States[perfThresholdStateWarning])
446 + alertThresholdMetric := "nagios.job.perfdata.threshold_state"
447 + alertThresholdLabels := metrix.Labels{"nagios_job": "period_job", perfdataValueLabelKey: "bytes_used"}
448 + alertPoint, ok := raw.StateSet(alertThresholdMetric, alertThresholdLabels)
449 + require.True(t, ok)
450 + assert.True(t, alertPoint.States[perfThresholdStateWarning])
451
452 *now = time.Date(2026, 3, 23, 20, 0, 0, 0, time.UTC)
453 runCollectCycle(t, coll)
454 assert.Equal(t, 1, runner.calls)
455
456 flat = coll.MetricStore().Read(metrix.ReadFlatten())
444 - assertMetricValue(t, flat, "nagios.job.state", metrix.Labels{"nagios_job": "period_job", "nagios.job.state": "paused"}, 1)
445 - assertMetricValue(t, flat, "nagios.true.bytes_used_value", metrix.Labels{"nagios_job": "period_job", metrix.MeasureSetFieldLabel: "value"}, 30000)
457 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.job.execution_state": "paused"}, 1)
458 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.job.execution_state": "retry"}, 0)
459 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.perfdata.true.job.execution_state": "paused"}, 1)
460 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.perfdata.true.job.execution_state": "retry"}, 0)
461 + assertMetricValue(t, flat, "nagios.perfdata.true.bytes_used_value", metrix.Labels{"nagios_job": "period_job", metrix.MeasureSetFieldLabel: "value"}, 30000)
462 assertMetricValue(t, flat, thresholdMetric, metrix.Labels{"nagios_job": "period_job", thresholdMetric: perfThresholdStateWarning}, 0)
463 assertMetricValue(t, flat, thresholdMetric, metrix.Labels{"nagios_job": "period_job", thresholdMetric: perfThresholdStateOK}, 0)
464 assertMetricValue(t, flat, thresholdMetric, metrix.Labels{"nagios_job": "period_job", thresholdMetric: perfThresholdStateCritical}, 0)
465 assertMetricValue(t, flat, thresholdMetric, metrix.Labels{"nagios_job": "period_job", thresholdMetric: perfThresholdStateNone}, 0)
466 + assertMetricValue(t, flat, alertThresholdMetric, metrix.Labels{"nagios_job": "period_job", perfdataValueLabelKey: "bytes_used", alertThresholdMetric: perfThresholdStateWarning}, 0)
467 + assertMetricValue(t, flat, alertThresholdMetric, metrix.Labels{"nagios_job": "period_job", perfdataValueLabelKey: "bytes_used", alertThresholdMetric: perfThresholdStateOK}, 0)
468 + assertMetricValue(t, flat, alertThresholdMetric, metrix.Labels{"nagios_job": "period_job", perfdataValueLabelKey: "bytes_used", alertThresholdMetric: perfThresholdStateCritical}, 0)
469 + assertMetricValue(t, flat, alertThresholdMetric, metrix.Labels{"nagios_job": "period_job", perfdataValueLabelKey: "bytes_used", alertThresholdMetric: perfThresholdStateNone}, 0)
470 + assertMetricValue(t, flat, alertThresholdMetric, metrix.Labels{"nagios_job": "period_job", perfdataValueLabelKey: "bytes_used", alertThresholdMetric: perfThresholdStateRetry}, 0)
471
472 raw = coll.MetricStore().Read()
473 point, ok = raw.StateSet(thresholdMetric, thresholdLabels)
@@ -455,14 +476,22 @@ func TestCollector_Collect(t *testing.T) {
476 assert.False(t, point.States[perfThresholdStateOK])
477 assert.False(t, point.States[perfThresholdStateWarning])
478 assert.False(t, point.States[perfThresholdStateCritical])
479 + alertPoint, ok = raw.StateSet(alertThresholdMetric, alertThresholdLabels)
480 + require.True(t, ok)
481 + assert.False(t, alertPoint.States[perfThresholdStateNone])
482 + assert.False(t, alertPoint.States[perfThresholdStateOK])
483 + assert.False(t, alertPoint.States[perfThresholdStateWarning])
484 + assert.False(t, alertPoint.States[perfThresholdStateCritical])
485 + assert.False(t, alertPoint.States[perfThresholdStateRetry])
486
487 *now = time.Date(2026, 3, 24, 9, 0, 0, 0, time.UTC)
488 runCollectCycle(t, coll)
489 assert.Equal(t, 2, runner.calls)
490
491 flat = coll.MetricStore().Read(metrix.ReadFlatten())
464 - assertMetricValue(t, flat, "nagios.job.state", metrix.Labels{"nagios_job": "period_job", "nagios.job.state": "ok"}, 1)
465 - assertMetricValue(t, flat, "nagios.true.bytes_used_value", metrix.Labels{"nagios_job": "period_job", metrix.MeasureSetFieldLabel: "value"}, 10000)
492 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.job.execution_state": "ok"}, 1)
493 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "period_job", "nagios.perfdata.true.job.execution_state": "ok"}, 1)
494 + assertMetricValue(t, flat, "nagios.perfdata.true.bytes_used_value", metrix.Labels{"nagios_job": "period_job", metrix.MeasureSetFieldLabel: "value"}, 10000)
495
496 raw = coll.MetricStore().Read()
497 point, ok = raw.StateSet(thresholdMetric, thresholdLabels)
@@ -471,16 +500,57 @@ func TestCollector_Collect(t *testing.T) {
500 assert.True(t, point.States[perfThresholdStateOK])
501 assert.False(t, point.States[perfThresholdStateWarning])
502 assert.False(t, point.States[perfThresholdStateCritical])
503 + alertPoint, ok = raw.StateSet(alertThresholdMetric, alertThresholdLabels)
504 + require.True(t, ok)
505 + assert.False(t, alertPoint.States[perfThresholdStateNone])
506 + assert.True(t, alertPoint.States[perfThresholdStateOK])
507 + assert.False(t, alertPoint.States[perfThresholdStateWarning])
508 + assert.False(t, alertPoint.States[perfThresholdStateCritical])
509 },
510 },
511 "uses retry interval for retry state": {
512 results: []fakeRun{
513 {
479 - result: checkRunResult{ServiceState: "WARNING", JobState: "WARNING", ExitCode: 1},
480 - err: errors.New("plugin returned warning"),
514 + result: checkRunResult{
515 + ServiceState: "WARNING",
516 + JobState: "WARNING",
517 + ExitCode: 1,
518 + Parsed: output.ParsedOutput{
519 + Perfdata: []output.PerfDatum{
520 + func() output.PerfDatum {
521 + low := 0.0
522 + high := 20.0
523 + return output.PerfDatum{
524 + Label: "used",
525 + Unit: "KB",
526 + Value: 30,
527 + Warn: &output.ThresholdRange{Low: &low, High: &high},
528 + }
529 + }(),
530 + },
531 + },
532 + },
533 + err: errors.New("plugin returned warning"),
534 },
535 {
483 - result: checkRunResult{ServiceState: "OK", JobState: "OK"},
536 + result: checkRunResult{
537 + ServiceState: "OK",
538 + JobState: "OK",
539 + Parsed: output.ParsedOutput{
540 + Perfdata: []output.PerfDatum{
541 + func() output.PerfDatum {
542 + low := 0.0
543 + high := 20.0
544 + return output.PerfDatum{
545 + Label: "used",
546 + Unit: "KB",
547 + Value: 10,
548 + Warn: &output.ThresholdRange{Low: &low, High: &high},
549 + }
550 + }(),
551 + },
552 + },
553 + },
554 },
555 },
556 config: Config{
@@ -498,6 +568,21 @@ func TestCollector_Collect(t *testing.T) {
568 runCollectCycle(t, coll)
569 assert.Equal(t, 1, runner.calls)
570 assert.Equal(t, 2, coll.state.currentAttempt())
571 + flat := coll.MetricStore().Read(metrix.ReadFlatten())
572 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "retry_job", "nagios.job.execution_state": "warning"}, 1)
573 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "retry_job", "nagios.job.execution_state": "retry"}, 1)
574 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "retry_job", "nagios.perfdata.true.job.execution_state": "warning"}, 1)
575 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "retry_job", "nagios.perfdata.true.job.execution_state": "retry"}, 1)
576 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
577 + "nagios_job": "retry_job",
578 + perfdataValueLabelKey: "bytes_used",
579 + "nagios.job.perfdata.threshold_state": perfThresholdStateWarning,
580 + }, 1)
581 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
582 + "nagios_job": "retry_job",
583 + perfdataValueLabelKey: "bytes_used",
584 + "nagios.job.perfdata.threshold_state": perfThresholdStateRetry,
585 + }, 1)
586
587 *now = now.Add(9 * time.Second)
588 runCollectCycle(t, coll)
@@ -507,6 +592,114 @@ func TestCollector_Collect(t *testing.T) {
592 runCollectCycle(t, coll)
593 assert.Equal(t, 2, runner.calls)
594 assert.Equal(t, 1, coll.state.currentAttempt())
595 + flat = coll.MetricStore().Read(metrix.ReadFlatten())
596 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "retry_job", "nagios.job.execution_state": "ok"}, 1)
597 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "retry_job", "nagios.job.execution_state": "retry"}, 0)
598 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
599 + "nagios_job": "retry_job",
600 + perfdataValueLabelKey: "bytes_used",
601 + "nagios.job.perfdata.threshold_state": perfThresholdStateOK,
602 + }, 1)
603 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
604 + "nagios_job": "retry_job",
605 + perfdataValueLabelKey: "bytes_used",
606 + "nagios.job.perfdata.threshold_state": perfThresholdStateRetry,
607 + }, 0)
608 + },
609 + },
610 + "period block suppresses public retry state": {
611 + results: []fakeRun{
612 + {
613 + result: checkRunResult{
614 + ServiceState: "WARNING",
615 + JobState: "WARNING",
616 + ExitCode: 1,
617 + Parsed: output.ParsedOutput{
618 + Perfdata: []output.PerfDatum{
619 + func() output.PerfDatum {
620 + low := 0.0
621 + high := 20.0
622 + return output.PerfDatum{
623 + Label: "used",
624 + Unit: "KB",
625 + Value: 30,
626 + Warn: &output.ThresholdRange{Low: &low, High: &high},
627 + }
628 + }(),
629 + },
630 + },
631 + },
632 + err: errors.New("plugin returned warning"),
633 + },
634 + },
635 + config: Config{
636 + UpdateEvery: 1,
637 + JobConfig: JobConfig{
638 + Name: "paused_retry_job",
639 + Plugin: "/bin/true",
640 + CheckInterval: confDuration(1 * time.Hour),
641 + RetryInterval: confDuration(10 * time.Second),
642 + MaxCheckAttempts: 3,
643 + CheckPeriod: "business",
644 + },
645 + TimePeriods: []timeperiod.Config{
646 + {
647 + Name: "business",
648 + Rules: []timeperiod.RuleConfig{
649 + {
650 + Type: "weekly",
651 + Days: []string{"sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"},
652 + Ranges: []string{"09:00-18:00"},
653 + },
654 + },
655 + },
656 + },
657 + },
658 + run: func(t *testing.T, coll *Collector, runner *fakeRunner, now *time.Time) {
659 + t.Helper()
660 + runCollectCycle(t, coll)
661 + assert.Equal(t, 1, runner.calls)
662 +
663 + flat := coll.MetricStore().Read(metrix.ReadFlatten())
664 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "paused_retry_job", "nagios.job.execution_state": "warning"}, 1)
665 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "paused_retry_job", "nagios.job.execution_state": "retry"}, 1)
666 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
667 + "nagios_job": "paused_retry_job",
668 + perfdataValueLabelKey: "bytes_used",
669 + "nagios.job.perfdata.threshold_state": perfThresholdStateWarning,
670 + }, 1)
671 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
672 + "nagios_job": "paused_retry_job",
673 + perfdataValueLabelKey: "bytes_used",
674 + "nagios.job.perfdata.threshold_state": perfThresholdStateRetry,
675 + }, 1)
676 +
677 + *now = time.Date(2026, 3, 23, 20, 0, 0, 0, time.UTC)
678 + runCollectCycle(t, coll)
679 + assert.Equal(t, 1, runner.calls)
680 +
681 + flat = coll.MetricStore().Read(metrix.ReadFlatten())
682 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "paused_retry_job", "nagios.job.execution_state": "paused"}, 1)
683 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "paused_retry_job", "nagios.job.execution_state": "retry"}, 0)
684 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "paused_retry_job", "nagios.perfdata.true.job.execution_state": "paused"}, 1)
685 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "paused_retry_job", "nagios.perfdata.true.job.execution_state": "retry"}, 0)
686 + for _, state := range perfThresholdAlertStateNames {
687 + assertMetricValue(t, flat, "nagios.job.perfdata.threshold_state", metrix.Labels{
688 + "nagios_job": "paused_retry_job",
689 + perfdataValueLabelKey: "bytes_used",
690 + "nagios.job.perfdata.threshold_state": state,
691 + }, 0)
692 + }
693 +
694 + raw := coll.MetricStore().Read()
695 + alertPoint, ok := raw.StateSet("nagios.job.perfdata.threshold_state", metrix.Labels{
696 + "nagios_job": "paused_retry_job",
697 + perfdataValueLabelKey: "bytes_used",
698 + })
699 + require.True(t, ok)
700 + for _, state := range perfThresholdAlertStateNames {
701 + assert.False(t, alertPoint.States[state])
702 + }
703 },
704 },
705 "timeout is exposed publicly but macros keep Nagios unknown": {
@@ -537,7 +730,10 @@ func TestCollector_Collect(t *testing.T) {
730 assert.Equal(t, jobStateTimeout, coll.state.currentJobState())
731
732 flat := coll.MetricStore().Read(metrix.ReadFlatten())
540 - assertMetricValue(t, flat, "nagios.job.state", metrix.Labels{"nagios_job": "timeout_job", "nagios.job.state": "timeout"}, 1)
733 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "timeout_job", "nagios.job.execution_state": "timeout"}, 1)
734 + assertMetricValue(t, flat, "nagios.job.execution_state", metrix.Labels{"nagios_job": "timeout_job", "nagios.job.execution_state": "retry"}, 1)
735 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "timeout_job", "nagios.perfdata.true.job.execution_state": "timeout"}, 1)
736 + assertMetricValue(t, flat, "nagios.perfdata.true.job.execution_state", metrix.Labels{"nagios_job": "timeout_job", "nagios.perfdata.true.job.execution_state": "retry"}, 1)
737
738 *now = now.Add(11 * time.Second)
739 runCollectCycle(t, coll)
src/go/plugin/scripts.d/collector/nagios/metadata.yaml
+38 -9
@@ -25,9 +25,9 @@ modules:
25 multi_instance: true
26 data_collection:
27 metrics_description: |
28 - This collector runs Nagios-compatible checks, tracks the state of each configured job, measures how long each check takes to run, and automatically charts any performance data the check prints. For non-counter perfdata, Netdata also derives a threshold-state chart and reports `no_threshold` when the check does not provide warning or critical ranges.
28 + This collector runs Nagios-compatible checks, tracks the execution state of each configured job, measures how long each check takes to run, and automatically charts any performance data the check prints. For non-counter perfdata, Netdata also derives a plugin-scoped threshold-state chart for visualization and a static `nagios.job.perfdata.threshold_state` duplicate for alerting. When the check does not provide warning or critical ranges, the threshold state is `no_threshold`.
29 method_description: |
30 - Netdata runs the configured Nagios-compatible command for each job, reads the process exit code to determine the check state, and parses the command output into a summary line, optional long output, and optional performance data. Any performance data found after the `|` separator is converted into charts automatically. The main perfdata value becomes a chart, and non-counter metrics also get a derived threshold-state chart. If the check does not provide warning or critical ranges, that derived state is `no_threshold`. You can use packaged Nagios plugins or your own scripts, and you can control how often checks run, how retries behave, and when checks are allowed to run by using the job configuration.
30 + Netdata runs the configured Nagios-compatible command for each job, reads the process exit code to determine the check state, and parses the command output into a summary line, optional long output, and optional performance data. Any performance data found after the `|` separator is converted into charts automatically. The main perfdata value becomes a chart, and non-counter metrics also get derived threshold-state output in two forms: a plugin-scoped chart for visualization and a static `nagios.job.perfdata.threshold_state` chart labeled by `perfdata_value` for stock alerting. If the check does not provide warning or critical ranges, that threshold state is `no_threshold`. You can use packaged Nagios plugins or your own scripts, and you can control how often checks run, how retries behave, and when checks are allowed to run by using the job configuration.
31 default_behavior:
32 auto_detection:
33 description: |
@@ -116,7 +116,7 @@ modules:
116 - `OK - 85.5% free memory` is the summary line
117 - `free_pct=85.5%;20;10;0;100` creates a percentage metric
118 - `free_kb=13999088KB;;;0;16380000` creates a size metric
119 - - the warning and critical ranges on non-counter metrics are also used to derive a threshold-state chart
119 + - the warning and critical ranges on non-counter metrics are also used to derive threshold-state output for both visualization and alerting
120
121 Good rules to follow:
122
@@ -301,19 +301,35 @@ modules:
301 - name: The script works in a shell but fails under Netdata
302 description: |
303 Nagios checks run with a limited execution environment rather than inheriting the full Netdata process environment. If the script depends on extra variables, set them explicitly in `environment` instead of relying on ambient shell state.
304 - - name: No built-in alerts are shipped yet
304 + - name: Built-in alerts cover warning and critical states only
305 description: |
306 - This preview collector does not currently install stock Netdata health alerts. Use the exposed `nagios.job.state` chart and the derived perfdata threshold-state charts to build alert rules that match your own checks.
306 + This collector installs stock Netdata health alerts for the `warning` and `critical` states on `nagios.job.execution_state` and `nagios.job.perfdata_threshold_state`. Both stock alert families suppress soft retry states by checking that `retry` is not active. If you also want alerts for `unknown`, `timeout`, `paused`, or more specific perfdata behavior, build your own rules on top of these contexts. The `job.perfdata.threshold_state` chart uses the `perfdata_value` label to identify which perfdata metric each threshold state belongs to.
307 - name: Windows checks need an executable entry point
308 description: |
309 The collector runs the command named in `plugin` directly. On Windows, point `plugin` to an executable or to an interpreter such as `powershell.exe` and pass the script path in `args`.
310 - alerts: []
310 + alerts:
311 + - name: nagios_job_execution_state_warn
312 + metric: nagios.job.execution_state
313 + info: "Nagios job ${label:nagios_job} is in WARNING state"
314 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/nagios.conf
315 + - name: nagios_job_execution_state_crit
316 + metric: nagios.job.execution_state
317 + info: "Nagios job ${label:nagios_job} is in CRITICAL state"
318 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/nagios.conf
319 + - name: nagios_job_perfdata_threshold_state_warn
320 + metric: nagios.job.perfdata_threshold_state
321 + info: "Nagios job ${label:nagios_job} perfdata ${label:perfdata_value} is in WARNING threshold state"
322 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/nagios.conf
323 + - name: nagios_job_perfdata_threshold_state_crit
324 + metric: nagios.job.perfdata_threshold_state
325 + info: "Nagios job ${label:nagios_job} perfdata ${label:perfdata_value} is in CRITICAL threshold state"
326 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/nagios.conf
327 metrics:
328 folding:
329 title: Metrics
330 enabled: false
331 description: |
316 - Each configured job exposes state and execution charts. If a check prints Nagios performance data, Netdata also creates additional value charts automatically from the values emitted by that check. For non-counter perfdata, Netdata also creates a derived threshold-state chart and uses `no_threshold` when the check does not define warning or critical ranges. Counter perfdata currently exposes only the value chart.
332 + Each configured job exposes execution-state and execution-resource charts. Netdata also emits a plugin-scoped copy of the job state named `nagios.perfdata.<plugin>.job.execution_state` so the state appears under each configured check section in the UI. If a check prints Nagios performance data, Netdata also creates additional value charts automatically from the values emitted by that check. For non-counter perfdata, Netdata creates both a plugin-scoped threshold-state chart for visualization and a static `nagios.job.perfdata.threshold_state` duplicate labeled by `perfdata_value` for alerting. Counter perfdata currently exposes only the value chart.
333 availability: []
334 scopes:
335 - name: job
@@ -321,9 +337,11 @@ modules:
337 labels:
338 - name: nagios_job
339 description: Job name as defined in the configuration.
340 + - name: perfdata_value
341 + description: Normalized perfdata identity in the form `<class>_<metric_key>`, used by the static threshold-state duplicate.
342 metrics:
325 - - name: nagios.job.state
326 - description: Current job state for the check. Normal plugin results use `ok`, `warning`, `critical`, or `unknown`; collector-detected check timeouts use `timeout`; jobs blocked by `check_period` use `paused`.
343 + - name: nagios.job.execution_state
344 + description: Current job execution state for the check. Normal plugin results use `ok`, `warning`, `critical`, or `unknown`; collector-detected check timeouts use `timeout`; jobs blocked by `check_period` use `paused`. While a non-OK result is still retrying, the `retry` flag is also active on this bitset chart. Netdata also emits a plugin-scoped duplicate named `nagios.perfdata.<plugin>.job.execution_state` for UI grouping.
345 unit: state
346 chart_type: line
347 dimensions:
@@ -333,6 +351,17 @@ modules:
351 - name: unknown
352 - name: timeout
353 - name: paused
354 + - name: retry
355 + - name: nagios.job.perfdata.threshold_state
356 + description: Static alert-oriented duplicate of the non-counter perfdata threshold-state signal. Use the `perfdata_value` label to select the normalized perfdata identity, such as `bytes_used` or `time_latency`. Values are `no_threshold`, `ok`, `warning`, or `critical`. While the source check result is still retrying, this bitset chart also exposes `retry`.
357 + unit: state
358 + chart_type: line
359 + dimensions:
360 + - name: no_threshold
361 + - name: ok
362 + - name: warning
363 + - name: critical
364 + - name: retry
365 - name: nagios.job.execution_duration
366 description: Wall-clock duration recorded when the check runs. Non-due cycles report zero.
367 unit: seconds
src/go/plugin/scripts.d/collector/nagios/perf_measureset.go
+33 -14
@@ -5,23 +5,30 @@ package nagios
5 import "github.com/netdata/netdata/go/plugins/pkg/metrix"
6
7 const (
8 - perfFieldValue = "value"
9 - perfThresholdStateNone = "no_threshold"
10 - perfThresholdStateOK = "ok"
11 - perfThresholdStateWarning = "warning"
12 - perfThresholdStateCritical = "critical"
8 + perfFieldValue = "value"
9 + perfThresholdStateNone = "no_threshold"
10 + perfThresholdStateOK = "ok"
11 + perfThresholdStateWarning = "warning"
12 + perfThresholdStateCritical = "critical"
13 + perfThresholdStateRetry = "retry"
14 + perfdataValueLabelKey = "perfdata_value"
15 + jobPerfdataThresholdMetricName = "job.perfdata.threshold_state"
16 )
17
18 var (
16 - perfMeasureSetFieldOrder = []string{
17 - perfFieldValue,
18 - }
19 perfThresholdStateNames = []string{
20 perfThresholdStateNone,
21 perfThresholdStateOK,
22 perfThresholdStateWarning,
23 perfThresholdStateCritical,
24 }
25 + perfThresholdAlertStateNames = []string{
26 + perfThresholdStateNone,
27 + perfThresholdStateOK,
28 + perfThresholdStateWarning,
29 + perfThresholdStateCritical,
30 + perfThresholdStateRetry,
31 + }
32 )
33
34 type perfValueMeasureSet struct {
@@ -33,9 +40,10 @@ type perfValueMeasureSet struct {
40 }
41
42 type perfThresholdStateSet struct {
36 - name string
37 - scriptName string
38 - state string
43 + name string
44 + scriptName string
45 + perfdataValue string
46 + state string
47 }
48
49 type perfRouteResult struct {
@@ -62,11 +70,22 @@ func perfMeasureSetValues(value metrix.SampleValue) map[string]metrix.SampleValu
70 }
71
72 func perfThresholdStatePoint(active string) metrix.StateSetPoint {
65 - states := make(map[string]bool, len(perfThresholdStateNames))
66 - for _, state := range perfThresholdStateNames {
73 + return stateSetPoint(perfThresholdStateNames, active)
74 +}
75 +
76 +func perfThresholdAlertStatePoint(actives ...string) metrix.StateSetPoint {
77 + return stateSetPoint(perfThresholdAlertStateNames, actives...)
78 +}
79 +
80 +func stateSetPoint(names []string, actives ...string) metrix.StateSetPoint {
81 + states := make(map[string]bool, len(names))
82 + for _, state := range names {
83 states[state] = false
84 }
69 - if active != "" {
85 + for _, active := range actives {
86 + if active == "" {
87 + continue
88 + }
89 states[active] = true
90 }
91 return metrix.StateSetPoint{States: states}
src/go/plugin/scripts.d/collector/nagios/perfdata_router.go
+10 -4
@@ -79,6 +79,7 @@ func (r *perfdataRouter) route(pluginPath string, perf []output.PerfDatum) perfR
79 }
80 for _, item := range deduped {
81 base := perfMetricIdentity(source, item)
82 + tail := perfMetricTail(item)
83 result.values = append(result.values, perfValueMeasureSet{
84 name: base,
85 scriptName: source,
@@ -95,9 +96,10 @@ func (r *perfdataRouter) route(pluginPath string, perf []output.PerfDatum) perfR
96 }
97
98 result.thresholdStates = append(result.thresholdStates, perfThresholdStateSet{
98 - name: perfThresholdStateMetricName(base),
99 - scriptName: source,
100 - state: thresholdStateForPerfDatum(item),
99 + name: perfThresholdStateMetricName(base),
100 + scriptName: source,
101 + perfdataValue: tail,
102 + state: thresholdStateForPerfDatum(item),
103 })
104 }
105
@@ -105,7 +107,11 @@ func (r *perfdataRouter) route(pluginPath string, perf []output.PerfDatum) perfR
107 }
108
109 func perfMetricIdentity(source string, item perfPreparedDatum) string {
108 - return fmt.Sprintf("%s.%s_%s", source, item.class, item.metricKey)
110 + return fmt.Sprintf("perfdata.%s.%s", source, perfMetricTail(item))
111 +}
112 +
113 +func perfMetricTail(item perfPreparedDatum) string {
114 + return fmt.Sprintf("%s_%s", item.class, item.metricKey)
115 }
116
117 func perfThresholdStateMetricName(base string) string {
src/go/plugin/scripts.d/collector/nagios/perfdata_router_test.go
+39 -28
@@ -40,28 +40,31 @@ func TestPerfdataRouterRoutesAndCanonicalizesUnits(t *testing.T) {
40 values := valueSampleMap(got.values)
41 units := valueSampleUnits(got.values)
42 thresholds := thresholdStateMap(got.thresholdStates)
43 -
44 - assertNear(t, values["check_memory.time_latency_value"], 0.12)
45 - assertNear(t, values["check_memory.bytes_throughput_value"], 30_000)
46 - assertNear(t, values["check_memory.bits_traffic_value"], 1_500_000)
47 - assertNear(t, values["check_memory.percent_free_pct_value"], 40)
48 - assertNear(t, values["check_memory.counter_checks_value"], 3)
49 - assertNear(t, values["check_memory.generic_custom_value"], 7.25)
50 -
51 - assertString(t, units["check_memory.time_latency_value"], "seconds")
52 - assertString(t, units["check_memory.bytes_throughput_value"], "bytes")
53 - assertString(t, units["check_memory.bits_traffic_value"], "bits")
54 - assertString(t, units["check_memory.percent_free_pct_value"], "%")
55 - assertString(t, units["check_memory.counter_checks_value"], "c")
56 - assertString(t, units["check_memory.generic_custom_value"], "generic")
57 -
58 - assertString(t, thresholds["check_memory.time_latency_threshold_state"], perfThresholdStateWarning)
59 - assertString(t, thresholds["check_memory.bytes_throughput_threshold_state"], perfThresholdStateNone)
60 - assertString(t, thresholds["check_memory.bits_traffic_threshold_state"], perfThresholdStateNone)
61 - assertString(t, thresholds["check_memory.percent_free_pct_threshold_state"], perfThresholdStateNone)
62 - assertString(t, thresholds["check_memory.generic_custom_threshold_state"], perfThresholdStateNone)
63 - _, hasCounterThreshold := thresholds["check_memory.counter_checks_threshold_state"]
43 + thresholdLabelValues := thresholdStateLabelValues(got.thresholdStates)
44 +
45 + assertNear(t, values["perfdata.check_memory.time_latency_value"], 0.12)
46 + assertNear(t, values["perfdata.check_memory.bytes_throughput_value"], 30_000)
47 + assertNear(t, values["perfdata.check_memory.bits_traffic_value"], 1_500_000)
48 + assertNear(t, values["perfdata.check_memory.percent_free_pct_value"], 40)
49 + assertNear(t, values["perfdata.check_memory.counter_checks_value"], 3)
50 + assertNear(t, values["perfdata.check_memory.generic_custom_value"], 7.25)
51 +
52 + assertString(t, units["perfdata.check_memory.time_latency_value"], "seconds")
53 + assertString(t, units["perfdata.check_memory.bytes_throughput_value"], "bytes")
54 + assertString(t, units["perfdata.check_memory.bits_traffic_value"], "bits")
55 + assertString(t, units["perfdata.check_memory.percent_free_pct_value"], "%")
56 + assertString(t, units["perfdata.check_memory.counter_checks_value"], "c")
57 + assertString(t, units["perfdata.check_memory.generic_custom_value"], "generic")
58 +
59 + assertString(t, thresholds["perfdata.check_memory.time_latency_threshold_state"], perfThresholdStateWarning)
60 + assertString(t, thresholds["perfdata.check_memory.bytes_throughput_threshold_state"], perfThresholdStateNone)
61 + assertString(t, thresholds["perfdata.check_memory.bits_traffic_threshold_state"], perfThresholdStateNone)
62 + assertString(t, thresholds["perfdata.check_memory.percent_free_pct_threshold_state"], perfThresholdStateNone)
63 + assertString(t, thresholds["perfdata.check_memory.generic_custom_threshold_state"], perfThresholdStateNone)
64 + _, hasCounterThreshold := thresholds["perfdata.check_memory.counter_checks_threshold_state"]
65 assert.False(t, hasCounterThreshold)
66 + assertString(t, thresholdLabelValues["perfdata.check_memory.time_latency_threshold_state"], "time_latency")
67 + assertString(t, thresholdLabelValues["perfdata.check_memory.bytes_throughput_threshold_state"], "bytes_throughput")
68 }
69
70 func TestPerfdataRouterPolicies(t *testing.T) {
@@ -80,7 +83,7 @@ func TestPerfdataRouterPolicies(t *testing.T) {
83 assert: func(t *testing.T, got perfRouteResult) {
84 t.Helper()
85 samples := valueSampleMap(got.values)
83 - assertNear(t, samples["check_memory.bytes_used_kb_value"], 1_000)
86 + assertNear(t, samples["perfdata.check_memory.bytes_used_kb_value"], 1_000)
87 },
88 },
89 "budget drops metrics beyond cap": {
@@ -93,9 +96,9 @@ func TestPerfdataRouterPolicies(t *testing.T) {
96 assert: func(t *testing.T, got perfRouteResult) {
97 t.Helper()
98 samples := valueSampleMap(got.values)
96 - _, okA := samples["check_memory.counter_a_value"]
97 - _, okB := samples["check_memory.counter_b_value"]
98 - _, okC := samples["check_memory.counter_c_value"]
99 + _, okA := samples["perfdata.check_memory.counter_a_value"]
100 + _, okB := samples["perfdata.check_memory.counter_b_value"]
101 + _, okC := samples["perfdata.check_memory.counter_c_value"]
102 assert.True(t, okA)
103 assert.True(t, okB)
104 assert.False(t, okC)
@@ -110,8 +113,8 @@ func TestPerfdataRouterPolicies(t *testing.T) {
113 assert: func(t *testing.T, got perfRouteResult) {
114 t.Helper()
115 samples := valueSampleMap(got.values)
113 - assertNear(t, samples["check_memory.bytes_latency_value"], 1_000)
114 - _, hasTime := samples["check_memory.time_latency_value"]
116 + assertNear(t, samples["perfdata.check_memory.bytes_latency_value"], 1_000)
117 + _, hasTime := samples["perfdata.check_memory.time_latency_value"]
118 assert.False(t, hasTime)
119 },
120 },
@@ -126,7 +129,7 @@ func TestPerfdataRouterPolicies(t *testing.T) {
129 assert: func(t *testing.T, got perfRouteResult) {
130 t.Helper()
131 samples := valueSampleMap(got.values)
129 - assertNear(t, samples["check_memory.bytes_latency_value"], 10_000)
132 + assertNear(t, samples["perfdata.check_memory.bytes_latency_value"], 10_000)
133 },
134 },
135 "invalid samples are ignored": {
@@ -214,6 +217,14 @@ func thresholdStateMap(sets []perfThresholdStateSet) map[string]string {
217 return out
218 }
219
220 +func thresholdStateLabelValues(sets []perfThresholdStateSet) map[string]string {
221 + out := make(map[string]string, len(sets))
222 + for _, set := range sets {
223 + out[set.name] = set.perfdataValue
224 + }
225 + return out
226 +}
227 +
228 func assertNear(t *testing.T, got, want float64) {
229 t.Helper()
230 assert.InDelta(t, want, got, 1e-9)
src/go/plugin/scripts.d/collector/nagios/public_state.go new
+68
@@ -0,0 +1,68 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package nagios
4 +
5 +import (
6 + "strings"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/metrix"
9 +)
10 +
11 +const (
12 + metricJobStateOK = "ok"
13 + metricJobStateWarning = "warning"
14 + metricJobStateCritical = "critical"
15 + metricJobStateUnknown = "unknown"
16 + metricJobStateTimeout = "timeout"
17 + metricJobStatePaused = "paused"
18 + metricStateRetry = "retry"
19 +)
20 +
21 +var jobExecutionStateNames = []string{
22 + metricJobStateOK,
23 + metricJobStateWarning,
24 + metricJobStateCritical,
25 + metricJobStateUnknown,
26 + metricJobStateTimeout,
27 + metricJobStatePaused,
28 + metricStateRetry,
29 +}
30 +
31 +// projectJobExecutionState maps runtime state into the public metric surface.
32 +func projectJobExecutionState(state string, retrying bool) metrix.StateSetPoint {
33 + normalized := normalizeJobStateForMetric(state)
34 + actives := []string{normalized}
35 + if retrying && normalized != metricJobStatePaused {
36 + actives = append(actives, metricStateRetry)
37 + }
38 + return stateSetPoint(jobExecutionStateNames, actives...)
39 +}
40 +
41 +// projectPerfThresholdAlertState maps routed threshold state into the alertable duplicate.
42 +func projectPerfThresholdAlertState(state string, retrying bool) metrix.StateSetPoint {
43 + if state == "" {
44 + return perfThresholdAlertStatePoint()
45 + }
46 + actives := []string{state}
47 + if retrying {
48 + actives = append(actives, perfThresholdStateRetry)
49 + }
50 + return perfThresholdAlertStatePoint(actives...)
51 +}
52 +
53 +func normalizeJobStateForMetric(state string) string {
54 + switch strings.ToUpper(strings.TrimSpace(state)) {
55 + case nagiosStateOK:
56 + return metricJobStateOK
57 + case nagiosStateWarning:
58 + return metricJobStateWarning
59 + case nagiosStateCritical:
60 + return metricJobStateCritical
61 + case jobStateTimeout:
62 + return metricJobStateTimeout
63 + case jobStatePaused:
64 + return metricJobStatePaused
65 + default:
66 + return metricJobStateUnknown
67 + }
68 +}
src/go/plugin/scripts.d/collector/nagios/public_state_test.go new
+38
@@ -0,0 +1,38 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package nagios
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/stretchr/testify/assert"
9 +)
10 +
11 +func TestProjectJobExecutionState(t *testing.T) {
12 + t.Run("paused suppresses retry", func(t *testing.T) {
13 + point := projectJobExecutionState(jobStatePaused, true)
14 + assert.True(t, point.States[metricJobStatePaused])
15 + assert.False(t, point.States[metricStateRetry])
16 + })
17 +
18 + t.Run("warning keeps retry", func(t *testing.T) {
19 + point := projectJobExecutionState(nagiosStateWarning, true)
20 + assert.True(t, point.States[metricJobStateWarning])
21 + assert.True(t, point.States[metricStateRetry])
22 + })
23 +}
24 +
25 +func TestProjectPerfThresholdAlertState(t *testing.T) {
26 + t.Run("empty state clears everything", func(t *testing.T) {
27 + point := projectPerfThresholdAlertState("", true)
28 + for _, state := range perfThresholdAlertStateNames {
29 + assert.False(t, point.States[state])
30 + }
31 + })
32 +
33 + t.Run("warning keeps retry", func(t *testing.T) {
34 + point := projectPerfThresholdAlertState(perfThresholdStateWarning, true)
35 + assert.True(t, point.States[perfThresholdStateWarning])
36 + assert.True(t, point.States[perfThresholdStateRetry])
37 + })
38 +}
src/go/plugin/scripts.d/collector/nagios/state.go
+4
@@ -49,6 +49,10 @@ func (s *collectState) currentJobState() string {
49 return s.jobState
50 }
51
52 +func (s *collectState) isRetrying() bool {
53 + return s != nil && s.retrying
54 +}
55 +
56 func (s *collectState) currentAttempt() int {
57 if s == nil {
58 return 1
src/go/plugin/scripts.d/collector/nagios/v2_gate_test.go
+72 -82
@@ -54,27 +54,27 @@ func TestV2Gate_G2_PerfdataRouting(t *testing.T) {
54 byName := valueSampleMap(samples.values)
55 byUnit := valueSampleUnits(samples.values)
56 byThreshold := thresholdStateMap(samples.thresholdStates)
57 - assertNear(t, byName["check_gate.time_latency_value"], 0.12)
58 - assertNear(t, byName["check_gate.bytes_throughput_value"], 30000)
59 - assertNear(t, byName["check_gate.bits_wire_rate_value"], 80000)
60 - assertNear(t, byName["check_gate.percent_free_pct_value"], 40)
61 - assertNear(t, byName["check_gate.counter_requests_value"], 42)
62 - assertNear(t, byName["check_gate.generic_custom_value"], 3.14)
63 - assertNear(t, byName["check_gate.generic_dup_one_value"], 11)
64 - assertString(t, byThreshold["check_gate.time_latency_threshold_state"], perfThresholdStateWarning)
65 - assertString(t, byThreshold["check_gate.bytes_throughput_threshold_state"], perfThresholdStateNone)
66 - assertString(t, byThreshold["check_gate.bits_wire_rate_threshold_state"], perfThresholdStateNone)
67 - assertString(t, byThreshold["check_gate.percent_free_pct_threshold_state"], perfThresholdStateNone)
68 - assertString(t, byThreshold["check_gate.generic_custom_threshold_state"], perfThresholdStateNone)
69 - _, hasCounterThreshold := byThreshold["check_gate.counter_requests_threshold_state"]
57 + assertNear(t, byName["perfdata.check_gate.time_latency_value"], 0.12)
58 + assertNear(t, byName["perfdata.check_gate.bytes_throughput_value"], 30000)
59 + assertNear(t, byName["perfdata.check_gate.bits_wire_rate_value"], 80000)
60 + assertNear(t, byName["perfdata.check_gate.percent_free_pct_value"], 40)
61 + assertNear(t, byName["perfdata.check_gate.counter_requests_value"], 42)
62 + assertNear(t, byName["perfdata.check_gate.generic_custom_value"], 3.14)
63 + assertNear(t, byName["perfdata.check_gate.generic_dup_one_value"], 11)
64 + assertString(t, byThreshold["perfdata.check_gate.time_latency_threshold_state"], perfThresholdStateWarning)
65 + assertString(t, byThreshold["perfdata.check_gate.bytes_throughput_threshold_state"], perfThresholdStateNone)
66 + assertString(t, byThreshold["perfdata.check_gate.bits_wire_rate_threshold_state"], perfThresholdStateNone)
67 + assertString(t, byThreshold["perfdata.check_gate.percent_free_pct_threshold_state"], perfThresholdStateNone)
68 + assertString(t, byThreshold["perfdata.check_gate.generic_custom_threshold_state"], perfThresholdStateNone)
69 + _, hasCounterThreshold := byThreshold["perfdata.check_gate.counter_requests_threshold_state"]
70 assert.False(t, hasCounterThreshold)
71
72 - assertString(t, byUnit["check_gate.time_latency_value"], "seconds")
73 - assertString(t, byUnit["check_gate.bytes_throughput_value"], "bytes")
74 - assertString(t, byUnit["check_gate.bits_wire_rate_value"], "bits")
75 - assertString(t, byUnit["check_gate.percent_free_pct_value"], "%")
76 - assertString(t, byUnit["check_gate.counter_requests_value"], "c")
77 - assertString(t, byUnit["check_gate.generic_custom_value"], "generic")
72 + assertString(t, byUnit["perfdata.check_gate.time_latency_value"], "seconds")
73 + assertString(t, byUnit["perfdata.check_gate.bytes_throughput_value"], "bytes")
74 + assertString(t, byUnit["perfdata.check_gate.bits_wire_rate_value"], "bits")
75 + assertString(t, byUnit["perfdata.check_gate.percent_free_pct_value"], "%")
76 + assertString(t, byUnit["perfdata.check_gate.counter_requests_value"], "c")
77 + assertString(t, byUnit["perfdata.check_gate.generic_custom_value"], "generic")
78
79 store := metrix.NewCollectorStore()
80 cc := gateCycleController(t, store)
@@ -89,7 +89,7 @@ func TestV2Gate_G2_PerfdataRouting(t *testing.T) {
89 sm.MeasureSetCounter(
90 measureSet.name,
91 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
92 - metrix.WithChartFamily(measureSet.scriptName),
92 + metrix.WithChartFamily(perfdataFamily(measureSet.scriptName)),
93 metrix.WithUnit(measureSet.unit),
94 ).ObserveTotalFields(fields, labels)
95 continue
@@ -97,7 +97,7 @@ func TestV2Gate_G2_PerfdataRouting(t *testing.T) {
97 sm.MeasureSetGauge(
98 measureSet.name,
99 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
100 - metrix.WithChartFamily(measureSet.scriptName),
100 + metrix.WithChartFamily(perfdataFamily(measureSet.scriptName)),
101 metrix.WithUnit(measureSet.unit),
102 ).ObserveFields(fields, labels)
103 }
@@ -106,31 +106,50 @@ func TestV2Gate_G2_PerfdataRouting(t *testing.T) {
106 thresholdState.name,
107 metrix.WithStateSetMode(metrix.ModeBitSet),
108 metrix.WithStateSetStates(perfThresholdStateNames...),
109 - metrix.WithChartFamily(thresholdState.scriptName),
109 + metrix.WithChartFamily(perfdataFamily(thresholdState.scriptName)),
110 + metrix.WithUnit("state"),
111 + ).Enable(thresholdState.state)
112 + sm.WithLabelSet(labels).WithLabels(
113 + metrix.Label{Key: perfdataValueLabelKey, Value: thresholdState.perfdataValue},
114 + ).StateSet(
115 + jobPerfdataThresholdMetricName,
116 + metrix.WithStateSetMode(metrix.ModeBitSet),
117 + metrix.WithStateSetStates(perfThresholdAlertStateNames...),
118 metrix.WithUnit("state"),
119 ).Enable(thresholdState.state)
120 }
121 cc.CommitCycleSuccess()
122
123 reader := store.Read(metrix.ReadFlatten())
116 - assertMetricMeta(t, reader, "nagios.check_gate.time_latency_value", "seconds", true)
117 - assertMetricMeta(t, reader, "nagios.check_gate.bytes_throughput_value", "bytes", true)
118 - assertMetricMeta(t, reader, "nagios.check_gate.bits_wire_rate_value", "bits", true)
119 - assertMetricMeta(t, reader, "nagios.check_gate.percent_free_pct_value", "%", true)
120 - assertMetricMeta(t, reader, "nagios.check_gate.counter_requests_value", "c", true)
121 - assertMetricMeta(t, reader, "nagios.check_gate.generic_custom_value", "generic", true)
122 - assertMetricMeta(t, reader, "nagios.check_gate.time_latency_threshold_state", "state", false)
123 - assertMetricChartFamily(t, reader, "nagios.check_gate.time_latency_value", "check_gate")
124 - assertMetricChartFamily(t, reader, "nagios.check_gate.time_latency_threshold_state", "check_gate")
125 - assertMetricValue(t, reader, "nagios.check_gate.time_latency_threshold_state", metrix.Labels{
124 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.time_latency_value", "seconds", true)
125 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.bytes_throughput_value", "bytes", true)
126 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.bits_wire_rate_value", "bits", true)
127 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.percent_free_pct_value", "%", true)
128 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.counter_requests_value", "c", true)
129 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.generic_custom_value", "generic", true)
130 + assertMetricMeta(t, reader, "nagios.perfdata.check_gate.time_latency_threshold_state", "state", false)
131 + assertMetricMeta(t, reader, "nagios.job.perfdata.threshold_state", "state", false)
132 + assertMetricChartFamily(t, reader, "nagios.perfdata.check_gate.time_latency_value", "Perfdata/check_gate")
133 + assertMetricChartFamily(t, reader, "nagios.perfdata.check_gate.time_latency_threshold_state", "Perfdata/check_gate")
134 + assertMetricValue(t, reader, "nagios.perfdata.check_gate.time_latency_threshold_state", metrix.Labels{
135 "nagios_job": "gate_job",
127 - "nagios.check_gate.time_latency_threshold_state": perfThresholdStateWarning,
136 + "nagios.perfdata.check_gate.time_latency_threshold_state": perfThresholdStateWarning,
137 }, 1)
129 - assertSeriesKind(t, reader, "nagios.check_gate.time_latency_value", metrix.Labels{
138 + assertMetricValue(t, reader, "nagios.job.perfdata.threshold_state", metrix.Labels{
139 + "nagios_job": "gate_job",
140 + perfdataValueLabelKey: "time_latency",
141 + "nagios.job.perfdata.threshold_state": perfThresholdStateWarning,
142 + }, 1)
143 + assertMetricValue(t, reader, "nagios.job.perfdata.threshold_state", metrix.Labels{
144 + "nagios_job": "gate_job",
145 + perfdataValueLabelKey: "time_latency",
146 + "nagios.job.perfdata.threshold_state": perfThresholdStateRetry,
147 + }, 0)
148 + assertSeriesKind(t, reader, "nagios.perfdata.check_gate.time_latency_value", metrix.Labels{
149 "nagios_job": "gate_job",
150 metrix.MeasureSetFieldLabel: perfFieldValue,
151 }, metrix.MetricKindGauge)
133 - assertSeriesKind(t, reader, "nagios.check_gate.counter_requests_value", metrix.Labels{
152 + assertSeriesKind(t, reader, "nagios.perfdata.check_gate.counter_requests_value", metrix.Labels{
153 "nagios_job": "gate_job",
154 metrix.MeasureSetFieldLabel: perfFieldValue,
155 }, metrix.MetricKindCounter)
@@ -139,7 +158,7 @@ func TestV2Gate_G2_PerfdataRouting(t *testing.T) {
158 {Label: "latency", Unit: "%", Value: 1}, // same label, different class => new identity
159 })
160 changedSamples := valueSampleMap(changedClass.values)
142 - assertNear(t, changedSamples["check_gate.percent_latency_value"], 1)
161 + assertNear(t, changedSamples["perfdata.check_gate.percent_latency_value"], 1)
162 }
163
164 func TestV2Gate_G3_ChartLifecycleChurn(t *testing.T) {
@@ -160,18 +179,18 @@ func TestV2Gate_G3_ChartLifecycleChurn(t *testing.T) {
179 aFields := defaultPerfMeasureSetValues()
180 aFields[perfFieldValue] = 1
181 sm.MeasureSetGauge(
163 - "check_gate.bytes_a",
182 + "perfdata.check_gate.bytes_a",
183 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
165 - metrix.WithChartFamily("check_gate"),
184 + metrix.WithChartFamily(perfdataFamily("check_gate")),
185 metrix.WithUnit("bytes"),
186 ).ObserveFields(aFields, ls)
187 if includeB {
188 bFields := defaultPerfMeasureSetValues()
189 bFields[perfFieldValue] = 2
190 sm.MeasureSetGauge(
172 - "check_gate.bytes_b",
191 + "perfdata.check_gate.bytes_b",
192 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
174 - metrix.WithChartFamily("check_gate"),
193 + metrix.WithChartFamily(perfdataFamily("check_gate")),
194 metrix.WithUnit("bytes"),
195 ).ObserveFields(bFields, ls)
196 }
@@ -199,9 +218,9 @@ func TestV2Gate_G3_ChartLifecycleChurn(t *testing.T) {
218 aFields := defaultPerfMeasureSetValues()
219 aFields[perfFieldValue] = 1
220 sm.MeasureSetGauge(
202 - "check_gate.bytes_a",
221 + "perfdata.check_gate.bytes_a",
222 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
204 - metrix.WithChartFamily("check_gate"),
223 + metrix.WithChartFamily(perfdataFamily("check_gate")),
224 metrix.WithUnit("bytes"),
225 ).ObserveFields(aFields, ls)
226 cc.AbortCycle()
@@ -212,15 +231,15 @@ func TestV2Gate_G3_ChartLifecycleChurn(t *testing.T) {
231 assert.Zero(t, removeActionsCount(planAbort.Actions))
232 })
233
215 - t.Run("failed-attempt gap contributes to expiry aging", func(t *testing.T) {
234 + t.Run("failed-attempt gap does not count toward expiry", func(t *testing.T) {
235 _, store, emit := newHarness(t)
236
237 plan1 := emit(true)
238 assert.NotZero(t, countActions[chartengine.CreateChartAction](plan1.Actions))
239 plan2 := emit(false)
240 assert.Zero(t, removeActionsCount(plan2.Actions))
222 - assertPlanHasUpdateForTarget(t, plan2, "nagios.check_gate.bytes_a")
223 - assertPlanHasNoRemoveForTarget(t, plan2, "nagios.check_gate.bytes_b")
241 + assertPlanHasUpdateForTarget(t, plan2, "nagios.perfdata.check_gate.bytes_a")
242 + assertPlanHasNoRemoveForTarget(t, plan2, "nagios.perfdata.check_gate.bytes_b")
243
244 cc := gateCycleController(t, store)
245 cc.BeginCycle()
@@ -231,20 +250,18 @@ func TestV2Gate_G3_ChartLifecycleChurn(t *testing.T) {
250 aFields := defaultPerfMeasureSetValues()
251 aFields[perfFieldValue] = 1
252 sm.MeasureSetGauge(
234 - "check_gate.bytes_a",
253 + "perfdata.check_gate.bytes_a",
254 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
236 - metrix.WithChartFamily("check_gate"),
255 + metrix.WithChartFamily(perfdataFamily("check_gate")),
256 metrix.WithUnit("bytes"),
257 ).ObserveFields(aFields, ls)
258 cc.AbortCycle()
259 assert.Equal(t, metrix.CollectStatusFailed, store.Read(metrix.ReadRaw()).CollectMeta().LastAttemptStatus)
260
261 plan3 := emit(false)
243 - assert.NotZero(t, removeActionsCount(plan3.Actions))
244 - assertPlanHasUpdateAndRemoveForTargets(t, plan3,
245 - "nagios.check_gate.bytes_a",
246 - "nagios.check_gate.bytes_b",
247 - )
262 + assert.Zero(t, removeActionsCount(plan3.Actions))
263 + assertPlanHasUpdateForTarget(t, plan3, "nagios.perfdata.check_gate.bytes_a")
264 + assertPlanHasNoRemoveForTarget(t, plan3, "nagios.perfdata.check_gate.bytes_b")
265 plan4 := emit(false)
266 assert.Zero(t, removeActionsCount(plan4.Actions))
267 })
@@ -308,7 +325,7 @@ func TestV2Gate_G5_ScalingPrecisionEquivalence(t *testing.T) {
325 sm.MeasureSetCounter(
326 measureSet.name,
327 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
311 - metrix.WithChartFamily(measureSet.scriptName),
328 + metrix.WithChartFamily(perfdataFamily(measureSet.scriptName)),
329 metrix.WithUnit(measureSet.unit),
330 ).ObserveTotalFields(fields, sm.LabelSet())
331 continue
@@ -316,14 +333,14 @@ func TestV2Gate_G5_ScalingPrecisionEquivalence(t *testing.T) {
333 sm.MeasureSetGauge(
334 measureSet.name,
335 metrix.WithMeasureSetFields(perfMeasureSetFieldSpecs()...),
319 - metrix.WithChartFamily(measureSet.scriptName),
336 + metrix.WithChartFamily(perfdataFamily(measureSet.scriptName)),
337 metrix.WithUnit(measureSet.unit),
338 ).ObserveFields(fields, sm.LabelSet())
339 }
340 cc.CommitCycleSuccess()
341 flat := store.Read(metrix.ReadFlatten())
342 assertMetricMeta(t, flat, "nagios."+candidateKey, tc.expectedUnit, true)
326 - assertMetricChartFamily(t, flat, "nagios."+candidateKey, "check_gate")
343 + assertMetricChartFamily(t, flat, "nagios."+candidateKey, "Perfdata/check_gate")
344 })
345 }
346 }
@@ -385,33 +402,6 @@ func assertSeriesKind(t *testing.T, reader metrix.Reader, metricName string, lab
402 assert.Equal(t, want, meta.Kind)
403 }
404
388 -func assertPlanHasUpdateAndRemoveForTargets(t *testing.T, plan chartengine.Plan, updateMetricPrefix, removeMetricPrefix string) {
389 - t.Helper()
390 -
391 - hasUpdate := false
392 - hasRemove := false
393 -
394 - for _, action := range plan.Actions {
395 - switch a := action.(type) {
396 - case chartengine.UpdateChartAction:
397 - if strings.HasPrefix(a.ChartID, updateMetricPrefix) {
398 - hasUpdate = true
399 - }
400 - case chartengine.RemoveDimensionAction:
401 - if strings.HasPrefix(a.ChartID, removeMetricPrefix) {
402 - hasRemove = true
403 - }
404 - case chartengine.RemoveChartAction:
405 - if strings.HasPrefix(a.ChartID, removeMetricPrefix) {
406 - hasRemove = true
407 - }
408 - }
409 - }
410 -
411 - assert.True(t, hasUpdate, "expected update action for %q", updateMetricPrefix)
412 - assert.True(t, hasRemove, "expected remove action for %q", removeMetricPrefix)
413 -}
414 -
405 func assertPlanHasUpdateForTarget(t *testing.T, plan chartengine.Plan, updateMetricPrefix string) {
406 t.Helper()
407 for _, action := range plan.Actions {
src/health/health.d/nagios.conf new
+57
@@ -0,0 +1,57 @@
1 +# you can disable an alarm notification by setting the 'to' line to: silent
2 +
3 +template: nagios_job_execution_state_warn
4 + on: nagios.job.execution_state
5 + class: Errors
6 + type: Other
7 +component: Nagios
8 + calc: $warning - $retry
9 + units: state
10 + every: 10s
11 + warn: $this != nan AND $this == 1
12 + delay: down 1m multiplier 1.5 max 1h
13 + summary: Nagios job ${label:nagios_job} execution state
14 + info: Nagios job ${label:nagios_job} is in WARNING state
15 + to: sysadmin
16 +
17 +template: nagios_job_execution_state_crit
18 + on: nagios.job.execution_state
19 + class: Errors
20 + type: Other
21 +component: Nagios
22 + calc: $critical - $retry
23 + units: state
24 + every: 10s
25 + crit: $this != nan AND $this == 1
26 + delay: down 1m multiplier 1.5 max 1h
27 + summary: Nagios job ${label:nagios_job} execution state
28 + info: Nagios job ${label:nagios_job} is in CRITICAL state
29 + to: sysadmin
30 +
31 +template: nagios_job_perfdata_threshold_state_warn
32 + on: nagios.job.perfdata_threshold_state
33 + class: Errors
34 + type: Other
35 +component: Nagios
36 + calc: $warning - $retry
37 + units: state
38 + every: 10s
39 + warn: $this != nan AND $this == 1
40 + delay: down 1m multiplier 1.5 max 1h
41 + summary: Nagios job ${label:nagios_job} perfdata ${label:perfdata_value} threshold state
42 + info: Nagios job ${label:nagios_job} perfdata ${label:perfdata_value} is in WARNING threshold state
43 + to: sysadmin
44 +
45 +template: nagios_job_perfdata_threshold_state_crit
46 + on: nagios.job.perfdata_threshold_state
47 + class: Errors
48 + type: Other
49 +component: Nagios
50 + calc: $critical - $retry
51 + units: state
52 + every: 10s
53 + crit: $this != nan AND $this == 1
54 + delay: down 1m multiplier 1.5 max 1h
55 + summary: Nagios job ${label:nagios_job} perfdata ${label:perfdata_value} threshold state
56 + info: Nagios job ${label:nagios_job} perfdata ${label:perfdata_value} is in CRITICAL threshold state
57 + to: sysadmin