@cryptotaxi247 / netdata-1 / commits / fbde7e017

feat(go.d/prometheus): add relabeling engine (#22660)

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

Ilya Mashchenko committed Jun 8, 2026 at 22:25 UTC fbde7e01709ee069ee017933af81e841bb944ab3
16 files changed +1868 -415
src/go/go.mod
+1 -1
@@ -78,6 +78,7 @@ require (
78 github.com/catonetworks/cato-go-sdk v0.2.6
79 github.com/cespare/xxhash/v2 v2.3.0
80 github.com/docker/go-units v0.5.0
81 + github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
82 github.com/ibm-messaging/mq-golang/v5 v5.7.1
83 github.com/maxmind/mmdbwriter v1.2.0
84 github.com/microsoft/go-mssqldb v1.10.0
@@ -126,7 +127,6 @@ require (
127 github.com/google/certificate-transparency-go v1.1.7 // indirect
128 github.com/google/gnostic-models v0.7.0 // indirect
129 github.com/google/go-cmp v0.7.0 // indirect
129 - github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
130 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
131 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
132 github.com/hashicorp/go-hclog v1.6.3 // indirect
src/go/pkg/prometheus/client.go
+21 -14
@@ -6,6 +6,7 @@ import (
6 "bufio"
7 "bytes"
8 "compress/gzip"
9 + "context"
10 "fmt"
11 "io"
12 "net/http"
@@ -23,17 +24,18 @@ type (
24 // ScrapeSeries and parse prometheus format metrics
25 ScrapeSeries() (Series, error)
26 Scrape() (MetricFamilies, error)
26 - // ScrapeStream scrapes and invokes onSample for every sample — a flat
27 - // [Sample] stream before typed-family assembly — with the selector (if
28 - // any) already applied; onHelp, if non-nil, receives per-family HELP. It
29 - // stops and returns the first error from a callback.
27 + // ScrapeWithTransform scrapes, runs transform on every sample (a flat
28 + // [Sample] stream before typed-family assembly, with the selector already
29 + // applied), then assembles the kept, transformed samples into MetricFamilies.
30 + // A nil transform behaves exactly like Scrape. The result aliases reused
31 + // buffers, valid until the next scrape on this instance (same as Scrape).
32 //
31 - // Order is exposition order, with one exception: a _sum/_count sample
32 - // whose family type is not yet known (its # TYPE, first bucket, or first
33 - // quantile has not appeared) is deferred and emitted once the type
34 - // resolves, or at end of stream — so it may arrive after a later,
35 - // unrelated sample. Each sample's Kind and FamilyType are always correct.
36 - ScrapeStream(onHelp func(name, help string), onSample func(Sample) error) error
33 + // Samples reach transform in exposition order, with one exception: a
34 + // _sum/_count sample whose family type is not yet known (its # TYPE, first
35 + // bucket, or first quantile has not appeared) is deferred and delivered once
36 + // the type resolves, or at end of scrape — so it may arrive after a later,
37 + // unrelated sample. A per-sample (stateless) transform is unaffected.
38 + ScrapeWithTransform(ctx context.Context, transform SampleTransform) (MetricFamilies, error)
39 HTTPClient() *http.Client
40 }
41
@@ -86,7 +88,7 @@ func (p *prometheus) HTTPClient() *http.Client {
88 func (p *prometheus) ScrapeSeries() (Series, error) {
89 p.buf.Reset()
90
89 - if err := p.fetch(p.buf); err != nil {
91 + if err := p.fetch(context.Background(), p.buf); err != nil {
92 return nil, err
93 }
94
@@ -94,16 +96,20 @@ func (p *prometheus) ScrapeSeries() (Series, error) {
96 }
97
98 func (p *prometheus) Scrape() (MetricFamilies, error) {
99 + return p.ScrapeWithTransform(context.Background(), nil)
100 +}
101 +
102 +func (p *prometheus) ScrapeWithTransform(ctx context.Context, transform SampleTransform) (MetricFamilies, error) {
103 p.buf.Reset()
104
99 - if err := p.fetch(p.buf); err != nil {
105 + if err := p.fetch(ctx, p.buf); err != nil {
106 return nil, err
107 }
108
103 - return p.parser.parseToMetricFamilies(p.buf.Bytes())
109 + return p.parser.parseToMetricFamilies(p.buf.Bytes(), transform)
110 }
111
106 -func (p *prometheus) fetch(w io.Writer) error {
112 +func (p *prometheus) fetch(ctx context.Context, w io.Writer) error {
113 // TODO: should be a separate text file prom client
114 if p.filepath != "" {
115 f, err := os.Open(p.filepath)
@@ -121,6 +127,7 @@ func (p *prometheus) fetch(w io.Writer) error {
127 if err != nil {
128 return err
129 }
130 + req = req.WithContext(ctx)
131
132 req.Header.Add("Accept", acceptHeader)
133 req.Header.Add("Accept-Encoding", "gzip")
src/go/pkg/prometheus/client_test.go
+39 -16
@@ -5,6 +5,7 @@ package prometheus
5 import (
6 "bytes"
7 "compress/gzip"
8 + "context"
9 "errors"
10 "net/http"
11 "net/http/httptest"
@@ -132,38 +133,60 @@ func TestPrometheusReadFromFile(t *testing.T) {
133 }
134 }
135
135 -func TestPrometheusScrapeStream(t *testing.T) {
136 +func TestPrometheusScrapeWithTransform(t *testing.T) {
137 errBoom := errors.New("boom")
138
139 tests := map[string]struct {
139 - onSampleErr error // returned by onSample (nil = stream everything)
140 - wantErr error
140 + transform func(seen *int) SampleTransform
141 + wantErr error
142 + check func(t *testing.T, mfs MetricFamilies, seen int)
143 }{
142 - "streams all samples and help": {},
143 - "onSample error propagates": {onSampleErr: errBoom, wantErr: errBoom},
144 + "nil transform assembles like Scrape": {
145 + transform: func(*int) SampleTransform { return nil },
146 + check: func(t *testing.T, mfs MetricFamilies, seen int) {
147 + assert.Positive(t, mfs.Len())
148 + assert.Zero(t, seen, "nil transform must not be invoked")
149 + },
150 + },
151 + "transform sees every sample and keeps them": {
152 + transform: func(seen *int) SampleTransform {
153 + return func(s Sample) (Sample, bool, error) { *seen++; return s, true, nil }
154 + },
155 + check: func(t *testing.T, mfs MetricFamilies, seen int) {
156 + assert.Positive(t, seen)
157 + assert.Positive(t, mfs.Len())
158 + },
159 + },
160 + "dropping every sample yields no families": {
161 + transform: func(seen *int) SampleTransform {
162 + return func(s Sample) (Sample, bool, error) { *seen++; return s, false, nil }
163 + },
164 + check: func(t *testing.T, mfs MetricFamilies, seen int) {
165 + assert.Positive(t, seen)
166 + assert.Zero(t, mfs.Len())
167 + },
168 + },
169 + "transform error aborts the scrape": {
170 + transform: func(*int) SampleTransform {
171 + return func(s Sample) (Sample, bool, error) { return s, false, errBoom }
172 + },
173 + wantErr: errBoom,
174 + },
175 }
176
177 for name, tc := range tests {
178 t.Run(name, func(t *testing.T) {
179 prom := New(http.DefaultClient, web.RequestConfig{URL: "file://testdata/testdata.txt"})
180
150 - var samples int
151 - var help []string
152 - err := prom.ScrapeStream(
153 - func(name, _ string) { help = append(help, name) },
154 - func(Sample) error {
155 - samples++
156 - return tc.onSampleErr
157 - },
158 - )
181 + var seen int
182 + mfs, err := prom.ScrapeWithTransform(context.Background(), tc.transform(&seen))
183
184 if tc.wantErr != nil {
185 assert.ErrorIs(t, err, tc.wantErr)
186 return
187 }
188 require.NoError(t, err)
165 - assert.Positive(t, samples)
166 - assert.Contains(t, help, "go_gc_duration_seconds")
189 + tc.check(t, mfs, seen)
190 })
191 }
192 }
src/go/pkg/prometheus/doc.go
+4 -2
@@ -10,8 +10,10 @@
10 // into their families.
11 // - [Prometheus.ScrapeSeries] returns the raw [Series]: one [SeriesSample] per
12 // scraped series, with labels in textparse-sorted order.
13 -// - [Prometheus.ScrapeStream] exposes a flat [Sample] stream before typed
14 -// assembly — the form a Prometheus metric-relabeling step operates on.
13 +// - [Prometheus.ScrapeWithTransform] runs a per-sample transform — the form a
14 +// Prometheus metric-relabeling step operates on — over the flat [Sample]
15 +// stream before assembling the kept samples into typed [MetricFamilies]; a
16 +// nil transform behaves like Scrape.
17 //
18 // Results are valid only until the next scrape on the same instance; buffers are
19 // reused across scrapes. An optional selector (see the selector subpackage)
src/go/pkg/prometheus/model.go
+11
@@ -42,3 +42,14 @@ type Sample struct {
42 Kind SampleKind
43 FamilyType model.MetricType
44 }
45 +
46 +// SampleTransform transforms or drops a single scraped Sample before typed-family
47 +// assembly. Return (sample, true, nil) to keep it (optionally mutated — rewrite Name
48 +// or mutate Labels in place), (_, false, nil) to drop it, or a non-nil error to abort
49 +// the scrape. Each Sample owns its Labels, so in-place mutation is safe and does not
50 +// affect other samples. It is the hook a Prometheus metric-relabeling step plugs into.
51 +//
52 +// Kind and FamilyType reflect the classification BEFORE the transform runs; rewriting
53 +// Name, le, or quantile does NOT reclassify the sample (matching Prometheus, where
54 +// relabeling cannot retype a series).
55 +type SampleTransform func(Sample) (Sample, bool, error)
src/go/pkg/prometheus/parse.go
+23 -16
@@ -39,13 +39,31 @@ type promTextParser struct {
39 series Series
40 }
41
42 -func (p *promTextParser) parseToMetricFamilies(text []byte) (MetricFamilies, error) {
42 +func (p *promTextParser) parseToMetricFamilies(text []byte, transform SampleTransform) (MetricFamilies, error) {
43 p.driver.sr = p.sr
44 p.asm.reset()
45
46 - // ownLabels=false: the assembler copies labels into its own buffers, so the
47 - // driver may lend the scratch label set (the no-allocation fast path).
48 - if err := p.driver.parseSamples(text, false, p.asm.applyHelp, p.asm.applySample); err != nil {
46 + // With no transform, ownLabels=false: the assembler copies labels into its own
47 + // buffers, so the driver may lend the scratch label set (the no-allocation fast
48 + // path). A transform may mutate or retain a sample's labels, so it needs the
49 + // sample to own them (ownLabels=true).
50 + onSample := p.asm.applySample
51 + ownLabels := false
52 + if transform != nil {
53 + ownLabels = true
54 + onSample = func(s Sample) error {
55 + s, keep, err := transform(s)
56 + if err != nil {
57 + return err
58 + }
59 + if !keep {
60 + return nil
61 + }
62 + return p.asm.applySample(s)
63 + }
64 + }
65 +
66 + if err := p.driver.parseSamples(text, ownLabels, p.asm.applyHelp, onSample); err != nil {
67 return nil, err
68 }
69
@@ -73,17 +91,6 @@ func (p *promTextParser) parseToSeries(text []byte) (Series, error) {
91 return p.series, nil
92 }
93
76 -func (p *promTextParser) parseToStream(text []byte, onHelp func(name, help string), onSample func(Sample) error) error {
77 - if onSample == nil {
78 - return nil
79 - }
80 - p.driver.sr = p.sr
81 -
82 - // ownLabels=true: each Sample must own its labels because the consumer may
83 - // retain or mutate them (e.g. relabeling) past the next sample.
84 - return p.driver.parseSamples(text, true, onHelp, onSample)
85 -}
86 -
94 // parseDriver runs the single exposition parse pass (iterate). On top of it,
95 // parseSamples emits a flat, classified sample stream, deferring a _sum/_count
96 // whose family type is not yet known and back-resolving it once the type appears
@@ -174,7 +181,7 @@ func (d *parseDriver) iterate(
181 // turns each series into a Sample (Kind + FamilyType) and defers a _sum/_count
182 // whose family type is not yet known, back-resolving it once the type appears (a
183 // later # TYPE, _bucket, or quantile series) or flushing it at EOF. Deferral can
177 -// emit a _sum/_count after a later, unrelated series — see ScrapeStream's doc.
184 +// emit a _sum/_count after a later, unrelated series — see ScrapeWithTransform's doc.
185 func (d *parseDriver) parseSamples(text []byte, ownLabels bool, onHelp func(name, help string), onSample func(Sample) error) error {
186 d.reset()
187
src/go/pkg/prometheus/parse_bench_test.go
+15 -13
@@ -28,20 +28,21 @@ import (
28 // parseToMetricFamilies 127868 ns/op 73404 B/op 1314 allocs/op
29 // parseToSeries 111382 ns/op 100444 B/op 1584 allocs/op
30 //
31 -// Unified driver+assembler stream parser (this rewrite — captured 2026-06-05):
31 +// Unified driver+assembler parser (this rewrite — captured 2026-06-05):
32 //
33 // parseToMetricFamilies 133911 ns/op 73408 B/op 1314 allocs/op
34 // parseToSeries 112803 ns/op 100448 B/op 1584 allocs/op
35 -// parseToStream 113248 ns/op 102807 B/op 1644 allocs/op
35 //
37 -// No allocation win and no allocation regression: the legacy parser already
38 -// parsed once per call and reused buffers, and both Scrape and ScrapeSeries match
39 -// it exactly (flat allocs). ScrapeSeries uses the raw-label path (identical to
40 -// legacy), so it is essentially free. Scrape/stream are a few percent slower on
41 -// CPU — the per-sample Sample struct + the shared iterate indirection + assembler
42 -// dispatch on the families/stream path (profiled: makeSample + applySample,
43 -// inherent to the single-driver model, not a fixable hotspot). The win is the
44 -// sample stream that metric relabeling consumes, not raw speed.
36 +// Scrape (parseToMetricFamilies) and ScrapeSeries keep the legacy allocation profile
37 +// (flat allocs); ScrapeSeries is the raw-label path (identical to legacy, essentially
38 +// free), Scrape a few percent slower on CPU (per-sample Sample + iterate indirection +
39 +// assembler dispatch — makeSample/applySample, inherent to the single-driver model).
40 +//
41 +// A transform runs with ownLabels=true, so each kept sample owns a copy of its labels
42 +// — the cost metric relabeling pays to mutate safely (captured 2026-06-08):
43 +//
44 +// parseToMetricFamilies 142135 ns/op 73412 B/op 1314 allocs/op
45 +// parseToMetricFamiliesWithTransform 147460 ns/op 93940 B/op 1709 allocs/op
46
47 func readBenchData(tb testing.TB) []byte {
48 tb.Helper()
@@ -64,7 +65,7 @@ func BenchmarkPromTextParser_parseToMetricFamilies(b *testing.B) {
65 b.SetBytes(int64(len(data)))
66 b.ResetTimer()
67 for range b.N {
67 - if _, err := p.parseToMetricFamilies(data); err != nil {
68 + if _, err := p.parseToMetricFamilies(data, nil); err != nil {
69 b.Fatal(err)
70 }
71 }
@@ -83,14 +84,15 @@ func BenchmarkPromTextParser_parseToSeries(b *testing.B) {
84 }
85 }
86
86 -func BenchmarkPromTextParser_parseToStream(b *testing.B) {
87 +func BenchmarkPromTextParser_parseToMetricFamiliesWithTransform(b *testing.B) {
88 data := readBenchData(b)
89 var p promTextParser
90 + keep := func(s Sample) (Sample, bool, error) { return s, true, nil }
91 b.ReportAllocs()
92 b.SetBytes(int64(len(data)))
93 b.ResetTimer()
94 for range b.N {
93 - if err := p.parseToStream(data, nil, func(Sample) error { return nil }); err != nil {
95 + if _, err := p.parseToMetricFamilies(data, keep); err != nil {
96 b.Fatal(err)
97 }
98 }
src/go/pkg/prometheus/parse_test.go
+296 -28
@@ -54,6 +54,105 @@ func TestPromTextParser_parseToMetricFamilies(t *testing.T) {
54 input []byte
55 want MetricFamilies
56 }{
57 + "summary _sum/_count before # TYPE fold into one summary": {
58 + input: []byte(`my_summary_sum{label1="value1"} 10
59 +my_summary_count{label1="value1"} 2
60 +# TYPE my_summary summary
61 +my_summary{label1="value1",quantile="0.5"} 0.5
62 +`),
63 + want: MetricFamilies{
64 + "my_summary": {
65 + name: "my_summary",
66 + typ: model.MetricTypeSummary,
67 + metrics: []Metric{
68 + {
69 + labels: labels.Labels{{Name: "label1", Value: "value1"}},
70 + summary: &Summary{
71 + sum: 10,
72 + count: 2,
73 + quantiles: []Quantile{{quantile: 0.5, value: 0.5}},
74 + },
75 + },
76 + },
77 + },
78 + },
79 + },
80 + "histogram _sum/_count before _bucket (no # TYPE) fold into one histogram": {
81 + input: []byte(`my_hist_sum{label1="value1"} 5
82 +my_hist_count{label1="value1"} 3
83 +my_hist_bucket{label1="value1",le="0.1"} 1
84 +my_hist_bucket{label1="value1",le="+Inf"} 3
85 +`),
86 + want: MetricFamilies{
87 + "my_hist": {
88 + name: "my_hist",
89 + typ: model.MetricTypeHistogram,
90 + metrics: []Metric{
91 + {
92 + labels: labels.Labels{{Name: "label1", Value: "value1"}},
93 + histogram: &Histogram{
94 + sum: 5,
95 + count: 3,
96 + buckets: []Bucket{
97 + {upperBound: 0.1, cumulativeCount: 1},
98 + {upperBound: math.Inf(1), cumulativeCount: 3},
99 + },
100 + },
101 + },
102 + },
103 + },
104 + },
105 + },
106 + "metric name found by lookup, not first label": {
107 + input: []byte(`
108 +# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization
109 +# TYPE DCGM_FI_DEV_GPU_UTIL gauge
110 +DCGM_FI_DEV_GPU_UTIL{UUID="GPU-aaa",gpu="0"} 80
111 +`),
112 + want: MetricFamilies{
113 + "DCGM_FI_DEV_GPU_UTIL": {
114 + name: "DCGM_FI_DEV_GPU_UTIL",
115 + help: "GPU utilization",
116 + typ: model.MetricTypeGauge,
117 + metrics: []Metric{
118 + {
119 + labels: labels.Labels{{Name: "UUID", Value: "GPU-aaa"}, {Name: "gpu", Value: "0"}},
120 + gauge: &Gauge{value: 80},
121 + },
122 + },
123 + },
124 + },
125 + },
126 + "valid _bucket (has le) folds into the histogram family": {
127 + input: []byte("# TYPE h histogram\nh_bucket{le=\"1\",label=\"x\"} 1\n"),
128 + want: MetricFamilies{
129 + "h": {
130 + name: "h",
131 + typ: model.MetricTypeHistogram,
132 + metrics: []Metric{
133 + {
134 + labels: labels.Labels{{Name: "label", Value: "x"}},
135 + histogram: &Histogram{buckets: []Bucket{{upperBound: 1, cumulativeCount: 1}}},
136 + },
137 + },
138 + },
139 + },
140 + },
141 + "_bucket without le is a plain metric, not a histogram bucket": {
142 + input: []byte("# TYPE h histogram\nh_bucket{label=\"x\"} 1\n"),
143 + want: MetricFamilies{
144 + "h_bucket": {
145 + name: "h_bucket",
146 + typ: model.MetricTypeUnknown,
147 + metrics: []Metric{
148 + {
149 + labels: labels.Labels{{Name: "label", Value: "x"}},
150 + untyped: &Untyped{value: 1},
151 + },
152 + },
153 + },
154 + },
155 + },
156 "Gauge with multiline HELP": {
157 input: dataMultilineHelp,
158 want: MetricFamilies{
@@ -1347,7 +1446,7 @@ func TestPromTextParser_parseToMetricFamilies(t *testing.T) {
1446
1447 for i := range 10 {
1448 t.Run(fmt.Sprintf("parse num %d", i+1), func(t *testing.T) {
1350 - mfs, err := p.parseToMetricFamilies(test.input)
1449 + mfs, err := p.parseToMetricFamilies(test.input, nil)
1450 if len(test.want) > 0 {
1451 assert.Equal(t, test.want, mfs)
1452 } else {
@@ -1382,7 +1481,7 @@ test_gauge_metric_2{label1="value2"} 1
1481 },
1482 }
1483
1385 - mfs, err := p.parseToMetricFamilies(txt)
1484 + mfs, err := p.parseToMetricFamilies(txt, nil)
1485
1486 require.NoError(t, err)
1487 assert.Equal(t, want, mfs)
@@ -1393,6 +1492,17 @@ func TestPromTextParser_parseToSeries(t *testing.T) {
1492 input []byte
1493 want Series
1494 }{
1495 + "label order matches textparse (__name__ not forced first)": {
1496 + input: []byte("m{UUID=\"x\",gpu=\"0\"} 5\n"),
1497 + want: Series{SeriesSample{
1498 + Labels: labels.Labels{
1499 + {Name: "UUID", Value: "x"},
1500 + {Name: "__name__", Value: "m"},
1501 + {Name: "gpu", Value: "0"},
1502 + },
1503 + Value: 5,
1504 + }},
1505 + },
1506 "All types": {
1507 input: []byte(`
1508 # HELP test_gauge_metric_1 Test Gauge Metric 1
@@ -1665,31 +1775,6 @@ test_gauge_metric_2{label1="value2"} 1
1775 assert.Equal(t, want, series)
1776 }
1777
1668 -func TestPromTextParser_parseToMetricFamilies_metricNameNotFirstLabel(t *testing.T) {
1669 - var p promTextParser
1670 -
1671 - txt := []byte(`
1672 -# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization
1673 -# TYPE DCGM_FI_DEV_GPU_UTIL gauge
1674 -DCGM_FI_DEV_GPU_UTIL{UUID="GPU-aaa",gpu="0"} 80
1675 -`)
1676 -
1677 - mfs, err := p.parseToMetricFamilies(txt)
1678 - require.NoError(t, err)
1679 -
1680 - require.Contains(t, mfs, "DCGM_FI_DEV_GPU_UTIL")
1681 - require.NotContains(t, mfs, "GPU-aaa")
1682 -
1683 - mf := mfs["DCGM_FI_DEV_GPU_UTIL"]
1684 - require.Len(t, mf.metrics, 1)
1685 - assert.Equal(t, model.MetricTypeGauge, mf.typ)
1686 - assert.Equal(t, 80.0, mf.metrics[0].gauge.value)
1687 - assert.EqualValues(t, labels.Labels{
1688 - {Name: "UUID", Value: "GPU-aaa"},
1689 - {Name: "gpu", Value: "0"},
1690 - }, mf.metrics[0].labels)
1691 -}
1692 -
1778 func TestPromTextParser_parseToMetricFamilies_failsOnInvalidSeriesValue(t *testing.T) {
1779 var p promTextParser
1780
@@ -1702,7 +1787,7 @@ DCGM_FI_DEV_GPU_UTIL{UUID="GPU-aaa",gpu="0"} 80
1787 DCGM_FI_DEV_REQUESTED_POWER_PROFILE_MASK{UUID="GPU-aaa",gpu="0"} ERROR - FAILED TO CONVERT TO STRING
1788 `)
1789
1705 - _, err := p.parseToMetricFamilies(txt)
1790 + _, err := p.parseToMetricFamilies(txt, nil)
1791 require.Error(t, err)
1792 assert.Contains(t, err.Error(), "failed to parse prometheus metrics")
1793 }
@@ -1728,3 +1813,186 @@ func joinData(data ...[]byte) []byte {
1813 }
1814 return buf.Bytes()
1815 }
1816 +
1817 +func TestPromTextParser_parseSamples(t *testing.T) {
1818 + type wantSample struct {
1819 + name string
1820 + labels string
1821 + value float64
1822 + kind SampleKind
1823 + familyType model.MetricType
1824 + }
1825 +
1826 + tests := map[string]struct {
1827 + input []byte
1828 + wantHelp []string
1829 + want []wantSample
1830 + }{
1831 + "deferred _sum emits after a later unrelated metric (cross-metric reorder)": {
1832 + input: []byte("a_sum 1\nb 2\n# TYPE a summary\na{quantile=\"0.5\"} 3\n"),
1833 + want: []wantSample{
1834 + {"b", `{}`, 2, SampleKindScalar, model.MetricTypeUnknown},
1835 + {"a_sum", `{}`, 1, SampleKindSummarySum, model.MetricTypeSummary},
1836 + {"a", `{quantile="0.5"}`, 3, SampleKindSummaryQuantile, model.MetricTypeSummary},
1837 + },
1838 + },
1839 + "all metric types are classified in exposition order": {
1840 + input: []byte(`# HELP test_gauge A gauge metric.
1841 +# TYPE test_gauge gauge
1842 +test_gauge{label="a"} 1
1843 +# TYPE test_counter_total counter
1844 +test_counter_total 5
1845 +# TYPE test_hist histogram
1846 +test_hist_bucket{le="0.1"} 1
1847 +test_hist_bucket{le="+Inf"} 2
1848 +test_hist_sum 3
1849 +test_hist_count 2
1850 +# TYPE test_summary summary
1851 +test_summary{quantile="0.5"} 0.2
1852 +test_summary_sum 1
1853 +test_summary_count 10
1854 +`),
1855 + wantHelp: []string{"test_gauge=A gauge metric."},
1856 + want: []wantSample{
1857 + {"test_gauge", `{label="a"}`, 1, SampleKindScalar, model.MetricTypeGauge},
1858 + {"test_counter_total", `{}`, 5, SampleKindScalar, model.MetricTypeCounter},
1859 + {"test_hist_bucket", `{le="0.1"}`, 1, SampleKindHistogramBucket, model.MetricTypeHistogram},
1860 + {"test_hist_bucket", `{le="+Inf"}`, 2, SampleKindHistogramBucket, model.MetricTypeHistogram},
1861 + {"test_hist_sum", `{}`, 3, SampleKindHistogramSum, model.MetricTypeHistogram},
1862 + {"test_hist_count", `{}`, 2, SampleKindHistogramCount, model.MetricTypeHistogram},
1863 + {"test_summary", `{quantile="0.5"}`, 0.2, SampleKindSummaryQuantile, model.MetricTypeSummary},
1864 + {"test_summary_sum", `{}`, 1, SampleKindSummarySum, model.MetricTypeSummary},
1865 + {"test_summary_count", `{}`, 10, SampleKindSummaryCount, model.MetricTypeSummary},
1866 + },
1867 + },
1868 + "__name__ is delivered via Name and excluded from Labels": {
1869 + input: []byte("# TYPE m gauge\nm{a=\"1\",b=\"2\"} 7\n"),
1870 + want: []wantSample{
1871 + {"m", `{a="1", b="2"}`, 7, SampleKindScalar, model.MetricTypeGauge},
1872 + },
1873 + },
1874 + "deferred sum/count before # TYPE are buffered and back-resolved": {
1875 + input: []byte(`my_summary_sum 10
1876 +my_summary_count 2
1877 +# TYPE my_summary summary
1878 +my_summary{quantile="0.5"} 0.5
1879 +`),
1880 + want: []wantSample{
1881 + {"my_summary_sum", `{}`, 10, SampleKindSummarySum, model.MetricTypeSummary},
1882 + {"my_summary_count", `{}`, 2, SampleKindSummaryCount, model.MetricTypeSummary},
1883 + {"my_summary", `{quantile="0.5"}`, 0.5, SampleKindSummaryQuantile, model.MetricTypeSummary},
1884 + },
1885 + },
1886 + }
1887 +
1888 + for name, test := range tests {
1889 + t.Run(name, func(t *testing.T) {
1890 + var p promTextParser
1891 +
1892 + for i := range 10 {
1893 + t.Run(fmt.Sprintf("parse num %d", i+1), func(t *testing.T) {
1894 + var got []wantSample
1895 + var help []string
1896 +
1897 + err := p.driver.parseSamples(test.input, true,
1898 + func(name, h string) { help = append(help, name+"="+h) },
1899 + func(s Sample) error {
1900 + assert.Falsef(t, s.Labels.Has(labels.MetricName),
1901 + "sample %q must not carry __name__ in Labels", s.Name)
1902 + got = append(got, wantSample{s.Name, s.Labels.String(), s.Value, s.Kind, s.FamilyType})
1903 + return nil
1904 + },
1905 + )
1906 + require.NoError(t, err)
1907 + assert.Equal(t, test.want, got)
1908 + for _, h := range test.wantHelp {
1909 + assert.Contains(t, help, h)
1910 + }
1911 + })
1912 + }
1913 + })
1914 + }
1915 +}
1916 +
1917 +// The stream supports Prometheus-style relabeling on __name__/le/quantile.
1918 +// ownLabels=true isolates each sample's labels, so a transform can rename via
1919 +// Name and mutate Labels in place without affecting later samples.
1920 +func TestPromTextParser_parseSamples_relabelStyle(t *testing.T) {
1921 + data := []byte(`# TYPE req_seconds histogram
1922 +req_seconds_bucket{le="0.1",path="/a"} 1
1923 +req_seconds_bucket{le="+Inf",path="/a"} 3
1924 +# TYPE rpc summary
1925 +rpc{quantile="0.99",path="/a"} 0.5
1926 +`)
1927 +
1928 + type out struct {
1929 + name string
1930 + le string
1931 + quantile string
1932 + labels string
1933 + }
1934 +
1935 + var p promTextParser
1936 + var got []out
1937 + err := p.driver.parseSamples(data, true, nil, func(s Sample) error {
1938 + o := out{
1939 + name: s.Name + ":relabeled", // __name__ is mutable via Name
1940 + le: s.Labels.Get(bucketLabel),
1941 + quantile: s.Labels.Get(quantileLabel),
1942 + }
1943 + // Drop the "path" target label in place (this sample owns its labels).
1944 + kept := s.Labels[:0]
1945 + for _, l := range s.Labels {
1946 + if l.Name == "path" {
1947 + continue
1948 + }
1949 + kept = append(kept, l)
1950 + }
1951 + o.labels = labels.Labels(kept).String()
1952 + got = append(got, o)
1953 + return nil
1954 + })
1955 + require.NoError(t, err)
1956 +
1957 + assert.Equal(t, []out{
1958 + {name: "req_seconds_bucket:relabeled", le: "0.1", quantile: "", labels: `{le="0.1"}`},
1959 + {name: "req_seconds_bucket:relabeled", le: "+Inf", quantile: "", labels: `{le="+Inf"}`},
1960 + {name: "rpc:relabeled", le: "", quantile: "0.99", labels: `{quantile="0.99"}`},
1961 + }, got)
1962 +}
1963 +
1964 +// A transform's mutations — a renamed Name and changed Labels — are what the
1965 +// assembler folds: parseToMetricFamilies must assemble the TRANSFORMED sample, not
1966 +// the original. Mutating Labels in place also exercises ownLabels=true.
1967 +func TestPromTextParser_parseToMetricFamilies_transformMutatesAssembledOutput(t *testing.T) {
1968 + input := []byte("# TYPE old_name gauge\nold_name{keep=\"yes\",drop=\"me\"} 42\n")
1969 +
1970 + // Rename the metric and drop the "drop" label in place (the sample owns its labels).
1971 + transform := func(s Sample) (Sample, bool, error) {
1972 + s.Name = "new_name"
1973 + kept := s.Labels[:0]
1974 + for _, l := range s.Labels {
1975 + if l.Name == "drop" {
1976 + continue
1977 + }
1978 + kept = append(kept, l)
1979 + }
1980 + s.Labels = kept
1981 + return s, true, nil
1982 + }
1983 +
1984 + want := MetricFamilies{
1985 + "new_name": {
1986 + name: "new_name",
1987 + typ: model.MetricTypeGauge,
1988 + metrics: []Metric{
1989 + {labels: labels.Labels{{Name: "keep", Value: "yes"}}, gauge: &Gauge{value: 42}},
1990 + },
1991 + },
1992 + }
1993 +
1994 + var p promTextParser
1995 + mfs, err := p.parseToMetricFamilies(input, transform)
1996 + require.NoError(t, err)
1997 + assert.Equal(t, want, mfs)
1998 +}
src/go/pkg/prometheus/stream.go deleted
-15
@@ -1,15 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package prometheus
4 -
5 -// ScrapeStream implements [Prometheus]. See the interface for the ordering and
6 -// callback contract. onHelp may be nil when per-family HELP is not needed.
7 -func (p *prometheus) ScrapeStream(onHelp func(name, help string), onSample func(Sample) error) error {
8 - p.buf.Reset()
9 -
10 - if err := p.fetch(p.buf); err != nil {
11 - return err
12 - }
13 -
14 - return p.parser.parseToStream(p.buf.Bytes(), onHelp, onSample)
15 -}
src/go/pkg/prometheus/stream_test.go deleted
-296
@@ -1,296 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package prometheus
4 -
5 -import (
6 - "fmt"
7 - "math"
8 - "testing"
9 -
10 - "github.com/prometheus/common/model"
11 - "github.com/prometheus/prometheus/model/labels"
12 - "github.com/stretchr/testify/assert"
13 - "github.com/stretchr/testify/require"
14 -)
15 -
16 -func TestPromTextParser_parseToStream(t *testing.T) {
17 - type wantSample struct {
18 - name string
19 - labels string
20 - value float64
21 - kind SampleKind
22 - familyType model.MetricType
23 - }
24 -
25 - tests := map[string]struct {
26 - input []byte
27 - wantHelp []string
28 - want []wantSample
29 - }{
30 - "all metric types are classified in exposition order": {
31 - input: []byte(`# HELP test_gauge A gauge metric.
32 -# TYPE test_gauge gauge
33 -test_gauge{label="a"} 1
34 -# TYPE test_counter_total counter
35 -test_counter_total 5
36 -# TYPE test_hist histogram
37 -test_hist_bucket{le="0.1"} 1
38 -test_hist_bucket{le="+Inf"} 2
39 -test_hist_sum 3
40 -test_hist_count 2
41 -# TYPE test_summary summary
42 -test_summary{quantile="0.5"} 0.2
43 -test_summary_sum 1
44 -test_summary_count 10
45 -`),
46 - wantHelp: []string{"test_gauge=A gauge metric."},
47 - want: []wantSample{
48 - {"test_gauge", `{label="a"}`, 1, SampleKindScalar, model.MetricTypeGauge},
49 - {"test_counter_total", `{}`, 5, SampleKindScalar, model.MetricTypeCounter},
50 - {"test_hist_bucket", `{le="0.1"}`, 1, SampleKindHistogramBucket, model.MetricTypeHistogram},
51 - {"test_hist_bucket", `{le="+Inf"}`, 2, SampleKindHistogramBucket, model.MetricTypeHistogram},
52 - {"test_hist_sum", `{}`, 3, SampleKindHistogramSum, model.MetricTypeHistogram},
53 - {"test_hist_count", `{}`, 2, SampleKindHistogramCount, model.MetricTypeHistogram},
54 - {"test_summary", `{quantile="0.5"}`, 0.2, SampleKindSummaryQuantile, model.MetricTypeSummary},
55 - {"test_summary_sum", `{}`, 1, SampleKindSummarySum, model.MetricTypeSummary},
56 - {"test_summary_count", `{}`, 10, SampleKindSummaryCount, model.MetricTypeSummary},
57 - },
58 - },
59 - "__name__ is delivered via Name and excluded from Labels": {
60 - input: []byte("# TYPE m gauge\nm{a=\"1\",b=\"2\"} 7\n"),
61 - want: []wantSample{
62 - {"m", `{a="1", b="2"}`, 7, SampleKindScalar, model.MetricTypeGauge},
63 - },
64 - },
65 - "deferred sum/count before # TYPE are buffered and back-resolved": {
66 - input: []byte(`my_summary_sum 10
67 -my_summary_count 2
68 -# TYPE my_summary summary
69 -my_summary{quantile="0.5"} 0.5
70 -`),
71 - want: []wantSample{
72 - {"my_summary_sum", `{}`, 10, SampleKindSummarySum, model.MetricTypeSummary},
73 - {"my_summary_count", `{}`, 2, SampleKindSummaryCount, model.MetricTypeSummary},
74 - {"my_summary", `{quantile="0.5"}`, 0.5, SampleKindSummaryQuantile, model.MetricTypeSummary},
75 - },
76 - },
77 - }
78 -
79 - for name, test := range tests {
80 - t.Run(name, func(t *testing.T) {
81 - var p promTextParser
82 -
83 - for i := range 10 {
84 - t.Run(fmt.Sprintf("parse num %d", i+1), func(t *testing.T) {
85 - var got []wantSample
86 - var help []string
87 -
88 - err := p.parseToStream(test.input,
89 - func(name, h string) { help = append(help, name+"="+h) },
90 - func(s Sample) error {
91 - assert.Falsef(t, s.Labels.Has(labels.MetricName),
92 - "sample %q must not carry __name__ in Labels", s.Name)
93 - got = append(got, wantSample{s.Name, s.Labels.String(), s.Value, s.Kind, s.FamilyType})
94 - return nil
95 - },
96 - )
97 - require.NoError(t, err)
98 - assert.Equal(t, test.want, got)
99 - for _, h := range test.wantHelp {
100 - assert.Contains(t, help, h)
101 - }
102 - })
103 - }
104 - })
105 - }
106 -}
107 -
108 -// Deferred _sum/_count (emitted before the family type is known) fold into the
109 -// typed family once it is resolved — criterion #5 at the assembled level.
110 -func TestPromTextParser_parseToMetricFamilies_deferredClassification(t *testing.T) {
111 - tests := map[string]struct {
112 - input []byte
113 - want MetricFamilies
114 - }{
115 - "summary _sum/_count before # TYPE fold into one summary": {
116 - input: []byte(`my_summary_sum{label1="value1"} 10
117 -my_summary_count{label1="value1"} 2
118 -# TYPE my_summary summary
119 -my_summary{label1="value1",quantile="0.5"} 0.5
120 -`),
121 - want: MetricFamilies{
122 - "my_summary": {
123 - name: "my_summary",
124 - typ: model.MetricTypeSummary,
125 - metrics: []Metric{
126 - {
127 - labels: labels.Labels{{Name: "label1", Value: "value1"}},
128 - summary: &Summary{
129 - sum: 10,
130 - count: 2,
131 - quantiles: []Quantile{{quantile: 0.5, value: 0.5}},
132 - },
133 - },
134 - },
135 - },
136 - },
137 - },
138 - "histogram _sum/_count before _bucket (no # TYPE) fold into one histogram": {
139 - input: []byte(`my_hist_sum{label1="value1"} 5
140 -my_hist_count{label1="value1"} 3
141 -my_hist_bucket{label1="value1",le="0.1"} 1
142 -my_hist_bucket{label1="value1",le="+Inf"} 3
143 -`),
144 - want: MetricFamilies{
145 - "my_hist": {
146 - name: "my_hist",
147 - typ: model.MetricTypeHistogram,
148 - metrics: []Metric{
149 - {
150 - labels: labels.Labels{{Name: "label1", Value: "value1"}},
151 - histogram: &Histogram{
152 - sum: 5,
153 - count: 3,
154 - buckets: []Bucket{
155 - {upperBound: 0.1, cumulativeCount: 1},
156 - {upperBound: math.Inf(1), cumulativeCount: 3},
157 - },
158 - },
159 - },
160 - },
161 - },
162 - },
163 - },
164 - }
165 -
166 - for name, test := range tests {
167 - t.Run(name, func(t *testing.T) {
168 - var p promTextParser
169 -
170 - for i := range 10 {
171 - t.Run(fmt.Sprintf("parse num %d", i+1), func(t *testing.T) {
172 - mfs, err := p.parseToMetricFamilies(test.input)
173 - require.NoError(t, err)
174 - assert.Equal(t, test.want, mfs)
175 - })
176 - }
177 - })
178 - }
179 -}
180 -
181 -// The stream supports Prometheus-style relabeling on __name__/le/quantile.
182 -// ownLabels=true isolates each sample's labels, so a transform can rename via
183 -// Name and mutate Labels in place without affecting later samples.
184 -func TestPromTextParser_parseToStream_relabelStyle(t *testing.T) {
185 - data := []byte(`# TYPE req_seconds histogram
186 -req_seconds_bucket{le="0.1",path="/a"} 1
187 -req_seconds_bucket{le="+Inf",path="/a"} 3
188 -# TYPE rpc summary
189 -rpc{quantile="0.99",path="/a"} 0.5
190 -`)
191 -
192 - type out struct {
193 - name string
194 - le string
195 - quantile string
196 - labels string
197 - }
198 -
199 - var p promTextParser
200 - var got []out
201 - err := p.parseToStream(data, nil, func(s Sample) error {
202 - o := out{
203 - name: s.Name + ":relabeled", // __name__ is mutable via Name
204 - le: s.Labels.Get(bucketLabel),
205 - quantile: s.Labels.Get(quantileLabel),
206 - }
207 - // Drop the "path" target label in place (this sample owns its labels).
208 - kept := s.Labels[:0]
209 - for _, l := range s.Labels {
210 - if l.Name == "path" {
211 - continue
212 - }
213 - kept = append(kept, l)
214 - }
215 - o.labels = labels.Labels(kept).String()
216 - got = append(got, o)
217 - return nil
218 - })
219 - require.NoError(t, err)
220 -
221 - assert.Equal(t, []out{
222 - {name: "req_seconds_bucket:relabeled", le: "0.1", quantile: "", labels: `{le="0.1"}`},
223 - {name: "req_seconds_bucket:relabeled", le: "+Inf", quantile: "", labels: `{le="+Inf"}`},
224 - {name: "rpc:relabeled", le: "", quantile: "0.99", labels: `{quantile="0.99"}`},
225 - }, got)
226 -}
227 -
228 -func TestPromTextParser_parseToStream_nilCallbackIsNoop(t *testing.T) {
229 - var p promTextParser
230 - require.NoError(t, p.parseToStream([]byte("metric 1\n"), nil, nil))
231 -}
232 -
233 -// Byte-identical series ordering: textparse sorts labels, so __name__ is NOT
234 -// always first — a label like "UUID" (0x55) sorts before "__name__" (0x5f).
235 -// ScrapeSeries must preserve that raw sorted order (same as the legacy parser),
236 -// not force __name__ first, which would also break the labels.Labels sorted
237 -// invariant.
238 -func TestPromTextParser_parseToSeries_labelOrderMatchesTextparse(t *testing.T) {
239 - var p promTextParser
240 - series, err := p.parseToSeries([]byte("m{UUID=\"x\",gpu=\"0\"} 5\n"))
241 - require.NoError(t, err)
242 - require.Len(t, series, 1)
243 - assert.Equal(t, `{UUID="x", __name__="m", gpu="0"}`, series[0].Labels.String())
244 -}
245 -
246 -// The deferred _sum/_count buffer (criterion #5) can emit a _sum after a later,
247 -// unrelated sample: here a_sum is buffered (type unknown), b is emitted, then
248 -// a_sum resolves on "# TYPE a summary". Documents the ScrapeStream order caveat.
249 -func TestPromTextParser_parseToStream_deferralReordersAcrossMetrics(t *testing.T) {
250 - data := []byte("a_sum 1\nb 2\n# TYPE a summary\na{quantile=\"0.5\"} 3\n")
251 -
252 - var p promTextParser
253 - var names []string
254 - err := p.parseToStream(data, nil, func(s Sample) error {
255 - names = append(names, s.Name)
256 - return nil
257 - })
258 - require.NoError(t, err)
259 - assert.Equal(t, []string{"b", "a_sum", "a"}, names)
260 -}
261 -
262 -// A _bucket-named series is a histogram bucket only if it carries an "le" label.
263 -// Without le it is malformed and is kept as a plain metric (value preserved) — not
264 -// folded into the histogram family. (Legacy folded it and dropped the value;
265 -// valid buckets always have le, so real histograms are unaffected.)
266 -func TestPromTextParser_parseToMetricFamilies_bucketRequiresLe(t *testing.T) {
267 - tests := map[string]struct {
268 - input []byte
269 - wantFam string
270 - wantType model.MetricType
271 - }{
272 - "valid _bucket (has le) folds into the histogram family": {
273 - input: []byte("# TYPE h histogram\nh_bucket{le=\"1\",label=\"x\"} 1\n"),
274 - wantFam: "h",
275 - wantType: model.MetricTypeHistogram,
276 - },
277 - "_bucket without le is a plain metric, not a histogram bucket": {
278 - input: []byte("# TYPE h histogram\nh_bucket{label=\"x\"} 1\n"),
279 - wantFam: "h_bucket",
280 - wantType: model.MetricTypeUnknown,
281 - },
282 - }
283 -
284 - for name, tc := range tests {
285 - t.Run(name, func(t *testing.T) {
286 - var p promTextParser
287 - mfs, err := p.parseToMetricFamilies(tc.input)
288 - require.NoError(t, err)
289 -
290 - mf := mfs.Get(tc.wantFam)
291 - require.NotNilf(t, mf, "expected family %q", tc.wantFam)
292 - assert.Equal(t, tc.wantType, mf.Type())
293 - assert.Len(t, mf.Metrics(), 1)
294 - })
295 - }
296 -}
src/go/plugin/go.d/collector/prometheus/collect.go
+17 -6
@@ -3,17 +3,19 @@
3 package prometheus
4
5 import (
6 + "context"
7 "fmt"
8 "strings"
9
10 "github.com/netdata/netdata/go/plugins/pkg/prometheus"
11 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/prometheus/relabel"
12 )
13
14 // collect scrapes the endpoint and writes the metric families to the metrix store. The
15 // store cycle (begin/commit) is driven by the framework around Collect, so this only
16 // writes observations and returns an error to abort the cycle.
15 -func (c *Collector) collect() error {
16 - mfs, err := c.scrape()
17 +func (c *Collector) collect(ctx context.Context) error {
18 + mfs, err := c.scrape(ctx)
19 if err != nil {
20 return err
21 }
@@ -24,8 +26,8 @@ func (c *Collector) collect() error {
26 // check probes the endpoint and enforces the startup gates the V1 collector applied once:
27 // the expected-prefix guard and the total time-series limit. Unlike V1 these are read-only
28 // (V1 mutated Config to make them one-shot); they run only at Check, i.e. autodetection.
27 -func (c *Collector) check() error {
28 - mfs, err := c.scrape()
29 +func (c *Collector) check(ctx context.Context) error {
30 + mfs, err := c.scrape(ctx)
31 if err != nil {
32 return err
33 }
@@ -45,8 +47,8 @@ func (c *Collector) check() error {
47
48 // scrape fetches the endpoint and enforces the empty-scrape contract: an empty scrape is
49 // an error (endpoint down or exposing nothing), not silent no-data.
48 -func (c *Collector) scrape() (prometheus.MetricFamilies, error) {
49 - mfs, err := c.prom.Scrape()
50 +func (c *Collector) scrape(ctx context.Context) (prometheus.MetricFamilies, error) {
51 + mfs, err := c.prom.ScrapeWithTransform(ctx, c.relabelTransform)
52 if err != nil {
53 return nil, err
54 }
@@ -72,3 +74,12 @@ func calcMetrics(mfs prometheus.MetricFamilies) int {
74 }
75 return n
76 }
77 +
78 +// onRelabelDrop logs why a relabel rule dropped a sample, at debug level. It logs
79 +// the metric name and the rule outcome, never label values (cardinality/PII).
80 +func (c *Collector) onRelabelDrop(s prometheus.Sample, d relabel.DropInfo) {
81 + c.When(d.RuleIndex >= 0).
82 + Debugf("relabel dropped metric %q: %s (rule %d, action %q)", s.Name, d.Reason, d.RuleIndex, d.Action).
83 + Else().
84 + Debugf("relabel dropped metric %q: %s", s.Name, d.Reason)
85 +}
src/go/plugin/go.d/collector/prometheus/collector.go
+20 -8
@@ -14,6 +14,7 @@ import (
14 "github.com/netdata/netdata/go/plugins/pkg/prometheus/selector"
15 "github.com/netdata/netdata/go/plugins/pkg/web"
16 "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi"
17 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/prometheus/relabel"
18 )
19
20 //go:embed "config_schema.json"
@@ -67,10 +68,12 @@ type Collector struct {
68 collectorapi.Base
69 Config `yaml:",inline" json:""`
70
70 - prom prometheus.Prometheus
71 - store metrix.CollectorStore
72 - writer *metricFamilyWriter
73 - chartTemplate string
71 + prom prometheus.Prometheus
72 + relabelConfigs []relabel.Config
73 + relabelTransform prometheus.SampleTransform
74 + store metrix.CollectorStore
75 + writer *metricFamilyWriter
76 + chartTemplate string
77 }
78
79 func (c *Collector) Configuration() any {
@@ -88,6 +91,15 @@ func (c *Collector) Init(context.Context) error {
91 }
92 c.prom = prom
93
94 + // relabelConfigs are empty in this PR (rules are set by tests; profiles populate
95 + // them later), so NewTransform returns a nil transform and Scrape keeps its
96 + // no-transform fast path. A non-empty, invalid rule set fails Init here.
97 + transform, err := relabel.NewTransform(c.relabelConfigs, c.onRelabelDrop)
98 + if err != nil {
99 + return fmt.Errorf("init relabel: %v", err)
100 + }
101 + c.relabelTransform = transform
102 +
103 gaugeFallback, err := c.initFallbackTypeMatcher(c.FallbackType.Gauge)
104 if err != nil {
105 return fmt.Errorf("init gauge fallback type matcher: %v", err)
@@ -113,12 +125,12 @@ func (c *Collector) Init(context.Context) error {
125 return nil
126 }
127
116 -func (c *Collector) Check(context.Context) error {
117 - return c.check()
128 +func (c *Collector) Check(ctx context.Context) error {
129 + return c.check(ctx)
130 }
131
120 -func (c *Collector) Collect(context.Context) error {
121 - return c.collect()
132 +func (c *Collector) Collect(ctx context.Context) error {
133 + return c.collect(ctx)
134 }
135
136 func (c *Collector) Cleanup(context.Context) {
src/go/plugin/go.d/collector/prometheus/collector_test.go
+59
@@ -15,6 +15,7 @@ import (
15 "github.com/netdata/netdata/go/plugins/pkg/metrix"
16 "github.com/netdata/netdata/go/plugins/pkg/prometheus/selector"
17 "github.com/netdata/netdata/go/plugins/pkg/web"
18 + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/prometheus/relabel"
19 "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/collecttest"
20 )
21
@@ -350,6 +351,64 @@ test_gauge_metric{label1="value2"} 12
351 assert.False(t, ok, "a family over the per-metric series limit must be skipped entirely")
352 },
353 },
354 + "relabel applies before assembly (drop + rename via __name__)": {
355 + prepare: func() *Collector {
356 + c := New()
357 + c.relabelConfigs = []relabel.Config{
358 + {
359 + SourceLabels: []string{"__name__"},
360 + Regex: relabel.MustNewRegexp("test_drop_me"),
361 + Action: relabel.Drop,
362 + },
363 + {
364 + SourceLabels: []string{"__name__"},
365 + Regex: relabel.MustNewRegexp("test_(.+)"),
366 + TargetLabel: "__name__",
367 + Replacement: "renamed_${1}",
368 + Action: relabel.Replace,
369 + },
370 + }
371 + return c
372 + },
373 + input: `
374 +# TYPE test_keep gauge
375 +test_keep{label1="value1"} 11
376 +# TYPE test_drop_me gauge
377 +test_drop_me{label1="value1"} 22
378 +`,
379 + want: func(t *testing.T, fr metrix.Reader) {
380 + assert.InDelta(t, 11, value(t, fr, "renamed_keep", metrix.Labels{"label1": "value1"}), 1e-9)
381 + _, ok := fr.Value("test_keep", metrix.Labels{"label1": "value1"})
382 + assert.False(t, ok, "the renamed metric must not appear under its original name")
383 + _, ok = fr.Value("test_drop_me", metrix.Labels{"label1": "value1"})
384 + assert.False(t, ok, "the drop rule must drop test_drop_me before assembly")
385 + _, ok = fr.Value("renamed_drop_me", metrix.Labels{"label1": "value1"})
386 + assert.False(t, ok, "a dropped sample must not be renamed or assembled")
387 + },
388 + },
389 + "relabel rewrites a regular label (copy via Replace)": {
390 + prepare: func() *Collector {
391 + c := New()
392 + c.relabelConfigs = []relabel.Config{
393 + {
394 + SourceLabels: []string{"method"},
395 + Regex: relabel.MustNewRegexp("(.+)"),
396 + TargetLabel: "verb",
397 + Replacement: "${1}",
398 + Action: relabel.Replace,
399 + },
400 + }
401 + return c
402 + },
403 + input: `
404 +# TYPE test_requests_total counter
405 +test_requests_total{method="get"} 5
406 +`,
407 + want: func(t *testing.T, fr metrix.Reader) {
408 + // Replace copies method -> verb; the series carries both labels.
409 + assert.InDelta(t, 5, value(t, fr, "test_requests_total", metrix.Labels{"method": "get", "verb": "get"}), 1e-9)
410 + },
411 + },
412 }
413
414 for name, tc := range tests {
src/go/plugin/go.d/collector/prometheus/relabel/relabel.go new
+567
@@ -0,0 +1,567 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +// Package relabel applies Prometheus-compatible metric-relabeling rules to
4 +// scraped samples (the metric name plus labels, including le/quantile) before
5 +// typed-family assembly. It is collector-local to the prometheus collector.
6 +package relabel
7 +
8 +import (
9 + "crypto/md5"
10 + "encoding/binary"
11 + "errors"
12 + "fmt"
13 + "strconv"
14 + "strings"
15 +
16 + "github.com/grafana/regexp"
17 + commonmodel "github.com/prometheus/common/model"
18 + "github.com/prometheus/prometheus/model/labels"
19 +
20 + prompkg "github.com/netdata/netdata/go/plugins/pkg/prometheus"
21 +)
22 +
23 +var (
24 + relabelTargetLegacy = regexp.MustCompile(`^(?:(?:[a-zA-Z_]|\$(?:\{\w+\}|\w+))+\w*)+$`)
25 +
26 + defaultConfig = Config{
27 + Action: Replace,
28 + Separator: ";",
29 + Regex: MustNewRegexp("(.*)"),
30 + Replacement: "$1",
31 + }
32 +)
33 +
34 +// defaultNameValidationScheme is the name-validation scheme applied when a rule
35 +// does not set one. It is UTF-8: relabeling may legitimately produce dotted or
36 +// otherwise non-legacy metric and label names, and only an empty name is
37 +// rejected. A rule may still opt into commonmodel.LegacyValidation via
38 +// Config.NameScheme.
39 +const defaultNameValidationScheme = commonmodel.UTF8Validation
40 +
41 +// Action is the operation a relabel rule performs on a sample's labels and metric
42 +// name. It mirrors Prometheus's relabel actions; New canonicalizes the value, so it
43 +// is case-insensitive. "Joined value" below means the SourceLabels values joined by
44 +// Separator.
45 +type Action string
46 +
47 +const (
48 + // Replace sets TargetLabel from the regex match of the joined value (Replacement
49 + // is the template; an empty result deletes the target label).
50 + Replace Action = "replace"
51 + // Keep keeps the sample only when Regex matches the joined value.
52 + Keep Action = "keep"
53 + // Drop drops the sample when Regex matches the joined value.
54 + Drop Action = "drop"
55 + // KeepEqual keeps the sample only when TargetLabel equals the joined value.
56 + KeepEqual Action = "keepequal"
57 + // DropEqual drops the sample when TargetLabel equals the joined value.
58 + DropEqual Action = "dropequal"
59 + // HashMod sets TargetLabel to the MD5 of the joined value modulo Modulus.
60 + HashMod Action = "hashmod"
61 + // LabelMap copies each label whose name matches Regex to a new name from Replacement.
62 + LabelMap Action = "labelmap"
63 + // LabelDrop removes every label whose name matches Regex.
64 + LabelDrop Action = "labeldrop"
65 + // LabelKeep removes every label whose name does not match Regex.
66 + LabelKeep Action = "labelkeep"
67 + // Lowercase sets TargetLabel to the lowercased joined value.
68 + Lowercase Action = "lowercase"
69 + // Uppercase sets TargetLabel to the uppercased joined value.
70 + Uppercase Action = "uppercase"
71 +)
72 +
73 +// DropReason explains why Apply dropped a sample, for the caller to log.
74 +type DropReason string
75 +
76 +const (
77 + DropReasonNone DropReason = ""
78 + DropReasonDropRuleMatched DropReason = "drop rule matched"
79 + DropReasonKeepRuleMismatch DropReason = "keep rule did not match"
80 + DropReasonDropEqualMatched DropReason = "dropequal rule matched"
81 + DropReasonKeepEqualMismatch DropReason = "keepequal rule did not match"
82 + DropReasonInvalidMetricName DropReason = "resulting metric name is empty or invalid"
83 +)
84 +
85 +// DropInfo is the outcome of Apply. Dropped reports whether the sample was
86 +// dropped; when it was, Reason says why and, for a rule-driven drop, RuleIndex
87 +// and Action identify the rule. RuleIndex is -1 when the drop is not tied to a
88 +// single rule (an invalid final metric name).
89 +type DropInfo struct {
90 + Reason DropReason
91 + RuleIndex int
92 + Action Action
93 +}
94 +
95 +// Dropped reports whether the sample was dropped.
96 +func (d DropInfo) Dropped() bool { return d.Reason != DropReasonNone }
97 +
98 +// DropObserver is called by the SampleTransform returned from NewTransform for
99 +// each dropped sample. Implementations SHOULD log the reason/rule/action but
100 +// MUST NOT log label values (cardinality and privacy).
101 +type DropObserver func(sample prompkg.Sample, drop DropInfo)
102 +
103 +// Config is one relabeling rule. Construct it directly using the Action constants
104 +// and exported fields; rules are validated (and the Action canonicalized) by New.
105 +//
106 +// The unexported separatorSet/replacementSet/sourceLabelsSet fields distinguish an
107 +// explicitly-empty field from an unset one (they are read by withDefaults, validate
108 +// and applyReplace). They are settable only within this package; callers outside the
109 +// package cannot express explicit-empty Separator/Replacement/SourceLabels via normal
110 +// struct literals or standard YAML/JSON unmarshaling (unexported fields are ignored),
111 +// so a dedicated config loader must set them when that behavior is needed.
112 +type Config struct {
113 + SourceLabels []string
114 + Separator string
115 + Regex Regexp
116 + Modulus uint64
117 + TargetLabel string
118 + Replacement string
119 + Action Action
120 + NameScheme commonmodel.ValidationScheme
121 +
122 + separatorSet bool
123 + replacementSet bool
124 + sourceLabelsSet bool
125 +}
126 +
127 +// Regexp is a relabel regular expression: a regexp.Regexp compiled fully anchored
128 +// (see NewRegexp). The zero value has no pattern; build one with NewRegexp or
129 +// MustNewRegexp. String returns the original, un-anchored source.
130 +type Regexp struct {
131 + *regexp.Regexp
132 + original string // un-anchored source passed to NewRegexp; returned by String
133 +}
134 +
135 +// Processor applies an ordered list of rules to samples. It reuses internal
136 +// buffers across calls, so a Processor is single-threaded per scrape and is NOT
137 +// goroutine-safe.
138 +type Processor struct {
139 + cfgs []Config
140 +
141 + builder *labels.Builder
142 + join strings.Builder
143 + buf []byte
144 + rangeBuf []labels.Label
145 +
146 + currentName string
147 + currentNameScheme commonmodel.ValidationScheme
148 + labelsChanged bool
149 + nameChanged bool
150 +}
151 +
152 +// New validates and compiles the rules into a Processor.
153 +func New(cfgs []Config) (*Processor, error) {
154 + compiled, err := normalizeAndValidateConfigs(cfgs)
155 + if err != nil {
156 + return nil, err
157 + }
158 +
159 + return &Processor{
160 + cfgs: compiled,
161 + builder: labels.NewBuilder(nil),
162 + }, nil
163 +}
164 +
165 +// NewTransform builds a prompkg.SampleTransform from the rules. It returns a nil
166 +// transform when there are no rules, so the scraper keeps its no-transform fast
167 +// path. onDrop, if non-nil, is called for each dropped sample. The returned
168 +// transform closes over a single reusable Processor, so it is NOT goroutine-safe;
169 +// use one transform per scrape goroutine.
170 +func NewTransform(cfgs []Config, onDrop DropObserver) (prompkg.SampleTransform, error) {
171 + p, err := New(cfgs)
172 + if err != nil {
173 + return nil, err
174 + }
175 + if len(p.cfgs) == 0 {
176 + return nil, nil
177 + }
178 +
179 + return func(s prompkg.Sample) (prompkg.Sample, bool, error) {
180 + out, drop := p.Apply(s)
181 + if drop.Dropped() {
182 + if onDrop != nil {
183 + onDrop(out, drop)
184 + }
185 + return prompkg.Sample{}, false, nil
186 + }
187 + return out, true, nil
188 + }, nil
189 +}
190 +
191 +func normalizeAndValidateConfigs(cfgs []Config) ([]Config, error) {
192 + compiled := make([]Config, 0, len(cfgs))
193 + for i, cfg := range cfgs {
194 + cfg = withDefaults(cfg)
195 + if err := cfg.validate(); err != nil {
196 + return nil, fmt.Errorf("rule %d: %w", i, err)
197 + }
198 + compiled = append(compiled, cfg)
199 + }
200 + return compiled, nil
201 +}
202 +
203 +func (c Config) validate() error {
204 + c = withDefaults(c)
205 +
206 + if _, err := parseAction(string(c.Action)); err != nil {
207 + return err
208 + }
209 +
210 + if err := validateNameScheme(c.NameScheme); err != nil {
211 + return err
212 + }
213 +
214 + scheme := c.NameScheme
215 + if scheme == commonmodel.UnsetValidation {
216 + scheme = defaultNameValidationScheme
217 + }
218 +
219 + if c.Modulus == 0 && c.Action == HashMod {
220 + return errors.New("relabel configuration for hashmod requires non-zero modulus")
221 + }
222 + if needsTargetLabel(c.Action) && c.TargetLabel == "" {
223 + return fmt.Errorf("relabel configuration for %s action requires 'target_label' value", c.Action)
224 + }
225 +
226 + if c.Action == Replace && !varInRegexTemplate(c.TargetLabel) && !scheme.IsValidLabelName(c.TargetLabel) {
227 + return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
228 + }
229 + if c.Action == Replace && varInRegexTemplate(c.TargetLabel) && !isValidLabelNameWithRegexVar(c.TargetLabel, scheme) {
230 + return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
231 + }
232 + if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) &&
233 + !scheme.IsValidLabelName(c.TargetLabel) {
234 + return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
235 + }
236 + if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) &&
237 + c.Replacement != defaultConfig.Replacement {
238 + return fmt.Errorf("'replacement' can not be set for %s action", c.Action)
239 + }
240 + if c.Action == LabelMap && !isValidLabelNameWithRegexVar(c.Replacement, scheme) {
241 + return fmt.Errorf("%q is invalid 'replacement' for %s action", c.Replacement, c.Action)
242 + }
243 + if c.Action == HashMod && !scheme.IsValidLabelName(c.TargetLabel) {
244 + return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
245 + }
246 + if c.Action == DropEqual || c.Action == KeepEqual {
247 + if c.Regex.String() != defaultConfig.Regex.String() ||
248 + c.Modulus != defaultConfig.Modulus ||
249 + c.Separator != defaultConfig.Separator ||
250 + c.Replacement != defaultConfig.Replacement {
251 + return fmt.Errorf("%s action requires only 'source_labels' and 'target_label', and no other fields", c.Action)
252 + }
253 + }
254 + if c.Action == LabelDrop || c.Action == LabelKeep {
255 + if c.sourceLabelsSet ||
256 + len(c.SourceLabels) > 0 ||
257 + c.TargetLabel != defaultConfig.TargetLabel ||
258 + c.Modulus != defaultConfig.Modulus ||
259 + c.Separator != defaultConfig.Separator ||
260 + c.Replacement != defaultConfig.Replacement {
261 + return fmt.Errorf("%s action requires only 'regex', and no other fields", c.Action)
262 + }
263 + }
264 +
265 + return nil
266 +}
267 +
268 +func NewRegexp(s string) (Regexp, error) {
269 + re, err := regexp.Compile("^(?s:" + s + ")$")
270 + return Regexp{Regexp: re, original: s}, err
271 +}
272 +
273 +func MustNewRegexp(s string) Regexp {
274 + re, err := NewRegexp(s)
275 + if err != nil {
276 + panic(err)
277 + }
278 + return re
279 +}
280 +
281 +// String returns the original, un-anchored pattern passed to NewRegexp. It returns
282 +// "" for the zero value or any Regexp not built via NewRegexp, and never inspects the
283 +// compiled form, so it is safe on a Regexp wrapping an arbitrary *regexp.Regexp.
284 +func (re Regexp) String() string {
285 + return re.original
286 +}
287 +
288 +// Apply runs the rules against one sample. It returns the (possibly mutated)
289 +// sample and a DropInfo. When DropInfo.Dropped() is true the sample must be
290 +// discarded; the returned sample is the original (unmutated) so the caller can
291 +// log its name. Value, Kind and FamilyType are passed through unchanged — a
292 +// relabeled sample is never re-typed.
293 +func (p *Processor) Apply(sample prompkg.Sample) (prompkg.Sample, DropInfo) {
294 + if len(p.cfgs) == 0 {
295 + return sample, DropInfo{}
296 + }
297 +
298 + p.builder.Reset(sample.Labels)
299 + p.currentName = sample.Name
300 + p.currentNameScheme = defaultNameValidationScheme
301 + p.labelsChanged = false
302 + p.nameChanged = false
303 +
304 + for i := range p.cfgs {
305 + if keep, drop := p.applyConfig(&p.cfgs[i], i); !keep {
306 + return sample, drop
307 + }
308 + }
309 +
310 + if !p.currentNameScheme.IsValidMetricName(p.currentName) {
311 + return sample, DropInfo{Reason: DropReasonInvalidMetricName, RuleIndex: -1}
312 + }
313 +
314 + if !p.nameChanged && !p.labelsChanged {
315 + return sample, DropInfo{}
316 + }
317 +
318 + sample.Name = p.currentName
319 + if p.labelsChanged {
320 + sample.Labels = p.builder.Labels()
321 + }
322 + return sample, DropInfo{}
323 +}
324 +
325 +func (p *Processor) applyConfig(cfg *Config, idx int) (bool, DropInfo) {
326 + val := p.joinSourceLabels(cfg.SourceLabels, cfg.Separator)
327 +
328 + switch cfg.Action {
329 + case Drop:
330 + if cfg.Regex.MatchString(val) {
331 + return false, DropInfo{Reason: DropReasonDropRuleMatched, RuleIndex: idx, Action: Drop}
332 + }
333 + case Keep:
334 + if !cfg.Regex.MatchString(val) {
335 + return false, DropInfo{Reason: DropReasonKeepRuleMismatch, RuleIndex: idx, Action: Keep}
336 + }
337 + case DropEqual:
338 + if p.getLabel(cfg.TargetLabel) == val {
339 + return false, DropInfo{Reason: DropReasonDropEqualMatched, RuleIndex: idx, Action: DropEqual}
340 + }
341 + case KeepEqual:
342 + if p.getLabel(cfg.TargetLabel) != val {
343 + return false, DropInfo{Reason: DropReasonKeepEqualMismatch, RuleIndex: idx, Action: KeepEqual}
344 + }
345 + case Replace:
346 + p.applyReplace(cfg, val)
347 + case Lowercase:
348 + p.setLabel(cfg.TargetLabel, strings.ToLower(val), cfg.NameScheme)
349 + case Uppercase:
350 + p.setLabel(cfg.TargetLabel, strings.ToUpper(val), cfg.NameScheme)
351 + case HashMod:
352 + hash := md5.Sum([]byte(val))
353 + mod := binary.BigEndian.Uint64(hash[8:]) % cfg.Modulus
354 + p.setLabel(cfg.TargetLabel, strconv.FormatUint(mod, 10), cfg.NameScheme)
355 + case LabelMap:
356 + p.rangeLabels(func(l labels.Label) {
357 + if cfg.Regex.MatchString(l.Name) {
358 + p.setLabel(cfg.Regex.ReplaceAllString(l.Name, cfg.Replacement), l.Value, cfg.NameScheme)
359 + }
360 + })
361 + case LabelDrop:
362 + p.rangeLabels(func(l labels.Label) {
363 + if cfg.Regex.MatchString(l.Name) {
364 + p.delLabel(l.Name)
365 + }
366 + })
367 + case LabelKeep:
368 + p.rangeLabels(func(l labels.Label) {
369 + if !cfg.Regex.MatchString(l.Name) {
370 + p.delLabel(l.Name)
371 + }
372 + })
373 + default:
374 + panic(fmt.Errorf("unknown relabel action %q", cfg.Action))
375 + }
376 +
377 + return true, DropInfo{}
378 +}
379 +
380 +func (p *Processor) applyReplace(cfg *Config, val string) {
381 + if val == "" &&
382 + cfg.Regex.String() == defaultConfig.Regex.String() &&
383 + !varInRegexTemplate(cfg.TargetLabel) &&
384 + !varInRegexTemplate(cfg.Replacement) {
385 + p.setLabel(cfg.TargetLabel, cfg.Replacement, cfg.NameScheme)
386 + return
387 + }
388 +
389 + indexes := cfg.Regex.FindStringSubmatchIndex(val)
390 + if indexes == nil {
391 + return
392 + }
393 +
394 + p.buf = cfg.Regex.ExpandString(p.buf[:0], cfg.TargetLabel, val, indexes)
395 + target := string(p.buf)
396 + if !cfg.NameScheme.IsValidLabelName(target) {
397 + return
398 + }
399 +
400 + p.buf = cfg.Regex.ExpandString(p.buf[:0], cfg.Replacement, val, indexes)
401 + if len(p.buf) == 0 {
402 + p.delLabel(target)
403 + return
404 + }
405 +
406 + p.setLabel(target, string(p.buf), cfg.NameScheme)
407 +}
408 +
409 +func (p *Processor) joinSourceLabels(sourceLabels []string, separator string) string {
410 + switch len(sourceLabels) {
411 + case 0:
412 + return ""
413 + case 1:
414 + return p.getLabel(sourceLabels[0])
415 + }
416 +
417 + p.join.Reset()
418 + for i, name := range sourceLabels {
419 + if i > 0 {
420 + p.join.WriteString(separator)
421 + }
422 + p.join.WriteString(p.getLabel(name))
423 + }
424 +
425 + return p.join.String()
426 +}
427 +
428 +func (p *Processor) getLabel(name string) string {
429 + if name == commonmodel.MetricNameLabel {
430 + return p.currentName
431 + }
432 + return p.builder.Get(name)
433 +}
434 +
435 +func (p *Processor) setLabel(name, value string, scheme commonmodel.ValidationScheme) {
436 + if name == commonmodel.MetricNameLabel {
437 + p.currentNameScheme = scheme
438 + if p.currentName != value {
439 + p.currentName = value
440 + p.nameChanged = true
441 + }
442 + return
443 + }
444 +
445 + if current, ok := p.lookupLabel(name); ok && current == value {
446 + return
447 + }
448 +
449 + p.builder.Set(name, value)
450 + p.labelsChanged = true
451 +}
452 +
453 +func (p *Processor) delLabel(name string) {
454 + if name == commonmodel.MetricNameLabel {
455 + if p.currentName != "" {
456 + p.currentName = ""
457 + p.nameChanged = true
458 + }
459 + return
460 + }
461 +
462 + if _, ok := p.lookupLabel(name); !ok {
463 + return
464 + }
465 +
466 + p.builder.Del(name)
467 + p.labelsChanged = true
468 +}
469 +
470 +func (p *Processor) rangeLabels(fn func(labels.Label)) {
471 + // Snapshot the current label set (including __name__) before invoking fn, so a
472 + // callback that adds labels (labelmap) does not re-process labels it creates in
473 + // the same rule — matching Prometheus, which ranges one snapshot per rule. The
474 + // scratch buffer is reused across calls.
475 + p.rangeBuf = p.rangeBuf[:0]
476 + if p.currentName != "" {
477 + p.rangeBuf = append(p.rangeBuf, labels.Label{Name: commonmodel.MetricNameLabel, Value: p.currentName})
478 + }
479 + p.builder.Range(func(l labels.Label) {
480 + p.rangeBuf = append(p.rangeBuf, l)
481 + })
482 + for _, l := range p.rangeBuf {
483 + fn(l)
484 + }
485 +}
486 +
487 +func (p *Processor) lookupLabel(name string) (string, bool) {
488 + if name == commonmodel.MetricNameLabel {
489 + if p.currentName == "" {
490 + return "", false
491 + }
492 + return p.currentName, true
493 + }
494 +
495 + var (
496 + value string
497 + ok bool
498 + )
499 + p.builder.Range(func(l labels.Label) {
500 + if l.Name == name {
501 + value = l.Value
502 + ok = true
503 + }
504 + })
505 + return value, ok
506 +}
507 +
508 +func parseAction(s string) (Action, error) {
509 + switch act := Action(strings.ToLower(s)); act {
510 + case Replace, Keep, Drop, KeepEqual, DropEqual, HashMod, LabelMap, LabelDrop, LabelKeep, Lowercase, Uppercase:
511 + return act, nil
512 + default:
513 + return "", fmt.Errorf("unknown relabel action %q", s)
514 + }
515 +}
516 +
517 +func withDefaults(cfg Config) Config {
518 + cfg.NameScheme = withNameScheme(cfg.NameScheme)
519 + if cfg.Action == "" {
520 + cfg.Action = defaultConfig.Action
521 + } else if act, err := parseAction(string(cfg.Action)); err == nil {
522 + // Canonicalize a valid action (e.g. "KEEP" -> "keep") so Apply's switch
523 + // matches; an invalid action is left for validate to reject.
524 + cfg.Action = act
525 + }
526 + if !cfg.separatorSet && cfg.Separator == "" {
527 + cfg.Separator = defaultConfig.Separator
528 + }
529 + if cfg.Regex.Regexp == nil {
530 + cfg.Regex = defaultConfig.Regex
531 + }
532 + if !cfg.replacementSet && cfg.Replacement == "" {
533 + cfg.Replacement = defaultConfig.Replacement
534 + }
535 + return cfg
536 +}
537 +
538 +func withNameScheme(scheme commonmodel.ValidationScheme) commonmodel.ValidationScheme {
539 + if scheme == commonmodel.UnsetValidation {
540 + return defaultNameValidationScheme
541 + }
542 + return scheme
543 +}
544 +
545 +func validateNameScheme(scheme commonmodel.ValidationScheme) error {
546 + switch scheme {
547 + case commonmodel.UnsetValidation, commonmodel.LegacyValidation, commonmodel.UTF8Validation:
548 + return nil
549 + default:
550 + return fmt.Errorf("unknown relabel config name validation method specified, must be either '', 'legacy' or 'utf8', got %s", scheme)
551 + }
552 +}
553 +
554 +func needsTargetLabel(action Action) bool {
555 + return action == Replace || action == HashMod || action == Lowercase || action == Uppercase || action == KeepEqual || action == DropEqual
556 +}
557 +
558 +func isValidLabelNameWithRegexVar(value string, scheme commonmodel.ValidationScheme) bool {
559 + if scheme == commonmodel.UTF8Validation {
560 + return scheme.IsValidLabelName(value)
561 + }
562 + return relabelTargetLegacy.MatchString(value)
563 +}
564 +
565 +func varInRegexTemplate(template string) bool {
566 + return strings.Contains(template, "$")
567 +}
src/go/plugin/go.d/collector/prometheus/relabel/relabel_bench_test.go new
+157
@@ -0,0 +1,157 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package relabel
4 +
5 +import (
6 + "strconv"
7 + "testing"
8 +
9 + commonmodel "github.com/prometheus/common/model"
10 + "github.com/prometheus/prometheus/model/labels"
11 +
12 + prompkg "github.com/netdata/netdata/go/plugins/pkg/prometheus"
13 +)
14 +
15 +var (
16 + benchSampleSink prompkg.Sample
17 + benchKeepSink bool
18 +)
19 +
20 +// relabel runs on every kept sample on the scrape hot path, so keep these
21 +// numbers current before/after changes to the engine. Run with:
22 +//
23 +// go test ./plugin/go.d/collector/prometheus/relabel -run '^$' -bench BenchmarkProcessorApply -benchmem
24 +//
25 +// Numbers are machine-specific; compare relative before/after deltas, and watch
26 +// the allocation columns (the no-rules and passthrough cases should stay at 0).
27 +func BenchmarkProcessorApply(b *testing.B) {
28 + type testCase struct {
29 + name string
30 + cfgs []Config
31 + sample prompkg.Sample
32 + }
33 +
34 + tests := []testCase{
35 + {
36 + name: "no_rules",
37 + cfgs: nil,
38 + sample: benchSample("http_requests_total", benchLabels(4), prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
39 + },
40 + {
41 + name: "keep_passthrough",
42 + cfgs: []Config{
43 + {
44 + SourceLabels: []string{"job"},
45 + Regex: MustNewRegexp("api"),
46 + Action: Keep,
47 + },
48 + },
49 + sample: benchSample("http_requests_total", map[string]string{
50 + "instance": "127.0.0.1:9090",
51 + "job": "api",
52 + "method": "GET",
53 + "status": "200",
54 + }, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
55 + },
56 + {
57 + name: "name_only_rewrite",
58 + cfgs: []Config{
59 + {
60 + SourceLabels: []string{commonmodel.MetricNameLabel},
61 + Regex: MustNewRegexp("(.*)_total"),
62 + TargetLabel: commonmodel.MetricNameLabel,
63 + Replacement: "${1}",
64 + Action: Replace,
65 + },
66 + },
67 + sample: benchSample("http_requests_total", benchLabels(4), prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
68 + },
69 + {
70 + name: "label_replace",
71 + cfgs: []Config{
72 + {
73 + SourceLabels: []string{"method"},
74 + TargetLabel: "http_method",
75 + Replacement: "$1",
76 + Action: Replace,
77 + },
78 + },
79 + sample: benchSample("http_requests_total", map[string]string{
80 + "instance": "127.0.0.1:9090",
81 + "job": "api",
82 + "method": "GET",
83 + "status": "200",
84 + }, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
85 + },
86 + {
87 + name: "multi_source_replace",
88 + cfgs: []Config{
89 + {
90 + SourceLabels: []string{"job", "instance", "method", "status"},
91 + Separator: "/",
92 + TargetLabel: "route_key",
93 + Replacement: "$1",
94 + Action: Replace,
95 + },
96 + },
97 + sample: benchSample("http_requests_total", map[string]string{
98 + "instance": "127.0.0.1:9090",
99 + "job": "api",
100 + "method": "GET",
101 + "status": "200",
102 + }, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
103 + },
104 + {
105 + name: "labeldrop_many_labels",
106 + cfgs: []Config{
107 + {
108 + Regex: MustNewRegexp("label_[02468]"),
109 + Action: LabelDrop,
110 + },
111 + },
112 + sample: benchSample("http_requests_total", benchLabels(12), prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
113 + },
114 + }
115 +
116 + for _, test := range tests {
117 + b.Run(test.name, func(b *testing.B) {
118 + p, err := New(test.cfgs)
119 + if err != nil {
120 + b.Fatalf("New() error = %v", err)
121 + }
122 +
123 + got, drop := p.Apply(test.sample)
124 + if drop.Dropped() {
125 + b.Fatal("unexpected drop in benchmark setup")
126 + }
127 + benchSampleSink = got
128 + benchKeepSink = !drop.Dropped()
129 +
130 + b.ReportAllocs()
131 + b.ResetTimer()
132 + for i := 0; i < b.N; i++ {
133 + got, drop = p.Apply(test.sample)
134 + benchSampleSink = got
135 + benchKeepSink = !drop.Dropped()
136 + }
137 + })
138 + }
139 +}
140 +
141 +func benchSample(name string, lbs map[string]string, kind prompkg.SampleKind, familyType commonmodel.MetricType) prompkg.Sample {
142 + return prompkg.Sample{
143 + Name: name,
144 + Labels: labels.FromMap(lbs),
145 + Value: 1,
146 + Kind: kind,
147 + FamilyType: familyType,
148 + }
149 +}
150 +
151 +func benchLabels(n int) map[string]string {
152 + lbs := make(map[string]string, n)
153 + for i := range n {
154 + lbs["label_"+strconv.Itoa(i)] = "value"
155 + }
156 + return lbs
157 +}
src/go/plugin/go.d/collector/prometheus/relabel/relabel_test.go new
+638
@@ -0,0 +1,638 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package relabel
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/grafana/regexp"
9 + commonmodel "github.com/prometheus/common/model"
10 + "github.com/prometheus/prometheus/model/labels"
11 + "github.com/stretchr/testify/assert"
12 + "github.com/stretchr/testify/require"
13 +
14 + prompkg "github.com/netdata/netdata/go/plugins/pkg/prometheus"
15 +)
16 +
17 +func TestRegexp_String(t *testing.T) {
18 + tests := map[string]struct {
19 + re Regexp
20 + want string
21 + }{
22 + "NewRegexp returns the un-anchored source": {re: MustNewRegexp("(.*)_total"), want: "(.*)_total"},
23 + "NewRegexp with empty source": {re: MustNewRegexp(""), want: ""},
24 + "zero value": {re: Regexp{}, want: ""},
25 + "raw non-anchored value is safe (no panic)": {re: Regexp{Regexp: regexp.MustCompile("x")}, want: ""},
26 + }
27 + for name, tc := range tests {
28 + t.Run(name, func(t *testing.T) {
29 + var got string
30 + require.NotPanics(t, func() { got = tc.re.String() })
31 + assert.Equal(t, tc.want, got)
32 + })
33 + }
34 +}
35 +
36 +func TestProcessor_Apply(t *testing.T) {
37 + tests := map[string]struct {
38 + cfgs []Config
39 + in prompkg.Sample
40 + want prompkg.Sample
41 + keep bool
42 + sameLabelBacking bool
43 + }{
44 + "labelmap matching __name__ maps it once, not its own output": {
45 + cfgs: []Config{{
46 + Regex: MustNewRegexp("(.+)"),
47 + Replacement: "copy_${1}",
48 + Action: LabelMap,
49 + }},
50 + in: sample("m", map[string]string{"job": "api"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
51 + want: sample("m", map[string]string{
52 + "job": "api",
53 + "copy___name__": "m",
54 + "copy_job": "api",
55 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
56 + keep: true,
57 + },
58 + "explicit empty separator joins without the default ';'": {
59 + cfgs: []Config{{
60 + SourceLabels: []string{"a", "b"},
61 + Separator: "",
62 + separatorSet: true,
63 + TargetLabel: "c",
64 + Action: Replace,
65 + }},
66 + in: sample("m", map[string]string{"a": "foo", "b": "bar"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
67 + want: sample("m", map[string]string{"a": "foo", "b": "bar", "c": "foobar"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
68 + keep: true,
69 + },
70 + "non-canonical action is canonicalized (KEEP -> keep)": {
71 + cfgs: []Config{{
72 + SourceLabels: []string{"a"},
73 + Regex: MustNewRegexp("foo"),
74 + Action: Action("KEEP"),
75 + }},
76 + in: sample("m", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
77 + want: sample("m", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
78 + keep: true,
79 + },
80 + "zero rules pass through without rematerializing labels": {
81 + in: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
82 + want: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
83 + keep: true,
84 + sameLabelBacking: true,
85 + },
86 + "replace rewrites label and preserves sample payload": {
87 + cfgs: []Config{
88 + {
89 + SourceLabels: []string{"a"},
90 + Regex: MustNewRegexp("f(.*)"),
91 + TargetLabel: "b",
92 + Replacement: "ch${1}",
93 + Action: Replace,
94 + },
95 + },
96 + in: sample("test_total", map[string]string{"a": "foo"}, 1.5, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
97 + want: sample("test_total", map[string]string{
98 + "a": "foo",
99 + "b": "choo",
100 + }, 1.5, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
101 + keep: true,
102 + },
103 + "replace can rewrite __name__": {
104 + cfgs: []Config{
105 + {
106 + SourceLabels: []string{commonmodel.MetricNameLabel},
107 + Regex: MustNewRegexp("(.*)_total"),
108 + TargetLabel: commonmodel.MetricNameLabel,
109 + Replacement: "${1}",
110 + Action: Replace,
111 + },
112 + },
113 + in: sample("http_requests_total", map[string]string{"method": "GET"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
114 + want: sample("http_requests", map[string]string{"method": "GET"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
115 + keep: true,
116 + },
117 + "label-only relabel preserves existing utf8 metric name": {
118 + cfgs: []Config{
119 + {
120 + SourceLabels: []string{"method"},
121 + TargetLabel: "method_upper",
122 + Action: Uppercase,
123 + },
124 + },
125 + in: sample("http.requests_total", map[string]string{"method": "get"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
126 + want: sample("http.requests_total", map[string]string{
127 + "method": "get",
128 + "method_upper": "GET",
129 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
130 + keep: true,
131 + },
132 + "replace rewrites __name__ with explicit utf8 scheme at runtime": {
133 + cfgs: []Config{
134 + {
135 + SourceLabels: []string{commonmodel.MetricNameLabel},
136 + Regex: MustNewRegexp("(.*)_total"),
137 + TargetLabel: commonmodel.MetricNameLabel,
138 + Replacement: "${1}.total",
139 + Action: Replace,
140 + NameScheme: commonmodel.UTF8Validation,
141 + },
142 + },
143 + in: sample("http_requests_total", map[string]string{"method": "GET"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
144 + want: sample("http_requests.total", map[string]string{"method": "GET"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
145 + keep: true,
146 + sameLabelBacking: true,
147 + },
148 + "histogram bucket relabel preserves kind and family type": {
149 + cfgs: []Config{
150 + {
151 + SourceLabels: []string{commonmodel.MetricNameLabel},
152 + Regex: MustNewRegexp("(.*)"),
153 + TargetLabel: commonmodel.MetricNameLabel,
154 + Replacement: "nginx_${1}",
155 + Action: Replace,
156 + },
157 + },
158 + in: sample("request_duration_seconds_bucket", map[string]string{
159 + "le": "0.5",
160 + "method": "GET",
161 + }, 42, prompkg.SampleKindHistogramBucket, commonmodel.MetricTypeHistogram),
162 + want: sample("nginx_request_duration_seconds_bucket", map[string]string{
163 + "le": "0.5",
164 + "method": "GET",
165 + }, 42, prompkg.SampleKindHistogramBucket, commonmodel.MetricTypeHistogram),
166 + keep: true,
167 + sameLabelBacking: true,
168 + },
169 + "summary quantile relabel preserves kind and family type": {
170 + cfgs: []Config{
171 + {
172 + SourceLabels: []string{commonmodel.MetricNameLabel},
173 + Regex: MustNewRegexp("(.*)"),
174 + TargetLabel: commonmodel.MetricNameLabel,
175 + Replacement: "nginx_${1}",
176 + Action: Replace,
177 + },
178 + },
179 + in: sample("request_duration_seconds", map[string]string{
180 + "method": "GET",
181 + "quantile": "0.9",
182 + }, 7, prompkg.SampleKindSummaryQuantile, commonmodel.MetricTypeSummary),
183 + want: sample("nginx_request_duration_seconds", map[string]string{
184 + "method": "GET",
185 + "quantile": "0.9",
186 + }, 7, prompkg.SampleKindSummaryQuantile, commonmodel.MetricTypeSummary),
187 + keep: true,
188 + sameLabelBacking: true,
189 + },
190 + "drop drops matching input": {
191 + cfgs: []Config{
192 + {
193 + SourceLabels: []string{"a"},
194 + Regex: MustNewRegexp(".*o.*"),
195 + Action: Drop,
196 + },
197 + },
198 + in: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
199 + keep: false,
200 + },
201 + "drop uses anchored regex semantics": {
202 + cfgs: []Config{
203 + {
204 + SourceLabels: []string{"a"},
205 + Regex: MustNewRegexp("f|o"),
206 + Action: Drop,
207 + },
208 + },
209 + in: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
210 + want: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
211 + keep: true,
212 + sameLabelBacking: true,
213 + },
214 + "keep drops non matching input": {
215 + cfgs: []Config{
216 + {
217 + SourceLabels: []string{"a"},
218 + Regex: MustNewRegexp("no-match"),
219 + Action: Keep,
220 + },
221 + },
222 + in: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
223 + keep: false,
224 + },
225 + "ordered rules see prior __name__ rewrite": {
226 + cfgs: []Config{
227 + {
228 + SourceLabels: []string{"rename_to"},
229 + TargetLabel: commonmodel.MetricNameLabel,
230 + Replacement: "$1",
231 + Action: Replace,
232 + NameScheme: commonmodel.UTF8Validation,
233 + },
234 + {
235 + SourceLabels: []string{commonmodel.MetricNameLabel},
236 + TargetLabel: "seen_name",
237 + Replacement: "prefix_$1",
238 + Action: Replace,
239 + },
240 + },
241 + in: sample("request_total", map[string]string{
242 + "rename_to": "request.total",
243 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
244 + want: sample("request.total", map[string]string{
245 + "rename_to": "request.total",
246 + "seen_name": "prefix_request.total",
247 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeCounter),
248 + keep: true,
249 + },
250 + "keepequal keeps when values match": {
251 + cfgs: []Config{
252 + {
253 + SourceLabels: []string{"__tmp_port"},
254 + TargetLabel: "__port1",
255 + Action: KeepEqual,
256 + },
257 + },
258 + in: sample("test_metric", map[string]string{"__tmp_port": "1234", "__port1": "1234"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
259 + want: sample("test_metric", map[string]string{
260 + "__tmp_port": "1234",
261 + "__port1": "1234",
262 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
263 + keep: true,
264 + },
265 + "dropequal drops when values match": {
266 + cfgs: []Config{
267 + {
268 + SourceLabels: []string{"__tmp_port"},
269 + TargetLabel: "__port1",
270 + Action: DropEqual,
271 + },
272 + },
273 + in: sample("test_metric", map[string]string{"__tmp_port": "1234", "__port1": "1234"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
274 + keep: false,
275 + },
276 + "hashmod matches upstream example": {
277 + cfgs: []Config{
278 + {
279 + SourceLabels: []string{"c"},
280 + TargetLabel: "d",
281 + Action: HashMod,
282 + Modulus: 1000,
283 + },
284 + },
285 + in: sample("test_metric", map[string]string{"a": "foo", "b": "bar", "c": "baz"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
286 + want: sample("test_metric", map[string]string{
287 + "a": "foo",
288 + "b": "bar",
289 + "c": "baz",
290 + "d": "976",
291 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
292 + keep: true,
293 + },
294 + "labelmap copies matching labels": {
295 + cfgs: []Config{
296 + {
297 + Regex: MustNewRegexp("(b.*)"),
298 + Replacement: "bar_${1}",
299 + Action: LabelMap,
300 + },
301 + },
302 + in: sample("test_metric", map[string]string{"a": "foo", "b1": "bar", "b2": "baz"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
303 + want: sample("test_metric", map[string]string{
304 + "a": "foo",
305 + "b1": "bar",
306 + "b2": "baz",
307 + "bar_b1": "bar",
308 + "bar_b2": "baz",
309 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
310 + keep: true,
311 + },
312 + "labeldrop removes matching labels": {
313 + cfgs: []Config{
314 + {
315 + Regex: MustNewRegexp("(b.*)"),
316 + Action: LabelDrop,
317 + },
318 + },
319 + in: sample("test_metric", map[string]string{"a": "foo", "b1": "bar", "b2": "baz"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
320 + want: sample("test_metric", map[string]string{
321 + "a": "foo",
322 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
323 + keep: true,
324 + },
325 + "labelkeep can drop __name__ and therefore drop the sample": {
326 + cfgs: []Config{
327 + {
328 + Regex: MustNewRegexp("(b.*)"),
329 + Action: LabelKeep,
330 + },
331 + },
332 + in: sample("test_metric", map[string]string{"b1": "bar"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
333 + keep: false,
334 + },
335 + "lowercase and uppercase write derived labels": {
336 + cfgs: []Config{
337 + {
338 + SourceLabels: []string{"foo"},
339 + TargetLabel: "foo_uppercase",
340 + Action: Uppercase,
341 + },
342 + {
343 + SourceLabels: []string{"foo"},
344 + TargetLabel: "foo_lowercase",
345 + Action: Lowercase,
346 + },
347 + },
348 + in: sample("test_metric", map[string]string{"foo": "bAr123Foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
349 + want: sample("test_metric", map[string]string{
350 + "foo": "bAr123Foo",
351 + "foo_lowercase": "bar123foo",
352 + "foo_uppercase": "BAR123FOO",
353 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
354 + keep: true,
355 + },
356 + "explicit empty replacement deletes target label": {
357 + cfgs: []Config{
358 + {
359 + SourceLabels: []string{"a"},
360 + TargetLabel: "b",
361 + Replacement: "",
362 + Action: Replace,
363 + replacementSet: true,
364 + },
365 + },
366 + in: sample("test_metric", map[string]string{"a": "foo", "b": "bar"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
367 + want: sample("test_metric", map[string]string{
368 + "a": "foo",
369 + }, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
370 + keep: true,
371 + },
372 + "invalid final metric name drops the sample": {
373 + cfgs: []Config{
374 + {
375 + SourceLabels: []string{commonmodel.MetricNameLabel},
376 + TargetLabel: commonmodel.MetricNameLabel,
377 + Replacement: "",
378 + Action: Replace,
379 + replacementSet: true,
380 + },
381 + },
382 + in: sample("test_metric", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
383 + keep: false,
384 + },
385 + }
386 +
387 + for name, test := range tests {
388 + t.Run(name, func(t *testing.T) {
389 + p, err := New(test.cfgs)
390 + require.NoError(t, err)
391 +
392 + got, drop := p.Apply(test.in)
393 + require.Equal(t, test.keep, !drop.Dropped())
394 + if !drop.Dropped() {
395 + assert.Equal(t, test.want, got)
396 + if test.sameLabelBacking && len(test.in.Labels) > 0 {
397 + require.NotEmpty(t, got.Labels)
398 + assert.True(t, &got.Labels[0] == &test.in.Labels[0])
399 + }
400 + }
401 + })
402 + }
403 +}
404 +
405 +func TestNew_Validate(t *testing.T) {
406 + tests := map[string]struct {
407 + cfgs []Config
408 + wantErrText string
409 + }{
410 + "rejects labeldrop with explicit empty source_labels": {
411 + cfgs: []Config{{
412 + Action: LabelDrop,
413 + Regex: MustNewRegexp("foo"),
414 + sourceLabelsSet: true,
415 + }},
416 + wantErrText: `requires only 'regex'`,
417 + },
418 + "rejects replace with invalid regex-var target label under legacy": {
419 + cfgs: []Config{{
420 + Action: Replace,
421 + SourceLabels: []string{"a"},
422 + TargetLabel: "${1}.x",
423 + NameScheme: commonmodel.LegacyValidation,
424 + }},
425 + wantErrText: `invalid 'target_label'`,
426 + },
427 + "rejects labelmap with invalid replacement under legacy": {
428 + cfgs: []Config{{
429 + Action: LabelMap,
430 + Regex: MustNewRegexp("(.+)"),
431 + Replacement: "bad.name",
432 + NameScheme: commonmodel.LegacyValidation,
433 + }},
434 + wantErrText: `invalid 'replacement'`,
435 + },
436 + "rejects unknown action": {
437 + cfgs: []Config{{Action: Action("wat")}},
438 + wantErrText: `unknown relabel action "wat"`,
439 + },
440 + "rejects missing target label for replace": {
441 + cfgs: []Config{{Action: Replace}},
442 + wantErrText: `requires 'target_label' value`,
443 + },
444 + "rejects hashmod without modulus": {
445 + cfgs: []Config{{
446 + Action: HashMod,
447 + TargetLabel: "d",
448 + }},
449 + wantErrText: `requires non-zero modulus`,
450 + },
451 + "rejects labeldrop extra fields": {
452 + cfgs: []Config{{
453 + Action: LabelDrop,
454 + Regex: MustNewRegexp("foo"),
455 + Replacement: "bar",
456 + replacementSet: true,
457 + }},
458 + wantErrText: `requires only 'regex'`,
459 + },
460 + "rejects keepequal with replacement": {
461 + cfgs: []Config{{
462 + Action: KeepEqual,
463 + TargetLabel: "__port1",
464 + Replacement: "bar",
465 + replacementSet: true,
466 + }},
467 + wantErrText: `'replacement' can not be set for keepequal action`,
468 + },
469 + "rejects legacy invalid target label": {
470 + cfgs: []Config{{
471 + Action: Lowercase,
472 + TargetLabel: "${3}",
473 + NameScheme: commonmodel.LegacyValidation,
474 + }},
475 + wantErrText: `"${3}" is invalid 'target_label' for lowercase action`,
476 + },
477 + "accepts utf8 target label for lowercase": {
478 + cfgs: []Config{{
479 + Action: Lowercase,
480 + TargetLabel: "${3}",
481 + NameScheme: commonmodel.UTF8Validation,
482 + }},
483 + },
484 + "defaults to utf8 validation when unset (accepts a target legacy would reject)": {
485 + cfgs: []Config{{
486 + Action: Lowercase,
487 + TargetLabel: "${3}",
488 + }},
489 + },
490 + "rejects invalid name scheme": {
491 + cfgs: []Config{{
492 + Action: Lowercase,
493 + TargetLabel: "foo",
494 + NameScheme: commonmodel.ValidationScheme(99),
495 + }},
496 + wantErrText: `unknown relabel config name validation method specified`,
497 + },
498 + }
499 +
500 + for name, test := range tests {
501 + t.Run(name, func(t *testing.T) {
502 + _, err := New(test.cfgs)
503 + if test.wantErrText == "" {
504 + require.NoError(t, err)
505 + return
506 + }
507 +
508 + require.ErrorContains(t, err, test.wantErrText)
509 + })
510 + }
511 +}
512 +
513 +func sample(name string, lbs map[string]string, value float64, kind prompkg.SampleKind, familyType commonmodel.MetricType) prompkg.Sample {
514 + return prompkg.Sample{
515 + Name: name,
516 + Labels: labels.FromMap(lbs),
517 + Value: value,
518 + Kind: kind,
519 + FamilyType: familyType,
520 + }
521 +}
522 +
523 +func TestProcessor_Apply_dropInfo(t *testing.T) {
524 + tests := map[string]struct {
525 + cfgs []Config
526 + in prompkg.Sample
527 + wantReason DropReason
528 + wantRule int
529 + wantAction Action
530 + }{
531 + "drop rule matched": {
532 + cfgs: []Config{{SourceLabels: []string{"a"}, Regex: MustNewRegexp("foo"), Action: Drop}},
533 + in: sample("m", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
534 + wantReason: DropReasonDropRuleMatched,
535 + wantRule: 0,
536 + wantAction: Drop,
537 + },
538 + "dropequal matched": {
539 + cfgs: []Config{{SourceLabels: []string{"a"}, TargetLabel: "b", Action: DropEqual}},
540 + in: sample("m", map[string]string{"a": "x", "b": "x"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
541 + wantReason: DropReasonDropEqualMatched,
542 + wantRule: 0,
543 + wantAction: DropEqual,
544 + },
545 + "keepequal did not match": {
546 + cfgs: []Config{{SourceLabels: []string{"a"}, TargetLabel: "b", Action: KeepEqual}},
547 + in: sample("m", map[string]string{"a": "x", "b": "y"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
548 + wantReason: DropReasonKeepEqualMismatch,
549 + wantRule: 0,
550 + wantAction: KeepEqual,
551 + },
552 + "keep rule did not match": {
553 + cfgs: []Config{{SourceLabels: []string{"a"}, Regex: MustNewRegexp("foo"), Action: Keep}},
554 + in: sample("m", map[string]string{"a": "bar"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
555 + wantReason: DropReasonKeepRuleMismatch,
556 + wantRule: 0,
557 + wantAction: Keep,
558 + },
559 + "invalid metric name (labelkeep drops __name__)": {
560 + cfgs: []Config{{Regex: MustNewRegexp("keep"), Action: LabelKeep}},
561 + in: sample("m", map[string]string{"keep": "v", "other": "x"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge),
562 + wantReason: DropReasonInvalidMetricName,
563 + wantRule: -1,
564 + wantAction: "",
565 + },
566 + }
567 +
568 + for name, test := range tests {
569 + t.Run(name, func(t *testing.T) {
570 + p, err := New(test.cfgs)
571 + require.NoError(t, err)
572 +
573 + got, drop := p.Apply(test.in)
574 + require.True(t, drop.Dropped())
575 + assert.Equal(t, test.wantReason, drop.Reason)
576 + assert.Equal(t, test.wantRule, drop.RuleIndex)
577 + assert.Equal(t, test.wantAction, drop.Action)
578 + // Apply returns the original sample on drop so the caller can log it.
579 + assert.Equal(t, test.in, got)
580 + })
581 + }
582 +}
583 +
584 +func TestNewTransform(t *testing.T) {
585 + gauge := func() prompkg.Sample {
586 + return sample("m", map[string]string{"a": "foo"}, 1, prompkg.SampleKindScalar, commonmodel.MetricTypeGauge)
587 + }
588 +
589 + t.Run("nil transform when there are no rules", func(t *testing.T) {
590 + tr, err := NewTransform(nil, nil)
591 + require.NoError(t, err)
592 + assert.Nil(t, tr)
593 + })
594 +
595 + t.Run("invalid rules return an error", func(t *testing.T) {
596 + _, err := NewTransform([]Config{{Action: HashMod, TargetLabel: "x"}}, nil)
597 + assert.Error(t, err)
598 + })
599 +
600 + t.Run("applies rules and keeps", func(t *testing.T) {
601 + tr, err := NewTransform([]Config{{
602 + SourceLabels: []string{"a"},
603 + Regex: MustNewRegexp("f(.*)"),
604 + TargetLabel: "b",
605 + Replacement: "x${1}",
606 + Action: Replace,
607 + }}, nil)
608 + require.NoError(t, err)
609 + require.NotNil(t, tr)
610 +
611 + out, keep, err := tr(gauge())
612 + require.NoError(t, err)
613 + assert.True(t, keep)
614 + assert.Equal(t, "xoo", out.Labels.Get("b"))
615 + })
616 +
617 + t.Run("drop calls onDrop with the original sample and no scrape error", func(t *testing.T) {
618 + var observed []DropInfo
619 + var observedSample prompkg.Sample
620 + tr, err := NewTransform([]Config{{
621 + SourceLabels: []string{"a"},
622 + Regex: MustNewRegexp("foo"),
623 + Action: Drop,
624 + }}, func(s prompkg.Sample, d DropInfo) {
625 + observedSample = s
626 + observed = append(observed, d)
627 + })
628 + require.NoError(t, err)
629 +
630 + in := gauge()
631 + _, keep, err := tr(in)
632 + require.NoError(t, err) // a drop is not a scrape error
633 + assert.False(t, keep)
634 + require.Len(t, observed, 1)
635 + assert.Equal(t, DropReasonDropRuleMatched, observed[0].Reason)
636 + assert.Equal(t, in, observedSample)
637 + })
638 +}