master
go 567 lines 17.5 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 // Package relabel applies Prometheus-compatible metric-relabeling rules to
4 // scraped samples (the metric name plus labels, including le/quantile) before
5 // typed-family assembly. It is collector-local to the prometheus collector.
6 package relabel
7
8 import (
9 "crypto/md5"
10 "encoding/binary"
11 "errors"
12 "fmt"
13 "strconv"
14 "strings"
15
16 "github.com/grafana/regexp"
17 commonmodel "github.com/prometheus/common/model"
18 "github.com/prometheus/prometheus/model/labels"
19
20 prompkg "github.com/netdata/netdata/go/plugins/pkg/prometheus"
21 )
22
23 var (
24 relabelTargetLegacy = regexp.MustCompile(`^(?:(?:[a-zA-Z_]|\$(?:\{\w+\}|\w+))+\w*)+$`)
25
26 defaultConfig = Config{
27 Action: Replace,
28 Separator: ";",
29 Regex: MustNewRegexp("(.*)"),
30 Replacement: "$1",
31 }
32 )
33
34 // defaultNameValidationScheme is the name-validation scheme applied when a rule
35 // does not set one. It is UTF-8: relabeling may legitimately produce dotted or
36 // otherwise non-legacy metric and label names, and only an empty name is
37 // rejected. A rule may still opt into commonmodel.LegacyValidation via
38 // Config.NameScheme.
39 const defaultNameValidationScheme = commonmodel.UTF8Validation
40
41 // Action is the operation a relabel rule performs on a sample's labels and metric
42 // name. It mirrors Prometheus's relabel actions; New canonicalizes the value, so it
43 // is case-insensitive. "Joined value" below means the SourceLabels values joined by
44 // Separator.
45 type Action string
46
47 const (
48 // Replace sets TargetLabel from the regex match of the joined value (Replacement
49 // is the template; an empty result deletes the target label).
50 Replace Action = "replace"
51 // Keep keeps the sample only when Regex matches the joined value.
52 Keep Action = "keep"
53 // Drop drops the sample when Regex matches the joined value.
54 Drop Action = "drop"
55 // KeepEqual keeps the sample only when TargetLabel equals the joined value.
56 KeepEqual Action = "keepequal"
57 // DropEqual drops the sample when TargetLabel equals the joined value.
58 DropEqual Action = "dropequal"
59 // HashMod sets TargetLabel to the MD5 of the joined value modulo Modulus.
60 HashMod Action = "hashmod"
61 // LabelMap copies each label whose name matches Regex to a new name from Replacement.
62 LabelMap Action = "labelmap"
63 // LabelDrop removes every label whose name matches Regex.
64 LabelDrop Action = "labeldrop"
65 // LabelKeep removes every label whose name does not match Regex.
66 LabelKeep Action = "labelkeep"
67 // Lowercase sets TargetLabel to the lowercased joined value.
68 Lowercase Action = "lowercase"
69 // Uppercase sets TargetLabel to the uppercased joined value.
70 Uppercase Action = "uppercase"
71 )
72
73 // DropReason explains why Apply dropped a sample, for the caller to log.
74 type DropReason string
75
76 const (
77 DropReasonNone DropReason = ""
78 DropReasonDropRuleMatched DropReason = "drop rule matched"
79 DropReasonKeepRuleMismatch DropReason = "keep rule did not match"
80 DropReasonDropEqualMatched DropReason = "dropequal rule matched"
81 DropReasonKeepEqualMismatch DropReason = "keepequal rule did not match"
82 DropReasonInvalidMetricName DropReason = "resulting metric name is empty or invalid"
83 )
84
85 // DropInfo is the outcome of Apply. Dropped reports whether the sample was
86 // dropped; when it was, Reason says why and, for a rule-driven drop, RuleIndex
87 // and Action identify the rule. RuleIndex is -1 when the drop is not tied to a
88 // single rule (an invalid final metric name).
89 type DropInfo struct {
90 Reason DropReason
91 RuleIndex int
92 Action Action
93 }
94
95 // Dropped reports whether the sample was dropped.
96 func (d DropInfo) Dropped() bool { return d.Reason != DropReasonNone }
97
98 // DropObserver is called by the SampleTransform returned from NewTransform for
99 // each dropped sample. Implementations SHOULD log the reason/rule/action but
100 // MUST NOT log label values (cardinality and privacy).
101 type DropObserver func(sample prompkg.Sample, drop DropInfo)
102
103 // Config is one relabeling rule. Construct it directly using the Action constants
104 // and exported fields; rules are validated (and the Action canonicalized) by New.
105 //
106 // The unexported separatorSet/replacementSet/sourceLabelsSet fields distinguish an
107 // explicitly-empty field from an unset one (they are read by withDefaults, validate
108 // and applyReplace). They are settable only within this package; callers outside the
109 // package cannot express explicit-empty Separator/Replacement/SourceLabels via normal
110 // struct literals or standard YAML/JSON unmarshaling (unexported fields are ignored),
111 // so a dedicated config loader must set them when that behavior is needed.
112 type Config struct {
113 SourceLabels []string
114 Separator string
115 Regex Regexp
116 Modulus uint64
117 TargetLabel string
118 Replacement string
119 Action Action
120 NameScheme commonmodel.ValidationScheme
121
122 separatorSet bool
123 replacementSet bool
124 sourceLabelsSet bool
125 }
126
127 // Regexp is a relabel regular expression: a regexp.Regexp compiled fully anchored
128 // (see NewRegexp). The zero value has no pattern; build one with NewRegexp or
129 // MustNewRegexp. String returns the original, un-anchored source.
130 type Regexp struct {
131 *regexp.Regexp
132 original string // un-anchored source passed to NewRegexp; returned by String
133 }
134
135 // Processor applies an ordered list of rules to samples. It reuses internal
136 // buffers across calls, so a Processor is single-threaded per scrape and is NOT
137 // goroutine-safe.
138 type Processor struct {
139 cfgs []Config
140
141 builder *labels.Builder
142 join strings.Builder
143 buf []byte
144 rangeBuf []labels.Label
145
146 currentName string
147 currentNameScheme commonmodel.ValidationScheme
148 labelsChanged bool
149 nameChanged bool
150 }
151
152 // New validates and compiles the rules into a Processor.
153 func New(cfgs []Config) (*Processor, error) {
154 compiled, err := normalizeAndValidateConfigs(cfgs)
155 if err != nil {
156 return nil, err
157 }
158
159 return &Processor{
160 cfgs: compiled,
161 builder: labels.NewBuilder(nil),
162 }, nil
163 }
164
165 // NewTransform builds a prompkg.SampleTransform from the rules. It returns a nil
166 // transform when there are no rules, so the scraper keeps its no-transform fast
167 // path. onDrop, if non-nil, is called for each dropped sample. The returned
168 // transform closes over a single reusable Processor, so it is NOT goroutine-safe;
169 // use one transform per scrape goroutine.
170 func NewTransform(cfgs []Config, onDrop DropObserver) (prompkg.SampleTransform, error) {
171 p, err := New(cfgs)
172 if err != nil {
173 return nil, err
174 }
175 if len(p.cfgs) == 0 {
176 return nil, nil
177 }
178
179 return func(s prompkg.Sample) (prompkg.Sample, bool, error) {
180 out, drop := p.Apply(s)
181 if drop.Dropped() {
182 if onDrop != nil {
183 onDrop(out, drop)
184 }
185 return prompkg.Sample{}, false, nil
186 }
187 return out, true, nil
188 }, nil
189 }
190
191 func normalizeAndValidateConfigs(cfgs []Config) ([]Config, error) {
192 compiled := make([]Config, 0, len(cfgs))
193 for i, cfg := range cfgs {
194 cfg = withDefaults(cfg)
195 if err := cfg.validate(); err != nil {
196 return nil, fmt.Errorf("rule %d: %w", i, err)
197 }
198 compiled = append(compiled, cfg)
199 }
200 return compiled, nil
201 }
202
203 func (c Config) validate() error {
204 c = withDefaults(c)
205
206 if _, err := parseAction(string(c.Action)); err != nil {
207 return err
208 }
209
210 if err := validateNameScheme(c.NameScheme); err != nil {
211 return err
212 }
213
214 scheme := c.NameScheme
215 if scheme == commonmodel.UnsetValidation {
216 scheme = defaultNameValidationScheme
217 }
218
219 if c.Modulus == 0 && c.Action == HashMod {
220 return errors.New("relabel configuration for hashmod requires non-zero modulus")
221 }
222 if needsTargetLabel(c.Action) && c.TargetLabel == "" {
223 return fmt.Errorf("relabel configuration for %s action requires 'target_label' value", c.Action)
224 }
225
226 if c.Action == Replace && !varInRegexTemplate(c.TargetLabel) && !scheme.IsValidLabelName(c.TargetLabel) {
227 return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
228 }
229 if c.Action == Replace && varInRegexTemplate(c.TargetLabel) && !isValidLabelNameWithRegexVar(c.TargetLabel, scheme) {
230 return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
231 }
232 if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) &&
233 !scheme.IsValidLabelName(c.TargetLabel) {
234 return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
235 }
236 if (c.Action == Lowercase || c.Action == Uppercase || c.Action == KeepEqual || c.Action == DropEqual) &&
237 c.Replacement != defaultConfig.Replacement {
238 return fmt.Errorf("'replacement' can not be set for %s action", c.Action)
239 }
240 if c.Action == LabelMap && !isValidLabelNameWithRegexVar(c.Replacement, scheme) {
241 return fmt.Errorf("%q is invalid 'replacement' for %s action", c.Replacement, c.Action)
242 }
243 if c.Action == HashMod && !scheme.IsValidLabelName(c.TargetLabel) {
244 return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action)
245 }
246 if c.Action == DropEqual || c.Action == KeepEqual {
247 if c.Regex.String() != defaultConfig.Regex.String() ||
248 c.Modulus != defaultConfig.Modulus ||
249 c.Separator != defaultConfig.Separator ||
250 c.Replacement != defaultConfig.Replacement {
251 return fmt.Errorf("%s action requires only 'source_labels' and 'target_label', and no other fields", c.Action)
252 }
253 }
254 if c.Action == LabelDrop || c.Action == LabelKeep {
255 if c.sourceLabelsSet ||
256 len(c.SourceLabels) > 0 ||
257 c.TargetLabel != defaultConfig.TargetLabel ||
258 c.Modulus != defaultConfig.Modulus ||
259 c.Separator != defaultConfig.Separator ||
260 c.Replacement != defaultConfig.Replacement {
261 return fmt.Errorf("%s action requires only 'regex', and no other fields", c.Action)
262 }
263 }
264
265 return nil
266 }
267
268 func NewRegexp(s string) (Regexp, error) {
269 re, err := regexp.Compile("^(?s:" + s + ")$")
270 return Regexp{Regexp: re, original: s}, err
271 }
272
273 func MustNewRegexp(s string) Regexp {
274 re, err := NewRegexp(s)
275 if err != nil {
276 panic(err)
277 }
278 return re
279 }
280
281 // String returns the original, un-anchored pattern passed to NewRegexp. It returns
282 // "" for the zero value or any Regexp not built via NewRegexp, and never inspects the
283 // compiled form, so it is safe on a Regexp wrapping an arbitrary *regexp.Regexp.
284 func (re Regexp) String() string {
285 return re.original
286 }
287
288 // Apply runs the rules against one sample. It returns the (possibly mutated)
289 // sample and a DropInfo. When DropInfo.Dropped() is true the sample must be
290 // discarded; the returned sample is the original (unmutated) so the caller can
291 // log its name. Value, Kind and FamilyType are passed through unchanged — a
292 // relabeled sample is never re-typed.
293 func (p *Processor) Apply(sample prompkg.Sample) (prompkg.Sample, DropInfo) {
294 if len(p.cfgs) == 0 {
295 return sample, DropInfo{}
296 }
297
298 p.builder.Reset(sample.Labels)
299 p.currentName = sample.Name
300 p.currentNameScheme = defaultNameValidationScheme
301 p.labelsChanged = false
302 p.nameChanged = false
303
304 for i := range p.cfgs {
305 if keep, drop := p.applyConfig(&p.cfgs[i], i); !keep {
306 return sample, drop
307 }
308 }
309
310 if !p.currentNameScheme.IsValidMetricName(p.currentName) {
311 return sample, DropInfo{Reason: DropReasonInvalidMetricName, RuleIndex: -1}
312 }
313
314 if !p.nameChanged && !p.labelsChanged {
315 return sample, DropInfo{}
316 }
317
318 sample.Name = p.currentName
319 if p.labelsChanged {
320 sample.Labels = p.builder.Labels()
321 }
322 return sample, DropInfo{}
323 }
324
325 func (p *Processor) applyConfig(cfg *Config, idx int) (bool, DropInfo) {
326 val := p.joinSourceLabels(cfg.SourceLabels, cfg.Separator)
327
328 switch cfg.Action {
329 case Drop:
330 if cfg.Regex.MatchString(val) {
331 return false, DropInfo{Reason: DropReasonDropRuleMatched, RuleIndex: idx, Action: Drop}
332 }
333 case Keep:
334 if !cfg.Regex.MatchString(val) {
335 return false, DropInfo{Reason: DropReasonKeepRuleMismatch, RuleIndex: idx, Action: Keep}
336 }
337 case DropEqual:
338 if p.getLabel(cfg.TargetLabel) == val {
339 return false, DropInfo{Reason: DropReasonDropEqualMatched, RuleIndex: idx, Action: DropEqual}
340 }
341 case KeepEqual:
342 if p.getLabel(cfg.TargetLabel) != val {
343 return false, DropInfo{Reason: DropReasonKeepEqualMismatch, RuleIndex: idx, Action: KeepEqual}
344 }
345 case Replace:
346 p.applyReplace(cfg, val)
347 case Lowercase:
348 p.setLabel(cfg.TargetLabel, strings.ToLower(val), cfg.NameScheme)
349 case Uppercase:
350 p.setLabel(cfg.TargetLabel, strings.ToUpper(val), cfg.NameScheme)
351 case HashMod:
352 hash := md5.Sum([]byte(val))
353 mod := binary.BigEndian.Uint64(hash[8:]) % cfg.Modulus
354 p.setLabel(cfg.TargetLabel, strconv.FormatUint(mod, 10), cfg.NameScheme)
355 case LabelMap:
356 p.rangeLabels(func(l labels.Label) {
357 if cfg.Regex.MatchString(l.Name) {
358 p.setLabel(cfg.Regex.ReplaceAllString(l.Name, cfg.Replacement), l.Value, cfg.NameScheme)
359 }
360 })
361 case LabelDrop:
362 p.rangeLabels(func(l labels.Label) {
363 if cfg.Regex.MatchString(l.Name) {
364 p.delLabel(l.Name)
365 }
366 })
367 case LabelKeep:
368 p.rangeLabels(func(l labels.Label) {
369 if !cfg.Regex.MatchString(l.Name) {
370 p.delLabel(l.Name)
371 }
372 })
373 default:
374 panic(fmt.Errorf("unknown relabel action %q", cfg.Action))
375 }
376
377 return true, DropInfo{}
378 }
379
380 func (p *Processor) applyReplace(cfg *Config, val string) {
381 if val == "" &&
382 cfg.Regex.String() == defaultConfig.Regex.String() &&
383 !varInRegexTemplate(cfg.TargetLabel) &&
384 !varInRegexTemplate(cfg.Replacement) {
385 p.setLabel(cfg.TargetLabel, cfg.Replacement, cfg.NameScheme)
386 return
387 }
388
389 indexes := cfg.Regex.FindStringSubmatchIndex(val)
390 if indexes == nil {
391 return
392 }
393
394 p.buf = cfg.Regex.ExpandString(p.buf[:0], cfg.TargetLabel, val, indexes)
395 target := string(p.buf)
396 if !cfg.NameScheme.IsValidLabelName(target) {
397 return
398 }
399
400 p.buf = cfg.Regex.ExpandString(p.buf[:0], cfg.Replacement, val, indexes)
401 if len(p.buf) == 0 {
402 p.delLabel(target)
403 return
404 }
405
406 p.setLabel(target, string(p.buf), cfg.NameScheme)
407 }
408
409 func (p *Processor) joinSourceLabels(sourceLabels []string, separator string) string {
410 switch len(sourceLabels) {
411 case 0:
412 return ""
413 case 1:
414 return p.getLabel(sourceLabels[0])
415 }
416
417 p.join.Reset()
418 for i, name := range sourceLabels {
419 if i > 0 {
420 p.join.WriteString(separator)
421 }
422 p.join.WriteString(p.getLabel(name))
423 }
424
425 return p.join.String()
426 }
427
428 func (p *Processor) getLabel(name string) string {
429 if name == commonmodel.MetricNameLabel {
430 return p.currentName
431 }
432 return p.builder.Get(name)
433 }
434
435 func (p *Processor) setLabel(name, value string, scheme commonmodel.ValidationScheme) {
436 if name == commonmodel.MetricNameLabel {
437 p.currentNameScheme = scheme
438 if p.currentName != value {
439 p.currentName = value
440 p.nameChanged = true
441 }
442 return
443 }
444
445 if current, ok := p.lookupLabel(name); ok && current == value {
446 return
447 }
448
449 p.builder.Set(name, value)
450 p.labelsChanged = true
451 }
452
453 func (p *Processor) delLabel(name string) {
454 if name == commonmodel.MetricNameLabel {
455 if p.currentName != "" {
456 p.currentName = ""
457 p.nameChanged = true
458 }
459 return
460 }
461
462 if _, ok := p.lookupLabel(name); !ok {
463 return
464 }
465
466 p.builder.Del(name)
467 p.labelsChanged = true
468 }
469
470 func (p *Processor) rangeLabels(fn func(labels.Label)) {
471 // Snapshot the current label set (including __name__) before invoking fn, so a
472 // callback that adds labels (labelmap) does not re-process labels it creates in
473 // the same rule — matching Prometheus, which ranges one snapshot per rule. The
474 // scratch buffer is reused across calls.
475 p.rangeBuf = p.rangeBuf[:0]
476 if p.currentName != "" {
477 p.rangeBuf = append(p.rangeBuf, labels.Label{Name: commonmodel.MetricNameLabel, Value: p.currentName})
478 }
479 p.builder.Range(func(l labels.Label) {
480 p.rangeBuf = append(p.rangeBuf, l)
481 })
482 for _, l := range p.rangeBuf {
483 fn(l)
484 }
485 }
486
487 func (p *Processor) lookupLabel(name string) (string, bool) {
488 if name == commonmodel.MetricNameLabel {
489 if p.currentName == "" {
490 return "", false
491 }
492 return p.currentName, true
493 }
494
495 var (
496 value string
497 ok bool
498 )
499 p.builder.Range(func(l labels.Label) {
500 if l.Name == name {
501 value = l.Value
502 ok = true
503 }
504 })
505 return value, ok
506 }
507
508 func parseAction(s string) (Action, error) {
509 switch act := Action(strings.ToLower(s)); act {
510 case Replace, Keep, Drop, KeepEqual, DropEqual, HashMod, LabelMap, LabelDrop, LabelKeep, Lowercase, Uppercase:
511 return act, nil
512 default:
513 return "", fmt.Errorf("unknown relabel action %q", s)
514 }
515 }
516
517 func withDefaults(cfg Config) Config {
518 cfg.NameScheme = withNameScheme(cfg.NameScheme)
519 if cfg.Action == "" {
520 cfg.Action = defaultConfig.Action
521 } else if act, err := parseAction(string(cfg.Action)); err == nil {
522 // Canonicalize a valid action (e.g. "KEEP" -> "keep") so Apply's switch
523 // matches; an invalid action is left for validate to reject.
524 cfg.Action = act
525 }
526 if !cfg.separatorSet && cfg.Separator == "" {
527 cfg.Separator = defaultConfig.Separator
528 }
529 if cfg.Regex.Regexp == nil {
530 cfg.Regex = defaultConfig.Regex
531 }
532 if !cfg.replacementSet && cfg.Replacement == "" {
533 cfg.Replacement = defaultConfig.Replacement
534 }
535 return cfg
536 }
537
538 func withNameScheme(scheme commonmodel.ValidationScheme) commonmodel.ValidationScheme {
539 if scheme == commonmodel.UnsetValidation {
540 return defaultNameValidationScheme
541 }
542 return scheme
543 }
544
545 func validateNameScheme(scheme commonmodel.ValidationScheme) error {
546 switch scheme {
547 case commonmodel.UnsetValidation, commonmodel.LegacyValidation, commonmodel.UTF8Validation:
548 return nil
549 default:
550 return fmt.Errorf("unknown relabel config name validation method specified, must be either '', 'legacy' or 'utf8', got %s", scheme)
551 }
552 }
553
554 func needsTargetLabel(action Action) bool {
555 return action == Replace || action == HashMod || action == Lowercase || action == Uppercase || action == KeepEqual || action == DropEqual
556 }
557
558 func isValidLabelNameWithRegexVar(value string, scheme commonmodel.ValidationScheme) bool {
559 if scheme == commonmodel.UTF8Validation {
560 return scheme.IsValidLabelName(value)
561 }
562 return relabelTargetLegacy.MatchString(value)
563 }
564
565 func varInRegexTemplate(template string) bool {
566 return strings.Contains(template, "$")
567 }