@cryptotaxi247 / netdata-1 / commits / 10cad04d2

Use chart labels to filter alerts (#14982)

* use chart labels to filter alerts * add entry to readme * support chart_label=val val2 val3 * docs updates * more docs * use rc not rt

Emmanuel Vasilakis committed May 22, 2023 at 14:14 UTC 10cad04d2d50261dae32b93bd5a5f2dfac5ceb5c

7 files changed +148 -2

database/rrdcalc.c

@@ -369,6 +369,10 @@ static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) {

369 st->rrdhost->rrdlabels, rc->host_labels_pattern, '=', NULL))

370 return false;

371

372 + if (st->rrdlabels && rc->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(

373 + st->rrdlabels, rc->chart_labels_pattern, '=', NULL))

374 + return false;

375 +

376 return true;

377 }

378

@@ -605,11 +609,13 @@ static void rrdcalc_free_internals(RRDCALC *rc) {

609 string_freez(rc->host_labels);

610 string_freez(rc->module_match);

611 string_freez(rc->plugin_match);

612 + string_freez(rc->chart_labels);

613

614 simple_pattern_free(rc->foreach_dimension_pattern);

615 simple_pattern_free(rc->host_labels_pattern);

616 simple_pattern_free(rc->module_pattern);

617 simple_pattern_free(rc->plugin_pattern);

618 + simple_pattern_free(rc->chart_labels_pattern);

619 }

620

621 static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdhost __maybe_unused) {

database/rrdcalc.h

@@ -109,6 +109,9 @@ struct rrdcalc {

109 STRING *host_labels; // the label read from an alarm file

110 SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels

111

112 + STRING *chart_labels; // the chart label read from an alarm file

113 + SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels

114 +

115 // ------------------------------------------------------------------------

116 // runtime information

117

@@ -168,6 +171,7 @@ struct rrdcalc {

171 #define rrdcalc_dimensions(rc) string2str((rc)->dimensions)

172 #define rrdcalc_foreachdim(rc) string2str((rc)->foreach_dimension)

173 #define rrdcalc_host_labels(rc) string2str((rc)->host_labels)

174 +#define rrdcalc_chart_labels(rc) string2str((rc)->chart_labels)

175

176 #define foreach_rrdcalc_in_rrdhost_read(host, rc) \

177 dfe_start_read((host)->rrdcalc_root_index, rc) \

@@ -206,6 +210,7 @@ struct alert_config {

210 STRING *options;

211 STRING *repeat;

212 STRING *host_labels;

213 + STRING *chart_labels;

214

215 STRING *p_db_lookup_dimensions;

216 STRING *p_db_lookup_method;

database/rrdcalctemplate.c

@@ -51,6 +51,11 @@ bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RR

51 '=', NULL))

52 return false;

54 + if(st->rrdlabels && rt->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdlabels,

55 + rt->chart_labels_pattern,

56 + '=', NULL))

57 + return false;

58 +

59 return true;

60 }

@@ -120,8 +125,10 @@ static void rrdcalctemplate_free_internals(RRDCALCTEMPLATE *rt) {

125 string_freez(rt->dimensions);

126 string_freez(rt->foreach_dimension);

127 string_freez(rt->host_labels);

128 + string_freez(rt->chart_labels);

129 simple_pattern_free(rt->foreach_dimension_pattern);

130 simple_pattern_free(rt->host_labels_pattern);

131 + simple_pattern_free(rt->chart_labels_pattern);

132 }

133

134 void rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(RRDCALCTEMPLATE *rt) {

database/rrdcalctemplate.h

@@ -74,6 +74,9 @@ struct rrdcalctemplate {

74 STRING *host_labels; // the label read from an alarm file

75 SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels

77 + STRING *chart_labels; // the chart label read from an alarm file

78 + SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels

79 +

80 // ------------------------------------------------------------------------

81 // expressions related to the alarm

@@ -107,6 +110,7 @@ struct rrdcalctemplate {

110 #define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions)

111 #define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreach_dimension)

112 #define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels)

113 +#define rrdcalctemplate_chart_labels(rt) string2str((rt)->chart_labels)

114

115 #define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after)

116

database/sqlite/sqlite_health.c

@@ -1070,6 +1070,7 @@ int alert_hash_and_store_config(

1070 DIGEST_ALERT_CONFIG_VAL(cfg->options);

1071 DIGEST_ALERT_CONFIG_VAL(cfg->repeat);

1072 DIGEST_ALERT_CONFIG_VAL(cfg->host_labels);

1073 + DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels);

1074

1075 EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);

1076 EVP_MD_CTX_destroy(evpctx);

health/REFERENCE.md

+34 -1

@@ -241,7 +241,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation

241 | [`delay`](#alarm-line-delay) | no | Optional hysteresis settings to prevent floods of notifications. |

242 | [`repeat`](#alarm-line-repeat) | no | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode. |

243 | [`options`](#alarm-line-options) | no | Add an option to not clear alarms. |

244 -| [`host labels`](#alarm-line-host-labels) | no | List of labels present on a host. |

244 +| [`host labels`](#alarm-line-host-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. |

245 +| [`chart labels`](#alarm-line-chart-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. |

246 | [`info`](#alarm-line-info) | no | A brief description of the alarm. |

247

248 The `alarm` or `template` line must be the first line of any entity.

@@ -446,6 +447,9 @@ For example, you can create a template on the `disk.io` context, but filter it t

447 families: sda sdb

448 ```

449

450 +Please note that the use of the `families` filter is planned to be deprecated in upcoming Netdata releases.

451 +Please use [`chart labels`](#alarm-line-chart-labels) instead.

452 +

453 #### Alarm line `lookup`

454

455 This line makes a database lookup to find a value. This result of this lookup is available as `$this`.

@@ -696,6 +700,35 @@ host labels: installed = 201*

700

701 See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.

702

703 +#### Alarm line `chart labels`

704 +

705 +Similar to host labels, the `chart labels` key can be used to filter if an alarm will load or not for a specific chart, based on

706 +whether these chart labels match or not.

707 +

708 +The list of chart labels present on each chart can be obtained from http://localhost:19999/api/v1/charts?all

709 +

710 +For example, each `disk_space` chart defines a chart label called `mount_point` with each instance of this chart having

711 +a value there of which mount point it monitors.

712 +

713 +If you have an e.g. external disk mounted on `/mnt/disk1` and you don't wish any related disk space alerts running for

714 +it (but you do for all other mount points), you can add the following to the alert's configuration:

715 +

716 +```yaml

717 +chart labels: mount_point=!/mnt/disk1 *`

718 +```

719 +

720 +The `chart labels` is a space-separated list that accepts simple patterns. If you use multiple different chart labels,

721 +then the result is an OR between them. i.e. the following:

722 +

723 +```yaml

724 +chart labels: mount_point=/mnt/disk1 device=sda`

725 +```

726 +

727 +Will create the alert if the `mount_point` is `/mnt/disk1` or the `device` is `sda`. Furthermore, if a chart label name

728 +is specified that does not exist in the chart, the chart won't be matched.

729 +

730 +See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.

731 +

732 #### Alarm line `info`

733

734 The info field can contain a small piece of text describing the alarm or template. This will be rendered in

health/health_config.c

+91 -1

@@ -32,6 +32,7 @@

32 #define HEALTH_REPEAT_KEY "repeat"

33 #define HEALTH_HOST_LABEL_KEY "host labels"

34 #define HEALTH_FOREACH_KEY "foreach"

35 +#define HEALTH_CHART_LABEL_KEY "chart labels"

37 static inline int health_parse_delay(

38 size_t line, const char *filename, char *string,

@@ -192,6 +193,49 @@ static inline int isvariableterm(const char s) {

193 return 1;

194 }

195

196 +// If needed, add a prefix key to all possible values in the range

197 +static inline char *health_config_add_key_to_values(char *value) {

198 + BUFFER *wb = buffer_create(HEALTH_CONF_MAX_LINE + 1, NULL);

199 + char key[HEALTH_CONF_MAX_LINE + 1];

200 + char data[HEALTH_CONF_MAX_LINE + 1];

201 +

202 + char *s = value;

203 + size_t i = 0;

204 +

205 + while(*s) {

206 + if (*s == '=') {

207 + //hold the key

208 + data[i]='\0';

209 + strncpyz(key, data, HEALTH_CONF_MAX_LINE);

210 + i=0;

211 + } else if (*s == ' ') {

212 + data[i]='\0';

213 + if (data[0]=='!')

214 + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);

215 + else

216 + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);

217 + i=0;

218 + } else {

219 + data[i++] = *s;

220 + }

221 + s++;

222 + }

223 +

224 + data[i]='\0';

225 + if (data[0]) {

226 + if (data[0]=='!')

227 + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);

228 + else

229 + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);

230 + }

231 +

232 + char *final = mallocz(HEALTH_CONF_MAX_LINE + 1);

233 + strncpyz(final, buffer_tostring(wb), HEALTH_CONF_MAX_LINE);

234 + buffer_free(wb);

235 +

236 + return final;

237 +}

238 +

239 static inline void parse_variables_and_store_in_health_rrdvars(char *value, size_t len) {

240 const char *s = value;

241 char buffer[RRDVAR_MAX_LENGTH];

@@ -453,6 +497,7 @@ static inline void alert_config_free(struct alert_config *cfg)

497 string_freez(cfg->host_labels);

498 string_freez(cfg->p_db_lookup_dimensions);

499 string_freez(cfg->p_db_lookup_method);

500 + string_freez(cfg->chart_labels);

501 freez(cfg);

502 }

503

@@ -489,7 +534,8 @@ static int health_readfile(const char *filename, void *data) {

534 hash_delay = 0,

535 hash_options = 0,

536 hash_repeat = 0,

492 - hash_host_label = 0;

537 + hash_host_label = 0,

538 + hash_chart_label = 0;

539

540 char buffer[HEALTH_CONF_MAX_LINE + 1];

541

@@ -521,6 +567,7 @@ static int health_readfile(const char *filename, void *data) {

567 hash_options = simple_uhash(HEALTH_OPTIONS_KEY);

568 hash_repeat = simple_uhash(HEALTH_REPEAT_KEY);

569 hash_host_label = simple_uhash(HEALTH_HOST_LABEL_KEY);

570 + hash_chart_label = simple_uhash(HEALTH_CHART_LABEL_KEY);

571 }

572

573 FILE *fp = fopen(filename, "r");

@@ -937,6 +984,27 @@ static int health_readfile(const char *filename, void *data) {

984 rc->module_match = string_strdupz(value);

985 rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true);

986 }

987 + else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {

988 + alert_cfg->chart_labels = string_strdupz(value);

989 + if(rc->chart_labels) {

990 + if(strcmp(rrdcalc_chart_labels(rc), value) != 0)

991 + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",

992 + line, filename, rrdcalc_name(rc), key, value, value);

993 +

994 + string_freez(rc->chart_labels);

995 + simple_pattern_free(rc->chart_labels_pattern);

996 + }

997 +

998 + {

999 + char *tmp = simple_pattern_trim_around_equal(value);

1000 + char *tmp_2 = health_config_add_key_to_values(tmp);

1001 + rc->chart_labels = string_strdupz(tmp_2);

1002 + freez(tmp);

1003 + freez(tmp_2);

1004 + }

1005 + rc->chart_labels_pattern = simple_pattern_create(rrdcalc_chart_labels(rc), NULL, SIMPLE_PATTERN_EXACT,

1006 + true);

1007 + }

1008 else {

1009 error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",

1010 line, filename, rrdcalc_name(rc), key);

@@ -1186,9 +1254,31 @@ static int health_readfile(const char *filename, void *data) {

1254 rt->host_labels = string_strdupz(tmp);

1255 freez(tmp);

1256 }

1257 +

1258 rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL,

1259 SIMPLE_PATTERN_EXACT, true);

1260 }

1261 + else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {

1262 + alert_cfg->chart_labels = string_strdupz(value);

1263 + if(rt->chart_labels) {

1264 + if(strcmp(rrdcalctemplate_chart_labels(rt), value) != 0)

1265 + error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",

1266 + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_chart_labels(rt), value, value);

1267 +

1268 + string_freez(rt->chart_labels);

1269 + simple_pattern_free(rt->chart_labels_pattern);

1270 + }

1271 +

1272 + {

1273 + char *tmp = simple_pattern_trim_around_equal(value);

1274 + char *tmp_2 = health_config_add_key_to_values(tmp);

1275 + rt->chart_labels = string_strdupz(tmp_2);

1276 + freez(tmp);

1277 + freez(tmp_2);

1278 + }

1279 + rt->chart_labels_pattern = simple_pattern_create(rrdcalctemplate_chart_labels(rt), NULL,

1280 + SIMPLE_PATTERN_EXACT, true);

1281 + }

1282 else {

1283 error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",

1284 line, filename, rrdcalctemplate_name(rt), key);