Add a health configuration option of what alarms to load (#14150)
* use a simple pattern to choose what alerts to load * rename config to alarms * add config entry doc * fix link * change wording * change wording 2 * add to analytics * build a list of disabled alarms for analytics * add silenced alarms * remove analytics related * remove analytics related 2
Emmanuel Vasilakis committed
Dec 19, 2022 at 13:52 UTC
e6c04cb4de6387c7ba1dbf80076ca4bd7578237b
4 files changed
+60
-48
daemon/config/README.md
+1
@@ -175,6 +175,7 @@ monitoring](/health/README.md).
175
| run at least every seconds | `10` | Controls how often all alarm conditions should be evaluated. |
176
| postpone alarms during hibernation for seconds | `60` | Prevents false alarms. May need to be increased if you get alarms during hibernation. |
177
| rotate log every lines | 2000 | Controls the number of alarm log entries stored in `<lib directory>/health-log.db`, where `<lib directory>` is the one configured in the [\[global\] section](#global-section-options) |
178
+| enabled alarms | * | Defines which alarms to load from both user and stock directories. This is a [simple pattern](/libnetdata/simple_pattern/README.md) list of alarm or template names. Can be used to disable specific alarms. For example, `enabled alarms = !oom_kill *` will load all alarms except `oom_kill`. |
179
180
### [web] section options
181
health/health.c
+3
-1
@@ -159,6 +159,7 @@ static bool prepare_command(BUFFER *wb,
159
160
unsigned int default_health_enabled = 1;
161
char *silencers_filename;
162
+SIMPLE_PATTERN *conf_enabled_alarms = NULL;
163
164
// the queue of executed alarm notifications that haven't been waited for yet
165
static __thread struct {
@@ -769,6 +770,8 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
770
else
771
host->health_log.max = (unsigned int)n;
772
773
+ conf_enabled_alarms = simple_pattern_create(config_get(CONFIG_SECTION_HEALTH, "enabled alarms", "*"), NULL, SIMPLE_PATTERN_EXACT);
774
+
775
netdata_rwlock_init(&host->health_log.alarm_log_rwlock);
776
777
char filename[FILENAME_MAX + 1];
@@ -1553,7 +1556,6 @@ void *health_main(void *ptr) {
1556
1557
void health_add_host_labels(void) {
1558
DICTIONARY *labels = localhost->rrdlabels;
1556
- enum rrdlabel_source src;
1559
1560
// The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).
1561
// Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.
health/health.h
+1
@@ -31,6 +31,7 @@ extern unsigned int default_health_enabled;
31
#define HEALTH_SILENCERS_MAX_FILE_LEN 10000
32
33
extern char *silencers_filename;
34
+extern SIMPLE_PATTERN *conf_enabled_alarms;
35
36
void health_init(void);
37
health/health_config.c
+55
-47
@@ -553,33 +553,37 @@ static int health_readfile(const char *filename, void *data) {
553
rt = NULL;
554
}
555
556
- rc = callocz(1, sizeof(RRDCALC));
557
- rc->next_event_id = 1;
558
-
559
- {
560
- char *tmp = strdupz(value);
561
- if(rrdvar_fix_name(tmp))
562
- error("Health configuration renamed alarm '%s' to '%s'", value, tmp);
563
-
564
- rc->name = string_strdupz(tmp);
565
- freez(tmp);
566
- }
567
-
568
- rc->source = health_source_file(line, filename);
569
- rc->green = NAN;
570
- rc->red = NAN;
571
- rc->value = NAN;
572
- rc->old_value = NAN;
573
- rc->delay_multiplier = 1.0;
574
- rc->old_status = RRDCALC_STATUS_UNINITIALIZED;
575
- rc->warn_repeat_every = host->health_default_warn_repeat_every;
576
- rc->crit_repeat_every = host->health_default_crit_repeat_every;
577
- if (alert_cfg)
578
- alert_config_free(alert_cfg);
579
- alert_cfg = callocz(1, sizeof(struct alert_config));
580
-
581
- alert_cfg->alarm = string_dup(rc->name);
582
- ignore_this = 0;
556
+ if (simple_pattern_matches(conf_enabled_alarms, value)) {
557
+ rc = callocz(1, sizeof(RRDCALC));
558
+ rc->next_event_id = 1;
559
+
560
+ {
561
+ char *tmp = strdupz(value);
562
+ if(rrdvar_fix_name(tmp))
563
+ error("Health configuration renamed alarm '%s' to '%s'", value, tmp);
564
+
565
+ rc->name = string_strdupz(tmp);
566
+ freez(tmp);
567
+ }
568
+
569
+ rc->source = health_source_file(line, filename);
570
+ rc->green = NAN;
571
+ rc->red = NAN;
572
+ rc->value = NAN;
573
+ rc->old_value = NAN;
574
+ rc->delay_multiplier = 1.0;
575
+ rc->old_status = RRDCALC_STATUS_UNINITIALIZED;
576
+ rc->warn_repeat_every = host->health_default_warn_repeat_every;
577
+ rc->crit_repeat_every = host->health_default_crit_repeat_every;
578
+ if (alert_cfg)
579
+ alert_config_free(alert_cfg);
580
+ alert_cfg = callocz(1, sizeof(struct alert_config));
581
+
582
+ alert_cfg->alarm = string_dup(rc->name);
583
+ ignore_this = 0;
584
+ } else {
585
+ rc = NULL;
586
+ }
587
}
588
else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
589
if(rc) {
@@ -599,29 +603,33 @@ static int health_readfile(const char *filename, void *data) {
603
rrdcalctemplate_add_from_config(host, rt);
604
}
605
602
- rt = callocz(1, sizeof(RRDCALCTEMPLATE));
606
+ if (simple_pattern_matches(conf_enabled_alarms, value)) {
607
+ rt = callocz(1, sizeof(RRDCALCTEMPLATE));
608
604
- {
605
- char *tmp = strdupz(value);
606
- if(rrdvar_fix_name(tmp))
607
- error("Health configuration renamed template '%s' to '%s'", value, tmp);
608
-
609
- rt->name = string_strdupz(tmp);
610
- freez(tmp);
611
- }
609
+ {
610
+ char *tmp = strdupz(value);
611
+ if(rrdvar_fix_name(tmp))
612
+ error("Health configuration renamed template '%s' to '%s'", value, tmp);
613
613
- rt->source = health_source_file(line, filename);
614
- rt->green = NAN;
615
- rt->red = NAN;
616
- rt->delay_multiplier = (float)1.0;
617
- rt->warn_repeat_every = host->health_default_warn_repeat_every;
618
- rt->crit_repeat_every = host->health_default_crit_repeat_every;
619
- if (alert_cfg)
620
- alert_config_free(alert_cfg);
621
- alert_cfg = callocz(1, sizeof(struct alert_config));
614
+ rt->name = string_strdupz(tmp);
615
+ freez(tmp);
616
+ }
617
623
- alert_cfg->template_key = string_dup(rt->name);
624
- ignore_this = 0;
618
+ rt->source = health_source_file(line, filename);
619
+ rt->green = NAN;
620
+ rt->red = NAN;
621
+ rt->delay_multiplier = (float)1.0;
622
+ rt->warn_repeat_every = host->health_default_warn_repeat_every;
623
+ rt->crit_repeat_every = host->health_default_crit_repeat_every;
624
+ if (alert_cfg)
625
+ alert_config_free(alert_cfg);
626
+ alert_cfg = callocz(1, sizeof(struct alert_config));
627
+
628
+ alert_cfg->template_key = string_dup(rt->name);
629
+ ignore_this = 0;
630
+ } else {
631
+ rt = NULL;
632
+ }
633
}
634
else if(hash == hash_os && !strcasecmp(key, HEALTH_OS_KEY)) {
635
char *os_match = value;