@cryptotaxi247 / netdata-1 / commits / 9f0fbff5b

Add a summary field to alerts (#15886)

* add a summary field to alerts * add summary field to db * rebase * better migration * rebase * change email notification * revert to silent * use macro * add the summary field to some alerts * add more summary fields * change migration function * add to postgres alerts * add summary to vernemq * more summary fields * more summary fields * fixes * add doc

Emmanuel Vasilakis committed Sep 19, 2023 at 15:35 UTC 9f0fbff5b8d014235b44d4460291d56128855f1d
39 files changed +423 -158
database/rrd.h
+3
@@ -1006,6 +1006,7 @@ struct alarm_entry {
1006
1007 STRING *source;
1008 STRING *units;
1009 + STRING *summary;
1010 STRING *info;
1011
1012 NETDATA_DOUBLE old_value;
@@ -1042,6 +1043,7 @@ struct alarm_entry {
1043 #define ae_recipient(ae) string2str((ae)->recipient)
1044 #define ae_source(ae) string2str((ae)->source)
1045 #define ae_units(ae) string2str((ae)->units)
1046 +#define ae_summary(ae) string2str((ae)->summary)
1047 #define ae_info(ae) string2str((ae)->info)
1048 #define ae_old_value_string(ae) string2str((ae)->old_value_string)
1049 #define ae_new_value_string(ae) string2str((ae)->new_value_string)
@@ -1064,6 +1066,7 @@ typedef struct health {
1066 uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
1067 uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
1068 unsigned int health_enabled; // 1 when this host has health enabled
1069 + bool use_summary_for_notifications; // whether or not to use the summary field as a subject for notifications
1070 } HEALTH;
1071
1072 // ----------------------------------------------------------------------------
database/rrdcalc.c
+24 -4
@@ -98,7 +98,7 @@ uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint3
98 }
99
100 // ----------------------------------------------------------------------------
101 -// RRDCALC replacing info text variables with RRDSET labels
101 +// RRDCALC replacing info/summary text variables with RRDSET labels
102
103 static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RRDCALC *rc) {
104 if (!line || !*line)
@@ -158,9 +158,17 @@ void rrdcalc_update_info_using_rrdset_labels(RRDCALC *rc) {
158 size_t labels_version = rrdlabels_version(rc->rrdset->rrdlabels);
159 if(rc->labels_version != labels_version) {
160
161 - STRING *old = rc->info;
162 - rc->info = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_info(rc), rc);
163 - string_freez(old);
161 + if (rc->original_info) {
162 + STRING *old = rc->info;
163 + rc->info = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_info(rc), rc);
164 + string_freez(old);
165 + }
166 +
167 + if (rc->original_summary) {
168 + STRING *old = rc->summary;
169 + rc->summary = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_summary(rc), rc);
170 + string_freez(old);
171 + }
172
173 rc->labels_version = labels_version;
174 }
@@ -285,6 +293,11 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) {
293
294 rrdcalc_update_info_using_rrdset_labels(rc);
295
296 + if(!rc->summary) {
297 + rc->summary = string_dup(rc->name);
298 + rc->original_summary = string_dup(rc->name);
299 + }
300 +
301 time_t now = now_realtime_sec();
302
303 ALARM_ENTRY *ae = health_create_alarm_entry(
@@ -310,6 +323,7 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) {
323 rc->status,
324 rc->source,
325 rc->units,
326 + rc->summary,
327 rc->info,
328 0,
329 rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
@@ -356,6 +370,7 @@ static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) {
370 RRDCALC_STATUS_REMOVED,
371 rc->source,
372 rc->units,
373 + rc->summary,
374 rc->info,
375 0,
376 0);
@@ -512,6 +527,11 @@ static void rrdcalc_rrdhost_insert_callback(const DICTIONARY_ITEM *item __maybe_
527 rc->info = string_dup(rt->info);
528 rc->original_info = string_dup(rt->info);
529
530 + if (!rt->summary)
531 + rt->summary = string_dup(rc->name);
532 + rc->summary = string_dup(rt->summary);
533 + rc->original_summary = string_dup(rt->summary);
534 +
535 rc->classification = string_dup(rt->classification);
536 rc->component = string_dup(rt->component);
537 rc->type = string_dup(rt->type);
database/rrdcalc.h
+6 -1
@@ -64,8 +64,10 @@ struct rrdcalc {
64
65 STRING *source; // the source of this alarm
66 STRING *units; // the units of the alarm
67 + STRING *summary; // a short alert summary
68 + STRING *original_summary; // the original summary field before any variable replacement
69 STRING *original_info; // the original info field before any variable replacement
68 - STRING *info; // a short description of the alarm
70 + STRING *info; // a description of the alarm
71
72 int update_every; // update frequency for the alarm
73
@@ -170,6 +172,8 @@ struct rrdcalc {
172 #define rrdcalc_module_match(rc) string2str((rc)->module_match)
173 #define rrdcalc_source(rc) string2str((rc)->source)
174 #define rrdcalc_units(rc) string2str((rc)->units)
175 +#define rrdcalc_original_summary(rc) string2str((rc)->original_summary)
176 +#define rrdcalc_summary(rc) string2str((rc)->summary)
177 #define rrdcalc_original_info(rc) string2str((rc)->original_info)
178 #define rrdcalc_info(rc) string2str((rc)->info)
179 #define rrdcalc_dimensions(rc) string2str((rc)->dimensions)
@@ -206,6 +210,7 @@ struct alert_config {
210 STRING *exec;
211 STRING *to;
212 STRING *units;
213 + STRING *summary;
214 STRING *info;
215 STRING *classification;
216 STRING *component;
database/rrdcalctemplate.h
+3 -1
@@ -36,7 +36,8 @@ struct rrdcalctemplate {
36
37 STRING *source; // the source of this alarm
38 STRING *units; // the units of the alarm
39 - STRING *info; // a short description of the alarm
39 + STRING *summary; // a short summary of the alarm
40 + STRING *info; // a description of the alarm
41
42 int update_every; // update frequency for the alarm
43
@@ -105,6 +106,7 @@ struct rrdcalctemplate {
106 #define rrdcalctemplate_module_match(rt) string2str((rt)->module_match)
107 #define rrdcalctemplate_charts_match(rt) string2str((rt)->charts_match)
108 #define rrdcalctemplate_units(rt) string2str((rt)->units)
109 +#define rrdcalctemplate_summary(rt) string2str((rt)->summary)
110 #define rrdcalctemplate_info(rt) string2str((rt)->info)
111 #define rrdcalctemplate_source(rt) string2str((rt)->source)
112 #define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions)
database/sqlite/sqlite_db_migration.c
+24
@@ -88,6 +88,12 @@ const char *database_migrate_v10_v11[] = {
88 NULL
89 };
90
91 +const char *database_migrate_v11_v12[] = {
92 + "ALTER TABLE health_log_detail ADD summary TEXT;",
93 + "ALTER TABLE alert_hash ADD summary TEXT;",
94 + NULL
95 +};
96 +
97 static int do_migration_v1_v2(sqlite3 *database, const char *name)
98 {
99 UNUSED(name);
@@ -315,6 +321,23 @@ static int do_migration_v10_v11(sqlite3 *database, const char *name)
321 return 0;
322 }
323
324 +#define MIGR_11_12_UPD_HEALTH_LOG_DETAIL "UPDATE health_log_detail SET summary = (select name from health_log where health_log_id = health_log_detail.health_log_id);"
325 +static int do_migration_v11_v12(sqlite3 *database, const char *name)
326 +{
327 + int rc = 0;
328 +
329 + netdata_log_info("Running \"%s\" database migration", name);
330 +
331 + if (table_exists_in_database("health_log_detail") && !column_exists_in_table("health_log_detail", "summary") &&
332 + table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "summary"))
333 + rc = init_database_batch(database, &database_migrate_v11_v12[0]);
334 +
335 + if (!rc)
336 + sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL);
337 +
338 + return rc;
339 +}
340 +
341 static int do_migration_noop(sqlite3 *database, const char *name)
342 {
343 UNUSED(database);
@@ -369,6 +392,7 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = {
392 {.name = "v8 to v9", .func = do_migration_v8_v9},
393 {.name = "v9 to v10", .func = do_migration_v9_v10},
394 {.name = "v10 to v11", .func = do_migration_v10_v11},
395 + {.name = "v11 to v12", .func = do_migration_v11_v12},
396 // the terminator of this array
397 {.name = NULL, .func = NULL}
398 };
database/sqlite/sqlite_functions.c
+3 -3
@@ -4,7 +4,7 @@
4 #include "sqlite3recover.h"
5 #include "sqlite_db_migration.h"
6
7 -#define DB_METADATA_VERSION 11
7 +#define DB_METADATA_VERSION 12
8
9 const char *database_config[] = {
10 "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "
@@ -33,7 +33,7 @@ const char *database_config[] = {
33 "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "
34 "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
35 "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
36 - "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text);",
36 + "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text);",
37
38 "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "
39 "date_created INT, PRIMARY KEY(host_id, system_key));",
@@ -54,7 +54,7 @@ const char *database_config[] = {
54 "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "
55 "flags int, exec_run_timestamp int, delay_up_to_timestamp int, "
56 "info text, exec_code int, new_status real, old_status real, delay int, "
57 - "new_value double, old_value double, last_repeat int, transition_id blob, global_id int);",
57 + "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text);",
58
59 "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id);",
60 "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id);",
database/sqlite/sqlite_health.c
+22 -10
@@ -105,9 +105,8 @@ failed:
105 #define SQL_INSERT_HEALTH_LOG_DETAIL \
106 "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, " \
107 "updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, " \
108 - "info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id) " \
109 - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,@global_id); "
110 -
108 + "info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id, summary) " \
109 + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,@global_id,?); "
110 static void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
111 sqlite3_stmt *res = NULL;
112 int rc;
@@ -347,6 +346,12 @@ static void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
346 goto failed;
347 }
348
349 + rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->summary, 23);
350 + if (unlikely(rc != SQLITE_OK)) {
351 + error_report("Failed to bind summary parameter for SQL_INSERT_HEALTH_LOG_DETAIL");
352 + goto failed;
353 + }
354 +
355 rc = execute_insert(res);
356 if (unlikely(rc != SQLITE_DONE)) {
357 error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG_DETAIL, rc = %d", rrdhost_hostname(host), rc);
@@ -500,9 +505,9 @@ done:
505 #define SQL_INJECT_REMOVED \
506 "insert into health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, updated_by_id, updates_id, when_key, " \
507 "duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, info, exec_code, new_status, old_status, " \
503 - "delay, new_value, old_value, last_repeat, transition_id, global_id) " \
508 + "delay, new_value, old_value, last_repeat, transition_id, global_id, summary) " \
509 "select health_log_id, ?1, ?2, ?3, 0, ?4, unixepoch(), 0, 0, flags, exec_run_timestamp, unixepoch(), info, exec_code, -2, " \
505 - "new_status, delay, NULL, new_value, 0, ?5, now_usec(0) from health_log_detail where unique_id = ?6 and transition_id = ?7;"
510 + "new_status, delay, NULL, new_value, 0, ?5, now_usec(0), summary from health_log_detail where unique_id = ?6 and transition_id = ?7;"
511
512 #define SQL_INJECT_REMOVED_UPDATE_DETAIL "update health_log_detail set flags = flags | ?1, updated_by_id = ?2 where unique_id = ?3 and transition_id = ?4;"
513
@@ -742,7 +747,7 @@ void sql_check_removed_alerts_state(RRDHOST *host)
747 "hld.updates_id, hld.when_key, hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, " \
748 "hld.delay_up_to_timestamp, hl.name, hl.chart, hl.family, hl.exec, hl.recipient, ah.source, hl.units, " \
749 "hld.info, hld.exec_code, hld.new_status, hld.old_status, hld.delay, hld.new_value, hld.old_value, " \
745 - "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.global_id, hl.chart_name " \
750 + "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.global_id, hl.chart_name, hld.summary " \
751 "FROM health_log hl, alert_hash ah, health_log_detail hld " \
752 "WHERE hl.config_hash_id = ah.hash_id and hl.host_id = @host_id and hl.last_transition_id = hld.transition_id;"
753
@@ -890,6 +895,7 @@ void sql_health_alarm_log_load(RRDHOST *host)
895 ae->global_id = sqlite3_column_int64(res, 32);
896
897 ae->chart_name = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 33);
898 + ae->summary = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 34);
899
900 char value_string[100 + 1];
901 string_freez(ae->old_value_string);
@@ -940,8 +946,8 @@ void sql_health_alarm_log_load(RRDHOST *host)
946 "on_key, class, component, type, os, hosts, lookup, every, units, calc, families, plugin, module, " \
947 "charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \
948 "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \
943 - "p_db_lookup_before, p_update_every, source, chart_labels) values (?1,unixepoch(),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12," \
944 - "?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34,?35,?36);"
949 + "p_db_lookup_before, p_update_every, source, chart_labels, summary) values (?1,unixepoch(),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12," \
950 + "?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34,?35,?36,?37);"
951
952 int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
953 {
@@ -1129,6 +1135,10 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
1135 if (unlikely(rc != SQLITE_OK))
1136 goto bind_fail;
1137
1138 + rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->summary, ++param);
1139 + if (unlikely(rc != SQLITE_OK))
1140 + goto bind_fail;
1141 +
1142 rc = execute_insert(res);
1143 if (unlikely(rc != SQLITE_DONE))
1144 error_report("Failed to store alert config, rc = %d", rc);
@@ -1195,6 +1205,7 @@ int alert_hash_and_store_config(
1205 DIGEST_ALERT_CONFIG_VAL(cfg->repeat);
1206 DIGEST_ALERT_CONFIG_VAL(cfg->host_labels);
1207 DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels);
1208 + DIGEST_ALERT_CONFIG_VAL(cfg->summary);
1209
1210 EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
1211 EVP_MD_CTX_destroy(evpctx);
@@ -1275,8 +1286,8 @@ done:
1286 "hld.when_key, hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, " \
1287 "hld.delay_up_to_timestamp, hl.name, hl.chart, hl.family, hl.exec, hl.recipient, ah.source, " \
1288 "hl.units, hld.info, hld.exec_code, hld.new_status, hld.old_status, hld.delay, hld.new_value, hld.old_value, " \
1278 - "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id FROM health_log hl, " \
1279 - "alert_hash ah, health_log_detail hld WHERE hl.config_hash_id = ah.hash_id and " \
1289 + "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.summary " \
1290 + "FROM health_log hl, alert_hash ah, health_log_detail hld WHERE hl.config_hash_id = ah.hash_id and " \
1291 "hl.health_log_id = hld.health_log_id and hl.host_id = @host_id "
1292
1293 void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart) {
@@ -1424,6 +1435,7 @@ void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *
1435 (long unsigned int)sqlite3_column_int64(res, 26),
1436 (sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false");
1437
1438 + health_string2json(wb, "\t\t", "summary", (char *) sqlite3_column_text(res, 32), ",\n");
1439 health_string2json(wb, "\t\t", "info", (char *) sqlite3_column_text(res, 19), ",\n");
1440
1441 if(unlikely(sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
health/REFERENCE.md
+51 -15
@@ -41,18 +41,22 @@ Each health configuration file contains one or more health _entities_, which alw
41 For example, here is the first health entity in `health.d/cpu.conf`:
42
43 ```yaml
44 -template: 10min_cpu_usage
45 - on: system.cpu
46 - os: linux
47 - hosts: *
48 - lookup: average -10m unaligned of user,system,softirq,irq,guest
49 - units: %
50 - every: 1m
51 - warn: $this > (($status >= $WARNING) ? (75) : (85))
52 - crit: $this > (($status == $CRITICAL) ? (85) : (95))
53 - delay: down 15m multiplier 1.5 max 1h
54 - info: average cpu utilization for the last 10 minutes (excluding iowait, nice and steal)
55 - to: sysadmin
44 + template: 10min_cpu_usage
45 + on: system.cpu
46 + class: Utilization
47 + type: System
48 +component: CPU
49 + os: linux
50 + hosts: *
51 + lookup: average -10m unaligned of user,system,softirq,irq,guest
52 + units: %
53 + every: 1m
54 + warn: $this > (($status >= $WARNING) ? (75) : (85))
55 + crit: $this > (($status == $CRITICAL) ? (85) : (95))
56 + delay: down 15m multiplier 1.5 max 1h
57 + summary: CPU utilization
58 + info: Average cpu utilization for the last 10 minutes (excluding iowait, nice and steal)
59 + to: sysadmin
60 ```
61
62 To tune this alert to trigger warning and critical alerts at a lower CPU utilization, change the `warn` and `crit` lines
@@ -243,7 +247,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation
247 | [`options`](#alert-line-options) | no | Add an option to not clear alerts. |
248 | [`host labels`](#alert-line-host-labels) | no | Restrict an alert or template to a list of matching labels present on a host. |
249 | [`chart labels`](#alert-line-chart-labels) | no | Restrict an alert or template to a list of matching labels present on a host. |
246 -| [`info`](#alert-line-info) | no | A brief description of the alert. |
250 +| [`summary`](#alert-line-summary) | no | A brief description of the alert. |
251 +| [`info`](#alert-line-info) | no | A longer text field that provides more information of this alert |
252
253 The `alarm` or `template` line must be the first line of any entity.
254
@@ -729,13 +734,44 @@ is specified that does not exist in the chart, the chart won't be matched.
734
735 See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
736
737 +#### Alert line `summary`
738 +
739 +The summary field contains a brief title of the alert. It is used as the subject for the notifications, and in
740 +dashboard list of alerts. An example for the `ram_available` alert is:
741 +
742 +```yaml
743 +summary: Available Ram
744 +```
745 +
746 +summary fields can contain special variables in their text that will be replaced during run-time to provide more specific
747 +alert information. Current variables supported are:
748 +
749 +| variable | description |
750 +|---------------------|-------------------------------------------------------------------|
751 +| ${family} | Will be replaced by the family instance for the alert (e.g. eth0) |
752 +| ${label:LABEL_NAME} | The variable will be replaced with the value of the chart label |
753 +
754 +For example, a summry field like the following:
755 +
756 +```yaml
757 +summary: 1 minute received traffic overflow for ${label:device}
758 +```
759 +
760 +Will be rendered on the alert acting on interface `eth0` as:
761 +
762 +```yaml
763 +info: 1 minute received traffic overflow for ${label:device}
764 +```
765 +
766 +> Please note that variable names are case-sensitive.
767 +
768 #### Alert line `info`
769
770 The info field can contain a small piece of text describing the alert or template. This will be rendered in
771 notifications and UI elements whenever the specific alert is in focus. An example for the `ram_available` alert is:
772
773 ```yaml
738 -info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
774 +info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
775 ```
776
777 info fields can contain special variables in their text that will be replaced during run-time to provide more specific
@@ -744,7 +780,7 @@ alert information. Current variables supported are:
780 | variable | description |
781 |---------------------|-------------------------------------------------------------------|
782 | ${family} | Will be replaced by the family instance for the alert (e.g. eth0) |
747 -| ${label:LABEL_NAME} | The variable will be replaced with the value of the label |
783 +| ${label:LABEL_NAME} | The variable will be replaced with the value of the chart label |
784
785 For example, an info field like the following:
786
health/health.c
+12 -2
@@ -82,7 +82,8 @@ static bool prepare_command(BUFFER *wb,
82 const char *classification,
83 const char *edit_command,
84 const char *machine_guid,
85 - uuid_t *transition_id
85 + uuid_t *transition_id,
86 + const char *summary
87 ) {
88 char buf[8192];
89 size_t n = 8192 - 1;
@@ -195,6 +196,10 @@ static bool prepare_command(BUFFER *wb,
196 return false;
197 buffer_sprintf(wb, " '%s'", buf);
198
199 + if (!sanitize_command_argument_string(buf, summary, n))
200 + return false;
201 + buffer_sprintf(wb, " '%s'", buf);
202 +
203 return true;
204 }
205
@@ -581,7 +586,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
586 ae->classification?ae_classification(ae):"Unknown",
587 edit_command,
588 host->machine_guid,
584 - &ae->transition_id);
589 + &ae->transition_id,
590 + host->health.use_summary_for_notifications && ae->summary?ae_summary(ae):ae_name(ae));
591
592 const char *command_to_run = buffer_tostring(wb);
593 if (ok) {
@@ -835,6 +841,7 @@ static void initialize_health(RRDHOST *host)
841 snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
842 host->health.health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
843 host->health.health_default_recipient = string_strdupz("root");
844 + host->health.use_summary_for_notifications = config_get_boolean(CONFIG_SECTION_HEALTH, "use summary for notifications", CONFIG_BOOLEAN_YES);
845
846 sql_health_alarm_log_load(host);
847
@@ -1157,6 +1164,7 @@ void *health_main(void *ptr) {
1164 RRDCALC_STATUS_REMOVED,
1165 rc->source,
1166 rc->units,
1167 + rc->summary,
1168 rc->info,
1169 0,
1170 rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
@@ -1424,6 +1432,7 @@ void *health_main(void *ptr) {
1432 status,
1433 rc->source,
1434 rc->units,
1435 + rc->summary,
1436 rc->info,
1437 rc->delay_last,
1438 (
@@ -1511,6 +1520,7 @@ void *health_main(void *ptr) {
1520 rc->status,
1521 rc->source,
1522 rc->units,
1523 + rc->summary,
1524 rc->info,
1525 rc->delay_last,
1526 (
health/health.d/btrfs.conf
+18 -9
@@ -11,7 +11,8 @@ component: File system
11 every: 10s
12 warn: $this > (($status == $CRITICAL) ? (95) : (98))
13 delay: up 1m down 15m multiplier 1.5 max 1h
14 - info: percentage of allocated BTRFS physical disk space
14 + summary: BTRFS space allocated
15 + info: Percentage of allocated BTRFS physical disk space
16 to: silent
17
18 template: btrfs_data
@@ -27,7 +28,8 @@ component: File system
28 warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
29 crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
30 delay: up 1m down 15m multiplier 1.5 max 1h
30 - info: utilization of BTRFS data space
31 + summary: BTRFS space utilization
32 + info: Utilization of BTRFS data space
33 to: sysadmin
34
35 template: btrfs_metadata
@@ -43,7 +45,8 @@ component: File system
45 warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
46 crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
47 delay: up 1m down 15m multiplier 1.5 max 1h
46 - info: utilization of BTRFS metadata space
48 + summary: BTRFS metadata space utilization
49 + info: Utilization of BTRFS metadata space
50 to: sysadmin
51
52 template: btrfs_system
@@ -59,7 +62,8 @@ component: File system
62 warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98
63 crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98
64 delay: up 1m down 15m multiplier 1.5 max 1h
62 - info: utilization of BTRFS system space
65 + summary: BTRFS system space utilization
66 + info: Utilization of BTRFS system space
67 to: sysadmin
68
69 template: btrfs_device_read_errors
@@ -73,7 +77,8 @@ component: File system
77 lookup: max -10m every 1m of read_errs
78 warn: $this > 0
79 delay: up 1m down 15m multiplier 1.5 max 1h
76 - info: number of encountered BTRFS read errors
80 + summary: BTRFS device read errors
81 + info: Number of encountered BTRFS read errors
82 to: sysadmin
83
84 template: btrfs_device_write_errors
@@ -87,7 +92,8 @@ component: File system
92 lookup: max -10m every 1m of write_errs
93 crit: $this > 0
94 delay: up 1m down 15m multiplier 1.5 max 1h
90 - info: number of encountered BTRFS write errors
95 + summary: BTRFS device write errors
96 + info: Number of encountered BTRFS write errors
97 to: sysadmin
98
99 template: btrfs_device_flush_errors
@@ -101,7 +107,8 @@ component: File system
107 lookup: max -10m every 1m of flush_errs
108 crit: $this > 0
109 delay: up 1m down 15m multiplier 1.5 max 1h
104 - info: number of encountered BTRFS flush errors
110 + summary: BTRFS device flush errors
111 + info: Number of encountered BTRFS flush errors
112 to: sysadmin
113
114 template: btrfs_device_corruption_errors
@@ -115,7 +122,8 @@ component: File system
122 lookup: max -10m every 1m of corruption_errs
123 warn: $this > 0
124 delay: up 1m down 15m multiplier 1.5 max 1h
118 - info: number of encountered BTRFS corruption errors
125 + summary: BTRFS device corruption errors
126 + info: Number of encountered BTRFS corruption errors
127 to: sysadmin
128
129 template: btrfs_device_generation_errors
@@ -129,5 +137,6 @@ component: File system
137 lookup: max -10m every 1m of generation_errs
138 warn: $this > 0
139 delay: up 1m down 15m multiplier 1.5 max 1h
132 - info: number of encountered BTRFS generation errors
140 + summary: BTRFS device generation errors
141 + info: Number of encountered BTRFS generation errors
142 to: sysadmin
health/health.d/cgroups.conf
+7 -3
@@ -13,7 +13,8 @@ component: CPU
13 every: 1m
14 warn: $this > (($status == $CRITICAL) ? (85) : (95))
15 delay: down 15m multiplier 1.5 max 1h
16 - info: average cgroup CPU utilization over the last 10 minutes
16 + summary: Cgroup CPU utilization
17 + info: Average cgroup CPU utilization over the last 10 minutes
18 to: silent
19
20 template: cgroup_ram_in_use
@@ -29,7 +30,8 @@ component: Memory
30 warn: $this > (($status >= $WARNING) ? (80) : (90))
31 crit: $this > (($status == $CRITICAL) ? (90) : (98))
32 delay: down 15m multiplier 1.5 max 1h
32 - info: cgroup memory utilization
33 + summary: Cgroup ram utilization
34 + info: Cgroup memory utilization
35 to: silent
36
37 # FIXME COMMENTED DUE TO A BUG IN NETDATA
@@ -83,7 +85,8 @@ component: CPU
85 every: 1m
86 warn: $this > (($status >= $WARNING) ? (75) : (85))
87 delay: down 15m multiplier 1.5 max 1h
86 - info: container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
88 + summary: Container ${label:k8s_container_name} CPU utilization
89 + info: Container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
90 average CPU utilization over the last 10 minutes
91 to: silent
92
@@ -100,6 +103,7 @@ component: Memory
103 warn: $this > (($status >= $WARNING) ? (80) : (90))
104 crit: $this > (($status == $CRITICAL) ? (90) : (98))
105 delay: down 15m multiplier 1.5 max 1h
106 + summary: Container ${label:k8s_container_name} ram utilization
107 info: container ${label:k8s_container_name} of pod ${label:k8s_pod_name} of namespace ${label:k8s_namespace}, \
108 memory utilization
109 to: silent
health/health.d/cpu.conf
+8 -4
@@ -14,7 +14,8 @@ component: CPU
14 warn: $this > (($status >= $WARNING) ? (75) : (85))
15 crit: $this > (($status == $CRITICAL) ? (85) : (95))
16 delay: down 15m multiplier 1.5 max 1h
17 - info: average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
17 + summary: CPU utilization
18 + info: Average CPU utilization over the last 10 minutes (excluding iowait, nice and steal)
19 to: silent
20
21 template: 10min_cpu_iowait
@@ -29,7 +30,8 @@ component: CPU
30 every: 1m
31 warn: $this > (($status >= $WARNING) ? (20) : (40))
32 delay: up 30m down 30m multiplier 1.5 max 2h
32 - info: average CPU iowait time over the last 10 minutes
33 + summary: CPU iowait time
34 + info: Average CPU iowait time over the last 10 minutes
35 to: silent
36
37 template: 20min_steal_cpu
@@ -44,7 +46,8 @@ component: CPU
46 every: 5m
47 warn: $this > (($status >= $WARNING) ? (5) : (10))
48 delay: down 1h multiplier 1.5 max 2h
47 - info: average CPU steal time over the last 20 minutes
49 + summary: CPU steal time
50 + info: Average CPU steal time over the last 20 minutes
51 to: silent
52
53 ## FreeBSD
@@ -61,5 +64,6 @@ component: CPU
64 warn: $this > (($status >= $WARNING) ? (75) : (85))
65 crit: $this > (($status == $CRITICAL) ? (85) : (95))
66 delay: down 15m multiplier 1.5 max 1h
64 - info: average CPU utilization over the last 10 minutes (excluding nice)
67 + summary: CPU utilization
68 + info: Average CPU utilization over the last 10 minutes (excluding nice)
69 to: silent
health/health.d/disks.conf
+12 -6
@@ -23,7 +23,8 @@ chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
23 warn: $this > (($status >= $WARNING ) ? (80) : (90))
24 crit: ($this > (($status == $CRITICAL) ? (90) : (98))) && $avail < 5
25 delay: up 1m down 15m multiplier 1.5 max 1h
26 - info: disk ${label:mount_point} space utilization
26 + summary: Disk ${label:mount_point} space usage
27 + info: Total space utilization of disk ${label:mount_point}
28 to: sysadmin
29
30 template: disk_inode_usage
@@ -40,7 +41,8 @@ chart labels: mount_point=!/dev !/dev/* !/run !/run/* *
41 warn: $this > (($status >= $WARNING) ? (80) : (90))
42 crit: $this > (($status == $CRITICAL) ? (90) : (98))
43 delay: up 1m down 15m multiplier 1.5 max 1h
43 - info: disk ${label:mount_point} inode utilization
44 + summary: Disk ${label:mount_point} inode usage
45 + info: Total inode utilization of disk ${label:mount_point}
46 to: sysadmin
47
48
@@ -79,7 +81,8 @@ template: out_of_disk_space_time
81 warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
82 crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
83 delay: down 15m multiplier 1.2 max 1h
82 - info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
84 + summary: Out of disk space time for ${label:mount_point}
85 + info: Estimated time the disk ${label:mount_point} will run out of space, if the system continues to add data with the rate of the last hour
86 to: silent
87
88
@@ -118,7 +121,8 @@ template: out_of_disk_inodes_time
121 warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8))
122 crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2))
123 delay: down 15m multiplier 1.2 max 1h
121 - info: estimated time the disk will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
124 + summary: Out of disk inodes time for ${label:mount_point}
125 + info: Estimated time the disk ${label:mount_point} will run out of inodes, if the system continues to allocate inodes with the rate of the last hour
126 to: silent
127
128
@@ -141,7 +145,8 @@ component: Disk
145 every: 1m
146 warn: $this > 98 * (($status >= $WARNING) ? (0.7) : (1))
147 delay: down 15m multiplier 1.2 max 1h
144 - info: average percentage of time ${label:device} disk was busy over the last 10 minutes
148 + summary: Disk ${label:device} utilization
149 + info: Average percentage of time ${label:device} disk was busy over the last 10 minutes
150 to: silent
151
152
@@ -162,5 +167,6 @@ component: Disk
167 every: 1m
168 warn: $this > 5000 * (($status >= $WARNING) ? (0.7) : (1))
169 delay: down 15m multiplier 1.2 max 1h
165 - info: average backlog size of the ${label:device} disk over the last 10 minutes
170 + summary: Disk ${label:device} backlog
171 + info: Average backlog size of the ${label:device} disk over the last 10 minutes
172 to: silent
health/health.d/docker.conf
+1
@@ -7,5 +7,6 @@ component: Docker
7 every: 10s
8 lookup: average -10s of unhealthy
9 warn: $this > 0
10 + summary: Docker container ${label:container_name} health
11 info: ${label:container_name} docker container health status is unhealthy
12 to: sysadmin
health/health.d/file_descriptors.conf
+4 -2
@@ -11,7 +11,8 @@
11 every: 1m
12 crit: $this > 90
13 delay: down 15m multiplier 1.5 max 1h
14 - info: system-wide utilization of open files
14 + summary: System open files utilization
15 + info: System-wide utilization of open files
16 to: sysadmin
17
18 template: apps_group_file_descriptors_utilization
@@ -27,5 +28,6 @@ component: Process
28 every: 10s
29 warn: $this > (($status >= $WARNING) ? (85) : (95))
30 delay: down 15m multiplier 1.5 max 1h
30 - info: open files percentage against the processes limits, among all PIDs in application group
31 + summary: Group open files utilization
32 + info: Open files percentage against the processes limits, among all PIDs in application group
33 to: sysadmin
health/health.d/httpcheck.conf
+8 -4
@@ -23,7 +23,8 @@ component: HTTP endpoint
23 warn: $this >= 10 AND $this < 40
24 crit: $this >= 40
25 delay: down 5m multiplier 1.5 max 1h
26 - info: percentage of HTTP responses from ${label:url} with unexpected content in the last 5 minutes
26 + summary: HTTP check for ${label:url} unexpected content
27 + info: Percentage of HTTP responses from ${label:url} with unexpected content in the last 5 minutes
28 to: webmaster
29
30 template: httpcheck_web_service_bad_status
@@ -37,7 +38,8 @@ component: HTTP endpoint
38 warn: $this >= 10 AND $this < 40
39 crit: $this >= 40
40 delay: down 5m multiplier 1.5 max 1h
40 - info: percentage of HTTP responses from ${label:url} with unexpected status in the last 5 minutes
41 + summary: HTTP check for ${label:url} unexpected status
42 + info: Percentage of HTTP responses from ${label:url} with unexpected status in the last 5 minutes
43 to: webmaster
44
45 template: httpcheck_web_service_timeouts
@@ -51,7 +53,8 @@ component: HTTP endpoint
53 warn: $this >= 10 AND $this < 40
54 crit: $this >= 40
55 delay: down 5m multiplier 1.5 max 1h
54 - info: percentage of timed-out HTTP requests to ${label:url} in the last 5 minutes
56 + summary: HTTP check for ${label:url} timeouts
57 + info: Percentage of timed-out HTTP requests to ${label:url} in the last 5 minutes
58 to: webmaster
59
60 template: httpcheck_web_service_no_connection
@@ -65,5 +68,6 @@ component: HTTP endpoint
68 warn: $this >= 10 AND $this < 40
69 crit: $this >= 40
70 delay: down 5m multiplier 1.5 max 1h
68 - info: percentage of failed HTTP requests to ${label:url} in the last 5 minutes
71 + summary: HTTP check for ${label:url} failed requests
72 + info: Percentage of failed HTTP requests to ${label:url} in the last 5 minutes
73 to: webmaster
health/health.d/ipc.conf
+2
@@ -13,6 +13,7 @@ component: IPC
13 every: 10s
14 warn: $this > (($status >= $WARNING) ? (70) : (80))
15 delay: down 5m multiplier 1.5 max 1h
16 + summary: IPC semaphores used
17 info: IPC semaphore utilization
18 to: sysadmin
19
@@ -28,5 +29,6 @@ component: IPC
29 every: 10s
30 warn: $this > (($status >= $WARNING) ? (70) : (80))
31 delay: down 5m multiplier 1.5 max 1h
32 + summary: IPC semaphore arrays used
33 info: IPC semaphore arrays utilization
34 to: sysadmin
health/health.d/ipmi.conf
+2
@@ -9,6 +9,7 @@ component: IPMI
9 warn: $warning > 0
10 crit: $critical > 0
11 delay: up 5m down 15m multiplier 1.5 max 1h
12 + summary: IPMI sensor ${label:sensor} state
13 info: IPMI sensor ${label:sensor} (${label:component}) state
14 to: sysadmin
15
@@ -22,5 +23,6 @@ component: IPMI
23 every: 10s
24 warn: $this > 0
25 delay: up 5m down 15m multiplier 1.5 max 1h
26 + summary: IPMI events
27 info: number of events in the IPMI System Event Log (SEL)
28 to: silent
health/health.d/linux_power_supply.conf
+2 -1
@@ -10,5 +10,6 @@ component: Battery
10 every: 10s
11 warn: $this < 10
12 delay: up 30s down 5m multiplier 1.2 max 1h
13 - info: percentage of remaining power supply capacity
13 + summary: Power supply capacity
14 + info: Percentage of remaining power supply capacity
15 to: silent
health/health.d/load.conf
+7 -4
@@ -14,7 +14,7 @@ component: Load
14 calc: ($active_processors == nan or $active_processors == 0) ? (nan) : ( ($active_processors < 2) ? ( 2 ) : ( $active_processors ) )
15 units: cpus
16 every: 1m
17 - info: number of active CPU cores in the system
17 + info: Number of active CPU cores in the system
18
19 # Send alarms if the load average is unusually high.
20 # These intentionally _do not_ calculate the average over the sampled
@@ -33,7 +33,8 @@ component: Load
33 every: 1m
34 warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 175 : 200)
35 delay: down 15m multiplier 1.5 max 1h
36 - info: system fifteen-minute load average
36 + summary: Load average (15 minutes)
37 + info: System load average for the past 15 minutes
38 to: silent
39
40 alarm: load_average_5
@@ -49,7 +50,8 @@ component: Load
50 every: 1m
51 warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 350 : 400)
52 delay: down 15m multiplier 1.5 max 1h
52 - info: system five-minute load average
53 + summary: Load average (5 minutes)
54 + info: System load average for the past 5 minutes
55 to: silent
56
57 alarm: load_average_1
@@ -65,5 +67,6 @@ component: Load
67 every: 1m
68 warn: ($this * 100 / $load_cpu_number) > (($status >= $WARNING) ? 700 : 800)
69 delay: down 15m multiplier 1.5 max 1h
68 - info: system one-minute load average
70 + summary: Load average (1 minute)
71 + info: System load average for the past 1 minute
72 to: silent
health/health.d/mdstat.conf
+6 -3
@@ -8,7 +8,8 @@ component: RAID
8 every: 10s
9 calc: $down
10 warn: $this > 0
11 - info: number of devices in the down state for the ${label:device} ${label:raid_level} array. \
11 + summary: Mdtat device ${label:device} down
12 + info: Number of devices in the down state for the ${label:device} ${label:raid_level} array. \
13 Any number > 0 indicates that the array is degraded.
14 to: sysadmin
15
@@ -23,7 +24,8 @@ chart labels: raid_level=!raid1 !raid10 *
24 every: 60s
25 warn: $this > 1024
26 delay: up 30m
26 - info: number of unsynchronized blocks for the ${label:device} ${label:raid_level} array
27 + summary: Mdstat device ${label:device} unsynchronized blocks
28 + info: Number of unsynchronized blocks for the ${label:device} ${label:raid_level} array
29 to: silent
30
31 template: mdstat_nonredundant_last_collected
@@ -36,5 +38,6 @@ component: RAID
38 every: 10s
39 warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
40 crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
39 - info: number of seconds since the last successful data collection
41 + summary: Mdstat last collected
42 + info: Number of seconds since the last successful data collection
43 to: sysadmin
health/health.d/net.conf
+24 -14
@@ -14,7 +14,7 @@ component: Network
14 calc: ( $nic_speed_max > 0 ) ? ( $nic_speed_max) : ( nan )
15 units: Mbit
16 every: 10s
17 - info: network interface ${label:device} current speed
17 + info: Network interface ${label:device} current speed
18
19 template: 1m_received_traffic_overflow
20 on: net.net
@@ -29,7 +29,8 @@ component: Network
29 every: 10s
30 warn: $this > (($status >= $WARNING) ? (85) : (90))
31 delay: up 1m down 1m multiplier 1.5 max 1h
32 - info: average inbound utilization for the network interface ${label:device} over the last minute
32 + summary: 1 minute received traffic overflow for ${label:device}
33 + info: Average inbound utilization for the network interface ${label:device} over the last minute
34 to: silent
35
36 template: 1m_sent_traffic_overflow
@@ -45,7 +46,8 @@ component: Network
46 every: 10s
47 warn: $this > (($status >= $WARNING) ? (85) : (90))
48 delay: up 1m down 1m multiplier 1.5 max 1h
48 - info: average outbound utilization for the network interface ${label:device} over the last minute
49 + summary: 1 minute sent traffic overflow for ${label:device}
50 + info: Average outbound utilization for the network interface ${label:device} over the last minute
51 to: silent
52
53 # -----------------------------------------------------------------------------
@@ -68,7 +70,7 @@ component: Network
70 lookup: sum -10m unaligned absolute of inbound
71 units: packets
72 every: 1m
71 - info: number of inbound dropped packets for the network interface ${label:device} in the last 10 minutes
73 + info: Number of inbound dropped packets for the network interface ${label:device} in the last 10 minutes
74
75 template: outbound_packets_dropped
76 on: net.drops
@@ -80,7 +82,7 @@ component: Network
82 lookup: sum -10m unaligned absolute of outbound
83 units: packets
84 every: 1m
83 - info: number of outbound dropped packets for the network interface ${label:device} in the last 10 minutes
85 + info: Number of outbound dropped packets for the network interface ${label:device} in the last 10 minutes
86
87 template: inbound_packets_dropped_ratio
88 on: net.packets
@@ -96,7 +98,8 @@ chart labels: device=!wl* *
98 every: 1m
99 warn: $this >= 2
100 delay: up 1m down 1h multiplier 1.5 max 2h
99 - info: ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
101 + summary: Inbound packets dropped ratio for ${label:device}
102 + info: Ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
103 to: silent
104
105 template: outbound_packets_dropped_ratio
@@ -113,7 +116,8 @@ chart labels: device=!wl* *
116 every: 1m
117 warn: $this >= 2
118 delay: up 1m down 1h multiplier 1.5 max 2h
116 - info: ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
119 + summary: Outbound packets dropped ratio for ${label:device}
120 + info: Ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
121 to: silent
122
123 template: wifi_inbound_packets_dropped_ratio
@@ -130,7 +134,8 @@ chart labels: device=wl*
134 every: 1m
135 warn: $this >= 10
136 delay: up 1m down 1h multiplier 1.5 max 2h
133 - info: ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
137 + summary: Inbound packets dropped ratio for ${label:device}
138 + info: Ratio of inbound dropped packets for the network interface ${label:device} over the last 10 minutes
139 to: silent
140
141 template: wifi_outbound_packets_dropped_ratio
@@ -147,7 +152,8 @@ chart labels: device=wl*
152 every: 1m
153 warn: $this >= 10
154 delay: up 1m down 1h multiplier 1.5 max 2h
150 - info: ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
155 + summary: Outbound packets dropped ratio for ${label:device}
156 + info: Ratio of outbound dropped packets for the network interface ${label:device} over the last 10 minutes
157 to: silent
158
159 # -----------------------------------------------------------------------------
@@ -165,7 +171,8 @@ component: Network
171 every: 1m
172 warn: $this >= 5
173 delay: down 1h multiplier 1.5 max 2h
168 - info: number of inbound errors for the network interface ${label:device} in the last 10 minutes
174 + summary: Inbound interface errors for ${label:device}
175 + info: Number of inbound errors for the network interface ${label:device} in the last 10 minutes
176 to: silent
177
178 template: interface_outbound_errors
@@ -180,7 +187,8 @@ component: Network
187 every: 1m
188 warn: $this >= 5
189 delay: down 1h multiplier 1.5 max 2h
183 - info: number of outbound errors for the network interface ${label:device} in the last 10 minutes
190 + summary: Outbound interface errors for ${label:device}
191 + info: Number of outbound errors for the network interface ${label:device} in the last 10 minutes
192 to: silent
193
194 # -----------------------------------------------------------------------------
@@ -203,7 +211,8 @@ component: Network
211 every: 1m
212 warn: $this > 0
213 delay: down 1h multiplier 1.5 max 2h
206 - info: number of FIFO errors for the network interface ${label:device} in the last 10 minutes
214 + summary: Net FIFO errors for ${label:device}
215 + info: Number of FIFO errors for the network interface ${label:device} in the last 10 minutes
216 to: silent
217
218 # -----------------------------------------------------------------------------
@@ -225,7 +234,7 @@ component: Network
234 lookup: average -1m unaligned of received
235 units: packets
236 every: 10s
228 - info: average number of packets received by the network interface ${label:device} over the last minute
237 + info: Average number of packets received by the network interface ${label:device} over the last minute
238
239 template: 10s_received_packets_storm
240 on: net.packets
@@ -241,6 +250,7 @@ component: Network
250 warn: $this > (($status >= $WARNING)?(200):(5000))
251 crit: $this > (($status == $CRITICAL)?(5000):(6000))
252 options: no-clear-notification
244 - info: ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
253 + summary: Received packets storm for ${label:device}
254 + info: Ratio of average number of received packets for the network interface ${label:device} over the last 10 seconds, \
255 compared to the rate over the last minute
256 to: silent
health/health.d/nut.conf
+5 -2
@@ -13,7 +13,8 @@ component: UPS
13 warn: $this > (($status >= $WARNING) ? (70) : (80))
14 crit: $this > (($status == $CRITICAL) ? (85) : (95))
15 delay: down 10m multiplier 1.5 max 1h
16 - info: average UPS load over the last 10 minutes
16 + summary: UPS load
17 + info: Average UPS load over the last 10 minutes
18 to: sitemgr
19
20 template: nut_ups_charge
@@ -29,7 +30,8 @@ component: UPS
30 warn: $this < 75
31 crit: $this < 40
32 delay: down 10m multiplier 1.5 max 1h
32 - info: average UPS charge over the last minute
33 + summary: UPS charge
34 + info: Average UPS charge over the last minute
35 to: sitemgr
36
37 template: nut_last_collected_secs
@@ -43,5 +45,6 @@ component: UPS device
45 warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
46 crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
47 delay: down 5m multiplier 1.5 max 1h
48 + summary: NUT last collected
49 info: number of seconds since the last successful data collection
50 to: sitemgr
health/health.d/nvme.conf
+1
@@ -10,5 +10,6 @@ component: Disk
10 every: 10s
11 crit: $this != nan AND $this != 0
12 delay: down 5m multiplier 1.5 max 2h
13 + summary: NVMe device ${label:device} state
14 info: NVMe device ${label:device} has critical warnings
15 to: sysadmin
health/health.d/ping.conf
+6 -3
@@ -11,7 +11,8 @@ component: Network
11 every: 10s
12 crit: $this == 0
13 delay: down 30m multiplier 1.5 max 2h
14 - info: network host ${label:host} reachability status
14 + summary: Host ${label:host} ping status
15 + info: Network host ${label:host} reachability status
16 to: sysadmin
17
18 template: ping_packet_loss
@@ -27,7 +28,8 @@ component: Network
28 warn: $this > $green
29 crit: $this > $red
30 delay: down 30m multiplier 1.5 max 2h
30 - info: packet loss percentage to the network host ${label:host} over the last 10 minutes
31 + summary: Host ${label:host} ping packet loss
32 + info: Packet loss percentage to the network host ${label:host} over the last 10 minutes
33 to: sysadmin
34
35 template: ping_host_latency
@@ -43,5 +45,6 @@ component: Network
45 warn: $this > $green OR $max > $red
46 crit: $this > $red
47 delay: down 30m multiplier 1.5 max 2h
46 - info: average latency to the network host ${label:host} over the last 10 seconds
48 + summary: Host ${label:host} ping latency
49 + info: Average latency to the network host ${label:host} over the last 10 seconds
50 to: sysadmin
health/health.d/postgres.conf
+27 -13
@@ -12,7 +12,8 @@ component: PostgreSQL
12 warn: $this > (($status >= $WARNING) ? (70) : (80))
13 crit: $this > (($status == $CRITICAL) ? (80) : (90))
14 delay: down 15m multiplier 1.5 max 1h
15 - info: average total connection utilization over the last minute
15 + summary: PostgreSQL connection utilization
16 + info: Average total connection utilization over the last minute
17 to: dba
18
19 template: postgres_acquired_locks_utilization
@@ -26,7 +27,8 @@ component: PostgreSQL
27 every: 1m
28 warn: $this > (($status >= $WARNING) ? (15) : (20))
29 delay: down 15m multiplier 1.5 max 1h
29 - info: average acquired locks utilization over the last minute
30 + summary: PostgreSQL acquired locks utilization
31 + info: Average acquired locks utilization over the last minute
32 to: dba
33
34 template: postgres_txid_exhaustion_perc
@@ -40,7 +42,8 @@ component: PostgreSQL
42 every: 1m
43 warn: $this > 90
44 delay: down 15m multiplier 1.5 max 1h
43 - info: percent towards TXID wraparound
45 + summary: PostgreSQL TXID exhaustion
46 + info: Percent towards TXID wraparound
47 to: dba
48
49 # Database alarms
@@ -58,7 +61,8 @@ component: PostgreSQL
61 warn: $this < (($status >= $WARNING) ? (70) : (60))
62 crit: $this < (($status == $CRITICAL) ? (60) : (50))
63 delay: down 15m multiplier 1.5 max 1h
61 - info: average cache hit ratio in db ${label:database} over the last minute
64 + summary: PostgreSQL DB ${label:database} cache hit ratio
65 + info: Average cache hit ratio in db ${label:database} over the last minute
66 to: dba
67
68 template: postgres_db_transactions_rollback_ratio
@@ -72,7 +76,8 @@ component: PostgreSQL
76 every: 1m
77 warn: $this > (($status >= $WARNING) ? (0) : (2))
78 delay: down 15m multiplier 1.5 max 1h
75 - info: average aborted transactions percentage in db ${label:database} over the last five minutes
79 + summary: PostgreSQL DB ${label:database} aborted transactions
80 + info: Average aborted transactions percentage in db ${label:database} over the last five minutes
81 to: dba
82
83 template: postgres_db_deadlocks_rate
@@ -86,7 +91,8 @@ component: PostgreSQL
91 every: 1m
92 warn: $this > (($status >= $WARNING) ? (0) : (10))
93 delay: down 15m multiplier 1.5 max 1h
89 - info: number of deadlocks detected in db ${label:database} in the last minute
94 + summary: PostgreSQL DB ${label:database} deadlocks rate
95 + info: Number of deadlocks detected in db ${label:database} in the last minute
96 to: dba
97
98 # Table alarms
@@ -104,7 +110,8 @@ component: PostgreSQL
110 warn: $this < (($status >= $WARNING) ? (70) : (60))
111 crit: $this < (($status == $CRITICAL) ? (60) : (50))
112 delay: down 15m multiplier 1.5 max 1h
107 - info: average cache hit ratio in db ${label:database} table ${label:table} over the last minute
113 + summary: PostgreSQL table ${label:table} cache hit ratio
114 + info: Average cache hit ratio in db ${label:database} table ${label:table} over the last minute
115 to: dba
116
117 template: postgres_table_index_cache_io_ratio
@@ -120,7 +127,8 @@ component: PostgreSQL
127 warn: $this < (($status >= $WARNING) ? (70) : (60))
128 crit: $this < (($status == $CRITICAL) ? (60) : (50))
129 delay: down 15m multiplier 1.5 max 1h
123 - info: average index cache hit ratio in db ${label:database} table ${label:table} over the last minute
130 + summary: PostgreSQL table ${label:table} index cache hit ratio
131 + info: Average index cache hit ratio in db ${label:database} table ${label:table} over the last minute
132 to: dba
133
134 template: postgres_table_toast_cache_io_ratio
@@ -136,7 +144,8 @@ component: PostgreSQL
144 warn: $this < (($status >= $WARNING) ? (70) : (60))
145 crit: $this < (($status == $CRITICAL) ? (60) : (50))
146 delay: down 15m multiplier 1.5 max 1h
139 - info: average TOAST hit ratio in db ${label:database} table ${label:table} over the last minute
147 + summary: PostgreSQL table ${label:table} toast cache hit ratio
148 + info: Average TOAST hit ratio in db ${label:database} table ${label:table} over the last minute
149 to: dba
150
151 template: postgres_table_toast_index_cache_io_ratio
@@ -152,6 +161,7 @@ component: PostgreSQL
161 warn: $this < (($status >= $WARNING) ? (70) : (60))
162 crit: $this < (($status == $CRITICAL) ? (60) : (50))
163 delay: down 15m multiplier 1.5 max 1h
164 + summary: PostgreSQL table ${label:table} index toast hit ratio
165 info: average index TOAST hit ratio in db ${label:database} table ${label:table} over the last minute
166 to: dba
167
@@ -167,7 +177,8 @@ component: PostgreSQL
177 warn: $this > (($status >= $WARNING) ? (60) : (70))
178 crit: $this > (($status == $CRITICAL) ? (70) : (80))
179 delay: down 15m multiplier 1.5 max 1h
170 - info: bloat size percentage in db ${label:database} table ${label:table}
180 + summary: PostgreSQL table ${label:table} bloat size
181 + info: Bloat size percentage in db ${label:database} table ${label:table}
182 to: dba
183
184 template: postgres_table_last_autovacuum_time
@@ -180,7 +191,8 @@ component: PostgreSQL
191 units: seconds
192 every: 1m
193 warn: $this != nan AND $this > (60 * 60 * 24 * 7)
183 - info: time elapsed since db ${label:database} table ${label:table} was vacuumed by the autovacuum daemon
194 + summary: PostgreSQL table ${label:table} last autovacuum
195 + info: Time elapsed since db ${label:database} table ${label:table} was vacuumed by the autovacuum daemon
196 to: dba
197
198 template: postgres_table_last_autoanalyze_time
@@ -193,7 +205,8 @@ component: PostgreSQL
205 units: seconds
206 every: 1m
207 warn: $this != nan AND $this > (60 * 60 * 24 * 7)
196 - info: time elapsed since db ${label:database} table ${label:table} was analyzed by the autovacuum daemon
208 + summary: PostgreSQL table ${label:table} last autoanalyze
209 + info: Time elapsed since db ${label:database} table ${label:table} was analyzed by the autovacuum daemon
210 to: dba
211
212 # Index alarms
@@ -210,5 +223,6 @@ component: PostgreSQL
223 warn: $this > (($status >= $WARNING) ? (60) : (70))
224 crit: $this > (($status == $CRITICAL) ? (70) : (80))
225 delay: down 15m multiplier 1.5 max 1h
213 - info: bloat size percentage in db ${label:database} table ${label:table} index ${label:index}
226 + summary: PostgreSQL table ${label:table} index bloat size
227 + info: Bloat size percentage in db ${label:database} table ${label:table} index ${label:index}
228 to: dba
health/health.d/qos.conf
+2 -1
@@ -13,5 +13,6 @@ template: 10min_qos_packet_drops
13 every: 30s
14 warn: $this > 0
15 units: packets
16 - info: dropped packets in the last 5 minutes
16 + summary: QOS packet drops
17 + info: Dropped packets in the last 5 minutes
18 to: silent
health/health.d/ram.conf
+20 -15
@@ -14,7 +14,8 @@ component: Memory
14 warn: $this > (($status >= $WARNING) ? (80) : (90))
15 crit: $this > (($status == $CRITICAL) ? (90) : (98))
16 delay: down 15m multiplier 1.5 max 1h
17 - info: system memory utilization
17 + summary: Ram utilization
18 + info: System memory utilization
19 to: sysadmin
20
21 alarm: ram_available
@@ -29,20 +30,22 @@ component: Memory
30 every: 10s
31 warn: $this < (($status >= $WARNING) ? (15) : (10))
32 delay: down 15m multiplier 1.5 max 1h
32 - info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
33 + summary: Available Ram
34 + info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
35 to: silent
36
35 - alarm: oom_kill
36 - on: mem.oom_kill
37 - os: linux
38 - hosts: *
39 - lookup: sum -30m unaligned
40 - units: kills
41 - every: 5m
42 - warn: $this > 0
43 - delay: down 10m
44 - info: number of out of memory kills in the last 30 minutes
45 - to: silent
37 + alarm: oom_kill
38 + on: mem.oom_kill
39 + os: linux
40 + hosts: *
41 + lookup: sum -30m unaligned
42 + units: kills
43 + every: 5m
44 + warn: $this > 0
45 + delay: down 10m
46 + summary: OOM kills
47 + info: Number of out of memory kills in the last 30 minutes
48 + to: silent
49
50 ## FreeBSD
51 alarm: ram_in_use
@@ -58,7 +61,8 @@ component: Memory
61 warn: $this > (($status >= $WARNING) ? (80) : (90))
62 crit: $this > (($status == $CRITICAL) ? (90) : (98))
63 delay: down 15m multiplier 1.5 max 1h
61 - info: system memory utilization
64 + summary: Ram utilization
65 + info: System memory utilization
66 to: sysadmin
67
68 alarm: ram_available
@@ -73,5 +77,6 @@ component: Memory
77 every: 10s
78 warn: $this < (($status >= $WARNING) ? (15) : (10))
79 delay: down 15m multiplier 1.5 max 1h
76 - info: percentage of estimated amount of RAM available for userspace processes, without causing swapping
80 + summary: Available Ram
81 + info: Percentage of estimated amount of RAM available for userspace processes, without causing swapping
82 to: silent
health/health.d/redis.conf
+8 -4
@@ -9,7 +9,8 @@ component: Redis
9 every: 10s
10 units: connections
11 warn: $this > 0
12 - info: connections rejected because of maxclients limit in the last minute
12 + summary: Redis rejected connections
13 + info: Connections rejected because of maxclients limit in the last minute
14 delay: down 5m multiplier 1.5 max 1h
15 to: dba
16
@@ -21,7 +22,8 @@ component: Redis
22 every: 10s
23 crit: $last_bgsave != nan AND $last_bgsave != 0
24 units: ok/failed
24 - info: status of the last RDB save operation (0: ok, 1: error)
25 + summary: Redis background save
26 + info: Status of the last RDB save operation (0: ok, 1: error)
27 delay: down 5m multiplier 1.5 max 1h
28 to: dba
29
@@ -35,7 +37,8 @@ component: Redis
37 warn: $this > 600
38 crit: $this > 1200
39 units: seconds
38 - info: duration of the on-going RDB save operation
40 + summary: Redis slow background save
41 + info: Duration of the on-going RDB save operation
42 delay: down 5m multiplier 1.5 max 1h
43 to: dba
44
@@ -48,6 +51,7 @@ component: Redis
51 calc: $time
52 units: seconds
53 crit: $this != nan AND $this > 0
51 - info: time elapsed since the link between master and slave is down
54 + summary: Redis master link down
55 + info: Time elapsed since the link between master and slave is down
56 delay: down 5m multiplier 1.5 max 1h
57 to: dba
health/health.d/swap.conf
+4 -2
@@ -15,7 +15,8 @@ component: Memory
15 every: 1m
16 warn: $this > (($status >= $WARNING) ? (20) : (30))
17 delay: down 15m multiplier 1.5 max 1h
18 - info: percentage of the system RAM swapped in the last 30 minutes
18 + summary: Ram swapped out
19 + info: Percentage of the system RAM swapped in the last 30 minutes
20 to: silent
21
22 alarm: used_swap
@@ -31,5 +32,6 @@ component: Memory
32 warn: $this > (($status >= $WARNING) ? (80) : (90))
33 crit: $this > (($status == $CRITICAL) ? (90) : (98))
34 delay: up 30s down 15m multiplier 1.5 max 1h
34 - info: swap memory utilization
35 + summary: Swap utilization
36 + info: Swap memory utilization
37 to: sysadmin
health/health.d/tcp_mem.conf
+1
@@ -19,5 +19,6 @@ component: Network
19 warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} ))
20 crit: ${mem} > (($status == $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 ))
21 delay: up 0 down 5m multiplier 1.5 max 1h
22 + summary: TCP memory utilization
23 info: TCP memory utilization
24 to: silent
health/health.d/timex.conf
+2 -1
@@ -13,5 +13,6 @@ component: Clock
13 every: 10s
14 warn: $system.uptime.uptime > 17 * 60 AND $this == 0
15 delay: down 5m
16 - info: when set to 0, the system kernel believes the system clock is not properly synchronized to a reliable server
16 + summary: System clock sync state
17 + info: When set to 0, the system kernel believes the system clock is not properly synchronized to a reliable server
18 to: silent
health/health.d/vernemq.conf
+47 -21
@@ -11,7 +11,8 @@ component: VerneMQ
11 every: 1m
12 warn: $this > (($status >= $WARNING) ? (0) : (5))
13 delay: up 2m down 5m multiplier 1.5 max 2h
14 - info: number of socket errors in the last minute
14 + summary: VerneMQ socket errors
15 + info: Number of socket errors in the last minute
16 to: sysadmin
17
18 # Queues dropped/expired/unhandled PUBLISH messages
@@ -26,7 +27,8 @@ component: VerneMQ
27 every: 1m
28 warn: $this > (($status >= $WARNING) ? (0) : (5))
29 delay: up 2m down 5m multiplier 1.5 max 2h
29 - info: number of dropped messaged due to full queues in the last minute
30 + summary: VerneMQ dropped messages
31 + info: Number of dropped messages due to full queues in the last minute
32 to: sysadmin
33
34 template: vernemq_queue_message_expired
@@ -39,6 +41,7 @@ component: VerneMQ
41 every: 1m
42 warn: $this > (($status >= $WARNING) ? (0) : (5))
43 delay: up 2m down 5m multiplier 1.5 max 2h
44 + summary: VerneMQ expired messages
45 info: number of messages which expired before delivery in the last minute
46 to: sysadmin
47
@@ -52,7 +55,8 @@ component: VerneMQ
55 every: 1m
56 warn: $this > (($status >= $WARNING) ? (0) : (5))
57 delay: up 2m down 5m multiplier 1.5 max 2h
55 - info: number of unhandled messages (connections with clean session=true) in the last minute
58 + summary: VerneMQ unhandled messages
59 + info: Number of unhandled messages (connections with clean session=true) in the last minute
60 to: sysadmin
61
62 # Erlang VM
@@ -68,7 +72,8 @@ component: VerneMQ
72 warn: $this > (($status >= $WARNING) ? (75) : (85))
73 crit: $this > (($status == $CRITICAL) ? (85) : (95))
74 delay: down 15m multiplier 1.5 max 1h
71 - info: average scheduler utilization over the last 10 minutes
75 + summary: VerneMQ scheduler utilization
76 + info: Average scheduler utilization over the last 10 minutes
77 to: sysadmin
78
79 # Cluster communication and netsplits
@@ -83,7 +88,8 @@ component: VerneMQ
88 every: 1m
89 warn: $this > 0
90 delay: up 5m down 5m multiplier 1.5 max 1h
86 - info: amount of traffic dropped during communication with the cluster nodes in the last minute
91 + summary: VerneMQ dropped traffic
92 + info: Amount of traffic dropped during communication with the cluster nodes in the last minute
93 to: sysadmin
94
95 template: vernemq_netsplits
@@ -96,7 +102,8 @@ component: VerneMQ
102 every: 10s
103 warn: $this > 0
104 delay: down 5m multiplier 1.5 max 2h
99 - info: number of detected netsplits (split brain situation) in the last minute
105 + summary: VerneMQ netsplits
106 + info: Number of detected netsplits (split brain situation) in the last minute
107 to: sysadmin
108
109 # Unsuccessful CONNACK
@@ -111,7 +118,8 @@ component: VerneMQ
118 every: 1m
119 warn: $this > (($status >= $WARNING) ? (0) : (5))
120 delay: up 2m down 5m multiplier 1.5 max 2h
114 - info: number of sent unsuccessful v3/v5 CONNACK packets in the last minute
121 + summary: VerneMQ unsuccessful CONNACK
122 + info: Number of sent unsuccessful v3/v5 CONNACK packets in the last minute
123 to: sysadmin
124
125 # Not normal DISCONNECT
@@ -126,7 +134,8 @@ component: VerneMQ
134 every: 1m
135 warn: $this > (($status >= $WARNING) ? (0) : (5))
136 delay: up 2m down 5m multiplier 1.5 max 2h
129 - info: number of received not normal v5 DISCONNECT packets in the last minute
137 + summary: VerneMQ received not normal DISCONNECT
138 + info: Number of received not normal v5 DISCONNECT packets in the last minute
139 to: sysadmin
140
141 template: vernemq_mqtt_disconnect_sent_reason_not_normal
@@ -139,7 +148,8 @@ component: VerneMQ
148 every: 1m
149 warn: $this > (($status >= $WARNING) ? (0) : (5))
150 delay: up 2m down 5m multiplier 1.5 max 2h
142 - info: number of sent not normal v5 DISCONNECT packets in the last minute
151 + summary: VerneMQ sent not normal DISCONNECT
152 + info: Number of sent not normal v5 DISCONNECT packets in the last minute
153 to: sysadmin
154
155 # SUBSCRIBE errors and unauthorized attempts
@@ -154,7 +164,8 @@ component: VerneMQ
164 every: 1m
165 warn: $this > (($status >= $WARNING) ? (0) : (5))
166 delay: up 2m down 5m multiplier 1.5 max 2h
157 - info: number of failed v3/v5 SUBSCRIBE operations in the last minute
167 + summary: VerneMQ failed SUBSCRIBE
168 + info: Number of failed v3/v5 SUBSCRIBE operations in the last minute
169 to: sysadmin
170
171 template: vernemq_mqtt_subscribe_auth_error
@@ -167,6 +178,7 @@ component: VerneMQ
178 every: 1m
179 warn: $this > (($status >= $WARNING) ? (0) : (5))
180 delay: up 2m down 5m multiplier 1.5 max 2h
181 + summary: VerneMQ unauthorized SUBSCRIBE
182 info: number of unauthorized v3/v5 SUBSCRIBE attempts in the last minute
183 to: sysadmin
184
@@ -182,7 +194,8 @@ component: VerneMQ
194 every: 1m
195 warn: $this > (($status >= $WARNING) ? (0) : (5))
196 delay: up 2m down 5m multiplier 1.5 max 2h
185 - info: number of failed v3/v5 UNSUBSCRIBE operations in the last minute
197 + summary: VerneMQ failed UNSUBSCRIBE
198 + info: Number of failed v3/v5 UNSUBSCRIBE operations in the last minute
199 to: sysadmin
200
201 # PUBLISH errors and unauthorized attempts
@@ -197,7 +210,8 @@ component: VerneMQ
210 every: 1m
211 warn: $this > (($status >= $WARNING) ? (0) : (5))
212 delay: up 2m down 5m multiplier 1.5 max 2h
200 - info: number of failed v3/v5 PUBLISH operations in the last minute
213 + summary: VerneMQ failed PUBLISH
214 + info: Number of failed v3/v5 PUBLISH operations in the last minute
215 to: sysadmin
216
217 template: vernemq_mqtt_publish_auth_errors
@@ -210,7 +224,8 @@ component: VerneMQ
224 every: 1m
225 warn: $this > (($status >= $WARNING) ? (0) : (5))
226 delay: up 2m down 5m multiplier 1.5 max 2h
213 - info: number of unauthorized v3/v5 PUBLISH attempts in the last minute
227 + summary: VerneMQ unauthorized PUBLISH
228 + info: Number of unauthorized v3/v5 PUBLISH attempts in the last minute
229 to: sysadmin
230
231 # Unsuccessful and unexpected PUBACK
@@ -225,7 +240,8 @@ component: VerneMQ
240 every: 1m
241 warn: $this > (($status >= $WARNING) ? (0) : (5))
242 delay: up 2m down 5m multiplier 1.5 max 2h
228 - info: number of received unsuccessful v5 PUBACK packets in the last minute
243 + summary: VerneMQ unsuccessful received PUBACK
244 + info: Number of received unsuccessful v5 PUBACK packets in the last minute
245 to: sysadmin
246
247 template: vernemq_mqtt_puback_sent_reason_unsuccessful
@@ -238,7 +254,8 @@ component: VerneMQ
254 every: 1m
255 warn: $this > (($status >= $WARNING) ? (0) : (5))
256 delay: up 2m down 5m multiplier 1.5 max 2h
241 - info: number of sent unsuccessful v5 PUBACK packets in the last minute
257 + summary: VerneMQ unsuccessful sent PUBACK
258 + info: Number of sent unsuccessful v5 PUBACK packets in the last minute
259 to: sysadmin
260
261 template: vernemq_mqtt_puback_unexpected
@@ -251,7 +268,8 @@ component: VerneMQ
268 every: 1m
269 warn: $this > (($status >= $WARNING) ? (0) : (5))
270 delay: up 2m down 5m multiplier 1.5 max 2h
254 - info: number of received unexpected v3/v5 PUBACK packets in the last minute
271 + summary: VerneMQ unnexpected recieved PUBACK
272 + info: Number of received unexpected v3/v5 PUBACK packets in the last minute
273 to: sysadmin
274
275 # Unsuccessful and unexpected PUBREC
@@ -266,7 +284,8 @@ component: VerneMQ
284 every: 1m
285 warn: $this > (($status >= $WARNING) ? (0) : (5))
286 delay: up 2m down 5m multiplier 1.5 max 2h
269 - info: number of received unsuccessful v5 PUBREC packets in the last minute
287 + summary: VerneMQ unsuccessful received PUBREC
288 + info: Number of received unsuccessful v5 PUBREC packets in the last minute
289 to: sysadmin
290
291 template: vernemq_mqtt_pubrec_sent_reason_unsuccessful
@@ -279,7 +298,8 @@ component: VerneMQ
298 every: 1m
299 warn: $this > (($status >= $WARNING) ? (0) : (5))
300 delay: up 2m down 5m multiplier 1.5 max 2h
282 - info: number of sent unsuccessful v5 PUBREC packets in the last minute
301 + summary: VerneMQ unsuccessful sent PUBREC
302 + info: Number of sent unsuccessful v5 PUBREC packets in the last minute
303 to: sysadmin
304
305 template: vernemq_mqtt_pubrec_invalid_error
@@ -292,7 +312,8 @@ component: VerneMQ
312 every: 1m
313 warn: $this > (($status >= $WARNING) ? (0) : (5))
314 delay: up 2m down 5m multiplier 1.5 max 2h
295 - info: number of received unexpected v3 PUBREC packets in the last minute
315 + summary: VerneMQ invalid received PUBREC
316 + info: Number of received invalid v3 PUBREC packets in the last minute
317 to: sysadmin
318
319 # Unsuccessful PUBREL
@@ -307,7 +328,8 @@ component: VerneMQ
328 every: 1m
329 warn: $this > (($status >= $WARNING) ? (0) : (5))
330 delay: up 2m down 5m multiplier 1.5 max 2h
310 - info: number of received unsuccessful v5 PUBREL packets in the last minute
331 + summary: VerneMQ unsuccessful received PUBREL
332 + info: Number of received unsuccessful v5 PUBREL packets in the last minute
333 to: sysadmin
334
335 template: vernemq_mqtt_pubrel_sent_reason_unsuccessful
@@ -320,6 +342,7 @@ component: VerneMQ
342 every: 1m
343 warn: $this > (($status >= $WARNING) ? (0) : (5))
344 delay: up 2m down 5m multiplier 1.5 max 2h
345 + summary: VerneMQ unsuccessful sent PUBREL
346 info: number of sent unsuccessful v5 PUBREL packets in the last minute
347 to: sysadmin
348
@@ -335,7 +358,8 @@ component: VerneMQ
358 every: 1m
359 warn: $this > (($status >= $WARNING) ? (0) : (5))
360 delay: up 2m down 5m multiplier 1.5 max 2h
338 - info: number of received unsuccessful v5 PUBCOMP packets in the last minute
361 + summary: VerneMQ unsuccessful received PUBCOMP
362 + info: Number of received unsuccessful v5 PUBCOMP packets in the last minute
363 to: sysadmin
364
365 template: vernemq_mqtt_pubcomp_sent_reason_unsuccessful
@@ -348,6 +372,7 @@ component: VerneMQ
372 every: 1m
373 warn: $this > (($status >= $WARNING) ? (0) : (5))
374 delay: up 2m down 5m multiplier 1.5 max 2h
375 + summary: VerneMQ unsuccessful sent PUBCOMP
376 info: number of sent unsuccessful v5 PUBCOMP packets in the last minute
377 to: sysadmin
378
@@ -361,5 +386,6 @@ component: VerneMQ
386 every: 1m
387 warn: $this > (($status >= $WARNING) ? (0) : (5))
388 delay: up 2m down 5m multiplier 1.5 max 2h
389 + summary: VerneMQ unexpected received PUBCOMP
390 info: number of received unexpected v3/v5 PUBCOMP packets in the last minute
391 to: sysadmin
health/health.d/zfs.conf
+3
@@ -9,6 +9,7 @@ component: File system
9 every: 1m
10 warn: $this > 0
11 delay: down 1h multiplier 1.5 max 2h
12 + summary: ZFS memory throttle
13 info: number of times ZFS had to limit the ARC growth in the last 10 minutes
14 to: silent
15
@@ -24,6 +25,7 @@ component: File system
25 every: 10s
26 warn: $this > 0
27 delay: down 1m multiplier 1.5 max 1h
28 + summary: ZFS pool ${label:pool} state
29 info: ZFS pool ${label:pool} state is degraded
30 to: sysadmin
31
@@ -37,5 +39,6 @@ component: File system
39 every: 10s
40 crit: $this > 0
41 delay: down 1m multiplier 1.5 max 1h
42 + summary: Critical ZFS pool ${label:pool} state
43 info: ZFS pool ${label:pool} state is faulted or unavail
44 to: sysadmin
health/health.h
+1
@@ -84,6 +84,7 @@ ALARM_ENTRY* health_create_alarm_entry(
84 RRDCALC_STATUS new_status,
85 STRING *source,
86 STRING *units,
87 + STRING *summary,
88 STRING *info,
89 int delay,
90 HEALTH_ENTRY_FLAGS flags);
health/health_config.c
+32
@@ -23,6 +23,7 @@
23 #define HEALTH_EXEC_KEY "exec"
24 #define HEALTH_RECIPIENT_KEY "to"
25 #define HEALTH_UNITS_KEY "units"
26 +#define HEALTH_SUMMARY_KEY "summary"
27 #define HEALTH_INFO_KEY "info"
28 #define HEALTH_CLASS_KEY "class"
29 #define HEALTH_COMPONENT_KEY "component"
@@ -488,6 +489,7 @@ static inline void alert_config_free(struct alert_config *cfg)
489 string_freez(cfg->exec);
490 string_freez(cfg->to);
491 string_freez(cfg->units);
492 + string_freez(cfg->summary);
493 string_freez(cfg->info);
494 string_freez(cfg->classification);
495 string_freez(cfg->component);
@@ -528,6 +530,7 @@ static int health_readfile(const char *filename, void *data) {
530 hash_every = 0,
531 hash_lookup = 0,
532 hash_units = 0,
533 + hash_summary = 0,
534 hash_info = 0,
535 hash_class = 0,
536 hash_component = 0,
@@ -560,6 +563,7 @@ static int health_readfile(const char *filename, void *data) {
563 hash_exec = simple_uhash(HEALTH_EXEC_KEY);
564 hash_every = simple_uhash(HEALTH_EVERY_KEY);
565 hash_units = simple_hash(HEALTH_UNITS_KEY);
566 + hash_summary = simple_hash(HEALTH_SUMMARY_KEY);
567 hash_info = simple_hash(HEALTH_INFO_KEY);
568 hash_class = simple_uhash(HEALTH_CLASS_KEY);
569 hash_component = simple_uhash(HEALTH_COMPONENT_KEY);
@@ -928,6 +932,21 @@ static int health_readfile(const char *filename, void *data) {
932 }
933 rc->units = string_strdupz(value);
934 }
935 + else if(hash == hash_summary && !strcasecmp(key, HEALTH_SUMMARY_KEY)) {
936 + strip_quotes(value);
937 +
938 + alert_cfg->summary = string_strdupz(value);
939 + if(rc->summary) {
940 + if(strcmp(rrdcalc_summary(rc), value) != 0)
941 + netdata_log_error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
942 + line, filename, rrdcalc_name(rc), key, rrdcalc_summary(rc), value, value);
943 +
944 + string_freez(rc->summary);
945 + string_freez(rc->original_summary);
946 + }
947 + rc->summary = string_strdupz(value);
948 + rc->original_summary = string_dup(rc->summary);
949 + }
950 else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
951 strip_quotes(value);
952
@@ -1219,6 +1238,19 @@ static int health_readfile(const char *filename, void *data) {
1238 }
1239 rt->units = string_strdupz(value);
1240 }
1241 + else if(hash == hash_summary && !strcasecmp(key, HEALTH_SUMMARY_KEY)) {
1242 + strip_quotes(value);
1243 +
1244 + alert_cfg->summary = string_strdupz(value);
1245 + if(rt->summary) {
1246 + if(strcmp(rrdcalctemplate_summary(rt), value) != 0)
1247 + netdata_log_error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1248 + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_summary(rt), value, value);
1249 +
1250 + string_freez(rt->summary);
1251 + }
1252 + rt->summary = string_strdupz(value);
1253 + }
1254 else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1255 strip_quotes(value);
1256
health/health_json.c
+2
@@ -60,6 +60,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
60 "\t\t\t\"recipient\": \"%s\",\n"
61 "\t\t\t\"source\": \"%s\",\n"
62 "\t\t\t\"units\": \"%s\",\n"
63 + "\t\t\t\"summary\": \"%s\",\n"
64 "\t\t\t\"info\": \"%s\",\n"
65 "\t\t\t\"status\": \"%s\",\n"
66 "\t\t\t\"last_status_change\": %lu,\n"
@@ -93,6 +94,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
94 , rc->recipient?rrdcalc_recipient(rc):string2str(host->health.health_default_recipient)
95 , rrdcalc_source(rc)
96 , rrdcalc_units(rc)
97 + , rrdcalc_summary(rc)
98 , rrdcalc_info(rc)
99 , rrdcalc_status2string(rc->status)
100 , (unsigned long)rc->last_status_change
health/health_log.c
+2
@@ -34,6 +34,7 @@ inline ALARM_ENTRY* health_create_alarm_entry(
34 RRDCALC_STATUS new_status,
35 STRING *source,
36 STRING *units,
37 + STRING *summary,
38 STRING *info,
39 int delay,
40 HEALTH_ENTRY_FLAGS flags
@@ -71,6 +72,7 @@ inline ALARM_ENTRY* health_create_alarm_entry(
72 ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1));
73 ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1));
74
75 + ae->summary = string_dup(summary);
76 ae->info = string_dup(info);
77 ae->old_status = old_status;
78 ae->new_status = new_status;
health/notifications/alarm-notify.sh.in
+11 -10
@@ -248,6 +248,7 @@ else
248 edit_command_line="${28}" # The command to edit the alarm, with the line number
249 child_machine_guid="${29}" # the machine_guid of the child
250 transition_id="${30}" # the transition_id of the alert
251 + summary="${31}" # the summary text field of the alert
252 fi
253
254 # -----------------------------------------------------------------------------
@@ -2531,7 +2532,7 @@ status_message="status unknown"
2532 color="grey"
2533
2534 # the alarm value
2534 -alarm="${name//_/ } = ${value_string}"
2535 +alarm="${summary//_/ } = ${value_string}"
2536
2537 # the image of the alarm
2538 image="${images_base_url}/images/banner-icon-144x144.png"
@@ -2582,7 +2583,7 @@ CLEAR)
2583 esac
2584
2585 # the html email subject
2585 -html_email_subject="${status_email_subject}, ${name} = ${value_string}, on ${host}"
2586 +html_email_subject="${status_email_subject}, ${summary} = ${value_string}, on ${host}"
2587
2588 if [ "${status}" = "CLEAR" ]; then
2589 severity="Recovered from ${old_status}"
@@ -2593,8 +2594,8 @@ if [ "${status}" = "CLEAR" ]; then
2594
2595 # don't show the value when the status is CLEAR
2596 # for certain alarms, this value might not have any meaning
2596 - alarm="${name//_/ } ${raised_for}"
2597 - html_email_subject="${status_email_subject}, ${name} ${raised_for}, on ${host}"
2597 + alarm="${summary//_/ } ${raised_for}"
2598 + html_email_subject="${status_email_subject}, ${summary} ${raised_for}, on ${host}"
2599
2600 elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
2601 severity="Escalated to ${status}"
@@ -3184,7 +3185,7 @@ Content-Transfer-Encoding: 8bit
3185 <tbody>
3186 <tr>
3187 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:15px;word-break:break-word;">
3187 - <div style="font-family:Open Sans, sans-serif;font-size:20px;font-weight:700;line-height:1;text-align:left;color:#35414A;">${name}</div>
3188 + <div style="font-family:Open Sans, sans-serif;font-size:20px;font-weight:700;line-height:1;text-align:left;color:#35414A;">${summary}</div>
3189 </td>
3190 </tr>
3191 </tbody>
@@ -3342,14 +3343,14 @@ Content-Transfer-Encoding: 8bit
3343 <tbody>
3344 <tr>
3345 <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
3345 - <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Chart:
3346 - <span style="font-weight:700; font-size:20px">${chart}</span></div>
3346 + <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Alert:
3347 + <span style="font-weight:700; font-size:20px">${name}</span></div>
3348 </td>
3349 </tr>
3350 <tr>
3351 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3351 - <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Family:
3352 - <span style="font-weight:700; font-size:20px">${family}</span></div>
3352 + <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Chart:
3353 + <span style="font-weight:700; font-size:20px">${chart}</span></div>
3354 </td>
3355 </tr>
3356 <tr>
@@ -3610,7 +3611,7 @@ Content-Transfer-Encoding: 8bit
3611 <tbody>
3612 <tr>
3613 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;padding-bottom:0;word-break:break-word;">
3613 - <div style="font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:center;color:#35414A;">© Netdata 2021 - The real-time performance and health monitoring</div>
3614 + <div style="font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:center;color:#35414A;">© Netdata $(date +'%Y') - The real-time performance and health monitoring</div>
3615 </td>
3616 </tr>
3617 </tbody>