@cryptotaxi247 / netdata-1 / commits / d2bd4d7ef

Fix: fix heap use after free in health (#19228)

* make health and service thread mutually exclusive * hide alarm-entry from the rest of netdata

Costa Tsaousis committed Dec 16, 2024 at 20:41 UTC d2bd4d7ef47e07ffedac2085b86bbc089a7163b4
12 files changed +553 -524
CMakeLists.txt
+2
@@ -1297,6 +1297,8 @@ set(HEALTH_PLUGIN_FILES
1297 src/health/rrdcalc.h
1298 src/health/rrdvar.c
1299 src/health/rrdvar.h
1300 + src/health/health-alert-entry.h
1301 + src/health/health-alert-log.h
1302 )
1303
1304 set(IDLEJITTER_PLUGIN_FILES src/collectors/idlejitter.plugin/plugin_idlejitter.c)
src/database/rrd.h
+14 -104
@@ -12,6 +12,7 @@ extern "C" {
12 #include "streaming/stream-traffic-types.h"
13 #include "streaming/stream-sender-commit.h"
14 #include "rrdhost-state-id.h"
15 +#include "health/health-alert-log.h"
16
17 // non-existing structs instead of voids
18 // to enable type checking at compile time
@@ -953,21 +954,23 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {
954 // Health
955 RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 17), // contains charts and dims with uninitialized variables
956 RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 18), // the host has initialized health structures
957 + RRDHOST_FLAG_HEALTH_RUNNING_NOW = (1 << 19), // health is currently executing health check on this host
958 +
959 // Exporting
957 - RRDHOST_FLAG_EXPORTING_SEND = (1 << 19), // send it to external databases
958 - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 20), // don't send it to external databases
960 + RRDHOST_FLAG_EXPORTING_SEND = (1 << 20), // send it to external databases
961 + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 21), // don't send it to external databases
962
963 // ACLK
961 - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 21), // when set, we should send ACLK stream context updates
962 - RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // Host should stream alerts
964 + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 22), // when set, we should send ACLK stream context updates
965 + RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 23), // Host should stream alerts
966
967 // Metadata
965 - RRDHOST_FLAG_METADATA_UPDATE = (1 << 23), // metadata needs to be stored in the database
966 - RRDHOST_FLAG_METADATA_LABELS = (1 << 24), // metadata needs to be stored in the database
967 - RRDHOST_FLAG_METADATA_INFO = (1 << 25), // metadata needs to be stored in the database
968 - RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 26), // Context needs to be loaded
968 + RRDHOST_FLAG_METADATA_UPDATE = (1 << 24), // metadata needs to be stored in the database
969 + RRDHOST_FLAG_METADATA_LABELS = (1 << 25), // metadata needs to be stored in the database
970 + RRDHOST_FLAG_METADATA_INFO = (1 << 26), // metadata needs to be stored in the database
971 + RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 27), // Context needs to be loaded
972
970 - RRDHOST_FLAG_METADATA_CLAIMID = (1 << 27), // metadata needs to be stored in the database
973 + RRDHOST_FLAG_METADATA_CLAIMID = (1 << 28), // metadata needs to be stored in the database
974
975 RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 29), // set when the host has updated global functions
976 } RRDHOST_FLAGS;
@@ -1009,100 +1012,6 @@ typedef enum __attribute__ ((__packed__)) {
1012 rrdhost_flag_check(host, RRDHOST_FLAG_COLLECTOR_ONLINE) \
1013 )
1014
1012 -// ----------------------------------------------------------------------------
1013 -// Health data
1014 -
1015 -struct alarm_entry {
1016 - uint32_t unique_id;
1017 - uint32_t alarm_id;
1018 - uint32_t alarm_event_id;
1019 - usec_t global_id;
1020 - nd_uuid_t config_hash_id;
1021 - nd_uuid_t transition_id;
1022 -
1023 - time_t when;
1024 - time_t duration;
1025 - time_t non_clear_duration;
1026 -
1027 - STRING *name;
1028 - STRING *chart;
1029 - STRING *chart_context;
1030 - STRING *chart_name;
1031 -
1032 - STRING *classification;
1033 - STRING *component;
1034 - STRING *type;
1035 -
1036 - STRING *exec;
1037 - STRING *recipient;
1038 - time_t exec_run_timestamp;
1039 - int exec_code;
1040 -
1041 - STRING *source;
1042 - STRING *units;
1043 - STRING *summary;
1044 - STRING *info;
1045 -
1046 - NETDATA_DOUBLE old_value;
1047 - NETDATA_DOUBLE new_value;
1048 -
1049 - STRING *old_value_string;
1050 - STRING *new_value_string;
1051 -
1052 - RRDCALC_STATUS old_status;
1053 - RRDCALC_STATUS new_status;
1054 -
1055 - uint32_t flags;
1056 - int32_t pending_save_count;
1057 -
1058 - int delay;
1059 - time_t delay_up_to_timestamp;
1060 -
1061 - uint32_t updated_by_id;
1062 - uint32_t updates_id;
1063 -
1064 - time_t last_repeat;
1065 -
1066 - POPEN_INSTANCE *popen_instance;
1067 -
1068 - struct alarm_entry *next;
1069 - struct alarm_entry *next_in_progress;
1070 - struct alarm_entry *prev_in_progress;
1071 -};
1072 -
1073 -#define ae_name(ae) string2str((ae)->name)
1074 -#define ae_chart_id(ae) string2str((ae)->chart)
1075 -#define ae_chart_name(ae) string2str((ae)->chart_name)
1076 -#define ae_chart_context(ae) string2str((ae)->chart_context)
1077 -#define ae_classification(ae) string2str((ae)->classification)
1078 -#define ae_exec(ae) string2str((ae)->exec)
1079 -#define ae_recipient(ae) string2str((ae)->recipient)
1080 -#define ae_source(ae) string2str((ae)->source)
1081 -#define ae_units(ae) string2str((ae)->units)
1082 -#define ae_summary(ae) string2str((ae)->summary)
1083 -#define ae_info(ae) string2str((ae)->info)
1084 -#define ae_old_value_string(ae) string2str((ae)->old_value_string)
1085 -#define ae_new_value_string(ae) string2str((ae)->new_value_string)
1086 -
1087 -typedef struct alarm_log {
1088 - uint32_t next_log_id;
1089 - uint32_t next_alarm_id;
1090 - unsigned int count;
1091 - unsigned int max;
1092 - uint32_t health_log_retention_s; // the health log retention in seconds to be kept in db
1093 - ALARM_ENTRY *alarms;
1094 - RW_SPINLOCK spinlock;
1095 -} ALARM_LOG;
1096 -
1097 -typedef struct health {
1098 - time_t delay_up_to; // a timestamp to delay alarms processing up to
1099 - STRING *default_exec; // the full path of the alarms notifications program
1100 - STRING *default_recipient; // the default recipient for all alarms
1101 - bool enabled; // 1 when this host has health enabled
1102 - bool use_summary_for_notifications; // whether to use the summary field as a subject for notifications
1103 - int32_t pending_transitions; // pending alert transitions to store
1104 -} HEALTH;
1105 -
1015 // ----------------------------------------------------------------------------
1016 // RRD HOST
1017
@@ -1520,7 +1429,8 @@ void rrdhost_free_all(void);
1429 void rrdhost_system_info_free(struct rrdhost_system_info *system_info);
1430 void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force);
1431
1523 -int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s);
1432 +bool rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s);
1433 +bool rrdhost_should_run_health(RRDHOST *host);
1434
1435 void rrdset_update_heterogeneous_flag(RRDSET *st);
1436
src/database/rrdhost.c
+13 -5
@@ -743,19 +743,27 @@ RRDHOST *rrdhost_find_or_create(
743 return host;
744 }
745
746 -inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s) {
746 +bool rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s) {
747 if(host != protected_host
748 && host != localhost
749 && rrdhost_receiver_replicating_charts(host) == 0
750 && rrdhost_sender_replicating_charts(host) == 0
751 && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)
752 - && !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)
753 - && !host->receiver
752 + && !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD | RRDHOST_FLAG_HEALTH_RUNNING_NOW | RRDHOST_FLAG_COLLECTOR_ONLINE)
753 && host->stream.rcv.status.last_disconnected
754 && host->stream.rcv.status.last_disconnected + rrdhost_free_orphan_time_s < now_s)
756 - return 1;
755 + return true;
756
758 - return 0;
757 + return false;
758 +}
759 +
760 +bool rrdhost_should_run_health(RRDHOST *host) {
761 + if (!host->health.enabled ||
762 + !rrdhost_flag_check(host, RRDHOST_FLAG_COLLECTOR_ONLINE) ||
763 + rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))
764 + return false;
765 +
766 + return true;
767 }
768
769 void api_v1_management_init(void);
src/database/sqlite/sqlite_health.c
+1
@@ -4,6 +4,7 @@
4 #include "sqlite_functions.h"
5 #include "sqlite_db_migration.h"
6 #include "health/health_internals.h"
7 +#include "health/health-alert-entry.h"
8
9 #define MAX_HEALTH_SQL_SIZE 2048
10 #define SQLITE3_BIND_STRING_OR_NULL(res, param, key) \
src/database/sqlite/sqlite_metadata.c
+2
@@ -2,6 +2,8 @@
2
3 #include "sqlite_metadata.h"
4 #include "database/sqlite/vendored/sqlite3recover.h"
5 +#include "health/health-alert-entry.h"
6 +
7 //#include "sqlite_db_migration.h"
8
9 #define DB_METADATA_VERSION 18
src/health/health-alert-entry.h new
+82
@@ -0,0 +1,82 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_HEALTH_ALERT_ENTRY_H
4 +#define NETDATA_HEALTH_ALERT_ENTRY_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "rrdcalc.h"
8 +
9 +struct alarm_entry {
10 + uint32_t unique_id;
11 + uint32_t alarm_id;
12 + uint32_t alarm_event_id;
13 + usec_t global_id;
14 + nd_uuid_t config_hash_id;
15 + nd_uuid_t transition_id;
16 +
17 + time_t when;
18 + time_t duration;
19 + time_t non_clear_duration;
20 +
21 + STRING *name;
22 + STRING *chart;
23 + STRING *chart_context;
24 + STRING *chart_name;
25 +
26 + STRING *classification;
27 + STRING *component;
28 + STRING *type;
29 +
30 + STRING *exec;
31 + STRING *recipient;
32 + time_t exec_run_timestamp;
33 + int exec_code;
34 +
35 + STRING *source;
36 + STRING *units;
37 + STRING *summary;
38 + STRING *info;
39 +
40 + NETDATA_DOUBLE old_value;
41 + NETDATA_DOUBLE new_value;
42 +
43 + STRING *old_value_string;
44 + STRING *new_value_string;
45 +
46 + RRDCALC_STATUS old_status;
47 + RRDCALC_STATUS new_status;
48 +
49 + uint32_t flags;
50 + int32_t pending_save_count;
51 +
52 + int delay;
53 + time_t delay_up_to_timestamp;
54 +
55 + uint32_t updated_by_id;
56 + uint32_t updates_id;
57 +
58 + time_t last_repeat;
59 +
60 + POPEN_INSTANCE *popen_instance;
61 +
62 + struct alarm_entry *next;
63 + struct alarm_entry *next_in_progress;
64 + struct alarm_entry *prev_in_progress;
65 +};
66 +
67 +
68 +#define ae_name(ae) string2str((ae)->name)
69 +#define ae_chart_id(ae) string2str((ae)->chart)
70 +#define ae_chart_name(ae) string2str((ae)->chart_name)
71 +#define ae_chart_context(ae) string2str((ae)->chart_context)
72 +#define ae_classification(ae) string2str((ae)->classification)
73 +#define ae_exec(ae) string2str((ae)->exec)
74 +#define ae_recipient(ae) string2str((ae)->recipient)
75 +#define ae_source(ae) string2str((ae)->source)
76 +#define ae_units(ae) string2str((ae)->units)
77 +#define ae_summary(ae) string2str((ae)->summary)
78 +#define ae_info(ae) string2str((ae)->info)
79 +#define ae_old_value_string(ae) string2str((ae)->old_value_string)
80 +#define ae_new_value_string(ae) string2str((ae)->new_value_string)
81 +
82 +#endif //NETDATA_HEALTH_ALERT_ENTRY_H
src/health/health-alert-log.h new
+27
@@ -0,0 +1,27 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_HEALTH_ALERT_LOG_H
4 +#define NETDATA_HEALTH_ALERT_LOG_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +
8 +typedef struct alarm_log {
9 + uint32_t next_log_id;
10 + uint32_t next_alarm_id;
11 + unsigned int count;
12 + unsigned int max;
13 + uint32_t health_log_retention_s; // the health log retention in seconds to be kept in db
14 + struct alarm_entry *alarms;
15 + RW_SPINLOCK spinlock;
16 +} ALARM_LOG;
17 +
18 +typedef struct health {
19 + time_t delay_up_to; // a timestamp to delay alarms processing up to
20 + STRING *default_exec; // the full path of the alarms notifications program
21 + STRING *default_recipient; // the default recipient for all alarms
22 + bool enabled; // 1 when this host has health enabled
23 + bool use_summary_for_notifications; // whether to use the summary field as a subject for notifications
24 + int32_t pending_transitions; // pending alert transitions to store
25 +} HEALTH;
26 +
27 +#endif //NETDATA_HEALTH_ALERT_LOG_H
src/health/health_event_loop.c
+408 -415
@@ -2,6 +2,7 @@
2
3 #include "health.h"
4 #include "health_internals.h"
5 +#include "health-alert-entry.h"
6
7 #define WORKER_HEALTH_JOB_RRD_LOCK 0
8 #define WORKER_HEALTH_JOB_HOST_LOCK 1
@@ -207,474 +208,466 @@ static void do_eval_expression(
208 *result = expression_result(expression);
209 }
210
210 -static void health_event_loop(void) {
211 - bool health_running_logged = false;
211 +// returns the number of runnable alerts
212 +static void health_event_loop_for_host(RRDHOST *host, bool apply_hibernation_delay, time_t now, time_t *next_run) {
213 + size_t runnable = 0;
214
213 - unsigned int loop = 0;
215 + if(unlikely(!rrdhost_should_run_health(host)))
216 + return;
217
215 - while(service_running(SERVICE_HEALTH)) {
216 - if(!stream_control_health_should_be_running()) {
217 - worker_is_idle();
218 - stream_control_throttle();
219 - continue;
218 + //#define rrdhost_pending_alert_transitions(host) (__atomic_load_n(&((host)->aclk_config.alert_transition.pending), __ATOMIC_RELAXED))
219 +
220 + if (unlikely(__atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED))) {
221 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
222 + "Host \"%s\" has pending alert transitions to save, postponing health checks",
223 + rrdhost_hostname(host));
224 + return;
225 + }
226 +
227 + if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
228 + health_initialize_rrdhost(host);
229 +
230 + health_execute_delayed_initializations(host);
231 +
232 + if (unlikely(apply_hibernation_delay)) {
233 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
234 + "[%s]: Postponing health checks for %"PRId32" seconds.",
235 + rrdhost_hostname(host),
236 + health_globals.config.postpone_alarms_during_hibernation_for_seconds);
237 +
238 + host->health.delay_up_to =
239 + now + health_globals.config.postpone_alarms_during_hibernation_for_seconds;
240 + }
241 +
242 + if (unlikely(host->health.delay_up_to)) {
243 + if (unlikely(now < host->health.delay_up_to))
244 + return;
245 +
246 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
247 + "[%s]: Resuming health checks after delay.",
248 + rrdhost_hostname(host));
249 +
250 + host->health.delay_up_to = 0;
251 + }
252 +
253 + // wait until cleanup of obsolete charts on children is complete
254 + if (host != localhost) {
255 + if (unlikely(host->stream.rcv.status.check_obsolete)) {
256 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
257 + "[%s]: Waiting for chart obsoletion check.",
258 + rrdhost_hostname(host));
259 + return;
260 }
261 + }
262
222 - loop++;
223 - netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
263 + worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
264 + {
265 + struct aclk_sync_cfg_t *wc = host->aclk_config;
266 + if (wc && wc->send_snapshot == 2)
267 + return;
268 + }
269
225 - time_t now = now_realtime_sec();
226 - int runnable = 0, apply_hibernation_delay = 0;
227 - time_t next_run = now + health_globals.config.run_at_least_every_seconds;
228 - RRDCALC *rc;
229 - RRDHOST *host;
270 + // the first loop is to lookup values from the db
271 + RRDCALC *rc;
272 + foreach_rrdcalc_in_rrdhost_read(host, rc) {
273 + if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host)))
274 + break;
275
231 - if (unlikely(check_if_resumed_from_suspension())) {
232 - apply_hibernation_delay = 1;
276 + rrdcalc_update_info_using_rrdset_labels(rc);
277
234 - nd_log(NDLS_DAEMON, NDLP_NOTICE,
235 - "Postponing alarm checks for %"PRId32" seconds, "
236 - "because it seems that the system was just resumed from suspension.",
237 - (int32_t)health_globals.config.postpone_alarms_during_hibernation_for_seconds);
238 - schedule_node_state_update(localhost, 0);
278 + if (health_silencers_update_disabled_silenced(host, rc))
279 + continue;
280 +
281 + // create an alert removed event if the chart is obsolete and
282 + // has stopped being collected for 60 seconds
283 + if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
284 + rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
285 + now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
286 +
287 + if (!rrdcalc_isrepeating(rc)) {
288 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
289 + time_t now_tmp = now_realtime_sec();
290 +
291 + ALARM_ENTRY *ae =
292 + health_create_alarm_entry(
293 + host,
294 + rc,
295 + now_tmp,
296 + now_tmp - rc->last_status_change,
297 + rc->value,
298 + NAN,
299 + rc->status,
300 + RRDCALC_STATUS_REMOVED,
301 + 0,
302 + rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
303 +
304 + if (ae) {
305 + health_log_alert(host, ae);
306 + health_alarm_log_add_entry(host, ae);
307 + rc->old_status = rc->status;
308 + rc->status = RRDCALC_STATUS_REMOVED;
309 + rc->last_status_change = now_tmp;
310 + rc->last_status_change_value = rc->value;
311 + rc->last_updated = now_tmp;
312 + rc->value = NAN;
313 + }
314 + }
315 }
316
241 - if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
242 - static int logged=0;
243 - if (!logged) {
244 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
245 - "Skipping health checks, because all alarms are disabled via API command.");
246 - logged = 1;
317 + if (unlikely(!rrdcalc_isrunnable(rc, now, next_run))) {
318 + if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
319 + rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
320 + continue;
321 + }
322 +
323 + runnable++;
324 + rc->old_value = rc->value;
325 + rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
326 +
327 + // ------------------------------------------------------------
328 + // if there is database lookup, do it
329 +
330 + if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
331 + worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
332 +
333 + /* time_t old_db_timestamp = rc->db_before; */
334 + int value_is_null = 0;
335 +
336 + char group_options_buf[100];
337 + const char *group_options = group_options_buf;
338 + switch(rc->config.time_group) {
339 + default:
340 + group_options = NULL;
341 + break;
342 +
343 + case RRDR_GROUPING_PERCENTILE:
344 + case RRDR_GROUPING_TRIMMED_MEAN:
345 + case RRDR_GROUPING_TRIMMED_MEDIAN:
346 + snprintfz(group_options_buf, sizeof(group_options_buf),
347 + NETDATA_DOUBLE_FORMAT_AUTO,
348 + rc->config.time_group_value);
349 + break;
350 +
351 + case RRDR_GROUPING_COUNTIF:
352 + snprintfz(group_options_buf, sizeof(group_options_buf),
353 + "%s" NETDATA_DOUBLE_FORMAT_AUTO,
354 + alerts_group_conditions_id2txt(rc->config.time_group_condition),
355 + rc->config.time_group_value);
356 + break;
357 }
358 +
359 + int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
360 + rc->config.after, rc->config.before, rc->config.time_group, group_options,
361 + 0, rc->config.options | RRDR_OPTION_SELECTED_TIER,
362 + &rc->db_after,&rc->db_before,
363 + NULL, NULL, NULL,
364 + &value_is_null, NULL, 0, 0,
365 + QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_SYNCHRONOUS);
366 +
367 + if (unlikely(ret != 200)) {
368 + // database lookup failed
369 + rc->value = NAN;
370 + rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
371 +
372 + netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
373 + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
374 + );
375 + } else
376 + rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
377 +
378 + if (unlikely(value_is_null)) {
379 + // collected value is null
380 + rc->value = NAN;
381 + rc->run_flags |= RRDCALC_FLAG_DB_NAN;
382 +
383 + netdata_log_debug(D_HEALTH,
384 + "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
385 + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
386 + );
387 + } else
388 + rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
389 +
390 + netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
391 + rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
392 + );
393 }
394
250 - worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
251 - dfe_start_reentrant(rrdhost_root_index, host) {
395 + // ------------------------------------------------------------
396 + // if there is calculation expression, run it
397
253 - if(unlikely(!service_running(SERVICE_HEALTH)))
398 + do_eval_expression(rc, rc->config.calculation, "calculation", WORKER_HEALTH_JOB_CALC_EVAL, RRDCALC_FLAG_CALC_ERROR, NULL, &rc->value);
399 + }
400 + foreach_rrdcalc_in_rrdhost_done(rc);
401 +
402 + struct health_raised_summary *hrm = alerts_raised_summary_create(host);
403 +
404 + if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
405 + foreach_rrdcalc_in_rrdhost_read(host, rc) {
406 + if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host)))
407 break;
408
256 - if (unlikely(!host->health.enabled))
409 + if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
410 continue;
411
259 -//#define rrdhost_pending_alert_transitions(host) (__atomic_load_n(&((host)->aclk_config.alert_transition.pending), __ATOMIC_RELAXED))
260 -
261 - if (unlikely(__atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED))) {
262 - nd_log(
263 - NDLS_DAEMON,
264 - NDLP_DEBUG,
265 - "Host \"%s\" has pending alert transitions to save, postponing health checks",
266 - rrdhost_hostname(host));
412 + if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
413 continue;
414 }
415 + RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
416 + RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
417
270 - if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
271 - health_initialize_rrdhost(host);
418 + do_eval_expression(rc, rc->config.warning, "warning", WORKER_HEALTH_JOB_WARNING_EVAL, RRDCALC_FLAG_WARN_ERROR, &warning_status, NULL);
419 + do_eval_expression(rc, rc->config.critical, "critical", WORKER_HEALTH_JOB_CRITICAL_EVAL, RRDCALC_FLAG_CRIT_ERROR, &critical_status, NULL);
420
273 - health_execute_delayed_initializations(host);
421 + // --------------------------------------------------------
422 + // decide the final alarm status
423
275 - if (unlikely(apply_hibernation_delay)) {
276 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
277 - "[%s]: Postponing health checks for %"PRId32" seconds.",
278 - rrdhost_hostname(host),
279 - health_globals.config.postpone_alarms_during_hibernation_for_seconds);
424 + RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
425 +
426 + switch (warning_status) {
427 + case RRDCALC_STATUS_CLEAR:
428 + status = RRDCALC_STATUS_CLEAR;
429 + break;
430 +
431 + case RRDCALC_STATUS_RAISED:
432 + status = RRDCALC_STATUS_WARNING;
433 + break;
434
281 - host->health.delay_up_to =
282 - now + health_globals.config.postpone_alarms_during_hibernation_for_seconds;
435 + default:
436 + break;
437 }
438
285 - if (unlikely(host->health.delay_up_to)) {
286 - if (unlikely(now < host->health.delay_up_to)) {
287 - continue;
288 - }
439 + switch (critical_status) {
440 + case RRDCALC_STATUS_CLEAR:
441 + if (status == RRDCALC_STATUS_UNDEFINED)
442 + status = RRDCALC_STATUS_CLEAR;
443 + break;
444
290 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
291 - "[%s]: Resuming health checks after delay.",
292 - rrdhost_hostname(host));
445 + case RRDCALC_STATUS_RAISED:
446 + status = RRDCALC_STATUS_CRITICAL;
447 + break;
448
294 - host->health.delay_up_to = 0;
449 + default:
450 + break;
451 }
452
297 - // wait until cleanup of obsolete charts on children is complete
298 - if (host != localhost) {
299 - if (unlikely(host->stream.rcv.status.check_obsolete)) {
453 + // --------------------------------------------------------
454 + // check if the new status and the old differ
455 +
456 + if (status != rc->status) {
457 +
458 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
459 + int delay;
460 +
461 + // apply trigger hysteresis
462
301 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
302 - "[%s]: Waiting for chart obsoletion check.",
303 - rrdhost_hostname(host));
463 + if (now > rc->delay_up_to_timestamp) {
464 + rc->delay_up_current = rc->config.delay_up_duration;
465 + rc->delay_down_current = rc->config.delay_down_duration;
466 + rc->delay_last = 0;
467 + rc->delay_up_to_timestamp = 0;
468 + } else {
469 + rc->delay_up_current = (int)((float)rc->delay_up_current * rc->config.delay_multiplier);
470 + if (rc->delay_up_current > rc->config.delay_max_duration)
471 + rc->delay_up_current = rc->config.delay_max_duration;
472
305 - continue;
473 + rc->delay_down_current = (int)((float)rc->delay_down_current * rc->config.delay_multiplier);
474 + if (rc->delay_down_current > rc->config.delay_max_duration)
475 + rc->delay_down_current = rc->config.delay_max_duration;
476 }
307 - }
477
309 - if (!health_running_logged) {
310 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
311 - "[%s]: Health is running.",
312 - rrdhost_hostname(host));
478 + if (status > rc->status)
479 + delay = rc->delay_up_current;
480 + else
481 + delay = rc->delay_down_current;
482 +
483 + // COMMENTED: because we do need to send raising alarms
484 + // if (now + delay < rc->delay_up_to_timestamp)
485 + // delay = (int)(rc->delay_up_to_timestamp - now);
486 +
487 + rc->delay_last = delay;
488 + rc->delay_up_to_timestamp = now + delay;
489 +
490 + ALARM_ENTRY *ae =
491 + health_create_alarm_entry(
492 + host,
493 + rc,
494 + now,
495 + now - rc->last_status_change,
496 + rc->old_value,
497 + rc->value,
498 + rc->status,
499 + status,
500 + rc->delay_last,
501 + (
502 + ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
503 + ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
504 + (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
505 + )
506 + );
507
314 - health_running_logged = true;
315 - }
508 + health_log_alert(host, ae);
509 + health_alarm_log_add_entry(host, ae);
510
317 - worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
318 - {
319 - struct aclk_sync_cfg_t *wc = host->aclk_config;
320 - if (wc && wc->send_snapshot == 2)
321 - continue;
511 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
512 + "[%s]: Alert event for [%s.%s], value [%s], status [%s].",
513 + rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), ae_new_value_string(ae),
514 + rrdcalc_status2string(ae->new_status));
515 +
516 + rc->last_status_change_value = rc->value;
517 + rc->last_status_change = now;
518 + rc->old_status = rc->status;
519 + rc->status = status;
520 +
521 + if(unlikely(rrdcalc_isrepeating(rc))) {
522 + rc->last_repeat = now;
523 + if (rc->status == RRDCALC_STATUS_CLEAR)
524 + rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
525 + }
526 }
527
324 - // the first loop is to lookup values from the db
325 - foreach_rrdcalc_in_rrdhost_read(host, rc) {
528 + rc->last_updated = now;
529 + rc->next_update = now + rc->config.update_every;
530
327 - if(unlikely(!service_running(SERVICE_HEALTH)))
328 - break;
531 + if (*next_run > rc->next_update)
532 + *next_run = rc->next_update;
533 + }
534 + foreach_rrdcalc_in_rrdhost_done(rc);
535
330 - rrdcalc_update_info_using_rrdset_labels(rc);
331 -
332 - if (health_silencers_update_disabled_silenced(host, rc))
333 - continue;
334 -
335 - // create an alert removed event if the chart is obsolete and
336 - // has stopped being collected for 60 seconds
337 - if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
338 - rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
339 - now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
340 -
341 - if (!rrdcalc_isrepeating(rc)) {
342 - worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
343 - time_t now_tmp = now_realtime_sec();
344 -
345 - ALARM_ENTRY *ae =
346 - health_create_alarm_entry(
347 - host,
348 - rc,
349 - now_tmp,
350 - now_tmp - rc->last_status_change,
351 - rc->value,
352 - NAN,
353 - rc->status,
354 - RRDCALC_STATUS_REMOVED,
355 - 0,
356 - rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
357 -
358 - if (ae) {
359 - health_log_alert(host, ae);
360 - health_alarm_log_add_entry(host, ae);
361 - rc->old_status = rc->status;
362 - rc->status = RRDCALC_STATUS_REMOVED;
363 - rc->last_status_change = now_tmp;
364 - rc->last_status_change_value = rc->value;
365 - rc->last_updated = now_tmp;
366 - rc->value = NAN;
367 - }
368 - }
369 - }
536 + alerts_raised_summary_populate(hrm);
537
371 - if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
372 - if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
373 - rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
374 - continue;
375 - }
538 + // process repeating alarms
539 + foreach_rrdcalc_in_rrdhost_read(host, rc) {
540 + if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host)))
541 + break;
542
377 - runnable++;
378 - rc->old_value = rc->value;
379 - rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
380 -
381 - // ------------------------------------------------------------
382 - // if there is database lookup, do it
383 -
384 - if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
385 - worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
386 -
387 - /* time_t old_db_timestamp = rc->db_before; */
388 - int value_is_null = 0;
389 -
390 - char group_options_buf[100];
391 - const char *group_options = group_options_buf;
392 - switch(rc->config.time_group) {
393 - default:
394 - group_options = NULL;
395 - break;
396 -
397 - case RRDR_GROUPING_PERCENTILE:
398 - case RRDR_GROUPING_TRIMMED_MEAN:
399 - case RRDR_GROUPING_TRIMMED_MEDIAN:
400 - snprintfz(group_options_buf, sizeof(group_options_buf),
401 - NETDATA_DOUBLE_FORMAT_AUTO,
402 - rc->config.time_group_value);
403 - break;
404 -
405 - case RRDR_GROUPING_COUNTIF:
406 - snprintfz(group_options_buf, sizeof(group_options_buf),
407 - "%s" NETDATA_DOUBLE_FORMAT_AUTO,
408 - alerts_group_conditions_id2txt(rc->config.time_group_condition),
409 - rc->config.time_group_value);
410 - break;
411 - }
412 -
413 - int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
414 - rc->config.after, rc->config.before, rc->config.time_group, group_options,
415 - 0, rc->config.options | RRDR_OPTION_SELECTED_TIER,
416 - &rc->db_after,&rc->db_before,
417 - NULL, NULL, NULL,
418 - &value_is_null, NULL, 0, 0,
419 - QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_SYNCHRONOUS);
420 -
421 - if (unlikely(ret != 200)) {
422 - // database lookup failed
423 - rc->value = NAN;
424 - rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
425 -
426 - netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
427 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
428 - );
429 - } else
430 - rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
431 -
432 - if (unlikely(value_is_null)) {
433 - // collected value is null
434 - rc->value = NAN;
435 - rc->run_flags |= RRDCALC_FLAG_DB_NAN;
436 -
437 - netdata_log_debug(D_HEALTH,
438 - "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
439 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
440 - );
441 - } else
442 - rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
443 -
444 - netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
445 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
446 - );
543 + int repeat_every = 0;
544 + if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
545 + if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
546 + rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
547 + repeat_every = (int)rc->config.warn_repeat_every;
548 }
549 + else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
550 + rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
551 + repeat_every = (int)rc->config.crit_repeat_every;
552 + }
553 + else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
554 + if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) &&
555 + (rc->old_status == RRDCALC_STATUS_CRITICAL || rc->old_status == RRDCALC_STATUS_WARNING))
556 + repeat_every = 1;
557 + }
558 + }
559 + else
560 + continue;
561
449 - // ------------------------------------------------------------
450 - // if there is calculation expression, run it
562 + if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
563 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
564 + rc->last_repeat = now;
565 + if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
566 + ALARM_ENTRY *ae =
567 + health_create_alarm_entry(
568 + host,
569 + rc,
570 + now,
571 + now - rc->last_status_change,
572 + rc->old_value,
573 + rc->value,
574 + rc->old_status,
575 + rc->status,
576 + rc->delay_last,
577 + (
578 + ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
579 + ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
580 + (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
581 + )
582 + );
583
452 - do_eval_expression(rc, rc->config.calculation, "calculation", WORKER_HEALTH_JOB_CALC_EVAL, RRDCALC_FLAG_CALC_ERROR, NULL, &rc->value);
453 - }
454 - foreach_rrdcalc_in_rrdhost_done(rc);
455 -
456 - struct health_raised_summary *hrm = alerts_raised_summary_create(host);
457 -
458 - if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
459 - foreach_rrdcalc_in_rrdhost_read(host, rc) {
460 - if(unlikely(!service_running(SERVICE_HEALTH)))
461 - break;
462 -
463 - if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
464 - continue;
465 -
466 - if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
467 - continue;
468 - }
469 - RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
470 - RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
471 -
472 - do_eval_expression(rc, rc->config.warning, "warning", WORKER_HEALTH_JOB_WARNING_EVAL, RRDCALC_FLAG_WARN_ERROR, &warning_status, NULL);
473 - do_eval_expression(rc, rc->config.critical, "critical", WORKER_HEALTH_JOB_CRITICAL_EVAL, RRDCALC_FLAG_CRIT_ERROR, &critical_status, NULL);
474 -
475 - // --------------------------------------------------------
476 - // decide the final alarm status
477 -
478 - RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
479 -
480 - switch (warning_status) {
481 - case RRDCALC_STATUS_CLEAR:
482 - status = RRDCALC_STATUS_CLEAR;
483 - break;
484 -
485 - case RRDCALC_STATUS_RAISED:
486 - status = RRDCALC_STATUS_WARNING;
487 - break;
488 -
489 - default:
490 - break;
491 - }
492 -
493 - switch (critical_status) {
494 - case RRDCALC_STATUS_CLEAR:
495 - if (status == RRDCALC_STATUS_UNDEFINED)
496 - status = RRDCALC_STATUS_CLEAR;
497 - break;
498 -
499 - case RRDCALC_STATUS_RAISED:
500 - status = RRDCALC_STATUS_CRITICAL;
501 - break;
502 -
503 - default:
504 - break;
505 - }
506 -
507 - // --------------------------------------------------------
508 - // check if the new status and the old differ
509 -
510 - if (status != rc->status) {
511 -
512 - worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
513 - int delay;
514 -
515 - // apply trigger hysteresis
516 -
517 - if (now > rc->delay_up_to_timestamp) {
518 - rc->delay_up_current = rc->config.delay_up_duration;
519 - rc->delay_down_current = rc->config.delay_down_duration;
520 - rc->delay_last = 0;
521 - rc->delay_up_to_timestamp = 0;
522 - } else {
523 - rc->delay_up_current = (int)((float)rc->delay_up_current * rc->config.delay_multiplier);
524 - if (rc->delay_up_current > rc->config.delay_max_duration)
525 - rc->delay_up_current = rc->config.delay_max_duration;
526 -
527 - rc->delay_down_current = (int)((float)rc->delay_down_current * rc->config.delay_multiplier);
528 - if (rc->delay_down_current > rc->config.delay_max_duration)
529 - rc->delay_down_current = rc->config.delay_max_duration;
530 - }
531 -
532 - if (status > rc->status)
533 - delay = rc->delay_up_current;
534 - else
535 - delay = rc->delay_down_current;
536 -
537 - // COMMENTED: because we do need to send raising alarms
538 - // if (now + delay < rc->delay_up_to_timestamp)
539 - // delay = (int)(rc->delay_up_to_timestamp - now);
540 -
541 - rc->delay_last = delay;
542 - rc->delay_up_to_timestamp = now + delay;
543 -
544 - ALARM_ENTRY *ae =
545 - health_create_alarm_entry(
546 - host,
547 - rc,
548 - now,
549 - now - rc->last_status_change,
550 - rc->old_value,
551 - rc->value,
552 - rc->status,
553 - status,
554 - rc->delay_last,
555 - (
556 - ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
557 - ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
558 - (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
559 - )
560 - );
561 -
562 - health_log_alert(host, ae);
563 - health_alarm_log_add_entry(host, ae);
564 -
565 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
566 - "[%s]: Alert event for [%s.%s], value [%s], status [%s].",
567 - rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), ae_new_value_string(ae),
568 - rrdcalc_status2string(ae->new_status));
569 -
570 - rc->last_status_change_value = rc->value;
571 - rc->last_status_change = now;
572 - rc->old_status = rc->status;
573 - rc->status = status;
574 -
575 - if(unlikely(rrdcalc_isrepeating(rc))) {
576 - rc->last_repeat = now;
577 - if (rc->status == RRDCALC_STATUS_CLEAR)
578 - rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
579 - }
580 - }
581 -
582 - rc->last_updated = now;
583 - rc->next_update = now + rc->config.update_every;
584 -
585 - if (next_run > rc->next_update)
586 - next_run = rc->next_update;
587 - }
588 - foreach_rrdcalc_in_rrdhost_done(rc);
589 -
590 - alerts_raised_summary_populate(hrm);
591 -
592 - // process repeating alarms
593 - foreach_rrdcalc_in_rrdhost_read(host, rc) {
594 - if(unlikely(!service_running(SERVICE_HEALTH)))
595 - break;
596 -
597 - int repeat_every = 0;
598 - if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
599 - if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
600 - rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
601 - repeat_every = (int)rc->config.warn_repeat_every;
602 - }
603 - else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
604 - rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
605 - repeat_every = (int)rc->config.crit_repeat_every;
606 - }
607 - else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
608 - if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) &&
609 - (rc->old_status == RRDCALC_STATUS_CRITICAL || rc->old_status == RRDCALC_STATUS_WARNING))
610 - repeat_every = 1;
611 - }
612 - }
613 - else
614 - continue;
615 -
616 - if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
617 - worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
618 - rc->last_repeat = now;
619 - if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
620 - ALARM_ENTRY *ae =
621 - health_create_alarm_entry(
622 - host,
623 - rc,
624 - now,
625 - now - rc->last_status_change,
626 - rc->old_value,
627 - rc->value,
628 - rc->old_status,
629 - rc->status,
630 - rc->delay_last,
631 - (
632 - ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
633 - ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
634 - (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
635 - )
636 - );
637 -
638 - health_log_alert(host, ae);
639 - ae->last_repeat = rc->last_repeat;
640 - if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
641 - ae->flags |= HEALTH_ENTRY_RUN_ONCE;
642 - }
643 - rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
644 - health_send_notification(host, ae, hrm);
645 - netdata_log_debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
646 - health_alarm_wait_for_execution(ae);
647 - health_alarm_log_free_one_nochecks_nounlink(ae);
648 - }
584 + health_log_alert(host, ae);
585 + ae->last_repeat = rc->last_repeat;
586 + if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
587 + ae->flags |= HEALTH_ENTRY_RUN_ONCE;
588 }
650 - foreach_rrdcalc_in_rrdhost_done(rc);
589 + rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
590 + health_send_notification(host, ae, hrm);
591 + netdata_log_debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
592 + health_alarm_wait_for_execution(ae);
593 + health_alarm_log_free_one_nochecks_nounlink(ae);
594 }
595 + }
596 + foreach_rrdcalc_in_rrdhost_done(rc);
597 + }
598
653 - if (unlikely(!service_running(SERVICE_HEALTH)))
654 - break;
599 + if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host)))
600 + return;
601
656 - // execute notifications
657 - // and cleanup
602 + // execute notifications
603 + // and cleanup
604 +
605 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
606 + health_alarm_log_process_to_send_notifications(host, hrm);
607 + alerts_raised_summary_free(hrm);
608 +
609 + int32_t pending = __atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED);
610 + if (pending)
611 + commit_alert_transitions(host);
612 +
613 + if (!__atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED)) {
614 + struct aclk_sync_cfg_t *wc = host->aclk_config;
615 + if (wc && wc->send_snapshot == 1) {
616 + wc->send_snapshot = 2;
617 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
618 + } else {
619 + if (process_alert_pending_queue(host))
620 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
621 + }
622 + }
623 +}
624
659 - worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
660 - health_alarm_log_process_to_send_notifications(host, hrm);
661 - alerts_raised_summary_free(hrm);
625 +static void health_event_loop(void) {
626 + unsigned int loop = 0;
627
663 - int32_t pending = __atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED);
664 - if (pending)
665 - commit_alert_transitions(host);
628 + while(service_running(SERVICE_HEALTH)) {
629 + if(!stream_control_health_should_be_running()) {
630 + worker_is_idle();
631 + stream_control_throttle();
632 + continue;
633 + }
634
667 - if (!__atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED)) {
668 - struct aclk_sync_cfg_t *wc = host->aclk_config;
669 - if (wc && wc->send_snapshot == 1) {
670 - wc->send_snapshot = 2;
671 - rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
672 - } else {
673 - if (process_alert_pending_queue(host))
674 - rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
675 - }
635 + loop++;
636 + netdata_log_debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
637 +
638 + time_t now = now_realtime_sec();
639 + bool apply_hibernation_delay = false;
640 + time_t next_run = now + health_globals.config.run_at_least_every_seconds;
641 +
642 + if (unlikely(check_if_resumed_from_suspension())) {
643 + apply_hibernation_delay = true;
644 +
645 + nd_log(NDLS_DAEMON, NDLP_NOTICE,
646 + "Postponing alarm checks for %"PRId32" seconds, "
647 + "because it seems that the system was just resumed from suspension.",
648 + (int32_t)health_globals.config.postpone_alarms_during_hibernation_for_seconds);
649 + schedule_node_state_update(localhost, 0);
650 + }
651 +
652 + if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
653 + static int logged=0;
654 + if (!logged) {
655 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
656 + "Skipping health checks, because all alarms are disabled via API command.");
657 + logged = 1;
658 }
659 }
660 +
661 + worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
662 + RRDHOST *host;
663 + dfe_start_reentrant(rrdhost_root_index, host) {
664 + if(unlikely(!service_running(SERVICE_HEALTH)))
665 + break;
666 +
667 + rrdhost_flag_set(host, RRDHOST_FLAG_HEALTH_RUNNING_NOW);
668 + health_event_loop_for_host(host, apply_hibernation_delay, now, &next_run);
669 + rrdhost_flag_clear(host, RRDHOST_FLAG_HEALTH_RUNNING_NOW);
670 + }
671 dfe_done(host);
672
673 if(unlikely(!service_running(SERVICE_HEALTH)))
src/health/health_log.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "health.h"
4 +#include "health-alert-entry.h"
5
6 // ----------------------------------------------------------------------------
7
src/health/health_notifications.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "health_internals.h"
4 +#include "health-alert-entry.h"
5
6 // the queue of executed alarm notifications that haven't been waited for yet
7 static struct {
src/health/health_prototypes.c
+1
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "health_internals.h"
4 +#include "health-alert-entry.h"
5
6 // ---------------------------------------------------------------------------------------------------------------------
7
src/health/rrdcalc.c
+1
@@ -2,6 +2,7 @@
2
3 #include "database/rrd.h"
4 #include "health_internals.h"
5 +#include "health-alert-entry.h"
6
7 // ----------------------------------------------------------------------------
8 // RRDCALC helpers