| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "health.h" |
| 4 | #include "health_internals.h" |
| 5 | #include "health-alert-entry.h" |
| 6 | |
| 7 | #define WORKER_HEALTH_JOB_RRD_LOCK 0 |
| 8 | #define WORKER_HEALTH_JOB_HOST_LOCK 1 |
| 9 | #define WORKER_HEALTH_JOB_DB_QUERY 2 |
| 10 | #define WORKER_HEALTH_JOB_CALC_EVAL 3 |
| 11 | #define WORKER_HEALTH_JOB_WARNING_EVAL 4 |
| 12 | #define WORKER_HEALTH_JOB_CRITICAL_EVAL 5 |
| 13 | #define WORKER_HEALTH_JOB_ALARM_LOG_ENTRY 6 |
| 14 | #define WORKER_HEALTH_JOB_ALARM_LOG_PROCESS 7 |
| 15 | #define WORKER_HEALTH_JOB_ALARM_LOG_QUEUE 8 |
| 16 | #define WORKER_HEALTH_JOB_WAIT_EXEC 9 |
| 17 | #define WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET 10 |
| 18 | #define WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM 11 |
| 19 | |
| 20 | #if WORKER_UTILIZATION_MAX_JOB_TYPES < 10 |
| 21 | #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 10 |
| 22 | #endif |
| 23 | |
| 24 | static uint64_t health_evloop_iteration = 0; |
| 25 | |
| 26 | uint64_t health_evloop_current_iteration(void) { |
| 27 | return __atomic_load_n(&health_evloop_iteration, __ATOMIC_RELAXED); |
| 28 | } |
| 29 | |
| 30 | uint64_t rrdhost_health_evloop_last_iteration(RRDHOST *host) { |
| 31 | return __atomic_load_n(&host->health.evloop_iteration, __ATOMIC_RELAXED); |
| 32 | } |
| 33 | |
| 34 | void rrdhost_set_health_evloop_iteration(RRDHOST *host) { |
| 35 | __atomic_store_n(&host->health.evloop_iteration, |
| 36 | health_evloop_current_iteration(), __ATOMIC_RELAXED); |
| 37 | } |
| 38 | |
| 39 | static inline void health_alert_status_counts_add(struct health_alert_status_counts *c, RRDCALC_STATUS status) { |
| 40 | switch(status) { |
| 41 | case RRDCALC_STATUS_CLEAR: |
| 42 | c->clear++; |
| 43 | break; |
| 44 | |
| 45 | case RRDCALC_STATUS_WARNING: |
| 46 | c->warning++; |
| 47 | break; |
| 48 | |
| 49 | case RRDCALC_STATUS_CRITICAL: |
| 50 | c->critical++; |
| 51 | break; |
| 52 | |
| 53 | case RRDCALC_STATUS_UNDEFINED: |
| 54 | c->undefined++; |
| 55 | break; |
| 56 | |
| 57 | case RRDCALC_STATUS_UNINITIALIZED: |
| 58 | c->uninitialized++; |
| 59 | break; |
| 60 | |
| 61 | default: |
| 62 | break; |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | static inline void health_alert_status_counts_sub(struct health_alert_status_counts *c, RRDCALC_STATUS status) { |
| 67 | switch(status) { |
| 68 | case RRDCALC_STATUS_CLEAR: |
| 69 | if(c->clear) c->clear--; |
| 70 | break; |
| 71 | |
| 72 | case RRDCALC_STATUS_WARNING: |
| 73 | if(c->warning) c->warning--; |
| 74 | break; |
| 75 | |
| 76 | case RRDCALC_STATUS_CRITICAL: |
| 77 | if(c->critical) c->critical--; |
| 78 | break; |
| 79 | |
| 80 | case RRDCALC_STATUS_UNDEFINED: |
| 81 | if(c->undefined) c->undefined--; |
| 82 | break; |
| 83 | |
| 84 | case RRDCALC_STATUS_UNINITIALIZED: |
| 85 | if(c->uninitialized) c->uninitialized--; |
| 86 | break; |
| 87 | |
| 88 | default: |
| 89 | break; |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | static inline uint64_t health_alert_status_snapshot_begin_update(RRDHOST *host) { |
| 94 | // Make generation odd (writer in progress) so readers can safely retry/fallback. |
| 95 | uint64_t generation = __atomic_add_fetch(&host->health.alert_status_snapshot.generation, 1, __ATOMIC_ACQ_REL); |
| 96 | if(!(generation & 1)) |
| 97 | generation = __atomic_add_fetch(&host->health.alert_status_snapshot.generation, 1, __ATOMIC_ACQ_REL); |
| 98 | |
| 99 | __atomic_store_n(&host->health.alert_status_snapshot.valid, 0, __ATOMIC_RELEASE); |
| 100 | return generation; |
| 101 | } |
| 102 | |
| 103 | static inline void health_alert_status_snapshot_finish_update( |
| 104 | RRDHOST *host, |
| 105 | const struct health_alert_status_counts *counts, |
| 106 | uint64_t odd_generation) { |
| 107 | |
| 108 | __atomic_store_n(&host->health.alert_status_snapshot.counts.clear, counts->clear, __ATOMIC_RELAXED); |
| 109 | __atomic_store_n(&host->health.alert_status_snapshot.counts.warning, counts->warning, __ATOMIC_RELAXED); |
| 110 | __atomic_store_n(&host->health.alert_status_snapshot.counts.critical, counts->critical, __ATOMIC_RELAXED); |
| 111 | __atomic_store_n(&host->health.alert_status_snapshot.counts.undefined, counts->undefined, __ATOMIC_RELAXED); |
| 112 | __atomic_store_n(&host->health.alert_status_snapshot.counts.uninitialized, counts->uninitialized, __ATOMIC_RELAXED); |
| 113 | |
| 114 | __atomic_store_n(&host->health.alert_status_snapshot.valid, 1, __ATOMIC_RELEASE); |
| 115 | __atomic_store_n(&host->health.alert_status_snapshot.generation, odd_generation + 1, __ATOMIC_RELEASE); |
| 116 | } |
| 117 | |
| 118 | // ---------------------------------------------------------------------------- |
| 119 | // health main thread and friends |
| 120 | |
| 121 | static inline RRDCALC_STATUS rrdcalc_value2status(NETDATA_DOUBLE n) { |
| 122 | if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED; |
| 123 | if(n) return RRDCALC_STATUS_RAISED; |
| 124 | return RRDCALC_STATUS_CLEAR; |
| 125 | } |
| 126 | |
| 127 | static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) { |
| 128 | if(unlikely(!rc->rrdset)) { |
| 129 | netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); |
| 130 | return 0; |
| 131 | } |
| 132 | |
| 133 | if(unlikely(rc->next_update > now)) { |
| 134 | if (unlikely(*next_run > rc->next_update)) { |
| 135 | // update the next_run time of the main loop |
| 136 | // to run this alarm precisely the time required |
| 137 | *next_run = rc->next_update; |
| 138 | } |
| 139 | |
| 140 | netdata_log_debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rrdcalc_chart_name(rc), rrdcalc_name(rc), (int) (rc->next_update - now)); |
| 141 | return 0; |
| 142 | } |
| 143 | |
| 144 | if(unlikely(!rc->config.update_every)) { |
| 145 | netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rrdcalc_chart_name(rc), rrdcalc_name(rc)); |
| 146 | return 0; |
| 147 | } |
| 148 | |
| 149 | if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE))) { |
| 150 | netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rrdcalc_chart_name(rc), rrdcalc_name(rc)); |
| 151 | return 0; |
| 152 | } |
| 153 | |
| 154 | if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) { |
| 155 | netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rrdcalc_chart_name(rc), rrdcalc_name(rc)); |
| 156 | return 0; |
| 157 | } |
| 158 | |
| 159 | int update_every = rc->rrdset->update_every; |
| 160 | time_t first = rrdset_first_entry_s(rc->rrdset); |
| 161 | time_t last = rrdset_last_entry_s(rc->rrdset); |
| 162 | |
| 163 | if(unlikely(now + update_every < first /* || now - update_every > last */)) { |
| 164 | netdata_log_debug(D_HEALTH |
| 165 | , "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)." |
| 166 | , rrdcalc_chart_name(rc), rrdcalc_name(rc), (unsigned long) now, (unsigned long) first |
| 167 | , (unsigned long) last); |
| 168 | return 0; |
| 169 | } |
| 170 | |
| 171 | if(RRDCALC_HAS_DB_LOOKUP(rc)) { |
| 172 | time_t needed = now + rc->config.before + rc->config.after; |
| 173 | |
| 174 | if(needed + update_every < first || needed - update_every > last) { |
| 175 | netdata_log_debug(D_HEALTH, |
| 176 | "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu).", |
| 177 | rrdcalc_chart_name(rc), |
| 178 | rrdcalc_name(rc), |
| 179 | (unsigned long) needed, |
| 180 | (unsigned long) first, |
| 181 | (unsigned long) last); |
| 182 | return 0; |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | return 1; |
| 187 | } |
| 188 | |
| 189 | static void health_sleep(time_t next_run, uint64_t loop __maybe_unused) { |
| 190 | time_t now = now_realtime_sec(); |
| 191 | if(now < next_run) { |
| 192 | worker_is_idle(); |
| 193 | netdata_log_debug(D_HEALTH, "Health monitoring iteration no %llu done. Next iteration in %d secs", |
| 194 | (unsigned long long)loop, (int) (next_run - now)); |
| 195 | while (now < next_run && service_running(SERVICE_HEALTH)) { |
| 196 | sleep_usec(USEC_PER_SEC); |
| 197 | now = now_realtime_sec(); |
| 198 | } |
| 199 | } |
| 200 | else { |
| 201 | netdata_log_debug(D_HEALTH, "Health monitoring iteration no %llu done. Next iteration now", |
| 202 | (unsigned long long)loop); |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | static void health_execute_delayed_initializations(RRDHOST *host) { |
| 207 | health_plugin_init(); |
| 208 | |
| 209 | RRDSET *st; |
| 210 | |
| 211 | // Atomically snapshot + clear the host pending flags. A separate check-then-clear |
| 212 | // would race with concurrent setters (e.g. label updaters in plugins/streaming), |
| 213 | // and a flag set between check and clear would be lost. |
| 214 | RRDHOST_FLAGS old_host_flags = rrdhost_flag_set_and_clear( |
| 215 | host, 0, |
| 216 | RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION | RRDHOST_FLAG_PENDING_LABEL_RECHECK); |
| 217 | |
| 218 | bool host_pending_init = old_host_flags & RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION; |
| 219 | bool host_pending_recheck = old_host_flags & RRDHOST_FLAG_PENDING_LABEL_RECHECK; |
| 220 | |
| 221 | if (!host_pending_init && !host_pending_recheck) return; |
| 222 | |
| 223 | rrdset_foreach_reentrant(st, host) { |
| 224 | // Same race applies per-chart: snapshot + clear atomically so a concurrent |
| 225 | // CLABEL stream commit or rrdset_update_rrdlabels() cannot have its |
| 226 | // pending-recheck request swallowed by a separate clear. |
| 227 | RRDSET_FLAGS old_st_flags = rrdset_flag_set_and_clear( |
| 228 | st, 0, |
| 229 | RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION | RRDSET_FLAG_PENDING_LABEL_RECHECK); |
| 230 | |
| 231 | bool needs_init = old_st_flags & RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION; |
| 232 | bool needs_recheck = host_pending_recheck || (old_st_flags & RRDSET_FLAG_PENDING_LABEL_RECHECK); |
| 233 | |
| 234 | if (!needs_init && !needs_recheck) |
| 235 | continue; |
| 236 | |
| 237 | worker_is_busy(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET); |
| 238 | |
| 239 | // recheck path subsumes init: reset detaches all current alerts on the |
| 240 | // chart and reattaches by re-evaluating every prototype against the |
| 241 | // current labels, which also picks up first-attach matches. |
| 242 | if (needs_recheck) |
| 243 | health_prototype_reset_alerts_for_rrdset(st); |
| 244 | else |
| 245 | health_prototype_alerts_for_rrdset_incrementally(st); |
| 246 | |
| 247 | if (!service_running(SERVICE_HEALTH)) |
| 248 | break; |
| 249 | } |
| 250 | rrdset_foreach_done(st); |
| 251 | } |
| 252 | |
| 253 | static void health_initialize_rrdhost(RRDHOST *host) { |
| 254 | health_plugin_init(); |
| 255 | |
| 256 | if(!host->health.enabled || |
| 257 | rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH) || |
| 258 | !service_running(SERVICE_HEALTH)) |
| 259 | return; |
| 260 | |
| 261 | host->health_log.max = health_globals.config.health_log_entries_max; |
| 262 | host->health_log.health_log_retention_s = health_globals.config.health_log_retention_s; |
| 263 | host->health.default_exec = string_dup(health_globals.config.default_exec); |
| 264 | host->health.default_recipient = string_dup(health_globals.config.default_recipient); |
| 265 | host->health.use_summary_for_notifications = health_globals.config.use_summary_for_notifications; |
| 266 | |
| 267 | host->health_log.next_log_id = get_uint32_id(); |
| 268 | host->health_log.next_alarm_id = 0; |
| 269 | |
| 270 | rw_spinlock_init(&host->health_log.spinlock); |
| 271 | sql_health_alarm_log_load(host); |
| 272 | rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH); |
| 273 | |
| 274 | |
| 275 | if (!service_running(SERVICE_HEALTH)) |
| 276 | return; |
| 277 | |
| 278 | health_apply_prototypes_to_host(host); |
| 279 | } |
| 280 | |
| 281 | static inline int check_if_resumed_from_suspension(void) { |
| 282 | static usec_t last_realtime = 0, last_monotonic = 0; |
| 283 | usec_t realtime = now_realtime_usec(), monotonic = now_monotonic_usec(); |
| 284 | int ret = 0; |
| 285 | |
| 286 | // detect if monotonic and realtime have twice the difference |
| 287 | // in which case we assume the system was just waken from hibernation |
| 288 | |
| 289 | if(last_realtime && last_monotonic && realtime - last_realtime > 2 * (monotonic - last_monotonic)) |
| 290 | ret = 1; |
| 291 | |
| 292 | last_realtime = realtime; |
| 293 | last_monotonic = monotonic; |
| 294 | |
| 295 | return ret; |
| 296 | } |
| 297 | |
| 298 | static void do_eval_expression( |
| 299 | RRDCALC *rc, |
| 300 | EVAL_EXPRESSION *expression, |
| 301 | const char *expression_type __maybe_unused, |
| 302 | size_t job_type, |
| 303 | RRDCALC_FLAGS error_type, |
| 304 | RRDCALC_STATUS *calc_status, |
| 305 | NETDATA_DOUBLE *result) |
| 306 | { |
| 307 | if (!expression || (!calc_status && !result)) |
| 308 | return; |
| 309 | |
| 310 | worker_is_busy(job_type); |
| 311 | |
| 312 | if (unlikely(!expression_evaluate(expression))) { |
| 313 | // calculation failed |
| 314 | rc->run_flags |= error_type; |
| 315 | if (result) |
| 316 | *result = NAN; |
| 317 | |
| 318 | netdata_log_debug(D_HEALTH, |
| 319 | "Health on host '%s', alarm '%s.%s': %s expression failed with error: %s", |
| 320 | rrdhost_hostname(rc->rrdset->rrdhost), rrdcalc_chart_name(rc), rrdcalc_name(rc), expression_type, |
| 321 | expression_error_msg(expression) |
| 322 | ); |
| 323 | return; |
| 324 | } |
| 325 | rc->run_flags &= ~error_type; |
| 326 | netdata_log_debug(D_HEALTH, |
| 327 | "Health on host '%s', alarm '%s.%s': %s expression gave value " |
| 328 | NETDATA_DOUBLE_FORMAT ": %s (source: %s)", |
| 329 | rrdhost_hostname(rc->rrdset->rrdhost), |
| 330 | rrdcalc_chart_name(rc), |
| 331 | rrdcalc_name(rc), |
| 332 | expression_type, |
| 333 | expression_result(expression), |
| 334 | expression_error_msg(expression), |
| 335 | rrdcalc_source(rc)); |
| 336 | if (calc_status) |
| 337 | *calc_status = rrdcalc_value2status(expression_result(expression)); |
| 338 | else |
| 339 | *result = expression_result(expression); |
| 340 | } |
| 341 | |
| 342 | // returns the number of runnable alerts |
| 343 | // |
| 344 | // The caller owns `owa` and provides it so a single arena can be reused |
| 345 | // across all hosts of one iteration (and, in the multi-threaded variant, |
| 346 | // across all hosts a single worker processes). The arena is reset between |
| 347 | // alerts inside this function, so peak memory stays bounded by one alert's |
| 348 | // scratch no matter how many hosts flow through it. |
| 349 | static void health_event_loop_for_host(RRDHOST *host, bool apply_hibernation_delay, time_t now, time_t *next_run, ONEWAYALLOC *owa) { |
| 350 | size_t runnable = 0; |
| 351 | struct health_alert_status_counts status_counts = { 0 }; |
| 352 | bool snapshot_complete = true; |
| 353 | |
| 354 | if(unlikely(!rrdhost_should_run_health(host))) |
| 355 | return; |
| 356 | |
| 357 | rrdhost_set_health_evloop_iteration(host); |
| 358 | |
| 359 | //#define rrdhost_pending_alert_transitions(host) (__atomic_load_n(&((host)->aclk_config.alert_transition.pending), __ATOMIC_RELAXED)) |
| 360 | |
| 361 | if (unlikely(__atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED))) { |
| 362 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 363 | "Host \"%s\" has pending alert transitions to save, postponing health checks", |
| 364 | rrdhost_hostname(host)); |
| 365 | return; |
| 366 | } |
| 367 | |
| 368 | if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) |
| 369 | health_initialize_rrdhost(host); |
| 370 | |
| 371 | health_execute_delayed_initializations(host); |
| 372 | |
| 373 | if (unlikely(apply_hibernation_delay)) { |
| 374 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 375 | "[%s]: Postponing health checks for %"PRId32" seconds.", |
| 376 | rrdhost_hostname(host), |
| 377 | health_globals.config.postpone_alarms_during_hibernation_for_seconds); |
| 378 | |
| 379 | host->health.delay_up_to = |
| 380 | now + health_globals.config.postpone_alarms_during_hibernation_for_seconds; |
| 381 | } |
| 382 | |
| 383 | if (unlikely(host->health.delay_up_to)) { |
| 384 | if (unlikely(now < host->health.delay_up_to)) |
| 385 | return; |
| 386 | |
| 387 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 388 | "[%s]: Resuming health checks after delay.", |
| 389 | rrdhost_hostname(host)); |
| 390 | |
| 391 | host->health.delay_up_to = 0; |
| 392 | } |
| 393 | |
| 394 | worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK); |
| 395 | { |
| 396 | struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_ACQUIRE); |
| 397 | if (aclk_host_config && aclk_host_config->send_snapshot == 2) |
| 398 | return; |
| 399 | } |
| 400 | |
| 401 | // Reuse the caller-provided arena across every alert's DB lookup. Each |
| 402 | // alert's scratch (RRDR + query state) is released via onewayalloc_reset |
| 403 | // at the top of the next iteration — trims the page list back to a |
| 404 | // single head page. Net effect: one mmap/munmap for the whole health |
| 405 | // iteration, regardless of host count or alert count. |
| 406 | |
| 407 | // the first loop is to lookup values from the db |
| 408 | RRDCALC *rc; |
| 409 | foreach_rrdcalc_in_rrdhost_read(host, rc) { |
| 410 | // Reclaim the previous alert's query scratch before starting the |
| 411 | // next one. The arena is reused across every alert of every host in |
| 412 | // this iteration, so this reset trims trailing pages left by the |
| 413 | // previous alert (same host or prior host). No-op only on the very |
| 414 | // first alert after the arena was created. |
| 415 | onewayalloc_reset(owa); |
| 416 | |
| 417 | if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host))) { |
| 418 | snapshot_complete = false; |
| 419 | break; |
| 420 | } |
| 421 | |
| 422 | if(likely(rc->rrdset)) |
| 423 | health_alert_status_counts_add(&status_counts, rc->status); |
| 424 | |
| 425 | rrdcalc_update_info_using_rrdset_labels(rc); |
| 426 | |
| 427 | if (health_silencers_update_disabled_silenced(host, rc)) |
| 428 | continue; |
| 429 | |
| 430 | // create an alert removed event if the chart is obsolete and |
| 431 | // has stopped being collected for 60 seconds |
| 432 | if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED && |
| 433 | rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) && |
| 434 | now > (rc->rrdset->last_collected_time.tv_sec + 60))) { |
| 435 | |
| 436 | if (!rrdcalc_isrepeating(rc)) { |
| 437 | worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY); |
| 438 | time_t now_tmp = now_realtime_sec(); |
| 439 | |
| 440 | ALARM_ENTRY *ae = |
| 441 | health_create_alarm_entry( |
| 442 | host, |
| 443 | rc, |
| 444 | now_tmp, |
| 445 | now_tmp - rc->last_status_change, |
| 446 | rc->value, |
| 447 | NAN, |
| 448 | rc->status, |
| 449 | RRDCALC_STATUS_REMOVED, |
| 450 | 0, |
| 451 | rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); |
| 452 | |
| 453 | if (ae) { |
| 454 | health_log_alert(host, ae); |
| 455 | health_alarm_log_add_entry(host, ae, false); |
| 456 | health_alert_status_counts_sub(&status_counts, rc->status); |
| 457 | rc->old_status = rc->status; |
| 458 | rc->status = RRDCALC_STATUS_REMOVED; |
| 459 | rc->last_status_change = now_tmp; |
| 460 | rc->last_status_change_value = rc->value; |
| 461 | rc->last_updated = now_tmp; |
| 462 | rc->value = NAN; |
| 463 | } |
| 464 | } |
| 465 | } |
| 466 | |
| 467 | if (unlikely(!rrdcalc_isrunnable(rc, now, next_run))) { |
| 468 | if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE)) |
| 469 | rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE; |
| 470 | continue; |
| 471 | } |
| 472 | |
| 473 | runnable++; |
| 474 | rc->old_value = rc->value; |
| 475 | rc->run_flags |= RRDCALC_FLAG_RUNNABLE; |
| 476 | |
| 477 | // ------------------------------------------------------------ |
| 478 | // if there is database lookup, do it |
| 479 | |
| 480 | if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) { |
| 481 | worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY); |
| 482 | |
| 483 | /* time_t old_db_timestamp = rc->db_before; */ |
| 484 | int value_is_null = 0; |
| 485 | |
| 486 | char group_options_buf[100]; |
| 487 | const char *group_options = group_options_buf; |
| 488 | switch(rc->config.time_group) { |
| 489 | default: |
| 490 | group_options = NULL; |
| 491 | break; |
| 492 | |
| 493 | case RRDR_GROUPING_PERCENTILE: |
| 494 | case RRDR_GROUPING_TRIMMED_MEAN: |
| 495 | case RRDR_GROUPING_TRIMMED_MEDIAN: |
| 496 | snprintfz(group_options_buf, sizeof(group_options_buf), |
| 497 | NETDATA_DOUBLE_FORMAT_AUTO, |
| 498 | rc->config.time_group_value); |
| 499 | break; |
| 500 | |
| 501 | case RRDR_GROUPING_COUNTIF: |
| 502 | snprintfz(group_options_buf, sizeof(group_options_buf), |
| 503 | "%s" NETDATA_DOUBLE_FORMAT_AUTO, |
| 504 | alerts_group_conditions_id2txt(rc->config.time_group_condition), |
| 505 | rc->config.time_group_value); |
| 506 | break; |
| 507 | } |
| 508 | |
| 509 | int ret = rrdset2value_api_v1_with_owa(owa, |
| 510 | rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1, |
| 511 | rc->config.after, rc->config.before, rc->config.time_group, group_options, |
| 512 | 0, rc->config.options | RRDR_OPTION_SELECTED_TIER, |
| 513 | &rc->db_after,&rc->db_before, |
| 514 | NULL, NULL, NULL, |
| 515 | &value_is_null, NULL, 0, 0, |
| 516 | QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_SYNCHRONOUS); |
| 517 | |
| 518 | if (unlikely(ret != 200)) { |
| 519 | // database lookup failed |
| 520 | rc->value = NAN; |
| 521 | rc->run_flags |= RRDCALC_FLAG_DB_ERROR; |
| 522 | |
| 523 | netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d", |
| 524 | rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret |
| 525 | ); |
| 526 | } else |
| 527 | rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR; |
| 528 | |
| 529 | if (unlikely(value_is_null)) { |
| 530 | // collected value is null |
| 531 | rc->value = NAN; |
| 532 | rc->run_flags |= RRDCALC_FLAG_DB_NAN; |
| 533 | |
| 534 | netdata_log_debug(D_HEALTH, |
| 535 | "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)", |
| 536 | rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc) |
| 537 | ); |
| 538 | } else |
| 539 | rc->run_flags &= ~RRDCALC_FLAG_DB_NAN; |
| 540 | |
| 541 | netdata_log_debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, |
| 542 | rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value |
| 543 | ); |
| 544 | } |
| 545 | |
| 546 | // ------------------------------------------------------------ |
| 547 | // if there is calculation expression, run it |
| 548 | |
| 549 | do_eval_expression(rc, rc->config.calculation, "calculation", WORKER_HEALTH_JOB_CALC_EVAL, RRDCALC_FLAG_CALC_ERROR, NULL, &rc->value); |
| 550 | } |
| 551 | foreach_rrdcalc_in_rrdhost_done(rc); |
| 552 | |
| 553 | struct health_raised_summary *hrm = alerts_raised_summary_create(host); |
| 554 | |
| 555 | if (unlikely(runnable && service_running(SERVICE_HEALTH))) { |
| 556 | foreach_rrdcalc_in_rrdhost_read(host, rc) { |
| 557 | if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host))) { |
| 558 | snapshot_complete = false; |
| 559 | break; |
| 560 | } |
| 561 | |
| 562 | if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE))) |
| 563 | continue; |
| 564 | |
| 565 | if (rc->run_flags & RRDCALC_FLAG_DISABLED) { |
| 566 | continue; |
| 567 | } |
| 568 | RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED; |
| 569 | RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED; |
| 570 | |
| 571 | do_eval_expression(rc, rc->config.warning, "warning", WORKER_HEALTH_JOB_WARNING_EVAL, RRDCALC_FLAG_WARN_ERROR, &warning_status, NULL); |
| 572 | do_eval_expression(rc, rc->config.critical, "critical", WORKER_HEALTH_JOB_CRITICAL_EVAL, RRDCALC_FLAG_CRIT_ERROR, &critical_status, NULL); |
| 573 | |
| 574 | // -------------------------------------------------------- |
| 575 | // decide the final alarm status |
| 576 | |
| 577 | RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED; |
| 578 | |
| 579 | switch (warning_status) { |
| 580 | case RRDCALC_STATUS_CLEAR: |
| 581 | status = RRDCALC_STATUS_CLEAR; |
| 582 | break; |
| 583 | |
| 584 | case RRDCALC_STATUS_RAISED: |
| 585 | status = RRDCALC_STATUS_WARNING; |
| 586 | break; |
| 587 | |
| 588 | default: |
| 589 | break; |
| 590 | } |
| 591 | |
| 592 | switch (critical_status) { |
| 593 | case RRDCALC_STATUS_CLEAR: |
| 594 | if (status == RRDCALC_STATUS_UNDEFINED) |
| 595 | status = RRDCALC_STATUS_CLEAR; |
| 596 | break; |
| 597 | |
| 598 | case RRDCALC_STATUS_RAISED: |
| 599 | status = RRDCALC_STATUS_CRITICAL; |
| 600 | break; |
| 601 | |
| 602 | default: |
| 603 | break; |
| 604 | } |
| 605 | |
| 606 | // -------------------------------------------------------- |
| 607 | // check if the new status and the old differ |
| 608 | |
| 609 | if (status != rc->status) { |
| 610 | |
| 611 | worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY); |
| 612 | int delay; |
| 613 | |
| 614 | // apply trigger hysteresis |
| 615 | |
| 616 | if (now > rc->delay_up_to_timestamp) { |
| 617 | rc->delay_up_current = rc->config.delay_up_duration; |
| 618 | rc->delay_down_current = rc->config.delay_down_duration; |
| 619 | rc->delay_last = 0; |
| 620 | rc->delay_up_to_timestamp = 0; |
| 621 | } else { |
| 622 | rc->delay_up_current = (int)((float)rc->delay_up_current * rc->config.delay_multiplier); |
| 623 | if (rc->delay_up_current > rc->config.delay_max_duration) |
| 624 | rc->delay_up_current = rc->config.delay_max_duration; |
| 625 | |
| 626 | rc->delay_down_current = (int)((float)rc->delay_down_current * rc->config.delay_multiplier); |
| 627 | if (rc->delay_down_current > rc->config.delay_max_duration) |
| 628 | rc->delay_down_current = rc->config.delay_max_duration; |
| 629 | } |
| 630 | |
| 631 | if (status > rc->status) |
| 632 | delay = rc->delay_up_current; |
| 633 | else |
| 634 | delay = rc->delay_down_current; |
| 635 | |
| 636 | // COMMENTED: because we do need to send raising alarms |
| 637 | // if (now + delay < rc->delay_up_to_timestamp) |
| 638 | // delay = (int)(rc->delay_up_to_timestamp - now); |
| 639 | |
| 640 | rc->delay_last = delay; |
| 641 | rc->delay_up_to_timestamp = now + delay; |
| 642 | |
| 643 | ALARM_ENTRY *ae = |
| 644 | health_create_alarm_entry( |
| 645 | host, |
| 646 | rc, |
| 647 | now, |
| 648 | now - rc->last_status_change, |
| 649 | rc->old_value, |
| 650 | rc->value, |
| 651 | rc->status, |
| 652 | status, |
| 653 | rc->delay_last, |
| 654 | ( |
| 655 | ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | |
| 656 | ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | |
| 657 | (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0) |
| 658 | ) |
| 659 | ); |
| 660 | |
| 661 | health_log_alert(host, ae); |
| 662 | health_alarm_log_add_entry(host, ae, false); |
| 663 | |
| 664 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 665 | "[%s]: Alert event for [%s.%s], value [%s], status [%s].", |
| 666 | rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), ae_new_value_string(ae), |
| 667 | rrdcalc_status2string(ae->new_status)); |
| 668 | |
| 669 | health_alert_status_counts_sub(&status_counts, rc->status); |
| 670 | health_alert_status_counts_add(&status_counts, status); |
| 671 | |
| 672 | rc->last_status_change_value = rc->value; |
| 673 | rc->last_status_change = now; |
| 674 | rc->old_status = rc->status; |
| 675 | rc->status = status; |
| 676 | |
| 677 | if(unlikely(rrdcalc_isrepeating(rc))) { |
| 678 | rc->last_repeat = now; |
| 679 | if (rc->status == RRDCALC_STATUS_CLEAR) |
| 680 | rc->run_flags |= RRDCALC_FLAG_RUN_ONCE; |
| 681 | } |
| 682 | } |
| 683 | |
| 684 | rc->last_updated = now; |
| 685 | rc->next_update = now + rc->config.update_every; |
| 686 | |
| 687 | if (*next_run > rc->next_update) |
| 688 | *next_run = rc->next_update; |
| 689 | } |
| 690 | foreach_rrdcalc_in_rrdhost_done(rc); |
| 691 | |
| 692 | alerts_raised_summary_populate(hrm); |
| 693 | |
| 694 | // process repeating alarms |
| 695 | foreach_rrdcalc_in_rrdhost_read(host, rc) { |
| 696 | if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host))) |
| 697 | break; |
| 698 | |
| 699 | int repeat_every = 0; |
| 700 | if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) { |
| 701 | if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) { |
| 702 | rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE; |
| 703 | repeat_every = (int)rc->config.warn_repeat_every; |
| 704 | } |
| 705 | else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) { |
| 706 | rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE; |
| 707 | repeat_every = (int)rc->config.crit_repeat_every; |
| 708 | } |
| 709 | else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) { |
| 710 | if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && |
| 711 | (rc->old_status == RRDCALC_STATUS_CRITICAL || rc->old_status == RRDCALC_STATUS_WARNING)) |
| 712 | repeat_every = 1; |
| 713 | } |
| 714 | } |
| 715 | else |
| 716 | continue; |
| 717 | |
| 718 | if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) { |
| 719 | worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY); |
| 720 | rc->last_repeat = now; |
| 721 | if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++; |
| 722 | ALARM_ENTRY *ae = |
| 723 | health_create_alarm_entry( |
| 724 | host, |
| 725 | rc, |
| 726 | now, |
| 727 | now - rc->last_status_change, |
| 728 | rc->old_value, |
| 729 | rc->value, |
| 730 | rc->old_status, |
| 731 | rc->status, |
| 732 | rc->delay_last, |
| 733 | ( |
| 734 | ((rc->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | |
| 735 | ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) | |
| 736 | (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0) |
| 737 | ) |
| 738 | ); |
| 739 | |
| 740 | health_log_alert(host, ae); |
| 741 | ae->last_repeat = rc->last_repeat; |
| 742 | if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) { |
| 743 | ae->flags |= HEALTH_ENTRY_RUN_ONCE; |
| 744 | } |
| 745 | rc->run_flags |= RRDCALC_FLAG_RUN_ONCE; |
| 746 | health_send_notification(host, ae, hrm); |
| 747 | netdata_log_debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id); |
| 748 | health_alarm_wait_for_execution(ae); |
| 749 | health_alarm_log_free_one_nochecks_nounlink(ae); |
| 750 | } |
| 751 | } |
| 752 | foreach_rrdcalc_in_rrdhost_done(rc); |
| 753 | } |
| 754 | |
| 755 | if(likely(snapshot_complete)) { |
| 756 | uint64_t snapshot_generation = health_alert_status_snapshot_begin_update(host); |
| 757 | health_alert_status_snapshot_finish_update(host, &status_counts, snapshot_generation); |
| 758 | } |
| 759 | |
| 760 | if(unlikely(!service_running(SERVICE_HEALTH) || !rrdhost_should_run_health(host))) { |
| 761 | alerts_raised_summary_free(hrm); |
| 762 | return; |
| 763 | } |
| 764 | |
| 765 | // execute notifications |
| 766 | // and cleanup |
| 767 | |
| 768 | worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS); |
| 769 | health_alarm_log_process_to_send_notifications(host, hrm); |
| 770 | alerts_raised_summary_free(hrm); |
| 771 | |
| 772 | int32_t pending = __atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED); |
| 773 | if (pending) |
| 774 | commit_alert_transitions(host); |
| 775 | |
| 776 | if (!__atomic_load_n(&host->health.pending_transitions, __ATOMIC_RELAXED)) { |
| 777 | struct aclk_sync_cfg_t *aclk_host_config = __atomic_load_n(&host->aclk_host_config, __ATOMIC_ACQUIRE); |
| 778 | if (aclk_host_config && aclk_host_config->send_snapshot == 1) { |
| 779 | aclk_host_config->send_snapshot = 2; |
| 780 | rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS); |
| 781 | } else { |
| 782 | worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_QUEUE); |
| 783 | if (process_alert_pending_queue(host)) |
| 784 | rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS); |
| 785 | } |
| 786 | } |
| 787 | worker_is_idle(); |
| 788 | } |
| 789 | |
| 790 | __thread bool is_health_thread = false; |
| 791 | static void health_event_loop(void) { |
| 792 | |
| 793 | is_health_thread = true; |
| 794 | while(service_running(SERVICE_HEALTH)) { |
| 795 | if(!stream_control_health_should_be_running()) { |
| 796 | worker_is_idle(); |
| 797 | stream_control_throttle(); |
| 798 | continue; |
| 799 | } |
| 800 | |
| 801 | time_t now = now_realtime_sec(); |
| 802 | bool apply_hibernation_delay = false; |
| 803 | time_t next_run = now + health_globals.config.run_at_least_every_seconds; |
| 804 | |
| 805 | if (unlikely(check_if_resumed_from_suspension())) { |
| 806 | apply_hibernation_delay = true; |
| 807 | |
| 808 | nd_log(NDLS_DAEMON, NDLP_NOTICE, |
| 809 | "Postponing alarm checks for %"PRId32" seconds, " |
| 810 | "because it seems that the system was just resumed from suspension.", |
| 811 | (int32_t)health_globals.config.postpone_alarms_during_hibernation_for_seconds); |
| 812 | schedule_node_state_update(localhost, 10); |
| 813 | } |
| 814 | |
| 815 | if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) { |
| 816 | static int logged=0; |
| 817 | if (!logged) { |
| 818 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 819 | "Skipping health checks, because all alarms are disabled via API command."); |
| 820 | logged = 1; |
| 821 | } |
| 822 | } |
| 823 | |
| 824 | worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK); |
| 825 | uint64_t loop = __atomic_add_fetch(&health_evloop_iteration, 1, __ATOMIC_RELAXED); |
| 826 | |
| 827 | // Single onewayalloc arena reused across every host in this iteration — |
| 828 | // one mmap/munmap pair for the whole cycle instead of per host. |
| 829 | ONEWAYALLOC *iter_owa = onewayalloc_create(0); |
| 830 | |
| 831 | RRDHOST *host; |
| 832 | dfe_start_reentrant(rrdhost_root_index, host) { |
| 833 | if(unlikely(!service_running(SERVICE_HEALTH))) |
| 834 | break; |
| 835 | |
| 836 | health_event_loop_for_host(host, apply_hibernation_delay, now, &next_run, iter_owa); |
| 837 | } |
| 838 | dfe_done(host); |
| 839 | |
| 840 | onewayalloc_destroy(iter_owa); |
| 841 | |
| 842 | if(unlikely(!service_running(SERVICE_HEALTH))) |
| 843 | break; |
| 844 | |
| 845 | // wait for all notifications to finish before allowing health to be cleaned up |
| 846 | worker_is_busy(WORKER_HEALTH_JOB_WAIT_EXEC); |
| 847 | wait_for_all_notifications_to_finish_before_allowing_health_to_be_cleaned_up(); |
| 848 | worker_is_idle(); |
| 849 | |
| 850 | health_sleep(next_run, loop); |
| 851 | } // forever |
| 852 | } |
| 853 | |
| 854 | |
| 855 | static void health_main_cleanup(void *pptr) { |
| 856 | struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); |
| 857 | if(!static_thread) return; |
| 858 | |
| 859 | worker_unregister(); |
| 860 | static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; |
| 861 | finalize_self_prepared_sql_statements(); |
| 862 | static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; |
| 863 | nd_log(NDLS_DAEMON, NDLP_DEBUG, "Health thread ended."); |
| 864 | } |
| 865 | |
| 866 | void *health_main(void *ptr) { |
| 867 | worker_register("HEALTH"); |
| 868 | worker_register_job_name(WORKER_HEALTH_JOB_RRD_LOCK, "rrd lock"); |
| 869 | worker_register_job_name(WORKER_HEALTH_JOB_HOST_LOCK, "host lock"); |
| 870 | worker_register_job_name(WORKER_HEALTH_JOB_DB_QUERY, "db lookup"); |
| 871 | worker_register_job_name(WORKER_HEALTH_JOB_CALC_EVAL, "calc eval"); |
| 872 | worker_register_job_name(WORKER_HEALTH_JOB_WARNING_EVAL, "warning eval"); |
| 873 | worker_register_job_name(WORKER_HEALTH_JOB_CRITICAL_EVAL, "critical eval"); |
| 874 | worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY, "alert log entry"); |
| 875 | worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS, "alert log process"); |
| 876 | worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_QUEUE, "alert log queue"); |
| 877 | worker_register_job_name(WORKER_HEALTH_JOB_WAIT_EXEC, "alert wait exec"); |
| 878 | worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init"); |
| 879 | worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init"); |
| 880 | |
| 881 | CLEANUP_FUNCTION_REGISTER(health_main_cleanup) cleanup_ptr = ptr; |
| 882 | health_event_loop(); |
| 883 | return NULL; |
| 884 | } |