| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "health_internals.h" |
| 4 | #include "health-alert-entry.h" |
| 5 | |
| 6 | // the queue of executed alarm notifications that haven't been waited for yet |
| 7 | static ALARM_ENTRY *alarm_notifications_in_progress = NULL; |
| 8 | |
| 9 | // how often the notification wait loop wakes up to re-check shutdown and the deadline |
| 10 | #define HEALTH_NOTIFICATION_WAIT_SLICE_MS 1000 |
| 11 | |
| 12 | struct health_raised_summary { |
| 13 | RRDHOST *host; |
| 14 | DICTIONARY *rrdcalc_dict; |
| 15 | |
| 16 | struct { |
| 17 | size_t size; |
| 18 | size_t used; |
| 19 | const DICTIONARY_ITEM **array; |
| 20 | } active_alerts; |
| 21 | }; |
| 22 | |
| 23 | void health_alarm_wait_for_execution(ALARM_ENTRY *ae) { |
| 24 | // this has to ALWAYS remove the given alarm entry from the queue |
| 25 | |
| 26 | int code = 0; |
| 27 | |
| 28 | bool in_process = ae->flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS; |
| 29 | if (!in_process) { |
| 30 | nd_log(NDLS_DAEMON, NDLP_ERR, "attempted to wait for the execution of alert that has not an execution in progress"); |
| 31 | code = 128; |
| 32 | goto cleanup; |
| 33 | } |
| 34 | |
| 35 | if(!ae->popen_instance) { |
| 36 | nd_log(NDLS_DAEMON, NDLP_ERR, "attempted to wait for the execution of alert that has not spawn a notification"); |
| 37 | code = 128; |
| 38 | goto cleanup; |
| 39 | } |
| 40 | |
| 41 | // bound the wait so a hung notification process (seen on Windows, where msys children can |
| 42 | // wedge during startup) cannot block the single health thread - and with it all health |
| 43 | // evaluation. Each slice is always bounded; the overall wait is bounded only when a non-zero |
| 44 | // timeout is configured. timeout == 0 means "wait forever" - the loop then breaks only on |
| 45 | // child exit or shutdown. The deadline is monotonic, so a wall-clock jump cannot extend it. |
| 46 | int32_t timeout = health_globals.config.notification_execution_timeout_seconds; |
| 47 | usec_t deadline_ut = now_monotonic_usec() + (usec_t)timeout * USEC_PER_SEC; |
| 48 | |
| 49 | while(true) { |
| 50 | SPAWN_TIMEDWAIT_RESULT r = spawn_popen_timedwait(ae->popen_instance, HEALTH_NOTIFICATION_WAIT_SLICE_MS, &code); |
| 51 | if(r == SPAWN_TIMEDWAIT_EXITED) |
| 52 | break; |
| 53 | |
| 54 | // RUNNING: keep waiting unless we should stop. ERROR: the wait broke and must never be |
| 55 | // looped on (it would spin forever at timeout == 0), so always fall through to the kill. |
| 56 | // re-check shutdown every slice, so a slow notification cannot block agent exit. |
| 57 | bool deadline_reached = (timeout > 0 && now_monotonic_usec() >= deadline_ut); |
| 58 | if(r == SPAWN_TIMEDWAIT_ERROR || unlikely(!service_running(SERVICE_HEALTH)) || deadline_reached) { |
| 59 | nd_log(NDLS_DAEMON, NDLP_ERR, |
| 60 | "HEALTH: alert notification '%s' (pid %d) %s - killing it", |
| 61 | ae_name(ae), (int)spawn_popen_pid(ae->popen_instance), |
| 62 | (r == SPAWN_TIMEDWAIT_ERROR) ? "could not be waited for (status channel error)" |
| 63 | : "is still running past its execution timeout"); |
| 64 | |
| 65 | spawn_popen_kill(ae->popen_instance, 0); |
| 66 | code = 128; |
| 67 | break; |
| 68 | } |
| 69 | } |
| 70 | ae->popen_instance = NULL; |
| 71 | netdata_log_debug(D_HEALTH, "done executing command - returned with code %d", code); |
| 72 | |
| 73 | cleanup: |
| 74 | ae->exec_code = code; |
| 75 | ae->flags &= ~HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS; |
| 76 | |
| 77 | if(ae->exec_code != 0) |
| 78 | ae->flags |= HEALTH_ENTRY_FLAG_EXEC_FAILED; |
| 79 | |
| 80 | if (in_process) |
| 81 | unlink_alarm_notify_in_progress(ae); |
| 82 | } |
| 83 | |
| 84 | void wait_for_all_notifications_to_finish_before_allowing_health_to_be_cleaned_up(void) { |
| 85 | ALARM_ENTRY *ae; |
| 86 | while (NULL != (ae = alarm_notifications_in_progress)) { |
| 87 | if(unlikely(!service_running(SERVICE_HEALTH))) |
| 88 | break; |
| 89 | |
| 90 | health_alarm_wait_for_execution(ae); |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | void unlink_alarm_notify_in_progress(ALARM_ENTRY *ae) |
| 95 | { |
| 96 | fatal_assert(ae->prev_in_progress || ae->next_in_progress); |
| 97 | DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(alarm_notifications_in_progress, ae, prev_in_progress, next_in_progress); |
| 98 | } |
| 99 | |
| 100 | static inline void enqueue_alarm_notify_in_progress(ALARM_ENTRY *ae) |
| 101 | { |
| 102 | fatal_assert(!ae->prev_in_progress && !ae->next_in_progress); |
| 103 | DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(alarm_notifications_in_progress, ae, prev_in_progress, next_in_progress); |
| 104 | } |
| 105 | |
| 106 | static bool prepare_command(BUFFER *wb, |
| 107 | const char *exec, |
| 108 | const char *recipient, |
| 109 | const char *registry_hostname, |
| 110 | uint32_t unique_id, |
| 111 | uint32_t alarm_id, |
| 112 | uint32_t alarm_event_id, |
| 113 | uint32_t when, |
| 114 | const char *alert_name, |
| 115 | const char *alert_chart_name, |
| 116 | const char *new_status, |
| 117 | const char *old_status, |
| 118 | NETDATA_DOUBLE new_value, |
| 119 | NETDATA_DOUBLE old_value, |
| 120 | const char *alert_source, |
| 121 | uint32_t duration, |
| 122 | uint32_t non_clear_duration, |
| 123 | const char *alert_units, |
| 124 | const char *alert_info, |
| 125 | const char *new_value_string, |
| 126 | const char *old_value_string, |
| 127 | const char *source, |
| 128 | const char *error_msg, |
| 129 | int n_warn, |
| 130 | int n_crit, |
| 131 | const char *warn_alarms, |
| 132 | const char *crit_alarms, |
| 133 | const char *classification, |
| 134 | const char *edit_command, |
| 135 | const char *machine_guid, |
| 136 | nd_uuid_t *transition_id, |
| 137 | const char *summary, |
| 138 | const char *context, |
| 139 | const char *component, |
| 140 | const char *type |
| 141 | ) { |
| 142 | char buf[8192]; |
| 143 | size_t n = sizeof(buf) - 1; |
| 144 | |
| 145 | buffer_strcat(wb, "exec"); |
| 146 | |
| 147 | if (!sanitize_command_argument_string(buf, exec, n)) |
| 148 | return false; |
| 149 | buffer_sprintf(wb, " '%s'", buf); |
| 150 | |
| 151 | if (!sanitize_command_argument_string(buf, recipient, n)) |
| 152 | return false; |
| 153 | buffer_sprintf(wb, " '%s'", buf); |
| 154 | |
| 155 | if (!sanitize_command_argument_string(buf, registry_hostname, n)) |
| 156 | return false; |
| 157 | buffer_sprintf(wb, " '%s'", buf); |
| 158 | |
| 159 | buffer_sprintf(wb, " '%u'", unique_id); |
| 160 | |
| 161 | buffer_sprintf(wb, " '%u'", alarm_id); |
| 162 | |
| 163 | buffer_sprintf(wb, " '%u'", alarm_event_id); |
| 164 | |
| 165 | buffer_sprintf(wb, " '%u'", when); |
| 166 | |
| 167 | if (!sanitize_command_argument_string(buf, alert_name, n)) |
| 168 | return false; |
| 169 | buffer_sprintf(wb, " '%s'", buf); |
| 170 | |
| 171 | if (!sanitize_command_argument_string(buf, alert_chart_name, n)) |
| 172 | return false; |
| 173 | buffer_sprintf(wb, " '%s'", buf); |
| 174 | |
| 175 | if (!sanitize_command_argument_string(buf, new_status, n)) |
| 176 | return false; |
| 177 | buffer_sprintf(wb, " '%s'", buf); |
| 178 | |
| 179 | if (!sanitize_command_argument_string(buf, old_status, n)) |
| 180 | return false; |
| 181 | buffer_sprintf(wb, " '%s'", buf); |
| 182 | |
| 183 | buffer_sprintf(wb, " '" NETDATA_DOUBLE_FORMAT_ZERO "'", new_value); |
| 184 | |
| 185 | buffer_sprintf(wb, " '" NETDATA_DOUBLE_FORMAT_ZERO "'", old_value); |
| 186 | |
| 187 | if (!sanitize_command_argument_string(buf, alert_source, n)) |
| 188 | return false; |
| 189 | buffer_sprintf(wb, " '%s'", buf); |
| 190 | |
| 191 | buffer_sprintf(wb, " '%u'", duration); |
| 192 | |
| 193 | buffer_sprintf(wb, " '%u'", non_clear_duration); |
| 194 | |
| 195 | if (!sanitize_command_argument_string(buf, alert_units, n)) |
| 196 | return false; |
| 197 | buffer_sprintf(wb, " '%s'", buf); |
| 198 | |
| 199 | if (!sanitize_command_argument_string(buf, alert_info, n)) |
| 200 | return false; |
| 201 | buffer_sprintf(wb, " '%s'", buf); |
| 202 | |
| 203 | if (!sanitize_command_argument_string(buf, new_value_string, n)) |
| 204 | return false; |
| 205 | buffer_sprintf(wb, " '%s'", buf); |
| 206 | |
| 207 | if (!sanitize_command_argument_string(buf, old_value_string, n)) |
| 208 | return false; |
| 209 | buffer_sprintf(wb, " '%s'", buf); |
| 210 | |
| 211 | if (!sanitize_command_argument_string(buf, source, n)) |
| 212 | return false; |
| 213 | buffer_sprintf(wb, " '%s'", buf); |
| 214 | |
| 215 | if (!sanitize_command_argument_string(buf, error_msg, n)) |
| 216 | return false; |
| 217 | buffer_sprintf(wb, " '%s'", buf); |
| 218 | |
| 219 | buffer_sprintf(wb, " '%d'", n_warn); |
| 220 | |
| 221 | buffer_sprintf(wb, " '%d'", n_crit); |
| 222 | |
| 223 | if (!sanitize_command_argument_string(buf, warn_alarms, n)) |
| 224 | return false; |
| 225 | buffer_sprintf(wb, " '%s'", buf); |
| 226 | |
| 227 | if (!sanitize_command_argument_string(buf, crit_alarms, n)) |
| 228 | return false; |
| 229 | buffer_sprintf(wb, " '%s'", buf); |
| 230 | |
| 231 | if (!sanitize_command_argument_string(buf, classification, n)) |
| 232 | return false; |
| 233 | buffer_sprintf(wb, " '%s'", buf); |
| 234 | |
| 235 | if (!sanitize_command_argument_string(buf, edit_command, n)) |
| 236 | return false; |
| 237 | buffer_sprintf(wb, " '%s'", buf); |
| 238 | |
| 239 | if (!sanitize_command_argument_string(buf, machine_guid, n)) |
| 240 | return false; |
| 241 | buffer_sprintf(wb, " '%s'", buf); |
| 242 | |
| 243 | char tr_id[UUID_STR_LEN]; |
| 244 | uuid_unparse_lower(*transition_id, tr_id); |
| 245 | if (!sanitize_command_argument_string(buf, tr_id, n)) |
| 246 | return false; |
| 247 | buffer_sprintf(wb, " '%s'", buf); |
| 248 | |
| 249 | if (!sanitize_command_argument_string(buf, summary, n)) |
| 250 | return false; |
| 251 | buffer_sprintf(wb, " '%s'", buf); |
| 252 | |
| 253 | if (!sanitize_command_argument_string(buf, context, n)) |
| 254 | return false; |
| 255 | buffer_sprintf(wb, " '%s'", buf); |
| 256 | |
| 257 | if (!sanitize_command_argument_string(buf, component, n)) |
| 258 | return false; |
| 259 | buffer_sprintf(wb, " '%s'", buf); |
| 260 | |
| 261 | if (!sanitize_command_argument_string(buf, type, n)) |
| 262 | return false; |
| 263 | buffer_sprintf(wb, " '%s'", buf); |
| 264 | |
| 265 | return true; |
| 266 | } |
| 267 | |
| 268 | static inline int compare_raised_alerts(const void *a, const void *b) { |
| 269 | const DICTIONARY_ITEM *item1 = *(const DICTIONARY_ITEM **)a; |
| 270 | const DICTIONARY_ITEM *item2 = *(const DICTIONARY_ITEM **)b; |
| 271 | |
| 272 | RRDCALC *rc1 = dictionary_acquired_item_value(item1); |
| 273 | RRDCALC *rc2 = dictionary_acquired_item_value(item2); |
| 274 | |
| 275 | return (int)(rc2->last_status_change - rc1->last_status_change); |
| 276 | } |
| 277 | |
| 278 | static void health_raised_summary_add_alert(struct health_raised_summary *hrm, const DICTIONARY_ITEM *item) { |
| 279 | if(hrm->active_alerts.used >= hrm->active_alerts.size) { |
| 280 | if(hrm->active_alerts.size == 0) |
| 281 | hrm->active_alerts.size = 2; |
| 282 | |
| 283 | hrm->active_alerts.size *= 2; |
| 284 | hrm->active_alerts.array = reallocz(hrm->active_alerts.array, sizeof(const DICTIONARY_ITEM *) * hrm->active_alerts.size); |
| 285 | } |
| 286 | |
| 287 | hrm->active_alerts.array[hrm->active_alerts.used++] = dictionary_acquired_item_dup(hrm->rrdcalc_dict, item); |
| 288 | } |
| 289 | |
| 290 | void alerts_raised_summary_free(struct health_raised_summary *hrm) { |
| 291 | for(size_t i = 0; i < hrm->active_alerts.used ;i++) |
| 292 | dictionary_acquired_item_release(hrm->rrdcalc_dict, hrm->active_alerts.array[i]); |
| 293 | |
| 294 | freez(hrm->active_alerts.array); |
| 295 | freez(hrm); |
| 296 | } |
| 297 | |
| 298 | struct health_raised_summary *alerts_raised_summary_create(RRDHOST *host) { |
| 299 | struct health_raised_summary *hrm = callocz(1, sizeof(*hrm)); |
| 300 | hrm->rrdcalc_dict = host->rrdcalc_root_index; |
| 301 | hrm->host = host; |
| 302 | return hrm; |
| 303 | } |
| 304 | |
| 305 | void alerts_raised_summary_populate(struct health_raised_summary *hrm) { |
| 306 | RRDCALC *rc; |
| 307 | foreach_rrdcalc_in_rrdhost_read(hrm->host, rc) { |
| 308 | if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; |
| 309 | health_raised_summary_add_alert(hrm, rc_dfe.item); |
| 310 | } |
| 311 | foreach_rrdcalc_in_rrdhost_done(rc); |
| 312 | |
| 313 | if (hrm->active_alerts.used > 1) |
| 314 | qsort(hrm->active_alerts.array, hrm->active_alerts.used, sizeof(const DICTIONARY_ITEM *), compare_raised_alerts); |
| 315 | } |
| 316 | |
| 317 | static size_t |
| 318 | health_raised_summary_entries(struct health_raised_summary *hrm, BUFFER *dst, ALARM_ENTRY *ae, RRDCALC_STATUS status) { |
| 319 | buffer_flush(dst); |
| 320 | |
| 321 | size_t count = 0; |
| 322 | for(size_t i = 0; i < hrm->active_alerts.used ;i++) { |
| 323 | RRDCALC *rc = dictionary_acquired_item_value(hrm->active_alerts.array[i]); |
| 324 | if(rc->status != status) continue; |
| 325 | if(rc->id == ae->alarm_id) continue; |
| 326 | |
| 327 | count++; |
| 328 | if(buffer_strlen(dst)) buffer_putc(dst, ','); |
| 329 | buffer_sprintf(dst, "%s=%" PRId64, string2str(rc->config.name), (int64_t)rc->last_status_change); |
| 330 | } |
| 331 | |
| 332 | return count; |
| 333 | } |
| 334 | |
| 335 | static const char *health_raised_summary_my_expression_source(struct health_raised_summary *hrm, ALARM_ENTRY *ae) { |
| 336 | for(size_t i = 0; i < hrm->active_alerts.used ;i++) { |
| 337 | RRDCALC *rc = dictionary_acquired_item_value(hrm->active_alerts.array[i]); |
| 338 | if(rc->id != ae->alarm_id) continue; |
| 339 | |
| 340 | if(rc->status == RRDCALC_STATUS_CRITICAL) |
| 341 | return expression_source(rc->config.critical); |
| 342 | else |
| 343 | return expression_source(rc->config.warning); |
| 344 | } |
| 345 | |
| 346 | return ""; |
| 347 | } |
| 348 | |
| 349 | static const char *health_raised_summary_my_expression_error(struct health_raised_summary *hrm, ALARM_ENTRY *ae) { |
| 350 | for(size_t i = 0; i < hrm->active_alerts.used ;i++) { |
| 351 | RRDCALC *rc = dictionary_acquired_item_value(hrm->active_alerts.array[i]); |
| 352 | if(rc->id != ae->alarm_id) continue; |
| 353 | |
| 354 | if(rc->status == RRDCALC_STATUS_CRITICAL) |
| 355 | return expression_error_msg(rc->config.critical); |
| 356 | else |
| 357 | return expression_error_msg(rc->config.warning); |
| 358 | } |
| 359 | |
| 360 | return ""; |
| 361 | } |
| 362 | |
| 363 | void health_send_notification(RRDHOST *host, ALARM_ENTRY *ae, struct health_raised_summary *hrm) { |
| 364 | netdata_log_debug(D_HEALTH, "Health alarm '%s.%s' = " NETDATA_DOUBLE_FORMAT_AUTO " - changed status from %s to %s", |
| 365 | ae->chart?ae_chart_id(ae):"NOCHART", ae_name(ae), |
| 366 | ae->new_value, |
| 367 | rrdcalc_status2string(ae->old_status), |
| 368 | rrdcalc_status2string(ae->new_status) |
| 369 | ); |
| 370 | |
| 371 | ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED; |
| 372 | |
| 373 | if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) { |
| 374 | // do not send notifications for internal statuses |
| 375 | netdata_log_debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); |
| 376 | goto done; |
| 377 | } |
| 378 | |
| 379 | if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) { |
| 380 | // do not send notifications for disabled statuses |
| 381 | |
| 382 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 383 | "[%s]: Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", |
| 384 | rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); |
| 385 | |
| 386 | // mark it as run, so that we will send the same alarm if it happens again |
| 387 | goto done; |
| 388 | } |
| 389 | |
| 390 | // find the previous notification for the same alarm |
| 391 | // which we have run the exec script |
| 392 | // exception: alarms with HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION set |
| 393 | RRDCALC_STATUS last_executed_status = -3; |
| 394 | if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) { |
| 395 | int ret = sql_health_get_last_executed_event(host, ae, &last_executed_status); |
| 396 | |
| 397 | if (likely(ret == 1)) { |
| 398 | // we have executed this alarm notification in the past |
| 399 | if(last_executed_status == ae->new_status && !(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING)) { |
| 400 | // don't send the notification for the same status again |
| 401 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 402 | "[%s]: Health not sending again notification for alarm '%s.%s' status %s", |
| 403 | rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), |
| 404 | rrdcalc_status2string(ae->new_status)); |
| 405 | goto done; |
| 406 | } |
| 407 | } |
| 408 | else { |
| 409 | // we have not executed this alarm notification in the past |
| 410 | // so, don't send CLEAR notifications |
| 411 | if(unlikely(ae->new_status == RRDCALC_STATUS_CLEAR)) { |
| 412 | if((!(ae->flags & HEALTH_ENTRY_RUN_ONCE)) || (ae->flags & HEALTH_ENTRY_RUN_ONCE && ae->old_status < RRDCALC_STATUS_RAISED) ) { |
| 413 | netdata_log_debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s" |
| 414 | , ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); |
| 415 | goto done; |
| 416 | } |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | // Check if alarm notifications are silenced |
| 422 | if (ae->flags & HEALTH_ENTRY_FLAG_SILENCED) { |
| 423 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 424 | "[%s]: Health not sending notification for alarm '%s.%s' status %s " |
| 425 | "(command API has disabled notifications)", |
| 426 | rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); |
| 427 | goto done; |
| 428 | } |
| 429 | |
| 430 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 431 | "[%s]: Sending notification for alarm '%s.%s' status %s.", |
| 432 | rrdhost_hostname(host), ae_chart_id(ae), ae_name(ae), rrdcalc_status2string(ae->new_status)); |
| 433 | |
| 434 | const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health.default_exec); |
| 435 | const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health.default_recipient); |
| 436 | |
| 437 | char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN"); |
| 438 | |
| 439 | BUFFER *warn_alarms = buffer_create(1024, &netdata_buffers_statistics.buffers_health); |
| 440 | BUFFER *crit_alarms = buffer_create(1024, &netdata_buffers_statistics.buffers_health); |
| 441 | |
| 442 | size_t n_warn = health_raised_summary_entries(hrm, warn_alarms, ae, RRDCALC_STATUS_WARNING); |
| 443 | size_t n_crit = health_raised_summary_entries(hrm, crit_alarms, ae, RRDCALC_STATUS_CRITICAL); |
| 444 | |
| 445 | BUFFER *wb = buffer_create(8192, &netdata_buffers_statistics.buffers_health); |
| 446 | bool ok = prepare_command(wb, |
| 447 | exec, |
| 448 | recipient, |
| 449 | rrdhost_registry_hostname(host), |
| 450 | ae->unique_id, |
| 451 | ae->alarm_id, |
| 452 | ae->alarm_event_id, |
| 453 | (unsigned long)ae->when, |
| 454 | ae_name(ae), |
| 455 | ae->chart?ae_chart_id(ae):"NOCHART", |
| 456 | rrdcalc_status2string(ae->new_status), |
| 457 | rrdcalc_status2string(ae->old_status), |
| 458 | ae->new_value, |
| 459 | ae->old_value, |
| 460 | ae->source?ae_source(ae):"UNKNOWN", |
| 461 | (uint32_t)ae->duration, |
| 462 | (ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING && ae->new_status >= RRDCALC_STATUS_WARNING) ? (uint32_t)ae->duration : (uint32_t)ae->non_clear_duration, |
| 463 | ae_units(ae), |
| 464 | ae_info(ae), |
| 465 | ae_new_value_string(ae), |
| 466 | ae_old_value_string(ae), |
| 467 | health_raised_summary_my_expression_source(hrm, ae), |
| 468 | health_raised_summary_my_expression_error(hrm, ae), |
| 469 | n_warn, |
| 470 | n_crit, |
| 471 | buffer_tostring(warn_alarms), |
| 472 | buffer_tostring(crit_alarms), |
| 473 | ae->classification?ae_classification(ae):"Unknown", |
| 474 | edit_command, |
| 475 | host->machine_guid, |
| 476 | &ae->transition_id, |
| 477 | host->health.use_summary_for_notifications && ae->summary?ae_summary(ae):ae_name(ae), |
| 478 | string2str(ae->chart_context), |
| 479 | string2str(ae->component), |
| 480 | string2str(ae->type) |
| 481 | ); |
| 482 | |
| 483 | const char *command_to_run = buffer_tostring(wb); |
| 484 | if (ok) { |
| 485 | ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN; |
| 486 | ae->exec_run_timestamp = now_realtime_sec(); /* will be updated by real time after spawning */ |
| 487 | |
| 488 | netdata_log_debug(D_HEALTH, "executing command '%s'", command_to_run); |
| 489 | ae->popen_instance = spawn_popen_run(command_to_run); |
| 490 | if(ae->popen_instance) { |
| 491 | ae->flags |= HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS; |
| 492 | enqueue_alarm_notify_in_progress(ae); |
| 493 | } |
| 494 | else |
| 495 | netdata_log_error("Failed to execute alarm notification"); |
| 496 | |
| 497 | health_alarm_log_save(host, ae, false); |
| 498 | } |
| 499 | else |
| 500 | netdata_log_error("Failed to format command arguments"); |
| 501 | |
| 502 | buffer_free(warn_alarms); |
| 503 | buffer_free(crit_alarms); |
| 504 | buffer_free(wb); |
| 505 | freez(edit_command); |
| 506 | |
| 507 | return; //health_alarm_wait_for_execution |
| 508 | done: |
| 509 | health_alarm_log_save(host, ae, false); |
| 510 | } |
| 511 | |
| 512 | bool health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(RRDCALC *rc, usec_t *global_id, nd_uuid_t *transitions_id) { |
| 513 | if(!rc->rrdset) |
| 514 | return false; |
| 515 | |
| 516 | RRDHOST *host = rc->rrdset->rrdhost; |
| 517 | |
| 518 | rw_spinlock_read_lock(&host->health_log.spinlock); |
| 519 | |
| 520 | ALARM_ENTRY *ae; |
| 521 | for(ae = host->health_log.alarms; ae ; ae = ae->next) { |
| 522 | if(unlikely(ae->alarm_id == rc->id)) |
| 523 | break; |
| 524 | } |
| 525 | |
| 526 | if(ae) { |
| 527 | *global_id = ae->global_id; |
| 528 | uuid_copy(*transitions_id, ae->transition_id); |
| 529 | } |
| 530 | else { |
| 531 | *global_id = 0; |
| 532 | uuid_clear(*transitions_id); |
| 533 | } |
| 534 | |
| 535 | rw_spinlock_read_unlock(&host->health_log.spinlock); |
| 536 | |
| 537 | return ae != NULL; |
| 538 | } |
| 539 | |
| 540 | void health_alarm_log_process_to_send_notifications(RRDHOST *host, struct health_raised_summary *hrm) { |
| 541 | uint32_t first_waiting = (host->health_log.alarms)?host->health_log.alarms->unique_id:0; |
| 542 | time_t now = now_realtime_sec(); |
| 543 | |
| 544 | rw_spinlock_read_lock(&host->health_log.spinlock); |
| 545 | |
| 546 | for(ALARM_ENTRY *ae = host->health_log.alarms; ae && ae->unique_id >= host->health_last_processed_id; ae = ae->next) { |
| 547 | if(unlikely( |
| 548 | !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) && |
| 549 | !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED) |
| 550 | )) { |
| 551 | if(unlikely(ae->unique_id < first_waiting)) |
| 552 | first_waiting = ae->unique_id; |
| 553 | |
| 554 | if(likely(now >= ae->delay_up_to_timestamp)) |
| 555 | health_send_notification(host, ae, hrm); |
| 556 | } |
| 557 | } |
| 558 | |
| 559 | rw_spinlock_read_unlock(&host->health_log.spinlock); |
| 560 | |
| 561 | // remember this for the next iteration |
| 562 | host->health_last_processed_id = first_waiting; |
| 563 | |
| 564 | //delete those that are updated, no in progress execution, and is not repeating |
| 565 | rw_spinlock_write_lock(&host->health_log.spinlock); |
| 566 | |
| 567 | ALARM_ENTRY *ae = host->health_log.alarms; |
| 568 | while(ae) { |
| 569 | ALARM_ENTRY *next = ae->next; // set it here, for the next iteration |
| 570 | |
| 571 | if((likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING)) && |
| 572 | (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) && |
| 573 | (ae->flags & HEALTH_ENTRY_FLAG_SAVED) && |
| 574 | !(ae->flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS)) |
| 575 | || |
| 576 | ((ae->new_status == RRDCALC_STATUS_REMOVED) && |
| 577 | (ae->flags & HEALTH_ENTRY_FLAG_SAVED) && |
| 578 | (ae->when + 86400 < now_realtime_sec()))) |
| 579 | { |
| 580 | DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->health_log.alarms, ae, prev, next); |
| 581 | health_alarm_log_free_one_nochecks_nounlink(ae); |
| 582 | } |
| 583 | |
| 584 | ae = next; |
| 585 | } |
| 586 | |
| 587 | rw_spinlock_write_unlock(&host->health_log.spinlock); |
| 588 | } |