| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "health.h" |
| 4 | #include "health-alert-entry.h" |
| 5 | |
| 6 | // ---------------------------------------------------------------------------- |
| 7 | // ARAL memory management for ALARM_ENTRY structures |
| 8 | |
| 9 | static struct { |
| 10 | ARAL *ar; |
| 11 | } health_alarm_entry_globals = { |
| 12 | .ar = NULL, |
| 13 | }; |
| 14 | |
| 15 | void health_alarm_entry_aral_init(void) { |
| 16 | health_alarm_entry_globals.ar = aral_create( |
| 17 | "health-alarm-entry", |
| 18 | sizeof(ALARM_ENTRY), |
| 19 | 0, |
| 20 | 0, |
| 21 | NULL, |
| 22 | NULL, NULL, false, false, true |
| 23 | ); |
| 24 | |
| 25 | pulse_aral_register(health_alarm_entry_globals.ar, "health_log"); |
| 26 | } |
| 27 | |
| 28 | struct aral_statistics *health_alarm_entry_aral_stats(void) { |
| 29 | return aral_get_statistics(health_alarm_entry_globals.ar); |
| 30 | } |
| 31 | |
| 32 | ALARM_ENTRY *health_alarm_entry_create(void) { |
| 33 | ALARM_ENTRY *ae = aral_mallocz(health_alarm_entry_globals.ar); |
| 34 | memset(ae, 0, sizeof(ALARM_ENTRY)); |
| 35 | return ae; |
| 36 | } |
| 37 | |
| 38 | void health_alarm_entry_destroy(ALARM_ENTRY *ae) { |
| 39 | aral_freez(health_alarm_entry_globals.ar, ae); |
| 40 | } |
| 41 | |
| 42 | // ---------------------------------------------------------------------------- |
| 43 | extern __thread bool is_health_thread; |
| 44 | |
| 45 | inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae, bool async) |
| 46 | { |
| 47 | if (async) { |
| 48 | bool queued = metadata_queue_ae_save(host, ae); |
| 49 | if (!queued && is_health_thread && service_running(SERVICE_HEALTH)) |
| 50 | sql_health_alarm_log_save(host, ae); |
| 51 | } else |
| 52 | sql_health_alarm_log_save(host, ae); |
| 53 | } |
| 54 | |
| 55 | void health_log_alert_transition_with_trace(RRDHOST *host, ALARM_ENTRY *ae, int line, const char *file, const char *function) { |
| 56 | if(!host || !ae) return; |
| 57 | |
| 58 | ND_LOG_STACK lgs[] = { |
| 59 | ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &health_alert_transition_msgid), |
| 60 | ND_LOG_FIELD_STR(NDF_NIDL_NODE, host->hostname), |
| 61 | ND_LOG_FIELD_STR(NDF_NIDL_INSTANCE, ae->chart_name), |
| 62 | ND_LOG_FIELD_STR(NDF_NIDL_CONTEXT, ae->chart_context), |
| 63 | ND_LOG_FIELD_U64(NDF_ALERT_ID, ae->alarm_id), |
| 64 | ND_LOG_FIELD_U64(NDF_ALERT_UNIQUE_ID, ae->unique_id), |
| 65 | ND_LOG_FIELD_U64(NDF_ALERT_EVENT_ID, ae->alarm_event_id), |
| 66 | ND_LOG_FIELD_UUID(NDF_ALERT_CONFIG_HASH, &ae->config_hash_id), |
| 67 | ND_LOG_FIELD_UUID(NDF_ALERT_TRANSITION_ID, &ae->transition_id), |
| 68 | ND_LOG_FIELD_STR(NDF_ALERT_NAME, ae->name), |
| 69 | ND_LOG_FIELD_STR(NDF_ALERT_CLASS, ae->classification), |
| 70 | ND_LOG_FIELD_STR(NDF_ALERT_COMPONENT, ae->component), |
| 71 | ND_LOG_FIELD_STR(NDF_ALERT_TYPE, ae->type), |
| 72 | ND_LOG_FIELD_STR(NDF_ALERT_EXEC, ae->exec), |
| 73 | ND_LOG_FIELD_STR(NDF_ALERT_RECIPIENT, ae->recipient), |
| 74 | ND_LOG_FIELD_STR(NDF_ALERT_SOURCE, ae->exec), |
| 75 | ND_LOG_FIELD_STR(NDF_ALERT_UNITS, ae->units), |
| 76 | ND_LOG_FIELD_STR(NDF_ALERT_SUMMARY, ae->summary), |
| 77 | ND_LOG_FIELD_STR(NDF_ALERT_INFO, ae->info), |
| 78 | ND_LOG_FIELD_DBL(NDF_ALERT_VALUE, ae->new_value), |
| 79 | ND_LOG_FIELD_DBL(NDF_ALERT_VALUE_OLD, ae->old_value), |
| 80 | ND_LOG_FIELD_TXT(NDF_ALERT_STATUS, rrdcalc_status2string(ae->new_status)), |
| 81 | ND_LOG_FIELD_TXT(NDF_ALERT_STATUS_OLD, rrdcalc_status2string(ae->old_status)), |
| 82 | ND_LOG_FIELD_I64(NDF_ALERT_DURATION, ae->duration), |
| 83 | ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, ae->exec_code), |
| 84 | ND_LOG_FIELD_U64(NDF_ALERT_NOTIFICATION_REALTIME_USEC, ae->delay_up_to_timestamp * USEC_PER_SEC), |
| 85 | ND_LOG_FIELD_END(), |
| 86 | }; |
| 87 | ND_LOG_STACK_PUSH(lgs); |
| 88 | |
| 89 | errno_clear(); |
| 90 | |
| 91 | ND_LOG_FIELD_PRIORITY priority = NDLP_INFO; |
| 92 | const char *emoji = "🌀"; |
| 93 | const char *transitioned = (ae->old_status < ae->new_status) ? "raised" : "lowered"; |
| 94 | |
| 95 | switch(ae->new_status) { |
| 96 | case RRDCALC_STATUS_UNDEFINED: |
| 97 | emoji = "❓"; |
| 98 | if(ae->old_status >= RRDCALC_STATUS_CLEAR) |
| 99 | priority = NDLP_NOTICE; |
| 100 | else |
| 101 | priority = NDLP_DEBUG; |
| 102 | break; |
| 103 | |
| 104 | default: |
| 105 | case RRDCALC_STATUS_UNINITIALIZED: |
| 106 | emoji = "⏳"; |
| 107 | priority = NDLP_DEBUG; |
| 108 | break; |
| 109 | |
| 110 | case RRDCALC_STATUS_REMOVED: |
| 111 | emoji = "🚫"; |
| 112 | priority = NDLP_DEBUG; |
| 113 | break; |
| 114 | |
| 115 | case RRDCALC_STATUS_CLEAR: |
| 116 | if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED) { |
| 117 | emoji = "💚"; |
| 118 | priority = NDLP_DEBUG; |
| 119 | } |
| 120 | else if(ae->old_status <= RRDCALC_STATUS_CLEAR) { |
| 121 | emoji = "✅"; |
| 122 | priority = NDLP_INFO; |
| 123 | } |
| 124 | else { |
| 125 | emoji = "💚"; |
| 126 | priority = NDLP_NOTICE; |
| 127 | } |
| 128 | break; |
| 129 | |
| 130 | case RRDCALC_STATUS_WARNING: |
| 131 | if(ae->old_status <= RRDCALC_STATUS_WARNING) { |
| 132 | emoji = "⚠️"; |
| 133 | priority = NDLP_WARNING; |
| 134 | } |
| 135 | else { |
| 136 | emoji = "🔶"; |
| 137 | priority = NDLP_INFO; |
| 138 | } |
| 139 | break; |
| 140 | |
| 141 | case RRDCALC_STATUS_CRITICAL: |
| 142 | emoji = "🔴"; |
| 143 | priority = NDLP_CRIT; |
| 144 | break; |
| 145 | } |
| 146 | |
| 147 | netdata_logger(NDLS_HEALTH, priority, file, function, line, |
| 148 | "ALERT '%s' of '%s' on node '%s', %s from %s to %s.\n" |
| 149 | "%s %s on %s.\n" |
| 150 | "%s:%s:%s value got from %f %s, to %f %s.", |
| 151 | string2str(ae->name), string2str(ae->chart), string2str(host->hostname), |
| 152 | transitioned, rrdcalc_status2string(ae->old_status), rrdcalc_status2string(ae->new_status), |
| 153 | emoji, string2str(ae->info), string2str(host->hostname), |
| 154 | string2str(host->hostname), string2str(ae->chart), string2str(ae->name), |
| 155 | ae->old_value, string2str(ae->units), |
| 156 | ae->new_value, string2str(ae->units)); |
| 157 | } |
| 158 | |
| 159 | // ---------------------------------------------------------------------------- |
| 160 | // health alarm log management |
| 161 | |
| 162 | inline ALARM_ENTRY* health_create_alarm_entry( |
| 163 | RRDHOST *host, |
| 164 | RRDCALC *rc, |
| 165 | time_t when, |
| 166 | time_t duration, |
| 167 | NETDATA_DOUBLE old_value, |
| 168 | NETDATA_DOUBLE new_value, |
| 169 | RRDCALC_STATUS old_status, |
| 170 | RRDCALC_STATUS new_status, |
| 171 | int delay, |
| 172 | HEALTH_ENTRY_FLAGS flags |
| 173 | ) { |
| 174 | uint32_t alarm_id = rc->id; |
| 175 | uint32_t alarm_event_id = rc->next_event_id++; |
| 176 | STRING *name = rc->config.name; |
| 177 | STRING *chart = rc->rrdset->id; |
| 178 | STRING *chart_context = rc->rrdset->context; |
| 179 | STRING *chart_name = rc->rrdset->name; |
| 180 | STRING *class = rc->config.classification; |
| 181 | STRING *component = rc->config.component; |
| 182 | STRING *type = rc->config.type; |
| 183 | STRING *exec = rc->config.exec; |
| 184 | STRING *recipient = rc->config.recipient; |
| 185 | STRING *source = rc->config.source; |
| 186 | STRING *units = rc->config.units; |
| 187 | STRING *summary = rc->summary; |
| 188 | STRING *info = rc->info; |
| 189 | |
| 190 | if (duration < 0) |
| 191 | duration = 0; |
| 192 | |
| 193 | netdata_log_debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id); |
| 194 | |
| 195 | ALARM_ENTRY *ae = health_alarm_entry_create(); |
| 196 | ae->name = string_dup(name); |
| 197 | ae->chart = string_dup(chart); |
| 198 | ae->chart_context = string_dup(chart_context); |
| 199 | ae->chart_name = string_dup(chart_name); |
| 200 | |
| 201 | uuid_copy(ae->config_hash_id, rc->config.hash_id); |
| 202 | |
| 203 | uuid_generate_random(ae->transition_id); |
| 204 | ae->global_id = now_realtime_usec(); |
| 205 | |
| 206 | ae->classification = string_dup(class); |
| 207 | ae->component = string_dup(component); |
| 208 | ae->type = string_dup(type); |
| 209 | ae->exec = string_dup(exec); |
| 210 | ae->recipient = string_dup(recipient); |
| 211 | ae->source = string_dup(source); |
| 212 | ae->units = string_dup(units); |
| 213 | |
| 214 | ae->unique_id = host->health_log.next_log_id++; |
| 215 | ae->alarm_id = alarm_id; |
| 216 | ae->alarm_event_id = alarm_event_id; |
| 217 | ae->when = when; |
| 218 | ae->old_value = old_value; |
| 219 | ae->new_value = new_value; |
| 220 | |
| 221 | char value_string[100 + 1]; |
| 222 | ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1)); |
| 223 | ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1)); |
| 224 | |
| 225 | ae->summary = string_dup(summary); |
| 226 | ae->info = string_dup(info); |
| 227 | ae->old_status = old_status; |
| 228 | ae->new_status = new_status; |
| 229 | ae->duration = duration; |
| 230 | ae->delay = delay; |
| 231 | ae->delay_up_to_timestamp = when + delay; |
| 232 | ae->flags |= flags; |
| 233 | |
| 234 | ae->last_repeat = 0; |
| 235 | ae->pending_save_count = 0; |
| 236 | |
| 237 | if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL) |
| 238 | ae->non_clear_duration += ae->duration; |
| 239 | |
| 240 | return ae; |
| 241 | } |
| 242 | |
| 243 | inline void health_alarm_log_add_entry(RRDHOST *host, ALARM_ENTRY *ae, bool async) |
| 244 | { |
| 245 | netdata_log_debug(D_HEALTH, "Health adding alarm log entry with id: %u", ae->unique_id); |
| 246 | |
| 247 | __atomic_add_fetch(&host->health_transitions, 1, __ATOMIC_RELAXED); |
| 248 | |
| 249 | // link it |
| 250 | rw_spinlock_write_lock(&host->health_log.spinlock); |
| 251 | DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(host->health_log.alarms, ae, prev, next); |
| 252 | rw_spinlock_write_unlock(&host->health_log.spinlock); |
| 253 | |
| 254 | // match previous alarms |
| 255 | rw_spinlock_read_lock(&host->health_log.spinlock); |
| 256 | ALARM_ENTRY *update_ae = NULL; |
| 257 | for(ALARM_ENTRY *t = host->health_log.alarms ; t ; t = t->next) { |
| 258 | if(t != ae && t->alarm_id == ae->alarm_id) { |
| 259 | if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) { |
| 260 | t->flags |= HEALTH_ENTRY_FLAG_UPDATED; |
| 261 | t->updated_by_id = ae->unique_id; |
| 262 | ae->updates_id = t->unique_id; |
| 263 | |
| 264 | if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) && |
| 265 | (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL)) |
| 266 | ae->non_clear_duration += t->non_clear_duration; |
| 267 | |
| 268 | update_ae = t; |
| 269 | } |
| 270 | |
| 271 | // no need to continue |
| 272 | break; |
| 273 | } |
| 274 | } |
| 275 | rw_spinlock_read_unlock(&host->health_log.spinlock); |
| 276 | if (update_ae) |
| 277 | health_alarm_log_save(host, update_ae, async); |
| 278 | |
| 279 | health_alarm_log_save(host, ae, async); |
| 280 | } |
| 281 | |
| 282 | inline void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae) { |
| 283 | if(__atomic_load_n(&ae->pending_save_count, __ATOMIC_RELAXED)) |
| 284 | metadata_queue_ae_deletion(ae); |
| 285 | else { |
| 286 | string_freez(ae->name); |
| 287 | string_freez(ae->chart); |
| 288 | string_freez(ae->chart_name); |
| 289 | string_freez(ae->chart_context); |
| 290 | string_freez(ae->classification); |
| 291 | string_freez(ae->component); |
| 292 | string_freez(ae->type); |
| 293 | string_freez(ae->exec); |
| 294 | string_freez(ae->recipient); |
| 295 | string_freez(ae->source); |
| 296 | string_freez(ae->units); |
| 297 | string_freez(ae->info); |
| 298 | string_freez(ae->old_value_string); |
| 299 | string_freez(ae->new_value_string); |
| 300 | string_freez(ae->summary); |
| 301 | |
| 302 | if(ae->popen_instance) { |
| 303 | spawn_popen_kill(ae->popen_instance, 0); |
| 304 | ae->popen_instance = NULL; |
| 305 | } |
| 306 | |
| 307 | if(ae->next || ae->prev) |
| 308 | fatal("HEALTH: alarm entry to delete is still linked!"); |
| 309 | |
| 310 | // if(ae->prev_in_progress || ae->next_in_progress) |
| 311 | // fatal("HEALTH: alarm entry to be delete is linked in progress!"); |
| 312 | |
| 313 | health_alarm_entry_destroy(ae); |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | inline void health_alarm_log_free(RRDHOST *host) { |
| 318 | rw_spinlock_write_lock(&host->health_log.spinlock); |
| 319 | |
| 320 | while(host->health_log.alarms) { |
| 321 | ALARM_ENTRY *ae = host->health_log.alarms; |
| 322 | DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->health_log.alarms, ae, prev, next); |
| 323 | health_alarm_log_free_one_nochecks_nounlink(ae); |
| 324 | } |
| 325 | |
| 326 | rw_spinlock_write_unlock(&host->health_log.spinlock); |
| 327 | } |
| 328 | |
| 329 | // Clean up old alarm entries from memory based on retention settings |
| 330 | void health_alarm_log_cleanup(RRDHOST *host) { |
| 331 | if(!host->health_log.alarms) |
| 332 | return; |
| 333 | |
| 334 | if (!rw_spinlock_trywrite_lock(&host->health_log.spinlock)) { |
| 335 | // If we can't get the lock, just return |
| 336 | return; |
| 337 | } |
| 338 | |
| 339 | time_t now = now_realtime_sec(); |
| 340 | time_t retention = host->health_log.health_log_retention_s; |
| 341 | |
| 342 | ALARM_ENTRY *ae = host->health_log.alarms; |
| 343 | while(ae) { |
| 344 | // Check if entry is old enough to be deleted |
| 345 | if(ae->when < now - retention && |
| 346 | (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) && // Only remove entries that have been processed/updated |
| 347 | __atomic_load_n(&ae->pending_save_count, __ATOMIC_RELAXED) == 0) { // Only remove entries not pending save |
| 348 | |
| 349 | // Remove from linked list |
| 350 | ALARM_ENTRY *next = ae->next; |
| 351 | DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->health_log.alarms, ae, prev, next); |
| 352 | |
| 353 | ALARM_ENTRY *to_free = ae; |
| 354 | ae = next; |
| 355 | |
| 356 | // Free memory |
| 357 | health_alarm_log_free_one_nochecks_nounlink(to_free); |
| 358 | } |
| 359 | else |
| 360 | ae = ae->next; |
| 361 | } |
| 362 | |
| 363 | rw_spinlock_write_unlock(&host->health_log.spinlock); |
| 364 | } |