| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "health_internals.h" |
| 4 | #include "health-alert-entry.h" |
| 5 | #include "database/sqlite/sqlite_aclk_alert.h" |
| 6 | |
| 7 | // --------------------------------------------------------------------------------------------------------------------- |
| 8 | |
| 9 | static struct { |
| 10 | ALERT_LOOKUP_DIMS_GROUPING group; |
| 11 | const char *name; |
| 12 | } dims_grouping[] = { |
| 13 | { .group = ALERT_LOOKUP_DIMS_SUM, .name = "sum" }, |
| 14 | { .group = ALERT_LOOKUP_DIMS_MIN, .name = "min" }, |
| 15 | { .group = ALERT_LOOKUP_DIMS_MAX, .name = "max" }, |
| 16 | { .group = ALERT_LOOKUP_DIMS_AVERAGE, .name = "average" }, |
| 17 | { .group = ALERT_LOOKUP_DIMS_MIN2MAX, .name = "min2max" }, |
| 18 | |
| 19 | // terminator |
| 20 | { .group = 0, .name = NULL }, |
| 21 | }; |
| 22 | |
| 23 | ALERT_LOOKUP_DIMS_GROUPING alerts_dims_grouping2id(const char *group) { |
| 24 | if(!group || !*group) |
| 25 | return dims_grouping[0].group; |
| 26 | |
| 27 | for(size_t i = 0; dims_grouping[i].name ;i++) { |
| 28 | if(strcmp(dims_grouping[i].name, group) == 0) |
| 29 | return dims_grouping[i].group; |
| 30 | } |
| 31 | |
| 32 | nd_log(NDLS_DAEMON, NDLP_WARNING, "Alert lookup dimensions grouping '%s' is not valid", group); |
| 33 | return dims_grouping[0].group; |
| 34 | } |
| 35 | |
| 36 | const char *alerts_dims_grouping_id2group(ALERT_LOOKUP_DIMS_GROUPING grouping) { |
| 37 | for(size_t i = 0; dims_grouping[i].name ;i++) { |
| 38 | if(grouping == dims_grouping[i].group) |
| 39 | return dims_grouping[i].name; |
| 40 | } |
| 41 | |
| 42 | nd_log(NDLS_DAEMON, NDLP_WARNING, "Alert lookup dimensions grouping %d is not valid", grouping); |
| 43 | return dims_grouping[0].name; |
| 44 | } |
| 45 | |
| 46 | // --------------------------------------------------------------------------------------------------------------------- |
| 47 | |
| 48 | static struct { |
| 49 | ALERT_LOOKUP_DATA_SOURCE source; |
| 50 | const char *name; |
| 51 | } data_sources[] = { |
| 52 | { .source = ALERT_LOOKUP_DATA_SOURCE_SAMPLES, .name = "samples" }, |
| 53 | { .source = ALERT_LOOKUP_DATA_SOURCE_PERCENTAGES, .name = "percentages" }, |
| 54 | { .source = ALERT_LOOKUP_DATA_SOURCE_ANOMALIES, .name = "anomalies" }, |
| 55 | |
| 56 | // terminator |
| 57 | { .source = 0, .name = NULL }, |
| 58 | }; |
| 59 | |
| 60 | ALERT_LOOKUP_DATA_SOURCE alerts_data_sources2id(const char *source) { |
| 61 | if(!source || !*source) |
| 62 | return data_sources[0].source; |
| 63 | |
| 64 | for(size_t i = 0; data_sources[i].name ;i++) { |
| 65 | if(strcmp(data_sources[i].name, source) == 0) |
| 66 | return data_sources[i].source; |
| 67 | } |
| 68 | |
| 69 | nd_log(NDLS_DAEMON, NDLP_WARNING, "Alert data source '%s' is not valid", source); |
| 70 | return data_sources[0].source; |
| 71 | } |
| 72 | |
| 73 | const char *alerts_data_source_id2source(ALERT_LOOKUP_DATA_SOURCE source) { |
| 74 | for(size_t i = 0; data_sources[i].name ;i++) { |
| 75 | if(source == data_sources[i].source) |
| 76 | return data_sources[i].name; |
| 77 | } |
| 78 | |
| 79 | nd_log(NDLS_DAEMON, NDLP_WARNING, "Alert data source %d is not valid", source); |
| 80 | return data_sources[0].name; |
| 81 | } |
| 82 | |
| 83 | // --------------------------------------------------------------------------------------------------------------------- |
| 84 | |
| 85 | static struct { |
| 86 | ALERT_LOOKUP_TIME_GROUP_CONDITION condition; |
| 87 | const char *name; |
| 88 | } group_conditions[] = { |
| 89 | { .condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_EQUAL, .name = "=" }, |
| 90 | { .condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_NOT_EQUAL, .name = "!=" }, |
| 91 | { .condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_GREATER, .name = ">" }, |
| 92 | { .condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_GREATER_EQUAL, .name = ">=" }, |
| 93 | { .condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_LESS, .name = "<" }, |
| 94 | { .condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_LESS_EQUAL, .name = "<=" }, |
| 95 | |
| 96 | // terminator |
| 97 | { .condition = 0, .name = NULL }, |
| 98 | }; |
| 99 | |
| 100 | ALERT_LOOKUP_TIME_GROUP_CONDITION alerts_group_condition2id(const char *source) { |
| 101 | if(!source || !*source) |
| 102 | return group_conditions[0].condition; |
| 103 | |
| 104 | for(size_t i = 0; group_conditions[i].name ;i++) { |
| 105 | if(strcmp(group_conditions[i].name, source) == 0) |
| 106 | return group_conditions[i].condition; |
| 107 | } |
| 108 | |
| 109 | nd_log(NDLS_DAEMON, NDLP_WARNING, "Alert data source '%s' is not valid", source); |
| 110 | return group_conditions[0].condition; |
| 111 | } |
| 112 | |
| 113 | const char *alerts_group_conditions_id2txt(ALERT_LOOKUP_TIME_GROUP_CONDITION source) { |
| 114 | for(size_t i = 0; group_conditions[i].name ;i++) { |
| 115 | if(source == group_conditions[i].condition) |
| 116 | return group_conditions[i].name; |
| 117 | } |
| 118 | |
| 119 | nd_log(NDLS_DAEMON, NDLP_WARNING, "Alert data source %d is not valid", source); |
| 120 | return group_conditions[0].name; |
| 121 | } |
| 122 | |
| 123 | // --------------------------------------------------------------------------------------------------------------------- |
| 124 | |
| 125 | static struct { |
| 126 | const char *name; |
| 127 | uint32_t hash; |
| 128 | ALERT_ACTION_OPTIONS value; |
| 129 | } alert_action_options[] = { |
| 130 | { "no-clear-notification", 0 , ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION} |
| 131 | |
| 132 | // terminator |
| 133 | , {NULL, 0, 0} |
| 134 | }; |
| 135 | |
| 136 | inline ALERT_ACTION_OPTIONS alert_action_options_parse_one(const char *o) { |
| 137 | ALERT_ACTION_OPTIONS ret = 0; |
| 138 | |
| 139 | if(!o || !*o) return ret; |
| 140 | |
| 141 | uint32_t hash = simple_hash(o); |
| 142 | int i; |
| 143 | for(i = 0; alert_action_options[i].name ; i++) { |
| 144 | if (unlikely(hash == alert_action_options[i].hash && !strcmp(o, alert_action_options[i].name))) { |
| 145 | ret |= alert_action_options[i].value; |
| 146 | break; |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | return ret; |
| 151 | } |
| 152 | |
| 153 | inline ALERT_ACTION_OPTIONS alert_action_options_parse(char *o) { |
| 154 | ALERT_ACTION_OPTIONS ret = 0; |
| 155 | char *tok; |
| 156 | |
| 157 | while(o && *o && (tok = strsep_skip_consecutive_separators(&o, ", |"))) { |
| 158 | if(!*tok) continue; |
| 159 | ret |= alert_action_options_parse_one(tok); |
| 160 | } |
| 161 | |
| 162 | return ret; |
| 163 | } |
| 164 | |
| 165 | void alert_action_options_to_buffer_json_array(BUFFER *wb, const char *key, ALERT_ACTION_OPTIONS options) { |
| 166 | buffer_json_member_add_array(wb, key); |
| 167 | |
| 168 | RRDR_OPTIONS used = 0; // to prevent adding duplicates |
| 169 | for(int i = 0; alert_action_options[i].name ; i++) { |
| 170 | if (unlikely((alert_action_options[i].value & options) && !(alert_action_options[i].value & used))) { |
| 171 | const char *name = alert_action_options[i].name; |
| 172 | used |= alert_action_options[i].value; |
| 173 | |
| 174 | buffer_json_add_array_item_string(wb, name); |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | buffer_json_array_close(wb); |
| 179 | } |
| 180 | |
| 181 | void alert_action_options_to_buffer(BUFFER *wb, ALERT_ACTION_OPTIONS options) { |
| 182 | RRDR_OPTIONS used = 0; // to prevent adding duplicates |
| 183 | for(int i = 0; alert_action_options[i].name ; i++) { |
| 184 | if (unlikely((alert_action_options[i].value & options) && !(alert_action_options[i].value & used))) { |
| 185 | if(used != 0) |
| 186 | buffer_strcat(wb, " "); |
| 187 | |
| 188 | const char *name = alert_action_options[i].name; |
| 189 | used |= alert_action_options[i].value; |
| 190 | |
| 191 | buffer_strcat(wb, name); |
| 192 | } |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | static void alert_action_options_init(void) { |
| 197 | for(int i = 0; alert_action_options[i].name ; i++) |
| 198 | alert_action_options[i].hash = simple_hash(alert_action_options[i].name); |
| 199 | } |
| 200 | |
| 201 | |
| 202 | // --------------------------------------------------------------------------------------------------------------------- |
| 203 | |
| 204 | static void health_prototype_cleanup_one_unsafe(RRD_ALERT_PROTOTYPE *ap) { |
| 205 | rrd_alert_match_cleanup(&ap->match); |
| 206 | rrd_alert_config_cleanup(&ap->config); |
| 207 | memset(ap, 0, sizeof(*ap)); |
| 208 | } |
| 209 | |
| 210 | void health_prototype_cleanup(RRD_ALERT_PROTOTYPE *ap) { |
| 211 | rw_spinlock_write_lock(&ap->_internal.rw_spinlock); |
| 212 | |
| 213 | while(ap->_internal.next) { |
| 214 | RRD_ALERT_PROTOTYPE *t = ap->_internal.next; |
| 215 | DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(ap->_internal.next, t, _internal.prev, _internal.next); |
| 216 | health_prototype_cleanup_one_unsafe(t); |
| 217 | freez(t); |
| 218 | } |
| 219 | |
| 220 | rw_spinlock_write_unlock(&ap->_internal.rw_spinlock); |
| 221 | |
| 222 | health_prototype_cleanup_one_unsafe(ap); |
| 223 | } |
| 224 | |
| 225 | void health_prototype_free(RRD_ALERT_PROTOTYPE *ap) { |
| 226 | if(!ap) return; |
| 227 | health_prototype_cleanup(ap); |
| 228 | freez(ap); |
| 229 | } |
| 230 | |
| 231 | void health_prototype_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { |
| 232 | RRD_ALERT_PROTOTYPE *ap = value; |
| 233 | rw_spinlock_init(&ap->_internal.rw_spinlock); |
| 234 | if(ap->config.source_type != DYNCFG_SOURCE_TYPE_DYNCFG) |
| 235 | ap->_internal.is_on_disk = true; |
| 236 | } |
| 237 | |
| 238 | bool health_prototype_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { |
| 239 | RRD_ALERT_PROTOTYPE *ap = old_value; |
| 240 | RRD_ALERT_PROTOTYPE *nap = new_value; |
| 241 | |
| 242 | bool replace = nap->config.source_type == DYNCFG_SOURCE_TYPE_DYNCFG; |
| 243 | |
| 244 | if(ap->config.source_type != DYNCFG_SOURCE_TYPE_DYNCFG || nap->config.source_type != DYNCFG_SOURCE_TYPE_DYNCFG) |
| 245 | ap->_internal.is_on_disk = nap->_internal.is_on_disk = true; |
| 246 | |
| 247 | if(!replace) { |
| 248 | if(ap->config.source_type != DYNCFG_SOURCE_TYPE_DYNCFG) { |
| 249 | // alerts with the same name are appended to the existing one |
| 250 | RRD_ALERT_PROTOTYPE *alloced = callocz(1, sizeof(*alloced)); |
| 251 | SWAP(*alloced, *nap); |
| 252 | |
| 253 | rw_spinlock_write_lock(&ap->_internal.rw_spinlock); |
| 254 | DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ap->_internal.next, alloced, _internal.prev, _internal.next); |
| 255 | rw_spinlock_write_unlock(&ap->_internal.rw_spinlock); |
| 256 | |
| 257 | if(alloced->_internal.enabled) |
| 258 | ap->_internal.enabled = true; |
| 259 | } |
| 260 | } |
| 261 | else { |
| 262 | // alerts with the same name replace the existing one |
| 263 | rw_spinlock_init(&nap->_internal.rw_spinlock); |
| 264 | |
| 265 | rw_spinlock_write_lock(&nap->_internal.rw_spinlock); |
| 266 | rw_spinlock_write_lock(&ap->_internal.rw_spinlock); |
| 267 | SWAP(*ap, *nap); |
| 268 | rw_spinlock_write_unlock(&ap->_internal.rw_spinlock); |
| 269 | rw_spinlock_write_unlock(&nap->_internal.rw_spinlock); |
| 270 | } |
| 271 | |
| 272 | health_prototype_cleanup(nap); |
| 273 | |
| 274 | return true; |
| 275 | } |
| 276 | |
| 277 | void health_prototype_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { |
| 278 | RRD_ALERT_PROTOTYPE *ap = value; |
| 279 | health_prototype_cleanup(ap); |
| 280 | } |
| 281 | |
| 282 | void health_init_prototypes(void) { |
| 283 | if(health_globals.prototypes.dict) |
| 284 | return; |
| 285 | |
| 286 | health_globals.prototypes.dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, &dictionary_stats_category_rrdhealth, sizeof(RRD_ALERT_PROTOTYPE)); |
| 287 | dictionary_register_insert_callback(health_globals.prototypes.dict, health_prototype_insert_cb, NULL); |
| 288 | dictionary_register_conflict_callback(health_globals.prototypes.dict, health_prototype_conflict_cb, NULL); |
| 289 | dictionary_register_delete_callback(health_globals.prototypes.dict, health_prototype_delete_cb, NULL); |
| 290 | |
| 291 | alert_action_options_init(); |
| 292 | } |
| 293 | |
| 294 | // --------------------------------------------------------------------------------------------------------------------- |
| 295 | |
| 296 | static inline struct pattern_array *health_config_add_key_to_values(struct pattern_array *pa, const char *input_key, char *value) |
| 297 | { |
| 298 | size_t value_len = strlen(value); |
| 299 | size_t input_key_len = input_key ? strlen(input_key) : 0; |
| 300 | size_t key_len = value_len > input_key_len ? value_len : input_key_len; |
| 301 | size_t pair_len = key_len + value_len + 4; |
| 302 | char *key = mallocz(key_len + 1); |
| 303 | char *data = mallocz(value_len + 1); |
| 304 | char *pair = mallocz(pair_len); |
| 305 | |
| 306 | char *s = value; |
| 307 | size_t i = 0; |
| 308 | |
| 309 | if (input_key) |
| 310 | strncpyz(key, input_key, key_len); |
| 311 | else |
| 312 | key[0] = '\0'; |
| 313 | |
| 314 | while(*s) { |
| 315 | if (*s == '=') { |
| 316 | //hold the key |
| 317 | data[i]='\0'; |
| 318 | strncpyz(key, data, key_len); |
| 319 | i=0; |
| 320 | } else if (*s == ' ') { |
| 321 | data[i]='\0'; |
| 322 | if (data[0]=='!') |
| 323 | snprintfz(pair, pair_len, "!%s=%s ", key, data + 1); |
| 324 | else |
| 325 | snprintfz(pair, pair_len, "%s=%s ", key, data); |
| 326 | |
| 327 | pa = pattern_array_add_key_simple_pattern(pa, key, simple_pattern_create(pair, NULL, SIMPLE_PATTERN_EXACT, true)); |
| 328 | i=0; |
| 329 | } else { |
| 330 | data[i++] = *s; |
| 331 | } |
| 332 | s++; |
| 333 | } |
| 334 | data[i]='\0'; |
| 335 | if (data[0]) { |
| 336 | if (data[0]=='!') |
| 337 | snprintfz(pair, pair_len, "!%s=%s ", key, data + 1); |
| 338 | else |
| 339 | snprintfz(pair, pair_len, "%s=%s ", key, data); |
| 340 | |
| 341 | pa = pattern_array_add_key_simple_pattern(pa, key, simple_pattern_create(pair, NULL, SIMPLE_PATTERN_EXACT, true)); |
| 342 | } |
| 343 | |
| 344 | freez(key); |
| 345 | freez(data); |
| 346 | freez(pair); |
| 347 | |
| 348 | return pa; |
| 349 | } |
| 350 | |
| 351 | static char *simple_pattern_trim_around_equal(const char *src) { |
| 352 | char *store = mallocz(strlen(src) + 1); |
| 353 | |
| 354 | char *dst = store; |
| 355 | while (*src) { |
| 356 | if (*src == '=') { |
| 357 | if (*(dst -1) == ' ') |
| 358 | dst--; |
| 359 | |
| 360 | *dst++ = *src++; |
| 361 | if (*src == ' ') |
| 362 | src++; |
| 363 | } |
| 364 | |
| 365 | *dst++ = *src++; |
| 366 | } |
| 367 | *dst = 0x00; |
| 368 | |
| 369 | return store; |
| 370 | } |
| 371 | |
| 372 | struct pattern_array *trim_and_add_key_to_values(struct pattern_array *pa, const char *key, STRING *input) { |
| 373 | char *tmp = simple_pattern_trim_around_equal(string2str(input)); |
| 374 | pa = health_config_add_key_to_values(pa, key, tmp); |
| 375 | freez(tmp); |
| 376 | return pa; |
| 377 | } |
| 378 | |
| 379 | static void health_prototype_activate_match_patterns(struct rrd_alert_match *am) { |
| 380 | if(am->host_labels) { |
| 381 | pattern_array_free(am->host_labels_pattern); |
| 382 | am->host_labels_pattern = NULL; |
| 383 | am->host_labels_pattern = trim_and_add_key_to_values(am->host_labels_pattern, NULL, am->host_labels); |
| 384 | } |
| 385 | |
| 386 | if(am->chart_labels) { |
| 387 | pattern_array_free(am->chart_labels_pattern); |
| 388 | am->chart_labels_pattern = NULL; |
| 389 | am->chart_labels_pattern = trim_and_add_key_to_values(am->chart_labels_pattern, NULL, am->chart_labels); |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | void health_prototype_hash_id(RRD_ALERT_PROTOTYPE *ap) { |
| 394 | CLEAN_BUFFER *wb = buffer_create(100, NULL); |
| 395 | health_prototype_to_json(wb, ap, true); |
| 396 | ND_UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb)); |
| 397 | uuid_copy(ap->config.hash_id, uuid.uuid); |
| 398 | |
| 399 | sql_alert_store_config(ap); |
| 400 | |
| 401 | if (ap->config.source_type == DYNCFG_SOURCE_TYPE_DYNCFG && |
| 402 | !__atomic_load_n(&health_globals.prototypes.registering, __ATOMIC_RELAXED) && |
| 403 | !alert_hash_has_transitioned(&ap->config.hash_id)) { |
| 404 | char hash_id_str[UUID_STR_LEN]; |
| 405 | uuid_unparse_lower(ap->config.hash_id, hash_id_str); |
| 406 | aclk_send_alert_configuration(hash_id_str); |
| 407 | } |
| 408 | } |
| 409 | |
| 410 | bool health_prototype_add(RRD_ALERT_PROTOTYPE *ap, char **msg) { |
| 411 | if(!ap->match.is_template) { |
| 412 | if(!ap->match.on.chart) { |
| 413 | netdata_log_error( |
| 414 | "HEALTH: alert '%s' does not define a instance (parameter 'on'). Source: %s", |
| 415 | string2str(ap->config.name), string2str(ap->config.source)); |
| 416 | if(msg) |
| 417 | *msg = "missing match 'on' parameter for instance"; |
| 418 | return false; |
| 419 | } |
| 420 | } |
| 421 | else { |
| 422 | if(!ap->match.on.context) { |
| 423 | netdata_log_error( |
| 424 | "HEALTH: alert '%s' does not define a context (parameter 'on'). Source: %s", |
| 425 | string2str(ap->config.name), string2str(ap->config.source)); |
| 426 | if(msg) |
| 427 | *msg = "missing match 'on' parameter for context"; |
| 428 | return false; |
| 429 | } |
| 430 | } |
| 431 | |
| 432 | if(!ap->config.update_every) { |
| 433 | netdata_log_error( |
| 434 | "HEALTH: alert '%s' has no frequency (parameter 'every'). Source: %s", |
| 435 | string2str(ap->config.name), string2str(ap->config.source)); |
| 436 | if(msg) |
| 437 | *msg = "missing update frequency"; |
| 438 | return false; |
| 439 | } |
| 440 | |
| 441 | if(!RRDCALC_HAS_DB_LOOKUP(ap) && !ap->config.calculation && !ap->config.warning && !ap->config.critical) { |
| 442 | netdata_log_error( |
| 443 | "HEALTH: alert '%s' is useless (no db lookup, no calculation, no warning and no critical expressions). Source: %s", |
| 444 | string2str(ap->config.name), string2str(ap->config.source)); |
| 445 | if(msg) |
| 446 | *msg = "no db lookup, calculation and warning/critical conditions"; |
| 447 | return false; |
| 448 | } |
| 449 | |
| 450 | // activate the match patterns in it |
| 451 | bool enabled = false; |
| 452 | for(RRD_ALERT_PROTOTYPE *t = ap; t ;t = t->_internal.next) { |
| 453 | // we need to generate config_hash_id for each instance included |
| 454 | // so, let's break the linked list for this iteration |
| 455 | |
| 456 | RRD_ALERT_PROTOTYPE *prev = t->_internal.prev; |
| 457 | RRD_ALERT_PROTOTYPE *next = t->_internal.next; |
| 458 | t->_internal.prev = t; |
| 459 | t->_internal.next = NULL; |
| 460 | |
| 461 | if(t->match.enabled) |
| 462 | enabled = true; |
| 463 | |
| 464 | if(!t->config.name) |
| 465 | t->config.name = string_dup(ap->config.name); |
| 466 | |
| 467 | health_prototype_hash_id(t); |
| 468 | |
| 469 | health_prototype_activate_match_patterns(&t->match); |
| 470 | |
| 471 | if (!t->config.exec) |
| 472 | t->config.exec = string_dup(health_globals.config.default_exec); |
| 473 | |
| 474 | if (!t->config.recipient) |
| 475 | t->config.recipient = string_dup(health_globals.config.default_recipient); |
| 476 | |
| 477 | // restore the linked list |
| 478 | t->_internal.prev = prev; |
| 479 | t->_internal.next = next; |
| 480 | } |
| 481 | ap->_internal.enabled = enabled; |
| 482 | |
| 483 | // add it to the prototypes |
| 484 | dictionary_set_advanced(health_globals.prototypes.dict, |
| 485 | string2str(ap->config.name), string_strlen(ap->config.name), |
| 486 | ap, sizeof(RRD_ALERT_PROTOTYPE), |
| 487 | NULL); |
| 488 | |
| 489 | return true; |
| 490 | } |
| 491 | |
| 492 | // --------------------------------------------------------------------------------------------------------------------- |
| 493 | |
| 494 | void health_reload_prototypes(void) { |
| 495 | // remove all dyncfg related to prototypes |
| 496 | health_dyncfg_unregister_all_prototypes(); |
| 497 | |
| 498 | // clear old prototypes from memory |
| 499 | dictionary_flush(health_globals.prototypes.dict); |
| 500 | |
| 501 | // load the prototypes from disk |
| 502 | recursive_config_double_dir_load( |
| 503 | health_user_config_dir(), |
| 504 | health_globals.config.stock_enabled ? health_stock_config_dir() : NULL, |
| 505 | NULL, |
| 506 | health_readfile, |
| 507 | NULL, 0); |
| 508 | |
| 509 | // register all loaded prototypes |
| 510 | __atomic_store_n(&health_globals.prototypes.registering, true, __ATOMIC_RELAXED); |
| 511 | health_dyncfg_register_all_prototypes(); |
| 512 | __atomic_store_n(&health_globals.prototypes.registering, false, __ATOMIC_RELAXED); |
| 513 | } |
| 514 | |
| 515 | // --------------------------------------------------------------------------------------------------------------------- |
| 516 | |
| 517 | static bool prototype_matches_host(RRDHOST *host, RRD_ALERT_PROTOTYPE *ap) { |
| 518 | if(health_globals.config.enabled_alerts && |
| 519 | !simple_pattern_matches(health_globals.config.enabled_alerts, string2str(ap->config.name))) |
| 520 | return false; |
| 521 | |
| 522 | if (host->rrdlabels && ap->match.host_labels_pattern && |
| 523 | !pattern_array_label_match(ap->match.host_labels_pattern, host->rrdlabels, '=', NULL)) |
| 524 | return false; |
| 525 | |
| 526 | return true; |
| 527 | } |
| 528 | |
| 529 | static bool prototype_matches_rrdset(RRDSET *st, RRD_ALERT_PROTOTYPE *ap) { |
| 530 | // match the chart id |
| 531 | if(!ap->match.is_template && ap->match.on.chart && |
| 532 | ap->match.on.chart != st->id && ap->match.on.chart != st->name) |
| 533 | return false; |
| 534 | |
| 535 | // match the chart context |
| 536 | if(ap->match.is_template && ap->match.on.context && |
| 537 | ap->match.on.context != st->context) |
| 538 | return false; |
| 539 | |
| 540 | if (st->rrdlabels && ap->match.chart_labels_pattern && |
| 541 | !pattern_array_label_match(ap->match.chart_labels_pattern, st->rrdlabels, '=', NULL)) |
| 542 | return false; |
| 543 | |
| 544 | return true; |
| 545 | } |
| 546 | |
| 547 | void health_prototype_copy_match_without_patterns(struct rrd_alert_match *dst, struct rrd_alert_match *src) { |
| 548 | dst->enabled = src->enabled; |
| 549 | dst->is_template = src->is_template; |
| 550 | |
| 551 | if(dst->is_template) |
| 552 | dst->on.context = string_dup(src->on.context); |
| 553 | else |
| 554 | dst->on.chart = string_dup(src->on.chart); |
| 555 | |
| 556 | dst->host_labels = string_dup(src->host_labels); |
| 557 | dst->chart_labels = string_dup(src->chart_labels); |
| 558 | } |
| 559 | |
| 560 | void health_prototype_copy_config(struct rrd_alert_config *dst, struct rrd_alert_config *src) { |
| 561 | uuid_copy(dst->hash_id, src->hash_id); |
| 562 | |
| 563 | dst->name = string_dup(src->name); |
| 564 | |
| 565 | dst->exec = string_dup(src->exec); |
| 566 | dst->recipient = string_dup(src->recipient); |
| 567 | |
| 568 | dst->classification = string_dup(src->classification); |
| 569 | dst->component = string_dup(src->component); |
| 570 | dst->type = string_dup(src->type); |
| 571 | |
| 572 | dst->source_type = src->source_type; |
| 573 | dst->source = string_dup(src->source); |
| 574 | dst->units = string_dup(src->units); |
| 575 | dst->summary = string_dup(src->summary); |
| 576 | dst->info = string_dup(src->info); |
| 577 | |
| 578 | dst->update_every = src->update_every; |
| 579 | |
| 580 | dst->alert_action_options = src->alert_action_options; |
| 581 | |
| 582 | dst->dimensions = string_dup(src->dimensions); |
| 583 | |
| 584 | dst->time_group = src->time_group; |
| 585 | dst->time_group_condition = src->time_group_condition; |
| 586 | dst->time_group_value = src->time_group_value; |
| 587 | dst->dims_group = src->dims_group; |
| 588 | dst->data_source = src->data_source; |
| 589 | dst->before = src->before; |
| 590 | dst->after = src->after; |
| 591 | dst->options = src->options; |
| 592 | |
| 593 | const char *failed_at = NULL; |
| 594 | int error = 0; |
| 595 | |
| 596 | dst->calculation = expression_parse(expression_source(src->calculation), &failed_at, &error); |
| 597 | dst->warning = expression_parse(expression_source(src->warning), &failed_at, &error); |
| 598 | dst->critical = expression_parse(expression_source(src->critical), &failed_at, &error); |
| 599 | |
| 600 | dst->delay_up_duration = src->delay_up_duration; |
| 601 | dst->delay_down_duration = src->delay_down_duration; |
| 602 | dst->delay_max_duration = src->delay_max_duration; |
| 603 | dst->delay_multiplier = src->delay_multiplier; |
| 604 | |
| 605 | dst->has_custom_repeat_config = src->has_custom_repeat_config; |
| 606 | dst->warn_repeat_every = src->warn_repeat_every; |
| 607 | dst->crit_repeat_every = src->crit_repeat_every; |
| 608 | } |
| 609 | |
| 610 | static void health_prototype_apply_to_rrdset(RRDSET *st, RRD_ALERT_PROTOTYPE *ap) { |
| 611 | if(!ap->_internal.enabled) |
| 612 | return; |
| 613 | |
| 614 | rw_spinlock_read_lock(&ap->_internal.rw_spinlock); |
| 615 | for(size_t template = 0; template < 2; template++) { |
| 616 | bool want_template = template ? true : false; |
| 617 | |
| 618 | for (RRD_ALERT_PROTOTYPE *t = ap; t; t = t->_internal.next) { |
| 619 | if (!t->match.enabled) |
| 620 | continue; |
| 621 | |
| 622 | bool is_template = t->match.is_template ? true : false; |
| 623 | |
| 624 | if (is_template != want_template) |
| 625 | continue; |
| 626 | |
| 627 | if (!prototype_matches_host(st->rrdhost, t)) |
| 628 | continue; |
| 629 | |
| 630 | if (!prototype_matches_rrdset(st, t)) |
| 631 | continue; |
| 632 | |
| 633 | rrdcalc_add_from_prototype(st->rrdhost, st, t); |
| 634 | } |
| 635 | } |
| 636 | rw_spinlock_read_unlock(&ap->_internal.rw_spinlock); |
| 637 | } |
| 638 | |
| 639 | extern __thread bool is_health_thread; |
| 640 | |
| 641 | void health_prototype_alerts_for_rrdset_incrementally(RRDSET *st) { |
| 642 | RRD_ALERT_PROTOTYPE *ap; |
| 643 | dfe_start_read(health_globals.prototypes.dict, ap) { |
| 644 | if (is_health_thread && !service_running(SERVICE_HEALTH)) |
| 645 | break; |
| 646 | health_prototype_apply_to_rrdset(st, ap); |
| 647 | } |
| 648 | dfe_done(ap); |
| 649 | } |
| 650 | |
| 651 | void health_prototype_reset_alerts_for_rrdset(RRDSET *st) { |
| 652 | rrdcalc_unlink_and_delete_all_rrdset_alerts(st); |
| 653 | health_prototype_alerts_for_rrdset_incrementally(st); |
| 654 | } |
| 655 | |
| 656 | // --------------------------------------------------------------------------------------------------------------------- |
| 657 | |
| 658 | void health_apply_prototype_to_host(RRDHOST *host, RRD_ALERT_PROTOTYPE *ap) { |
| 659 | if(!ap->_internal.enabled) |
| 660 | return; |
| 661 | |
| 662 | if(unlikely(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) |
| 663 | return; |
| 664 | |
| 665 | RRDSET *st; |
| 666 | rrdset_foreach_read(st, host) { |
| 667 | health_prototype_apply_to_rrdset(st, ap); |
| 668 | } |
| 669 | rrdset_foreach_done(st); |
| 670 | } |
| 671 | |
| 672 | void health_prototype_apply_to_all_hosts(RRD_ALERT_PROTOTYPE *ap) { |
| 673 | RRDHOST *host; |
| 674 | dfe_start_reentrant(rrdhost_root_index, host){ |
| 675 | if(unlikely(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) |
| 676 | continue; |
| 677 | |
| 678 | RRDCALC *rc; |
| 679 | foreach_rrdcalc_in_rrdhost_reentrant(host, rc) { |
| 680 | if(rc->config.name != ap->config.name) |
| 681 | continue; |
| 682 | |
| 683 | rrdcalc_unlink_and_delete(host, rc, false); |
| 684 | } |
| 685 | foreach_rrdcalc_in_rrdhost_done(rc); |
| 686 | dictionary_garbage_collect(host->rrdcalc_root_index); |
| 687 | |
| 688 | if(ap->_internal.enabled) |
| 689 | health_apply_prototype_to_host(host, ap); |
| 690 | } |
| 691 | dfe_done(host); |
| 692 | } |
| 693 | |
| 694 | // --------------------------------------------------------------------------------------------------------------------- |
| 695 | |
| 696 | void health_apply_prototypes_to_host(RRDHOST *host) { |
| 697 | if(unlikely(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) |
| 698 | return; |
| 699 | |
| 700 | // free all running alarms |
| 701 | rrdcalc_delete_all(host); |
| 702 | |
| 703 | // invalidate all previous entries in the alarm log |
| 704 | rw_spinlock_read_lock(&host->health_log.spinlock); |
| 705 | ALARM_ENTRY *t; |
| 706 | for(t = host->health_log.alarms ; t ; t = t->next) { |
| 707 | if(t->new_status != RRDCALC_STATUS_REMOVED) |
| 708 | t->flags |= HEALTH_ENTRY_FLAG_UPDATED; |
| 709 | } |
| 710 | rw_spinlock_read_unlock(&host->health_log.spinlock); |
| 711 | |
| 712 | // apply all the prototypes for the charts of the host |
| 713 | RRDSET *st; |
| 714 | rrdset_foreach_reentrant(st, host) { |
| 715 | if (is_health_thread && !service_running(SERVICE_HEALTH)) |
| 716 | break; |
| 717 | health_prototype_reset_alerts_for_rrdset(st); |
| 718 | } |
| 719 | rrdset_foreach_done(st); |
| 720 | dictionary_garbage_collect(host->rrdcalc_root_index); |
| 721 | } |
| 722 | |
| 723 | void health_apply_prototypes_to_all_hosts(void) { |
| 724 | RRDHOST *host; |
| 725 | dfe_start_reentrant(rrdhost_root_index, host){ |
| 726 | health_apply_prototypes_to_host(host); |
| 727 | } |
| 728 | dfe_done(host); |
| 729 | } |
| 730 | |
| 731 | // --------------------------------------------------------------------------------------------------------------------- |
| 732 | |
| 733 | void health_prototype_metadata_foreach(void *data, prototype_metadata_cb_t cb) { |
| 734 | RRD_ALERT_PROTOTYPE *ap; |
| 735 | dfe_start_read(health_globals.prototypes.dict, ap) { |
| 736 | cb(data, ap->config.type, ap->config.component, ap->config.classification, ap->config.recipient); |
| 737 | } |
| 738 | dfe_done(ap); |
| 739 | } |