master
c 526 lines 17.2 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "database/rrd.h"
4 #include "health_internals.h"
5 #include "health-alert-entry.h"
6
7 // ----------------------------------------------------------------------------
8 // RRDCALC helpers
9
10 void rrdcalc_flags_to_json_array(BUFFER *wb, const char *key, RRDCALC_FLAGS flags) {
11 buffer_json_member_add_array(wb, key);
12
13 if(flags & RRDCALC_FLAG_DB_ERROR)
14 buffer_json_add_array_item_string(wb, "DB_ERROR");
15 if(flags & RRDCALC_FLAG_DB_NAN)
16 buffer_json_add_array_item_string(wb, "DB_NAN");
17 if(flags & RRDCALC_FLAG_CALC_ERROR)
18 buffer_json_add_array_item_string(wb, "CALC_ERROR");
19 if(flags & RRDCALC_FLAG_WARN_ERROR)
20 buffer_json_add_array_item_string(wb, "WARN_ERROR");
21 if(flags & RRDCALC_FLAG_CRIT_ERROR)
22 buffer_json_add_array_item_string(wb, "CRIT_ERROR");
23 if(flags & RRDCALC_FLAG_RUNNABLE)
24 buffer_json_add_array_item_string(wb, "RUNNABLE");
25 if(flags & RRDCALC_FLAG_DISABLED)
26 buffer_json_add_array_item_string(wb, "DISABLED");
27 if(flags & RRDCALC_FLAG_SILENCED)
28 buffer_json_add_array_item_string(wb, "SILENCED");
29 if(flags & RRDCALC_FLAG_RUN_ONCE)
30 buffer_json_add_array_item_string(wb, "RUN_ONCE");
31
32 buffer_json_array_close(wb);
33 }
34
35 inline const char *rrdcalc_status2string(RRDCALC_STATUS status) {
36 switch(status) {
37 case RRDCALC_STATUS_REMOVED:
38 return "REMOVED";
39
40 case RRDCALC_STATUS_UNDEFINED:
41 return "UNDEFINED";
42
43 case RRDCALC_STATUS_UNINITIALIZED:
44 return "UNINITIALIZED";
45
46 case RRDCALC_STATUS_CLEAR:
47 return "CLEAR";
48
49 case RRDCALC_STATUS_RAISED:
50 return "RAISED";
51
52 case RRDCALC_STATUS_WARNING:
53 return "WARNING";
54
55 case RRDCALC_STATUS_CRITICAL:
56 return "CRITICAL";
57
58 default:
59 netdata_log_error("Unknown alarm status %d", status);
60 return "UNKNOWN";
61 }
62 }
63
64 uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, nd_uuid_t *config_hash_id) {
65 rw_spinlock_read_lock(&host->health_log.spinlock);
66
67 // re-use old IDs, by looking them up in the alarm log
68 ALARM_ENTRY *ae = NULL;
69 for(ae = host->health_log.alarms; ae ;ae = ae->next) {
70 if(unlikely(name == ae->name && chart == ae->chart && uuid_eq(ae->config_hash_id, *config_hash_id))) {
71 if(next_event_id) *next_event_id = ae->alarm_event_id + 1;
72 break;
73 }
74 }
75
76 uint32_t alarm_id;
77
78 if(ae)
79 alarm_id = ae->alarm_id;
80 else {
81 alarm_id = sql_get_alarm_id(host, chart, name, next_event_id);
82 if (!alarm_id) {
83 if (unlikely(!host->health_log.next_alarm_id))
84 host->health_log.next_alarm_id = get_uint32_id();
85 alarm_id = host->health_log.next_alarm_id++;
86 }
87 }
88
89 rw_spinlock_read_unlock(&host->health_log.spinlock);
90 return alarm_id;
91 }
92
93 // ----------------------------------------------------------------------------
94 // RRDCALC replacing info/summary text variables with RRDSET labels
95
96 static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RRDCALC *rc) {
97 if (!line || !*line)
98 return NULL;
99
100 size_t pos = 0;
101 char *temp = strdupz(line);
102 char var[RRDCALC_VAR_MAX];
103 char *m, *lbl_value = NULL;
104
105 while ((m = strchr(temp + pos, '$')) && *(m+1) == '{') {
106 int i = 0;
107 char *e = m;
108 while (*e) {
109 var[i++] = *e;
110
111 if (*e == '}' || i == RRDCALC_VAR_MAX - 1)
112 break;
113
114 e++;
115 }
116
117 var[i] = '\0';
118 pos = m - temp + 1;
119
120 if (!strcmp(var, RRDCALC_VAR_FAMILY)) {
121 char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rrdset_family(rc->rrdset) : "", m);
122 freez(temp);
123 temp = buf;
124 }
125 else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN) &&
126 i > (int)RRDCALC_VAR_LABEL_LEN && var[i - 1] == '}') {
127 char label_val[RRDCALC_VAR_MAX + RRDCALC_VAR_LABEL_LEN + 1] = { 0 };
128 strcpy(label_val, var+RRDCALC_VAR_LABEL_LEN);
129 label_val[i - RRDCALC_VAR_LABEL_LEN - 1] = '\0';
130
131 if(likely(rc->rrdset && rc->rrdset->rrdlabels)) {
132 lbl_value = NULL;
133 rrdlabels_get_value_strdup_or_null(rc->rrdset->rrdlabels, &lbl_value, label_val);
134 if (lbl_value) {
135 char *buf = find_and_replace(temp, var, lbl_value, m);
136 freez(temp);
137 temp = buf;
138 freez(lbl_value);
139 }
140 }
141 }
142 }
143
144 STRING *ret = string_strdupz(temp);
145 freez(temp);
146
147 return ret;
148 }
149
150 void rrdcalc_update_info_using_rrdset_labels(RRDCALC *rc) {
151 if(rc->rrdset && rc->rrdset->rrdlabels) {
152 uint32_t labels_version = rrdlabels_version(rc->rrdset->rrdlabels);
153 if (rc->labels_version != labels_version) {
154 STRING *old;
155
156 old = rc->info;
157 rc->info = rrdcalc_replace_variables_with_rrdset_labels(string2str(rc->config.info), rc);
158 string_freez(old);
159
160 old = rc->summary;
161 rc->summary = rrdcalc_replace_variables_with_rrdset_labels(string2str(rc->config.summary), rc);
162 string_freez(old);
163
164 rc->labels_version = labels_version;
165 }
166 }
167
168 if(!rc->summary)
169 rc->summary = string_dup(rc->config.summary);
170
171 if(!rc->info)
172 rc->info = string_dup(rc->config.info);
173 }
174
175 // ----------------------------------------------------------------------------
176 // RRDCALC index management for RRDSET
177
178 // the dictionary requires a unique key for every item
179 // we use {chart id}.{alert name} for both the RRDHOST and RRDSET alert indexes.
180
181 #define RRDCALC_MAX_KEY_SIZE 1024
182 static size_t rrdcalc_key(char *dst, size_t dst_len, const char *chart, const char *alert) {
183 return snprintfz(dst, dst_len, "%s,on[%s]", alert, chart);
184 }
185
186 const RRDCALC_ACQUIRED *rrdcalc_from_rrdset_get(RRDSET *st, const char *alert_name) {
187 char key[RRDCALC_MAX_KEY_SIZE + 1];
188 size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_id(st), alert_name);
189
190 const RRDCALC_ACQUIRED *rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)key_len);
191
192 if(!rca) {
193 key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_name(st), alert_name);
194 rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)key_len);
195 }
196
197 return rca;
198 }
199
200 void rrdcalc_from_rrdset_release(RRDSET *st, const RRDCALC_ACQUIRED *rca) {
201 if(!rca) return;
202
203 dictionary_acquired_item_release(st->rrdhost->rrdcalc_root_index, (const DICTIONARY_ITEM *)rca);
204 }
205
206 RRDCALC *rrdcalc_acquired_to_rrdcalc(const RRDCALC_ACQUIRED *rca) {
207 if(rca)
208 return dictionary_acquired_item_value((const DICTIONARY_ITEM *)rca);
209
210 return NULL;
211 }
212
213 // ----------------------------------------------------------------------------
214 // RRDCALC managing the linking with RRDSET
215
216 static void rrdcalc_link_to_rrdset(RRDCALC *rc) {
217 RRDSET *st = rc->rrdset;
218 RRDHOST *host = st->rrdhost;
219
220 rw_spinlock_write_lock(&st->alerts.spinlock);
221 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(st->alerts.base, rc, prev, next);
222 rw_spinlock_write_unlock(&st->alerts.spinlock);
223
224 char buf[RRDVAR_MAX_LENGTH + 1];
225 snprintfz(buf, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc));
226 STRING *rrdset_name_rrdcalc_name = string_strdupz(buf);
227 snprintfz(buf, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc));
228 STRING *rrdset_id_rrdcalc_name = string_strdupz(buf);
229
230 string_freez(rrdset_id_rrdcalc_name);
231 string_freez(rrdset_name_rrdcalc_name);
232
233 time_t now = now_realtime_sec();
234 ALARM_ENTRY *ae = health_create_alarm_entry(
235 host,
236 rc,
237 now,
238 now - rc->last_status_change,
239 rc->old_value,
240 rc->value,
241 RRDCALC_STATUS_REMOVED,
242 rc->status,
243 0,
244 rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
245
246 health_log_alert(host, ae);
247 health_alarm_log_add_entry(host, ae, true);
248 rrdset_flag_set(st, RRDSET_FLAG_HAS_RRDCALC_LINKED);
249 }
250
251 static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) {
252 RRDSET *st = rc->rrdset;
253 if(!st) return;
254
255 if (!exit_initiated_get()) {
256 time_t now = now_realtime_sec();
257
258 if (likely(rc->status != RRDCALC_STATUS_REMOVED)) {
259 RRDHOST *host = st->rrdhost;
260 ALARM_ENTRY *ae = health_create_alarm_entry(
261 host,
262 rc,
263 now,
264 now - rc->last_status_change,
265 rc->old_value,
266 rc->value,
267 rc->status,
268 RRDCALC_STATUS_REMOVED,
269 0,
270 0);
271
272 health_log_alert(host, ae);
273 health_alarm_log_add_entry(host, ae, true);
274 }
275 }
276
277 if(!having_ll_wrlock)
278 rw_spinlock_write_lock(&st->alerts.spinlock);
279
280 if(rc->prev)
281 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(st->alerts.base, rc, prev, next);
282
283 rc->rrdset = NULL;
284
285 if(!having_ll_wrlock)
286 rw_spinlock_write_unlock(&st->alerts.spinlock);
287 }
288
289 // ----------------------------------------------------------------------------
290 // RRDCALC rrdhost index management - constructor
291
292 struct rrdcalc_constructor {
293 RRDSET *rrdset;
294 RRD_ALERT_PROTOTYPE *ap;
295
296 enum {
297 RRDCALC_REACT_NONE,
298 RRDCALC_REACT_NEW,
299 } react_action;
300 };
301
302 static void rrdcalc_rrdhost_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *constructor_data) {
303 RRDCALC *rc = rrdcalc;
304 struct rrdcalc_constructor *ctr = constructor_data;
305 RRDSET *st = ctr->rrdset;
306 RRDHOST *host = st->rrdhost;
307 RRD_ALERT_PROTOTYPE *ap = ctr->ap;
308
309 rc->key = string_strdupz(dictionary_acquired_item_name(item));
310 rc->rrdset = st;
311 rc->chart = string_dup(st->id);
312
313 health_prototype_copy_config(&rc->config, &ap->config);
314
315 rc->next_event_id = 1;
316 rc->value = NAN;
317 rc->old_value = NAN;
318 rc->last_repeat = 0;
319 rc->times_repeat = 0;
320 rc->last_status_change_value = rc->value;
321 rc->last_status_change = now_realtime_sec();
322
323 if(!rc->config.units)
324 rc->config.units = string_dup(st->units);
325
326 // the following interferes with replication, changing the alert frequency to unexpected values
327 // let's respect user configuration, so we disable it
328
329 // if(rc->config.update_every < rc->rrdset->update_every) {
330 // netdata_log_info(
331 // "HEALTH: alert '%s.%s' has update every %d, less than chart update every %d. "
332 // "Setting alarm update frequency to %d.",
333 // string2str(st->id), string2str(rc->config.name),
334 // rc->config.update_every, rc->rrdset->update_every, rc->rrdset->update_every);
335 //
336 // rc->config.update_every = st->update_every;
337 // }
338
339 rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->config.name, &rc->next_event_id, &rc->config.hash_id);
340
341 expression_set_variable_lookup_callback(rc->config.calculation, alert_variable_lookup, rc);
342 expression_set_variable_lookup_callback(rc->config.warning, alert_variable_lookup, rc);
343 expression_set_variable_lookup_callback(rc->config.critical, alert_variable_lookup, rc);
344
345 rrdcalc_update_info_using_rrdset_labels(rc);
346
347 ctr->react_action = RRDCALC_REACT_NEW;
348 }
349
350 static bool rrdcalc_rrdhost_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc __maybe_unused, void *rrdcalc_new __maybe_unused, void *constructor_data) {
351 struct rrdcalc_constructor *ctr = constructor_data;
352 ctr->react_action = RRDCALC_REACT_NONE;
353 return false;
354 }
355
356 static void rrdcalc_rrdhost_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *constructor_data) {
357 RRDCALC *rc = rrdcalc;
358 struct rrdcalc_constructor *ctr = constructor_data;
359
360 if(ctr->react_action == RRDCALC_REACT_NEW)
361 rrdcalc_link_to_rrdset(rc);
362 }
363
364 // ----------------------------------------------------------------------------
365 // RRDCALC rrdhost index management - destructor
366
367 static __thread bool thread_having_ll_wrlock = false;
368
369 static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdhost __maybe_unused) {
370 RRDCALC *rc = rrdcalc;
371 //RRDHOST *host = rrdhost;
372
373 rrdcalc_unlink_from_rrdset(rc, thread_having_ll_wrlock);
374
375 // any destruction actions that require other locks
376 // have to be placed in rrdcalc_del(), because the object is actually locked for deletion
377
378 rrd_alert_config_cleanup(&rc->config);
379
380 string_freez(rc->key);
381 string_freez(rc->chart);
382
383 string_freez(rc->info);
384 string_freez(rc->summary);
385
386 memset(rc, 0, sizeof(*rc));
387 }
388
389 // ----------------------------------------------------------------------------
390 // RRDCALC rrdhost index management - index API
391
392 void rrdcalc_rrdhost_index_init(RRDHOST *host) {
393 if(!host->rrdcalc_root_index) {
394 host->rrdcalc_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
395 &dictionary_stats_category_rrdhealth, sizeof(RRDCALC));
396
397 dictionary_register_insert_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_insert_callback, NULL);
398 dictionary_register_conflict_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_conflict_callback, NULL);
399 dictionary_register_react_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_react_callback, NULL);
400 dictionary_register_delete_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_delete_callback, host);
401 }
402 }
403
404 void rrdcalc_rrdhost_index_destroy(RRDHOST *host) {
405 rrdcalc_delete_all(host);
406 dictionary_destroy(host->rrdcalc_root_index);
407 host->rrdcalc_root_index = NULL;
408 }
409
410 bool rrdcalc_add_from_prototype(RRDHOST *host, RRDSET *st, RRD_ALERT_PROTOTYPE *ap) {
411 char key[RRDCALC_MAX_KEY_SIZE + 1];
412 size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE,
413 string2str(st->id), string2str(ap->config.name));
414
415 struct rrdcalc_constructor tmp = {
416 .ap = ap,
417 .rrdset = st,
418 .react_action = RRDCALC_REACT_NONE,
419 };
420
421 bool ret = true;
422
423 dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)key_len,
424 NULL, sizeof(RRDCALC), &tmp);
425
426 if(tmp.react_action != RRDCALC_REACT_NEW)
427 ret = false;
428
429 return ret;
430 }
431
432 void rrdcalc_unlink_and_delete(RRDHOST *host, RRDCALC *rc, bool having_ll_wrlock) {
433 rrdcalc_unlink_from_rrdset(rc, having_ll_wrlock);
434
435 thread_having_ll_wrlock = having_ll_wrlock;
436 dictionary_del_advanced(host->rrdcalc_root_index, string2str(rc->key), (ssize_t)string_strlen(rc->key));
437 thread_having_ll_wrlock = false;
438 }
439
440 // ----------------------------------------------------------------------------
441 // RRDCALC cleanup API functions
442
443 void rrdcalc_unlink_and_delete_all_rrdset_alerts(RRDSET *st) {
444 RRDHOST *host = st->rrdhost;
445 RRDCALC *rc, *last = NULL;
446
447 // Acquire the host alert dictionary write lock to exclude the health
448 // thread (which mutates rc fields under foreach_rrdcalc_in_rrdhost_read),
449 // then walk only this chart's alert list to keep the work O(chart-alerts).
450 // The lock is recursive, so dictionary_del_advanced() inside the loop
451 // re-enters safely.
452 dictionary_write_lock(host->rrdcalc_root_index);
453 rw_spinlock_write_lock(&st->alerts.spinlock);
454
455 while((rc = st->alerts.base)) {
456 if(last == rc) {
457 netdata_log_error("RRDCALC: malformed list of alerts linked to chart - cannot cleanup - giving up.");
458 break;
459 }
460 last = rc;
461
462 rrdcalc_unlink_and_delete(host, rc, true);
463 }
464
465 rw_spinlock_write_unlock(&st->alerts.spinlock);
466 dictionary_write_unlock(host->rrdcalc_root_index);
467 }
468
469 void rrdcalc_delete_all(RRDHOST *host) {
470 RRDCALC *rc;
471 foreach_rrdcalc_in_rrdhost_write(host, rc) {
472 rrdcalc_unlink_and_delete(host, rc, false);
473 }
474 foreach_rrdcalc_in_rrdhost_done(rc);
475 dictionary_garbage_collect(host->rrdcalc_root_index);
476 }
477
478 void rrdcalc_child_disconnected(RRDHOST *host) {
479 rrdcalc_delete_all(host);
480
481 rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
482 RRDSET *st;
483 rrdset_foreach_read(st, host) {
484 rrdset_flag_clear(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
485 }
486 rrdset_foreach_done(st);
487 }
488
489 void rrd_alert_match_cleanup(struct rrd_alert_match *am) {
490 if(am->is_template)
491 string_freez(am->on.context);
492 else
493 string_freez(am->on.chart);
494
495 string_freez(am->host_labels);
496 pattern_array_free(am->host_labels_pattern);
497
498 string_freez(am->chart_labels);
499 pattern_array_free(am->chart_labels_pattern);
500
501 memset(am, 0, sizeof(*am));
502 }
503
504 void rrd_alert_config_cleanup(struct rrd_alert_config *ac) {
505 string_freez(ac->name);
506
507 string_freez(ac->exec);
508 string_freez(ac->recipient);
509
510 string_freez(ac->classification);
511 string_freez(ac->component);
512 string_freez(ac->type);
513
514 string_freez(ac->source);
515 string_freez(ac->units);
516 string_freez(ac->summary);
517 string_freez(ac->info);
518
519 string_freez(ac->dimensions);
520
521 expression_free(ac->calculation);
522 expression_free(ac->warning);
523 expression_free(ac->critical);
524
525 memset(ac, 0, sizeof(*ac));
526 }