Initialize foreach alarms of dimensions in health thread. (#12452)
The previous approach required us to try wr-lock the host after locking a chart and sleeping on failure. Lock contention would lead to alarms not being created and the agent to become unresponsive.
vkalintiris committed
Mar 31, 2022 at 17:49 UTC
af72ed83f929b312a254b9e8b9b74c7403b689ac
3 files changed
+50
-33
database/rrd.h
+6
-1
@@ -167,7 +167,9 @@ typedef enum rrddim_flags {
167
// No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at
168
// least rrdset_free_obsolete_time seconds ago.
169
RRDDIM_FLAG_ARCHIVED = (1 << 3),
170
- RRDDIM_FLAG_ACLK = (1 << 4)
170
+ RRDDIM_FLAG_ACLK = (1 << 4),
171
+
172
+ RRDDIM_FLAG_PENDING_FOREACH_ALARM = (1 << 5), // set when foreach alarm has not been initialized yet
173
} RRDDIM_FLAGS;
174
175
#ifdef HAVE_C___ATOMIC
@@ -476,6 +478,8 @@ typedef enum rrdset_flags {
478
// least rrdset_free_obsolete_time seconds ago.
479
RRDSET_FLAG_ARCHIVED = 1 << 15,
480
RRDSET_FLAG_ACLK = 1 << 16,
481
+
482
+ RRDSET_FLAG_PENDING_FOREACH_ALARMS = 1 << 17, // contains dims with uninitialized foreach alarms
483
} RRDSET_FLAGS;
484
485
#ifdef HAVE_C___ATOMIC
@@ -634,6 +638,7 @@ typedef enum rrdhost_flags {
638
RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases
639
RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet
640
RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb
641
+ RRDHOST_FLAG_PENDING_FOREACH_ALARMS = 1 << 7, // contains dims with uninitialized foreach alarms
642
} RRDHOST_FLAGS;
643
644
#ifdef HAVE_C___ATOMIC
database/rrddim.c
+7
-32
@@ -3,36 +3,6 @@
3
#define NETDATA_RRD_INTERNALS
4
#include "rrd.h"
5
6
-static inline void calc_link_to_rrddim(RRDDIM *rd)
7
-{
8
- RRDHOST *host = rd->rrdset->rrdhost;
9
- RRDSET *st = rd->rrdset;
10
-
11
- if (st->state && st->state->is_ar_chart)
12
- return;
13
-
14
- if (host->alarms_with_foreach || host->alarms_template_with_foreach) {
15
- int count = 0;
16
- int hostlocked;
17
- for (count = 0; count < 5; count++) {
18
- hostlocked = netdata_rwlock_trywrlock(&host->rrdhost_rwlock);
19
- if (!hostlocked) {
20
- rrdcalc_link_to_rrddim(rd, st, host);
21
- rrdhost_unlock(host);
22
- break;
23
- } else if (hostlocked != EBUSY) {
24
- error("Cannot lock host to create an alarm for the dimension.");
25
- }
26
- sleep_usec(USEC_PER_MS * 200);
27
- }
28
-
29
- if (count == 5) {
30
- error(
31
- "Failed to create an alarm for dimension %s of chart %s 5 times. Skipping alarm.", rd->name, st->name);
32
- }
33
- }
34
-}
35
-
6
// ----------------------------------------------------------------------------
7
// RRDDIM index
8
@@ -248,7 +218,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
218
rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
219
rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT);
220
rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
251
- calc_link_to_rrddim(rd);
221
+
222
+ rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM);
223
+ rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
224
+ rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
225
}
226
if (unlikely(rc)) {
227
debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
@@ -457,7 +430,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
430
if(unlikely(rrddim_index_add(st, rd) != rd))
431
error("RRDDIM: INTERNAL ERROR: attempt to index duplicate dimension '%s' on chart '%s'", rd->id, st->id);
432
460
- calc_link_to_rrddim(rd);
433
+ rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM);
434
+ rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
435
+ rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
436
437
ml_new_dimension(rd);
438
health/health.c
+37
@@ -655,6 +655,41 @@ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) {
655
return 0;
656
}
657
658
+// Create alarms for dimensions that have been added to charts
659
+// since the previous iteration.
660
+static void init_pending_foreach_alarms(RRDHOST *host) {
661
+ rrdhost_wrlock(host);
662
+
663
+ if (host->alarms_with_foreach || host->alarms_template_with_foreach) {
664
+ if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS)) {
665
+ RRDSET *st;
666
+
667
+ rrdset_foreach_read(st, host) {
668
+ rrdset_wrlock(st);
669
+
670
+ if (rrdset_flag_check(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS)) {
671
+ RRDDIM *rd;
672
+
673
+ rrddim_foreach_write(rd, st) {
674
+ if (rrddim_flag_check(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM)) {
675
+ rrdcalc_link_to_rrddim(rd, st, host);
676
+ rrddim_flag_clear(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM);
677
+ }
678
+ }
679
+
680
+ rrdset_flag_clear(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
681
+ }
682
+
683
+ rrdset_unlock(st);
684
+ }
685
+
686
+ rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
687
+ }
688
+ }
689
+
690
+ rrdhost_unlock(host);
691
+}
692
+
693
/**
694
* Health Main
695
*
@@ -739,6 +774,8 @@ void *health_main(void *ptr) {
774
if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0))
775
sql_health_alarm_log_cleanup(host);
776
777
+ init_pending_foreach_alarms(host);
778
+
779
rrdhost_rdlock(host);
780
781
// the first loop is to lookup values from the db