remove deadlock from dyncfg health (#19890)
* remove deadlock from dyncfg health * do not apply dyncfg prototypes to hosts that are not yet loaded * Add entries with async=true when linking rrdcalc to rrdset --------- Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Costa Tsaousis committed
Mar 17, 2025 at 19:48 UTC
aabcfb8e6be8ee7a88970993be2f5ef8921bcd23
4 files changed
+28
-13
src/health/health_dyncfg.c
+5
-2
@@ -526,8 +526,11 @@ static size_t dyncfg_health_remove_all_rrdcalc_of_prototype(STRING *alert_name)
526
527
RRDHOST *host;
528
dfe_start_reentrant(rrdhost_root_index, host) {
529
+ if(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
530
+ continue;
531
+
532
RRDCALC *rc;
530
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
533
+ foreach_rrdcalc_in_rrdhost_reentrant(host, rc) {
534
if(rc->config.name != alert_name)
535
continue;
536
@@ -535,6 +538,7 @@ static size_t dyncfg_health_remove_all_rrdcalc_of_prototype(STRING *alert_name)
538
removed++;
539
}
540
foreach_rrdcalc_in_rrdhost_done(rc);
541
+ dictionary_garbage_collect(host->rrdcalc_root_index);
542
}
543
dfe_done(host);
544
@@ -542,7 +546,6 @@ static size_t dyncfg_health_remove_all_rrdcalc_of_prototype(STRING *alert_name)
546
}
547
548
static void dyncfg_health_prototype_reapply(RRD_ALERT_PROTOTYPE *ap) {
545
- dyncfg_health_remove_all_rrdcalc_of_prototype(ap->config.name);
549
health_prototype_apply_to_all_hosts(ap);
550
}
551
src/health/health_event_loop.c
+3
-3
@@ -150,8 +150,6 @@ static void health_initialize_rrdhost(RRDHOST *host) {
150
!service_running(SERVICE_HEALTH))
151
return;
152
153
- rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);
154
-
153
host->health_log.max = health_globals.config.health_log_entries_max;
154
host->health_log.health_log_retention_s = health_globals.config.health_log_retention_s;
155
host->health.default_exec = string_dup(health_globals.config.default_exec);
@@ -161,8 +159,10 @@ static void health_initialize_rrdhost(RRDHOST *host) {
159
host->health_log.next_log_id = get_uint32_id();
160
host->health_log.next_alarm_id = 0;
161
164
- rw_spinlock_init(&host->health_log.spinlock);
162
sql_health_alarm_log_load(host);
163
+ rw_spinlock_init(&host->health_log.spinlock);
164
+ rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);
165
+
166
health_apply_prototypes_to_host(host);
167
}
168
src/health/health_prototypes.c
+19
-7
@@ -641,7 +641,7 @@ void health_apply_prototype_to_host(RRDHOST *host, RRD_ALERT_PROTOTYPE *ap) {
641
if(!ap->_internal.enabled)
642
return;
643
644
- if(unlikely(!host->health.enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
644
+ if(unlikely(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
645
return;
646
647
RRDSET *st;
@@ -652,12 +652,23 @@ void health_apply_prototype_to_host(RRDHOST *host, RRD_ALERT_PROTOTYPE *ap) {
652
}
653
654
void health_prototype_apply_to_all_hosts(RRD_ALERT_PROTOTYPE *ap) {
655
- if(!ap->_internal.enabled)
656
- return;
657
-
655
RRDHOST *host;
656
dfe_start_reentrant(rrdhost_root_index, host){
660
- health_apply_prototype_to_host(host, ap);
657
+ if(unlikely(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
658
+ continue;
659
+
660
+ RRDCALC *rc;
661
+ foreach_rrdcalc_in_rrdhost_reentrant(host, rc) {
662
+ if(rc->config.name != ap->config.name)
663
+ continue;
664
+
665
+ rrdcalc_unlink_and_delete(host, rc, false);
666
+ }
667
+ foreach_rrdcalc_in_rrdhost_done(rc);
668
+ dictionary_garbage_collect(host->rrdcalc_root_index);
669
+
670
+ if(ap->_internal.enabled)
671
+ health_apply_prototype_to_host(host, ap);
672
}
673
dfe_done(host);
674
}
@@ -665,7 +676,7 @@ void health_prototype_apply_to_all_hosts(RRD_ALERT_PROTOTYPE *ap) {
676
// ---------------------------------------------------------------------------------------------------------------------
677
678
void health_apply_prototypes_to_host(RRDHOST *host) {
668
- if(unlikely(!host->health.enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
679
+ if(unlikely(!host->health.enabled || !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)))
680
return;
681
682
// free all running alarms
@@ -682,10 +693,11 @@ void health_apply_prototypes_to_host(RRDHOST *host) {
693
694
// apply all the prototypes for the charts of the host
695
RRDSET *st;
685
- rrdset_foreach_read(st, host) {
696
+ rrdset_foreach_reentrant(st, host) {
697
health_prototype_reset_alerts_for_rrdset(st);
698
}
699
rrdset_foreach_done(st);
700
+ dictionary_garbage_collect(host->rrdcalc_root_index);
701
}
702
703
void health_apply_prototypes_to_all_hosts(void) {
src/health/rrdcalc.c
+1
-1
@@ -243,7 +243,7 @@ static void rrdcalc_link_to_rrdset(RRDCALC *rc) {
243
rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
244
245
health_log_alert(host, ae);
246
- health_alarm_log_add_entry(host, ae, false);
246
+ health_alarm_log_add_entry(host, ae, true);
247
rrdset_flag_set(st, RRDSET_FLAG_HAS_RRDCALC_LINKED);
248
}
249