reset health when children disconnect (#17612)
* reset health when children disconnect * cleanup when health is enabled somehow * fix cleanup * fix cleanup * reset health flags on disconnect * cleanup obsolete states * use the host variable * set health variables on every chart update
Costa Tsaousis committed
May 7, 2024 at 23:26 UTC
9628f80713820ff77a6154339f3728e1b1e8401f
7 files changed
+32
-26
src/database/rrd.h
+6
-6
@@ -226,16 +226,16 @@ typedef enum __attribute__ ((__packed__)) rrddim_options {
226
// flags are runtime changing status flags (atomics are required to alter/access them)
227
typedef enum __attribute__ ((__packed__)) rrddim_flags {
228
RRDDIM_FLAG_NONE = 0,
229
- RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 0),
229
231
- RRDDIM_FLAG_OBSOLETE = (1 << 1), // this is marked by the collector/module as obsolete
230
+ RRDDIM_FLAG_OBSOLETE = (1 << 0), // this is marked by the collector/module as obsolete
231
// No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at
232
// least rrdset_free_obsolete_time seconds ago.
234
- RRDDIM_FLAG_ARCHIVED = (1 << 2),
235
- RRDDIM_FLAG_METADATA_UPDATE = (1 << 3), // Metadata needs to go to the database
233
237
- RRDDIM_FLAG_META_HIDDEN = (1 << 4), // Status of hidden option in the metadata database
238
- RRDDIM_FLAG_ML_MODEL_LOAD = (1 << 5), // Do ML LOAD for this dimension
234
+ RRDDIM_FLAG_ARCHIVED = (1 << 1),
235
+ RRDDIM_FLAG_METADATA_UPDATE = (1 << 2), // Metadata needs to go to the database
236
+
237
+ RRDDIM_FLAG_META_HIDDEN = (1 << 3), // Status of hidden option in the metadata database
238
+ RRDDIM_FLAG_ML_MODEL_LOAD = (1 << 4), // Do ML LOAD for this dimension
239
240
// this is 8 bit
241
} RRDDIM_FLAGS;
src/database/rrddim.c
+1
-10
@@ -150,10 +150,6 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
150
}
151
}
152
153
- rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION);
154
- rrdset_flag_set(rd->rrdset, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
155
- rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
156
-
153
// let the chart resync
154
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
155
@@ -259,14 +255,9 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
255
storage_metric_store_init(rd->tiers[tier].seb, rd->tiers[tier].smh, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->smg[tier]);
256
}
257
262
- if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
258
+ if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
259
rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
260
265
- rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_HEALTH_INITIALIZATION);
266
- rrdset_flag_set(rd->rrdset, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
267
- rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
268
- }
269
-
261
if(unlikely(rc))
262
ctr->react_action = RRDDIM_REACT_UPDATED;
263
src/database/rrdset.c
+6
-6
@@ -290,6 +290,9 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
290
291
rrdset_pluginsd_receive_slots_initialize(st);
292
293
+ rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
294
+ rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
295
+
296
ctr->react_action = RRDSET_REACT_NEW;
297
298
ml_chart_new(st);
@@ -465,6 +468,8 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
468
rrdset_update_permanent_labels(st);
469
470
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
471
+ rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
472
+ rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
473
474
return ctr->react_action != RRDSET_REACT_NONE;
475
}
@@ -478,18 +483,13 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
483
484
st->last_accessed_time_s = now_realtime_sec();
485
481
- if(host->health.health_enabled && (ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_CHART_ACTIVATED))) {
482
- rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
483
- rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
484
- }
485
-
486
if(ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
487
if (ctr->react_action & RRDSET_REACT_NEW) {
488
if(unlikely(rrdcontext_find_chart_uuid(st, &st->chart_uuid)))
489
uuid_generate(st->chart_uuid);
490
}
491
rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
492
- rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
492
+ rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_UPDATE);
493
}
494
495
rrdset_metadata_updated(st);
src/health/health_event_loop.c
+1
@@ -287,6 +287,7 @@ static void health_event_loop(void) {
287
if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
288
rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
289
now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
290
+
291
if (!rrdcalc_isrepeating(rc)) {
292
worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
293
time_t now_tmp = now_realtime_sec();
src/health/rrdcalc.c
+11
@@ -463,6 +463,17 @@ void rrdcalc_delete_all(RRDHOST *host) {
463
dictionary_flush(host->rrdcalc_root_index);
464
}
465
466
+void rrdcalc_child_disconnected(RRDHOST *host) {
467
+ rrdcalc_delete_all(host);
468
+
469
+ rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
470
+ RRDSET *st;
471
+ rrdset_foreach_read(st, host) {
472
+ rrdset_flag_clear(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
473
+ }
474
+ rrdset_foreach_done(st);
475
+}
476
+
477
void rrd_alert_match_cleanup(struct rrd_alert_match *am) {
478
if(am->is_template)
479
string_freez(am->on.context);
src/health/rrdcalc.h
+2
@@ -143,4 +143,6 @@ void rrdcalc_unlink_and_delete(RRDHOST *host, RRDCALC *rc, bool having_ll_wrlock
143
#define RRDCALC_VAR_LABEL "${label:"
144
#define RRDCALC_VAR_LABEL_LEN (sizeof(RRDCALC_VAR_LABEL)-1)
145
146
+void rrdcalc_child_disconnected(RRDHOST *host);
147
+
148
#endif //NETDATA_RRDCALC_H
src/streaming/receiver.c
+5
-4
@@ -430,10 +430,9 @@ static bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
430
}
431
432
static void rrdhost_clear_receiver(struct receiver_state *rpt) {
433
- bool signal_rrdcontext = false;
434
-
433
RRDHOST *host = rpt->host;
434
if(host) {
435
+ bool signal_rrdcontext = false;
436
netdata_mutex_lock(&host->receiver_lock);
437
438
// Make sure that we detach this thread and don't kill a freshly arriving receiver
@@ -445,8 +444,7 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
444
host->child_connect_time = 0;
445
host->child_disconnected_time = now_realtime_sec();
446
448
- if (rpt->config.health_enabled == CONFIG_BOOLEAN_AUTO)
449
- host->health.health_enabled = 0;
447
+ host->health.health_enabled = 0;
448
449
rrdpush_sender_thread_stop(host, STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, false);
450
@@ -456,6 +454,9 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
454
rrdhost_flag_set(host, RRDHOST_FLAG_ORPHAN);
455
host->receiver = NULL;
456
host->rrdpush_last_receiver_exit_reason = rpt->exit.reason;
457
+
458
+ if(rpt->config.health_enabled)
459
+ rrdcalc_child_disconnected(host);
460
}
461
462
netdata_mutex_unlock(&host->receiver_lock);