Delay children chart obsoletion check (#12992)
* wait untill after 2 minutes of last chart received to run obsoletion check * turn write to read locks
Emmanuel Vasilakis committed
May 25, 2022 at 11:31 UTC
bf85ce801417edff5fe7e9eb6bfae0caba34b41e
4 files changed
+19
-7
database/rrd.h
+3
-1
@@ -797,11 +797,13 @@ struct rrdhost {
797
volatile size_t connected_senders; // when remote hosts are streaming to this
798
// host, this is the counter of connected clients
799
800
+ time_t senders_connect_time; // the time the last sender was connected
801
+ time_t senders_last_chart_command; // the time of the last CHART streaming command
802
time_t senders_disconnected_time; // the time the last sender was disconnected
803
804
struct receiver_state *receiver;
805
netdata_mutex_t receiver_lock;
804
- time_t trigger_chart_obsoletion_check; // set when child connects, will instruct parent to
806
+ int trigger_chart_obsoletion_check; // set when child connects, will instruct parent to
807
// trigger a check for obsoleted charts since previous connect
808
809
// ------------------------------------------------------------------------
database/rrdhost.c
+7
-4
@@ -1533,8 +1533,10 @@ restart_after_removal:
1533
void rrdset_check_obsoletion(RRDHOST *host)
1534
{
1535
RRDSET *st;
1536
- rrdset_foreach_write(st, host) {
1537
- if (rrdset_last_entry_t(st) < host->trigger_chart_obsoletion_check) {
1536
+ time_t last_entry_t;
1537
+ rrdset_foreach_read(st, host) {
1538
+ last_entry_t = rrdset_last_entry_t(st);
1539
+ if (last_entry_t && last_entry_t < host->senders_connect_time) {
1540
rrdset_is_obsolete(st);
1541
}
1542
}
@@ -1562,8 +1564,9 @@ void rrd_cleanup_obsolete_charts()
1564
1565
if (host != localhost &&
1566
host->trigger_chart_obsoletion_check &&
1565
- host->trigger_chart_obsoletion_check + 120 < now_realtime_sec()) {
1566
- rrdhost_wrlock(host);
1567
+ host->senders_last_chart_command &&
1568
+ host->senders_last_chart_command + 120 < now_realtime_sec()) {
1569
+ rrdhost_rdlock(host);
1570
rrdset_check_obsoletion(host);
1571
rrdhost_unlock(host);
1572
host->trigger_chart_obsoletion_check = 0;
database/rrdset.c
+4
@@ -552,6 +552,10 @@ RRDSET *rrdset_create_custom(
552
return NULL;
553
}
554
555
+ if (host != localhost) {
556
+ host->senders_last_chart_command = now_realtime_sec();
557
+ }
558
+
559
// ------------------------------------------------------------------------
560
// check if it already exists
561
streaming/receiver.c
+5
-2
@@ -671,7 +671,9 @@ static int rrdpush_receive(struct receiver_state *rpt)
671
rpt->host->hostname);
672
}
673
}
674
- rpt->host->trigger_chart_obsoletion_check = now_realtime_sec();
674
+ rpt->host->senders_connect_time = now_realtime_sec();
675
+ rpt->host->senders_last_chart_command = 0;
676
+ rpt->host->trigger_chart_obsoletion_check = 1;
677
rrdhost_unlock(rpt->host);
678
679
// call the plugins.d processor to receive the metrics
@@ -707,12 +709,13 @@ static int rrdpush_receive(struct receiver_state *rpt)
709
rrdhost_wrlock(rpt->host);
710
netdata_mutex_lock(&rpt->host->receiver_lock);
711
if (rpt->host->receiver == rpt) {
712
+ rpt->host->senders_connect_time = 0;
713
+ rpt->host->trigger_chart_obsoletion_check = 0;
714
rpt->host->senders_disconnected_time = now_realtime_sec();
715
rrdhost_flag_set(rpt->host, RRDHOST_FLAG_ORPHAN);
716
if(health_enabled == CONFIG_BOOLEAN_AUTO)
717
rpt->host->health_enabled = 0;
718
}
715
- rpt->host->trigger_chart_obsoletion_check = 0;
719
rrdhost_unlock(rpt->host);
720
if (rpt->host->receiver == rpt) {
721
rrdpush_sender_thread_stop(rpt->host);