Remove redundant per-host replication gate from obsolete chart cleanup (#22467)
* refactor: improve obsolete chart cleanup logic for streaming children * refactor: enhance obsolete chart cleanup logic for replicating charts
Stelios Fragkakis committed
May 13, 2026 at 08:55 UTC
e31ede15cd0ca164d04294e36dc973f2a91e01f9
3 files changed
+30
-16
src/daemon/service.c
+24
-11
@@ -110,25 +110,34 @@ static inline size_t svc_rrdhost_cleanup_charts_marked_obsolete(RRDHOST *host) {
110
time_t now = now_realtime_sec();
111
RRDSET *st;
112
rrdset_foreach_reentrant(st, host) {
113
- if(rrdset_is_replicating(st))
114
- continue;
115
-
113
+ bool is_replicating = rrdset_is_replicating(st);
114
RRDSET_FLAGS flags = rrdset_flag_get(st);
115
116
+ // A replicating chart with pending obsolete work must still be
117
+ // counted as a candidate, even though we cannot archive it this
118
+ // pass. The PENDING_OBSOLETE_* host flags are cleared up-front;
119
+ // if we did not count this chart, candidates == archives == 0
120
+ // for it and the host flag would not be re-armed -- the cleanup
121
+ // would never retry once replication finishes. Counting it as a
122
+ // candidate without an archive forces archives != candidates at
123
+ // end-of-loop, which re-arms the host flag for the next pass.
124
+
125
if(flags & RRDSET_FLAG_OBSOLETE_DIMENSIONS) {
126
partial_candidates++;
127
121
- archived_items += svc_rrdset_archive_obsolete_dimensions(st, false);
128
+ if(!is_replicating) {
129
+ archived_items += svc_rrdset_archive_obsolete_dimensions(st, false);
130
123
- // "all candidates archived" -> flag was not re-set inside.
124
- if(!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))
125
- partial_archives++;
131
+ // "all candidates archived" -> flag was not re-set inside.
132
+ if(!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))
133
+ partial_archives++;
134
+ }
135
}
136
137
if(flags & RRDSET_FLAG_OBSOLETE) {
138
full_candidates++;
139
131
- if(svc_rrdset_lock_for_deletion(st, now)) {
140
+ if(!is_replicating && svc_rrdset_lock_for_deletion(st, now)) {
141
archived_items += svc_rrdset_archive_obsolete_dimensions(st, true);
142
143
if(!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) {
@@ -175,9 +184,13 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
184
185
RRDHOST *host;
186
rrdhost_foreach_read(host) {
178
- if(rrdhost_receiver_replicating_charts(host) || rrdhost_sender_replicating_charts(host))
179
- continue;
180
-
187
+ // Per-chart correctness gate is rrdset_is_replicating(st) inside
188
+ // svc_rrdhost_cleanup_charts_marked_obsolete. The previous host-level
189
+ // gate here (rrdhost_*_replicating_charts(host) > 0) was a defensive
190
+ // holdover from before the sender replication counter was accurate;
191
+ // on streaming children with continuous chart churn it permanently
192
+ // tripped and blocked all obsolete-chart cleanup on the host, piling
193
+ // up obsolete-but-still-live charts and their RAM-mode dim mmaps.
194
archived += svc_rrdhost_cleanup_charts_marked_obsolete(host);
195
196
if (rrdhost_is_local(host) || IS_VIRTUAL_HOST_OS(host))
src/database/rrdset.c
+3
-3
@@ -131,9 +131,9 @@ void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) {
131
// "replication finished" decrement at stream-replication-sender.c will
132
// never fire for this chart. Release any pending replication slot now,
133
// otherwise rrdhost_sender_replicating_charts pins above zero and
134
- // permanently blocks svc_rrd_cleanup_obsolete_charts_from_all_hosts.
135
- // Mirror the natural-finalize path's pulse-status flip on the 0/1
136
- // boundary so SND_REPLICATING -> SND_RUNNING is observed by pulse.
134
+ // observability (SND_REPLICATING vs SND_RUNNING) stays stuck on the
135
+ // wrong state. Mirror the natural-finalize path's pulse-status flip on
136
+ // the 0/1 boundary so SND_REPLICATING -> SND_RUNNING is observed.
137
RRDSET_FLAGS old_repl = rrdset_flag_set_and_clear(
138
st,
139
RRDSET_FLAG_SENDER_REPLICATION_FINISHED,
src/streaming/protocol/command-chart-definition.c
+3
-2
@@ -119,8 +119,9 @@ bool stream_sender_send_rrdset_definition(BUFFER *wb, RRDSET *st) {
119
120
// The receiver skips replication for obsolete charts (stream-receiver.c),
121
// so do not enter the replication bookkeeping here either: it would pin
122
- // rrdhost_sender_replicating_charts and permanently gate the cleanup loop
123
- // in svc_rrd_cleanup_obsolete_charts_from_all_hosts.
122
+ // rrdhost_sender_replicating_charts above zero and keep the host's
123
+ // pulse SND_REPLICATING / SND_RUNNING observability stuck on the wrong
124
+ // state.
125
if(!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)) {
126
// Claim before publish: increment the host counter BEFORE setting
127
// RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS, so any concurrent