Prevent host status stalls during obsolete-chart cleanup (#22539)
* Gate reconnecting receivers during obsolete chart cleanup to prevent race conditions * Add result-based return value for `rrdhost_set_receiver` and handle cleanup scenarios * Update `rrdhost_set_receiver` to return a status for cleanup contention and refine related comments
Stelios Fragkakis committed
May 25, 2026 at 09:21 UTC
ff099b324be8529299800c8691ed6f717b7159e8
5 files changed
+62
-6
src/daemon/service.c
+15
-2
@@ -196,17 +196,30 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
196
if (rrdhost_is_local(host) || IS_VIRTUAL_HOST_OS(host))
197
continue;
198
199
+ // Two-phase obsolete-all: decide under receiver_lock (short held),
200
+ // run the O(charts) walk + ml_host_disconnected without the lock,
201
+ // gated by RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS so a reconnecting
202
+ // receiver bails out in rrdhost_set_receiver() instead of overlapping.
203
+ bool obsolete_all = false;
204
+
205
rrdhost_receiver_lock(host);
206
207
time_t now = now_realtime_sec();
208
209
if (!host->receiver &&
210
host->stream.rcv.status.last_connected == 0 &&
205
- (host->stream.rcv.status.last_disconnected + rrdset_free_obsolete_time_s < now)) {
206
- svc_rrdhost_obsolete_all_charts(host);
211
+ (host->stream.rcv.status.last_disconnected + rrdset_free_obsolete_time_s < now) &&
212
+ !rrdhost_flag_check(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS)) {
213
+ rrdhost_flag_set(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS);
214
+ obsolete_all = true;
215
}
216
217
rrdhost_receiver_unlock(host);
218
+
219
+ if (obsolete_all) {
220
+ svc_rrdhost_obsolete_all_charts(host);
221
+ rrdhost_flag_clear(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS);
222
+ }
223
}
224
225
rrd_rdunlock();
src/database/rrdhost.h
+17
@@ -96,6 +96,11 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {
96
97
RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 28), // set when the host has updated global functions
98
RRDHOST_FLAG_RRDCONTEXT_GET_RETENTION = (1 << 29), // set when rrdcontext needs to update the retention of the host
99
+
100
+ RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS = (1 << 30), // svc_rrdhost_obsolete_all_charts() is running on this host;
101
+ // gates rrdhost_set_receiver() so a reconnect cannot attach
102
+ // mid-pass. Set under receiver_lock; the heavy work runs
103
+ // without holding receiver_lock so readers stay unblocked.
104
} RRDHOST_FLAGS;
105
106
#define rrdhost_flag_get(host) atomic_flags_get(&((host)->flags))
@@ -341,6 +346,18 @@ struct rrdhost {
346
347
extern RRDHOST *localhost;
348
349
+// receiver_lock protects host->receiver and the host->stream.rcv.status fields.
350
+//
351
+// Hold time must stay short-bounded: no caller may hold receiver_lock across
352
+// O(charts), I/O, sends, ML stop/start, or any wait on other subsystems. The
353
+// obsolete-all cleanup pass (service.c) used to violate this; it now sets
354
+// RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS under the lock and runs the heavy
355
+// work without holding it. rrdhost_set_receiver() checks that flag under
356
+// the lock and returns RRDHOST_SET_RECEIVER_CLEANUP_BUSY when the cleanup
357
+// pass is in progress; the caller answers the child with BUSY_TRY_LATER and
358
+// the child reconnects via normal backoff. With cleanup off the lock, all
359
+// other readers (status, ACLK, capabilities, paths, event-driven sends)
360
+// keep blocking-lock semantics and stay truthful.
361
#define rrdhost_receiver_lock(host) spinlock_lock(&(host)->receiver_lock)
362
#define rrdhost_receiver_unlock(host) spinlock_unlock(&(host)->receiver_lock)
363
src/streaming/stream-receiver-connection.c
+11
-1
@@ -229,7 +229,17 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
229
// return false;
230
// }
231
232
- if(!rrdhost_set_receiver(host, rpt)) {
232
+ RRDHOST_SET_RECEIVER_RESULT result = rrdhost_set_receiver(host, rpt);
233
+ if (result == RRDHOST_SET_RECEIVER_CLEANUP_BUSY) {
234
+ stream_receiver_log_status(
235
+ rpt,
236
+ "rejecting streaming connection; internal cleanup is in progress for this node, please retry shortly",
237
+ STREAM_HANDSHAKE_PARENT_BUSY_TRY_LATER, NDLP_INFO);
238
+
239
+ stream_send_error_on_taken_over_connection(rpt, START_STREAMING_ERROR_BUSY_TRY_LATER);
240
+ return false;
241
+ }
242
+ if (result == RRDHOST_SET_RECEIVER_ALREADY_ATTACHED) {
243
stream_receiver_log_status(
244
rpt,
245
"rejecting streaming connection; host is already served by another receiver",
src/streaming/stream-receiver-internals.h
+7
-1
@@ -101,7 +101,13 @@ struct receiver_state {
101
#endif
102
};
103
104
-bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt);
104
+typedef enum {
105
+ RRDHOST_SET_RECEIVER_OK, // attached
106
+ RRDHOST_SET_RECEIVER_ALREADY_ATTACHED, // another receiver already attached
107
+ RRDHOST_SET_RECEIVER_CLEANUP_BUSY, // obsolete-all cleanup is running; caller should answer BUSY_TRY_LATER
108
+} RRDHOST_SET_RECEIVER_RESULT;
109
+
110
+RRDHOST_SET_RECEIVER_RESULT rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt);
111
void rrdhost_clear_receiver(struct receiver_state *rpt, STREAM_HANDSHAKE reason);
112
void stream_receiver_log_status(struct receiver_state *rpt, const char *msg, STREAM_HANDSHAKE reason, ND_LOG_FIELD_PRIORITY priority);
113
src/streaming/stream-receiver.c
+12
-2
@@ -1128,12 +1128,22 @@ static void stream_receiver_replication_reset(RRDHOST *host) {
1128
__atomic_store_n(&host->stream.rcv.status.replication.backfill_pending, 0, __ATOMIC_RELAXED);
1129
}
1130
1131
-bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
1131
+RRDHOST_SET_RECEIVER_RESULT rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
1132
bool signal_rrdcontext = false;
1133
bool set_this = false;
1134
1135
rrdhost_receiver_lock(host);
1136
1137
+ // If the obsolete-all cleanup is running on this host, refuse the attach.
1138
+ // The cleanup walks all charts marking them obsolete without holding
1139
+ // receiver_lock; attaching mid-pass would let it mark the new receiver's
1140
+ // charts obsolete and call ml_host_disconnected() on a connected host.
1141
+ // The child reconnects via normal backoff; by then the pass is done.
1142
+ if (rrdhost_flag_check(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS)) {
1143
+ rrdhost_receiver_unlock(host);
1144
+ return RRDHOST_SET_RECEIVER_CLEANUP_BUSY;
1145
+ }
1146
+
1147
if (!host->receiver) {
1148
object_state_activate_if_not_activated(&host->state_id);
1149
@@ -1188,7 +1198,7 @@ bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
1198
if(set_this)
1199
ml_host_start(host);
1200
1191
- return set_this;
1201
+ return set_this ? RRDHOST_SET_RECEIVER_OK : RRDHOST_SET_RECEIVER_ALREADY_ATTACHED;
1202
}
1203
1204
void rrdhost_clear_receiver(struct receiver_state *rpt, STREAM_HANDSHAKE reason) {