Improve ML shutdown checks (#21250)
* Proper check before rrdhost_acquired_release * Ensure safe deletion of ML dimensions by waiting for in-progress training to complete
Stelios Fragkakis committed
Nov 2, 2025 at 21:58 UTC
a5ea333c24129d3e8d12abe5a94886e5a23eb4f9
2 files changed
+23
-1
src/ml/ml_dimension.h
+1
-1
@@ -151,7 +151,7 @@ public:
151
if (AcqRS)
152
rrdset_acquired_release(AcqRS);
153
154
- if (AcqRD)
154
+ if (AcqRH)
155
rrdhost_acquired_release(AcqRH);
156
}
157
src/ml/ml_public.cc
+22
@@ -315,6 +315,28 @@ void ml_dimension_delete(RRDDIM *rd)
315
if (!dim)
316
return;
317
318
+ // Wait for any in-progress training to complete before deleting
319
+ // This prevents use-after-free crashes when training thread accesses dim->rd
320
+ size_t wait_iterations = 0;
321
+ const size_t max_wait_iterations = 3000; // 30 seconds max (3000 * 10ms)
322
+
323
+ spinlock_lock(&dim->slock);
324
+ while (dim->training_in_progress && wait_iterations < max_wait_iterations) {
325
+ spinlock_unlock(&dim->slock);
326
+ sleep_usec(10000); // Wait 10ms
327
+ wait_iterations++;
328
+ spinlock_lock(&dim->slock);
329
+ }
330
+
331
+ if (dim->training_in_progress) {
332
+ // Training is stuck, but we can't wait forever
333
+ // Log the issue but proceed with deletion
334
+ netdata_log_error("ML: Dimension '%s' of chart '%s' is being deleted while training is in progress after waiting %zu ms",
335
+ rrddim_id(rd), rrdset_id(rd->rrdset), wait_iterations * 10);
336
+ }
337
+
338
+ spinlock_unlock(&dim->slock);
339
+
340
delete dim;
341
rd->ml_dimension = NULL;
342
}