Fix data race in ML training during host stop (#21844)
Prevent concurrent ML activity during host reset and improve thread safety in k-means dimension handling
Stelios Fragkakis committed
Mar 2, 2026 at 16:34 UTC
cee7787b2553e25f1ba37933d35a55d82e3e00ed
2 files changed
+13
-4
src/ml/ml.cc
+9
@@ -620,6 +620,13 @@ static void ml_dimension_update_models(ml_worker_t *worker, ml_dimension_t *dim)
620
621
spinlock_lock(&dim->slock);
622
623
+ ml_host_t *host = (ml_host_t *) dim->rd->rrdset->rrdhost->ml_host;
624
+ if (!host || !host->ml_running) {
625
+ dim->training_in_progress = false;
626
+ spinlock_unlock(&dim->slock);
627
+ return;
628
+ }
629
+
630
if (dim->km_contexts.size() < Cfg.num_models_to_use) {
631
dim->km_contexts.emplace_back(dim->kmeans);
632
} else {
@@ -1133,6 +1140,8 @@ static enum ml_worker_result ml_worker_add_existing_model(ml_worker_t *worker, m
1140
}
1141
spinlock_unlock(&Dim->slock);
1142
1143
+ // Safe without Dim->slock: per-host work is serialized through a single worker queue,
1144
+ // and stop/reset no longer writes Dim->kmeans from non-worker threads.
1145
Dim->kmeans = req.inlined_km;
1146
ml_dimension_update_models(worker, Dim);
1147
pulse_ml_models_received();
src/ml/ml_public.cc
+4
-4
@@ -79,6 +79,9 @@ void ml_host_stop(RRDHOST *rh) {
79
if (!host || !host->ml_running)
80
return;
81
82
+ // Prevent new ML activity from publishing while we reset host/dimension state.
83
+ host->ml_running = false;
84
+
85
netdata_mutex_lock(&host->mutex);
86
87
// reset host stats
@@ -112,8 +115,7 @@ void ml_host_stop(RRDHOST *rh) {
115
dim->suppression_anomaly_counter = 0;
116
dim->suppression_window_counter = 0;
117
dim->cns.clear();
115
-
116
- ml_kmeans_init(&dim->kmeans);
118
+ dim->km_contexts.clear();
119
120
spinlock_unlock(&dim->slock);
121
}
@@ -122,8 +124,6 @@ void ml_host_stop(RRDHOST *rh) {
124
rrdset_foreach_done(rsp);
125
126
netdata_mutex_unlock(&host->mutex);
125
-
126
- host->ml_running = false;
127
}
128
129
void ml_host_get_info(RRDHOST *rh, BUFFER *wb)