@cryptotaxi247 / netdata-1 / commits / 4c33144f9

Fix ML concurrent dimension training (#21183)

Remove unused fields Add training in progress per dimension

Stelios Fragkakis committed Oct 22, 2025 at 22:47 UTC 4c33144f969b688f83ca199d0906ac513d576e85

6 files changed +50 -29

src/ml/ml.cc

+23 -3

@@ -643,8 +643,6 @@ static void ml_dimension_update_models(ml_worker_t *worker, ml_dimension_t *dim)

643 dim->suppression_anomaly_counter = 0;

644 dim->suppression_window_counter = 0;

645

646 - dim->last_training_time = rrddim_last_entry_s(dim->rd);

647 -

646 // Add the newly generated model to the list of pending models to flush

647 ml_model_info_t model_info;

648 nd_uuid_t *rd_uuid = uuidmap_uuid_ptr(dim->rd->uuid);

@@ -654,6 +652,9 @@ static void ml_dimension_update_models(ml_worker_t *worker, ml_dimension_t *dim)

652

653 ml_dimension_stream_kmeans(dim);

654

655 + // Clear the training in progress flag

656 + dim->training_in_progress = false;

657 +

658 spinlock_unlock(&dim->slock);

659 }

660

@@ -667,6 +668,16 @@ ml_dimension_train_model(ml_worker_t *worker, ml_dimension_t *dim)

668 spinlock_unlock(&dim->slock);

669 return ML_WORKER_RESULT_OK;

670 }

671 +

672 + // Check if training is already in progress for this dimension

673 + // If so, skip this training request to prevent concurrent access to dim->kmeans

674 + if (dim->training_in_progress) {

675 + spinlock_unlock(&dim->slock);

676 + return ML_WORKER_RESULT_OK;

677 + }

678 +

679 + // Mark training as in progress

680 + dim->training_in_progress = true;

681 spinlock_unlock(&dim->slock);

682

683 auto P = ml_dimension_calculated_numbers(worker, dim);

@@ -679,7 +690,7 @@ ml_dimension_train_model(ml_worker_t *worker, ml_dimension_t *dim)

690 dim->mt = METRIC_TYPE_CONSTANT;

691 dim->suppression_anomaly_counter = 0;

692 dim->suppression_window_counter = 0;

682 - dim->last_training_time = training_response.last_entry_on_response;

693 + dim->training_in_progress = false;

694

695 spinlock_unlock(&dim->slock);

696

@@ -1107,6 +1118,15 @@ static enum ml_worker_result ml_worker_add_existing_model(ml_worker_t *worker, m

1118 return ML_WORKER_RESULT_OK;

1119 }

1120

1121 + // Check if training is in progress and skip if so to avoid race condition

1122 + spinlock_lock(&Dim->slock);

1123 + if (Dim->training_in_progress) {

1124 + spinlock_unlock(&Dim->slock);

1125 + pulse_ml_models_ignored();

1126 + return ML_WORKER_RESULT_OK;

1127 + }

1128 + spinlock_unlock(&Dim->slock);

1129 +

1130 Dim->kmeans = req.inlined_km;

1131 ml_dimension_update_models(worker, Dim);

1132 pulse_ml_models_received();

src/ml/ml_dimension.h

+4 -6

@@ -14,18 +14,16 @@ struct ml_dimension_t {

14 enum ml_metric_type mt;

15 enum ml_training_status ts;

16 enum ml_machine_learning_status mls;

17 -

18 - time_t last_training_time;

17 + SPINLOCK slock;

18 + uint32_t suppression_window_counter;

19 + uint32_t suppression_anomaly_counter;

20 + bool training_in_progress;

22 std::vector<calculated_number_t> cns;

24 std::vector<ml_kmeans_inlined_t> km_contexts;

23 - SPINLOCK slock;

25 ml_kmeans_t kmeans;

26 std::vector<DSample> feature;

26 -

27 - uint32_t suppression_window_counter;

28 - uint32_t suppression_anomaly_counter;

27 };

29 bool

src/ml/ml_enums.h

+3 -3

@@ -3,7 +3,7 @@

3 #ifndef NETDATA_ML_ENUMS_H

4 #define NETDATA_ML_ENUMS_H

6 -enum ml_metric_type {

6 +enum ml_metric_type : unsigned char {

7 // The dimension has constant values, no need to train

8 METRIC_TYPE_CONSTANT,

@@ -13,7 +13,7 @@ enum ml_metric_type {

14 const char *ml_metric_type_to_string(enum ml_metric_type mt);

16 -enum ml_machine_learning_status {

16 +enum ml_machine_learning_status : unsigned char {

17 // Enable training/prediction

18 MACHINE_LEARNING_STATUS_ENABLED,

@@ -23,7 +23,7 @@ enum ml_machine_learning_status {

24 const char *ml_machine_learning_status_to_string(enum ml_machine_learning_status mls);

26 -enum ml_training_status {

26 +enum ml_training_status : unsigned char {

27 // We don't have a model for this dimension

28 TRAINING_STATUS_UNTRAINED,

src/ml/ml_host.h

+13 -13

@@ -13,26 +13,26 @@

13 struct ml_queue_t;

15 typedef struct machine_learning_stats_t {

16 - size_t num_machine_learning_status_enabled;

17 - size_t num_machine_learning_status_disabled_sp;

16 + uint32_t num_machine_learning_status_enabled;

17 + uint32_t num_machine_learning_status_disabled_sp;

19 - size_t num_metric_type_constant;

20 - size_t num_metric_type_variable;

19 + uint32_t num_metric_type_constant;

20 + uint32_t num_metric_type_variable;

22 - size_t num_training_status_untrained;

23 - size_t num_training_status_pending_without_model;

24 - size_t num_training_status_trained;

25 - size_t num_training_status_pending_with_model;

26 - size_t num_training_status_silenced;

22 + uint32_t num_training_status_untrained;

23 + uint32_t num_training_status_pending_without_model;

24 + uint32_t num_training_status_trained;

25 + uint32_t num_training_status_pending_with_model;

26 + uint32_t num_training_status_silenced;

28 - size_t num_anomalous_dimensions;

29 - size_t num_normal_dimensions;

28 + uint32_t num_anomalous_dimensions;

29 + uint32_t num_normal_dimensions;

30 } ml_machine_learning_stats_t;

32 typedef struct {

33 RRDDIM *rd;

34 - size_t normal_dimensions;

35 - size_t anomalous_dimensions;

34 + uint32_t normal_dimensions;

35 + uint32_t anomalous_dimensions;

36 } ml_context_anomaly_rate_t;

38 typedef struct {

src/ml/ml_kmeans.cc

@@ -37,6 +37,12 @@ ml_kmeans_train(ml_kmeans_t *kmeans, const ml_features_t *features, unsigned max

37 return;

38 }

40 + // Reserve capacity for cluster centers BEFORE calling dlib functions to prevent

41 + // reallocation during lazy evaluation. dlib uses expression templates that hold

42 + // references to vector elements, and reallocation would invalidate those references,

43 + // causing heap-use-after-free when multiple threads train models concurrently.

44 + //kmeans->cluster_centers.reserve(2);

45 +

46 dlib::pick_initial_centers(2, kmeans->cluster_centers, features->preprocessed_features);

47 dlib::find_clusters_using_kmeans(features->preprocessed_features, kmeans->cluster_centers, max_iters);

src/ml/ml_public.cc

+1 -4

@@ -109,9 +109,6 @@ void ml_host_stop(RRDHOST *rh) {

109 dim->mt = METRIC_TYPE_CONSTANT;

110 dim->ts = TRAINING_STATUS_UNTRAINED;

111

112 - // TODO: Check if we can remove this field.

113 - dim->last_training_time = 0;

114 -

112 dim->suppression_anomaly_counter = 0;

113 dim->suppression_window_counter = 0;

114 dim->cns.clear();

@@ -273,9 +270,9 @@ void ml_dimension_new(RRDDIM *rd)

270

271 dim->mt = METRIC_TYPE_CONSTANT;

272 dim->ts = TRAINING_STATUS_UNTRAINED;

276 - dim->last_training_time = 0;

273 dim->suppression_anomaly_counter = 0;

274 dim->suppression_window_counter = 0;

275 + dim->training_in_progress = false;

276

277 ml_kmeans_init(&dim->kmeans);

278