@cryptotaxi247 / netdata-1 / commits / 216f91b91

Silence dimensions with noisy ML models (#15011)

* Add suppression options. * Silence noisy dimensions

vkalintiris committed May 10, 2023 at 11:40 UTC 216f91b91f24467b6379435e99438df104bebfda

4 files changed +52 -1

ml/Config.cc

+10 -1

@@ -46,6 +46,9 @@ void ml_config_load(ml_config_t *cfg) {

46 size_t num_training_threads = config_get_number(config_section_ml, "num training threads", 4);

47 size_t flush_models_batch_size = config_get_number(config_section_ml, "flush models batch size", 128);

49 + size_t suppression_window = config_get_number(config_section_ml, "dimension anomaly rate suppression window", 1800);

50 + size_t suppression_threshold = config_get_number(config_section_ml, "dimension anomaly rate suppression threshold", suppression_window / 2);

51 +

52 bool enable_statistics_charts = config_get_boolean(config_section_ml, "enable statistics charts", true);

54 /*

@@ -72,7 +75,10 @@ void ml_config_load(ml_config_t *cfg) {

75 num_training_threads = clamp<size_t>(num_training_threads, 1, 128);

76 flush_models_batch_size = clamp<size_t>(flush_models_batch_size, 8, 512);

75 - /*

78 + suppression_window = clamp<size_t>(suppression_window, 1, max_train_samples);

79 + suppression_threshold = clamp<size_t>(suppression_threshold, 1, suppression_window);

80 +

81 + /*

82 * Validate

83 */

@@ -121,5 +127,8 @@ void ml_config_load(ml_config_t *cfg) {

127 cfg->num_training_threads = num_training_threads;

128 cfg->flush_models_batch_size = flush_models_batch_size;

129

130 + cfg->suppression_window = suppression_window;

131 + cfg->suppression_threshold = suppression_threshold;

132 +

133 cfg->enable_statistics_charts = enable_statistics_charts;

134 }

ml/ad_charts.cc

@@ -124,6 +124,8 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats

124 rrddim_add(host->training_status_rs, "trained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

125 host->training_status_pending_with_model_rd =

126 rrddim_add(host->training_status_rs, "pending-with-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

127 + host->training_status_silenced_rd =

128 + rrddim_add(host->training_status_rs, "silenced", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

129 }

130

131 rrddim_set_by_pointer(host->training_status_rs,

@@ -134,6 +136,8 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats

136 host->training_status_trained_rd, mls.num_training_status_trained);

137 rrddim_set_by_pointer(host->training_status_rs,

138 host->training_status_pending_with_model_rd, mls.num_training_status_pending_with_model);

139 + rrddim_set_by_pointer(host->training_status_rs,

140 + host->training_status_silenced_rd, mls.num_training_status_silenced);

141

142 rrdset_done(host->training_status_rs);

143 }

ml/ml-private.h

+11

@@ -55,6 +55,7 @@ typedef struct machine_learning_stats_t {

55 size_t num_training_status_pending_without_model;

56 size_t num_training_status_trained;

57 size_t num_training_status_pending_with_model;

58 + size_t num_training_status_silenced;

60 size_t num_anomalous_dimensions;

61 size_t num_normal_dimensions;

@@ -103,6 +104,9 @@ enum ml_training_status {

104

105 // Have a valid, up-to-date model

106 TRAINING_STATUS_TRAINED,

107 +

108 + // Have a valid, up-to-date model that is silenced because its too noisy

109 + TRAINING_STATUS_SILENCED,

110 };

111

112 enum ml_training_result {

@@ -194,6 +198,9 @@ typedef struct {

198 netdata_mutex_t mutex;

199 ml_kmeans_t kmeans;

200 std::vector<DSample> feature;

201 +

202 + uint32_t suppression_window_counter;

203 + uint32_t suppression_anomaly_counter;

204 } ml_dimension_t;

205

206 typedef struct {

@@ -233,6 +240,7 @@ typedef struct {

240 RRDDIM *training_status_pending_without_model_rd;

241 RRDDIM *training_status_trained_rd;

242 RRDDIM *training_status_pending_with_model_rd;

243 + RRDDIM *training_status_silenced_rd;

244

245 RRDSET *dimensions_rs;

246 RRDDIM *dimensions_anomalous_rd;

@@ -325,6 +333,9 @@ typedef struct {

333 std::vector<ml_training_thread_t> training_threads;

334 std::atomic<bool> training_stop;

335

336 + size_t suppression_window;

337 + size_t suppression_threshold;

338 +

339 bool enable_statistics_charts;

340 } ml_config_t;

341

ml/ml.cc

+27

@@ -63,6 +63,8 @@ ml_training_status_to_string(enum ml_training_status ts)

63 return "trained";

64 case TRAINING_STATUS_UNTRAINED:

65 return "untrained";

66 + case TRAINING_STATUS_SILENCED:

67 + return "silenced";

68 default:

69 return "unknown";

70 }

@@ -679,6 +681,8 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *

681 break;

682 }

683

684 + dim->suppression_anomaly_counter = 0;

685 + dim->suppression_window_counter = 0;

686 dim->tr = training_response;

687

688 dim->last_training_time = training_response.last_entry_on_response;

@@ -735,6 +739,10 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *

739

740 dim->mt = METRIC_TYPE_CONSTANT;

741 dim->ts = TRAINING_STATUS_TRAINED;

742 +

743 + dim->suppression_anomaly_counter = 0;

744 + dim->suppression_window_counter = 0;

745 +

746 dim->tr = training_response;

747 dim->last_training_time = rrddim_last_entry_s(dim->rd);

748

@@ -771,6 +779,7 @@ ml_dimension_schedule_for_training(ml_dimension_t *dim, time_t curr_time)

779 schedule_for_training = true;

780 dim->ts = TRAINING_STATUS_PENDING_WITHOUT_MODEL;

781 break;

782 + case TRAINING_STATUS_SILENCED:

783 case TRAINING_STATUS_TRAINED:

784 if ((dim->last_training_time + (Cfg.train_every * dim->rd->update_every)) < curr_time) {

785 schedule_for_training = true;

@@ -856,6 +865,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

865 switch (dim->ts) {

866 case TRAINING_STATUS_UNTRAINED:

867 case TRAINING_STATUS_PENDING_WITHOUT_MODEL: {

868 + case TRAINING_STATUS_SILENCED:

869 netdata_mutex_unlock(&dim->mutex);

870 return false;

871 }

@@ -863,6 +873,8 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

873 break;

874 }

875

876 + dim->suppression_window_counter++;

877 +

878 /*

879 * Use the KMeans models to check if the value is anomalous

880 */

@@ -886,6 +898,13 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

898 sum += 1;

899 }

900

901 + dim->suppression_anomaly_counter += sum ? 1 : 0;

902 +

903 + if ((dim->suppression_anomaly_counter >= Cfg.suppression_threshold) &&

904 + (dim->suppression_window_counter >= Cfg.suppression_window)) {

905 + dim->ts = TRAINING_STATUS_SILENCED;

906 + }

907 +

908 netdata_mutex_unlock(&dim->mutex);

909

910 global_statistics_ml_models_consulted(models_consulted);

@@ -939,6 +958,13 @@ ml_chart_update_dimension(ml_chart_t *chart, ml_dimension_t *dim, bool is_anomal

958 case TRAINING_STATUS_PENDING_WITH_MODEL:

959 chart->mls.num_training_status_pending_with_model++;

960

961 + chart->mls.num_anomalous_dimensions += is_anomalous;

962 + chart->mls.num_normal_dimensions += !is_anomalous;

963 + return;

964 + case TRAINING_STATUS_SILENCED:

965 + chart->mls.num_training_status_silenced++;

966 + chart->mls.num_training_status_trained++;

967 +

968 chart->mls.num_anomalous_dimensions += is_anomalous;

969 chart->mls.num_normal_dimensions += !is_anomalous;

970 return;

@@ -995,6 +1021,7 @@ ml_host_detect_once(ml_host_t *host)

1021 host->mls.num_training_status_pending_without_model += chart_mls.num_training_status_pending_without_model;

1022 host->mls.num_training_status_trained += chart_mls.num_training_status_trained;

1023 host->mls.num_training_status_pending_with_model += chart_mls.num_training_status_pending_with_model;

1024 + host->mls.num_training_status_silenced += chart_mls.num_training_status_silenced;

1025

1026 host->mls.num_anomalous_dimensions += chart_mls.num_anomalous_dimensions;

1027 host->mls.num_normal_dimensions += chart_mls.num_normal_dimensions;