@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
cc 257 lines 12.9 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "ml_config.h"
4
5 static void ml_config_migrate() {
6 const char *config_section_ml = CONFIG_SECTION_ML;
7
8 // Check if migration is needed by looking for old keys
9 bool has_old_keys = false;
10 if (inicfg_exists(&netdata_config, config_section_ml, "maximum num samples to train") ||
11 inicfg_exists(&netdata_config, config_section_ml, "minimum num samples to train") ||
12 inicfg_exists(&netdata_config, config_section_ml, "num samples to diff") ||
13 inicfg_exists(&netdata_config, config_section_ml, "num samples to smooth") ||
14 inicfg_exists(&netdata_config, config_section_ml, "num samples to lag") ||
15 inicfg_exists(&netdata_config, config_section_ml, "random sampling ratio")) {
16 has_old_keys = true;
17 }
18
19 // Check if new keys already exist (user manually migrated)
20 bool has_new_keys = false;
21 if (inicfg_exists(&netdata_config, config_section_ml, "training window") ||
22 inicfg_exists(&netdata_config, config_section_ml, "max training vectors")) {
23 has_new_keys = true;
24 }
25
26 // Only migrate if we have old keys but no new keys
27 if (!has_old_keys || has_new_keys) {
28 return;
29 }
30
31 // Get the user's "high resolution" setting
32 // This is what their configuration was designed for
33 time_t global_update_every = nd_profile.update_every;
34
35 // Read all old configuration values with defaults
36 // Users may have changed only some values, so we need proper defaults
37 unsigned old_max_train_samples = inicfg_get_number(&netdata_config, config_section_ml,
38 "maximum num samples to train", 21600);
39 unsigned old_min_train_samples = inicfg_get_number(&netdata_config, config_section_ml,
40 "minimum num samples to train", 900);
41 unsigned old_train_every = inicfg_get_duration_seconds(&netdata_config, config_section_ml,
42 "train every", 10800);
43 unsigned old_diff_n = inicfg_get_number(&netdata_config, config_section_ml,
44 "num samples to diff", 1);
45 unsigned old_smooth_n = inicfg_get_number(&netdata_config, config_section_ml,
46 "num samples to smooth", 3);
47 unsigned old_lag_n = inicfg_get_number(&netdata_config, config_section_ml,
48 "num samples to lag", 5);
49 double old_sampling_ratio = inicfg_get_double(&netdata_config, config_section_ml,
50 "random sampling ratio", 0.2);
51
52 // Calculate time-based equivalents
53 // These preserve the exact behavior the user had configured
54 time_t training_window = old_max_train_samples * global_update_every;
55 time_t min_training_window = old_min_train_samples * global_update_every;
56
57 // Calculate target training vectors based on old pipeline
58 // Account for data reduction from diff, smooth, and sampling
59 size_t effective_samples = old_max_train_samples;
60 if (old_diff_n > 0) effective_samples--; // Lose one sample to differencing
61 size_t max_training_vectors = (size_t)(effective_samples * old_sampling_ratio);
62
63 // Write new configuration values
64 char window_str[32];
65 snprintf(window_str, sizeof(window_str), "%ldh", training_window / 3600);
66 inicfg_set(&netdata_config, config_section_ml, "training window", window_str);
67
68 snprintf(window_str, sizeof(window_str), "%ldm", min_training_window / 60);
69 inicfg_set(&netdata_config, config_section_ml, "min training window", window_str);
70
71 inicfg_set_number(&netdata_config, config_section_ml, "max training vectors", max_training_vectors);
72 inicfg_set_number(&netdata_config, config_section_ml, "max samples to smooth", old_smooth_n);
73
74 // Migrate unchanged values
75 inicfg_set_duration_seconds(&netdata_config, config_section_ml, "train every", old_train_every);
76 inicfg_set_number(&netdata_config, config_section_ml, "num samples to diff", old_diff_n);
77 inicfg_set_number(&netdata_config, config_section_ml, "num samples to lag", old_lag_n);
78
79 // Mark old keys as migrated by moving them to avoid showing in netdata.conf
80 // This uses Netdata's config migration pattern
81 inicfg_move(&netdata_config, config_section_ml, "maximum num samples to train",
82 config_section_ml, "obsolete maximum num samples to train");
83 inicfg_move(&netdata_config, config_section_ml, "minimum num samples to train",
84 config_section_ml, "obsolete minimum num samples to train");
85 inicfg_move(&netdata_config, config_section_ml, "num samples to smooth",
86 config_section_ml, "obsolete num samples to smooth");
87 inicfg_move(&netdata_config, config_section_ml, "random sampling ratio",
88 config_section_ml, "obsolete random sampling ratio");
89
90 // Log the migration
91 nd_log(NDLS_DAEMON, NDLP_NOTICE,
92 "ML configuration migrated from sample-based to time-based:");
93 nd_log(NDLS_DAEMON, NDLP_NOTICE,
94 " Training window: %ld seconds (%ld hours) - was %u samples at %ld second intervals",
95 training_window, training_window / 3600, old_max_train_samples, global_update_every);
96 nd_log(NDLS_DAEMON, NDLP_NOTICE,
97 " Target training vectors: %zu - calculated from smoothing and sampling",
98 max_training_vectors);
99 }
100
101 /*
102 * Global configuration instance to be shared between training and
103 * prediction threads.
104 */
105 ml_config_t Cfg;
106
107 template <typename T>
108 static T clamp(const T& Value, const T& Min, const T& Max) {
109 return std::max(Min, std::min(Value, Max));
110 }
111
112 /*
113 * Initialize global configuration variable.
114 */
115 void ml_config_load(ml_config_t *cfg) {
116 const char *config_section_ml = CONFIG_SECTION_ML;
117
118 // Migrate old configuration if needed
119 ml_config_migrate();
120
121 int enable_anomaly_detection = inicfg_get_boolean_ondemand(&netdata_config, config_section_ml, "enabled", nd_profile.ml_enabled);
122
123 /*
124 * Read values
125 */
126
127 time_t training_window = inicfg_get_duration_seconds(&netdata_config, config_section_ml, "training window", 6 * 3600);
128 time_t min_training_window = inicfg_get_duration_seconds(&netdata_config, config_section_ml, "min training window", 15 * 60);
129 size_t max_training_vectors = inicfg_get_number(&netdata_config, config_section_ml, "max training vectors", 1440);
130 size_t max_samples_to_smooth = inicfg_get_number(&netdata_config, config_section_ml, "max samples to smooth", 3);
131 unsigned train_every = inicfg_get_duration_seconds(&netdata_config, config_section_ml, "train every", 3 * 3600);
132
133 unsigned num_models_to_use = inicfg_get_number(&netdata_config, config_section_ml, "number of models per dimension", 18);
134 unsigned delete_models_older_than = inicfg_get_duration_seconds(&netdata_config, config_section_ml, "delete models older than", 60 * 60 * 24 * 7);
135
136 unsigned diff_n = inicfg_get_number(&netdata_config, config_section_ml, "num samples to diff", 1);
137 unsigned lag_n = inicfg_get_number(&netdata_config, config_section_ml, "num samples to lag", 5);
138
139 unsigned max_kmeans_iters = inicfg_get_number(&netdata_config, config_section_ml, "maximum number of k-means iterations", 1000);
140
141 double dimension_anomaly_rate_threshold = inicfg_get_double(&netdata_config, config_section_ml, "dimension anomaly score threshold", 0.99);
142
143 double host_anomaly_rate_threshold = inicfg_get_double(&netdata_config, config_section_ml, "host anomaly rate threshold", 1.0);
144 std::string anomaly_detection_grouping_method = inicfg_get(&netdata_config, config_section_ml, "anomaly detection grouping method", "average");
145 time_t anomaly_detection_query_duration = inicfg_get_duration_seconds(&netdata_config, config_section_ml, "anomaly detection grouping duration", 5 * 60);
146
147 size_t num_worker_threads = netdata_conf_is_parent() ? netdata_conf_cpus() / 4 : 1;
148 if (num_worker_threads < 1) num_worker_threads = 1;
149 else if (num_worker_threads > 256) num_worker_threads = 256;
150 num_worker_threads = inicfg_get_number(&netdata_config, config_section_ml, "num training threads", num_worker_threads);
151
152 size_t flush_models_batch_size = inicfg_get_number(&netdata_config, config_section_ml, "flush models batch size", 256);
153
154 size_t suppression_window =
155 inicfg_get_duration_seconds(&netdata_config, config_section_ml, "dimension anomaly rate suppression window", 900);
156
157 size_t suppression_threshold =
158 inicfg_get_number(&netdata_config, config_section_ml, "dimension anomaly rate suppression threshold", suppression_window / 2);
159
160 bool enable_statistics_charts = inicfg_get_boolean(&netdata_config, config_section_ml, "enable statistics charts", true);
161
162 /*
163 * Clamp
164 */
165
166 training_window = clamp<time_t>(training_window, 1 * 3600, 24 * 3600);
167 min_training_window = clamp<time_t>(min_training_window, 1 * 900, 6 * 3600);
168 train_every = clamp<unsigned>(train_every, 1 * 3600, 6 * 3600);
169
170 num_models_to_use = clamp<unsigned>(num_models_to_use, 1, 7 * 24);
171 delete_models_older_than = clamp<unsigned>(delete_models_older_than, 60 * 60 * 24 * 1, 60 * 60 * 24 * 7);
172
173 diff_n = clamp(diff_n, 0u, 1u);
174 max_samples_to_smooth = clamp<size_t>(max_samples_to_smooth, 0, 5);
175 lag_n = clamp(lag_n, 1u, 5u);
176 // max_training_vectors drives the lag-extraction sampling ratio (not a
177 // hard cap on output). A floor of 2 keeps the sampler from being starved
178 // by a misconfigured value; the ceiling of 86400 (24 hours of 1-second
179 // samples) is well past anything training_window can supply. The runtime
180 // guard in ml_dimension_train_model is still required because sampling is
181 // probabilistic and can drop the emitted vector count below 2.
182 max_training_vectors = clamp<size_t>(max_training_vectors, 2, 86400);
183
184 max_kmeans_iters = clamp(max_kmeans_iters, 500u, 1000u);
185
186 dimension_anomaly_rate_threshold = clamp(dimension_anomaly_rate_threshold, 0.01, 5.00);
187
188 host_anomaly_rate_threshold = clamp(host_anomaly_rate_threshold, 0.1, 10.0);
189 anomaly_detection_query_duration = clamp<time_t>(anomaly_detection_query_duration, 60, 15 * 60);
190
191 num_worker_threads = clamp<size_t>(num_worker_threads, 4, netdata_conf_cpus());
192 flush_models_batch_size = clamp<size_t>(flush_models_batch_size, 8, 512);
193
194 suppression_window = clamp<size_t>(suppression_window, 1, training_window);
195 suppression_threshold = clamp<size_t>(suppression_threshold, 1, suppression_window);
196
197 /*
198 * Validate
199 */
200
201 if (min_training_window >= training_window) {
202 netdata_log_error("invalid min/max training window found (%ld >= %ld)", min_training_window, training_window);
203
204 min_training_window = 1 * 3600;
205 training_window = 6 * 3600;
206 }
207
208 /*
209 * Assign to config instance
210 */
211
212 cfg->enable_anomaly_detection = enable_anomaly_detection;
213
214 cfg->training_window = training_window;
215 cfg->min_training_window = min_training_window;
216 cfg->max_training_vectors = max_training_vectors;
217 cfg->max_samples_to_smooth = max_samples_to_smooth;
218 cfg->train_every = train_every;
219
220 cfg->num_models_to_use = num_models_to_use;
221 cfg->delete_models_older_than = delete_models_older_than;
222
223 cfg->diff_n = diff_n;
224 cfg->lag_n = lag_n;
225
226 cfg->max_kmeans_iters = max_kmeans_iters;
227
228 cfg->host_anomaly_rate_threshold = host_anomaly_rate_threshold;
229 cfg->anomaly_detection_grouping_method =
230 time_grouping_parse(anomaly_detection_grouping_method.c_str(), RRDR_GROUPING_AVERAGE);
231 cfg->anomaly_detection_query_duration = anomaly_detection_query_duration;
232 cfg->dimension_anomaly_score_threshold = dimension_anomaly_rate_threshold;
233
234 cfg->hosts_to_skip = inicfg_get(&netdata_config, config_section_ml, "hosts to skip from training", "!*");
235 cfg->sp_host_to_skip = simple_pattern_create(cfg->hosts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT, true);
236
237 // Always exclude anomaly_detection charts from training.
238 cfg->charts_to_skip = "anomaly_detection.* ";
239 cfg->charts_to_skip += inicfg_get(&netdata_config, config_section_ml, "charts to skip from training", "netdata.*");
240 cfg->sp_charts_to_skip = simple_pattern_create(cfg->charts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT, true);
241
242 cfg->stream_anomaly_detection_charts = inicfg_get_boolean(&netdata_config, config_section_ml, "stream anomaly detection charts", true);
243
244 cfg->num_worker_threads = num_worker_threads;
245 cfg->flush_models_batch_size = flush_models_batch_size;
246
247 cfg->suppression_window = suppression_window;
248 cfg->suppression_threshold = suppression_threshold;
249
250 cfg->enable_statistics_charts = enable_statistics_charts;
251
252 if (cfg->enable_anomaly_detection == CONFIG_BOOLEAN_AUTO && default_rrd_memory_mode != RRD_DB_MODE_DBENGINE) {
253 Cfg.enable_anomaly_detection = 0;
254 inicfg_set_boolean(&netdata_config, config_section_ml, "enabled", CONFIG_BOOLEAN_NO);
255 return;
256 }
257 }