1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include <dlib/clustering.h>
4
+
5
+#include "nml.h"
6
+
7
+#include <random>
8
+
9
+#include "ad_charts.h"
10
+
11
+typedef struct {
12
+ calculated_number_t *training_cns;
13
+ calculated_number_t *scratch_training_cns;
14
+
15
+ std::vector<DSample> training_samples;
16
+} nml_tls_data_t;
17
+
18
+static thread_local nml_tls_data_t tls_data;
19
+
20
+/*
21
+ * Functions to convert enums to strings
22
+*/
23
+
24
+static const char *nml_machine_learning_status_to_string(enum nml_machine_learning_status mls) {
25
+ switch (mls) {
26
+ case MACHINE_LEARNING_STATUS_ENABLED:
27
+ return "enabled";
28
+ case MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART:
29
+ return "disabled-sp";
30
+ default:
31
+ return "unknown";
32
+ }
33
+}
34
+
35
+static const char *nml_metric_type_to_string(enum nml_metric_type mt) {
36
+ switch (mt) {
37
+ case METRIC_TYPE_CONSTANT:
38
+ return "constant";
39
+ case METRIC_TYPE_VARIABLE:
40
+ return "variable";
41
+ default:
42
+ return "unknown";
43
+ }
44
+}
45
+
46
+static const char *nml_training_status_to_string(enum nml_training_status ts) {
47
+ switch (ts) {
48
+ case TRAINING_STATUS_PENDING_WITH_MODEL:
49
+ return "pending-with-model";
50
+ case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
51
+ return "pending-without-model";
52
+ case TRAINING_STATUS_TRAINED:
53
+ return "trained";
54
+ case TRAINING_STATUS_UNTRAINED:
55
+ return "untrained";
56
+ default:
57
+ return "unknown";
58
+ }
59
+}
60
+
61
+static const char *nml_training_result_to_string(enum nml_training_result tr) {
62
+ switch (tr) {
63
+ case TRAINING_RESULT_OK:
64
+ return "ok";
65
+ case TRAINING_RESULT_INVALID_QUERY_TIME_RANGE:
66
+ return "invalid-query";
67
+ case TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES:
68
+ return "missing-values";
69
+ case TRAINING_RESULT_NULL_ACQUIRED_DIMENSION:
70
+ return "null-acquired-dim";
71
+ case TRAINING_RESULT_CHART_UNDER_REPLICATION:
72
+ return "chart-under-replication";
73
+ default:
74
+ return "unknown";
75
+ }
76
+}
77
+
78
+/*
79
+ * Features
80
+*/
81
+
82
+// subtract elements that are `diff_n` positions apart
83
+static void nml_features_diff(nml_features_t *features) {
84
+ if (features->diff_n == 0)
85
+ return;
86
+
87
+ for (size_t idx = 0; idx != (features->src_n - features->diff_n); idx++) {
88
+ size_t high = (features->src_n - 1) - idx;
89
+ size_t low = high - features->diff_n;
90
+
91
+ features->dst[low] = features->src[high] - features->src[low];
92
+ }
93
+
94
+ size_t n = features->src_n - features->diff_n;
95
+ memcpy(features->src, features->dst, n * sizeof(calculated_number_t));
96
+
97
+ for (size_t idx = features->src_n - features->diff_n; idx != features->src_n; idx++)
98
+ features->src[idx] = 0.0;
99
+}
100
+
101
+// a function that computes the window average of an array inplace
102
+static void nml_features_smooth(nml_features_t *features) {
103
+ calculated_number_t sum = 0.0;
104
+
105
+ size_t idx = 0;
106
+ for (; idx != features->smooth_n - 1; idx++)
107
+ sum += features->src[idx];
108
+
109
+ for (; idx != (features->src_n - features->diff_n); idx++) {
110
+ sum += features->src[idx];
111
+ calculated_number_t prev_cn = features->src[idx - (features->smooth_n - 1)];
112
+ features->src[idx - (features->smooth_n - 1)] = sum / features->smooth_n;
113
+ sum -= prev_cn;
114
+ }
115
+
116
+ for (idx = 0; idx != features->smooth_n; idx++)
117
+ features->src[(features->src_n - 1) - idx] = 0.0;
118
+}
119
+
120
+// create lag'd vectors out of the preprocessed buffer
121
+static void nml_features_lag(nml_features_t *features) {
122
+ size_t n = features->src_n - features->diff_n - features->smooth_n + 1 - features->lag_n;
123
+ features->preprocessed_features.resize(n);
124
+
125
+ unsigned target_num_samples = Cfg.max_train_samples * Cfg.random_sampling_ratio;
126
+ double sampling_ratio = std::min(static_cast<double>(target_num_samples) / n, 1.0);
127
+
128
+ uint32_t max_mt = std::numeric_limits<uint32_t>::max();
129
+ uint32_t cutoff = static_cast<double>(max_mt) * sampling_ratio;
130
+
131
+ size_t sample_idx = 0;
132
+
133
+ for (size_t idx = 0; idx != n; idx++) {
134
+ DSample &DS = features->preprocessed_features[sample_idx++];
135
+ DS.set_size(features->lag_n);
136
+
137
+ if (Cfg.random_nums[idx] > cutoff) {
138
+ sample_idx--;
139
+ continue;
140
+ }
141
+
142
+ for (size_t feature_idx = 0; feature_idx != features->lag_n + 1; feature_idx++)
143
+ DS(feature_idx) = features->src[idx + feature_idx];
144
+ }
145
+
146
+ features->preprocessed_features.resize(sample_idx);
147
+}
148
+
149
+static void nml_features_preprocess(nml_features_t *features) {
150
+ nml_features_diff(features);
151
+ nml_features_smooth(features);
152
+ nml_features_lag(features);
153
+}
154
+
155
+/*
156
+ * KMeans
157
+*/
158
+
159
+static void nml_kmeans_init(nml_kmeans_t *kmeans, size_t num_clusters, size_t max_iterations) {
160
+ kmeans->num_clusters = num_clusters;
161
+ kmeans->max_iterations = max_iterations;
162
+
163
+ kmeans->cluster_centers.reserve(kmeans->num_clusters);
164
+ kmeans->min_dist = std::numeric_limits<calculated_number_t>::max();
165
+ kmeans->max_dist = std::numeric_limits<calculated_number_t>::min();
166
+}
167
+
168
+static void nml_kmeans_train(nml_kmeans_t *kmeans, const nml_features_t *features) {
169
+ kmeans->min_dist = std::numeric_limits<calculated_number_t>::max();
170
+ kmeans->max_dist = std::numeric_limits<calculated_number_t>::min();
171
+
172
+ kmeans->cluster_centers.clear();
173
+
174
+ dlib::pick_initial_centers(kmeans->num_clusters, kmeans->cluster_centers, features->preprocessed_features);
175
+ dlib::find_clusters_using_kmeans(features->preprocessed_features, kmeans->cluster_centers, kmeans->max_iterations);
176
+
177
+ for (const auto &preprocessed_feature : features->preprocessed_features) {
178
+ calculated_number_t mean_dist = 0.0;
179
+
180
+ for (const auto &cluster_center : kmeans->cluster_centers) {
181
+ mean_dist += dlib::length(cluster_center - preprocessed_feature);
182
+ }
183
+
184
+ mean_dist /= kmeans->num_clusters;
185
+
186
+ if (mean_dist < kmeans->min_dist)
187
+ kmeans->min_dist = mean_dist;
188
+
189
+ if (mean_dist > kmeans->max_dist)
190
+ kmeans->max_dist = mean_dist;
191
+ }
192
+}
193
+
194
+static calculated_number_t nml_kmeans_anomaly_score(const nml_kmeans_t *kmeans, const DSample &DS) {
195
+ calculated_number_t mean_dist = 0.0;
196
+ for (const auto &CC: kmeans->cluster_centers)
197
+ mean_dist += dlib::length(CC - DS);
198
+
199
+ mean_dist /= kmeans->num_clusters;
200
+
201
+ if (kmeans->max_dist == kmeans->min_dist)
202
+ return 0.0;
203
+
204
+ calculated_number_t anomaly_score = 100.0 * std::abs((mean_dist - kmeans->min_dist) / (kmeans->max_dist - kmeans->min_dist));
205
+ return (anomaly_score > 100.0) ? 100.0 : anomaly_score;
206
+}
207
+
208
+/*
209
+ * Queue
210
+*/
211
+
212
+nml_queue_t *nml_queue_init() {
213
+ nml_queue_t *q = new nml_queue_t();
214
+
215
+ netdata_mutex_init(&q->mutex);
216
+ pthread_cond_init(&q->cond_var, NULL);
217
+ q->exit = false;
218
+ return q;
219
+}
220
+
221
+void nml_queue_destroy(nml_queue_t *q) {
222
+ netdata_mutex_destroy(&q->mutex);
223
+ pthread_cond_destroy(&q->cond_var);
224
+ delete q;
225
+}
226
+
227
+void nml_queue_push(nml_queue_t *q, const nml_training_request_t req) {
228
+ netdata_mutex_lock(&q->mutex);
229
+ q->internal.push(req);
230
+ pthread_cond_signal(&q->cond_var);
231
+ netdata_mutex_unlock(&q->mutex);
232
+}
233
+
234
+nml_training_request_t nml_queue_pop(nml_queue_t *q) {
235
+ netdata_mutex_lock(&q->mutex);
236
+
237
+ nml_training_request_t req = { NULL, NULL, 0, 0, 0 };
238
+
239
+ while (q->internal.empty()) {
240
+ pthread_cond_wait(&q->cond_var, &q->mutex);
241
+
242
+ if (q->exit) {
243
+ netdata_mutex_unlock(&q->mutex);
244
+
245
+ // We return a dummy request because the queue has been signaled
246
+ return req;
247
+ }
248
+ }
249
+
250
+ req = q->internal.front();
251
+ q->internal.pop();
252
+
253
+ netdata_mutex_unlock(&q->mutex);
254
+ return req;
255
+}
256
+
257
+size_t nml_queue_size(nml_queue_t *q) {
258
+ netdata_mutex_lock(&q->mutex);
259
+ size_t size = q->internal.size();
260
+ netdata_mutex_unlock(&q->mutex);
261
+ return size;
262
+}
263
+
264
+void nml_queue_signal(nml_queue_t *q) {
265
+ netdata_mutex_lock(&q->mutex);
266
+ q->exit = true;
267
+ pthread_cond_signal(&q->cond_var);
268
+ netdata_mutex_unlock(&q->mutex);
269
+}
270
+
271
+/*
272
+ * Dimension
273
+*/
274
+
275
+static std::pair<calculated_number_t *, nml_training_response_t>
276
+nml_dimension_calculated_numbers(nml_dimension_t *dim, const nml_training_request_t &training_request) {
277
+ nml_training_response_t training_response = {};
278
+
279
+ training_response.request_time = training_request.request_time;
280
+ training_response.first_entry_on_request = training_request.first_entry_on_request;
281
+ training_response.last_entry_on_request = training_request.last_entry_on_request;
282
+
283
+ training_response.first_entry_on_response = rrddim_first_entry_s_of_tier(dim->rd, 0);
284
+ training_response.last_entry_on_response = rrddim_last_entry_s_of_tier(dim->rd, 0);
285
+
286
+ size_t min_n = Cfg.min_train_samples;
287
+ size_t max_n = Cfg.max_train_samples;
288
+
289
+ // Figure out what our time window should be.
290
+ training_response.query_before_t = training_response.last_entry_on_response;
291
+ training_response.query_after_t = std::max(
292
+ training_response.query_before_t - static_cast<time_t>((max_n - 1) * dim->rd->update_every),
293
+ training_response.first_entry_on_response
294
+ );
295
+
296
+ if (training_response.query_after_t >= training_response.query_before_t) {
297
+ training_response.result = TRAINING_RESULT_INVALID_QUERY_TIME_RANGE;
298
+ return { NULL, training_response };
299
+ }
300
+
301
+ if (rrdset_is_replicating(dim->rd->rrdset)) {
302
+ training_response.result = TRAINING_RESULT_CHART_UNDER_REPLICATION;
303
+ return { NULL, training_response };
304
+ }
305
+
306
+ /*
307
+ * Execute the query
308
+ */
309
+ struct storage_engine_query_ops *ops = dim->rd->tiers[0].query_ops;
310
+ struct storage_engine_query_handle handle;
311
+
312
+ ops->init(dim->rd->tiers[0].db_metric_handle,
313
+ &handle,
314
+ training_response.query_after_t,
315
+ training_response.query_before_t,
316
+ STORAGE_PRIORITY_BEST_EFFORT);
317
+
318
+ size_t idx = 0;
319
+ memset(tls_data.training_cns, 0, sizeof(calculated_number_t) * max_n * (Cfg.lag_n + 1));
320
+ calculated_number_t last_value = std::numeric_limits<calculated_number_t>::quiet_NaN();
321
+
322
+ while (!ops->is_finished(&handle)) {
323
+ if (idx == max_n)
324
+ break;
325
+
326
+ STORAGE_POINT sp = ops->next_metric(&handle);
327
+
328
+ time_t timestamp = sp.end_time_s;
329
+ calculated_number_t value = sp.sum / sp.count;
330
+
331
+ if (netdata_double_isnumber(value)) {
332
+ if (!training_response.db_after_t)
333
+ training_response.db_after_t = timestamp;
334
+ training_response.db_before_t = timestamp;
335
+
336
+ tls_data.training_cns[idx] = value;
337
+ last_value = tls_data.training_cns[idx];
338
+ training_response.collected_values++;
339
+ } else
340
+ tls_data.training_cns[idx] = last_value;
341
+
342
+ idx++;
343
+ }
344
+ ops->finalize(&handle);
345
+
346
+ global_statistics_ml_query_completed(/* points_read */ idx);
347
+
348
+ training_response.total_values = idx;
349
+ if (training_response.collected_values < min_n) {
350
+ training_response.result = TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES;
351
+ return { NULL, training_response };
352
+ }
353
+
354
+ // Find first non-NaN value.
355
+ for (idx = 0; std::isnan(tls_data.training_cns[idx]); idx++, training_response.total_values--) { }
356
+
357
+ // Overwrite NaN values.
358
+ if (idx != 0)
359
+ memmove(tls_data.training_cns, &tls_data.training_cns[idx], sizeof(calculated_number_t) * training_response.total_values);
360
+
361
+ training_response.result = TRAINING_RESULT_OK;
362
+ return { tls_data.training_cns, training_response };
363
+}
364
+
365
+static enum nml_training_result
366
+nml_dimension_train_model(nml_dimension_t *dim, const nml_training_request_t &training_request) {
367
+ auto P = nml_dimension_calculated_numbers(dim, training_request);
368
+ nml_training_response_t training_response = P.second;
369
+
370
+ if (training_response.result != TRAINING_RESULT_OK) {
371
+ netdata_mutex_lock(&dim->mutex);
372
+
373
+ dim->mt = METRIC_TYPE_CONSTANT;
374
+
375
+ switch (dim->ts) {
376
+ case TRAINING_STATUS_PENDING_WITH_MODEL:
377
+ dim->ts = TRAINING_STATUS_TRAINED;
378
+ break;
379
+ case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
380
+ dim->ts = TRAINING_STATUS_UNTRAINED;
381
+ break;
382
+ default:
383
+ break;
384
+ }
385
+
386
+ dim->tr = training_response;
387
+
388
+ dim->last_training_time = training_response.last_entry_on_response;
389
+ enum nml_training_result result = training_response.result;
390
+ netdata_mutex_unlock(&dim->mutex);
391
+
392
+ return result;
393
+ }
394
+
395
+ // compute kmeans
396
+ {
397
+ memcpy(tls_data.scratch_training_cns, tls_data.training_cns,
398
+ training_response.total_values * sizeof(calculated_number_t));
399
+
400
+ nml_features_t features = {
401
+ Cfg.diff_n, Cfg.smooth_n, Cfg.lag_n,
402
+ tls_data.scratch_training_cns, training_response.total_values,
403
+ tls_data.training_cns, training_response.total_values,
404
+ tls_data.training_samples
405
+ };
406
+ nml_features_preprocess(&features);
407
+
408
+ nml_kmeans_init(&dim->kmeans, 2, 1000);
409
+ nml_kmeans_train(&dim->kmeans, &features);
410
+ }
411
+
412
+ // update kmeans models
413
+ {
414
+ netdata_mutex_lock(&dim->mutex);
415
+
416
+ if (dim->km_contexts.size() < Cfg.num_models_to_use) {
417
+ dim->km_contexts.push_back(std::move(dim->kmeans));
418
+ } else {
419
+ std::rotate(std::begin(dim->km_contexts), std::begin(dim->km_contexts) + 1, std::end(dim->km_contexts));
420
+ dim->km_contexts[dim->km_contexts.size() - 1] = std::move(dim->kmeans);
421
+ }
422
+
423
+ dim->mt = METRIC_TYPE_CONSTANT;
424
+ dim->ts = TRAINING_STATUS_TRAINED;
425
+ dim->tr = training_response;
426
+ dim->last_training_time = rrddim_last_entry_s(dim->rd);
427
+
428
+ netdata_mutex_unlock(&dim->mutex);
429
+ }
430
+
431
+ return training_response.result;
432
+}
433
+
434
+static void nml_dimension_schedule_for_training(nml_dimension_t *dim, time_t curr_time) {
435
+ switch (dim->mt) {
436
+ case METRIC_TYPE_CONSTANT:
437
+ return;
438
+ default:
439
+ break;
440
+ }
441
+
442
+ bool schedule_for_training = false;
443
+
444
+ switch (dim->ts) {
445
+ case TRAINING_STATUS_PENDING_WITH_MODEL:
446
+ case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
447
+ schedule_for_training = false;
448
+ break;
449
+ case TRAINING_STATUS_UNTRAINED:
450
+ schedule_for_training = true;
451
+ dim->ts = TRAINING_STATUS_PENDING_WITHOUT_MODEL;
452
+ break;
453
+ case TRAINING_STATUS_TRAINED:
454
+ if ((dim->last_training_time + (Cfg.train_every * dim->rd->update_every)) < curr_time) {
455
+ schedule_for_training = true;
456
+ dim->ts = TRAINING_STATUS_PENDING_WITH_MODEL;
457
+ }
458
+ break;
459
+ }
460
+
461
+ if (schedule_for_training) {
462
+ nml_host_t *host = reinterpret_cast<nml_host_t *>(dim->rd->rrdset->rrdhost->ml_host);
463
+ nml_training_request_t req = {
464
+ string_dup(dim->rd->rrdset->id), string_dup(dim->rd->id),
465
+ curr_time, rrddim_first_entry_s(dim->rd), rrddim_last_entry_s(dim->rd),
466
+ };
467
+ nml_queue_push(host->training_queue, req);
468
+ }
469
+}
470
+
471
+bool nml_dimension_predict(nml_dimension_t *dim, time_t curr_time, calculated_number_t value, bool exists) {
472
+ // Nothing to do if ML is disabled for this dimension
473
+ if (dim->mls != MACHINE_LEARNING_STATUS_ENABLED)
474
+ return false;
475
+
476
+ // Don't treat values that don't exist as anomalous
477
+ if (!exists) {
478
+ dim->cns.clear();
479
+ return false;
480
+ }
481
+
482
+ // Save the value and return if we don't have enough values for a sample
483
+ unsigned n = Cfg.diff_n + Cfg.smooth_n + Cfg.lag_n;
484
+ if (dim->cns.size() < n) {
485
+ dim->cns.push_back(value);
486
+ return false;
487
+ }
488
+
489
+ // Push the value and check if it's different from the last one
490
+ bool same_value = true;
491
+ std::rotate(std::begin(dim->cns), std::begin(dim->cns) + 1, std::end(dim->cns));
492
+ if (dim->cns[n - 1] != value)
493
+ same_value = false;
494
+ dim->cns[n - 1] = value;
495
+
496
+ // Create the sample
497
+ assert((n * (Cfg.lag_n + 1) <= 128) &&
498
+ "Static buffers too small to perform prediction. "
499
+ "This should not be possible with the default clamping of feature extraction options");
500
+ calculated_number_t src_cns[128];
501
+ calculated_number_t dst_cns[128];
502
+
503
+ memset(src_cns, 0, n * (Cfg.lag_n + 1) * sizeof(calculated_number_t));
504
+ memcpy(src_cns, dim->cns.data(), n * sizeof(calculated_number_t));
505
+ memcpy(dst_cns, dim->cns.data(), n * sizeof(calculated_number_t));
506
+
507
+ nml_features_t features = {
508
+ Cfg.diff_n, Cfg.smooth_n, Cfg.lag_n,
509
+ dst_cns, n, src_cns, n,
510
+ dim->feature
511
+ };
512
+ nml_features_preprocess(&features);
513
+
514
+ /*
515
+ * Lock to predict and possibly schedule the dimension for training
516
+ */
517
+ if (netdata_mutex_trylock(&dim->mutex) != 0)
518
+ return false;
519
+
520
+ // Mark the metric time as variable if we received different values
521
+ if (!same_value)
522
+ dim->mt = METRIC_TYPE_VARIABLE;
523
+
524
+ // Decide if the dimension needs to be scheduled for training
525
+ nml_dimension_schedule_for_training(dim, curr_time);
526
+
527
+ // Nothing to do if we don't have a model
528
+ switch (dim->ts) {
529
+ case TRAINING_STATUS_UNTRAINED:
530
+ case TRAINING_STATUS_PENDING_WITHOUT_MODEL: {
531
+ netdata_mutex_unlock(&dim->mutex);
532
+ return false;
533
+ }
534
+ default:
535
+ break;
536
+ }
537
+
538
+ /*
539
+ * Use the KMeans models to check if the value is anomalous
540
+ */
541
+
542
+ size_t sum = 0;
543
+ size_t models_consulted = 0;
544
+
545
+ for (const auto &km_ctx : dim->km_contexts) {
546
+ models_consulted++;
547
+
548
+ calculated_number_t anomaly_score = nml_kmeans_anomaly_score(&km_ctx, features.preprocessed_features[0]);
549
+ if (anomaly_score == std::numeric_limits<calculated_number_t>::quiet_NaN())
550
+ continue;
551
+
552
+ if (anomaly_score < (100 * Cfg.dimension_anomaly_score_threshold)) {
553
+ global_statistics_ml_models_consulted(models_consulted);
554
+ netdata_mutex_unlock(&dim->mutex);
555
+ return false;
556
+ }
557
+
558
+ sum += 1;
559
+ }
560
+
561
+ netdata_mutex_unlock(&dim->mutex);
562
+
563
+ global_statistics_ml_models_consulted(models_consulted);
564
+ return sum;
565
+}
566
+
567
+void nml_dimension_dump(nml_dimension_t *dim) {
568
+ const char *chart_id = rrdset_id(dim->rd->rrdset);
569
+ const char *dimension_id = rrddim_id(dim->rd);
570
+
571
+ const char *mls_str = nml_machine_learning_status_to_string(dim->mls);
572
+ const char *mt_str = nml_metric_type_to_string(dim->mt);
573
+ const char *ts_str = nml_training_status_to_string(dim->ts);
574
+ const char *tr_str = nml_training_result_to_string(dim->tr.result);
575
+
576
+ const char *fmt =
577
+ "[ML] %s.%s: MLS=%s, MT=%s, TS=%s, Result=%s, "
578
+ "ReqTime=%ld, FEOReq=%ld, LEOReq=%ld, "
579
+ "FEOResp=%ld, LEOResp=%ld, QTR=<%ld, %ld>, DBTR=<%ld, %ld>, Collected=%zu, Total=%zu";
580
+
581
+ error(fmt,
582
+ chart_id, dimension_id, mls_str, mt_str, ts_str, tr_str,
583
+ dim->tr.request_time, dim->tr.first_entry_on_request, dim->tr.last_entry_on_request,
584
+ dim->tr.first_entry_on_response, dim->tr.last_entry_on_response,
585
+ dim->tr.query_after_t, dim->tr.query_before_t, dim->tr.db_after_t, dim->tr.db_before_t, dim->tr.collected_values, dim->tr.total_values
586
+ );
587
+}
588
+
589
+nml_dimension_t *nml_dimension_new(RRDDIM *rd) {
590
+ nml_dimension_t *dim = new nml_dimension_t();
591
+
592
+ dim->rd = rd;
593
+
594
+ dim->mt = METRIC_TYPE_CONSTANT;
595
+ dim->ts = TRAINING_STATUS_UNTRAINED;
596
+
597
+ dim->last_training_time = 0;
598
+
599
+ nml_kmeans_init(&dim->kmeans, 2, 1000);
600
+
601
+ if (simple_pattern_matches(Cfg.sp_charts_to_skip, rrdset_name(rd->rrdset)))
602
+ dim->mls = MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART;
603
+ else
604
+ dim->mls = MACHINE_LEARNING_STATUS_ENABLED;
605
+
606
+ netdata_mutex_init(&dim->mutex);
607
+
608
+ dim->km_contexts.reserve(Cfg.num_models_to_use);
609
+
610
+ return dim;
611
+}
612
+
613
+void nml_dimension_delete(nml_dimension_t *dim) {
614
+ netdata_mutex_destroy(&dim->mutex);
615
+ delete dim;
616
+}
617
+
618
+nml_chart_t *nml_chart_new(RRDSET *rs) {
619
+ nml_chart_t *chart = new nml_chart_t();
620
+
621
+ chart->rs = rs;
622
+ chart->mls = nml_machine_learning_stats_t();
623
+
624
+ netdata_mutex_init(&chart->mutex);
625
+
626
+ return chart;
627
+}
628
+
629
+void nml_chart_delete(nml_chart_t *chart) {
630
+ netdata_mutex_destroy(&chart->mutex);
631
+ delete chart;
632
+}
633
+
634
+static bool nml_chart_is_available_for_ml(nml_chart_t *chart) {
635
+ return rrdset_is_available_for_exporting_and_alarms(chart->rs);
636
+}
637
+
638
+static std::string ml_dimension_get_id(RRDDIM *rd) {
639
+ RRDSET *rs = rd->rrdset;
640
+
641
+ std::stringstream ss;
642
+ ss << rrdset_context(rs) << "|" << rrdset_id(rs) << "|" << rrddim_name(rd);
643
+ return ss.str();
644
+}
645
+
646
+static void nml_chart_get_models_as_json(nml_chart_t *chart, nlohmann::json &j) {
647
+ netdata_mutex_lock(&chart->mutex);
648
+
649
+ void *rdp = NULL;
650
+ rrddim_foreach_read(rdp, chart->rs) {
651
+ RRDDIM *rd = static_cast<RRDDIM *>(rdp);
652
+ nml_dimension_t *dim = reinterpret_cast<nml_dimension_t *>(rd->ml_dimension);
653
+ if (!dim)
654
+ continue;
655
+
656
+ nlohmann::json jarray = nlohmann::json::array();
657
+#if 0
658
+ for (const KMeans &KM : nml_dimension_models(D)) {
659
+ nlohmann::json tmp;
660
+
661
+ KM.toJson(tmp);
662
+ jarray.push_back(tmp);
663
+ j[ml_dimension_get_id(D->rd)] = jarray;
664
+ }
665
+#else
666
+ j[ml_dimension_get_id(dim->rd)] = jarray;
667
+#endif
668
+ }
669
+ rrdset_foreach_done(rdp);
670
+
671
+ netdata_mutex_unlock(&chart->mutex);
672
+}
673
+
674
+void nml_chart_update_begin(nml_chart_t *chart) {
675
+ netdata_mutex_lock(&chart->mutex);
676
+ chart->mls = {};
677
+}
678
+
679
+void nml_chart_update_end(nml_chart_t *chart) {
680
+ netdata_mutex_unlock(&chart->mutex);
681
+}
682
+
683
+void nml_chart_update_dimension(nml_chart_t *chart, nml_dimension_t *dim, bool is_anomalous) {
684
+ switch (dim->mls) {
685
+ case MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART:
686
+ chart->mls.num_machine_learning_status_disabled_sp++;
687
+ return;
688
+ case MACHINE_LEARNING_STATUS_ENABLED: {
689
+ chart->mls.num_machine_learning_status_enabled++;
690
+
691
+ switch (dim->mt) {
692
+ case METRIC_TYPE_CONSTANT:
693
+ chart->mls.num_metric_type_constant++;
694
+ chart->mls.num_training_status_trained++;
695
+ chart->mls.num_normal_dimensions++;
696
+ return;
697
+ case METRIC_TYPE_VARIABLE:
698
+ chart->mls.num_metric_type_variable++;
699
+ break;
700
+ }
701
+
702
+ switch (dim->ts) {
703
+ case TRAINING_STATUS_UNTRAINED:
704
+ chart->mls.num_training_status_untrained++;
705
+ return;
706
+ case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
707
+ chart->mls.num_training_status_pending_without_model++;
708
+ return;
709
+ case TRAINING_STATUS_TRAINED:
710
+ chart->mls.num_training_status_trained++;
711
+
712
+ chart->mls.num_anomalous_dimensions += is_anomalous;
713
+ chart->mls.num_normal_dimensions += !is_anomalous;
714
+ return;
715
+ case TRAINING_STATUS_PENDING_WITH_MODEL:
716
+ chart->mls.num_training_status_pending_with_model++;
717
+
718
+ chart->mls.num_anomalous_dimensions += is_anomalous;
719
+ chart->mls.num_normal_dimensions += !is_anomalous;
720
+ return;
721
+ }
722
+
723
+ return;
724
+ }
725
+ }
726
+}
727
+
728
+nml_host_t *nml_host_new(RRDHOST *rh) {
729
+ nml_host_t *host = new nml_host_t();
730
+
731
+ host->rh = rh;
732
+ host->mls = nml_machine_learning_stats_t();
733
+ host->ts = nml_training_stats_t();
734
+
735
+ host->host_anomaly_rate = 0.0;
736
+ host->threads_running = false;
737
+ host->threads_cancelled = false;
738
+ host->threads_joined = false;
739
+
740
+ host->training_queue = nml_queue_init();
741
+
742
+ netdata_mutex_init(&host->mutex);
743
+
744
+ return host;
745
+}
746
+
747
+void nml_host_delete(nml_host_t *host) {
748
+ netdata_mutex_destroy(&host->mutex);
749
+ nml_queue_destroy(host->training_queue);
750
+ delete host;
751
+}
752
+
753
+void nml_host_get_config_as_json(nml_host_t *host, BUFFER *wb) {
754
+ // Unused for now, until we add support for per-host configs
755
+ (void) host;
756
+
757
+ buffer_json_member_add_uint64(wb, "version", 1);
758
+
759
+ buffer_json_member_add_boolean(wb, "enabled", Cfg.enable_anomaly_detection);
760
+
761
+ buffer_json_member_add_uint64(wb, "min-train-samples", Cfg.min_train_samples);
762
+ buffer_json_member_add_uint64(wb, "max-train-samples", Cfg.max_train_samples);
763
+ buffer_json_member_add_uint64(wb, "train-every", Cfg.train_every);
764
+
765
+ buffer_json_member_add_uint64(wb, "diff-n", Cfg.diff_n);
766
+ buffer_json_member_add_uint64(wb, "smooth-n", Cfg.smooth_n);
767
+ buffer_json_member_add_uint64(wb, "lag-n", Cfg.lag_n);
768
+
769
+ buffer_json_member_add_double(wb, "random-sampling-ratio", Cfg.random_sampling_ratio);
770
+ buffer_json_member_add_uint64(wb, "max-kmeans-iters", Cfg.random_sampling_ratio);
771
+
772
+ buffer_json_member_add_double(wb, "dimension-anomaly-score-threshold", Cfg.dimension_anomaly_score_threshold);
773
+
774
+ buffer_json_member_add_string(wb, "anomaly-detection-grouping-method",
775
+ time_grouping_method2string(Cfg.anomaly_detection_grouping_method));
776
+
777
+ buffer_json_member_add_int64(wb, "anomaly-detection-query-duration", Cfg.anomaly_detection_query_duration);
778
+
779
+ buffer_json_member_add_string(wb, "hosts-to-skip", Cfg.hosts_to_skip.c_str());
780
+ buffer_json_member_add_string(wb, "charts-to-skip", Cfg.charts_to_skip.c_str());
781
+}
782
+
783
+void nml_host_get_models_as_json(nml_host_t *host, nlohmann::json &j) {
784
+ netdata_mutex_lock(&host->mutex);
785
+
786
+ void* rsp = NULL;
787
+ rrdset_foreach_read(rsp, host->rh) {
788
+ RRDSET *rs = static_cast<RRDSET *>(rsp);
789
+ nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
790
+
791
+ if (!chart)
792
+ continue;
793
+
794
+ nml_chart_get_models_as_json(chart, j);
795
+ }
796
+ rrdset_foreach_done(rsp);
797
+
798
+ netdata_mutex_unlock(&host->mutex);
799
+}
800
+
801
+#define WORKER_JOB_DETECTION_PREP 0
802
+#define WORKER_JOB_DETECTION_DIM_CHART 1
803
+#define WORKER_JOB_DETECTION_HOST_CHART 2
804
+#define WORKER_JOB_DETECTION_STATS 3
805
+#define WORKER_JOB_DETECTION_RESOURCES 4
806
+
807
+static void nml_host_detect_once(nml_host_t *host) {
808
+ worker_is_busy(WORKER_JOB_DETECTION_PREP);
809
+
810
+ host->mls = {};
811
+ nml_machine_learning_stats_t mls_copy = {};
812
+ nml_training_stats_t ts_copy = {};
813
+
814
+ {
815
+ netdata_mutex_lock(&host->mutex);
816
+
817
+ /*
818
+ * prediction/detection stats
819
+ */
820
+ void *rsp = NULL;
821
+ rrdset_foreach_read(rsp, host->rh) {
822
+ RRDSET *rs = static_cast<RRDSET *>(rsp);
823
+
824
+ nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
825
+ if (!chart)
826
+ continue;
827
+
828
+ if (!nml_chart_is_available_for_ml(chart))
829
+ continue;
830
+
831
+ nml_machine_learning_stats_t chart_mls = chart->mls;
832
+
833
+ host->mls.num_machine_learning_status_enabled += chart_mls.num_machine_learning_status_enabled;
834
+ host->mls.num_machine_learning_status_disabled_sp += chart_mls.num_machine_learning_status_disabled_sp;
835
+
836
+ host->mls.num_metric_type_constant += chart_mls.num_metric_type_constant;
837
+ host->mls.num_metric_type_variable += chart_mls.num_metric_type_variable;
838
+
839
+ host->mls.num_training_status_untrained += chart_mls.num_training_status_untrained;
840
+ host->mls.num_training_status_pending_without_model += chart_mls.num_training_status_pending_without_model;
841
+ host->mls.num_training_status_trained += chart_mls.num_training_status_trained;
842
+ host->mls.num_training_status_pending_with_model += chart_mls.num_training_status_pending_with_model;
843
+
844
+ host->mls.num_anomalous_dimensions += chart_mls.num_anomalous_dimensions;
845
+ host->mls.num_normal_dimensions += chart_mls.num_normal_dimensions;
846
+ }
847
+ rrdset_foreach_done(rsp);
848
+
849
+ host->host_anomaly_rate = 0.0;
850
+ size_t NumActiveDimensions = host->mls.num_anomalous_dimensions + host->mls.num_normal_dimensions;
851
+ if (NumActiveDimensions)
852
+ host->host_anomaly_rate = static_cast<double>(host->mls.num_anomalous_dimensions) / NumActiveDimensions;
853
+
854
+ mls_copy = host->mls;
855
+
856
+ /*
857
+ * training stats
858
+ */
859
+ ts_copy = host->ts;
860
+
861
+ host->ts.queue_size = 0;
862
+ host->ts.num_popped_items = 0;
863
+
864
+ host->ts.allotted_ut = 0;
865
+ host->ts.consumed_ut = 0;
866
+ host->ts.remaining_ut = 0;
867
+
868
+ host->ts.training_result_ok = 0;
869
+ host->ts.training_result_invalid_query_time_range = 0;
870
+ host->ts.training_result_not_enough_collected_values = 0;
871
+ host->ts.training_result_null_acquired_dimension = 0;
872
+ host->ts.training_result_chart_under_replication = 0;
873
+
874
+ netdata_mutex_unlock(&host->mutex);
875
+ }
876
+
877
+ // Calc the avg values
878
+ if (ts_copy.num_popped_items) {
879
+ ts_copy.queue_size /= ts_copy.num_popped_items;
880
+ ts_copy.allotted_ut /= ts_copy.num_popped_items;
881
+ ts_copy.consumed_ut /= ts_copy.num_popped_items;
882
+ ts_copy.remaining_ut /= ts_copy.num_popped_items;
883
+
884
+ ts_copy.training_result_ok /= ts_copy.num_popped_items;
885
+ ts_copy.training_result_invalid_query_time_range /= ts_copy.num_popped_items;
886
+ ts_copy.training_result_not_enough_collected_values /= ts_copy.num_popped_items;
887
+ ts_copy.training_result_null_acquired_dimension /= ts_copy.num_popped_items;
888
+ ts_copy.training_result_chart_under_replication /= ts_copy.num_popped_items;
889
+ } else {
890
+ ts_copy.queue_size = 0;
891
+ ts_copy.allotted_ut = 0;
892
+ ts_copy.consumed_ut = 0;
893
+ ts_copy.remaining_ut = 0;
894
+ }
895
+
896
+ worker_is_busy(WORKER_JOB_DETECTION_DIM_CHART);
897
+ nml_update_dimensions_chart(host, mls_copy);
898
+
899
+ worker_is_busy(WORKER_JOB_DETECTION_HOST_CHART);
900
+ nml_update_host_and_detection_rate_charts(host, host->host_anomaly_rate * 10000.0);
901
+
902
+#ifdef NETDATA_ML_RESOURCE_CHARTS
903
+ worker_is_busy(WORKER_JOB_DETECTION_RESOURCES);
904
+ struct rusage PredictionRU;
905
+ getrusage(RUSAGE_THREAD, &PredictionRU);
906
+ updateResourceUsageCharts(RH, PredictionRU, TSCopy.TrainingRU);
907
+#endif
908
+
909
+ worker_is_busy(WORKER_JOB_DETECTION_STATS);
910
+ nml_update_training_statistics_chart(host, ts_copy);
911
+}
912
+
913
+typedef struct {
914
+ RRDDIM_ACQUIRED *acq_rd;
915
+ nml_dimension_t *dim;
916
+} nml_acquired_dimension_t;
917
+
918
+static nml_acquired_dimension_t nml_acquired_dimension_get(RRDHOST *rh, STRING *chart_id, STRING *dimension_id) {
919
+ RRDDIM_ACQUIRED *acq_rd = NULL;
920
+ nml_dimension_t *dim = NULL;
921
+
922
+ RRDSET *rs = rrdset_find(rh, string2str(chart_id));
923
+ if (rs) {
924
+ acq_rd = rrddim_find_and_acquire(rs, string2str(dimension_id));
925
+ if (acq_rd) {
926
+ RRDDIM *rd = rrddim_acquired_to_rrddim(acq_rd);
927
+ if (rd)
928
+ dim = reinterpret_cast<nml_dimension_t *>(rd->ml_dimension);
929
+ }
930
+ }
931
+
932
+ nml_acquired_dimension_t acq_dim = {
933
+ acq_rd, dim
934
+ };
935
+
936
+ return acq_dim;
937
+}
938
+
939
+static void nml_acquired_dimension_release(nml_acquired_dimension_t acq_dim) {
940
+ if (!acq_dim.acq_rd)
941
+ return;
942
+
943
+ rrddim_acquired_release(acq_dim.acq_rd);
944
+}
945
+
946
+static enum nml_training_result nml_acquired_dimension_train(nml_acquired_dimension_t acq_dim, const nml_training_request_t &TR) {
947
+ if (!acq_dim.dim)
948
+ return TRAINING_RESULT_NULL_ACQUIRED_DIMENSION;
949
+
950
+ return nml_dimension_train_model(acq_dim.dim, TR);
951
+}
952
+
953
+#define WORKER_JOB_TRAINING_FIND 0
954
+#define WORKER_JOB_TRAINING_TRAIN 1
955
+#define WORKER_JOB_TRAINING_STATS 2
956
+
957
+void nml_host_get_detection_info_as_json(nml_host_t *host, nlohmann::json &j) {
958
+ j["version"] = 1;
959
+ j["anomalous-dimensions"] = host->mls.num_anomalous_dimensions;
960
+ j["normal-dimensions"] = host->mls.num_normal_dimensions;
961
+ j["total-dimensions"] = host->mls.num_anomalous_dimensions + host->mls.num_normal_dimensions;
962
+ j["trained-dimensions"] = host->mls.num_training_status_trained + host->mls.num_training_status_pending_with_model;
963
+}
964
+
965
+void nml_host_train(nml_host_t *host) {
966
+ worker_register("MLTRAIN");
967
+ worker_register_job_name(WORKER_JOB_TRAINING_FIND, "find");
968
+ worker_register_job_name(WORKER_JOB_TRAINING_TRAIN, "train");
969
+ worker_register_job_name(WORKER_JOB_TRAINING_STATS, "stats");
970
+
971
+ service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, (force_quit_t )ml_cancel_training_thread, host->rh, true);
972
+
973
+ while (service_running(SERVICE_ML_TRAINING)) {
974
+ nml_training_request_t training_req = nml_queue_pop(host->training_queue);
975
+ size_t queue_size = nml_queue_size(host->training_queue) + 1;
976
+
977
+ if (host->threads_cancelled) {
978
+ info("Stopping training thread for host %s because it was cancelled", rrdhost_hostname(host->rh));
979
+ break;
980
+ }
981
+
982
+ usec_t allotted_ut = (Cfg.train_every * host->rh->rrd_update_every * USEC_PER_SEC) / queue_size;
983
+ if (allotted_ut > USEC_PER_SEC)
984
+ allotted_ut = USEC_PER_SEC;
985
+
986
+ usec_t start_ut = now_monotonic_usec();
987
+ enum nml_training_result training_res;
988
+ {
989
+ worker_is_busy(WORKER_JOB_TRAINING_FIND);
990
+ nml_acquired_dimension_t acq_dim = nml_acquired_dimension_get(host->rh, training_req.chart_id, training_req.dimension_id);
991
+
992
+ worker_is_busy(WORKER_JOB_TRAINING_TRAIN);
993
+ training_res = nml_acquired_dimension_train(acq_dim, training_req);
994
+
995
+ string_freez(training_req.chart_id);
996
+ string_freez(training_req.dimension_id);
997
+
998
+ nml_acquired_dimension_release(acq_dim);
999
+ }
1000
+ usec_t consumed_ut = now_monotonic_usec() - start_ut;
1001
+
1002
+ worker_is_busy(WORKER_JOB_TRAINING_STATS);
1003
+
1004
+ usec_t remaining_ut = 0;
1005
+ if (consumed_ut < allotted_ut)
1006
+ remaining_ut = allotted_ut - consumed_ut;
1007
+
1008
+ {
1009
+ netdata_mutex_lock(&host->mutex);
1010
+
1011
+ if (host->ts.allotted_ut == 0) {
1012
+ struct rusage TRU;
1013
+ getrusage(RUSAGE_THREAD, &TRU);
1014
+ host->ts.training_ru = TRU;
1015
+ }
1016
+
1017
+ host->ts.queue_size += queue_size;
1018
+ host->ts.num_popped_items += 1;
1019
+
1020
+ host->ts.allotted_ut += allotted_ut;
1021
+ host->ts.consumed_ut += consumed_ut;
1022
+ host->ts.remaining_ut += remaining_ut;
1023
+
1024
+ switch (training_res) {
1025
+ case TRAINING_RESULT_OK:
1026
+ host->ts.training_result_ok += 1;
1027
+ break;
1028
+ case TRAINING_RESULT_INVALID_QUERY_TIME_RANGE:
1029
+ host->ts.training_result_invalid_query_time_range += 1;
1030
+ break;
1031
+ case TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES:
1032
+ host->ts.training_result_not_enough_collected_values += 1;
1033
+ break;
1034
+ case TRAINING_RESULT_NULL_ACQUIRED_DIMENSION:
1035
+ host->ts.training_result_null_acquired_dimension += 1;
1036
+ break;
1037
+ case TRAINING_RESULT_CHART_UNDER_REPLICATION:
1038
+ host->ts.training_result_chart_under_replication += 1;
1039
+ break;
1040
+ }
1041
+
1042
+ netdata_mutex_unlock(&host->mutex);
1043
+ }
1044
+
1045
+ worker_is_idle();
1046
+ std::this_thread::sleep_for(std::chrono::microseconds{remaining_ut});
1047
+ worker_is_busy(0);
1048
+ }
1049
+}
1050
+
1051
+static void *train_main(void *arg) {
1052
+ size_t max_elements_needed_for_training = Cfg.max_train_samples * (Cfg.lag_n + 1);
1053
+ tls_data.training_cns = new calculated_number_t[max_elements_needed_for_training]();
1054
+ tls_data.scratch_training_cns = new calculated_number_t[max_elements_needed_for_training]();
1055
+
1056
+ nml_host_t *host = reinterpret_cast<nml_host_t *>(arg);
1057
+ nml_host_train(host);
1058
+ return NULL;
1059
+}
1060
+
1061
+void nml_host_start_training_thread(nml_host_t *host) {
1062
+ if (host->threads_running) {
1063
+ error("Anomaly detections threads for host %s are already-up and running.", rrdhost_hostname(host->rh));
1064
+ return;
1065
+ }
1066
+
1067
+ host->threads_running = true;
1068
+ host->threads_cancelled = false;
1069
+ host->threads_joined = false;
1070
+
1071
+ char tag[NETDATA_THREAD_TAG_MAX + 1];
1072
+
1073
+ snprintfz(tag, NETDATA_THREAD_TAG_MAX, "MLTR[%s]", rrdhost_hostname(host->rh));
1074
+ netdata_thread_create(&host->training_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, train_main, static_cast<void *>(host));
1075
+}
1076
+
1077
+void nml_host_stop_training_thread(nml_host_t *host, bool join) {
1078
+ if (!host->threads_running) {
1079
+ error("Anomaly detections threads for host %s have already been stopped.", rrdhost_hostname(host->rh));
1080
+ return;
1081
+ }
1082
+
1083
+ if (!host->threads_cancelled) {
1084
+ host->threads_cancelled = true;
1085
+
1086
+ // Signal the training queue to stop popping-items
1087
+ nml_queue_signal(host->training_queue);
1088
+ netdata_thread_cancel(host->training_thread);
1089
+ }
1090
+
1091
+ if (join && !host->threads_joined) {
1092
+ host->threads_joined = true;
1093
+ host->threads_running = false;
1094
+
1095
+ delete[] tls_data.training_cns;
1096
+ delete[] tls_data.scratch_training_cns;
1097
+
1098
+ netdata_thread_join(host->training_thread, NULL);
1099
+ }
1100
+}
1101
+
1102
+void *nml_detect_main(void *arg) {
1103
+ UNUSED(arg);
1104
+
1105
+ worker_register("MLDETECT");
1106
+ worker_register_job_name(WORKER_JOB_DETECTION_PREP, "prep");
1107
+ worker_register_job_name(WORKER_JOB_DETECTION_DIM_CHART, "dim chart");
1108
+ worker_register_job_name(WORKER_JOB_DETECTION_HOST_CHART, "host chart");
1109
+ worker_register_job_name(WORKER_JOB_DETECTION_STATS, "stats");
1110
+ worker_register_job_name(WORKER_JOB_DETECTION_RESOURCES, "resources");
1111
+
1112
+ service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, NULL, NULL, true);
1113
+
1114
+ heartbeat_t hb;
1115
+ heartbeat_init(&hb);
1116
+
1117
+ while (service_running((SERVICE_TYPE)(SERVICE_ML_PREDICTION | SERVICE_COLLECTORS))) {
1118
+ worker_is_idle();
1119
+ heartbeat_next(&hb, USEC_PER_SEC);
1120
+
1121
+ rrd_rdlock();
1122
+
1123
+ RRDHOST *rh;
1124
+ rrdhost_foreach_read(rh) {
1125
+ if (!rh->ml_host)
1126
+ continue;
1127
+
1128
+ nml_host_detect_once(reinterpret_cast<nml_host_t *>(rh->ml_host));
1129
+ }
1130
+
1131
+ rrd_unlock();
1132
+ }
1133
+
1134
+ return NULL;
1135
+}