@cryptotaxi247 / netdata-1 / commits / 45981cb73

Port ML from C++ to C. (#14567)

* Port ML from C++ to C. Pretty much everything is a non-functional change, ie. the functionality is identical to the one provided by the existing implementation that is written in C++. Performance-wise, this implementation: - Eliminates/reduces the number of allocations and deallocations we have to do for training/detection, - Uses just a single thread to perform detection for *all* the hosts (ie. reduces the number of required threads by 50% on parents), and - Allows training, prediction and detection of dimensions that have an update_every that is different from that of the localhost. The only C++ functionality that we still use is vectors, because they make our life easier and they are pretty much a requirement imposed by dlib. * Remove profile.plugin It was useful only for testing during development. * Limit logs to 200 lines per period * Properly generate ml_info in /api/v1/info endpoint. * Remove resource usage charts since we use worker charts. * Use a temporary to make linters happy. * Rebase. * Fix builds that have ML functionality disabled.

vkalintiris committed Feb 28, 2023 at 15:53 UTC 45981cb7347a5fed7ebbabba0fe193d0d9471eff
28 files changed +2121 -2561
Makefile.am
+4 -21
@@ -235,37 +235,20 @@ ML_FILES = \
235 if ENABLE_ML
236
237 ML_FILES += \
238 - ml/ADCharts.h \
239 - ml/ADCharts.cc \
240 - ml/Config.h \
238 + ml/ad_charts.h \
239 + ml/ad_charts.cc \
240 ml/Config.cc \
242 - ml/Chart.cc \
243 - ml/Chart.h \
244 - ml/Stats.h \
245 - ml/Dimension.cc \
246 - ml/Dimension.h \
247 - ml/Host.h \
248 - ml/Host.cc \
249 - ml/Mutex.h \
250 - ml/Queue.h \
251 - ml/Query.h \
252 - ml/KMeans.h \
253 - ml/KMeans.cc \
254 - ml/SamplesBuffer.h \
255 - ml/SamplesBuffer.cc \
241 ml/dlib/dlib/all/source.cpp \
242 ml/json/single_include/nlohmann/json.hpp \
243 + ml/nml.h \
244 + ml/nml.cc \
245 ml/ml.cc \
259 - ml/ml-private.h \
246 $(NULL)
247
248 # Disable warnings from dlib library
249 ml/dlib/dlib/all/source.$(OBJEXT) : CXXFLAGS += -Wno-sign-compare -Wno-type-limits -Wno-aggressive-loop-optimizations -Wno-stringop-overflow -Wno-psabi
250
251 # Disable ml warnings
266 -ml/Dimension.$(OBJEXT) : CXXFLAGS += -Wno-psabi
267 -ml/Host.$(OBJEXT) : CXXFLAGS += -Wno-psabi
268 -ml/KMeans.$(OBJEXT) : CXXFLAGS += -Wno-psabi
252 ml/ml.$(OBJEXT) : CXXFLAGS += -Wno-psabi
253
254 endif
database/rrdhost.c
+3 -3
@@ -525,7 +525,7 @@ int is_legacy = 1;
525 rrdhost_load_rrdcontext_data(host);
526 if (!archived) {
527 ml_host_new(host);
528 - ml_start_anomaly_detection_threads(host);
528 + ml_start_training_thread(host);
529 } else
530 rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED | RRDHOST_FLAG_ORPHAN);
531
@@ -642,7 +642,7 @@ static void rrdhost_update(RRDHOST *host
642 host->rrdpush_replication_step = rrdpush_replication_step;
643
644 ml_host_new(host);
645 - ml_start_anomaly_detection_threads(host);
645 + ml_start_training_thread(host);
646
647 rrdhost_load_rrdcontext_data(host);
648 info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
@@ -1145,7 +1145,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1145 rrdcalctemplate_index_destroy(host);
1146
1147 // cleanup ML resources
1148 - ml_stop_anomaly_detection_threads(host);
1148 + ml_stop_training_thread(host);
1149 ml_host_delete(host);
1150
1151 freez(host->exporting_flags);
ml/ADCharts.cc deleted
-518
@@ -1,518 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "ADCharts.h"
4 -#include "Config.h"
5 -
6 -void ml::updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS) {
7 - /*
8 - * Machine learning status
9 - */
10 - {
11 - static thread_local RRDSET *MachineLearningStatusRS = nullptr;
12 -
13 - static thread_local RRDDIM *Enabled = nullptr;
14 - static thread_local RRDDIM *DisabledUE = nullptr;
15 - static thread_local RRDDIM *DisabledSP = nullptr;
16 -
17 - if (!MachineLearningStatusRS) {
18 - std::stringstream IdSS, NameSS;
19 -
20 - IdSS << "machine_learning_status_on_" << localhost->machine_guid;
21 - NameSS << "machine_learning_status_on_" << rrdhost_hostname(localhost);
22 -
23 - MachineLearningStatusRS = rrdset_create(
24 - RH,
25 - "netdata", // type
26 - IdSS.str().c_str(), // id
27 - NameSS.str().c_str(), // name
28 - NETDATA_ML_CHART_FAMILY, // family
29 - "netdata.machine_learning_status", // ctx
30 - "Machine learning status", // title
31 - "dimensions", // units
32 - NETDATA_ML_PLUGIN, // plugin
33 - NETDATA_ML_MODULE_TRAINING, // module
34 - NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
35 - RH->rrd_update_every, // update_every
36 - RRDSET_TYPE_LINE // chart_type
37 - );
38 - rrdset_flag_set(MachineLearningStatusRS , RRDSET_FLAG_ANOMALY_DETECTION);
39 -
40 - Enabled = rrddim_add(MachineLearningStatusRS, "enabled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
41 - DisabledUE = rrddim_add(MachineLearningStatusRS, "disabled-ue", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
42 - DisabledSP = rrddim_add(MachineLearningStatusRS, "disabled-sp", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
43 - }
44 -
45 - rrddim_set_by_pointer(MachineLearningStatusRS, Enabled, MLS.NumMachineLearningStatusEnabled);
46 - rrddim_set_by_pointer(MachineLearningStatusRS, DisabledUE, MLS.NumMachineLearningStatusDisabledUE);
47 - rrddim_set_by_pointer(MachineLearningStatusRS, DisabledSP, MLS.NumMachineLearningStatusDisabledSP);
48 -
49 - rrdset_done(MachineLearningStatusRS);
50 - }
51 -
52 - /*
53 - * Metric type
54 - */
55 - {
56 - static thread_local RRDSET *MetricTypesRS = nullptr;
57 -
58 - static thread_local RRDDIM *Constant = nullptr;
59 - static thread_local RRDDIM *Variable = nullptr;
60 -
61 - if (!MetricTypesRS) {
62 - std::stringstream IdSS, NameSS;
63 -
64 - IdSS << "metric_types_on_" << localhost->machine_guid;
65 - NameSS << "metric_types_on_" << rrdhost_hostname(localhost);
66 -
67 - MetricTypesRS = rrdset_create(
68 - RH,
69 - "netdata", // type
70 - IdSS.str().c_str(), // id
71 - NameSS.str().c_str(), // name
72 - NETDATA_ML_CHART_FAMILY, // family
73 - "netdata.metric_types", // ctx
74 - "Dimensions by metric type", // title
75 - "dimensions", // units
76 - NETDATA_ML_PLUGIN, // plugin
77 - NETDATA_ML_MODULE_TRAINING, // module
78 - NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
79 - RH->rrd_update_every, // update_every
80 - RRDSET_TYPE_LINE // chart_type
81 - );
82 - rrdset_flag_set(MetricTypesRS, RRDSET_FLAG_ANOMALY_DETECTION);
83 -
84 - Constant = rrddim_add(MetricTypesRS, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
85 - Variable = rrddim_add(MetricTypesRS, "variable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
86 - }
87 -
88 - rrddim_set_by_pointer(MetricTypesRS, Constant, MLS.NumMetricTypeConstant);
89 - rrddim_set_by_pointer(MetricTypesRS, Variable, MLS.NumMetricTypeVariable);
90 -
91 - rrdset_done(MetricTypesRS);
92 - }
93 -
94 - /*
95 - * Training status
96 - */
97 - {
98 - static thread_local RRDSET *TrainingStatusRS = nullptr;
99 -
100 - static thread_local RRDDIM *Untrained = nullptr;
101 - static thread_local RRDDIM *PendingWithoutModel = nullptr;
102 - static thread_local RRDDIM *Trained = nullptr;
103 - static thread_local RRDDIM *PendingWithModel = nullptr;
104 -
105 - if (!TrainingStatusRS) {
106 - std::stringstream IdSS, NameSS;
107 -
108 - IdSS << "training_status_on_" << localhost->machine_guid;
109 - NameSS << "training_status_on_" << rrdhost_hostname(localhost);
110 -
111 - TrainingStatusRS = rrdset_create(
112 - RH,
113 - "netdata", // type
114 - IdSS.str().c_str(), // id
115 - NameSS.str().c_str(), // name
116 - NETDATA_ML_CHART_FAMILY, // family
117 - "netdata.training_status", // ctx
118 - "Training status of dimensions", // title
119 - "dimensions", // units
120 - NETDATA_ML_PLUGIN, // plugin
121 - NETDATA_ML_MODULE_TRAINING, // module
122 - NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
123 - RH->rrd_update_every, // update_every
124 - RRDSET_TYPE_LINE // chart_type
125 - );
126 -
127 - rrdset_flag_set(TrainingStatusRS, RRDSET_FLAG_ANOMALY_DETECTION);
128 -
129 - Untrained = rrddim_add(TrainingStatusRS, "untrained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
130 - PendingWithoutModel = rrddim_add(TrainingStatusRS, "pending-without-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
131 - Trained = rrddim_add(TrainingStatusRS, "trained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
132 - PendingWithModel = rrddim_add(TrainingStatusRS, "pending-with-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
133 - }
134 -
135 - rrddim_set_by_pointer(TrainingStatusRS, Untrained, MLS.NumTrainingStatusUntrained);
136 - rrddim_set_by_pointer(TrainingStatusRS, PendingWithoutModel, MLS.NumTrainingStatusPendingWithoutModel);
137 - rrddim_set_by_pointer(TrainingStatusRS, Trained, MLS.NumTrainingStatusTrained);
138 - rrddim_set_by_pointer(TrainingStatusRS, PendingWithModel, MLS.NumTrainingStatusPendingWithModel);
139 -
140 - rrdset_done(TrainingStatusRS);
141 - }
142 -
143 - /*
144 - * Prediction status
145 - */
146 - {
147 - static thread_local RRDSET *PredictionRS = nullptr;
148 -
149 - static thread_local RRDDIM *Anomalous = nullptr;
150 - static thread_local RRDDIM *Normal = nullptr;
151 -
152 - if (!PredictionRS) {
153 - std::stringstream IdSS, NameSS;
154 -
155 - IdSS << "dimensions_on_" << localhost->machine_guid;
156 - NameSS << "dimensions_on_" << rrdhost_hostname(localhost);
157 -
158 - PredictionRS = rrdset_create(
159 - RH,
160 - "anomaly_detection", // type
161 - IdSS.str().c_str(), // id
162 - NameSS.str().c_str(), // name
163 - "dimensions", // family
164 - "anomaly_detection.dimensions", // ctx
165 - "Anomaly detection dimensions", // title
166 - "dimensions", // units
167 - NETDATA_ML_PLUGIN, // plugin
168 - NETDATA_ML_MODULE_TRAINING, // module
169 - ML_CHART_PRIO_DIMENSIONS, // priority
170 - RH->rrd_update_every, // update_every
171 - RRDSET_TYPE_LINE // chart_type
172 - );
173 - rrdset_flag_set(PredictionRS, RRDSET_FLAG_ANOMALY_DETECTION);
174 -
175 - Anomalous = rrddim_add(PredictionRS, "anomalous", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
176 - Normal = rrddim_add(PredictionRS, "normal", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
177 - }
178 -
179 - rrddim_set_by_pointer(PredictionRS, Anomalous, MLS.NumAnomalousDimensions);
180 - rrddim_set_by_pointer(PredictionRS, Normal, MLS.NumNormalDimensions);
181 -
182 - rrdset_done(PredictionRS);
183 - }
184 -
185 -}
186 -
187 -void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) {
188 - static thread_local RRDSET *HostRateRS = nullptr;
189 - static thread_local RRDDIM *AnomalyRateRD = nullptr;
190 -
191 - if (!HostRateRS) {
192 - std::stringstream IdSS, NameSS;
193 -
194 - IdSS << "anomaly_rate_on_" << localhost->machine_guid;
195 - NameSS << "anomaly_rate_on_" << rrdhost_hostname(localhost);
196 -
197 - HostRateRS = rrdset_create(
198 - RH,
199 - "anomaly_detection", // type
200 - IdSS.str().c_str(), // id
201 - NameSS.str().c_str(), // name
202 - "anomaly_rate", // family
203 - "anomaly_detection.anomaly_rate", // ctx
204 - "Percentage of anomalous dimensions", // title
205 - "percentage", // units
206 - NETDATA_ML_PLUGIN, // plugin
207 - NETDATA_ML_MODULE_DETECTION, // module
208 - ML_CHART_PRIO_ANOMALY_RATE, // priority
209 - RH->rrd_update_every, // update_every
210 - RRDSET_TYPE_LINE // chart_type
211 - );
212 - rrdset_flag_set(HostRateRS, RRDSET_FLAG_ANOMALY_DETECTION);
213 -
214 - AnomalyRateRD = rrddim_add(HostRateRS, "anomaly_rate", NULL,
215 - 1, 100, RRD_ALGORITHM_ABSOLUTE);
216 - }
217 -
218 - rrddim_set_by_pointer(HostRateRS, AnomalyRateRD, AnomalyRate);
219 - rrdset_done(HostRateRS);
220 -
221 - static thread_local RRDSET *AnomalyDetectionRS = nullptr;
222 - static thread_local RRDDIM *AboveThresholdRD = nullptr;
223 - static thread_local RRDDIM *NewAnomalyEventRD = nullptr;
224 -
225 - if (!AnomalyDetectionRS) {
226 - std::stringstream IdSS, NameSS;
227 -
228 - IdSS << "anomaly_detection_on_" << localhost->machine_guid;
229 - NameSS << "anomaly_detection_on_" << rrdhost_hostname(localhost);
230 -
231 - AnomalyDetectionRS = rrdset_create(
232 - RH,
233 - "anomaly_detection", // type
234 - IdSS.str().c_str(), // id
235 - NameSS.str().c_str(), // name
236 - "anomaly_detection", // family
237 - "anomaly_detection.detector_events", // ctx
238 - "Anomaly detection events", // title
239 - "percentage", // units
240 - NETDATA_ML_PLUGIN, // plugin
241 - NETDATA_ML_MODULE_DETECTION, // module
242 - ML_CHART_PRIO_DETECTOR_EVENTS, // priority
243 - RH->rrd_update_every, // update_every
244 - RRDSET_TYPE_LINE // chart_type
245 - );
246 - rrdset_flag_set(AnomalyDetectionRS, RRDSET_FLAG_ANOMALY_DETECTION);
247 -
248 - AboveThresholdRD = rrddim_add(AnomalyDetectionRS, "above_threshold", NULL,
249 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
250 - NewAnomalyEventRD = rrddim_add(AnomalyDetectionRS, "new_anomaly_event", NULL,
251 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
252 - }
253 -
254 - /*
255 - * Compute the values of the dimensions based on the host rate chart
256 - */
257 - ONEWAYALLOC *OWA = onewayalloc_create(0);
258 - time_t Now = now_realtime_sec();
259 - time_t Before = Now - RH->rrd_update_every;
260 - time_t After = Before - Cfg.AnomalyDetectionQueryDuration;
261 - RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
262 -
263 - RRDR *R = rrd2rrdr_legacy(
264 - OWA, HostRateRS,
265 - 1 /* points wanted */,
266 - After,
267 - Before,
268 - Cfg.AnomalyDetectionGroupingMethod,
269 - 0 /* resampling time */,
270 - Options, "anomaly_rate",
271 - NULL /* group options */,
272 - 0, /* timeout */
273 - 0, /* tier */
274 - QUERY_SOURCE_ML,
275 - STORAGE_PRIORITY_BEST_EFFORT
276 - );
277 -
278 - if(R) {
279 - if(R->d == 1 && R->n == 1 && R->rows == 1) {
280 - static thread_local bool PrevAboveThreshold = false;
281 - bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
282 - bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
283 - PrevAboveThreshold = AboveThreshold;
284 -
285 - rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
286 - rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
287 - rrdset_done(AnomalyDetectionRS);
288 - }
289 -
290 - rrdr_free(OWA, R);
291 - }
292 -
293 - onewayalloc_destroy(OWA);
294 -}
295 -
296 -void ml::updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU) {
297 - /*
298 - * prediction rusage
299 - */
300 - {
301 - static thread_local RRDSET *RS = nullptr;
302 -
303 - static thread_local RRDDIM *User = nullptr;
304 - static thread_local RRDDIM *System = nullptr;
305 -
306 - if (!RS) {
307 - std::stringstream IdSS, NameSS;
308 -
309 - IdSS << "prediction_usage_for_" << RH->machine_guid;
310 - NameSS << "prediction_usage_for_" << rrdhost_hostname(RH);
311 -
312 - RS = rrdset_create_localhost(
313 - "netdata", // type
314 - IdSS.str().c_str(), // id
315 - NameSS.str().c_str(), // name
316 - NETDATA_ML_CHART_FAMILY, // family
317 - "netdata.prediction_usage", // ctx
318 - "Prediction resource usage", // title
319 - "milliseconds/s", // units
320 - NETDATA_ML_PLUGIN, // plugin
321 - NETDATA_ML_MODULE_PREDICTION, // module
322 - NETDATA_ML_CHART_PRIO_PREDICTION_USAGE, // priority
323 - RH->rrd_update_every, // update_every
324 - RRDSET_TYPE_STACKED // chart_type
325 - );
326 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
327 -
328 - User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
329 - System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
330 - }
331 -
332 - rrddim_set_by_pointer(RS, User, PredictionRU.ru_utime.tv_sec * 1000000ULL + PredictionRU.ru_utime.tv_usec);
333 - rrddim_set_by_pointer(RS, System, PredictionRU.ru_stime.tv_sec * 1000000ULL + PredictionRU.ru_stime.tv_usec);
334 -
335 - rrdset_done(RS);
336 - }
337 -
338 - /*
339 - * training rusage
340 - */
341 - {
342 - static thread_local RRDSET *RS = nullptr;
343 -
344 - static thread_local RRDDIM *User = nullptr;
345 - static thread_local RRDDIM *System = nullptr;
346 -
347 - if (!RS) {
348 - std::stringstream IdSS, NameSS;
349 -
350 - IdSS << "training_usage_for_" << RH->machine_guid;
351 - NameSS << "training_usage_for_" << rrdhost_hostname(RH);
352 -
353 - RS = rrdset_create_localhost(
354 - "netdata", // type
355 - IdSS.str().c_str(), // id
356 - NameSS.str().c_str(), // name
357 - NETDATA_ML_CHART_FAMILY, // family
358 - "netdata.training_usage", // ctx
359 - "Training resource usage", // title
360 - "milliseconds/s", // units
361 - NETDATA_ML_PLUGIN, // plugin
362 - NETDATA_ML_MODULE_TRAINING, // module
363 - NETDATA_ML_CHART_PRIO_TRAINING_USAGE, // priority
364 - RH->rrd_update_every, // update_every
365 - RRDSET_TYPE_STACKED // chart_type
366 - );
367 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
368 -
369 - User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
370 - System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
371 - }
372 -
373 - rrddim_set_by_pointer(RS, User, TrainingRU.ru_utime.tv_sec * 1000000ULL + TrainingRU.ru_utime.tv_usec);
374 - rrddim_set_by_pointer(RS, System, TrainingRU.ru_stime.tv_sec * 1000000ULL + TrainingRU.ru_stime.tv_usec);
375 -
376 - rrdset_done(RS);
377 - }
378 -}
379 -
380 -void ml::updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS) {
381 - /*
382 - * queue stats
383 - */
384 - {
385 - static thread_local RRDSET *RS = nullptr;
386 -
387 - static thread_local RRDDIM *QueueSize = nullptr;
388 - static thread_local RRDDIM *PoppedItems = nullptr;
389 -
390 - if (!RS) {
391 - std::stringstream IdSS, NameSS;
392 -
393 - IdSS << "queue_stats_on_" << localhost->machine_guid;
394 - NameSS << "queue_stats_on_" << rrdhost_hostname(localhost);
395 -
396 - RS = rrdset_create(
397 - RH,
398 - "netdata", // type
399 - IdSS.str().c_str(), // id
400 - NameSS.str().c_str(), // name
401 - NETDATA_ML_CHART_FAMILY, // family
402 - "netdata.queue_stats", // ctx
403 - "Training queue stats", // title
404 - "items", // units
405 - NETDATA_ML_PLUGIN, // plugin
406 - NETDATA_ML_MODULE_TRAINING, // module
407 - NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
408 - RH->rrd_update_every, // update_every
409 - RRDSET_TYPE_LINE// chart_type
410 - );
411 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
412 -
413 - QueueSize = rrddim_add(RS, "queue_size", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
414 - PoppedItems = rrddim_add(RS, "popped_items", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
415 - }
416 -
417 - rrddim_set_by_pointer(RS, QueueSize, TS.QueueSize);
418 - rrddim_set_by_pointer(RS, PoppedItems, TS.NumPoppedItems);
419 -
420 - rrdset_done(RS);
421 - }
422 -
423 - /*
424 - * training stats
425 - */
426 - {
427 - static thread_local RRDSET *RS = nullptr;
428 -
429 - static thread_local RRDDIM *Allotted = nullptr;
430 - static thread_local RRDDIM *Consumed = nullptr;
431 - static thread_local RRDDIM *Remaining = nullptr;
432 -
433 - if (!RS) {
434 - std::stringstream IdSS, NameSS;
435 -
436 - IdSS << "training_time_stats_on_" << localhost->machine_guid;
437 - NameSS << "training_time_stats_on_" << rrdhost_hostname(localhost);
438 -
439 - RS = rrdset_create(
440 - RH,
441 - "netdata", // type
442 - IdSS.str().c_str(), // id
443 - NameSS.str().c_str(), // name
444 - NETDATA_ML_CHART_FAMILY, // family
445 - "netdata.training_time_stats", // ctx
446 - "Training time stats", // title
447 - "milliseconds", // units
448 - NETDATA_ML_PLUGIN, // plugin
449 - NETDATA_ML_MODULE_TRAINING, // module
450 - NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
451 - RH->rrd_update_every, // update_every
452 - RRDSET_TYPE_LINE// chart_type
453 - );
454 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
455 -
456 - Allotted = rrddim_add(RS, "allotted", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
457 - Consumed = rrddim_add(RS, "consumed", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
458 - Remaining = rrddim_add(RS, "remaining", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
459 - }
460 -
461 - rrddim_set_by_pointer(RS, Allotted, TS.AllottedUT);
462 - rrddim_set_by_pointer(RS, Consumed, TS.ConsumedUT);
463 - rrddim_set_by_pointer(RS, Remaining, TS.RemainingUT);
464 -
465 - rrdset_done(RS);
466 - }
467 -
468 - /*
469 - * training result stats
470 - */
471 - {
472 - static thread_local RRDSET *RS = nullptr;
473 -
474 - static thread_local RRDDIM *Ok = nullptr;
475 - static thread_local RRDDIM *InvalidQueryTimeRange = nullptr;
476 - static thread_local RRDDIM *NotEnoughCollectedValues = nullptr;
477 - static thread_local RRDDIM *NullAcquiredDimension = nullptr;
478 - static thread_local RRDDIM *ChartUnderReplication = nullptr;
479 -
480 - if (!RS) {
481 - std::stringstream IdSS, NameSS;
482 -
483 - IdSS << "training_results_on_" << localhost->machine_guid;
484 - NameSS << "training_results_on_" << rrdhost_hostname(localhost);
485 -
486 - RS = rrdset_create(
487 - RH,
488 - "netdata", // type
489 - IdSS.str().c_str(), // id
490 - NameSS.str().c_str(), // name
491 - NETDATA_ML_CHART_FAMILY, // family
492 - "netdata.training_results", // ctx
493 - "Training results", // title
494 - "events", // units
495 - NETDATA_ML_PLUGIN, // plugin
496 - NETDATA_ML_MODULE_TRAINING, // module
497 - NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
498 - RH->rrd_update_every, // update_every
499 - RRDSET_TYPE_LINE// chart_type
500 - );
501 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
502 -
503 - Ok = rrddim_add(RS, "ok", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
504 - InvalidQueryTimeRange = rrddim_add(RS, "invalid-queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
505 - NotEnoughCollectedValues = rrddim_add(RS, "not-enough-values", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
506 - NullAcquiredDimension = rrddim_add(RS, "null-acquired-dimensions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
507 - ChartUnderReplication = rrddim_add(RS, "chart-under-replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
508 - }
509 -
510 - rrddim_set_by_pointer(RS, Ok, TS.TrainingResultOk);
511 - rrddim_set_by_pointer(RS, InvalidQueryTimeRange, TS.TrainingResultInvalidQueryTimeRange);
512 - rrddim_set_by_pointer(RS, NotEnoughCollectedValues, TS.TrainingResultNotEnoughCollectedValues);
513 - rrddim_set_by_pointer(RS, NullAcquiredDimension, TS.TrainingResultNullAcquiredDimension);
514 - rrddim_set_by_pointer(RS, ChartUnderReplication, TS.TrainingResultChartUnderReplication);
515 -
516 - rrdset_done(RS);
517 - }
518 -}
ml/ADCharts.h deleted
-21
@@ -1,21 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_ADCHARTS_H
4 -#define ML_ADCHARTS_H
5 -
6 -#include "Stats.h"
7 -#include "ml-private.h"
8 -
9 -namespace ml {
10 -
11 -void updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS);
12 -
13 -void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate);
14 -
15 -void updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU);
16 -
17 -void updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS);
18 -
19 -} // namespace ml
20 -
21 -#endif /* ML_ADCHARTS_H */
ml/Chart.cc
ml/Chart.h deleted
-128
@@ -1,128 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_CHART_H
4 -#define ML_CHART_H
5 -
6 -#include "Config.h"
7 -#include "Dimension.h"
8 -
9 -#include "ml-private.h"
10 -#include "json/single_include/nlohmann/json.hpp"
11 -
12 -namespace ml
13 -{
14 -
15 -class Chart {
16 -public:
17 - Chart(RRDSET *RS) :
18 - RS(RS),
19 - MLS()
20 - { }
21 -
22 - RRDSET *getRS() const {
23 - return RS;
24 - }
25 -
26 - bool isAvailableForML() {
27 - return rrdset_is_available_for_exporting_and_alarms(RS);
28 - }
29 -
30 - void addDimension(Dimension *D) {
31 - std::lock_guard<Mutex> L(M);
32 - Dimensions[D->getRD()] = D;
33 - }
34 -
35 - void removeDimension(Dimension *D) {
36 - std::lock_guard<Mutex> L(M);
37 - Dimensions.erase(D->getRD());
38 - }
39 -
40 - void getModelsAsJson(nlohmann::json &Json) {
41 - std::lock_guard<Mutex> L(M);
42 -
43 - for (auto &DP : Dimensions) {
44 - Dimension *D = DP.second;
45 - nlohmann::json JsonArray = nlohmann::json::array();
46 - for (const KMeans &KM : D->getModels()) {
47 - nlohmann::json J;
48 - KM.toJson(J);
49 - JsonArray.push_back(J);
50 - }
51 -
52 - Json[getMLDimensionID(D->getRD())] = JsonArray;
53 - }
54 - }
55 -
56 - void updateBegin() {
57 - M.lock();
58 - MLS = {};
59 - }
60 -
61 - void updateDimension(Dimension *D, bool IsAnomalous) {
62 - switch (D->getMLS()) {
63 - case MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
64 - MLS.NumMachineLearningStatusDisabledUE++;
65 - return;
66 - case MachineLearningStatus::DisabledDueToExcludedChart:
67 - MLS.NumMachineLearningStatusDisabledSP++;
68 - return;
69 - case MachineLearningStatus::Enabled: {
70 - MLS.NumMachineLearningStatusEnabled++;
71 -
72 - switch (D->getMT()) {
73 - case MetricType::Constant:
74 - MLS.NumMetricTypeConstant++;
75 - MLS.NumTrainingStatusTrained++;
76 - MLS.NumNormalDimensions++;
77 - return;
78 - case MetricType::Variable:
79 - MLS.NumMetricTypeVariable++;
80 - break;
81 - }
82 -
83 - switch (D->getTS()) {
84 - case TrainingStatus::Untrained:
85 - MLS.NumTrainingStatusUntrained++;
86 - return;
87 - case TrainingStatus::PendingWithoutModel:
88 - MLS.NumTrainingStatusPendingWithoutModel++;
89 - return;
90 - case TrainingStatus::Trained:
91 - MLS.NumTrainingStatusTrained++;
92 -
93 - MLS.NumAnomalousDimensions += IsAnomalous;
94 - MLS.NumNormalDimensions += !IsAnomalous;
95 - return;
96 - case TrainingStatus::PendingWithModel:
97 - MLS.NumTrainingStatusPendingWithModel++;
98 -
99 - MLS.NumAnomalousDimensions += IsAnomalous;
100 - MLS.NumNormalDimensions += !IsAnomalous;
101 - return;
102 - }
103 -
104 - return;
105 - }
106 - }
107 - }
108 -
109 - void updateEnd() {
110 - M.unlock();
111 - }
112 -
113 - MachineLearningStats getMLS() {
114 - std::lock_guard<Mutex> L(M);
115 - return MLS;
116 - }
117 -
118 -private:
119 - RRDSET *RS;
120 - MachineLearningStats MLS;
121 -
122 - Mutex M;
123 - std::unordered_map<RRDDIM *, Dimension *> Dimensions;
124 -};
125 -
126 -} // namespace ml
127 -
128 -#endif /* ML_CHART_H */
ml/Config.cc
+55 -58
@@ -1,15 +1,12 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "Config.h"
4 -#include "ml-private.h"
5 -
6 -using namespace ml;
3 +#include "nml.h"
4
5 /*
6 * Global configuration instance to be shared between training and
7 * prediction threads.
8 */
12 -Config ml::Cfg;
9 +nml_config_t Cfg;
10
11 template <typename T>
12 static T clamp(const T& Value, const T& Min, const T& Max) {
@@ -19,97 +16,97 @@ static T clamp(const T& Value, const T& Min, const T& Max) {
16 /*
17 * Initialize global configuration variable.
18 */
22 -void Config::readMLConfig(void) {
23 - const char *ConfigSectionML = CONFIG_SECTION_ML;
19 +void nml_config_load(nml_config_t *cfg) {
20 + const char *config_section_ml = CONFIG_SECTION_ML;
21
25 - bool EnableAnomalyDetection = config_get_boolean(ConfigSectionML, "enabled", true);
22 + bool enable_anomaly_detection = config_get_boolean(config_section_ml, "enabled", true);
23
24 /*
25 * Read values
26 */
27
31 - unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
32 - unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
33 - unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
34 - unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1);
28 + unsigned max_train_samples = config_get_number(config_section_ml, "maximum num samples to train", 4 * 3600);
29 + unsigned min_train_samples = config_get_number(config_section_ml, "minimum num samples to train", 1 * 900);
30 + unsigned train_every = config_get_number(config_section_ml, "train every", 1 * 3600);
31 + unsigned num_models_to_use = config_get_number(config_section_ml, "number of models per dimension", 1);
32
36 - unsigned DiffN = config_get_number(ConfigSectionML, "num samples to diff", 1);
37 - unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);
38 - unsigned LagN = config_get_number(ConfigSectionML, "num samples to lag", 5);
33 + unsigned diff_n = config_get_number(config_section_ml, "num samples to diff", 1);
34 + unsigned smooth_n = config_get_number(config_section_ml, "num samples to smooth", 3);
35 + unsigned lag_n = config_get_number(config_section_ml, "num samples to lag", 5);
36
40 - double RandomSamplingRatio = config_get_float(ConfigSectionML, "random sampling ratio", 1.0 / LagN);
41 - unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000);
37 + double random_sampling_ratio = config_get_float(config_section_ml, "random sampling ratio", 1.0 / lag_n);
38 + unsigned max_kmeans_iters = config_get_number(config_section_ml, "maximum number of k-means iterations", 1000);
39
43 - double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99);
40 + double dimension_anomaly_rate_threshold = config_get_float(config_section_ml, "dimension anomaly score threshold", 0.99);
41
45 - double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 1.0);
46 - std::string AnomalyDetectionGroupingMethod = config_get(ConfigSectionML, "anomaly detection grouping method", "average");
47 - time_t AnomalyDetectionQueryDuration = config_get_number(ConfigSectionML, "anomaly detection grouping duration", 5 * 60);
42 + double host_anomaly_rate_threshold = config_get_float(config_section_ml, "host anomaly rate threshold", 1.0);
43 + std::string anomaly_detection_grouping_method = config_get(config_section_ml, "anomaly detection grouping method", "average");
44 + time_t anomaly_detection_query_duration = config_get_number(config_section_ml, "anomaly detection grouping duration", 5 * 60);
45
46 /*
47 * Clamp
48 */
49
53 - MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
54 - MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
55 - TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
56 - NumModelsToUse = clamp<unsigned>(NumModelsToUse, 1, 7 * 24);
50 + max_train_samples = clamp<unsigned>(max_train_samples, 1 * 3600, 24 * 3600);
51 + min_train_samples = clamp<unsigned>(min_train_samples, 1 * 900, 6 * 3600);
52 + train_every = clamp<unsigned>(train_every, 1 * 3600, 6 * 3600);
53 + num_models_to_use = clamp<unsigned>(num_models_to_use, 1, 7 * 24);
54
58 - DiffN = clamp(DiffN, 0u, 1u);
59 - SmoothN = clamp(SmoothN, 0u, 5u);
60 - LagN = clamp(LagN, 1u, 5u);
55 + diff_n = clamp(diff_n, 0u, 1u);
56 + smooth_n = clamp(smooth_n, 0u, 5u);
57 + lag_n = clamp(lag_n, 1u, 5u);
58
62 - RandomSamplingRatio = clamp(RandomSamplingRatio, 0.2, 1.0);
63 - MaxKMeansIters = clamp(MaxKMeansIters, 500u, 1000u);
59 + random_sampling_ratio = clamp(random_sampling_ratio, 0.2, 1.0);
60 + max_kmeans_iters = clamp(max_kmeans_iters, 500u, 1000u);
61
65 - DimensionAnomalyScoreThreshold = clamp(DimensionAnomalyScoreThreshold, 0.01, 5.00);
62 + dimension_anomaly_rate_threshold = clamp(dimension_anomaly_rate_threshold, 0.01, 5.00);
63
67 - HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.1, 10.0);
68 - AnomalyDetectionQueryDuration = clamp<time_t>(AnomalyDetectionQueryDuration, 60, 15 * 60);
64 + host_anomaly_rate_threshold = clamp(host_anomaly_rate_threshold, 0.1, 10.0);
65 + anomaly_detection_query_duration = clamp<time_t>(anomaly_detection_query_duration, 60, 15 * 60);
66
67 /*
68 * Validate
69 */
70
74 - if (MinTrainSamples >= MaxTrainSamples) {
75 - error("invalid min/max train samples found (%u >= %u)", MinTrainSamples, MaxTrainSamples);
71 + if (min_train_samples >= max_train_samples) {
72 + error("invalid min/max train samples found (%u >= %u)", min_train_samples, max_train_samples);
73
77 - MinTrainSamples = 1 * 3600;
78 - MaxTrainSamples = 4 * 3600;
74 + min_train_samples = 1 * 3600;
75 + max_train_samples = 4 * 3600;
76 }
77
78 /*
79 * Assign to config instance
80 */
81
85 - Cfg.EnableAnomalyDetection = EnableAnomalyDetection;
82 + cfg->enable_anomaly_detection = enable_anomaly_detection;
83
87 - Cfg.MaxTrainSamples = MaxTrainSamples;
88 - Cfg.MinTrainSamples = MinTrainSamples;
89 - Cfg.TrainEvery = TrainEvery;
90 - Cfg.NumModelsToUse = NumModelsToUse;
84 + cfg->max_train_samples = max_train_samples;
85 + cfg->min_train_samples = min_train_samples;
86 + cfg->train_every = train_every;
87
92 - Cfg.DiffN = DiffN;
93 - Cfg.SmoothN = SmoothN;
94 - Cfg.LagN = LagN;
88 + cfg->num_models_to_use = num_models_to_use;
89
96 - Cfg.RandomSamplingRatio = RandomSamplingRatio;
97 - Cfg.MaxKMeansIters = MaxKMeansIters;
90 + cfg->diff_n = diff_n;
91 + cfg->smooth_n = smooth_n;
92 + cfg->lag_n = lag_n;
93
99 - Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;
94 + cfg->random_sampling_ratio = random_sampling_ratio;
95 + cfg->max_kmeans_iters = max_kmeans_iters;
96
101 - Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
102 - Cfg.AnomalyDetectionGroupingMethod = time_grouping_parse(
103 - AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
104 - Cfg.AnomalyDetectionQueryDuration = AnomalyDetectionQueryDuration;
97 + cfg->host_anomaly_rate_threshold = host_anomaly_rate_threshold;
98 + cfg->anomaly_detection_grouping_method =
99 + time_grouping_parse(anomaly_detection_grouping_method.c_str(), RRDR_GROUPING_AVERAGE);
100 + cfg->anomaly_detection_query_duration = anomaly_detection_query_duration;
101 + cfg->dimension_anomaly_score_threshold = dimension_anomaly_rate_threshold;
102
106 - Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
107 - Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
103 + cfg->hosts_to_skip = config_get(config_section_ml, "hosts to skip from training", "!*");
104 + cfg->sp_host_to_skip = simple_pattern_create(cfg->hosts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
105
106 // Always exclude anomaly_detection charts from training.
110 - Cfg.ChartsToSkip = "anomaly_detection.* ";
111 - Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");
112 - Cfg.SP_ChartsToSkip = simple_pattern_create(Cfg.ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
107 + cfg->charts_to_skip = "anomaly_detection.* ";
108 + cfg->charts_to_skip += config_get(config_section_ml, "charts to skip from training", "netdata.*");
109 + cfg->sp_charts_to_skip = simple_pattern_create(cfg->charts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
110
114 - Cfg.StreamADCharts = config_get_boolean(ConfigSectionML, "stream anomaly detection charts", true);
111 + cfg->stream_anomaly_detection_charts = config_get_boolean(config_section_ml, "stream anomaly detection charts", true);
112 }
ml/Config.h deleted
-52
@@ -1,52 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_CONFIG_H
4 -#define ML_CONFIG_H
5 -
6 -#include "ml-private.h"
7 -
8 -namespace ml {
9 -
10 -class Config {
11 -public:
12 - bool EnableAnomalyDetection;
13 -
14 - unsigned MaxTrainSamples;
15 - unsigned MinTrainSamples;
16 - unsigned TrainEvery;
17 -
18 - unsigned NumModelsToUse;
19 -
20 - unsigned DBEngineAnomalyRateEvery;
21 -
22 - unsigned DiffN;
23 - unsigned SmoothN;
24 - unsigned LagN;
25 -
26 - double RandomSamplingRatio;
27 - unsigned MaxKMeansIters;
28 -
29 - double DimensionAnomalyScoreThreshold;
30 -
31 - double HostAnomalyRateThreshold;
32 - RRDR_TIME_GROUPING AnomalyDetectionGroupingMethod;
33 - time_t AnomalyDetectionQueryDuration;
34 -
35 - bool StreamADCharts;
36 -
37 - std::string HostsToSkip;
38 - SIMPLE_PATTERN *SP_HostsToSkip;
39 -
40 - std::string ChartsToSkip;
41 - SIMPLE_PATTERN *SP_ChartsToSkip;
42 -
43 - std::vector<uint32_t> RandomNums;
44 -
45 - void readMLConfig();
46 -};
47 -
48 -extern Config Cfg;
49 -
50 -} // namespace ml
51 -
52 -#endif /* ML_CONFIG_H */
ml/Dimension.cc deleted
-346
@@ -1,346 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "Config.h"
4 -#include "Dimension.h"
5 -#include "Query.h"
6 -#include "Host.h"
7 -
8 -using namespace ml;
9 -
10 -static const char *mls2str(MachineLearningStatus MLS) {
11 - switch (MLS) {
12 - case ml::MachineLearningStatus::Enabled:
13 - return "enabled";
14 - case ml::MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
15 - return "disabled-ue";
16 - case ml::MachineLearningStatus::DisabledDueToExcludedChart:
17 - return "disabled-sp";
18 - default:
19 - return "unknown";
20 - }
21 -}
22 -
23 -static const char *mt2str(MetricType MT) {
24 - switch (MT) {
25 - case ml::MetricType::Constant:
26 - return "constant";
27 - case ml::MetricType::Variable:
28 - return "variable";
29 - default:
30 - return "unknown";
31 - }
32 -}
33 -
34 -static const char *ts2str(TrainingStatus TS) {
35 - switch (TS) {
36 - case ml::TrainingStatus::PendingWithModel:
37 - return "pending-with-model";
38 - case ml::TrainingStatus::PendingWithoutModel:
39 - return "pending-without-model";
40 - case ml::TrainingStatus::Trained:
41 - return "trained";
42 - case ml::TrainingStatus::Untrained:
43 - return "untrained";
44 - default:
45 - return "unknown";
46 - }
47 -}
48 -
49 -static const char *tr2str(TrainingResult TR) {
50 - switch (TR) {
51 - case ml::TrainingResult::Ok:
52 - return "ok";
53 - case ml::TrainingResult::InvalidQueryTimeRange:
54 - return "invalid-query";
55 - case ml::TrainingResult::NotEnoughCollectedValues:
56 - return "missing-values";
57 - case ml::TrainingResult::NullAcquiredDimension:
58 - return "null-acquired-dim";
59 - case ml::TrainingResult::ChartUnderReplication:
60 - return "chart-under-replication";
61 - default:
62 - return "unknown";
63 - }
64 -}
65 -
66 -std::pair<CalculatedNumber *, TrainingResponse> Dimension::getCalculatedNumbers(const TrainingRequest &TrainingReq) {
67 - TrainingResponse TrainingResp = {};
68 -
69 - TrainingResp.RequestTime = TrainingReq.RequestTime;
70 - TrainingResp.FirstEntryOnRequest = TrainingReq.FirstEntryOnRequest;
71 - TrainingResp.LastEntryOnRequest = TrainingReq.LastEntryOnRequest;
72 -
73 - TrainingResp.FirstEntryOnResponse = rrddim_first_entry_s_of_tier(RD, 0);
74 - TrainingResp.LastEntryOnResponse = rrddim_last_entry_s_of_tier(RD, 0);
75 -
76 - size_t MinN = Cfg.MinTrainSamples;
77 - size_t MaxN = Cfg.MaxTrainSamples;
78 -
79 - // Figure out what our time window should be.
80 - TrainingResp.QueryBeforeT = TrainingResp.LastEntryOnResponse;
81 - TrainingResp.QueryAfterT = std::max(
82 - TrainingResp.QueryBeforeT - static_cast<time_t>((MaxN - 1) * updateEvery()),
83 - TrainingResp.FirstEntryOnResponse
84 - );
85 -
86 - if (TrainingResp.QueryAfterT >= TrainingResp.QueryBeforeT) {
87 - TrainingResp.Result = TrainingResult::InvalidQueryTimeRange;
88 - return { nullptr, TrainingResp };
89 - }
90 -
91 - if (rrdset_is_replicating(RD->rrdset)) {
92 - TrainingResp.Result = TrainingResult::ChartUnderReplication;
93 - return { nullptr, TrainingResp };
94 - }
95 -
96 - CalculatedNumber *CNs = new CalculatedNumber[MaxN * (Cfg.LagN + 1)]();
97 -
98 - // Start the query.
99 - size_t Idx = 0;
100 -
101 - CalculatedNumber LastValue = std::numeric_limits<CalculatedNumber>::quiet_NaN();
102 - Query Q = Query(getRD());
103 -
104 - Q.init(TrainingResp.QueryAfterT, TrainingResp.QueryBeforeT);
105 - while (!Q.isFinished()) {
106 - if (Idx == MaxN)
107 - break;
108 -
109 - auto P = Q.nextMetric();
110 -
111 - CalculatedNumber Value = P.second;
112 -
113 - if (netdata_double_isnumber(Value)) {
114 - if (!TrainingResp.DbAfterT)
115 - TrainingResp.DbAfterT = P.first;
116 - TrainingResp.DbBeforeT = P.first;
117 -
118 - CNs[Idx] = Value;
119 - LastValue = CNs[Idx];
120 - TrainingResp.CollectedValues++;
121 - } else
122 - CNs[Idx] = LastValue;
123 -
124 - Idx++;
125 - }
126 - TrainingResp.TotalValues = Idx;
127 -
128 - if (TrainingResp.CollectedValues < MinN) {
129 - TrainingResp.Result = TrainingResult::NotEnoughCollectedValues;
130 -
131 - delete[] CNs;
132 - return { nullptr, TrainingResp };
133 - }
134 -
135 - // Find first non-NaN value.
136 - for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TrainingResp.TotalValues--) { }
137 -
138 - // Overwrite NaN values.
139 - if (Idx != 0)
140 - memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TrainingResp.TotalValues);
141 -
142 - TrainingResp.Result = TrainingResult::Ok;
143 - return { CNs, TrainingResp };
144 -}
145 -
146 -TrainingResult Dimension::trainModel(const TrainingRequest &TrainingReq) {
147 - auto P = getCalculatedNumbers(TrainingReq);
148 - CalculatedNumber *CNs = P.first;
149 - TrainingResponse TrainingResp = P.second;
150 -
151 - if (TrainingResp.Result != TrainingResult::Ok) {
152 - std::lock_guard<Mutex> L(M);
153 -
154 - MT = MetricType::Constant;
155 -
156 - switch (TS) {
157 - case TrainingStatus::PendingWithModel:
158 - TS = TrainingStatus::Trained;
159 - break;
160 - case TrainingStatus::PendingWithoutModel:
161 - TS = TrainingStatus::Untrained;
162 - break;
163 - default:
164 - break;
165 - }
166 -
167 - TR = TrainingResp;
168 -
169 - LastTrainingTime = TrainingResp.LastEntryOnResponse;
170 - return TrainingResp.Result;
171 - }
172 -
173 - unsigned N = TrainingResp.TotalValues;
174 - unsigned TargetNumSamples = Cfg.MaxTrainSamples * Cfg.RandomSamplingRatio;
175 - double SamplingRatio = std::min(static_cast<double>(TargetNumSamples) / N, 1.0);
176 -
177 - SamplesBuffer SB = SamplesBuffer(CNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
178 - SamplingRatio, Cfg.RandomNums);
179 - std::vector<DSample> Samples;
180 - SB.preprocess(Samples);
181 -
182 - KMeans KM;
183 - KM.train(Samples, Cfg.MaxKMeansIters);
184 -
185 - {
186 - std::lock_guard<Mutex> L(M);
187 -
188 - if (Models.size() < Cfg.NumModelsToUse) {
189 - Models.push_back(std::move(KM));
190 - } else {
191 - std::rotate(std::begin(Models), std::begin(Models) + 1, std::end(Models));
192 - Models[Models.size() - 1] = std::move(KM);
193 - }
194 -
195 - MT = MetricType::Constant;
196 - TS = TrainingStatus::Trained;
197 - TR = TrainingResp;
198 - LastTrainingTime = rrddim_last_entry_s(RD);
199 - }
200 -
201 - delete[] CNs;
202 - return TrainingResp.Result;
203 -}
204 -
205 -void Dimension::scheduleForTraining(time_t CurrT) {
206 - switch (MT) {
207 - case MetricType::Constant: {
208 - return;
209 - } default:
210 - break;
211 - }
212 -
213 - switch (TS) {
214 - case TrainingStatus::PendingWithModel:
215 - case TrainingStatus::PendingWithoutModel:
216 - break;
217 - case TrainingStatus::Untrained: {
218 - Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
219 - TS = TrainingStatus::PendingWithoutModel;
220 - H->scheduleForTraining(getTrainingRequest(CurrT));
221 - break;
222 - }
223 - case TrainingStatus::Trained: {
224 - bool NeedsTraining = (time_t)(LastTrainingTime + (Cfg.TrainEvery * updateEvery())) < CurrT;
225 -
226 - if (NeedsTraining) {
227 - Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
228 - TS = TrainingStatus::PendingWithModel;
229 - H->scheduleForTraining(getTrainingRequest(CurrT));
230 - }
231 - break;
232 - }
233 - }
234 -}
235 -
236 -bool Dimension::predict(time_t CurrT, CalculatedNumber Value, bool Exists) {
237 - // Nothing to do if ML is disabled for this dimension
238 - if (MLS != MachineLearningStatus::Enabled)
239 - return false;
240 -
241 - // Don't treat values that don't exist as anomalous
242 - if (!Exists) {
243 - CNs.clear();
244 - return false;
245 - }
246 -
247 - // Save the value and return if we don't have enough values for a sample
248 - unsigned N = Cfg.DiffN + Cfg.SmoothN + Cfg.LagN;
249 - if (CNs.size() < N) {
250 - CNs.push_back(Value);
251 - return false;
252 - }
253 -
254 - // Push the value and check if it's different from the last one
255 - bool SameValue = true;
256 - std::rotate(std::begin(CNs), std::begin(CNs) + 1, std::end(CNs));
257 - if (CNs[N - 1] != Value)
258 - SameValue = false;
259 - CNs[N - 1] = Value;
260 -
261 - // Create the sample
262 - CalculatedNumber TmpCNs[N * (Cfg.LagN + 1)];
263 - memset(TmpCNs, 0, N * (Cfg.LagN + 1) * sizeof(CalculatedNumber));
264 - std::memcpy(TmpCNs, CNs.data(), N * sizeof(CalculatedNumber));
265 - SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1,
266 - Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
267 - 1.0, Cfg.RandomNums);
268 - SB.preprocess(Feature);
269 -
270 - /*
271 - * Lock to predict and possibly schedule the dimension for training
272 - */
273 -
274 - std::unique_lock<Mutex> L(M, std::defer_lock);
275 - if (!L.try_lock()) {
276 - return false;
277 - }
278 -
279 - // Mark the metric time as variable if we received different values
280 - if (!SameValue)
281 - MT = MetricType::Variable;
282 -
283 - // Decide if the dimension needs to be scheduled for training
284 - scheduleForTraining(CurrT);
285 -
286 - // Nothing to do if we don't have a model
287 - switch (TS) {
288 - case TrainingStatus::Untrained:
289 - case TrainingStatus::PendingWithoutModel:
290 - return false;
291 - default:
292 - break;
293 - }
294 -
295 - /*
296 - * Use the KMeans models to check if the value is anomalous
297 - */
298 -
299 - size_t ModelsConsulted = 0;
300 - size_t Sum = 0;
301 -
302 - for (const auto &KM : Models) {
303 - ModelsConsulted++;
304 -
305 - double AnomalyScore = KM.anomalyScore(Feature);
306 - if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN())
307 - continue;
308 -
309 - if (AnomalyScore < (100 * Cfg.DimensionAnomalyScoreThreshold)) {
310 - global_statistics_ml_models_consulted(ModelsConsulted);
311 - return false;
312 - }
313 -
314 - Sum += 1;
315 - }
316 -
317 - global_statistics_ml_models_consulted(ModelsConsulted);
318 - return Sum;
319 -}
320 -
321 -std::vector<KMeans> Dimension::getModels() {
322 - std::unique_lock<Mutex> L(M);
323 - return Models;
324 -}
325 -
326 -void Dimension::dump() const {
327 - const char *ChartId = rrdset_id(RD->rrdset);
328 - const char *DimensionId = rrddim_id(RD);
329 -
330 - const char *MLS_Str = mls2str(MLS);
331 - const char *MT_Str = mt2str(MT);
332 - const char *TS_Str = ts2str(TS);
333 - const char *TR_Str = tr2str(TR.Result);
334 -
335 - const char *fmt =
336 - "[ML] %s.%s: MLS=%s, MT=%s, TS=%s, Result=%s, "
337 - "ReqTime=%ld, FEOReq=%ld, LEOReq=%ld, "
338 - "FEOResp=%ld, LEOResp=%ld, QTR=<%ld, %ld>, DBTR=<%ld, %ld>, Collected=%zu, Total=%zu";
339 -
340 - error(fmt,
341 - ChartId, DimensionId, MLS_Str, MT_Str, TS_Str, TR_Str,
342 - TR.RequestTime, TR.FirstEntryOnRequest, TR.LastEntryOnRequest,
343 - TR.FirstEntryOnResponse, TR.LastEntryOnResponse,
344 - TR.QueryAfterT, TR.QueryBeforeT, TR.DbAfterT, TR.DbBeforeT, TR.CollectedValues, TR.TotalValues
345 - );
346 -}
ml/Dimension.h deleted
-198
@@ -1,198 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_DIMENSION_H
4 -#define ML_DIMENSION_H
5 -
6 -#include "Mutex.h"
7 -#include "Stats.h"
8 -#include "Query.h"
9 -#include "Config.h"
10 -
11 -#include "ml-private.h"
12 -
13 -namespace ml {
14 -
15 -static inline std::string getMLDimensionID(RRDDIM *RD) {
16 - RRDSET *RS = RD->rrdset;
17 -
18 - std::stringstream SS;
19 - SS << rrdset_context(RS) << "|" << rrdset_id(RS) << "|" << rrddim_name(RD);
20 - return SS.str();
21 -}
22 -
23 -enum class MachineLearningStatus {
24 - // Enable training/prediction
25 - Enabled,
26 -
27 - // Disable due to update every being different from the host's
28 - DisabledDueToUniqueUpdateEvery,
29 -
30 - // Disable because configuration pattern matches the chart's id
31 - DisabledDueToExcludedChart,
32 -};
33 -
34 -enum class TrainingStatus {
35 - // We don't have a model for this dimension
36 - Untrained,
37 -
38 - // Request for training sent, but we don't have any models yet
39 - PendingWithoutModel,
40 -
41 - // Request to update existing models sent
42 - PendingWithModel,
43 -
44 - // Have a valid, up-to-date model
45 - Trained,
46 -};
47 -
48 -enum class MetricType {
49 - // The dimension has constant values, no need to train
50 - Constant,
51 -
52 - // The dimension's values fluctuate, we need to generate a model
53 - Variable,
54 -};
55 -
56 -struct TrainingRequest {
57 - // Chart/dimension we want to train
58 - STRING *ChartId;
59 - STRING *DimensionId;
60 -
61 - // Creation time of request
62 - time_t RequestTime;
63 -
64 - // First/last entry of this dimension in DB
65 - // at the point the request was made
66 - time_t FirstEntryOnRequest;
67 - time_t LastEntryOnRequest;
68 -};
69 -
70 -void dumpTrainingRequest(const TrainingRequest &TrainingReq, const char *Prefix);
71 -
72 -enum TrainingResult {
73 - // We managed to create a KMeans model
74 - Ok,
75 - // Could not query DB with a correct time range
76 - InvalidQueryTimeRange,
77 - // Did not gather enough data from DB to run KMeans
78 - NotEnoughCollectedValues,
79 - // Acquired a null dimension
80 - NullAcquiredDimension,
81 - // Chart is under replication
82 - ChartUnderReplication,
83 -};
84 -
85 -struct TrainingResponse {
86 - // Time when the request for this response was made
87 - time_t RequestTime;
88 -
89 - // First/last entry of the dimension in DB when generating the request
90 - time_t FirstEntryOnRequest;
91 - time_t LastEntryOnRequest;
92 -
93 - // First/last entry of the dimension in DB when generating the response
94 - time_t FirstEntryOnResponse;
95 - time_t LastEntryOnResponse;
96 -
97 - // After/Before timestamps of our DB query
98 - time_t QueryAfterT;
99 - time_t QueryBeforeT;
100 -
101 - // Actual after/before returned by the DB query ops
102 - time_t DbAfterT;
103 - time_t DbBeforeT;
104 -
105 - // Number of doubles returned by the DB query
106 - size_t CollectedValues;
107 -
108 - // Number of values we return to the caller
109 - size_t TotalValues;
110 -
111 - // Result of training response
112 - TrainingResult Result;
113 -};
114 -
115 -void dumpTrainingResponse(const TrainingResponse &TrainingResp, const char *Prefix);
116 -
117 -class Dimension {
118 -public:
119 - Dimension(RRDDIM *RD) :
120 - RD(RD),
121 - MT(MetricType::Constant),
122 - TS(TrainingStatus::Untrained),
123 - TR(),
124 - LastTrainingTime(0)
125 - {
126 - if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RD->rrdset)))
127 - MLS = MachineLearningStatus::DisabledDueToExcludedChart;
128 - else if (RD->update_every != RD->rrdset->rrdhost->rrd_update_every)
129 - MLS = MachineLearningStatus::DisabledDueToUniqueUpdateEvery;
130 - else
131 - MLS = MachineLearningStatus::Enabled;
132 -
133 - Models.reserve(Cfg.NumModelsToUse);
134 - }
135 -
136 - RRDDIM *getRD() const {
137 - return RD;
138 - }
139 -
140 - unsigned updateEvery() const {
141 - return RD->update_every;
142 - }
143 -
144 - MetricType getMT() const {
145 - return MT;
146 - }
147 -
148 - TrainingStatus getTS() const {
149 - return TS;
150 - }
151 -
152 - MachineLearningStatus getMLS() const {
153 - return MLS;
154 - }
155 -
156 - TrainingResult trainModel(const TrainingRequest &TR);
157 -
158 - void scheduleForTraining(time_t CurrT);
159 -
160 - bool predict(time_t CurrT, CalculatedNumber Value, bool Exists);
161 -
162 - std::vector<KMeans> getModels();
163 -
164 - void dump() const;
165 -
166 -private:
167 - TrainingRequest getTrainingRequest(time_t CurrT) const {
168 - return TrainingRequest {
169 - string_dup(RD->rrdset->id),
170 - string_dup(RD->id),
171 - CurrT,
172 - rrddim_first_entry_s(RD),
173 - rrddim_last_entry_s(RD)
174 - };
175 - }
176 -
177 -private:
178 - std::pair<CalculatedNumber *, TrainingResponse> getCalculatedNumbers(const TrainingRequest &TrainingReq);
179 -
180 -public:
181 - RRDDIM *RD;
182 - MetricType MT;
183 - TrainingStatus TS;
184 - TrainingResponse TR;
185 -
186 - time_t LastTrainingTime;
187 -
188 - MachineLearningStatus MLS;
189 -
190 - std::vector<CalculatedNumber> CNs;
191 - DSample Feature;
192 - std::vector<KMeans> Models;
193 - Mutex M;
194 -};
195 -
196 -} // namespace ml
197 -
198 -#endif /* ML_DIMENSION_H */
ml/Host.cc deleted
-387
@@ -1,387 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "Config.h"
4 -#include "Host.h"
5 -#include "Queue.h"
6 -#include "ADCharts.h"
7 -
8 -#include "json/single_include/nlohmann/json.hpp"
9 -
10 -using namespace ml;
11 -
12 -void Host::addChart(Chart *C) {
13 - std::lock_guard<Mutex> L(M);
14 - Charts[C->getRS()] = C;
15 -}
16 -
17 -void Host::removeChart(Chart *C) {
18 - std::lock_guard<Mutex> L(M);
19 - Charts.erase(C->getRS());
20 -}
21 -
22 -void Host::getConfigAsJson(BUFFER *wb) const {
23 - buffer_json_member_add_uint64(wb, "version", 1);
24 -
25 - buffer_json_member_add_boolean(wb, "enabled", Cfg.EnableAnomalyDetection);
26 -
27 - buffer_json_member_add_uint64(wb, "min-train-samples", Cfg.MinTrainSamples);
28 - buffer_json_member_add_uint64(wb, "max-train-samples", Cfg.MaxTrainSamples);
29 - buffer_json_member_add_uint64(wb, "train-every", Cfg.TrainEvery);
30 -
31 - buffer_json_member_add_uint64(wb, "diff-n", Cfg.DiffN);
32 - buffer_json_member_add_uint64(wb, "smooth-n", Cfg.SmoothN);
33 - buffer_json_member_add_uint64(wb, "lag-n", Cfg.LagN);
34 -
35 - buffer_json_member_add_double(wb, "random-sampling-ratio", Cfg.RandomSamplingRatio);
36 - buffer_json_member_add_uint64(wb, "max-kmeans-iters", Cfg.MaxKMeansIters);
37 -
38 - buffer_json_member_add_double(wb, "dimension-anomaly-score-threshold", Cfg.DimensionAnomalyScoreThreshold);
39 -
40 - buffer_json_member_add_double(wb, "host-anomaly-rate-threshold", Cfg.HostAnomalyRateThreshold);
41 - buffer_json_member_add_string(wb, "anomaly-detection-grouping-method", time_grouping_method2string(Cfg.AnomalyDetectionGroupingMethod));
42 - buffer_json_member_add_time_t(wb, "anomaly-detection-query-duration", Cfg.AnomalyDetectionQueryDuration);
43 -
44 - buffer_json_member_add_string(wb, "hosts-to-skip", Cfg.HostsToSkip.c_str());
45 - buffer_json_member_add_string(wb, "charts-to-skip", Cfg.ChartsToSkip.c_str());
46 -}
47 -
48 -void Host::getModelsAsJson(nlohmann::json &Json) {
49 - std::lock_guard<Mutex> L(M);
50 -
51 - for (auto &CP : Charts) {
52 - Chart *C = CP.second;
53 - C->getModelsAsJson(Json);
54 - }
55 -}
56 -
57 -#define WORKER_JOB_DETECTION_PREP 0
58 -#define WORKER_JOB_DETECTION_DIM_CHART 1
59 -#define WORKER_JOB_DETECTION_HOST_CHART 2
60 -#define WORKER_JOB_DETECTION_STATS 3
61 -#define WORKER_JOB_DETECTION_RESOURCES 4
62 -
63 -void Host::detectOnce() {
64 - worker_is_busy(WORKER_JOB_DETECTION_PREP);
65 -
66 - MLS = {};
67 - MachineLearningStats MLSCopy = {};
68 - TrainingStats TSCopy = {};
69 -
70 - {
71 - std::lock_guard<Mutex> L(M);
72 -
73 - /*
74 - * prediction/detection stats
75 - */
76 - for (auto &CP : Charts) {
77 - Chart *C = CP.second;
78 -
79 - if (!C->isAvailableForML())
80 - continue;
81 -
82 - MachineLearningStats ChartMLS = C->getMLS();
83 -
84 - MLS.NumMachineLearningStatusEnabled += ChartMLS.NumMachineLearningStatusEnabled;
85 - MLS.NumMachineLearningStatusDisabledUE += ChartMLS.NumMachineLearningStatusDisabledUE;
86 - MLS.NumMachineLearningStatusDisabledSP += ChartMLS.NumMachineLearningStatusDisabledSP;
87 -
88 - MLS.NumMetricTypeConstant += ChartMLS.NumMetricTypeConstant;
89 - MLS.NumMetricTypeVariable += ChartMLS.NumMetricTypeVariable;
90 -
91 - MLS.NumTrainingStatusUntrained += ChartMLS.NumTrainingStatusUntrained;
92 - MLS.NumTrainingStatusPendingWithoutModel += ChartMLS.NumTrainingStatusPendingWithoutModel;
93 - MLS.NumTrainingStatusTrained += ChartMLS.NumTrainingStatusTrained;
94 - MLS.NumTrainingStatusPendingWithModel += ChartMLS.NumTrainingStatusPendingWithModel;
95 -
96 - MLS.NumAnomalousDimensions += ChartMLS.NumAnomalousDimensions;
97 - MLS.NumNormalDimensions += ChartMLS.NumNormalDimensions;
98 - }
99 -
100 - HostAnomalyRate = 0.0;
101 - size_t NumActiveDimensions = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
102 - if (NumActiveDimensions)
103 - HostAnomalyRate = static_cast<double>(MLS.NumAnomalousDimensions) / NumActiveDimensions;
104 -
105 - MLSCopy = MLS;
106 -
107 - /*
108 - * training stats
109 - */
110 - TSCopy = TS;
111 -
112 - TS.QueueSize = 0;
113 - TS.NumPoppedItems = 0;
114 -
115 - TS.AllottedUT = 0;
116 - TS.ConsumedUT = 0;
117 - TS.RemainingUT = 0;
118 -
119 - TS.TrainingResultOk = 0;
120 - TS.TrainingResultInvalidQueryTimeRange = 0;
121 - TS.TrainingResultNotEnoughCollectedValues = 0;
122 - TS.TrainingResultNullAcquiredDimension = 0;
123 - TS.TrainingResultChartUnderReplication = 0;
124 - }
125 -
126 - // Calc the avg values
127 - if (TSCopy.NumPoppedItems) {
128 - TSCopy.QueueSize /= TSCopy.NumPoppedItems;
129 - TSCopy.AllottedUT /= TSCopy.NumPoppedItems;
130 - TSCopy.ConsumedUT /= TSCopy.NumPoppedItems;
131 - TSCopy.RemainingUT /= TSCopy.NumPoppedItems;
132 -
133 - TSCopy.TrainingResultOk /= TSCopy.NumPoppedItems;
134 - TSCopy.TrainingResultInvalidQueryTimeRange /= TSCopy.NumPoppedItems;
135 - TSCopy.TrainingResultNotEnoughCollectedValues /= TSCopy.NumPoppedItems;
136 - TSCopy.TrainingResultNullAcquiredDimension /= TSCopy.NumPoppedItems;
137 - TSCopy.TrainingResultChartUnderReplication /= TSCopy.NumPoppedItems;
138 - } else {
139 - TSCopy.QueueSize = 0;
140 - TSCopy.AllottedUT = 0;
141 - TSCopy.ConsumedUT = 0;
142 - TSCopy.RemainingUT = 0;
143 - }
144 -
145 - if(!RH)
146 - return;
147 -
148 - worker_is_busy(WORKER_JOB_DETECTION_DIM_CHART);
149 - updateDimensionsChart(RH, MLSCopy);
150 -
151 - worker_is_busy(WORKER_JOB_DETECTION_HOST_CHART);
152 - updateHostAndDetectionRateCharts(RH, HostAnomalyRate * 10000.0);
153 -
154 -#ifdef NETDATA_ML_RESOURCE_CHARTS
155 - worker_is_busy(WORKER_JOB_DETECTION_RESOURCES);
156 - struct rusage PredictionRU;
157 - getrusage(RUSAGE_THREAD, &PredictionRU);
158 - updateResourceUsageCharts(RH, PredictionRU, TSCopy.TrainingRU);
159 -#endif
160 -
161 - worker_is_busy(WORKER_JOB_DETECTION_STATS);
162 - updateTrainingStatisticsChart(RH, TSCopy);
163 -}
164 -
165 -class AcquiredDimension {
166 -public:
167 - static AcquiredDimension find(RRDHOST *RH, STRING *ChartId, STRING *DimensionId) {
168 - RRDDIM_ACQUIRED *AcqRD = nullptr;
169 - Dimension *D = nullptr;
170 -
171 - RRDSET *RS = rrdset_find(RH, string2str(ChartId));
172 - if (RS) {
173 - AcqRD = rrddim_find_and_acquire(RS, string2str(DimensionId));
174 - if (AcqRD) {
175 - RRDDIM *RD = rrddim_acquired_to_rrddim(AcqRD);
176 - if (RD)
177 - D = reinterpret_cast<Dimension *>(RD->ml_dimension);
178 - }
179 - }
180 -
181 - return AcquiredDimension(AcqRD, D);
182 - }
183 -
184 -private:
185 - AcquiredDimension(RRDDIM_ACQUIRED *AcqRD, Dimension *D) : AcqRD(AcqRD), D(D) {}
186 -
187 -public:
188 - TrainingResult train(const TrainingRequest &TR) {
189 - if (!D)
190 - return TrainingResult::NullAcquiredDimension;
191 -
192 - return D->trainModel(TR);
193 - }
194 -
195 - ~AcquiredDimension() {
196 - if (AcqRD)
197 - rrddim_acquired_release(AcqRD);
198 - }
199 -
200 -private:
201 - RRDDIM_ACQUIRED *AcqRD;
202 - Dimension *D;
203 -};
204 -
205 -void Host::scheduleForTraining(TrainingRequest TR) {
206 - TrainingQueue.push(TR);
207 -}
208 -
209 -#define WORKER_JOB_TRAINING_FIND 0
210 -#define WORKER_JOB_TRAINING_TRAIN 1
211 -#define WORKER_JOB_TRAINING_STATS 2
212 -
213 -void Host::train() {
214 - worker_register("MLTRAIN");
215 - worker_register_job_name(WORKER_JOB_TRAINING_FIND, "find");
216 - worker_register_job_name(WORKER_JOB_TRAINING_TRAIN, "train");
217 - worker_register_job_name(WORKER_JOB_TRAINING_STATS, "stats");
218 -
219 - service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, (force_quit_t )ml_cancel_anomaly_detection_threads, RH, true);
220 -
221 - while (service_running(SERVICE_ML_TRAINING)) {
222 - auto P = TrainingQueue.pop();
223 - TrainingRequest TrainingReq = P.first;
224 - size_t Size = P.second;
225 -
226 - if (ThreadsCancelled) {
227 - info("Stopping training thread because it was cancelled.");
228 - break;
229 - }
230 -
231 - usec_t AllottedUT = (Cfg.TrainEvery * RH->rrd_update_every * USEC_PER_SEC) / Size;
232 - if (AllottedUT > USEC_PER_SEC)
233 - AllottedUT = USEC_PER_SEC;
234 -
235 - usec_t StartUT = now_monotonic_usec();
236 - TrainingResult TrainingRes;
237 - {
238 - worker_is_busy(WORKER_JOB_TRAINING_FIND);
239 - AcquiredDimension AcqDim = AcquiredDimension::find(RH, TrainingReq.ChartId, TrainingReq.DimensionId);
240 -
241 - worker_is_busy(WORKER_JOB_TRAINING_TRAIN);
242 - TrainingRes = AcqDim.train(TrainingReq);
243 -
244 - string_freez(TrainingReq.ChartId);
245 - string_freez(TrainingReq.DimensionId);
246 - }
247 - usec_t ConsumedUT = now_monotonic_usec() - StartUT;
248 -
249 - worker_is_busy(WORKER_JOB_TRAINING_STATS);
250 -
251 - usec_t RemainingUT = 0;
252 - if (ConsumedUT < AllottedUT)
253 - RemainingUT = AllottedUT - ConsumedUT;
254 -
255 - {
256 - std::lock_guard<Mutex> L(M);
257 -
258 - if (TS.AllottedUT == 0) {
259 - struct rusage TRU;
260 - getrusage(RUSAGE_THREAD, &TRU);
261 - TS.TrainingRU = TRU;
262 - }
263 -
264 - TS.QueueSize += Size;
265 - TS.NumPoppedItems += 1;
266 -
267 - TS.AllottedUT += AllottedUT;
268 - TS.ConsumedUT += ConsumedUT;
269 - TS.RemainingUT += RemainingUT;
270 -
271 - switch (TrainingRes) {
272 - case TrainingResult::Ok:
273 - TS.TrainingResultOk += 1;
274 - break;
275 - case TrainingResult::InvalidQueryTimeRange:
276 - TS.TrainingResultInvalidQueryTimeRange += 1;
277 - break;
278 - case TrainingResult::NotEnoughCollectedValues:
279 - TS.TrainingResultNotEnoughCollectedValues += 1;
280 - break;
281 - case TrainingResult::NullAcquiredDimension:
282 - TS.TrainingResultNullAcquiredDimension += 1;
283 - break;
284 - case TrainingResult::ChartUnderReplication:
285 - TS.TrainingResultChartUnderReplication += 1;
286 - break;
287 - }
288 - }
289 -
290 - worker_is_idle();
291 - std::this_thread::sleep_for(std::chrono::microseconds{RemainingUT});
292 - worker_is_busy(0);
293 - }
294 -}
295 -
296 -void Host::detect() {
297 - worker_register("MLDETECT");
298 - worker_register_job_name(WORKER_JOB_DETECTION_PREP, "prep");
299 - worker_register_job_name(WORKER_JOB_DETECTION_DIM_CHART, "dim chart");
300 - worker_register_job_name(WORKER_JOB_DETECTION_HOST_CHART, "host chart");
301 - worker_register_job_name(WORKER_JOB_DETECTION_STATS, "stats");
302 - worker_register_job_name(WORKER_JOB_DETECTION_RESOURCES, "resources");
303 -
304 - service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, (force_quit_t )ml_cancel_anomaly_detection_threads, RH, true);
305 -
306 - heartbeat_t HB;
307 - heartbeat_init(&HB);
308 -
309 - while (service_running((SERVICE_TYPE)(SERVICE_ML_PREDICTION | SERVICE_COLLECTORS))) {
310 - worker_is_idle();
311 - heartbeat_next(&HB, (RH ? RH->rrd_update_every : default_rrd_update_every) * USEC_PER_SEC);
312 - detectOnce();
313 - }
314 -}
315 -
316 -void Host::getDetectionInfoAsJson(nlohmann::json &Json) const {
317 - Json["version"] = 1;
318 - Json["anomalous-dimensions"] = MLS.NumAnomalousDimensions;
319 - Json["normal-dimensions"] = MLS.NumNormalDimensions;
320 - Json["total-dimensions"] = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
321 - Json["trained-dimensions"] = MLS.NumTrainingStatusTrained + MLS.NumTrainingStatusPendingWithModel;
322 -}
323 -
324 -void *train_main(void *Arg) {
325 - Host *H = reinterpret_cast<Host *>(Arg);
326 - H->train();
327 - return nullptr;
328 -}
329 -
330 -void *detect_main(void *Arg) {
331 - Host *H = reinterpret_cast<Host *>(Arg);
332 - H->detect();
333 - return nullptr;
334 -}
335 -
336 -void Host::startAnomalyDetectionThreads() {
337 - if (ThreadsRunning) {
338 - error("Anomaly detections threads for host %s are already-up and running.", rrdhost_hostname(RH));
339 - return;
340 - }
341 -
342 - ThreadsRunning = true;
343 - ThreadsCancelled = false;
344 - ThreadsJoined = false;
345 -
346 - char Tag[NETDATA_THREAD_TAG_MAX + 1];
347 -
348 -// #define ML_DISABLE_JOINING
349 -
350 - snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "MLTR[%s]", rrdhost_hostname(RH));
351 - netdata_thread_create(&TrainingThread, Tag, NETDATA_THREAD_OPTION_JOINABLE, train_main, static_cast<void *>(this));
352 -
353 - snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "MLDT[%s]", rrdhost_hostname(RH));
354 - netdata_thread_create(&DetectionThread, Tag, NETDATA_THREAD_OPTION_JOINABLE, detect_main, static_cast<void *>(this));
355 -}
356 -
357 -void Host::stopAnomalyDetectionThreads(bool join) {
358 - if (!ThreadsRunning) {
359 - error("Anomaly detections threads for host %s have already been stopped.", rrdhost_hostname(RH));
360 - return;
361 - }
362 -
363 - if(!ThreadsCancelled) {
364 - ThreadsCancelled = true;
365 -
366 - // Signal the training queue to stop popping-items
367 - TrainingQueue.signal();
368 - netdata_thread_cancel(TrainingThread);
369 - netdata_thread_cancel(DetectionThread);
370 - }
371 -
372 - if (join && !ThreadsJoined) {
373 - ThreadsJoined = true;
374 - ThreadsRunning = false;
375 -
376 - // these fail on alpine linux and our CI hangs forever
377 - // failing to compile static builds
378 -
379 - // commenting them, until we find a solution
380 -
381 - // to enable again:
382 - // NETDATA_THREAD_OPTION_DEFAULT needs to become NETDATA_THREAD_OPTION_JOINABLE
383 -
384 - netdata_thread_join(TrainingThread, nullptr);
385 - netdata_thread_join(DetectionThread, nullptr);
386 - }
387 -}
ml/Host.h deleted
-70
@@ -1,70 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_HOST_H
4 -#define ML_HOST_H
5 -
6 -#include "Mutex.h"
7 -#include "Config.h"
8 -#include "Dimension.h"
9 -#include "Chart.h"
10 -#include "Queue.h"
11 -
12 -#include "ml-private.h"
13 -#include "json/single_include/nlohmann/json.hpp"
14 -
15 -namespace ml
16 -{
17 -
18 -class Host {
19 -
20 -friend void* train_main(void *);
21 -friend void *detect_main(void *);
22 -
23 -public:
24 - Host(RRDHOST *RH) :
25 - RH(RH),
26 - MLS(),
27 - TS(),
28 - HostAnomalyRate(0.0),
29 - ThreadsRunning(false),
30 - ThreadsCancelled(false),
31 - ThreadsJoined(false)
32 - {}
33 -
34 - void addChart(Chart *C);
35 - void removeChart(Chart *C);
36 -
37 - void getConfigAsJson(BUFFER *wb) const;
38 - void getModelsAsJson(nlohmann::json &Json);
39 - void getDetectionInfoAsJson(nlohmann::json &Json) const;
40 -
41 - void startAnomalyDetectionThreads();
42 - void stopAnomalyDetectionThreads(bool join);
43 -
44 - void scheduleForTraining(TrainingRequest TR);
45 - void train();
46 -
47 - void detect();
48 - void detectOnce();
49 -
50 -private:
51 - RRDHOST *RH;
52 - MachineLearningStats MLS;
53 - TrainingStats TS;
54 - CalculatedNumber HostAnomalyRate{0.0};
55 - std::atomic<bool> ThreadsRunning;
56 - std::atomic<bool> ThreadsCancelled;
57 - std::atomic<bool> ThreadsJoined;
58 -
59 - Queue<TrainingRequest> TrainingQueue;
60 -
61 - Mutex M;
62 - std::unordered_map<RRDSET *, Chart *> Charts;
63 -
64 - netdata_thread_t TrainingThread;
65 - netdata_thread_t DetectionThread;
66 -};
67 -
68 -} // namespace ml
69 -
70 -#endif /* ML_HOST_H */
ml/KMeans.cc deleted
-43
@@ -1,43 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "KMeans.h"
4 -#include <dlib/clustering.h>
5 -
6 -void KMeans::train(const std::vector<DSample> &Samples, size_t MaxIterations) {
7 - MinDist = std::numeric_limits<CalculatedNumber>::max();
8 - MaxDist = std::numeric_limits<CalculatedNumber>::min();
9 -
10 - ClusterCenters.clear();
11 -
12 - dlib::pick_initial_centers(NumClusters, ClusterCenters, Samples);
13 - dlib::find_clusters_using_kmeans(Samples, ClusterCenters, MaxIterations);
14 -
15 - for (const auto &S : Samples) {
16 - CalculatedNumber MeanDist = 0.0;
17 -
18 - for (const auto &KMCenter : ClusterCenters)
19 - MeanDist += dlib::length(KMCenter - S);
20 -
21 - MeanDist /= NumClusters;
22 -
23 - if (MeanDist < MinDist)
24 - MinDist = MeanDist;
25 -
26 - if (MeanDist > MaxDist)
27 - MaxDist = MeanDist;
28 - }
29 -}
30 -
31 -CalculatedNumber KMeans::anomalyScore(const DSample &Sample) const {
32 - CalculatedNumber MeanDist = 0.0;
33 - for (const auto &CC: ClusterCenters)
34 - MeanDist += dlib::length(CC - Sample);
35 -
36 - MeanDist /= NumClusters;
37 -
38 - if (MaxDist == MinDist)
39 - return 0.0;
40 -
41 - CalculatedNumber AnomalyScore = 100.0 * std::abs((MeanDist - MinDist) / (MaxDist - MinDist));
42 - return (AnomalyScore > 100.0) ? 100.0 : AnomalyScore;
43 -}
ml/KMeans.h deleted
-41
@@ -1,41 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef KMEANS_H
4 -#define KMEANS_H
5 -
6 -#include <atomic>
7 -#include <vector>
8 -#include <limits>
9 -#include <mutex>
10 -
11 -#include "SamplesBuffer.h"
12 -#include "json/single_include/nlohmann/json.hpp"
13 -
14 -class KMeans {
15 -public:
16 - KMeans(size_t NumClusters = 2) : NumClusters(NumClusters) {
17 - MinDist = std::numeric_limits<CalculatedNumber>::max();
18 - MaxDist = std::numeric_limits<CalculatedNumber>::min();
19 - };
20 -
21 - void train(const std::vector<DSample> &Samples, size_t MaxIterations);
22 - CalculatedNumber anomalyScore(const DSample &Sample) const;
23 -
24 - void toJson(nlohmann::json &J) const {
25 - J = nlohmann::json{
26 - {"CCs", ClusterCenters},
27 - {"MinDist", MinDist},
28 - {"MaxDist", MaxDist}
29 - };
30 - }
31 -
32 -private:
33 - size_t NumClusters;
34 -
35 - std::vector<DSample> ClusterCenters;
36 -
37 - CalculatedNumber MinDist;
38 - CalculatedNumber MaxDist;
39 -};
40 -
41 -#endif /* KMEANS_H */
ml/Mutex.h deleted
-36
@@ -1,36 +0,0 @@
1 -#ifndef ML_MUTEX_H
2 -#define ML_MUTEX_H
3 -
4 -#include "ml-private.h"
5 -
6 -class Mutex {
7 -public:
8 - Mutex() {
9 - netdata_mutex_init(&M);
10 - }
11 -
12 - void lock() {
13 - netdata_mutex_lock(&M);
14 - }
15 -
16 - void unlock() {
17 - netdata_mutex_unlock(&M);
18 - }
19 -
20 - bool try_lock() {
21 - return netdata_mutex_trylock(&M) == 0;
22 - }
23 -
24 - netdata_mutex_t *inner() {
25 - return &M;
26 - }
27 -
28 - ~Mutex() {
29 - netdata_mutex_destroy(&M);
30 - }
31 -
32 -private:
33 - netdata_mutex_t M;
34 -};
35 -
36 -#endif /* ML_MUTEX_H */
ml/Query.h deleted
-57
@@ -1,57 +0,0 @@
1 -#ifndef QUERY_H
2 -#define QUERY_H
3 -
4 -#include "ml-private.h"
5 -
6 -namespace ml {
7 -
8 -class Query {
9 -public:
10 - Query(RRDDIM *RD) : RD(RD), Initialized(false) {
11 - Ops = RD->tiers[0].query_ops;
12 - }
13 -
14 - time_t latestTime() {
15 - return Ops->latest_time_s(RD->tiers[0].db_metric_handle);
16 - }
17 -
18 - time_t oldestTime() {
19 - return Ops->oldest_time_s(RD->tiers[0].db_metric_handle);
20 - }
21 -
22 - void init(time_t AfterT, time_t BeforeT) {
23 - Ops->init(RD->tiers[0].db_metric_handle, &Handle, AfterT, BeforeT, STORAGE_PRIORITY_BEST_EFFORT);
24 - Initialized = true;
25 - points_read = 0;
26 - }
27 -
28 - bool isFinished() {
29 - return Ops->is_finished(&Handle);
30 - }
31 -
32 - ~Query() {
33 - if (Initialized) {
34 - Ops->finalize(&Handle);
35 - global_statistics_ml_query_completed(points_read);
36 - points_read = 0;
37 - }
38 - }
39 -
40 - std::pair<time_t, CalculatedNumber> nextMetric() {
41 - points_read++;
42 - STORAGE_POINT sp = Ops->next_metric(&Handle);
43 - return {sp.end_time_s, sp.sum / sp.count };
44 - }
45 -
46 -private:
47 - RRDDIM *RD;
48 - bool Initialized;
49 - size_t points_read;
50 -
51 - struct storage_engine_query_ops *Ops;
52 - struct storage_engine_query_handle Handle;
53 -};
54 -
55 -} // namespace ml
56 -
57 -#endif /* QUERY_H */
ml/Queue.h deleted
-66
@@ -1,66 +0,0 @@
1 -#ifndef QUEUE_H
2 -#define QUEUE_H
3 -
4 -#include "ml-private.h"
5 -#include "Mutex.h"
6 -#include <queue>
7 -#include <mutex>
8 -#include <condition_variable>
9 -
10 -template<typename T>
11 -class Queue {
12 -public:
13 - Queue(void) : Q(), M() {
14 - pthread_cond_init(&CV, nullptr);
15 - Exit = false;
16 - }
17 -
18 - ~Queue() {
19 - pthread_cond_destroy(&CV);
20 - }
21 -
22 - void push(T t) {
23 - std::lock_guard<Mutex> L(M);
24 -
25 - Q.push(t);
26 - pthread_cond_signal(&CV);
27 - }
28 -
29 - std::pair<T, size_t> pop(void) {
30 - std::lock_guard<Mutex> L(M);
31 -
32 - while (Q.empty()) {
33 - pthread_cond_wait(&CV, M.inner());
34 -
35 - if (Exit) {
36 - // This should happen only when we are destroying a host.
37 - // Callers should use a flag dedicated to checking if we
38 - // are about to delete the host or exit the agent. The original
39 - // implementation would call pthread_exit which would cause
40 - // the queue's mutex to be destroyed twice (and fail on the
41 - // 2nd time)
42 - return { T(), 0 };
43 - }
44 - }
45 -
46 - T V = Q.front();
47 - size_t Size = Q.size();
48 - Q.pop();
49 -
50 - return { V, Size };
51 - }
52 -
53 - void signal() {
54 - std::lock_guard<Mutex> L(M);
55 - Exit = true;
56 - pthread_cond_signal(&CV);
57 - }
58 -
59 -private:
60 - std::queue<T> Q;
61 - Mutex M;
62 - pthread_cond_t CV;
63 - std::atomic<bool> Exit;
64 -};
65 -
66 -#endif /* QUEUE_H */
ml/SamplesBuffer.cc deleted
-183
@@ -1,183 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -//
3 -#include "SamplesBuffer.h"
4 -
5 -#include <fstream>
6 -#include <sstream>
7 -#include <string>
8 -
9 -void Sample::print(std::ostream &OS) const {
10 - for (size_t Idx = 0; Idx != NumDims - 1; Idx++)
11 - OS << CNs[Idx] << ", ";
12 -
13 - OS << CNs[NumDims - 1];
14 -}
15 -
16 -void SamplesBuffer::print(std::ostream &OS) const {
17 - for (size_t Idx = Preprocessed ? (DiffN + (SmoothN - 1) + (LagN)) : 0;
18 - Idx != NumSamples; Idx++) {
19 - Sample S = Preprocessed ? getPreprocessedSample(Idx) : getSample(Idx);
20 - OS << S << std::endl;
21 - }
22 -}
23 -
24 -std::vector<Sample> SamplesBuffer::getPreprocessedSamples() const {
25 - std::vector<Sample> V;
26 -
27 - for (size_t Idx = Preprocessed ? (DiffN + (SmoothN - 1) + (LagN)) : 0;
28 - Idx != NumSamples; Idx++) {
29 - Sample S = Preprocessed ? getPreprocessedSample(Idx) : getSample(Idx);
30 - V.push_back(S);
31 - }
32 -
33 - return V;
34 -}
35 -
36 -void SamplesBuffer::diffSamples() {
37 - // Panda's DataFrame default behaviour is to subtract each element from
38 - // itself. For us `DiffN = 0` means "disable diff-ing" when preprocessing
39 - // the samples buffer. This deviation will make it easier for us to test
40 - // the KMeans implementation.
41 - if (DiffN == 0)
42 - return;
43 -
44 - for (size_t Idx = 0; Idx != (NumSamples - DiffN); Idx++) {
45 - size_t High = (NumSamples - 1) - Idx;
46 - size_t Low = High - DiffN;
47 -
48 - Sample LHS = getSample(High);
49 - Sample RHS = getSample(Low);
50 -
51 - LHS.diff(RHS);
52 - }
53 -}
54 -
55 -void SamplesBuffer::smoothSamples() {
56 - // Holds the mean value of each window
57 - CalculatedNumber AccCNs[1] = { 0 };
58 - Sample Acc(AccCNs, 1);
59 -
60 - // Used to avoid clobbering the accumulator when moving the window
61 - CalculatedNumber TmpCNs[1] = { 0 };
62 - Sample Tmp(TmpCNs, 1);
63 -
64 - CalculatedNumber Factor = (CalculatedNumber) 1 / SmoothN;
65 -
66 - // Calculate the value of the 1st window
67 - for (size_t Idx = 0; Idx != std::min(SmoothN, NumSamples); Idx++) {
68 - Tmp.add(getSample(NumSamples - (Idx + 1)));
69 - }
70 -
71 - Acc.add(Tmp);
72 - Acc.scale(Factor);
73 -
74 - // Move the window and update the samples
75 - for (size_t Idx = NumSamples; Idx != (DiffN + SmoothN - 1); Idx--) {
76 - Sample S = getSample(Idx - 1);
77 -
78 - // Tmp <- Next window (if any)
79 - if (Idx >= (SmoothN + 1)) {
80 - Tmp.diff(S);
81 - Tmp.add(getSample(Idx - (SmoothN + 1)));
82 - }
83 -
84 - // S <- Acc
85 - S.copy(Acc);
86 -
87 - // Acc <- Tmp
88 - Acc.copy(Tmp);
89 - Acc.scale(Factor);
90 - }
91 -}
92 -
93 -void SamplesBuffer::lagSamples() {
94 - if (LagN == 0)
95 - return;
96 -
97 - for (size_t Idx = NumSamples; Idx != LagN; Idx--) {
98 - Sample PS = getPreprocessedSample(Idx - 1);
99 - PS.lag(getSample(Idx - 1), LagN);
100 - }
101 -}
102 -
103 -void SamplesBuffer::preprocess(std::vector<DSample> &Samples) {
104 - assert(Preprocessed == false);
105 -
106 - size_t OutN = NumSamples;
107 -
108 - // Diff
109 - if (DiffN >= OutN)
110 - return;
111 - OutN -= DiffN;
112 - diffSamples();
113 -
114 - // Smooth
115 - if (SmoothN == 0 || SmoothN > OutN)
116 - return;
117 - OutN -= (SmoothN - 1);
118 - smoothSamples();
119 -
120 - // Lag
121 - if (LagN >= OutN)
122 - return;
123 - OutN -= LagN;
124 - lagSamples();
125 -
126 - Samples.reserve(OutN);
127 - Preprocessed = true;
128 -
129 - uint32_t MaxMT = std::numeric_limits<uint32_t>::max();
130 - uint32_t CutOff = static_cast<double>(MaxMT) * SamplingRatio;
131 -
132 - for (size_t Idx = NumSamples - OutN; Idx != NumSamples; Idx++) {
133 - if (RandNums[Idx] > CutOff)
134 - continue;
135 -
136 - DSample DS;
137 - DS.set_size(NumDimsPerSample * (LagN + 1));
138 -
139 - const Sample PS = getPreprocessedSample(Idx);
140 - PS.initDSample(DS);
141 -
142 - Samples.push_back(std::move(DS));
143 - }
144 -}
145 -
146 -void SamplesBuffer::preprocess(DSample &Feature) {
147 - assert(Preprocessed == false);
148 -
149 - size_t OutN = NumSamples;
150 -
151 - // Diff
152 - if (DiffN >= OutN)
153 - return;
154 - OutN -= DiffN;
155 - diffSamples();
156 -
157 - // Smooth
158 - if (SmoothN == 0 || SmoothN > OutN)
159 - return;
160 - OutN -= (SmoothN - 1);
161 - smoothSamples();
162 -
163 - // Lag
164 - if (LagN >= OutN)
165 - return;
166 - OutN -= LagN;
167 - lagSamples();
168 -
169 - Preprocessed = true;
170 -
171 - uint32_t MaxMT = std::numeric_limits<uint32_t>::max();
172 - uint32_t CutOff = static_cast<double>(MaxMT) * SamplingRatio;
173 -
174 - for (size_t Idx = NumSamples - OutN; Idx != NumSamples; Idx++) {
175 - if (RandNums[Idx] > CutOff)
176 - continue;
177 -
178 - Feature.set_size(NumDimsPerSample * (LagN + 1));
179 -
180 - const Sample PS = getPreprocessedSample(Idx);
181 - PS.initDSample(Feature);
182 - }
183 -}
ml/SamplesBuffer.h deleted
-149
@@ -1,149 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef SAMPLES_BUFFER_H
4 -#define SAMPLES_BUFFER_H
5 -
6 -#include <iostream>
7 -#include <vector>
8 -
9 -#include <cassert>
10 -#include <cstdlib>
11 -#include <cstring>
12 -
13 -#include <dlib/matrix.h>
14 -
15 -typedef double CalculatedNumber;
16 -typedef dlib::matrix<CalculatedNumber, 0, 1> DSample;
17 -
18 -class Sample {
19 -public:
20 - Sample(CalculatedNumber *Buf, size_t N) : CNs(Buf), NumDims(N) {}
21 -
22 - void initDSample(DSample &DS) const {
23 - for (size_t Idx = 0; Idx != NumDims; Idx++) {
24 - DS(Idx) = std::abs(CNs[Idx]);
25 - }
26 - }
27 -
28 - void add(const Sample &RHS) const {
29 - assert(NumDims == RHS.NumDims);
30 -
31 - for (size_t Idx = 0; Idx != NumDims; Idx++)
32 - CNs[Idx] += RHS.CNs[Idx];
33 - };
34 -
35 - void diff(const Sample &RHS) const {
36 - assert(NumDims == RHS.NumDims);
37 -
38 - for (size_t Idx = 0; Idx != NumDims; Idx++)
39 - CNs[Idx] -= RHS.CNs[Idx];
40 - };
41 -
42 - void copy(const Sample &RHS) const {
43 - assert(NumDims == RHS.NumDims);
44 -
45 - std::memcpy(CNs, RHS.CNs, NumDims * sizeof(CalculatedNumber));
46 - }
47 -
48 - void scale(CalculatedNumber Factor) {
49 - for (size_t Idx = 0; Idx != NumDims; Idx++)
50 - CNs[Idx] *= Factor;
51 - }
52 -
53 - void lag(const Sample &S, size_t LagN) {
54 - size_t N = S.NumDims;
55 -
56 - for (size_t Idx = 0; Idx != (LagN + 1); Idx++) {
57 - Sample Src(S.CNs - (Idx * N), N);
58 - Sample Dst(CNs + (Idx * N), N);
59 - Dst.copy(Src);
60 - }
61 - }
62 -
63 - const CalculatedNumber *getCalculatedNumbers() const {
64 - return CNs;
65 - };
66 -
67 - void print(std::ostream &OS) const;
68 -
69 -private:
70 - CalculatedNumber *CNs;
71 - size_t NumDims;
72 -};
73 -
74 -inline std::ostream& operator<<(std::ostream &OS, const Sample &S) {
75 - S.print(OS);
76 - return OS;
77 -}
78 -
79 -class SamplesBuffer {
80 -public:
81 - SamplesBuffer(CalculatedNumber *CNs,
82 - size_t NumSamples, size_t NumDimsPerSample,
83 - size_t DiffN, size_t SmoothN, size_t LagN,
84 - double SamplingRatio, std::vector<uint32_t> &RandNums) :
85 - CNs(CNs), NumSamples(NumSamples), NumDimsPerSample(NumDimsPerSample),
86 - DiffN(DiffN), SmoothN(SmoothN), LagN(LagN),
87 - SamplingRatio(SamplingRatio), RandNums(RandNums),
88 - BytesPerSample(NumDimsPerSample * sizeof(CalculatedNumber)),
89 - Preprocessed(false) {
90 - assert(NumDimsPerSample == 1 && "SamplesBuffer supports only one dimension per sample");
91 - };
92 -
93 - void preprocess(std::vector<DSample> &Samples);
94 - void preprocess(DSample &Feature);
95 - std::vector<Sample> getPreprocessedSamples() const;
96 -
97 - size_t capacity() const { return NumSamples; }
98 - void print(std::ostream &OS) const;
99 -
100 -private:
101 - size_t getSampleOffset(size_t Index) const {
102 - assert(Index < NumSamples);
103 - return Index * NumDimsPerSample;
104 - }
105 -
106 - size_t getPreprocessedSampleOffset(size_t Index) const {
107 - assert(Index < NumSamples);
108 - return getSampleOffset(Index) * (LagN + 1);
109 - }
110 -
111 - void setSample(size_t Index, const Sample &S) const {
112 - size_t Offset = getSampleOffset(Index);
113 - std::memcpy(&CNs[Offset], S.getCalculatedNumbers(), BytesPerSample);
114 - }
115 -
116 - const Sample getSample(size_t Index) const {
117 - size_t Offset = getSampleOffset(Index);
118 - return Sample(&CNs[Offset], NumDimsPerSample);
119 - };
120 -
121 - const Sample getPreprocessedSample(size_t Index) const {
122 - size_t Offset = getPreprocessedSampleOffset(Index);
123 - return Sample(&CNs[Offset], NumDimsPerSample * (LagN + 1));
124 - };
125 -
126 - void diffSamples();
127 - void smoothSamples();
128 - void lagSamples();
129 -
130 -private:
131 - CalculatedNumber *CNs;
132 - size_t NumSamples;
133 - size_t NumDimsPerSample;
134 - size_t DiffN;
135 - size_t SmoothN;
136 - size_t LagN;
137 - double SamplingRatio;
138 - std::vector<uint32_t> &RandNums;
139 -
140 - size_t BytesPerSample;
141 - bool Preprocessed;
142 -};
143 -
144 -inline std::ostream& operator<<(std::ostream& OS, const SamplesBuffer &SB) {
145 - SB.print(OS);
146 - return OS;
147 -}
148 -
149 -#endif /* SAMPLES_BUFFER_H */
ml/Stats.h deleted
-46
@@ -1,46 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_STATS_H
4 -#define ML_STATS_H
5 -
6 -#include "ml-private.h"
7 -
8 -namespace ml {
9 -
10 -struct MachineLearningStats {
11 - size_t NumMachineLearningStatusEnabled;
12 - size_t NumMachineLearningStatusDisabledUE;
13 - size_t NumMachineLearningStatusDisabledSP;
14 -
15 - size_t NumMetricTypeConstant;
16 - size_t NumMetricTypeVariable;
17 -
18 - size_t NumTrainingStatusUntrained;
19 - size_t NumTrainingStatusPendingWithoutModel;
20 - size_t NumTrainingStatusTrained;
21 - size_t NumTrainingStatusPendingWithModel;
22 -
23 - size_t NumAnomalousDimensions;
24 - size_t NumNormalDimensions;
25 -};
26 -
27 -struct TrainingStats {
28 - struct rusage TrainingRU;
29 -
30 - size_t QueueSize;
31 - size_t NumPoppedItems;
32 -
33 - usec_t AllottedUT;
34 - usec_t ConsumedUT;
35 - usec_t RemainingUT;
36 -
37 - size_t TrainingResultOk;
38 - size_t TrainingResultInvalidQueryTimeRange;
39 - size_t TrainingResultNotEnoughCollectedValues;
40 - size_t TrainingResultNullAcquiredDimension;
41 - size_t TrainingResultChartUnderReplication;
42 -};
43 -
44 -} // namespace ml
45 -
46 -#endif /* ML_STATS_H */
ml/ad_charts.cc new
+446
@@ -0,0 +1,446 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "ad_charts.h"
4 +
5 +void nml_update_dimensions_chart(nml_host_t *host, const nml_machine_learning_stats_t &mls) {
6 + /*
7 + * Machine learning status
8 + */
9 + {
10 + if (!host->machine_learning_status_rs) {
11 + char id_buf[1024];
12 + char name_buf[1024];
13 +
14 + snprintfz(id_buf, 1024, "machine_learning_status_on_%s", localhost->machine_guid);
15 + snprintfz(name_buf, 1024, "machine_learning_status_on_%s", rrdhost_hostname(localhost));
16 +
17 + host->machine_learning_status_rs = rrdset_create(
18 + host->rh,
19 + "netdata", // type
20 + id_buf,
21 + name_buf, // name
22 + NETDATA_ML_CHART_FAMILY, // family
23 + "netdata.machine_learning_status", // ctx
24 + "Machine learning status", // title
25 + "dimensions", // units
26 + NETDATA_ML_PLUGIN, // plugin
27 + NETDATA_ML_MODULE_TRAINING, // module
28 + NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
29 + localhost->rrd_update_every, // update_every
30 + RRDSET_TYPE_LINE // chart_type
31 + );
32 + rrdset_flag_set(host->machine_learning_status_rs , RRDSET_FLAG_ANOMALY_DETECTION);
33 +
34 + host->machine_learning_status_enabled_rd =
35 + rrddim_add(host->machine_learning_status_rs, "enabled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
36 + host->machine_learning_status_disabled_sp_rd =
37 + rrddim_add(host->machine_learning_status_rs, "disabled-sp", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
38 + }
39 +
40 + rrddim_set_by_pointer(host->machine_learning_status_rs,
41 + host->machine_learning_status_enabled_rd, mls.num_machine_learning_status_enabled);
42 + rrddim_set_by_pointer(host->machine_learning_status_rs,
43 + host->machine_learning_status_disabled_sp_rd, mls.num_machine_learning_status_disabled_sp);
44 +
45 + rrdset_done(host->machine_learning_status_rs);
46 + }
47 +
48 + /*
49 + * Metric type
50 + */
51 + {
52 + if (!host->metric_type_rs) {
53 + char id_buf[1024];
54 + char name_buf[1024];
55 +
56 + snprintfz(id_buf, 1024, "metric_types_on_%s", localhost->machine_guid);
57 + snprintfz(name_buf, 1024, "metric_types_on_%s", rrdhost_hostname(localhost));
58 +
59 + host->metric_type_rs = rrdset_create(
60 + host->rh,
61 + "netdata", // type
62 + id_buf, // id
63 + name_buf, // name
64 + NETDATA_ML_CHART_FAMILY, // family
65 + "netdata.metric_types", // ctx
66 + "Dimensions by metric type", // title
67 + "dimensions", // units
68 + NETDATA_ML_PLUGIN, // plugin
69 + NETDATA_ML_MODULE_TRAINING, // module
70 + NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
71 + localhost->rrd_update_every, // update_every
72 + RRDSET_TYPE_LINE // chart_type
73 + );
74 + rrdset_flag_set(host->metric_type_rs, RRDSET_FLAG_ANOMALY_DETECTION);
75 +
76 + host->metric_type_constant_rd =
77 + rrddim_add(host->metric_type_rs, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
78 + host->metric_type_variable_rd =
79 + rrddim_add(host->metric_type_rs, "variable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
80 + }
81 +
82 + rrddim_set_by_pointer(host->metric_type_rs,
83 + host->metric_type_constant_rd, mls.num_metric_type_constant);
84 + rrddim_set_by_pointer(host->metric_type_rs,
85 + host->metric_type_variable_rd, mls.num_metric_type_variable);
86 +
87 + rrdset_done(host->metric_type_rs);
88 + }
89 +
90 + /*
91 + * Training status
92 + */
93 + {
94 + if (!host->training_status_rs) {
95 + char id_buf[1024];
96 + char name_buf[1024];
97 +
98 + snprintfz(id_buf, 1024, "training_status_on_%s", localhost->machine_guid);
99 + snprintfz(name_buf, 1024, "training_status_on_%s", rrdhost_hostname(localhost));
100 +
101 + host->training_status_rs = rrdset_create(
102 + host->rh,
103 + "netdata", // type
104 + id_buf, // id
105 + name_buf, // name
106 + NETDATA_ML_CHART_FAMILY, // family
107 + "netdata.training_status", // ctx
108 + "Training status of dimensions", // title
109 + "dimensions", // units
110 + NETDATA_ML_PLUGIN, // plugin
111 + NETDATA_ML_MODULE_TRAINING, // module
112 + NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
113 + localhost->rrd_update_every, // update_every
114 + RRDSET_TYPE_LINE // chart_type
115 + );
116 +
117 + rrdset_flag_set(host->training_status_rs, RRDSET_FLAG_ANOMALY_DETECTION);
118 +
119 + host->training_status_untrained_rd =
120 + rrddim_add(host->training_status_rs, "untrained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
121 + host->training_status_pending_without_model_rd =
122 + rrddim_add(host->training_status_rs, "pending-without-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
123 + host->training_status_trained_rd =
124 + rrddim_add(host->training_status_rs, "trained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
125 + host->training_status_pending_with_model_rd =
126 + rrddim_add(host->training_status_rs, "pending-with-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
127 + }
128 +
129 + rrddim_set_by_pointer(host->training_status_rs,
130 + host->training_status_untrained_rd, mls.num_training_status_untrained);
131 + rrddim_set_by_pointer(host->training_status_rs,
132 + host->training_status_pending_without_model_rd, mls.num_training_status_pending_without_model);
133 + rrddim_set_by_pointer(host->training_status_rs,
134 + host->training_status_trained_rd, mls.num_training_status_trained);
135 + rrddim_set_by_pointer(host->training_status_rs,
136 + host->training_status_pending_with_model_rd, mls.num_training_status_pending_with_model);
137 +
138 + rrdset_done(host->training_status_rs);
139 + }
140 +
141 + /*
142 + * Prediction status
143 + */
144 + {
145 + if (!host->dimensions_rs) {
146 + char id_buf[1024];
147 + char name_buf[1024];
148 +
149 + snprintfz(id_buf, 1024, "dimensions_on_%s", localhost->machine_guid);
150 + snprintfz(name_buf, 1024, "dimensions_on_%s", rrdhost_hostname(localhost));
151 +
152 + host->dimensions_rs = rrdset_create(
153 + host->rh,
154 + "anomaly_detection", // type
155 + id_buf, // id
156 + name_buf, // name
157 + "dimensions", // family
158 + "anomaly_detection.dimensions", // ctx
159 + "Anomaly detection dimensions", // title
160 + "dimensions", // units
161 + NETDATA_ML_PLUGIN, // plugin
162 + NETDATA_ML_MODULE_TRAINING, // module
163 + ML_CHART_PRIO_DIMENSIONS, // priority
164 + localhost->rrd_update_every, // update_every
165 + RRDSET_TYPE_LINE // chart_type
166 + );
167 + rrdset_flag_set(host->dimensions_rs, RRDSET_FLAG_ANOMALY_DETECTION);
168 +
169 + host->dimensions_anomalous_rd =
170 + rrddim_add(host->dimensions_rs, "anomalous", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
171 + host->dimensions_normal_rd =
172 + rrddim_add(host->dimensions_rs, "normal", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
173 + }
174 +
175 + rrddim_set_by_pointer(host->dimensions_rs,
176 + host->dimensions_anomalous_rd, mls.num_anomalous_dimensions);
177 + rrddim_set_by_pointer(host->dimensions_rs,
178 + host->dimensions_normal_rd, mls.num_normal_dimensions);
179 +
180 + rrdset_done(host->dimensions_rs);
181 + }
182 +
183 +}
184 +
185 +void nml_update_host_and_detection_rate_charts(nml_host_t *host, collected_number AnomalyRate) {
186 + /*
187 + * Anomaly rate
188 + */
189 + {
190 + if (!host->anomaly_rate_rs) {
191 + char id_buf[1024];
192 + char name_buf[1024];
193 +
194 + snprintfz(id_buf, 1024, "anomaly_rate_on_%s", localhost->machine_guid);
195 + snprintfz(name_buf, 1024, "anomaly_rate_on_%s", rrdhost_hostname(localhost));
196 +
197 + host->anomaly_rate_rs = rrdset_create(
198 + host->rh,
199 + "anomaly_detection", // type
200 + id_buf, // id
201 + name_buf, // name
202 + "anomaly_rate", // family
203 + "anomaly_detection.anomaly_rate", // ctx
204 + "Percentage of anomalous dimensions", // title
205 + "percentage", // units
206 + NETDATA_ML_PLUGIN, // plugin
207 + NETDATA_ML_MODULE_DETECTION, // module
208 + ML_CHART_PRIO_ANOMALY_RATE, // priority
209 + localhost->rrd_update_every, // update_every
210 + RRDSET_TYPE_LINE // chart_type
211 + );
212 + rrdset_flag_set(host->anomaly_rate_rs, RRDSET_FLAG_ANOMALY_DETECTION);
213 +
214 + host->anomaly_rate_rd =
215 + rrddim_add(host->anomaly_rate_rs, "anomaly_rate", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
216 + }
217 +
218 + rrddim_set_by_pointer(host->anomaly_rate_rs, host->anomaly_rate_rd, AnomalyRate);
219 +
220 + rrdset_done(host->anomaly_rate_rs);
221 + }
222 +
223 + /*
224 + * Detector Events
225 + */
226 + {
227 + if (!host->detector_events_rs) {
228 + char id_buf[1024];
229 + char name_buf[1024];
230 +
231 + snprintfz(id_buf, 1024, "anomaly_detection_on_%s", localhost->machine_guid);
232 + snprintfz(name_buf, 1024, "anomaly_detection_on_%s", rrdhost_hostname(localhost));
233 +
234 + host->detector_events_rs = rrdset_create(
235 + host->rh,
236 + "anomaly_detection", // type
237 + id_buf, // id
238 + name_buf, // name
239 + "anomaly_detection", // family
240 + "anomaly_detection.detector_events", // ctx
241 + "Anomaly detection events", // title
242 + "percentage", // units
243 + NETDATA_ML_PLUGIN, // plugin
244 + NETDATA_ML_MODULE_DETECTION, // module
245 + ML_CHART_PRIO_DETECTOR_EVENTS, // priority
246 + localhost->rrd_update_every, // update_every
247 + RRDSET_TYPE_LINE // chart_type
248 + );
249 + rrdset_flag_set(host->detector_events_rs, RRDSET_FLAG_ANOMALY_DETECTION);
250 +
251 + host->detector_events_above_threshold_rd =
252 + rrddim_add(host->detector_events_rs, "above_threshold", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
253 + host->detector_events_new_anomaly_event_rd =
254 + rrddim_add(host->detector_events_rs, "new_anomaly_event", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
255 + }
256 +
257 + /*
258 + * Compute the values of the dimensions based on the host rate chart
259 + */
260 + ONEWAYALLOC *OWA = onewayalloc_create(0);
261 + time_t Now = now_realtime_sec();
262 + time_t Before = Now - host->rh->rrd_update_every;
263 + time_t After = Before - Cfg.anomaly_detection_query_duration;
264 + RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
265 +
266 + RRDR *R = rrd2rrdr_legacy(
267 + OWA,
268 + host->anomaly_rate_rs,
269 + 1 /* points wanted */,
270 + After,
271 + Before,
272 + Cfg.anomaly_detection_grouping_method,
273 + 0 /* resampling time */,
274 + Options, "anomaly_rate",
275 + NULL /* group options */,
276 + 0, /* timeout */
277 + 0, /* tier */
278 + QUERY_SOURCE_ML,
279 + STORAGE_PRIORITY_BEST_EFFORT
280 + );
281 +
282 + if (R) {
283 + if (R->d == 1 && R->n == 1 && R->rows == 1) {
284 + static thread_local bool prev_above_threshold = false;
285 + bool above_threshold = R->v[0] >= Cfg.host_anomaly_rate_threshold;
286 + bool new_anomaly_event = above_threshold && !prev_above_threshold;
287 + prev_above_threshold = above_threshold;
288 +
289 + rrddim_set_by_pointer(host->detector_events_rs,
290 + host->detector_events_above_threshold_rd, above_threshold);
291 + rrddim_set_by_pointer(host->detector_events_rs,
292 + host->detector_events_new_anomaly_event_rd, new_anomaly_event);
293 +
294 + rrdset_done(host->detector_events_rs);
295 + }
296 +
297 + rrdr_free(OWA, R);
298 + }
299 +
300 + onewayalloc_destroy(OWA);
301 + }
302 +}
303 +
304 +void nml_update_training_statistics_chart(nml_host_t *host, const nml_training_stats_t &ts) {
305 + /*
306 + * queue stats
307 + */
308 + {
309 + if (!host->queue_stats_rs) {
310 + char id_buf[1024];
311 + char name_buf[1024];
312 +
313 + snprintfz(id_buf, 1024, "queue_stats_on_%s", localhost->machine_guid);
314 + snprintfz(name_buf, 1024, "queue_stats_on_%s", rrdhost_hostname(localhost));
315 +
316 + host->queue_stats_rs = rrdset_create(
317 + host->rh,
318 + "netdata", // type
319 + id_buf, // id
320 + name_buf, // name
321 + NETDATA_ML_CHART_FAMILY, // family
322 + "netdata.queue_stats", // ctx
323 + "Training queue stats", // title
324 + "items", // units
325 + NETDATA_ML_PLUGIN, // plugin
326 + NETDATA_ML_MODULE_TRAINING, // module
327 + NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
328 + localhost->rrd_update_every, // update_every
329 + RRDSET_TYPE_LINE// chart_type
330 + );
331 + rrdset_flag_set(host->queue_stats_rs, RRDSET_FLAG_ANOMALY_DETECTION);
332 +
333 + host->queue_stats_queue_size_rd =
334 + rrddim_add(host->queue_stats_rs, "queue_size", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
335 + host->queue_stats_popped_items_rd =
336 + rrddim_add(host->queue_stats_rs, "popped_items", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
337 + }
338 +
339 + rrddim_set_by_pointer(host->queue_stats_rs,
340 + host->queue_stats_queue_size_rd, ts.queue_size);
341 + rrddim_set_by_pointer(host->queue_stats_rs,
342 + host->queue_stats_popped_items_rd, ts.num_popped_items);
343 +
344 + rrdset_done(host->queue_stats_rs);
345 + }
346 +
347 + /*
348 + * training stats
349 + */
350 + {
351 + if (!host->training_time_stats_rs) {
352 + char id_buf[1024];
353 + char name_buf[1024];
354 +
355 + snprintfz(id_buf, 1024, "training_time_stats_on_%s", localhost->machine_guid);
356 + snprintfz(name_buf, 1024, "training_time_stats_on_%s", rrdhost_hostname(localhost));
357 +
358 + host->training_time_stats_rs = rrdset_create(
359 + host->rh,
360 + "netdata", // type
361 + id_buf, // id
362 + name_buf, // name
363 + NETDATA_ML_CHART_FAMILY, // family
364 + "netdata.training_time_stats", // ctx
365 + "Training time stats", // title
366 + "milliseconds", // units
367 + NETDATA_ML_PLUGIN, // plugin
368 + NETDATA_ML_MODULE_TRAINING, // module
369 + NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
370 + localhost->rrd_update_every, // update_every
371 + RRDSET_TYPE_LINE// chart_type
372 + );
373 + rrdset_flag_set(host->training_time_stats_rs, RRDSET_FLAG_ANOMALY_DETECTION);
374 +
375 + host->training_time_stats_allotted_rd =
376 + rrddim_add(host->training_time_stats_rs, "allotted", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
377 + host->training_time_stats_consumed_rd =
378 + rrddim_add(host->training_time_stats_rs, "consumed", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
379 + host->training_time_stats_remaining_rd =
380 + rrddim_add(host->training_time_stats_rs, "remaining", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
381 + }
382 +
383 + rrddim_set_by_pointer(host->training_time_stats_rs,
384 + host->training_time_stats_allotted_rd, ts.allotted_ut);
385 + rrddim_set_by_pointer(host->training_time_stats_rs,
386 + host->training_time_stats_consumed_rd, ts.consumed_ut);
387 + rrddim_set_by_pointer(host->training_time_stats_rs,
388 + host->training_time_stats_remaining_rd, ts.remaining_ut);
389 +
390 + rrdset_done(host->training_time_stats_rs);
391 + }
392 +
393 + /*
394 + * training result stats
395 + */
396 + {
397 + if (!host->training_results_rs) {
398 + char id_buf[1024];
399 + char name_buf[1024];
400 +
401 + snprintfz(id_buf, 1024, "training_results_on_%s", localhost->machine_guid);
402 + snprintfz(name_buf, 1024, "training_results_on_%s", rrdhost_hostname(localhost));
403 +
404 + host->training_results_rs = rrdset_create(
405 + host->rh,
406 + "netdata", // type
407 + id_buf, // id
408 + name_buf, // name
409 + NETDATA_ML_CHART_FAMILY, // family
410 + "netdata.training_results", // ctx
411 + "Training results", // title
412 + "events", // units
413 + NETDATA_ML_PLUGIN, // plugin
414 + NETDATA_ML_MODULE_TRAINING, // module
415 + NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
416 + localhost->rrd_update_every, // update_every
417 + RRDSET_TYPE_LINE// chart_type
418 + );
419 + rrdset_flag_set(host->training_results_rs, RRDSET_FLAG_ANOMALY_DETECTION);
420 +
421 + host->training_results_ok_rd =
422 + rrddim_add(host->training_results_rs, "ok", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
423 + host->training_results_invalid_query_time_range_rd =
424 + rrddim_add(host->training_results_rs, "invalid-queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
425 + host->training_results_not_enough_collected_values_rd =
426 + rrddim_add(host->training_results_rs, "not-enough-values", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
427 + host->training_results_null_acquired_dimension_rd =
428 + rrddim_add(host->training_results_rs, "null-acquired-dimensions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
429 + host->training_results_chart_under_replication_rd =
430 + rrddim_add(host->training_results_rs, "chart-under-replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
431 + }
432 +
433 + rrddim_set_by_pointer(host->training_results_rs,
434 + host->training_results_ok_rd, ts.training_result_ok);
435 + rrddim_set_by_pointer(host->training_results_rs,
436 + host->training_results_invalid_query_time_range_rd, ts.training_result_invalid_query_time_range);
437 + rrddim_set_by_pointer(host->training_results_rs,
438 + host->training_results_not_enough_collected_values_rd, ts.training_result_not_enough_collected_values);
439 + rrddim_set_by_pointer(host->training_results_rs,
440 + host->training_results_null_acquired_dimension_rd, ts.training_result_null_acquired_dimension);
441 + rrddim_set_by_pointer(host->training_results_rs,
442 + host->training_results_chart_under_replication_rd, ts.training_result_chart_under_replication);
443 +
444 + rrdset_done(host->training_results_rs);
445 + }
446 +}
ml/ad_charts.h new
+14
@@ -0,0 +1,14 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef ML_ADCHARTS_H
4 +#define ML_ADCHARTS_H
5 +
6 +#include "nml.h"
7 +
8 +void nml_update_dimensions_chart(nml_host_t *host, const nml_machine_learning_stats_t &mls);
9 +
10 +void nml_update_host_and_detection_rate_charts(nml_host_t *host, collected_number anomaly_rate);
11 +
12 +void nml_update_training_statistics_chart(nml_host_t *host, const nml_training_stats_t &ts);
13 +
14 +#endif /* ML_ADCHARTS_H */
ml/ml-dummy.c
+2 -2
@@ -39,11 +39,11 @@ void ml_dimension_delete(RRDDIM *RD) {
39 UNUSED(RD);
40 }
41
42 -void ml_start_anomaly_detection_threads(RRDHOST *RH) {
42 +void ml_start_training_thread(RRDHOST *RH) {
43 UNUSED(RH);
44 }
45
46 -void ml_stop_anomaly_detection_threads(RRDHOST *RH) {
46 +void ml_stop_training_thread(RRDHOST *RH) {
47 UNUSED(RH);
48 }
49
ml/ml-private.h deleted
-13
@@ -1,13 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_PRIVATE_H
4 -#define ML_PRIVATE_H
5 -
6 -#include "KMeans.h"
7 -#include "ml/ml.h"
8 -
9 -#include <map>
10 -#include <mutex>
11 -#include <sstream>
12 -
13 -#endif /* ML_PRIVATE_H */
ml/ml.cc
+101 -108
@@ -1,23 +1,18 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "Config.h"
4 -#include "Dimension.h"
5 -#include "Chart.h"
6 -#include "Host.h"
3 +#include "nml.h"
4
5 #include <random>
6
10 -using namespace ml;
11 -
7 bool ml_capable() {
8 return true;
9 }
10
16 -bool ml_enabled(RRDHOST *RH) {
17 - if (!Cfg.EnableAnomalyDetection)
11 +bool ml_enabled(RRDHOST *rh) {
12 + if (!Cfg.enable_anomaly_detection)
13 return false;
14
20 - if (simple_pattern_matches(Cfg.SP_HostsToSkip, rrdhost_hostname(RH)))
15 + if (simple_pattern_matches(Cfg.sp_host_to_skip, rrdhost_hostname(rh)))
16 return false;
17
18 return true;
@@ -31,9 +26,9 @@ bool ml_enabled(RRDHOST *RH) {
26
27 void ml_init(void) {
28 // Read config values
34 - Cfg.readMLConfig();
29 + nml_config_load(&Cfg);
30
36 - if (!Cfg.EnableAnomalyDetection)
31 + if (!Cfg.enable_anomaly_detection)
32 return;
33
34 // Generate random numbers to efficiently sample the features we need
@@ -41,160 +36,158 @@ void ml_init(void) {
36 std::random_device RD;
37 std::mt19937 Gen(RD());
38
44 - Cfg.RandomNums.reserve(Cfg.MaxTrainSamples);
45 - for (size_t Idx = 0; Idx != Cfg.MaxTrainSamples; Idx++)
46 - Cfg.RandomNums.push_back(Gen());
39 + Cfg.random_nums.reserve(Cfg.max_train_samples);
40 + for (size_t Idx = 0; Idx != Cfg.max_train_samples; Idx++)
41 + Cfg.random_nums.push_back(Gen());
42 +
43 +
44 + // start detection & training threads
45 + char tag[NETDATA_THREAD_TAG_MAX + 1];
46 +
47 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "%s", "PREDICT");
48 + netdata_thread_create(&Cfg.detection_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, nml_detect_main, NULL);
49 }
50
49 -void ml_host_new(RRDHOST *RH) {
50 - if (!ml_enabled(RH))
51 +void ml_host_new(RRDHOST *rh) {
52 + if (!ml_enabled(rh))
53 return;
54
53 - Host *H = new Host(RH);
54 - RH->ml_host = reinterpret_cast<ml_host_t *>(H);
55 + nml_host_t *host = nml_host_new(rh);
56 + rh->ml_host = reinterpret_cast<ml_host_t *>(host);
57 }
58
57 -void ml_host_delete(RRDHOST *RH) {
58 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
59 - if (!H)
59 +void ml_host_delete(RRDHOST *rh) {
60 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
61 + if (!host)
62 return;
63
62 - delete H;
63 - RH->ml_host = nullptr;
64 + nml_host_delete(host);
65 + rh->ml_host = NULL;
66 }
67
66 -void ml_chart_new(RRDSET *RS) {
67 - Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
68 - if (!H)
68 +void ml_chart_new(RRDSET *rs) {
69 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rs->rrdhost->ml_host);
70 + if (!host)
71 return;
72
71 - Chart *C = new Chart(RS);
72 - RS->ml_chart = reinterpret_cast<ml_chart_t *>(C);
73 -
74 - H->addChart(C);
73 + nml_chart_t *chart = nml_chart_new(rs);
74 + rs->ml_chart = reinterpret_cast<ml_chart_t *>(chart);
75 }
76
77 -void ml_chart_delete(RRDSET *RS) {
78 - Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
79 - if (!H)
77 +void ml_chart_delete(RRDSET *rs) {
78 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rs->rrdhost->ml_host);
79 + if (!host)
80 return;
81
82 - Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
83 - H->removeChart(C);
82 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
83
85 - delete C;
86 - RS->ml_chart = nullptr;
84 + nml_chart_delete(chart);
85 + rs->ml_chart = NULL;
86 }
87
89 -void ml_dimension_new(RRDDIM *RD) {
90 - Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
91 - if (!C)
88 +void ml_dimension_new(RRDDIM *rd) {
89 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rd->rrdset->ml_chart);
90 + if (!chart)
91 return;
92
94 - Dimension *D = new Dimension(RD);
95 - RD->ml_dimension = reinterpret_cast<ml_dimension_t *>(D);
96 - C->addDimension(D);
93 + nml_dimension_t *dim = nml_dimension_new(rd);
94 + rd->ml_dimension = reinterpret_cast<ml_dimension_t *>(dim);
95 }
96
99 -void ml_dimension_delete(RRDDIM *RD) {
100 - Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
101 - if (!D)
97 +void ml_dimension_delete(RRDDIM *rd) {
98 + nml_dimension_t *dim = reinterpret_cast<nml_dimension_t *>(rd->ml_dimension);
99 + if (!dim)
100 return;
101
104 - Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
105 - C->removeDimension(D);
106 -
107 - delete D;
108 - RD->ml_dimension = nullptr;
102 + nml_dimension_delete(dim);
103 + rd->ml_dimension = NULL;
104 }
105
111 -void ml_get_host_info(RRDHOST *RH, BUFFER *wb) {
112 - if (RH && RH->ml_host) {
113 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
114 - H->getConfigAsJson(wb);
106 +void ml_get_host_info(RRDHOST *rh, BUFFER *wb) {
107 + if (rh && rh->ml_host) {
108 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
109 + nml_host_get_config_as_json(host, wb);
110 } else {
111 buffer_json_member_add_boolean(wb, "enabled", false);
112 }
113 }
114
120 -char *ml_get_host_runtime_info(RRDHOST *RH) {
121 - nlohmann::json ConfigJson;
115 +char *ml_get_host_runtime_info(RRDHOST *rh) {
116 + nlohmann::json config_json;
117
123 - if (RH && RH->ml_host) {
124 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
125 - H->getDetectionInfoAsJson(ConfigJson);
118 + if (rh && rh->ml_host) {
119 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
120 + nml_host_get_detection_info_as_json(host, config_json);
121 } else {
127 - return nullptr;
122 + return NULL;
123 }
124
130 - return strdup(ConfigJson.dump(1, '\t').c_str());
125 + return strdup(config_json.dump(1, '\t').c_str());
126 }
127
133 -char *ml_get_host_models(RRDHOST *RH) {
134 - nlohmann::json ModelsJson;
128 +char *ml_get_host_models(RRDHOST *rh) {
129 + nlohmann::json j;
130
136 - if (RH && RH->ml_host) {
137 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
138 - H->getModelsAsJson(ModelsJson);
139 - return strdup(ModelsJson.dump(2, '\t').c_str());
131 + if (rh && rh->ml_host) {
132 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
133 + nml_host_get_models_as_json(host, j);
134 + return strdup(j.dump(2, '\t').c_str());
135 }
136
142 - return nullptr;
137 + return NULL;
138 }
139
145 -void ml_start_anomaly_detection_threads(RRDHOST *RH) {
146 - if (RH && RH->ml_host) {
147 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
148 - H->startAnomalyDetectionThreads();
149 - }
150 -}
151 -
152 -void ml_stop_anomaly_detection_threads(RRDHOST *RH) {
153 - if (RH && RH->ml_host) {
154 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
155 - H->stopAnomalyDetectionThreads(true);
156 - }
157 -}
158 -
159 -void ml_cancel_anomaly_detection_threads(RRDHOST *RH) {
160 - if (RH && RH->ml_host) {
161 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
162 - H->stopAnomalyDetectionThreads(false);
163 - }
164 -}
165 -
166 -bool ml_chart_update_begin(RRDSET *RS) {
167 - Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
168 - if (!C)
140 +bool ml_chart_update_begin(RRDSET *rs) {
141 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
142 + if (!chart)
143 return false;
144
171 - C->updateBegin();
172 -
145 + nml_chart_update_begin(chart);
146 return true;
147 }
148
176 -void ml_chart_update_end(RRDSET *RS) {
177 - Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
178 - if (!C)
149 +void ml_chart_update_end(RRDSET *rs) {
150 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
151 + if (!chart)
152 return;
153
181 - C->updateEnd();
154 + nml_chart_update_end(chart);
155 }
156
184 -bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
185 - Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
186 - if (!D)
157 +bool ml_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool exists) {
158 + nml_dimension_t *dim = reinterpret_cast<nml_dimension_t *>(rd->ml_dimension);
159 + if (!dim)
160 return false;
161
189 - Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
162 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rd->rrdset->ml_chart);
163
191 - bool IsAnomalous = D->predict(CurrT, Value, Exists);
192 - C->updateDimension(D, IsAnomalous);
193 - return IsAnomalous;
164 + bool is_anomalous = nml_dimension_predict(dim, curr_time, value, exists);
165 + nml_chart_update_dimension(chart, dim, is_anomalous);
166 +
167 + return is_anomalous;
168 }
169
170 bool ml_streaming_enabled() {
197 - return Cfg.StreamADCharts;
171 + return Cfg.stream_anomaly_detection_charts;
172 +}
173 +
174 +void ml_start_training_thread(RRDHOST *rh) {
175 + if (rh && rh->ml_host) {
176 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
177 + nml_host_start_training_thread(host);
178 + }
179 +}
180 +
181 +void ml_stop_training_thread(RRDHOST *rh) {
182 + if (rh && rh->ml_host) {
183 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
184 + nml_host_stop_training_thread(host, /* join */ true);
185 + }
186 }
187
200 -#include "ml-private.h"
188 +void ml_cancel_training_thread(RRDHOST *rh) {
189 + if (rh && rh->ml_host) {
190 + nml_host_t *host = reinterpret_cast<nml_host_t *>(rh->ml_host);
191 + nml_host_stop_training_thread(host, /* join */ false);
192 + }
193 +}
ml/ml.h
+15 -15
@@ -12,38 +12,38 @@ extern "C" {
12
13 // This is a DBEngine function redeclared here so that we can free
14 // the anomaly rate dimension, whenever its backing dimension is freed.
15 -void rrddim_free(RRDSET *st, RRDDIM *rd);
15 +void rrddim_free(RRDSET *rs, RRDDIM *rd);
16
17 bool ml_capable();
18
19 -bool ml_enabled(RRDHOST *RH);
19 +bool ml_enabled(RRDHOST *rh);
20
21 void ml_init(void);
22
23 -void ml_host_new(RRDHOST *RH);
24 -void ml_host_delete(RRDHOST *RH);
23 +void ml_host_new(RRDHOST *rh);
24 +void ml_host_delete(RRDHOST *rh);
25
26 -void ml_chart_new(RRDSET *RS);
27 -void ml_chart_delete(RRDSET *RS);
26 +void ml_chart_new(RRDSET *rs);
27 +void ml_chart_delete(RRDSET *rs);
28
29 -void ml_dimension_new(RRDDIM *RD);
30 -void ml_dimension_delete(RRDDIM *RD);
31 -
32 -void ml_start_anomaly_detection_threads(RRDHOST *RH);
33 -void ml_stop_anomaly_detection_threads(RRDHOST *RH);
34 -void ml_cancel_anomaly_detection_threads(RRDHOST *RH);
29 +void ml_dimension_new(RRDDIM *rd);
30 +void ml_dimension_delete(RRDDIM *rd);
31
32 void ml_get_host_info(RRDHOST *RH, BUFFER *wb);
33 char *ml_get_host_runtime_info(RRDHOST *RH);
34 char *ml_get_host_models(RRDHOST *RH);
35
40 -bool ml_chart_update_begin(RRDSET *RS);
41 -void ml_chart_update_end(RRDSET *RS);
36 +bool ml_chart_update_begin(RRDSET *rs);
37 +void ml_chart_update_end(RRDSET *rs);
38
43 -bool ml_is_anomalous(RRDDIM *RD, time_t curr_t, double value, bool exists);
39 +bool ml_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool exists);
40
41 bool ml_streaming_enabled();
42
43 +void ml_start_training_thread(RRDHOST *rh);
44 +void ml_cancel_training_thread(RRDHOST *rh);
45 +void ml_stop_training_thread(RRDHOST *rh);
46 +
47 #ifdef __cplusplus
48 };
49 #endif
ml/nml.cc new
+1135
@@ -0,0 +1,1135 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include <dlib/clustering.h>
4 +
5 +#include "nml.h"
6 +
7 +#include <random>
8 +
9 +#include "ad_charts.h"
10 +
11 +typedef struct {
12 + calculated_number_t *training_cns;
13 + calculated_number_t *scratch_training_cns;
14 +
15 + std::vector<DSample> training_samples;
16 +} nml_tls_data_t;
17 +
18 +static thread_local nml_tls_data_t tls_data;
19 +
20 +/*
21 + * Functions to convert enums to strings
22 +*/
23 +
24 +static const char *nml_machine_learning_status_to_string(enum nml_machine_learning_status mls) {
25 + switch (mls) {
26 + case MACHINE_LEARNING_STATUS_ENABLED:
27 + return "enabled";
28 + case MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART:
29 + return "disabled-sp";
30 + default:
31 + return "unknown";
32 + }
33 +}
34 +
35 +static const char *nml_metric_type_to_string(enum nml_metric_type mt) {
36 + switch (mt) {
37 + case METRIC_TYPE_CONSTANT:
38 + return "constant";
39 + case METRIC_TYPE_VARIABLE:
40 + return "variable";
41 + default:
42 + return "unknown";
43 + }
44 +}
45 +
46 +static const char *nml_training_status_to_string(enum nml_training_status ts) {
47 + switch (ts) {
48 + case TRAINING_STATUS_PENDING_WITH_MODEL:
49 + return "pending-with-model";
50 + case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
51 + return "pending-without-model";
52 + case TRAINING_STATUS_TRAINED:
53 + return "trained";
54 + case TRAINING_STATUS_UNTRAINED:
55 + return "untrained";
56 + default:
57 + return "unknown";
58 + }
59 +}
60 +
61 +static const char *nml_training_result_to_string(enum nml_training_result tr) {
62 + switch (tr) {
63 + case TRAINING_RESULT_OK:
64 + return "ok";
65 + case TRAINING_RESULT_INVALID_QUERY_TIME_RANGE:
66 + return "invalid-query";
67 + case TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES:
68 + return "missing-values";
69 + case TRAINING_RESULT_NULL_ACQUIRED_DIMENSION:
70 + return "null-acquired-dim";
71 + case TRAINING_RESULT_CHART_UNDER_REPLICATION:
72 + return "chart-under-replication";
73 + default:
74 + return "unknown";
75 + }
76 +}
77 +
78 +/*
79 + * Features
80 +*/
81 +
82 +// subtract elements that are `diff_n` positions apart
83 +static void nml_features_diff(nml_features_t *features) {
84 + if (features->diff_n == 0)
85 + return;
86 +
87 + for (size_t idx = 0; idx != (features->src_n - features->diff_n); idx++) {
88 + size_t high = (features->src_n - 1) - idx;
89 + size_t low = high - features->diff_n;
90 +
91 + features->dst[low] = features->src[high] - features->src[low];
92 + }
93 +
94 + size_t n = features->src_n - features->diff_n;
95 + memcpy(features->src, features->dst, n * sizeof(calculated_number_t));
96 +
97 + for (size_t idx = features->src_n - features->diff_n; idx != features->src_n; idx++)
98 + features->src[idx] = 0.0;
99 +}
100 +
101 +// a function that computes the window average of an array inplace
102 +static void nml_features_smooth(nml_features_t *features) {
103 + calculated_number_t sum = 0.0;
104 +
105 + size_t idx = 0;
106 + for (; idx != features->smooth_n - 1; idx++)
107 + sum += features->src[idx];
108 +
109 + for (; idx != (features->src_n - features->diff_n); idx++) {
110 + sum += features->src[idx];
111 + calculated_number_t prev_cn = features->src[idx - (features->smooth_n - 1)];
112 + features->src[idx - (features->smooth_n - 1)] = sum / features->smooth_n;
113 + sum -= prev_cn;
114 + }
115 +
116 + for (idx = 0; idx != features->smooth_n; idx++)
117 + features->src[(features->src_n - 1) - idx] = 0.0;
118 +}
119 +
120 +// create lag'd vectors out of the preprocessed buffer
121 +static void nml_features_lag(nml_features_t *features) {
122 + size_t n = features->src_n - features->diff_n - features->smooth_n + 1 - features->lag_n;
123 + features->preprocessed_features.resize(n);
124 +
125 + unsigned target_num_samples = Cfg.max_train_samples * Cfg.random_sampling_ratio;
126 + double sampling_ratio = std::min(static_cast<double>(target_num_samples) / n, 1.0);
127 +
128 + uint32_t max_mt = std::numeric_limits<uint32_t>::max();
129 + uint32_t cutoff = static_cast<double>(max_mt) * sampling_ratio;
130 +
131 + size_t sample_idx = 0;
132 +
133 + for (size_t idx = 0; idx != n; idx++) {
134 + DSample &DS = features->preprocessed_features[sample_idx++];
135 + DS.set_size(features->lag_n);
136 +
137 + if (Cfg.random_nums[idx] > cutoff) {
138 + sample_idx--;
139 + continue;
140 + }
141 +
142 + for (size_t feature_idx = 0; feature_idx != features->lag_n + 1; feature_idx++)
143 + DS(feature_idx) = features->src[idx + feature_idx];
144 + }
145 +
146 + features->preprocessed_features.resize(sample_idx);
147 +}
148 +
149 +static void nml_features_preprocess(nml_features_t *features) {
150 + nml_features_diff(features);
151 + nml_features_smooth(features);
152 + nml_features_lag(features);
153 +}
154 +
155 +/*
156 + * KMeans
157 +*/
158 +
159 +static void nml_kmeans_init(nml_kmeans_t *kmeans, size_t num_clusters, size_t max_iterations) {
160 + kmeans->num_clusters = num_clusters;
161 + kmeans->max_iterations = max_iterations;
162 +
163 + kmeans->cluster_centers.reserve(kmeans->num_clusters);
164 + kmeans->min_dist = std::numeric_limits<calculated_number_t>::max();
165 + kmeans->max_dist = std::numeric_limits<calculated_number_t>::min();
166 +}
167 +
168 +static void nml_kmeans_train(nml_kmeans_t *kmeans, const nml_features_t *features) {
169 + kmeans->min_dist = std::numeric_limits<calculated_number_t>::max();
170 + kmeans->max_dist = std::numeric_limits<calculated_number_t>::min();
171 +
172 + kmeans->cluster_centers.clear();
173 +
174 + dlib::pick_initial_centers(kmeans->num_clusters, kmeans->cluster_centers, features->preprocessed_features);
175 + dlib::find_clusters_using_kmeans(features->preprocessed_features, kmeans->cluster_centers, kmeans->max_iterations);
176 +
177 + for (const auto &preprocessed_feature : features->preprocessed_features) {
178 + calculated_number_t mean_dist = 0.0;
179 +
180 + for (const auto &cluster_center : kmeans->cluster_centers) {
181 + mean_dist += dlib::length(cluster_center - preprocessed_feature);
182 + }
183 +
184 + mean_dist /= kmeans->num_clusters;
185 +
186 + if (mean_dist < kmeans->min_dist)
187 + kmeans->min_dist = mean_dist;
188 +
189 + if (mean_dist > kmeans->max_dist)
190 + kmeans->max_dist = mean_dist;
191 + }
192 +}
193 +
194 +static calculated_number_t nml_kmeans_anomaly_score(const nml_kmeans_t *kmeans, const DSample &DS) {
195 + calculated_number_t mean_dist = 0.0;
196 + for (const auto &CC: kmeans->cluster_centers)
197 + mean_dist += dlib::length(CC - DS);
198 +
199 + mean_dist /= kmeans->num_clusters;
200 +
201 + if (kmeans->max_dist == kmeans->min_dist)
202 + return 0.0;
203 +
204 + calculated_number_t anomaly_score = 100.0 * std::abs((mean_dist - kmeans->min_dist) / (kmeans->max_dist - kmeans->min_dist));
205 + return (anomaly_score > 100.0) ? 100.0 : anomaly_score;
206 +}
207 +
208 +/*
209 + * Queue
210 +*/
211 +
212 +nml_queue_t *nml_queue_init() {
213 + nml_queue_t *q = new nml_queue_t();
214 +
215 + netdata_mutex_init(&q->mutex);
216 + pthread_cond_init(&q->cond_var, NULL);
217 + q->exit = false;
218 + return q;
219 +}
220 +
221 +void nml_queue_destroy(nml_queue_t *q) {
222 + netdata_mutex_destroy(&q->mutex);
223 + pthread_cond_destroy(&q->cond_var);
224 + delete q;
225 +}
226 +
227 +void nml_queue_push(nml_queue_t *q, const nml_training_request_t req) {
228 + netdata_mutex_lock(&q->mutex);
229 + q->internal.push(req);
230 + pthread_cond_signal(&q->cond_var);
231 + netdata_mutex_unlock(&q->mutex);
232 +}
233 +
234 +nml_training_request_t nml_queue_pop(nml_queue_t *q) {
235 + netdata_mutex_lock(&q->mutex);
236 +
237 + nml_training_request_t req = { NULL, NULL, 0, 0, 0 };
238 +
239 + while (q->internal.empty()) {
240 + pthread_cond_wait(&q->cond_var, &q->mutex);
241 +
242 + if (q->exit) {
243 + netdata_mutex_unlock(&q->mutex);
244 +
245 + // We return a dummy request because the queue has been signaled
246 + return req;
247 + }
248 + }
249 +
250 + req = q->internal.front();
251 + q->internal.pop();
252 +
253 + netdata_mutex_unlock(&q->mutex);
254 + return req;
255 +}
256 +
257 +size_t nml_queue_size(nml_queue_t *q) {
258 + netdata_mutex_lock(&q->mutex);
259 + size_t size = q->internal.size();
260 + netdata_mutex_unlock(&q->mutex);
261 + return size;
262 +}
263 +
264 +void nml_queue_signal(nml_queue_t *q) {
265 + netdata_mutex_lock(&q->mutex);
266 + q->exit = true;
267 + pthread_cond_signal(&q->cond_var);
268 + netdata_mutex_unlock(&q->mutex);
269 +}
270 +
271 +/*
272 + * Dimension
273 +*/
274 +
275 +static std::pair<calculated_number_t *, nml_training_response_t>
276 +nml_dimension_calculated_numbers(nml_dimension_t *dim, const nml_training_request_t &training_request) {
277 + nml_training_response_t training_response = {};
278 +
279 + training_response.request_time = training_request.request_time;
280 + training_response.first_entry_on_request = training_request.first_entry_on_request;
281 + training_response.last_entry_on_request = training_request.last_entry_on_request;
282 +
283 + training_response.first_entry_on_response = rrddim_first_entry_s_of_tier(dim->rd, 0);
284 + training_response.last_entry_on_response = rrddim_last_entry_s_of_tier(dim->rd, 0);
285 +
286 + size_t min_n = Cfg.min_train_samples;
287 + size_t max_n = Cfg.max_train_samples;
288 +
289 + // Figure out what our time window should be.
290 + training_response.query_before_t = training_response.last_entry_on_response;
291 + training_response.query_after_t = std::max(
292 + training_response.query_before_t - static_cast<time_t>((max_n - 1) * dim->rd->update_every),
293 + training_response.first_entry_on_response
294 + );
295 +
296 + if (training_response.query_after_t >= training_response.query_before_t) {
297 + training_response.result = TRAINING_RESULT_INVALID_QUERY_TIME_RANGE;
298 + return { NULL, training_response };
299 + }
300 +
301 + if (rrdset_is_replicating(dim->rd->rrdset)) {
302 + training_response.result = TRAINING_RESULT_CHART_UNDER_REPLICATION;
303 + return { NULL, training_response };
304 + }
305 +
306 + /*
307 + * Execute the query
308 + */
309 + struct storage_engine_query_ops *ops = dim->rd->tiers[0].query_ops;
310 + struct storage_engine_query_handle handle;
311 +
312 + ops->init(dim->rd->tiers[0].db_metric_handle,
313 + &handle,
314 + training_response.query_after_t,
315 + training_response.query_before_t,
316 + STORAGE_PRIORITY_BEST_EFFORT);
317 +
318 + size_t idx = 0;
319 + memset(tls_data.training_cns, 0, sizeof(calculated_number_t) * max_n * (Cfg.lag_n + 1));
320 + calculated_number_t last_value = std::numeric_limits<calculated_number_t>::quiet_NaN();
321 +
322 + while (!ops->is_finished(&handle)) {
323 + if (idx == max_n)
324 + break;
325 +
326 + STORAGE_POINT sp = ops->next_metric(&handle);
327 +
328 + time_t timestamp = sp.end_time_s;
329 + calculated_number_t value = sp.sum / sp.count;
330 +
331 + if (netdata_double_isnumber(value)) {
332 + if (!training_response.db_after_t)
333 + training_response.db_after_t = timestamp;
334 + training_response.db_before_t = timestamp;
335 +
336 + tls_data.training_cns[idx] = value;
337 + last_value = tls_data.training_cns[idx];
338 + training_response.collected_values++;
339 + } else
340 + tls_data.training_cns[idx] = last_value;
341 +
342 + idx++;
343 + }
344 + ops->finalize(&handle);
345 +
346 + global_statistics_ml_query_completed(/* points_read */ idx);
347 +
348 + training_response.total_values = idx;
349 + if (training_response.collected_values < min_n) {
350 + training_response.result = TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES;
351 + return { NULL, training_response };
352 + }
353 +
354 + // Find first non-NaN value.
355 + for (idx = 0; std::isnan(tls_data.training_cns[idx]); idx++, training_response.total_values--) { }
356 +
357 + // Overwrite NaN values.
358 + if (idx != 0)
359 + memmove(tls_data.training_cns, &tls_data.training_cns[idx], sizeof(calculated_number_t) * training_response.total_values);
360 +
361 + training_response.result = TRAINING_RESULT_OK;
362 + return { tls_data.training_cns, training_response };
363 +}
364 +
365 +static enum nml_training_result
366 +nml_dimension_train_model(nml_dimension_t *dim, const nml_training_request_t &training_request) {
367 + auto P = nml_dimension_calculated_numbers(dim, training_request);
368 + nml_training_response_t training_response = P.second;
369 +
370 + if (training_response.result != TRAINING_RESULT_OK) {
371 + netdata_mutex_lock(&dim->mutex);
372 +
373 + dim->mt = METRIC_TYPE_CONSTANT;
374 +
375 + switch (dim->ts) {
376 + case TRAINING_STATUS_PENDING_WITH_MODEL:
377 + dim->ts = TRAINING_STATUS_TRAINED;
378 + break;
379 + case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
380 + dim->ts = TRAINING_STATUS_UNTRAINED;
381 + break;
382 + default:
383 + break;
384 + }
385 +
386 + dim->tr = training_response;
387 +
388 + dim->last_training_time = training_response.last_entry_on_response;
389 + enum nml_training_result result = training_response.result;
390 + netdata_mutex_unlock(&dim->mutex);
391 +
392 + return result;
393 + }
394 +
395 + // compute kmeans
396 + {
397 + memcpy(tls_data.scratch_training_cns, tls_data.training_cns,
398 + training_response.total_values * sizeof(calculated_number_t));
399 +
400 + nml_features_t features = {
401 + Cfg.diff_n, Cfg.smooth_n, Cfg.lag_n,
402 + tls_data.scratch_training_cns, training_response.total_values,
403 + tls_data.training_cns, training_response.total_values,
404 + tls_data.training_samples
405 + };
406 + nml_features_preprocess(&features);
407 +
408 + nml_kmeans_init(&dim->kmeans, 2, 1000);
409 + nml_kmeans_train(&dim->kmeans, &features);
410 + }
411 +
412 + // update kmeans models
413 + {
414 + netdata_mutex_lock(&dim->mutex);
415 +
416 + if (dim->km_contexts.size() < Cfg.num_models_to_use) {
417 + dim->km_contexts.push_back(std::move(dim->kmeans));
418 + } else {
419 + std::rotate(std::begin(dim->km_contexts), std::begin(dim->km_contexts) + 1, std::end(dim->km_contexts));
420 + dim->km_contexts[dim->km_contexts.size() - 1] = std::move(dim->kmeans);
421 + }
422 +
423 + dim->mt = METRIC_TYPE_CONSTANT;
424 + dim->ts = TRAINING_STATUS_TRAINED;
425 + dim->tr = training_response;
426 + dim->last_training_time = rrddim_last_entry_s(dim->rd);
427 +
428 + netdata_mutex_unlock(&dim->mutex);
429 + }
430 +
431 + return training_response.result;
432 +}
433 +
434 +static void nml_dimension_schedule_for_training(nml_dimension_t *dim, time_t curr_time) {
435 + switch (dim->mt) {
436 + case METRIC_TYPE_CONSTANT:
437 + return;
438 + default:
439 + break;
440 + }
441 +
442 + bool schedule_for_training = false;
443 +
444 + switch (dim->ts) {
445 + case TRAINING_STATUS_PENDING_WITH_MODEL:
446 + case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
447 + schedule_for_training = false;
448 + break;
449 + case TRAINING_STATUS_UNTRAINED:
450 + schedule_for_training = true;
451 + dim->ts = TRAINING_STATUS_PENDING_WITHOUT_MODEL;
452 + break;
453 + case TRAINING_STATUS_TRAINED:
454 + if ((dim->last_training_time + (Cfg.train_every * dim->rd->update_every)) < curr_time) {
455 + schedule_for_training = true;
456 + dim->ts = TRAINING_STATUS_PENDING_WITH_MODEL;
457 + }
458 + break;
459 + }
460 +
461 + if (schedule_for_training) {
462 + nml_host_t *host = reinterpret_cast<nml_host_t *>(dim->rd->rrdset->rrdhost->ml_host);
463 + nml_training_request_t req = {
464 + string_dup(dim->rd->rrdset->id), string_dup(dim->rd->id),
465 + curr_time, rrddim_first_entry_s(dim->rd), rrddim_last_entry_s(dim->rd),
466 + };
467 + nml_queue_push(host->training_queue, req);
468 + }
469 +}
470 +
471 +bool nml_dimension_predict(nml_dimension_t *dim, time_t curr_time, calculated_number_t value, bool exists) {
472 + // Nothing to do if ML is disabled for this dimension
473 + if (dim->mls != MACHINE_LEARNING_STATUS_ENABLED)
474 + return false;
475 +
476 + // Don't treat values that don't exist as anomalous
477 + if (!exists) {
478 + dim->cns.clear();
479 + return false;
480 + }
481 +
482 + // Save the value and return if we don't have enough values for a sample
483 + unsigned n = Cfg.diff_n + Cfg.smooth_n + Cfg.lag_n;
484 + if (dim->cns.size() < n) {
485 + dim->cns.push_back(value);
486 + return false;
487 + }
488 +
489 + // Push the value and check if it's different from the last one
490 + bool same_value = true;
491 + std::rotate(std::begin(dim->cns), std::begin(dim->cns) + 1, std::end(dim->cns));
492 + if (dim->cns[n - 1] != value)
493 + same_value = false;
494 + dim->cns[n - 1] = value;
495 +
496 + // Create the sample
497 + assert((n * (Cfg.lag_n + 1) <= 128) &&
498 + "Static buffers too small to perform prediction. "
499 + "This should not be possible with the default clamping of feature extraction options");
500 + calculated_number_t src_cns[128];
501 + calculated_number_t dst_cns[128];
502 +
503 + memset(src_cns, 0, n * (Cfg.lag_n + 1) * sizeof(calculated_number_t));
504 + memcpy(src_cns, dim->cns.data(), n * sizeof(calculated_number_t));
505 + memcpy(dst_cns, dim->cns.data(), n * sizeof(calculated_number_t));
506 +
507 + nml_features_t features = {
508 + Cfg.diff_n, Cfg.smooth_n, Cfg.lag_n,
509 + dst_cns, n, src_cns, n,
510 + dim->feature
511 + };
512 + nml_features_preprocess(&features);
513 +
514 + /*
515 + * Lock to predict and possibly schedule the dimension for training
516 + */
517 + if (netdata_mutex_trylock(&dim->mutex) != 0)
518 + return false;
519 +
520 + // Mark the metric time as variable if we received different values
521 + if (!same_value)
522 + dim->mt = METRIC_TYPE_VARIABLE;
523 +
524 + // Decide if the dimension needs to be scheduled for training
525 + nml_dimension_schedule_for_training(dim, curr_time);
526 +
527 + // Nothing to do if we don't have a model
528 + switch (dim->ts) {
529 + case TRAINING_STATUS_UNTRAINED:
530 + case TRAINING_STATUS_PENDING_WITHOUT_MODEL: {
531 + netdata_mutex_unlock(&dim->mutex);
532 + return false;
533 + }
534 + default:
535 + break;
536 + }
537 +
538 + /*
539 + * Use the KMeans models to check if the value is anomalous
540 + */
541 +
542 + size_t sum = 0;
543 + size_t models_consulted = 0;
544 +
545 + for (const auto &km_ctx : dim->km_contexts) {
546 + models_consulted++;
547 +
548 + calculated_number_t anomaly_score = nml_kmeans_anomaly_score(&km_ctx, features.preprocessed_features[0]);
549 + if (anomaly_score == std::numeric_limits<calculated_number_t>::quiet_NaN())
550 + continue;
551 +
552 + if (anomaly_score < (100 * Cfg.dimension_anomaly_score_threshold)) {
553 + global_statistics_ml_models_consulted(models_consulted);
554 + netdata_mutex_unlock(&dim->mutex);
555 + return false;
556 + }
557 +
558 + sum += 1;
559 + }
560 +
561 + netdata_mutex_unlock(&dim->mutex);
562 +
563 + global_statistics_ml_models_consulted(models_consulted);
564 + return sum;
565 +}
566 +
567 +void nml_dimension_dump(nml_dimension_t *dim) {
568 + const char *chart_id = rrdset_id(dim->rd->rrdset);
569 + const char *dimension_id = rrddim_id(dim->rd);
570 +
571 + const char *mls_str = nml_machine_learning_status_to_string(dim->mls);
572 + const char *mt_str = nml_metric_type_to_string(dim->mt);
573 + const char *ts_str = nml_training_status_to_string(dim->ts);
574 + const char *tr_str = nml_training_result_to_string(dim->tr.result);
575 +
576 + const char *fmt =
577 + "[ML] %s.%s: MLS=%s, MT=%s, TS=%s, Result=%s, "
578 + "ReqTime=%ld, FEOReq=%ld, LEOReq=%ld, "
579 + "FEOResp=%ld, LEOResp=%ld, QTR=<%ld, %ld>, DBTR=<%ld, %ld>, Collected=%zu, Total=%zu";
580 +
581 + error(fmt,
582 + chart_id, dimension_id, mls_str, mt_str, ts_str, tr_str,
583 + dim->tr.request_time, dim->tr.first_entry_on_request, dim->tr.last_entry_on_request,
584 + dim->tr.first_entry_on_response, dim->tr.last_entry_on_response,
585 + dim->tr.query_after_t, dim->tr.query_before_t, dim->tr.db_after_t, dim->tr.db_before_t, dim->tr.collected_values, dim->tr.total_values
586 + );
587 +}
588 +
589 +nml_dimension_t *nml_dimension_new(RRDDIM *rd) {
590 + nml_dimension_t *dim = new nml_dimension_t();
591 +
592 + dim->rd = rd;
593 +
594 + dim->mt = METRIC_TYPE_CONSTANT;
595 + dim->ts = TRAINING_STATUS_UNTRAINED;
596 +
597 + dim->last_training_time = 0;
598 +
599 + nml_kmeans_init(&dim->kmeans, 2, 1000);
600 +
601 + if (simple_pattern_matches(Cfg.sp_charts_to_skip, rrdset_name(rd->rrdset)))
602 + dim->mls = MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART;
603 + else
604 + dim->mls = MACHINE_LEARNING_STATUS_ENABLED;
605 +
606 + netdata_mutex_init(&dim->mutex);
607 +
608 + dim->km_contexts.reserve(Cfg.num_models_to_use);
609 +
610 + return dim;
611 +}
612 +
613 +void nml_dimension_delete(nml_dimension_t *dim) {
614 + netdata_mutex_destroy(&dim->mutex);
615 + delete dim;
616 +}
617 +
618 +nml_chart_t *nml_chart_new(RRDSET *rs) {
619 + nml_chart_t *chart = new nml_chart_t();
620 +
621 + chart->rs = rs;
622 + chart->mls = nml_machine_learning_stats_t();
623 +
624 + netdata_mutex_init(&chart->mutex);
625 +
626 + return chart;
627 +}
628 +
629 +void nml_chart_delete(nml_chart_t *chart) {
630 + netdata_mutex_destroy(&chart->mutex);
631 + delete chart;
632 +}
633 +
634 +static bool nml_chart_is_available_for_ml(nml_chart_t *chart) {
635 + return rrdset_is_available_for_exporting_and_alarms(chart->rs);
636 +}
637 +
638 +static std::string ml_dimension_get_id(RRDDIM *rd) {
639 + RRDSET *rs = rd->rrdset;
640 +
641 + std::stringstream ss;
642 + ss << rrdset_context(rs) << "|" << rrdset_id(rs) << "|" << rrddim_name(rd);
643 + return ss.str();
644 +}
645 +
646 +static void nml_chart_get_models_as_json(nml_chart_t *chart, nlohmann::json &j) {
647 + netdata_mutex_lock(&chart->mutex);
648 +
649 + void *rdp = NULL;
650 + rrddim_foreach_read(rdp, chart->rs) {
651 + RRDDIM *rd = static_cast<RRDDIM *>(rdp);
652 + nml_dimension_t *dim = reinterpret_cast<nml_dimension_t *>(rd->ml_dimension);
653 + if (!dim)
654 + continue;
655 +
656 + nlohmann::json jarray = nlohmann::json::array();
657 +#if 0
658 + for (const KMeans &KM : nml_dimension_models(D)) {
659 + nlohmann::json tmp;
660 +
661 + KM.toJson(tmp);
662 + jarray.push_back(tmp);
663 + j[ml_dimension_get_id(D->rd)] = jarray;
664 + }
665 +#else
666 + j[ml_dimension_get_id(dim->rd)] = jarray;
667 +#endif
668 + }
669 + rrdset_foreach_done(rdp);
670 +
671 + netdata_mutex_unlock(&chart->mutex);
672 +}
673 +
674 +void nml_chart_update_begin(nml_chart_t *chart) {
675 + netdata_mutex_lock(&chart->mutex);
676 + chart->mls = {};
677 +}
678 +
679 +void nml_chart_update_end(nml_chart_t *chart) {
680 + netdata_mutex_unlock(&chart->mutex);
681 +}
682 +
683 +void nml_chart_update_dimension(nml_chart_t *chart, nml_dimension_t *dim, bool is_anomalous) {
684 + switch (dim->mls) {
685 + case MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART:
686 + chart->mls.num_machine_learning_status_disabled_sp++;
687 + return;
688 + case MACHINE_LEARNING_STATUS_ENABLED: {
689 + chart->mls.num_machine_learning_status_enabled++;
690 +
691 + switch (dim->mt) {
692 + case METRIC_TYPE_CONSTANT:
693 + chart->mls.num_metric_type_constant++;
694 + chart->mls.num_training_status_trained++;
695 + chart->mls.num_normal_dimensions++;
696 + return;
697 + case METRIC_TYPE_VARIABLE:
698 + chart->mls.num_metric_type_variable++;
699 + break;
700 + }
701 +
702 + switch (dim->ts) {
703 + case TRAINING_STATUS_UNTRAINED:
704 + chart->mls.num_training_status_untrained++;
705 + return;
706 + case TRAINING_STATUS_PENDING_WITHOUT_MODEL:
707 + chart->mls.num_training_status_pending_without_model++;
708 + return;
709 + case TRAINING_STATUS_TRAINED:
710 + chart->mls.num_training_status_trained++;
711 +
712 + chart->mls.num_anomalous_dimensions += is_anomalous;
713 + chart->mls.num_normal_dimensions += !is_anomalous;
714 + return;
715 + case TRAINING_STATUS_PENDING_WITH_MODEL:
716 + chart->mls.num_training_status_pending_with_model++;
717 +
718 + chart->mls.num_anomalous_dimensions += is_anomalous;
719 + chart->mls.num_normal_dimensions += !is_anomalous;
720 + return;
721 + }
722 +
723 + return;
724 + }
725 + }
726 +}
727 +
728 +nml_host_t *nml_host_new(RRDHOST *rh) {
729 + nml_host_t *host = new nml_host_t();
730 +
731 + host->rh = rh;
732 + host->mls = nml_machine_learning_stats_t();
733 + host->ts = nml_training_stats_t();
734 +
735 + host->host_anomaly_rate = 0.0;
736 + host->threads_running = false;
737 + host->threads_cancelled = false;
738 + host->threads_joined = false;
739 +
740 + host->training_queue = nml_queue_init();
741 +
742 + netdata_mutex_init(&host->mutex);
743 +
744 + return host;
745 +}
746 +
747 +void nml_host_delete(nml_host_t *host) {
748 + netdata_mutex_destroy(&host->mutex);
749 + nml_queue_destroy(host->training_queue);
750 + delete host;
751 +}
752 +
753 +void nml_host_get_config_as_json(nml_host_t *host, BUFFER *wb) {
754 + // Unused for now, until we add support for per-host configs
755 + (void) host;
756 +
757 + buffer_json_member_add_uint64(wb, "version", 1);
758 +
759 + buffer_json_member_add_boolean(wb, "enabled", Cfg.enable_anomaly_detection);
760 +
761 + buffer_json_member_add_uint64(wb, "min-train-samples", Cfg.min_train_samples);
762 + buffer_json_member_add_uint64(wb, "max-train-samples", Cfg.max_train_samples);
763 + buffer_json_member_add_uint64(wb, "train-every", Cfg.train_every);
764 +
765 + buffer_json_member_add_uint64(wb, "diff-n", Cfg.diff_n);
766 + buffer_json_member_add_uint64(wb, "smooth-n", Cfg.smooth_n);
767 + buffer_json_member_add_uint64(wb, "lag-n", Cfg.lag_n);
768 +
769 + buffer_json_member_add_double(wb, "random-sampling-ratio", Cfg.random_sampling_ratio);
770 + buffer_json_member_add_uint64(wb, "max-kmeans-iters", Cfg.random_sampling_ratio);
771 +
772 + buffer_json_member_add_double(wb, "dimension-anomaly-score-threshold", Cfg.dimension_anomaly_score_threshold);
773 +
774 + buffer_json_member_add_string(wb, "anomaly-detection-grouping-method",
775 + time_grouping_method2string(Cfg.anomaly_detection_grouping_method));
776 +
777 + buffer_json_member_add_int64(wb, "anomaly-detection-query-duration", Cfg.anomaly_detection_query_duration);
778 +
779 + buffer_json_member_add_string(wb, "hosts-to-skip", Cfg.hosts_to_skip.c_str());
780 + buffer_json_member_add_string(wb, "charts-to-skip", Cfg.charts_to_skip.c_str());
781 +}
782 +
783 +void nml_host_get_models_as_json(nml_host_t *host, nlohmann::json &j) {
784 + netdata_mutex_lock(&host->mutex);
785 +
786 + void* rsp = NULL;
787 + rrdset_foreach_read(rsp, host->rh) {
788 + RRDSET *rs = static_cast<RRDSET *>(rsp);
789 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
790 +
791 + if (!chart)
792 + continue;
793 +
794 + nml_chart_get_models_as_json(chart, j);
795 + }
796 + rrdset_foreach_done(rsp);
797 +
798 + netdata_mutex_unlock(&host->mutex);
799 +}
800 +
801 +#define WORKER_JOB_DETECTION_PREP 0
802 +#define WORKER_JOB_DETECTION_DIM_CHART 1
803 +#define WORKER_JOB_DETECTION_HOST_CHART 2
804 +#define WORKER_JOB_DETECTION_STATS 3
805 +#define WORKER_JOB_DETECTION_RESOURCES 4
806 +
807 +static void nml_host_detect_once(nml_host_t *host) {
808 + worker_is_busy(WORKER_JOB_DETECTION_PREP);
809 +
810 + host->mls = {};
811 + nml_machine_learning_stats_t mls_copy = {};
812 + nml_training_stats_t ts_copy = {};
813 +
814 + {
815 + netdata_mutex_lock(&host->mutex);
816 +
817 + /*
818 + * prediction/detection stats
819 + */
820 + void *rsp = NULL;
821 + rrdset_foreach_read(rsp, host->rh) {
822 + RRDSET *rs = static_cast<RRDSET *>(rsp);
823 +
824 + nml_chart_t *chart = reinterpret_cast<nml_chart_t *>(rs->ml_chart);
825 + if (!chart)
826 + continue;
827 +
828 + if (!nml_chart_is_available_for_ml(chart))
829 + continue;
830 +
831 + nml_machine_learning_stats_t chart_mls = chart->mls;
832 +
833 + host->mls.num_machine_learning_status_enabled += chart_mls.num_machine_learning_status_enabled;
834 + host->mls.num_machine_learning_status_disabled_sp += chart_mls.num_machine_learning_status_disabled_sp;
835 +
836 + host->mls.num_metric_type_constant += chart_mls.num_metric_type_constant;
837 + host->mls.num_metric_type_variable += chart_mls.num_metric_type_variable;
838 +
839 + host->mls.num_training_status_untrained += chart_mls.num_training_status_untrained;
840 + host->mls.num_training_status_pending_without_model += chart_mls.num_training_status_pending_without_model;
841 + host->mls.num_training_status_trained += chart_mls.num_training_status_trained;
842 + host->mls.num_training_status_pending_with_model += chart_mls.num_training_status_pending_with_model;
843 +
844 + host->mls.num_anomalous_dimensions += chart_mls.num_anomalous_dimensions;
845 + host->mls.num_normal_dimensions += chart_mls.num_normal_dimensions;
846 + }
847 + rrdset_foreach_done(rsp);
848 +
849 + host->host_anomaly_rate = 0.0;
850 + size_t NumActiveDimensions = host->mls.num_anomalous_dimensions + host->mls.num_normal_dimensions;
851 + if (NumActiveDimensions)
852 + host->host_anomaly_rate = static_cast<double>(host->mls.num_anomalous_dimensions) / NumActiveDimensions;
853 +
854 + mls_copy = host->mls;
855 +
856 + /*
857 + * training stats
858 + */
859 + ts_copy = host->ts;
860 +
861 + host->ts.queue_size = 0;
862 + host->ts.num_popped_items = 0;
863 +
864 + host->ts.allotted_ut = 0;
865 + host->ts.consumed_ut = 0;
866 + host->ts.remaining_ut = 0;
867 +
868 + host->ts.training_result_ok = 0;
869 + host->ts.training_result_invalid_query_time_range = 0;
870 + host->ts.training_result_not_enough_collected_values = 0;
871 + host->ts.training_result_null_acquired_dimension = 0;
872 + host->ts.training_result_chart_under_replication = 0;
873 +
874 + netdata_mutex_unlock(&host->mutex);
875 + }
876 +
877 + // Calc the avg values
878 + if (ts_copy.num_popped_items) {
879 + ts_copy.queue_size /= ts_copy.num_popped_items;
880 + ts_copy.allotted_ut /= ts_copy.num_popped_items;
881 + ts_copy.consumed_ut /= ts_copy.num_popped_items;
882 + ts_copy.remaining_ut /= ts_copy.num_popped_items;
883 +
884 + ts_copy.training_result_ok /= ts_copy.num_popped_items;
885 + ts_copy.training_result_invalid_query_time_range /= ts_copy.num_popped_items;
886 + ts_copy.training_result_not_enough_collected_values /= ts_copy.num_popped_items;
887 + ts_copy.training_result_null_acquired_dimension /= ts_copy.num_popped_items;
888 + ts_copy.training_result_chart_under_replication /= ts_copy.num_popped_items;
889 + } else {
890 + ts_copy.queue_size = 0;
891 + ts_copy.allotted_ut = 0;
892 + ts_copy.consumed_ut = 0;
893 + ts_copy.remaining_ut = 0;
894 + }
895 +
896 + worker_is_busy(WORKER_JOB_DETECTION_DIM_CHART);
897 + nml_update_dimensions_chart(host, mls_copy);
898 +
899 + worker_is_busy(WORKER_JOB_DETECTION_HOST_CHART);
900 + nml_update_host_and_detection_rate_charts(host, host->host_anomaly_rate * 10000.0);
901 +
902 +#ifdef NETDATA_ML_RESOURCE_CHARTS
903 + worker_is_busy(WORKER_JOB_DETECTION_RESOURCES);
904 + struct rusage PredictionRU;
905 + getrusage(RUSAGE_THREAD, &PredictionRU);
906 + updateResourceUsageCharts(RH, PredictionRU, TSCopy.TrainingRU);
907 +#endif
908 +
909 + worker_is_busy(WORKER_JOB_DETECTION_STATS);
910 + nml_update_training_statistics_chart(host, ts_copy);
911 +}
912 +
913 +typedef struct {
914 + RRDDIM_ACQUIRED *acq_rd;
915 + nml_dimension_t *dim;
916 +} nml_acquired_dimension_t;
917 +
918 +static nml_acquired_dimension_t nml_acquired_dimension_get(RRDHOST *rh, STRING *chart_id, STRING *dimension_id) {
919 + RRDDIM_ACQUIRED *acq_rd = NULL;
920 + nml_dimension_t *dim = NULL;
921 +
922 + RRDSET *rs = rrdset_find(rh, string2str(chart_id));
923 + if (rs) {
924 + acq_rd = rrddim_find_and_acquire(rs, string2str(dimension_id));
925 + if (acq_rd) {
926 + RRDDIM *rd = rrddim_acquired_to_rrddim(acq_rd);
927 + if (rd)
928 + dim = reinterpret_cast<nml_dimension_t *>(rd->ml_dimension);
929 + }
930 + }
931 +
932 + nml_acquired_dimension_t acq_dim = {
933 + acq_rd, dim
934 + };
935 +
936 + return acq_dim;
937 +}
938 +
939 +static void nml_acquired_dimension_release(nml_acquired_dimension_t acq_dim) {
940 + if (!acq_dim.acq_rd)
941 + return;
942 +
943 + rrddim_acquired_release(acq_dim.acq_rd);
944 +}
945 +
946 +static enum nml_training_result nml_acquired_dimension_train(nml_acquired_dimension_t acq_dim, const nml_training_request_t &TR) {
947 + if (!acq_dim.dim)
948 + return TRAINING_RESULT_NULL_ACQUIRED_DIMENSION;
949 +
950 + return nml_dimension_train_model(acq_dim.dim, TR);
951 +}
952 +
953 +#define WORKER_JOB_TRAINING_FIND 0
954 +#define WORKER_JOB_TRAINING_TRAIN 1
955 +#define WORKER_JOB_TRAINING_STATS 2
956 +
957 +void nml_host_get_detection_info_as_json(nml_host_t *host, nlohmann::json &j) {
958 + j["version"] = 1;
959 + j["anomalous-dimensions"] = host->mls.num_anomalous_dimensions;
960 + j["normal-dimensions"] = host->mls.num_normal_dimensions;
961 + j["total-dimensions"] = host->mls.num_anomalous_dimensions + host->mls.num_normal_dimensions;
962 + j["trained-dimensions"] = host->mls.num_training_status_trained + host->mls.num_training_status_pending_with_model;
963 +}
964 +
965 +void nml_host_train(nml_host_t *host) {
966 + worker_register("MLTRAIN");
967 + worker_register_job_name(WORKER_JOB_TRAINING_FIND, "find");
968 + worker_register_job_name(WORKER_JOB_TRAINING_TRAIN, "train");
969 + worker_register_job_name(WORKER_JOB_TRAINING_STATS, "stats");
970 +
971 + service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, (force_quit_t )ml_cancel_training_thread, host->rh, true);
972 +
973 + while (service_running(SERVICE_ML_TRAINING)) {
974 + nml_training_request_t training_req = nml_queue_pop(host->training_queue);
975 + size_t queue_size = nml_queue_size(host->training_queue) + 1;
976 +
977 + if (host->threads_cancelled) {
978 + info("Stopping training thread for host %s because it was cancelled", rrdhost_hostname(host->rh));
979 + break;
980 + }
981 +
982 + usec_t allotted_ut = (Cfg.train_every * host->rh->rrd_update_every * USEC_PER_SEC) / queue_size;
983 + if (allotted_ut > USEC_PER_SEC)
984 + allotted_ut = USEC_PER_SEC;
985 +
986 + usec_t start_ut = now_monotonic_usec();
987 + enum nml_training_result training_res;
988 + {
989 + worker_is_busy(WORKER_JOB_TRAINING_FIND);
990 + nml_acquired_dimension_t acq_dim = nml_acquired_dimension_get(host->rh, training_req.chart_id, training_req.dimension_id);
991 +
992 + worker_is_busy(WORKER_JOB_TRAINING_TRAIN);
993 + training_res = nml_acquired_dimension_train(acq_dim, training_req);
994 +
995 + string_freez(training_req.chart_id);
996 + string_freez(training_req.dimension_id);
997 +
998 + nml_acquired_dimension_release(acq_dim);
999 + }
1000 + usec_t consumed_ut = now_monotonic_usec() - start_ut;
1001 +
1002 + worker_is_busy(WORKER_JOB_TRAINING_STATS);
1003 +
1004 + usec_t remaining_ut = 0;
1005 + if (consumed_ut < allotted_ut)
1006 + remaining_ut = allotted_ut - consumed_ut;
1007 +
1008 + {
1009 + netdata_mutex_lock(&host->mutex);
1010 +
1011 + if (host->ts.allotted_ut == 0) {
1012 + struct rusage TRU;
1013 + getrusage(RUSAGE_THREAD, &TRU);
1014 + host->ts.training_ru = TRU;
1015 + }
1016 +
1017 + host->ts.queue_size += queue_size;
1018 + host->ts.num_popped_items += 1;
1019 +
1020 + host->ts.allotted_ut += allotted_ut;
1021 + host->ts.consumed_ut += consumed_ut;
1022 + host->ts.remaining_ut += remaining_ut;
1023 +
1024 + switch (training_res) {
1025 + case TRAINING_RESULT_OK:
1026 + host->ts.training_result_ok += 1;
1027 + break;
1028 + case TRAINING_RESULT_INVALID_QUERY_TIME_RANGE:
1029 + host->ts.training_result_invalid_query_time_range += 1;
1030 + break;
1031 + case TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES:
1032 + host->ts.training_result_not_enough_collected_values += 1;
1033 + break;
1034 + case TRAINING_RESULT_NULL_ACQUIRED_DIMENSION:
1035 + host->ts.training_result_null_acquired_dimension += 1;
1036 + break;
1037 + case TRAINING_RESULT_CHART_UNDER_REPLICATION:
1038 + host->ts.training_result_chart_under_replication += 1;
1039 + break;
1040 + }
1041 +
1042 + netdata_mutex_unlock(&host->mutex);
1043 + }
1044 +
1045 + worker_is_idle();
1046 + std::this_thread::sleep_for(std::chrono::microseconds{remaining_ut});
1047 + worker_is_busy(0);
1048 + }
1049 +}
1050 +
1051 +static void *train_main(void *arg) {
1052 + size_t max_elements_needed_for_training = Cfg.max_train_samples * (Cfg.lag_n + 1);
1053 + tls_data.training_cns = new calculated_number_t[max_elements_needed_for_training]();
1054 + tls_data.scratch_training_cns = new calculated_number_t[max_elements_needed_for_training]();
1055 +
1056 + nml_host_t *host = reinterpret_cast<nml_host_t *>(arg);
1057 + nml_host_train(host);
1058 + return NULL;
1059 +}
1060 +
1061 +void nml_host_start_training_thread(nml_host_t *host) {
1062 + if (host->threads_running) {
1063 + error("Anomaly detections threads for host %s are already-up and running.", rrdhost_hostname(host->rh));
1064 + return;
1065 + }
1066 +
1067 + host->threads_running = true;
1068 + host->threads_cancelled = false;
1069 + host->threads_joined = false;
1070 +
1071 + char tag[NETDATA_THREAD_TAG_MAX + 1];
1072 +
1073 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "MLTR[%s]", rrdhost_hostname(host->rh));
1074 + netdata_thread_create(&host->training_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, train_main, static_cast<void *>(host));
1075 +}
1076 +
1077 +void nml_host_stop_training_thread(nml_host_t *host, bool join) {
1078 + if (!host->threads_running) {
1079 + error("Anomaly detections threads for host %s have already been stopped.", rrdhost_hostname(host->rh));
1080 + return;
1081 + }
1082 +
1083 + if (!host->threads_cancelled) {
1084 + host->threads_cancelled = true;
1085 +
1086 + // Signal the training queue to stop popping-items
1087 + nml_queue_signal(host->training_queue);
1088 + netdata_thread_cancel(host->training_thread);
1089 + }
1090 +
1091 + if (join && !host->threads_joined) {
1092 + host->threads_joined = true;
1093 + host->threads_running = false;
1094 +
1095 + delete[] tls_data.training_cns;
1096 + delete[] tls_data.scratch_training_cns;
1097 +
1098 + netdata_thread_join(host->training_thread, NULL);
1099 + }
1100 +}
1101 +
1102 +void *nml_detect_main(void *arg) {
1103 + UNUSED(arg);
1104 +
1105 + worker_register("MLDETECT");
1106 + worker_register_job_name(WORKER_JOB_DETECTION_PREP, "prep");
1107 + worker_register_job_name(WORKER_JOB_DETECTION_DIM_CHART, "dim chart");
1108 + worker_register_job_name(WORKER_JOB_DETECTION_HOST_CHART, "host chart");
1109 + worker_register_job_name(WORKER_JOB_DETECTION_STATS, "stats");
1110 + worker_register_job_name(WORKER_JOB_DETECTION_RESOURCES, "resources");
1111 +
1112 + service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, NULL, NULL, true);
1113 +
1114 + heartbeat_t hb;
1115 + heartbeat_init(&hb);
1116 +
1117 + while (service_running((SERVICE_TYPE)(SERVICE_ML_PREDICTION | SERVICE_COLLECTORS))) {
1118 + worker_is_idle();
1119 + heartbeat_next(&hb, USEC_PER_SEC);
1120 +
1121 + rrd_rdlock();
1122 +
1123 + RRDHOST *rh;
1124 + rrdhost_foreach_read(rh) {
1125 + if (!rh->ml_host)
1126 + continue;
1127 +
1128 + nml_host_detect_once(reinterpret_cast<nml_host_t *>(rh->ml_host));
1129 + }
1130 +
1131 + rrd_unlock();
1132 + }
1133 +
1134 + return NULL;
1135 +}
ml/nml.h new
+346
@@ -0,0 +1,346 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_NML_H
4 +#define NETDATA_NML_H
5 +
6 +#include "dlib/matrix.h"
7 +#include "ml/ml.h"
8 +
9 +#include <vector>
10 +#include <queue>
11 +
12 +typedef double calculated_number_t;
13 +typedef dlib::matrix<calculated_number_t, 6, 1> DSample;
14 +
15 +/*
16 + * Features
17 + */
18 +
19 +typedef struct {
20 + size_t diff_n;
21 + size_t smooth_n;
22 + size_t lag_n;
23 +
24 + calculated_number_t *dst;
25 + size_t dst_n;
26 +
27 + calculated_number_t *src;
28 + size_t src_n;
29 +
30 + std::vector<DSample> &preprocessed_features;
31 +} nml_features_t;
32 +
33 +/*
34 + * KMeans
35 + */
36 +typedef struct {
37 + size_t num_clusters;
38 + size_t max_iterations;
39 +
40 + std::vector<DSample> cluster_centers;
41 +
42 + calculated_number_t min_dist;
43 + calculated_number_t max_dist;
44 +} nml_kmeans_t;
45 +
46 +#include "json/single_include/nlohmann/json.hpp"
47 +
48 +typedef struct machine_learning_stats_t {
49 + size_t num_machine_learning_status_enabled;
50 + size_t num_machine_learning_status_disabled_sp;
51 +
52 + size_t num_metric_type_constant;
53 + size_t num_metric_type_variable;
54 +
55 + size_t num_training_status_untrained;
56 + size_t num_training_status_pending_without_model;
57 + size_t num_training_status_trained;
58 + size_t num_training_status_pending_with_model;
59 +
60 + size_t num_anomalous_dimensions;
61 + size_t num_normal_dimensions;
62 +} nml_machine_learning_stats_t;
63 +
64 +typedef struct training_stats_t {
65 + struct rusage training_ru;
66 +
67 + size_t queue_size;
68 + size_t num_popped_items;
69 +
70 + usec_t allotted_ut;
71 + usec_t consumed_ut;
72 + usec_t remaining_ut;
73 +
74 + size_t training_result_ok;
75 + size_t training_result_invalid_query_time_range;
76 + size_t training_result_not_enough_collected_values;
77 + size_t training_result_null_acquired_dimension;
78 + size_t training_result_chart_under_replication;
79 +} nml_training_stats_t;
80 +
81 +enum nml_metric_type {
82 + // The dimension has constant values, no need to train
83 + METRIC_TYPE_CONSTANT,
84 +
85 + // The dimension's values fluctuate, we need to generate a model
86 + METRIC_TYPE_VARIABLE,
87 +};
88 +
89 +enum nml_machine_learning_status {
90 + // Enable training/prediction
91 + MACHINE_LEARNING_STATUS_ENABLED,
92 +
93 + // Disable because configuration pattern matches the chart's id
94 + MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART,
95 +};
96 +
97 +enum nml_training_status {
98 + // We don't have a model for this dimension
99 + TRAINING_STATUS_UNTRAINED,
100 +
101 + // Request for training sent, but we don't have any models yet
102 + TRAINING_STATUS_PENDING_WITHOUT_MODEL,
103 +
104 + // Request to update existing models sent
105 + TRAINING_STATUS_PENDING_WITH_MODEL,
106 +
107 + // Have a valid, up-to-date model
108 + TRAINING_STATUS_TRAINED,
109 +};
110 +
111 +enum nml_training_result {
112 + // We managed to create a KMeans model
113 + TRAINING_RESULT_OK,
114 +
115 + // Could not query DB with a correct time range
116 + TRAINING_RESULT_INVALID_QUERY_TIME_RANGE,
117 +
118 + // Did not gather enough data from DB to run KMeans
119 + TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES,
120 +
121 + // Acquired a null dimension
122 + TRAINING_RESULT_NULL_ACQUIRED_DIMENSION,
123 +
124 + // Chart is under replication
125 + TRAINING_RESULT_CHART_UNDER_REPLICATION,
126 +};
127 +
128 +typedef struct {
129 + // Chart/dimension we want to train
130 + STRING *chart_id;
131 + STRING *dimension_id;
132 +
133 + // Creation time of request
134 + time_t request_time;
135 +
136 + // First/last entry of this dimension in DB
137 + // at the point the request was made
138 + time_t first_entry_on_request;
139 + time_t last_entry_on_request;
140 +} nml_training_request_t;
141 +
142 +typedef struct {
143 + // Time when the request for this response was made
144 + time_t request_time;
145 +
146 + // First/last entry of the dimension in DB when generating the request
147 + time_t first_entry_on_request;
148 + time_t last_entry_on_request;
149 +
150 + // First/last entry of the dimension in DB when generating the response
151 + time_t first_entry_on_response;
152 + time_t last_entry_on_response;
153 +
154 + // After/Before timestamps of our DB query
155 + time_t query_after_t;
156 + time_t query_before_t;
157 +
158 + // Actual after/before returned by the DB query ops
159 + time_t db_after_t;
160 + time_t db_before_t;
161 +
162 + // Number of doubles returned by the DB query
163 + size_t collected_values;
164 +
165 + // Number of values we return to the caller
166 + size_t total_values;
167 +
168 + // Result of training response
169 + enum nml_training_result result;
170 +} nml_training_response_t;
171 +
172 +/*
173 + * Queue
174 +*/
175 +
176 +typedef struct {
177 + std::queue<nml_training_request_t> internal;
178 + netdata_mutex_t mutex;
179 + pthread_cond_t cond_var;
180 + std::atomic<bool> exit;
181 +} nml_queue_t;
182 +
183 +nml_queue_t *nml_queue_init(void);
184 +void nml_queue_destroy(nml_queue_t *q);
185 +
186 +void nml_queue_push(nml_queue_t *q, const nml_training_request_t req);
187 +nml_training_request_t nml_queue_pop(nml_queue_t *q);
188 +size_t nml_queue_size(nml_queue_t *q);
189 +
190 +void nml_queue_signal(nml_queue_t *q);
191 +
192 +typedef struct {
193 + RRDDIM *rd;
194 +
195 + enum nml_metric_type mt;
196 + enum nml_training_status ts;
197 + enum nml_machine_learning_status mls;
198 +
199 + nml_training_response_t tr;
200 + time_t last_training_time;
201 +
202 + std::vector<calculated_number_t> cns;
203 +
204 + std::vector<nml_kmeans_t> km_contexts;
205 + netdata_mutex_t mutex;
206 + nml_kmeans_t kmeans;
207 + std::vector<DSample> feature;
208 +} nml_dimension_t;
209 +
210 +nml_dimension_t *nml_dimension_new(RRDDIM *rd);
211 +void nml_dimension_delete(nml_dimension_t *dim);
212 +
213 +bool nml_dimension_predict(nml_dimension_t *d, time_t curr_t, calculated_number_t value, bool exists);
214 +
215 +typedef struct {
216 + RRDSET *rs;
217 + nml_machine_learning_stats_t mls;
218 +
219 + netdata_mutex_t mutex;
220 +} nml_chart_t;
221 +
222 +nml_chart_t *nml_chart_new(RRDSET *rs);
223 +void nml_chart_delete(nml_chart_t *chart);
224 +
225 +void nml_chart_update_begin(nml_chart_t *chart);
226 +void nml_chart_update_end(nml_chart_t *chart);
227 +void nml_chart_update_dimension(nml_chart_t *chart, nml_dimension_t *dim, bool is_anomalous);
228 +
229 +typedef struct {
230 + RRDHOST *rh;
231 +
232 + nml_machine_learning_stats_t mls;
233 + nml_training_stats_t ts;
234 +
235 + calculated_number_t host_anomaly_rate;
236 +
237 + std::atomic<bool> threads_running;
238 + std::atomic<bool> threads_cancelled;
239 + std::atomic<bool> threads_joined;
240 +
241 + nml_queue_t *training_queue;
242 +
243 + netdata_mutex_t mutex;
244 +
245 + netdata_thread_t training_thread;
246 +
247 + /*
248 + * bookkeeping for anomaly detection charts
249 + */
250 +
251 + RRDSET *machine_learning_status_rs;
252 + RRDDIM *machine_learning_status_enabled_rd;
253 + RRDDIM *machine_learning_status_disabled_sp_rd;
254 +
255 + RRDSET *metric_type_rs;
256 + RRDDIM *metric_type_constant_rd;
257 + RRDDIM *metric_type_variable_rd;
258 +
259 + RRDSET *training_status_rs;
260 + RRDDIM *training_status_untrained_rd;
261 + RRDDIM *training_status_pending_without_model_rd;
262 + RRDDIM *training_status_trained_rd;
263 + RRDDIM *training_status_pending_with_model_rd;
264 +
265 + RRDSET *dimensions_rs;
266 + RRDDIM *dimensions_anomalous_rd;
267 + RRDDIM *dimensions_normal_rd;
268 +
269 + RRDSET *anomaly_rate_rs;
270 + RRDDIM *anomaly_rate_rd;
271 +
272 + RRDSET *detector_events_rs;
273 + RRDDIM *detector_events_above_threshold_rd;
274 + RRDDIM *detector_events_new_anomaly_event_rd;
275 +
276 + RRDSET *queue_stats_rs;
277 + RRDDIM *queue_stats_queue_size_rd;
278 + RRDDIM *queue_stats_popped_items_rd;
279 +
280 + RRDSET *training_time_stats_rs;
281 + RRDDIM *training_time_stats_allotted_rd;
282 + RRDDIM *training_time_stats_consumed_rd;
283 + RRDDIM *training_time_stats_remaining_rd;
284 +
285 + RRDSET *training_results_rs;
286 + RRDDIM *training_results_ok_rd;
287 + RRDDIM *training_results_invalid_query_time_range_rd;
288 + RRDDIM *training_results_not_enough_collected_values_rd;
289 + RRDDIM *training_results_null_acquired_dimension_rd;
290 + RRDDIM *training_results_chart_under_replication_rd;
291 +} nml_host_t;
292 +
293 +nml_host_t *nml_host_new(RRDHOST *rh);
294 +void nml_host_delete(nml_host_t *host);
295 +
296 +void nml_host_start_training_thread(nml_host_t *host);
297 +void nml_host_stop_training_thread(nml_host_t *host, bool join);
298 +
299 +void nml_host_get_config_as_json(nml_host_t *host, BUFFER *wb);
300 +void nml_host_get_models_as_json(nml_host_t *host, nlohmann::json &j);
301 +void nml_host_get_detection_info_as_json(nml_host_t *host, nlohmann::json &j);
302 +
303 +typedef struct {
304 + bool enable_anomaly_detection;
305 +
306 + unsigned max_train_samples;
307 + unsigned min_train_samples;
308 + unsigned train_every;
309 +
310 + unsigned num_models_to_use;
311 +
312 + unsigned db_engine_anomaly_rate_every;
313 +
314 + unsigned diff_n;
315 + unsigned smooth_n;
316 + unsigned lag_n;
317 +
318 + double random_sampling_ratio;
319 + unsigned max_kmeans_iters;
320 +
321 + double dimension_anomaly_score_threshold;
322 +
323 + double host_anomaly_rate_threshold;
324 + RRDR_TIME_GROUPING anomaly_detection_grouping_method;
325 + time_t anomaly_detection_query_duration;
326 +
327 + bool stream_anomaly_detection_charts;
328 +
329 + std::string hosts_to_skip;
330 + SIMPLE_PATTERN *sp_host_to_skip;
331 +
332 + std::string charts_to_skip;
333 + SIMPLE_PATTERN *sp_charts_to_skip;
334 +
335 + std::vector<uint32_t> random_nums;
336 +
337 + netdata_thread_t detection_thread;
338 +} nml_config_t;
339 +
340 +void nml_config_load(nml_config_t *cfg);
341 +
342 +void *nml_detect_main(void *arg);
343 +
344 +extern nml_config_t Cfg;
345 +
346 +#endif /* NETDATA_NML_H */