@cryptotaxi247 / netdata-1 / commits / 689dc6b7f

Refactor ML code and add support for multiple KMeans models. (#14065)

* Add profile.plugin Creates the specified number of charts/dimensions, and supports backfilling with pseudo-historical data. * Bump * Remove wrongly merged line. * Use the number of models specified from the config section. * Add option to consult all ML models. * Remove profiling option consuming all models. * Add underscore after chart name prefix. * prediction -> dimensions chart * reorder funcs * Split charts across types with correct priority * Ignore training request when chart is under replication. * Track global number of models consulted. * Cleanup config. * initial readme updates * fix readme * readme * Fix function definition when ML is disabled. * Add dummy ml_chart_update_{begin,end} * Remove profile_plugin * Define chart priorities under collectors/all.h * s/curr_t/current_time/ Co-authored-by: Andrew Maguire <andrewm4894@gmail.com>

vkalintiris committed Dec 21, 2022 at 15:03 UTC 689dc6b7fbbf495ce3e020dcff0d014a8d338c52
31 files changed +1411 -785
Makefile.am
+4 -13
@@ -236,10 +236,14 @@ ML_FILES += \
236 ml/ADCharts.cc \
237 ml/Config.h \
238 ml/Config.cc \
239 + ml/Chart.cc \
240 + ml/Chart.h \
241 + ml/Stats.h \
242 ml/Dimension.cc \
243 ml/Dimension.h \
244 ml/Host.h \
245 ml/Host.cc \
246 + ml/Queue.h \
247 ml/Query.h \
248 ml/KMeans.h \
249 ml/KMeans.cc \
@@ -262,13 +266,6 @@ ml/ml.$(OBJEXT) : CXXFLAGS += -Wno-psabi
266
267 endif
268
265 -
266 -if ENABLE_ML_TESTS
267 -ML_TESTS_FILES = \
268 - ml/SamplesBufferTests.cc \
269 - $(NULL)
270 -endif
271 -
269 IDLEJITTER_PLUGIN_FILES = \
270 collectors/idlejitter.plugin/plugin_idlejitter.c \
271 $(NULL)
@@ -920,7 +917,6 @@ NETDATA_FILES = \
917 $(EXPORTING_ENGINE_FILES) \
918 $(HEALTH_PLUGIN_FILES) \
919 $(ML_FILES) \
923 - $(ML_TESTS_FILES) \
920 $(IDLEJITTER_PLUGIN_FILES) \
921 $(PLUGINSD_PLUGIN_FILES) \
922 $(REGISTRY_PLUGIN_FILES) \
@@ -1008,11 +1004,6 @@ if ENABLE_ACLK
1004 $(NULL)
1005 endif
1006
1011 -if ENABLE_ML_TESTS
1012 - netdata_LDADD += $(OPTIONAL_ML_TESTS_LIBS) \
1013 - $(NULL)
1014 -endif
1015 -
1007 netdata_LINK = $(CXXLD) $(CXXFLAGS) $(LDFLAGS) -o $@
1008
1009 sbin_PROGRAMS += netdatacli
collectors/all.h
+18
@@ -363,5 +363,23 @@
363 #define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030
364 #define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100
365
366 +// NETDATA ML CHARTS
367 +
368 +// [ml] charts
369 +#define ML_CHART_PRIO_DIMENSIONS 39181
370 +#define ML_CHART_PRIO_ANOMALY_RATE 39182
371 +#define ML_CHART_PRIO_DETECTOR_EVENTS 39183
372 +
373 +// [netdata.ml] charts
374 +#define NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS 890001
375 +#define NETDATA_ML_CHART_PRIO_METRIC_TYPES 890002
376 +#define NETDATA_ML_CHART_PRIO_TRAINING_STATUS 890003
377 +
378 +#define NETDATA_ML_CHART_PRIO_PREDICTION_USAGE 890004
379 +#define NETDATA_ML_CHART_PRIO_TRAINING_USAGE 890005
380 +
381 +#define NETDATA_ML_CHART_PRIO_QUEUE_STATS 890006
382 +#define NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS 890007
383 +#define NETDATA_ML_CHART_PRIO_TRAINING_RESULTS 890008
384
385 #endif //NETDATA_ALL_H
configure.ac
+1 -22
@@ -207,12 +207,6 @@ AC_ARG_ENABLE(
207 ,
208 [enable_ml="detect"]
209 )
210 -AC_ARG_ENABLE(
211 - [ml_tests],
212 - [AS_HELP_STRING([--enable-ml-tests], [Enable anomaly detection tests @<:@no@:>@])],
213 - [enable_ml_tests="yes"],
214 - [enable_ml_tests="no"]
215 -)
210 AC_ARG_ENABLE(
211 [aclk_ssl_debug],
212 [AS_HELP_STRING([--enable-aclk-ssl-debug], [Enables possibility for SSL key logging @<:@default no@:>@])],
@@ -1180,19 +1174,6 @@ if test "${build_ml}" = "yes"; then
1174 OPTIONAL_ML_LIBS=""
1175 fi
1176
1183 -# Decide if we should build ML tests.
1184 -if test "${build_ml}" = "yes" -a "${enable_ml_tests}" = "yes" -a "${have_gtest}" = "yes"; then
1185 - build_ml_tests="yes"
1186 -else
1187 - build_ml_tests="no"
1188 -fi
1189 -
1190 -AM_CONDITIONAL([ENABLE_ML_TESTS], [test "${build_ml_tests}" = "yes"])
1191 -if test "${build_ml_tests}" = "yes"; then
1192 - AC_DEFINE([ENABLE_ML_TESTS], [1], [anomaly detection tests])
1193 - OPTIONAL_ML_TESTS_CFLAGS="${OPTIONAL_GTEST_CFLAGS}"
1194 - OPTIONAL_ML_TESTS_LIBS="${OPTIONAL_GTEST_LIBS}"
1195 -fi
1177
1178 # -----------------------------------------------------------------------------
1179 # ebpf.plugin
@@ -1612,7 +1593,7 @@ CFLAGS="${originalCFLAGS} ${OPTIONAL_LTO_CFLAGS} ${OPTIONAL_PROTOBUF_CFLAGS} ${O
1593 ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
1594 ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PUBSUB_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} \
1595 ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
1615 - ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_ML_TESTS_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1596 + ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1597
1598 CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
1599
@@ -1666,8 +1647,6 @@ AC_SUBST([OPTIONAL_GTEST_CFLAGS])
1647 AC_SUBST([OPTIONAL_GTEST_LIBS])
1648 AC_SUBST([OPTIONAL_ML_CFLAGS])
1649 AC_SUBST([OPTIONAL_ML_LIBS])
1669 -AC_SUBST([OPTIONAL_ML_TESTS_CFLAGS])
1670 -AC_SUBST([OPTIONAL_ML_TESTS_LIBS])
1650
1651 # -----------------------------------------------------------------------------
1652 # Check if cmocka is available - needed for unit testing
daemon/global_statistics.c
+34
@@ -52,6 +52,7 @@ static struct global_statistics {
52 uint64_t ml_queries_made;
53 uint64_t ml_db_points_read;
54 uint64_t ml_result_points_generated;
55 + uint64_t ml_models_consulted;
56
57 uint64_t exporters_queries_made;
58 uint64_t exporters_db_points_read;
@@ -88,6 +89,10 @@ void global_statistics_ml_query_completed(size_t points_read) {
89 __atomic_fetch_add(&global_statistics.ml_db_points_read, points_read, __ATOMIC_RELAXED);
90 }
91
92 +void global_statistics_ml_models_consulted(size_t models_consulted) {
93 + __atomic_fetch_add(&global_statistics.ml_models_consulted, models_consulted, __ATOMIC_RELAXED);
94 +}
95 +
96 void global_statistics_exporters_query_completed(size_t points_read) {
97 __atomic_fetch_add(&global_statistics.exporters_queries_made, 1, __ATOMIC_RELAXED);
98 __atomic_fetch_add(&global_statistics.exporters_db_points_read, points_read, __ATOMIC_RELAXED);
@@ -193,6 +198,7 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
198 gs->ml_queries_made = __atomic_load_n(&global_statistics.ml_queries_made, __ATOMIC_RELAXED);
199 gs->ml_db_points_read = __atomic_load_n(&global_statistics.ml_db_points_read, __ATOMIC_RELAXED);
200 gs->ml_result_points_generated = __atomic_load_n(&global_statistics.ml_result_points_generated, __ATOMIC_RELAXED);
201 + gs->ml_models_consulted = __atomic_load_n(&global_statistics.ml_models_consulted, __ATOMIC_RELAXED);
202
203 gs->exporters_queries_made = __atomic_load_n(&global_statistics.exporters_queries_made, __ATOMIC_RELAXED);
204 gs->exporters_db_points_read = __atomic_load_n(&global_statistics.exporters_db_points_read, __ATOMIC_RELAXED);
@@ -653,6 +659,34 @@ static void global_statistics_charts(void) {
659
660 rrdset_done(st_points_stored);
661 }
662 +
663 + {
664 + static RRDSET *st = NULL;
665 + static RRDDIM *rd = NULL;
666 +
667 + if (unlikely(!st)) {
668 + st = rrdset_create_localhost(
669 + "netdata" // type
670 + , "ml_models_consulted" // id
671 + , NULL // name
672 + , "ml" // family
673 + , NULL // context
674 + , "KMeans models used for prediction" // title
675 + , "models" // units
676 + , "netdata" // plugin
677 + , "ml" // module
678 + , 131004 // priority
679 + , localhost->rrd_update_every // update_every
680 + , RRDSET_TYPE_STACKED // chart_type
681 + );
682 +
683 + rd = rrddim_add(st, "num_models_consulted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
684 + }
685 +
686 + rrddim_set_by_pointer(st, rd, (collected_number) gs.ml_models_consulted);
687 +
688 + rrdset_done(st);
689 + }
690 }
691
692 // ----------------------------------------------------------------------------
daemon/global_statistics.h
+1
@@ -9,6 +9,7 @@
9 // global statistics
10
11 void global_statistics_ml_query_completed(size_t points_read);
12 +void global_statistics_ml_models_consulted(size_t models_consulted);
13 void global_statistics_exporters_query_completed(size_t points_read);
14 void global_statistics_backfill_query_completed(size_t points_read);
15 void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source);
daemon/main.c
-5
@@ -1027,11 +1027,6 @@ int main(int argc, char **argv) {
1027 else if(strcmp(optarg, "escapetest") == 0) {
1028 return command_argument_sanitization_tests();
1029 }
1030 -#ifdef ENABLE_ML_TESTS
1031 - else if(strcmp(optarg, "mltest") == 0) {
1032 - return test_ml(argc, argv);
1033 - }
1034 -#endif
1030 #ifdef ENABLE_DBENGINE
1031 else if(strcmp(optarg, "mctest") == 0) {
1032 unittest_running = true;
database/rrd.h
+12 -6
@@ -30,8 +30,9 @@ typedef struct rrdhost_acquired RRDHOST_ACQUIRED;
30 typedef struct rrdset_acquired RRDSET_ACQUIRED;
31 typedef struct rrddim_acquired RRDDIM_ACQUIRED;
32
33 -typedef void *ml_host_t;
34 -typedef void *ml_dimension_t;
33 +typedef struct ml_host ml_host_t;
34 +typedef struct ml_chart ml_chart_t;
35 +typedef struct ml_dimension ml_dimension_t;
36
37 typedef enum {
38 QUERY_SOURCE_UNKNOWN,
@@ -296,7 +297,7 @@ struct rrddim {
297 // ------------------------------------------------------------------------
298 // operational state members
299
299 - ml_dimension_t ml_dimension; // machine learning data about this dimension
300 + ml_dimension_t *ml_dimension; // machine learning data about this dimension
301
302 // ------------------------------------------------------------------------
303 // linking to siblings and parents
@@ -595,6 +596,8 @@ struct rrdset {
596 DICTIONARY *rrddimvar_root_index; // dimension variables
597 // we use this dictionary to manage their allocation
598
599 + ml_chart_t *ml_chart;
600 +
601 // ------------------------------------------------------------------------
602 // operational state members
603
@@ -1028,7 +1031,7 @@ struct rrdhost {
1031
1032 // ------------------------------------------------------------------------
1033 // ML handle
1031 - ml_host_t ml_host;
1034 + ml_host_t *ml_host;
1035
1036 // ------------------------------------------------------------------------
1037 // Support for host-level labels
@@ -1301,9 +1304,12 @@ void rrdset_isnot_obsolete(RRDSET *st);
1304 time_t rrddim_first_entry_t(RRDDIM *rd);
1305 time_t rrddim_first_entry_t_of_tier(RRDDIM *rd, size_t tier);
1306 time_t rrddim_last_entry_t(RRDDIM *rd);
1304 -time_t rrdset_last_entry_t(RRDSET *st);
1305 -time_t rrdset_first_entry_t_of_tier(RRDSET *st, size_t tier);
1307 +time_t rrddim_last_entry_t_of_tier(RRDDIM *rd, size_t tier);
1308 +
1309 time_t rrdset_first_entry_t(RRDSET *st);
1310 +time_t rrdset_first_entry_t_of_tier(RRDSET *st, size_t tier);
1311 +time_t rrdset_last_entry_t(RRDSET *st);
1312 +
1313 time_t rrdhost_last_entry_t(RRDHOST *h);
1314
1315 // ----------------------------------------------------------------------------
database/rrdcontext.c
-9
@@ -750,11 +750,6 @@ static void rrdinstance_free(RRDINSTANCE *ri) {
750 }
751
752 static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) {
753 - static STRING *ml_anomaly_rates_id = NULL;
754 -
755 - if(unlikely(!ml_anomaly_rates_id))
756 - ml_anomaly_rates_id = string_strdupz(ML_ANOMALY_RATES_CHART_ID);
757 -
753 RRDINSTANCE *ri = value;
754
755 // link it to its parent
@@ -781,10 +776,6 @@ static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unus
776 ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor
777 }
778
784 - // we need this when loading from SQL
785 - if(unlikely(ri->id == ml_anomaly_rates_id))
786 - ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
787 -
779 rrdmetrics_create_in_rrdinstance(ri);
780
781 // signal the react callback to do the job
database/rrddim.c
+9 -3
@@ -172,7 +172,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
172 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
173 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
174
175 - ml_new_dimension(rd);
175 + ml_dimension_new(rd);
176
177 ctr->react_action = RRDDIM_REACT_NEW;
178
@@ -191,7 +191,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
191
192 rrdcontext_removed_rrddim(rd);
193
194 - ml_delete_dimension(rd);
194 + ml_dimension_delete(rd);
195
196 debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
197
@@ -420,7 +420,13 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
420
421 // ----------------------------------------------------------------------------
422
423 -// get the timestamp of the last entry in the round-robin database
423 +time_t rrddim_last_entry_t_of_tier(RRDDIM *rd, size_t tier) {
424 + if(unlikely(tier > storage_tiers || !rd->tiers[tier]))
425 + return 0;
426 +
427 + return rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle);
428 +}
429 +
430 time_t rrddim_last_entry_t(RRDDIM *rd) {
431 time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
432
database/rrdhost.c
+3 -3
@@ -518,7 +518,7 @@ int is_legacy = 1;
518
519 rrdhost_load_rrdcontext_data(host);
520 if (!archived)
521 - ml_new_host(host);
521 + ml_host_new(host);
522 else
523 rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
524
@@ -629,7 +629,7 @@ void rrdhost_update(RRDHOST *host
629 host->rrdpush_replication_step = rrdpush_replication_step;
630
631 rrd_hosts_available++;
632 - ml_new_host(host);
632 + ml_host_new(host);
633 rrdhost_load_rrdcontext_data(host);
634 info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
635 }
@@ -1089,7 +1089,7 @@ void rrdhost_free(RRDHOST *host, bool force) {
1089 rrd_check_wrlock(); // make sure the RRDs are write locked
1090
1091 rrdhost_wrlock(host);
1092 - ml_delete_host(host);
1092 + ml_host_delete(host);
1093 rrdhost_unlock(host);
1094
1095 // ------------------------------------------------------------------------
database/rrdset.c
+15 -3
@@ -178,6 +178,8 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
178 st->red = NAN;
179
180 ctr->react_action = RRDSET_REACT_NEW;
181 +
182 + ml_chart_new(st);
183 }
184
185 // the destructor - the dictionary is write locked while this runs
@@ -232,6 +234,9 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
234 // 7. destroy the chart labels
235 rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
236
237 + // 8. destroy the ml handle
238 + ml_chart_delete(st);
239 +
240 rrdset_memory_file_free(st); // remove files of db mode save and map
241
242 // ------------------------------------------------------------------------
@@ -1253,6 +1258,8 @@ static inline size_t rrdset_done_interpolate(
1258
1259 last_ut = next_store_ut;
1260
1261 + ml_chart_update_begin(st);
1262 +
1263 struct rda_item *rda;
1264 size_t dim_id;
1265 for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
@@ -1332,8 +1339,11 @@ static inline size_t rrdset_done_interpolate(
1339 break;
1340 }
1341
1342 + time_t current_time = (time_t) (next_store_ut / USEC_PER_SEC);
1343 +
1344 if(unlikely(!store_this_entry)) {
1336 - (void) ml_is_anomalous(rd, 0, false);
1345 + (void) ml_is_anomalous(rd, current_time, 0, false);
1346 +
1347 rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1348 rrdcontext_collected_rrddim(rd);
1349 continue;
@@ -1342,7 +1352,7 @@ static inline size_t rrdset_done_interpolate(
1352 if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) {
1353 uint32_t dim_storage_flags = storage_flags;
1354
1345 - if (ml_is_anomalous(rd, new_value, true)) {
1355 + if (ml_is_anomalous(rd, current_time, new_value, true)) {
1356 // clear anomaly bit: 0 -> is anomalous, 1 -> not anomalous
1357 dim_storage_flags &= ~((storage_number)SN_FLAG_NOT_ANOMALOUS);
1358 }
@@ -1352,7 +1362,7 @@ static inline size_t rrdset_done_interpolate(
1362 rd->last_stored_value = new_value;
1363 }
1364 else {
1355 - (void) ml_is_anomalous(rd, 0, false);
1365 + (void) ml_is_anomalous(rd, current_time, 0, false);
1366
1367 rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
1368
@@ -1364,6 +1374,8 @@ static inline size_t rrdset_done_interpolate(
1374 stored_entries++;
1375 }
1376
1377 + ml_chart_update_end(st);
1378 +
1379 // reset the storage flags for the next point, if any;
1380 storage_flags = SN_DEFAULT_FLAGS;
1381
ml/ADCharts.cc
+387 -109
@@ -3,55 +3,182 @@
3 #include "ADCharts.h"
4 #include "Config.h"
5
6 -void ml::updateDimensionsChart(RRDHOST *RH,
7 - collected_number NumTrainedDimensions,
8 - collected_number NumNormalDimensions,
9 - collected_number NumAnomalousDimensions) {
10 - static thread_local RRDSET *RS = nullptr;
11 - static thread_local RRDDIM *NumTotalDimensionsRD = nullptr;
12 - static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr;
13 - static thread_local RRDDIM *NumNormalDimensionsRD = nullptr;
14 - static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
15 -
16 - if (!RS) {
17 - std::stringstream IdSS, NameSS;
6 +void ml::updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS) {
7 + /*
8 + * Machine learning status
9 + */
10 + {
11 + static thread_local RRDSET *MachineLearningStatusRS = nullptr;
12 +
13 + static thread_local RRDDIM *Enabled = nullptr;
14 + static thread_local RRDDIM *DisabledUE = nullptr;
15 + static thread_local RRDDIM *DisabledSP = nullptr;
16 +
17 + if (!MachineLearningStatusRS) {
18 + std::stringstream IdSS, NameSS;
19 +
20 + IdSS << "machine_learning_status_for_" << localhost->machine_guid;
21 + NameSS << "machine_learning_status_for_" << localhost->hostname;
22 +
23 + MachineLearningStatusRS = rrdset_create_localhost(
24 + "netdata", // type
25 + IdSS.str().c_str(), // id
26 + NameSS.str().c_str(), // name
27 + "ml", // family
28 + "netdata.machine_learning_status", // ctx
29 + "Machine learning status", // title
30 + "dimensions", // units
31 + "netdata", // plugin
32 + "ml", // module
33 + NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
34 + RH->rrd_update_every, // update_every
35 + RRDSET_TYPE_LINE // chart_type
36 + );
37 + rrdset_flag_set(MachineLearningStatusRS , RRDSET_FLAG_ANOMALY_DETECTION);
38 +
39 + Enabled = rrddim_add(MachineLearningStatusRS, "enabled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
40 + DisabledUE = rrddim_add(MachineLearningStatusRS, "disabled-ue", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
41 + DisabledSP = rrddim_add(MachineLearningStatusRS, "disabled-sp", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
42 + }
43 +
44 + rrddim_set_by_pointer(MachineLearningStatusRS, Enabled, MLS.NumMachineLearningStatusEnabled);
45 + rrddim_set_by_pointer(MachineLearningStatusRS, DisabledUE, MLS.NumMachineLearningStatusDisabledUE);
46 + rrddim_set_by_pointer(MachineLearningStatusRS, DisabledSP, MLS.NumMachineLearningStatusDisabledSP);
47 +
48 + rrdset_done(MachineLearningStatusRS);
49 + }
50
19 - IdSS << "dimensions_on_" << localhost->machine_guid;
20 - NameSS << "dimensions_on_" << localhost->hostname;
51 + /*
52 + * Metric type
53 + */
54 + {
55 + static thread_local RRDSET *MetricTypesRS = nullptr;
56 +
57 + static thread_local RRDDIM *Constant = nullptr;
58 + static thread_local RRDDIM *Variable = nullptr;
59 +
60 + if (!MetricTypesRS) {
61 + std::stringstream IdSS, NameSS;
62 +
63 + IdSS << "metric_types_for_" << localhost->machine_guid;
64 + NameSS << "metric_types_for_" << localhost->hostname;
65 +
66 + MetricTypesRS = rrdset_create_localhost(
67 + "netdata", // type
68 + IdSS.str().c_str(), // id
69 + NameSS.str().c_str(), // name
70 + "ml", // family
71 + "netdata.metric_types", // ctx
72 + "Dimensions by metric type", // title
73 + "dimensions", // units
74 + "netdata", // plugin
75 + "ml", // module
76 + NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
77 + RH->rrd_update_every, // update_every
78 + RRDSET_TYPE_LINE // chart_type
79 + );
80 + rrdset_flag_set(MetricTypesRS, RRDSET_FLAG_ANOMALY_DETECTION);
81 +
82 + Constant = rrddim_add(MetricTypesRS, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
83 + Variable = rrddim_add(MetricTypesRS, "variable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
84 + }
85 +
86 + rrddim_set_by_pointer(MetricTypesRS, Constant, MLS.NumMetricTypeConstant);
87 + rrddim_set_by_pointer(MetricTypesRS, Variable, MLS.NumMetricTypeVariable);
88 +
89 + rrdset_done(MetricTypesRS);
90 + }
91
22 - RS = rrdset_create(
23 - RH,
24 - "anomaly_detection", // type
25 - IdSS.str().c_str(), // id
26 - NameSS.str().c_str(), // name
27 - "dimensions", // family
28 - "anomaly_detection.dimensions", // ctx
29 - "Anomaly detection dimensions", // title
30 - "dimensions", // units
31 - "netdata", // plugin
32 - "ml", // module
33 - 39183, // priority
34 - RH->rrd_update_every, // update_every
35 - RRDSET_TYPE_LINE // chart_type
36 - );
37 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
38 -
39 - NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
40 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
41 - NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL,
42 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
43 - NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL,
44 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
45 - NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL,
46 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
92 + /*
93 + * Training status
94 + */
95 + {
96 + static thread_local RRDSET *TrainingStatusRS = nullptr;
97 +
98 + static thread_local RRDDIM *Untrained = nullptr;
99 + static thread_local RRDDIM *PendingWithoutModel = nullptr;
100 + static thread_local RRDDIM *Trained = nullptr;
101 + static thread_local RRDDIM *PendingWithModel = nullptr;
102 +
103 + if (!TrainingStatusRS) {
104 + std::stringstream IdSS, NameSS;
105 +
106 + IdSS << "training_status_for_" << localhost->machine_guid;
107 + NameSS << "training_status_for_" << localhost->hostname;
108 +
109 + TrainingStatusRS = rrdset_create_localhost(
110 + "netdata", // type
111 + IdSS.str().c_str(), // id
112 + NameSS.str().c_str(), // name
113 + "ml", // family
114 + "netdata.training_status", // ctx
115 + "Training status of dimensions", // title
116 + "dimensions", // units
117 + "netdata", // plugin
118 + "ml", // module
119 + NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
120 + RH->rrd_update_every, // update_every
121 + RRDSET_TYPE_LINE // chart_type
122 + );
123 +
124 + rrdset_flag_set(TrainingStatusRS, RRDSET_FLAG_ANOMALY_DETECTION);
125 +
126 + Untrained = rrddim_add(TrainingStatusRS, "untrained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
127 + PendingWithoutModel = rrddim_add(TrainingStatusRS, "pending-without-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
128 + Trained = rrddim_add(TrainingStatusRS, "trained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
129 + PendingWithModel = rrddim_add(TrainingStatusRS, "pending-with-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
130 + }
131 +
132 + rrddim_set_by_pointer(TrainingStatusRS, Untrained, MLS.NumTrainingStatusUntrained);
133 + rrddim_set_by_pointer(TrainingStatusRS, PendingWithoutModel, MLS.NumTrainingStatusPendingWithoutModel);
134 + rrddim_set_by_pointer(TrainingStatusRS, Trained, MLS.NumTrainingStatusTrained);
135 + rrddim_set_by_pointer(TrainingStatusRS, PendingWithModel, MLS.NumTrainingStatusPendingWithModel);
136 +
137 + rrdset_done(TrainingStatusRS);
138 }
139
49 - rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions);
50 - rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions);
51 - rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions);
52 - rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions);
140 + /*
141 + * Prediction status
142 + */
143 + {
144 + static thread_local RRDSET *PredictionRS = nullptr;
145 +
146 + static thread_local RRDDIM *Anomalous = nullptr;
147 + static thread_local RRDDIM *Normal = nullptr;
148 +
149 + if (!PredictionRS) {
150 + std::stringstream IdSS, NameSS;
151 +
152 + IdSS << "dimensions_on_" << localhost->machine_guid;
153 + NameSS << "dimensions_on_" << localhost->hostname;
154 +
155 + PredictionRS = rrdset_create(
156 + RH,
157 + "anomaly_detection", // type
158 + IdSS.str().c_str(), // id
159 + NameSS.str().c_str(), // name
160 + "dimensions", // family
161 + "anomaly_detection.dimensions", // ctx
162 + "Anomaly detection dimensions", // title
163 + "dimensions", // units
164 + "netdata", // plugin
165 + "ml", // module
166 + ML_CHART_PRIO_DIMENSIONS, // priority
167 + RH->rrd_update_every, // update_every
168 + RRDSET_TYPE_LINE // chart_type
169 + );
170 + rrdset_flag_set(PredictionRS, RRDSET_FLAG_ANOMALY_DETECTION);
171 +
172 + Anomalous = rrddim_add(PredictionRS, "anomalous", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
173 + Normal = rrddim_add(PredictionRS, "normal", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
174 + }
175 +
176 + rrddim_set_by_pointer(PredictionRS, Anomalous, MLS.NumAnomalousDimensions);
177 + rrddim_set_by_pointer(PredictionRS, Normal, MLS.NumNormalDimensions);
178 +
179 + rrdset_done(PredictionRS);
180 + }
181
54 - rrdset_done(RS);
182 }
183
184 void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) {
@@ -75,7 +202,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
202 "percentage", // units
203 "netdata", // plugin
204 "ml", // module
78 - 39184, // priority
205 + ML_CHART_PRIO_ANOMALY_RATE, // priority
206 RH->rrd_update_every, // update_every
207 RRDSET_TYPE_LINE // chart_type
208 );
@@ -109,7 +236,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
236 "percentage", // units
237 "netdata", // plugin
238 "ml", // module
112 - 39185, // priority
239 + ML_CHART_PRIO_DETECTOR_EVENTS, // priority
240 RH->rrd_update_every, // update_every
241 RRDSET_TYPE_LINE // chart_type
242 );
@@ -143,6 +270,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
270 0, /* tier */
271 QUERY_SOURCE_ML
272 );
273 +
274 if(R) {
275 assert(R->d == 1 && R->n == 1 && R->rows == 1);
276
@@ -157,77 +285,227 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
285
286 rrdr_free(OWA, R);
287 }
288 +
289 onewayalloc_destroy(OWA);
290 }
291
163 -void ml::updateDetectionChart(RRDHOST *RH) {
164 - static thread_local RRDSET *RS = nullptr;
165 - static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
166 -
167 - if (!RS) {
168 - std::stringstream IdSS, NameSS;
169 -
170 - IdSS << "prediction_stats_" << RH->machine_guid;
171 - NameSS << "prediction_stats_for_" << RH->hostname;
172 -
173 - RS = rrdset_create_localhost(
174 - "netdata", // type
175 - IdSS.str().c_str(), // id
176 - NameSS.str().c_str(), // name
177 - "ml", // family
178 - "netdata.prediction_stats", // ctx
179 - "Prediction thread CPU usage", // title
180 - "milliseconds/s", // units
181 - "netdata", // plugin
182 - "ml", // module
183 - 136000, // priority
184 - RH->rrd_update_every, // update_every
185 - RRDSET_TYPE_STACKED // chart_type
186 - );
187 -
188 - UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
189 - SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
292 +void ml::updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU) {
293 + /*
294 + * prediction rusage
295 + */
296 + {
297 + static thread_local RRDSET *RS = nullptr;
298 +
299 + static thread_local RRDDIM *User = nullptr;
300 + static thread_local RRDDIM *System = nullptr;
301 +
302 + if (!RS) {
303 + std::stringstream IdSS, NameSS;
304 +
305 + IdSS << "prediction_usage_for_" << localhost->machine_guid;
306 + NameSS << "prediction_usage_for_" << localhost->hostname;
307 +
308 + RS = rrdset_create_localhost(
309 + "netdata", // type
310 + IdSS.str().c_str(), // id
311 + NameSS.str().c_str(), // name
312 + "ml", // family
313 + "netdata.prediction_usage", // ctx
314 + "Prediction resource usage", // title
315 + "milliseconds/s", // units
316 + "netdata", // plugin
317 + "ml", // module
318 + NETDATA_ML_CHART_PRIO_PREDICTION_USAGE, // priority
319 + RH->rrd_update_every, // update_every
320 + RRDSET_TYPE_STACKED // chart_type
321 + );
322 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
323 +
324 + User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
325 + System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
326 + }
327 +
328 + rrddim_set_by_pointer(RS, User, PredictionRU.ru_utime.tv_sec * 1000000ULL + PredictionRU.ru_utime.tv_usec);
329 + rrddim_set_by_pointer(RS, System, PredictionRU.ru_stime.tv_sec * 1000000ULL + PredictionRU.ru_stime.tv_usec);
330 +
331 + rrdset_done(RS);
332 }
333
192 - struct rusage TRU;
193 - getrusage(RUSAGE_THREAD, &TRU);
194 -
195 - rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);
196 - rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);
197 - rrdset_done(RS);
334 + /*
335 + * training rusage
336 + */
337 + {
338 + static thread_local RRDSET *RS = nullptr;
339 +
340 + static thread_local RRDDIM *User = nullptr;
341 + static thread_local RRDDIM *System = nullptr;
342 +
343 + if (!RS) {
344 + std::stringstream IdSS, NameSS;
345 +
346 + IdSS << "training_usage_for_" << localhost->machine_guid;
347 + NameSS << "training_usage_for_" << localhost->hostname;
348 +
349 + RS = rrdset_create_localhost(
350 + "netdata", // type
351 + IdSS.str().c_str(), // id
352 + NameSS.str().c_str(), // name
353 + "ml", // family
354 + "netdata.training_usage", // ctx
355 + "Training resource usage", // title
356 + "milliseconds/s", // units
357 + "netdata", // plugin
358 + "ml", // module
359 + NETDATA_ML_CHART_PRIO_TRAINING_USAGE, // priority
360 + RH->rrd_update_every, // update_every
361 + RRDSET_TYPE_STACKED // chart_type
362 + );
363 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
364 +
365 + User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
366 + System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
367 + }
368 +
369 + rrddim_set_by_pointer(RS, User, TrainingRU.ru_utime.tv_sec * 1000000ULL + TrainingRU.ru_utime.tv_usec);
370 + rrddim_set_by_pointer(RS, System, TrainingRU.ru_stime.tv_sec * 1000000ULL + TrainingRU.ru_stime.tv_usec);
371 +
372 + rrdset_done(RS);
373 + }
374 }
375
200 -void ml::updateTrainingChart(RRDHOST *RH, struct rusage *TRU) {
201 - static thread_local RRDSET *RS = nullptr;
202 - static thread_local RRDDIM *UserRD = nullptr;
203 - static thread_local RRDDIM *SystemRD = nullptr;
204 -
205 - if (!RS) {
206 - std::stringstream IdSS, NameSS;
207 -
208 - IdSS << "training_stats_" << RH->machine_guid;
209 - NameSS << "training_stats_for_" << RH->hostname;
210 -
211 - RS = rrdset_create_localhost(
212 - "netdata", // type
213 - IdSS.str().c_str(), // id
214 - NameSS.str().c_str(), // name
215 - "ml", // family
216 - "netdata.training_stats", // ctx
217 - "Training thread CPU usage", // title
218 - "milliseconds/s", // units
219 - "netdata", // plugin
220 - "ml", // module
221 - 136001, // priority
222 - RH->rrd_update_every, // update_every
223 - RRDSET_TYPE_STACKED // chart_type
224 - );
376 +void ml::updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS) {
377 + /*
378 + * queue stats
379 + */
380 + {
381 + static thread_local RRDSET *RS = nullptr;
382 +
383 + static thread_local RRDDIM *QueueSize = nullptr;
384 + static thread_local RRDDIM *PoppedItems = nullptr;
385 +
386 + if (!RS) {
387 + std::stringstream IdSS, NameSS;
388 +
389 + IdSS << "queue_stats_for_" << localhost->machine_guid;
390 + NameSS << "queue_stats_for_" << localhost->hostname;
391 +
392 + RS = rrdset_create_localhost(
393 + "netdata", // type
394 + IdSS.str().c_str(), // id
395 + NameSS.str().c_str(), // name
396 + "ml", // family
397 + "netdata.queue_stats", // ctx
398 + "Training queue stats", // title
399 + "items", // units
400 + "netdata", // plugin
401 + "ml", // module
402 + NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
403 + RH->rrd_update_every, // update_every
404 + RRDSET_TYPE_LINE// chart_type
405 + );
406 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
407 +
408 + QueueSize = rrddim_add(RS, "queue_size", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
409 + PoppedItems = rrddim_add(RS, "popped_items", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
410 + }
411 +
412 + rrddim_set_by_pointer(RS, QueueSize, TS.QueueSize);
413 + rrddim_set_by_pointer(RS, PoppedItems, TS.NumPoppedItems);
414 +
415 + rrdset_done(RS);
416 + }
417
226 - UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
227 - SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
418 + /*
419 + * training stats
420 + */
421 + {
422 + static thread_local RRDSET *RS = nullptr;
423 +
424 + static thread_local RRDDIM *Allotted = nullptr;
425 + static thread_local RRDDIM *Consumed = nullptr;
426 + static thread_local RRDDIM *Remaining = nullptr;
427 +
428 + if (!RS) {
429 + std::stringstream IdSS, NameSS;
430 +
431 + IdSS << "training_time_stats_for_" << localhost->machine_guid;
432 + NameSS << "training_time_stats_for_" << localhost->hostname;
433 +
434 + RS = rrdset_create_localhost(
435 + "netdata", // type
436 + IdSS.str().c_str(), // id
437 + NameSS.str().c_str(), // name
438 + "ml", // family
439 + "netdata.training_time_stats", // ctx
440 + "Training time stats", // title
441 + "milliseconds", // units
442 + "netdata", // plugin
443 + "ml", // module
444 + NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
445 + RH->rrd_update_every, // update_every
446 + RRDSET_TYPE_LINE// chart_type
447 + );
448 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
449 +
450 + Allotted = rrddim_add(RS, "allotted", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
451 + Consumed = rrddim_add(RS, "consumed", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
452 + Remaining = rrddim_add(RS, "remaining", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
453 + }
454 +
455 + rrddim_set_by_pointer(RS, Allotted, TS.AllottedUT);
456 + rrddim_set_by_pointer(RS, Consumed, TS.ConsumedUT);
457 + rrddim_set_by_pointer(RS, Remaining, TS.RemainingUT);
458 +
459 + rrdset_done(RS);
460 }
461
230 - rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);
231 - rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);
232 - rrdset_done(RS);
462 + /*
463 + * training result stats
464 + */
465 + {
466 + static thread_local RRDSET *RS = nullptr;
467 +
468 + static thread_local RRDDIM *Ok = nullptr;
469 + static thread_local RRDDIM *InvalidQueryTimeRange = nullptr;
470 + static thread_local RRDDIM *NotEnoughCollectedValues = nullptr;
471 + static thread_local RRDDIM *NullAcquiredDimension = nullptr;
472 + static thread_local RRDDIM *ChartUnderReplication = nullptr;
473 +
474 + if (!RS) {
475 + std::stringstream IdSS, NameSS;
476 +
477 + IdSS << "training_results_for_" << localhost->machine_guid;
478 + NameSS << "training_results_for_" << localhost->hostname;
479 +
480 + RS = rrdset_create_localhost(
481 + "netdata", // type
482 + IdSS.str().c_str(), // id
483 + NameSS.str().c_str(), // name
484 + "ml", // family
485 + "netdata.training_results", // ctx
486 + "Training results", // title
487 + "events", // units
488 + "netdata", // plugin
489 + "ml", // module
490 + NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
491 + RH->rrd_update_every, // update_every
492 + RRDSET_TYPE_LINE// chart_type
493 + );
494 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
495 +
496 + Ok = rrddim_add(RS, "ok", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
497 + InvalidQueryTimeRange = rrddim_add(RS, "invalid-queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
498 + NotEnoughCollectedValues = rrddim_add(RS, "not-enough-values", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
499 + NullAcquiredDimension = rrddim_add(RS, "null-acquired-dimensions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
500 + ChartUnderReplication = rrddim_add(RS, "chart-under-replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
501 + }
502 +
503 + rrddim_set_by_pointer(RS, Ok, TS.TrainingResultOk);
504 + rrddim_set_by_pointer(RS, InvalidQueryTimeRange, TS.TrainingResultInvalidQueryTimeRange);
505 + rrddim_set_by_pointer(RS, NotEnoughCollectedValues, TS.TrainingResultNotEnoughCollectedValues);
506 + rrddim_set_by_pointer(RS, NullAcquiredDimension, TS.TrainingResultNullAcquiredDimension);
507 + rrddim_set_by_pointer(RS, ChartUnderReplication, TS.TrainingResultChartUnderReplication);
508 +
509 + rrdset_done(RS);
510 + }
511 }
ml/ADCharts.h
+4 -6
@@ -3,20 +3,18 @@
3 #ifndef ML_ADCHARTS_H
4 #define ML_ADCHARTS_H
5
6 +#include "Stats.h"
7 #include "ml-private.h"
8
9 namespace ml {
10
10 -void updateDimensionsChart(RRDHOST *RH,
11 - collected_number NumTrainedDimensions,
12 - collected_number NumNormalDimensions,
13 - collected_number NumAnomalousDimensions);
11 +void updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS);
12
13 void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate);
14
17 -void updateDetectionChart(RRDHOST *RH);
15 +void updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU);
16
19 -void updateTrainingChart(RRDHOST *RH, struct rusage *TRU);
17 +void updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS);
18
19 } // namespace ml
20
ml/Chart.cc
ml/Chart.h new
+128
@@ -0,0 +1,128 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef ML_CHART_H
4 +#define ML_CHART_H
5 +
6 +#include "Config.h"
7 +#include "Dimension.h"
8 +
9 +#include "ml-private.h"
10 +#include "json/single_include/nlohmann/json.hpp"
11 +
12 +namespace ml
13 +{
14 +
15 +class Chart {
16 +public:
17 + Chart(RRDSET *RS) :
18 + RS(RS),
19 + MLS()
20 + { }
21 +
22 + RRDSET *getRS() const {
23 + return RS;
24 + }
25 +
26 + bool isAvailableForML() {
27 + return rrdset_is_available_for_exporting_and_alarms(RS);
28 + }
29 +
30 + void addDimension(Dimension *D) {
31 + std::lock_guard<std::mutex> Lock(Mutex);
32 + Dimensions[D->getRD()] = D;
33 + }
34 +
35 + void removeDimension(Dimension *D) {
36 + std::lock_guard<std::mutex> Lock(Mutex);
37 + Dimensions.erase(D->getRD());
38 + }
39 +
40 + void getModelsAsJson(nlohmann::json &Json) {
41 + std::lock_guard<std::mutex> Lock(Mutex);
42 +
43 + for (auto &DP : Dimensions) {
44 + Dimension *D = DP.second;
45 + nlohmann::json JsonArray = nlohmann::json::array();
46 + for (const KMeans &KM : D->getModels()) {
47 + nlohmann::json J;
48 + KM.toJson(J);
49 + JsonArray.push_back(J);
50 + }
51 +
52 + Json[getMLDimensionID(D->getRD())] = JsonArray;
53 + }
54 + }
55 +
56 + void updateBegin() {
57 + Mutex.lock();
58 + MLS = {};
59 + }
60 +
61 + void updateDimension(Dimension *D, bool IsAnomalous) {
62 + switch (D->getMLS()) {
63 + case MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
64 + MLS.NumMachineLearningStatusDisabledUE++;
65 + return;
66 + case MachineLearningStatus::DisabledDueToExcludedChart:
67 + MLS.NumMachineLearningStatusDisabledSP++;
68 + return;
69 + case MachineLearningStatus::Enabled: {
70 + MLS.NumMachineLearningStatusEnabled++;
71 +
72 + switch (D->getMT()) {
73 + case MetricType::Constant:
74 + MLS.NumMetricTypeConstant++;
75 + MLS.NumTrainingStatusTrained++;
76 + MLS.NumNormalDimensions++;
77 + return;
78 + case MetricType::Variable:
79 + MLS.NumMetricTypeVariable++;
80 + break;
81 + }
82 +
83 + switch (D->getTS()) {
84 + case TrainingStatus::Untrained:
85 + MLS.NumTrainingStatusUntrained++;
86 + return;
87 + case TrainingStatus::PendingWithoutModel:
88 + MLS.NumTrainingStatusPendingWithoutModel++;
89 + return;
90 + case TrainingStatus::Trained:
91 + MLS.NumTrainingStatusTrained++;
92 +
93 + MLS.NumAnomalousDimensions += IsAnomalous;
94 + MLS.NumNormalDimensions += !IsAnomalous;
95 + return;
96 + case TrainingStatus::PendingWithModel:
97 + MLS.NumTrainingStatusPendingWithModel++;
98 +
99 + MLS.NumAnomalousDimensions += IsAnomalous;
100 + MLS.NumNormalDimensions += !IsAnomalous;
101 + return;
102 + }
103 +
104 + return;
105 + }
106 + }
107 + }
108 +
109 + void updateEnd() {
110 + Mutex.unlock();
111 + }
112 +
113 + MachineLearningStats getMLS() {
114 + std::lock_guard<std::mutex> Lock(Mutex);
115 + return MLS;
116 + }
117 +
118 +private:
119 + RRDSET *RS;
120 + MachineLearningStats MLS;
121 +
122 + std::mutex Mutex;
123 + std::unordered_map<RRDDIM *, Dimension *> Dimensions;
124 +};
125 +
126 +} // namespace ml
127 +
128 +#endif /* ML_CHART_H */
ml/Config.cc
+3 -3
@@ -31,7 +31,7 @@ void Config::readMLConfig(void) {
31 unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
32 unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
33 unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
34 - unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24);
34 + unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1);
35
36 unsigned DiffN = config_get_number(ConfigSectionML, "num samples to diff", 1);
37 unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);
@@ -53,7 +53,7 @@ void Config::readMLConfig(void) {
53 MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
54 MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
55 TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
56 - NumModelsToUse = clamp<unsigned>(TrainEvery, 1, 7 * 24);
56 + NumModelsToUse = clamp<unsigned>(NumModelsToUse, 1, 7 * 24);
57
58 DiffN = clamp(DiffN, 0u, 1u);
59 SmoothN = clamp(SmoothN, 0u, 5u);
@@ -108,7 +108,7 @@ void Config::readMLConfig(void) {
108 // Always exclude anomaly_detection charts from training.
109 Cfg.ChartsToSkip = "anomaly_detection.* ";
110 Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");
111 - Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
111 + Cfg.SP_ChartsToSkip = simple_pattern_create(Cfg.ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
112
113 Cfg.StreamADCharts = config_get_boolean(ConfigSectionML, "stream anomaly detection charts", true);
114 }
ml/Config.h
+1
@@ -14,6 +14,7 @@ public:
14 unsigned MaxTrainSamples;
15 unsigned MinTrainSamples;
16 unsigned TrainEvery;
17 +
18 unsigned NumModelsToUse;
19
20 unsigned DBEngineAnomalyRateEvery;
ml/Dimension.cc
+231 -59
@@ -3,84 +3,174 @@
3 #include "Config.h"
4 #include "Dimension.h"
5 #include "Query.h"
6 +#include "Host.h"
7
8 using namespace ml;
9
9 -bool Dimension::isActive() const {
10 - bool SetObsolete = rrdset_flag_check(RD->rrdset, RRDSET_FLAG_OBSOLETE);
11 - bool DimObsolete = rrddim_flag_check(RD, RRDDIM_FLAG_OBSOLETE);
12 - return !SetObsolete && !DimObsolete;
10 +static const char *mls2str(MachineLearningStatus MLS) {
11 + switch (MLS) {
12 + case ml::MachineLearningStatus::Enabled:
13 + return "enabled";
14 + case ml::MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
15 + return "disabled-ue";
16 + case ml::MachineLearningStatus::DisabledDueToExcludedChart:
17 + return "disabled-sp";
18 + default:
19 + return "unknown";
20 + }
21 +}
22 +
23 +static const char *mt2str(MetricType MT) {
24 + switch (MT) {
25 + case ml::MetricType::Constant:
26 + return "constant";
27 + case ml::MetricType::Variable:
28 + return "variable";
29 + default:
30 + return "unknown";
31 + }
32 }
33
15 -std::pair<CalculatedNumber *, size_t> Dimension::getCalculatedNumbers() {
34 +static const char *ts2str(TrainingStatus TS) {
35 + switch (TS) {
36 + case ml::TrainingStatus::PendingWithModel:
37 + return "pending-with-model";
38 + case ml::TrainingStatus::PendingWithoutModel:
39 + return "pending-without-model";
40 + case ml::TrainingStatus::Trained:
41 + return "trained";
42 + case ml::TrainingStatus::Untrained:
43 + return "untrained";
44 + default:
45 + return "unknown";
46 + }
47 +}
48 +
49 +static const char *tr2str(TrainingResult TR) {
50 + switch (TR) {
51 + case ml::TrainingResult::Ok:
52 + return "ok";
53 + case ml::TrainingResult::InvalidQueryTimeRange:
54 + return "invalid-query";
55 + case ml::TrainingResult::NotEnoughCollectedValues:
56 + return "missing-values";
57 + case ml::TrainingResult::NullAcquiredDimension:
58 + return "null-acquired-dim";
59 + case ml::TrainingResult::ChartUnderReplication:
60 + return "chart-under-replication";
61 + default:
62 + return "unknown";
63 + }
64 +}
65 +
66 +std::pair<CalculatedNumber *, TrainingResponse> Dimension::getCalculatedNumbers(const TrainingRequest &TrainingReq) {
67 + TrainingResponse TrainingResp = {};
68 +
69 + TrainingResp.RequestTime = TrainingReq.RequestTime;
70 + TrainingResp.FirstEntryOnRequest = TrainingReq.FirstEntryOnRequest;
71 + TrainingResp.LastEntryOnRequest = TrainingReq.LastEntryOnRequest;
72 +
73 + TrainingResp.FirstEntryOnResponse = rrddim_first_entry_t_of_tier(RD, 0);
74 + TrainingResp.LastEntryOnResponse = rrddim_last_entry_t_of_tier(RD, 0);
75 +
76 size_t MinN = Cfg.MinTrainSamples;
77 size_t MaxN = Cfg.MaxTrainSamples;
78
79 // Figure out what our time window should be.
20 - time_t BeforeT = now_realtime_sec() - 1;
21 - time_t AfterT = BeforeT - (MaxN * updateEvery());
22 -
23 - BeforeT -= (BeforeT % updateEvery());
24 - AfterT -= (AfterT % updateEvery());
25 -
26 - BeforeT = std::min(BeforeT, latestTime());
27 - AfterT = std::max(AfterT, oldestTime());
80 + TrainingResp.QueryBeforeT = TrainingResp.LastEntryOnResponse;
81 + TrainingResp.QueryAfterT = std::max(
82 + TrainingResp.QueryBeforeT - static_cast<time_t>((MaxN - 1) * updateEvery()),
83 + TrainingResp.FirstEntryOnResponse
84 + );
85 +
86 + if (TrainingResp.QueryAfterT >= TrainingResp.QueryBeforeT) {
87 + TrainingResp.Result = TrainingResult::InvalidQueryTimeRange;
88 + return { nullptr, TrainingResp };
89 + }
90
29 - if (AfterT >= BeforeT)
30 - return { nullptr, 0 };
91 + if (rrdset_is_replicating(RD->rrdset)) {
92 + TrainingResp.Result = TrainingResult::ChartUnderReplication;
93 + return { nullptr, TrainingResp };
94 + }
95
96 CalculatedNumber *CNs = new CalculatedNumber[MaxN * (Cfg.LagN + 1)]();
97
98 // Start the query.
35 - unsigned Idx = 0;
36 - unsigned CollectedValues = 0;
37 - unsigned TotalValues = 0;
99 + size_t Idx = 0;
100
101 CalculatedNumber LastValue = std::numeric_limits<CalculatedNumber>::quiet_NaN();
102 Query Q = Query(getRD());
103
42 - Q.init(AfterT, BeforeT);
104 + Q.init(TrainingResp.QueryAfterT, TrainingResp.QueryBeforeT);
105 while (!Q.isFinished()) {
106 if (Idx == MaxN)
107 break;
108
109 auto P = Q.nextMetric();
110 +
111 CalculatedNumber Value = P.second;
112
113 if (netdata_double_isnumber(Value)) {
114 + if (!TrainingResp.DbAfterT)
115 + TrainingResp.DbAfterT = P.first;
116 + TrainingResp.DbBeforeT = P.first;
117 +
118 CNs[Idx] = Value;
119 LastValue = CNs[Idx];
53 - CollectedValues++;
120 + TrainingResp.CollectedValues++;
121 } else
122 CNs[Idx] = LastValue;
123
124 Idx++;
125 }
59 - TotalValues = Idx;
126 + TrainingResp.TotalValues = Idx;
127 +
128 + if (TrainingResp.CollectedValues < MinN) {
129 + TrainingResp.Result = TrainingResult::NotEnoughCollectedValues;
130
61 - if (CollectedValues < MinN) {
131 delete[] CNs;
63 - return { nullptr, 0 };
132 + return { nullptr, TrainingResp };
133 }
134
135 // Find first non-NaN value.
67 - for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TotalValues--) { }
136 + for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TrainingResp.TotalValues--) { }
137
138 // Overwrite NaN values.
139 if (Idx != 0)
71 - memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TotalValues);
140 + memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TrainingResp.TotalValues);
141
73 - return { CNs, TotalValues };
142 + TrainingResp.Result = TrainingResult::Ok;
143 + return { CNs, TrainingResp };
144 }
145
76 -MLResult Dimension::trainModel() {
77 - auto P = getCalculatedNumbers();
146 +TrainingResult Dimension::trainModel(const TrainingRequest &TrainingReq) {
147 + auto P = getCalculatedNumbers(TrainingReq);
148 CalculatedNumber *CNs = P.first;
79 - unsigned N = P.second;
149 + TrainingResponse TrainingResp = P.second;
150 +
151 + if (TrainingResp.Result != TrainingResult::Ok) {
152 + std::lock_guard<std::mutex> Lock(Mutex);
153
81 - if (!CNs)
82 - return MLResult::MissingData;
154 + MT = MetricType::Constant;
155 +
156 + switch (TS) {
157 + case TrainingStatus::PendingWithModel:
158 + TS = TrainingStatus::Trained;
159 + break;
160 + case TrainingStatus::PendingWithoutModel:
161 + TS = TrainingStatus::Untrained;
162 + break;
163 + default:
164 + break;
165 + }
166
167 + TR = TrainingResp;
168 +
169 + LastTrainingTime = TrainingResp.LastEntryOnResponse;
170 + return TrainingResp.Result;
171 + }
172 +
173 + unsigned N = TrainingResp.TotalValues;
174 unsigned TargetNumSamples = Cfg.MaxTrainSamples * Cfg.RandomSamplingRatio;
175 double SamplingRatio = std::min(static_cast<double>(TargetNumSamples) / N, 1.0);
176
@@ -93,49 +183,81 @@ MLResult Dimension::trainModel() {
183
184 {
185 std::lock_guard<std::mutex> Lock(Mutex);
96 - Models[0] = KM;
97 - }
186
99 - Trained = true;
100 - ConstantModel = true;
187 + if (Models.size() < Cfg.NumModelsToUse) {
188 + Models.push_back(std::move(KM));
189 + } else {
190 + std::rotate(std::begin(Models), std::begin(Models) + 1, std::end(Models));
191 + Models[Models.size() - 1] = std::move(KM);
192 + }
193 +
194 + MT = MetricType::Constant;
195 + TS = TrainingStatus::Trained;
196 + TR = TrainingResp;
197 + LastTrainingTime = rrddim_last_entry_t(RD);
198 + }
199
200 delete[] CNs;
103 - return MLResult::Success;
201 + return TrainingResp.Result;
202 }
203
106 -bool Dimension::shouldTrain(const TimePoint &TP) const {
107 - if (ConstantModel)
108 - return false;
204 +void Dimension::scheduleForTraining(time_t CurrT) {
205 + switch (MT) {
206 + case MetricType::Constant: {
207 + return;
208 + } default:
209 + break;
210 + }
211
110 - return (LastTrainedAt + Seconds(Cfg.TrainEvery * updateEvery())) < TP;
212 + switch (TS) {
213 + case TrainingStatus::PendingWithModel:
214 + case TrainingStatus::PendingWithoutModel:
215 + break;
216 + case TrainingStatus::Untrained: {
217 + Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
218 + TS = TrainingStatus::PendingWithoutModel;
219 + H->scheduleForTraining(getTrainingRequest(CurrT));
220 + break;
221 + }
222 + case TrainingStatus::Trained: {
223 + bool NeedsTraining = LastTrainingTime + (Cfg.TrainEvery * updateEvery()) < CurrT;
224 +
225 + if (NeedsTraining) {
226 + Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
227 + TS = TrainingStatus::PendingWithModel;
228 + H->scheduleForTraining(getTrainingRequest(CurrT));
229 + }
230 + break;
231 + }
232 + }
233 }
234
113 -bool Dimension::predict(CalculatedNumber Value, bool Exists) {
235 +bool Dimension::predict(time_t CurrT, CalculatedNumber Value, bool Exists) {
236 + // Nothing to do if ML is disabled for this dimension
237 + if (MLS != MachineLearningStatus::Enabled)
238 + return false;
239 +
240 + // Don't treat values that don't exist as anomalous
241 if (!Exists) {
242 CNs.clear();
116 - AnomalyBit = false;
243 return false;
244 }
245
246 + // Save the value and return if we don't have enough values for a sample
247 unsigned N = Cfg.DiffN + Cfg.SmoothN + Cfg.LagN;
248 if (CNs.size() < N) {
249 CNs.push_back(Value);
123 - AnomalyBit = false;
250 return false;
251 }
252
253 + // Push the value and check if it's different from the last one
254 + bool SameValue = true;
255 std::rotate(std::begin(CNs), std::begin(CNs) + 1, std::end(CNs));
128 -
256 if (CNs[N - 1] != Value)
130 - ConstantModel = false;
131 -
257 + SameValue = false;
258 CNs[N - 1] = Value;
259
134 - if (!isTrained() || ConstantModel) {
135 - AnomalyBit = false;
136 - return false;
137 - }
138 -
260 + // Create the sample
261 CalculatedNumber *TmpCNs = new CalculatedNumber[N * (Cfg.LagN + 1)]();
262 std::memcpy(TmpCNs, CNs.data(), N * sizeof(CalculatedNumber));
263 SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1,
@@ -144,30 +266,80 @@ bool Dimension::predict(CalculatedNumber Value, bool Exists) {
266 const DSample Sample = SB.preprocess().back();
267 delete[] TmpCNs;
268
269 + /*
270 + * Lock to predict and possibly schedule the dimension for training
271 + */
272 +
273 std::unique_lock<std::mutex> Lock(Mutex, std::defer_lock);
274 if (!Lock.try_lock()) {
149 - AnomalyBit = false;
275 return false;
276 }
277
278 + // Mark the metric time as variable if we received different values
279 + if (!SameValue)
280 + MT = MetricType::Variable;
281 +
282 + // Decide if the dimension needs to be scheduled for training
283 + scheduleForTraining(CurrT);
284 +
285 + // Nothing to do if we don't have a model
286 + switch (TS) {
287 + case TrainingStatus::Untrained:
288 + case TrainingStatus::PendingWithoutModel:
289 + return false;
290 + default:
291 + break;
292 + }
293 +
294 + /*
295 + * Use the KMeans models to check if the value is anomalous
296 + */
297 +
298 + size_t ModelsConsulted = 0;
299 + size_t Sum = 0;
300 +
301 for (const auto &KM : Models) {
302 + ModelsConsulted++;
303 +
304 double AnomalyScore = KM.anomalyScore(Sample);
155 - if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN()) {
156 - AnomalyBit = false;
305 + if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN())
306 continue;
158 - }
307
308 if (AnomalyScore < (100 * Cfg.DimensionAnomalyScoreThreshold)) {
161 - AnomalyBit = false;
309 + global_statistics_ml_models_consulted(ModelsConsulted);
310 return false;
311 }
312 +
313 + Sum += 1;
314 }
315
166 - AnomalyBit = true;
167 - return true;
316 + global_statistics_ml_models_consulted(ModelsConsulted);
317 + return Sum;
318 }
319
170 -std::array<KMeans, 1> Dimension::getModels() {
320 +std::vector<KMeans> Dimension::getModels() {
321 std::unique_lock<std::mutex> Lock(Mutex);
322 return Models;
323 }
324 +
325 +void Dimension::dump() const {
326 + const char *ChartId = rrdset_id(RD->rrdset);
327 + const char *DimensionId = rrddim_id(RD);
328 +
329 + const char *MLS_Str = mls2str(MLS);
330 + const char *MT_Str = mt2str(MT);
331 + const char *TS_Str = ts2str(TS);
332 + const char *TR_Str = tr2str(TR.Result);
333 +
334 + const char *fmt =
335 + "[ML] %s.%s: MLS=%s, MT=%s, TS=%s, Result=%s, "
336 + "ReqTime=%ld, FEOReq=%ld, LEOReq=%ld, "
337 + "FEOResp=%ld, LEOResp=%ld, QTR=<%ld, %ld>, DBTR=<%ld, %ld>, Collected=%zu, Total=%zu";
338 +
339 + error(fmt,
340 + ChartId, DimensionId, MLS_Str, MT_Str, TS_Str, TR_Str,
341 + TR.RequestTime, TR.FirstEntryOnRequest, TR.LastEntryOnRequest,
342 + TR.FirstEntryOnResponse, TR.LastEntryOnResponse,
343 + TR.QueryAfterT, TR.QueryBeforeT, TR.DbAfterT, TR.DbBeforeT, TR.CollectedValues, TR.TotalValues
344 + );
345 +}
ml/Dimension.h
+138 -36
@@ -3,6 +3,7 @@
3 #ifndef ML_DIMENSION_H
4 #define ML_DIMENSION_H
5
6 +#include "Stats.h"
7 #include "Query.h"
8 #include "Config.h"
9
@@ -10,12 +11,6 @@
11
12 namespace ml {
13
13 -enum class MLResult {
14 - Success = 0,
15 - MissingData,
16 - NaN,
17 -};
18 -
14 static inline std::string getMLDimensionID(RRDDIM *RD) {
15 RRDSET *RS = RD->rrdset;
16
@@ -24,16 +19,118 @@ static inline std::string getMLDimensionID(RRDDIM *RD) {
19 return SS.str();
20 }
21
22 +enum class MachineLearningStatus {
23 + // Enable training/prediction
24 + Enabled,
25 +
26 + // Disable due to update every being different from the host's
27 + DisabledDueToUniqueUpdateEvery,
28 +
29 + // Disable because configuration pattern matches the chart's id
30 + DisabledDueToExcludedChart,
31 +};
32 +
33 +enum class TrainingStatus {
34 + // We don't have a model for this dimension
35 + Untrained,
36 +
37 + // Request for training sent, but we don't have any models yet
38 + PendingWithoutModel,
39 +
40 + // Request to update existing models sent
41 + PendingWithModel,
42 +
43 + // Have a valid, up-to-date model
44 + Trained,
45 +};
46 +
47 +enum class MetricType {
48 + // The dimension has constant values, no need to train
49 + Constant,
50 +
51 + // The dimension's values fluctuate, we need to generate a model
52 + Variable,
53 +};
54 +
55 +struct TrainingRequest {
56 + // Chart/dimension we want to train
57 + STRING *ChartId;
58 + STRING *DimensionId;
59 +
60 + // Creation time of request
61 + time_t RequestTime;
62 +
63 + // First/last entry of this dimension in DB
64 + // at the point the request was made
65 + time_t FirstEntryOnRequest;
66 + time_t LastEntryOnRequest;
67 +};
68 +
69 +void dumpTrainingRequest(const TrainingRequest &TrainingReq, const char *Prefix);
70 +
71 +enum TrainingResult {
72 + // We managed to create a KMeans model
73 + Ok,
74 + // Could not query DB with a correct time range
75 + InvalidQueryTimeRange,
76 + // Did not gather enough data from DB to run KMeans
77 + NotEnoughCollectedValues,
78 + // Acquired a null dimension
79 + NullAcquiredDimension,
80 + // Chart is under replication
81 + ChartUnderReplication,
82 +};
83 +
84 +struct TrainingResponse {
85 + // Time when the request for this response was made
86 + time_t RequestTime;
87 +
88 + // First/last entry of the dimension in DB when generating the request
89 + time_t FirstEntryOnRequest;
90 + time_t LastEntryOnRequest;
91 +
92 + // First/last entry of the dimension in DB when generating the response
93 + time_t FirstEntryOnResponse;
94 + time_t LastEntryOnResponse;
95 +
96 + // After/Before timestamps of our DB query
97 + time_t QueryAfterT;
98 + time_t QueryBeforeT;
99 +
100 + // Actual after/before returned by the DB query ops
101 + time_t DbAfterT;
102 + time_t DbBeforeT;
103 +
104 + // Number of doubles returned by the DB query
105 + size_t CollectedValues;
106 +
107 + // Number of values we return to the caller
108 + size_t TotalValues;
109 +
110 + // Result of training response
111 + TrainingResult Result;
112 +};
113 +
114 +void dumpTrainingResponse(const TrainingResponse &TrainingResp, const char *Prefix);
115 +
116 class Dimension {
117 public:
118 Dimension(RRDDIM *RD) :
119 RD(RD),
31 - LastTrainedAt(Seconds(0)),
32 - Trained(false),
33 - ConstantModel(false),
34 - AnomalyScore(0.0),
35 - AnomalyBit(0)
36 - { }
120 + MT(MetricType::Constant),
121 + TS(TrainingStatus::Untrained),
122 + TR(),
123 + LastTrainingTime(0)
124 + {
125 + if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RD->rrdset)))
126 + MLS = MachineLearningStatus::DisabledDueToExcludedChart;
127 + else if (RD->update_every != RD->rrdset->rrdhost->rrd_update_every)
128 + MLS = MachineLearningStatus::DisabledDueToUniqueUpdateEvery;
129 + else
130 + MLS = MachineLearningStatus::Enabled;
131 +
132 + Models.reserve(Cfg.NumModelsToUse);
133 + }
134
135 RRDDIM *getRD() const {
136 return RD;
@@ -43,49 +140,54 @@ public:
140 return RD->update_every;
141 }
142
46 - time_t latestTime() const {
47 - return Query(RD).latestTime();
48 - }
49 -
50 - time_t oldestTime() const {
51 - return Query(RD).oldestTime();
143 + MetricType getMT() const {
144 + return MT;
145 }
146
54 - bool isTrained() const {
55 - return Trained;
147 + TrainingStatus getTS() const {
148 + return TS;
149 }
150
58 - bool isAnomalous() const {
59 - return AnomalyBit;
151 + MachineLearningStatus getMLS() const {
152 + return MLS;
153 }
154
62 - bool shouldTrain(const TimePoint &TP) const;
155 + TrainingResult trainModel(const TrainingRequest &TR);
156
64 - bool isActive() const;
157 + void scheduleForTraining(time_t CurrT);
158
66 - MLResult trainModel();
159 + bool predict(time_t CurrT, CalculatedNumber Value, bool Exists);
160
68 - bool predict(CalculatedNumber Value, bool Exists);
161 + std::vector<KMeans> getModels();
162 +
163 + void dump() const;
164
70 - std::pair<bool, double> detect(size_t WindowLength, bool Reset);
71 -
72 - std::array<KMeans, 1> getModels();
165 +private:
166 + TrainingRequest getTrainingRequest(time_t CurrT) const {
167 + return TrainingRequest {
168 + string_dup(RD->rrdset->id),
169 + string_dup(RD->id),
170 + CurrT,
171 + rrddim_first_entry_t(RD),
172 + rrddim_last_entry_t(RD)
173 + };
174 + }
175
176 private:
75 - std::pair<CalculatedNumber *, size_t> getCalculatedNumbers();
177 + std::pair<CalculatedNumber *, TrainingResponse> getCalculatedNumbers(const TrainingRequest &TrainingReq);
178
179 public:
180 RRDDIM *RD;
181 + MetricType MT;
182 + TrainingStatus TS;
183 + TrainingResponse TR;
184
80 - TimePoint LastTrainedAt;
81 - std::atomic<bool> Trained;
82 - std::atomic<bool> ConstantModel;
185 + time_t LastTrainingTime;
186
84 - CalculatedNumber AnomalyScore;
85 - std::atomic<bool> AnomalyBit;
187 + MachineLearningStatus MLS;
188
189 std::vector<CalculatedNumber> CNs;
88 - std::array<KMeans, 1> Models;
190 + std::vector<KMeans> Models;
191 std::mutex Mutex;
192 };
193
ml/Host.cc
+194 -158
@@ -2,42 +2,24 @@
2
3 #include "Config.h"
4 #include "Host.h"
5 +#include "Queue.h"
6 #include "ADCharts.h"
7
8 #include "json/single_include/nlohmann/json.hpp"
9
10 using namespace ml;
11
11 -void RrdHost::addDimension(Dimension *D) {
12 +void Host::addChart(Chart *C) {
13 std::lock_guard<std::mutex> Lock(Mutex);
13 -
14 - DimensionsMap[D->getRD()] = D;
15 -
16 - // Default construct mutex for dimension
17 - LocksMap[D];
14 + Charts[C->getRS()] = C;
15 }
16
20 -void RrdHost::removeDimension(Dimension *D) {
21 - // Remove the dimension from the hosts map.
22 - {
23 - std::lock_guard<std::mutex> Lock(Mutex);
24 - DimensionsMap.erase(D->getRD());
25 - }
26 -
27 - // Delete the dimension by locking the mutex that protects it.
28 - {
29 - std::lock_guard<std::mutex> Lock(LocksMap[D]);
30 - delete D;
31 - }
32 -
33 - // Remove the lock entry for the deleted dimension.
34 - {
35 - std::lock_guard<std::mutex> Lock(Mutex);
36 - LocksMap.erase(D);
37 - }
17 +void Host::removeChart(Chart *C) {
18 + std::lock_guard<std::mutex> Lock(Mutex);
19 + Charts.erase(C->getRS());
20 }
21
40 -void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
22 +void Host::getConfigAsJson(nlohmann::json &Json) const {
23 Json["version"] = 1;
24
25 Json["enabled"] = Cfg.EnableAnomalyDetection;
@@ -63,193 +45,247 @@ void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
45 Json["charts-to-skip"] = Cfg.ChartsToSkip;
46 }
47
66 -void TrainableHost::getModelsAsJson(nlohmann::json &Json) {
67 - std::lock_guard<std::mutex> Lock(Mutex);
68 -
69 - for (auto &DP : DimensionsMap) {
70 - Dimension *D = DP.second;
71 -
72 - nlohmann::json JsonArray = nlohmann::json::array();
73 - for (const KMeans &KM : D->getModels()) {
74 - nlohmann::json J;
75 - KM.toJson(J);
76 - JsonArray.push_back(J);
77 - }
78 - Json[getMLDimensionID(D->getRD())] = JsonArray;
79 - }
80 -
81 - return;
82 -}
83 -
84 -std::pair<Dimension *, Duration<double>>
85 -TrainableHost::findDimensionToTrain(const TimePoint &NowTP) {
48 +void Host::getModelsAsJson(nlohmann::json &Json) {
49 std::lock_guard<std::mutex> Lock(Mutex);
50
88 - Duration<double> AllottedDuration = Duration<double>{Cfg.TrainEvery * updateEvery()} / (DimensionsMap.size() + 1);
89 -
90 - for (auto &DP : DimensionsMap) {
91 - Dimension *D = DP.second;
92 -
93 - if (D->shouldTrain(NowTP)) {
94 - LocksMap[D].lock();
95 - return { D, AllottedDuration };
96 - }
51 + for (auto &CP : Charts) {
52 + Chart *C = CP.second;
53 + C->getModelsAsJson(Json);
54 }
98 -
99 - return { nullptr, AllottedDuration };
55 }
56
102 -void TrainableHost::trainDimension(Dimension *D, const TimePoint &NowTP) {
103 - if (D == nullptr)
104 - return;
105 -
106 - D->LastTrainedAt = NowTP + Seconds{D->updateEvery()};
107 - D->trainModel();
57 +void Host::detectOnce() {
58 + MLS = {};
59 + MachineLearningStats MLSCopy = {};
60 + TrainingStats TSCopy = {};
61
62 {
63 std::lock_guard<std::mutex> Lock(Mutex);
111 - LocksMap[D].unlock();
112 - }
113 -}
114 -
115 -void TrainableHost::train() {
116 - Duration<double> MaxSleepFor = Seconds{10 * updateEvery()};
64
118 - worker_register("MLTRAIN");
119 - worker_register_job_name(0, "dimensions");
65 + /*
66 + * prediction/detection stats
67 + */
68 + for (auto &CP : Charts) {
69 + Chart *C = CP.second;
70
121 - worker_is_busy(0);
122 - while (!netdata_exit) {
123 - netdata_thread_testcancel();
124 - netdata_thread_disable_cancelability();
71 + if (!C->isAvailableForML())
72 + continue;
73
126 - updateResourceUsage();
74 + MachineLearningStats ChartMLS = C->getMLS();
75
128 - TimePoint NowTP = SteadyClock::now();
76 + MLS.NumMachineLearningStatusEnabled += ChartMLS.NumMachineLearningStatusEnabled;
77 + MLS.NumMachineLearningStatusDisabledUE += ChartMLS.NumMachineLearningStatusDisabledUE;
78 + MLS.NumMachineLearningStatusDisabledSP += ChartMLS.NumMachineLearningStatusDisabledSP;
79
130 - auto P = findDimensionToTrain(NowTP);
131 - trainDimension(P.first, NowTP);
80 + MLS.NumMetricTypeConstant += ChartMLS.NumMetricTypeConstant;
81 + MLS.NumMetricTypeVariable += ChartMLS.NumMetricTypeVariable;
82
133 - netdata_thread_enable_cancelability();
83 + MLS.NumTrainingStatusUntrained += ChartMLS.NumTrainingStatusUntrained;
84 + MLS.NumTrainingStatusPendingWithoutModel += ChartMLS.NumTrainingStatusPendingWithoutModel;
85 + MLS.NumTrainingStatusTrained += ChartMLS.NumTrainingStatusTrained;
86 + MLS.NumTrainingStatusPendingWithModel += ChartMLS.NumTrainingStatusPendingWithModel;
87
135 - Duration<double> AllottedDuration = P.second;
136 - Duration<double> RealDuration = SteadyClock::now() - NowTP;
88 + MLS.NumAnomalousDimensions += ChartMLS.NumAnomalousDimensions;
89 + MLS.NumNormalDimensions += ChartMLS.NumNormalDimensions;
90 + }
91
138 - Duration<double> SleepFor;
139 - if (RealDuration >= AllottedDuration)
140 - continue;
92 + HostAnomalyRate = 0.0;
93 + size_t NumActiveDimensions = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
94 + if (NumActiveDimensions)
95 + HostAnomalyRate = static_cast<double>(MLS.NumAnomalousDimensions) / NumActiveDimensions;
96
142 - worker_is_idle();
143 - SleepFor = std::min(AllottedDuration - RealDuration, MaxSleepFor);
144 - TimePoint Now = SteadyClock::now();
145 - auto Until = Now + SleepFor;
146 - while (Now < Until && !netdata_exit) {
147 - std::this_thread::sleep_for(std::chrono::milliseconds(1000));
148 - Now = SteadyClock::now();
149 - }
150 - worker_is_busy(0);
151 - }
152 -}
97 + MLSCopy = MLS;
98
154 -#define WORKER_JOB_DETECT_DIMENSION 0
155 -#define WORKER_JOB_UPDATE_DETECTION_CHART 1
156 -#define WORKER_JOB_UPDATE_ANOMALY_RATES 2
157 -#define WORKER_JOB_UPDATE_CHARTS 3
99 + /*
100 + * training stats
101 + */
102 + TSCopy = TS;
103
159 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
160 -#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
161 -#endif
104 + TS.QueueSize = 0;
105 + TS.NumPoppedItems = 0;
106
163 -void DetectableHost::detectOnce() {
164 - size_t NumAnomalousDimensions = 0;
165 - size_t NumNormalDimensions = 0;
166 - size_t NumTrainedDimensions = 0;
167 - size_t NumActiveDimensions = 0;
107 + TS.AllottedUT = 0;
108 + TS.ConsumedUT = 0;
109 + TS.RemainingUT = 0;
110
169 - {
170 - std::lock_guard<std::mutex> Lock(Mutex);
111 + TS.TrainingResultOk = 0;
112 + TS.TrainingResultInvalidQueryTimeRange = 0;
113 + TS.TrainingResultNotEnoughCollectedValues = 0;
114 + TS.TrainingResultNullAcquiredDimension = 0;
115 + TS.TrainingResultChartUnderReplication = 0;
116 + }
117
172 - for (auto &DP : DimensionsMap) {
173 - worker_is_busy(WORKER_JOB_DETECT_DIMENSION);
118 + // Calc the avg values
119 + if (TSCopy.NumPoppedItems) {
120 + TSCopy.QueueSize /= TSCopy.NumPoppedItems;
121 + TSCopy.AllottedUT /= TSCopy.NumPoppedItems;
122 + TSCopy.ConsumedUT /= TSCopy.NumPoppedItems;
123 + TSCopy.RemainingUT /= TSCopy.NumPoppedItems;
124 +
125 + TSCopy.TrainingResultOk /= TSCopy.NumPoppedItems;
126 + TSCopy.TrainingResultInvalidQueryTimeRange /= TSCopy.NumPoppedItems;
127 + TSCopy.TrainingResultNotEnoughCollectedValues /= TSCopy.NumPoppedItems;
128 + TSCopy.TrainingResultNullAcquiredDimension /= TSCopy.NumPoppedItems;
129 + TSCopy.TrainingResultChartUnderReplication /= TSCopy.NumPoppedItems;
130 + } else {
131 + TSCopy.QueueSize = 0;
132 + TSCopy.AllottedUT = 0;
133 + TSCopy.ConsumedUT = 0;
134 + TSCopy.RemainingUT = 0;
135 + }
136
175 - Dimension *D = DP.second;
137 + updateDimensionsChart(RH, MLSCopy);
138 + updateHostAndDetectionRateCharts(RH, HostAnomalyRate * 10000.0);
139
177 - if (!D->isActive())
178 - continue;
140 + struct rusage PredictionRU;
141 + getrusage(RUSAGE_THREAD, &PredictionRU);
142 + updateResourceUsageCharts(RH, PredictionRU, TSCopy.TrainingRU);
143
180 - NumActiveDimensions++;
181 - NumTrainedDimensions += D->isTrained();
144 + updateTrainingStatisticsChart(RH, TSCopy);
145 +}
146
183 - bool IsAnomalous = D->isAnomalous();
184 - if (IsAnomalous)
185 - NumAnomalousDimensions += 1;
147 +class AcquiredDimension {
148 +public:
149 + static AcquiredDimension find(RRDHOST *RH, STRING *ChartId, STRING *DimensionId) {
150 + RRDDIM_ACQUIRED *AcqRD = nullptr;
151 + Dimension *D = nullptr;
152 +
153 + rrdhost_rdlock(RH);
154 + RRDSET *RS = rrdset_find(RH, string2str(ChartId));
155 + if (RS) {
156 + AcqRD = rrddim_find_and_acquire(RS, string2str(DimensionId));
157 + if (AcqRD) {
158 + RRDDIM *RD = rrddim_acquired_to_rrddim(AcqRD);
159 + if (RD)
160 + D = reinterpret_cast<Dimension *>(RD->ml_dimension);
161 + }
162 }
163 + rrdhost_unlock(RH);
164 +
165 + return AcquiredDimension(AcqRD, D);
166 + }
167
188 - if (NumAnomalousDimensions)
189 - HostAnomalyRate = static_cast<double>(NumAnomalousDimensions) / NumActiveDimensions;
190 - else
191 - HostAnomalyRate = 0.0;
168 +private:
169 + AcquiredDimension(RRDDIM_ACQUIRED *AcqRD, Dimension *D) : AcqRD(AcqRD), D(D) {}
170
193 - NumNormalDimensions = NumActiveDimensions - NumAnomalousDimensions;
171 +public:
172 + TrainingResult train(const TrainingRequest &TR) {
173 + if (!D)
174 + return TrainingResult::NullAcquiredDimension;
175 +
176 + return D->trainModel(TR);
177 }
178
196 - this->NumAnomalousDimensions = NumAnomalousDimensions;
197 - this->NumNormalDimensions = NumNormalDimensions;
198 - this->NumTrainedDimensions = NumTrainedDimensions;
199 - this->NumActiveDimensions = NumActiveDimensions;
179 + ~AcquiredDimension() {
180 + if (AcqRD)
181 + rrddim_acquired_release(AcqRD);
182 + }
183
201 - worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
202 - updateDimensionsChart(getRH(), NumTrainedDimensions, NumNormalDimensions, NumAnomalousDimensions);
203 - updateHostAndDetectionRateCharts(getRH(), HostAnomalyRate * 10000.0);
184 +private:
185 + RRDDIM_ACQUIRED *AcqRD;
186 + Dimension *D;
187 +};
188
205 - struct rusage TRU;
206 - getResourceUsage(&TRU);
207 - updateTrainingChart(getRH(), &TRU);
189 +void Host::scheduleForTraining(TrainingRequest TR) {
190 + TrainingQueue.push(TR);
191 }
192
210 -void DetectableHost::detect() {
211 - worker_register("MLDETECT");
212 - worker_register_job_name(WORKER_JOB_DETECT_DIMENSION, "dimensions");
213 - worker_register_job_name(WORKER_JOB_UPDATE_DETECTION_CHART, "detection chart");
214 - worker_register_job_name(WORKER_JOB_UPDATE_ANOMALY_RATES, "anomaly rates");
215 - worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "charts");
193 +void Host::train() {
194 + while (!netdata_exit) {
195 + netdata_thread_disable_cancelability();
196
217 - std::this_thread::sleep_for(Seconds{10});
197 + auto P = TrainingQueue.pop();
198 + TrainingRequest TrainingReq = P.first;
199 + size_t Size = P.second;
200 +
201 + usec_t AllottedUT = (Cfg.TrainEvery * RH->rrd_update_every * USEC_PER_SEC) / Size;
202 + if (AllottedUT > USEC_PER_SEC)
203 + AllottedUT = USEC_PER_SEC;
204 +
205 + usec_t StartUT = now_realtime_usec();
206 + TrainingResult TrainingRes;
207 + {
208 + AcquiredDimension AcqDim = AcquiredDimension::find(RH, TrainingReq.ChartId, TrainingReq.DimensionId);
209 + TrainingRes = AcqDim.train(TrainingReq);
210 + string_freez(TrainingReq.ChartId);
211 + string_freez(TrainingReq.DimensionId);
212 + }
213 + usec_t ConsumedUT = now_realtime_usec() - StartUT;
214 +
215 + usec_t RemainingUT = 0;
216 + if (ConsumedUT < AllottedUT)
217 + RemainingUT = AllottedUT - ConsumedUT;
218 +
219 + {
220 + std::lock_guard<std::mutex> Lock(Mutex);
221 +
222 + if (TS.AllottedUT == 0) {
223 + struct rusage TRU;
224 + getrusage(RUSAGE_THREAD, &TRU);
225 + TS.TrainingRU = TRU;
226 + }
227 +
228 + TS.QueueSize += Size;
229 + TS.NumPoppedItems += 1;
230 +
231 + TS.AllottedUT += AllottedUT;
232 + TS.ConsumedUT += ConsumedUT;
233 + TS.RemainingUT += RemainingUT;
234 +
235 + switch (TrainingRes) {
236 + case TrainingResult::Ok:
237 + TS.TrainingResultOk += 1;
238 + break;
239 + case TrainingResult::InvalidQueryTimeRange:
240 + TS.TrainingResultInvalidQueryTimeRange += 1;
241 + break;
242 + case TrainingResult::NotEnoughCollectedValues:
243 + TS.TrainingResultNotEnoughCollectedValues += 1;
244 + break;
245 + case TrainingResult::NullAcquiredDimension:
246 + TS.TrainingResultNullAcquiredDimension += 1;
247 + break;
248 + case TrainingResult::ChartUnderReplication:
249 + TS.TrainingResultChartUnderReplication += 1;
250 + break;
251 + }
252 + }
253
254 + netdata_thread_enable_cancelability();
255 + std::this_thread::sleep_for(std::chrono::microseconds{RemainingUT});
256 + }
257 +}
258 +
259 +void Host::detect() {
260 heartbeat_t HB;
261 heartbeat_init(&HB);
262
263 while (!netdata_exit) {
223 - netdata_thread_testcancel();
224 - worker_is_idle();
225 - heartbeat_next(&HB, updateEvery() * USEC_PER_SEC);
264 + heartbeat_next(&HB, RH->rrd_update_every * USEC_PER_SEC);
265
266 netdata_thread_disable_cancelability();
267 detectOnce();
229 -
230 - worker_is_busy(WORKER_JOB_UPDATE_DETECTION_CHART);
231 - updateDetectionChart(getRH());
268 netdata_thread_enable_cancelability();
269 }
270 }
271
236 -void DetectableHost::getDetectionInfoAsJson(nlohmann::json &Json) const {
272 +void Host::getDetectionInfoAsJson(nlohmann::json &Json) const {
273 Json["version"] = 1;
238 - Json["anomalous-dimensions"] = NumAnomalousDimensions;
239 - Json["normal-dimensions"] = NumNormalDimensions;
240 - Json["total-dimensions"] = NumAnomalousDimensions + NumNormalDimensions;
241 - Json["trained-dimensions"] = NumTrainedDimensions;
274 + Json["anomalous-dimensions"] = MLS.NumAnomalousDimensions;
275 + Json["normal-dimensions"] = MLS.NumNormalDimensions;
276 + Json["total-dimensions"] = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
277 + Json["trained-dimensions"] = MLS.NumTrainingStatusTrained + MLS.NumTrainingStatusPendingWithModel;
278 }
279
244 -void DetectableHost::startAnomalyDetectionThreads() {
245 - TrainingThread = std::thread(&TrainableHost::train, this);
246 - DetectionThread = std::thread(&DetectableHost::detect, this);
280 +void Host::startAnomalyDetectionThreads() {
281 + TrainingThread = std::thread(&Host::train, this);
282 + DetectionThread = std::thread(&Host::detect, this);
283 }
284
249 -void DetectableHost::stopAnomalyDetectionThreads() {
285 +void Host::stopAnomalyDetectionThreads() {
286 netdata_thread_cancel(TrainingThread.native_handle());
251 - netdata_thread_cancel(DetectionThread.native_handle());
252 -
287 TrainingThread.join();
288 +
289 + netdata_thread_cancel(DetectionThread.native_handle());
290 DetectionThread.join();
291 }
ml/Host.h
+26 -67
@@ -5,95 +5,54 @@
5
6 #include "Config.h"
7 #include "Dimension.h"
8 +#include "Chart.h"
9 +#include "Queue.h"
10
11 #include "ml-private.h"
12 #include "json/single_include/nlohmann/json.hpp"
13
12 -namespace ml {
14 +namespace ml
15 +{
16
14 -class RrdHost {
17 +class Host {
18 public:
16 - RrdHost(RRDHOST *RH) : RH(RH) {};
19 + Host(RRDHOST *RH) :
20 + RH(RH),
21 + MLS(),
22 + TS(),
23 + HostAnomalyRate(0.0)
24 + { }
25
18 - RRDHOST *getRH() { return RH; }
19 -
20 - unsigned updateEvery() { return RH->rrd_update_every; }
21 -
22 - std::string getUUID() {
23 - char S[UUID_STR_LEN];
24 - uuid_unparse_lower(RH->host_uuid, S);
25 - return S;
26 - }
27 -
28 - void addDimension(Dimension *D);
29 - void removeDimension(Dimension *D);
26 + void addChart(Chart *C);
27 + void removeChart(Chart *C);
28
29 void getConfigAsJson(nlohmann::json &Json) const;
32 -
33 - virtual ~RrdHost() {};
34 -
35 -protected:
36 - RRDHOST *RH;
37 -
38 - // Protect dimension and lock maps
39 - std::mutex Mutex;
40 -
41 - std::unordered_map<RRDDIM *, Dimension *> DimensionsMap;
42 - std::unordered_map<Dimension *, std::mutex> LocksMap;
43 -};
44 -
45 -class TrainableHost : public RrdHost {
46 -public:
47 - TrainableHost(RRDHOST *RH) : RrdHost(RH) {}
48 -
49 - void train();
50 -
51 - void updateResourceUsage() {
52 - std::lock_guard<std::mutex> Lock(ResourceUsageMutex);
53 - getrusage(RUSAGE_THREAD, &ResourceUsage);
54 - }
55 -
56 - void getResourceUsage(struct rusage *RU) {
57 - std::lock_guard<std::mutex> Lock(ResourceUsageMutex);
58 - memcpy(RU, &ResourceUsage, sizeof(struct rusage));
59 - }
60 -
30 void getModelsAsJson(nlohmann::json &Json);
62 -
63 -private:
64 - std::pair<Dimension *, Duration<double>> findDimensionToTrain(const TimePoint &NowTP);
65 - void trainDimension(Dimension *D, const TimePoint &NowTP);
66 -
67 - struct rusage ResourceUsage{};
68 - std::mutex ResourceUsageMutex;
69 -};
70 -
71 -class DetectableHost : public TrainableHost {
72 -public:
73 - DetectableHost(RRDHOST *RH) : TrainableHost(RH) {}
31 + void getDetectionInfoAsJson(nlohmann::json &Json) const;
32
33 void startAnomalyDetectionThreads();
34 void stopAnomalyDetectionThreads();
35
78 - void getDetectionInfoAsJson(nlohmann::json &Json) const;
36 + void scheduleForTraining(TrainingRequest TR);
37 + void train();
38
80 -private:
39 void detect();
40 void detectOnce();
41
42 private:
85 - std::thread TrainingThread;
86 - std::thread DetectionThread;
87 -
43 + RRDHOST *RH;
44 + MachineLearningStats MLS;
45 + TrainingStats TS;
46 CalculatedNumber HostAnomalyRate{0.0};
47
90 - size_t NumAnomalousDimensions{0};
91 - size_t NumNormalDimensions{0};
92 - size_t NumTrainedDimensions{0};
93 - size_t NumActiveDimensions{0};
94 -};
48 + Queue<TrainingRequest> TrainingQueue;
49
96 -using Host = DetectableHost;
50 + std::mutex Mutex;
51 + std::unordered_map<RRDSET *, Chart *> Charts;
52 +
53 + std::thread TrainingThread;
54 + std::thread DetectionThread;
55 +};
56
57 } // namespace ml
58
ml/Query.h
+1 -1
@@ -40,7 +40,7 @@ public:
40 std::pair<time_t, CalculatedNumber> nextMetric() {
41 points_read++;
42 STORAGE_POINT sp = Ops->next_metric(&Handle);
43 - return { sp.start_time, sp.sum / sp.count };
43 + return { sp.end_time, sp.sum / sp.count };
44 }
45
46 private:
ml/Queue.h new
+37
@@ -0,0 +1,37 @@
1 +#ifndef QUEUE_H
2 +#define QUEUE_H
3 +
4 +#include <queue>
5 +#include <mutex>
6 +#include <condition_variable>
7 +
8 +template<typename T>
9 +class Queue {
10 +public:
11 + Queue(void) : Q(), Mutex(), CondVar() { }
12 +
13 + void push(T t) {
14 + std::lock_guard<std::mutex> Lock(Mutex);
15 + Q.push(t);
16 + CondVar.notify_one();
17 + }
18 +
19 + std::pair<T, size_t> pop(void) {
20 + std::unique_lock<std::mutex> Lock(Mutex);
21 + while (Q.empty())
22 + CondVar.wait(Lock);
23 +
24 + T V = Q.front();
25 + size_t Size = Q.size();
26 +
27 + Q.pop();
28 + return { V, Size };
29 + }
30 +
31 +private:
32 + std::queue<T> Q;
33 + std::mutex Mutex;
34 + std::condition_variable CondVar;
35 +};
36 +
37 +#endif /* QUEUE_H */
ml/README.md
+15 -61
@@ -8,7 +8,7 @@ keywords: [machine learning, anomaly detection, Netdata ML]
8
9 ## Overview
10
11 -As of [`v1.32.0`](https://github.com/netdata/netdata/releases/tag/v1.32.0), Netdata comes with some ML powered [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) capabilities built into it and available to use out of the box, with zero configuration required (ML was enabled by default in `v1.35.0-29-nightly` in [this PR](https://github.com/netdata/netdata/pull/13158), previously it required a one line config change).
11 +As of [`v1.32.0`](https://github.com/netdata/netdata/releases/tag/v1.32.0), Netdata comes with ML powered [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) capabilities built into it and available to use out of the box, with zero configuration required (ML was enabled by default in `v1.35.0-29-nightly` in [this PR](https://github.com/netdata/netdata/pull/13158), previously it required a one line config change).
12
13 🚧 **Note**: If you would like to get involved and help us with some feedback, email us at analytics-ml-team@netdata.cloud, comment on the [beta launch post](https://community.netdata.cloud/t/anomaly-advisor-beta-launch/2717) in the Netdata community, or come join us in the [🤖-ml-powered-monitoring](https://discord.gg/4eRSEUpJnc) channel of the Netdata discord.
14
@@ -99,49 +99,7 @@ An ["anomaly detector"](#anomaly-detector) looks at all anomaly bits of a node.
99
100 Essentially if the ["Node Anomaly Rate"](#node-anomaly-rate) (NAR) passes a defined threshold and stays above that threshold for a persistent amount of time, a "Node [Anomaly Event](#anomaly-event)" will be triggered.
101
102 -These anomaly events are currently exposed via `/api/v1/anomaly_events`
103 -
104 -**Note**: Clicking the link below will likely return an empty list of `[]`. This is the response when no anomaly events exist in the specified range. The example response below is illustrative of what the response would be when one or more anomaly events exist within the range of `after` to `before`.
105 -
106 -https://london.my-netdata.io/api/v1/anomaly_events?after=1638365182000&before=1638365602000
107 -
108 -If an event exists within the window, the result would be a list of start and end times.
109 -
110 -```
111 -[
112 - [
113 - 1638367788,
114 - 1638367851
115 - ]
116 -]
117 -```
118 -
119 -Information about each anomaly event can then be found at the `/api/v1/anomaly_event_info` endpoint (making sure to pass the `after` and `before` params):
120 -
121 -**Note**: If you click the below url you will get a `null` since no such anomaly event exists as the response is just an illustrative example taken from a node that did have such an anomaly event.
122 -
123 -https://london.my-netdata.io/api/v1/anomaly_event_info?after=1638367788&before=1638367851
124 -
125 -```
126 -[
127 - [
128 - 0.66,
129 - "netdata.response_time|max"
130 - ],
131 - [
132 - 0.63,
133 - "netdata.response_time|average"
134 - ],
135 - [
136 - 0.54,
137 - "netdata.requests|requests"
138 - ],
139 - ...
140 -```
141 -
142 -The query returns a list of dimension anomaly rates for all dimensions that were considered part of the detected anomaly event.
143 -
144 -**Note**: We plan to build additional anomaly detection and exploration features into both Netdata Agent and Netdata Cloud. The current endpoints are still under active development to power the upcoming features.
102 +These anomaly events are currently exposed via the `new_anomaly_event` dimension on the `anomaly_detection.anomaly_detection` chart.
103
104 ## Configuration
105
@@ -162,6 +120,7 @@ Below is a list of all the available configuration params and their default valu
120 # maximum num samples to train = 14400
121 # minimum num samples to train = 3600
122 # train every = 3600
123 + # number of models per dimension = 1
124 # dbengine anomaly rate every = 30
125 # num samples to diff = 1
126 # num samples to smooth = 3
@@ -169,12 +128,9 @@ Below is a list of all the available configuration params and their default valu
128 # random sampling ratio = 0.2
129 # maximum number of k-means iterations = 1000
130 # dimension anomaly score threshold = 0.99
172 - # host anomaly rate threshold = 0.01000
173 - # minimum window size = 30.00000
174 - # maximum window size = 600.00000
175 - # idle window size = 30.00000
176 - # window minimum anomaly rate = 0.25000
177 - # anomaly event min dimension rate threshold = 0.05000
131 + # host anomaly rate threshold = 1.0
132 + # anomaly detection grouping method = average
133 + # anomaly detection grouping duration = 300
134 # hosts to skip from training = !*
135 # charts to skip from training = netdata.*
136 ```
@@ -221,6 +177,7 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
177 - `maximum num samples to train`: (`3600`/`86400`) This is the maximum amount of time you would like to train each model on. For example, the default of `14400` trains on the preceding 4 hours of data, assuming an `update every` of 1 second.
178 - `minimum num samples to train`: (`900`/`21600`) This is the minimum amount of data required to be able to train a model. For example, the default of `900` implies that once at least 15 minutes of data is available for training, a model is trained, otherwise it is skipped and checked again at the next training run.
179 - `train every`: (`1800`/`21600`) This is how often each model will be retrained. For example, the default of `3600` means that each model is retrained every hour. Note: The training of all models is spread out across the `train every` period for efficiency, so in reality, it means that each model will be trained in a staggered manner within each `train every` period.
180 +- `number of models per dimension`: (`1`/`168`) This is the number of trained models that will be used for scoring. For example the default `number of models per dimension = 1` means that just the most recently trained model (covering up to the most recent `maximum num samples to train` of training data) for the dimension will be used to determine the corresponding anomaly bit. Alternatively, if you have `train every = 3600` and `number of models per dimension = 24` this means that netdata will store and use the last 24 trained models for each dimension when determining the anomaly bit, this means that for the latest feature vector in this configuration to be considered anomalous it would need to look anomalous across _all_ the models trained for that dimension in the last 24 hours. As such, increasing `number of models per dimension` may reduce some false positives since it will result in more models (covering a wider time frame of training) being used during scoring.
181 - `dbengine anomaly rate every`: (`30`/`900`) This is how often netdata will aggregate all the anomaly bits into a single chart (`anomaly_detection.anomaly_rates`). The aggregation into a single chart allows enabling anomaly rate ranking over _all_ metrics with one API call as opposed to a call per chart.
182 - `num samples to diff`: (`0`/`1`) This is a `0` or `1` to determine if you want the model to operate on differences of the raw data or just the raw data. For example, the default of `1` means that we take differences of the raw values. Using differences is more general and works on dimensions that might naturally tend to have some trends or cycles in them that is normal behavior to which we don't want to be too sensitive.
183 - `num samples to smooth`: (`0`/`5`) This is a small integer that controls the amount of smoothing applied as part of the feature processing used by the model. For example, the default of `3` means that the rolling average of the last 3 values is used. Smoothing like this helps the model be a little more robust to spiky types of dimensions that naturally "jump" up or down as part of their normal behavior.
@@ -228,11 +185,9 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
185 - `random sampling ratio`: (`0.2`/`1.0`) This parameter determines how much of the available training data is randomly sampled when training a model. The default of `0.2` means that Netdata will train on a random 20% of training data. This parameter influences cost efficiency. At `0.2` the model is still reasonably trained while minimizing system overhead costs caused by the training.
186 - `maximum number of k-means iterations`: This is a parameter that can be passed to the model to limit the number of iterations in training the k-means model. Vast majority of cases can ignore and leave as default.
187 - `dimension anomaly score threshold`: (`0.01`/`5.00`) This is the threshold at which an individual dimension at a specific timestep is considered anomalous or not. For example, the default of `0.99` means that a dimension with an anomaly score of 99% or higher is flagged as anomalous. This is a normalized probability based on the training data, so the default of 99% means that anything that is as strange (based on distance measure) or more strange as the most strange 1% of data observed during training will be flagged as anomalous. If you wanted to make the anomaly detection on individual dimensions more sensitive you could try a value like `0.90` (90%) or to make it less sensitive you could try `1.5` (150%).
231 -- `host anomaly rate threshold`: (`0.0`/`1.0`) This is the percentage of dimensions (based on all those enabled for anomaly detection) that need to be considered anomalous at specific timestep for the host itself to be considered anomalous. For example, the default value of `0.01` means that if more than 1% of dimensions are anomalous at the same time then the host itself is considered in an anomalous state.
232 -- `minimum window size`: The Netdata "Anomaly Detector" logic works over a rolling window of data. This parameter defines the minimum length of window to consider. If over this window the host is in an anomalous state then an anomaly detection event will be triggered. For example, the default of `30` means that the detector will initially work over a rolling window of 30 seconds. Note: The length of this window will be dynamic once an anomaly event has been triggered such that it will expand as needed until either the max length of an anomaly event is hit or the host settles back into a normal state with sufficiently decreased host level anomaly states in the rolling window. Note: If you wanted to adjust the higher level anomaly detector behavior then this is one parameter you might adjust to see the impact of on anomaly detection events.
233 -- `maximum window size`: This parameter defines the maximum length of window to consider. If an anomaly event reaches this size, it will be closed. This is to provide an upper bound on the length of an anomaly event and cost of the anomaly detector logic for that event.
234 -- `window minimum anomaly rate`: (`0.0`/`1.0`) This parameter corresponds to a threshold on the percentage of time in the rolling window that the host was considered in an anomalous state. For example, the default of `0.25` means that if the host is in an anomalous state for 25% of more of the rolling window then and anomaly event will be triggered or extended if one is already active. Note: If you want to make the anomaly detector itself less sensitive, you can adjust this value to something like `0.75` which would mean the host needs to be much more consistently in an anomalous state to trigger an anomaly detection event. Likewise, a lower value like `0.1` would make the anomaly detector more sensitive.
235 -- `anomaly event min dimension rate threshold`: (`0.0`/`1.0`) This is a parameter that helps filter out irrelevant dimensions from anomaly events. For example, the default of `0.05` means that only dimensions that were considered anomalous for at least 5% of the anomaly event itself will be included in that anomaly event. The idea here is to just include dimensions that were consistently anomalous as opposed to those that may have just randomly happened to be anomalous at the same time.
188 +- `host anomaly rate threshold`: (`0.1`/`10.0`) This is the percentage of dimensions (based on all those enabled for anomaly detection) that need to be considered anomalous at specific timestep for the host itself to be considered anomalous. For example, the default value of `1.0` means that if more than 1% of dimensions are anomalous at the same time then the host itself is considered in an anomalous state.
189 +- `anomaly detection grouping method`: The grouping method used when calculating node level anomaly rate.
190 +- `anomaly detection grouping duration`: (`60`/`900`) The duration across which to calculate the node level anomaly rate, the default of `900` means that the node level anomaly rate is calculated across a rolling 5 minute window.
191 - `hosts to skip from training`: This parameter allows you to turn off anomaly detection for any child hosts on a parent host by defining those you would like to skip from training here. For example, a value like `dev-*` skips all hosts on a parent that begin with the "dev-" prefix. The default value of `!*` means "don't skip any".
192 - `charts to skip from training`: This parameter allows you to exclude certain charts from anomaly detection. By default, only netdata related charts are excluded. This is to avoid the scenario where accessing the netdata dashboard could itself tigger some anomalies if you don't access them regularly. If you want to include charts that are excluded by default, add them in small groups and then measure any impact on performance before adding additional ones. Example: If you want to include system, apps, and user charts:`!system.* !apps.* !user.* *`.
193
@@ -240,28 +195,27 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
195
196 Once enabled, the "Anomaly Detection" menu and charts will be available on the dashboard.
197
243 -![anomaly_detection_menu](https://user-images.githubusercontent.com/2178292/144255721-4568aabf-39c7-4855-bf1c-31b1d60e28e6.png)
198 +![anomaly_detection_menu](https://user-images.githubusercontent.com/2178292/207584589-2e984786-5e01-404b-a20a-58573884d6df.png)
199
200 In terms of anomaly detection, the most interesting charts would be the `anomaly_detection.dimensions` and `anomaly_detection.anomaly_rate` ones, which hold the `anomalous` and `anomaly_rate` dimensions that show the overall number of dimensions considered anomalous at any time and the corresponding anomaly rate.
201
202 - `anomaly_detection.dimensions`: Total count of dimensions considered anomalous or normal.
203 - `anomaly_detection.dimensions`: Percentage of anomalous dimensions.
249 -- `anomaly_detection.detector_window`: The length of the active window used by the detector.
250 -- `anomaly_detection.detector_events`: Flags (0 or 1) to show when an anomaly event has been triggered by the detector.
204 +- `anomaly_detection.anomaly_detection`: Flags (0 or 1) to show when an anomaly event has been triggered by the detector.
205
206 Below is an example of how these charts may look in the presence of an anomaly event.
207
208 Initially we see a jump in `anomalous` dimensions:
209
256 -![anomalous](https://user-images.githubusercontent.com/2178292/144256036-c89fa768-5e5f-4278-9725-c67521c0d95e.png)
210 +![anomalous](https://user-images.githubusercontent.com/2178292/207589021-c0d2926f-bb55-4c5c-9e32-be1851558fa8.png)
211
212 And a corresponding jump in the `anomaly_rate`:
213
260 -![anomaly_rate](https://user-images.githubusercontent.com/2178292/144256071-7d157438-31f3-4b23-a795-0fd3b2e2e85c.png)
214 +![anomaly_rate](https://user-images.githubusercontent.com/2178292/207589172-8853804b-6826-4731-8d06-b9e32d3071af.png)
215
216 After a short while the rolling node anomaly rate goes `above_threshold`, and once it stays above threshold for long enough a `new_anomaly_event` is created:
217
264 -![anomaly_event](https://user-images.githubusercontent.com/2178292/144256152-910b06ec-26b8-45b4-bcb7-4c2acdf9af15.png)
218 +![anomaly_event](https://user-images.githubusercontent.com/2178292/207589308-931a3c76-440a-48c1-970e-191743d26607.png)
219
220 ## Glossary
221
ml/SamplesBufferTests.cc deleted
-146
@@ -1,146 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "ml/ml-private.h"
4 -#include <gtest/gtest.h>
5 -
6 -/*
7 - * The SamplesBuffer class implements the functionality of the following python
8 - * code:
9 - * >> df = pd.DataFrame(data=samples)
10 - * >> df = df.diff(diff_n).dropna()
11 - * >> df = df.rolling(smooth_n).mean().dropna()
12 - * >> df = pd.concat([df.shift(n) for n in range(lag_n + 1)], axis=1).dropna()
13 - *
14 - * Its correctness has been verified by automatically generating random
15 - * data frames in Python and comparing them with the correspondent preprocessed
16 - * SampleBuffers.
17 - *
18 - * The following tests are meant to catch unintended changes in the SamplesBuffer
19 - * implementation. For development purposes, one should compare changes against
20 - * the aforementioned python code.
21 -*/
22 -
23 -TEST(SamplesBufferTest, NS_8_NDPS_1_DN_1_SN_3_LN_1) {
24 - size_t NumSamples = 8, NumDimsPerSample = 1;
25 - size_t DiffN = 1, SmoothN = 3, LagN = 3;
26 -
27 - size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
28 - CalculatedNumber *CNs = new CalculatedNumber[N]();
29 -
30 - CNs[0] = 0.7568336679490107;
31 - CNs[1] = 0.4814406581763254;
32 - CNs[2] = 0.40073555156221874;
33 - CNs[3] = 0.5973257298194408;
34 - CNs[4] = 0.5334727814345868;
35 - CNs[5] = 0.2632477193454843;
36 - CNs[6] = 0.2684839023122384;
37 - CNs[7] = 0.851332948637479;
38 -
39 - std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
40 - SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
41 - SB.preprocess();
42 -
43 - std::vector<Sample> Samples = SB.getPreprocessedSamples();
44 - EXPECT_EQ(Samples.size(), 2);
45 -
46 - Sample S0 = Samples[0];
47 - const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
48 - Sample S1 = Samples[1];
49 - const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
50 -
51 - EXPECT_NEAR(S0_CNs[0], -0.109614, 0.001);
52 - EXPECT_NEAR(S0_CNs[1], -0.0458293, 0.001);
53 - EXPECT_NEAR(S0_CNs[2], 0.017344, 0.001);
54 - EXPECT_NEAR(S0_CNs[3], -0.0531693, 0.001);
55 -
56 - EXPECT_NEAR(S1_CNs[0], 0.105953, 0.001);
57 - EXPECT_NEAR(S1_CNs[1], -0.109614, 0.001);
58 - EXPECT_NEAR(S1_CNs[2], -0.0458293, 0.001);
59 - EXPECT_NEAR(S1_CNs[3], 0.017344, 0.001);
60 -
61 - delete[] CNs;
62 -}
63 -
64 -TEST(SamplesBufferTest, NS_8_NDPS_1_DN_2_SN_3_LN_2) {
65 - size_t NumSamples = 8, NumDimsPerSample = 1;
66 - size_t DiffN = 2, SmoothN = 3, LagN = 2;
67 -
68 - size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
69 - CalculatedNumber *CNs = new CalculatedNumber[N]();
70 -
71 - CNs[0] = 0.20511885291342846;
72 - CNs[1] = 0.13151717360306558;
73 - CNs[2] = 0.6017085062423134;
74 - CNs[3] = 0.46256882933941545;
75 - CNs[4] = 0.7887758447877941;
76 - CNs[5] = 0.9237989080034406;
77 - CNs[6] = 0.15552559051428083;
78 - CNs[7] = 0.6309750314597955;
79 -
80 - std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
81 - SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
82 - SB.preprocess();
83 -
84 - std::vector<Sample> Samples = SB.getPreprocessedSamples();
85 - EXPECT_EQ(Samples.size(), 2);
86 -
87 - Sample S0 = Samples[0];
88 - const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
89 - Sample S1 = Samples[1];
90 - const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
91 -
92 - EXPECT_NEAR(S0_CNs[0], 0.005016, 0.001);
93 - EXPECT_NEAR(S0_CNs[1], 0.326450, 0.001);
94 - EXPECT_NEAR(S0_CNs[2], 0.304903, 0.001);
95 -
96 - EXPECT_NEAR(S1_CNs[0], -0.154948, 0.001);
97 - EXPECT_NEAR(S1_CNs[1], 0.005016, 0.001);
98 - EXPECT_NEAR(S1_CNs[2], 0.326450, 0.001);
99 -
100 - delete[] CNs;
101 -}
102 -
103 -TEST(SamplesBufferTest, NS_8_NDPS_3_DN_2_SN_4_LN_1) {
104 - size_t NumSamples = 8, NumDimsPerSample = 3;
105 - size_t DiffN = 2, SmoothN = 4, LagN = 1;
106 -
107 - size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
108 - CalculatedNumber *CNs = new CalculatedNumber[N]();
109 -
110 - CNs[0] = 0.34310900399667765; CNs[1] = 0.14694315994488194; CNs[2] = 0.8246677800938796;
111 - CNs[3] = 0.48249504592307835; CNs[4] = 0.23241087965531182; CNs[5] = 0.9595348555892567;
112 - CNs[6] = 0.44281094035598334; CNs[7] = 0.5143142171362715; CNs[8] = 0.06391303014242555;
113 - CNs[9] = 0.7460491027783901; CNs[10] = 0.43887217459032923; CNs[11] = 0.2814395025355999;
114 - CNs[12] = 0.9231114281214198; CNs[13] = 0.326882401786898; CNs[14] = 0.26747939220376216;
115 - CNs[15] = 0.7787571209969636; CNs[16] =0.5851700001235088; CNs[17] = 0.34410728945321567;
116 - CNs[18] = 0.9394494507088997; CNs[19] =0.17567223681734334; CNs[20] = 0.42732886195446984;
117 - CNs[21] = 0.9460522396152958; CNs[22] =0.23462747016780894; CNs[23] = 0.35983249900892145;
118 -
119 - std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
120 - SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
121 - SB.preprocess();
122 -
123 - std::vector<Sample> Samples = SB.getPreprocessedSamples();
124 - EXPECT_EQ(Samples.size(), 2);
125 -
126 - Sample S0 = Samples[0];
127 - const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
128 - Sample S1 = Samples[1];
129 - const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
130 -
131 - EXPECT_NEAR(S0_CNs[0], 0.198225, 0.001);
132 - EXPECT_NEAR(S0_CNs[1], 0.003529, 0.001);
133 - EXPECT_NEAR(S0_CNs[2], -0.063003, 0.001);
134 - EXPECT_NEAR(S0_CNs[3], 0.219066, 0.001);
135 - EXPECT_NEAR(S0_CNs[4], 0.133175, 0.001);
136 - EXPECT_NEAR(S0_CNs[5], -0.293154, 0.001);
137 -
138 - EXPECT_NEAR(S1_CNs[0], 0.174160, 0.001);
139 - EXPECT_NEAR(S1_CNs[1], -0.135722, 0.001);
140 - EXPECT_NEAR(S1_CNs[2], 0.110452, 0.001);
141 - EXPECT_NEAR(S1_CNs[3], 0.198225, 0.001);
142 - EXPECT_NEAR(S1_CNs[4], 0.003529, 0.001);
143 - EXPECT_NEAR(S1_CNs[5], -0.063003, 0.001);
144 -
145 - delete[] CNs;
146 -}
ml/Stats.h new
+46
@@ -0,0 +1,46 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef ML_STATS_H
4 +#define ML_STATS_H
5 +
6 +#include "ml-private.h"
7 +
8 +namespace ml {
9 +
10 +struct MachineLearningStats {
11 + size_t NumMachineLearningStatusEnabled;
12 + size_t NumMachineLearningStatusDisabledUE;
13 + size_t NumMachineLearningStatusDisabledSP;
14 +
15 + size_t NumMetricTypeConstant;
16 + size_t NumMetricTypeVariable;
17 +
18 + size_t NumTrainingStatusUntrained;
19 + size_t NumTrainingStatusPendingWithoutModel;
20 + size_t NumTrainingStatusTrained;
21 + size_t NumTrainingStatusPendingWithModel;
22 +
23 + size_t NumAnomalousDimensions;
24 + size_t NumNormalDimensions;
25 +};
26 +
27 +struct TrainingStats {
28 + struct rusage TrainingRU;
29 +
30 + size_t QueueSize;
31 + size_t NumPoppedItems;
32 +
33 + usec_t AllottedUT;
34 + usec_t ConsumedUT;
35 + usec_t RemainingUT;
36 +
37 + size_t TrainingResultOk;
38 + size_t TrainingResultInvalidQueryTimeRange;
39 + size_t TrainingResultNotEnoughCollectedValues;
40 + size_t TrainingResultNullAcquiredDimension;
41 + size_t TrainingResultChartUnderReplication;
42 +};
43 +
44 +} // namespace ml
45 +
46 +#endif /* ML_STATS_H */
ml/ml-dummy.c
+35 -8
@@ -15,9 +15,29 @@ bool ml_enabled(RRDHOST *RH) {
15
16 void ml_init(void) {}
17
18 -void ml_new_host(RRDHOST *RH) { (void) RH; }
18 +void ml_host_new(RRDHOST *RH) {
19 + UNUSED(RH);
20 +}
21 +
22 +void ml_host_delete(RRDHOST *RH) {
23 + UNUSED(RH);
24 +}
25 +
26 +void ml_chart_new(RRDSET *RS) {
27 + UNUSED(RS);
28 +}
29 +
30 +void ml_chart_delete(RRDSET *RS) {
31 + UNUSED(RS);
32 +}
33 +
34 +void ml_dimension_new(RRDDIM *RD) {
35 + UNUSED(RD);
36 +}
37
20 -void ml_delete_host(RRDHOST *RH) { (void) RH; }
38 +void ml_dimension_delete(RRDDIM *RD) {
39 + UNUSED(RD);
40 +}
41
42 char *ml_get_host_info(RRDHOST *RH) {
43 (void) RH;
@@ -29,17 +49,24 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
49 return NULL;
50 }
51
52 +void ml_chart_update_begin(RRDSET *RS) {
53 + (void) RS;
54 +}
55 +
56 +void ml_chart_update_end(RRDSET *RS) {
57 + (void) RS;
58 +}
59 +
60 char *ml_get_host_models(RRDHOST *RH) {
61 (void) RH;
62 return NULL;
63 }
64
37 -void ml_new_dimension(RRDDIM *RD) { (void) RD; }
38 -
39 -void ml_delete_dimension(RRDDIM *RD) { (void) RD; }
40 -
41 -bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
42 - (void) RD; (void) Value; (void) Exists;
65 +bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
66 + (void) RD;
67 + (void) CurrT;
68 + (void) Value;
69 + (void) Exists;
70 return false;
71 }
72
ml/ml-private.h
-13
@@ -6,21 +6,8 @@
6 #include "KMeans.h"
7 #include "ml/ml.h"
8
9 -#include <chrono>
9 #include <map>
10 #include <mutex>
11 #include <sstream>
12
14 -namespace ml {
15 -
16 -using SteadyClock = std::chrono::steady_clock;
17 -using TimePoint = std::chrono::time_point<SteadyClock>;
18 -
19 -template<typename T>
20 -using Duration = std::chrono::duration<T>;
21 -
22 -using Seconds = std::chrono::seconds;
23 -
24 -} // namespace ml
25 -
13 #endif /* ML_PRIVATE_H */
ml/ml.cc
+57 -38
@@ -2,6 +2,7 @@
2
3 #include "Config.h"
4 #include "Dimension.h"
5 +#include "Chart.h"
6 #include "Host.h"
7
8 #include <random>
@@ -45,18 +46,18 @@ void ml_init(void) {
46 Cfg.RandomNums.push_back(Gen());
47 }
48
48 -void ml_new_host(RRDHOST *RH) {
49 +void ml_host_new(RRDHOST *RH) {
50 if (!ml_enabled(RH))
51 return;
52
53 Host *H = new Host(RH);
53 - RH->ml_host = static_cast<ml_host_t>(H);
54 + RH->ml_host = reinterpret_cast<ml_host_t *>(H);
55
56 H->startAnomalyDetectionThreads();
57 }
58
58 -void ml_delete_host(RRDHOST *RH) {
59 - Host *H = static_cast<Host *>(RH->ml_host);
59 +void ml_host_delete(RRDHOST *RH) {
60 + Host *H = reinterpret_cast<Host *>(RH->ml_host);
61 if (!H)
62 return;
63
@@ -66,34 +67,46 @@ void ml_delete_host(RRDHOST *RH) {
67 RH->ml_host = nullptr;
68 }
69
69 -void ml_new_dimension(RRDDIM *RD) {
70 - RRDSET *RS = RD->rrdset;
71 -
72 - Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host);
70 +void ml_chart_new(RRDSET *RS) {
71 + Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
72 if (!H)
73 return;
74
76 - if (static_cast<unsigned>(RD->update_every) != H->updateEvery())
75 + Chart *C = new Chart(RS);
76 + RS->ml_chart = reinterpret_cast<ml_chart_t *>(C);
77 +
78 + H->addChart(C);
79 +}
80 +
81 +void ml_chart_delete(RRDSET *RS) {
82 + Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
83 + if (!H)
84 return;
85
79 - if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RS)))
86 + Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
87 + H->removeChart(C);
88 +
89 + delete C;
90 + RS->ml_chart = nullptr;
91 +}
92 +
93 +void ml_dimension_new(RRDDIM *RD) {
94 + Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
95 + if (!C)
96 return;
97
98 Dimension *D = new Dimension(RD);
83 - RD->ml_dimension = static_cast<ml_dimension_t>(D);
84 - H->addDimension(D);
99 + RD->ml_dimension = reinterpret_cast<ml_dimension_t *>(D);
100 + C->addDimension(D);
101 }
102
87 -void ml_delete_dimension(RRDDIM *RD) {
88 - Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
103 +void ml_dimension_delete(RRDDIM *RD) {
104 + Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
105 if (!D)
106 return;
107
92 - Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host);
93 - if (!H)
94 - delete D;
95 - else
96 - H->removeDimension(D);
108 + Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
109 + C->removeDimension(D);
110
111 RD->ml_dimension = nullptr;
112 }
@@ -102,7 +115,7 @@ char *ml_get_host_info(RRDHOST *RH) {
115 nlohmann::json ConfigJson;
116
117 if (RH && RH->ml_host) {
105 - Host *H = static_cast<Host *>(RH->ml_host);
118 + Host *H = reinterpret_cast<Host *>(RH->ml_host);
119 H->getConfigAsJson(ConfigJson);
120 } else {
121 ConfigJson["enabled"] = false;
@@ -115,7 +128,7 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
128 nlohmann::json ConfigJson;
129
130 if (RH && RH->ml_host) {
118 - Host *H = static_cast<Host *>(RH->ml_host);
131 + Host *H = reinterpret_cast<Host *>(RH->ml_host);
132 H->getDetectionInfoAsJson(ConfigJson);
133 } else {
134 return nullptr;
@@ -128,7 +141,7 @@ char *ml_get_host_models(RRDHOST *RH) {
141 nlohmann::json ModelsJson;
142
143 if (RH && RH->ml_host) {
131 - Host *H = static_cast<Host *>(RH->ml_host);
144 + Host *H = reinterpret_cast<Host *>(RH->ml_host);
145 H->getModelsAsJson(ModelsJson);
146 return strdup(ModelsJson.dump(2, '\t').c_str());
147 }
@@ -136,30 +149,36 @@ char *ml_get_host_models(RRDHOST *RH) {
149 return nullptr;
150 }
151
139 -bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
140 - Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
141 - if (!D)
142 - return false;
152 +void ml_chart_update_begin(RRDSET *RS) {
153 + Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
154 + if (!C)
155 + return;
156
144 - return D->predict(Value, Exists);
157 + C->updateBegin();
158 }
159
147 -bool ml_streaming_enabled() {
148 - return Cfg.StreamADCharts;
149 -}
160 +void ml_chart_update_end(RRDSET *RS) {
161 + Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
162 + if (!C)
163 + return;
164
151 -#if defined(ENABLE_ML_TESTS)
165 + C->updateEnd();
166 +}
167
153 -#include "gtest/gtest.h"
168 +bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
169 + Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
170 + if (!D)
171 + return false;
172
155 -int test_ml(int argc, char *argv[]) {
156 - (void) argc;
157 - (void) argv;
173 + Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
174
159 - ::testing::InitGoogleTest(&argc, argv);
160 - return RUN_ALL_TESTS();
175 + bool IsAnomalous = D->predict(CurrT, Value, Exists);
176 + C->updateDimension(D, IsAnomalous);
177 + return IsAnomalous;
178 }
179
163 -#endif // ENABLE_ML_TESTS
180 +bool ml_streaming_enabled() {
181 + return Cfg.StreamADCharts;
182 +}
183
184 #include "ml-private.h"
ml/ml.h
+11 -14
@@ -14,35 +14,32 @@ extern "C" {
14 // the anomaly rate dimension, whenever its backing dimension is freed.
15 void rrddim_free(RRDSET *st, RRDDIM *rd);
16
17 -typedef void* ml_host_t;
18 -typedef void* ml_dimension_t;
19 -
17 bool ml_capable();
18
19 bool ml_enabled(RRDHOST *RH);
20
21 void ml_init(void);
22
26 -void ml_new_host(RRDHOST *RH);
27 -void ml_delete_host(RRDHOST *RH);
23 +void ml_host_new(RRDHOST *RH);
24 +void ml_host_delete(RRDHOST *RH);
25 +
26 +void ml_chart_new(RRDSET *RS);
27 +void ml_chart_delete(RRDSET *RS);
28 +
29 +void ml_dimension_new(RRDDIM *RD);
30 +void ml_dimension_delete(RRDDIM *RD);
31
32 char *ml_get_host_info(RRDHOST *RH);
33 char *ml_get_host_runtime_info(RRDHOST *RH);
34 char *ml_get_host_models(RRDHOST *RH);
35
33 -void ml_new_dimension(RRDDIM *RD);
34 -void ml_delete_dimension(RRDDIM *RD);
36 +void ml_chart_update_begin(RRDSET *RS);
37 +void ml_chart_update_end(RRDSET *RS);
38
36 -bool ml_is_anomalous(RRDDIM *RD, double value, bool exists);
39 +bool ml_is_anomalous(RRDDIM *RD, time_t curr_t, double value, bool exists);
40
41 bool ml_streaming_enabled();
42
40 -#define ML_ANOMALY_RATES_CHART_ID "anomaly_detection.anomaly_rates"
41 -
42 -#if defined(ENABLE_ML_TESTS)
43 -int test_ml(int argc, char *argv[]);
44 -#endif
45 -
43 #ifdef __cplusplus
44 };
45 #endif
netdata-installer.sh
-2
@@ -337,8 +337,6 @@ while [ -n "${1}" ]; do
337 NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-ml)}" | sed 's/$/ --disable-ml/g')"
338 NETDATA_ENABLE_ML=0
339 ;;
340 - "--enable-ml-tests") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-ml-tests)}" | sed 's/$/ --enable-ml-tests/g')" ;;
341 - "--disable-ml-tests") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-ml-tests)}" | sed 's/$/ --disable-ml-tests/g')" ;;
340 "--disable-lto") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-lto)}" | sed 's/$/ --disable-lto/g')" ;;
341 "--disable-x86-sse") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-x86-sse)}" | sed 's/$/ --disable-x86-sse/g')" ;;
342 "--disable-telemetry") NETDATA_DISABLE_TELEMETRY=1 ;;