@cryptotaxi247 / netdata-1 / commits / 6f42311c4

Revert "Refactor ML code and add support for multiple KMeans models. … (#14172)

vkalintiris committed Dec 22, 2022 at 13:18 UTC 6f42311c4b32d42798f78de1fd43f53694f24e6e
31 files changed +785 -1411
Makefile.am
+13 -4
@@ -236,14 +236,10 @@ ML_FILES += \
236 ml/ADCharts.cc \
237 ml/Config.h \
238 ml/Config.cc \
239 - ml/Chart.cc \
240 - ml/Chart.h \
241 - ml/Stats.h \
239 ml/Dimension.cc \
240 ml/Dimension.h \
241 ml/Host.h \
242 ml/Host.cc \
246 - ml/Queue.h \
243 ml/Query.h \
244 ml/KMeans.h \
245 ml/KMeans.cc \
@@ -266,6 +262,13 @@ ml/ml.$(OBJEXT) : CXXFLAGS += -Wno-psabi
262
263 endif
264
265 +
266 +if ENABLE_ML_TESTS
267 +ML_TESTS_FILES = \
268 + ml/SamplesBufferTests.cc \
269 + $(NULL)
270 +endif
271 +
272 IDLEJITTER_PLUGIN_FILES = \
273 collectors/idlejitter.plugin/plugin_idlejitter.c \
274 $(NULL)
@@ -917,6 +920,7 @@ NETDATA_FILES = \
920 $(EXPORTING_ENGINE_FILES) \
921 $(HEALTH_PLUGIN_FILES) \
922 $(ML_FILES) \
923 + $(ML_TESTS_FILES) \
924 $(IDLEJITTER_PLUGIN_FILES) \
925 $(PLUGINSD_PLUGIN_FILES) \
926 $(REGISTRY_PLUGIN_FILES) \
@@ -1004,6 +1008,11 @@ if ENABLE_ACLK
1008 $(NULL)
1009 endif
1010
1011 +if ENABLE_ML_TESTS
1012 + netdata_LDADD += $(OPTIONAL_ML_TESTS_LIBS) \
1013 + $(NULL)
1014 +endif
1015 +
1016 netdata_LINK = $(CXXLD) $(CXXFLAGS) $(LDFLAGS) -o $@
1017
1018 sbin_PROGRAMS += netdatacli
collectors/all.h
-18
@@ -363,23 +363,5 @@
363 #define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030
364 #define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100
365
366 -// NETDATA ML CHARTS
367 -
368 -// [ml] charts
369 -#define ML_CHART_PRIO_DIMENSIONS 39181
370 -#define ML_CHART_PRIO_ANOMALY_RATE 39182
371 -#define ML_CHART_PRIO_DETECTOR_EVENTS 39183
372 -
373 -// [netdata.ml] charts
374 -#define NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS 890001
375 -#define NETDATA_ML_CHART_PRIO_METRIC_TYPES 890002
376 -#define NETDATA_ML_CHART_PRIO_TRAINING_STATUS 890003
377 -
378 -#define NETDATA_ML_CHART_PRIO_PREDICTION_USAGE 890004
379 -#define NETDATA_ML_CHART_PRIO_TRAINING_USAGE 890005
380 -
381 -#define NETDATA_ML_CHART_PRIO_QUEUE_STATS 890006
382 -#define NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS 890007
383 -#define NETDATA_ML_CHART_PRIO_TRAINING_RESULTS 890008
366
367 #endif //NETDATA_ALL_H
configure.ac
+22 -1
@@ -207,6 +207,12 @@ AC_ARG_ENABLE(
207 ,
208 [enable_ml="detect"]
209 )
210 +AC_ARG_ENABLE(
211 + [ml_tests],
212 + [AS_HELP_STRING([--enable-ml-tests], [Enable anomaly detection tests @<:@no@:>@])],
213 + [enable_ml_tests="yes"],
214 + [enable_ml_tests="no"]
215 +)
216 AC_ARG_ENABLE(
217 [aclk_ssl_debug],
218 [AS_HELP_STRING([--enable-aclk-ssl-debug], [Enables possibility for SSL key logging @<:@default no@:>@])],
@@ -1174,6 +1180,19 @@ if test "${build_ml}" = "yes"; then
1180 OPTIONAL_ML_LIBS=""
1181 fi
1182
1183 +# Decide if we should build ML tests.
1184 +if test "${build_ml}" = "yes" -a "${enable_ml_tests}" = "yes" -a "${have_gtest}" = "yes"; then
1185 + build_ml_tests="yes"
1186 +else
1187 + build_ml_tests="no"
1188 +fi
1189 +
1190 +AM_CONDITIONAL([ENABLE_ML_TESTS], [test "${build_ml_tests}" = "yes"])
1191 +if test "${build_ml_tests}" = "yes"; then
1192 + AC_DEFINE([ENABLE_ML_TESTS], [1], [anomaly detection tests])
1193 + OPTIONAL_ML_TESTS_CFLAGS="${OPTIONAL_GTEST_CFLAGS}"
1194 + OPTIONAL_ML_TESTS_LIBS="${OPTIONAL_GTEST_LIBS}"
1195 +fi
1196
1197 # -----------------------------------------------------------------------------
1198 # ebpf.plugin
@@ -1593,7 +1612,7 @@ CFLAGS="${originalCFLAGS} ${OPTIONAL_LTO_CFLAGS} ${OPTIONAL_PROTOBUF_CFLAGS} ${O
1612 ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
1613 ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PUBSUB_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} \
1614 ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
1596 - ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1615 + ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_ML_TESTS_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1616
1617 CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
1618
@@ -1647,6 +1666,8 @@ AC_SUBST([OPTIONAL_GTEST_CFLAGS])
1666 AC_SUBST([OPTIONAL_GTEST_LIBS])
1667 AC_SUBST([OPTIONAL_ML_CFLAGS])
1668 AC_SUBST([OPTIONAL_ML_LIBS])
1669 +AC_SUBST([OPTIONAL_ML_TESTS_CFLAGS])
1670 +AC_SUBST([OPTIONAL_ML_TESTS_LIBS])
1671
1672 # -----------------------------------------------------------------------------
1673 # Check if cmocka is available - needed for unit testing
daemon/global_statistics.c
-34
@@ -52,7 +52,6 @@ static struct global_statistics {
52 uint64_t ml_queries_made;
53 uint64_t ml_db_points_read;
54 uint64_t ml_result_points_generated;
55 - uint64_t ml_models_consulted;
55
56 uint64_t exporters_queries_made;
57 uint64_t exporters_db_points_read;
@@ -89,10 +88,6 @@ void global_statistics_ml_query_completed(size_t points_read) {
88 __atomic_fetch_add(&global_statistics.ml_db_points_read, points_read, __ATOMIC_RELAXED);
89 }
90
92 -void global_statistics_ml_models_consulted(size_t models_consulted) {
93 - __atomic_fetch_add(&global_statistics.ml_models_consulted, models_consulted, __ATOMIC_RELAXED);
94 -}
95 -
91 void global_statistics_exporters_query_completed(size_t points_read) {
92 __atomic_fetch_add(&global_statistics.exporters_queries_made, 1, __ATOMIC_RELAXED);
93 __atomic_fetch_add(&global_statistics.exporters_db_points_read, points_read, __ATOMIC_RELAXED);
@@ -198,7 +193,6 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
193 gs->ml_queries_made = __atomic_load_n(&global_statistics.ml_queries_made, __ATOMIC_RELAXED);
194 gs->ml_db_points_read = __atomic_load_n(&global_statistics.ml_db_points_read, __ATOMIC_RELAXED);
195 gs->ml_result_points_generated = __atomic_load_n(&global_statistics.ml_result_points_generated, __ATOMIC_RELAXED);
201 - gs->ml_models_consulted = __atomic_load_n(&global_statistics.ml_models_consulted, __ATOMIC_RELAXED);
196
197 gs->exporters_queries_made = __atomic_load_n(&global_statistics.exporters_queries_made, __ATOMIC_RELAXED);
198 gs->exporters_db_points_read = __atomic_load_n(&global_statistics.exporters_db_points_read, __ATOMIC_RELAXED);
@@ -659,34 +653,6 @@ static void global_statistics_charts(void) {
653
654 rrdset_done(st_points_stored);
655 }
662 -
663 - {
664 - static RRDSET *st = NULL;
665 - static RRDDIM *rd = NULL;
666 -
667 - if (unlikely(!st)) {
668 - st = rrdset_create_localhost(
669 - "netdata" // type
670 - , "ml_models_consulted" // id
671 - , NULL // name
672 - , "ml" // family
673 - , NULL // context
674 - , "KMeans models used for prediction" // title
675 - , "models" // units
676 - , "netdata" // plugin
677 - , "ml" // module
678 - , 131004 // priority
679 - , localhost->rrd_update_every // update_every
680 - , RRDSET_TYPE_STACKED // chart_type
681 - );
682 -
683 - rd = rrddim_add(st, "num_models_consulted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
684 - }
685 -
686 - rrddim_set_by_pointer(st, rd, (collected_number) gs.ml_models_consulted);
687 -
688 - rrdset_done(st);
689 - }
656 }
657
658 // ----------------------------------------------------------------------------
daemon/global_statistics.h
-1
@@ -9,7 +9,6 @@
9 // global statistics
10
11 void global_statistics_ml_query_completed(size_t points_read);
12 -void global_statistics_ml_models_consulted(size_t models_consulted);
12 void global_statistics_exporters_query_completed(size_t points_read);
13 void global_statistics_backfill_query_completed(size_t points_read);
14 void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source);
daemon/main.c
+5
@@ -1027,6 +1027,11 @@ int main(int argc, char **argv) {
1027 else if(strcmp(optarg, "escapetest") == 0) {
1028 return command_argument_sanitization_tests();
1029 }
1030 +#ifdef ENABLE_ML_TESTS
1031 + else if(strcmp(optarg, "mltest") == 0) {
1032 + return test_ml(argc, argv);
1033 + }
1034 +#endif
1035 #ifdef ENABLE_DBENGINE
1036 else if(strcmp(optarg, "mctest") == 0) {
1037 unittest_running = true;
database/rrd.h
+6 -12
@@ -30,9 +30,8 @@ typedef struct rrdhost_acquired RRDHOST_ACQUIRED;
30 typedef struct rrdset_acquired RRDSET_ACQUIRED;
31 typedef struct rrddim_acquired RRDDIM_ACQUIRED;
32
33 -typedef struct ml_host ml_host_t;
34 -typedef struct ml_chart ml_chart_t;
35 -typedef struct ml_dimension ml_dimension_t;
33 +typedef void *ml_host_t;
34 +typedef void *ml_dimension_t;
35
36 typedef enum {
37 QUERY_SOURCE_UNKNOWN,
@@ -297,7 +296,7 @@ struct rrddim {
296 // ------------------------------------------------------------------------
297 // operational state members
298
300 - ml_dimension_t *ml_dimension; // machine learning data about this dimension
299 + ml_dimension_t ml_dimension; // machine learning data about this dimension
300
301 // ------------------------------------------------------------------------
302 // linking to siblings and parents
@@ -596,8 +595,6 @@ struct rrdset {
595 DICTIONARY *rrddimvar_root_index; // dimension variables
596 // we use this dictionary to manage their allocation
597
599 - ml_chart_t *ml_chart;
600 -
598 // ------------------------------------------------------------------------
599 // operational state members
600
@@ -1031,7 +1028,7 @@ struct rrdhost {
1028
1029 // ------------------------------------------------------------------------
1030 // ML handle
1034 - ml_host_t *ml_host;
1031 + ml_host_t ml_host;
1032
1033 // ------------------------------------------------------------------------
1034 // Support for host-level labels
@@ -1304,12 +1301,9 @@ void rrdset_isnot_obsolete(RRDSET *st);
1301 time_t rrddim_first_entry_t(RRDDIM *rd);
1302 time_t rrddim_first_entry_t_of_tier(RRDDIM *rd, size_t tier);
1303 time_t rrddim_last_entry_t(RRDDIM *rd);
1307 -time_t rrddim_last_entry_t_of_tier(RRDDIM *rd, size_t tier);
1308 -
1309 -time_t rrdset_first_entry_t(RRDSET *st);
1310 -time_t rrdset_first_entry_t_of_tier(RRDSET *st, size_t tier);
1304 time_t rrdset_last_entry_t(RRDSET *st);
1312 -
1305 +time_t rrdset_first_entry_t_of_tier(RRDSET *st, size_t tier);
1306 +time_t rrdset_first_entry_t(RRDSET *st);
1307 time_t rrdhost_last_entry_t(RRDHOST *h);
1308
1309 // ----------------------------------------------------------------------------
database/rrdcontext.c
+9
@@ -750,6 +750,11 @@ static void rrdinstance_free(RRDINSTANCE *ri) {
750 }
751
752 static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) {
753 + static STRING *ml_anomaly_rates_id = NULL;
754 +
755 + if(unlikely(!ml_anomaly_rates_id))
756 + ml_anomaly_rates_id = string_strdupz(ML_ANOMALY_RATES_CHART_ID);
757 +
758 RRDINSTANCE *ri = value;
759
760 // link it to its parent
@@ -776,6 +781,10 @@ static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unus
781 ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor
782 }
783
784 + // we need this when loading from SQL
785 + if(unlikely(ri->id == ml_anomaly_rates_id))
786 + ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
787 +
788 rrdmetrics_create_in_rrdinstance(ri);
789
790 // signal the react callback to do the job
database/rrddim.c
+3 -9
@@ -172,7 +172,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
172 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
173 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
174
175 - ml_dimension_new(rd);
175 + ml_new_dimension(rd);
176
177 ctr->react_action = RRDDIM_REACT_NEW;
178
@@ -191,7 +191,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
191
192 rrdcontext_removed_rrddim(rd);
193
194 - ml_dimension_delete(rd);
194 + ml_delete_dimension(rd);
195
196 debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
197
@@ -420,13 +420,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
420
421 // ----------------------------------------------------------------------------
422
423 -time_t rrddim_last_entry_t_of_tier(RRDDIM *rd, size_t tier) {
424 - if(unlikely(tier > storage_tiers || !rd->tiers[tier]))
425 - return 0;
426 -
427 - return rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle);
428 -}
429 -
423 +// get the timestamp of the last entry in the round-robin database
424 time_t rrddim_last_entry_t(RRDDIM *rd) {
425 time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
426
database/rrdhost.c
+3 -3
@@ -518,7 +518,7 @@ int is_legacy = 1;
518
519 rrdhost_load_rrdcontext_data(host);
520 if (!archived)
521 - ml_host_new(host);
521 + ml_new_host(host);
522 else
523 rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
524
@@ -629,7 +629,7 @@ void rrdhost_update(RRDHOST *host
629 host->rrdpush_replication_step = rrdpush_replication_step;
630
631 rrd_hosts_available++;
632 - ml_host_new(host);
632 + ml_new_host(host);
633 rrdhost_load_rrdcontext_data(host);
634 info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
635 }
@@ -1089,7 +1089,7 @@ void rrdhost_free(RRDHOST *host, bool force) {
1089 rrd_check_wrlock(); // make sure the RRDs are write locked
1090
1091 rrdhost_wrlock(host);
1092 - ml_host_delete(host);
1092 + ml_delete_host(host);
1093 rrdhost_unlock(host);
1094
1095 // ------------------------------------------------------------------------
database/rrdset.c
+3 -15
@@ -178,8 +178,6 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
178 st->red = NAN;
179
180 ctr->react_action = RRDSET_REACT_NEW;
181 -
182 - ml_chart_new(st);
181 }
182
183 // the destructor - the dictionary is write locked while this runs
@@ -234,9 +232,6 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
232 // 7. destroy the chart labels
233 rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
234
237 - // 8. destroy the ml handle
238 - ml_chart_delete(st);
239 -
235 rrdset_memory_file_free(st); // remove files of db mode save and map
236
237 // ------------------------------------------------------------------------
@@ -1258,8 +1253,6 @@ static inline size_t rrdset_done_interpolate(
1253
1254 last_ut = next_store_ut;
1255
1261 - ml_chart_update_begin(st);
1262 -
1256 struct rda_item *rda;
1257 size_t dim_id;
1258 for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
@@ -1339,11 +1332,8 @@ static inline size_t rrdset_done_interpolate(
1332 break;
1333 }
1334
1342 - time_t current_time = (time_t) (next_store_ut / USEC_PER_SEC);
1343 -
1335 if(unlikely(!store_this_entry)) {
1345 - (void) ml_is_anomalous(rd, current_time, 0, false);
1346 -
1336 + (void) ml_is_anomalous(rd, 0, false);
1337 rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1338 rrdcontext_collected_rrddim(rd);
1339 continue;
@@ -1352,7 +1342,7 @@ static inline size_t rrdset_done_interpolate(
1342 if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) {
1343 uint32_t dim_storage_flags = storage_flags;
1344
1355 - if (ml_is_anomalous(rd, current_time, new_value, true)) {
1345 + if (ml_is_anomalous(rd, new_value, true)) {
1346 // clear anomaly bit: 0 -> is anomalous, 1 -> not anomalous
1347 dim_storage_flags &= ~((storage_number)SN_FLAG_NOT_ANOMALOUS);
1348 }
@@ -1362,7 +1352,7 @@ static inline size_t rrdset_done_interpolate(
1352 rd->last_stored_value = new_value;
1353 }
1354 else {
1365 - (void) ml_is_anomalous(rd, current_time, 0, false);
1355 + (void) ml_is_anomalous(rd, 0, false);
1356
1357 rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
1358
@@ -1374,8 +1364,6 @@ static inline size_t rrdset_done_interpolate(
1364 stored_entries++;
1365 }
1366
1377 - ml_chart_update_end(st);
1378 -
1367 // reset the storage flags for the next point, if any;
1368 storage_flags = SN_DEFAULT_FLAGS;
1369
ml/ADCharts.cc
+109 -387
@@ -3,182 +3,55 @@
3 #include "ADCharts.h"
4 #include "Config.h"
5
6 -void ml::updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS) {
7 - /*
8 - * Machine learning status
9 - */
10 - {
11 - static thread_local RRDSET *MachineLearningStatusRS = nullptr;
12 -
13 - static thread_local RRDDIM *Enabled = nullptr;
14 - static thread_local RRDDIM *DisabledUE = nullptr;
15 - static thread_local RRDDIM *DisabledSP = nullptr;
16 -
17 - if (!MachineLearningStatusRS) {
18 - std::stringstream IdSS, NameSS;
19 -
20 - IdSS << "machine_learning_status_for_" << localhost->machine_guid;
21 - NameSS << "machine_learning_status_for_" << localhost->hostname;
22 -
23 - MachineLearningStatusRS = rrdset_create_localhost(
24 - "netdata", // type
25 - IdSS.str().c_str(), // id
26 - NameSS.str().c_str(), // name
27 - "ml", // family
28 - "netdata.machine_learning_status", // ctx
29 - "Machine learning status", // title
30 - "dimensions", // units
31 - "netdata", // plugin
32 - "ml", // module
33 - NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
34 - RH->rrd_update_every, // update_every
35 - RRDSET_TYPE_LINE // chart_type
36 - );
37 - rrdset_flag_set(MachineLearningStatusRS , RRDSET_FLAG_ANOMALY_DETECTION);
38 -
39 - Enabled = rrddim_add(MachineLearningStatusRS, "enabled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
40 - DisabledUE = rrddim_add(MachineLearningStatusRS, "disabled-ue", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
41 - DisabledSP = rrddim_add(MachineLearningStatusRS, "disabled-sp", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
42 - }
43 -
44 - rrddim_set_by_pointer(MachineLearningStatusRS, Enabled, MLS.NumMachineLearningStatusEnabled);
45 - rrddim_set_by_pointer(MachineLearningStatusRS, DisabledUE, MLS.NumMachineLearningStatusDisabledUE);
46 - rrddim_set_by_pointer(MachineLearningStatusRS, DisabledSP, MLS.NumMachineLearningStatusDisabledSP);
47 -
48 - rrdset_done(MachineLearningStatusRS);
49 - }
6 +void ml::updateDimensionsChart(RRDHOST *RH,
7 + collected_number NumTrainedDimensions,
8 + collected_number NumNormalDimensions,
9 + collected_number NumAnomalousDimensions) {
10 + static thread_local RRDSET *RS = nullptr;
11 + static thread_local RRDDIM *NumTotalDimensionsRD = nullptr;
12 + static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr;
13 + static thread_local RRDDIM *NumNormalDimensionsRD = nullptr;
14 + static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
15 +
16 + if (!RS) {
17 + std::stringstream IdSS, NameSS;
18
51 - /*
52 - * Metric type
53 - */
54 - {
55 - static thread_local RRDSET *MetricTypesRS = nullptr;
56 -
57 - static thread_local RRDDIM *Constant = nullptr;
58 - static thread_local RRDDIM *Variable = nullptr;
59 -
60 - if (!MetricTypesRS) {
61 - std::stringstream IdSS, NameSS;
62 -
63 - IdSS << "metric_types_for_" << localhost->machine_guid;
64 - NameSS << "metric_types_for_" << localhost->hostname;
65 -
66 - MetricTypesRS = rrdset_create_localhost(
67 - "netdata", // type
68 - IdSS.str().c_str(), // id
69 - NameSS.str().c_str(), // name
70 - "ml", // family
71 - "netdata.metric_types", // ctx
72 - "Dimensions by metric type", // title
73 - "dimensions", // units
74 - "netdata", // plugin
75 - "ml", // module
76 - NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
77 - RH->rrd_update_every, // update_every
78 - RRDSET_TYPE_LINE // chart_type
79 - );
80 - rrdset_flag_set(MetricTypesRS, RRDSET_FLAG_ANOMALY_DETECTION);
81 -
82 - Constant = rrddim_add(MetricTypesRS, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
83 - Variable = rrddim_add(MetricTypesRS, "variable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
84 - }
85 -
86 - rrddim_set_by_pointer(MetricTypesRS, Constant, MLS.NumMetricTypeConstant);
87 - rrddim_set_by_pointer(MetricTypesRS, Variable, MLS.NumMetricTypeVariable);
88 -
89 - rrdset_done(MetricTypesRS);
90 - }
19 + IdSS << "dimensions_on_" << localhost->machine_guid;
20 + NameSS << "dimensions_on_" << localhost->hostname;
21
92 - /*
93 - * Training status
94 - */
95 - {
96 - static thread_local RRDSET *TrainingStatusRS = nullptr;
97 -
98 - static thread_local RRDDIM *Untrained = nullptr;
99 - static thread_local RRDDIM *PendingWithoutModel = nullptr;
100 - static thread_local RRDDIM *Trained = nullptr;
101 - static thread_local RRDDIM *PendingWithModel = nullptr;
102 -
103 - if (!TrainingStatusRS) {
104 - std::stringstream IdSS, NameSS;
105 -
106 - IdSS << "training_status_for_" << localhost->machine_guid;
107 - NameSS << "training_status_for_" << localhost->hostname;
108 -
109 - TrainingStatusRS = rrdset_create_localhost(
110 - "netdata", // type
111 - IdSS.str().c_str(), // id
112 - NameSS.str().c_str(), // name
113 - "ml", // family
114 - "netdata.training_status", // ctx
115 - "Training status of dimensions", // title
116 - "dimensions", // units
117 - "netdata", // plugin
118 - "ml", // module
119 - NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
120 - RH->rrd_update_every, // update_every
121 - RRDSET_TYPE_LINE // chart_type
122 - );
123 -
124 - rrdset_flag_set(TrainingStatusRS, RRDSET_FLAG_ANOMALY_DETECTION);
125 -
126 - Untrained = rrddim_add(TrainingStatusRS, "untrained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
127 - PendingWithoutModel = rrddim_add(TrainingStatusRS, "pending-without-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
128 - Trained = rrddim_add(TrainingStatusRS, "trained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
129 - PendingWithModel = rrddim_add(TrainingStatusRS, "pending-with-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
130 - }
131 -
132 - rrddim_set_by_pointer(TrainingStatusRS, Untrained, MLS.NumTrainingStatusUntrained);
133 - rrddim_set_by_pointer(TrainingStatusRS, PendingWithoutModel, MLS.NumTrainingStatusPendingWithoutModel);
134 - rrddim_set_by_pointer(TrainingStatusRS, Trained, MLS.NumTrainingStatusTrained);
135 - rrddim_set_by_pointer(TrainingStatusRS, PendingWithModel, MLS.NumTrainingStatusPendingWithModel);
136 -
137 - rrdset_done(TrainingStatusRS);
22 + RS = rrdset_create(
23 + RH,
24 + "anomaly_detection", // type
25 + IdSS.str().c_str(), // id
26 + NameSS.str().c_str(), // name
27 + "dimensions", // family
28 + "anomaly_detection.dimensions", // ctx
29 + "Anomaly detection dimensions", // title
30 + "dimensions", // units
31 + "netdata", // plugin
32 + "ml", // module
33 + 39183, // priority
34 + RH->rrd_update_every, // update_every
35 + RRDSET_TYPE_LINE // chart_type
36 + );
37 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
38 +
39 + NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
40 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
41 + NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL,
42 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
43 + NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL,
44 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
45 + NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL,
46 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
47 }
48
140 - /*
141 - * Prediction status
142 - */
143 - {
144 - static thread_local RRDSET *PredictionRS = nullptr;
145 -
146 - static thread_local RRDDIM *Anomalous = nullptr;
147 - static thread_local RRDDIM *Normal = nullptr;
148 -
149 - if (!PredictionRS) {
150 - std::stringstream IdSS, NameSS;
151 -
152 - IdSS << "dimensions_on_" << localhost->machine_guid;
153 - NameSS << "dimensions_on_" << localhost->hostname;
154 -
155 - PredictionRS = rrdset_create(
156 - RH,
157 - "anomaly_detection", // type
158 - IdSS.str().c_str(), // id
159 - NameSS.str().c_str(), // name
160 - "dimensions", // family
161 - "anomaly_detection.dimensions", // ctx
162 - "Anomaly detection dimensions", // title
163 - "dimensions", // units
164 - "netdata", // plugin
165 - "ml", // module
166 - ML_CHART_PRIO_DIMENSIONS, // priority
167 - RH->rrd_update_every, // update_every
168 - RRDSET_TYPE_LINE // chart_type
169 - );
170 - rrdset_flag_set(PredictionRS, RRDSET_FLAG_ANOMALY_DETECTION);
171 -
172 - Anomalous = rrddim_add(PredictionRS, "anomalous", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
173 - Normal = rrddim_add(PredictionRS, "normal", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
174 - }
175 -
176 - rrddim_set_by_pointer(PredictionRS, Anomalous, MLS.NumAnomalousDimensions);
177 - rrddim_set_by_pointer(PredictionRS, Normal, MLS.NumNormalDimensions);
178 -
179 - rrdset_done(PredictionRS);
180 - }
49 + rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions);
50 + rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions);
51 + rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions);
52 + rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions);
53
54 + rrdset_done(RS);
55 }
56
57 void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) {
@@ -202,7 +75,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
75 "percentage", // units
76 "netdata", // plugin
77 "ml", // module
205 - ML_CHART_PRIO_ANOMALY_RATE, // priority
78 + 39184, // priority
79 RH->rrd_update_every, // update_every
80 RRDSET_TYPE_LINE // chart_type
81 );
@@ -236,7 +109,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
109 "percentage", // units
110 "netdata", // plugin
111 "ml", // module
239 - ML_CHART_PRIO_DETECTOR_EVENTS, // priority
112 + 39185, // priority
113 RH->rrd_update_every, // update_every
114 RRDSET_TYPE_LINE // chart_type
115 );
@@ -270,7 +143,6 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
143 0, /* tier */
144 QUERY_SOURCE_ML
145 );
273 -
146 if(R) {
147 assert(R->d == 1 && R->n == 1 && R->rows == 1);
148
@@ -285,227 +157,77 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
157
158 rrdr_free(OWA, R);
159 }
288 -
160 onewayalloc_destroy(OWA);
161 }
162
292 -void ml::updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU) {
293 - /*
294 - * prediction rusage
295 - */
296 - {
297 - static thread_local RRDSET *RS = nullptr;
298 -
299 - static thread_local RRDDIM *User = nullptr;
300 - static thread_local RRDDIM *System = nullptr;
301 -
302 - if (!RS) {
303 - std::stringstream IdSS, NameSS;
304 -
305 - IdSS << "prediction_usage_for_" << localhost->machine_guid;
306 - NameSS << "prediction_usage_for_" << localhost->hostname;
307 -
308 - RS = rrdset_create_localhost(
309 - "netdata", // type
310 - IdSS.str().c_str(), // id
311 - NameSS.str().c_str(), // name
312 - "ml", // family
313 - "netdata.prediction_usage", // ctx
314 - "Prediction resource usage", // title
315 - "milliseconds/s", // units
316 - "netdata", // plugin
317 - "ml", // module
318 - NETDATA_ML_CHART_PRIO_PREDICTION_USAGE, // priority
319 - RH->rrd_update_every, // update_every
320 - RRDSET_TYPE_STACKED // chart_type
321 - );
322 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
323 -
324 - User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
325 - System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
326 - }
327 -
328 - rrddim_set_by_pointer(RS, User, PredictionRU.ru_utime.tv_sec * 1000000ULL + PredictionRU.ru_utime.tv_usec);
329 - rrddim_set_by_pointer(RS, System, PredictionRU.ru_stime.tv_sec * 1000000ULL + PredictionRU.ru_stime.tv_usec);
330 -
331 - rrdset_done(RS);
332 - }
163 +void ml::updateDetectionChart(RRDHOST *RH) {
164 + static thread_local RRDSET *RS = nullptr;
165 + static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
166
334 - /*
335 - * training rusage
336 - */
337 - {
338 - static thread_local RRDSET *RS = nullptr;
339 -
340 - static thread_local RRDDIM *User = nullptr;
341 - static thread_local RRDDIM *System = nullptr;
342 -
343 - if (!RS) {
344 - std::stringstream IdSS, NameSS;
345 -
346 - IdSS << "training_usage_for_" << localhost->machine_guid;
347 - NameSS << "training_usage_for_" << localhost->hostname;
348 -
349 - RS = rrdset_create_localhost(
350 - "netdata", // type
351 - IdSS.str().c_str(), // id
352 - NameSS.str().c_str(), // name
353 - "ml", // family
354 - "netdata.training_usage", // ctx
355 - "Training resource usage", // title
356 - "milliseconds/s", // units
357 - "netdata", // plugin
358 - "ml", // module
359 - NETDATA_ML_CHART_PRIO_TRAINING_USAGE, // priority
360 - RH->rrd_update_every, // update_every
361 - RRDSET_TYPE_STACKED // chart_type
362 - );
363 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
364 -
365 - User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
366 - System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
367 - }
368 -
369 - rrddim_set_by_pointer(RS, User, TrainingRU.ru_utime.tv_sec * 1000000ULL + TrainingRU.ru_utime.tv_usec);
370 - rrddim_set_by_pointer(RS, System, TrainingRU.ru_stime.tv_sec * 1000000ULL + TrainingRU.ru_stime.tv_usec);
371 -
372 - rrdset_done(RS);
167 + if (!RS) {
168 + std::stringstream IdSS, NameSS;
169 +
170 + IdSS << "prediction_stats_" << RH->machine_guid;
171 + NameSS << "prediction_stats_for_" << RH->hostname;
172 +
173 + RS = rrdset_create_localhost(
174 + "netdata", // type
175 + IdSS.str().c_str(), // id
176 + NameSS.str().c_str(), // name
177 + "ml", // family
178 + "netdata.prediction_stats", // ctx
179 + "Prediction thread CPU usage", // title
180 + "milliseconds/s", // units
181 + "netdata", // plugin
182 + "ml", // module
183 + 136000, // priority
184 + RH->rrd_update_every, // update_every
185 + RRDSET_TYPE_STACKED // chart_type
186 + );
187 +
188 + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
189 + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
190 }
191 +
192 + struct rusage TRU;
193 + getrusage(RUSAGE_THREAD, &TRU);
194 +
195 + rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);
196 + rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);
197 + rrdset_done(RS);
198 }
199
376 -void ml::updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS) {
377 - /*
378 - * queue stats
379 - */
380 - {
381 - static thread_local RRDSET *RS = nullptr;
382 -
383 - static thread_local RRDDIM *QueueSize = nullptr;
384 - static thread_local RRDDIM *PoppedItems = nullptr;
385 -
386 - if (!RS) {
387 - std::stringstream IdSS, NameSS;
388 -
389 - IdSS << "queue_stats_for_" << localhost->machine_guid;
390 - NameSS << "queue_stats_for_" << localhost->hostname;
391 -
392 - RS = rrdset_create_localhost(
393 - "netdata", // type
394 - IdSS.str().c_str(), // id
395 - NameSS.str().c_str(), // name
396 - "ml", // family
397 - "netdata.queue_stats", // ctx
398 - "Training queue stats", // title
399 - "items", // units
400 - "netdata", // plugin
401 - "ml", // module
402 - NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
403 - RH->rrd_update_every, // update_every
404 - RRDSET_TYPE_LINE// chart_type
405 - );
406 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
407 -
408 - QueueSize = rrddim_add(RS, "queue_size", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
409 - PoppedItems = rrddim_add(RS, "popped_items", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
410 - }
411 -
412 - rrddim_set_by_pointer(RS, QueueSize, TS.QueueSize);
413 - rrddim_set_by_pointer(RS, PoppedItems, TS.NumPoppedItems);
414 -
415 - rrdset_done(RS);
416 - }
200 +void ml::updateTrainingChart(RRDHOST *RH, struct rusage *TRU) {
201 + static thread_local RRDSET *RS = nullptr;
202 + static thread_local RRDDIM *UserRD = nullptr;
203 + static thread_local RRDDIM *SystemRD = nullptr;
204
418 - /*
419 - * training stats
420 - */
421 - {
422 - static thread_local RRDSET *RS = nullptr;
423 -
424 - static thread_local RRDDIM *Allotted = nullptr;
425 - static thread_local RRDDIM *Consumed = nullptr;
426 - static thread_local RRDDIM *Remaining = nullptr;
427 -
428 - if (!RS) {
429 - std::stringstream IdSS, NameSS;
430 -
431 - IdSS << "training_time_stats_for_" << localhost->machine_guid;
432 - NameSS << "training_time_stats_for_" << localhost->hostname;
433 -
434 - RS = rrdset_create_localhost(
435 - "netdata", // type
436 - IdSS.str().c_str(), // id
437 - NameSS.str().c_str(), // name
438 - "ml", // family
439 - "netdata.training_time_stats", // ctx
440 - "Training time stats", // title
441 - "milliseconds", // units
442 - "netdata", // plugin
443 - "ml", // module
444 - NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
445 - RH->rrd_update_every, // update_every
446 - RRDSET_TYPE_LINE// chart_type
447 - );
448 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
449 -
450 - Allotted = rrddim_add(RS, "allotted", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
451 - Consumed = rrddim_add(RS, "consumed", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
452 - Remaining = rrddim_add(RS, "remaining", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
453 - }
454 -
455 - rrddim_set_by_pointer(RS, Allotted, TS.AllottedUT);
456 - rrddim_set_by_pointer(RS, Consumed, TS.ConsumedUT);
457 - rrddim_set_by_pointer(RS, Remaining, TS.RemainingUT);
458 -
459 - rrdset_done(RS);
460 - }
205 + if (!RS) {
206 + std::stringstream IdSS, NameSS;
207
462 - /*
463 - * training result stats
464 - */
465 - {
466 - static thread_local RRDSET *RS = nullptr;
467 -
468 - static thread_local RRDDIM *Ok = nullptr;
469 - static thread_local RRDDIM *InvalidQueryTimeRange = nullptr;
470 - static thread_local RRDDIM *NotEnoughCollectedValues = nullptr;
471 - static thread_local RRDDIM *NullAcquiredDimension = nullptr;
472 - static thread_local RRDDIM *ChartUnderReplication = nullptr;
473 -
474 - if (!RS) {
475 - std::stringstream IdSS, NameSS;
476 -
477 - IdSS << "training_results_for_" << localhost->machine_guid;
478 - NameSS << "training_results_for_" << localhost->hostname;
479 -
480 - RS = rrdset_create_localhost(
481 - "netdata", // type
482 - IdSS.str().c_str(), // id
483 - NameSS.str().c_str(), // name
484 - "ml", // family
485 - "netdata.training_results", // ctx
486 - "Training results", // title
487 - "events", // units
488 - "netdata", // plugin
489 - "ml", // module
490 - NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
491 - RH->rrd_update_every, // update_every
492 - RRDSET_TYPE_LINE// chart_type
493 - );
494 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
495 -
496 - Ok = rrddim_add(RS, "ok", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
497 - InvalidQueryTimeRange = rrddim_add(RS, "invalid-queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
498 - NotEnoughCollectedValues = rrddim_add(RS, "not-enough-values", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
499 - NullAcquiredDimension = rrddim_add(RS, "null-acquired-dimensions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
500 - ChartUnderReplication = rrddim_add(RS, "chart-under-replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
501 - }
502 -
503 - rrddim_set_by_pointer(RS, Ok, TS.TrainingResultOk);
504 - rrddim_set_by_pointer(RS, InvalidQueryTimeRange, TS.TrainingResultInvalidQueryTimeRange);
505 - rrddim_set_by_pointer(RS, NotEnoughCollectedValues, TS.TrainingResultNotEnoughCollectedValues);
506 - rrddim_set_by_pointer(RS, NullAcquiredDimension, TS.TrainingResultNullAcquiredDimension);
507 - rrddim_set_by_pointer(RS, ChartUnderReplication, TS.TrainingResultChartUnderReplication);
508 -
509 - rrdset_done(RS);
208 + IdSS << "training_stats_" << RH->machine_guid;
209 + NameSS << "training_stats_for_" << RH->hostname;
210 +
211 + RS = rrdset_create_localhost(
212 + "netdata", // type
213 + IdSS.str().c_str(), // id
214 + NameSS.str().c_str(), // name
215 + "ml", // family
216 + "netdata.training_stats", // ctx
217 + "Training thread CPU usage", // title
218 + "milliseconds/s", // units
219 + "netdata", // plugin
220 + "ml", // module
221 + 136001, // priority
222 + RH->rrd_update_every, // update_every
223 + RRDSET_TYPE_STACKED // chart_type
224 + );
225 +
226 + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
227 + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
228 }
229 +
230 + rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);
231 + rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);
232 + rrdset_done(RS);
233 }
ml/ADCharts.h
+6 -4
@@ -3,18 +3,20 @@
3 #ifndef ML_ADCHARTS_H
4 #define ML_ADCHARTS_H
5
6 -#include "Stats.h"
6 #include "ml-private.h"
7
8 namespace ml {
9
11 -void updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS);
10 +void updateDimensionsChart(RRDHOST *RH,
11 + collected_number NumTrainedDimensions,
12 + collected_number NumNormalDimensions,
13 + collected_number NumAnomalousDimensions);
14
15 void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate);
16
15 -void updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU);
17 +void updateDetectionChart(RRDHOST *RH);
18
17 -void updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS);
19 +void updateTrainingChart(RRDHOST *RH, struct rusage *TRU);
20
21 } // namespace ml
22
ml/Chart.cc
ml/Chart.h deleted
-128
@@ -1,128 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_CHART_H
4 -#define ML_CHART_H
5 -
6 -#include "Config.h"
7 -#include "Dimension.h"
8 -
9 -#include "ml-private.h"
10 -#include "json/single_include/nlohmann/json.hpp"
11 -
12 -namespace ml
13 -{
14 -
15 -class Chart {
16 -public:
17 - Chart(RRDSET *RS) :
18 - RS(RS),
19 - MLS()
20 - { }
21 -
22 - RRDSET *getRS() const {
23 - return RS;
24 - }
25 -
26 - bool isAvailableForML() {
27 - return rrdset_is_available_for_exporting_and_alarms(RS);
28 - }
29 -
30 - void addDimension(Dimension *D) {
31 - std::lock_guard<std::mutex> Lock(Mutex);
32 - Dimensions[D->getRD()] = D;
33 - }
34 -
35 - void removeDimension(Dimension *D) {
36 - std::lock_guard<std::mutex> Lock(Mutex);
37 - Dimensions.erase(D->getRD());
38 - }
39 -
40 - void getModelsAsJson(nlohmann::json &Json) {
41 - std::lock_guard<std::mutex> Lock(Mutex);
42 -
43 - for (auto &DP : Dimensions) {
44 - Dimension *D = DP.second;
45 - nlohmann::json JsonArray = nlohmann::json::array();
46 - for (const KMeans &KM : D->getModels()) {
47 - nlohmann::json J;
48 - KM.toJson(J);
49 - JsonArray.push_back(J);
50 - }
51 -
52 - Json[getMLDimensionID(D->getRD())] = JsonArray;
53 - }
54 - }
55 -
56 - void updateBegin() {
57 - Mutex.lock();
58 - MLS = {};
59 - }
60 -
61 - void updateDimension(Dimension *D, bool IsAnomalous) {
62 - switch (D->getMLS()) {
63 - case MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
64 - MLS.NumMachineLearningStatusDisabledUE++;
65 - return;
66 - case MachineLearningStatus::DisabledDueToExcludedChart:
67 - MLS.NumMachineLearningStatusDisabledSP++;
68 - return;
69 - case MachineLearningStatus::Enabled: {
70 - MLS.NumMachineLearningStatusEnabled++;
71 -
72 - switch (D->getMT()) {
73 - case MetricType::Constant:
74 - MLS.NumMetricTypeConstant++;
75 - MLS.NumTrainingStatusTrained++;
76 - MLS.NumNormalDimensions++;
77 - return;
78 - case MetricType::Variable:
79 - MLS.NumMetricTypeVariable++;
80 - break;
81 - }
82 -
83 - switch (D->getTS()) {
84 - case TrainingStatus::Untrained:
85 - MLS.NumTrainingStatusUntrained++;
86 - return;
87 - case TrainingStatus::PendingWithoutModel:
88 - MLS.NumTrainingStatusPendingWithoutModel++;
89 - return;
90 - case TrainingStatus::Trained:
91 - MLS.NumTrainingStatusTrained++;
92 -
93 - MLS.NumAnomalousDimensions += IsAnomalous;
94 - MLS.NumNormalDimensions += !IsAnomalous;
95 - return;
96 - case TrainingStatus::PendingWithModel:
97 - MLS.NumTrainingStatusPendingWithModel++;
98 -
99 - MLS.NumAnomalousDimensions += IsAnomalous;
100 - MLS.NumNormalDimensions += !IsAnomalous;
101 - return;
102 - }
103 -
104 - return;
105 - }
106 - }
107 - }
108 -
109 - void updateEnd() {
110 - Mutex.unlock();
111 - }
112 -
113 - MachineLearningStats getMLS() {
114 - std::lock_guard<std::mutex> Lock(Mutex);
115 - return MLS;
116 - }
117 -
118 -private:
119 - RRDSET *RS;
120 - MachineLearningStats MLS;
121 -
122 - std::mutex Mutex;
123 - std::unordered_map<RRDDIM *, Dimension *> Dimensions;
124 -};
125 -
126 -} // namespace ml
127 -
128 -#endif /* ML_CHART_H */
ml/Config.cc
+3 -3
@@ -31,7 +31,7 @@ void Config::readMLConfig(void) {
31 unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
32 unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
33 unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
34 - unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1);
34 + unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24);
35
36 unsigned DiffN = config_get_number(ConfigSectionML, "num samples to diff", 1);
37 unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);
@@ -53,7 +53,7 @@ void Config::readMLConfig(void) {
53 MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
54 MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
55 TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
56 - NumModelsToUse = clamp<unsigned>(NumModelsToUse, 1, 7 * 24);
56 + NumModelsToUse = clamp<unsigned>(TrainEvery, 1, 7 * 24);
57
58 DiffN = clamp(DiffN, 0u, 1u);
59 SmoothN = clamp(SmoothN, 0u, 5u);
@@ -108,7 +108,7 @@ void Config::readMLConfig(void) {
108 // Always exclude anomaly_detection charts from training.
109 Cfg.ChartsToSkip = "anomaly_detection.* ";
110 Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");
111 - Cfg.SP_ChartsToSkip = simple_pattern_create(Cfg.ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
111 + Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
112
113 Cfg.StreamADCharts = config_get_boolean(ConfigSectionML, "stream anomaly detection charts", true);
114 }
ml/Config.h
-1
@@ -14,7 +14,6 @@ public:
14 unsigned MaxTrainSamples;
15 unsigned MinTrainSamples;
16 unsigned TrainEvery;
17 -
17 unsigned NumModelsToUse;
18
19 unsigned DBEngineAnomalyRateEvery;
ml/Dimension.cc
+59 -231
@@ -3,174 +3,84 @@
3 #include "Config.h"
4 #include "Dimension.h"
5 #include "Query.h"
6 -#include "Host.h"
6
7 using namespace ml;
8
10 -static const char *mls2str(MachineLearningStatus MLS) {
11 - switch (MLS) {
12 - case ml::MachineLearningStatus::Enabled:
13 - return "enabled";
14 - case ml::MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
15 - return "disabled-ue";
16 - case ml::MachineLearningStatus::DisabledDueToExcludedChart:
17 - return "disabled-sp";
18 - default:
19 - return "unknown";
20 - }
21 -}
22 -
23 -static const char *mt2str(MetricType MT) {
24 - switch (MT) {
25 - case ml::MetricType::Constant:
26 - return "constant";
27 - case ml::MetricType::Variable:
28 - return "variable";
29 - default:
30 - return "unknown";
31 - }
9 +bool Dimension::isActive() const {
10 + bool SetObsolete = rrdset_flag_check(RD->rrdset, RRDSET_FLAG_OBSOLETE);
11 + bool DimObsolete = rrddim_flag_check(RD, RRDDIM_FLAG_OBSOLETE);
12 + return !SetObsolete && !DimObsolete;
13 }
14
34 -static const char *ts2str(TrainingStatus TS) {
35 - switch (TS) {
36 - case ml::TrainingStatus::PendingWithModel:
37 - return "pending-with-model";
38 - case ml::TrainingStatus::PendingWithoutModel:
39 - return "pending-without-model";
40 - case ml::TrainingStatus::Trained:
41 - return "trained";
42 - case ml::TrainingStatus::Untrained:
43 - return "untrained";
44 - default:
45 - return "unknown";
46 - }
47 -}
48 -
49 -static const char *tr2str(TrainingResult TR) {
50 - switch (TR) {
51 - case ml::TrainingResult::Ok:
52 - return "ok";
53 - case ml::TrainingResult::InvalidQueryTimeRange:
54 - return "invalid-query";
55 - case ml::TrainingResult::NotEnoughCollectedValues:
56 - return "missing-values";
57 - case ml::TrainingResult::NullAcquiredDimension:
58 - return "null-acquired-dim";
59 - case ml::TrainingResult::ChartUnderReplication:
60 - return "chart-under-replication";
61 - default:
62 - return "unknown";
63 - }
64 -}
65 -
66 -std::pair<CalculatedNumber *, TrainingResponse> Dimension::getCalculatedNumbers(const TrainingRequest &TrainingReq) {
67 - TrainingResponse TrainingResp = {};
68 -
69 - TrainingResp.RequestTime = TrainingReq.RequestTime;
70 - TrainingResp.FirstEntryOnRequest = TrainingReq.FirstEntryOnRequest;
71 - TrainingResp.LastEntryOnRequest = TrainingReq.LastEntryOnRequest;
72 -
73 - TrainingResp.FirstEntryOnResponse = rrddim_first_entry_t_of_tier(RD, 0);
74 - TrainingResp.LastEntryOnResponse = rrddim_last_entry_t_of_tier(RD, 0);
75 -
15 +std::pair<CalculatedNumber *, size_t> Dimension::getCalculatedNumbers() {
16 size_t MinN = Cfg.MinTrainSamples;
17 size_t MaxN = Cfg.MaxTrainSamples;
18
19 // Figure out what our time window should be.
80 - TrainingResp.QueryBeforeT = TrainingResp.LastEntryOnResponse;
81 - TrainingResp.QueryAfterT = std::max(
82 - TrainingResp.QueryBeforeT - static_cast<time_t>((MaxN - 1) * updateEvery()),
83 - TrainingResp.FirstEntryOnResponse
84 - );
85 -
86 - if (TrainingResp.QueryAfterT >= TrainingResp.QueryBeforeT) {
87 - TrainingResp.Result = TrainingResult::InvalidQueryTimeRange;
88 - return { nullptr, TrainingResp };
89 - }
20 + time_t BeforeT = now_realtime_sec() - 1;
21 + time_t AfterT = BeforeT - (MaxN * updateEvery());
22
91 - if (rrdset_is_replicating(RD->rrdset)) {
92 - TrainingResp.Result = TrainingResult::ChartUnderReplication;
93 - return { nullptr, TrainingResp };
94 - }
23 + BeforeT -= (BeforeT % updateEvery());
24 + AfterT -= (AfterT % updateEvery());
25 +
26 + BeforeT = std::min(BeforeT, latestTime());
27 + AfterT = std::max(AfterT, oldestTime());
28 +
29 + if (AfterT >= BeforeT)
30 + return { nullptr, 0 };
31
32 CalculatedNumber *CNs = new CalculatedNumber[MaxN * (Cfg.LagN + 1)]();
33
34 // Start the query.
99 - size_t Idx = 0;
35 + unsigned Idx = 0;
36 + unsigned CollectedValues = 0;
37 + unsigned TotalValues = 0;
38
39 CalculatedNumber LastValue = std::numeric_limits<CalculatedNumber>::quiet_NaN();
40 Query Q = Query(getRD());
41
104 - Q.init(TrainingResp.QueryAfterT, TrainingResp.QueryBeforeT);
42 + Q.init(AfterT, BeforeT);
43 while (!Q.isFinished()) {
44 if (Idx == MaxN)
45 break;
46
47 auto P = Q.nextMetric();
110 -
48 CalculatedNumber Value = P.second;
49
50 if (netdata_double_isnumber(Value)) {
114 - if (!TrainingResp.DbAfterT)
115 - TrainingResp.DbAfterT = P.first;
116 - TrainingResp.DbBeforeT = P.first;
117 -
51 CNs[Idx] = Value;
52 LastValue = CNs[Idx];
120 - TrainingResp.CollectedValues++;
53 + CollectedValues++;
54 } else
55 CNs[Idx] = LastValue;
56
57 Idx++;
58 }
126 - TrainingResp.TotalValues = Idx;
127 -
128 - if (TrainingResp.CollectedValues < MinN) {
129 - TrainingResp.Result = TrainingResult::NotEnoughCollectedValues;
59 + TotalValues = Idx;
60
61 + if (CollectedValues < MinN) {
62 delete[] CNs;
132 - return { nullptr, TrainingResp };
63 + return { nullptr, 0 };
64 }
65
66 // Find first non-NaN value.
136 - for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TrainingResp.TotalValues--) { }
67 + for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TotalValues--) { }
68
69 // Overwrite NaN values.
70 if (Idx != 0)
140 - memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TrainingResp.TotalValues);
71 + memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TotalValues);
72
142 - TrainingResp.Result = TrainingResult::Ok;
143 - return { CNs, TrainingResp };
73 + return { CNs, TotalValues };
74 }
75
146 -TrainingResult Dimension::trainModel(const TrainingRequest &TrainingReq) {
147 - auto P = getCalculatedNumbers(TrainingReq);
76 +MLResult Dimension::trainModel() {
77 + auto P = getCalculatedNumbers();
78 CalculatedNumber *CNs = P.first;
149 - TrainingResponse TrainingResp = P.second;
150 -
151 - if (TrainingResp.Result != TrainingResult::Ok) {
152 - std::lock_guard<std::mutex> Lock(Mutex);
79 + unsigned N = P.second;
80
154 - MT = MetricType::Constant;
155 -
156 - switch (TS) {
157 - case TrainingStatus::PendingWithModel:
158 - TS = TrainingStatus::Trained;
159 - break;
160 - case TrainingStatus::PendingWithoutModel:
161 - TS = TrainingStatus::Untrained;
162 - break;
163 - default:
164 - break;
165 - }
81 + if (!CNs)
82 + return MLResult::MissingData;
83
167 - TR = TrainingResp;
168 -
169 - LastTrainingTime = TrainingResp.LastEntryOnResponse;
170 - return TrainingResp.Result;
171 - }
172 -
173 - unsigned N = TrainingResp.TotalValues;
84 unsigned TargetNumSamples = Cfg.MaxTrainSamples * Cfg.RandomSamplingRatio;
85 double SamplingRatio = std::min(static_cast<double>(TargetNumSamples) / N, 1.0);
86
@@ -183,81 +93,49 @@ TrainingResult Dimension::trainModel(const TrainingRequest &TrainingReq) {
93
94 {
95 std::lock_guard<std::mutex> Lock(Mutex);
186 -
187 - if (Models.size() < Cfg.NumModelsToUse) {
188 - Models.push_back(std::move(KM));
189 - } else {
190 - std::rotate(std::begin(Models), std::begin(Models) + 1, std::end(Models));
191 - Models[Models.size() - 1] = std::move(KM);
192 - }
193 -
194 - MT = MetricType::Constant;
195 - TS = TrainingStatus::Trained;
196 - TR = TrainingResp;
197 - LastTrainingTime = rrddim_last_entry_t(RD);
96 + Models[0] = KM;
97 }
98
99 + Trained = true;
100 + ConstantModel = true;
101 +
102 delete[] CNs;
201 - return TrainingResp.Result;
103 + return MLResult::Success;
104 }
105
204 -void Dimension::scheduleForTraining(time_t CurrT) {
205 - switch (MT) {
206 - case MetricType::Constant: {
207 - return;
208 - } default:
209 - break;
210 - }
106 +bool Dimension::shouldTrain(const TimePoint &TP) const {
107 + if (ConstantModel)
108 + return false;
109
212 - switch (TS) {
213 - case TrainingStatus::PendingWithModel:
214 - case TrainingStatus::PendingWithoutModel:
215 - break;
216 - case TrainingStatus::Untrained: {
217 - Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
218 - TS = TrainingStatus::PendingWithoutModel;
219 - H->scheduleForTraining(getTrainingRequest(CurrT));
220 - break;
221 - }
222 - case TrainingStatus::Trained: {
223 - bool NeedsTraining = LastTrainingTime + (Cfg.TrainEvery * updateEvery()) < CurrT;
224 -
225 - if (NeedsTraining) {
226 - Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
227 - TS = TrainingStatus::PendingWithModel;
228 - H->scheduleForTraining(getTrainingRequest(CurrT));
229 - }
230 - break;
231 - }
232 - }
110 + return (LastTrainedAt + Seconds(Cfg.TrainEvery * updateEvery())) < TP;
111 }
112
235 -bool Dimension::predict(time_t CurrT, CalculatedNumber Value, bool Exists) {
236 - // Nothing to do if ML is disabled for this dimension
237 - if (MLS != MachineLearningStatus::Enabled)
238 - return false;
239 -
240 - // Don't treat values that don't exist as anomalous
113 +bool Dimension::predict(CalculatedNumber Value, bool Exists) {
114 if (!Exists) {
115 CNs.clear();
116 + AnomalyBit = false;
117 return false;
118 }
119
246 - // Save the value and return if we don't have enough values for a sample
120 unsigned N = Cfg.DiffN + Cfg.SmoothN + Cfg.LagN;
121 if (CNs.size() < N) {
122 CNs.push_back(Value);
123 + AnomalyBit = false;
124 return false;
125 }
126
253 - // Push the value and check if it's different from the last one
254 - bool SameValue = true;
127 std::rotate(std::begin(CNs), std::begin(CNs) + 1, std::end(CNs));
128 +
129 if (CNs[N - 1] != Value)
257 - SameValue = false;
130 + ConstantModel = false;
131 +
132 CNs[N - 1] = Value;
133
260 - // Create the sample
134 + if (!isTrained() || ConstantModel) {
135 + AnomalyBit = false;
136 + return false;
137 + }
138 +
139 CalculatedNumber *TmpCNs = new CalculatedNumber[N * (Cfg.LagN + 1)]();
140 std::memcpy(TmpCNs, CNs.data(), N * sizeof(CalculatedNumber));
141 SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1,
@@ -266,80 +144,30 @@ bool Dimension::predict(time_t CurrT, CalculatedNumber Value, bool Exists) {
144 const DSample Sample = SB.preprocess().back();
145 delete[] TmpCNs;
146
269 - /*
270 - * Lock to predict and possibly schedule the dimension for training
271 - */
272 -
147 std::unique_lock<std::mutex> Lock(Mutex, std::defer_lock);
148 if (!Lock.try_lock()) {
149 + AnomalyBit = false;
150 return false;
151 }
152
278 - // Mark the metric time as variable if we received different values
279 - if (!SameValue)
280 - MT = MetricType::Variable;
281 -
282 - // Decide if the dimension needs to be scheduled for training
283 - scheduleForTraining(CurrT);
284 -
285 - // Nothing to do if we don't have a model
286 - switch (TS) {
287 - case TrainingStatus::Untrained:
288 - case TrainingStatus::PendingWithoutModel:
289 - return false;
290 - default:
291 - break;
292 - }
293 -
294 - /*
295 - * Use the KMeans models to check if the value is anomalous
296 - */
297 -
298 - size_t ModelsConsulted = 0;
299 - size_t Sum = 0;
300 -
153 for (const auto &KM : Models) {
302 - ModelsConsulted++;
303 -
154 double AnomalyScore = KM.anomalyScore(Sample);
305 - if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN())
155 + if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN()) {
156 + AnomalyBit = false;
157 continue;
158 + }
159
160 if (AnomalyScore < (100 * Cfg.DimensionAnomalyScoreThreshold)) {
309 - global_statistics_ml_models_consulted(ModelsConsulted);
161 + AnomalyBit = false;
162 return false;
163 }
312 -
313 - Sum += 1;
164 }
165
316 - global_statistics_ml_models_consulted(ModelsConsulted);
317 - return Sum;
166 + AnomalyBit = true;
167 + return true;
168 }
169
320 -std::vector<KMeans> Dimension::getModels() {
170 +std::array<KMeans, 1> Dimension::getModels() {
171 std::unique_lock<std::mutex> Lock(Mutex);
172 return Models;
173 }
324 -
325 -void Dimension::dump() const {
326 - const char *ChartId = rrdset_id(RD->rrdset);
327 - const char *DimensionId = rrddim_id(RD);
328 -
329 - const char *MLS_Str = mls2str(MLS);
330 - const char *MT_Str = mt2str(MT);
331 - const char *TS_Str = ts2str(TS);
332 - const char *TR_Str = tr2str(TR.Result);
333 -
334 - const char *fmt =
335 - "[ML] %s.%s: MLS=%s, MT=%s, TS=%s, Result=%s, "
336 - "ReqTime=%ld, FEOReq=%ld, LEOReq=%ld, "
337 - "FEOResp=%ld, LEOResp=%ld, QTR=<%ld, %ld>, DBTR=<%ld, %ld>, Collected=%zu, Total=%zu";
338 -
339 - error(fmt,
340 - ChartId, DimensionId, MLS_Str, MT_Str, TS_Str, TR_Str,
341 - TR.RequestTime, TR.FirstEntryOnRequest, TR.LastEntryOnRequest,
342 - TR.FirstEntryOnResponse, TR.LastEntryOnResponse,
343 - TR.QueryAfterT, TR.QueryBeforeT, TR.DbAfterT, TR.DbBeforeT, TR.CollectedValues, TR.TotalValues
344 - );
345 -}
ml/Dimension.h
+36 -138
@@ -3,7 +3,6 @@
3 #ifndef ML_DIMENSION_H
4 #define ML_DIMENSION_H
5
6 -#include "Stats.h"
6 #include "Query.h"
7 #include "Config.h"
8
@@ -11,6 +10,12 @@
10
11 namespace ml {
12
13 +enum class MLResult {
14 + Success = 0,
15 + MissingData,
16 + NaN,
17 +};
18 +
19 static inline std::string getMLDimensionID(RRDDIM *RD) {
20 RRDSET *RS = RD->rrdset;
21
@@ -19,118 +24,16 @@ static inline std::string getMLDimensionID(RRDDIM *RD) {
24 return SS.str();
25 }
26
22 -enum class MachineLearningStatus {
23 - // Enable training/prediction
24 - Enabled,
25 -
26 - // Disable due to update every being different from the host's
27 - DisabledDueToUniqueUpdateEvery,
28 -
29 - // Disable because configuration pattern matches the chart's id
30 - DisabledDueToExcludedChart,
31 -};
32 -
33 -enum class TrainingStatus {
34 - // We don't have a model for this dimension
35 - Untrained,
36 -
37 - // Request for training sent, but we don't have any models yet
38 - PendingWithoutModel,
39 -
40 - // Request to update existing models sent
41 - PendingWithModel,
42 -
43 - // Have a valid, up-to-date model
44 - Trained,
45 -};
46 -
47 -enum class MetricType {
48 - // The dimension has constant values, no need to train
49 - Constant,
50 -
51 - // The dimension's values fluctuate, we need to generate a model
52 - Variable,
53 -};
54 -
55 -struct TrainingRequest {
56 - // Chart/dimension we want to train
57 - STRING *ChartId;
58 - STRING *DimensionId;
59 -
60 - // Creation time of request
61 - time_t RequestTime;
62 -
63 - // First/last entry of this dimension in DB
64 - // at the point the request was made
65 - time_t FirstEntryOnRequest;
66 - time_t LastEntryOnRequest;
67 -};
68 -
69 -void dumpTrainingRequest(const TrainingRequest &TrainingReq, const char *Prefix);
70 -
71 -enum TrainingResult {
72 - // We managed to create a KMeans model
73 - Ok,
74 - // Could not query DB with a correct time range
75 - InvalidQueryTimeRange,
76 - // Did not gather enough data from DB to run KMeans
77 - NotEnoughCollectedValues,
78 - // Acquired a null dimension
79 - NullAcquiredDimension,
80 - // Chart is under replication
81 - ChartUnderReplication,
82 -};
83 -
84 -struct TrainingResponse {
85 - // Time when the request for this response was made
86 - time_t RequestTime;
87 -
88 - // First/last entry of the dimension in DB when generating the request
89 - time_t FirstEntryOnRequest;
90 - time_t LastEntryOnRequest;
91 -
92 - // First/last entry of the dimension in DB when generating the response
93 - time_t FirstEntryOnResponse;
94 - time_t LastEntryOnResponse;
95 -
96 - // After/Before timestamps of our DB query
97 - time_t QueryAfterT;
98 - time_t QueryBeforeT;
99 -
100 - // Actual after/before returned by the DB query ops
101 - time_t DbAfterT;
102 - time_t DbBeforeT;
103 -
104 - // Number of doubles returned by the DB query
105 - size_t CollectedValues;
106 -
107 - // Number of values we return to the caller
108 - size_t TotalValues;
109 -
110 - // Result of training response
111 - TrainingResult Result;
112 -};
113 -
114 -void dumpTrainingResponse(const TrainingResponse &TrainingResp, const char *Prefix);
115 -
27 class Dimension {
28 public:
29 Dimension(RRDDIM *RD) :
30 RD(RD),
120 - MT(MetricType::Constant),
121 - TS(TrainingStatus::Untrained),
122 - TR(),
123 - LastTrainingTime(0)
124 - {
125 - if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RD->rrdset)))
126 - MLS = MachineLearningStatus::DisabledDueToExcludedChart;
127 - else if (RD->update_every != RD->rrdset->rrdhost->rrd_update_every)
128 - MLS = MachineLearningStatus::DisabledDueToUniqueUpdateEvery;
129 - else
130 - MLS = MachineLearningStatus::Enabled;
131 -
132 - Models.reserve(Cfg.NumModelsToUse);
133 - }
31 + LastTrainedAt(Seconds(0)),
32 + Trained(false),
33 + ConstantModel(false),
34 + AnomalyScore(0.0),
35 + AnomalyBit(0)
36 + { }
37
38 RRDDIM *getRD() const {
39 return RD;
@@ -140,54 +43,49 @@ public:
43 return RD->update_every;
44 }
45
143 - MetricType getMT() const {
144 - return MT;
46 + time_t latestTime() const {
47 + return Query(RD).latestTime();
48 }
49
147 - TrainingStatus getTS() const {
148 - return TS;
50 + time_t oldestTime() const {
51 + return Query(RD).oldestTime();
52 }
53
151 - MachineLearningStatus getMLS() const {
152 - return MLS;
54 + bool isTrained() const {
55 + return Trained;
56 }
57
155 - TrainingResult trainModel(const TrainingRequest &TR);
58 + bool isAnomalous() const {
59 + return AnomalyBit;
60 + }
61
157 - void scheduleForTraining(time_t CurrT);
62 + bool shouldTrain(const TimePoint &TP) const;
63
159 - bool predict(time_t CurrT, CalculatedNumber Value, bool Exists);
64 + bool isActive() const;
65
161 - std::vector<KMeans> getModels();
162 -
163 - void dump() const;
66 + MLResult trainModel();
67
165 -private:
166 - TrainingRequest getTrainingRequest(time_t CurrT) const {
167 - return TrainingRequest {
168 - string_dup(RD->rrdset->id),
169 - string_dup(RD->id),
170 - CurrT,
171 - rrddim_first_entry_t(RD),
172 - rrddim_last_entry_t(RD)
173 - };
174 - }
68 + bool predict(CalculatedNumber Value, bool Exists);
69 +
70 + std::pair<bool, double> detect(size_t WindowLength, bool Reset);
71 +
72 + std::array<KMeans, 1> getModels();
73
74 private:
177 - std::pair<CalculatedNumber *, TrainingResponse> getCalculatedNumbers(const TrainingRequest &TrainingReq);
75 + std::pair<CalculatedNumber *, size_t> getCalculatedNumbers();
76
77 public:
78 RRDDIM *RD;
181 - MetricType MT;
182 - TrainingStatus TS;
183 - TrainingResponse TR;
79
185 - time_t LastTrainingTime;
80 + TimePoint LastTrainedAt;
81 + std::atomic<bool> Trained;
82 + std::atomic<bool> ConstantModel;
83
187 - MachineLearningStatus MLS;
84 + CalculatedNumber AnomalyScore;
85 + std::atomic<bool> AnomalyBit;
86
87 std::vector<CalculatedNumber> CNs;
190 - std::vector<KMeans> Models;
88 + std::array<KMeans, 1> Models;
89 std::mutex Mutex;
90 };
91
ml/Host.cc
+158 -194
@@ -2,24 +2,42 @@
2
3 #include "Config.h"
4 #include "Host.h"
5 -#include "Queue.h"
5 #include "ADCharts.h"
6
7 #include "json/single_include/nlohmann/json.hpp"
8
9 using namespace ml;
10
12 -void Host::addChart(Chart *C) {
11 +void RrdHost::addDimension(Dimension *D) {
12 std::lock_guard<std::mutex> Lock(Mutex);
14 - Charts[C->getRS()] = C;
13 +
14 + DimensionsMap[D->getRD()] = D;
15 +
16 + // Default construct mutex for dimension
17 + LocksMap[D];
18 }
19
17 -void Host::removeChart(Chart *C) {
18 - std::lock_guard<std::mutex> Lock(Mutex);
19 - Charts.erase(C->getRS());
20 +void RrdHost::removeDimension(Dimension *D) {
21 + // Remove the dimension from the hosts map.
22 + {
23 + std::lock_guard<std::mutex> Lock(Mutex);
24 + DimensionsMap.erase(D->getRD());
25 + }
26 +
27 + // Delete the dimension by locking the mutex that protects it.
28 + {
29 + std::lock_guard<std::mutex> Lock(LocksMap[D]);
30 + delete D;
31 + }
32 +
33 + // Remove the lock entry for the deleted dimension.
34 + {
35 + std::lock_guard<std::mutex> Lock(Mutex);
36 + LocksMap.erase(D);
37 + }
38 }
39
22 -void Host::getConfigAsJson(nlohmann::json &Json) const {
40 +void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
41 Json["version"] = 1;
42
43 Json["enabled"] = Cfg.EnableAnomalyDetection;
@@ -45,247 +63,193 @@ void Host::getConfigAsJson(nlohmann::json &Json) const {
63 Json["charts-to-skip"] = Cfg.ChartsToSkip;
64 }
65
48 -void Host::getModelsAsJson(nlohmann::json &Json) {
66 +void TrainableHost::getModelsAsJson(nlohmann::json &Json) {
67 std::lock_guard<std::mutex> Lock(Mutex);
68
51 - for (auto &CP : Charts) {
52 - Chart *C = CP.second;
53 - C->getModelsAsJson(Json);
54 - }
55 -}
69 + for (auto &DP : DimensionsMap) {
70 + Dimension *D = DP.second;
71
57 -void Host::detectOnce() {
58 - MLS = {};
59 - MachineLearningStats MLSCopy = {};
60 - TrainingStats TSCopy = {};
72 + nlohmann::json JsonArray = nlohmann::json::array();
73 + for (const KMeans &KM : D->getModels()) {
74 + nlohmann::json J;
75 + KM.toJson(J);
76 + JsonArray.push_back(J);
77 + }
78 + Json[getMLDimensionID(D->getRD())] = JsonArray;
79 + }
80
62 - {
63 - std::lock_guard<std::mutex> Lock(Mutex);
81 + return;
82 +}
83
65 - /*
66 - * prediction/detection stats
67 - */
68 - for (auto &CP : Charts) {
69 - Chart *C = CP.second;
84 +std::pair<Dimension *, Duration<double>>
85 +TrainableHost::findDimensionToTrain(const TimePoint &NowTP) {
86 + std::lock_guard<std::mutex> Lock(Mutex);
87
71 - if (!C->isAvailableForML())
72 - continue;
88 + Duration<double> AllottedDuration = Duration<double>{Cfg.TrainEvery * updateEvery()} / (DimensionsMap.size() + 1);
89
74 - MachineLearningStats ChartMLS = C->getMLS();
90 + for (auto &DP : DimensionsMap) {
91 + Dimension *D = DP.second;
92
76 - MLS.NumMachineLearningStatusEnabled += ChartMLS.NumMachineLearningStatusEnabled;
77 - MLS.NumMachineLearningStatusDisabledUE += ChartMLS.NumMachineLearningStatusDisabledUE;
78 - MLS.NumMachineLearningStatusDisabledSP += ChartMLS.NumMachineLearningStatusDisabledSP;
93 + if (D->shouldTrain(NowTP)) {
94 + LocksMap[D].lock();
95 + return { D, AllottedDuration };
96 + }
97 + }
98
80 - MLS.NumMetricTypeConstant += ChartMLS.NumMetricTypeConstant;
81 - MLS.NumMetricTypeVariable += ChartMLS.NumMetricTypeVariable;
99 + return { nullptr, AllottedDuration };
100 +}
101
83 - MLS.NumTrainingStatusUntrained += ChartMLS.NumTrainingStatusUntrained;
84 - MLS.NumTrainingStatusPendingWithoutModel += ChartMLS.NumTrainingStatusPendingWithoutModel;
85 - MLS.NumTrainingStatusTrained += ChartMLS.NumTrainingStatusTrained;
86 - MLS.NumTrainingStatusPendingWithModel += ChartMLS.NumTrainingStatusPendingWithModel;
102 +void TrainableHost::trainDimension(Dimension *D, const TimePoint &NowTP) {
103 + if (D == nullptr)
104 + return;
105
88 - MLS.NumAnomalousDimensions += ChartMLS.NumAnomalousDimensions;
89 - MLS.NumNormalDimensions += ChartMLS.NumNormalDimensions;
90 - }
106 + D->LastTrainedAt = NowTP + Seconds{D->updateEvery()};
107 + D->trainModel();
108
92 - HostAnomalyRate = 0.0;
93 - size_t NumActiveDimensions = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
94 - if (NumActiveDimensions)
95 - HostAnomalyRate = static_cast<double>(MLS.NumAnomalousDimensions) / NumActiveDimensions;
109 + {
110 + std::lock_guard<std::mutex> Lock(Mutex);
111 + LocksMap[D].unlock();
112 + }
113 +}
114
97 - MLSCopy = MLS;
115 +void TrainableHost::train() {
116 + Duration<double> MaxSleepFor = Seconds{10 * updateEvery()};
117
99 - /*
100 - * training stats
101 - */
102 - TSCopy = TS;
118 + worker_register("MLTRAIN");
119 + worker_register_job_name(0, "dimensions");
120
104 - TS.QueueSize = 0;
105 - TS.NumPoppedItems = 0;
121 + worker_is_busy(0);
122 + while (!netdata_exit) {
123 + netdata_thread_testcancel();
124 + netdata_thread_disable_cancelability();
125
107 - TS.AllottedUT = 0;
108 - TS.ConsumedUT = 0;
109 - TS.RemainingUT = 0;
126 + updateResourceUsage();
127
111 - TS.TrainingResultOk = 0;
112 - TS.TrainingResultInvalidQueryTimeRange = 0;
113 - TS.TrainingResultNotEnoughCollectedValues = 0;
114 - TS.TrainingResultNullAcquiredDimension = 0;
115 - TS.TrainingResultChartUnderReplication = 0;
116 - }
128 + TimePoint NowTP = SteadyClock::now();
129
118 - // Calc the avg values
119 - if (TSCopy.NumPoppedItems) {
120 - TSCopy.QueueSize /= TSCopy.NumPoppedItems;
121 - TSCopy.AllottedUT /= TSCopy.NumPoppedItems;
122 - TSCopy.ConsumedUT /= TSCopy.NumPoppedItems;
123 - TSCopy.RemainingUT /= TSCopy.NumPoppedItems;
124 -
125 - TSCopy.TrainingResultOk /= TSCopy.NumPoppedItems;
126 - TSCopy.TrainingResultInvalidQueryTimeRange /= TSCopy.NumPoppedItems;
127 - TSCopy.TrainingResultNotEnoughCollectedValues /= TSCopy.NumPoppedItems;
128 - TSCopy.TrainingResultNullAcquiredDimension /= TSCopy.NumPoppedItems;
129 - TSCopy.TrainingResultChartUnderReplication /= TSCopy.NumPoppedItems;
130 - } else {
131 - TSCopy.QueueSize = 0;
132 - TSCopy.AllottedUT = 0;
133 - TSCopy.ConsumedUT = 0;
134 - TSCopy.RemainingUT = 0;
135 - }
130 + auto P = findDimensionToTrain(NowTP);
131 + trainDimension(P.first, NowTP);
132
137 - updateDimensionsChart(RH, MLSCopy);
138 - updateHostAndDetectionRateCharts(RH, HostAnomalyRate * 10000.0);
133 + netdata_thread_enable_cancelability();
134
140 - struct rusage PredictionRU;
141 - getrusage(RUSAGE_THREAD, &PredictionRU);
142 - updateResourceUsageCharts(RH, PredictionRU, TSCopy.TrainingRU);
135 + Duration<double> AllottedDuration = P.second;
136 + Duration<double> RealDuration = SteadyClock::now() - NowTP;
137
144 - updateTrainingStatisticsChart(RH, TSCopy);
145 -}
138 + Duration<double> SleepFor;
139 + if (RealDuration >= AllottedDuration)
140 + continue;
141
147 -class AcquiredDimension {
148 -public:
149 - static AcquiredDimension find(RRDHOST *RH, STRING *ChartId, STRING *DimensionId) {
150 - RRDDIM_ACQUIRED *AcqRD = nullptr;
151 - Dimension *D = nullptr;
152 -
153 - rrdhost_rdlock(RH);
154 - RRDSET *RS = rrdset_find(RH, string2str(ChartId));
155 - if (RS) {
156 - AcqRD = rrddim_find_and_acquire(RS, string2str(DimensionId));
157 - if (AcqRD) {
158 - RRDDIM *RD = rrddim_acquired_to_rrddim(AcqRD);
159 - if (RD)
160 - D = reinterpret_cast<Dimension *>(RD->ml_dimension);
161 - }
142 + worker_is_idle();
143 + SleepFor = std::min(AllottedDuration - RealDuration, MaxSleepFor);
144 + TimePoint Now = SteadyClock::now();
145 + auto Until = Now + SleepFor;
146 + while (Now < Until && !netdata_exit) {
147 + std::this_thread::sleep_for(std::chrono::milliseconds(1000));
148 + Now = SteadyClock::now();
149 }
163 - rrdhost_unlock(RH);
164 -
165 - return AcquiredDimension(AcqRD, D);
150 + worker_is_busy(0);
151 }
152 +}
153
168 -private:
169 - AcquiredDimension(RRDDIM_ACQUIRED *AcqRD, Dimension *D) : AcqRD(AcqRD), D(D) {}
154 +#define WORKER_JOB_DETECT_DIMENSION 0
155 +#define WORKER_JOB_UPDATE_DETECTION_CHART 1
156 +#define WORKER_JOB_UPDATE_ANOMALY_RATES 2
157 +#define WORKER_JOB_UPDATE_CHARTS 3
158
171 -public:
172 - TrainingResult train(const TrainingRequest &TR) {
173 - if (!D)
174 - return TrainingResult::NullAcquiredDimension;
159 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
160 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
161 +#endif
162
176 - return D->trainModel(TR);
177 - }
163 +void DetectableHost::detectOnce() {
164 + size_t NumAnomalousDimensions = 0;
165 + size_t NumNormalDimensions = 0;
166 + size_t NumTrainedDimensions = 0;
167 + size_t NumActiveDimensions = 0;
168
179 - ~AcquiredDimension() {
180 - if (AcqRD)
181 - rrddim_acquired_release(AcqRD);
182 - }
169 + {
170 + std::lock_guard<std::mutex> Lock(Mutex);
171
184 -private:
185 - RRDDIM_ACQUIRED *AcqRD;
186 - Dimension *D;
187 -};
172 + for (auto &DP : DimensionsMap) {
173 + worker_is_busy(WORKER_JOB_DETECT_DIMENSION);
174
189 -void Host::scheduleForTraining(TrainingRequest TR) {
190 - TrainingQueue.push(TR);
191 -}
175 + Dimension *D = DP.second;
176
193 -void Host::train() {
194 - while (!netdata_exit) {
195 - netdata_thread_disable_cancelability();
177 + if (!D->isActive())
178 + continue;
179
197 - auto P = TrainingQueue.pop();
198 - TrainingRequest TrainingReq = P.first;
199 - size_t Size = P.second;
200 -
201 - usec_t AllottedUT = (Cfg.TrainEvery * RH->rrd_update_every * USEC_PER_SEC) / Size;
202 - if (AllottedUT > USEC_PER_SEC)
203 - AllottedUT = USEC_PER_SEC;
204 -
205 - usec_t StartUT = now_realtime_usec();
206 - TrainingResult TrainingRes;
207 - {
208 - AcquiredDimension AcqDim = AcquiredDimension::find(RH, TrainingReq.ChartId, TrainingReq.DimensionId);
209 - TrainingRes = AcqDim.train(TrainingReq);
210 - string_freez(TrainingReq.ChartId);
211 - string_freez(TrainingReq.DimensionId);
212 - }
213 - usec_t ConsumedUT = now_realtime_usec() - StartUT;
214 -
215 - usec_t RemainingUT = 0;
216 - if (ConsumedUT < AllottedUT)
217 - RemainingUT = AllottedUT - ConsumedUT;
218 -
219 - {
220 - std::lock_guard<std::mutex> Lock(Mutex);
221 -
222 - if (TS.AllottedUT == 0) {
223 - struct rusage TRU;
224 - getrusage(RUSAGE_THREAD, &TRU);
225 - TS.TrainingRU = TRU;
226 - }
227 -
228 - TS.QueueSize += Size;
229 - TS.NumPoppedItems += 1;
230 -
231 - TS.AllottedUT += AllottedUT;
232 - TS.ConsumedUT += ConsumedUT;
233 - TS.RemainingUT += RemainingUT;
234 -
235 - switch (TrainingRes) {
236 - case TrainingResult::Ok:
237 - TS.TrainingResultOk += 1;
238 - break;
239 - case TrainingResult::InvalidQueryTimeRange:
240 - TS.TrainingResultInvalidQueryTimeRange += 1;
241 - break;
242 - case TrainingResult::NotEnoughCollectedValues:
243 - TS.TrainingResultNotEnoughCollectedValues += 1;
244 - break;
245 - case TrainingResult::NullAcquiredDimension:
246 - TS.TrainingResultNullAcquiredDimension += 1;
247 - break;
248 - case TrainingResult::ChartUnderReplication:
249 - TS.TrainingResultChartUnderReplication += 1;
250 - break;
251 - }
180 + NumActiveDimensions++;
181 + NumTrainedDimensions += D->isTrained();
182 +
183 + bool IsAnomalous = D->isAnomalous();
184 + if (IsAnomalous)
185 + NumAnomalousDimensions += 1;
186 }
187
254 - netdata_thread_enable_cancelability();
255 - std::this_thread::sleep_for(std::chrono::microseconds{RemainingUT});
188 + if (NumAnomalousDimensions)
189 + HostAnomalyRate = static_cast<double>(NumAnomalousDimensions) / NumActiveDimensions;
190 + else
191 + HostAnomalyRate = 0.0;
192 +
193 + NumNormalDimensions = NumActiveDimensions - NumAnomalousDimensions;
194 }
195 +
196 + this->NumAnomalousDimensions = NumAnomalousDimensions;
197 + this->NumNormalDimensions = NumNormalDimensions;
198 + this->NumTrainedDimensions = NumTrainedDimensions;
199 + this->NumActiveDimensions = NumActiveDimensions;
200 +
201 + worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
202 + updateDimensionsChart(getRH(), NumTrainedDimensions, NumNormalDimensions, NumAnomalousDimensions);
203 + updateHostAndDetectionRateCharts(getRH(), HostAnomalyRate * 10000.0);
204 +
205 + struct rusage TRU;
206 + getResourceUsage(&TRU);
207 + updateTrainingChart(getRH(), &TRU);
208 }
209
259 -void Host::detect() {
210 +void DetectableHost::detect() {
211 + worker_register("MLDETECT");
212 + worker_register_job_name(WORKER_JOB_DETECT_DIMENSION, "dimensions");
213 + worker_register_job_name(WORKER_JOB_UPDATE_DETECTION_CHART, "detection chart");
214 + worker_register_job_name(WORKER_JOB_UPDATE_ANOMALY_RATES, "anomaly rates");
215 + worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "charts");
216 +
217 + std::this_thread::sleep_for(Seconds{10});
218 +
219 heartbeat_t HB;
220 heartbeat_init(&HB);
221
222 while (!netdata_exit) {
264 - heartbeat_next(&HB, RH->rrd_update_every * USEC_PER_SEC);
223 + netdata_thread_testcancel();
224 + worker_is_idle();
225 + heartbeat_next(&HB, updateEvery() * USEC_PER_SEC);
226
227 netdata_thread_disable_cancelability();
228 detectOnce();
229 +
230 + worker_is_busy(WORKER_JOB_UPDATE_DETECTION_CHART);
231 + updateDetectionChart(getRH());
232 netdata_thread_enable_cancelability();
233 }
234 }
235
272 -void Host::getDetectionInfoAsJson(nlohmann::json &Json) const {
236 +void DetectableHost::getDetectionInfoAsJson(nlohmann::json &Json) const {
237 Json["version"] = 1;
274 - Json["anomalous-dimensions"] = MLS.NumAnomalousDimensions;
275 - Json["normal-dimensions"] = MLS.NumNormalDimensions;
276 - Json["total-dimensions"] = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
277 - Json["trained-dimensions"] = MLS.NumTrainingStatusTrained + MLS.NumTrainingStatusPendingWithModel;
238 + Json["anomalous-dimensions"] = NumAnomalousDimensions;
239 + Json["normal-dimensions"] = NumNormalDimensions;
240 + Json["total-dimensions"] = NumAnomalousDimensions + NumNormalDimensions;
241 + Json["trained-dimensions"] = NumTrainedDimensions;
242 }
243
280 -void Host::startAnomalyDetectionThreads() {
281 - TrainingThread = std::thread(&Host::train, this);
282 - DetectionThread = std::thread(&Host::detect, this);
244 +void DetectableHost::startAnomalyDetectionThreads() {
245 + TrainingThread = std::thread(&TrainableHost::train, this);
246 + DetectionThread = std::thread(&DetectableHost::detect, this);
247 }
248
285 -void Host::stopAnomalyDetectionThreads() {
249 +void DetectableHost::stopAnomalyDetectionThreads() {
250 netdata_thread_cancel(TrainingThread.native_handle());
287 - TrainingThread.join();
288 -
251 netdata_thread_cancel(DetectionThread.native_handle());
252 +
253 + TrainingThread.join();
254 DetectionThread.join();
255 }
ml/Host.h
+67 -26
@@ -5,55 +5,96 @@
5
6 #include "Config.h"
7 #include "Dimension.h"
8 -#include "Chart.h"
9 -#include "Queue.h"
8
9 #include "ml-private.h"
10 #include "json/single_include/nlohmann/json.hpp"
11
14 -namespace ml
15 -{
12 +namespace ml {
13
17 -class Host {
14 +class RrdHost {
15 public:
19 - Host(RRDHOST *RH) :
20 - RH(RH),
21 - MLS(),
22 - TS(),
23 - HostAnomalyRate(0.0)
24 - { }
16 + RrdHost(RRDHOST *RH) : RH(RH) {};
17
26 - void addChart(Chart *C);
27 - void removeChart(Chart *C);
18 + RRDHOST *getRH() { return RH; }
19 +
20 + unsigned updateEvery() { return RH->rrd_update_every; }
21 +
22 + std::string getUUID() {
23 + char S[UUID_STR_LEN];
24 + uuid_unparse_lower(RH->host_uuid, S);
25 + return S;
26 + }
27 +
28 + void addDimension(Dimension *D);
29 + void removeDimension(Dimension *D);
30
31 void getConfigAsJson(nlohmann::json &Json) const;
32 +
33 + virtual ~RrdHost() {};
34 +
35 +protected:
36 + RRDHOST *RH;
37 +
38 + // Protect dimension and lock maps
39 + std::mutex Mutex;
40 +
41 + std::unordered_map<RRDDIM *, Dimension *> DimensionsMap;
42 + std::unordered_map<Dimension *, std::mutex> LocksMap;
43 +};
44 +
45 +class TrainableHost : public RrdHost {
46 +public:
47 + TrainableHost(RRDHOST *RH) : RrdHost(RH) {}
48 +
49 + void train();
50 +
51 + void updateResourceUsage() {
52 + std::lock_guard<std::mutex> Lock(ResourceUsageMutex);
53 + getrusage(RUSAGE_THREAD, &ResourceUsage);
54 + }
55 +
56 + void getResourceUsage(struct rusage *RU) {
57 + std::lock_guard<std::mutex> Lock(ResourceUsageMutex);
58 + memcpy(RU, &ResourceUsage, sizeof(struct rusage));
59 + }
60 +
61 void getModelsAsJson(nlohmann::json &Json);
31 - void getDetectionInfoAsJson(nlohmann::json &Json) const;
62 +
63 +private:
64 + std::pair<Dimension *, Duration<double>> findDimensionToTrain(const TimePoint &NowTP);
65 + void trainDimension(Dimension *D, const TimePoint &NowTP);
66 +
67 + struct rusage ResourceUsage{};
68 + std::mutex ResourceUsageMutex;
69 +};
70 +
71 +class DetectableHost : public TrainableHost {
72 +public:
73 + DetectableHost(RRDHOST *RH) : TrainableHost(RH) {}
74
75 void startAnomalyDetectionThreads();
76 void stopAnomalyDetectionThreads();
77
36 - void scheduleForTraining(TrainingRequest TR);
37 - void train();
78 + void getDetectionInfoAsJson(nlohmann::json &Json) const;
79
80 +private:
81 void detect();
82 void detectOnce();
83
84 private:
43 - RRDHOST *RH;
44 - MachineLearningStats MLS;
45 - TrainingStats TS;
46 - CalculatedNumber HostAnomalyRate{0.0};
47 -
48 - Queue<TrainingRequest> TrainingQueue;
49 -
50 - std::mutex Mutex;
51 - std::unordered_map<RRDSET *, Chart *> Charts;
52 -
85 std::thread TrainingThread;
86 std::thread DetectionThread;
87 +
88 + CalculatedNumber HostAnomalyRate{0.0};
89 +
90 + size_t NumAnomalousDimensions{0};
91 + size_t NumNormalDimensions{0};
92 + size_t NumTrainedDimensions{0};
93 + size_t NumActiveDimensions{0};
94 };
95
96 +using Host = DetectableHost;
97 +
98 } // namespace ml
99
100 #endif /* ML_HOST_H */
ml/Query.h
+1 -1
@@ -40,7 +40,7 @@ public:
40 std::pair<time_t, CalculatedNumber> nextMetric() {
41 points_read++;
42 STORAGE_POINT sp = Ops->next_metric(&Handle);
43 - return { sp.end_time, sp.sum / sp.count };
43 + return { sp.start_time, sp.sum / sp.count };
44 }
45
46 private:
ml/Queue.h deleted
-37
@@ -1,37 +0,0 @@
1 -#ifndef QUEUE_H
2 -#define QUEUE_H
3 -
4 -#include <queue>
5 -#include <mutex>
6 -#include <condition_variable>
7 -
8 -template<typename T>
9 -class Queue {
10 -public:
11 - Queue(void) : Q(), Mutex(), CondVar() { }
12 -
13 - void push(T t) {
14 - std::lock_guard<std::mutex> Lock(Mutex);
15 - Q.push(t);
16 - CondVar.notify_one();
17 - }
18 -
19 - std::pair<T, size_t> pop(void) {
20 - std::unique_lock<std::mutex> Lock(Mutex);
21 - while (Q.empty())
22 - CondVar.wait(Lock);
23 -
24 - T V = Q.front();
25 - size_t Size = Q.size();
26 -
27 - Q.pop();
28 - return { V, Size };
29 - }
30 -
31 -private:
32 - std::queue<T> Q;
33 - std::mutex Mutex;
34 - std::condition_variable CondVar;
35 -};
36 -
37 -#endif /* QUEUE_H */
ml/README.md
+61 -15
@@ -8,7 +8,7 @@ keywords: [machine learning, anomaly detection, Netdata ML]
8
9 ## Overview
10
11 -As of [`v1.32.0`](https://github.com/netdata/netdata/releases/tag/v1.32.0), Netdata comes with ML powered [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) capabilities built into it and available to use out of the box, with zero configuration required (ML was enabled by default in `v1.35.0-29-nightly` in [this PR](https://github.com/netdata/netdata/pull/13158), previously it required a one line config change).
11 +As of [`v1.32.0`](https://github.com/netdata/netdata/releases/tag/v1.32.0), Netdata comes with some ML powered [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) capabilities built into it and available to use out of the box, with zero configuration required (ML was enabled by default in `v1.35.0-29-nightly` in [this PR](https://github.com/netdata/netdata/pull/13158), previously it required a one line config change).
12
13 🚧 **Note**: If you would like to get involved and help us with some feedback, email us at analytics-ml-team@netdata.cloud, comment on the [beta launch post](https://community.netdata.cloud/t/anomaly-advisor-beta-launch/2717) in the Netdata community, or come join us in the [🤖-ml-powered-monitoring](https://discord.gg/4eRSEUpJnc) channel of the Netdata discord.
14
@@ -99,7 +99,49 @@ An ["anomaly detector"](#anomaly-detector) looks at all anomaly bits of a node.
99
100 Essentially if the ["Node Anomaly Rate"](#node-anomaly-rate) (NAR) passes a defined threshold and stays above that threshold for a persistent amount of time, a "Node [Anomaly Event](#anomaly-event)" will be triggered.
101
102 -These anomaly events are currently exposed via the `new_anomaly_event` dimension on the `anomaly_detection.anomaly_detection` chart.
102 +These anomaly events are currently exposed via `/api/v1/anomaly_events`
103 +
104 +**Note**: Clicking the link below will likely return an empty list of `[]`. This is the response when no anomaly events exist in the specified range. The example response below is illustrative of what the response would be when one or more anomaly events exist within the range of `after` to `before`.
105 +
106 +https://london.my-netdata.io/api/v1/anomaly_events?after=1638365182000&before=1638365602000
107 +
108 +If an event exists within the window, the result would be a list of start and end times.
109 +
110 +```
111 +[
112 + [
113 + 1638367788,
114 + 1638367851
115 + ]
116 +]
117 +```
118 +
119 +Information about each anomaly event can then be found at the `/api/v1/anomaly_event_info` endpoint (making sure to pass the `after` and `before` params):
120 +
121 +**Note**: If you click the below url you will get a `null` since no such anomaly event exists as the response is just an illustrative example taken from a node that did have such an anomaly event.
122 +
123 +https://london.my-netdata.io/api/v1/anomaly_event_info?after=1638367788&before=1638367851
124 +
125 +```
126 +[
127 + [
128 + 0.66,
129 + "netdata.response_time|max"
130 + ],
131 + [
132 + 0.63,
133 + "netdata.response_time|average"
134 + ],
135 + [
136 + 0.54,
137 + "netdata.requests|requests"
138 + ],
139 + ...
140 +```
141 +
142 +The query returns a list of dimension anomaly rates for all dimensions that were considered part of the detected anomaly event.
143 +
144 +**Note**: We plan to build additional anomaly detection and exploration features into both Netdata Agent and Netdata Cloud. The current endpoints are still under active development to power the upcoming features.
145
146 ## Configuration
147
@@ -120,7 +162,6 @@ Below is a list of all the available configuration params and their default valu
162 # maximum num samples to train = 14400
163 # minimum num samples to train = 3600
164 # train every = 3600
123 - # number of models per dimension = 1
165 # dbengine anomaly rate every = 30
166 # num samples to diff = 1
167 # num samples to smooth = 3
@@ -128,9 +169,12 @@ Below is a list of all the available configuration params and their default valu
169 # random sampling ratio = 0.2
170 # maximum number of k-means iterations = 1000
171 # dimension anomaly score threshold = 0.99
131 - # host anomaly rate threshold = 1.0
132 - # anomaly detection grouping method = average
133 - # anomaly detection grouping duration = 300
172 + # host anomaly rate threshold = 0.01000
173 + # minimum window size = 30.00000
174 + # maximum window size = 600.00000
175 + # idle window size = 30.00000
176 + # window minimum anomaly rate = 0.25000
177 + # anomaly event min dimension rate threshold = 0.05000
178 # hosts to skip from training = !*
179 # charts to skip from training = netdata.*
180 ```
@@ -177,7 +221,6 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
221 - `maximum num samples to train`: (`3600`/`86400`) This is the maximum amount of time you would like to train each model on. For example, the default of `14400` trains on the preceding 4 hours of data, assuming an `update every` of 1 second.
222 - `minimum num samples to train`: (`900`/`21600`) This is the minimum amount of data required to be able to train a model. For example, the default of `900` implies that once at least 15 minutes of data is available for training, a model is trained, otherwise it is skipped and checked again at the next training run.
223 - `train every`: (`1800`/`21600`) This is how often each model will be retrained. For example, the default of `3600` means that each model is retrained every hour. Note: The training of all models is spread out across the `train every` period for efficiency, so in reality, it means that each model will be trained in a staggered manner within each `train every` period.
180 -- `number of models per dimension`: (`1`/`168`) This is the number of trained models that will be used for scoring. For example the default `number of models per dimension = 1` means that just the most recently trained model (covering up to the most recent `maximum num samples to train` of training data) for the dimension will be used to determine the corresponding anomaly bit. Alternatively, if you have `train every = 3600` and `number of models per dimension = 24` this means that netdata will store and use the last 24 trained models for each dimension when determining the anomaly bit, this means that for the latest feature vector in this configuration to be considered anomalous it would need to look anomalous across _all_ the models trained for that dimension in the last 24 hours. As such, increasing `number of models per dimension` may reduce some false positives since it will result in more models (covering a wider time frame of training) being used during scoring.
224 - `dbengine anomaly rate every`: (`30`/`900`) This is how often netdata will aggregate all the anomaly bits into a single chart (`anomaly_detection.anomaly_rates`). The aggregation into a single chart allows enabling anomaly rate ranking over _all_ metrics with one API call as opposed to a call per chart.
225 - `num samples to diff`: (`0`/`1`) This is a `0` or `1` to determine if you want the model to operate on differences of the raw data or just the raw data. For example, the default of `1` means that we take differences of the raw values. Using differences is more general and works on dimensions that might naturally tend to have some trends or cycles in them that is normal behavior to which we don't want to be too sensitive.
226 - `num samples to smooth`: (`0`/`5`) This is a small integer that controls the amount of smoothing applied as part of the feature processing used by the model. For example, the default of `3` means that the rolling average of the last 3 values is used. Smoothing like this helps the model be a little more robust to spiky types of dimensions that naturally "jump" up or down as part of their normal behavior.
@@ -185,9 +228,11 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
228 - `random sampling ratio`: (`0.2`/`1.0`) This parameter determines how much of the available training data is randomly sampled when training a model. The default of `0.2` means that Netdata will train on a random 20% of training data. This parameter influences cost efficiency. At `0.2` the model is still reasonably trained while minimizing system overhead costs caused by the training.
229 - `maximum number of k-means iterations`: This is a parameter that can be passed to the model to limit the number of iterations in training the k-means model. Vast majority of cases can ignore and leave as default.
230 - `dimension anomaly score threshold`: (`0.01`/`5.00`) This is the threshold at which an individual dimension at a specific timestep is considered anomalous or not. For example, the default of `0.99` means that a dimension with an anomaly score of 99% or higher is flagged as anomalous. This is a normalized probability based on the training data, so the default of 99% means that anything that is as strange (based on distance measure) or more strange as the most strange 1% of data observed during training will be flagged as anomalous. If you wanted to make the anomaly detection on individual dimensions more sensitive you could try a value like `0.90` (90%) or to make it less sensitive you could try `1.5` (150%).
188 -- `host anomaly rate threshold`: (`0.1`/`10.0`) This is the percentage of dimensions (based on all those enabled for anomaly detection) that need to be considered anomalous at specific timestep for the host itself to be considered anomalous. For example, the default value of `1.0` means that if more than 1% of dimensions are anomalous at the same time then the host itself is considered in an anomalous state.
189 -- `anomaly detection grouping method`: The grouping method used when calculating node level anomaly rate.
190 -- `anomaly detection grouping duration`: (`60`/`900`) The duration across which to calculate the node level anomaly rate, the default of `900` means that the node level anomaly rate is calculated across a rolling 5 minute window.
231 +- `host anomaly rate threshold`: (`0.0`/`1.0`) This is the percentage of dimensions (based on all those enabled for anomaly detection) that need to be considered anomalous at specific timestep for the host itself to be considered anomalous. For example, the default value of `0.01` means that if more than 1% of dimensions are anomalous at the same time then the host itself is considered in an anomalous state.
232 +- `minimum window size`: The Netdata "Anomaly Detector" logic works over a rolling window of data. This parameter defines the minimum length of window to consider. If over this window the host is in an anomalous state then an anomaly detection event will be triggered. For example, the default of `30` means that the detector will initially work over a rolling window of 30 seconds. Note: The length of this window will be dynamic once an anomaly event has been triggered such that it will expand as needed until either the max length of an anomaly event is hit or the host settles back into a normal state with sufficiently decreased host level anomaly states in the rolling window. Note: If you wanted to adjust the higher level anomaly detector behavior then this is one parameter you might adjust to see the impact of on anomaly detection events.
233 +- `maximum window size`: This parameter defines the maximum length of window to consider. If an anomaly event reaches this size, it will be closed. This is to provide an upper bound on the length of an anomaly event and cost of the anomaly detector logic for that event.
234 +- `window minimum anomaly rate`: (`0.0`/`1.0`) This parameter corresponds to a threshold on the percentage of time in the rolling window that the host was considered in an anomalous state. For example, the default of `0.25` means that if the host is in an anomalous state for 25% of more of the rolling window then and anomaly event will be triggered or extended if one is already active. Note: If you want to make the anomaly detector itself less sensitive, you can adjust this value to something like `0.75` which would mean the host needs to be much more consistently in an anomalous state to trigger an anomaly detection event. Likewise, a lower value like `0.1` would make the anomaly detector more sensitive.
235 +- `anomaly event min dimension rate threshold`: (`0.0`/`1.0`) This is a parameter that helps filter out irrelevant dimensions from anomaly events. For example, the default of `0.05` means that only dimensions that were considered anomalous for at least 5% of the anomaly event itself will be included in that anomaly event. The idea here is to just include dimensions that were consistently anomalous as opposed to those that may have just randomly happened to be anomalous at the same time.
236 - `hosts to skip from training`: This parameter allows you to turn off anomaly detection for any child hosts on a parent host by defining those you would like to skip from training here. For example, a value like `dev-*` skips all hosts on a parent that begin with the "dev-" prefix. The default value of `!*` means "don't skip any".
237 - `charts to skip from training`: This parameter allows you to exclude certain charts from anomaly detection. By default, only netdata related charts are excluded. This is to avoid the scenario where accessing the netdata dashboard could itself tigger some anomalies if you don't access them regularly. If you want to include charts that are excluded by default, add them in small groups and then measure any impact on performance before adding additional ones. Example: If you want to include system, apps, and user charts:`!system.* !apps.* !user.* *`.
238
@@ -195,27 +240,28 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
240
241 Once enabled, the "Anomaly Detection" menu and charts will be available on the dashboard.
242
198 -![anomaly_detection_menu](https://user-images.githubusercontent.com/2178292/207584589-2e984786-5e01-404b-a20a-58573884d6df.png)
243 +![anomaly_detection_menu](https://user-images.githubusercontent.com/2178292/144255721-4568aabf-39c7-4855-bf1c-31b1d60e28e6.png)
244
245 In terms of anomaly detection, the most interesting charts would be the `anomaly_detection.dimensions` and `anomaly_detection.anomaly_rate` ones, which hold the `anomalous` and `anomaly_rate` dimensions that show the overall number of dimensions considered anomalous at any time and the corresponding anomaly rate.
246
247 - `anomaly_detection.dimensions`: Total count of dimensions considered anomalous or normal.
248 - `anomaly_detection.dimensions`: Percentage of anomalous dimensions.
204 -- `anomaly_detection.anomaly_detection`: Flags (0 or 1) to show when an anomaly event has been triggered by the detector.
249 +- `anomaly_detection.detector_window`: The length of the active window used by the detector.
250 +- `anomaly_detection.detector_events`: Flags (0 or 1) to show when an anomaly event has been triggered by the detector.
251
252 Below is an example of how these charts may look in the presence of an anomaly event.
253
254 Initially we see a jump in `anomalous` dimensions:
255
210 -![anomalous](https://user-images.githubusercontent.com/2178292/207589021-c0d2926f-bb55-4c5c-9e32-be1851558fa8.png)
256 +![anomalous](https://user-images.githubusercontent.com/2178292/144256036-c89fa768-5e5f-4278-9725-c67521c0d95e.png)
257
258 And a corresponding jump in the `anomaly_rate`:
259
214 -![anomaly_rate](https://user-images.githubusercontent.com/2178292/207589172-8853804b-6826-4731-8d06-b9e32d3071af.png)
260 +![anomaly_rate](https://user-images.githubusercontent.com/2178292/144256071-7d157438-31f3-4b23-a795-0fd3b2e2e85c.png)
261
262 After a short while the rolling node anomaly rate goes `above_threshold`, and once it stays above threshold for long enough a `new_anomaly_event` is created:
263
218 -![anomaly_event](https://user-images.githubusercontent.com/2178292/207589308-931a3c76-440a-48c1-970e-191743d26607.png)
264 +![anomaly_event](https://user-images.githubusercontent.com/2178292/144256152-910b06ec-26b8-45b4-bcb7-4c2acdf9af15.png)
265
266 ## Glossary
267
ml/SamplesBufferTests.cc new
+146
@@ -0,0 +1,146 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "ml/ml-private.h"
4 +#include <gtest/gtest.h>
5 +
6 +/*
7 + * The SamplesBuffer class implements the functionality of the following python
8 + * code:
9 + * >> df = pd.DataFrame(data=samples)
10 + * >> df = df.diff(diff_n).dropna()
11 + * >> df = df.rolling(smooth_n).mean().dropna()
12 + * >> df = pd.concat([df.shift(n) for n in range(lag_n + 1)], axis=1).dropna()
13 + *
14 + * Its correctness has been verified by automatically generating random
15 + * data frames in Python and comparing them with the correspondent preprocessed
16 + * SampleBuffers.
17 + *
18 + * The following tests are meant to catch unintended changes in the SamplesBuffer
19 + * implementation. For development purposes, one should compare changes against
20 + * the aforementioned python code.
21 +*/
22 +
23 +TEST(SamplesBufferTest, NS_8_NDPS_1_DN_1_SN_3_LN_1) {
24 + size_t NumSamples = 8, NumDimsPerSample = 1;
25 + size_t DiffN = 1, SmoothN = 3, LagN = 3;
26 +
27 + size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
28 + CalculatedNumber *CNs = new CalculatedNumber[N]();
29 +
30 + CNs[0] = 0.7568336679490107;
31 + CNs[1] = 0.4814406581763254;
32 + CNs[2] = 0.40073555156221874;
33 + CNs[3] = 0.5973257298194408;
34 + CNs[4] = 0.5334727814345868;
35 + CNs[5] = 0.2632477193454843;
36 + CNs[6] = 0.2684839023122384;
37 + CNs[7] = 0.851332948637479;
38 +
39 + std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
40 + SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
41 + SB.preprocess();
42 +
43 + std::vector<Sample> Samples = SB.getPreprocessedSamples();
44 + EXPECT_EQ(Samples.size(), 2);
45 +
46 + Sample S0 = Samples[0];
47 + const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
48 + Sample S1 = Samples[1];
49 + const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
50 +
51 + EXPECT_NEAR(S0_CNs[0], -0.109614, 0.001);
52 + EXPECT_NEAR(S0_CNs[1], -0.0458293, 0.001);
53 + EXPECT_NEAR(S0_CNs[2], 0.017344, 0.001);
54 + EXPECT_NEAR(S0_CNs[3], -0.0531693, 0.001);
55 +
56 + EXPECT_NEAR(S1_CNs[0], 0.105953, 0.001);
57 + EXPECT_NEAR(S1_CNs[1], -0.109614, 0.001);
58 + EXPECT_NEAR(S1_CNs[2], -0.0458293, 0.001);
59 + EXPECT_NEAR(S1_CNs[3], 0.017344, 0.001);
60 +
61 + delete[] CNs;
62 +}
63 +
64 +TEST(SamplesBufferTest, NS_8_NDPS_1_DN_2_SN_3_LN_2) {
65 + size_t NumSamples = 8, NumDimsPerSample = 1;
66 + size_t DiffN = 2, SmoothN = 3, LagN = 2;
67 +
68 + size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
69 + CalculatedNumber *CNs = new CalculatedNumber[N]();
70 +
71 + CNs[0] = 0.20511885291342846;
72 + CNs[1] = 0.13151717360306558;
73 + CNs[2] = 0.6017085062423134;
74 + CNs[3] = 0.46256882933941545;
75 + CNs[4] = 0.7887758447877941;
76 + CNs[5] = 0.9237989080034406;
77 + CNs[6] = 0.15552559051428083;
78 + CNs[7] = 0.6309750314597955;
79 +
80 + std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
81 + SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
82 + SB.preprocess();
83 +
84 + std::vector<Sample> Samples = SB.getPreprocessedSamples();
85 + EXPECT_EQ(Samples.size(), 2);
86 +
87 + Sample S0 = Samples[0];
88 + const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
89 + Sample S1 = Samples[1];
90 + const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
91 +
92 + EXPECT_NEAR(S0_CNs[0], 0.005016, 0.001);
93 + EXPECT_NEAR(S0_CNs[1], 0.326450, 0.001);
94 + EXPECT_NEAR(S0_CNs[2], 0.304903, 0.001);
95 +
96 + EXPECT_NEAR(S1_CNs[0], -0.154948, 0.001);
97 + EXPECT_NEAR(S1_CNs[1], 0.005016, 0.001);
98 + EXPECT_NEAR(S1_CNs[2], 0.326450, 0.001);
99 +
100 + delete[] CNs;
101 +}
102 +
103 +TEST(SamplesBufferTest, NS_8_NDPS_3_DN_2_SN_4_LN_1) {
104 + size_t NumSamples = 8, NumDimsPerSample = 3;
105 + size_t DiffN = 2, SmoothN = 4, LagN = 1;
106 +
107 + size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
108 + CalculatedNumber *CNs = new CalculatedNumber[N]();
109 +
110 + CNs[0] = 0.34310900399667765; CNs[1] = 0.14694315994488194; CNs[2] = 0.8246677800938796;
111 + CNs[3] = 0.48249504592307835; CNs[4] = 0.23241087965531182; CNs[5] = 0.9595348555892567;
112 + CNs[6] = 0.44281094035598334; CNs[7] = 0.5143142171362715; CNs[8] = 0.06391303014242555;
113 + CNs[9] = 0.7460491027783901; CNs[10] = 0.43887217459032923; CNs[11] = 0.2814395025355999;
114 + CNs[12] = 0.9231114281214198; CNs[13] = 0.326882401786898; CNs[14] = 0.26747939220376216;
115 + CNs[15] = 0.7787571209969636; CNs[16] =0.5851700001235088; CNs[17] = 0.34410728945321567;
116 + CNs[18] = 0.9394494507088997; CNs[19] =0.17567223681734334; CNs[20] = 0.42732886195446984;
117 + CNs[21] = 0.9460522396152958; CNs[22] =0.23462747016780894; CNs[23] = 0.35983249900892145;
118 +
119 + std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
120 + SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
121 + SB.preprocess();
122 +
123 + std::vector<Sample> Samples = SB.getPreprocessedSamples();
124 + EXPECT_EQ(Samples.size(), 2);
125 +
126 + Sample S0 = Samples[0];
127 + const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
128 + Sample S1 = Samples[1];
129 + const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
130 +
131 + EXPECT_NEAR(S0_CNs[0], 0.198225, 0.001);
132 + EXPECT_NEAR(S0_CNs[1], 0.003529, 0.001);
133 + EXPECT_NEAR(S0_CNs[2], -0.063003, 0.001);
134 + EXPECT_NEAR(S0_CNs[3], 0.219066, 0.001);
135 + EXPECT_NEAR(S0_CNs[4], 0.133175, 0.001);
136 + EXPECT_NEAR(S0_CNs[5], -0.293154, 0.001);
137 +
138 + EXPECT_NEAR(S1_CNs[0], 0.174160, 0.001);
139 + EXPECT_NEAR(S1_CNs[1], -0.135722, 0.001);
140 + EXPECT_NEAR(S1_CNs[2], 0.110452, 0.001);
141 + EXPECT_NEAR(S1_CNs[3], 0.198225, 0.001);
142 + EXPECT_NEAR(S1_CNs[4], 0.003529, 0.001);
143 + EXPECT_NEAR(S1_CNs[5], -0.063003, 0.001);
144 +
145 + delete[] CNs;
146 +}
ml/Stats.h deleted
-46
@@ -1,46 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_STATS_H
4 -#define ML_STATS_H
5 -
6 -#include "ml-private.h"
7 -
8 -namespace ml {
9 -
10 -struct MachineLearningStats {
11 - size_t NumMachineLearningStatusEnabled;
12 - size_t NumMachineLearningStatusDisabledUE;
13 - size_t NumMachineLearningStatusDisabledSP;
14 -
15 - size_t NumMetricTypeConstant;
16 - size_t NumMetricTypeVariable;
17 -
18 - size_t NumTrainingStatusUntrained;
19 - size_t NumTrainingStatusPendingWithoutModel;
20 - size_t NumTrainingStatusTrained;
21 - size_t NumTrainingStatusPendingWithModel;
22 -
23 - size_t NumAnomalousDimensions;
24 - size_t NumNormalDimensions;
25 -};
26 -
27 -struct TrainingStats {
28 - struct rusage TrainingRU;
29 -
30 - size_t QueueSize;
31 - size_t NumPoppedItems;
32 -
33 - usec_t AllottedUT;
34 - usec_t ConsumedUT;
35 - usec_t RemainingUT;
36 -
37 - size_t TrainingResultOk;
38 - size_t TrainingResultInvalidQueryTimeRange;
39 - size_t TrainingResultNotEnoughCollectedValues;
40 - size_t TrainingResultNullAcquiredDimension;
41 - size_t TrainingResultChartUnderReplication;
42 -};
43 -
44 -} // namespace ml
45 -
46 -#endif /* ML_STATS_H */
ml/ml-dummy.c
+8 -35
@@ -15,29 +15,9 @@ bool ml_enabled(RRDHOST *RH) {
15
16 void ml_init(void) {}
17
18 -void ml_host_new(RRDHOST *RH) {
19 - UNUSED(RH);
20 -}
21 -
22 -void ml_host_delete(RRDHOST *RH) {
23 - UNUSED(RH);
24 -}
25 -
26 -void ml_chart_new(RRDSET *RS) {
27 - UNUSED(RS);
28 -}
29 -
30 -void ml_chart_delete(RRDSET *RS) {
31 - UNUSED(RS);
32 -}
33 -
34 -void ml_dimension_new(RRDDIM *RD) {
35 - UNUSED(RD);
36 -}
18 +void ml_new_host(RRDHOST *RH) { (void) RH; }
19
38 -void ml_dimension_delete(RRDDIM *RD) {
39 - UNUSED(RD);
40 -}
20 +void ml_delete_host(RRDHOST *RH) { (void) RH; }
21
22 char *ml_get_host_info(RRDHOST *RH) {
23 (void) RH;
@@ -49,24 +29,17 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
29 return NULL;
30 }
31
52 -void ml_chart_update_begin(RRDSET *RS) {
53 - (void) RS;
54 -}
55 -
56 -void ml_chart_update_end(RRDSET *RS) {
57 - (void) RS;
58 -}
59 -
32 char *ml_get_host_models(RRDHOST *RH) {
33 (void) RH;
34 return NULL;
35 }
36
65 -bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
66 - (void) RD;
67 - (void) CurrT;
68 - (void) Value;
69 - (void) Exists;
37 +void ml_new_dimension(RRDDIM *RD) { (void) RD; }
38 +
39 +void ml_delete_dimension(RRDDIM *RD) { (void) RD; }
40 +
41 +bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
42 + (void) RD; (void) Value; (void) Exists;
43 return false;
44 }
45
ml/ml-private.h
+13
@@ -6,8 +6,21 @@
6 #include "KMeans.h"
7 #include "ml/ml.h"
8
9 +#include <chrono>
10 #include <map>
11 #include <mutex>
12 #include <sstream>
13
14 +namespace ml {
15 +
16 +using SteadyClock = std::chrono::steady_clock;
17 +using TimePoint = std::chrono::time_point<SteadyClock>;
18 +
19 +template<typename T>
20 +using Duration = std::chrono::duration<T>;
21 +
22 +using Seconds = std::chrono::seconds;
23 +
24 +} // namespace ml
25 +
26 #endif /* ML_PRIVATE_H */
ml/ml.cc
+38 -57
@@ -2,7 +2,6 @@
2
3 #include "Config.h"
4 #include "Dimension.h"
5 -#include "Chart.h"
5 #include "Host.h"
6
7 #include <random>
@@ -46,18 +45,18 @@ void ml_init(void) {
45 Cfg.RandomNums.push_back(Gen());
46 }
47
49 -void ml_host_new(RRDHOST *RH) {
48 +void ml_new_host(RRDHOST *RH) {
49 if (!ml_enabled(RH))
50 return;
51
52 Host *H = new Host(RH);
54 - RH->ml_host = reinterpret_cast<ml_host_t *>(H);
53 + RH->ml_host = static_cast<ml_host_t>(H);
54
55 H->startAnomalyDetectionThreads();
56 }
57
59 -void ml_host_delete(RRDHOST *RH) {
60 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
58 +void ml_delete_host(RRDHOST *RH) {
59 + Host *H = static_cast<Host *>(RH->ml_host);
60 if (!H)
61 return;
62
@@ -67,46 +66,34 @@ void ml_host_delete(RRDHOST *RH) {
66 RH->ml_host = nullptr;
67 }
68
70 -void ml_chart_new(RRDSET *RS) {
71 - Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
72 - if (!H)
73 - return;
74 -
75 - Chart *C = new Chart(RS);
76 - RS->ml_chart = reinterpret_cast<ml_chart_t *>(C);
69 +void ml_new_dimension(RRDDIM *RD) {
70 + RRDSET *RS = RD->rrdset;
71
78 - H->addChart(C);
79 -}
80 -
81 -void ml_chart_delete(RRDSET *RS) {
82 - Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
72 + Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host);
73 if (!H)
74 return;
75
86 - Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
87 - H->removeChart(C);
88 -
89 - delete C;
90 - RS->ml_chart = nullptr;
91 -}
76 + if (static_cast<unsigned>(RD->update_every) != H->updateEvery())
77 + return;
78
93 -void ml_dimension_new(RRDDIM *RD) {
94 - Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
95 - if (!C)
79 + if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RS)))
80 return;
81
82 Dimension *D = new Dimension(RD);
99 - RD->ml_dimension = reinterpret_cast<ml_dimension_t *>(D);
100 - C->addDimension(D);
83 + RD->ml_dimension = static_cast<ml_dimension_t>(D);
84 + H->addDimension(D);
85 }
86
103 -void ml_dimension_delete(RRDDIM *RD) {
104 - Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
87 +void ml_delete_dimension(RRDDIM *RD) {
88 + Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
89 if (!D)
90 return;
91
108 - Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
109 - C->removeDimension(D);
92 + Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host);
93 + if (!H)
94 + delete D;
95 + else
96 + H->removeDimension(D);
97
98 RD->ml_dimension = nullptr;
99 }
@@ -115,7 +102,7 @@ char *ml_get_host_info(RRDHOST *RH) {
102 nlohmann::json ConfigJson;
103
104 if (RH && RH->ml_host) {
118 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
105 + Host *H = static_cast<Host *>(RH->ml_host);
106 H->getConfigAsJson(ConfigJson);
107 } else {
108 ConfigJson["enabled"] = false;
@@ -128,7 +115,7 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
115 nlohmann::json ConfigJson;
116
117 if (RH && RH->ml_host) {
131 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
118 + Host *H = static_cast<Host *>(RH->ml_host);
119 H->getDetectionInfoAsJson(ConfigJson);
120 } else {
121 return nullptr;
@@ -141,7 +128,7 @@ char *ml_get_host_models(RRDHOST *RH) {
128 nlohmann::json ModelsJson;
129
130 if (RH && RH->ml_host) {
144 - Host *H = reinterpret_cast<Host *>(RH->ml_host);
131 + Host *H = static_cast<Host *>(RH->ml_host);
132 H->getModelsAsJson(ModelsJson);
133 return strdup(ModelsJson.dump(2, '\t').c_str());
134 }
@@ -149,36 +136,30 @@ char *ml_get_host_models(RRDHOST *RH) {
136 return nullptr;
137 }
138
152 -void ml_chart_update_begin(RRDSET *RS) {
153 - Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
154 - if (!C)
155 - return;
139 +bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
140 + Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
141 + if (!D)
142 + return false;
143
157 - C->updateBegin();
144 + return D->predict(Value, Exists);
145 }
146
160 -void ml_chart_update_end(RRDSET *RS) {
161 - Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
162 - if (!C)
163 - return;
164 -
165 - C->updateEnd();
147 +bool ml_streaming_enabled() {
148 + return Cfg.StreamADCharts;
149 }
150
168 -bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
169 - Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
170 - if (!D)
171 - return false;
151 +#if defined(ENABLE_ML_TESTS)
152
173 - Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
153 +#include "gtest/gtest.h"
154
175 - bool IsAnomalous = D->predict(CurrT, Value, Exists);
176 - C->updateDimension(D, IsAnomalous);
177 - return IsAnomalous;
178 -}
155 +int test_ml(int argc, char *argv[]) {
156 + (void) argc;
157 + (void) argv;
158
180 -bool ml_streaming_enabled() {
181 - return Cfg.StreamADCharts;
159 + ::testing::InitGoogleTest(&argc, argv);
160 + return RUN_ALL_TESTS();
161 }
162
163 +#endif // ENABLE_ML_TESTS
164 +
165 #include "ml-private.h"
ml/ml.h
+14 -11
@@ -14,32 +14,35 @@ extern "C" {
14 // the anomaly rate dimension, whenever its backing dimension is freed.
15 void rrddim_free(RRDSET *st, RRDDIM *rd);
16
17 +typedef void* ml_host_t;
18 +typedef void* ml_dimension_t;
19 +
20 bool ml_capable();
21
22 bool ml_enabled(RRDHOST *RH);
23
24 void ml_init(void);
25
23 -void ml_host_new(RRDHOST *RH);
24 -void ml_host_delete(RRDHOST *RH);
25 -
26 -void ml_chart_new(RRDSET *RS);
27 -void ml_chart_delete(RRDSET *RS);
28 -
29 -void ml_dimension_new(RRDDIM *RD);
30 -void ml_dimension_delete(RRDDIM *RD);
26 +void ml_new_host(RRDHOST *RH);
27 +void ml_delete_host(RRDHOST *RH);
28
29 char *ml_get_host_info(RRDHOST *RH);
30 char *ml_get_host_runtime_info(RRDHOST *RH);
31 char *ml_get_host_models(RRDHOST *RH);
32
36 -void ml_chart_update_begin(RRDSET *RS);
37 -void ml_chart_update_end(RRDSET *RS);
33 +void ml_new_dimension(RRDDIM *RD);
34 +void ml_delete_dimension(RRDDIM *RD);
35
39 -bool ml_is_anomalous(RRDDIM *RD, time_t curr_t, double value, bool exists);
36 +bool ml_is_anomalous(RRDDIM *RD, double value, bool exists);
37
38 bool ml_streaming_enabled();
39
40 +#define ML_ANOMALY_RATES_CHART_ID "anomaly_detection.anomaly_rates"
41 +
42 +#if defined(ENABLE_ML_TESTS)
43 +int test_ml(int argc, char *argv[]);
44 +#endif
45 +
46 #ifdef __cplusplus
47 };
48 #endif
netdata-installer.sh
+2
@@ -337,6 +337,8 @@ while [ -n "${1}" ]; do
337 NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-ml)}" | sed 's/$/ --disable-ml/g')"
338 NETDATA_ENABLE_ML=0
339 ;;
340 + "--enable-ml-tests") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-ml-tests)}" | sed 's/$/ --enable-ml-tests/g')" ;;
341 + "--disable-ml-tests") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-ml-tests)}" | sed 's/$/ --disable-ml-tests/g')" ;;
342 "--disable-lto") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-lto)}" | sed 's/$/ --disable-lto/g')" ;;
343 "--disable-x86-sse") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-x86-sse)}" | sed 's/$/ --disable-x86-sse/g')" ;;
344 "--disable-telemetry") NETDATA_DISABLE_TELEMETRY=1 ;;