Refactor ML code and add support for multiple KMeans models (#14198)
* Add profile.plugin Creates the specified number of charts/dimensions, and supports backfilling with pseudo-historical data. * Bump * Remove wrongly merged line. * Use the number of models specified from the config section. * Add option to consult all ML models. * Remove profiling option consuming all models. * Add underscore after chart name prefix. * prediction -> dimensions chart * reorder funcs * Split charts across types with correct priority * Ignore training request when chart is under replication. * Track global number of models consulted. * Cleanup config. * initial readme updates * fix readme * readme * Fix function definition when ML is disabled. * Add dummy ml_chart_update_{begin,end} * Remove profile_plugin * Define chart priorities under collectors/all.h * s/curr_t/current_time/ * Use libnetdata's lock/thread wrappers. * Fix autotools & cmake builds. * Delete ML dimensions & charts. * Let users of buffer preprocessing to handle memory. * Add separate API calls to start/stop ML threads. Co-authored-by: Andrew Maguire <andrewm4894@gmail.com>
vkalintiris committed
Jan 4, 2023 at 14:51 UTC
78359cd375d0b2c285741e6f934a681d0a0c3c15
35 files changed
+1608
-815
CMakeLists.txt
+6
-1
@@ -1030,12 +1030,17 @@ set(ML_FILES
1030
IF(ENABLE_ML)
1031
message(STATUS "ML: enabled")
1032
list(APPEND ML_FILES
1033
+ ml/ADCharts.h
1034
+ ml/ADCharts.cc
1035
+ ml/Chart.h
1036
+ ml/Chart.cc
1037
ml/Config.h
1038
ml/Config.cc
1035
- ml/Dimension.cc
1039
ml/Dimension.h
1040
+ ml/Dimension.cc
1041
ml/Host.h
1042
ml/Host.cc
1043
+ ml/Mutex.h
1044
ml/Query.h
1045
ml/KMeans.h
1046
ml/KMeans.cc
Makefile.am
+5
-13
@@ -236,10 +236,15 @@ ML_FILES += \
236
ml/ADCharts.cc \
237
ml/Config.h \
238
ml/Config.cc \
239
+ ml/Chart.cc \
240
+ ml/Chart.h \
241
+ ml/Stats.h \
242
ml/Dimension.cc \
243
ml/Dimension.h \
244
ml/Host.h \
245
ml/Host.cc \
246
+ ml/Mutex.h \
247
+ ml/Queue.h \
248
ml/Query.h \
249
ml/KMeans.h \
250
ml/KMeans.cc \
@@ -262,13 +267,6 @@ ml/ml.$(OBJEXT) : CXXFLAGS += -Wno-psabi
267
268
endif
269
265
-
266
-if ENABLE_ML_TESTS
267
-ML_TESTS_FILES = \
268
- ml/SamplesBufferTests.cc \
269
- $(NULL)
270
-endif
271
-
270
IDLEJITTER_PLUGIN_FILES = \
271
collectors/idlejitter.plugin/plugin_idlejitter.c \
272
$(NULL)
@@ -920,7 +918,6 @@ NETDATA_FILES = \
918
$(EXPORTING_ENGINE_FILES) \
919
$(HEALTH_PLUGIN_FILES) \
920
$(ML_FILES) \
923
- $(ML_TESTS_FILES) \
921
$(IDLEJITTER_PLUGIN_FILES) \
922
$(PLUGINSD_PLUGIN_FILES) \
923
$(REGISTRY_PLUGIN_FILES) \
@@ -1008,11 +1005,6 @@ if ENABLE_ACLK
1005
$(NULL)
1006
endif
1007
1011
-if ENABLE_ML_TESTS
1012
- netdata_LDADD += $(OPTIONAL_ML_TESTS_LIBS) \
1013
- $(NULL)
1014
-endif
1015
-
1008
netdata_LINK = $(CXXLD) $(CXXFLAGS) $(LDFLAGS) -o $@
1009
1010
sbin_PROGRAMS += netdatacli
collectors/all.h
+18
@@ -363,5 +363,23 @@
363
#define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030
364
#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100
365
366
+// NETDATA ML CHARTS
367
+
368
+// [ml] charts
369
+#define ML_CHART_PRIO_DIMENSIONS 39181
370
+#define ML_CHART_PRIO_ANOMALY_RATE 39182
371
+#define ML_CHART_PRIO_DETECTOR_EVENTS 39183
372
+
373
+// [netdata.ml] charts
374
+#define NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS 890001
375
+#define NETDATA_ML_CHART_PRIO_METRIC_TYPES 890002
376
+#define NETDATA_ML_CHART_PRIO_TRAINING_STATUS 890003
377
+
378
+#define NETDATA_ML_CHART_PRIO_PREDICTION_USAGE 890004
379
+#define NETDATA_ML_CHART_PRIO_TRAINING_USAGE 890005
380
+
381
+#define NETDATA_ML_CHART_PRIO_QUEUE_STATS 890006
382
+#define NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS 890007
383
+#define NETDATA_ML_CHART_PRIO_TRAINING_RESULTS 890008
384
385
#endif //NETDATA_ALL_H
configure.ac
+1
-22
@@ -207,12 +207,6 @@ AC_ARG_ENABLE(
207
,
208
[enable_ml="detect"]
209
)
210
-AC_ARG_ENABLE(
211
- [ml_tests],
212
- [AS_HELP_STRING([--enable-ml-tests], [Enable anomaly detection tests @<:@no@:>@])],
213
- [enable_ml_tests="yes"],
214
- [enable_ml_tests="no"]
215
-)
210
AC_ARG_ENABLE(
211
[aclk_ssl_debug],
212
[AS_HELP_STRING([--enable-aclk-ssl-debug], [Enables possibility for SSL key logging @<:@default no@:>@])],
@@ -1180,19 +1174,6 @@ if test "${build_ml}" = "yes"; then
1174
OPTIONAL_ML_LIBS=""
1175
fi
1176
1183
-# Decide if we should build ML tests.
1184
-if test "${build_ml}" = "yes" -a "${enable_ml_tests}" = "yes" -a "${have_gtest}" = "yes"; then
1185
- build_ml_tests="yes"
1186
-else
1187
- build_ml_tests="no"
1188
-fi
1189
-
1190
-AM_CONDITIONAL([ENABLE_ML_TESTS], [test "${build_ml_tests}" = "yes"])
1191
-if test "${build_ml_tests}" = "yes"; then
1192
- AC_DEFINE([ENABLE_ML_TESTS], [1], [anomaly detection tests])
1193
- OPTIONAL_ML_TESTS_CFLAGS="${OPTIONAL_GTEST_CFLAGS}"
1194
- OPTIONAL_ML_TESTS_LIBS="${OPTIONAL_GTEST_LIBS}"
1195
-fi
1177
1178
# -----------------------------------------------------------------------------
1179
# ebpf.plugin
@@ -1612,7 +1593,7 @@ CFLAGS="${originalCFLAGS} ${OPTIONAL_LTO_CFLAGS} ${OPTIONAL_PROTOBUF_CFLAGS} ${O
1593
${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
1594
${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PUBSUB_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} \
1595
${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
1615
- ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_ML_TESTS_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1596
+ ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1597
1598
CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
1599
@@ -1666,8 +1647,6 @@ AC_SUBST([OPTIONAL_GTEST_CFLAGS])
1647
AC_SUBST([OPTIONAL_GTEST_LIBS])
1648
AC_SUBST([OPTIONAL_ML_CFLAGS])
1649
AC_SUBST([OPTIONAL_ML_LIBS])
1669
-AC_SUBST([OPTIONAL_ML_TESTS_CFLAGS])
1670
-AC_SUBST([OPTIONAL_ML_TESTS_LIBS])
1650
1651
# -----------------------------------------------------------------------------
1652
# Check if cmocka is available - needed for unit testing
daemon/global_statistics.c
+34
@@ -52,6 +52,7 @@ static struct global_statistics {
52
uint64_t ml_queries_made;
53
uint64_t ml_db_points_read;
54
uint64_t ml_result_points_generated;
55
+ uint64_t ml_models_consulted;
56
57
uint64_t exporters_queries_made;
58
uint64_t exporters_db_points_read;
@@ -88,6 +89,10 @@ void global_statistics_ml_query_completed(size_t points_read) {
89
__atomic_fetch_add(&global_statistics.ml_db_points_read, points_read, __ATOMIC_RELAXED);
90
}
91
92
+void global_statistics_ml_models_consulted(size_t models_consulted) {
93
+ __atomic_fetch_add(&global_statistics.ml_models_consulted, models_consulted, __ATOMIC_RELAXED);
94
+}
95
+
96
void global_statistics_exporters_query_completed(size_t points_read) {
97
__atomic_fetch_add(&global_statistics.exporters_queries_made, 1, __ATOMIC_RELAXED);
98
__atomic_fetch_add(&global_statistics.exporters_db_points_read, points_read, __ATOMIC_RELAXED);
@@ -193,6 +198,7 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
198
gs->ml_queries_made = __atomic_load_n(&global_statistics.ml_queries_made, __ATOMIC_RELAXED);
199
gs->ml_db_points_read = __atomic_load_n(&global_statistics.ml_db_points_read, __ATOMIC_RELAXED);
200
gs->ml_result_points_generated = __atomic_load_n(&global_statistics.ml_result_points_generated, __ATOMIC_RELAXED);
201
+ gs->ml_models_consulted = __atomic_load_n(&global_statistics.ml_models_consulted, __ATOMIC_RELAXED);
202
203
gs->exporters_queries_made = __atomic_load_n(&global_statistics.exporters_queries_made, __ATOMIC_RELAXED);
204
gs->exporters_db_points_read = __atomic_load_n(&global_statistics.exporters_db_points_read, __ATOMIC_RELAXED);
@@ -653,6 +659,34 @@ static void global_statistics_charts(void) {
659
660
rrdset_done(st_points_stored);
661
}
662
+
663
+ {
664
+ static RRDSET *st = NULL;
665
+ static RRDDIM *rd = NULL;
666
+
667
+ if (unlikely(!st)) {
668
+ st = rrdset_create_localhost(
669
+ "netdata" // type
670
+ , "ml_models_consulted" // id
671
+ , NULL // name
672
+ , "ml" // family
673
+ , NULL // context
674
+ , "KMeans models used for prediction" // title
675
+ , "models" // units
676
+ , "netdata" // plugin
677
+ , "ml" // module
678
+ , 131004 // priority
679
+ , localhost->rrd_update_every // update_every
680
+ , RRDSET_TYPE_STACKED // chart_type
681
+ );
682
+
683
+ rd = rrddim_add(st, "num_models_consulted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
684
+ }
685
+
686
+ rrddim_set_by_pointer(st, rd, (collected_number) gs.ml_models_consulted);
687
+
688
+ rrdset_done(st);
689
+ }
690
}
691
692
// ----------------------------------------------------------------------------
daemon/global_statistics.h
+1
@@ -9,6 +9,7 @@
9
// global statistics
10
11
void global_statistics_ml_query_completed(size_t points_read);
12
+void global_statistics_ml_models_consulted(size_t models_consulted);
13
void global_statistics_exporters_query_completed(size_t points_read);
14
void global_statistics_backfill_query_completed(size_t points_read);
15
void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source);
daemon/main.c
-5
@@ -1027,11 +1027,6 @@ int main(int argc, char **argv) {
1027
else if(strcmp(optarg, "escapetest") == 0) {
1028
return command_argument_sanitization_tests();
1029
}
1030
-#ifdef ENABLE_ML_TESTS
1031
- else if(strcmp(optarg, "mltest") == 0) {
1032
- return test_ml(argc, argv);
1033
- }
1034
-#endif
1030
#ifdef ENABLE_DBENGINE
1031
else if(strcmp(optarg, "mctest") == 0) {
1032
unittest_running = true;
database/rrd.h
+12
-6
@@ -30,8 +30,9 @@ typedef struct rrdhost_acquired RRDHOST_ACQUIRED;
30
typedef struct rrdset_acquired RRDSET_ACQUIRED;
31
typedef struct rrddim_acquired RRDDIM_ACQUIRED;
32
33
-typedef void *ml_host_t;
34
-typedef void *ml_dimension_t;
33
+typedef struct ml_host ml_host_t;
34
+typedef struct ml_chart ml_chart_t;
35
+typedef struct ml_dimension ml_dimension_t;
36
37
typedef enum {
38
QUERY_SOURCE_UNKNOWN,
@@ -296,7 +297,7 @@ struct rrddim {
297
// ------------------------------------------------------------------------
298
// operational state members
299
299
- ml_dimension_t ml_dimension; // machine learning data about this dimension
300
+ ml_dimension_t *ml_dimension; // machine learning data about this dimension
301
302
// ------------------------------------------------------------------------
303
// linking to siblings and parents
@@ -595,6 +596,8 @@ struct rrdset {
596
DICTIONARY *rrddimvar_root_index; // dimension variables
597
// we use this dictionary to manage their allocation
598
599
+ ml_chart_t *ml_chart;
600
+
601
// ------------------------------------------------------------------------
602
// operational state members
603
@@ -1028,7 +1031,7 @@ struct rrdhost {
1031
1032
// ------------------------------------------------------------------------
1033
// ML handle
1031
- ml_host_t ml_host;
1034
+ ml_host_t *ml_host;
1035
1036
// ------------------------------------------------------------------------
1037
// Support for host-level labels
@@ -1301,9 +1304,12 @@ void rrdset_isnot_obsolete(RRDSET *st);
1304
time_t rrddim_first_entry_t(RRDDIM *rd);
1305
time_t rrddim_first_entry_t_of_tier(RRDDIM *rd, size_t tier);
1306
time_t rrddim_last_entry_t(RRDDIM *rd);
1304
-time_t rrdset_last_entry_t(RRDSET *st);
1305
-time_t rrdset_first_entry_t_of_tier(RRDSET *st, size_t tier);
1307
+time_t rrddim_last_entry_t_of_tier(RRDDIM *rd, size_t tier);
1308
+
1309
time_t rrdset_first_entry_t(RRDSET *st);
1310
+time_t rrdset_first_entry_t_of_tier(RRDSET *st, size_t tier);
1311
+time_t rrdset_last_entry_t(RRDSET *st);
1312
+
1313
time_t rrdhost_last_entry_t(RRDHOST *h);
1314
1315
// ----------------------------------------------------------------------------
database/rrdcontext.c
-9
@@ -750,11 +750,6 @@ static void rrdinstance_free(RRDINSTANCE *ri) {
750
}
751
752
static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) {
753
- static STRING *ml_anomaly_rates_id = NULL;
754
-
755
- if(unlikely(!ml_anomaly_rates_id))
756
- ml_anomaly_rates_id = string_strdupz(ML_ANOMALY_RATES_CHART_ID);
757
-
753
RRDINSTANCE *ri = value;
754
755
// link it to its parent
@@ -781,10 +776,6 @@ static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unus
776
ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor
777
}
778
784
- // we need this when loading from SQL
785
- if(unlikely(ri->id == ml_anomaly_rates_id))
786
- ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
787
-
779
rrdmetrics_create_in_rrdinstance(ri);
780
781
// signal the react callback to do the job
database/rrddim.c
+9
-3
@@ -172,7 +172,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
172
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
173
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
174
175
- ml_new_dimension(rd);
175
+ ml_dimension_new(rd);
176
177
ctr->react_action = RRDDIM_REACT_NEW;
178
@@ -191,7 +191,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
191
192
rrdcontext_removed_rrddim(rd);
193
194
- ml_delete_dimension(rd);
194
+ ml_dimension_delete(rd);
195
196
debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
197
@@ -420,7 +420,13 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
420
421
// ----------------------------------------------------------------------------
422
423
-// get the timestamp of the last entry in the round-robin database
423
+time_t rrddim_last_entry_t_of_tier(RRDDIM *rd, size_t tier) {
424
+ if(unlikely(tier > storage_tiers || !rd->tiers[tier]))
425
+ return 0;
426
+
427
+ return rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle);
428
+}
429
+
430
time_t rrddim_last_entry_t(RRDDIM *rd) {
431
time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
432
database/rrdhost.c
+12
-8
@@ -517,9 +517,10 @@ int is_legacy = 1;
517
rrd_hosts_available++;
518
519
rrdhost_load_rrdcontext_data(host);
520
- if (!archived)
521
- ml_new_host(host);
522
- else
520
+ if (!archived) {
521
+ ml_host_new(host);
522
+ ml_start_anomaly_detection_threads(host);
523
+ } else
524
rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
525
526
@@ -629,7 +630,10 @@ void rrdhost_update(RRDHOST *host
630
host->rrdpush_replication_step = rrdpush_replication_step;
631
632
rrd_hosts_available++;
632
- ml_new_host(host);
633
+
634
+ ml_host_new(host);
635
+ ml_start_anomaly_detection_threads(host);
636
+
637
rrdhost_load_rrdcontext_data(host);
638
info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
639
}
@@ -1088,10 +1092,6 @@ void rrdhost_free(RRDHOST *host, bool force) {
1092
1093
rrd_check_wrlock(); // make sure the RRDs are write locked
1094
1091
- rrdhost_wrlock(host);
1092
- ml_delete_host(host);
1093
- rrdhost_unlock(host);
1094
-
1095
// ------------------------------------------------------------------------
1096
// clean up streaming
1097
@@ -1126,6 +1126,10 @@ void rrdhost_free(RRDHOST *host, bool force) {
1126
rrdcalc_rrdhost_index_destroy(host);
1127
rrdcalctemplate_index_destroy(host);
1128
1129
+ // cleanup ML resources
1130
+ ml_stop_anomaly_detection_threads(host);
1131
+ ml_host_delete(host);
1132
+
1133
freez(host->exporting_flags);
1134
1135
health_alarm_log_free(host);
database/rrdset.c
+15
-3
@@ -178,6 +178,8 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
178
st->red = NAN;
179
180
ctr->react_action = RRDSET_REACT_NEW;
181
+
182
+ ml_chart_new(st);
183
}
184
185
// the destructor - the dictionary is write locked while this runs
@@ -232,6 +234,9 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
234
// 7. destroy the chart labels
235
rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
236
237
+ // 8. destroy the ml handle
238
+ ml_chart_delete(st);
239
+
240
rrdset_memory_file_free(st); // remove files of db mode save and map
241
242
// ------------------------------------------------------------------------
@@ -1253,6 +1258,8 @@ static inline size_t rrdset_done_interpolate(
1258
1259
last_ut = next_store_ut;
1260
1261
+ ml_chart_update_begin(st);
1262
+
1263
struct rda_item *rda;
1264
size_t dim_id;
1265
for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
@@ -1332,8 +1339,11 @@ static inline size_t rrdset_done_interpolate(
1339
break;
1340
}
1341
1342
+ time_t current_time = (time_t) (next_store_ut / USEC_PER_SEC);
1343
+
1344
if(unlikely(!store_this_entry)) {
1336
- (void) ml_is_anomalous(rd, 0, false);
1345
+ (void) ml_is_anomalous(rd, current_time, 0, false);
1346
+
1347
rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1348
rrdcontext_collected_rrddim(rd);
1349
continue;
@@ -1342,7 +1352,7 @@ static inline size_t rrdset_done_interpolate(
1352
if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) {
1353
uint32_t dim_storage_flags = storage_flags;
1354
1345
- if (ml_is_anomalous(rd, new_value, true)) {
1355
+ if (ml_is_anomalous(rd, current_time, new_value, true)) {
1356
// clear anomaly bit: 0 -> is anomalous, 1 -> not anomalous
1357
dim_storage_flags &= ~((storage_number)SN_FLAG_NOT_ANOMALOUS);
1358
}
@@ -1352,7 +1362,7 @@ static inline size_t rrdset_done_interpolate(
1362
rd->last_stored_value = new_value;
1363
}
1364
else {
1355
- (void) ml_is_anomalous(rd, 0, false);
1365
+ (void) ml_is_anomalous(rd, current_time, 0, false);
1366
1367
rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
1368
@@ -1364,6 +1374,8 @@ static inline size_t rrdset_done_interpolate(
1374
stored_entries++;
1375
}
1376
1377
+ ml_chart_update_end(st);
1378
+
1379
// reset the storage flags for the next point, if any;
1380
storage_flags = SN_DEFAULT_FLAGS;
1381
ml/ADCharts.cc
+387
-109
@@ -3,55 +3,182 @@
3
#include "ADCharts.h"
4
#include "Config.h"
5
6
-void ml::updateDimensionsChart(RRDHOST *RH,
7
- collected_number NumTrainedDimensions,
8
- collected_number NumNormalDimensions,
9
- collected_number NumAnomalousDimensions) {
10
- static thread_local RRDSET *RS = nullptr;
11
- static thread_local RRDDIM *NumTotalDimensionsRD = nullptr;
12
- static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr;
13
- static thread_local RRDDIM *NumNormalDimensionsRD = nullptr;
14
- static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
15
-
16
- if (!RS) {
17
- std::stringstream IdSS, NameSS;
6
+void ml::updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS) {
7
+ /*
8
+ * Machine learning status
9
+ */
10
+ {
11
+ static thread_local RRDSET *MachineLearningStatusRS = nullptr;
12
+
13
+ static thread_local RRDDIM *Enabled = nullptr;
14
+ static thread_local RRDDIM *DisabledUE = nullptr;
15
+ static thread_local RRDDIM *DisabledSP = nullptr;
16
+
17
+ if (!MachineLearningStatusRS) {
18
+ std::stringstream IdSS, NameSS;
19
+
20
+ IdSS << "machine_learning_status_for_" << localhost->machine_guid;
21
+ NameSS << "machine_learning_status_for_" << localhost->hostname;
22
+
23
+ MachineLearningStatusRS = rrdset_create_localhost(
24
+ "netdata", // type
25
+ IdSS.str().c_str(), // id
26
+ NameSS.str().c_str(), // name
27
+ "ml", // family
28
+ "netdata.machine_learning_status", // ctx
29
+ "Machine learning status", // title
30
+ "dimensions", // units
31
+ "netdata", // plugin
32
+ "ml", // module
33
+ NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS, // priority
34
+ RH->rrd_update_every, // update_every
35
+ RRDSET_TYPE_LINE // chart_type
36
+ );
37
+ rrdset_flag_set(MachineLearningStatusRS , RRDSET_FLAG_ANOMALY_DETECTION);
38
+
39
+ Enabled = rrddim_add(MachineLearningStatusRS, "enabled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
40
+ DisabledUE = rrddim_add(MachineLearningStatusRS, "disabled-ue", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
41
+ DisabledSP = rrddim_add(MachineLearningStatusRS, "disabled-sp", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
42
+ }
43
+
44
+ rrddim_set_by_pointer(MachineLearningStatusRS, Enabled, MLS.NumMachineLearningStatusEnabled);
45
+ rrddim_set_by_pointer(MachineLearningStatusRS, DisabledUE, MLS.NumMachineLearningStatusDisabledUE);
46
+ rrddim_set_by_pointer(MachineLearningStatusRS, DisabledSP, MLS.NumMachineLearningStatusDisabledSP);
47
+
48
+ rrdset_done(MachineLearningStatusRS);
49
+ }
50
19
- IdSS << "dimensions_on_" << localhost->machine_guid;
20
- NameSS << "dimensions_on_" << localhost->hostname;
51
+ /*
52
+ * Metric type
53
+ */
54
+ {
55
+ static thread_local RRDSET *MetricTypesRS = nullptr;
56
+
57
+ static thread_local RRDDIM *Constant = nullptr;
58
+ static thread_local RRDDIM *Variable = nullptr;
59
+
60
+ if (!MetricTypesRS) {
61
+ std::stringstream IdSS, NameSS;
62
+
63
+ IdSS << "metric_types_for_" << localhost->machine_guid;
64
+ NameSS << "metric_types_for_" << localhost->hostname;
65
+
66
+ MetricTypesRS = rrdset_create_localhost(
67
+ "netdata", // type
68
+ IdSS.str().c_str(), // id
69
+ NameSS.str().c_str(), // name
70
+ "ml", // family
71
+ "netdata.metric_types", // ctx
72
+ "Dimensions by metric type", // title
73
+ "dimensions", // units
74
+ "netdata", // plugin
75
+ "ml", // module
76
+ NETDATA_ML_CHART_PRIO_METRIC_TYPES, // priority
77
+ RH->rrd_update_every, // update_every
78
+ RRDSET_TYPE_LINE // chart_type
79
+ );
80
+ rrdset_flag_set(MetricTypesRS, RRDSET_FLAG_ANOMALY_DETECTION);
81
+
82
+ Constant = rrddim_add(MetricTypesRS, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
83
+ Variable = rrddim_add(MetricTypesRS, "variable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
84
+ }
85
+
86
+ rrddim_set_by_pointer(MetricTypesRS, Constant, MLS.NumMetricTypeConstant);
87
+ rrddim_set_by_pointer(MetricTypesRS, Variable, MLS.NumMetricTypeVariable);
88
+
89
+ rrdset_done(MetricTypesRS);
90
+ }
91
22
- RS = rrdset_create(
23
- RH,
24
- "anomaly_detection", // type
25
- IdSS.str().c_str(), // id
26
- NameSS.str().c_str(), // name
27
- "dimensions", // family
28
- "anomaly_detection.dimensions", // ctx
29
- "Anomaly detection dimensions", // title
30
- "dimensions", // units
31
- "netdata", // plugin
32
- "ml", // module
33
- 39183, // priority
34
- RH->rrd_update_every, // update_every
35
- RRDSET_TYPE_LINE // chart_type
36
- );
37
- rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
38
-
39
- NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
40
- 1, 1, RRD_ALGORITHM_ABSOLUTE);
41
- NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL,
42
- 1, 1, RRD_ALGORITHM_ABSOLUTE);
43
- NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL,
44
- 1, 1, RRD_ALGORITHM_ABSOLUTE);
45
- NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL,
46
- 1, 1, RRD_ALGORITHM_ABSOLUTE);
92
+ /*
93
+ * Training status
94
+ */
95
+ {
96
+ static thread_local RRDSET *TrainingStatusRS = nullptr;
97
+
98
+ static thread_local RRDDIM *Untrained = nullptr;
99
+ static thread_local RRDDIM *PendingWithoutModel = nullptr;
100
+ static thread_local RRDDIM *Trained = nullptr;
101
+ static thread_local RRDDIM *PendingWithModel = nullptr;
102
+
103
+ if (!TrainingStatusRS) {
104
+ std::stringstream IdSS, NameSS;
105
+
106
+ IdSS << "training_status_for_" << localhost->machine_guid;
107
+ NameSS << "training_status_for_" << localhost->hostname;
108
+
109
+ TrainingStatusRS = rrdset_create_localhost(
110
+ "netdata", // type
111
+ IdSS.str().c_str(), // id
112
+ NameSS.str().c_str(), // name
113
+ "ml", // family
114
+ "netdata.training_status", // ctx
115
+ "Training status of dimensions", // title
116
+ "dimensions", // units
117
+ "netdata", // plugin
118
+ "ml", // module
119
+ NETDATA_ML_CHART_PRIO_TRAINING_STATUS, // priority
120
+ RH->rrd_update_every, // update_every
121
+ RRDSET_TYPE_LINE // chart_type
122
+ );
123
+
124
+ rrdset_flag_set(TrainingStatusRS, RRDSET_FLAG_ANOMALY_DETECTION);
125
+
126
+ Untrained = rrddim_add(TrainingStatusRS, "untrained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
127
+ PendingWithoutModel = rrddim_add(TrainingStatusRS, "pending-without-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
128
+ Trained = rrddim_add(TrainingStatusRS, "trained", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
129
+ PendingWithModel = rrddim_add(TrainingStatusRS, "pending-with-model", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
130
+ }
131
+
132
+ rrddim_set_by_pointer(TrainingStatusRS, Untrained, MLS.NumTrainingStatusUntrained);
133
+ rrddim_set_by_pointer(TrainingStatusRS, PendingWithoutModel, MLS.NumTrainingStatusPendingWithoutModel);
134
+ rrddim_set_by_pointer(TrainingStatusRS, Trained, MLS.NumTrainingStatusTrained);
135
+ rrddim_set_by_pointer(TrainingStatusRS, PendingWithModel, MLS.NumTrainingStatusPendingWithModel);
136
+
137
+ rrdset_done(TrainingStatusRS);
138
}
139
49
- rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions);
50
- rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions);
51
- rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions);
52
- rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions);
140
+ /*
141
+ * Prediction status
142
+ */
143
+ {
144
+ static thread_local RRDSET *PredictionRS = nullptr;
145
+
146
+ static thread_local RRDDIM *Anomalous = nullptr;
147
+ static thread_local RRDDIM *Normal = nullptr;
148
+
149
+ if (!PredictionRS) {
150
+ std::stringstream IdSS, NameSS;
151
+
152
+ IdSS << "dimensions_on_" << localhost->machine_guid;
153
+ NameSS << "dimensions_on_" << localhost->hostname;
154
+
155
+ PredictionRS = rrdset_create(
156
+ RH,
157
+ "anomaly_detection", // type
158
+ IdSS.str().c_str(), // id
159
+ NameSS.str().c_str(), // name
160
+ "dimensions", // family
161
+ "anomaly_detection.dimensions", // ctx
162
+ "Anomaly detection dimensions", // title
163
+ "dimensions", // units
164
+ "netdata", // plugin
165
+ "ml", // module
166
+ ML_CHART_PRIO_DIMENSIONS, // priority
167
+ RH->rrd_update_every, // update_every
168
+ RRDSET_TYPE_LINE // chart_type
169
+ );
170
+ rrdset_flag_set(PredictionRS, RRDSET_FLAG_ANOMALY_DETECTION);
171
+
172
+ Anomalous = rrddim_add(PredictionRS, "anomalous", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
173
+ Normal = rrddim_add(PredictionRS, "normal", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
174
+ }
175
+
176
+ rrddim_set_by_pointer(PredictionRS, Anomalous, MLS.NumAnomalousDimensions);
177
+ rrddim_set_by_pointer(PredictionRS, Normal, MLS.NumNormalDimensions);
178
+
179
+ rrdset_done(PredictionRS);
180
+ }
181
54
- rrdset_done(RS);
182
}
183
184
void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) {
@@ -75,7 +202,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
202
"percentage", // units
203
"netdata", // plugin
204
"ml", // module
78
- 39184, // priority
205
+ ML_CHART_PRIO_ANOMALY_RATE, // priority
206
RH->rrd_update_every, // update_every
207
RRDSET_TYPE_LINE // chart_type
208
);
@@ -109,7 +236,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
236
"percentage", // units
237
"netdata", // plugin
238
"ml", // module
112
- 39185, // priority
239
+ ML_CHART_PRIO_DETECTOR_EVENTS, // priority
240
RH->rrd_update_every, // update_every
241
RRDSET_TYPE_LINE // chart_type
242
);
@@ -143,6 +270,7 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
270
0, /* tier */
271
QUERY_SOURCE_ML
272
);
273
+
274
if(R) {
275
assert(R->d == 1 && R->n == 1 && R->rows == 1);
276
@@ -157,77 +285,227 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
285
286
rrdr_free(OWA, R);
287
}
288
+
289
onewayalloc_destroy(OWA);
290
}
291
163
-void ml::updateDetectionChart(RRDHOST *RH) {
164
- static thread_local RRDSET *RS = nullptr;
165
- static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
166
-
167
- if (!RS) {
168
- std::stringstream IdSS, NameSS;
169
-
170
- IdSS << "prediction_stats_" << RH->machine_guid;
171
- NameSS << "prediction_stats_for_" << RH->hostname;
172
-
173
- RS = rrdset_create_localhost(
174
- "netdata", // type
175
- IdSS.str().c_str(), // id
176
- NameSS.str().c_str(), // name
177
- "ml", // family
178
- "netdata.prediction_stats", // ctx
179
- "Prediction thread CPU usage", // title
180
- "milliseconds/s", // units
181
- "netdata", // plugin
182
- "ml", // module
183
- 136000, // priority
184
- RH->rrd_update_every, // update_every
185
- RRDSET_TYPE_STACKED // chart_type
186
- );
187
-
188
- UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
189
- SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
292
+void ml::updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU) {
293
+ /*
294
+ * prediction rusage
295
+ */
296
+ {
297
+ static thread_local RRDSET *RS = nullptr;
298
+
299
+ static thread_local RRDDIM *User = nullptr;
300
+ static thread_local RRDDIM *System = nullptr;
301
+
302
+ if (!RS) {
303
+ std::stringstream IdSS, NameSS;
304
+
305
+ IdSS << "prediction_usage_for_" << localhost->machine_guid;
306
+ NameSS << "prediction_usage_for_" << localhost->hostname;
307
+
308
+ RS = rrdset_create_localhost(
309
+ "netdata", // type
310
+ IdSS.str().c_str(), // id
311
+ NameSS.str().c_str(), // name
312
+ "ml", // family
313
+ "netdata.prediction_usage", // ctx
314
+ "Prediction resource usage", // title
315
+ "milliseconds/s", // units
316
+ "netdata", // plugin
317
+ "ml", // module
318
+ NETDATA_ML_CHART_PRIO_PREDICTION_USAGE, // priority
319
+ RH->rrd_update_every, // update_every
320
+ RRDSET_TYPE_STACKED // chart_type
321
+ );
322
+ rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
323
+
324
+ User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
325
+ System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
326
+ }
327
+
328
+ rrddim_set_by_pointer(RS, User, PredictionRU.ru_utime.tv_sec * 1000000ULL + PredictionRU.ru_utime.tv_usec);
329
+ rrddim_set_by_pointer(RS, System, PredictionRU.ru_stime.tv_sec * 1000000ULL + PredictionRU.ru_stime.tv_usec);
330
+
331
+ rrdset_done(RS);
332
}
333
192
- struct rusage TRU;
193
- getrusage(RUSAGE_THREAD, &TRU);
194
-
195
- rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);
196
- rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);
197
- rrdset_done(RS);
334
+ /*
335
+ * training rusage
336
+ */
337
+ {
338
+ static thread_local RRDSET *RS = nullptr;
339
+
340
+ static thread_local RRDDIM *User = nullptr;
341
+ static thread_local RRDDIM *System = nullptr;
342
+
343
+ if (!RS) {
344
+ std::stringstream IdSS, NameSS;
345
+
346
+ IdSS << "training_usage_for_" << localhost->machine_guid;
347
+ NameSS << "training_usage_for_" << localhost->hostname;
348
+
349
+ RS = rrdset_create_localhost(
350
+ "netdata", // type
351
+ IdSS.str().c_str(), // id
352
+ NameSS.str().c_str(), // name
353
+ "ml", // family
354
+ "netdata.training_usage", // ctx
355
+ "Training resource usage", // title
356
+ "milliseconds/s", // units
357
+ "netdata", // plugin
358
+ "ml", // module
359
+ NETDATA_ML_CHART_PRIO_TRAINING_USAGE, // priority
360
+ RH->rrd_update_every, // update_every
361
+ RRDSET_TYPE_STACKED // chart_type
362
+ );
363
+ rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
364
+
365
+ User = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
366
+ System = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
367
+ }
368
+
369
+ rrddim_set_by_pointer(RS, User, TrainingRU.ru_utime.tv_sec * 1000000ULL + TrainingRU.ru_utime.tv_usec);
370
+ rrddim_set_by_pointer(RS, System, TrainingRU.ru_stime.tv_sec * 1000000ULL + TrainingRU.ru_stime.tv_usec);
371
+
372
+ rrdset_done(RS);
373
+ }
374
}
375
200
-void ml::updateTrainingChart(RRDHOST *RH, struct rusage *TRU) {
201
- static thread_local RRDSET *RS = nullptr;
202
- static thread_local RRDDIM *UserRD = nullptr;
203
- static thread_local RRDDIM *SystemRD = nullptr;
204
-
205
- if (!RS) {
206
- std::stringstream IdSS, NameSS;
207
-
208
- IdSS << "training_stats_" << RH->machine_guid;
209
- NameSS << "training_stats_for_" << RH->hostname;
210
-
211
- RS = rrdset_create_localhost(
212
- "netdata", // type
213
- IdSS.str().c_str(), // id
214
- NameSS.str().c_str(), // name
215
- "ml", // family
216
- "netdata.training_stats", // ctx
217
- "Training thread CPU usage", // title
218
- "milliseconds/s", // units
219
- "netdata", // plugin
220
- "ml", // module
221
- 136001, // priority
222
- RH->rrd_update_every, // update_every
223
- RRDSET_TYPE_STACKED // chart_type
224
- );
376
+void ml::updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS) {
377
+ /*
378
+ * queue stats
379
+ */
380
+ {
381
+ static thread_local RRDSET *RS = nullptr;
382
+
383
+ static thread_local RRDDIM *QueueSize = nullptr;
384
+ static thread_local RRDDIM *PoppedItems = nullptr;
385
+
386
+ if (!RS) {
387
+ std::stringstream IdSS, NameSS;
388
+
389
+ IdSS << "queue_stats_for_" << localhost->machine_guid;
390
+ NameSS << "queue_stats_for_" << localhost->hostname;
391
+
392
+ RS = rrdset_create_localhost(
393
+ "netdata", // type
394
+ IdSS.str().c_str(), // id
395
+ NameSS.str().c_str(), // name
396
+ "ml", // family
397
+ "netdata.queue_stats", // ctx
398
+ "Training queue stats", // title
399
+ "items", // units
400
+ "netdata", // plugin
401
+ "ml", // module
402
+ NETDATA_ML_CHART_PRIO_QUEUE_STATS, // priority
403
+ RH->rrd_update_every, // update_every
404
+ RRDSET_TYPE_LINE// chart_type
405
+ );
406
+ rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
407
+
408
+ QueueSize = rrddim_add(RS, "queue_size", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
409
+ PoppedItems = rrddim_add(RS, "popped_items", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
410
+ }
411
+
412
+ rrddim_set_by_pointer(RS, QueueSize, TS.QueueSize);
413
+ rrddim_set_by_pointer(RS, PoppedItems, TS.NumPoppedItems);
414
+
415
+ rrdset_done(RS);
416
+ }
417
226
- UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
227
- SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
418
+ /*
419
+ * training stats
420
+ */
421
+ {
422
+ static thread_local RRDSET *RS = nullptr;
423
+
424
+ static thread_local RRDDIM *Allotted = nullptr;
425
+ static thread_local RRDDIM *Consumed = nullptr;
426
+ static thread_local RRDDIM *Remaining = nullptr;
427
+
428
+ if (!RS) {
429
+ std::stringstream IdSS, NameSS;
430
+
431
+ IdSS << "training_time_stats_for_" << localhost->machine_guid;
432
+ NameSS << "training_time_stats_for_" << localhost->hostname;
433
+
434
+ RS = rrdset_create_localhost(
435
+ "netdata", // type
436
+ IdSS.str().c_str(), // id
437
+ NameSS.str().c_str(), // name
438
+ "ml", // family
439
+ "netdata.training_time_stats", // ctx
440
+ "Training time stats", // title
441
+ "milliseconds", // units
442
+ "netdata", // plugin
443
+ "ml", // module
444
+ NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS, // priority
445
+ RH->rrd_update_every, // update_every
446
+ RRDSET_TYPE_LINE// chart_type
447
+ );
448
+ rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
449
+
450
+ Allotted = rrddim_add(RS, "allotted", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
451
+ Consumed = rrddim_add(RS, "consumed", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
452
+ Remaining = rrddim_add(RS, "remaining", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
453
+ }
454
+
455
+ rrddim_set_by_pointer(RS, Allotted, TS.AllottedUT);
456
+ rrddim_set_by_pointer(RS, Consumed, TS.ConsumedUT);
457
+ rrddim_set_by_pointer(RS, Remaining, TS.RemainingUT);
458
+
459
+ rrdset_done(RS);
460
}
461
230
- rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);
231
- rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);
232
- rrdset_done(RS);
462
+ /*
463
+ * training result stats
464
+ */
465
+ {
466
+ static thread_local RRDSET *RS = nullptr;
467
+
468
+ static thread_local RRDDIM *Ok = nullptr;
469
+ static thread_local RRDDIM *InvalidQueryTimeRange = nullptr;
470
+ static thread_local RRDDIM *NotEnoughCollectedValues = nullptr;
471
+ static thread_local RRDDIM *NullAcquiredDimension = nullptr;
472
+ static thread_local RRDDIM *ChartUnderReplication = nullptr;
473
+
474
+ if (!RS) {
475
+ std::stringstream IdSS, NameSS;
476
+
477
+ IdSS << "training_results_for_" << localhost->machine_guid;
478
+ NameSS << "training_results_for_" << localhost->hostname;
479
+
480
+ RS = rrdset_create_localhost(
481
+ "netdata", // type
482
+ IdSS.str().c_str(), // id
483
+ NameSS.str().c_str(), // name
484
+ "ml", // family
485
+ "netdata.training_results", // ctx
486
+ "Training results", // title
487
+ "events", // units
488
+ "netdata", // plugin
489
+ "ml", // module
490
+ NETDATA_ML_CHART_PRIO_TRAINING_RESULTS, // priority
491
+ RH->rrd_update_every, // update_every
492
+ RRDSET_TYPE_LINE// chart_type
493
+ );
494
+ rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
495
+
496
+ Ok = rrddim_add(RS, "ok", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
497
+ InvalidQueryTimeRange = rrddim_add(RS, "invalid-queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
498
+ NotEnoughCollectedValues = rrddim_add(RS, "not-enough-values", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
499
+ NullAcquiredDimension = rrddim_add(RS, "null-acquired-dimensions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
500
+ ChartUnderReplication = rrddim_add(RS, "chart-under-replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
501
+ }
502
+
503
+ rrddim_set_by_pointer(RS, Ok, TS.TrainingResultOk);
504
+ rrddim_set_by_pointer(RS, InvalidQueryTimeRange, TS.TrainingResultInvalidQueryTimeRange);
505
+ rrddim_set_by_pointer(RS, NotEnoughCollectedValues, TS.TrainingResultNotEnoughCollectedValues);
506
+ rrddim_set_by_pointer(RS, NullAcquiredDimension, TS.TrainingResultNullAcquiredDimension);
507
+ rrddim_set_by_pointer(RS, ChartUnderReplication, TS.TrainingResultChartUnderReplication);
508
+
509
+ rrdset_done(RS);
510
+ }
511
}
ml/ADCharts.h
+4
-6
@@ -3,20 +3,18 @@
3
#ifndef ML_ADCHARTS_H
4
#define ML_ADCHARTS_H
5
6
+#include "Stats.h"
7
#include "ml-private.h"
8
9
namespace ml {
10
10
-void updateDimensionsChart(RRDHOST *RH,
11
- collected_number NumTrainedDimensions,
12
- collected_number NumNormalDimensions,
13
- collected_number NumAnomalousDimensions);
11
+void updateDimensionsChart(RRDHOST *RH, const MachineLearningStats &MLS);
12
13
void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate);
14
17
-void updateDetectionChart(RRDHOST *RH);
15
+void updateResourceUsageCharts(RRDHOST *RH, const struct rusage &PredictionRU, const struct rusage &TrainingRU);
16
19
-void updateTrainingChart(RRDHOST *RH, struct rusage *TRU);
17
+void updateTrainingStatisticsChart(RRDHOST *RH, const TrainingStats &TS);
18
19
} // namespace ml
20
ml/Chart.cc
ml/Chart.h
new
+128
@@ -0,0 +1,128 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef ML_CHART_H
4
+#define ML_CHART_H
5
+
6
+#include "Config.h"
7
+#include "Dimension.h"
8
+
9
+#include "ml-private.h"
10
+#include "json/single_include/nlohmann/json.hpp"
11
+
12
+namespace ml
13
+{
14
+
15
+class Chart {
16
+public:
17
+ Chart(RRDSET *RS) :
18
+ RS(RS),
19
+ MLS()
20
+ { }
21
+
22
+ RRDSET *getRS() const {
23
+ return RS;
24
+ }
25
+
26
+ bool isAvailableForML() {
27
+ return rrdset_is_available_for_exporting_and_alarms(RS);
28
+ }
29
+
30
+ void addDimension(Dimension *D) {
31
+ std::lock_guard<Mutex> L(M);
32
+ Dimensions[D->getRD()] = D;
33
+ }
34
+
35
+ void removeDimension(Dimension *D) {
36
+ std::lock_guard<Mutex> L(M);
37
+ Dimensions.erase(D->getRD());
38
+ }
39
+
40
+ void getModelsAsJson(nlohmann::json &Json) {
41
+ std::lock_guard<Mutex> L(M);
42
+
43
+ for (auto &DP : Dimensions) {
44
+ Dimension *D = DP.second;
45
+ nlohmann::json JsonArray = nlohmann::json::array();
46
+ for (const KMeans &KM : D->getModels()) {
47
+ nlohmann::json J;
48
+ KM.toJson(J);
49
+ JsonArray.push_back(J);
50
+ }
51
+
52
+ Json[getMLDimensionID(D->getRD())] = JsonArray;
53
+ }
54
+ }
55
+
56
+ void updateBegin() {
57
+ M.lock();
58
+ MLS = {};
59
+ }
60
+
61
+ void updateDimension(Dimension *D, bool IsAnomalous) {
62
+ switch (D->getMLS()) {
63
+ case MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
64
+ MLS.NumMachineLearningStatusDisabledUE++;
65
+ return;
66
+ case MachineLearningStatus::DisabledDueToExcludedChart:
67
+ MLS.NumMachineLearningStatusDisabledSP++;
68
+ return;
69
+ case MachineLearningStatus::Enabled: {
70
+ MLS.NumMachineLearningStatusEnabled++;
71
+
72
+ switch (D->getMT()) {
73
+ case MetricType::Constant:
74
+ MLS.NumMetricTypeConstant++;
75
+ MLS.NumTrainingStatusTrained++;
76
+ MLS.NumNormalDimensions++;
77
+ return;
78
+ case MetricType::Variable:
79
+ MLS.NumMetricTypeVariable++;
80
+ break;
81
+ }
82
+
83
+ switch (D->getTS()) {
84
+ case TrainingStatus::Untrained:
85
+ MLS.NumTrainingStatusUntrained++;
86
+ return;
87
+ case TrainingStatus::PendingWithoutModel:
88
+ MLS.NumTrainingStatusPendingWithoutModel++;
89
+ return;
90
+ case TrainingStatus::Trained:
91
+ MLS.NumTrainingStatusTrained++;
92
+
93
+ MLS.NumAnomalousDimensions += IsAnomalous;
94
+ MLS.NumNormalDimensions += !IsAnomalous;
95
+ return;
96
+ case TrainingStatus::PendingWithModel:
97
+ MLS.NumTrainingStatusPendingWithModel++;
98
+
99
+ MLS.NumAnomalousDimensions += IsAnomalous;
100
+ MLS.NumNormalDimensions += !IsAnomalous;
101
+ return;
102
+ }
103
+
104
+ return;
105
+ }
106
+ }
107
+ }
108
+
109
+ void updateEnd() {
110
+ M.unlock();
111
+ }
112
+
113
+ MachineLearningStats getMLS() {
114
+ std::lock_guard<Mutex> L(M);
115
+ return MLS;
116
+ }
117
+
118
+private:
119
+ RRDSET *RS;
120
+ MachineLearningStats MLS;
121
+
122
+ Mutex M;
123
+ std::unordered_map<RRDDIM *, Dimension *> Dimensions;
124
+};
125
+
126
+} // namespace ml
127
+
128
+#endif /* ML_CHART_H */
ml/Config.cc
+3
-3
@@ -31,7 +31,7 @@ void Config::readMLConfig(void) {
31
unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
32
unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
33
unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
34
- unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24);
34
+ unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1);
35
36
unsigned DiffN = config_get_number(ConfigSectionML, "num samples to diff", 1);
37
unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);
@@ -53,7 +53,7 @@ void Config::readMLConfig(void) {
53
MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
54
MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
55
TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
56
- NumModelsToUse = clamp<unsigned>(TrainEvery, 1, 7 * 24);
56
+ NumModelsToUse = clamp<unsigned>(NumModelsToUse, 1, 7 * 24);
57
58
DiffN = clamp(DiffN, 0u, 1u);
59
SmoothN = clamp(SmoothN, 0u, 5u);
@@ -108,7 +108,7 @@ void Config::readMLConfig(void) {
108
// Always exclude anomaly_detection charts from training.
109
Cfg.ChartsToSkip = "anomaly_detection.* ";
110
Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");
111
- Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
111
+ Cfg.SP_ChartsToSkip = simple_pattern_create(Cfg.ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
112
113
Cfg.StreamADCharts = config_get_boolean(ConfigSectionML, "stream anomaly detection charts", true);
114
}
ml/Config.h
+1
@@ -14,6 +14,7 @@ public:
14
unsigned MaxTrainSamples;
15
unsigned MinTrainSamples;
16
unsigned TrainEvery;
17
+
18
unsigned NumModelsToUse;
19
20
unsigned DBEngineAnomalyRateEvery;
ml/Dimension.cc
+241
-68
@@ -3,171 +3,344 @@
3
#include "Config.h"
4
#include "Dimension.h"
5
#include "Query.h"
6
+#include "Host.h"
7
8
using namespace ml;
9
9
-bool Dimension::isActive() const {
10
- bool SetObsolete = rrdset_flag_check(RD->rrdset, RRDSET_FLAG_OBSOLETE);
11
- bool DimObsolete = rrddim_flag_check(RD, RRDDIM_FLAG_OBSOLETE);
12
- return !SetObsolete && !DimObsolete;
10
+static const char *mls2str(MachineLearningStatus MLS) {
11
+ switch (MLS) {
12
+ case ml::MachineLearningStatus::Enabled:
13
+ return "enabled";
14
+ case ml::MachineLearningStatus::DisabledDueToUniqueUpdateEvery:
15
+ return "disabled-ue";
16
+ case ml::MachineLearningStatus::DisabledDueToExcludedChart:
17
+ return "disabled-sp";
18
+ default:
19
+ return "unknown";
20
+ }
21
+}
22
+
23
+static const char *mt2str(MetricType MT) {
24
+ switch (MT) {
25
+ case ml::MetricType::Constant:
26
+ return "constant";
27
+ case ml::MetricType::Variable:
28
+ return "variable";
29
+ default:
30
+ return "unknown";
31
+ }
32
}
33
15
-std::pair<CalculatedNumber *, size_t> Dimension::getCalculatedNumbers() {
34
+static const char *ts2str(TrainingStatus TS) {
35
+ switch (TS) {
36
+ case ml::TrainingStatus::PendingWithModel:
37
+ return "pending-with-model";
38
+ case ml::TrainingStatus::PendingWithoutModel:
39
+ return "pending-without-model";
40
+ case ml::TrainingStatus::Trained:
41
+ return "trained";
42
+ case ml::TrainingStatus::Untrained:
43
+ return "untrained";
44
+ default:
45
+ return "unknown";
46
+ }
47
+}
48
+
49
+static const char *tr2str(TrainingResult TR) {
50
+ switch (TR) {
51
+ case ml::TrainingResult::Ok:
52
+ return "ok";
53
+ case ml::TrainingResult::InvalidQueryTimeRange:
54
+ return "invalid-query";
55
+ case ml::TrainingResult::NotEnoughCollectedValues:
56
+ return "missing-values";
57
+ case ml::TrainingResult::NullAcquiredDimension:
58
+ return "null-acquired-dim";
59
+ case ml::TrainingResult::ChartUnderReplication:
60
+ return "chart-under-replication";
61
+ default:
62
+ return "unknown";
63
+ }
64
+}
65
+
66
+std::pair<CalculatedNumber *, TrainingResponse> Dimension::getCalculatedNumbers(const TrainingRequest &TrainingReq) {
67
+ TrainingResponse TrainingResp = {};
68
+
69
+ TrainingResp.RequestTime = TrainingReq.RequestTime;
70
+ TrainingResp.FirstEntryOnRequest = TrainingReq.FirstEntryOnRequest;
71
+ TrainingResp.LastEntryOnRequest = TrainingReq.LastEntryOnRequest;
72
+
73
+ TrainingResp.FirstEntryOnResponse = rrddim_first_entry_t_of_tier(RD, 0);
74
+ TrainingResp.LastEntryOnResponse = rrddim_last_entry_t_of_tier(RD, 0);
75
+
76
size_t MinN = Cfg.MinTrainSamples;
77
size_t MaxN = Cfg.MaxTrainSamples;
78
79
// Figure out what our time window should be.
20
- time_t BeforeT = now_realtime_sec() - 1;
21
- time_t AfterT = BeforeT - (MaxN * updateEvery());
22
-
23
- BeforeT -= (BeforeT % updateEvery());
24
- AfterT -= (AfterT % updateEvery());
25
-
26
- BeforeT = std::min(BeforeT, latestTime());
27
- AfterT = std::max(AfterT, oldestTime());
80
+ TrainingResp.QueryBeforeT = TrainingResp.LastEntryOnResponse;
81
+ TrainingResp.QueryAfterT = std::max(
82
+ TrainingResp.QueryBeforeT - static_cast<time_t>((MaxN - 1) * updateEvery()),
83
+ TrainingResp.FirstEntryOnResponse
84
+ );
85
+
86
+ if (TrainingResp.QueryAfterT >= TrainingResp.QueryBeforeT) {
87
+ TrainingResp.Result = TrainingResult::InvalidQueryTimeRange;
88
+ return { nullptr, TrainingResp };
89
+ }
90
29
- if (AfterT >= BeforeT)
30
- return { nullptr, 0 };
91
+ if (rrdset_is_replicating(RD->rrdset)) {
92
+ TrainingResp.Result = TrainingResult::ChartUnderReplication;
93
+ return { nullptr, TrainingResp };
94
+ }
95
96
CalculatedNumber *CNs = new CalculatedNumber[MaxN * (Cfg.LagN + 1)]();
97
98
// Start the query.
35
- unsigned Idx = 0;
36
- unsigned CollectedValues = 0;
37
- unsigned TotalValues = 0;
99
+ size_t Idx = 0;
100
101
CalculatedNumber LastValue = std::numeric_limits<CalculatedNumber>::quiet_NaN();
102
Query Q = Query(getRD());
103
42
- Q.init(AfterT, BeforeT);
104
+ Q.init(TrainingResp.QueryAfterT, TrainingResp.QueryBeforeT);
105
while (!Q.isFinished()) {
106
if (Idx == MaxN)
107
break;
108
109
auto P = Q.nextMetric();
110
+
111
CalculatedNumber Value = P.second;
112
113
if (netdata_double_isnumber(Value)) {
114
+ if (!TrainingResp.DbAfterT)
115
+ TrainingResp.DbAfterT = P.first;
116
+ TrainingResp.DbBeforeT = P.first;
117
+
118
CNs[Idx] = Value;
119
LastValue = CNs[Idx];
53
- CollectedValues++;
120
+ TrainingResp.CollectedValues++;
121
} else
122
CNs[Idx] = LastValue;
123
124
Idx++;
125
}
59
- TotalValues = Idx;
126
+ TrainingResp.TotalValues = Idx;
127
+
128
+ if (TrainingResp.CollectedValues < MinN) {
129
+ TrainingResp.Result = TrainingResult::NotEnoughCollectedValues;
130
61
- if (CollectedValues < MinN) {
131
delete[] CNs;
63
- return { nullptr, 0 };
132
+ return { nullptr, TrainingResp };
133
}
134
135
// Find first non-NaN value.
67
- for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TotalValues--) { }
136
+ for (Idx = 0; std::isnan(CNs[Idx]); Idx++, TrainingResp.TotalValues--) { }
137
138
// Overwrite NaN values.
139
if (Idx != 0)
71
- memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TotalValues);
140
+ memmove(CNs, &CNs[Idx], sizeof(CalculatedNumber) * TrainingResp.TotalValues);
141
73
- return { CNs, TotalValues };
142
+ TrainingResp.Result = TrainingResult::Ok;
143
+ return { CNs, TrainingResp };
144
}
145
76
-MLResult Dimension::trainModel() {
77
- auto P = getCalculatedNumbers();
146
+TrainingResult Dimension::trainModel(const TrainingRequest &TrainingReq) {
147
+ auto P = getCalculatedNumbers(TrainingReq);
148
CalculatedNumber *CNs = P.first;
79
- unsigned N = P.second;
149
+ TrainingResponse TrainingResp = P.second;
150
+
151
+ if (TrainingResp.Result != TrainingResult::Ok) {
152
+ std::lock_guard<Mutex> L(M);
153
+
154
+ MT = MetricType::Constant;
155
+
156
+ switch (TS) {
157
+ case TrainingStatus::PendingWithModel:
158
+ TS = TrainingStatus::Trained;
159
+ break;
160
+ case TrainingStatus::PendingWithoutModel:
161
+ TS = TrainingStatus::Untrained;
162
+ break;
163
+ default:
164
+ break;
165
+ }
166
+
167
+ TR = TrainingResp;
168
81
- if (!CNs)
82
- return MLResult::MissingData;
169
+ LastTrainingTime = TrainingResp.LastEntryOnResponse;
170
+ return TrainingResp.Result;
171
+ }
172
173
+ unsigned N = TrainingResp.TotalValues;
174
unsigned TargetNumSamples = Cfg.MaxTrainSamples * Cfg.RandomSamplingRatio;
175
double SamplingRatio = std::min(static_cast<double>(TargetNumSamples) / N, 1.0);
176
177
SamplesBuffer SB = SamplesBuffer(CNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
178
SamplingRatio, Cfg.RandomNums);
89
- std::vector<DSample> Samples = SB.preprocess();
179
+ std::vector<DSample> Samples;
180
+ SB.preprocess(Samples);
181
182
KMeans KM;
183
KM.train(Samples, Cfg.MaxKMeansIters);
184
185
{
95
- std::lock_guard<std::mutex> Lock(Mutex);
96
- Models[0] = KM;
97
- }
186
+ std::lock_guard<Mutex> L(M);
187
99
- Trained = true;
100
- ConstantModel = true;
188
+ if (Models.size() < Cfg.NumModelsToUse) {
189
+ Models.push_back(std::move(KM));
190
+ } else {
191
+ std::rotate(std::begin(Models), std::begin(Models) + 1, std::end(Models));
192
+ Models[Models.size() - 1] = std::move(KM);
193
+ }
194
+
195
+ MT = MetricType::Constant;
196
+ TS = TrainingStatus::Trained;
197
+ TR = TrainingResp;
198
+ LastTrainingTime = rrddim_last_entry_t(RD);
199
+ }
200
201
delete[] CNs;
103
- return MLResult::Success;
202
+ return TrainingResp.Result;
203
}
204
106
-bool Dimension::shouldTrain(const TimePoint &TP) const {
107
- if (ConstantModel)
108
- return false;
205
+void Dimension::scheduleForTraining(time_t CurrT) {
206
+ switch (MT) {
207
+ case MetricType::Constant: {
208
+ return;
209
+ } default:
210
+ break;
211
+ }
212
110
- return (LastTrainedAt + Seconds(Cfg.TrainEvery * updateEvery())) < TP;
213
+ switch (TS) {
214
+ case TrainingStatus::PendingWithModel:
215
+ case TrainingStatus::PendingWithoutModel:
216
+ break;
217
+ case TrainingStatus::Untrained: {
218
+ Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
219
+ TS = TrainingStatus::PendingWithoutModel;
220
+ H->scheduleForTraining(getTrainingRequest(CurrT));
221
+ break;
222
+ }
223
+ case TrainingStatus::Trained: {
224
+ bool NeedsTraining = LastTrainingTime + (Cfg.TrainEvery * updateEvery()) < CurrT;
225
+
226
+ if (NeedsTraining) {
227
+ Host *H = reinterpret_cast<Host *>(RD->rrdset->rrdhost->ml_host);
228
+ TS = TrainingStatus::PendingWithModel;
229
+ H->scheduleForTraining(getTrainingRequest(CurrT));
230
+ }
231
+ break;
232
+ }
233
+ }
234
}
235
113
-bool Dimension::predict(CalculatedNumber Value, bool Exists) {
236
+bool Dimension::predict(time_t CurrT, CalculatedNumber Value, bool Exists) {
237
+ // Nothing to do if ML is disabled for this dimension
238
+ if (MLS != MachineLearningStatus::Enabled)
239
+ return false;
240
+
241
+ // Don't treat values that don't exist as anomalous
242
if (!Exists) {
243
CNs.clear();
116
- AnomalyBit = false;
244
return false;
245
}
246
247
+ // Save the value and return if we don't have enough values for a sample
248
unsigned N = Cfg.DiffN + Cfg.SmoothN + Cfg.LagN;
249
if (CNs.size() < N) {
250
CNs.push_back(Value);
123
- AnomalyBit = false;
251
return false;
252
}
253
254
+ // Push the value and check if it's different from the last one
255
+ bool SameValue = true;
256
std::rotate(std::begin(CNs), std::begin(CNs) + 1, std::end(CNs));
128
-
257
if (CNs[N - 1] != Value)
130
- ConstantModel = false;
131
-
258
+ SameValue = false;
259
CNs[N - 1] = Value;
260
134
- if (!isTrained() || ConstantModel) {
135
- AnomalyBit = false;
136
- return false;
137
- }
138
-
139
- CalculatedNumber *TmpCNs = new CalculatedNumber[N * (Cfg.LagN + 1)]();
261
+ // Create the sample
262
+ CalculatedNumber TmpCNs[N * (Cfg.LagN + 1)];
263
+ memset(TmpCNs, 0, N * (Cfg.LagN + 1) * sizeof(CalculatedNumber));
264
std::memcpy(TmpCNs, CNs.data(), N * sizeof(CalculatedNumber));
265
SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1,
266
Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
267
1.0, Cfg.RandomNums);
144
- const DSample Sample = SB.preprocess().back();
145
- delete[] TmpCNs;
268
+ SB.preprocess(Feature);
269
147
- std::unique_lock<std::mutex> Lock(Mutex, std::defer_lock);
148
- if (!Lock.try_lock()) {
149
- AnomalyBit = false;
270
+ /*
271
+ * Lock to predict and possibly schedule the dimension for training
272
+ */
273
+
274
+ std::unique_lock<Mutex> L(M, std::defer_lock);
275
+ if (!L.try_lock()) {
276
return false;
277
}
278
279
+ // Mark the metric time as variable if we received different values
280
+ if (!SameValue)
281
+ MT = MetricType::Variable;
282
+
283
+ // Decide if the dimension needs to be scheduled for training
284
+ scheduleForTraining(CurrT);
285
+
286
+ // Nothing to do if we don't have a model
287
+ switch (TS) {
288
+ case TrainingStatus::Untrained:
289
+ case TrainingStatus::PendingWithoutModel:
290
+ return false;
291
+ default:
292
+ break;
293
+ }
294
+
295
+ /*
296
+ * Use the KMeans models to check if the value is anomalous
297
+ */
298
+
299
+ size_t ModelsConsulted = 0;
300
+ size_t Sum = 0;
301
+
302
for (const auto &KM : Models) {
154
- double AnomalyScore = KM.anomalyScore(Sample);
155
- if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN()) {
156
- AnomalyBit = false;
303
+ ModelsConsulted++;
304
+
305
+ double AnomalyScore = KM.anomalyScore(Feature);
306
+ if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN())
307
continue;
158
- }
308
309
if (AnomalyScore < (100 * Cfg.DimensionAnomalyScoreThreshold)) {
161
- AnomalyBit = false;
310
+ global_statistics_ml_models_consulted(ModelsConsulted);
311
return false;
312
}
313
+
314
+ Sum += 1;
315
}
316
166
- AnomalyBit = true;
167
- return true;
317
+ global_statistics_ml_models_consulted(ModelsConsulted);
318
+ return Sum;
319
}
320
170
-std::array<KMeans, 1> Dimension::getModels() {
171
- std::unique_lock<std::mutex> Lock(Mutex);
321
+std::vector<KMeans> Dimension::getModels() {
322
+ std::unique_lock<Mutex> L(M);
323
return Models;
324
}
325
+
326
+void Dimension::dump() const {
327
+ const char *ChartId = rrdset_id(RD->rrdset);
328
+ const char *DimensionId = rrddim_id(RD);
329
+
330
+ const char *MLS_Str = mls2str(MLS);
331
+ const char *MT_Str = mt2str(MT);
332
+ const char *TS_Str = ts2str(TS);
333
+ const char *TR_Str = tr2str(TR.Result);
334
+
335
+ const char *fmt =
336
+ "[ML] %s.%s: MLS=%s, MT=%s, TS=%s, Result=%s, "
337
+ "ReqTime=%ld, FEOReq=%ld, LEOReq=%ld, "
338
+ "FEOResp=%ld, LEOResp=%ld, QTR=<%ld, %ld>, DBTR=<%ld, %ld>, Collected=%zu, Total=%zu";
339
+
340
+ error(fmt,
341
+ ChartId, DimensionId, MLS_Str, MT_Str, TS_Str, TR_Str,
342
+ TR.RequestTime, TR.FirstEntryOnRequest, TR.LastEntryOnRequest,
343
+ TR.FirstEntryOnResponse, TR.LastEntryOnResponse,
344
+ TR.QueryAfterT, TR.QueryBeforeT, TR.DbAfterT, TR.DbBeforeT, TR.CollectedValues, TR.TotalValues
345
+ );
346
+}
ml/Dimension.h
+141
-37
@@ -3,6 +3,8 @@
3
#ifndef ML_DIMENSION_H
4
#define ML_DIMENSION_H
5
6
+#include "Mutex.h"
7
+#include "Stats.h"
8
#include "Query.h"
9
#include "Config.h"
10
@@ -10,12 +12,6 @@
12
13
namespace ml {
14
13
-enum class MLResult {
14
- Success = 0,
15
- MissingData,
16
- NaN,
17
-};
18
-
15
static inline std::string getMLDimensionID(RRDDIM *RD) {
16
RRDSET *RS = RD->rrdset;
17
@@ -24,16 +20,118 @@ static inline std::string getMLDimensionID(RRDDIM *RD) {
20
return SS.str();
21
}
22
23
+enum class MachineLearningStatus {
24
+ // Enable training/prediction
25
+ Enabled,
26
+
27
+ // Disable due to update every being different from the host's
28
+ DisabledDueToUniqueUpdateEvery,
29
+
30
+ // Disable because configuration pattern matches the chart's id
31
+ DisabledDueToExcludedChart,
32
+};
33
+
34
+enum class TrainingStatus {
35
+ // We don't have a model for this dimension
36
+ Untrained,
37
+
38
+ // Request for training sent, but we don't have any models yet
39
+ PendingWithoutModel,
40
+
41
+ // Request to update existing models sent
42
+ PendingWithModel,
43
+
44
+ // Have a valid, up-to-date model
45
+ Trained,
46
+};
47
+
48
+enum class MetricType {
49
+ // The dimension has constant values, no need to train
50
+ Constant,
51
+
52
+ // The dimension's values fluctuate, we need to generate a model
53
+ Variable,
54
+};
55
+
56
+struct TrainingRequest {
57
+ // Chart/dimension we want to train
58
+ STRING *ChartId;
59
+ STRING *DimensionId;
60
+
61
+ // Creation time of request
62
+ time_t RequestTime;
63
+
64
+ // First/last entry of this dimension in DB
65
+ // at the point the request was made
66
+ time_t FirstEntryOnRequest;
67
+ time_t LastEntryOnRequest;
68
+};
69
+
70
+void dumpTrainingRequest(const TrainingRequest &TrainingReq, const char *Prefix);
71
+
72
+enum TrainingResult {
73
+ // We managed to create a KMeans model
74
+ Ok,
75
+ // Could not query DB with a correct time range
76
+ InvalidQueryTimeRange,
77
+ // Did not gather enough data from DB to run KMeans
78
+ NotEnoughCollectedValues,
79
+ // Acquired a null dimension
80
+ NullAcquiredDimension,
81
+ // Chart is under replication
82
+ ChartUnderReplication,
83
+};
84
+
85
+struct TrainingResponse {
86
+ // Time when the request for this response was made
87
+ time_t RequestTime;
88
+
89
+ // First/last entry of the dimension in DB when generating the request
90
+ time_t FirstEntryOnRequest;
91
+ time_t LastEntryOnRequest;
92
+
93
+ // First/last entry of the dimension in DB when generating the response
94
+ time_t FirstEntryOnResponse;
95
+ time_t LastEntryOnResponse;
96
+
97
+ // After/Before timestamps of our DB query
98
+ time_t QueryAfterT;
99
+ time_t QueryBeforeT;
100
+
101
+ // Actual after/before returned by the DB query ops
102
+ time_t DbAfterT;
103
+ time_t DbBeforeT;
104
+
105
+ // Number of doubles returned by the DB query
106
+ size_t CollectedValues;
107
+
108
+ // Number of values we return to the caller
109
+ size_t TotalValues;
110
+
111
+ // Result of training response
112
+ TrainingResult Result;
113
+};
114
+
115
+void dumpTrainingResponse(const TrainingResponse &TrainingResp, const char *Prefix);
116
+
117
class Dimension {
118
public:
119
Dimension(RRDDIM *RD) :
120
RD(RD),
31
- LastTrainedAt(Seconds(0)),
32
- Trained(false),
33
- ConstantModel(false),
34
- AnomalyScore(0.0),
35
- AnomalyBit(0)
36
- { }
121
+ MT(MetricType::Constant),
122
+ TS(TrainingStatus::Untrained),
123
+ TR(),
124
+ LastTrainingTime(0)
125
+ {
126
+ if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RD->rrdset)))
127
+ MLS = MachineLearningStatus::DisabledDueToExcludedChart;
128
+ else if (RD->update_every != RD->rrdset->rrdhost->rrd_update_every)
129
+ MLS = MachineLearningStatus::DisabledDueToUniqueUpdateEvery;
130
+ else
131
+ MLS = MachineLearningStatus::Enabled;
132
+
133
+ Models.reserve(Cfg.NumModelsToUse);
134
+ }
135
136
RRDDIM *getRD() const {
137
return RD;
@@ -43,50 +141,56 @@ public:
141
return RD->update_every;
142
}
143
46
- time_t latestTime() const {
47
- return Query(RD).latestTime();
48
- }
49
-
50
- time_t oldestTime() const {
51
- return Query(RD).oldestTime();
144
+ MetricType getMT() const {
145
+ return MT;
146
}
147
54
- bool isTrained() const {
55
- return Trained;
148
+ TrainingStatus getTS() const {
149
+ return TS;
150
}
151
58
- bool isAnomalous() const {
59
- return AnomalyBit;
152
+ MachineLearningStatus getMLS() const {
153
+ return MLS;
154
}
155
62
- bool shouldTrain(const TimePoint &TP) const;
156
+ TrainingResult trainModel(const TrainingRequest &TR);
157
64
- bool isActive() const;
158
+ void scheduleForTraining(time_t CurrT);
159
66
- MLResult trainModel();
160
+ bool predict(time_t CurrT, CalculatedNumber Value, bool Exists);
161
68
- bool predict(CalculatedNumber Value, bool Exists);
162
+ std::vector<KMeans> getModels();
163
+
164
+ void dump() const;
165
70
- std::pair<bool, double> detect(size_t WindowLength, bool Reset);
71
-
72
- std::array<KMeans, 1> getModels();
166
+private:
167
+ TrainingRequest getTrainingRequest(time_t CurrT) const {
168
+ return TrainingRequest {
169
+ string_dup(RD->rrdset->id),
170
+ string_dup(RD->id),
171
+ CurrT,
172
+ rrddim_first_entry_t(RD),
173
+ rrddim_last_entry_t(RD)
174
+ };
175
+ }
176
177
private:
75
- std::pair<CalculatedNumber *, size_t> getCalculatedNumbers();
178
+ std::pair<CalculatedNumber *, TrainingResponse> getCalculatedNumbers(const TrainingRequest &TrainingReq);
179
180
public:
181
RRDDIM *RD;
182
+ MetricType MT;
183
+ TrainingStatus TS;
184
+ TrainingResponse TR;
185
80
- TimePoint LastTrainedAt;
81
- std::atomic<bool> Trained;
82
- std::atomic<bool> ConstantModel;
186
+ time_t LastTrainingTime;
187
84
- CalculatedNumber AnomalyScore;
85
- std::atomic<bool> AnomalyBit;
188
+ MachineLearningStatus MLS;
189
190
std::vector<CalculatedNumber> CNs;
88
- std::array<KMeans, 1> Models;
89
- std::mutex Mutex;
191
+ DSample Feature;
192
+ std::vector<KMeans> Models;
193
+ Mutex M;
194
};
195
196
} // namespace ml
ml/Host.cc
+228
-165
@@ -2,42 +2,24 @@
2
3
#include "Config.h"
4
#include "Host.h"
5
+#include "Queue.h"
6
#include "ADCharts.h"
7
8
#include "json/single_include/nlohmann/json.hpp"
9
10
using namespace ml;
11
11
-void RrdHost::addDimension(Dimension *D) {
12
- std::lock_guard<std::mutex> Lock(Mutex);
13
-
14
- DimensionsMap[D->getRD()] = D;
15
-
16
- // Default construct mutex for dimension
17
- LocksMap[D];
12
+void Host::addChart(Chart *C) {
13
+ std::lock_guard<Mutex> L(M);
14
+ Charts[C->getRS()] = C;
15
}
16
20
-void RrdHost::removeDimension(Dimension *D) {
21
- // Remove the dimension from the hosts map.
22
- {
23
- std::lock_guard<std::mutex> Lock(Mutex);
24
- DimensionsMap.erase(D->getRD());
25
- }
26
-
27
- // Delete the dimension by locking the mutex that protects it.
28
- {
29
- std::lock_guard<std::mutex> Lock(LocksMap[D]);
30
- delete D;
31
- }
32
-
33
- // Remove the lock entry for the deleted dimension.
34
- {
35
- std::lock_guard<std::mutex> Lock(Mutex);
36
- LocksMap.erase(D);
37
- }
17
+void Host::removeChart(Chart *C) {
18
+ std::lock_guard<Mutex> L(M);
19
+ Charts.erase(C->getRS());
20
}
21
40
-void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
22
+void Host::getConfigAsJson(nlohmann::json &Json) const {
23
Json["version"] = 1;
24
25
Json["enabled"] = Cfg.EnableAnomalyDetection;
@@ -63,193 +45,274 @@ void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
45
Json["charts-to-skip"] = Cfg.ChartsToSkip;
46
}
47
66
-void TrainableHost::getModelsAsJson(nlohmann::json &Json) {
67
- std::lock_guard<std::mutex> Lock(Mutex);
48
+void Host::getModelsAsJson(nlohmann::json &Json) {
49
+ std::lock_guard<Mutex> L(M);
50
69
- for (auto &DP : DimensionsMap) {
70
- Dimension *D = DP.second;
71
-
72
- nlohmann::json JsonArray = nlohmann::json::array();
73
- for (const KMeans &KM : D->getModels()) {
74
- nlohmann::json J;
75
- KM.toJson(J);
76
- JsonArray.push_back(J);
77
- }
78
- Json[getMLDimensionID(D->getRD())] = JsonArray;
51
+ for (auto &CP : Charts) {
52
+ Chart *C = CP.second;
53
+ C->getModelsAsJson(Json);
54
}
80
-
81
- return;
55
}
56
84
-std::pair<Dimension *, Duration<double>>
85
-TrainableHost::findDimensionToTrain(const TimePoint &NowTP) {
86
- std::lock_guard<std::mutex> Lock(Mutex);
87
-
88
- Duration<double> AllottedDuration = Duration<double>{Cfg.TrainEvery * updateEvery()} / (DimensionsMap.size() + 1);
89
-
90
- for (auto &DP : DimensionsMap) {
91
- Dimension *D = DP.second;
57
+void Host::detectOnce() {
58
+ MLS = {};
59
+ MachineLearningStats MLSCopy = {};
60
+ TrainingStats TSCopy = {};
61
93
- if (D->shouldTrain(NowTP)) {
94
- LocksMap[D].lock();
95
- return { D, AllottedDuration };
96
- }
97
- }
62
+ {
63
+ std::lock_guard<Mutex> L(M);
64
99
- return { nullptr, AllottedDuration };
100
-}
65
+ /*
66
+ * prediction/detection stats
67
+ */
68
+ for (auto &CP : Charts) {
69
+ Chart *C = CP.second;
70
102
-void TrainableHost::trainDimension(Dimension *D, const TimePoint &NowTP) {
103
- if (D == nullptr)
104
- return;
71
+ if (!C->isAvailableForML())
72
+ continue;
73
106
- D->LastTrainedAt = NowTP + Seconds{D->updateEvery()};
107
- D->trainModel();
74
+ MachineLearningStats ChartMLS = C->getMLS();
75
109
- {
110
- std::lock_guard<std::mutex> Lock(Mutex);
111
- LocksMap[D].unlock();
112
- }
113
-}
114
-
115
-void TrainableHost::train() {
116
- Duration<double> MaxSleepFor = Seconds{10 * updateEvery()};
76
+ MLS.NumMachineLearningStatusEnabled += ChartMLS.NumMachineLearningStatusEnabled;
77
+ MLS.NumMachineLearningStatusDisabledUE += ChartMLS.NumMachineLearningStatusDisabledUE;
78
+ MLS.NumMachineLearningStatusDisabledSP += ChartMLS.NumMachineLearningStatusDisabledSP;
79
118
- worker_register("MLTRAIN");
119
- worker_register_job_name(0, "dimensions");
80
+ MLS.NumMetricTypeConstant += ChartMLS.NumMetricTypeConstant;
81
+ MLS.NumMetricTypeVariable += ChartMLS.NumMetricTypeVariable;
82
121
- worker_is_busy(0);
122
- while (!netdata_exit) {
123
- netdata_thread_testcancel();
124
- netdata_thread_disable_cancelability();
83
+ MLS.NumTrainingStatusUntrained += ChartMLS.NumTrainingStatusUntrained;
84
+ MLS.NumTrainingStatusPendingWithoutModel += ChartMLS.NumTrainingStatusPendingWithoutModel;
85
+ MLS.NumTrainingStatusTrained += ChartMLS.NumTrainingStatusTrained;
86
+ MLS.NumTrainingStatusPendingWithModel += ChartMLS.NumTrainingStatusPendingWithModel;
87
126
- updateResourceUsage();
88
+ MLS.NumAnomalousDimensions += ChartMLS.NumAnomalousDimensions;
89
+ MLS.NumNormalDimensions += ChartMLS.NumNormalDimensions;
90
+ }
91
128
- TimePoint NowTP = SteadyClock::now();
92
+ HostAnomalyRate = 0.0;
93
+ size_t NumActiveDimensions = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
94
+ if (NumActiveDimensions)
95
+ HostAnomalyRate = static_cast<double>(MLS.NumAnomalousDimensions) / NumActiveDimensions;
96
130
- auto P = findDimensionToTrain(NowTP);
131
- trainDimension(P.first, NowTP);
97
+ MLSCopy = MLS;
98
133
- netdata_thread_enable_cancelability();
99
+ /*
100
+ * training stats
101
+ */
102
+ TSCopy = TS;
103
135
- Duration<double> AllottedDuration = P.second;
136
- Duration<double> RealDuration = SteadyClock::now() - NowTP;
104
+ TS.QueueSize = 0;
105
+ TS.NumPoppedItems = 0;
106
138
- Duration<double> SleepFor;
139
- if (RealDuration >= AllottedDuration)
140
- continue;
107
+ TS.AllottedUT = 0;
108
+ TS.ConsumedUT = 0;
109
+ TS.RemainingUT = 0;
110
142
- worker_is_idle();
143
- SleepFor = std::min(AllottedDuration - RealDuration, MaxSleepFor);
144
- TimePoint Now = SteadyClock::now();
145
- auto Until = Now + SleepFor;
146
- while (Now < Until && !netdata_exit) {
147
- std::this_thread::sleep_for(std::chrono::milliseconds(1000));
148
- Now = SteadyClock::now();
149
- }
150
- worker_is_busy(0);
111
+ TS.TrainingResultOk = 0;
112
+ TS.TrainingResultInvalidQueryTimeRange = 0;
113
+ TS.TrainingResultNotEnoughCollectedValues = 0;
114
+ TS.TrainingResultNullAcquiredDimension = 0;
115
+ TS.TrainingResultChartUnderReplication = 0;
116
}
152
-}
153
-
154
-#define WORKER_JOB_DETECT_DIMENSION 0
155
-#define WORKER_JOB_UPDATE_DETECTION_CHART 1
156
-#define WORKER_JOB_UPDATE_ANOMALY_RATES 2
157
-#define WORKER_JOB_UPDATE_CHARTS 3
117
159
-#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
160
-#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
161
-#endif
162
-
163
-void DetectableHost::detectOnce() {
164
- size_t NumAnomalousDimensions = 0;
165
- size_t NumNormalDimensions = 0;
166
- size_t NumTrainedDimensions = 0;
167
- size_t NumActiveDimensions = 0;
118
+ // Calc the avg values
119
+ if (TSCopy.NumPoppedItems) {
120
+ TSCopy.QueueSize /= TSCopy.NumPoppedItems;
121
+ TSCopy.AllottedUT /= TSCopy.NumPoppedItems;
122
+ TSCopy.ConsumedUT /= TSCopy.NumPoppedItems;
123
+ TSCopy.RemainingUT /= TSCopy.NumPoppedItems;
124
+
125
+ TSCopy.TrainingResultOk /= TSCopy.NumPoppedItems;
126
+ TSCopy.TrainingResultInvalidQueryTimeRange /= TSCopy.NumPoppedItems;
127
+ TSCopy.TrainingResultNotEnoughCollectedValues /= TSCopy.NumPoppedItems;
128
+ TSCopy.TrainingResultNullAcquiredDimension /= TSCopy.NumPoppedItems;
129
+ TSCopy.TrainingResultChartUnderReplication /= TSCopy.NumPoppedItems;
130
+ } else {
131
+ TSCopy.QueueSize = 0;
132
+ TSCopy.AllottedUT = 0;
133
+ TSCopy.ConsumedUT = 0;
134
+ TSCopy.RemainingUT = 0;
135
+ }
136
169
- {
170
- std::lock_guard<std::mutex> Lock(Mutex);
137
+ updateDimensionsChart(RH, MLSCopy);
138
+ updateHostAndDetectionRateCharts(RH, HostAnomalyRate * 10000.0);
139
172
- for (auto &DP : DimensionsMap) {
173
- worker_is_busy(WORKER_JOB_DETECT_DIMENSION);
140
+ struct rusage PredictionRU;
141
+ getrusage(RUSAGE_THREAD, &PredictionRU);
142
+ updateResourceUsageCharts(RH, PredictionRU, TSCopy.TrainingRU);
143
175
- Dimension *D = DP.second;
144
+ updateTrainingStatisticsChart(RH, TSCopy);
145
+}
146
177
- if (!D->isActive())
178
- continue;
147
+class AcquiredDimension {
148
+public:
149
+ static AcquiredDimension find(RRDHOST *RH, STRING *ChartId, STRING *DimensionId) {
150
+ RRDDIM_ACQUIRED *AcqRD = nullptr;
151
+ Dimension *D = nullptr;
152
+
153
+ rrdhost_rdlock(RH);
154
+ RRDSET *RS = rrdset_find(RH, string2str(ChartId));
155
+ if (RS) {
156
+ AcqRD = rrddim_find_and_acquire(RS, string2str(DimensionId));
157
+ if (AcqRD) {
158
+ RRDDIM *RD = rrddim_acquired_to_rrddim(AcqRD);
159
+ if (RD)
160
+ D = reinterpret_cast<Dimension *>(RD->ml_dimension);
161
+ }
162
+ }
163
+ rrdhost_unlock(RH);
164
180
- NumActiveDimensions++;
181
- NumTrainedDimensions += D->isTrained();
165
+ return AcquiredDimension(AcqRD, D);
166
+ }
167
183
- bool IsAnomalous = D->isAnomalous();
184
- if (IsAnomalous)
185
- NumAnomalousDimensions += 1;
186
- }
168
+private:
169
+ AcquiredDimension(RRDDIM_ACQUIRED *AcqRD, Dimension *D) : AcqRD(AcqRD), D(D) {}
170
188
- if (NumAnomalousDimensions)
189
- HostAnomalyRate = static_cast<double>(NumAnomalousDimensions) / NumActiveDimensions;
190
- else
191
- HostAnomalyRate = 0.0;
171
+public:
172
+ TrainingResult train(const TrainingRequest &TR) {
173
+ if (!D)
174
+ return TrainingResult::NullAcquiredDimension;
175
193
- NumNormalDimensions = NumActiveDimensions - NumAnomalousDimensions;
176
+ return D->trainModel(TR);
177
}
178
196
- this->NumAnomalousDimensions = NumAnomalousDimensions;
197
- this->NumNormalDimensions = NumNormalDimensions;
198
- this->NumTrainedDimensions = NumTrainedDimensions;
199
- this->NumActiveDimensions = NumActiveDimensions;
179
+ ~AcquiredDimension() {
180
+ if (AcqRD)
181
+ rrddim_acquired_release(AcqRD);
182
+ }
183
201
- worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
202
- updateDimensionsChart(getRH(), NumTrainedDimensions, NumNormalDimensions, NumAnomalousDimensions);
203
- updateHostAndDetectionRateCharts(getRH(), HostAnomalyRate * 10000.0);
184
+private:
185
+ RRDDIM_ACQUIRED *AcqRD;
186
+ Dimension *D;
187
+};
188
205
- struct rusage TRU;
206
- getResourceUsage(&TRU);
207
- updateTrainingChart(getRH(), &TRU);
189
+void Host::scheduleForTraining(TrainingRequest TR) {
190
+ TrainingQueue.push(TR);
191
}
192
210
-void DetectableHost::detect() {
211
- worker_register("MLDETECT");
212
- worker_register_job_name(WORKER_JOB_DETECT_DIMENSION, "dimensions");
213
- worker_register_job_name(WORKER_JOB_UPDATE_DETECTION_CHART, "detection chart");
214
- worker_register_job_name(WORKER_JOB_UPDATE_ANOMALY_RATES, "anomaly rates");
215
- worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "charts");
193
+void Host::train() {
194
+ while (!netdata_exit) {
195
+ auto P = TrainingQueue.pop();
196
+ TrainingRequest TrainingReq = P.first;
197
+ size_t Size = P.second;
198
+
199
+ usec_t AllottedUT = (Cfg.TrainEvery * RH->rrd_update_every * USEC_PER_SEC) / Size;
200
+ if (AllottedUT > USEC_PER_SEC)
201
+ AllottedUT = USEC_PER_SEC;
202
+
203
+ usec_t StartUT = now_realtime_usec();
204
+ TrainingResult TrainingRes;
205
+ {
206
+ AcquiredDimension AcqDim = AcquiredDimension::find(RH, TrainingReq.ChartId, TrainingReq.DimensionId);
207
+ TrainingRes = AcqDim.train(TrainingReq);
208
+ string_freez(TrainingReq.ChartId);
209
+ string_freez(TrainingReq.DimensionId);
210
+ }
211
+ usec_t ConsumedUT = now_realtime_usec() - StartUT;
212
+
213
+ usec_t RemainingUT = 0;
214
+ if (ConsumedUT < AllottedUT)
215
+ RemainingUT = AllottedUT - ConsumedUT;
216
+
217
+ {
218
+ std::lock_guard<Mutex> L(M);
219
+
220
+ if (TS.AllottedUT == 0) {
221
+ struct rusage TRU;
222
+ getrusage(RUSAGE_THREAD, &TRU);
223
+ TS.TrainingRU = TRU;
224
+ }
225
+
226
+ TS.QueueSize += Size;
227
+ TS.NumPoppedItems += 1;
228
+
229
+ TS.AllottedUT += AllottedUT;
230
+ TS.ConsumedUT += ConsumedUT;
231
+ TS.RemainingUT += RemainingUT;
232
+
233
+ switch (TrainingRes) {
234
+ case TrainingResult::Ok:
235
+ TS.TrainingResultOk += 1;
236
+ break;
237
+ case TrainingResult::InvalidQueryTimeRange:
238
+ TS.TrainingResultInvalidQueryTimeRange += 1;
239
+ break;
240
+ case TrainingResult::NotEnoughCollectedValues:
241
+ TS.TrainingResultNotEnoughCollectedValues += 1;
242
+ break;
243
+ case TrainingResult::NullAcquiredDimension:
244
+ TS.TrainingResultNullAcquiredDimension += 1;
245
+ break;
246
+ case TrainingResult::ChartUnderReplication:
247
+ TS.TrainingResultChartUnderReplication += 1;
248
+ break;
249
+ }
250
+ }
251
217
- std::this_thread::sleep_for(Seconds{10});
252
+ std::this_thread::sleep_for(std::chrono::microseconds{RemainingUT});
253
+ }
254
+}
255
256
+void Host::detect() {
257
heartbeat_t HB;
258
heartbeat_init(&HB);
259
260
while (!netdata_exit) {
223
- netdata_thread_testcancel();
224
- worker_is_idle();
225
- heartbeat_next(&HB, updateEvery() * USEC_PER_SEC);
226
-
227
- netdata_thread_disable_cancelability();
261
+ heartbeat_next(&HB, RH->rrd_update_every * USEC_PER_SEC);
262
detectOnce();
229
-
230
- worker_is_busy(WORKER_JOB_UPDATE_DETECTION_CHART);
231
- updateDetectionChart(getRH());
232
- netdata_thread_enable_cancelability();
263
}
264
}
265
236
-void DetectableHost::getDetectionInfoAsJson(nlohmann::json &Json) const {
266
+void Host::getDetectionInfoAsJson(nlohmann::json &Json) const {
267
Json["version"] = 1;
238
- Json["anomalous-dimensions"] = NumAnomalousDimensions;
239
- Json["normal-dimensions"] = NumNormalDimensions;
240
- Json["total-dimensions"] = NumAnomalousDimensions + NumNormalDimensions;
241
- Json["trained-dimensions"] = NumTrainedDimensions;
268
+ Json["anomalous-dimensions"] = MLS.NumAnomalousDimensions;
269
+ Json["normal-dimensions"] = MLS.NumNormalDimensions;
270
+ Json["total-dimensions"] = MLS.NumAnomalousDimensions + MLS.NumNormalDimensions;
271
+ Json["trained-dimensions"] = MLS.NumTrainingStatusTrained + MLS.NumTrainingStatusPendingWithModel;
272
+}
273
+
274
+void *train_main(void *Arg) {
275
+ Host *H = reinterpret_cast<Host *>(Arg);
276
+ H->train();
277
+ return nullptr;
278
}
279
244
-void DetectableHost::startAnomalyDetectionThreads() {
245
- TrainingThread = std::thread(&TrainableHost::train, this);
246
- DetectionThread = std::thread(&DetectableHost::detect, this);
280
+void *detect_main(void *Arg) {
281
+ Host *H = reinterpret_cast<Host *>(Arg);
282
+ H->detect();
283
+ return nullptr;
284
}
285
249
-void DetectableHost::stopAnomalyDetectionThreads() {
250
- netdata_thread_cancel(TrainingThread.native_handle());
251
- netdata_thread_cancel(DetectionThread.native_handle());
286
+void Host::startAnomalyDetectionThreads() {
287
+ if (ThreadsRunning) {
288
+ error("Anomaly detections threads for host %s are already-up and running.", rrdhost_hostname(RH));
289
+ return;
290
+ }
291
+
292
+ ThreadsRunning = true;
293
+
294
+ char Tag[NETDATA_THREAD_TAG_MAX + 1];
295
+
296
+ snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "TRAIN[%s]", rrdhost_hostname(RH));
297
+ netdata_thread_create(&TrainingThread, Tag, NETDATA_THREAD_OPTION_JOINABLE, train_main, static_cast<void *>(this));
298
+
299
+ snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "DETECT[%s]", rrdhost_hostname(RH));
300
+ netdata_thread_create(&DetectionThread, Tag, NETDATA_THREAD_OPTION_JOINABLE, detect_main, static_cast<void *>(this));
301
+}
302
+
303
+void Host::stopAnomalyDetectionThreads() {
304
+ if (!ThreadsRunning) {
305
+ error("Anomaly detections threads for host %s have already been stopped.", rrdhost_hostname(RH));
306
+ return;
307
+ }
308
+
309
+ ThreadsRunning = false;
310
+
311
+ // Signal the training queue to stop popping-items
312
+ TrainingQueue.signal();
313
+ netdata_thread_cancel(TrainingThread);
314
+ netdata_thread_join(TrainingThread, nullptr);
315
253
- TrainingThread.join();
254
- DetectionThread.join();
316
+ netdata_thread_cancel(DetectionThread);
317
+ netdata_thread_join(DetectionThread, nullptr);
318
}
ml/Host.h
+31
-66
@@ -3,97 +3,62 @@
3
#ifndef ML_HOST_H
4
#define ML_HOST_H
5
6
+#include "Mutex.h"
7
#include "Config.h"
8
#include "Dimension.h"
9
+#include "Chart.h"
10
+#include "Queue.h"
11
12
#include "ml-private.h"
13
#include "json/single_include/nlohmann/json.hpp"
14
12
-namespace ml {
15
+namespace ml
16
+{
17
14
-class RrdHost {
15
-public:
16
- RrdHost(RRDHOST *RH) : RH(RH) {};
17
-
18
- RRDHOST *getRH() { return RH; }
19
-
20
- unsigned updateEvery() { return RH->rrd_update_every; }
21
-
22
- std::string getUUID() {
23
- char S[UUID_STR_LEN];
24
- uuid_unparse_lower(RH->host_uuid, S);
25
- return S;
26
- }
27
-
28
- void addDimension(Dimension *D);
29
- void removeDimension(Dimension *D);
30
-
31
- void getConfigAsJson(nlohmann::json &Json) const;
32
-
33
- virtual ~RrdHost() {};
18
+class Host {
19
35
-protected:
36
- RRDHOST *RH;
37
-
38
- // Protect dimension and lock maps
39
- std::mutex Mutex;
40
-
41
- std::unordered_map<RRDDIM *, Dimension *> DimensionsMap;
42
- std::unordered_map<Dimension *, std::mutex> LocksMap;
43
-};
20
+friend void* train_main(void *);
21
+friend void *detect_main(void *);
22
45
-class TrainableHost : public RrdHost {
23
public:
47
- TrainableHost(RRDHOST *RH) : RrdHost(RH) {}
24
+ Host(RRDHOST *RH) :
25
+ RH(RH),
26
+ MLS(),
27
+ TS(),
28
+ HostAnomalyRate(0.0),
29
+ ThreadsRunning(false) {}
30
49
- void train();
50
-
51
- void updateResourceUsage() {
52
- std::lock_guard<std::mutex> Lock(ResourceUsageMutex);
53
- getrusage(RUSAGE_THREAD, &ResourceUsage);
54
- }
55
-
56
- void getResourceUsage(struct rusage *RU) {
57
- std::lock_guard<std::mutex> Lock(ResourceUsageMutex);
58
- memcpy(RU, &ResourceUsage, sizeof(struct rusage));
59
- }
31
+ void addChart(Chart *C);
32
+ void removeChart(Chart *C);
33
34
+ void getConfigAsJson(nlohmann::json &Json) const;
35
void getModelsAsJson(nlohmann::json &Json);
62
-
63
-private:
64
- std::pair<Dimension *, Duration<double>> findDimensionToTrain(const TimePoint &NowTP);
65
- void trainDimension(Dimension *D, const TimePoint &NowTP);
66
-
67
- struct rusage ResourceUsage{};
68
- std::mutex ResourceUsageMutex;
69
-};
70
-
71
-class DetectableHost : public TrainableHost {
72
-public:
73
- DetectableHost(RRDHOST *RH) : TrainableHost(RH) {}
36
+ void getDetectionInfoAsJson(nlohmann::json &Json) const;
37
38
void startAnomalyDetectionThreads();
39
void stopAnomalyDetectionThreads();
40
78
- void getDetectionInfoAsJson(nlohmann::json &Json) const;
41
+ void scheduleForTraining(TrainingRequest TR);
42
+ void train();
43
80
-private:
44
void detect();
45
void detectOnce();
46
47
private:
85
- std::thread TrainingThread;
86
- std::thread DetectionThread;
87
-
48
+ RRDHOST *RH;
49
+ MachineLearningStats MLS;
50
+ TrainingStats TS;
51
CalculatedNumber HostAnomalyRate{0.0};
52
+ std::atomic<bool> ThreadsRunning;
53
90
- size_t NumAnomalousDimensions{0};
91
- size_t NumNormalDimensions{0};
92
- size_t NumTrainedDimensions{0};
93
- size_t NumActiveDimensions{0};
94
-};
54
+ Queue<TrainingRequest> TrainingQueue;
55
+
56
+ Mutex M;
57
+ std::unordered_map<RRDSET *, Chart *> Charts;
58
96
-using Host = DetectableHost;
59
+ netdata_thread_t TrainingThread;
60
+ netdata_thread_t DetectionThread;
61
+};
62
63
} // namespace ml
64
ml/Mutex.h
new
+36
@@ -0,0 +1,36 @@
1
+#ifndef ML_MUTEX_H
2
+#define ML_MUTEX_H
3
+
4
+#include "ml-private.h"
5
+
6
+class Mutex {
7
+public:
8
+ Mutex() {
9
+ netdata_mutex_init(&M);
10
+ }
11
+
12
+ void lock() {
13
+ netdata_mutex_lock(&M);
14
+ }
15
+
16
+ void unlock() {
17
+ netdata_mutex_unlock(&M);
18
+ }
19
+
20
+ bool try_lock() {
21
+ return netdata_mutex_trylock(&M) == 0;
22
+ }
23
+
24
+ netdata_mutex_t *inner() {
25
+ return &M;
26
+ }
27
+
28
+ ~Mutex() {
29
+ netdata_mutex_destroy(&M);
30
+ }
31
+
32
+private:
33
+ netdata_mutex_t M;
34
+};
35
+
36
+#endif /* ML_MUTEX_H */
ml/Query.h
+1
-1
@@ -40,7 +40,7 @@ public:
40
std::pair<time_t, CalculatedNumber> nextMetric() {
41
points_read++;
42
STORAGE_POINT sp = Ops->next_metric(&Handle);
43
- return { sp.start_time, sp.sum / sp.count };
43
+ return { sp.end_time, sp.sum / sp.count };
44
}
45
46
private:
ml/Queue.h
new
+59
@@ -0,0 +1,59 @@
1
+#ifndef QUEUE_H
2
+#define QUEUE_H
3
+
4
+#include "ml-private.h"
5
+#include "Mutex.h"
6
+#include <queue>
7
+#include <mutex>
8
+#include <condition_variable>
9
+
10
+template<typename T>
11
+class Queue {
12
+public:
13
+ Queue(void) : Q(), M() {
14
+ pthread_cond_init(&CV, nullptr);
15
+ Exit = false;
16
+ }
17
+
18
+ ~Queue() {
19
+ pthread_cond_destroy(&CV);
20
+ }
21
+
22
+ void push(T t) {
23
+ std::lock_guard<Mutex> L(M);
24
+
25
+ Q.push(t);
26
+ pthread_cond_signal(&CV);
27
+ }
28
+
29
+ std::pair<T, size_t> pop(void) {
30
+ std::lock_guard<Mutex> L(M);
31
+
32
+ while (Q.empty()) {
33
+ pthread_cond_wait(&CV, M.inner());
34
+
35
+ if (Exit)
36
+ pthread_exit(nullptr);
37
+ }
38
+
39
+ T V = Q.front();
40
+ size_t Size = Q.size();
41
+ Q.pop();
42
+
43
+ return { V, Size };
44
+ }
45
+
46
+ void signal() {
47
+ std::lock_guard<Mutex> L(M);
48
+ Exit = true;
49
+ pthread_cond_signal(&CV);
50
+ }
51
+
52
+private:
53
+ std::queue<T> Q;
54
+ Mutex M;
55
+ pthread_cond_t CV;
56
+ std::atomic<bool> Exit;
57
+};
58
+
59
+#endif /* QUEUE_H */
ml/README.md
+15
-61
@@ -8,7 +8,7 @@ keywords: [machine learning, anomaly detection, Netdata ML]
8
9
## Overview
10
11
-As of [`v1.32.0`](https://github.com/netdata/netdata/releases/tag/v1.32.0), Netdata comes with some ML powered [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) capabilities built into it and available to use out of the box, with zero configuration required (ML was enabled by default in `v1.35.0-29-nightly` in [this PR](https://github.com/netdata/netdata/pull/13158), previously it required a one line config change).
11
+As of [`v1.32.0`](https://github.com/netdata/netdata/releases/tag/v1.32.0), Netdata comes with ML powered [anomaly detection](https://en.wikipedia.org/wiki/Anomaly_detection) capabilities built into it and available to use out of the box, with zero configuration required (ML was enabled by default in `v1.35.0-29-nightly` in [this PR](https://github.com/netdata/netdata/pull/13158), previously it required a one line config change).
12
13
🚧 **Note**: If you would like to get involved and help us with some feedback, email us at analytics-ml-team@netdata.cloud, comment on the [beta launch post](https://community.netdata.cloud/t/anomaly-advisor-beta-launch/2717) in the Netdata community, or come join us in the [🤖-ml-powered-monitoring](https://discord.gg/4eRSEUpJnc) channel of the Netdata discord.
14
@@ -99,49 +99,7 @@ An ["anomaly detector"](#anomaly-detector) looks at all anomaly bits of a node.
99
100
Essentially if the ["Node Anomaly Rate"](#node-anomaly-rate) (NAR) passes a defined threshold and stays above that threshold for a persistent amount of time, a "Node [Anomaly Event](#anomaly-event)" will be triggered.
101
102
-These anomaly events are currently exposed via `/api/v1/anomaly_events`
103
-
104
-**Note**: Clicking the link below will likely return an empty list of `[]`. This is the response when no anomaly events exist in the specified range. The example response below is illustrative of what the response would be when one or more anomaly events exist within the range of `after` to `before`.
105
-
106
-https://london.my-netdata.io/api/v1/anomaly_events?after=1638365182000&before=1638365602000
107
-
108
-If an event exists within the window, the result would be a list of start and end times.
109
-
110
-```
111
-[
112
- [
113
- 1638367788,
114
- 1638367851
115
- ]
116
-]
117
-```
118
-
119
-Information about each anomaly event can then be found at the `/api/v1/anomaly_event_info` endpoint (making sure to pass the `after` and `before` params):
120
-
121
-**Note**: If you click the below url you will get a `null` since no such anomaly event exists as the response is just an illustrative example taken from a node that did have such an anomaly event.
122
-
123
-https://london.my-netdata.io/api/v1/anomaly_event_info?after=1638367788&before=1638367851
124
-
125
-```
126
-[
127
- [
128
- 0.66,
129
- "netdata.response_time|max"
130
- ],
131
- [
132
- 0.63,
133
- "netdata.response_time|average"
134
- ],
135
- [
136
- 0.54,
137
- "netdata.requests|requests"
138
- ],
139
- ...
140
-```
141
-
142
-The query returns a list of dimension anomaly rates for all dimensions that were considered part of the detected anomaly event.
143
-
144
-**Note**: We plan to build additional anomaly detection and exploration features into both Netdata Agent and Netdata Cloud. The current endpoints are still under active development to power the upcoming features.
102
+These anomaly events are currently exposed via the `new_anomaly_event` dimension on the `anomaly_detection.anomaly_detection` chart.
103
104
## Configuration
105
@@ -162,6 +120,7 @@ Below is a list of all the available configuration params and their default valu
120
# maximum num samples to train = 14400
121
# minimum num samples to train = 3600
122
# train every = 3600
123
+ # number of models per dimension = 1
124
# dbengine anomaly rate every = 30
125
# num samples to diff = 1
126
# num samples to smooth = 3
@@ -169,12 +128,9 @@ Below is a list of all the available configuration params and their default valu
128
# random sampling ratio = 0.2
129
# maximum number of k-means iterations = 1000
130
# dimension anomaly score threshold = 0.99
172
- # host anomaly rate threshold = 0.01000
173
- # minimum window size = 30.00000
174
- # maximum window size = 600.00000
175
- # idle window size = 30.00000
176
- # window minimum anomaly rate = 0.25000
177
- # anomaly event min dimension rate threshold = 0.05000
131
+ # host anomaly rate threshold = 1.0
132
+ # anomaly detection grouping method = average
133
+ # anomaly detection grouping duration = 300
134
# hosts to skip from training = !*
135
# charts to skip from training = netdata.*
136
```
@@ -221,6 +177,7 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
177
- `maximum num samples to train`: (`3600`/`86400`) This is the maximum amount of time you would like to train each model on. For example, the default of `14400` trains on the preceding 4 hours of data, assuming an `update every` of 1 second.
178
- `minimum num samples to train`: (`900`/`21600`) This is the minimum amount of data required to be able to train a model. For example, the default of `900` implies that once at least 15 minutes of data is available for training, a model is trained, otherwise it is skipped and checked again at the next training run.
179
- `train every`: (`1800`/`21600`) This is how often each model will be retrained. For example, the default of `3600` means that each model is retrained every hour. Note: The training of all models is spread out across the `train every` period for efficiency, so in reality, it means that each model will be trained in a staggered manner within each `train every` period.
180
+- `number of models per dimension`: (`1`/`168`) This is the number of trained models that will be used for scoring. For example the default `number of models per dimension = 1` means that just the most recently trained model (covering up to the most recent `maximum num samples to train` of training data) for the dimension will be used to determine the corresponding anomaly bit. Alternatively, if you have `train every = 3600` and `number of models per dimension = 24` this means that netdata will store and use the last 24 trained models for each dimension when determining the anomaly bit, this means that for the latest feature vector in this configuration to be considered anomalous it would need to look anomalous across _all_ the models trained for that dimension in the last 24 hours. As such, increasing `number of models per dimension` may reduce some false positives since it will result in more models (covering a wider time frame of training) being used during scoring.
181
- `dbengine anomaly rate every`: (`30`/`900`) This is how often netdata will aggregate all the anomaly bits into a single chart (`anomaly_detection.anomaly_rates`). The aggregation into a single chart allows enabling anomaly rate ranking over _all_ metrics with one API call as opposed to a call per chart.
182
- `num samples to diff`: (`0`/`1`) This is a `0` or `1` to determine if you want the model to operate on differences of the raw data or just the raw data. For example, the default of `1` means that we take differences of the raw values. Using differences is more general and works on dimensions that might naturally tend to have some trends or cycles in them that is normal behavior to which we don't want to be too sensitive.
183
- `num samples to smooth`: (`0`/`5`) This is a small integer that controls the amount of smoothing applied as part of the feature processing used by the model. For example, the default of `3` means that the rolling average of the last 3 values is used. Smoothing like this helps the model be a little more robust to spiky types of dimensions that naturally "jump" up or down as part of their normal behavior.
@@ -228,11 +185,9 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
185
- `random sampling ratio`: (`0.2`/`1.0`) This parameter determines how much of the available training data is randomly sampled when training a model. The default of `0.2` means that Netdata will train on a random 20% of training data. This parameter influences cost efficiency. At `0.2` the model is still reasonably trained while minimizing system overhead costs caused by the training.
186
- `maximum number of k-means iterations`: This is a parameter that can be passed to the model to limit the number of iterations in training the k-means model. Vast majority of cases can ignore and leave as default.
187
- `dimension anomaly score threshold`: (`0.01`/`5.00`) This is the threshold at which an individual dimension at a specific timestep is considered anomalous or not. For example, the default of `0.99` means that a dimension with an anomaly score of 99% or higher is flagged as anomalous. This is a normalized probability based on the training data, so the default of 99% means that anything that is as strange (based on distance measure) or more strange as the most strange 1% of data observed during training will be flagged as anomalous. If you wanted to make the anomaly detection on individual dimensions more sensitive you could try a value like `0.90` (90%) or to make it less sensitive you could try `1.5` (150%).
231
-- `host anomaly rate threshold`: (`0.0`/`1.0`) This is the percentage of dimensions (based on all those enabled for anomaly detection) that need to be considered anomalous at specific timestep for the host itself to be considered anomalous. For example, the default value of `0.01` means that if more than 1% of dimensions are anomalous at the same time then the host itself is considered in an anomalous state.
232
-- `minimum window size`: The Netdata "Anomaly Detector" logic works over a rolling window of data. This parameter defines the minimum length of window to consider. If over this window the host is in an anomalous state then an anomaly detection event will be triggered. For example, the default of `30` means that the detector will initially work over a rolling window of 30 seconds. Note: The length of this window will be dynamic once an anomaly event has been triggered such that it will expand as needed until either the max length of an anomaly event is hit or the host settles back into a normal state with sufficiently decreased host level anomaly states in the rolling window. Note: If you wanted to adjust the higher level anomaly detector behavior then this is one parameter you might adjust to see the impact of on anomaly detection events.
233
-- `maximum window size`: This parameter defines the maximum length of window to consider. If an anomaly event reaches this size, it will be closed. This is to provide an upper bound on the length of an anomaly event and cost of the anomaly detector logic for that event.
234
-- `window minimum anomaly rate`: (`0.0`/`1.0`) This parameter corresponds to a threshold on the percentage of time in the rolling window that the host was considered in an anomalous state. For example, the default of `0.25` means that if the host is in an anomalous state for 25% of more of the rolling window then and anomaly event will be triggered or extended if one is already active. Note: If you want to make the anomaly detector itself less sensitive, you can adjust this value to something like `0.75` which would mean the host needs to be much more consistently in an anomalous state to trigger an anomaly detection event. Likewise, a lower value like `0.1` would make the anomaly detector more sensitive.
235
-- `anomaly event min dimension rate threshold`: (`0.0`/`1.0`) This is a parameter that helps filter out irrelevant dimensions from anomaly events. For example, the default of `0.05` means that only dimensions that were considered anomalous for at least 5% of the anomaly event itself will be included in that anomaly event. The idea here is to just include dimensions that were consistently anomalous as opposed to those that may have just randomly happened to be anomalous at the same time.
188
+- `host anomaly rate threshold`: (`0.1`/`10.0`) This is the percentage of dimensions (based on all those enabled for anomaly detection) that need to be considered anomalous at specific timestep for the host itself to be considered anomalous. For example, the default value of `1.0` means that if more than 1% of dimensions are anomalous at the same time then the host itself is considered in an anomalous state.
189
+- `anomaly detection grouping method`: The grouping method used when calculating node level anomaly rate.
190
+- `anomaly detection grouping duration`: (`60`/`900`) The duration across which to calculate the node level anomaly rate, the default of `900` means that the node level anomaly rate is calculated across a rolling 5 minute window.
191
- `hosts to skip from training`: This parameter allows you to turn off anomaly detection for any child hosts on a parent host by defining those you would like to skip from training here. For example, a value like `dev-*` skips all hosts on a parent that begin with the "dev-" prefix. The default value of `!*` means "don't skip any".
192
- `charts to skip from training`: This parameter allows you to exclude certain charts from anomaly detection. By default, only netdata related charts are excluded. This is to avoid the scenario where accessing the netdata dashboard could itself tigger some anomalies if you don't access them regularly. If you want to include charts that are excluded by default, add them in small groups and then measure any impact on performance before adding additional ones. Example: If you want to include system, apps, and user charts:`!system.* !apps.* !user.* *`.
193
@@ -240,28 +195,27 @@ This example assumes 3 child nodes [streaming](https://learn.netdata.cloud/docs/
195
196
Once enabled, the "Anomaly Detection" menu and charts will be available on the dashboard.
197
243
-
198
+
199
200
In terms of anomaly detection, the most interesting charts would be the `anomaly_detection.dimensions` and `anomaly_detection.anomaly_rate` ones, which hold the `anomalous` and `anomaly_rate` dimensions that show the overall number of dimensions considered anomalous at any time and the corresponding anomaly rate.
201
202
- `anomaly_detection.dimensions`: Total count of dimensions considered anomalous or normal.
203
- `anomaly_detection.dimensions`: Percentage of anomalous dimensions.
249
-- `anomaly_detection.detector_window`: The length of the active window used by the detector.
250
-- `anomaly_detection.detector_events`: Flags (0 or 1) to show when an anomaly event has been triggered by the detector.
204
+- `anomaly_detection.anomaly_detection`: Flags (0 or 1) to show when an anomaly event has been triggered by the detector.
205
206
Below is an example of how these charts may look in the presence of an anomaly event.
207
208
Initially we see a jump in `anomalous` dimensions:
209
256
-
210
+
211
212
And a corresponding jump in the `anomaly_rate`:
213
260
-
214
+
215
216
After a short while the rolling node anomaly rate goes `above_threshold`, and once it stays above threshold for long enough a `new_anomaly_event` is created:
217
264
-
218
+
219
220
## Glossary
221
ml/SamplesBuffer.cc
+44
-8
@@ -103,31 +103,30 @@ void SamplesBuffer::lagSamples() {
103
}
104
}
105
106
-std::vector<DSample> SamplesBuffer::preprocess() {
106
+void SamplesBuffer::preprocess(std::vector<DSample> &Samples) {
107
assert(Preprocessed == false);
108
109
- std::vector<DSample> DSamples;
109
size_t OutN = NumSamples;
110
111
// Diff
112
if (DiffN >= OutN)
114
- return DSamples;
113
+ return;
114
OutN -= DiffN;
115
diffSamples();
116
117
// Smooth
118
if (SmoothN == 0 || SmoothN > OutN)
120
- return DSamples;
119
+ return;
120
OutN -= (SmoothN - 1);
121
smoothSamples();
122
123
// Lag
124
if (LagN >= OutN)
126
- return DSamples;
125
+ return;
126
OutN -= LagN;
127
lagSamples();
128
130
- DSamples.reserve(OutN);
129
+ Samples.reserve(OutN);
130
Preprocessed = true;
131
132
uint32_t MaxMT = std::numeric_limits<uint32_t>::max();
@@ -143,8 +142,45 @@ std::vector<DSample> SamplesBuffer::preprocess() {
142
const Sample PS = getPreprocessedSample(Idx);
143
PS.initDSample(DS);
144
146
- DSamples.push_back(DS);
145
+ Samples.push_back(DS);
146
}
147
+}
148
+
149
+void SamplesBuffer::preprocess(DSample &Feature) {
150
+ assert(Preprocessed == false);
151
+
152
+ size_t OutN = NumSamples;
153
+
154
+ // Diff
155
+ if (DiffN >= OutN)
156
+ return;
157
+ OutN -= DiffN;
158
+ diffSamples();
159
+
160
+ // Smooth
161
+ if (SmoothN == 0 || SmoothN > OutN)
162
+ return;
163
+ OutN -= (SmoothN - 1);
164
+ smoothSamples();
165
+
166
+ // Lag
167
+ if (LagN >= OutN)
168
+ return;
169
+ OutN -= LagN;
170
+ lagSamples();
171
149
- return DSamples;
172
+ Preprocessed = true;
173
+
174
+ uint32_t MaxMT = std::numeric_limits<uint32_t>::max();
175
+ uint32_t CutOff = static_cast<double>(MaxMT) * SamplingRatio;
176
+
177
+ for (size_t Idx = NumSamples - OutN; Idx != NumSamples; Idx++) {
178
+ if (RandNums[Idx] > CutOff)
179
+ continue;
180
+
181
+ Feature.set_size(NumDimsPerSample * (LagN + 1));
182
+
183
+ const Sample PS = getPreprocessedSample(Idx);
184
+ PS.initDSample(Feature);
185
+ }
186
}
ml/SamplesBuffer.h
+2
-1
@@ -88,7 +88,8 @@ public:
88
BytesPerSample(NumDimsPerSample * sizeof(CalculatedNumber)),
89
Preprocessed(false) {};
90
91
- std::vector<DSample> preprocess();
91
+ void preprocess(std::vector<DSample> &Samples);
92
+ void preprocess(DSample &Feature);
93
std::vector<Sample> getPreprocessedSamples() const;
94
95
size_t capacity() const { return NumSamples; }
ml/SamplesBufferTests.cc
deleted
-146
@@ -1,146 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "ml/ml-private.h"
4
-#include <gtest/gtest.h>
5
-
6
-/*
7
- * The SamplesBuffer class implements the functionality of the following python
8
- * code:
9
- * >> df = pd.DataFrame(data=samples)
10
- * >> df = df.diff(diff_n).dropna()
11
- * >> df = df.rolling(smooth_n).mean().dropna()
12
- * >> df = pd.concat([df.shift(n) for n in range(lag_n + 1)], axis=1).dropna()
13
- *
14
- * Its correctness has been verified by automatically generating random
15
- * data frames in Python and comparing them with the correspondent preprocessed
16
- * SampleBuffers.
17
- *
18
- * The following tests are meant to catch unintended changes in the SamplesBuffer
19
- * implementation. For development purposes, one should compare changes against
20
- * the aforementioned python code.
21
-*/
22
-
23
-TEST(SamplesBufferTest, NS_8_NDPS_1_DN_1_SN_3_LN_1) {
24
- size_t NumSamples = 8, NumDimsPerSample = 1;
25
- size_t DiffN = 1, SmoothN = 3, LagN = 3;
26
-
27
- size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
28
- CalculatedNumber *CNs = new CalculatedNumber[N]();
29
-
30
- CNs[0] = 0.7568336679490107;
31
- CNs[1] = 0.4814406581763254;
32
- CNs[2] = 0.40073555156221874;
33
- CNs[3] = 0.5973257298194408;
34
- CNs[4] = 0.5334727814345868;
35
- CNs[5] = 0.2632477193454843;
36
- CNs[6] = 0.2684839023122384;
37
- CNs[7] = 0.851332948637479;
38
-
39
- std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
40
- SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
41
- SB.preprocess();
42
-
43
- std::vector<Sample> Samples = SB.getPreprocessedSamples();
44
- EXPECT_EQ(Samples.size(), 2);
45
-
46
- Sample S0 = Samples[0];
47
- const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
48
- Sample S1 = Samples[1];
49
- const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
50
-
51
- EXPECT_NEAR(S0_CNs[0], -0.109614, 0.001);
52
- EXPECT_NEAR(S0_CNs[1], -0.0458293, 0.001);
53
- EXPECT_NEAR(S0_CNs[2], 0.017344, 0.001);
54
- EXPECT_NEAR(S0_CNs[3], -0.0531693, 0.001);
55
-
56
- EXPECT_NEAR(S1_CNs[0], 0.105953, 0.001);
57
- EXPECT_NEAR(S1_CNs[1], -0.109614, 0.001);
58
- EXPECT_NEAR(S1_CNs[2], -0.0458293, 0.001);
59
- EXPECT_NEAR(S1_CNs[3], 0.017344, 0.001);
60
-
61
- delete[] CNs;
62
-}
63
-
64
-TEST(SamplesBufferTest, NS_8_NDPS_1_DN_2_SN_3_LN_2) {
65
- size_t NumSamples = 8, NumDimsPerSample = 1;
66
- size_t DiffN = 2, SmoothN = 3, LagN = 2;
67
-
68
- size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
69
- CalculatedNumber *CNs = new CalculatedNumber[N]();
70
-
71
- CNs[0] = 0.20511885291342846;
72
- CNs[1] = 0.13151717360306558;
73
- CNs[2] = 0.6017085062423134;
74
- CNs[3] = 0.46256882933941545;
75
- CNs[4] = 0.7887758447877941;
76
- CNs[5] = 0.9237989080034406;
77
- CNs[6] = 0.15552559051428083;
78
- CNs[7] = 0.6309750314597955;
79
-
80
- std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
81
- SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
82
- SB.preprocess();
83
-
84
- std::vector<Sample> Samples = SB.getPreprocessedSamples();
85
- EXPECT_EQ(Samples.size(), 2);
86
-
87
- Sample S0 = Samples[0];
88
- const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
89
- Sample S1 = Samples[1];
90
- const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
91
-
92
- EXPECT_NEAR(S0_CNs[0], 0.005016, 0.001);
93
- EXPECT_NEAR(S0_CNs[1], 0.326450, 0.001);
94
- EXPECT_NEAR(S0_CNs[2], 0.304903, 0.001);
95
-
96
- EXPECT_NEAR(S1_CNs[0], -0.154948, 0.001);
97
- EXPECT_NEAR(S1_CNs[1], 0.005016, 0.001);
98
- EXPECT_NEAR(S1_CNs[2], 0.326450, 0.001);
99
-
100
- delete[] CNs;
101
-}
102
-
103
-TEST(SamplesBufferTest, NS_8_NDPS_3_DN_2_SN_4_LN_1) {
104
- size_t NumSamples = 8, NumDimsPerSample = 3;
105
- size_t DiffN = 2, SmoothN = 4, LagN = 1;
106
-
107
- size_t N = NumSamples * NumDimsPerSample * (LagN + 1);
108
- CalculatedNumber *CNs = new CalculatedNumber[N]();
109
-
110
- CNs[0] = 0.34310900399667765; CNs[1] = 0.14694315994488194; CNs[2] = 0.8246677800938796;
111
- CNs[3] = 0.48249504592307835; CNs[4] = 0.23241087965531182; CNs[5] = 0.9595348555892567;
112
- CNs[6] = 0.44281094035598334; CNs[7] = 0.5143142171362715; CNs[8] = 0.06391303014242555;
113
- CNs[9] = 0.7460491027783901; CNs[10] = 0.43887217459032923; CNs[11] = 0.2814395025355999;
114
- CNs[12] = 0.9231114281214198; CNs[13] = 0.326882401786898; CNs[14] = 0.26747939220376216;
115
- CNs[15] = 0.7787571209969636; CNs[16] =0.5851700001235088; CNs[17] = 0.34410728945321567;
116
- CNs[18] = 0.9394494507088997; CNs[19] =0.17567223681734334; CNs[20] = 0.42732886195446984;
117
- CNs[21] = 0.9460522396152958; CNs[22] =0.23462747016780894; CNs[23] = 0.35983249900892145;
118
-
119
- std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
120
- SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
121
- SB.preprocess();
122
-
123
- std::vector<Sample> Samples = SB.getPreprocessedSamples();
124
- EXPECT_EQ(Samples.size(), 2);
125
-
126
- Sample S0 = Samples[0];
127
- const CalculatedNumber *S0_CNs = S0.getCalculatedNumbers();
128
- Sample S1 = Samples[1];
129
- const CalculatedNumber *S1_CNs = S1.getCalculatedNumbers();
130
-
131
- EXPECT_NEAR(S0_CNs[0], 0.198225, 0.001);
132
- EXPECT_NEAR(S0_CNs[1], 0.003529, 0.001);
133
- EXPECT_NEAR(S0_CNs[2], -0.063003, 0.001);
134
- EXPECT_NEAR(S0_CNs[3], 0.219066, 0.001);
135
- EXPECT_NEAR(S0_CNs[4], 0.133175, 0.001);
136
- EXPECT_NEAR(S0_CNs[5], -0.293154, 0.001);
137
-
138
- EXPECT_NEAR(S1_CNs[0], 0.174160, 0.001);
139
- EXPECT_NEAR(S1_CNs[1], -0.135722, 0.001);
140
- EXPECT_NEAR(S1_CNs[2], 0.110452, 0.001);
141
- EXPECT_NEAR(S1_CNs[3], 0.198225, 0.001);
142
- EXPECT_NEAR(S1_CNs[4], 0.003529, 0.001);
143
- EXPECT_NEAR(S1_CNs[5], -0.063003, 0.001);
144
-
145
- delete[] CNs;
146
-}
ml/Stats.h
new
+46
@@ -0,0 +1,46 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef ML_STATS_H
4
+#define ML_STATS_H
5
+
6
+#include "ml-private.h"
7
+
8
+namespace ml {
9
+
10
+struct MachineLearningStats {
11
+ size_t NumMachineLearningStatusEnabled;
12
+ size_t NumMachineLearningStatusDisabledUE;
13
+ size_t NumMachineLearningStatusDisabledSP;
14
+
15
+ size_t NumMetricTypeConstant;
16
+ size_t NumMetricTypeVariable;
17
+
18
+ size_t NumTrainingStatusUntrained;
19
+ size_t NumTrainingStatusPendingWithoutModel;
20
+ size_t NumTrainingStatusTrained;
21
+ size_t NumTrainingStatusPendingWithModel;
22
+
23
+ size_t NumAnomalousDimensions;
24
+ size_t NumNormalDimensions;
25
+};
26
+
27
+struct TrainingStats {
28
+ struct rusage TrainingRU;
29
+
30
+ size_t QueueSize;
31
+ size_t NumPoppedItems;
32
+
33
+ usec_t AllottedUT;
34
+ usec_t ConsumedUT;
35
+ usec_t RemainingUT;
36
+
37
+ size_t TrainingResultOk;
38
+ size_t TrainingResultInvalidQueryTimeRange;
39
+ size_t TrainingResultNotEnoughCollectedValues;
40
+ size_t TrainingResultNullAcquiredDimension;
41
+ size_t TrainingResultChartUnderReplication;
42
+};
43
+
44
+} // namespace ml
45
+
46
+#endif /* ML_STATS_H */
ml/ml-dummy.c
+43
-8
@@ -15,9 +15,37 @@ bool ml_enabled(RRDHOST *RH) {
15
16
void ml_init(void) {}
17
18
-void ml_new_host(RRDHOST *RH) { (void) RH; }
18
+void ml_host_new(RRDHOST *RH) {
19
+ UNUSED(RH);
20
+}
21
+
22
+void ml_host_delete(RRDHOST *RH) {
23
+ UNUSED(RH);
24
+}
25
+
26
+void ml_chart_new(RRDSET *RS) {
27
+ UNUSED(RS);
28
+}
29
+
30
+void ml_chart_delete(RRDSET *RS) {
31
+ UNUSED(RS);
32
+}
33
+
34
+void ml_dimension_new(RRDDIM *RD) {
35
+ UNUSED(RD);
36
+}
37
+
38
+void ml_dimension_delete(RRDDIM *RD) {
39
+ UNUSED(RD);
40
+}
41
20
-void ml_delete_host(RRDHOST *RH) { (void) RH; }
42
+void ml_start_anomaly_detection_threads(RRDHOST *RH) {
43
+ UNUSED(RH);
44
+}
45
+
46
+void ml_stop_anomaly_detection_threads(RRDHOST *RH) {
47
+ UNUSED(RH);
48
+}
49
50
char *ml_get_host_info(RRDHOST *RH) {
51
(void) RH;
@@ -29,17 +57,24 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
57
return NULL;
58
}
59
60
+void ml_chart_update_begin(RRDSET *RS) {
61
+ (void) RS;
62
+}
63
+
64
+void ml_chart_update_end(RRDSET *RS) {
65
+ (void) RS;
66
+}
67
+
68
char *ml_get_host_models(RRDHOST *RH) {
69
(void) RH;
70
return NULL;
71
}
72
37
-void ml_new_dimension(RRDDIM *RD) { (void) RD; }
38
-
39
-void ml_delete_dimension(RRDDIM *RD) { (void) RD; }
40
-
41
-bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
42
- (void) RD; (void) Value; (void) Exists;
73
+bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
74
+ (void) RD;
75
+ (void) CurrT;
76
+ (void) Value;
77
+ (void) Exists;
78
return false;
79
}
80
ml/ml-private.h
-13
@@ -6,21 +6,8 @@
6
#include "KMeans.h"
7
#include "ml/ml.h"
8
9
-#include <chrono>
9
#include <map>
10
#include <mutex>
11
#include <sstream>
12
14
-namespace ml {
15
-
16
-using SteadyClock = std::chrono::steady_clock;
17
-using TimePoint = std::chrono::time_point<SteadyClock>;
18
-
19
-template<typename T>
20
-using Duration = std::chrono::duration<T>;
21
-
22
-using Seconds = std::chrono::seconds;
23
-
24
-} // namespace ml
25
-
13
#endif /* ML_PRIVATE_H */
ml/ml.cc
+71
-37
@@ -2,6 +2,7 @@
2
3
#include "Config.h"
4
#include "Dimension.h"
5
+#include "Chart.h"
6
#include "Host.h"
7
8
#include <random>
@@ -45,18 +46,18 @@ void ml_init(void) {
46
Cfg.RandomNums.push_back(Gen());
47
}
48
48
-void ml_new_host(RRDHOST *RH) {
49
+void ml_host_new(RRDHOST *RH) {
50
if (!ml_enabled(RH))
51
return;
52
53
Host *H = new Host(RH);
53
- RH->ml_host = static_cast<ml_host_t>(H);
54
+ RH->ml_host = reinterpret_cast<ml_host_t *>(H);
55
56
H->startAnomalyDetectionThreads();
57
}
58
58
-void ml_delete_host(RRDHOST *RH) {
59
- Host *H = static_cast<Host *>(RH->ml_host);
59
+void ml_host_delete(RRDHOST *RH) {
60
+ Host *H = reinterpret_cast<Host *>(RH->ml_host);
61
if (!H)
62
return;
63
@@ -66,35 +67,48 @@ void ml_delete_host(RRDHOST *RH) {
67
RH->ml_host = nullptr;
68
}
69
69
-void ml_new_dimension(RRDDIM *RD) {
70
- RRDSET *RS = RD->rrdset;
71
-
72
- Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host);
70
+void ml_chart_new(RRDSET *RS) {
71
+ Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
72
if (!H)
73
return;
74
76
- if (static_cast<unsigned>(RD->update_every) != H->updateEvery())
75
+ Chart *C = new Chart(RS);
76
+ RS->ml_chart = reinterpret_cast<ml_chart_t *>(C);
77
+
78
+ H->addChart(C);
79
+}
80
+
81
+void ml_chart_delete(RRDSET *RS) {
82
+ Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host);
83
+ if (!H)
84
return;
85
79
- if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RS)))
86
+ Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
87
+ H->removeChart(C);
88
+
89
+ delete C;
90
+ RS->ml_chart = nullptr;
91
+}
92
+
93
+void ml_dimension_new(RRDDIM *RD) {
94
+ Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
95
+ if (!C)
96
return;
97
98
Dimension *D = new Dimension(RD);
83
- RD->ml_dimension = static_cast<ml_dimension_t>(D);
84
- H->addDimension(D);
99
+ RD->ml_dimension = reinterpret_cast<ml_dimension_t *>(D);
100
+ C->addDimension(D);
101
}
102
87
-void ml_delete_dimension(RRDDIM *RD) {
88
- Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
103
+void ml_dimension_delete(RRDDIM *RD) {
104
+ Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
105
if (!D)
106
return;
107
92
- Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host);
93
- if (!H)
94
- delete D;
95
- else
96
- H->removeDimension(D);
108
+ Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
109
+ C->removeDimension(D);
110
111
+ delete D;
112
RD->ml_dimension = nullptr;
113
}
114
@@ -102,7 +116,7 @@ char *ml_get_host_info(RRDHOST *RH) {
116
nlohmann::json ConfigJson;
117
118
if (RH && RH->ml_host) {
105
- Host *H = static_cast<Host *>(RH->ml_host);
119
+ Host *H = reinterpret_cast<Host *>(RH->ml_host);
120
H->getConfigAsJson(ConfigJson);
121
} else {
122
ConfigJson["enabled"] = false;
@@ -115,7 +129,7 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
129
nlohmann::json ConfigJson;
130
131
if (RH && RH->ml_host) {
118
- Host *H = static_cast<Host *>(RH->ml_host);
132
+ Host *H = reinterpret_cast<Host *>(RH->ml_host);
133
H->getDetectionInfoAsJson(ConfigJson);
134
} else {
135
return nullptr;
@@ -128,7 +142,7 @@ char *ml_get_host_models(RRDHOST *RH) {
142
nlohmann::json ModelsJson;
143
144
if (RH && RH->ml_host) {
131
- Host *H = static_cast<Host *>(RH->ml_host);
145
+ Host *H = reinterpret_cast<Host *>(RH->ml_host);
146
H->getModelsAsJson(ModelsJson);
147
return strdup(ModelsJson.dump(2, '\t').c_str());
148
}
@@ -136,30 +150,50 @@ char *ml_get_host_models(RRDHOST *RH) {
150
return nullptr;
151
}
152
139
-bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
140
- Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
141
- if (!D)
142
- return false;
153
+void ml_start_anomaly_detection_threads(RRDHOST *RH) {
154
+ if (RH && RH->ml_host) {
155
+ Host *H = reinterpret_cast<Host *>(RH->ml_host);
156
+ H->startAnomalyDetectionThreads();
157
+ }
158
+}
159
144
- return D->predict(Value, Exists);
160
+void ml_stop_anomaly_detection_threads(RRDHOST *RH) {
161
+ if (RH && RH->ml_host) {
162
+ Host *H = reinterpret_cast<Host *>(RH->ml_host);
163
+ H->stopAnomalyDetectionThreads();
164
+ }
165
}
166
147
-bool ml_streaming_enabled() {
148
- return Cfg.StreamADCharts;
167
+void ml_chart_update_begin(RRDSET *RS) {
168
+ Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
169
+ if (!C)
170
+ return;
171
+
172
+ C->updateBegin();
173
}
174
151
-#if defined(ENABLE_ML_TESTS)
175
+void ml_chart_update_end(RRDSET *RS) {
176
+ Chart *C = reinterpret_cast<Chart *>(RS->ml_chart);
177
+ if (!C)
178
+ return;
179
+
180
+ C->updateEnd();
181
+}
182
153
-#include "gtest/gtest.h"
183
+bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) {
184
+ Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension);
185
+ if (!D)
186
+ return false;
187
155
-int test_ml(int argc, char *argv[]) {
156
- (void) argc;
157
- (void) argv;
188
+ Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart);
189
159
- ::testing::InitGoogleTest(&argc, argv);
160
- return RUN_ALL_TESTS();
190
+ bool IsAnomalous = D->predict(CurrT, Value, Exists);
191
+ C->updateDimension(D, IsAnomalous);
192
+ return IsAnomalous;
193
}
194
163
-#endif // ENABLE_ML_TESTS
195
+bool ml_streaming_enabled() {
196
+ return Cfg.StreamADCharts;
197
+}
198
199
#include "ml-private.h"
ml/ml.h
+14
-14
@@ -14,35 +14,35 @@ extern "C" {
14
// the anomaly rate dimension, whenever its backing dimension is freed.
15
void rrddim_free(RRDSET *st, RRDDIM *rd);
16
17
-typedef void* ml_host_t;
18
-typedef void* ml_dimension_t;
19
-
17
bool ml_capable();
18
19
bool ml_enabled(RRDHOST *RH);
20
21
void ml_init(void);
22
26
-void ml_new_host(RRDHOST *RH);
27
-void ml_delete_host(RRDHOST *RH);
23
+void ml_host_new(RRDHOST *RH);
24
+void ml_host_delete(RRDHOST *RH);
25
+
26
+void ml_chart_new(RRDSET *RS);
27
+void ml_chart_delete(RRDSET *RS);
28
+
29
+void ml_dimension_new(RRDDIM *RD);
30
+void ml_dimension_delete(RRDDIM *RD);
31
+
32
+void ml_start_anomaly_detection_threads(RRDHOST *RH);
33
+void ml_stop_anomaly_detection_threads(RRDHOST *RH);
34
35
char *ml_get_host_info(RRDHOST *RH);
36
char *ml_get_host_runtime_info(RRDHOST *RH);
37
char *ml_get_host_models(RRDHOST *RH);
38
33
-void ml_new_dimension(RRDDIM *RD);
34
-void ml_delete_dimension(RRDDIM *RD);
39
+void ml_chart_update_begin(RRDSET *RS);
40
+void ml_chart_update_end(RRDSET *RS);
41
36
-bool ml_is_anomalous(RRDDIM *RD, double value, bool exists);
42
+bool ml_is_anomalous(RRDDIM *RD, time_t curr_t, double value, bool exists);
43
44
bool ml_streaming_enabled();
45
40
-#define ML_ANOMALY_RATES_CHART_ID "anomaly_detection.anomaly_rates"
41
-
42
-#if defined(ENABLE_ML_TESTS)
43
-int test_ml(int argc, char *argv[]);
44
-#endif
45
-
46
#ifdef __cplusplus
47
};
48
#endif
netdata-installer.sh
-2
@@ -332,8 +332,6 @@ while [ -n "${1}" ]; do
332
NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-ml)}" | sed 's/$/ --disable-ml/g')"
333
NETDATA_ENABLE_ML=0
334
;;
335
- "--enable-ml-tests") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-ml-tests)}" | sed 's/$/ --enable-ml-tests/g')" ;;
336
- "--disable-ml-tests") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-ml-tests)}" | sed 's/$/ --disable-ml-tests/g')" ;;
335
"--disable-lto") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-lto)}" | sed 's/$/ --disable-lto/g')" ;;
336
"--disable-x86-sse") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-x86-sse)}" | sed 's/$/ --disable-x86-sse/g')" ;;
337
"--disable-telemetry") NETDATA_DISABLE_TELEMETRY=1 ;;