@cryptotaxi247 / netdata-1 / commits / 6850878e6

Remove anomaly detector (#13657)

* Move all dims under one class. * Dimension owns anomaly rate RD. * Remove Dimension::isAnomalous() * Remove Dimension::trainEvery() * Rm ml/kmeans * Remove anomaly detector The same logic can be implemented by using the host anomaly rate dim. * Profile plugin. * Revert "Profile plugin." This reverts commit e3db37cb49c514502c5216cfe7bca2a003fb90f1. * Add separate source files for anomaly detection charts. * Handle training/prediction sync at the dimension level. * Keep multiple KMeans models in mem. * Move feature extraction outside KMeans class. * Use multiple models. * Add /api/v1/ml_models endpoint. * Remove Dimension::getID() * Use just 1 model and fix tests. * Add detection logic based on rrdr. * Remove config options related to anomaly detection. * Make anomaly detection queries configurable. * Fix ad query duration option. * Finalize queries in all code paths. * Check if query was initialized before finalizing it * Do not leak OWA * Profile plugin. * Revert "Profile plugin." This reverts commit 5c77145d0df7e091d030476c480ab8d9cbceb89e. * Change context from anomaly_detection to detector_events.

vkalintiris committed Oct 5, 2022 at 10:11 UTC 6850878e697d66dc90b9af1e750b22238c63c292
34 files changed +565 -1658
.gitmodules
+1 -1
@@ -5,7 +5,7 @@
5 path = aclk/aclk-schemas
6 url = https://github.com/netdata/aclk-schemas.git
7 [submodule "ml/kmeans/dlib"]
8 - path = ml/kmeans/dlib
8 + path = ml/dlib
9 url = https://github.com/davisking/dlib.git
10 shallow = true
11 ignore = dirty
CMakeLists.txt
+7 -13
@@ -348,11 +348,11 @@ ENDIF()
348 # Detect ml dependencies
349 file(STRINGS "${CMAKE_SOURCE_DIR}/config.h" DEFINE_ENABLE_ML REGEX "^#define ENABLE_ML 1$")
350 IF(DEFINE_ENABLE_ML MATCHES ".+" AND
351 - EXISTS "${CMAKE_SOURCE_DIR}/ml/kmeans/dlib/dlib/all/source.cpp" AND
351 + EXISTS "${CMAKE_SOURCE_DIR}/ml/dlib/dlib/all/source.cpp" AND
352 EXISTS "${CMAKE_SOURCE_DIR}/ml/json/single_include/nlohmann/json.hpp")
353 set(ENABLE_ML True)
354 list(APPEND NETDATA_COMMON_CFLAGS "-DDLIB_NO_GUI_SUPPORT")
355 - list(APPEND NETDATA_COMMON_INCLUDE_DIRS "ml/kmeans/dlib")
355 + list(APPEND NETDATA_COMMON_INCLUDE_DIRS "ml/dlib")
356 ELSE()
357 set(ENABLE_ML False)
358 ENDIF()
@@ -1025,24 +1025,18 @@ set(ML_FILES
1025
1026 IF(ENABLE_ML)
1027 list(APPEND ML_FILES
1028 - ml/BitBufferCounter.h
1029 - ml/BitBufferCounter.cc
1030 - ml/BitRateWindow.h
1031 - ml/BitRateWindow.cc
1028 ml/Config.h
1029 ml/Config.cc
1034 - ml/Database.h
1035 - ml/Database.cc
1030 ml/Dimension.cc
1031 ml/Dimension.h
1032 ml/Host.h
1033 ml/Host.cc
1034 ml/Query.h
1041 - ml/kmeans/KMeans.h
1042 - ml/kmeans/KMeans.cc
1043 - ml/kmeans/SamplesBuffer.h
1044 - ml/kmeans/SamplesBuffer.cc
1045 - ml/kmeans/dlib/dlib/all/source.cpp
1035 + ml/KMeans.h
1036 + ml/KMeans.cc
1037 + ml/SamplesBuffer.h
1038 + ml/SamplesBuffer.cc
1039 + ml/dlib/dlib/all/source.cpp
1040 ml/json/single_include/nlohmann/json.hpp
1041 ml/ml.cc
1042 ml/ml-private.h
Makefile.am
+10 -16
@@ -38,7 +38,7 @@ EXTRA_DIST = \
38 build/m4/ax_c_mallopt.m4 \
39 build/m4/tcmalloc.m4 \
40 build/m4/ax_c__generic.m4 \
41 - ml/kmeans/dlib \
41 + ml/dlib \
42 README.md \
43 LICENSE \
44 REDISTRIBUTED.md \
@@ -109,7 +109,6 @@ SUBDIRS += \
109 claim \
110 parser \
111 spawn \
112 - ml \
112 $(NULL)
113
114 AM_CFLAGS = \
@@ -237,39 +236,34 @@ ML_FILES = \
236 if ENABLE_ML
237
238 ML_FILES += \
240 - ml/BitBufferCounter.h \
241 - ml/BitBufferCounter.cc \
242 - ml/BitRateWindow.h \
243 - ml/BitRateWindow.cc \
239 + ml/ADCharts.h \
240 + ml/ADCharts.cc \
241 ml/Config.h \
242 ml/Config.cc \
246 - ml/Database.h \
247 - ml/Database.cc \
243 ml/Dimension.cc \
244 ml/Dimension.h \
245 ml/Host.h \
246 ml/Host.cc \
247 ml/Query.h \
253 - ml/kmeans/KMeans.h \
254 - ml/kmeans/KMeans.cc \
255 - ml/kmeans/SamplesBuffer.h \
256 - ml/kmeans/SamplesBuffer.cc \
257 - ml/kmeans/dlib/dlib/all/source.cpp \
248 + ml/KMeans.h \
249 + ml/KMeans.cc \
250 + ml/SamplesBuffer.h \
251 + ml/SamplesBuffer.cc \
252 + ml/dlib/dlib/all/source.cpp \
253 ml/json/single_include/nlohmann/json.hpp \
254 ml/ml.cc \
255 ml/ml-private.h \
256 $(NULL)
257
258 # Disable warnings from dlib library
264 -ml/kmeans/dlib/dlib/all/source.$(OBJEXT) : CXXFLAGS += -Wno-sign-compare -Wno-type-limits -Wno-aggressive-loop-optimizations -Wno-stringop-overflow
259 +ml/dlib/dlib/all/source.$(OBJEXT) : CXXFLAGS += -Wno-sign-compare -Wno-type-limits -Wno-aggressive-loop-optimizations -Wno-stringop-overflow
260
261 endif
262
263
264 if ENABLE_ML_TESTS
265 ML_TESTS_FILES = \
271 - ml/kmeans/Tests.cc \
272 - ml/Tests.cc \
266 + ml/SamplesBufferTests.cc \
267 $(NULL)
268 endif
269
configure.ac
+2 -4
@@ -1109,7 +1109,7 @@ fi
1109
1110 # Check if submodules have not been fetched. Fail if ML was explicitly requested.
1111 AC_MSG_CHECKING([if git submodules are present for machine learning functionality])
1112 -if test -f "ml/kmeans/dlib/dlib/all/source.cpp" -a -f "ml/json/single_include/nlohmann/json.hpp"; then
1112 +if test -f "ml/dlib/dlib/all/source.cpp" -a -f "ml/json/single_include/nlohmann/json.hpp"; then
1113 AC_MSG_RESULT([yes])
1114 have_ml_submodules="yes"
1115 else
@@ -1149,7 +1149,7 @@ fi
1149 AM_CONDITIONAL([ENABLE_ML], [test "${build_ml}" = "yes"])
1150 if test "${build_ml}" = "yes"; then
1151 AC_DEFINE([ENABLE_ML], [1], [anomaly detection usability])
1152 - OPTIONAL_ML_CFLAGS="-DDLIB_NO_GUI_SUPPORT -I \$(abs_top_srcdir)/ml/kmeans/dlib"
1152 + OPTIONAL_ML_CFLAGS="-DDLIB_NO_GUI_SUPPORT -I \$(abs_top_srcdir)/ml/dlib"
1153 OPTIONAL_ML_LIBS=""
1154 fi
1155
@@ -1695,8 +1695,6 @@ AC_CONFIG_FILES([
1695 exporting/tests/Makefile
1696 health/Makefile
1697 health/notifications/Makefile
1698 - ml/Makefile
1699 - ml/kmeans/Makefile
1698 libnetdata/Makefile
1699 libnetdata/tests/Makefile
1700 libnetdata/adaptive_resortable_list/Makefile
ml/ADCharts.cc new
+236
@@ -0,0 +1,236 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "ADCharts.h"
4 +#include "Config.h"
5 +
6 +void ml::updateDimensionsChart(RRDHOST *RH,
7 + collected_number NumTrainedDimensions,
8 + collected_number NumNormalDimensions,
9 + collected_number NumAnomalousDimensions) {
10 + static thread_local RRDSET *RS = nullptr;
11 + static thread_local RRDDIM *NumTotalDimensionsRD = nullptr;
12 + static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr;
13 + static thread_local RRDDIM *NumNormalDimensionsRD = nullptr;
14 + static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
15 +
16 + if (!RS) {
17 + std::stringstream IdSS, NameSS;
18 +
19 + IdSS << "dimensions_on_" << localhost->machine_guid;
20 + NameSS << "dimensions_on_" << localhost->hostname;
21 +
22 + RS = rrdset_create(
23 + RH,
24 + "anomaly_detection", // type
25 + IdSS.str().c_str(), // id
26 + NameSS.str().c_str(), // name
27 + "dimensions", // family
28 + "anomaly_detection.dimensions", // ctx
29 + "Anomaly detection dimensions", // title
30 + "dimensions", // units
31 + "netdata", // plugin
32 + "ml", // module
33 + 39183, // priority
34 + RH->rrd_update_every, // update_every
35 + RRDSET_TYPE_LINE // chart_type
36 + );
37 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
38 +
39 + NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
40 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
41 + NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL,
42 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
43 + NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL,
44 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
45 + NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL,
46 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
47 + } else
48 + rrdset_next(RS);
49 +
50 + rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions);
51 + rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions);
52 + rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions);
53 + rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions);
54 +
55 + rrdset_done(RS);
56 +}
57 +
58 +void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) {
59 + static thread_local RRDSET *HostRateRS = nullptr;
60 + static thread_local RRDDIM *AnomalyRateRD = nullptr;
61 +
62 + if (!HostRateRS) {
63 + std::stringstream IdSS, NameSS;
64 +
65 + IdSS << "anomaly_rate_on_" << localhost->machine_guid;
66 + NameSS << "anomaly_rate_on_" << localhost->hostname;
67 +
68 + HostRateRS = rrdset_create(
69 + RH,
70 + "anomaly_detection", // type
71 + IdSS.str().c_str(), // id
72 + NameSS.str().c_str(), // name
73 + "anomaly_rate", // family
74 + "anomaly_detection.anomaly_rate", // ctx
75 + "Percentage of anomalous dimensions", // title
76 + "percentage", // units
77 + "netdata", // plugin
78 + "ml", // module
79 + 39184, // priority
80 + RH->rrd_update_every, // update_every
81 + RRDSET_TYPE_LINE // chart_type
82 + );
83 + rrdset_flag_set(HostRateRS, RRDSET_FLAG_ANOMALY_DETECTION);
84 +
85 + AnomalyRateRD = rrddim_add(HostRateRS, "anomaly_rate", NULL,
86 + 1, 100, RRD_ALGORITHM_ABSOLUTE);
87 + } else
88 + rrdset_next(HostRateRS);
89 +
90 + rrddim_set_by_pointer(HostRateRS, AnomalyRateRD, AnomalyRate);
91 + rrdset_done(HostRateRS);
92 +
93 + static thread_local RRDSET *AnomalyDetectionRS = nullptr;
94 + static thread_local RRDDIM *AboveThresholdRD = nullptr;
95 + static thread_local RRDDIM *NewAnomalyEventRD = nullptr;
96 +
97 + if (!AnomalyDetectionRS) {
98 + std::stringstream IdSS, NameSS;
99 +
100 + IdSS << "anomaly_detection_on_" << localhost->machine_guid;
101 + NameSS << "anomaly_detection_on_" << localhost->hostname;
102 +
103 + AnomalyDetectionRS = rrdset_create(
104 + RH,
105 + "anomaly_detection", // type
106 + IdSS.str().c_str(), // id
107 + NameSS.str().c_str(), // name
108 + "anomaly_detection", // family
109 + "anomaly_detection.detector_events", // ctx
110 + "Anomaly detection events", // title
111 + "percentage", // units
112 + "netdata", // plugin
113 + "ml", // module
114 + 39185, // priority
115 + RH->rrd_update_every, // update_every
116 + RRDSET_TYPE_LINE // chart_type
117 + );
118 + rrdset_flag_set(AnomalyDetectionRS, RRDSET_FLAG_ANOMALY_DETECTION);
119 +
120 + AboveThresholdRD = rrddim_add(AnomalyDetectionRS, "above_threshold", NULL,
121 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
122 + NewAnomalyEventRD = rrddim_add(AnomalyDetectionRS, "new_anomaly_event", NULL,
123 + 1, 1, RRD_ALGORITHM_ABSOLUTE);
124 + } else
125 + rrdset_next(AnomalyDetectionRS);
126 +
127 + /*
128 + * Compute the values of the dimensions based on the host rate chart
129 + */
130 + ONEWAYALLOC *OWA = onewayalloc_create(0);
131 + time_t Now = now_realtime_sec();
132 + time_t Before = Now - RH->rrd_update_every;
133 + time_t After = Before - Cfg.AnomalyDetectionQueryDuration;
134 + RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
135 +
136 + RRDR *R = rrd2rrdr(
137 + OWA, HostRateRS,
138 + 1 /* points wanted */,
139 + After,
140 + Before,
141 + Cfg.AnomalyDetectionGroupingMethod,
142 + 0 /* resampling time */,
143 + Options, "anomaly_rate",
144 + NULL /* context param list */,
145 + NULL /* group options */,
146 + 0, /* timeout */
147 + 0 /* tier */
148 + );
149 + assert(R->d == 1 && R->n == 1 && R->rows == 1);
150 +
151 + static thread_local bool PrevAboveThreshold = false;
152 + bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
153 + bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
154 + PrevAboveThreshold = AboveThreshold;
155 +
156 + rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
157 + rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
158 + rrdset_done(AnomalyDetectionRS);
159 +
160 + rrdr_free(OWA, R);
161 + onewayalloc_destroy(OWA);
162 +}
163 +
164 +void ml::updateDetectionChart(RRDHOST *RH) {
165 + static thread_local RRDSET *RS = nullptr;
166 + static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
167 +
168 + if (!RS) {
169 + std::stringstream IdSS, NameSS;
170 +
171 + IdSS << "prediction_stats_" << RH->machine_guid;
172 + NameSS << "prediction_stats_for_" << RH->hostname;
173 +
174 + RS = rrdset_create_localhost(
175 + "netdata", // type
176 + IdSS.str().c_str(), // id
177 + NameSS.str().c_str(), // name
178 + "ml", // family
179 + "netdata.prediction_stats", // ctx
180 + "Prediction thread CPU usage", // title
181 + "milliseconds/s", // units
182 + "netdata", // plugin
183 + "ml", // module
184 + 136000, // priority
185 + RH->rrd_update_every, // update_every
186 + RRDSET_TYPE_STACKED // chart_type
187 + );
188 +
189 + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
190 + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
191 + } else
192 + rrdset_next(RS);
193 +
194 + struct rusage TRU;
195 + getrusage(RUSAGE_THREAD, &TRU);
196 +
197 + rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);
198 + rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);
199 + rrdset_done(RS);
200 +}
201 +
202 +void ml::updateTrainingChart(RRDHOST *RH, struct rusage *TRU) {
203 + static thread_local RRDSET *RS = nullptr;
204 + static thread_local RRDDIM *UserRD = nullptr;
205 + static thread_local RRDDIM *SystemRD = nullptr;
206 +
207 + if (!RS) {
208 + std::stringstream IdSS, NameSS;
209 +
210 + IdSS << "training_stats_" << RH->machine_guid;
211 + NameSS << "training_stats_for_" << RH->hostname;
212 +
213 + RS = rrdset_create_localhost(
214 + "netdata", // type
215 + IdSS.str().c_str(), // id
216 + NameSS.str().c_str(), // name
217 + "ml", // family
218 + "netdata.training_stats", // ctx
219 + "Training thread CPU usage", // title
220 + "milliseconds/s", // units
221 + "netdata", // plugin
222 + "ml", // module
223 + 136001, // priority
224 + RH->rrd_update_every, // update_every
225 + RRDSET_TYPE_STACKED // chart_type
226 + );
227 +
228 + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
229 + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
230 + } else
231 + rrdset_next(RS);
232 +
233 + rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);
234 + rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);
235 + rrdset_done(RS);
236 +}
ml/ADCharts.h new
+23
@@ -0,0 +1,23 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef ML_ADCHARTS_H
4 +#define ML_ADCHARTS_H
5 +
6 +#include "ml-private.h"
7 +
8 +namespace ml {
9 +
10 +void updateDimensionsChart(RRDHOST *RH,
11 + collected_number NumTrainedDimensions,
12 + collected_number NumNormalDimensions,
13 + collected_number NumAnomalousDimensions);
14 +
15 +void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate);
16 +
17 +void updateDetectionChart(RRDHOST *RH);
18 +
19 +void updateTrainingChart(RRDHOST *RH, struct rusage *TRU);
20 +
21 +} // namespace ml
22 +
23 +#endif /* ML_ADCHARTS_H */
ml/BitBufferCounter.cc deleted
-29
@@ -1,29 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "BitBufferCounter.h"
4 -
5 -using namespace ml;
6 -
7 -std::vector<bool> BitBufferCounter::getBuffer() const {
8 - std::vector<bool> Buffer;
9 -
10 - for (size_t Idx = start(); Idx != (start() + size()); Idx++)
11 - Buffer.push_back(V[Idx % V.size()]);
12 -
13 - return Buffer;
14 -}
15 -
16 -void BitBufferCounter::insert(bool Bit) {
17 - if (N >= V.size())
18 - NumSetBits -= (V[start()] == true);
19 -
20 - NumSetBits += (Bit == true);
21 - V[N++ % V.size()] = Bit;
22 -}
23 -
24 -void BitBufferCounter::print(std::ostream &OS) const {
25 - std::vector<bool> Buffer = getBuffer();
26 -
27 - for (bool B : Buffer)
28 - OS << B;
29 -}
ml/BitBufferCounter.h deleted
-54
@@ -1,54 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef BIT_BUFFER_COUNTER_H
4 -#define BIT_BUFFER_COUNTER_H
5 -
6 -#include "ml-private.h"
7 -
8 -namespace ml {
9 -
10 -class BitBufferCounter {
11 -public:
12 - BitBufferCounter(size_t Capacity) : V(Capacity, 0), NumSetBits(0), N(0) {}
13 -
14 - std::vector<bool> getBuffer() const;
15 -
16 - void insert(bool Bit);
17 -
18 - void print(std::ostream &OS) const;
19 -
20 - bool isFilled() const {
21 - return N >= V.size();
22 - }
23 -
24 - size_t numSetBits() const {
25 - return NumSetBits;
26 - }
27 -
28 -private:
29 - inline size_t size() const {
30 - return N < V.size() ? N : V.size();
31 - }
32 -
33 - inline size_t start() const {
34 - if (N <= V.size())
35 - return 0;
36 -
37 - return N % V.size();
38 - }
39 -
40 -private:
41 - std::vector<bool> V;
42 - size_t NumSetBits;
43 -
44 - size_t N;
45 -};
46 -
47 -} // namespace ml
48 -
49 -inline std::ostream& operator<<(std::ostream &OS, const ml::BitBufferCounter &BBC) {
50 - BBC.print(OS);
51 - return OS;
52 -}
53 -
54 -#endif /* BIT_BUFFER_COUNTER_H */
ml/BitRateWindow.cc deleted
-75
@@ -1,75 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "BitRateWindow.h"
4 -
5 -using namespace ml;
6 -
7 -std::pair<BitRateWindow::Edge, size_t> BitRateWindow::insert(bool Bit) {
8 - Edge E;
9 -
10 - BBC.insert(Bit);
11 - switch (CurrState) {
12 - case State::NotFilled: {
13 - if (BBC.isFilled()) {
14 - if (BBC.numSetBits() < SetBitsThreshold) {
15 - CurrState = State::BelowThreshold;
16 - } else {
17 - CurrState = State::AboveThreshold;
18 - }
19 - } else {
20 - CurrState = State::NotFilled;
21 - }
22 -
23 - E = {State::NotFilled, CurrState};
24 - break;
25 - } case State::BelowThreshold: {
26 - if (BBC.numSetBits() >= SetBitsThreshold) {
27 - CurrState = State::AboveThreshold;
28 - }
29 -
30 - E = {State::BelowThreshold, CurrState};
31 - break;
32 - } case State::AboveThreshold: {
33 - if ((BBC.numSetBits() < SetBitsThreshold) ||
34 - (CurrLength == MaxLength)) {
35 - CurrState = State::Idle;
36 - }
37 -
38 - E = {State::AboveThreshold, CurrState};
39 - break;
40 - } case State::Idle: {
41 - if (CurrLength == IdleLength) {
42 - CurrState = State::NotFilled;
43 - }
44 -
45 - E = {State::Idle, CurrState};
46 - break;
47 - }
48 - }
49 -
50 - Action A = EdgeActions[E];
51 - size_t L = (this->*A)(E.first, Bit);
52 - return {E, L};
53 -}
54 -
55 -void BitRateWindow::print(std::ostream &OS) const {
56 - switch (CurrState) {
57 - case State::NotFilled:
58 - OS << "NotFilled";
59 - break;
60 - case State::BelowThreshold:
61 - OS << "BelowThreshold";
62 - break;
63 - case State::AboveThreshold:
64 - OS << "AboveThreshold";
65 - break;
66 - case State::Idle:
67 - OS << "Idle";
68 - break;
69 - default:
70 - OS << "UnknownState";
71 - break;
72 - }
73 -
74 - OS << ": " << BBC << " (Current Length: " << CurrLength << ")";
75 -}
ml/BitRateWindow.h deleted
-170
@@ -1,170 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef BIT_RATE_WINDOW_H
4 -#define BIT_RATE_WINDOW_H
5 -
6 -#include "BitBufferCounter.h"
7 -#include "ml-private.h"
8 -
9 -namespace ml {
10 -
11 -class BitRateWindow {
12 -public:
13 - enum class State {
14 - NotFilled,
15 - BelowThreshold,
16 - AboveThreshold,
17 - Idle
18 - };
19 -
20 - using Edge = std::pair<State, State>;
21 - using Action = size_t (BitRateWindow::*)(State PrevState, bool NewBit);
22 -
23 -private:
24 - std::map<Edge, Action> EdgeActions = {
25 - // From == To
26 - {
27 - Edge(State::NotFilled, State::NotFilled),
28 - &BitRateWindow::onRoundtripNotFilled,
29 - },
30 - {
31 - Edge(State::BelowThreshold, State::BelowThreshold),
32 - &BitRateWindow::onRoundtripBelowThreshold,
33 - },
34 - {
35 - Edge(State::AboveThreshold, State::AboveThreshold),
36 - &BitRateWindow::onRoundtripAboveThreshold,
37 - },
38 - {
39 - Edge(State::Idle, State::Idle),
40 - &BitRateWindow::onRoundtripIdle,
41 - },
42 -
43 -
44 - // NotFilled => {BelowThreshold, AboveThreshold}
45 - {
46 - Edge(State::NotFilled, State::BelowThreshold),
47 - &BitRateWindow::onNotFilledToBelowThreshold
48 - },
49 - {
50 - Edge(State::NotFilled, State::AboveThreshold),
51 - &BitRateWindow::onNotFilledToAboveThreshold
52 - },
53 -
54 - // BelowThreshold => AboveThreshold
55 - {
56 - Edge(State::BelowThreshold, State::AboveThreshold),
57 - &BitRateWindow::onBelowToAboveThreshold
58 - },
59 -
60 - // AboveThreshold => Idle
61 - {
62 - Edge(State::AboveThreshold, State::Idle),
63 - &BitRateWindow::onAboveThresholdToIdle
64 - },
65 -
66 - // Idle => NotFilled
67 - {
68 - Edge(State::Idle, State::NotFilled),
69 - &BitRateWindow::onIdleToNotFilled
70 - },
71 - };
72 -
73 -public:
74 - BitRateWindow(size_t MinLength, size_t MaxLength, size_t IdleLength,
75 - size_t SetBitsThreshold) :
76 - MinLength(MinLength), MaxLength(MaxLength), IdleLength(IdleLength),
77 - SetBitsThreshold(SetBitsThreshold),
78 - CurrState(State::NotFilled), CurrLength(0), BBC(MinLength) {}
79 -
80 - std::pair<Edge, size_t> insert(bool Bit);
81 -
82 - void print(std::ostream &OS) const;
83 -
84 -private:
85 - size_t onRoundtripNotFilled(State PrevState, bool NewBit) {
86 - (void) PrevState, (void) NewBit;
87 -
88 - CurrLength += 1;
89 - return CurrLength;
90 - }
91 -
92 - size_t onRoundtripBelowThreshold(State PrevState, bool NewBit) {
93 - (void) PrevState, (void) NewBit;
94 -
95 - CurrLength = MinLength;
96 - return CurrLength;
97 - }
98 -
99 - size_t onRoundtripAboveThreshold(State PrevState, bool NewBit) {
100 - (void) PrevState, (void) NewBit;
101 -
102 - CurrLength += 1;
103 - return CurrLength;
104 - }
105 -
106 - size_t onRoundtripIdle(State PrevState, bool NewBit) {
107 - (void) PrevState, (void) NewBit;
108 -
109 - CurrLength += 1;
110 - return CurrLength;
111 - }
112 -
113 - size_t onNotFilledToBelowThreshold(State PrevState, bool NewBit) {
114 - (void) PrevState, (void) NewBit;
115 -
116 - CurrLength = MinLength;
117 - return CurrLength;
118 - }
119 -
120 - size_t onNotFilledToAboveThreshold(State PrevState, bool NewBit) {
121 - (void) PrevState, (void) NewBit;
122 -
123 - CurrLength += 1;
124 - return CurrLength;
125 - }
126 -
127 - size_t onBelowToAboveThreshold(State PrevState, bool NewBit) {
128 - (void) PrevState, (void) NewBit;
129 -
130 - CurrLength = MinLength;
131 - return CurrLength;
132 - }
133 -
134 - size_t onAboveThresholdToIdle(State PrevState, bool NewBit) {
135 - (void) PrevState, (void) NewBit;
136 -
137 - size_t PrevLength = CurrLength;
138 - CurrLength = 1;
139 - return PrevLength;
140 - }
141 -
142 - size_t onIdleToNotFilled(State PrevState, bool NewBit) {
143 - (void) PrevState, (void) NewBit;
144 -
145 - BBC = BitBufferCounter(MinLength);
146 - BBC.insert(NewBit);
147 -
148 - CurrLength = 1;
149 - return CurrLength;
150 - }
151 -
152 -private:
153 - size_t MinLength;
154 - size_t MaxLength;
155 - size_t IdleLength;
156 - size_t SetBitsThreshold;
157 -
158 - State CurrState;
159 - size_t CurrLength;
160 - BitBufferCounter BBC;
161 -};
162 -
163 -} // namespace ml
164 -
165 -inline std::ostream& operator<<(std::ostream &OS, const ml::BitRateWindow BRW) {
166 - BRW.print(OS);
167 - return OS;
168 -}
169 -
170 -#endif /* BIT_RATE_WINDOW_H */
ml/Config.cc
+14 -32
@@ -31,6 +31,7 @@ void Config::readMLConfig(void) {
31 unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
32 unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
33 unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
34 + unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24);
35
36 unsigned DBEngineAnomalyRateEvery = config_get_number(ConfigSectionML, "dbengine anomaly rate every", 30);
37
@@ -42,25 +43,19 @@ void Config::readMLConfig(void) {
43 unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000);
44
45 double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99);
45 - double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 0.01);
46
47 - double ADMinWindowSize = config_get_float(ConfigSectionML, "minimum window size", 30);
48 - double ADMaxWindowSize = config_get_float(ConfigSectionML, "maximum window size", 600);
49 - double ADIdleWindowSize = config_get_float(ConfigSectionML, "idle window size", 30);
50 - double ADWindowRateThreshold = config_get_float(ConfigSectionML, "window minimum anomaly rate", 0.25);
51 - double ADDimensionRateThreshold = config_get_float(ConfigSectionML, "anomaly event min dimension rate threshold", 0.05);
52 -
53 - std::stringstream SS;
54 - SS << netdata_configured_cache_dir << "/anomaly-detection.db";
55 - Cfg.AnomalyDBPath = SS.str();
47 + double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 1.0);
48 + std::string AnomalyDetectionGroupingMethod = config_get(ConfigSectionML, "anomaly detection grouping method", "average");
49 + time_t AnomalyDetectionQueryDuration = config_get_number(ConfigSectionML, "anomaly detection grouping duration", 5 * 60);
50
51 /*
52 * Clamp
53 */
54
61 - MaxTrainSamples = clamp(MaxTrainSamples, 1 * 3600u, 24 * 3600u);
62 - MinTrainSamples = clamp(MinTrainSamples, 1 * 900u, 6 * 3600u);
63 - TrainEvery = clamp(TrainEvery, 1 * 3600u, 6 * 3600u);
55 + MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
56 + MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
57 + TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
58 + NumModelsToUse = clamp<unsigned>(TrainEvery, 1, 7 * 24);
59
60 DBEngineAnomalyRateEvery = clamp(DBEngineAnomalyRateEvery, 1 * 30u, 15 * 60u);
61
@@ -72,13 +67,9 @@ void Config::readMLConfig(void) {
67 MaxKMeansIters = clamp(MaxKMeansIters, 500u, 1000u);
68
69 DimensionAnomalyScoreThreshold = clamp(DimensionAnomalyScoreThreshold, 0.01, 5.00);
75 - HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.01, 1.0);
70
77 - ADMinWindowSize = clamp(ADMinWindowSize, 30.0, 300.0);
78 - ADMaxWindowSize = clamp(ADMaxWindowSize, 60.0, 900.0);
79 - ADIdleWindowSize = clamp(ADIdleWindowSize, 30.0, 900.0);
80 - ADWindowRateThreshold = clamp(ADWindowRateThreshold, 0.01, 0.99);
81 - ADDimensionRateThreshold = clamp(ADDimensionRateThreshold, 0.01, 0.99);
71 + HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.1, 10.0);
72 + AnomalyDetectionQueryDuration = clamp<time_t>(AnomalyDetectionQueryDuration, 60, 15 * 60);
73
74 /*
75 * Validate
@@ -91,13 +82,6 @@ void Config::readMLConfig(void) {
82 MaxTrainSamples = 4 * 3600;
83 }
84
94 - if (ADMinWindowSize >= ADMaxWindowSize) {
95 - error("invalid min/max anomaly window size found (%lf >= %lf)", ADMinWindowSize, ADMaxWindowSize);
96 -
97 - ADMinWindowSize = 30.0;
98 - ADMaxWindowSize = 600.0;
99 - }
100 -
85 /*
86 * Assign to config instance
87 */
@@ -107,6 +91,7 @@ void Config::readMLConfig(void) {
91 Cfg.MaxTrainSamples = MaxTrainSamples;
92 Cfg.MinTrainSamples = MinTrainSamples;
93 Cfg.TrainEvery = TrainEvery;
94 + Cfg.NumModelsToUse = NumModelsToUse;
95
96 Cfg.DBEngineAnomalyRateEvery = DBEngineAnomalyRateEvery;
97
@@ -118,13 +103,10 @@ void Config::readMLConfig(void) {
103 Cfg.MaxKMeansIters = MaxKMeansIters;
104
105 Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;
121 - Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
106
123 - Cfg.ADMinWindowSize = ADMinWindowSize;
124 - Cfg.ADMaxWindowSize = ADMaxWindowSize;
125 - Cfg.ADIdleWindowSize = ADIdleWindowSize;
126 - Cfg.ADWindowRateThreshold = ADWindowRateThreshold;
127 - Cfg.ADDimensionRateThreshold = ADDimensionRateThreshold;
107 + Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
108 + Cfg.AnomalyDetectionGroupingMethod = web_client_api_request_v1_data_group(AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
109 + Cfg.AnomalyDetectionQueryDuration = AnomalyDetectionQueryDuration;
110
111 Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
112 Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
ml/Config.h
+4 -7
@@ -14,6 +14,7 @@ public:
14 unsigned MaxTrainSamples;
15 unsigned MinTrainSamples;
16 unsigned TrainEvery;
17 + unsigned NumModelsToUse;
18
19 unsigned DBEngineAnomalyRateEvery;
20
@@ -25,13 +26,10 @@ public:
26 unsigned MaxKMeansIters;
27
28 double DimensionAnomalyScoreThreshold;
28 - double HostAnomalyRateThreshold;
29
30 - double ADMinWindowSize;
31 - double ADMaxWindowSize;
32 - double ADIdleWindowSize;
33 - double ADWindowRateThreshold;
34 - double ADDimensionRateThreshold;
30 + double HostAnomalyRateThreshold;
31 + RRDR_GROUPING AnomalyDetectionGroupingMethod;
32 + time_t AnomalyDetectionQueryDuration;
33
34 bool StreamADCharts;
35
@@ -41,7 +39,6 @@ public:
39 std::string ChartsToSkip;
40 SIMPLE_PATTERN *SP_ChartsToSkip;
41
44 - std::string AnomalyDBPath;
42 std::vector<uint32_t> RandomNums;
43
44 void readMLConfig();
ml/Database.cc deleted
-127
@@ -1,127 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "Database.h"
4 -
5 -const char *ml::Database::SQL_CREATE_ANOMALIES_TABLE =
6 - "CREATE TABLE IF NOT EXISTS anomaly_events( "
7 - " anomaly_detector_name text NOT NULL, "
8 - " anomaly_detector_version int NOT NULL, "
9 - " host_id text NOT NULL, "
10 - " after int NOT NULL, "
11 - " before int NOT NULL, "
12 - " anomaly_event_info text, "
13 - " PRIMARY KEY( "
14 - " anomaly_detector_name, anomaly_detector_version, "
15 - " host_id, after, before "
16 - " ) "
17 - ");";
18 -
19 -const char *ml::Database::SQL_INSERT_ANOMALY =
20 - "INSERT INTO anomaly_events( "
21 - " anomaly_detector_name, anomaly_detector_version, "
22 - " host_id, after, before, anomaly_event_info) "
23 - "VALUES (?1, ?2, ?3, ?4, ?5, ?6);";
24 -
25 -const char *ml::Database::SQL_SELECT_ANOMALY =
26 - "SELECT anomaly_event_info FROM anomaly_events WHERE"
27 - " anomaly_detector_name == ?1 AND"
28 - " anomaly_detector_version == ?2 AND"
29 - " host_id == ?3 AND"
30 - " after == ?4 AND"
31 - " before == ?5;";
32 -
33 -const char *ml::Database::SQL_SELECT_ANOMALY_EVENTS =
34 - "SELECT after, before FROM anomaly_events WHERE"
35 - " anomaly_detector_name == ?1 AND"
36 - " anomaly_detector_version == ?2 AND"
37 - " host_id == ?3 AND"
38 - " after >= ?4 AND"
39 - " before <= ?5;";
40 -
41 -using namespace ml;
42 -
43 -bool Statement::prepare(sqlite3 *Conn) {
44 - if (!Conn)
45 - return false;
46 -
47 - if (ParsedStmt)
48 - return true;
49 -
50 - int RC = sqlite3_prepare_v2(Conn, RawStmt, -1, &ParsedStmt, nullptr);
51 - if (RC == SQLITE_OK)
52 - return true;
53 -
54 - std::string Msg = "Statement \"%s\" preparation failed due to \"%s\"";
55 - error(Msg.c_str(), RawStmt, sqlite3_errstr(RC));
56 -
57 - return false;
58 -}
59 -
60 -bool Statement::bindValue(size_t Pos, const std::string &Value) {
61 - int RC = sqlite3_bind_text(ParsedStmt, Pos, Value.c_str(), -1, SQLITE_TRANSIENT);
62 - if (RC == SQLITE_OK)
63 - return true;
64 -
65 - error("Failed to bind text '%s' (pos = %zu) in statement '%s'.", Value.c_str(), Pos, RawStmt);
66 - return false;
67 -}
68 -
69 -bool Statement::bindValue(size_t Pos, const int Value) {
70 - int RC = sqlite3_bind_int(ParsedStmt, Pos, Value);
71 - if (RC == SQLITE_OK)
72 - return true;
73 -
74 - error("Failed to bind integer %d (pos = %zu) in statement '%s'.", Value, Pos, RawStmt);
75 - return false;
76 -}
77 -
78 -bool Statement::resetAndClear(bool Ret) {
79 - int RC = sqlite3_reset(ParsedStmt);
80 - if (RC != SQLITE_OK) {
81 - error("Could not reset statement: '%s'", RawStmt);
82 - return false;
83 - }
84 -
85 - RC = sqlite3_clear_bindings(ParsedStmt);
86 - if (RC != SQLITE_OK) {
87 - error("Could not clear bindings in statement: '%s'", RawStmt);
88 - return false;
89 - }
90 -
91 - return Ret;
92 -}
93 -
94 -Database::Database(const std::string &Path) {
95 - // Get sqlite3 connection handle.
96 - int RC = sqlite3_open(Path.c_str(), &Conn);
97 - if (RC != SQLITE_OK) {
98 - std::string Msg = "Failed to initialize ML DB at %s, due to \"%s\"";
99 - error(Msg.c_str(), Path.c_str(), sqlite3_errstr(RC));
100 -
101 - sqlite3_close(Conn);
102 - Conn = nullptr;
103 - return;
104 - }
105 -
106 - // Create anomaly events table if it does not exist.
107 - char *ErrMsg;
108 - RC = sqlite3_exec_monitored(Conn, SQL_CREATE_ANOMALIES_TABLE, nullptr, nullptr, &ErrMsg);
109 - if (RC == SQLITE_OK)
110 - return;
111 -
112 - error("SQLite error during database initialization, rc = %d (%s)", RC, ErrMsg);
113 - error("SQLite failed statement: %s", SQL_CREATE_ANOMALIES_TABLE);
114 -
115 - sqlite3_free(ErrMsg);
116 - sqlite3_close(Conn);
117 - Conn = nullptr;
118 -}
119 -
120 -Database::~Database() {
121 - if (!Conn)
122 - return;
123 -
124 - int RC = sqlite3_close(Conn);
125 - if (RC != SQLITE_OK)
126 - error("Could not close connection properly (rc=%d)", RC);
127 -}
ml/Database.h deleted
-131
@@ -1,131 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#ifndef ML_DATABASE_H
4 -#define ML_DATABASE_H
5 -
6 -#include "Dimension.h"
7 -#include "ml-private.h"
8 -
9 -#include "json/single_include/nlohmann/json.hpp"
10 -
11 -namespace ml {
12 -
13 -class Statement {
14 -public:
15 - using RowCallback = std::function<void(sqlite3_stmt *Stmt)>;
16 -
17 -public:
18 - Statement(const char *RawStmt) : RawStmt(RawStmt), ParsedStmt(nullptr) {}
19 -
20 - template<typename ...ArgTypes>
21 - bool exec(sqlite3 *Conn, RowCallback RowCb, ArgTypes ...Args) {
22 - if (!prepare(Conn))
23 - return false;
24 -
25 - switch (bind(1, Args...)) {
26 - case 0:
27 - return false;
28 - case sizeof...(Args):
29 - break;
30 - default:
31 - return resetAndClear(false);
32 - }
33 -
34 - while (true) {
35 - switch (int RC = sqlite3_step_monitored(ParsedStmt)) {
36 - case SQLITE_BUSY: case SQLITE_LOCKED:
37 - usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
38 - continue;
39 - case SQLITE_ROW:
40 - RowCb(ParsedStmt);
41 - continue;
42 - case SQLITE_DONE:
43 - return resetAndClear(true);
44 - default:
45 - error("Stepping through '%s' returned rc=%d", RawStmt, RC);
46 - return resetAndClear(false);
47 - }
48 - }
49 - }
50 -
51 - ~Statement() {
52 - if (!ParsedStmt)
53 - return;
54 -
55 - int RC = sqlite3_finalize(ParsedStmt);
56 - if (RC != SQLITE_OK)
57 - error("Could not properly finalize statement (rc=%d)", RC);
58 - }
59 -
60 -private:
61 - bool prepare(sqlite3 *Conn);
62 -
63 - bool bindValue(size_t Pos, const int Value);
64 - bool bindValue(size_t Pos, const std::string &Value);
65 -
66 - template<typename ArgType, typename ...ArgTypes>
67 - size_t bind(size_t Pos, ArgType T) {
68 - return bindValue(Pos, T);
69 - }
70 -
71 - template<typename ArgType, typename ...ArgTypes>
72 - size_t bind(size_t Pos, ArgType T, ArgTypes ...Args) {
73 - return bindValue(Pos, T) + bind(Pos + 1, Args...);
74 - }
75 -
76 - bool resetAndClear(bool Ret);
77 -
78 -private:
79 - const char *RawStmt;
80 - sqlite3_stmt *ParsedStmt;
81 -};
82 -
83 -class Database {
84 -private:
85 - static const char *SQL_CREATE_ANOMALIES_TABLE;
86 - static const char *SQL_INSERT_ANOMALY;
87 - static const char *SQL_SELECT_ANOMALY;
88 - static const char *SQL_SELECT_ANOMALY_EVENTS;
89 -
90 -public:
91 - Database(const std::string &Path);
92 -
93 - ~Database();
94 -
95 - template<typename ...ArgTypes>
96 - bool insertAnomaly(ArgTypes... Args) {
97 - Statement::RowCallback RowCb = [](sqlite3_stmt *Stmt) { (void) Stmt; };
98 - return InsertAnomalyStmt.exec(Conn, RowCb, Args...);
99 - }
100 -
101 - template<typename ...ArgTypes>
102 - bool getAnomalyInfo(nlohmann::json &Json, ArgTypes&&... Args) {
103 - Statement::RowCallback RowCb = [&](sqlite3_stmt *Stmt) {
104 - const char *Text = static_cast<const char *>(sqlite3_column_blob(Stmt, 0));
105 - Json = nlohmann::json::parse(Text);
106 - };
107 - return GetAnomalyInfoStmt.exec(Conn, RowCb, Args...);
108 - }
109 -
110 - template<typename ...ArgTypes>
111 - bool getAnomaliesInRange(std::vector<std::pair<time_t, time_t>> &V, ArgTypes&&... Args) {
112 - Statement::RowCallback RowCb = [&](sqlite3_stmt *Stmt) {
113 - V.push_back({
114 - sqlite3_column_int64(Stmt, 0),
115 - sqlite3_column_int64(Stmt, 1)
116 - });
117 - };
118 - return GetAnomaliesInRangeStmt.exec(Conn, RowCb, Args...);
119 - }
120 -
121 -private:
122 - sqlite3 *Conn;
123 -
124 - Statement InsertAnomalyStmt{SQL_INSERT_ANOMALY};
125 - Statement GetAnomalyInfoStmt{SQL_SELECT_ANOMALY};
126 - Statement GetAnomaliesInRangeStmt{SQL_SELECT_ANOMALY_EVENTS};
127 -};
128 -
129 -}
130 -
131 -#endif /* ML_DATABASE_H */
ml/Dimension.cc
+67 -19
@@ -6,8 +6,13 @@
6
7 using namespace ml;
8
9 -std::pair<CalculatedNumber *, size_t>
10 -TrainableDimension::getCalculatedNumbers() {
9 +bool Dimension::isActive() const {
10 + bool SetObsolete = rrdset_flag_check(RD->rrdset, RRDSET_FLAG_OBSOLETE);
11 + bool DimObsolete = rrddim_flag_check(RD, RRDDIM_FLAG_OBSOLETE);
12 + return !SetObsolete && !DimObsolete;
13 +}
14 +
15 +std::pair<CalculatedNumber *, size_t> Dimension::getCalculatedNumbers() {
16 size_t MinN = Cfg.MinTrainSamples;
17 size_t MaxN = Cfg.MaxTrainSamples;
18
@@ -68,7 +73,7 @@ TrainableDimension::getCalculatedNumbers() {
73 return { CNs, TotalValues };
74 }
75
71 -MLResult TrainableDimension::trainModel() {
76 +MLResult Dimension::trainModel() {
77 auto P = getCalculatedNumbers();
78 CalculatedNumber *CNs = P.first;
79 unsigned N = P.second;
@@ -81,7 +86,15 @@ MLResult TrainableDimension::trainModel() {
86
87 SamplesBuffer SB = SamplesBuffer(CNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
88 SamplingRatio, Cfg.RandomNums);
84 - KM.train(SB, Cfg.MaxKMeansIters);
89 + std::vector<DSample> Samples = SB.preprocess();
90 +
91 + KMeans KM;
92 + KM.train(Samples, Cfg.MaxKMeansIters);
93 +
94 + {
95 + std::lock_guard<std::mutex> Lock(Mutex);
96 + Models[0] = KM;
97 + }
98
99 Trained = true;
100 ConstantModel = true;
@@ -90,16 +103,25 @@ MLResult TrainableDimension::trainModel() {
103 return MLResult::Success;
104 }
105
93 -void PredictableDimension::addValue(CalculatedNumber Value, bool Exists) {
106 +bool Dimension::shouldTrain(const TimePoint &TP) const {
107 + if (ConstantModel)
108 + return false;
109 +
110 + return (LastTrainedAt + Seconds(Cfg.TrainEvery * updateEvery())) < TP;
111 +}
112 +
113 +bool Dimension::predict(CalculatedNumber Value, bool Exists) {
114 if (!Exists) {
115 CNs.clear();
96 - return;
116 + AnomalyBit = false;
117 + return false;
118 }
119
120 unsigned N = Cfg.DiffN + Cfg.SmoothN + Cfg.LagN;
121 if (CNs.size() < N) {
122 CNs.push_back(Value);
102 - return;
123 + AnomalyBit = false;
124 + return false;
125 }
126
127 std::rotate(std::begin(CNs), std::begin(CNs) + 1, std::end(CNs));
@@ -108,28 +130,54 @@ void PredictableDimension::addValue(CalculatedNumber Value, bool Exists) {
130 ConstantModel = false;
131
132 CNs[N - 1] = Value;
111 -}
133
113 -std::pair<MLResult, bool> PredictableDimension::predict() {
114 - unsigned N = Cfg.DiffN + Cfg.SmoothN + Cfg.LagN;
115 - if (CNs.size() != N) {
134 + if (!isTrained() || ConstantModel) {
135 AnomalyBit = false;
117 - return { MLResult::MissingData, AnomalyBit };
136 + return false;
137 }
138
139 CalculatedNumber *TmpCNs = new CalculatedNumber[N * (Cfg.LagN + 1)]();
140 std::memcpy(TmpCNs, CNs.data(), N * sizeof(CalculatedNumber));
122 -
123 - SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
141 + SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1,
142 + Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,
143 1.0, Cfg.RandomNums);
125 - AnomalyScore = computeAnomalyScore(SB);
144 + const DSample Sample = SB.preprocess().back();
145 delete[] TmpCNs;
146
128 - if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN()) {
147 + std::unique_lock<std::mutex> Lock(Mutex, std::defer_lock);
148 + if (!Lock.try_lock()) {
149 AnomalyBit = false;
130 - return { MLResult::NaN, AnomalyBit };
150 + return false;
151 + }
152 +
153 + for (const auto &KM : Models) {
154 + double AnomalyScore = KM.anomalyScore(Sample);
155 + if (AnomalyScore == std::numeric_limits<CalculatedNumber>::quiet_NaN()) {
156 + AnomalyBit = false;
157 + continue;
158 + }
159 +
160 + if (AnomalyScore < (100 * Cfg.DimensionAnomalyScoreThreshold)) {
161 + AnomalyBit = false;
162 + return false;
163 + }
164 + }
165 +
166 + AnomalyBit = true;
167 + return true;
168 +}
169 +
170 +void Dimension::updateAnomalyBitCounter(RRDSET *RS, unsigned Elapsed, bool IsAnomalous) {
171 + AnomalyBitCounter += IsAnomalous;
172 +
173 + if (Elapsed == Cfg.DBEngineAnomalyRateEvery) {
174 + double AR = static_cast<double>(AnomalyBitCounter) / Cfg.DBEngineAnomalyRateEvery;
175 + rrddim_set_by_pointer(RS, getAnomalyRateRD(), AR * 1000);
176 + AnomalyBitCounter = 0;
177 }
178 +}
179
133 - AnomalyBit = AnomalyScore >= (100 * Cfg.DimensionAnomalyScoreThreshold);
134 - return { MLResult::Success, AnomalyBit };
180 +std::array<KMeans, 1> Dimension::getModels() {
181 + std::unique_lock<std::mutex> Lock(Mutex);
182 + return Models;
183 }
ml/Dimension.h
+60 -109
@@ -3,155 +3,106 @@
3 #ifndef ML_DIMENSION_H
4 #define ML_DIMENSION_H
5
6 -#include "BitBufferCounter.h"
6 +#include "Query.h"
7 #include "Config.h"
8
9 #include "ml-private.h"
10
11 namespace ml {
12
13 -class RrdDimension {
14 -public:
15 - RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->tiers[0]->query_ops) { }
16 -
17 - RRDDIM *getRD() const { return RD; }
18 -
19 - time_t latestTime() { return Ops->latest_time(RD->tiers[0]->db_metric_handle); }
20 -
21 - time_t oldestTime() { return Ops->oldest_time(RD->tiers[0]->db_metric_handle); }
13 +enum class MLResult {
14 + Success = 0,
15 + MissingData,
16 + NaN,
17 +};
18
23 - unsigned updateEvery() const { return RD->update_every; }
19 +static inline std::string getMLDimensionID(RRDDIM *RD) {
20 + RRDSET *RS = RD->rrdset;
21
25 - const std::string getID() const {
26 - RRDSET *RS = RD->rrdset;
22 + std::stringstream SS;
23 + SS << rrdset_context(RS) << "|" << rrdset_id(RS) << "|" << rrddim_name(RD);
24 + return SS.str();
25 +}
26
28 - std::stringstream SS;
29 - SS << rrdset_context(RS) << "|" << rrdset_id(RS) << "|" << rrddim_name(RD);
30 - return SS.str();
27 +class Dimension {
28 +public:
29 + Dimension(RRDDIM *RD, RRDSET *AnomalyRateRS) :
30 + RD(RD),
31 + AnomalyRateRD(rrddim_add(AnomalyRateRS, ml::getMLDimensionID(RD).c_str(), NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE)),
32 + LastTrainedAt(Seconds(0)),
33 + Trained(false),
34 + ConstantModel(false),
35 + AnomalyScore(0.0),
36 + AnomalyBit(0),
37 + AnomalyBitCounter(0)
38 + { }
39 +
40 + RRDDIM *getRD() const {
41 + return RD;
42 }
43
33 - bool isActive() const {
34 - if (rrdset_flag_check(RD->rrdset, RRDSET_FLAG_OBSOLETE))
35 - return false;
36 -
37 - if (rrddim_flag_check(RD, RRDDIM_FLAG_OBSOLETE))
38 - return false;
44 + unsigned updateEvery() const {
45 + return RD->update_every;
46 + }
47
40 - return true;
48 + time_t latestTime() const {
49 + return Query(RD).latestTime();
50 }
51
43 - void setAnomalyRateRD(RRDDIM *ARRD) { AnomalyRateRD = ARRD; }
44 - RRDDIM *getAnomalyRateRD() const { return AnomalyRateRD; }
52 + time_t oldestTime() const {
53 + return Query(RD).oldestTime();
54 + }
55
56 void setAnomalyRateRDName(const char *Name) const {
57 rrddim_reset_name(AnomalyRateRD->rrdset, AnomalyRateRD, Name);
58 }
59
50 - virtual ~RrdDimension() {}
51 -
52 -private:
53 - RRDDIM *RD;
54 - RRDDIM *AnomalyRateRD;
60 + RRDDIM *getAnomalyRateRD() const {
61 + return AnomalyRateRD;
62 + }
63
56 - struct rrddim_query_ops *Ops;
64 + bool isTrained() const {
65 + return Trained;
66 + }
67
58 - std::string ID;
59 -};
68 + bool isAnomalous() const {
69 + return AnomalyBit;
70 + }
71
61 -enum class MLResult {
62 - Success = 0,
63 - MissingData,
64 - NaN,
65 -};
72 + bool shouldTrain(const TimePoint &TP) const;
73
67 -class TrainableDimension : public RrdDimension {
68 -public:
69 - TrainableDimension(RRDDIM *RD) :
70 - RrdDimension(RD), TrainEvery(Cfg.TrainEvery * updateEvery()) {}
74 + bool isActive() const;
75
76 MLResult trainModel();
77
74 - CalculatedNumber computeAnomalyScore(SamplesBuffer &SB) {
75 - return Trained ? KM.anomalyScore(SB) : 0.0;
76 - }
78 + bool predict(CalculatedNumber Value, bool Exists);
79
78 - bool shouldTrain(const TimePoint &TP) const {
79 - if (ConstantModel)
80 - return false;
80 + void updateAnomalyBitCounter(RRDSET *RS, unsigned Elapsed, bool IsAnomalous);
81
82 - return (LastTrainedAt + TrainEvery) < TP;
83 - }
82 + std::pair<bool, double> detect(size_t WindowLength, bool Reset);
83
85 - bool isTrained() const { return Trained; }
84 + std::array<KMeans, 1> getModels();
85
86 private:
87 std::pair<CalculatedNumber *, size_t> getCalculatedNumbers();
88
89 public:
91 - TimePoint LastTrainedAt{Seconds{0}};
92 -
93 -protected:
94 - std::atomic<bool> ConstantModel{false};
95 -
96 -private:
97 - Seconds TrainEvery;
98 - KMeans KM;
99 -
100 - std::atomic<bool> Trained{false};
101 -};
102 -
103 -class PredictableDimension : public TrainableDimension {
104 -public:
105 - PredictableDimension(RRDDIM *RD) : TrainableDimension(RD) {}
106 -
107 - std::pair<MLResult, bool> predict();
108 -
109 - void addValue(CalculatedNumber Value, bool Exists);
110 -
111 - bool isAnomalous() { return AnomalyBit; }
90 + RRDDIM *RD;
91 + RRDDIM *AnomalyRateRD;
92
113 - void updateAnomalyBitCounter(RRDSET *RS, unsigned Elapsed, bool IsAnomalous) {
114 - AnomalyBitCounter += IsAnomalous;
93 + TimePoint LastTrainedAt;
94 + std::atomic<bool> Trained;
95 + std::atomic<bool> ConstantModel;
96
116 - if (Elapsed == Cfg.DBEngineAnomalyRateEvery) {
117 - double AR = static_cast<double>(AnomalyBitCounter) / Cfg.DBEngineAnomalyRateEvery;
118 - rrddim_set_by_pointer(RS, getAnomalyRateRD(), AR * 1000);
119 - AnomalyBitCounter = 0;
120 - }
121 - }
122 -
123 -private:
124 - CalculatedNumber AnomalyScore{0.0};
125 - std::atomic<bool> AnomalyBit{false};
126 - unsigned AnomalyBitCounter{0};
97 + CalculatedNumber AnomalyScore;
98 + std::atomic<bool> AnomalyBit;
99 + unsigned AnomalyBitCounter;
100
101 std::vector<CalculatedNumber> CNs;
102 + std::array<KMeans, 1> Models;
103 + std::mutex Mutex;
104 };
105
131 -class DetectableDimension : public PredictableDimension {
132 -public:
133 - DetectableDimension(RRDDIM *RD) : PredictableDimension(RD) {}
134 -
135 - std::pair<bool, double> detect(size_t WindowLength, bool Reset) {
136 - bool AnomalyBit = isAnomalous();
137 -
138 - if (Reset)
139 - NumSetBits = BBC.numSetBits();
140 -
141 - NumSetBits += AnomalyBit;
142 - BBC.insert(AnomalyBit);
143 -
144 - double AnomalyRate = static_cast<double>(NumSetBits) / WindowLength;
145 - return { AnomalyBit, AnomalyRate };
146 - }
147 -
148 -private:
149 - BitBufferCounter BBC{static_cast<size_t>(Cfg.ADMinWindowSize)};
150 - size_t NumSetBits{0};
151 -};
152 -
153 -using Dimension = DetectableDimension;
154 -
106 } // namespace ml
107
108 #endif /* ML_DIMENSION_H */
ml/Host.cc
+30 -322
@@ -1,278 +1,20 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include <dlib/statistics.h>
4 -
3 #include "Config.h"
4 #include "Host.h"
5 +#include "ADCharts.h"
6
7 #include "json/single_include/nlohmann/json.hpp"
8
9 using namespace ml;
10
12 -static void updateDimensionsChart(RRDHOST *RH,
13 - collected_number NumTrainedDimensions,
14 - collected_number NumNormalDimensions,
15 - collected_number NumAnomalousDimensions) {
16 - static thread_local RRDSET *RS = nullptr;
17 - static thread_local RRDDIM *NumTotalDimensionsRD = nullptr;
18 - static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr;
19 - static thread_local RRDDIM *NumNormalDimensionsRD = nullptr;
20 - static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
21 -
22 - if (!RS) {
23 - std::stringstream IdSS, NameSS;
24 -
25 - IdSS << "dimensions_on_" << localhost->machine_guid;
26 - NameSS << "dimensions_on_" << rrdhost_hostname(localhost);
27 -
28 - RS = rrdset_create(
29 - RH,
30 - "anomaly_detection", // type
31 - IdSS.str().c_str(), // id
32 - NameSS.str().c_str(), // name
33 - "dimensions", // family
34 - "anomaly_detection.dimensions", // ctx
35 - "Anomaly detection dimensions", // title
36 - "dimensions", // units
37 - "netdata", // plugin
38 - "ml", // module
39 - 39183, // priority
40 - RH->rrd_update_every, // update_every
41 - RRDSET_TYPE_LINE // chart_type
42 - );
43 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
44 -
45 - NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
46 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
47 - NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL,
48 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
49 - NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL,
50 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
51 - NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL,
52 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
53 - } else
54 - rrdset_next(RS);
55 -
56 - rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions);
57 - rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions);
58 - rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions);
59 - rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions);
60 -
61 - rrdset_done(RS);
62 -}
63 -
64 -static void updateRateChart(RRDHOST *RH, collected_number AnomalyRate) {
65 - static thread_local RRDSET *RS = nullptr;
66 - static thread_local RRDDIM *AnomalyRateRD = nullptr;
67 -
68 - if (!RS) {
69 - std::stringstream IdSS, NameSS;
70 -
71 - IdSS << "anomaly_rate_on_" << localhost->machine_guid;
72 - NameSS << "anomaly_rate_on_" << rrdhost_hostname(localhost);
73 -
74 - RS = rrdset_create(
75 - RH,
76 - "anomaly_detection", // type
77 - IdSS.str().c_str(), // id
78 - NameSS.str().c_str(), // name
79 - "anomaly_rate", // family
80 - "anomaly_detection.anomaly_rate", // ctx
81 - "Percentage of anomalous dimensions", // title
82 - "percentage", // units
83 - "netdata", // plugin
84 - "ml", // module
85 - 39184, // priority
86 - RH->rrd_update_every, // update_every
87 - RRDSET_TYPE_LINE // chart_type
88 - );
89 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
90 -
91 - AnomalyRateRD = rrddim_add(RS, "anomaly_rate", NULL,
92 - 1, 100, RRD_ALGORITHM_ABSOLUTE);
93 - } else
94 - rrdset_next(RS);
95 -
96 - rrddim_set_by_pointer(RS, AnomalyRateRD, AnomalyRate);
97 -
98 - rrdset_done(RS);
99 -}
100 -
101 -static void updateWindowLengthChart(RRDHOST *RH, collected_number WindowLength) {
102 - static thread_local RRDSET *RS = nullptr;
103 - static thread_local RRDDIM *WindowLengthRD = nullptr;
104 -
105 - if (!RS) {
106 - std::stringstream IdSS, NameSS;
107 -
108 - IdSS << "detector_window_on_" << localhost->machine_guid;
109 - NameSS << "detector_window_on_" << rrdhost_hostname(localhost);
110 -
111 - RS = rrdset_create(
112 - RH,
113 - "anomaly_detection", // type
114 - IdSS.str().c_str(), // id
115 - NameSS.str().c_str(), // name
116 - "detector_window", // family
117 - "anomaly_detection.detector_window", // ctx
118 - "Anomaly detector window length", // title
119 - "seconds", // units
120 - "netdata", // plugin
121 - "ml", // module
122 - 39185, // priority
123 - RH->rrd_update_every, // update_every
124 - RRDSET_TYPE_LINE // chart_type
125 - );
126 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
127 -
128 - WindowLengthRD = rrddim_add(RS, "duration", NULL,
129 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
130 - } else
131 - rrdset_next(RS);
132 -
133 - rrddim_set_by_pointer(RS, WindowLengthRD, WindowLength * RH->rrd_update_every);
134 - rrdset_done(RS);
135 -}
136 -
137 -static void updateEventsChart(RRDHOST *RH,
138 - std::pair<BitRateWindow::Edge, size_t> P,
139 - bool ResetBitCounter,
140 - bool NewAnomalyEvent) {
141 - static thread_local RRDSET *RS = nullptr;
142 - static thread_local RRDDIM *AboveThresholdRD = nullptr;
143 - static thread_local RRDDIM *ResetBitCounterRD = nullptr;
144 - static thread_local RRDDIM *NewAnomalyEventRD = nullptr;
145 -
146 - if (!RS) {
147 - std::stringstream IdSS, NameSS;
148 -
149 - IdSS << "detector_events_on_" << localhost->machine_guid;
150 - NameSS << "detector_events_on_" << rrdhost_hostname(localhost);
151 -
152 - RS = rrdset_create(
153 - RH,
154 - "anomaly_detection", // type
155 - IdSS.str().c_str(), // id
156 - NameSS.str().c_str(), // name
157 - "detector_events", // family
158 - "anomaly_detection.detector_events", // ctx
159 - "Anomaly events triggered", // title
160 - "boolean", // units
161 - "netdata", // plugin
162 - "ml", // module
163 - 39186, // priority
164 - RH->rrd_update_every, // update_every
165 - RRDSET_TYPE_LINE // chart_type
166 - );
167 - rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
168 -
169 - AboveThresholdRD = rrddim_add(RS, "above_threshold", NULL,
170 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
171 - ResetBitCounterRD = rrddim_add(RS, "reset_bit_counter", NULL,
172 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
173 - NewAnomalyEventRD = rrddim_add(RS, "new_anomaly_event", NULL,
174 - 1, 1, RRD_ALGORITHM_ABSOLUTE);
175 - } else
176 - rrdset_next(RS);
177 -
178 - BitRateWindow::Edge E = P.first;
179 - bool AboveThreshold = E.second == BitRateWindow::State::AboveThreshold;
180 -
181 - rrddim_set_by_pointer(RS, AboveThresholdRD, AboveThreshold);
182 - rrddim_set_by_pointer(RS, ResetBitCounterRD, ResetBitCounter);
183 - rrddim_set_by_pointer(RS, NewAnomalyEventRD, NewAnomalyEvent);
184 -
185 - rrdset_done(RS);
186 -}
187 -
188 -static void updateDetectionChart(RRDHOST *RH) {
189 - static thread_local RRDSET *RS = nullptr;
190 - static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
191 -
192 - if (!RS) {
193 - std::stringstream IdSS, NameSS;
194 -
195 - IdSS << "prediction_stats_" << RH->machine_guid;
196 - NameSS << "prediction_stats_for_" << rrdhost_hostname(RH);
197 -
198 - RS = rrdset_create_localhost(
199 - "netdata", // type
200 - IdSS.str().c_str(), // id
201 - NameSS.str().c_str(), // name
202 - "ml", // family
203 - "netdata.prediction_stats", // ctx
204 - "Prediction thread CPU usage", // title
205 - "milliseconds/s", // units
206 - "netdata", // plugin
207 - "ml", // module
208 - 136000, // priority
209 - RH->rrd_update_every, // update_every
210 - RRDSET_TYPE_STACKED // chart_type
211 - );
212 -
213 - UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
214 - SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
215 - } else
216 - rrdset_next(RS);
217 -
218 - struct rusage TRU;
219 - getrusage(RUSAGE_THREAD, &TRU);
220 -
221 - rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);
222 - rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);
223 - rrdset_done(RS);
224 -}
225 -
226 -static void updateTrainingChart(RRDHOST *RH, struct rusage *TRU)
227 -{
228 - static thread_local RRDSET *RS = nullptr;
229 - static thread_local RRDDIM *UserRD = nullptr;
230 - static thread_local RRDDIM *SystemRD = nullptr;
231 -
232 - if (!RS) {
233 - std::stringstream IdSS, NameSS;
234 -
235 - IdSS << "training_stats_" << RH->machine_guid;
236 - NameSS << "training_stats_for_" << rrdhost_hostname(RH);
237 -
238 - RS = rrdset_create_localhost(
239 - "netdata", // type
240 - IdSS.str().c_str(), // id
241 - NameSS.str().c_str(), // name
242 - "ml", // family
243 - "netdata.training_stats", // ctx
244 - "Training thread CPU usage", // title
245 - "milliseconds/s", // units
246 - "netdata", // plugin
247 - "ml", // module
248 - 136001, // priority
249 - RH->rrd_update_every, // update_every
250 - RRDSET_TYPE_STACKED // chart_type
251 - );
252 -
253 - UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
254 - SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
255 - } else
256 - rrdset_next(RS);
257 -
258 - rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);
259 - rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);
260 - rrdset_done(RS);
261 -}
262 -
11 void RrdHost::addDimension(Dimension *D) {
264 - RRDDIM *AnomalyRateRD = rrddim_add(AnomalyRateRS, D->getID().c_str(), NULL,
265 - 1, 1000, RRD_ALGORITHM_ABSOLUTE);
266 - D->setAnomalyRateRD(AnomalyRateRD);
267 -
268 - {
269 - std::lock_guard<std::mutex> Lock(Mutex);
12 + std::lock_guard<std::mutex> Lock(Mutex);
13
271 - DimensionsMap[D->getRD()] = D;
14 + DimensionsMap[D->getRD()] = D;
15
273 - // Default construct mutex for dimension
274 - LocksMap[D];
275 - }
16 + // Default construct mutex for dimension
17 + LocksMap[D];
18 }
19
20 void RrdHost::removeDimension(Dimension *D) {
@@ -312,18 +54,33 @@ void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
54 Json["max-kmeans-iters"] = Cfg.MaxKMeansIters;
55
56 Json["dimension-anomaly-score-threshold"] = Cfg.DimensionAnomalyScoreThreshold;
315 - Json["host-anomaly-rate-threshold"] = Cfg.HostAnomalyRateThreshold;
57
317 - Json["min-window-size"] = Cfg.ADMinWindowSize;
318 - Json["max-window-size"] = Cfg.ADMaxWindowSize;
319 - Json["idle-window-size"] = Cfg.ADIdleWindowSize;
320 - Json["window-rate-threshold"] = Cfg.ADWindowRateThreshold;
321 - Json["dimension-rate-threshold"] = Cfg.ADDimensionRateThreshold;
58 + Json["host-anomaly-rate-threshold"] = Cfg.HostAnomalyRateThreshold;
59 + Json["anomaly-detection-grouping-method"] = group_method2string(Cfg.AnomalyDetectionGroupingMethod);
60 + Json["anomaly-detection-query-duration"] = Cfg.AnomalyDetectionQueryDuration;
61
62 Json["hosts-to-skip"] = Cfg.HostsToSkip;
63 Json["charts-to-skip"] = Cfg.ChartsToSkip;
64 }
65
66 +void TrainableHost::getModelsAsJson(nlohmann::json &Json) {
67 + std::lock_guard<std::mutex> Lock(Mutex);
68 +
69 + for (auto &DP : DimensionsMap) {
70 + Dimension *D = DP.second;
71 +
72 + nlohmann::json JsonArray = nlohmann::json::array();
73 + for (const KMeans &KM : D->getModels()) {
74 + nlohmann::json J;
75 + KM.toJson(J);
76 + JsonArray.push_back(J);
77 + }
78 + Json[getMLDimensionID(D->getRD())] = JsonArray;
79 + }
80 +
81 + return;
82 +}
83 +
84 std::pair<Dimension *, Duration<double>>
85 TrainableHost::findDimensionToTrain(const TimePoint &NowTP) {
86 std::lock_guard<std::mutex> Lock(Mutex);
@@ -393,23 +150,12 @@ void TrainableHost::train() {
150 #define WORKER_JOB_UPDATE_DETECTION_CHART 1
151 #define WORKER_JOB_UPDATE_ANOMALY_RATES 2
152 #define WORKER_JOB_UPDATE_CHARTS 3
396 -#define WORKER_JOB_SAVE_ANOMALY_EVENT 4
153
154 #if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
155 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
156 #endif
157
158 void DetectableHost::detectOnce() {
403 - auto P = BRW.insert(WindowAnomalyRate >= Cfg.HostAnomalyRateThreshold);
404 - BitRateWindow::Edge Edge = P.first;
405 - size_t WindowLength = P.second;
406 -
407 - bool ResetBitCounter = (Edge.first != BitRateWindow::State::AboveThreshold);
408 - bool NewAnomalyEvent = (Edge.first == BitRateWindow::State::AboveThreshold) &&
409 - (Edge.second == BitRateWindow::State::Idle);
410 -
411 - std::vector<std::pair<double, std::string>> DimsOverThreshold;
412 -
159 size_t NumAnomalousDimensions = 0;
160 size_t NumNormalDimensions = 0;
161 size_t NumTrainedDimensions = 0;
@@ -422,8 +168,6 @@ void DetectableHost::detectOnce() {
168 {
169 std::lock_guard<std::mutex> Lock(Mutex);
170
425 - DimsOverThreshold.reserve(DimensionsMap.size());
426 -
171 for (auto &DP : DimensionsMap) {
172 worker_is_busy(WORKER_JOB_DETECT_DIMENSION);
173
@@ -435,26 +179,18 @@ void DetectableHost::detectOnce() {
179 }
180
181 NumActiveDimensions++;
438 -
439 - auto P = D->detect(WindowLength, ResetBitCounter);
440 - bool IsAnomalous = P.first;
441 - double AnomalyScore = P.second;
442 -
182 NumTrainedDimensions += D->isTrained();
183
184 + bool IsAnomalous = D->isAnomalous();
185 if (IsAnomalous)
186 NumAnomalousDimensions += 1;
447 -
448 - if (NewAnomalyEvent && (AnomalyScore >= Cfg.ADDimensionRateThreshold))
449 - DimsOverThreshold.push_back({ AnomalyScore, D->getID() });
450 -
187 D->updateAnomalyBitCounter(AnomalyRateRS, AnomalyRateTimer, IsAnomalous);
188 }
189
190 if (NumAnomalousDimensions)
455 - WindowAnomalyRate = static_cast<double>(NumAnomalousDimensions) / NumActiveDimensions;
191 + HostAnomalyRate = static_cast<double>(NumAnomalousDimensions) / NumActiveDimensions;
192 else
457 - WindowAnomalyRate = 0.0;
193 + HostAnomalyRate = 0.0;
194
195 NumNormalDimensions = NumActiveDimensions - NumAnomalousDimensions;
196 }
@@ -472,38 +208,11 @@ void DetectableHost::detectOnce() {
208
209 worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
210 updateDimensionsChart(getRH(), NumTrainedDimensions, NumNormalDimensions, NumAnomalousDimensions);
475 - updateRateChart(getRH(), WindowAnomalyRate * 10000.0);
476 - updateWindowLengthChart(getRH(), WindowLength);
477 - updateEventsChart(getRH(), P, ResetBitCounter, NewAnomalyEvent);
211 + updateHostAndDetectionRateCharts(getRH(), HostAnomalyRate * 10000.0);
212
213 struct rusage TRU;
214 getResourceUsage(&TRU);
215 updateTrainingChart(getRH(), &TRU);
482 -
483 - if (!NewAnomalyEvent || (DimsOverThreshold.size() == 0))
484 - return;
485 -
486 - worker_is_busy(WORKER_JOB_SAVE_ANOMALY_EVENT);
487 -
488 - std::sort(DimsOverThreshold.begin(), DimsOverThreshold.end());
489 - std::reverse(DimsOverThreshold.begin(), DimsOverThreshold.end());
490 -
491 - // Make sure the JSON response won't grow beyond a specific number
492 - // of dimensions. Log an error message if this happens, because it
493 - // most likely means that the user specified a very-low anomaly rate
494 - // threshold.
495 - size_t NumMaxDimsOverThreshold = 2000;
496 - if (DimsOverThreshold.size() > NumMaxDimsOverThreshold) {
497 - error("Found %zu dimensions over threshold. Reducing JSON result to %zu dimensions.",
498 - DimsOverThreshold.size(), NumMaxDimsOverThreshold);
499 - DimsOverThreshold.resize(NumMaxDimsOverThreshold);
500 - }
501 -
502 - nlohmann::json JsonResult = DimsOverThreshold;
503 -
504 - time_t Before = now_realtime_sec();
505 - time_t After = Before - (WindowLength * updateEvery());
506 - DB.insertAnomaly("AD1", 1, getUUID(), After, Before, JsonResult.dump(4));
216 }
217
218 void DetectableHost::detect() {
@@ -512,7 +221,6 @@ void DetectableHost::detect() {
221 worker_register_job_name(WORKER_JOB_UPDATE_DETECTION_CHART, "detection chart");
222 worker_register_job_name(WORKER_JOB_UPDATE_ANOMALY_RATES, "anomaly rates");
223 worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "charts");
515 - worker_register_job_name(WORKER_JOB_SAVE_ANOMALY_EVENT, "anomaly event");
224
225 std::this_thread::sleep_for(Seconds{10});
226
ml/Host.h
+8 -22
@@ -3,12 +3,11 @@
3 #ifndef ML_HOST_H
4 #define ML_HOST_H
5
6 -#include "BitRateWindow.h"
6 #include "Config.h"
8 -#include "Database.h"
7 #include "Dimension.h"
8
9 #include "ml-private.h"
10 +#include "json/single_include/nlohmann/json.hpp"
11
12 namespace ml {
13
@@ -44,6 +43,10 @@ public:
43 return S;
44 }
45
46 + RRDSET *getAnomalyRateRS() const {
47 + return AnomalyRateRS;
48 + }
49 +
50 void addDimension(Dimension *D);
51 void removeDimension(Dimension *D);
52
@@ -78,6 +81,8 @@ public:
81 memcpy(RU, &ResourceUsage, sizeof(struct rusage));
82 }
83
84 + void getModelsAsJson(nlohmann::json &Json);
85 +
86 private:
87 std::pair<Dimension *, Duration<double>> findDimensionToTrain(const TimePoint &NowTP);
88 void trainDimension(Dimension *D, const TimePoint &NowTP);
@@ -93,16 +98,6 @@ public:
98 void startAnomalyDetectionThreads();
99 void stopAnomalyDetectionThreads();
100
96 - template<typename ...ArgTypes>
97 - bool getAnomalyInfo(ArgTypes&&... Args) {
98 - return DB.getAnomalyInfo(Args...);
99 - }
100 -
101 - template<typename ...ArgTypes>
102 - bool getAnomaliesInRange(ArgTypes&&... Args) {
103 - return DB.getAnomaliesInRange(Args...);
104 - }
105 -
101 void getDetectionInfoAsJson(nlohmann::json &Json) const;
102
103 private:
@@ -113,14 +108,7 @@ private:
108 std::thread TrainingThread;
109 std::thread DetectionThread;
110
116 - BitRateWindow BRW{
117 - static_cast<size_t>(Cfg.ADMinWindowSize),
118 - static_cast<size_t>(Cfg.ADMaxWindowSize),
119 - static_cast<size_t>(Cfg.ADIdleWindowSize),
120 - static_cast<size_t>(Cfg.ADMinWindowSize * Cfg.ADWindowRateThreshold)
121 - };
122 -
123 - CalculatedNumber WindowAnomalyRate{0.0};
111 + CalculatedNumber HostAnomalyRate{0.0};
112
113 size_t NumAnomalousDimensions{0};
114 size_t NumNormalDimensions{0};
@@ -128,8 +116,6 @@ private:
116 size_t NumActiveDimensions{0};
117
118 unsigned AnomalyRateTimer{0};
131 -
132 - Database DB{Cfg.AnomalyDBPath};
119 };
120
121 using Host = DetectableHost;
ml/KMeans.cc new
+43
@@ -0,0 +1,43 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "KMeans.h"
4 +#include <dlib/clustering.h>
5 +
6 +void KMeans::train(const std::vector<DSample> &Samples, size_t MaxIterations) {
7 + MinDist = std::numeric_limits<CalculatedNumber>::max();
8 + MaxDist = std::numeric_limits<CalculatedNumber>::min();
9 +
10 + ClusterCenters.clear();
11 +
12 + dlib::pick_initial_centers(NumClusters, ClusterCenters, Samples);
13 + dlib::find_clusters_using_kmeans(Samples, ClusterCenters, MaxIterations);
14 +
15 + for (const auto &S : Samples) {
16 + CalculatedNumber MeanDist = 0.0;
17 +
18 + for (const auto &KMCenter : ClusterCenters)
19 + MeanDist += dlib::length(KMCenter - S);
20 +
21 + MeanDist /= NumClusters;
22 +
23 + if (MeanDist < MinDist)
24 + MinDist = MeanDist;
25 +
26 + if (MeanDist > MaxDist)
27 + MaxDist = MeanDist;
28 + }
29 +}
30 +
31 +CalculatedNumber KMeans::anomalyScore(const DSample &Sample) const {
32 + CalculatedNumber MeanDist = 0.0;
33 + for (const auto &CC: ClusterCenters)
34 + MeanDist += dlib::length(CC - Sample);
35 +
36 + MeanDist /= NumClusters;
37 +
38 + if (MaxDist == MinDist)
39 + return 0.0;
40 +
41 + CalculatedNumber AnomalyScore = 100.0 * std::abs((MeanDist - MinDist) / (MaxDist - MinDist));
42 + return (AnomalyScore > 100.0) ? 100.0 : AnomalyScore;
43 +}
ml/KMeans.h renamed
+11 -4
@@ -9,6 +9,7 @@
9 #include <mutex>
10
11 #include "SamplesBuffer.h"
12 +#include "json/single_include/nlohmann/json.hpp"
13
14 class KMeans {
15 public:
@@ -17,8 +18,16 @@ public:
18 MaxDist = std::numeric_limits<CalculatedNumber>::min();
19 };
20
20 - void train(SamplesBuffer &SB, size_t MaxIterations);
21 - CalculatedNumber anomalyScore(SamplesBuffer &SB);
21 + void train(const std::vector<DSample> &Samples, size_t MaxIterations);
22 + CalculatedNumber anomalyScore(const DSample &Sample) const;
23 +
24 + void toJson(nlohmann::json &J) const {
25 + J = nlohmann::json{
26 + {"CCs", ClusterCenters},
27 + {"MinDist", MinDist},
28 + {"MaxDist", MaxDist}
29 + };
30 + }
31
32 private:
33 size_t NumClusters;
@@ -27,8 +36,6 @@ private:
36
37 CalculatedNumber MinDist;
38 CalculatedNumber MaxDist;
30 -
31 - std::mutex Mutex;
39 };
40
41 #endif /* KMEANS_H */
ml/Makefile.am deleted
-8
@@ -1,8 +0,0 @@
1 -# SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -AUTOMAKE_OPTIONS = subdir-objects
4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 -
6 -SUBDIRS = \
7 - kmeans \
8 - $(NULL)
ml/Query.h
+8 -5
@@ -7,7 +7,7 @@ namespace ml {
7
8 class Query {
9 public:
10 - Query(RRDDIM *RD) : RD(RD) {
10 + Query(RRDDIM *RD) : RD(RD), Initialized(false) {
11 Ops = &RD->tiers[0]->query_ops;
12 }
13
@@ -21,23 +21,26 @@ public:
21
22 void init(time_t AfterT, time_t BeforeT) {
23 Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT, TIER_QUERY_FETCH_SUM);
24 + Initialized = true;
25 }
26
27 bool isFinished() {
28 return Ops->is_finished(&Handle);
29 }
30
31 + ~Query() {
32 + if (Initialized)
33 + Ops->finalize(&Handle);
34 + }
35 +
36 std::pair<time_t, CalculatedNumber> nextMetric() {
37 STORAGE_POINT sp = Ops->next_metric(&Handle);
38 return { sp.start_time, sp.sum / sp.count };
39 }
40
35 - ~Query() {
36 - Ops->finalize(&Handle);
37 - }
38 -
41 private:
42 RRDDIM *RD;
43 + bool Initialized;
44
45 struct rrddim_query_ops *Ops;
46 struct rrddim_query_handle Handle;
ml/SamplesBuffer.cc renamed
ml/SamplesBuffer.h renamed
ml/SamplesBufferTests.cc renamed
+6 -3
@@ -36,7 +36,8 @@ TEST(SamplesBufferTest, NS_8_NDPS_1_DN_1_SN_3_LN_1) {
36 CNs[6] = 0.2684839023122384;
37 CNs[7] = 0.851332948637479;
38
39 - SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN);
39 + std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
40 + SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
41 SB.preprocess();
42
43 std::vector<Sample> Samples = SB.getPreprocessedSamples();
@@ -76,7 +77,8 @@ TEST(SamplesBufferTest, NS_8_NDPS_1_DN_2_SN_3_LN_2) {
77 CNs[6] = 0.15552559051428083;
78 CNs[7] = 0.6309750314597955;
79
79 - SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN);
80 + std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
81 + SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
82 SB.preprocess();
83
84 std::vector<Sample> Samples = SB.getPreprocessedSamples();
@@ -114,7 +116,8 @@ TEST(SamplesBufferTest, NS_8_NDPS_3_DN_2_SN_4_LN_1) {
116 CNs[18] = 0.9394494507088997; CNs[19] =0.17567223681734334; CNs[20] = 0.42732886195446984;
117 CNs[21] = 0.9460522396152958; CNs[22] =0.23462747016780894; CNs[23] = 0.35983249900892145;
118
117 - SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN);
119 + std::vector<uint32_t> RandNums(NumSamples, std::numeric_limits<uint32_t>::max());
120 + SamplesBuffer SB(CNs, NumSamples, NumDimsPerSample, DiffN, SmoothN, LagN, 1.0, RandNums);
121 SB.preprocess();
122
123 std::vector<Sample> Samples = SB.getPreprocessedSamples();
ml/Tests.cc deleted
-301
@@ -1,301 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "BitBufferCounter.h"
4 -#include "BitRateWindow.h"
5 -
6 -#include "gtest/gtest.h"
7 -
8 -using namespace ml;
9 -
10 -TEST(BitBufferCounterTest, Cap_4) {
11 - size_t Capacity = 4;
12 - BitBufferCounter BBC(Capacity);
13 -
14 - // No bits set
15 - EXPECT_EQ(BBC.numSetBits(), 0);
16 -
17 - // All ones
18 - for (size_t Idx = 0; Idx != (2 * Capacity); Idx++) {
19 - BBC.insert(true);
20 -
21 - EXPECT_EQ(BBC.numSetBits(), std::min(Idx + 1, Capacity));
22 - }
23 -
24 - // All zeroes
25 - for (size_t Idx = 0; Idx != Capacity; Idx++) {
26 - BBC.insert(false);
27 -
28 - if (Idx < Capacity)
29 - EXPECT_EQ(BBC.numSetBits(), Capacity - (Idx + 1));
30 - else
31 - EXPECT_EQ(BBC.numSetBits(), 0);
32 - }
33 -
34 - // Even ones/zeroes
35 - for (size_t Idx = 0; Idx != (2 * Capacity); Idx++)
36 - BBC.insert(Idx % 2 == 0);
37 - EXPECT_EQ(BBC.numSetBits(), Capacity / 2);
38 -}
39 -
40 -using State = BitRateWindow::State;
41 -using Edge = BitRateWindow::Edge;
42 -using Result = std::pair<Edge, size_t>;
43 -
44 -TEST(BitRateWindowTest, Cycles) {
45 - /* Test the FSM by going through its two cycles:
46 - * 1) NotFilled -> AboveThreshold -> Idle -> NotFilled
47 - * 2) NotFilled -> BelowThreshold -> AboveThreshold -> Idle -> NotFilled
48 - *
49 - * Check the window's length on every new state transition.
50 - */
51 -
52 - size_t MinLength = 4, MaxLength = 6, IdleLength = 5;
53 - size_t SetBitsThreshold = 3;
54 -
55 - Result R;
56 - BitRateWindow BRW(MinLength, MaxLength, IdleLength, SetBitsThreshold);
57 -
58 - /*
59 - * 1st cycle
60 - */
61 -
62 - // NotFilled -> AboveThreshold
63 - R = BRW.insert(true);
64 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
65 - R = BRW.insert(true);
66 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
67 - R = BRW.insert(true);
68 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
69 - R = BRW.insert(true);
70 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::AboveThreshold));
71 - EXPECT_EQ(R.second, MinLength);
72 -
73 - // AboveThreshold -> Idle
74 - R = BRW.insert(true);
75 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::AboveThreshold));
76 - R = BRW.insert(true);
77 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::AboveThreshold));
78 -
79 - R = BRW.insert(true);
80 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::Idle));
81 - EXPECT_EQ(R.second, MaxLength);
82 -
83 -
84 - // Idle -> NotFilled
85 - R = BRW.insert(true);
86 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
87 - R = BRW.insert(true);
88 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
89 - R = BRW.insert(true);
90 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
91 - R = BRW.insert(true);
92 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
93 - R = BRW.insert(true);
94 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::NotFilled));
95 - EXPECT_EQ(R.second, 1);
96 -
97 - // NotFilled -> AboveThreshold
98 - R = BRW.insert(true);
99 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
100 - R = BRW.insert(true);
101 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
102 - R = BRW.insert(true);
103 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::AboveThreshold));
104 - EXPECT_EQ(R.second, MinLength);
105 -
106 - /*
107 - * 2nd cycle
108 - */
109 -
110 - BRW = BitRateWindow(MinLength, MaxLength, IdleLength, SetBitsThreshold);
111 -
112 - // NotFilled -> BelowThreshold
113 - R = BRW.insert(false);
114 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
115 - R = BRW.insert(false);
116 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
117 - R = BRW.insert(false);
118 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
119 - R = BRW.insert(false);
120 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::BelowThreshold));
121 - EXPECT_EQ(R.second, MinLength);
122 -
123 - // BelowThreshold -> BelowThreshold:
124 - // Check the state's self loop by adding set bits that will keep the
125 - // bit buffer below the specified threshold.
126 - //
127 - for (size_t Idx = 0; Idx != 2 * MaxLength; Idx++) {
128 - R = BRW.insert(Idx % 2 == 0);
129 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::BelowThreshold));
130 - EXPECT_EQ(R.second, MinLength);
131 - }
132 -
133 - // Verify that at the end of the loop the internal bit buffer contains
134 - // "1010". Do so by adding one set bit and checking that we remain below
135 - // the specified threshold.
136 - R = BRW.insert(true);
137 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::BelowThreshold));
138 - EXPECT_EQ(R.second, MinLength);
139 -
140 - // BelowThreshold -> AboveThreshold
141 - R = BRW.insert(true);
142 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::AboveThreshold));
143 - EXPECT_EQ(R.second, MinLength);
144 -
145 - // AboveThreshold -> Idle:
146 - // Do the transition without filling the max window size this time.
147 - R = BRW.insert(false);
148 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::Idle));
149 - EXPECT_EQ(R.second, MinLength);
150 -
151 - // Idle -> NotFilled
152 - R = BRW.insert(false);
153 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
154 - R = BRW.insert(false);
155 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
156 - R = BRW.insert(false);
157 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
158 - R = BRW.insert(false);
159 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
160 - R = BRW.insert(false);
161 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::NotFilled));
162 - EXPECT_EQ(R.second, 1);
163 -
164 - // NotFilled -> AboveThreshold
165 - R = BRW.insert(true);
166 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
167 - R = BRW.insert(true);
168 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::NotFilled));
169 - R = BRW.insert(true);
170 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::AboveThreshold));
171 - EXPECT_EQ(R.second, MinLength);
172 -}
173 -
174 -TEST(BitRateWindowTest, ConsecutiveOnes) {
175 - size_t MinLength = 120, MaxLength = 240, IdleLength = 30;
176 - size_t SetBitsThreshold = 30;
177 -
178 - Result R;
179 - BitRateWindow BRW(MinLength, MaxLength, IdleLength, SetBitsThreshold);
180 -
181 - for (size_t Idx = 0; Idx != MaxLength; Idx++)
182 - R = BRW.insert(false);
183 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::BelowThreshold));
184 - EXPECT_EQ(R.second, MinLength);
185 -
186 - for (size_t Idx = 0; Idx != SetBitsThreshold; Idx++) {
187 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::BelowThreshold));
188 - R = BRW.insert(true);
189 - }
190 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::AboveThreshold));
191 - EXPECT_EQ(R.second, MinLength);
192 -
193 - // At this point the window's buffer contains:
194 - // (MinLength - SetBitsThreshold = 90) 0s, followed by
195 - // (SetBitsThreshold = 30) 1s.
196 - //
197 - // To go below the threshold, we need to add (90 + 1) more 0s in the window's
198 - // buffer. At that point, the the window's buffer will contain:
199 - // (SetBitsThreshold = 29) 1s, followed by
200 - // (MinLength - SetBitsThreshold = 91) 0s.
201 - //
202 - // Right before adding the last 0, we expect the window's length to be equal to 210,
203 - // because the bit buffer has gone through these bits:
204 - // (MinLength - SetBitsThreshold = 90) 0s, followed by
205 - // (SetBitsThreshold = 30) 1s, followed by
206 - // (MinLength - SetBitsThreshold = 90) 0s.
207 -
208 - for (size_t Idx = 0; Idx != (MinLength - SetBitsThreshold); Idx++) {
209 - R = BRW.insert(false);
210 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::AboveThreshold));
211 - }
212 - EXPECT_EQ(R.second, 2 * MinLength - SetBitsThreshold);
213 - R = BRW.insert(false);
214 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::Idle));
215 -
216 - // Continue with the Idle -> NotFilled edge.
217 - for (size_t Idx = 0; Idx != IdleLength - 1; Idx++) {
218 - R = BRW.insert(false);
219 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::Idle));
220 - }
221 - R = BRW.insert(false);
222 - EXPECT_EQ(R.first, std::make_pair(State::Idle, State::NotFilled));
223 - EXPECT_EQ(R.second, 1);
224 -}
225 -
226 -TEST(BitRateWindowTest, WithHoles) {
227 - size_t MinLength = 120, MaxLength = 240, IdleLength = 30;
228 - size_t SetBitsThreshold = 30;
229 -
230 - Result R;
231 - BitRateWindow BRW(MinLength, MaxLength, IdleLength, SetBitsThreshold);
232 -
233 - for (size_t Idx = 0; Idx != MaxLength; Idx++)
234 - R = BRW.insert(false);
235 -
236 - for (size_t Idx = 0; Idx != SetBitsThreshold / 3; Idx++)
237 - R = BRW.insert(true);
238 - for (size_t Idx = 0; Idx != SetBitsThreshold / 3; Idx++)
239 - R = BRW.insert(false);
240 - for (size_t Idx = 0; Idx != SetBitsThreshold / 3; Idx++)
241 - R = BRW.insert(true);
242 - for (size_t Idx = 0; Idx != SetBitsThreshold / 3; Idx++)
243 - R = BRW.insert(false);
244 - for (size_t Idx = 0; Idx != SetBitsThreshold / 3; Idx++)
245 - R = BRW.insert(true);
246 -
247 - EXPECT_EQ(R.first, std::make_pair(State::BelowThreshold, State::AboveThreshold));
248 - EXPECT_EQ(R.second, MinLength);
249 -
250 - // The window's bit buffer contains:
251 - // 70 0s, 10 1s, 10 0s, 10 1s, 10 0s, 10 1s.
252 - // Where: 70 = MinLength - (5 / 3) * SetBitsThresholds, ie. we need
253 - // to add (70 + 1) more zeros to make the bit buffer go below the
254 - // threshold and then the window's length should be:
255 - // 70 + 50 + 70 = 190.
256 -
257 - BitRateWindow::Edge E;
258 - do {
259 - R = BRW.insert(false);
260 - E = R.first;
261 - } while (E.first != State::AboveThreshold || E.second != State::Idle);
262 - EXPECT_EQ(R.second, 2 * MinLength - (5 * SetBitsThreshold) / 3);
263 -}
264 -
265 -TEST(BitRateWindowTest, MinWindow) {
266 - size_t MinLength = 120, MaxLength = 240, IdleLength = 30;
267 - size_t SetBitsThreshold = 30;
268 -
269 - Result R;
270 - BitRateWindow BRW(MinLength, MaxLength, IdleLength, SetBitsThreshold);
271 -
272 - BRW.insert(true);
273 - BRW.insert(false);
274 - for (size_t Idx = 2; Idx != SetBitsThreshold; Idx++)
275 - BRW.insert(true);
276 - for (size_t Idx = SetBitsThreshold; Idx != MinLength - 1; Idx++)
277 - BRW.insert(false);
278 -
279 - R = BRW.insert(true);
280 - EXPECT_EQ(R.first, std::make_pair(State::NotFilled, State::AboveThreshold));
281 - EXPECT_EQ(R.second, MinLength);
282 -
283 - R = BRW.insert(false);
284 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::Idle));
285 -}
286 -
287 -TEST(BitRateWindowTest, MaxWindow) {
288 - size_t MinLength = 100, MaxLength = 200, IdleLength = 30;
289 - size_t SetBitsThreshold = 50;
290 -
291 - Result R;
292 - BitRateWindow BRW(MinLength, MaxLength, IdleLength, SetBitsThreshold);
293 -
294 - for (size_t Idx = 0; Idx != MaxLength; Idx++)
295 - R = BRW.insert(Idx % 2 == 0);
296 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::AboveThreshold));
297 - EXPECT_EQ(R.second, MaxLength);
298 -
299 - R = BRW.insert(false);
300 - EXPECT_EQ(R.first, std::make_pair(State::AboveThreshold, State::Idle));
301 -}
ml/dlib renamed
ml/kmeans/KMeans.cc deleted
-55
@@ -1,55 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "KMeans.h"
4 -#include <dlib/clustering.h>
5 -
6 -void KMeans::train(SamplesBuffer &SB, size_t MaxIterations) {
7 - std::vector<DSample> Samples = SB.preprocess();
8 -
9 - MinDist = std::numeric_limits<CalculatedNumber>::max();
10 - MaxDist = std::numeric_limits<CalculatedNumber>::min();
11 -
12 - {
13 - std::lock_guard<std::mutex> Lock(Mutex);
14 -
15 - ClusterCenters.clear();
16 -
17 - dlib::pick_initial_centers(NumClusters, ClusterCenters, Samples);
18 - dlib::find_clusters_using_kmeans(Samples, ClusterCenters, MaxIterations);
19 -
20 - for (const auto &S : Samples) {
21 - CalculatedNumber MeanDist = 0.0;
22 -
23 - for (const auto &KMCenter : ClusterCenters)
24 - MeanDist += dlib::length(KMCenter - S);
25 -
26 - MeanDist /= NumClusters;
27 -
28 - if (MeanDist < MinDist)
29 - MinDist = MeanDist;
30 -
31 - if (MeanDist > MaxDist)
32 - MaxDist = MeanDist;
33 - }
34 - }
35 -}
36 -
37 -CalculatedNumber KMeans::anomalyScore(SamplesBuffer &SB) {
38 - std::vector<DSample> DSamples = SB.preprocess();
39 -
40 - std::unique_lock<std::mutex> Lock(Mutex, std::defer_lock);
41 - if (!Lock.try_lock())
42 - return std::numeric_limits<CalculatedNumber>::quiet_NaN();
43 -
44 - CalculatedNumber MeanDist = 0.0;
45 - for (const auto &CC: ClusterCenters)
46 - MeanDist += dlib::length(CC - DSamples.back());
47 -
48 - MeanDist /= NumClusters;
49 -
50 - if (MaxDist == MinDist)
51 - return 0.0;
52 -
53 - CalculatedNumber AnomalyScore = 100.0 * std::abs((MeanDist - MinDist) / (MaxDist - MinDist));
54 - return (AnomalyScore > 100.0) ? 100.0 : AnomalyScore;
55 -}
ml/kmeans/Makefile.am deleted
-4
@@ -1,4 +0,0 @@
1 -# SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -AUTOMAKE_OPTIONS = subdir-objects
4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
ml/ml-dummy.c
+10 -14
@@ -24,6 +24,16 @@ char *ml_get_host_info(RRDHOST *RH) {
24 return NULL;
25 }
26
27 +char *ml_get_host_runtime_info(RRDHOST *RH) {
28 + (void) RH;
29 + return NULL;
30 +}
31 +
32 +char *ml_get_host_models(RRDHOST *RH) {
33 + (void) RH;
34 + return NULL;
35 +}
36 +
37 void ml_new_dimension(RRDDIM *RD) { (void) RD; }
38
39 void ml_delete_dimension(RRDDIM *RD) { (void) RD; }
@@ -33,20 +43,6 @@ bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
43 return false;
44 }
45
36 -char *ml_get_anomaly_events(RRDHOST *RH, const char *AnomalyDetectorName,
37 - int AnomalyDetectorVersion, time_t After, time_t Before) {
38 - (void) RH; (void) AnomalyDetectorName;
39 - (void) AnomalyDetectorVersion; (void) After; (void) Before;
40 - return NULL;
41 -}
42 -
43 -char *ml_get_anomaly_event_info(RRDHOST *RH, const char *AnomalyDetectorName,
44 - int AnomalyDetectorVersion, time_t After, time_t Before) {
45 - (void) RH; (void) AnomalyDetectorName;
46 - (void) AnomalyDetectorVersion; (void) After; (void) Before;
47 - return NULL;
48 -}
49 -
46 void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
47 (void) R;
48 (void) MaxAnomalyRates;
ml/ml-private.h
+1 -1
@@ -3,7 +3,7 @@
3 #ifndef ML_PRIVATE_H
4 #define ML_PRIVATE_H
5
6 -#include "kmeans/KMeans.h"
6 +#include "KMeans.h"
7 #include "ml/ml.h"
8
9 #include <chrono>
ml/ml.cc
+13 -48
@@ -82,7 +82,7 @@ void ml_new_dimension(RRDDIM *RD) {
82 if (rrdset_is_ar_chart(RS))
83 return;
84
85 - Dimension *D = new Dimension(RD);
85 + Dimension *D = new Dimension(RD, H->getAnomalyRateRS());
86 RD->ml_dimension = static_cast<ml_dimension_t>(D);
87 H->addDimension(D);
88 }
@@ -127,59 +127,24 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
127 return strdup(ConfigJson.dump(1, '\t').c_str());
128 }
129
130 -bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
131 - Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
132 - if (!D)
133 - return false;
134 -
135 - D->addValue(Value, Exists);
136 - bool Result = D->predict().second;
137 - return Result;
138 -}
130 +char *ml_get_host_models(RRDHOST *RH) {
131 + nlohmann::json ModelsJson;
132
140 -char *ml_get_anomaly_events(RRDHOST *RH, const char *AnomalyDetectorName,
141 - int AnomalyDetectorVersion, time_t After, time_t Before) {
142 - if (!RH || !RH->ml_host) {
143 - error("No host");
144 - return nullptr;
145 - }
146 -
147 - Host *H = static_cast<Host *>(RH->ml_host);
148 - std::vector<std::pair<time_t, time_t>> TimeRanges;
149 -
150 - bool Res = H->getAnomaliesInRange(TimeRanges, AnomalyDetectorName,
151 - AnomalyDetectorVersion,
152 - H->getUUID(),
153 - After, Before);
154 - if (!Res) {
155 - error("DB result is empty");
156 - return nullptr;
133 + if (RH && RH->ml_host) {
134 + Host *H = static_cast<Host *>(RH->ml_host);
135 + H->getModelsAsJson(ModelsJson);
136 + return strdup(ModelsJson.dump(2, '\t').c_str());
137 }
138
159 - nlohmann::json Json = TimeRanges;
160 - return strdup(Json.dump(4).c_str());
139 + return nullptr;
140 }
141
163 -char *ml_get_anomaly_event_info(RRDHOST *RH, const char *AnomalyDetectorName,
164 - int AnomalyDetectorVersion, time_t After, time_t Before) {
165 - if (!RH || !RH->ml_host) {
166 - error("No host");
167 - return nullptr;
168 - }
169 -
170 - Host *H = static_cast<Host *>(RH->ml_host);
171 -
172 - nlohmann::json Json;
173 - bool Res = H->getAnomalyInfo(Json, AnomalyDetectorName,
174 - AnomalyDetectorVersion,
175 - H->getUUID(),
176 - After, Before);
177 - if (!Res) {
178 - error("DB result is empty");
179 - return nullptr;
180 - }
142 +bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
143 + Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
144 + if (!D)
145 + return false;
146
182 - return strdup(Json.dump(4, '\t').c_str());
147 + return D->predict(Value, Exists);
148 }
149
150 void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
ml/ml.h
+1 -6
@@ -28,18 +28,13 @@ void ml_delete_host(RRDHOST *RH);
28
29 char *ml_get_host_info(RRDHOST *RH);
30 char *ml_get_host_runtime_info(RRDHOST *RH);
31 +char *ml_get_host_models(RRDHOST *RH);
32
33 void ml_new_dimension(RRDDIM *RD);
34 void ml_delete_dimension(RRDDIM *RD);
35
36 bool ml_is_anomalous(RRDDIM *RD, double value, bool exists);
37
37 -char *ml_get_anomaly_events(RRDHOST *RH, const char *AnomalyDetectorName,
38 - int AnomalyDetectorVersion, time_t After, time_t Before);
39 -
40 -char *ml_get_anomaly_event_info(RRDHOST *RH, const char *AnomalyDetectorName,
41 - int AnomalyDetectorVersion, time_t After, time_t Before);
42 -
38 void ml_process_rrdr(RRDR *R, int MaxAnomalyRates);
39
40 void ml_dimension_update_name(RRDSET *RS, RRDDIM *RD, const char *name);
web/api/web_api_v1.c
+10 -78
@@ -1329,81 +1329,15 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
1329 }
1330
1331 #if defined(ENABLE_ML)
1332 -int web_client_api_request_v1_anomaly_events(RRDHOST *host, struct web_client *w, char *url) {
1333 - if (!netdata_ready)
1334 - return HTTP_RESP_BACKEND_FETCH_FAILED;
1335 -
1336 - uint32_t after = 0, before = 0;
1337 -
1338 - while (url) {
1339 - char *value = mystrsep(&url, "&");
1340 - if (!value || !*value)
1341 - continue;
1342 -
1343 - char *name = mystrsep(&value, "=");
1344 - if (!name || !*name)
1345 - continue;
1346 - if (!value || !*value)
1347 - continue;
1348 -
1349 - if (!strcmp(name, "after"))
1350 - after = (uint32_t) (strtoul(value, NULL, 0) / 1000);
1351 - else if (!strcmp(name, "before"))
1352 - before = (uint32_t) (strtoul(value, NULL, 0) / 1000);
1353 - }
1354 -
1355 - char *s;
1356 - if (!before || !after)
1357 - s = strdupz("{\"error\": \"missing after/before parameters\" }\n");
1358 - else {
1359 - s = ml_get_anomaly_events(host, "AD1", 1, after, before);
1360 - if (!s)
1361 - s = strdupz("{\"error\": \"json string is empty\" }\n");
1362 - }
1363 -
1364 - BUFFER *wb = w->response.data;
1365 - buffer_flush(wb);
1366 -
1367 - wb->contenttype = CT_APPLICATION_JSON;
1368 - buffer_strcat(wb, s);
1369 - buffer_no_cacheable(wb);
1370 -
1371 - freez(s);
1372 -
1373 - return HTTP_RESP_OK;
1374 -}
1332 +int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char *url) {
1333 + (void) url;
1334
1376 -int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_client *w, char *url) {
1335 if (!netdata_ready)
1336 return HTTP_RESP_BACKEND_FETCH_FAILED;
1337
1380 - uint32_t after = 0, before = 0;
1381 -
1382 - while (url) {
1383 - char *value = mystrsep(&url, "&");
1384 - if (!value || !*value)
1385 - continue;
1386 -
1387 - char *name = mystrsep(&value, "=");
1388 - if (!name || !*name)
1389 - continue;
1390 - if (!value || !*value)
1391 - continue;
1392 -
1393 - if (!strcmp(name, "after"))
1394 - after = (uint32_t) strtoul(value, NULL, 0);
1395 - else if (!strcmp(name, "before"))
1396 - before = (uint32_t) strtoul(value, NULL, 0);
1397 - }
1398 -
1399 - char *s;
1400 - if (!before || !after)
1401 - s = strdupz("{\"error\": \"missing after/before parameters\" }\n");
1402 - else {
1403 - s = ml_get_anomaly_event_info(host, "AD1", 1, after, before);
1404 - if (!s)
1405 - s = strdupz("{\"error\": \"json string is empty\" }\n");
1406 - }
1338 + char *s = ml_get_host_runtime_info(host);
1339 + if (!s)
1340 + s = strdupz("{\"error\": \"json string is empty\" }\n");
1341
1342 BUFFER *wb = w->response.data;
1343 buffer_flush(wb);
@@ -1415,13 +1349,13 @@ int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_clien
1349 return HTTP_RESP_OK;
1350 }
1351
1418 -int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char *url) {
1352 +int web_client_api_request_v1_ml_models(RRDHOST *host, struct web_client *w, char *url) {
1353 (void) url;
1354
1355 if (!netdata_ready)
1356 return HTTP_RESP_BACKEND_FETCH_FAILED;
1357
1424 - char *s = ml_get_host_runtime_info(host);
1358 + char *s = ml_get_host_models(host);
1359 if (!s)
1360 s = strdupz("{\"error\": \"json string is empty\" }\n");
1361
@@ -1434,8 +1368,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
1368 freez(s);
1369 return HTTP_RESP_OK;
1370 }
1437 -
1438 -#endif // defined(ENABLE_ML)
1371 +#endif
1372
1373 inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url) {
1374 (void)url;
@@ -1680,9 +1613,8 @@ static struct api_command {
1613 { "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_allmetrics },
1614
1615 #if defined(ENABLE_ML)
1683 - { "anomaly_events", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_events },
1684 - { "anomaly_event_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_event_info },
1685 - { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info },
1616 + { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info },
1617 + { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },
1618 #endif
1619
1620 { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health },