@cryptotaxi247 / netdata-1 / commits / 9c834eff9

Update ML-related charts (#12574)

* Move CPU usage stats under netdata charts Use the hostname in each chart's name, and the machine GUID in each chart's id. * Move anomaly_detection.* charts to child host instance. * Add option to enable/disable streaming of ML-related charts. * Update priority of prediction/training charts.

vkalintiris committed Apr 4, 2022 at 14:22 UTC 9c834eff9715d1347b33f9f1f8c3deaf5c162443
8 files changed +86 -60
database/rrd.h
+1 -1
@@ -478,8 +478,8 @@ typedef enum rrdset_flags {
478 // least rrdset_free_obsolete_time seconds ago.
479 RRDSET_FLAG_ARCHIVED = 1 << 15,
480 RRDSET_FLAG_ACLK = 1 << 16,
481 -
481 RRDSET_FLAG_PENDING_FOREACH_ALARMS = 1 << 17, // contains dims with uninitialized foreach alarms
482 + RRDSET_FLAG_ANOMALY_DETECTION = 1 << 18 // flag to identify anomaly detection charts.
483 } RRDSET_FLAGS;
484
485 #ifdef HAVE_C___ATOMIC
ml/Config.cc
+2
@@ -133,4 +133,6 @@ void Config::readMLConfig(void) {
133 Cfg.ChartsToSkip = "anomaly_detection.* ";
134 Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");
135 Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
136 +
137 + Cfg.StreamADCharts = config_get_boolean(ConfigSectionML, "stream anomaly detection charts", false);
138 }
ml/Config.h
+2
@@ -33,6 +33,8 @@ public:
33 double ADWindowRateThreshold;
34 double ADDimensionRateThreshold;
35
36 + bool StreamADCharts;
37 +
38 std::string HostsToSkip;
39 SIMPLE_PATTERN *SP_HostsToSkip;
40
ml/Host.cc
+68 -59
@@ -9,17 +9,6 @@
9
10 using namespace ml;
11
12 -static std::pair<std::string, std::string>
13 -getHostSpecificIdAndTitle(RRDHOST *RH, const std::string &IdPrefix,
14 - const std::string &TitlePrefix) {
15 - std::stringstream IdSS, TitleSS;
16 -
17 - IdSS << IdPrefix << "_" << RH->machine_guid;
18 - TitleSS << TitlePrefix << " " << RH->hostname;
19 -
20 - return {IdSS.str(), TitleSS.str()};
21 -}
22 -
12 static void updateDimensionsChart(RRDHOST *RH,
13 collected_number NumTrainedDimensions,
14 collected_number NumNormalDimensions,
@@ -31,17 +20,20 @@ static void updateDimensionsChart(RRDHOST *RH,
20 static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
21
22 if (!RS) {
34 - std::string IdPrefix = "dimensions";
35 - std::string TitlePrefix = "Anomaly detection dimensions for host";
36 - auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);
23 + std::stringstream IdSS, NameSS, TitleSS;
24
38 - RS = rrdset_create_localhost(
25 + IdSS << "dimensions_on_" << localhost->machine_guid;
26 + NameSS << "dimensions_on_" << localhost->hostname;
27 + TitleSS << "Anomaly detection dimensions for host " << RH->hostname;
28 +
29 + RS = rrdset_create(
30 + RH,
31 "anomaly_detection", // type
40 - IdTitlePair.first.c_str(), // id
41 - NULL, // name
32 + IdSS.str().c_str(), // id
33 + NameSS.str().c_str(), // name
34 "dimensions", // family
35 "anomaly_detection.dimensions", // ctx
44 - IdTitlePair.second.c_str(), // title
36 + TitleSS.str().c_str(), // title
37 "dimensions", // units
38 "netdata", // plugin
39 "ml", // module
@@ -49,6 +41,7 @@ static void updateDimensionsChart(RRDHOST *RH,
41 RH->rrd_update_every, // update_every
42 RRDSET_TYPE_LINE // chart_type
43 );
44 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
45
46 NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
47 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -74,17 +67,20 @@ static void updateRateChart(RRDHOST *RH, collected_number AnomalyRate) {
67 static thread_local RRDDIM *AnomalyRateRD = nullptr;
68
69 if (!RS) {
77 - std::string IdPrefix = "anomaly_rate";
78 - std::string TitlePrefix = "Percentage of anomalous dimensions for host";
79 - auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);
70 + std::stringstream IdSS, NameSS, TitleSS;
71
81 - RS = rrdset_create_localhost(
72 + IdSS << "anomaly_rate_on_" << localhost->machine_guid;
73 + NameSS << "anomaly_rate_on_" << localhost->hostname;
74 + TitleSS << "Percentage of anomalous dimensions for host " << RH->hostname;
75 +
76 + RS = rrdset_create(
77 + RH,
78 "anomaly_detection", // type
83 - IdTitlePair.first.c_str(), // id
84 - NULL, // name
79 + IdSS.str().c_str(), // id
80 + NameSS.str().c_str(), // name
81 "anomaly_rate", // family
82 "anomaly_detection.anomaly_rate", // ctx
87 - IdTitlePair.second.c_str(), // title
83 + TitleSS.str().c_str(), // title
84 "percentage", // units
85 "netdata", // plugin
86 "ml", // module
@@ -92,6 +88,7 @@ static void updateRateChart(RRDHOST *RH, collected_number AnomalyRate) {
88 RH->rrd_update_every, // update_every
89 RRDSET_TYPE_LINE // chart_type
90 );
91 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
92
93 AnomalyRateRD = rrddim_add(RS, "anomaly_rate", NULL,
94 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -108,17 +105,20 @@ static void updateWindowLengthChart(RRDHOST *RH, collected_number WindowLength)
105 static thread_local RRDDIM *WindowLengthRD = nullptr;
106
107 if (!RS) {
111 - std::string IdPrefix = "detector_window";
112 - std::string TitlePrefix = "Anomaly detector window length for host";
113 - auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);
108 + std::stringstream IdSS, NameSS, TitleSS;
109
115 - RS = rrdset_create_localhost(
110 + IdSS << "detector_window_on_" << localhost->machine_guid;
111 + NameSS << "detector_window_on_" << localhost->hostname;
112 + TitleSS << "Anomaly detector window length for host " << RH->hostname;
113 +
114 + RS = rrdset_create(
115 + RH,
116 "anomaly_detection", // type
117 - IdTitlePair.first.c_str(), // id
118 - NULL, // name
117 + IdSS.str().c_str(), // id
118 + NameSS.str().c_str(), // name
119 "detector_window", // family
120 "anomaly_detection.detector_window", // ctx
121 - IdTitlePair.second.c_str(), // title
121 + TitleSS.str().c_str(), // title
122 "seconds", // units
123 "netdata", // plugin
124 "ml", // module
@@ -126,6 +126,7 @@ static void updateWindowLengthChart(RRDHOST *RH, collected_number WindowLength)
126 RH->rrd_update_every, // update_every
127 RRDSET_TYPE_LINE // chart_type
128 );
129 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
130
131 WindowLengthRD = rrddim_add(RS, "duration", NULL,
132 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -146,17 +147,20 @@ static void updateEventsChart(RRDHOST *RH,
147 static thread_local RRDDIM *NewAnomalyEventRD = nullptr;
148
149 if (!RS) {
149 - std::string IdPrefix = "detector_events";
150 - std::string TitlePrefix = "Anomaly events triggered for host";
151 - auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);
150 + std::stringstream IdSS, NameSS, TitleSS;
151
153 - RS = rrdset_create_localhost(
152 + IdSS << "detector_events_on_" << localhost->machine_guid;
153 + NameSS << "detector_events_on_" << localhost->hostname;
154 + TitleSS << "Anomaly events triggered for host " << RH->hostname;
155 +
156 + RS = rrdset_create(
157 + RH,
158 "anomaly_detection", // type
155 - IdTitlePair.first.c_str(), // id
156 - NULL, // name
159 + IdSS.str().c_str(), // id
160 + NameSS.str().c_str(), // name
161 "detector_events", // family
162 "anomaly_detection.detector_events", // ctx
159 - IdTitlePair.second.c_str(), // title
163 + TitleSS.str().c_str(), // title
164 "boolean", // units
165 "netdata", // plugin
166 "ml", // module
@@ -164,6 +168,7 @@ static void updateEventsChart(RRDHOST *RH,
168 RH->rrd_update_every, // update_every
169 RRDSET_TYPE_LINE // chart_type
170 );
171 + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
172
173 AboveThresholdRD = rrddim_add(RS, "above_threshold", NULL,
174 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -189,21 +194,23 @@ static void updateDetectionChart(RRDHOST *RH) {
194 static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
195
196 if (!RS) {
192 - std::string IdPrefix = "prediction_stats";
193 - std::string TitlePrefix = "Prediction thread CPU usage for host";
194 - auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);
197 + std::stringstream IdSS, NameSS, TitleSS;
198 +
199 + IdSS << "prediction_stats_" << RH->machine_guid;
200 + NameSS << "prediction_stats_for_" << RH->hostname;
201 + TitleSS << "Prediction thread CPU usage for host " << RH->hostname;
202
203 RS = rrdset_create_localhost(
197 - "anomaly_detection", // type
198 - IdTitlePair.first.c_str(), // id
199 - NULL, // name
200 - "prediction_stats", // family
201 - "anomaly_detection.prediction_stats", // ctx
202 - IdTitlePair.second.c_str(), // title
204 + "netdata", // type
205 + IdSS.str().c_str(), // id
206 + NameSS.str().c_str(), // name
207 + "ml", // family
208 + "prediction_stats", // ctx
209 + TitleSS.str().c_str(), // title
210 "milliseconds/s", // units
211 "netdata", // plugin
212 "ml", // module
206 - 39187, // priority
213 + 136000, // priority
214 RH->rrd_update_every, // update_every
215 RRDSET_TYPE_STACKED // chart_type
216 );
@@ -228,21 +235,23 @@ static void updateTrainingChart(RRDHOST *RH, struct rusage *TRU)
235 static thread_local RRDDIM *SystemRD = nullptr;
236
237 if (!RS) {
231 - std::string IdPrefix = "training_stats";
232 - std::string TitlePrefix = "Training thread CPU usage for host";
233 - auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);
238 + std::stringstream IdSS, NameSS, TitleSS;
239 +
240 + IdSS << "training_stats_" << RH->machine_guid;
241 + NameSS << "training_stats_for_" << RH->hostname;
242 + TitleSS << "Training thread CPU usage for host " << RH->hostname;
243
244 RS = rrdset_create_localhost(
236 - "anomaly_detection", // type
237 - IdTitlePair.first.c_str(), // id
238 - NULL, // name
239 - "training_stats", // family
240 - "anomaly_detection.training_stats", // ctx
241 - IdTitlePair.second.c_str(), // title
245 + "netdata", // type
246 + IdSS.str().c_str(), // id
247 + NameSS.str().c_str(), // name
248 + "ml", // family
249 + "training_stats", // ctx
250 + TitleSS.str().c_str(), // title
251 "milliseconds/s", // units
252 "netdata", // plugin
253 "ml", // module
245 - 39188, // priority
254 + 136001, // priority
255 RH->rrd_update_every, // update_every
256 RRDSET_TYPE_STACKED // chart_type
257 );
ml/ml-dummy.c
+4
@@ -58,4 +58,8 @@ void ml_dimension_update_name(RRDSET *RS, RRDDIM *RD, const char *name) {
58 (void) name;
59 }
60
61 +bool ml_streaming_enabled() {
62 + return false;
63 +}
64 +
65 #endif
ml/ml.cc
+4
@@ -214,6 +214,10 @@ void ml_dimension_update_name(RRDSET *RS, RRDDIM *RD, const char *Name) {
214 D->setAnomalyRateRDName(Name);
215 }
216
217 +bool ml_streaming_enabled() {
218 + return Cfg.StreamADCharts;
219 +}
220 +
221 #if defined(ENABLE_ML_TESTS)
222
223 #include "gtest/gtest.h"
ml/ml.h
+2
@@ -44,6 +44,8 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates);
44
45 void ml_dimension_update_name(RRDSET *RS, RRDDIM *RD, const char *name);
46
47 +bool ml_streaming_enabled();
48 +
49 #define ML_ANOMALY_RATES_CHART_ID "anomaly_detection.anomaly_rates"
50
51 #if defined(ENABLE_ML_TESTS)
streaming/rrdpush.c
+3
@@ -133,6 +133,9 @@ static inline int should_send_chart_matching(RRDSET *st) {
133 if (unlikely(st->state->is_ar_chart))
134 return false;
135
136 + if (rrdset_flag_check(st, RRDSET_FLAG_ANOMALY_DETECTION))
137 + return ml_streaming_enabled();
138 +
139 if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ENABLED))) {
140 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
141 rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_IGNORE);