@cryptotaxi247 / netdata-1 / commits / c76538e2f

Add two functions that allow someone to start/stop ML. (#15185)

* Add two functions that allow someone to start/stop ML. * Shutdown ML after stopping collector services * Remove unnecessary mutex from ml charts. There's already a spinlock that protects the chart when a someone calls rrdset_done(). * Use a lightweight spinlock instead of a mutext for ML dimensions.

vkalintiris committed Jun 19, 2023 at 15:24 UTC c76538e2f0ef4450e7c5f12b44f620d74fdcde47
8 files changed +203 -83
collectors/all.h
+9 -8
@@ -395,16 +395,17 @@
395 #define ML_CHART_PRIO_DETECTOR_EVENTS 39183
396
397 // [netdata.ml] charts
398 -#define NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS 890001
399 -#define NETDATA_ML_CHART_PRIO_METRIC_TYPES 890002
400 -#define NETDATA_ML_CHART_PRIO_TRAINING_STATUS 890003
398 +#define NETDATA_ML_CHART_RUNNING 890001
399 +#define NETDATA_ML_CHART_PRIO_MACHINE_LEARNING_STATUS 890002
400 +#define NETDATA_ML_CHART_PRIO_METRIC_TYPES 890003
401 +#define NETDATA_ML_CHART_PRIO_TRAINING_STATUS 890004
402
402 -#define NETDATA_ML_CHART_PRIO_PREDICTION_USAGE 890004
403 -#define NETDATA_ML_CHART_PRIO_TRAINING_USAGE 890005
403 +#define NETDATA_ML_CHART_PRIO_PREDICTION_USAGE 890005
404 +#define NETDATA_ML_CHART_PRIO_TRAINING_USAGE 890006
405
405 -#define NETDATA_ML_CHART_PRIO_QUEUE_STATS 890006
406 -#define NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS 890007
407 -#define NETDATA_ML_CHART_PRIO_TRAINING_RESULTS 890008
406 +#define NETDATA_ML_CHART_PRIO_QUEUE_STATS 890007
407 +#define NETDATA_ML_CHART_PRIO_TRAINING_TIME_STATS 890008
408 +#define NETDATA_ML_CHART_PRIO_TRAINING_RESULTS 890009
409
410 #define NETDATA_ML_CHART_FAMILY "machine learning"
411 #define NETDATA_ML_PLUGIN "ml.plugin"
daemon/main.c
+5 -5
@@ -344,11 +344,6 @@ void netdata_cleanup_and_exit(int ret) {
344
345 webrtc_close_all_connections();
346
347 - delta_shutdown_time("disable ML detection and training threads");
348 -
349 - ml_stop_threads();
350 - ml_fini();
351 -
347 delta_shutdown_time("disable maintenance, new queries, new web requests, new streaming connections and aclk");
348
349 service_signal_exit(
@@ -377,6 +372,11 @@ void netdata_cleanup_and_exit(int ret) {
372 | SERVICE_STREAMING
373 , 3 * USEC_PER_SEC);
374
375 + delta_shutdown_time("disable ML detection and training threads");
376 +
377 + ml_stop_threads();
378 + ml_fini();
379 +
380 delta_shutdown_time("stop context thread");
381
382 timeout = !service_wait_exit(
database/sqlite/sqlite_metadata.c
-1
@@ -1485,7 +1485,6 @@ static inline void queue_metadata_cmd(enum metadata_opcode opcode, const void *p
1485 cmd.param[1] = param1;
1486 cmd.completion = NULL;
1487 metadata_enq_cmd(&metasync_worker, &cmd);
1488 -
1488 }
1489
1490 // Public
ml/ad_charts.cc
+81 -38
@@ -183,6 +183,41 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
183
184 rrdset_done(host->dimensions_rs);
185 }
186 +
187 + // ML running
188 + {
189 + if (!host->ml_running_rs) {
190 + char id_buf[1024];
191 + char name_buf[1024];
192 +
193 + snprintfz(id_buf, 1024, "ml_running_on_%s", localhost->machine_guid);
194 + snprintfz(name_buf, 1024, "ml_running_on_%s", rrdhost_hostname(localhost));
195 +
196 + host->ml_running_rs = rrdset_create(
197 + host->rh,
198 + "anomaly_detection", // type
199 + id_buf, // id
200 + name_buf, // name
201 + "anomaly_detection", // family
202 + "anomaly_detection.ml_running", // ctx
203 + "ML running", // title
204 + "boolean", // units
205 + NETDATA_ML_PLUGIN, // plugin
206 + NETDATA_ML_MODULE_DETECTION, // module
207 + NETDATA_ML_CHART_RUNNING, // priority
208 + localhost->rrd_update_every, // update_every
209 + RRDSET_TYPE_LINE // chart_type
210 + );
211 + rrdset_flag_set(host->ml_running_rs, RRDSET_FLAG_ANOMALY_DETECTION);
212 +
213 + host->ml_running_rd =
214 + rrddim_add(host->ml_running_rs, "ml_running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
215 + }
216 +
217 + rrddim_set_by_pointer(host->ml_running_rs,
218 + host->ml_running_rd, host->ml_running);
219 + rrdset_done(host->ml_running_rs);
220 + }
221 }
222
223 void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number AnomalyRate) {
@@ -260,47 +295,55 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
295 /*
296 * Compute the values of the dimensions based on the host rate chart
297 */
263 - ONEWAYALLOC *OWA = onewayalloc_create(0);
264 - time_t Now = now_realtime_sec();
265 - time_t Before = Now - host->rh->rrd_update_every;
266 - time_t After = Before - Cfg.anomaly_detection_query_duration;
267 - RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
268 -
269 - RRDR *R = rrd2rrdr_legacy(
270 - OWA,
271 - host->anomaly_rate_rs,
272 - 1 /* points wanted */,
273 - After,
274 - Before,
275 - Cfg.anomaly_detection_grouping_method,
276 - 0 /* resampling time */,
277 - Options, "anomaly_rate",
278 - NULL /* group options */,
279 - 0, /* timeout */
280 - 0, /* tier */
281 - QUERY_SOURCE_ML,
282 - STORAGE_PRIORITY_SYNCHRONOUS
283 - );
284 -
285 - if (R) {
286 - if (R->d == 1 && R->n == 1 && R->rows == 1) {
287 - static thread_local bool prev_above_threshold = false;
288 - bool above_threshold = R->v[0] >= Cfg.host_anomaly_rate_threshold;
289 - bool new_anomaly_event = above_threshold && !prev_above_threshold;
290 - prev_above_threshold = above_threshold;
291 -
292 - rrddim_set_by_pointer(host->detector_events_rs,
293 - host->detector_events_above_threshold_rd, above_threshold);
294 - rrddim_set_by_pointer(host->detector_events_rs,
295 - host->detector_events_new_anomaly_event_rd, new_anomaly_event);
296 -
297 - rrdset_done(host->detector_events_rs);
298 + if (host->ml_running) {
299 + ONEWAYALLOC *OWA = onewayalloc_create(0);
300 + time_t Now = now_realtime_sec();
301 + time_t Before = Now - host->rh->rrd_update_every;
302 + time_t After = Before - Cfg.anomaly_detection_query_duration;
303 + RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
304 +
305 + RRDR *R = rrd2rrdr_legacy(
306 + OWA,
307 + host->anomaly_rate_rs,
308 + 1 /* points wanted */,
309 + After,
310 + Before,
311 + Cfg.anomaly_detection_grouping_method,
312 + 0 /* resampling time */,
313 + Options, "anomaly_rate",
314 + NULL /* group options */,
315 + 0, /* timeout */
316 + 0, /* tier */
317 + QUERY_SOURCE_ML,
318 + STORAGE_PRIORITY_SYNCHRONOUS
319 + );
320 +
321 + if (R) {
322 + if (R->d == 1 && R->n == 1 && R->rows == 1) {
323 + static thread_local bool prev_above_threshold = false;
324 + bool above_threshold = R->v[0] >= Cfg.host_anomaly_rate_threshold;
325 + bool new_anomaly_event = above_threshold && !prev_above_threshold;
326 + prev_above_threshold = above_threshold;
327 +
328 + rrddim_set_by_pointer(host->detector_events_rs,
329 + host->detector_events_above_threshold_rd, above_threshold);
330 + rrddim_set_by_pointer(host->detector_events_rs,
331 + host->detector_events_new_anomaly_event_rd, new_anomaly_event);
332 +
333 + rrdset_done(host->detector_events_rs);
334 + }
335 +
336 + rrdr_free(OWA, R);
337 }
338
300 - rrdr_free(OWA, R);
339 + onewayalloc_destroy(OWA);
340 + } else {
341 + rrddim_set_by_pointer(host->detector_events_rs,
342 + host->detector_events_above_threshold_rd, 0);
343 + rrddim_set_by_pointer(host->detector_events_rs,
344 + host->detector_events_new_anomaly_event_rd, 0);
345 + rrdset_done(host->detector_events_rs);
346 }
302 -
303 - onewayalloc_destroy(OWA);
347 }
348 }
349
ml/ml-dummy.c
+8
@@ -33,6 +33,14 @@ void ml_host_delete(RRDHOST *rh) {
33 UNUSED(rh);
34 }
35
36 +void ml_host_start(RRDHOST *rh) {
37 + UNUSED(rh);
38 +}
39 +
40 +void ml_host_stop(RRDHOST *rh) {
41 + UNUSED(rh);
42 +}
43 +
44 void ml_host_start_training_thread(RRDHOST *rh) {
45 UNUSED(rh);
46 }
ml/ml-private.h
+6 -3
@@ -195,7 +195,7 @@ typedef struct {
195 std::vector<calculated_number_t> cns;
196
197 std::vector<ml_kmeans_t> km_contexts;
198 - netdata_mutex_t mutex;
198 + SPINLOCK slock;
199 ml_kmeans_t kmeans;
200 std::vector<DSample> feature;
201
@@ -206,8 +206,6 @@ typedef struct {
206 typedef struct {
207 RRDSET *rs;
208 ml_machine_learning_stats_t mls;
209 -
210 - netdata_mutex_t mutex;
209 } ml_chart_t;
210
211 void ml_chart_update_dimension(ml_chart_t *chart, ml_dimension_t *dim, bool is_anomalous);
@@ -215,6 +213,8 @@ void ml_chart_update_dimension(ml_chart_t *chart, ml_dimension_t *dim, bool is_a
213 typedef struct {
214 RRDHOST *rh;
215
216 + std::atomic<bool> ml_running;
217 +
218 ml_machine_learning_stats_t mls;
219
220 calculated_number_t host_anomaly_rate;
@@ -227,6 +227,9 @@ typedef struct {
227 * bookkeeping for anomaly detection charts
228 */
229
230 + RRDSET *ml_running_rs;
231 + RRDDIM *ml_running_rd;
232 +
233 RRDSET *machine_learning_status_rs;
234 RRDDIM *machine_learning_status_enabled_rd;
235 RRDDIM *machine_learning_status_disabled_sp_rd;
ml/ml.cc
+91 -28
@@ -568,9 +568,9 @@ int ml_dimension_load_models(RRDDIM *rd) {
568 if (!dim)
569 return 0;
570
571 - netdata_mutex_lock(&dim->mutex);
571 + netdata_spinlock_lock(&dim->slock);
572 bool is_empty = dim->km_contexts.empty();
573 - netdata_mutex_unlock(&dim->mutex);
573 + netdata_spinlock_unlock(&dim->slock);
574
575 if (!is_empty)
576 return 0;
@@ -602,7 +602,7 @@ int ml_dimension_load_models(RRDDIM *rd) {
602 if (unlikely(rc != SQLITE_OK))
603 goto bind_fail;
604
605 - netdata_mutex_lock(&dim->mutex);
605 + netdata_spinlock_lock(&dim->slock);
606
607 dim->km_contexts.reserve(Cfg.num_models_to_use);
608 while ((rc = sqlite3_step_monitored(res)) == SQLITE_ROW) {
@@ -639,7 +639,7 @@ int ml_dimension_load_models(RRDDIM *rd) {
639 dim->ts = TRAINING_STATUS_TRAINED;
640 }
641
642 - netdata_mutex_unlock(&dim->mutex);
642 + netdata_spinlock_unlock(&dim->slock);
643
644 if (unlikely(rc != SQLITE_DONE))
645 error_report("Failed to load models, rc = %d", rc);
@@ -666,7 +666,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *
666 ml_training_response_t training_response = P.second;
667
668 if (training_response.result != TRAINING_RESULT_OK) {
669 - netdata_mutex_lock(&dim->mutex);
669 + netdata_spinlock_lock(&dim->slock);
670
671 dim->mt = METRIC_TYPE_CONSTANT;
672
@@ -687,7 +687,8 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *
687
688 dim->last_training_time = training_response.last_entry_on_response;
689 enum ml_training_result result = training_response.result;
690 - netdata_mutex_unlock(&dim->mutex);
690 +
691 + netdata_spinlock_unlock(&dim->slock);
692
693 return result;
694 }
@@ -713,7 +714,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *
714 // update models
715 worker_is_busy(WORKER_TRAIN_UPDATE_MODELS);
716 {
716 - netdata_mutex_lock(&dim->mutex);
717 + netdata_spinlock_lock(&dim->slock);
718
719 if (dim->km_contexts.size() < Cfg.num_models_to_use) {
720 dim->km_contexts.push_back(std::move(dim->kmeans));
@@ -752,7 +753,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *
753 model_info.kmeans = dim->km_contexts.back();
754 training_thread->pending_model_info.push_back(model_info);
755
755 - netdata_mutex_unlock(&dim->mutex);
756 + netdata_spinlock_unlock(&dim->slock);
757 }
758
759 return training_response.result;
@@ -851,7 +852,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t
852 /*
853 * Lock to predict and possibly schedule the dimension for training
854 */
854 - if (netdata_mutex_trylock(&dim->mutex) != 0)
855 + if (netdata_spinlock_trylock(&dim->slock) == 0)
856 return false;
857
858 // Mark the metric time as variable if we received different values
@@ -866,7 +867,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t
867 case TRAINING_STATUS_UNTRAINED:
868 case TRAINING_STATUS_PENDING_WITHOUT_MODEL: {
869 case TRAINING_STATUS_SILENCED:
869 - netdata_mutex_unlock(&dim->mutex);
870 + netdata_spinlock_unlock(&dim->slock);
871 return false;
872 }
873 default:
@@ -891,7 +892,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t
892
893 if (anomaly_score < (100 * Cfg.dimension_anomaly_score_threshold)) {
894 global_statistics_ml_models_consulted(models_consulted);
894 - netdata_mutex_unlock(&dim->mutex);
895 + netdata_spinlock_unlock(&dim->slock);
896 return false;
897 }
898
@@ -905,7 +906,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t
906 dim->ts = TRAINING_STATUS_SILENCED;
907 }
908
908 - netdata_mutex_unlock(&dim->mutex);
909 + netdata_spinlock_unlock(&dim->slock);
910
911 global_statistics_ml_models_consulted(models_consulted);
912 return sum;
@@ -992,7 +993,7 @@ ml_host_detect_once(ml_host_t *host)
993 host->mls = {};
994 ml_machine_learning_stats_t mls_copy = {};
995
995 - {
996 + if (host->ml_running) {
997 netdata_mutex_lock(&host->mutex);
998
999 /*
@@ -1036,6 +1037,8 @@ ml_host_detect_once(ml_host_t *host)
1037 mls_copy = host->mls;
1038
1039 netdata_mutex_unlock(&host->mutex);
1040 + } else {
1041 + host->host_anomaly_rate = 0.0;
1042 }
1043
1044 worker_is_busy(WORKER_JOB_DETECTION_DIM_CHART);
@@ -1213,15 +1216,14 @@ void ml_host_new(RRDHOST *rh)
1216
1217 host->rh = rh;
1218 host->mls = ml_machine_learning_stats_t();
1216 - //host->ts = ml_training_stats_t();
1219 + host->host_anomaly_rate = 0.0;
1220
1221 static std::atomic<size_t> times_called(0);
1222 host->training_queue = Cfg.training_threads[times_called++ % Cfg.num_training_threads].training_queue;
1223
1221 - host->host_anomaly_rate = 0.0;
1222 -
1224 netdata_mutex_init(&host->mutex);
1225
1226 + host->ml_running = true;
1227 rh->ml_host = (rrd_ml_host_t *) host;
1228 }
1229
@@ -1237,6 +1239,70 @@ void ml_host_delete(RRDHOST *rh)
1239 rh->ml_host = NULL;
1240 }
1241
1242 +void ml_host_start(RRDHOST *rh) {
1243 + ml_host_t *host = (ml_host_t *) rh->ml_host;
1244 + if (!host)
1245 + return;
1246 +
1247 + host->ml_running = true;
1248 +}
1249 +
1250 +void ml_host_stop(RRDHOST *rh) {
1251 + ml_host_t *host = (ml_host_t *) rh->ml_host;
1252 + if (!host || !host->ml_running)
1253 + return;
1254 +
1255 + netdata_mutex_lock(&host->mutex);
1256 +
1257 + // reset host stats
1258 + host->mls = ml_machine_learning_stats_t();
1259 +
1260 + // reset charts/dims
1261 + void *rsp = NULL;
1262 + rrdset_foreach_read(rsp, host->rh) {
1263 + RRDSET *rs = static_cast<RRDSET *>(rsp);
1264 +
1265 + ml_chart_t *chart = (ml_chart_t *) rs->ml_chart;
1266 + if (!chart)
1267 + continue;
1268 +
1269 + // reset chart
1270 + chart->mls = ml_machine_learning_stats_t();
1271 +
1272 + void *rdp = NULL;
1273 + rrddim_foreach_read(rdp, rs) {
1274 + RRDDIM *rd = static_cast<RRDDIM *>(rdp);
1275 +
1276 + ml_dimension_t *dim = (ml_dimension_t *) rd->ml_dimension;
1277 + if (!dim)
1278 + continue;
1279 +
1280 + netdata_spinlock_lock(&dim->slock);
1281 +
1282 + // reset dim
1283 + // TODO: should we drop in-mem models, or mark them as stale? Is it
1284 + // okay to resume training straight away?
1285 +
1286 + dim->mt = METRIC_TYPE_CONSTANT;
1287 + dim->ts = TRAINING_STATUS_UNTRAINED;
1288 + dim->last_training_time = 0;
1289 + dim->suppression_anomaly_counter = 0;
1290 + dim->suppression_window_counter = 0;
1291 + dim->cns.clear();
1292 +
1293 + ml_kmeans_init(&dim->kmeans);
1294 +
1295 + netdata_spinlock_unlock(&dim->slock);
1296 + }
1297 + rrddim_foreach_done(rdp);
1298 + }
1299 + rrdset_foreach_done(rsp);
1300 +
1301 + netdata_mutex_unlock(&host->mutex);
1302 +
1303 + host->ml_running = false;
1304 +}
1305 +
1306 void ml_host_get_info(RRDHOST *rh, BUFFER *wb)
1307 {
1308 ml_host_t *host = (ml_host_t *) rh->ml_host;
@@ -1279,7 +1345,8 @@ void ml_host_get_detection_info(RRDHOST *rh, BUFFER *wb)
1345
1346 netdata_mutex_lock(&host->mutex);
1347
1282 - buffer_json_member_add_uint64(wb, "version", 1);
1348 + buffer_json_member_add_uint64(wb, "version", 2);
1349 + buffer_json_member_add_uint64(wb, "ml-running", host->ml_running);
1350 buffer_json_member_add_uint64(wb, "anomalous-dimensions", host->mls.num_anomalous_dimensions);
1351 buffer_json_member_add_uint64(wb, "normal-dimensions", host->mls.num_normal_dimensions);
1352 buffer_json_member_add_uint64(wb, "total-dimensions", host->mls.num_anomalous_dimensions +
@@ -1309,8 +1376,6 @@ void ml_chart_new(RRDSET *rs)
1376 chart->rs = rs;
1377 chart->mls = ml_machine_learning_stats_t();
1378
1312 - netdata_mutex_init(&chart->mutex);
1313 -
1379 rs->ml_chart = (rrd_ml_chart_t *) chart;
1380 }
1381
@@ -1322,8 +1387,6 @@ void ml_chart_delete(RRDSET *rs)
1387
1388 ml_chart_t *chart = (ml_chart_t *) rs->ml_chart;
1389
1325 - netdata_mutex_destroy(&chart->mutex);
1326 -
1390 delete chart;
1391 rs->ml_chart = NULL;
1392 }
@@ -1334,7 +1397,6 @@ bool ml_chart_update_begin(RRDSET *rs)
1397 if (!chart)
1398 return false;
1399
1337 - netdata_mutex_lock(&chart->mutex);
1400 chart->mls = {};
1401 return true;
1402 }
@@ -1344,8 +1406,6 @@ void ml_chart_update_end(RRDSET *rs)
1406 ml_chart_t *chart = (ml_chart_t *) rs->ml_chart;
1407 if (!chart)
1408 return;
1347 -
1348 - netdata_mutex_unlock(&chart->mutex);
1409 }
1410
1411 void ml_dimension_new(RRDDIM *rd)
@@ -1360,8 +1420,9 @@ void ml_dimension_new(RRDDIM *rd)
1420
1421 dim->mt = METRIC_TYPE_CONSTANT;
1422 dim->ts = TRAINING_STATUS_UNTRAINED;
1363 -
1423 dim->last_training_time = 0;
1424 + dim->suppression_anomaly_counter = 0;
1425 + dim->suppression_window_counter = 0;
1426
1427 ml_kmeans_init(&dim->kmeans);
1428
@@ -1370,7 +1431,7 @@ void ml_dimension_new(RRDDIM *rd)
1431 else
1432 dim->mls = MACHINE_LEARNING_STATUS_ENABLED;
1433
1373 - netdata_mutex_init(&dim->mutex);
1434 + netdata_spinlock_init(&dim->slock);
1435
1436 dim->km_contexts.reserve(Cfg.num_models_to_use);
1437
@@ -1385,8 +1446,6 @@ void ml_dimension_delete(RRDDIM *rd)
1446 if (!dim)
1447 return;
1448
1388 - netdata_mutex_destroy(&dim->mutex);
1389 -
1449 delete dim;
1450 rd->ml_dimension = NULL;
1451 }
@@ -1397,6 +1456,10 @@ bool ml_dimension_is_anomalous(RRDDIM *rd, time_t curr_time, double value, bool
1456 if (!dim)
1457 return false;
1458
1459 + ml_host_t *host = (ml_host_t *) rd->rrdset->rrdhost->ml_host;
1460 + if (!host->ml_running)
1461 + return false;
1462 +
1463 ml_chart_t *chart = (ml_chart_t *) rd->rrdset->ml_chart;
1464
1465 bool is_anomalous = ml_dimension_predict(dim, curr_time, value, exists);
ml/ml.h
+3
@@ -23,6 +23,9 @@ void ml_stop_threads(void);
23 void ml_host_new(RRDHOST *rh);
24 void ml_host_delete(RRDHOST *rh);
25
26 +void ml_host_start(RRDHOST *RH);
27 +void ml_host_stop(RRDHOST *RH);
28 +
29 void ml_host_get_info(RRDHOST *RH, BUFFER *wb);
30 void ml_host_get_detection_info(RRDHOST *RH, BUFFER *wb);
31 void ml_host_get_models(RRDHOST *RH, BUFFER *wb);