@cryptotaxi247 / netdata-1 / commits / c234d2366

Cancel anomaly detection threads before joining. (#12681)

Originally, the main training/detection thread loops where meant to be run only for the localhost host. They would stop when `netdata_exit` was set to true during the shutdown process. By enabling training/detection for children, we have to explicitly cancel ML threads because the service thread can free a child host at any point in time without setting `netdata_exit` to true. To support this: - We send a cancellation request to the training and the detection threads when we call rrdhost_free. - We disable/enable cancelation for the actual training/detection step on every iteration (in order to protect locks and shared data structures).

vkalintiris committed Apr 13, 2022 at 20:44 UTC c234d2366cc9c42389e66a7559e31a7da553d506

1 file changed +11 -1

ml/Host.cc

+11 -1

@@ -359,6 +359,9 @@ void TrainableHost::train() {

359 Duration<double> MaxSleepFor = Seconds{10 * updateEvery()};

360

361 while (!netdata_exit) {

362 + netdata_thread_testcancel();

363 + netdata_thread_disable_cancelability();

364 +

365 updateResourceUsage();

366

367 TimePoint NowTP = SteadyClock::now();

@@ -366,6 +369,8 @@ void TrainableHost::train() {

369 auto P = findDimensionToTrain(NowTP);

370 trainDimension(P.first, NowTP);

371

372 + netdata_thread_enable_cancelability();

373 +

374 Duration<double> AllottedDuration = P.second;

375 Duration<double> RealDuration = SteadyClock::now() - NowTP;

376

@@ -477,11 +482,13 @@ void DetectableHost::detect() {

482 heartbeat_init(&HB);

483

484 while (!netdata_exit) {

485 + netdata_thread_testcancel();

486 heartbeat_next(&HB, updateEvery() * USEC_PER_SEC);

487

488 + netdata_thread_disable_cancelability();

489 detectOnce();

483 -

490 updateDetectionChart(getRH());

491 + netdata_thread_enable_cancelability();

492 }

493 }

494

@@ -499,6 +506,9 @@ void DetectableHost::startAnomalyDetectionThreads() {

506 }

507

508 void DetectableHost::stopAnomalyDetectionThreads() {

509 + netdata_thread_cancel(TrainingThread.native_handle());

510 + netdata_thread_cancel(DetectionThread.native_handle());

511 +

512 TrainingThread.join();

513 DetectionThread.join();

514 }