@cryptotaxi247 / netdata-1 / commits / 41a40dc3a

ML-related changes to address issue/discussion comments. (#12494)

* Increase training thread's max sleep time. With this change we will only cap the allotted time when it is more than ten seconds. The previous limit was one second, which had the effect of scheduling dimensions near the beggining of each training window. This was not desirable because it would cause high CPU usage on parents with many children. * Only exclude netdata.* charts from training. * Use heartbeat in detection thread. * Track rusage of prediction thread. * Track rusage of training thread. * Add support for random sampling of extracted features. * Rebase * Skip RNG when ML is disabled and fix undef behaviour

vkalintiris committed Mar 30, 2022 at 13:38 UTC 41a40dc3a406c3c8dc70f41038e7d75ef2601f8b

9 files changed +89 -54

ml/Config.cc

+4 -4

@@ -38,6 +38,7 @@ void Config::readMLConfig(void) {

38 unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);

39 unsigned LagN = config_get_number(ConfigSectionML, "num samples to lag", 5);

41 + double RandomSamplingRatio = config_get_float(ConfigSectionML, "random sampling ratio", 1.0 / LagN);

42 unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000);

44 double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99);

@@ -67,6 +68,7 @@ void Config::readMLConfig(void) {

68 SmoothN = clamp(SmoothN, 0u, 5u);

69 LagN = clamp(LagN, 0u, 5u);

71 + RandomSamplingRatio = clamp(RandomSamplingRatio, 0.2, 1.0);

72 MaxKMeansIters = clamp(MaxKMeansIters, 500u, 1000u);

74 DimensionAnomalyScoreThreshold = clamp(DimensionAnomalyScoreThreshold, 0.01, 5.00);

@@ -112,6 +114,7 @@ void Config::readMLConfig(void) {

114 Cfg.SmoothN = SmoothN;

115 Cfg.LagN = LagN;

116

117 + Cfg.RandomSamplingRatio = RandomSamplingRatio;

118 Cfg.MaxKMeansIters = MaxKMeansIters;

119

120 Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;

@@ -128,9 +131,6 @@ void Config::readMLConfig(void) {

131

132 // Always exclude anomaly_detection charts from training.

133 Cfg.ChartsToSkip = "anomaly_detection.* ";

131 - Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training",

132 - "!system.* !cpu.* !mem.* !disk.* !disk_* "

133 - "!ip.* !ipv4.* !ipv6.* !net.* !net_* !netfilter.* "

134 - "!services.* !apps.* !groups.* !user.* !ebpf.* !netdata.* *");

134 + Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");

135 Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);

136 }

ml/Config.h

@@ -21,6 +21,7 @@ public:

21 unsigned SmoothN;

22 unsigned LagN;

24 + double RandomSamplingRatio;

25 unsigned MaxKMeansIters;

27 double DimensionAnomalyScoreThreshold;

@@ -39,6 +40,7 @@ public:

40 SIMPLE_PATTERN *SP_ChartsToSkip;

42 std::string AnomalyDBPath;

43 + std::vector<uint32_t> RandomNums;

45 void readMLConfig();

46 };

ml/Dimension.cc

+8 -2

@@ -125,8 +125,13 @@ MLResult TrainableDimension::trainModel() {

125 if (!CNs)

126 return MLResult::MissingData;

127

128 - SamplesBuffer SB = SamplesBuffer(CNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN);

128 + unsigned TargetNumSamples = Cfg.MaxTrainSamples * Cfg.RandomSamplingRatio;

129 + double SamplingRatio = std::min(static_cast<double>(TargetNumSamples) / N, 1.0);

130 +

131 + SamplesBuffer SB = SamplesBuffer(CNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,

132 + SamplingRatio, Cfg.RandomNums);

133 KM.train(SB, Cfg.MaxKMeansIters);

134 +

135 Trained = true;

136 ConstantModel = true;

137

@@ -162,7 +167,8 @@ std::pair<MLResult, bool> PredictableDimension::predict() {

167 CalculatedNumber *TmpCNs = new CalculatedNumber[N * (Cfg.LagN + 1)]();

168 std::memcpy(TmpCNs, CNs.data(), N * sizeof(CalculatedNumber));

169

165 - SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN);

170 + SamplesBuffer SB = SamplesBuffer(TmpCNs, N, 1, Cfg.DiffN, Cfg.SmoothN, Cfg.LagN,

171 + 1.0, Cfg.RandomNums);

172 AnomalyScore = computeAnomalyScore(SB);

173 delete[] TmpCNs;

174

ml/Dimension.h

-5

@@ -76,10 +76,6 @@ public:

77 bool isTrained() const { return Trained; }

79 - double updateTrainingDuration(double Duration) {

80 - return TrainingDuration.exchange(Duration);

81 - }

82 -

79 private:

80 std::pair<CalculatedNumber *, size_t> getCalculatedNumbers();

@@ -94,7 +90,6 @@ private:

90 KMeans KM;

92 std::atomic<bool> Trained{false};

97 - std::atomic<double> TrainingDuration{0.0};

93 };

95 class PredictableDimension : public TrainableDimension {

ml/Host.cc

+35 -42

@@ -184,13 +184,13 @@ static void updateEventsChart(RRDHOST *RH,

184 rrdset_done(RS);

185 }

186

187 -static void updateDetectionChart(RRDHOST *RH, collected_number PredictionDuration) {

187 +static void updateDetectionChart(RRDHOST *RH) {

188 static thread_local RRDSET *RS = nullptr;

189 - static thread_local RRDDIM *PredictiobDurationRD = nullptr;

189 + static thread_local RRDDIM *UserRD, *SystemRD = nullptr;

190

191 if (!RS) {

192 std::string IdPrefix = "prediction_stats";

193 - std::string TitlePrefix = "Time it took to run prediction for host";

193 + std::string TitlePrefix = "Prediction thread CPU usage for host";

194 auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);

195

196 RS = rrdset_create_localhost(

@@ -200,35 +200,36 @@ static void updateDetectionChart(RRDHOST *RH, collected_number PredictionDuratio

200 "prediction_stats", // family

201 "anomaly_detection.prediction_stats", // ctx

202 IdTitlePair.second.c_str(), // title

203 - "milliseconds", // units

203 + "milliseconds/s", // units

204 "netdata", // plugin

205 "ml", // module

206 39187, // priority

207 RH->rrd_update_every, // update_every

208 - RRDSET_TYPE_LINE // chart_type

208 + RRDSET_TYPE_STACKED // chart_type

209 );

210

211 - PredictiobDurationRD = rrddim_add(RS, "duration", NULL,

212 - 1, 1, RRD_ALGORITHM_ABSOLUTE);

211 + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

212 + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

213 } else

214 rrdset_next(RS);

215

216 - rrddim_set_by_pointer(RS, PredictiobDurationRD, PredictionDuration);

216 + struct rusage TRU;

217 + getrusage(RUSAGE_THREAD, &TRU);

218

219 + rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);

220 + rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);

221 rrdset_done(RS);

222 }

223

221 -static void updateTrainingChart(RRDHOST *RH,

222 - collected_number TotalTrainingDuration,

223 - collected_number MaxTrainingDuration)

224 +static void updateTrainingChart(RRDHOST *RH, struct rusage *TRU)

225 {

226 static thread_local RRDSET *RS = nullptr;

226 - static thread_local RRDDIM *TotalTrainingDurationRD = nullptr;

227 - static thread_local RRDDIM *MaxTrainingDurationRD = nullptr;

227 + static thread_local RRDDIM *UserRD = nullptr;

228 + static thread_local RRDDIM *SystemRD = nullptr;

229

230 if (!RS) {

231 std::string IdPrefix = "training_stats";

231 - std::string TitlePrefix = "Training step statistics for host";

232 + std::string TitlePrefix = "Training thread CPU usage for host";

233 auto IdTitlePair = getHostSpecificIdAndTitle(RH, IdPrefix, TitlePrefix);

234

235 RS = rrdset_create_localhost(

@@ -238,24 +239,21 @@ static void updateTrainingChart(RRDHOST *RH,

239 "training_stats", // family

240 "anomaly_detection.training_stats", // ctx

241 IdTitlePair.second.c_str(), // title

241 - "milliseconds", // units

242 + "milliseconds/s", // units

243 "netdata", // plugin

244 "ml", // module

245 39188, // priority

246 RH->rrd_update_every, // update_every

246 - RRDSET_TYPE_LINE // chart_type

247 + RRDSET_TYPE_STACKED // chart_type

248 );

249

249 - TotalTrainingDurationRD = rrddim_add(RS, "total_training_duration", NULL,

250 - 1, 1, RRD_ALGORITHM_ABSOLUTE);

251 - MaxTrainingDurationRD = rrddim_add(RS, "max_training_duration", NULL,

252 - 1, 1, RRD_ALGORITHM_ABSOLUTE);

250 + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

251 + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

252 } else

253 rrdset_next(RS);

254

256 - rrddim_set_by_pointer(RS, TotalTrainingDurationRD, TotalTrainingDuration);

257 - rrddim_set_by_pointer(RS, MaxTrainingDurationRD, MaxTrainingDuration);

258 -

255 + rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);

256 + rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);

257 rrdset_done(RS);

258 }

259

@@ -307,6 +305,7 @@ void RrdHost::getConfigAsJson(nlohmann::json &Json) const {

305 Json["smooth-n"] = Cfg.SmoothN;

306 Json["lag-n"] = Cfg.LagN;

307

308 + Json["random-sampling-ratio"] = Cfg.RandomSamplingRatio;

309 Json["max-kmeans-iters"] = Cfg.MaxKMeansIters;

310

311 Json["dimension-anomaly-score-threshold"] = Cfg.DimensionAnomalyScoreThreshold;

@@ -345,11 +344,7 @@ void TrainableHost::trainDimension(Dimension *D, const TimePoint &NowTP) {

344 return;

345

346 D->LastTrainedAt = NowTP + Seconds{D->updateEvery()};

348 -

349 - TimePoint StartTP = SteadyClock::now();

347 D->trainModel();

351 - Duration<double> Duration = SteadyClock::now() - StartTP;

352 - D->updateTrainingDuration(Duration.count());

348

349 {

350 std::lock_guard<std::mutex> Lock(Mutex);

@@ -358,9 +353,11 @@ void TrainableHost::trainDimension(Dimension *D, const TimePoint &NowTP) {

353 }

354

355 void TrainableHost::train() {

361 - Duration<double> MaxSleepFor = Seconds{updateEvery()};

356 + Duration<double> MaxSleepFor = Seconds{10 * updateEvery()};

357

358 while (!netdata_exit) {

359 + updateResourceUsage();

360 +

361 TimePoint NowTP = SteadyClock::now();

362

363 auto P = findDimensionToTrain(NowTP);

@@ -393,9 +390,6 @@ void DetectableHost::detectOnce() {

390 size_t NumNormalDimensions = 0;

391 size_t NumTrainedDimensions = 0;

392

396 - double TotalTrainingDuration = 0.0;

397 - double MaxTrainingDuration = 0.0;

398 -

393 bool CollectAnomalyRates = (++AnomalyRateTimer == Cfg.DBEngineAnomalyRateEvery);

394 if (CollectAnomalyRates)

395 rrdset_next(AnomalyRateRS);

@@ -414,10 +408,6 @@ void DetectableHost::detectOnce() {

408

409 NumTrainedDimensions += D->isTrained();

410

417 - double DimTrainingDuration = D->updateTrainingDuration(0.0);

418 - MaxTrainingDuration = std::max(MaxTrainingDuration, DimTrainingDuration);

419 - TotalTrainingDuration += DimTrainingDuration;

420 -

411 if (IsAnomalous)

412 NumAnomalousDimensions += 1;

413

@@ -448,7 +438,10 @@ void DetectableHost::detectOnce() {

438 updateRateChart(getRH(), WindowAnomalyRate * 10000.0);

439 updateWindowLengthChart(getRH(), WindowLength);

440 updateEventsChart(getRH(), P, ResetBitCounter, NewAnomalyEvent);

451 - updateTrainingChart(getRH(), TotalTrainingDuration * 1000.0, MaxTrainingDuration * 1000.0);

441 +

442 + struct rusage TRU;

443 + getResourceUsage(&TRU);

444 + updateTrainingChart(getRH(), &TRU);

445

446 if (!NewAnomalyEvent || (DimsOverThreshold.size() == 0))

447 return;

@@ -477,15 +470,15 @@ void DetectableHost::detectOnce() {

470 void DetectableHost::detect() {

471 std::this_thread::sleep_for(Seconds{10});

472

473 + heartbeat_t HB;

474 + heartbeat_init(&HB);

475 +

476 while (!netdata_exit) {

481 - TimePoint StartTP = SteadyClock::now();

482 - detectOnce();

483 - TimePoint EndTP = SteadyClock::now();

477 + heartbeat_next(&HB, updateEvery() * USEC_PER_SEC);

478

485 - Duration<double> Dur = EndTP - StartTP;

486 - updateDetectionChart(getRH(), Dur.count() * 1000);

479 + detectOnce();

480

488 - std::this_thread::sleep_for(Seconds{updateEvery()});

481 + updateDetectionChart(getRH());

482 }

483 }

484

ml/Host.h

+13

@@ -70,9 +70,22 @@ public:

71 void train();

73 + void updateResourceUsage() {

74 + std::lock_guard<std::mutex> Lock(ResourceUsageMutex);

75 + getrusage(RUSAGE_THREAD, &ResourceUsage);

76 + }

77 +

78 + void getResourceUsage(struct rusage *RU) {

79 + std::lock_guard<std::mutex> Lock(ResourceUsageMutex);

80 + memcpy(RU, &ResourceUsage, sizeof(struct rusage));

81 + }

82 +

83 private:

84 std::pair<Dimension *, Duration<double>> findDimensionToTrain(const TimePoint &NowTP);

85 void trainDimension(Dimension *D, const TimePoint &NowTP);

86 +

87 + std::mutex ResourceUsageMutex;

88 + struct rusage ResourceUsage;

89 };

91 class DetectableHost : public TrainableHost {

ml/kmeans/SamplesBuffer.cc

@@ -130,7 +130,13 @@ std::vector<DSample> SamplesBuffer::preprocess() {

130 DSamples.reserve(OutN);

131 Preprocessed = true;

132

133 + uint32_t MaxMT = std::numeric_limits<uint32_t>::max();

134 + uint32_t CutOff = static_cast<double>(MaxMT) * SamplingRatio;

135 +

136 for (size_t Idx = NumSamples - OutN; Idx != NumSamples; Idx++) {

137 + if (RandNums[Idx] > CutOff)

138 + continue;

139 +

140 DSample DS;

141 DS.set_size(NumDimsPerSample * (LagN + 1));

142

ml/kmeans/SamplesBuffer.h

+6 -1

@@ -80,9 +80,11 @@ class SamplesBuffer {

80 public:

81 SamplesBuffer(CalculatedNumber *CNs,

82 size_t NumSamples, size_t NumDimsPerSample,

83 - size_t DiffN = 1, size_t SmoothN = 3, size_t LagN = 3) :

83 + size_t DiffN, size_t SmoothN, size_t LagN,

84 + double SamplingRatio, std::vector<uint32_t> &RandNums) :

85 CNs(CNs), NumSamples(NumSamples), NumDimsPerSample(NumDimsPerSample),

86 DiffN(DiffN), SmoothN(SmoothN), LagN(LagN),

87 + SamplingRatio(SamplingRatio), RandNums(RandNums),

88 BytesPerSample(NumDimsPerSample * sizeof(CalculatedNumber)),

89 Preprocessed(false) {};

@@ -129,6 +131,9 @@ private:

131 size_t DiffN;

132 size_t SmoothN;

133 size_t LagN;

134 + double SamplingRatio;

135 + std::vector<uint32_t> &RandNums;

136 +

137 size_t BytesPerSample;

138 bool Preprocessed;

139 };

ml/ml.cc

+15

@@ -4,6 +4,8 @@

4 #include "Dimension.h"

5 #include "Host.h"

7 +#include <random>

8 +

9 using namespace ml;

11 bool ml_capable() {

@@ -27,7 +29,20 @@ bool ml_enabled(RRDHOST *RH) {

29 */

31 void ml_init(void) {

32 + // Read config values

33 Cfg.readMLConfig();

34 +

35 + if (!Cfg.EnableAnomalyDetection)

36 + return;

37 +

38 + // Generate random numbers to efficiently sample the features we need

39 + // for KMeans clustering.

40 + std::random_device RD;

41 + std::mt19937 Gen(RD());

42 +

43 + Cfg.RandomNums.reserve(Cfg.MaxTrainSamples);

44 + for (size_t Idx = 0; Idx != Cfg.MaxTrainSamples; Idx++)

45 + Cfg.RandomNums.push_back(Gen());

46 }

48 void ml_new_host(RRDHOST *RH) {