@cryptotaxi247 / netdata-1 / commits / 84b50fdb3

allow delaying alarm checks when remote netdata starts streaming metrics or when system resumes from hybernation; fixes #1296

Costa Tsaousis committed Feb 23, 2017 at 18:22 UTC 84b50fdb3bc566ae0d117715f184287e3535153e

3 files changed +23 -1

src/health.c

+16 -1

@@ -340,9 +340,14 @@ void *health_main(void *ptr) {

340

341 int oldstate, runnable = 0;

342 time_t now = now_realtime_sec();

343 + time_t now_boot = now_boottime_sec();

344 time_t next_run = now + min_run_every;

345 RRDCALC *rc;

346

347 + time_t last_now = now;

348 + time_t last_now_boot = now_boot;

349 + time_t hibernation_delay = config_get_number("health", "postpone alarms during hibernation for seconds", 60);

350 +

351 if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))

352 error("Cannot set pthread cancel state to DISABLE.");

353

@@ -350,7 +355,15 @@ void *health_main(void *ptr) {

355

356 RRDHOST *host;

357 rrdhost_foreach_read(host) {

353 - if(unlikely(!host->health_enabled)) continue;

358 + if(now - last_now > 2 * (now_boot - last_now_boot)) {

359 + info("Postponing alarm checks for %ld seconds, on host '%s', due to boottime discrepancy (realtime dt: %ld, boottime dt: %ld).",

360 + hibernation_delay, host->hostname, (long)(now - last_now), (long)(now_boot - last_now_boot));

361 + host->health_delay_up_to = now + hibernation_delay;

362 + }

363 + last_now = now;

364 + last_now_boot = now_boot;

365 +

366 + if(unlikely(!host->health_enabled || now < host->health_delay_up_to)) continue;

367

368 rrdhost_rdlock(host);

369

@@ -611,6 +624,8 @@ void *health_main(void *ptr) {

624 }

625 else

626 debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);

627 +

628 + now_boot = now_boottime_sec();

629 }

630

631 buffer_free(wb);

src/rrd.h

@@ -340,6 +340,7 @@ struct rrdhost {

340 int rrd_history_entries; // the number of history entries for the host's charts

341

342 int health_enabled; // 1 when this host has health enabled

343 + time_t health_delay_up_to; // a timestamp to delay alarms processing up to

344 RRD_MEMORY_MODE rrd_memory_mode; // the memory more for the charts of this host

345

346 RRDSET *rrdset_root; // the host charts

src/rrdpush.c

@@ -404,6 +404,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m

404 int history = default_rrd_history_entries;

405 RRD_MEMORY_MODE mode = default_rrd_memory_mode;

406 int health_enabled = default_health_enabled;

407 + time_t alarms_delay = 60;

408

409 update_every = (int)appconfig_get_number(&stream_config, machine_guid, "update every", update_every);

410 if(update_every < 0) update_every = 1;

@@ -418,6 +419,9 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m

419 health_enabled = appconfig_get_boolean_ondemand(&stream_config, key, "health enabled by default", health_enabled);

420 health_enabled = appconfig_get_boolean_ondemand(&stream_config, machine_guid, "health enabled", health_enabled);

421

422 + alarms_delay = appconfig_get_number(&stream_config, key, "default postpone alarms on connect seconds", alarms_delay);

423 + alarms_delay = appconfig_get_number(&stream_config, machine_guid, "postpone alarms on connect seconds", alarms_delay);

424 +

425 if(!strcmp(machine_guid, "localhost"))

426 host = localhost;

427 else

@@ -468,6 +472,8 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m

472

473 rrdhost_wrlock(host);

474 host->use_counter++;

475 + if(health_enabled != CONFIG_BOOLEAN_NO)

476 + host->health_delay_up_to = now_realtime_sec() + alarms_delay;

477 rrdhost_unlock(host);

478

479 // call the plugins.d processor to receive the metrics