prevent a condition when an alarm is half-checked
Costa Tsaousis (ktsaou) committed
Nov 8, 2016 at 21:41 UTC
9dcadc4d61073d05f67fc5c5ff79a3c93bee69ba
2 files changed
+27
-19
src/health.c
+26
-19
@@ -2559,21 +2559,32 @@ static inline void health_alarm_log_process(RRDHOST *host) {
2559
}
2560
2561
static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
2562
- if (unlikely(!rc->rrdset)) {
2562
+ if(unlikely(!rc->rrdset)) {
2563
debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
2564
return 0;
2565
}
2566
2567
- if (unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
2568
- debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
2567
+ if(unlikely(rc->next_update > now)) {
2568
+ if (unlikely(*next_run > rc->next_update)) {
2569
+ // update the next_run time of the main loop
2570
+ // to run this alarm precisely the time required
2571
+ *next_run = rc->next_update;
2572
+ }
2573
+
2574
+ debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
2575
return 0;
2576
}
2577
2572
- if (unlikely(!rc->update_every)) {
2578
+ if(unlikely(!rc->update_every)) {
2579
debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name);
2580
return 0;
2581
}
2582
2583
+ if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
2584
+ debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
2585
+ return 0;
2586
+ }
2587
+
2588
int update_every = rc->rrdset->update_every;
2589
time_t first = rrdset_first_entry_t(rc->rrdset);
2590
time_t last = rrdset_last_entry_t(rc->rrdset);
@@ -2586,7 +2597,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
2597
return 0;
2598
}
2599
2589
- if (RRDCALC_HAS_DB_LOOKUP(rc)) {
2600
+ if(RRDCALC_HAS_DB_LOOKUP(rc)) {
2601
time_t needed = now + rc->before + rc->after;
2602
2603
if(needed + update_every < first || needed - update_every > last) {
@@ -2598,14 +2609,6 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
2609
}
2610
}
2611
2601
- if (unlikely(rc->next_update > now)) {
2602
- if (unlikely(*next_run > rc->next_update))
2603
- *next_run = rc->next_update;
2604
-
2605
- debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
2606
- return 0;
2607
- }
2608
-
2612
return 1;
2613
}
2614
@@ -2635,18 +2638,22 @@ void *health_main(void *ptr) {
2638
time_t next_run = now + min_run_every;
2639
RRDCALC *rc;
2640
2638
- if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
2641
+ if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
2642
error("Cannot set pthread cancel state to DISABLE.");
2643
2644
rrdhost_rdlock(&localhost);
2645
2646
// the first loop is to lookup values from the db
2644
- for (rc = localhost.alarms; rc; rc = rc->next) {
2645
- if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2647
+ for(rc = localhost.alarms; rc; rc = rc->next) {
2648
+ if(unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
2649
+ if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))
2650
+ rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE;
2651
continue;
2652
+ }
2653
2654
runnable++;
2655
rc->old_value = rc->value;
2656
+ rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE;
2657
2658
// 1. if there is database lookup, do it
2659
// 2. if there is calculation expression, run it
@@ -2743,11 +2750,11 @@ void *health_main(void *ptr) {
2750
}
2751
rrdhost_unlock(&localhost);
2752
2746
- if (unlikely(runnable && !netdata_exit)) {
2753
+ if(unlikely(runnable && !netdata_exit)) {
2754
rrdhost_rdlock(&localhost);
2755
2749
- for (rc = localhost.alarms; rc; rc = rc->next) {
2750
- if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2756
+ for(rc = localhost.alarms; rc; rc = rc->next) {
2757
+ if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))
2758
continue;
2759
2760
int warning_status = RRDCALC_STATUS_UNDEFINED;
src/health.h
+1
@@ -119,6 +119,7 @@ typedef struct rrddimvar {
119
#define RRDCALC_FLAG_CALC_ERROR 0x00000008
120
#define RRDCALC_FLAG_WARN_ERROR 0x00000010
121
#define RRDCALC_FLAG_CRIT_ERROR 0x00000020
122
+#define RRDCALC_FLAG_RUNNABLE 0x00000040
123
124
typedef struct rrdcalc {
125
uint32_t id; // the unique id of this alarm