@cryptotaxi247 / netdata-1 / commits / f1f4652c5

added support for notifications delays; fixes #945

Costa Tsaousis (ktsaou) committed Sep 14, 2016 at 00:13 UTC f1f4652c5c8cd7717b62d27e6bd8f0d412733299
17 files changed +373 -168
conf.d/health.d/apache.conf
+2 -1
@@ -4,10 +4,11 @@
4 template: apache_last_collected_secs
5 on: apache.requests
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: webmaster
14
conf.d/health.d/cpu.conf
+6 -3
@@ -2,29 +2,32 @@
2 template: 10min_cpu_usage
3 on: system.cpu
4 lookup: average -10m unaligned of user,system,nice,softirq,irq,guest,guest_nice
5 + units: %
6 every: 1m
7 warn: $this > 80
8 crit: $this > 90
8 - units: %
9 + delay: up 0 down 15m multiplier 1.5 max 1h
10 info: average cpu utilization for the last 10 minutes
11 to: sysadmin
12
13 template: 10min_cpu_iowait
14 on: system.cpu
15 lookup: average -10m unaligned of iowait
16 + units: %
17 every: 1m
18 warn: $this > 10
19 crit: $this > 30
18 - units: %
20 + delay: up 0 down 15m multiplier 1.5 max 1h
21 info: average CPU wait I/O for the last 10 minutes
22 to: sysadmin
23
24 template: 20min_steal_cpu
25 on: system.cpu
26 lookup: average -20m unaligned of steal
27 + units: %
28 every: 5m
29 warn: $this > 10
30 crit: $this > 30
28 - units: %
31 + delay: up 0 down 15m multiplier 1.5 max 1h
32 info: average CPU steal time for the last 20 minutes
33 to: sysadmin
conf.d/health.d/disks.conf
+14 -7
@@ -5,10 +5,11 @@
5 template: disk_space_last_collected_secs
6 on: disk.space
7 calc: $now - $last_collected_t
8 + units: seconds ago
9 every: 10s
10 warn: $this > ( 5 * $update_every)
11 crit: $this > (60 * $update_every)
11 - units: seconds ago
12 + delay: up 0 down 15m multiplier 1.5 max 1h
13 info: number of seconds since the last successful data collection of the mount point
14 to: sysadmin
15
@@ -16,10 +17,11 @@ template: disk_space_last_collected_secs
17 template: disk_last_collected_secs
18 on: disk.io
19 calc: $now - $last_collected_t
20 + units: seconds ago
21 every: 10s
22 warn: $this > ( 5 * $update_every)
23 crit: $this > (60 * $update_every)
22 - units: seconds ago
24 + delay: up 0 down 15m multiplier 1.5 max 1h
25 info: number of seconds since the last successful data collection of the block device
26 to: sysadmin
27
@@ -34,20 +36,22 @@ template: disk_last_collected_secs
36 template: disk_space_usage
37 on: disk.space
38 calc: $used * 100 / ($avail + $used)
39 + units: %
40 every: 1m
41 warn: $this > 80
42 crit: $this > 95
40 - units: %
43 + delay: up 1m down 15m multiplier 1.5 max 1h
44 info: current disk space usage
45 to: sysadmin
46
47 template: disk_inode_usage
48 on: disk.inodes
49 calc: $used * 100 / ($avail + $used)
50 + units: %
51 every: 1m
52 warn: $this > 80
53 crit: $this > 95
50 - units: %
54 + delay: up 1m down 15m multiplier 1.5 max 1h
55 info: current disk inode usage
56 to: sysadmin
57
@@ -79,10 +83,11 @@ template: disk_fill_rate
83 template: out_of_disk_space_time
84 on: disk.space
85 calc: $avail / $disk_fill_rate
86 + units: hours
87 every: 10s
88 warn: $this > 0 and $this < 8
89 crit: $this > 0 and $this < 2
85 - units: hours
90 + delay: up 0 down 15m multiplier 1.5 max 1h
91 info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last hour
92 to: sysadmin
93
@@ -97,12 +102,13 @@ template: out_of_disk_space_time
102 template: 10min_disk_utilization
103 on: disk.util
104 lookup: average -10m unaligned
105 + units: %
106 every: 1m
107 green: 90
108 red: 98
109 warn: $this > $green
110 crit: $this > $red
105 - units: %
111 + delay: up 0 down 30m multiplier 1.5 max 1h
112 info: the percentage of time the disk was busy, during the last 10 minutes
113 to: sysadmin
114
@@ -115,11 +121,12 @@ template: 10min_disk_utilization
121 template: 10min_disk_backlog
122 on: disk.backlog
123 lookup: average -10m unaligned
124 + units: ms
125 every: 1m
126 green: 2000
127 red: 5000
128 warn: $this > $green
129 crit: $this > $red
123 - units: ms
130 + delay: up 1m down 30m multiplier 1.5 max 1h
131 info: average of the kernel estimated disk backlog, for the last 10 minutes
132 to: sysadmin
conf.d/health.d/entropy.conf
+6 -5
@@ -1,13 +1,14 @@
1
2 # check if entropy is too low
3 # the alarm is checked every 1 minute
4 -# and examines the last 30 minutes of data
4 +# and examines the last hour of data
5
6 - alarm: 30min_lowest_entropy
6 + alarm: 1hour_lowest_entropy
7 on: system.entropy
8 - lookup: min -30m unaligned
9 - every: 1m
10 - warn: $this < 100
8 + lookup: min -1h unaligned
9 units: entries
10 + every: 5m
11 + warn: $this < 100
12 + delay: up 0 down 1h multiplier 1.5 max 1h
13 info: minimum entries in the random numbers pool in the last 30 minutes
14 to: sysadmin
conf.d/health.d/memcached.conf
+7 -4
@@ -4,10 +4,11 @@
4 template: memcached_last_collected_secs
5 on: memcached.cache
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: dba
14
@@ -17,10 +18,11 @@ template: memcached_last_collected_secs
18 template: memcached_cache_memory_usage
19 on: memcached.cache
20 calc: $used * 100 / ($used + $available)
21 + units: %
22 every: 10s
23 warn: $this > 80
24 crit: $this > 90
23 - units: %
25 + delay: up 0 down 15m multiplier 1.5 max 1h
26 info: current cache memory usage
27 to: dba
28
@@ -31,8 +33,8 @@ template: cache_fill_rate
33 on: memcached.cache
34 lookup: min -10m at -50m unaligned of available
35 calc: ($this - $available) / (($now - $after) / 3600)
34 - every: 1m
36 units: KB/hour
37 + every: 1m
38 info: average rate the cache fills up (positive), or frees up (negative) space, for the last hour
39
40
@@ -41,9 +43,10 @@ template: cache_fill_rate
43 template: out_of_cache_space_time
44 on: memcached.cache
45 calc: $available / $cache_fill_rate
46 + units: hours
47 every: 10s
48 warn: $this > 0 and $this < 8
49 crit: $this > 0 and $this < 2
47 - units: hours
50 + delay: up 0 down 15m multiplier 1.5 max 1h
51 info: estimated time the cache will run out of space, if the system continues to add data with the rate of the last hour
52 to: dba
conf.d/health.d/mysql.conf
+2 -1
@@ -4,9 +4,10 @@
4 template: mysql_last_collected_secs
5 on: mysql.queries
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: dba
conf.d/health.d/named.conf
+2 -1
@@ -4,10 +4,11 @@
4 template: named_last_collected_secs
5 on: named.global_queries
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: domainadmin
14
conf.d/health.d/net.conf
+6 -4
@@ -4,10 +4,11 @@
4 template: interface_last_collected_secs
5 on: net.net
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: sysadmin
14
@@ -21,9 +22,10 @@ template: interface_last_collected_secs
22 template: 1hour_packet_drops
23 on: net.drops
24 lookup: sum -1h unaligned absolute
25 + units: packets
26 every: 1m
27 warn: $this > 0
26 - units: packets
28 + delay: up 0 down 15m multiplier 1.5 max 1h
29 info: interface dropped packets in the last hour
30 to: sysadmin
31
@@ -38,9 +40,9 @@ template: 1hour_packet_drops
40 template: 1hour_fifo_errors
41 on: net.fifo
42 lookup: sum -1h unaligned absolute
43 + units: errors
44 every: 1m
45 warn: $this > 0
43 - units: errors
46 + delay: up 0 down 15m multiplier 1.5 max 1h
47 info: interface fifo errors in the last hour
48 to: sysadmin
46 -
conf.d/health.d/nginx.conf
+2 -1
@@ -4,10 +4,11 @@
4 template: nginx_last_collected_secs
5 on: nginx.requests
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: webmaster
14
conf.d/health.d/qos.conf
+1
@@ -8,6 +8,7 @@
8 # lookup: sum -10m unaligned absolute
9 # every: 30s
10 # warn: $this > 0
11 +# delay: up 0 down 15m multiplier 1.5 max 1h
12 # units: packets
13 # info: dropped packets in the last 30 minutes
14 # to: sysadmin
conf.d/health.d/ram.conf
+2 -1
@@ -2,9 +2,10 @@
2 alarm: ram_in_use
3 on: system.ram
4 calc: $used * 100 / ($used + $cached + $free)
5 + units: %
6 every: 10s
7 warn: $this > 80
8 crit: $this > 90
8 - units: %
9 + delay: up 1m down 15m multiplier 1.5 max 1h
10 info: system RAM usage
11 to: sysadmin
conf.d/health.d/redis.conf
+2 -1
@@ -4,10 +4,11 @@
4 template: redis_last_collected_secs
5 on: redis.operations
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: dba
14
conf.d/health.d/retroshare.conf
+4 -2
@@ -3,10 +3,11 @@
3 template: retroshare_last_collected_secs
4 on: retroshare.peers
5 calc: $now - $last_collected_t
6 + units: seconds ago
7 every: 10s
8 warn: $this > ( 5 * $update_every)
9 crit: $this > (60 * $update_every)
9 - units: seconds ago
10 + delay: up 0 down 15m multiplier 1.5 max 1h
11 info: number of seconds since the last successful data collection
12 to: sysadmin
13
@@ -15,9 +16,10 @@ template: retroshare_last_collected_secs
16 template: retroshare_dht_working
17 on: retroshare.dht
18 calc: $dht_size_all
19 + units: peers
20 every: 1m
21 warn: $this < 100
22 crit: $this == 0
21 - units: peers
23 + delay: up 0 down 15m multiplier 1.5 max 1h
24 info: Checks if the DHT has enough peers to operate
25 to: sysadmin
conf.d/health.d/squid.conf
+2 -1
@@ -4,10 +4,11 @@
4 template: squid_last_collected_secs
5 on: squid.clients_requests
6 calc: $now - $last_collected_t
7 + units: seconds ago
8 every: 10s
9 warn: $this > ( 5 * $update_every)
10 crit: $this > (60 * $update_every)
10 - units: seconds ago
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: number of seconds since the last successful data collection
13 to: proxyadmin
14
conf.d/health.d/swap.conf
+4 -2
@@ -4,19 +4,21 @@
4 lookup: sum -30m unaligned absolute of out
5 # we have to convert KB to MB by dividing $this (i.e. the result of the lookup) with 1024
6 calc: $this / 1024 * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
7 + units: % of RAM
8 every: 1m
9 warn: $this > 10
10 crit: $this > 20
10 - units: % of RAM
11 + delay: up 0 down 15m multiplier 1.5 max 1h
12 info: the amount of memory swapped in the last 30 minutes, as a percentage of the system RAM
13 to: sysadmin
14
15 alarm: used_swap_space
16 on: system.swap
17 calc: $used * 100 / ( $system.ram.used + $system.ram.cached + $system.ram.free )
18 + units: % of RAM
19 every: 10s
20 warn: $this > 20
21 crit: $this > 50
20 - units: % of RAM
22 + delay: up 0 down 15m multiplier 1.5 max 1h
23 info: the swap memory used, as a percentage of the system RAM
24 to: sysadmin
src/health.c
+217 -92
@@ -18,7 +18,8 @@ static inline void health_alarm_log(RRDHOST *host,
18 int old_status, int new_status,
19 const char *source,
20 const char *units,
21 - const char *info
21 + const char *info,
22 + int delay
23 ) {
24 debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
25
@@ -49,6 +50,8 @@ static inline void health_alarm_log(RRDHOST *host,
50 ae->old_status = old_status;
51 ae->new_status = new_status;
52 ae->duration = duration;
53 + ae->delay = delay;
54 + ae->delay_up_to_timestamp = when + delay;
55
56 if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
57 ae->non_clear_duration += ae->duration;
@@ -557,7 +560,7 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
560
561 {
562 time_t now = time(NULL);
560 - health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_UNINITIALIZED, rc->source, rc->units, rc->info);
563 + health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_UNINITIALIZED, rc->source, rc->units, rc->info, 0);
564 }
565 }
566
@@ -595,7 +598,7 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
598
599 {
600 time_t now = time(NULL);
598 - health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info);
601 + health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0);
602 }
603
604 RRDHOST *host = st->rrdhost;
@@ -668,7 +671,7 @@ static inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *n
671 return 0;
672 }
673
671 -static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name) {
674 +static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) {
675 if(chart && name) {
676 uint32_t hash_chart = simple_hash(chart);
677 uint32_t hash_name = simple_hash(name);
@@ -676,8 +679,10 @@ static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, c
679 // re-use old IDs, by looking them up in the alarm log
680 ALARM_ENTRY *ae;
681 for(ae = host->health_log.alarms; ae ;ae = ae->next) {
679 - if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart)))
682 + if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart))) {
683 + if(next_event_id) *next_event_id = ae->alarm_event_id + 1;
684 return ae->alarm_id;
685 + }
686 }
687 }
688
@@ -729,62 +734,62 @@ static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
734 }
735 }
736
732 -static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, const char *dimensions,
733 - const char *units, const char *info,
734 - int group_method, int after, int before, int update_every, uint32_t options,
735 - calculated_number green, calculated_number red,
736 - const char *exec, const char *recipient, const char *source,
737 - const char *calc, const char *warn, const char *crit) {
737 +static inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) {
738
739 - debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, name);
739 + debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name);
740
741 - if(rrdcalc_exists(host, chart, name, 0, 0))
741 + if(rrdcalc_exists(host, chart, rt->name, 0, 0))
742 return NULL;
743
744 RRDCALC *rc = callocz(1, sizeof(RRDCALC));
745 - rc->id = rrdcalc_get_unique_id(host, chart, name);
745 rc->next_event_id = 1;
747 - rc->name = strdupz(name);
746 + rc->id = rrdcalc_get_unique_id(host, chart, rt->name, &rc->next_event_id);
747 + rc->name = strdupz(rt->name);
748 rc->hash = simple_hash(rc->name);
749 rc->chart = strdupz(chart);
750 rc->hash_chart = simple_hash(rc->chart);
751
752 - if(dimensions) rc->dimensions = strdupz(dimensions);
752 + if(rt->dimensions) rc->dimensions = strdupz(rt->dimensions);
753
754 - rc->green = green;
755 - rc->red = red;
754 + rc->green = rt->green;
755 + rc->red = rt->red;
756 rc->value = NAN;
757 rc->old_value = NAN;
758
759 - rc->group = group_method;
760 - rc->after = after;
761 - rc->before = before;
762 - rc->update_every = update_every;
763 - rc->options = options;
764 -
765 - if(exec) rc->exec = strdupz(exec);
766 - if(recipient) rc->recipient = strdupz(recipient);
767 - if(source) rc->source = strdupz(source);
768 - if(units) rc->units = strdupz(units);
769 - if(info) rc->info = strdupz(info);
770 -
771 - if(calc) {
772 - rc->calculation = expression_parse(calc, NULL, NULL);
759 + rc->delay_up_duration = rt->delay_up_duration;
760 + rc->delay_down_duration = rt->delay_down_duration;
761 + rc->delay_max_duration = rt->delay_max_duration;
762 + rc->delay_multiplier = rt->delay_multiplier;
763 +
764 + rc->group = rt->group;
765 + rc->after = rt->after;
766 + rc->before = rt->before;
767 + rc->update_every = rt->update_every;
768 + rc->options = rt->options;
769 +
770 + if(rt->exec) rc->exec = strdupz(rt->exec);
771 + if(rt->recipient) rc->recipient = strdupz(rt->recipient);
772 + if(rt->source) rc->source = strdupz(rt->source);
773 + if(rt->units) rc->units = strdupz(rt->units);
774 + if(rt->info) rc->info = strdupz(rt->info);
775 +
776 + if(rt->calculation) {
777 + rc->calculation = expression_parse(rt->calculation->source, NULL, NULL);
778 if(!rc->calculation)
774 - error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, name, calc);
779 + error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rt->name, rt->calculation->source);
780 }
776 - if(warn) {
777 - rc->warning = expression_parse(warn, NULL, NULL);
781 + if(rt->warning) {
782 + rc->warning = expression_parse(rt->warning->source, NULL, NULL);
783 if(!rc->warning)
779 - error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, name, warn);
784 + error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rt->name, rt->warning->source);
785 }
781 - if(crit) {
782 - rc->critical = expression_parse(crit, NULL, NULL);
786 + if(rt->critical) {
787 + rc->critical = expression_parse(rt->critical->source, NULL, NULL);
788 if(!rc->critical)
784 - error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, name, crit);
789 + error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);
790 }
791
787 - debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
792 + debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
793 (rc->chart)?rc->chart:"NOCHART",
794 rc->name,
795 (rc->exec)?rc->exec:"DEFAULT",
@@ -800,7 +805,11 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
805 (rc->calculation)?rc->calculation->parsed_as:"NONE",
806 (rc->warning)?rc->warning->parsed_as:"NONE",
807 (rc->critical)?rc->critical->parsed_as:"NONE",
803 - rc->source
808 + rc->source,
809 + rc->delay_up_duration,
810 + rc->delay_down_duration,
811 + rc->delay_max_duration,
812 + rc->delay_multiplier
813 );
814
815 rrdcalc_create_part2(host, rc);
@@ -854,14 +863,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) {
863
864 for(rt = st->rrdhost->templates; rt ; rt = rt->next) {
865 if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
857 - RRDCALC *rc = rrdcalc_create(st->rrdhost, rt->name, st->id,
858 - rt->dimensions, rt->units, rt->info, rt->group, rt->after, rt->before, rt->update_every, rt->options,
859 - rt->green, rt->red,
860 - rt->exec, rt->recipient, rt->source,
861 - (rt->calculation)?rt->calculation->source:NULL,
862 - (rt->warning)?rt->warning->source:NULL,
863 - (rt->critical)?rt->critical->source:NULL);
864 -
866 + RRDCALC *rc = rrdcalc_create(st->rrdhost, rt, st->id);
867 if(unlikely(!rc))
868 error("Health tried to create alarm from template '%s', but it failed", rt->name);
869
@@ -926,6 +928,7 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
928 #define HEALTH_RECIPIENT_KEY "to"
929 #define HEALTH_UNITS_KEY "units"
930 #define HEALTH_INFO_KEY "info"
931 +#define HEALTH_DELAY_KEY "delay"
932
933 static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
934 if(!rc->chart) {
@@ -946,9 +949,9 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
949 if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash))
950 return 0;
951
949 - rc->id = rrdcalc_get_unique_id(&localhost, rc->chart, rc->name);
952 + rc->id = rrdcalc_get_unique_id(&localhost, rc->chart, rc->name, &rc->next_event_id);
953
951 - debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
954 + debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
955 rc->chart?rc->chart:"NOCHART",
956 rc->name,
957 rc->id,
@@ -965,7 +968,11 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
968 (rc->calculation)?rc->calculation->parsed_as:"NONE",
969 (rc->warning)?rc->warning->parsed_as:"NONE",
970 (rc->critical)?rc->critical->parsed_as:"NONE",
968 - rc->source
971 + rc->source,
972 + rc->delay_up_duration,
973 + rc->delay_down_duration,
974 + rc->delay_max_duration,
975 + rc->delay_multiplier
976 );
977
978 rrdcalc_create_part2(host, rc);
@@ -996,7 +1003,7 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL
1003 }
1004 }
1005
999 - debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
1006 + debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1007 rt->name,
1008 (rt->context)?rt->context:"NONE",
1009 (rt->exec)?rt->exec:"DEFAULT",
@@ -1012,7 +1019,11 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL
1019 (rt->calculation)?rt->calculation->parsed_as:"NONE",
1020 (rt->warning)?rt->warning->parsed_as:"NONE",
1021 (rt->critical)?rt->critical->parsed_as:"NONE",
1015 - rt->source
1022 + rt->source,
1023 + rt->delay_up_duration,
1024 + rt->delay_down_duration,
1025 + rt->delay_max_duration,
1026 + rt->delay_multiplier
1027 );
1028
1029 if(likely(last)) {
@@ -1068,6 +1079,60 @@ static inline int health_parse_duration(char *string, int *result) {
1079 return 1;
1080 }
1081
1082 +static inline int health_parse_delay(
1083 + size_t line, const char *path, const char *file, char *string,
1084 + int *delay_up_duration,
1085 + int *delay_down_duration,
1086 + int *delay_max_duration,
1087 + float *delay_multiplier) {
1088 +
1089 + char *s = string;
1090 + while(*s) {
1091 + char *key = s;
1092 +
1093 + while(*s && !isspace(*s)) s++;
1094 + while(*s && isspace(*s)) *s++ = '\0';
1095 +
1096 + if(!*key) break;
1097 +
1098 + char *value = s;
1099 + while(*s && !isspace(*s)) s++;
1100 + while(*s && isspace(*s)) *s++ = '\0';
1101 +
1102 + if(!strcasecmp(key, "up")) {
1103 + if (!health_parse_duration(value, delay_up_duration)) {
1104 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1105 + line, path, file, value, key);
1106 + }
1107 + }
1108 + else if(!strcasecmp(key, "down")) {
1109 + if (!health_parse_duration(value, delay_down_duration)) {
1110 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1111 + line, path, file, value, key);
1112 + }
1113 + }
1114 + else if(!strcasecmp(key, "multiplier")) {
1115 + *delay_multiplier = strtof(value, NULL);
1116 + if(isnan(*delay_multiplier) || isinf(*delay_multiplier) || islessequal(*delay_multiplier, 0)) {
1117 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1118 + line, path, file, value, key);
1119 + }
1120 + }
1121 + else if(!strcasecmp(key, "max")) {
1122 + if (!health_parse_duration(value, delay_max_duration)) {
1123 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1124 + line, path, file, value, key);
1125 + }
1126 + }
1127 + else {
1128 + error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1129 + line, path, file, key);
1130 + }
1131 + }
1132 +
1133 + return 1;
1134 +}
1135 +
1136 static inline int health_parse_db_lookup(
1137 size_t line, const char *path, const char *file, char *string,
1138 int *group_method, int *after, int *before, int *every,
@@ -1186,7 +1251,7 @@ static inline void strip_quotes(char *s) {
1251 int health_readfile(const char *path, const char *filename) {
1252 debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1253
1189 - static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0;
1254 + static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0, hash_delay = 0;
1255 char buffer[HEALTH_CONF_MAX_LINE + 1];
1256
1257 if(unlikely(!hash_alarm)) {
@@ -1204,6 +1269,7 @@ int health_readfile(const char *path, const char *filename) {
1269 hash_units = simple_hash(HEALTH_UNITS_KEY);
1270 hash_info = simple_hash(HEALTH_INFO_KEY);
1271 hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1272 + hash_delay = simple_uhash(HEALTH_DELAY_KEY);
1273 }
1274
1275 snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
@@ -1283,6 +1349,7 @@ int health_readfile(const char *path, const char *filename) {
1349 rc->red = NAN;
1350 rc->value = NAN;
1351 rc->old_value = NAN;
1352 + rc->delay_multiplier = 1.0;
1353
1354 if(rrdvar_fix_name(rc->name))
1355 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
@@ -1303,6 +1370,7 @@ int health_readfile(const char *path, const char *filename) {
1370 rt->source = health_source_file(line, path, filename);
1371 rt->green = NAN;
1372 rt->red = NAN;
1373 + rt->delay_multiplier = 1.0;
1374
1375 if(rrdvar_fix_name(rt->name))
1376 error("Health configuration renamed template '%s' to '%s'", value, rt->name);
@@ -1414,6 +1482,9 @@ int health_readfile(const char *path, const char *filename) {
1482 rc->info = strdupz(value);
1483 strip_quotes(rc->info);
1484 }
1485 + else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
1486 + health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
1487 + }
1488 else {
1489 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
1490 line, path, filename, rc->name, key);
@@ -1526,6 +1597,9 @@ int health_readfile(const char *path, const char *filename) {
1597 rt->info = strdupz(value);
1598 strip_quotes(rt->info);
1599 }
1600 + else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
1601 + health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
1602 + }
1603 else {
1604 error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
1605 line, path, filename, rt->name, key);
@@ -1615,7 +1689,7 @@ void health_init(void) {
1689 }
1690
1691 long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1618 - if(n < 2) {
1692 + if(n < 10) {
1693 error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
1694 config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1695 }
@@ -1638,28 +1712,30 @@ static inline void health_string2json(BUFFER *wb, const char *prefix, const char
1712
1713 static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
1714 buffer_sprintf(wb, "\n\t{\n"
1641 - "\t\t\"hostname\":\"%s\",\n"
1642 - "\t\t\"unique_id\":%u,\n"
1643 - "\t\t\"alarm_id\":%u,\n"
1644 - "\t\t\"alarm_event_id\":%u,\n"
1645 - "\t\t\"name\":\"%s\",\n"
1646 - "\t\t\"chart\":\"%s\",\n"
1647 - "\t\t\"family\":\"%s\",\n"
1648 - "\t\t\"processed\":%s,\n"
1649 - "\t\t\"updated\":%s,\n"
1650 - "\t\t\"exec_run\":%s,\n"
1651 - "\t\t\"exec_failed\":%s,\n"
1652 - "\t\t\"exec\":\"%s\",\n"
1653 - "\t\t\"recipient\":\"%s\",\n"
1654 - "\t\t\"exec_code\":%d,\n"
1655 - "\t\t\"source\":\"%s\",\n"
1656 - "\t\t\"units\":\"%s\",\n"
1657 - "\t\t\"info\":\"%s\",\n"
1658 - "\t\t\"when\":%lu,\n"
1659 - "\t\t\"duration\":%lu,\n"
1660 - "\t\t\"non_clear_duration\":%lu,\n"
1661 - "\t\t\"status\":\"%s\",\n"
1662 - "\t\t\"old_status\":\"%s\",\n",
1715 + "\t\t\"hostname\": \"%s\",\n"
1716 + "\t\t\"unique_id\": %u,\n"
1717 + "\t\t\"alarm_id\": %u,\n"
1718 + "\t\t\"alarm_event_id\": %u,\n"
1719 + "\t\t\"name\": \"%s\",\n"
1720 + "\t\t\"chart\": \"%s\",\n"
1721 + "\t\t\"family\": \"%s\",\n"
1722 + "\t\t\"processed\": %s,\n"
1723 + "\t\t\"updated\": %s,\n"
1724 + "\t\t\"exec_run\": %lu,\n"
1725 + "\t\t\"exec_failed\": %s,\n"
1726 + "\t\t\"exec\": \"%s\",\n"
1727 + "\t\t\"recipient\": \"%s\",\n"
1728 + "\t\t\"exec_code\": %d,\n"
1729 + "\t\t\"source\": \"%s\",\n"
1730 + "\t\t\"units\": \"%s\",\n"
1731 + "\t\t\"info\": \"%s\",\n"
1732 + "\t\t\"when\": %lu,\n"
1733 + "\t\t\"duration\": %lu,\n"
1734 + "\t\t\"non_clear_duration\": %lu,\n"
1735 + "\t\t\"status\": \"%s\",\n"
1736 + "\t\t\"old_status\": \"%s\",\n"
1737 + "\t\t\"delay\": %d,\n"
1738 + "\t\t\"delay_up_to_timestamp\": %lu,\n",
1739 host->hostname,
1740 ae->unique_id,
1741 ae->alarm_id,
@@ -1669,7 +1745,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R
1745 ae->family,
1746 (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)?"true":"false",
1747 (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false",
1672 - (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN)?"true":"false",
1748 + (unsigned long)ae->exec_run_timestamp,
1749 (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
1750 ae->exec?ae->exec:health_default_exec,
1751 ae->recipient?ae->recipient:health_default_recipient,
@@ -1681,7 +1757,9 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R
1757 (unsigned long)ae->duration,
1758 (unsigned long)ae->non_clear_duration,
1759 rrdcalc_status2string(ae->new_status),
1684 - rrdcalc_status2string(ae->old_status)
1760 + rrdcalc_status2string(ae->old_status),
1761 + ae->delay,
1762 + (unsigned long)ae->delay_up_to_timestamp
1763 );
1764
1765 buffer_strcat(wb, "\t\t\"value\":");
@@ -1732,6 +1810,12 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1810 "\t\t\t\"last_updated\": %lu,\n"
1811 "\t\t\t\"next_update\": %lu,\n"
1812 "\t\t\t\"update_every\": %d,\n"
1813 + "\t\t\t\"delay_up_duration\": %d,\n"
1814 + "\t\t\t\"delay_down_duration\": %d,\n"
1815 + "\t\t\t\"delay_max_duration\": %d,\n"
1816 + "\t\t\t\"delay_multiplier\": %f,\n"
1817 + "\t\t\t\"delay\": %d,\n"
1818 + "\t\t\t\"delay_up_to_timestamp\": %lu,\n"
1819 , rc->chart, rc->name
1820 , rc->name
1821 , rc->chart
@@ -1747,6 +1831,12 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1831 , (unsigned long)rc->last_updated
1832 , (unsigned long)rc->next_update
1833 , rc->update_every
1834 + , rc->delay_up_duration
1835 + , rc->delay_down_duration
1836 + , rc->delay_max_duration
1837 + , rc->delay_multiplier
1838 + , rc->delay_last
1839 + , rc->delay_up_to_timestamp
1840 );
1841
1842 if(RRDCALC_HAS_DB_LOOKUP(rc)) {
@@ -1906,6 +1996,8 @@ static inline int rrdcalc_value2status(calculated_number n) {
1996 }
1997
1998 static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
1999 + ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
2000 +
2001 if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)
2002 return;
2003
@@ -1941,6 +2033,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2033 );
2034
2035 ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN;
2036 + ae->exec_run_timestamp = time(NULL);
2037
2038 debug(D_HEALTH, "executing command '%s'", buffer);
2039 FILE *fp = mypopen(buffer, &command_pid);
@@ -1971,23 +2064,29 @@ static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae)
2064 }
2065
2066 static inline void health_alarm_log_process(RRDHOST *host) {
1974 - static uint32_t last_processed = 0;
1975 - ALARM_ENTRY *ae;
2067 + static uint32_t stop_at_id = 0;
2068 + uint32_t first_waiting = (host->health_log.alarms)?host->health_log.alarms->unique_id:0;
2069 + time_t now = time(NULL);
2070
2071 pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
2072
1979 - for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1980 - if(last_processed >= ae->unique_id) break;
2073 + ALARM_ENTRY *ae;
2074 + for(ae = host->health_log.alarms; ae && ae->unique_id >= stop_at_id ; ae = ae->next) {
2075 + if(unlikely(
2076 + !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
2077 + !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)
2078 + )) {
2079
1982 - if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
1983 - !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)) {
1984 - ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
1985 - health_process_notifications(host, ae);
2080 + if(unlikely(ae->unique_id < first_waiting))
2081 + first_waiting = ae->unique_id;
2082 +
2083 + if(likely(now > ae->delay_up_to_timestamp))
2084 + health_process_notifications(host, ae);
2085 }
2086 }
2087
1989 - if(host->health_log.alarms)
1990 - last_processed = host->health_log.alarms->unique_id;
2088 + // remember this for the next iteration
2089 + stop_at_id = first_waiting;
2090
2091 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2092
@@ -1998,7 +2097,7 @@ static inline void health_alarm_log_process(RRDHOST *host) {
2097 pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
2098
2099 ALARM_ENTRY *last = NULL;
2001 - unsigned int count = host->health_log.max;
2100 + unsigned int count = host->health_log.max * 2 / 3;
2101 for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
2102
2103 if(ae && last && last->next == ae)
@@ -2293,7 +2392,33 @@ void *health_main(void *ptr) {
2392 }
2393
2394 if(status != rc->status) {
2296 - health_alarm_log(&localhost, rc->id, rc->next_event_id++, time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info);
2395 + int delay = 0;
2396 +
2397 + if(now > rc->delay_up_to_timestamp) {
2398 + rc->delay_up_current = rc->delay_up_duration;
2399 + rc->delay_down_current = rc->delay_down_duration;
2400 + rc->delay_last = 0;
2401 + rc->delay_up_to_timestamp = 0;
2402 + }
2403 + else {
2404 + rc->delay_up_current = (int)(rc->delay_up_current * rc->delay_multiplier);
2405 + if(rc->delay_up_current > rc->delay_max_duration) rc->delay_up_current = rc->delay_max_duration;
2406 +
2407 + rc->delay_down_current = (int)(rc->delay_down_current * rc->delay_multiplier);
2408 + if(rc->delay_down_current > rc->delay_max_duration) rc->delay_down_current = rc->delay_max_duration;
2409 + }
2410 +
2411 + if(status > rc->status)
2412 + delay = rc->delay_up_current;
2413 + else
2414 + delay = rc->delay_down_current;
2415 +
2416 + if(now + delay < rc->delay_up_to_timestamp)
2417 + delay = rc->delay_up_to_timestamp - now;
2418 +
2419 + rc->delay_last = delay;
2420 + rc->delay_up_to_timestamp = now + delay;
2421 + health_alarm_log(&localhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info, rc->delay_last);
2422 rc->last_status_change = now;
2423 rc->status = status;
2424 }
src/health.h
+94 -42
@@ -121,56 +121,89 @@ typedef struct rrddimvar {
121 #define RRDCALC_FLAG_CRIT_ERROR 0x00000020
122
123 typedef struct rrdcalc {
124 - uint32_t id;
125 - uint32_t next_event_id;
124 + uint32_t id; // the unique id of this alarm
125 + uint32_t next_event_id; // the next event id that will be used for this alarm
126
127 - char *name;
128 - uint32_t hash;
127 + char *name; // the name of this alarm
128 + uint32_t hash;
129
130 - char *exec;
131 - char *recipient;
130 + char *exec; // the command to execute when this alarm switches state
131 + char *recipient; // the recipient of the alarm (the first parameter to exec)
132
133 - char *chart; // the chart id this should be linked to
133 + char *chart; // the chart id this should be linked to
134 uint32_t hash_chart;
135
136 - char *source; // the source of this calculation
137 - char *units;
138 - char *info;
136 + char *source; // the source of this alarm
137 + char *units; // the units of the alarm
138 + char *info; // a short description of the alarm
139
140 - char *dimensions; // the chart dimensions
140 + int update_every; // update frequency for the alarm
141
142 - int group; // grouping method: average, max, etc.
143 - int before; // ending point in time-series
144 - int after; // starting point in time-series
145 - uint32_t options; // calculation options
146 - int update_every; // update frequency for the calculation
142 + // the red and green threshold of this alarm (to be set to the chart)
143 + calculated_number green;
144 + calculated_number red;
145
148 - time_t last_updated;
149 - time_t next_update;
146 + // ------------------------------------------------------------------------
147 + // database lookup settings
148
151 - EVAL_EXPRESSION *calculation;
152 - EVAL_EXPRESSION *warning;
153 - EVAL_EXPRESSION *critical;
149 + char *dimensions; // the chart dimensions
150 + int group; // grouping method: average, max, etc.
151 + int before; // ending point in time-series
152 + int after; // starting point in time-series
153 + uint32_t options; // calculation options
154
155 - uint32_t rrdcalc_flags;
156 - int status;
155 + // ------------------------------------------------------------------------
156 + // expressions related to the alarm
157
158 - time_t db_after;
159 - time_t db_before;
160 - time_t last_status_change;
158 + EVAL_EXPRESSION *calculation; // expression to calculate the value of the alarm
159 + EVAL_EXPRESSION *warning; // expression to check the warning condition
160 + EVAL_EXPRESSION *critical; // expression to check the critical condition
161
162 - calculated_number value;
163 - calculated_number old_value;
162 + // ------------------------------------------------------------------------
163 + // notification delay settings
164
165 - calculated_number green;
166 - calculated_number red;
165 + int delay_up_duration; // duration to delay notifications when alarm raises
166 + int delay_down_duration; // duration to delay notifications when alarm lowers
167 + int delay_max_duration; // the absolute max delay to apply to this alarm
168 + float delay_multiplier; // multiplier for all delays when alarms switch status
169 + // while now < delay_up_to
170 +
171 + // ------------------------------------------------------------------------
172 + // runtime information
173 +
174 + int status; // the current status of the alarm
175 +
176 + calculated_number value; // the current value of the alarm
177 + calculated_number old_value; // the previous value of the alarm
178 +
179 + uint32_t rrdcalc_flags; // check RRDCALC_FLAG_*
180 +
181 + time_t last_updated; // the last update timestamp of the alarm
182 + time_t next_update; // the next update timestamp of the alarm
183 + time_t last_status_change; // the timestamp of the last time this alarm changed status
184 +
185 + time_t db_after; // the first timestamp evaluated by the db lookup
186 + time_t db_before; // the last timestamp evaluated by the db lookup
187 +
188 + time_t delay_up_to_timestamp; // the timestamp up to which we should delay notifications
189 + int delay_up_current; // the current up notification delay duration
190 + int delay_down_current; // the current down notification delay duration
191 + int delay_last; // the last delay we used
192 +
193 + // ------------------------------------------------------------------------
194 + // variables this alarm exposes to the rest of the alarms
195
196 RRDVAR *local;
197 RRDVAR *family;
198 RRDVAR *hostid;
199 RRDVAR *hostname;
200
201 + // ------------------------------------------------------------------------
202 + // the chart this alarm it is linked to
203 +
204 struct rrdset *rrdset;
205 +
206 + // linking of this alarm on its chart
207 struct rrdcalc *rrdset_next;
208 struct rrdcalc *rrdset_prev;
209
@@ -192,25 +225,40 @@ typedef struct rrdcalctemplate {
225 char *context;
226 uint32_t hash_context;
227
195 - char *source; // the source of this template
196 - char *units;
197 - char *info;
228 + char *source; // the source of this alarm
229 + char *units; // the units of the alarm
230 + char *info; // a short description of the alarm
231
199 - char *dimensions;
232 + int update_every; // update frequency for the alarm
233
201 - int group; // grouping method: average, max, etc.
202 - int before; // ending point in time-series
203 - int after; // starting point in time-series
204 - uint32_t options; // calculation options
205 - int update_every; // update frequency for the calculation
234 + // the red and green threshold of this alarm (to be set to the chart)
235 + calculated_number green;
236 + calculated_number red;
237 +
238 + // ------------------------------------------------------------------------
239 + // database lookup settings
240 +
241 + char *dimensions; // the chart dimensions
242 + int group; // grouping method: average, max, etc.
243 + int before; // ending point in time-series
244 + int after; // starting point in time-series
245 + uint32_t options; // calculation options
246 +
247 + // ------------------------------------------------------------------------
248 + // notification delay settings
249 +
250 + int delay_up_duration; // duration to delay notifications when alarm raises
251 + int delay_down_duration; // duration to delay notifications when alarm lowers
252 + int delay_max_duration; // the absolute max delay to apply to this alarm
253 + float delay_multiplier; // multiplier for all delays when alarms switch status
254 +
255 + // ------------------------------------------------------------------------
256 + // expressions related to the alarm
257
258 EVAL_EXPRESSION *calculation;
259 EVAL_EXPRESSION *warning;
260 EVAL_EXPRESSION *critical;
261
211 - calculated_number green;
212 - calculated_number red;
213 -
262 struct rrdcalctemplate *next;
263 } RRDCALCTEMPLATE;
264
@@ -240,6 +288,7 @@ typedef struct alarm_entry {
288
289 char *exec;
290 char *recipient;
291 + time_t exec_run_timestamp;
292 int exec_code;
293
294 char *source;
@@ -253,6 +302,9 @@ typedef struct alarm_entry {
302
303 uint32_t notifications;
304
305 + int delay;
306 + time_t delay_up_to_timestamp;
307 +
308 struct alarm_entry *updated_by;
309 struct alarm_entry *next;
310 } ALARM_ENTRY;