allow health configuration to be expressed in a way that will allow us to have additional charts from the calculated values; alarms that do not do database lookups can now have an update every setting
Costa Tsaousis committed
Aug 14, 2016 at 05:05 UTC
ecf21c63ff39bc71be39aa2f3d3696417105caf9
3 files changed
+120
-31
conf.d/health.d/disks.conf
+54
-17
@@ -1,3 +1,9 @@
1
+# -----------------------------------------------------------------------------
2
+# low disk space
3
+
4
+# checking the latest collected values
5
+# raise an alarm if the disk is low on
6
+# available disk space
7
8
template: low_disk_space
9
on: disk.space
@@ -5,26 +11,57 @@ template: low_disk_space
11
warn: $avail * 100 / ($avail + $used) > 80
12
crit: $avail * 100 / ($avail + $used) > 90
13
8
-template: avail_10m_before
9
- on: disk.space
10
- calc: max -1s at -10m unaligned
11
- every: 1m
14
+# -----------------------------------------------------------------------------
15
+# disk fill rate
16
13
-template: avail_59m_before
17
+# calculate the rate the disk fills
18
+# use as base, the available space change
19
+# during the last minute
20
+
21
+template: disk_fill_rate_1m
22
on: disk.space
15
- calc: max -1s at -59m unaligned
23
+ lookup: max -1s at -1m unaligned
24
+ calc: ($this - $avail) / (1 * 60)
25
every: 1m
26
+ warn: $this * 2 * 86400 > $avail
27
+ crit: $this * 1 * 86400 > $avail
28
18
-template: disk_fills_quickly
29
+# calculate the rate the disk fills
30
+# use as base, the available space change
31
+# during the last hour
32
+template: disk_fill_rate_59m
33
on: disk.space
34
+ lookup: max -1s at -59m unaligned
35
every: 1m
21
- warn: ($avail < $avail_10m_before OR $avail < $avail_59m_before) \
22
- AND ( \
23
- (7 * 24 * 3600 * ($avail_10m_before - $avail) / (10 * 60) > $avail) OR \
24
- (7 * 24 * 3600 * ($avail_59m_before - $avail) / (59 * 60) > $avail) \
25
- )
26
- crit: ($avail < $avail_10m_before OR $avail < $avail_45m_before) \
27
- AND ( \
28
- (1 * 24 * 3600 * ($avail_10m_before - $avail) / (10 * 60) > $avail) OR \
29
- (1 * 24 * 3600 * ($avail_59m_before - $avail) / (59 * 60) > $avail) \
30
- )
36
+ calc: ($this - $avail) / (59 * 60)
37
+ warn: $this * 2 * 86400 > $avail
38
+ crit: $this * 1 * 86400 > $avail
39
+
40
+# -----------------------------------------------------------------------------
41
+# disk congestion
42
+
43
+# raise an alarm if the disk is congested
44
+# by calculating the average disk utilization
45
+# for the last minute
46
+
47
+template: disk_congested
48
+ on: disk.util
49
+ lookup: average -1m every 1m unaligned
50
+ green: 70
51
+ red: 95
52
+ warn: $this > $green
53
+ crit: $this > $red
54
+
55
+# raise an alarm if the disk backlog
56
+# is above 1000ms (1s) per second
57
+# for 1 minute
58
+# (i.e. the disk cannot catch up)
59
+
60
+template: disk_not_catching_up
61
+ on: disk.backlog
62
+ lookup: average -1m every 1m unaligned
63
+ green: 500
64
+ red: 1000
65
+ warn: $this > $green
66
+ crit: $this > $red
67
+
src/health.c
+59
-12
@@ -265,6 +265,9 @@ RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char
265
rs->local_id = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->hash, rs->type, rs->value);
266
rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->hash_name, rs->type, rs->value);
267
268
+ rs->context_id = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->id, rs->hash, rs->type, rs->value);
269
+ rs->context_name = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->name, rs->hash_name, rs->type, rs->value);
270
+
271
rs->context_fullidid = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->fullidid, rs->hash_fullidid, rs->type, rs->value);
272
rs->context_fullidname = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->fullidname, rs->hash_fullidname, rs->type, rs->value);
273
rs->context_fullnameid = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rs->fullnameid, rs->hash_fullnameid, rs->type, rs->value);
@@ -375,6 +378,14 @@ void rrddimvar_free(RRDDIMVAR *rs) {
378
rrdvar_free(st->rrdhost, rs->local_name);
379
}
380
381
+ if(rs->context_id) {
382
+ rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_id);
383
+ rrdvar_free(st->rrdhost, rs->context_id);
384
+ }
385
+ if(rs->context_name) {
386
+ rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_name);
387
+ rrdvar_free(st->rrdhost, rs->context_name);
388
+ }
389
if(rs->context_fullidid) {
390
rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullidid);
391
rrdvar_free(st->rrdhost, rs->context_fullidid);
@@ -714,7 +725,9 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
725
#define HEALTH_ALARM_KEY "alarm"
726
#define HEALTH_TEMPLATE_KEY "template"
727
#define HEALTH_ON_KEY "on"
728
+#define HEALTH_LOOKUP_KEY "lookup"
729
#define HEALTH_CALC_KEY "calc"
730
+#define HEALTH_EVERY_KEY "every"
731
#define HEALTH_GREEN_KEY "green"
732
#define HEALTH_RED_KEY "red"
733
#define HEALTH_WARN_KEY "warn"
@@ -722,7 +735,7 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
735
#define HEALTH_EXEC_KEY "exec"
736
737
static inline int rrdcalc_add(RRDHOST *host, RRDCALC *rc) {
725
- info("Health configuration examining alarm '%s': chart '%s', exec '%s', green %Lf, red %Lf, calculation: group %d, after %d, before %d, options %u, update every %d, dimensions '%s', warning '%s', critical '%s', source '%s",
738
+ info("Health configuration examining alarm '%s': chart '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
739
rc->name,
740
(rc->chart)?rc->chart:"NONE",
741
(rc->exec)?rc->exec:"DEFAULT",
@@ -732,8 +745,9 @@ static inline int rrdcalc_add(RRDHOST *host, RRDCALC *rc) {
745
rc->after,
746
rc->before,
747
rc->options,
735
- rc->update_every,
748
(rc->dimensions)?rc->dimensions:"NONE",
749
+ rc->update_every,
750
+ (rc->calculation)?rc->calculation->parsed_as:"NONE",
751
(rc->warning)?rc->warning->parsed_as:"NONE",
752
(rc->critical)?rc->critical->parsed_as:"NONE",
753
rc->source
@@ -757,7 +771,7 @@ static inline int rrdcalc_add(RRDHOST *host, RRDCALC *rc) {
771
}
772
773
static inline int rrdcalctemplate_add(RRDHOST *host, RRDCALCTEMPLATE *rt) {
760
- info("Health configuration examining template '%s': context '%s', exec '%s', green %Lf, red %Lf, calculation: group %d, after %d, before %d, options %u, update every %d, dimensions '%s', warning '%s', critical '%s', source '%s'",
774
+ info("Health configuration examining template '%s': context '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
775
rt->name,
776
(rt->context)?rt->context:"NONE",
777
(rt->exec)?rt->exec:"DEFAULT",
@@ -767,8 +781,9 @@ static inline int rrdcalctemplate_add(RRDHOST *host, RRDCALCTEMPLATE *rt) {
781
rt->after,
782
rt->before,
783
rt->options,
770
- rt->update_every,
784
(rt->dimensions)?rt->dimensions:"NONE",
785
+ rt->update_every,
786
+ (rt->calculation)?rt->calculation->parsed_as:"NONE",
787
(rt->warning)?rt->warning->parsed_as:"NONE",
788
(rt->critical)?rt->critical->parsed_as:"NONE",
789
rt->source
@@ -839,7 +854,7 @@ static inline int health_parse_time(char *string, int *result) {
854
return 1;
855
}
856
842
-static inline int health_parse_chart_calc(
857
+static inline int health_parse_lookup(
858
size_t line, const char *path, const char *file, char *string,
859
int *group_method, int *after, int *before, int *every,
860
uint32_t *options, char **dimensions
@@ -900,7 +915,7 @@ static inline int health_parse_chart_calc(
915
line, path, file, value, key);
916
}
917
}
903
- else if(!strcasecmp(key, "every")) {
918
+ else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
919
char *value = s;
920
while(*s && !isspace(*s)) s++;
921
while(*s && isspace(*s)) *s++ = '\0';
@@ -942,7 +957,7 @@ static inline char *health_source_file(int line, const char *path, const char *f
957
}
958
959
int health_readfile(const char *path, const char *filename) {
945
- static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0;
960
+ static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0;
961
char buffer[HEALTH_CONF_MAX_LINE + 1];
962
963
if(unlikely(!hash_alarm)) {
@@ -950,11 +965,13 @@ int health_readfile(const char *path, const char *filename) {
965
hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
966
hash_on = simple_uhash(HEALTH_ON_KEY);
967
hash_calc = simple_uhash(HEALTH_CALC_KEY);
968
+ hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
969
hash_green = simple_uhash(HEALTH_GREEN_KEY);
970
hash_red = simple_uhash(HEALTH_RED_KEY);
971
hash_warn = simple_uhash(HEALTH_WARN_KEY);
972
hash_crit = simple_uhash(HEALTH_CRIT_KEY);
973
hash_exec = simple_uhash(HEALTH_EXEC_KEY);
974
+ hash_every = simple_uhash(HEALTH_EVERY_KEY);
975
}
976
977
// info("Reading file '%s/%s'", path, filename);
@@ -1057,8 +1074,14 @@ int health_readfile(const char *path, const char *filename) {
1074
rc->chart = strdupz(value);
1075
rc->hash_chart = simple_hash(rc->chart);
1076
}
1060
- else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1061
- health_parse_chart_calc(line, path, filename, value, &rc->group, &rc->after, &rc->before, &rc->update_every, &rc->options, &rc->dimensions);
1077
+ else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1078
+ health_parse_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before, &rc->update_every,
1079
+ &rc->options, &rc->dimensions);
1080
+ }
1081
+ else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1082
+ if(!health_parse_time(value, &rc->update_every))
1083
+ info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
1084
+ line, path, filename, rc->name, key, value);
1085
}
1086
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1087
char *e;
@@ -1076,13 +1099,22 @@ int health_readfile(const char *path, const char *filename) {
1099
line, path, filename, rc->name, key, e);
1100
}
1101
}
1102
+ else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1103
+ const char *failed_at = NULL;
1104
+ int error = 0;
1105
+ rc->calculation = expression_parse(value, &failed_at, &error);
1106
+ if(!rc->calculation) {
1107
+ error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1108
+ line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1109
+ }
1110
+ }
1111
else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1112
const char *failed_at = NULL;
1113
int error = 0;
1114
rc->warning = expression_parse(value, &failed_at, &error);
1115
if(!rc->warning) {
1116
error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1085
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1117
+ line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1118
}
1119
}
1120
else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
@@ -1121,8 +1153,14 @@ int health_readfile(const char *path, const char *filename) {
1153
rt->context = strdupz(value);
1154
rt->hash_context = simple_hash(rt->context);
1155
}
1124
- else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1125
- health_parse_chart_calc(line, path, filename, value, &rt->group, &rt->after, &rt->before, &rt->update_every, &rt->options, &rt->dimensions);
1156
+ else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1157
+ health_parse_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before, &rt->update_every,
1158
+ &rt->options, &rt->dimensions);
1159
+ }
1160
+ else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1161
+ if(!health_parse_time(value, &rt->update_every))
1162
+ info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
1163
+ line, path, filename, rt->name, key, value);
1164
}
1165
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1166
char *e;
@@ -1140,6 +1178,15 @@ int health_readfile(const char *path, const char *filename) {
1178
line, path, filename, rt->name, key, e);
1179
}
1180
}
1181
+ else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1182
+ const char *failed_at = NULL;
1183
+ int error = 0;
1184
+ rt->calculation = expression_parse(value, &failed_at, &error);
1185
+ if(!rt->calculation) {
1186
+ error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1187
+ line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1188
+ }
1189
+ }
1190
else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1191
const char *failed_at = NULL;
1192
int error = 0;
src/health.h
+7
-2
@@ -114,6 +114,9 @@ typedef struct rrddimvar {
114
RRDVAR *local_id;
115
RRDVAR *local_name;
116
117
+ RRDVAR *context_id;
118
+ RRDVAR *context_name;
119
+
120
RRDVAR *context_fullidid;
121
RRDVAR *context_fullidname;
122
RRDVAR *context_fullnameid;
@@ -163,6 +166,7 @@ typedef struct rrdcalc {
166
time_t last_updated;
167
time_t next_update;
168
169
+ EVAL_EXPRESSION *calculation;
170
EVAL_EXPRESSION *warning;
171
EVAL_EXPRESSION *critical;
172
@@ -182,7 +186,7 @@ typedef struct rrdcalc {
186
struct rrdcalc *next;
187
} RRDCALC;
188
185
-#define RRDCALC_HAS_CALCULATION(rc) ((rc)->after && (rc)->update_every)
189
+#define RRDCALC_HAS_CALCULATION(rc) ((rc)->after)
190
191
// RRDCALCTEMPLATE
192
// these are to be applied to charts found dynamically
@@ -206,6 +210,7 @@ typedef struct rrdcalctemplate {
210
uint32_t options; // calculation options
211
int update_every; // update frequency for the calculation
212
213
+ EVAL_EXPRESSION *calculation;
214
EVAL_EXPRESSION *warning;
215
EVAL_EXPRESSION *critical;
216
@@ -215,7 +220,7 @@ typedef struct rrdcalctemplate {
220
struct rrdcalctemplate *next;
221
} RRDCALCTEMPLATE;
222
218
-#define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after && (rt)->update_every)
223
+#define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
224
225
226
#include "rrd.h"