@cryptotaxi247 / netdata-1 / commits / 31d02cfae

every host has its own health

Costa Tsaousis (ktsaou) committed Feb 18, 2017 at 16:26 UTC 31d02cfae33331b29d915dbe19e5baf3bfd72df8
10 files changed +1948 -1916
CMakeLists.txt
+1 -1
@@ -123,7 +123,7 @@ set(NETDATA_SOURCE_FILES
123 src/web_client.h
124 src/web_server.c
125 src/web_server.h
126 - src/rrdhost.c src/rrdfamily.c src/rrdset.c src/rrddim.c)
126 + src/rrdhost.c src/rrdfamily.c src/rrdset.c src/rrddim.c src/health_log.c src/health_config.c src/health_json.c)
127
128 set(APPS_PLUGIN_SOURCE_FILES
129 src/appconfig.c
src/Makefile.am
+1 -1
@@ -39,7 +39,7 @@ netdata_SOURCES = \
39 dictionary.c dictionary.h \
40 eval.c eval.h \
41 global_statistics.c global_statistics.h \
42 - health.c health.h \
42 + health.c health.h health_log.c health_config.c health_json.c \
43 inlined.h \
44 log.c log.h \
45 main.c main.h \
src/health.c
+268 -1898
@@ -1,464 +1,14 @@
1 +#define NETDATA_HEALTH_INTERNALS
2 #include "common.h"
3
4 #define RRDVAR_MAX_LENGTH 1024
5
5 -struct health_options {
6 - const char *health_default_exec;
7 - const char *health_default_recipient;
8 - const char *log_filename;
9 - size_t log_entries_written;
10 - FILE *log_fp;
11 -};
12 -
13 -static struct health_options health = {
14 - .health_default_exec = NULL,
15 - .health_default_recipient = "root",
16 - .log_filename = NULL,
17 - .log_entries_written = 0,
18 - .log_fp = NULL
19 -};
20 -
6 int health_enabled = 1;
7
23 -// ----------------------------------------------------------------------------
24 -// health alarm log load/save
25 -// no need for locking - only one thread is reading / writing the alarms log
26 -
27 -static inline int health_alarm_log_open(void) {
28 - if(health.log_fp)
29 - fclose(health.log_fp);
30 -
31 - health.log_fp = fopen(health.log_filename, "a");
32 -
33 - if(health.log_fp) {
34 - if (setvbuf(health.log_fp, NULL, _IOLBF, 0) != 0)
35 - error("Health: cannot set line buffering on health log file.");
36 - return 0;
37 - }
38 -
39 - error("Health: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename);
40 - return -1;
41 -}
42 -
43 -static inline void health_alarm_log_close(void) {
44 - if(health.log_fp) {
45 - fclose(health.log_fp);
46 - health.log_fp = NULL;
47 - }
48 -}
49 -
50 -static inline void health_log_rotate(void) {
51 - static size_t rotate_every = 0;
52 -
53 - if(unlikely(rotate_every == 0)) {
54 - rotate_every = (size_t)config_get_number("health", "rotate log every lines", 2000);
55 - if(rotate_every < 100) rotate_every = 100;
56 - }
57 -
58 - if(unlikely(health.log_entries_written > rotate_every)) {
59 - health_alarm_log_close();
60 -
61 - char old_filename[FILENAME_MAX + 1];
62 - snprintfz(old_filename, FILENAME_MAX, "%s.old", health.log_filename);
63 -
64 - if(unlink(old_filename) == -1 && errno != ENOENT)
65 - error("Health: cannot remove old alarms log file '%s'", old_filename);
66 -
67 - if(link(health.log_filename, old_filename) == -1 && errno != ENOENT)
68 - error("Health: cannot move file '%s' to '%s'.", health.log_filename, old_filename);
69 -
70 - if(unlink(health.log_filename) == -1 && errno != ENOENT)
71 - error("Health: cannot remove old alarms log file '%s'", health.log_filename);
72 -
73 - // open it with truncate
74 - health.log_fp = fopen(health.log_filename, "w");
75 -
76 - if(health.log_fp)
77 - fclose(health.log_fp);
78 - else
79 - error("Health: cannot truncate health log '%s'", health.log_filename);
80 -
81 - health.log_fp = NULL;
82 -
83 - health.log_entries_written = 0;
84 - health_alarm_log_open();
85 - }
86 -}
87 -
88 -static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
89 - health_log_rotate();
90 -
91 - if(likely(health.log_fp)) {
92 - if(unlikely(fprintf(health.log_fp
93 - , "%c\t%s"
94 - "\t%08x\t%08x\t%08x\t%08x\t%08x"
95 - "\t%08x\t%08x\t%08x"
96 - "\t%08x\t%08x\t%08x"
97 - "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
98 - "\t%d\t%d\t%d\t%d"
99 - "\t%Lf\t%Lf"
100 - "\n"
101 - , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
102 - , host->hostname
103 -
104 - , ae->unique_id
105 - , ae->alarm_id
106 - , ae->alarm_event_id
107 - , ae->updated_by_id
108 - , ae->updates_id
109 -
110 - , (uint32_t)ae->when
111 - , (uint32_t)ae->duration
112 - , (uint32_t)ae->non_clear_duration
113 - , (uint32_t)ae->flags
114 - , (uint32_t)ae->exec_run_timestamp
115 - , (uint32_t)ae->delay_up_to_timestamp
116 -
117 - , (ae->name)?ae->name:""
118 - , (ae->chart)?ae->chart:""
119 - , (ae->family)?ae->family:""
120 - , (ae->exec)?ae->exec:""
121 - , (ae->recipient)?ae->recipient:""
122 - , (ae->source)?ae->source:""
123 - , (ae->units)?ae->units:""
124 - , (ae->info)?ae->info:""
125 -
126 - , ae->exec_code
127 - , ae->new_status
128 - , ae->old_status
129 - , ae->delay
130 -
131 - , (long double)ae->new_value
132 - , (long double)ae->old_value
133 - ) < 0))
134 - error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
135 - else {
136 - ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
137 - health.log_entries_written++;
138 - }
139 - }
140 -}
141 -
142 -static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
143 - static uint32_t max_unique_id = 0, max_alarm_id = 0;
144 -
145 - errno = 0;
146 -
147 - char *s, *buf = mallocz(65536 + 1);
148 - size_t line = 0, len = 0;
149 - ssize_t loaded = 0, updated = 0, errored = 0, duplicate = 0;
150 -
151 - pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
152 -
153 - while((s = fgets_trim_len(buf, 65536, fp, &len))) {
154 - health.log_entries_written++;
155 - line++;
156 -
157 - int max_entries = 30, entries = 0;
158 - char *pointers[max_entries];
159 -
160 - pointers[entries++] = s++;
161 - while(*s) {
162 - if(unlikely(*s == '\t')) {
163 - *s = '\0';
164 - pointers[entries++] = ++s;
165 - if(entries >= max_entries) {
166 - error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, filename, max_entries);
167 - break;
168 - }
169 - }
170 - else s++;
171 - }
172 -
173 - if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
174 - ALARM_ENTRY *ae = NULL;
175 -
176 - if(entries < 26) {
177 - error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring it.", line, filename, entries);
178 - errored++;
179 - continue;
180 - }
181 -
182 - // check that we have valid ids
183 - uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
184 - if(!unique_id) {
185 - error("Health: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", line, filename, unique_id, pointers[2]);
186 - errored++;
187 - continue;
188 - }
189 -
190 - uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
191 - if(!alarm_id) {
192 - error("Health: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", line, filename, alarm_id, pointers[3]);
193 - errored++;
194 - continue;
195 - }
196 -
197 - if(unlikely(*pointers[0] == 'A')) {
198 - // make sure it is properly numbered
199 - if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) {
200 - error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order. Ignoring it.", line, filename, unique_id);
201 - errored++;
202 - continue;
203 - }
204 -
205 - ae = callocz(1, sizeof(ALARM_ENTRY));
206 - }
207 - else if(unlikely(*pointers[0] == 'U')) {
208 - // find the original
209 - for(ae = host->health_log.alarms; ae; ae = ae->next) {
210 - if(unlikely(unique_id == ae->unique_id)) {
211 - if(unlikely(*pointers[0] == 'A')) {
212 - error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u. Using the later."
213 - , line, filename, unique_id);
214 - *pointers[0] = 'U';
215 - duplicate++;
216 - }
217 - break;
218 - }
219 - else if(unlikely(unique_id > ae->unique_id)) {
220 - // no need to continue
221 - // the linked list is sorted
222 - ae = NULL;
223 - break;
224 - }
225 - }
226 -
227 - // if not found, skip this line
228 - if(!ae) {
229 - // error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, filename, unique_id);
230 - continue;
231 - }
232 - }
233 -
234 - // check for a possible host missmatch
235 - //if(strcmp(pointers[1], host->hostname))
236 - // error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, filename, pointers[1], host->hostname);
237 -
238 - ae->unique_id = unique_id;
239 - ae->alarm_id = alarm_id;
240 - ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
241 - ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
242 - ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
243 -
244 - ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
245 - ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
246 - ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
247 -
248 - ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
249 - ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
250 -
251 - ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
252 - ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
253 -
254 - freez(ae->name);
255 - ae->name = strdupz(pointers[13]);
256 - ae->hash_name = simple_hash(ae->name);
257 -
258 - freez(ae->chart);
259 - ae->chart = strdupz(pointers[14]);
260 - ae->hash_chart = simple_hash(ae->chart);
261 -
262 - freez(ae->family);
263 - ae->family = strdupz(pointers[15]);
264 -
265 - freez(ae->exec);
266 - ae->exec = strdupz(pointers[16]);
267 - if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
268 -
269 - freez(ae->recipient);
270 - ae->recipient = strdupz(pointers[17]);
271 - if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
272 -
273 - freez(ae->source);
274 - ae->source = strdupz(pointers[18]);
275 - if(!*ae->source) { freez(ae->source); ae->source = NULL; }
276 -
277 - freez(ae->units);
278 - ae->units = strdupz(pointers[19]);
279 - if(!*ae->units) { freez(ae->units); ae->units = NULL; }
280 -
281 - freez(ae->info);
282 - ae->info = strdupz(pointers[20]);
283 - if(!*ae->info) { freez(ae->info); ae->info = NULL; }
284 -
285 - ae->exec_code = str2i(pointers[21]);
286 - ae->new_status = str2i(pointers[22]);
287 - ae->old_status = str2i(pointers[23]);
288 - ae->delay = str2i(pointers[24]);
289 -
290 - ae->new_value = str2l(pointers[25]);
291 - ae->old_value = str2l(pointers[26]);
292 -
293 - static char value_string[100 + 1];
294 - freez(ae->old_value_string);
295 - freez(ae->new_value_string);
296 - ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
297 - ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
298 -
299 - // add it to host if not already there
300 - if(unlikely(*pointers[0] == 'A')) {
301 - ae->next = host->health_log.alarms;
302 - host->health_log.alarms = ae;
303 - loaded++;
304 - }
305 - else updated++;
306 -
307 - if(unlikely(ae->unique_id > max_unique_id))
308 - max_unique_id = ae->unique_id;
309 -
310 - if(unlikely(ae->alarm_id >= max_alarm_id))
311 - max_alarm_id = ae->alarm_id;
312 - }
313 - else {
314 - error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, filename, pointers[0]);
315 - errored++;
316 - }
317 - }
318 -
319 - pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
320 -
321 - freez(buf);
322 -
323 - if(!max_unique_id) max_unique_id = (uint32_t)now_realtime_sec();
324 - if(!max_alarm_id) max_alarm_id = (uint32_t)now_realtime_sec();
325 -
326 - host->health_log.next_log_id = max_unique_id + 1;
327 - host->health_log.next_alarm_id = max_alarm_id + 1;
328 -
329 - debug(D_HEALTH, "Health: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", filename, loaded, updated, errored, duplicate);
330 - return loaded;
331 -}
332 -
333 -static inline void health_alarm_log_load(RRDHOST *host) {
334 - health_alarm_log_close();
335 -
336 - char filename[FILENAME_MAX + 1];
337 - snprintfz(filename, FILENAME_MAX, "%s.old", health.log_filename);
338 - FILE *fp = fopen(filename, "r");
339 - if(!fp)
340 - error("Health: cannot open health file: %s", filename);
341 - else {
342 - health_alarm_log_read(host, fp, filename);
343 - fclose(fp);
344 - }
345 -
346 - health.log_entries_written = 0;
347 - fp = fopen(health.log_filename, "r");
348 - if(!fp)
349 - error("Health: cannot open health file: %s", health.log_filename);
350 - else {
351 - health_alarm_log_read(host, fp, health.log_filename);
352 - fclose(fp);
353 - }
354 -
355 - health_alarm_log_open();
356 -}
357 -
358 -
359 -// ----------------------------------------------------------------------------
360 -// health alarm log management
361 -
362 -static inline void health_alarm_log(
363 - RRDHOST *host,
364 - uint32_t alarm_id,
365 - uint32_t alarm_event_id,
366 - time_t when,
367 - const char *name,
368 - const char *chart,
369 - const char *family,
370 - const char *exec,
371 - const char *recipient,
372 - time_t duration,
373 - calculated_number old_value,
374 - calculated_number new_value,
375 - int old_status,
376 - int new_status,
377 - const char *source,
378 - const char *units,
379 - const char *info,
380 - int delay,
381 - uint32_t flags
382 -) {
383 - debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
384 -
385 - ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
386 - ae->name = strdupz(name);
387 - ae->hash_name = simple_hash(ae->name);
388 -
389 - if(chart) {
390 - ae->chart = strdupz(chart);
391 - ae->hash_chart = simple_hash(ae->chart);
392 - }
393 -
394 - if(family)
395 - ae->family = strdupz(family);
396 -
397 - if(exec) ae->exec = strdupz(exec);
398 - if(recipient) ae->recipient = strdupz(recipient);
399 - if(source) ae->source = strdupz(source);
400 - if(units) ae->units = strdupz(units);
401 - if(info) ae->info = strdupz(info);
402 -
403 - ae->unique_id = host->health_log.next_log_id++;
404 - ae->alarm_id = alarm_id;
405 - ae->alarm_event_id = alarm_event_id;
406 - ae->when = when;
407 - ae->old_value = old_value;
408 - ae->new_value = new_value;
409 -
410 - static char value_string[100 + 1];
411 - ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
412 - ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
413 -
414 - ae->old_status = old_status;
415 - ae->new_status = new_status;
416 - ae->duration = duration;
417 - ae->delay = delay;
418 - ae->delay_up_to_timestamp = when + delay;
419 -
420 - ae->flags |= flags;
421 -
422 - if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
423 - ae->non_clear_duration += ae->duration;
424 -
425 - // link it
426 - pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
427 - ae->next = host->health_log.alarms;
428 - host->health_log.alarms = ae;
429 - host->health_log.count++;
430 - pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
431 -
432 - // match previous alarms
433 - pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
434 - ALARM_ENTRY *t;
435 - for(t = host->health_log.alarms ; t ; t = t->next) {
436 - if(t != ae && t->alarm_id == ae->alarm_id) {
437 - if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) {
438 - t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
439 - t->updated_by_id = ae->unique_id;
440 - ae->updates_id = t->unique_id;
441 -
442 - if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
443 - (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
444 - ae->non_clear_duration += t->non_clear_duration;
445 -
446 - health_alarm_log_save(host, t);
447 - }
448 -
449 - // no need to continue
450 - break;
451 - }
452 - }
453 - pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
454 -
455 - health_alarm_log_save(host, ae);
456 -}
457 -
8 // ----------------------------------------------------------------------------
9 // RRDVAR management
10
461 -static inline int rrdvar_fix_name(char *variable) {
11 +inline int rrdvar_fix_name(char *variable) {
12 int fixed = 0;
13 while(*variable) {
14 if (!isalnum(*variable) && *variable != '.' && *variable != '_') {
@@ -1245,7 +795,7 @@ RRDCALC *rrdcalc_find(RRDSET *st, const char *name) {
795 return NULL;
796 }
797
1248 -static inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) {
798 +inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) {
799 RRDCALC *rc;
800
801 if(unlikely(!chart)) {
@@ -1268,7 +818,7 @@ static inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *n
818 return 0;
819 }
820
1271 -static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) {
821 +inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) {
822 if(chart && name) {
823 uint32_t hash_chart = simple_hash(chart);
824 uint32_t hash_name = simple_hash(name);
@@ -1286,7 +836,7 @@ static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, c
836 return host->health_log.next_alarm_id++;
837 }
838
1289 -static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
839 +inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
840 rrdhost_check_rdlock(host);
841
842 if(rc->calculation) {
@@ -1476,7 +1026,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) {
1026 }
1027 }
1028
1479 -static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1029 +inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1030 debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
1031
1032 if(host->templates) {
@@ -1514,1249 +1064,107 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1064 }
1065
1066 // ----------------------------------------------------------------------------
1517 -// load health configuration
1518 -
1519 -#define HEALTH_CONF_MAX_LINE 4096
1520 -
1521 -#define HEALTH_ALARM_KEY "alarm"
1522 -#define HEALTH_TEMPLATE_KEY "template"
1523 -#define HEALTH_ON_KEY "on"
1524 -#define HEALTH_FAMILIES_KEY "families"
1525 -#define HEALTH_LOOKUP_KEY "lookup"
1526 -#define HEALTH_CALC_KEY "calc"
1527 -#define HEALTH_EVERY_KEY "every"
1528 -#define HEALTH_GREEN_KEY "green"
1529 -#define HEALTH_RED_KEY "red"
1530 -#define HEALTH_WARN_KEY "warn"
1531 -#define HEALTH_CRIT_KEY "crit"
1532 -#define HEALTH_EXEC_KEY "exec"
1533 -#define HEALTH_RECIPIENT_KEY "to"
1534 -#define HEALTH_UNITS_KEY "units"
1535 -#define HEALTH_INFO_KEY "info"
1536 -#define HEALTH_DELAY_KEY "delay"
1537 -#define HEALTH_OPTIONS_KEY "options"
1538 -
1539 -static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
1540 - if(!rc->chart) {
1541 - error("Health configuration for alarm '%s' does not have a chart", rc->name);
1542 - return 0;
1543 - }
1067 +// health initialization
1068
1545 - if(!rc->update_every) {
1546 - error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name);
1547 - return 0;
1548 - }
1069 +inline char *health_config_dir(void) {
1070 + char buffer[FILENAME_MAX + 1];
1071 + snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_config_dir);
1072 + return config_get("health", "health configuration directory", buffer);
1073 +}
1074
1550 - if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
1551 - error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
1552 - return 0;
1075 +void health_init(void) {
1076 + debug(D_HEALTH, "Health configuration initializing");
1077 +
1078 + if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
1079 + debug(D_HEALTH, "Health is disabled.");
1080 + return;
1081 }
1082
1555 - if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash))
1556 - return 0;
1083 + char pathname[FILENAME_MAX + 1];
1084 + snprintfz(pathname, FILENAME_MAX, "%s/health", netdata_configured_varlib_dir);
1085 + if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
1086 + fatal("Cannot create directory '%s'.", pathname);
1087 +}
1088
1558 - rc->id = rrdcalc_get_unique_id(localhost, rc->chart, rc->name, &rc->next_event_id);
1089 +// ----------------------------------------------------------------------------
1090 +// re-load health configuration
1091
1560 - debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1561 - rc->chart?rc->chart:"NOCHART",
1562 - rc->name,
1563 - rc->id,
1564 - (rc->exec)?rc->exec:"DEFAULT",
1565 - (rc->recipient)?rc->recipient:"DEFAULT",
1566 - rc->green,
1567 - rc->red,
1568 - rc->group,
1569 - rc->after,
1570 - rc->before,
1571 - rc->options,
1572 - (rc->dimensions)?rc->dimensions:"NONE",
1573 - rc->update_every,
1574 - (rc->calculation)?rc->calculation->parsed_as:"NONE",
1575 - (rc->warning)?rc->warning->parsed_as:"NONE",
1576 - (rc->critical)?rc->critical->parsed_as:"NONE",
1577 - rc->source,
1578 - rc->delay_up_duration,
1579 - rc->delay_down_duration,
1580 - rc->delay_max_duration,
1581 - rc->delay_multiplier
1582 - );
1092 +inline void health_free_host_nolock(RRDHOST *host) {
1093 + while(host->templates)
1094 + rrdcalctemplate_free(host, host->templates);
1095
1584 - rrdcalc_create_part2(host, rc);
1585 - return 1;
1096 + while(host->alarms)
1097 + rrdcalc_free(host, host->alarms);
1098 }
1099
1588 -static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1589 - if(unlikely(!rt->context)) {
1590 - error("Health configuration for template '%s' does not have a context", rt->name);
1591 - return 0;
1100 +void health_reload_host(RRDHOST *host) {
1101 + if(!health_enabled) {
1102 + error("Health reload is requested, but health is not enabled.");
1103 + return;
1104 }
1105
1594 - if(unlikely(!rt->update_every)) {
1595 - error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name);
1596 - return 0;
1597 - }
1106 + char *path = health_config_dir();
1107
1599 - if(unlikely(!RRDCALCTEMPLATE_HAS_CALCULATION(rt) && !rt->warning && !rt->critical)) {
1600 - error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name);
1601 - return 0;
1602 - }
1108 + // free all running alarms
1109 + rrdhost_rwlock(host);
1110 + health_free_host_nolock(host);
1111 + rrdhost_unlock(host);
1112
1604 - RRDCALCTEMPLATE *t, *last = NULL;
1605 - for (t = host->templates; t ; last = t, t = t->next) {
1606 - if(unlikely(t->hash_name == rt->hash_name && !strcmp(t->name, rt->name))) {
1607 - error("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname);
1608 - return 0;
1609 - }
1113 + // invalidate all previous entries in the alarm log
1114 + ALARM_ENTRY *t;
1115 + for(t = host->health_log.alarms ; t ; t = t->next) {
1116 + if(t->new_status != RRDCALC_STATUS_REMOVED)
1117 + t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
1118 }
1119
1612 - debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1613 - rt->name,
1614 - (rt->context)?rt->context:"NONE",
1615 - (rt->exec)?rt->exec:"DEFAULT",
1616 - (rt->recipient)?rt->recipient:"DEFAULT",
1617 - rt->green,
1618 - rt->red,
1619 - rt->group,
1620 - rt->after,
1621 - rt->before,
1622 - rt->options,
1623 - (rt->dimensions)?rt->dimensions:"NONE",
1624 - rt->update_every,
1625 - (rt->calculation)?rt->calculation->parsed_as:"NONE",
1626 - (rt->warning)?rt->warning->parsed_as:"NONE",
1627 - (rt->critical)?rt->critical->parsed_as:"NONE",
1628 - rt->source,
1629 - rt->delay_up_duration,
1630 - rt->delay_down_duration,
1631 - rt->delay_max_duration,
1632 - rt->delay_multiplier
1633 - );
1634 -
1635 - if(likely(last)) {
1636 - last->next = rt;
1637 - }
1638 - else {
1639 - rt->next = host->templates;
1640 - host->templates = rt;
1120 + // reset all thresholds to all charts
1121 + RRDSET *st;
1122 + for(st = host->rrdset_root; st ; st = st->next) {
1123 + st->green = NAN;
1124 + st->red = NAN;
1125 }
1126
1643 - return 1;
1644 -}
1127 + // load the new alarms
1128 + rrdhost_rwlock(host);
1129 + health_readdir(host, path);
1130 + rrdhost_unlock(host);
1131
1646 -static inline int health_parse_duration(char *string, int *result) {
1647 - // make sure it is a number
1648 - if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
1649 - *result = 0;
1650 - return 0;
1651 - }
1132 + // link the loaded alarms to their charts
1133 + for(st = host->rrdset_root; st ; st = st->next) {
1134 + rrdhost_rwlock(host);
1135
1653 - char *e = NULL;
1654 - calculated_number n = strtold(string, &e);
1655 - if(e && *e) {
1656 - switch (*e) {
1657 - case 'Y':
1658 - *result = (int) (n * 86400 * 365);
1659 - break;
1660 - case 'M':
1661 - *result = (int) (n * 86400 * 30);
1662 - break;
1663 - case 'w':
1664 - *result = (int) (n * 86400 * 7);
1665 - break;
1666 - case 'd':
1667 - *result = (int) (n * 86400);
1668 - break;
1669 - case 'h':
1670 - *result = (int) (n * 3600);
1671 - break;
1672 - case 'm':
1673 - *result = (int) (n * 60);
1674 - break;
1136 + rrdsetcalc_link_matching(st);
1137 + rrdcalctemplate_link_matching(st);
1138
1676 - default:
1677 - case 's':
1678 - *result = (int) (n);
1679 - break;
1680 - }
1139 + rrdhost_unlock(host);
1140 }
1682 - else
1683 - *result = (int)(n);
1684 -
1685 - return 1;
1141 }
1142
1688 -static inline int health_parse_delay(
1689 - size_t line, const char *path, const char *file, char *string,
1690 - int *delay_up_duration,
1691 - int *delay_down_duration,
1692 - int *delay_max_duration,
1693 - float *delay_multiplier) {
1143 +void health_reload(void) {
1144 + RRDHOST *host;
1145
1695 - char given_up = 0;
1696 - char given_down = 0;
1697 - char given_max = 0;
1698 - char given_multiplier = 0;
1146 + for(host = localhost; host ; host = host->next)
1147 + health_reload_host(host);
1148 +}
1149
1700 - char *s = string;
1701 - while(*s) {
1702 - char *key = s;
1150 +// ----------------------------------------------------------------------------
1151 +// health main thread and friends
1152
1704 - while(*s && !isspace(*s)) s++;
1705 - while(*s && isspace(*s)) *s++ = '\0';
1153 +static inline int rrdcalc_value2status(calculated_number n) {
1154 + if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
1155 + if(n) return RRDCALC_STATUS_RAISED;
1156 + return RRDCALC_STATUS_CLEAR;
1157 +}
1158
1707 - if(!*key) break;
1159 +#define ALARM_EXEC_COMMAND_LENGTH 8192
1160
1709 - char *value = s;
1710 - while(*s && !isspace(*s)) s++;
1711 - while(*s && isspace(*s)) *s++ = '\0';
1161 +static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
1162 + ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
1163
1713 - if(!strcasecmp(key, "up")) {
1714 - if (!health_parse_duration(value, delay_up_duration)) {
1715 - error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1716 - line, path, file, value, key);
1717 - }
1718 - else given_up = 1;
1719 - }
1720 - else if(!strcasecmp(key, "down")) {
1721 - if (!health_parse_duration(value, delay_down_duration)) {
1722 - error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1723 - line, path, file, value, key);
1724 - }
1725 - else given_down = 1;
1726 - }
1727 - else if(!strcasecmp(key, "multiplier")) {
1728 - *delay_multiplier = strtof(value, NULL);
1729 - if(isnan(*delay_multiplier) || isinf(*delay_multiplier) || islessequal(*delay_multiplier, 0)) {
1730 - error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1731 - line, path, file, value, key);
1732 - }
1733 - else given_multiplier = 1;
1734 - }
1735 - else if(!strcasecmp(key, "max")) {
1736 - if (!health_parse_duration(value, delay_max_duration)) {
1737 - error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1738 - line, path, file, value, key);
1739 - }
1740 - else given_max = 1;
1741 - }
1742 - else {
1743 - error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1744 - line, path, file, key);
1745 - }
1746 - }
1747 -
1748 - if(!given_up)
1749 - *delay_up_duration = 0;
1750 -
1751 - if(!given_down)
1752 - *delay_down_duration = 0;
1753 -
1754 - if(!given_multiplier)
1755 - *delay_multiplier = 1.0;
1756 -
1757 - if(!given_max) {
1758 - if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier))
1759 - *delay_max_duration = (*delay_up_duration) * (*delay_multiplier);
1760 -
1761 - if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier))
1762 - *delay_max_duration = (*delay_down_duration) * (*delay_multiplier);
1763 - }
1764 -
1765 - return 1;
1766 -}
1767 -
1768 -static inline uint32_t health_parse_options(const char *s) {
1769 - uint32_t options = 0;
1770 - char buf[100+1] = "";
1771 -
1772 - while(*s) {
1773 - buf[0] = '\0';
1774 -
1775 - // skip spaces
1776 - while(*s && isspace(*s))
1777 - s++;
1778 -
1779 - // find the next space
1780 - size_t count = 0;
1781 - while(*s && count < 100 && !isspace(*s))
1782 - buf[count++] = *s++;
1783 -
1784 - if(buf[0]) {
1785 - buf[count] = '\0';
1786 -
1787 - if(!strcasecmp(buf, "no-clear-notification") || !strcasecmp(buf, "no-clear"))
1788 - options |= RRDCALC_FLAG_NO_CLEAR_NOTIFICATION;
1789 - else
1790 - error("Ignoring unknown alarm option '%s'", buf);
1791 - }
1792 - }
1793 -
1794 - return options;
1795 -}
1796 -
1797 -static inline int health_parse_db_lookup(
1798 - size_t line, const char *path, const char *file, char *string,
1799 - int *group_method, int *after, int *before, int *every,
1800 - uint32_t *options, char **dimensions
1801 -) {
1802 - debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
1803 -
1804 - if(*dimensions) freez(*dimensions);
1805 - *dimensions = NULL;
1806 - *after = 0;
1807 - *before = 0;
1808 - *every = 0;
1809 - *options = 0;
1810 -
1811 - char *s = string, *key;
1812 -
1813 - // first is the group method
1814 - key = s;
1815 - while(*s && !isspace(*s)) s++;
1816 - while(*s && isspace(*s)) *s++ = '\0';
1817 - if(!*s) {
1818 - error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
1819 - line, path, file, key);
1820 - return 0;
1821 - }
1822 -
1823 - if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
1824 - error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
1825 - line, path, file, key);
1826 - return 0;
1827 - }
1828 -
1829 - // then is the 'after' time
1830 - key = s;
1831 - while(*s && !isspace(*s)) s++;
1832 - while(*s && isspace(*s)) *s++ = '\0';
1833 -
1834 - if(!health_parse_duration(key, after)) {
1835 - error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
1836 - line, path, file, key);
1837 - return 0;
1838 - }
1839 -
1840 - // sane defaults
1841 - *every = abs(*after);
1842 -
1843 - // now we may have optional parameters
1844 - while(*s) {
1845 - key = s;
1846 - while(*s && !isspace(*s)) s++;
1847 - while(*s && isspace(*s)) *s++ = '\0';
1848 - if(!*key) break;
1849 -
1850 - if(!strcasecmp(key, "at")) {
1851 - char *value = s;
1852 - while(*s && !isspace(*s)) s++;
1853 - while(*s && isspace(*s)) *s++ = '\0';
1854 -
1855 - if (!health_parse_duration(value, before)) {
1856 - error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1857 - line, path, file, value, key);
1858 - }
1859 - }
1860 - else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
1861 - char *value = s;
1862 - while(*s && !isspace(*s)) s++;
1863 - while(*s && isspace(*s)) *s++ = '\0';
1864 -
1865 - if (!health_parse_duration(value, every)) {
1866 - error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1867 - line, path, file, value, key);
1868 - }
1869 - }
1870 - else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
1871 - *options |= RRDR_OPTION_ABSOLUTE;
1872 - }
1873 - else if(!strcasecmp(key, "min2max")) {
1874 - *options |= RRDR_OPTION_MIN2MAX;
1875 - }
1876 - else if(!strcasecmp(key, "null2zero")) {
1877 - *options |= RRDR_OPTION_NULL2ZERO;
1878 - }
1879 - else if(!strcasecmp(key, "percentage")) {
1880 - *options |= RRDR_OPTION_PERCENTAGE;
1881 - }
1882 - else if(!strcasecmp(key, "unaligned")) {
1883 - *options |= RRDR_OPTION_NOT_ALIGNED;
1884 - }
1885 - else if(!strcasecmp(key, "of")) {
1886 - if(*s && strcasecmp(s, "all"))
1887 - *dimensions = strdupz(s);
1888 - break;
1889 - }
1890 - else {
1891 - error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1892 - line, path, file, key);
1893 - }
1894 - }
1895 -
1896 - return 1;
1897 -}
1898 -
1899 -static inline char *trim_all_spaces(char *buffer) {
1900 - char *d = buffer, *s = buffer;
1901 -
1902 - // skip spaces
1903 - while(isspace(*s)) s++;
1904 -
1905 - while(*s) {
1906 - // copy the non-space part
1907 - while(*s && !isspace(*s)) *d++ = *s++;
1908 -
1909 - // add a space if we have to
1910 - if(*s && isspace(*s)) {
1911 - *d++ = ' ';
1912 - s++;
1913 - }
1914 -
1915 - // skip spaces
1916 - while(isspace(*s)) s++;
1917 - }
1918 -
1919 - *d = '\0';
1920 -
1921 - if(d > buffer) {
1922 - d--;
1923 - if(isspace(*d)) *d = '\0';
1924 - }
1925 -
1926 - if(!buffer[0]) return NULL;
1927 - return buffer;
1928 -}
1929 -
1930 -static inline char *health_source_file(size_t line, const char *path, const char *filename) {
1931 - char buffer[FILENAME_MAX + 1];
1932 - snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
1933 - return strdupz(buffer);
1934 -}
1935 -
1936 -static inline void strip_quotes(char *s) {
1937 - while(*s) {
1938 - if(*s == '\'' || *s == '"') *s = ' ';
1939 - s++;
1940 - }
1941 -}
1942 -
1943 -int health_readfile(const char *path, const char *filename) {
1944 - debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1945 -
1946 - static uint32_t
1947 - hash_alarm = 0,
1948 - hash_template = 0,
1949 - hash_on = 0,
1950 - hash_families = 0,
1951 - hash_calc = 0,
1952 - hash_green = 0,
1953 - hash_red = 0,
1954 - hash_warn = 0,
1955 - hash_crit = 0,
1956 - hash_exec = 0,
1957 - hash_every = 0,
1958 - hash_lookup = 0,
1959 - hash_units = 0,
1960 - hash_info = 0,
1961 - hash_recipient = 0,
1962 - hash_delay = 0,
1963 - hash_options = 0;
1964 -
1965 - char buffer[HEALTH_CONF_MAX_LINE + 1];
1966 -
1967 - if(unlikely(!hash_alarm)) {
1968 - hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
1969 - hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
1970 - hash_on = simple_uhash(HEALTH_ON_KEY);
1971 - hash_families = simple_uhash(HEALTH_FAMILIES_KEY);
1972 - hash_calc = simple_uhash(HEALTH_CALC_KEY);
1973 - hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
1974 - hash_green = simple_uhash(HEALTH_GREEN_KEY);
1975 - hash_red = simple_uhash(HEALTH_RED_KEY);
1976 - hash_warn = simple_uhash(HEALTH_WARN_KEY);
1977 - hash_crit = simple_uhash(HEALTH_CRIT_KEY);
1978 - hash_exec = simple_uhash(HEALTH_EXEC_KEY);
1979 - hash_every = simple_uhash(HEALTH_EVERY_KEY);
1980 - hash_units = simple_hash(HEALTH_UNITS_KEY);
1981 - hash_info = simple_hash(HEALTH_INFO_KEY);
1982 - hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1983 - hash_delay = simple_uhash(HEALTH_DELAY_KEY);
1984 - hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
1985 - }
1986 -
1987 - snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
1988 - FILE *fp = fopen(buffer, "r");
1989 - if(!fp) {
1990 - error("Health configuration cannot read file '%s'.", buffer);
1991 - return 0;
1992 - }
1993 -
1994 - RRDCALC *rc = NULL;
1995 - RRDCALCTEMPLATE *rt = NULL;
1996 -
1997 - size_t line = 0, append = 0;
1998 - char *s;
1999 - while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
2000 - int stop_appending = !s;
2001 - line++;
2002 - s = trim(buffer);
2003 - if(!s) continue;
2004 -
2005 - append = strlen(s);
2006 - if(!stop_appending && s[append - 1] == '\\') {
2007 - s[append - 1] = ' ';
2008 - append = &s[append] - buffer;
2009 - if(append < HEALTH_CONF_MAX_LINE)
2010 - continue;
2011 - else {
2012 - error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
2013 - }
2014 - }
2015 - append = 0;
2016 -
2017 - char *key = s;
2018 - while(*s && *s != ':') s++;
2019 - if(!*s) {
2020 - error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
2021 - continue;
2022 - }
2023 - *s = '\0';
2024 - s++;
2025 -
2026 - char *value = s;
2027 - key = trim_all_spaces(key);
2028 - value = trim_all_spaces(value);
2029 -
2030 - if(!key) {
2031 - error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
2032 - continue;
2033 - }
2034 -
2035 - if(!value) {
2036 - error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
2037 - continue;
2038 - }
2039 -
2040 - uint32_t hash = simple_uhash(key);
2041 -
2042 - if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
2043 - if(rc && !rrdcalc_add_alarm_from_config(localhost, rc))
2044 - rrdcalc_free(localhost, rc);
2045 -
2046 - if(rt) {
2047 - if (!rrdcalctemplate_add_template_from_config(localhost, rt))
2048 - rrdcalctemplate_free(localhost, rt);
2049 - rt = NULL;
2050 - }
2051 -
2052 - rc = callocz(1, sizeof(RRDCALC));
2053 - rc->next_event_id = 1;
2054 - rc->name = strdupz(value);
2055 - rc->hash = simple_hash(rc->name);
2056 - rc->source = health_source_file(line, path, filename);
2057 - rc->green = NAN;
2058 - rc->red = NAN;
2059 - rc->value = NAN;
2060 - rc->old_value = NAN;
2061 - rc->delay_multiplier = 1.0;
2062 -
2063 - if(rrdvar_fix_name(rc->name))
2064 - error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
2065 - }
2066 - else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
2067 - if(rc) {
2068 - if(!rrdcalc_add_alarm_from_config(localhost, rc))
2069 - rrdcalc_free(localhost, rc);
2070 - rc = NULL;
2071 - }
2072 -
2073 - if(rt && !rrdcalctemplate_add_template_from_config(localhost, rt))
2074 - rrdcalctemplate_free(localhost, rt);
2075 -
2076 - rt = callocz(1, sizeof(RRDCALCTEMPLATE));
2077 - rt->name = strdupz(value);
2078 - rt->hash_name = simple_hash(rt->name);
2079 - rt->source = health_source_file(line, path, filename);
2080 - rt->green = NAN;
2081 - rt->red = NAN;
2082 - rt->delay_multiplier = 1.0;
2083 -
2084 - if(rrdvar_fix_name(rt->name))
2085 - error("Health configuration renamed template '%s' to '%s'", value, rt->name);
2086 - }
2087 - else if(rc) {
2088 - if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
2089 - if(rc->chart) {
2090 - if(strcmp(rc->chart, value))
2091 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2092 - line, path, filename, rc->name, key, rc->chart, value, value);
2093 -
2094 - freez(rc->chart);
2095 - }
2096 - rc->chart = strdupz(value);
2097 - rc->hash_chart = simple_hash(rc->chart);
2098 - }
2099 - else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
2100 - health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
2101 - &rc->update_every,
2102 - &rc->options, &rc->dimensions);
2103 - }
2104 - else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
2105 - if(!health_parse_duration(value, &rc->update_every))
2106 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
2107 - line, path, filename, rc->name, key, value);
2108 - }
2109 - else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
2110 - char *e;
2111 - rc->green = strtold(value, &e);
2112 - if(e && *e) {
2113 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
2114 - line, path, filename, rc->name, key, e);
2115 - }
2116 - }
2117 - else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
2118 - char *e;
2119 - rc->red = strtold(value, &e);
2120 - if(e && *e) {
2121 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
2122 - line, path, filename, rc->name, key, e);
2123 - }
2124 - }
2125 - else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
2126 - const char *failed_at = NULL;
2127 - int error = 0;
2128 - rc->calculation = expression_parse(value, &failed_at, &error);
2129 - if(!rc->calculation) {
2130 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
2131 - line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
2132 - }
2133 - }
2134 - else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
2135 - const char *failed_at = NULL;
2136 - int error = 0;
2137 - rc->warning = expression_parse(value, &failed_at, &error);
2138 - if(!rc->warning) {
2139 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
2140 - line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
2141 - }
2142 - }
2143 - else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
2144 - const char *failed_at = NULL;
2145 - int error = 0;
2146 - rc->critical = expression_parse(value, &failed_at, &error);
2147 - if(!rc->critical) {
2148 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
2149 - line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
2150 - }
2151 - }
2152 - else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
2153 - if(rc->exec) {
2154 - if(strcmp(rc->exec, value))
2155 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2156 - line, path, filename, rc->name, key, rc->exec, value, value);
2157 -
2158 - freez(rc->exec);
2159 - }
2160 - rc->exec = strdupz(value);
2161 - }
2162 - else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
2163 - if(rc->recipient) {
2164 - if(strcmp(rc->recipient, value))
2165 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2166 - line, path, filename, rc->name, key, rc->recipient, value, value);
2167 -
2168 - freez(rc->recipient);
2169 - }
2170 - rc->recipient = strdupz(value);
2171 - }
2172 - else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
2173 - if(rc->units) {
2174 - if(strcmp(rc->units, value))
2175 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2176 - line, path, filename, rc->name, key, rc->units, value, value);
2177 -
2178 - freez(rc->units);
2179 - }
2180 - rc->units = strdupz(value);
2181 - strip_quotes(rc->units);
2182 - }
2183 - else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
2184 - if(rc->info) {
2185 - if(strcmp(rc->info, value))
2186 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2187 - line, path, filename, rc->name, key, rc->info, value, value);
2188 -
2189 - freez(rc->info);
2190 - }
2191 - rc->info = strdupz(value);
2192 - strip_quotes(rc->info);
2193 - }
2194 - else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
2195 - health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
2196 - }
2197 - else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
2198 - rc->options |= health_parse_options(value);
2199 - }
2200 - else {
2201 - error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
2202 - line, path, filename, rc->name, key);
2203 - }
2204 - }
2205 - else if(rt) {
2206 - if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
2207 - if(rt->context) {
2208 - if(strcmp(rt->context, value))
2209 - error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2210 - line, path, filename, rt->name, key, rt->context, value, value);
2211 -
2212 - freez(rt->context);
2213 - }
2214 - rt->context = strdupz(value);
2215 - rt->hash_context = simple_hash(rt->context);
2216 - }
2217 - else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
2218 - freez(rt->family_match);
2219 - simple_pattern_free(rt->family_pattern);
2220 -
2221 - rt->family_match = strdupz(value);
2222 - rt->family_pattern = simple_pattern_create(rt->family_match, SIMPLE_PATTERN_EXACT);
2223 - }
2224 - else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
2225 - health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
2226 - &rt->update_every, &rt->options, &rt->dimensions);
2227 - }
2228 - else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
2229 - if(!health_parse_duration(value, &rt->update_every))
2230 - error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
2231 - line, path, filename, rt->name, key, value);
2232 - }
2233 - else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
2234 - char *e;
2235 - rt->green = strtold(value, &e);
2236 - if(e && *e) {
2237 - error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
2238 - line, path, filename, rt->name, key, e);
2239 - }
2240 - }
2241 - else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
2242 - char *e;
2243 - rt->red = strtold(value, &e);
2244 - if(e && *e) {
2245 - error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
2246 - line, path, filename, rt->name, key, e);
2247 - }
2248 - }
2249 - else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
2250 - const char *failed_at = NULL;
2251 - int error = 0;
2252 - rt->calculation = expression_parse(value, &failed_at, &error);
2253 - if(!rt->calculation) {
2254 - error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
2255 - line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
2256 - }
2257 - }
2258 - else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
2259 - const char *failed_at = NULL;
2260 - int error = 0;
2261 - rt->warning = expression_parse(value, &failed_at, &error);
2262 - if(!rt->warning) {
2263 - error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
2264 - line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
2265 - }
2266 - }
2267 - else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
2268 - const char *failed_at = NULL;
2269 - int error = 0;
2270 - rt->critical = expression_parse(value, &failed_at, &error);
2271 - if(!rt->critical) {
2272 - error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
2273 - line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
2274 - }
2275 - }
2276 - else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
2277 - if(rt->exec) {
2278 - if(strcmp(rt->exec, value))
2279 - error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2280 - line, path, filename, rt->name, key, rt->exec, value, value);
2281 -
2282 - freez(rt->exec);
2283 - }
2284 - rt->exec = strdupz(value);
2285 - }
2286 - else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
2287 - if(rt->recipient) {
2288 - if(strcmp(rt->recipient, value))
2289 - error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2290 - line, path, filename, rt->name, key, rt->recipient, value, value);
2291 -
2292 - freez(rt->recipient);
2293 - }
2294 - rt->recipient = strdupz(value);
2295 - }
2296 - else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
2297 - if(rt->units) {
2298 - if(strcmp(rt->units, value))
2299 - error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2300 - line, path, filename, rt->name, key, rt->units, value, value);
2301 -
2302 - freez(rt->units);
2303 - }
2304 - rt->units = strdupz(value);
2305 - strip_quotes(rt->units);
2306 - }
2307 - else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
2308 - if(rt->info) {
2309 - if(strcmp(rt->info, value))
2310 - error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
2311 - line, path, filename, rt->name, key, rt->info, value, value);
2312 -
2313 - freez(rt->info);
2314 - }
2315 - rt->info = strdupz(value);
2316 - strip_quotes(rt->info);
2317 - }
2318 - else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
2319 - health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
2320 - }
2321 - else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
2322 - rt->options |= health_parse_options(value);
2323 - }
2324 - else {
2325 - error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
2326 - line, path, filename, rt->name, key);
2327 - }
2328 - }
2329 - else {
2330 - error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
2331 - line, path, filename, key);
2332 - }
2333 - }
2334 -
2335 - if(rc && !rrdcalc_add_alarm_from_config(localhost, rc))
2336 - rrdcalc_free(localhost, rc);
2337 -
2338 - if(rt && !rrdcalctemplate_add_template_from_config(localhost, rt))
2339 - rrdcalctemplate_free(localhost, rt);
2340 -
2341 - fclose(fp);
2342 - return 1;
2343 -}
2344 -
2345 -void health_readdir(const char *path) {
2346 - size_t pathlen = strlen(path);
2347 -
2348 - debug(D_HEALTH, "Health configuration reading directory '%s'", path);
2349 -
2350 - DIR *dir = opendir(path);
2351 - if (!dir) {
2352 - error("Health configuration cannot open directory '%s'.", path);
2353 - return;
2354 - }
2355 -
2356 - struct dirent *de = NULL;
2357 - while ((de = readdir(dir))) {
2358 - size_t len = strlen(de->d_name);
2359 -
2360 - if(de->d_type == DT_DIR
2361 - && (
2362 - (de->d_name[0] == '.' && de->d_name[1] == '\0')
2363 - || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
2364 - )) {
2365 - debug(D_HEALTH, "Ignoring directory '%s'", de->d_name);
2366 - continue;
2367 - }
2368 -
2369 - else if(de->d_type == DT_DIR) {
2370 - char *s = mallocz(pathlen + strlen(de->d_name) + 2);
2371 - strcpy(s, path);
2372 - strcat(s, "/");
2373 - strcat(s, de->d_name);
2374 - health_readdir(s);
2375 - freez(s);
2376 - continue;
2377 - }
2378 -
2379 - else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
2380 - len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
2381 - health_readfile(path, de->d_name);
2382 - }
2383 -
2384 - else debug(D_HEALTH, "Ignoring file '%s'", de->d_name);
2385 - }
2386 -
2387 - closedir(dir);
2388 -}
2389 -
2390 -static inline char *health_config_dir(void) {
2391 - char buffer[FILENAME_MAX + 1];
2392 - snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_config_dir);
2393 - return config_get("health", "health configuration directory", buffer);
2394 -}
2395 -
2396 -void health_init(void) {
2397 - debug(D_HEALTH, "Health configuration initializing");
2398 -
2399 - if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
2400 - debug(D_HEALTH, "Health is disabled.");
2401 - return;
2402 - }
2403 -
2404 - char pathname[FILENAME_MAX + 1];
2405 - snprintfz(pathname, FILENAME_MAX, "%s/health", netdata_configured_varlib_dir);
2406 - if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
2407 - fatal("Cannot create directory '%s'.", pathname);
2408 -
2409 - char filename[FILENAME_MAX + 1];
2410 - snprintfz(filename, FILENAME_MAX, "%s/health-log.db", pathname);
2411 - health.log_filename = config_get("health", "health db file", filename);
2412 -
2413 - health_alarm_log_load(localhost);
2414 - health_alarm_log_open();
2415 -
2416 - char *path = health_config_dir();
2417 -
2418 - snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_plugins_dir);
2419 - health.health_default_exec = config_get("health", "script to execute on alarm", filename);
2420 -
2421 - long n = config_get_number("health", "in memory max health log entries", (long)localhost->health_log.max);
2422 - if(n < 10) {
2423 - error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost->health_log.max);
2424 - config_set_number("health", "in memory max health log entries", (long)localhost->health_log.max);
2425 - }
2426 - else localhost->health_log.max = (unsigned int)n;
2427 -
2428 - rrdhost_rwlock(localhost);
2429 - health_readdir(path);
2430 - rrdhost_unlock(localhost);
2431 -}
2432 -
2433 -// ----------------------------------------------------------------------------
2434 -// JSON generation
2435 -
2436 -static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
2437 - if(value && *value)
2438 - buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
2439 - else
2440 - buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
2441 -}
2442 -
2443 -static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
2444 - buffer_sprintf(wb,
2445 - "\n\t{\n"
2446 - "\t\t\"hostname\": \"%s\",\n"
2447 - "\t\t\"unique_id\": %u,\n"
2448 - "\t\t\"alarm_id\": %u,\n"
2449 - "\t\t\"alarm_event_id\": %u,\n"
2450 - "\t\t\"name\": \"%s\",\n"
2451 - "\t\t\"chart\": \"%s\",\n"
2452 - "\t\t\"family\": \"%s\",\n"
2453 - "\t\t\"processed\": %s,\n"
2454 - "\t\t\"updated\": %s,\n"
2455 - "\t\t\"exec_run\": %lu,\n"
2456 - "\t\t\"exec_failed\": %s,\n"
2457 - "\t\t\"exec\": \"%s\",\n"
2458 - "\t\t\"recipient\": \"%s\",\n"
2459 - "\t\t\"exec_code\": %d,\n"
2460 - "\t\t\"source\": \"%s\",\n"
2461 - "\t\t\"units\": \"%s\",\n"
2462 - "\t\t\"info\": \"%s\",\n"
2463 - "\t\t\"when\": %lu,\n"
2464 - "\t\t\"duration\": %lu,\n"
2465 - "\t\t\"non_clear_duration\": %lu,\n"
2466 - "\t\t\"status\": \"%s\",\n"
2467 - "\t\t\"old_status\": \"%s\",\n"
2468 - "\t\t\"delay\": %d,\n"
2469 - "\t\t\"delay_up_to_timestamp\": %lu,\n"
2470 - "\t\t\"updated_by_id\": %u,\n"
2471 - "\t\t\"updates_id\": %u,\n"
2472 - "\t\t\"value_string\": \"%s\",\n"
2473 - "\t\t\"old_value_string\": \"%s\",\n"
2474 - , host->hostname
2475 - , ae->unique_id
2476 - , ae->alarm_id
2477 - , ae->alarm_event_id
2478 - , ae->name
2479 - , ae->chart
2480 - , ae->family
2481 - , (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false"
2482 - , (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"
2483 - , (unsigned long)ae->exec_run_timestamp
2484 - , (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false"
2485 - , ae->exec?ae->exec:health.health_default_exec
2486 - , ae->recipient?ae->recipient:health.health_default_recipient
2487 - , ae->exec_code
2488 - , ae->source
2489 - , ae->units?ae->units:""
2490 - , ae->info?ae->info:""
2491 - , (unsigned long)ae->when
2492 - , (unsigned long)ae->duration
2493 - , (unsigned long)ae->non_clear_duration
2494 - , rrdcalc_status2string(ae->new_status)
2495 - , rrdcalc_status2string(ae->old_status)
2496 - , ae->delay
2497 - , (unsigned long)ae->delay_up_to_timestamp
2498 - , ae->updated_by_id
2499 - , ae->updates_id
2500 - , ae->new_value_string
2501 - , ae->old_value_string
2502 - );
2503 -
2504 - if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
2505 - buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");
2506 - }
2507 -
2508 - buffer_strcat(wb, "\t\t\"value\":");
2509 - buffer_rrd_value(wb, ae->new_value);
2510 - buffer_strcat(wb, ",\n");
2511 -
2512 - buffer_strcat(wb, "\t\t\"old_value\":");
2513 - buffer_rrd_value(wb, ae->old_value);
2514 - buffer_strcat(wb, "\n");
2515 -
2516 - buffer_strcat(wb, "\t}");
2517 -}
2518 -
2519 -void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after) {
2520 - pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
2521 -
2522 - buffer_strcat(wb, "[");
2523 -
2524 - unsigned int max = host->health_log.max;
2525 - unsigned int count = 0;
2526 - ALARM_ENTRY *ae;
2527 - for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
2528 - if(ae->unique_id > after) {
2529 - if(likely(count)) buffer_strcat(wb, ",");
2530 - health_alarm_entry2json_nolock(wb, ae, host);
2531 - }
2532 - }
2533 -
2534 - buffer_strcat(wb, "\n]\n");
2535 -
2536 - pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2537 -}
2538 -
2539 -static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
2540 - char value_string[100 + 1];
2541 - format_value_and_unit(value_string, 100, rc->value, rc->units, -1);
2542 -
2543 - buffer_sprintf(wb,
2544 - "\t\t\"%s.%s\": {\n"
2545 - "\t\t\t\"id\": %lu,\n"
2546 - "\t\t\t\"name\": \"%s\",\n"
2547 - "\t\t\t\"chart\": \"%s\",\n"
2548 - "\t\t\t\"family\": \"%s\",\n"
2549 - "\t\t\t\"active\": %s,\n"
2550 - "\t\t\t\"exec\": \"%s\",\n"
2551 - "\t\t\t\"recipient\": \"%s\",\n"
2552 - "\t\t\t\"source\": \"%s\",\n"
2553 - "\t\t\t\"units\": \"%s\",\n"
2554 - "\t\t\t\"info\": \"%s\",\n"
2555 - "\t\t\t\"status\": \"%s\",\n"
2556 - "\t\t\t\"last_status_change\": %lu,\n"
2557 - "\t\t\t\"last_updated\": %lu,\n"
2558 - "\t\t\t\"next_update\": %lu,\n"
2559 - "\t\t\t\"update_every\": %d,\n"
2560 - "\t\t\t\"delay_up_duration\": %d,\n"
2561 - "\t\t\t\"delay_down_duration\": %d,\n"
2562 - "\t\t\t\"delay_max_duration\": %d,\n"
2563 - "\t\t\t\"delay_multiplier\": %f,\n"
2564 - "\t\t\t\"delay\": %d,\n"
2565 - "\t\t\t\"delay_up_to_timestamp\": %lu,\n"
2566 - "\t\t\t\"value_string\": \"%s\",\n"
2567 - , rc->chart, rc->name
2568 - , (unsigned long)rc->id
2569 - , rc->name
2570 - , rc->chart
2571 - , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
2572 - , (rc->rrdset)?"true":"false"
2573 - , rc->exec?rc->exec:health.health_default_exec
2574 - , rc->recipient?rc->recipient:health.health_default_recipient
2575 - , rc->source
2576 - , rc->units?rc->units:""
2577 - , rc->info?rc->info:""
2578 - , rrdcalc_status2string(rc->status)
2579 - , (unsigned long)rc->last_status_change
2580 - , (unsigned long)rc->last_updated
2581 - , (unsigned long)rc->next_update
2582 - , rc->update_every
2583 - , rc->delay_up_duration
2584 - , rc->delay_down_duration
2585 - , rc->delay_max_duration
2586 - , rc->delay_multiplier
2587 - , rc->delay_last
2588 - , (unsigned long)rc->delay_up_to_timestamp
2589 - , value_string
2590 - );
2591 -
2592 - if(unlikely(rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)) {
2593 - buffer_strcat(wb, "\t\t\t\"no_clear_notification\": true,\n");
2594 - }
2595 -
2596 - if(RRDCALC_HAS_DB_LOOKUP(rc)) {
2597 - if(rc->dimensions && *rc->dimensions)
2598 - health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
2599 -
2600 - buffer_sprintf(wb,
2601 - "\t\t\t\"db_after\": %lu,\n"
2602 - "\t\t\t\"db_before\": %lu,\n"
2603 - "\t\t\t\"lookup_method\": \"%s\",\n"
2604 - "\t\t\t\"lookup_after\": %d,\n"
2605 - "\t\t\t\"lookup_before\": %d,\n"
2606 - "\t\t\t\"lookup_options\": \"",
2607 - (unsigned long) rc->db_after,
2608 - (unsigned long) rc->db_before,
2609 - group_method2string(rc->group),
2610 - rc->after,
2611 - rc->before
2612 - );
2613 - buffer_data_options2string(wb, rc->options);
2614 - buffer_strcat(wb, "\",\n");
2615 - }
2616 -
2617 - if(rc->calculation) {
2618 - health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
2619 - health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
2620 - }
2621 -
2622 - if(rc->warning) {
2623 - health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
2624 - health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
2625 - }
2626 -
2627 - if(rc->critical) {
2628 - health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
2629 - health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
2630 - }
2631 -
2632 - buffer_strcat(wb, "\t\t\t\"green\":");
2633 - buffer_rrd_value(wb, rc->green);
2634 - buffer_strcat(wb, ",\n");
2635 -
2636 - buffer_strcat(wb, "\t\t\t\"red\":");
2637 - buffer_rrd_value(wb, rc->red);
2638 - buffer_strcat(wb, ",\n");
2639 -
2640 - buffer_strcat(wb, "\t\t\t\"value\":");
2641 - buffer_rrd_value(wb, rc->value);
2642 - buffer_strcat(wb, "\n");
2643 -
2644 - buffer_strcat(wb, "\t\t}");
2645 -}
2646 -
2647 -//void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
2648 -//
2649 -//}
2650 -
2651 -void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
2652 - int i;
2653 -
2654 - rrdhost_rdlock(localhost);
2655 - buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
2656 - "\n\t\"latest_alarm_log_unique_id\": %u,"
2657 - "\n\t\"status\": %s,"
2658 - "\n\t\"now\": %lu,"
2659 - "\n\t\"alarms\": {\n",
2660 - host->hostname,
2661 - (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
2662 - health_enabled?"true":"false",
2663 - (unsigned long)now_realtime_sec());
2664 -
2665 - RRDCALC *rc;
2666 - for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
2667 - if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
2668 - continue;
2669 -
2670 - if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
2671 - continue;
2672 -
2673 - if(likely(i)) buffer_strcat(wb, ",\n");
2674 - health_rrdcalc2json_nolock(wb, rc);
2675 - i++;
2676 - }
2677 -
2678 -// buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
2679 -// RRDCALCTEMPLATE *rt;
2680 -// for(rt = host->templates; rt ; rt = rt->next)
2681 -// health_rrdcalctemplate2json_nolock(wb, rt);
2682 -
2683 - buffer_strcat(wb, "\n\t}\n}\n");
2684 - rrdhost_unlock(localhost);
2685 -}
2686 -
2687 -
2688 -// ----------------------------------------------------------------------------
2689 -// re-load health configuration
2690 -
2691 -static inline void health_free_all_nolock(RRDHOST *host) {
2692 - while(host->templates)
2693 - rrdcalctemplate_free(host, host->templates);
2694 -
2695 - while(host->alarms)
2696 - rrdcalc_free(host, host->alarms);
2697 -}
2698 -
2699 -void health_reload(void) {
2700 - if(!health_enabled) {
2701 - error("Health reload is requested, but health is not enabled.");
2702 - return;
2703 - }
2704 -
2705 - char *path = health_config_dir();
2706 -
2707 - // free all running alarms
2708 - rrdhost_rwlock(localhost);
2709 - health_free_all_nolock(localhost);
2710 - rrdhost_unlock(localhost);
2711 -
2712 - // invalidate all previous entries in the alarm log
2713 - ALARM_ENTRY *t;
2714 - for(t = localhost->health_log.alarms ; t ; t = t->next) {
2715 - if(t->new_status != RRDCALC_STATUS_REMOVED)
2716 - t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
2717 - }
2718 -
2719 - // reset all thresholds to all charts
2720 - RRDSET *st;
2721 - for(st = localhost->rrdset_root; st ; st = st->next) {
2722 - st->green = NAN;
2723 - st->red = NAN;
2724 - }
2725 -
2726 - // load the new alarms
2727 - rrdhost_rwlock(localhost);
2728 - health_readdir(path);
2729 - rrdhost_unlock(localhost);
2730 -
2731 - // link the loaded alarms to their charts
2732 - for(st = localhost->rrdset_root; st ; st = st->next) {
2733 - rrdhost_rwlock(localhost);
2734 -
2735 - rrdsetcalc_link_matching(st);
2736 - rrdcalctemplate_link_matching(st);
2737 -
2738 - rrdhost_unlock(localhost);
2739 - }
2740 -}
2741 -
2742 -// ----------------------------------------------------------------------------
2743 -// health main thread and friends
2744 -
2745 -static inline int rrdcalc_value2status(calculated_number n) {
2746 - if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
2747 - if(n) return RRDCALC_STATUS_RAISED;
2748 - return RRDCALC_STATUS_CLEAR;
2749 -}
2750 -
2751 -#define ALARM_EXEC_COMMAND_LENGTH 8192
2752 -
2753 -static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2754 - ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
2755 -
2756 - if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
2757 - // do not send notifications for internal statuses
2758 - debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
2759 - goto done;
1164 + if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
1165 + // do not send notifications for internal statuses
1166 + debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
1167 + goto done;
1168 }
1169
1170 if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {
@@ -2800,11 +1208,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
1208 static char command_to_run[ALARM_EXEC_COMMAND_LENGTH + 1];
1209 pid_t command_pid;
1210
2803 - const char *exec = ae->exec;
2804 - if(!exec) exec = health.health_default_exec;
2805 -
2806 - const char *recipient = ae->recipient;
2807 - if(!recipient) recipient = health.health_default_recipient;
1211 + const char *exec = (ae->exec) ? ae->exec : host->health_default_exec;
1212 + const char *recipient = (ae->recipient) ? ae->recipient : host->health_default_recipient;
1213
1214 snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s' '%s' '%s'",
1215 exec,
@@ -3013,300 +1418,265 @@ void *health_main(void *ptr) {
1418 if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
1419 error("Cannot set pthread cancel state to DISABLE.");
1420
3016 - rrdhost_rdlock(localhost);
1421 + RRDHOST *host;
1422 + for(host = localhost; host ; host = host->next) {
1423 + rrdhost_rdlock(host);
1424
3018 - // the first loop is to lookup values from the db
3019 - for(rc = localhost->alarms; rc; rc = rc->next) {
3020 - if(unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
3021 - if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))
3022 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE;
3023 - continue;
3024 - }
1425 + // the first loop is to lookup values from the db
1426 + for(rc = host->alarms; rc; rc = rc->next) {
1427 + if(unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
1428 + if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))
1429 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE;
1430 + continue;
1431 + }
1432
3026 - runnable++;
3027 - rc->old_value = rc->value;
3028 - rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE;
1433 + runnable++;
1434 + rc->old_value = rc->value;
1435 + rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE;
1436
3030 - // 1. if there is database lookup, do it
3031 - // 2. if there is calculation expression, run it
1437 + // 1. if there is database lookup, do it
1438 + // 2. if there is calculation expression, run it
1439
3033 - if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
3034 - /* time_t old_db_timestamp = rc->db_before; */
3035 - int value_is_null = 0;
1440 + if(unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
1441 + /* time_t old_db_timestamp = rc->db_before; */
1442 + int value_is_null = 0;
1443
3037 - int ret = rrd2value(rc->rrdset, wb, &rc->value,
3038 - rc->dimensions, 1, rc->after, rc->before, rc->group,
3039 - rc->options, &rc->db_after, &rc->db_before, &value_is_null);
1444 + int ret = rrd2value(rc->rrdset, wb, &rc->value, rc->dimensions, 1, rc->after, rc->before, rc->group, rc->options, &rc->db_after, &rc->db_before, &value_is_null);
1445
3041 - if (unlikely(ret != 200)) {
3042 - // database lookup failed
3043 - rc->value = NAN;
1446 + if(unlikely(ret != 200)) {
1447 + // database lookup failed
1448 + rc->value = NAN;
1449
3045 - debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
1450 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, ret);
1451
3047 - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))) {
3048 - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
3049 - error("Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
1452 + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))) {
1453 + rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
1454 + error("Health on host '%s', alarm '%s.%s': database lookup returned error %d", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, ret);
1455 + }
1456 }
3051 - }
3052 - else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))
3053 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
1457 + else if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))
1458 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
1459
3055 - /* - RRDCALC_FLAG_DB_STALE not currently used
3056 - if (unlikely(old_db_timestamp == rc->db_before)) {
3057 - // database is stale
1460 + /* - RRDCALC_FLAG_DB_STALE not currently used
1461 + if (unlikely(old_db_timestamp == rc->db_before)) {
1462 + // database is stale
1463
3059 - debug(D_HEALTH, "Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
1464 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1465
3061 - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
3062 - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
3063 - error("Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
1466 + if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
1467 + rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
1468 + error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1469 + }
1470 }
3065 - }
3066 - else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
3067 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
3068 - */
1471 + else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
1472 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
1473 + */
1474
3070 - if (unlikely(value_is_null)) {
3071 - // collected value is null
1475 + if(unlikely(value_is_null)) {
1476 + // collected value is null
1477
3073 - rc->value = NAN;
1478 + rc->value = NAN;
1479
3075 - debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
3076 - rc->chart?rc->chart:"NOCHART", rc->name);
1480 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name);
1481
3078 - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))) {
3079 - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN;
3080 - error("Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
3081 - rc->chart?rc->chart:"NOCHART", rc->name);
1482 + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))) {
1483 + rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN;
1484 + error("Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name);
1485 + }
1486 }
1487 + else if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))
1488 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
1489 +
1490 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " CALCULATED_NUMBER_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, rc->value);
1491 }
3084 - else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))
3085 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
1492
3087 - debug(D_HEALTH, "Health alarm '%s.%s': database lookup gave value "
3088 - CALCULATED_NUMBER_FORMAT, rc->chart?rc->chart:"NOCHART", rc->name, rc->value);
3089 - }
1493 + if(unlikely(rc->calculation)) {
1494 + if(unlikely(!expression_evaluate(rc->calculation))) {
1495 + // calculation failed
1496
3091 - if(unlikely(rc->calculation)) {
3092 - if (unlikely(!expression_evaluate(rc->calculation))) {
3093 - // calculation failed
1497 + rc->value = NAN;
1498
3095 - rc->value = NAN;
1499 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg));
1500
3097 - debug(D_HEALTH, "Health alarm '%s.%s': expression '%s' failed: %s",
3098 - rc->chart?rc->chart:"NOCHART", rc->name, rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg));
1501 + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) {
1502 + rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
1503 + error("Health on host '%s', alarm '%s.%s': expression '%s' failed: %s", rc->chart ? rc->chart : "NOCHART", host->hostname, rc->name, rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg));
1504 + }
1505 + }
1506 + else {
1507 + if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))
1508 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
1509 +
1510 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
1511 + CALCULATED_NUMBER_FORMAT
1512 + ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name
1513 + , rc->calculation->parsed_as, rc->calculation->result,
1514 + buffer_tostring(rc->calculation->error_msg), rc->source
1515 + );
1516
3100 - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) {
3101 - rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
3102 - error("Health alarm '%s.%s': expression '%s' failed: %s",
3103 - rc->chart?rc->chart:"NOCHART", rc->name, rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg));
1517 + rc->value = rc->calculation->result;
1518 }
1519 }
3106 - else {
3107 - if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))
3108 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
3109 -
3110 - debug(D_HEALTH, "Health alarm '%s.%s': expression '%s' gave value "
3111 - CALCULATED_NUMBER_FORMAT
3112 - ": %s (source: %s)",
3113 - rc->chart?rc->chart:"NOCHART", rc->name,
3114 - rc->calculation->parsed_as,
3115 - rc->calculation->result,
3116 - buffer_tostring(rc->calculation->error_msg),
3117 - rc->source
3118 - );
3119 -
3120 - rc->value = rc->calculation->result;
3121 - }
1520 }
3123 - }
3124 - rrdhost_unlock(localhost);
1521 + rrdhost_unlock(host);
1522
3126 - if(unlikely(runnable && !netdata_exit)) {
3127 - rrdhost_rdlock(localhost);
1523 + if(unlikely(runnable && !netdata_exit)) {
1524 + rrdhost_rdlock(host);
1525
3129 - for(rc = localhost->alarms; rc; rc = rc->next) {
3130 - if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)))
3131 - continue;
1526 + for(rc = host->alarms; rc; rc = rc->next) {
1527 + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)))
1528 + continue;
1529
3133 - int warning_status = RRDCALC_STATUS_UNDEFINED;
3134 - int critical_status = RRDCALC_STATUS_UNDEFINED;
1530 + int warning_status = RRDCALC_STATUS_UNDEFINED;
1531 + int critical_status = RRDCALC_STATUS_UNDEFINED;
1532
3136 - if(likely(rc->warning)) {
3137 - if(unlikely(!expression_evaluate(rc->warning))) {
3138 - // calculation failed
1533 + if(likely(rc->warning)) {
1534 + if(unlikely(!expression_evaluate(rc->warning))) {
1535 + // calculation failed
1536
3140 - debug(D_HEALTH, "Health alarm '%s.%s': warning expression failed with error: %s",
3141 - rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
1537 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
1538
3143 - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))) {
3144 - rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
3145 - error("Health alarm '%s.%s': warning expression failed with error: %s",
3146 - rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
1539 + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))) {
1540 + rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
1541 + error("Health on host '%s', alarm '%s.%s': warning expression failed with error: %s", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
1542 + }
1543 }
3148 - }
3149 - else {
3150 - if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))
3151 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
1544 + else {
1545 + if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))
1546 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
1547
3153 - debug(D_HEALTH, "Health alarm '%s.%s': warning expression gave value "
3154 - CALCULATED_NUMBER_FORMAT
3155 - ": %s (source: %s)",
3156 - rc->chart?rc->chart:"NOCHART", rc->name,
3157 - rc->warning->result,
3158 - buffer_tostring(rc->warning->error_msg),
3159 - rc->source
3160 - );
1548 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value " CALCULATED_NUMBER_FORMAT ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source);
1549
3162 - warning_status = rrdcalc_value2status(rc->warning->result);
1550 + warning_status = rrdcalc_value2status(rc->warning->result);
1551 + }
1552 }
3164 - }
1553
3166 - if(likely(rc->critical)) {
3167 - if(unlikely(!expression_evaluate(rc->critical))) {
3168 - // calculation failed
1554 + if(likely(rc->critical)) {
1555 + if(unlikely(!expression_evaluate(rc->critical))) {
1556 + // calculation failed
1557
3170 - debug(D_HEALTH, "Health alarm '%s.%s': critical expression failed with error: %s",
3171 - rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
1558 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
1559
3173 - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))) {
3174 - rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
3175 - error("Health alarm '%s.%s': critical expression failed with error: %s",
3176 - rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
1560 + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))) {
1561 + rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
1562 + error("Health on host '%s', alarm '%s.%s': critical expression failed with error: %s", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
1563 + }
1564 }
3178 - }
3179 - else {
3180 - if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))
3181 - rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
1565 + else {
1566 + if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))
1567 + rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
1568
3183 - debug(D_HEALTH, "Health alarm '%s.%s': critical expression gave value "
3184 - CALCULATED_NUMBER_FORMAT
3185 - ": %s (source: %s)",
3186 - rc->chart?rc->chart:"NOCHART", rc->name,
3187 - rc->critical->result,
3188 - buffer_tostring(rc->critical->error_msg),
3189 - rc->source
3190 - );
1569 + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value " CALCULATED_NUMBER_FORMAT ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name , rc->critical->result, buffer_tostring(rc->critical->error_msg), rc->source);
1570
3192 - critical_status = rrdcalc_value2status(rc->critical->result);
1571 + critical_status = rrdcalc_value2status(rc->critical->result);
1572 + }
1573 }
3194 - }
1574
3196 - int status = RRDCALC_STATUS_UNDEFINED;
1575 + int status = RRDCALC_STATUS_UNDEFINED;
1576
3198 - switch(warning_status) {
3199 - case RRDCALC_STATUS_CLEAR:
3200 - status = RRDCALC_STATUS_CLEAR;
3201 - break;
1577 + switch(warning_status) {
1578 + case RRDCALC_STATUS_CLEAR:
1579 + status = RRDCALC_STATUS_CLEAR;
1580 + break;
1581
3203 - case RRDCALC_STATUS_RAISED:
3204 - status = RRDCALC_STATUS_WARNING;
3205 - break;
1582 + case RRDCALC_STATUS_RAISED:
1583 + status = RRDCALC_STATUS_WARNING;
1584 + break;
1585
3207 - default:
3208 - break;
3209 - }
1586 + default:
1587 + break;
1588 + }
1589
3211 - switch(critical_status) {
3212 - case RRDCALC_STATUS_CLEAR:
3213 - if(status == RRDCALC_STATUS_UNDEFINED)
3214 - status = RRDCALC_STATUS_CLEAR;
3215 - break;
1590 + switch(critical_status) {
1591 + case RRDCALC_STATUS_CLEAR:
1592 + if(status == RRDCALC_STATUS_UNDEFINED)
1593 + status = RRDCALC_STATUS_CLEAR;
1594 + break;
1595
3217 - case RRDCALC_STATUS_RAISED:
3218 - status = RRDCALC_STATUS_CRITICAL;
3219 - break;
1596 + case RRDCALC_STATUS_RAISED:
1597 + status = RRDCALC_STATUS_CRITICAL;
1598 + break;
1599
3221 - default:
3222 - break;
3223 - }
1600 + default:
1601 + break;
1602 + }
1603
3225 - if(status != rc->status) {
3226 - int delay = 0;
1604 + if(status != rc->status) {
1605 + int delay = 0;
1606
3228 - if(now > rc->delay_up_to_timestamp) {
3229 - rc->delay_up_current = rc->delay_up_duration;
3230 - rc->delay_down_current = rc->delay_down_duration;
3231 - rc->delay_last = 0;
3232 - rc->delay_up_to_timestamp = 0;
3233 - }
3234 - else {
3235 - rc->delay_up_current = (int)(rc->delay_up_current * rc->delay_multiplier);
3236 - if(rc->delay_up_current > rc->delay_max_duration) rc->delay_up_current = rc->delay_max_duration;
1607 + if(now > rc->delay_up_to_timestamp) {
1608 + rc->delay_up_current = rc->delay_up_duration;
1609 + rc->delay_down_current = rc->delay_down_duration;
1610 + rc->delay_last = 0;
1611 + rc->delay_up_to_timestamp = 0;
1612 + }
1613 + else {
1614 + rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
1615 + if(rc->delay_up_current > rc->delay_max_duration)
1616 + rc->delay_up_current = rc->delay_max_duration;
1617 +
1618 + rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
1619 + if(rc->delay_down_current > rc->delay_max_duration)
1620 + rc->delay_down_current = rc->delay_max_duration;
1621 + }
1622
3238 - rc->delay_down_current = (int)(rc->delay_down_current * rc->delay_multiplier);
3239 - if(rc->delay_down_current > rc->delay_max_duration) rc->delay_down_current = rc->delay_max_duration;
1623 + if(status > rc->status)
1624 + delay = rc->delay_up_current;
1625 + else
1626 + delay = rc->delay_down_current;
1627 +
1628 + // COMMENTED: because we do need to send raising alarms
1629 + // if(now + delay < rc->delay_up_to_timestamp)
1630 + // delay = (int)(rc->delay_up_to_timestamp - now);
1631 +
1632 + rc->delay_last = delay;
1633 + rc->delay_up_to_timestamp = now + delay;
1634 + health_alarm_log(
1635 + host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id
1636 + , rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change
1637 + , rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info
1638 + , rc->delay_last, (rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)
1639 + ? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0
1640 + );
1641 + rc->last_status_change = now;
1642 + rc->status = status;
1643 }
1644
3242 - if(status > rc->status)
3243 - delay = rc->delay_up_current;
3244 - else
3245 - delay = rc->delay_down_current;
3246 -
3247 - // COMMENTED: because we do need to send raising alarms
3248 - // if(now + delay < rc->delay_up_to_timestamp)
3249 - // delay = (int)(rc->delay_up_to_timestamp - now);
3250 -
3251 - rc->delay_last = delay;
3252 - rc->delay_up_to_timestamp = now + delay;
3253 - health_alarm_log(
3254 - localhost,
3255 - rc->id,
3256 - rc->next_event_id++,
3257 - now,
3258 - rc->name,
3259 - rc->rrdset->id,
3260 - rc->rrdset->family,
3261 - rc->exec,
3262 - rc->recipient,
3263 - now - rc->last_status_change,
3264 - rc->old_value,
3265 - rc->value,
3266 - rc->status,
3267 - status,
3268 - rc->source,
3269 - rc->units,
3270 - rc->info,
3271 - rc->delay_last,
3272 - (rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)?HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION:0
3273 - );
3274 - rc->last_status_change = now;
3275 - rc->status = status;
3276 - }
1645 + rc->last_updated = now;
1646 + rc->next_update = now + rc->update_every;
1647
3278 - rc->last_updated = now;
3279 - rc->next_update = now + rc->update_every;
1648 + if(next_run > rc->next_update)
1649 + next_run = rc->next_update;
1650 + }
1651
3281 - if (next_run > rc->next_update)
3282 - next_run = rc->next_update;
1652 + rrdhost_unlock(host);
1653 }
1654
3285 - rrdhost_unlock(localhost);
3286 - }
1655 + if(unlikely(netdata_exit))
1656 + break;
1657
3288 - if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
3289 - error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1658 + // execute notifications
1659 + // and cleanup
1660 + health_alarm_log_process(host);
1661
3291 - if(unlikely(netdata_exit))
3292 - break;
1662 + if(unlikely(netdata_exit))
1663 + break;
1664
3294 - // execute notifications
3295 - // and cleanup
3296 - health_alarm_log_process(localhost);
1665 + } /* host loop */
1666 +
1667 + if(unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
1668 + error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1669
1670 if(unlikely(netdata_exit))
1671 break;
3300 -
1672 +
1673 now = now_realtime_sec();
1674 if(now < next_run) {
3303 - debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
3304 - loop, (int) (next_run - now));
1675 + debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
1676 sleep_usec(USEC_PER_SEC * (usec_t) (next_run - now));
1677 }
3307 - else {
1678 + else
1679 debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
3309 - }
1680 }
1681
1682 buffer_free(wb);
src/health.h
+43
@@ -369,4 +369,47 @@ extern void rrdvar_custom_host_variable_set(RRDVAR *rv, calculated_number value)
369
370 extern const char *rrdcalc_status2string(int status);
371
372 +
373 +extern int health_alarm_log_open(RRDHOST *host);
374 +extern void health_alarm_log_close(RRDHOST *host);
375 +extern void health_log_rotate(RRDHOST *host);
376 +extern void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
377 +extern ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename);
378 +extern void health_alarm_log_load(RRDHOST *host);
379 +extern void health_alarm_log(
380 + RRDHOST *host,
381 + uint32_t alarm_id,
382 + uint32_t alarm_event_id,
383 + time_t when,
384 + const char *name,
385 + const char *chart,
386 + const char *family,
387 + const char *exec,
388 + const char *recipient,
389 + time_t duration,
390 + calculated_number old_value,
391 + calculated_number new_value,
392 + int old_status,
393 + int new_status,
394 + const char *source,
395 + const char *units,
396 + const char *info,
397 + int delay,
398 + uint32_t flags
399 +);
400 +
401 +extern void health_readdir(RRDHOST *host, const char *path);
402 +extern char *health_config_dir(void);
403 +extern void health_free_host_nolock(RRDHOST *host);
404 +extern void health_reload_host(RRDHOST *host);
405 +
406 +#ifdef NETDATA_HEALTH_INTERNALS
407 +extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name);
408 +extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id);
409 +extern void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc);
410 +extern void rrdcalc_free(RRDHOST *host, RRDCALC *rc);
411 +extern void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt);
412 +extern int rrdvar_fix_name(char *variable);
413 +#endif
414 +
415 #endif //NETDATA_HEALTH_H
src/health_config.c new
+875
@@ -0,0 +1,875 @@
1 +#define NETDATA_HEALTH_INTERNALS
2 +#include "common.h"
3 +
4 +#define HEALTH_CONF_MAX_LINE 4096
5 +
6 +#define HEALTH_ALARM_KEY "alarm"
7 +#define HEALTH_TEMPLATE_KEY "template"
8 +#define HEALTH_ON_KEY "on"
9 +#define HEALTH_FAMILIES_KEY "families"
10 +#define HEALTH_LOOKUP_KEY "lookup"
11 +#define HEALTH_CALC_KEY "calc"
12 +#define HEALTH_EVERY_KEY "every"
13 +#define HEALTH_GREEN_KEY "green"
14 +#define HEALTH_RED_KEY "red"
15 +#define HEALTH_WARN_KEY "warn"
16 +#define HEALTH_CRIT_KEY "crit"
17 +#define HEALTH_EXEC_KEY "exec"
18 +#define HEALTH_RECIPIENT_KEY "to"
19 +#define HEALTH_UNITS_KEY "units"
20 +#define HEALTH_INFO_KEY "info"
21 +#define HEALTH_DELAY_KEY "delay"
22 +#define HEALTH_OPTIONS_KEY "options"
23 +
24 +static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
25 + if(!rc->chart) {
26 + error("Health configuration for alarm '%s' does not have a chart", rc->name);
27 + return 0;
28 + }
29 +
30 + if(!rc->update_every) {
31 + error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name);
32 + return 0;
33 + }
34 +
35 + if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
36 + error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
37 + return 0;
38 + }
39 +
40 + if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash))
41 + return 0;
42 +
43 + rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id);
44 +
45 + debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
46 + rc->chart?rc->chart:"NOCHART",
47 + rc->name,
48 + rc->id,
49 + (rc->exec)?rc->exec:"DEFAULT",
50 + (rc->recipient)?rc->recipient:"DEFAULT",
51 + rc->green,
52 + rc->red,
53 + rc->group,
54 + rc->after,
55 + rc->before,
56 + rc->options,
57 + (rc->dimensions)?rc->dimensions:"NONE",
58 + rc->update_every,
59 + (rc->calculation)?rc->calculation->parsed_as:"NONE",
60 + (rc->warning)?rc->warning->parsed_as:"NONE",
61 + (rc->critical)?rc->critical->parsed_as:"NONE",
62 + rc->source,
63 + rc->delay_up_duration,
64 + rc->delay_down_duration,
65 + rc->delay_max_duration,
66 + rc->delay_multiplier
67 + );
68 +
69 + rrdcalc_create_part2(host, rc);
70 + return 1;
71 +}
72 +
73 +static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
74 + if(unlikely(!rt->context)) {
75 + error("Health configuration for template '%s' does not have a context", rt->name);
76 + return 0;
77 + }
78 +
79 + if(unlikely(!rt->update_every)) {
80 + error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name);
81 + return 0;
82 + }
83 +
84 + if(unlikely(!RRDCALCTEMPLATE_HAS_CALCULATION(rt) && !rt->warning && !rt->critical)) {
85 + error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name);
86 + return 0;
87 + }
88 +
89 + RRDCALCTEMPLATE *t, *last = NULL;
90 + for (t = host->templates; t ; last = t, t = t->next) {
91 + if(unlikely(t->hash_name == rt->hash_name && !strcmp(t->name, rt->name))) {
92 + error("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname);
93 + return 0;
94 + }
95 + }
96 +
97 + debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
98 + rt->name,
99 + (rt->context)?rt->context:"NONE",
100 + (rt->exec)?rt->exec:"DEFAULT",
101 + (rt->recipient)?rt->recipient:"DEFAULT",
102 + rt->green,
103 + rt->red,
104 + rt->group,
105 + rt->after,
106 + rt->before,
107 + rt->options,
108 + (rt->dimensions)?rt->dimensions:"NONE",
109 + rt->update_every,
110 + (rt->calculation)?rt->calculation->parsed_as:"NONE",
111 + (rt->warning)?rt->warning->parsed_as:"NONE",
112 + (rt->critical)?rt->critical->parsed_as:"NONE",
113 + rt->source,
114 + rt->delay_up_duration,
115 + rt->delay_down_duration,
116 + rt->delay_max_duration,
117 + rt->delay_multiplier
118 + );
119 +
120 + if(likely(last)) {
121 + last->next = rt;
122 + }
123 + else {
124 + rt->next = host->templates;
125 + host->templates = rt;
126 + }
127 +
128 + return 1;
129 +}
130 +
131 +static inline int health_parse_duration(char *string, int *result) {
132 + // make sure it is a number
133 + if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
134 + *result = 0;
135 + return 0;
136 + }
137 +
138 + char *e = NULL;
139 + calculated_number n = strtold(string, &e);
140 + if(e && *e) {
141 + switch (*e) {
142 + case 'Y':
143 + *result = (int) (n * 86400 * 365);
144 + break;
145 + case 'M':
146 + *result = (int) (n * 86400 * 30);
147 + break;
148 + case 'w':
149 + *result = (int) (n * 86400 * 7);
150 + break;
151 + case 'd':
152 + *result = (int) (n * 86400);
153 + break;
154 + case 'h':
155 + *result = (int) (n * 3600);
156 + break;
157 + case 'm':
158 + *result = (int) (n * 60);
159 + break;
160 +
161 + default:
162 + case 's':
163 + *result = (int) (n);
164 + break;
165 + }
166 + }
167 + else
168 + *result = (int)(n);
169 +
170 + return 1;
171 +}
172 +
173 +static inline int health_parse_delay(
174 + size_t line, const char *path, const char *file, char *string,
175 + int *delay_up_duration,
176 + int *delay_down_duration,
177 + int *delay_max_duration,
178 + float *delay_multiplier) {
179 +
180 + char given_up = 0;
181 + char given_down = 0;
182 + char given_max = 0;
183 + char given_multiplier = 0;
184 +
185 + char *s = string;
186 + while(*s) {
187 + char *key = s;
188 +
189 + while(*s && !isspace(*s)) s++;
190 + while(*s && isspace(*s)) *s++ = '\0';
191 +
192 + if(!*key) break;
193 +
194 + char *value = s;
195 + while(*s && !isspace(*s)) s++;
196 + while(*s && isspace(*s)) *s++ = '\0';
197 +
198 + if(!strcasecmp(key, "up")) {
199 + if (!health_parse_duration(value, delay_up_duration)) {
200 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
201 + line, path, file, value, key);
202 + }
203 + else given_up = 1;
204 + }
205 + else if(!strcasecmp(key, "down")) {
206 + if (!health_parse_duration(value, delay_down_duration)) {
207 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
208 + line, path, file, value, key);
209 + }
210 + else given_down = 1;
211 + }
212 + else if(!strcasecmp(key, "multiplier")) {
213 + *delay_multiplier = strtof(value, NULL);
214 + if(isnan(*delay_multiplier) || isinf(*delay_multiplier) || islessequal(*delay_multiplier, 0)) {
215 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
216 + line, path, file, value, key);
217 + }
218 + else given_multiplier = 1;
219 + }
220 + else if(!strcasecmp(key, "max")) {
221 + if (!health_parse_duration(value, delay_max_duration)) {
222 + error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
223 + line, path, file, value, key);
224 + }
225 + else given_max = 1;
226 + }
227 + else {
228 + error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
229 + line, path, file, key);
230 + }
231 + }
232 +
233 + if(!given_up)
234 + *delay_up_duration = 0;
235 +
236 + if(!given_down)
237 + *delay_down_duration = 0;
238 +
239 + if(!given_multiplier)
240 + *delay_multiplier = 1.0;
241 +
242 + if(!given_max) {
243 + if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier))
244 + *delay_max_duration = (*delay_up_duration) * (*delay_multiplier);
245 +
246 + if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier))
247 + *delay_max_duration = (*delay_down_duration) * (*delay_multiplier);
248 + }
249 +
250 + return 1;
251 +}
252 +
253 +static inline uint32_t health_parse_options(const char *s) {
254 + uint32_t options = 0;
255 + char buf[100+1] = "";
256 +
257 + while(*s) {
258 + buf[0] = '\0';
259 +
260 + // skip spaces
261 + while(*s && isspace(*s))
262 + s++;
263 +
264 + // find the next space
265 + size_t count = 0;
266 + while(*s && count < 100 && !isspace(*s))
267 + buf[count++] = *s++;
268 +
269 + if(buf[0]) {
270 + buf[count] = '\0';
271 +
272 + if(!strcasecmp(buf, "no-clear-notification") || !strcasecmp(buf, "no-clear"))
273 + options |= RRDCALC_FLAG_NO_CLEAR_NOTIFICATION;
274 + else
275 + error("Ignoring unknown alarm option '%s'", buf);
276 + }
277 + }
278 +
279 + return options;
280 +}
281 +
282 +static inline int health_parse_db_lookup(
283 + size_t line, const char *path, const char *file, char *string,
284 + int *group_method, int *after, int *before, int *every,
285 + uint32_t *options, char **dimensions
286 +) {
287 + debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
288 +
289 + if(*dimensions) freez(*dimensions);
290 + *dimensions = NULL;
291 + *after = 0;
292 + *before = 0;
293 + *every = 0;
294 + *options = 0;
295 +
296 + char *s = string, *key;
297 +
298 + // first is the group method
299 + key = s;
300 + while(*s && !isspace(*s)) s++;
301 + while(*s && isspace(*s)) *s++ = '\0';
302 + if(!*s) {
303 + error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
304 + line, path, file, key);
305 + return 0;
306 + }
307 +
308 + if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
309 + error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
310 + line, path, file, key);
311 + return 0;
312 + }
313 +
314 + // then is the 'after' time
315 + key = s;
316 + while(*s && !isspace(*s)) s++;
317 + while(*s && isspace(*s)) *s++ = '\0';
318 +
319 + if(!health_parse_duration(key, after)) {
320 + error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
321 + line, path, file, key);
322 + return 0;
323 + }
324 +
325 + // sane defaults
326 + *every = abs(*after);
327 +
328 + // now we may have optional parameters
329 + while(*s) {
330 + key = s;
331 + while(*s && !isspace(*s)) s++;
332 + while(*s && isspace(*s)) *s++ = '\0';
333 + if(!*key) break;
334 +
335 + if(!strcasecmp(key, "at")) {
336 + char *value = s;
337 + while(*s && !isspace(*s)) s++;
338 + while(*s && isspace(*s)) *s++ = '\0';
339 +
340 + if (!health_parse_duration(value, before)) {
341 + error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
342 + line, path, file, value, key);
343 + }
344 + }
345 + else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
346 + char *value = s;
347 + while(*s && !isspace(*s)) s++;
348 + while(*s && isspace(*s)) *s++ = '\0';
349 +
350 + if (!health_parse_duration(value, every)) {
351 + error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
352 + line, path, file, value, key);
353 + }
354 + }
355 + else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
356 + *options |= RRDR_OPTION_ABSOLUTE;
357 + }
358 + else if(!strcasecmp(key, "min2max")) {
359 + *options |= RRDR_OPTION_MIN2MAX;
360 + }
361 + else if(!strcasecmp(key, "null2zero")) {
362 + *options |= RRDR_OPTION_NULL2ZERO;
363 + }
364 + else if(!strcasecmp(key, "percentage")) {
365 + *options |= RRDR_OPTION_PERCENTAGE;
366 + }
367 + else if(!strcasecmp(key, "unaligned")) {
368 + *options |= RRDR_OPTION_NOT_ALIGNED;
369 + }
370 + else if(!strcasecmp(key, "of")) {
371 + if(*s && strcasecmp(s, "all"))
372 + *dimensions = strdupz(s);
373 + break;
374 + }
375 + else {
376 + error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
377 + line, path, file, key);
378 + }
379 + }
380 +
381 + return 1;
382 +}
383 +
384 +static inline char *trim_all_spaces(char *buffer) {
385 + char *d = buffer, *s = buffer;
386 +
387 + // skip spaces
388 + while(isspace(*s)) s++;
389 +
390 + while(*s) {
391 + // copy the non-space part
392 + while(*s && !isspace(*s)) *d++ = *s++;
393 +
394 + // add a space if we have to
395 + if(*s && isspace(*s)) {
396 + *d++ = ' ';
397 + s++;
398 + }
399 +
400 + // skip spaces
401 + while(isspace(*s)) s++;
402 + }
403 +
404 + *d = '\0';
405 +
406 + if(d > buffer) {
407 + d--;
408 + if(isspace(*d)) *d = '\0';
409 + }
410 +
411 + if(!buffer[0]) return NULL;
412 + return buffer;
413 +}
414 +
415 +static inline char *health_source_file(size_t line, const char *path, const char *filename) {
416 + char buffer[FILENAME_MAX + 1];
417 + snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
418 + return strdupz(buffer);
419 +}
420 +
421 +static inline void strip_quotes(char *s) {
422 + while(*s) {
423 + if(*s == '\'' || *s == '"') *s = ' ';
424 + s++;
425 + }
426 +}
427 +
428 +int health_readfile(RRDHOST *host, const char *path, const char *filename) {
429 + debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
430 +
431 + static uint32_t
432 + hash_alarm = 0,
433 + hash_template = 0,
434 + hash_on = 0,
435 + hash_families = 0,
436 + hash_calc = 0,
437 + hash_green = 0,
438 + hash_red = 0,
439 + hash_warn = 0,
440 + hash_crit = 0,
441 + hash_exec = 0,
442 + hash_every = 0,
443 + hash_lookup = 0,
444 + hash_units = 0,
445 + hash_info = 0,
446 + hash_recipient = 0,
447 + hash_delay = 0,
448 + hash_options = 0;
449 +
450 + char buffer[HEALTH_CONF_MAX_LINE + 1];
451 +
452 + if(unlikely(!hash_alarm)) {
453 + hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
454 + hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
455 + hash_on = simple_uhash(HEALTH_ON_KEY);
456 + hash_families = simple_uhash(HEALTH_FAMILIES_KEY);
457 + hash_calc = simple_uhash(HEALTH_CALC_KEY);
458 + hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
459 + hash_green = simple_uhash(HEALTH_GREEN_KEY);
460 + hash_red = simple_uhash(HEALTH_RED_KEY);
461 + hash_warn = simple_uhash(HEALTH_WARN_KEY);
462 + hash_crit = simple_uhash(HEALTH_CRIT_KEY);
463 + hash_exec = simple_uhash(HEALTH_EXEC_KEY);
464 + hash_every = simple_uhash(HEALTH_EVERY_KEY);
465 + hash_units = simple_hash(HEALTH_UNITS_KEY);
466 + hash_info = simple_hash(HEALTH_INFO_KEY);
467 + hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
468 + hash_delay = simple_uhash(HEALTH_DELAY_KEY);
469 + hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
470 + }
471 +
472 + snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
473 + FILE *fp = fopen(buffer, "r");
474 + if(!fp) {
475 + error("Health configuration cannot read file '%s'.", buffer);
476 + return 0;
477 + }
478 +
479 + RRDCALC *rc = NULL;
480 + RRDCALCTEMPLATE *rt = NULL;
481 +
482 + size_t line = 0, append = 0;
483 + char *s;
484 + while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
485 + int stop_appending = !s;
486 + line++;
487 + s = trim(buffer);
488 + if(!s) continue;
489 +
490 + append = strlen(s);
491 + if(!stop_appending && s[append - 1] == '\\') {
492 + s[append - 1] = ' ';
493 + append = &s[append] - buffer;
494 + if(append < HEALTH_CONF_MAX_LINE)
495 + continue;
496 + else {
497 + error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
498 + }
499 + }
500 + append = 0;
501 +
502 + char *key = s;
503 + while(*s && *s != ':') s++;
504 + if(!*s) {
505 + error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
506 + continue;
507 + }
508 + *s = '\0';
509 + s++;
510 +
511 + char *value = s;
512 + key = trim_all_spaces(key);
513 + value = trim_all_spaces(value);
514 +
515 + if(!key) {
516 + error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
517 + continue;
518 + }
519 +
520 + if(!value) {
521 + error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
522 + continue;
523 + }
524 +
525 + uint32_t hash = simple_uhash(key);
526 +
527 + if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
528 + if(rc && !rrdcalc_add_alarm_from_config(host, rc))
529 + rrdcalc_free(host, rc);
530 +
531 + if(rt) {
532 + if (!rrdcalctemplate_add_template_from_config(host, rt))
533 + rrdcalctemplate_free(host, rt);
534 + rt = NULL;
535 + }
536 +
537 + rc = callocz(1, sizeof(RRDCALC));
538 + rc->next_event_id = 1;
539 + rc->name = strdupz(value);
540 + rc->hash = simple_hash(rc->name);
541 + rc->source = health_source_file(line, path, filename);
542 + rc->green = NAN;
543 + rc->red = NAN;
544 + rc->value = NAN;
545 + rc->old_value = NAN;
546 + rc->delay_multiplier = 1.0;
547 +
548 + if(rrdvar_fix_name(rc->name))
549 + error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
550 + }
551 + else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
552 + if(rc) {
553 + if(!rrdcalc_add_alarm_from_config(host, rc))
554 + rrdcalc_free(host, rc);
555 + rc = NULL;
556 + }
557 +
558 + if(rt && !rrdcalctemplate_add_template_from_config(host, rt))
559 + rrdcalctemplate_free(host, rt);
560 +
561 + rt = callocz(1, sizeof(RRDCALCTEMPLATE));
562 + rt->name = strdupz(value);
563 + rt->hash_name = simple_hash(rt->name);
564 + rt->source = health_source_file(line, path, filename);
565 + rt->green = NAN;
566 + rt->red = NAN;
567 + rt->delay_multiplier = 1.0;
568 +
569 + if(rrdvar_fix_name(rt->name))
570 + error("Health configuration renamed template '%s' to '%s'", value, rt->name);
571 + }
572 + else if(rc) {
573 + if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
574 + if(rc->chart) {
575 + if(strcmp(rc->chart, value))
576 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
577 + line, path, filename, rc->name, key, rc->chart, value, value);
578 +
579 + freez(rc->chart);
580 + }
581 + rc->chart = strdupz(value);
582 + rc->hash_chart = simple_hash(rc->chart);
583 + }
584 + else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
585 + health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
586 + &rc->update_every,
587 + &rc->options, &rc->dimensions);
588 + }
589 + else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
590 + if(!health_parse_duration(value, &rc->update_every))
591 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
592 + line, path, filename, rc->name, key, value);
593 + }
594 + else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
595 + char *e;
596 + rc->green = strtold(value, &e);
597 + if(e && *e) {
598 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
599 + line, path, filename, rc->name, key, e);
600 + }
601 + }
602 + else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
603 + char *e;
604 + rc->red = strtold(value, &e);
605 + if(e && *e) {
606 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
607 + line, path, filename, rc->name, key, e);
608 + }
609 + }
610 + else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
611 + const char *failed_at = NULL;
612 + int error = 0;
613 + rc->calculation = expression_parse(value, &failed_at, &error);
614 + if(!rc->calculation) {
615 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
616 + line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
617 + }
618 + }
619 + else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
620 + const char *failed_at = NULL;
621 + int error = 0;
622 + rc->warning = expression_parse(value, &failed_at, &error);
623 + if(!rc->warning) {
624 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
625 + line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
626 + }
627 + }
628 + else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
629 + const char *failed_at = NULL;
630 + int error = 0;
631 + rc->critical = expression_parse(value, &failed_at, &error);
632 + if(!rc->critical) {
633 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
634 + line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
635 + }
636 + }
637 + else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
638 + if(rc->exec) {
639 + if(strcmp(rc->exec, value))
640 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
641 + line, path, filename, rc->name, key, rc->exec, value, value);
642 +
643 + freez(rc->exec);
644 + }
645 + rc->exec = strdupz(value);
646 + }
647 + else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
648 + if(rc->recipient) {
649 + if(strcmp(rc->recipient, value))
650 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
651 + line, path, filename, rc->name, key, rc->recipient, value, value);
652 +
653 + freez(rc->recipient);
654 + }
655 + rc->recipient = strdupz(value);
656 + }
657 + else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
658 + if(rc->units) {
659 + if(strcmp(rc->units, value))
660 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
661 + line, path, filename, rc->name, key, rc->units, value, value);
662 +
663 + freez(rc->units);
664 + }
665 + rc->units = strdupz(value);
666 + strip_quotes(rc->units);
667 + }
668 + else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
669 + if(rc->info) {
670 + if(strcmp(rc->info, value))
671 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
672 + line, path, filename, rc->name, key, rc->info, value, value);
673 +
674 + freez(rc->info);
675 + }
676 + rc->info = strdupz(value);
677 + strip_quotes(rc->info);
678 + }
679 + else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
680 + health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
681 + }
682 + else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
683 + rc->options |= health_parse_options(value);
684 + }
685 + else {
686 + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
687 + line, path, filename, rc->name, key);
688 + }
689 + }
690 + else if(rt) {
691 + if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
692 + if(rt->context) {
693 + if(strcmp(rt->context, value))
694 + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
695 + line, path, filename, rt->name, key, rt->context, value, value);
696 +
697 + freez(rt->context);
698 + }
699 + rt->context = strdupz(value);
700 + rt->hash_context = simple_hash(rt->context);
701 + }
702 + else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
703 + freez(rt->family_match);
704 + simple_pattern_free(rt->family_pattern);
705 +
706 + rt->family_match = strdupz(value);
707 + rt->family_pattern = simple_pattern_create(rt->family_match, SIMPLE_PATTERN_EXACT);
708 + }
709 + else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
710 + health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
711 + &rt->update_every, &rt->options, &rt->dimensions);
712 + }
713 + else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
714 + if(!health_parse_duration(value, &rt->update_every))
715 + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
716 + line, path, filename, rt->name, key, value);
717 + }
718 + else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
719 + char *e;
720 + rt->green = strtold(value, &e);
721 + if(e && *e) {
722 + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
723 + line, path, filename, rt->name, key, e);
724 + }
725 + }
726 + else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
727 + char *e;
728 + rt->red = strtold(value, &e);
729 + if(e && *e) {
730 + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
731 + line, path, filename, rt->name, key, e);
732 + }
733 + }
734 + else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
735 + const char *failed_at = NULL;
736 + int error = 0;
737 + rt->calculation = expression_parse(value, &failed_at, &error);
738 + if(!rt->calculation) {
739 + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
740 + line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
741 + }
742 + }
743 + else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
744 + const char *failed_at = NULL;
745 + int error = 0;
746 + rt->warning = expression_parse(value, &failed_at, &error);
747 + if(!rt->warning) {
748 + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
749 + line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
750 + }
751 + }
752 + else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
753 + const char *failed_at = NULL;
754 + int error = 0;
755 + rt->critical = expression_parse(value, &failed_at, &error);
756 + if(!rt->critical) {
757 + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
758 + line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
759 + }
760 + }
761 + else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
762 + if(rt->exec) {
763 + if(strcmp(rt->exec, value))
764 + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
765 + line, path, filename, rt->name, key, rt->exec, value, value);
766 +
767 + freez(rt->exec);
768 + }
769 + rt->exec = strdupz(value);
770 + }
771 + else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
772 + if(rt->recipient) {
773 + if(strcmp(rt->recipient, value))
774 + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
775 + line, path, filename, rt->name, key, rt->recipient, value, value);
776 +
777 + freez(rt->recipient);
778 + }
779 + rt->recipient = strdupz(value);
780 + }
781 + else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
782 + if(rt->units) {
783 + if(strcmp(rt->units, value))
784 + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
785 + line, path, filename, rt->name, key, rt->units, value, value);
786 +
787 + freez(rt->units);
788 + }
789 + rt->units = strdupz(value);
790 + strip_quotes(rt->units);
791 + }
792 + else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
793 + if(rt->info) {
794 + if(strcmp(rt->info, value))
795 + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
796 + line, path, filename, rt->name, key, rt->info, value, value);
797 +
798 + freez(rt->info);
799 + }
800 + rt->info = strdupz(value);
801 + strip_quotes(rt->info);
802 + }
803 + else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
804 + health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
805 + }
806 + else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
807 + rt->options |= health_parse_options(value);
808 + }
809 + else {
810 + error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
811 + line, path, filename, rt->name, key);
812 + }
813 + }
814 + else {
815 + error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
816 + line, path, filename, key);
817 + }
818 + }
819 +
820 + if(rc && !rrdcalc_add_alarm_from_config(host, rc))
821 + rrdcalc_free(host, rc);
822 +
823 + if(rt && !rrdcalctemplate_add_template_from_config(host, rt))
824 + rrdcalctemplate_free(host, rt);
825 +
826 + fclose(fp);
827 + return 1;
828 +}
829 +
830 +void health_readdir(RRDHOST *host, const char *path) {
831 + size_t pathlen = strlen(path);
832 +
833 + debug(D_HEALTH, "Health configuration reading directory '%s'", path);
834 +
835 + DIR *dir = opendir(path);
836 + if (!dir) {
837 + error("Health configuration cannot open directory '%s'.", path);
838 + return;
839 + }
840 +
841 + struct dirent *de = NULL;
842 + while ((de = readdir(dir))) {
843 + size_t len = strlen(de->d_name);
844 +
845 + if(de->d_type == DT_DIR
846 + && (
847 + (de->d_name[0] == '.' && de->d_name[1] == '\0')
848 + || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
849 + )) {
850 + debug(D_HEALTH, "Ignoring directory '%s'", de->d_name);
851 + continue;
852 + }
853 +
854 + else if(de->d_type == DT_DIR) {
855 + char *s = mallocz(pathlen + strlen(de->d_name) + 2);
856 + strcpy(s, path);
857 + strcat(s, "/");
858 + strcat(s, de->d_name);
859 + health_readdir(host, s);
860 + freez(s);
861 + continue;
862 + }
863 +
864 + else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
865 + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
866 + health_readfile(host, path, de->d_name);
867 + }
868 +
869 + else debug(D_HEALTH, "Ignoring file '%s'", de->d_name);
870 + }
871 +
872 + closedir(dir);
873 +}
874 +
875 +
src/health_json.c new
+256
@@ -0,0 +1,256 @@
1 +#define NETDATA_HEALTH_INTERNALS
2 +#include "common.h"
3 +
4 +static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
5 + if(value && *value)
6 + buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
7 + else
8 + buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
9 +}
10 +
11 +static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
12 + buffer_sprintf(wb,
13 + "\n\t{\n"
14 + "\t\t\"hostname\": \"%s\",\n"
15 + "\t\t\"unique_id\": %u,\n"
16 + "\t\t\"alarm_id\": %u,\n"
17 + "\t\t\"alarm_event_id\": %u,\n"
18 + "\t\t\"name\": \"%s\",\n"
19 + "\t\t\"chart\": \"%s\",\n"
20 + "\t\t\"family\": \"%s\",\n"
21 + "\t\t\"processed\": %s,\n"
22 + "\t\t\"updated\": %s,\n"
23 + "\t\t\"exec_run\": %lu,\n"
24 + "\t\t\"exec_failed\": %s,\n"
25 + "\t\t\"exec\": \"%s\",\n"
26 + "\t\t\"recipient\": \"%s\",\n"
27 + "\t\t\"exec_code\": %d,\n"
28 + "\t\t\"source\": \"%s\",\n"
29 + "\t\t\"units\": \"%s\",\n"
30 + "\t\t\"info\": \"%s\",\n"
31 + "\t\t\"when\": %lu,\n"
32 + "\t\t\"duration\": %lu,\n"
33 + "\t\t\"non_clear_duration\": %lu,\n"
34 + "\t\t\"status\": \"%s\",\n"
35 + "\t\t\"old_status\": \"%s\",\n"
36 + "\t\t\"delay\": %d,\n"
37 + "\t\t\"delay_up_to_timestamp\": %lu,\n"
38 + "\t\t\"updated_by_id\": %u,\n"
39 + "\t\t\"updates_id\": %u,\n"
40 + "\t\t\"value_string\": \"%s\",\n"
41 + "\t\t\"old_value_string\": \"%s\",\n"
42 + , host->hostname
43 + , ae->unique_id
44 + , ae->alarm_id
45 + , ae->alarm_event_id
46 + , ae->name
47 + , ae->chart
48 + , ae->family
49 + , (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false"
50 + , (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"
51 + , (unsigned long)ae->exec_run_timestamp
52 + , (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false"
53 + , ae->exec?ae->exec:host->health_default_exec
54 + , ae->recipient?ae->recipient:host->health_default_recipient
55 + , ae->exec_code
56 + , ae->source
57 + , ae->units?ae->units:""
58 + , ae->info?ae->info:""
59 + , (unsigned long)ae->when
60 + , (unsigned long)ae->duration
61 + , (unsigned long)ae->non_clear_duration
62 + , rrdcalc_status2string(ae->new_status)
63 + , rrdcalc_status2string(ae->old_status)
64 + , ae->delay
65 + , (unsigned long)ae->delay_up_to_timestamp
66 + , ae->updated_by_id
67 + , ae->updates_id
68 + , ae->new_value_string
69 + , ae->old_value_string
70 + );
71 +
72 + if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
73 + buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");
74 + }
75 +
76 + buffer_strcat(wb, "\t\t\"value\":");
77 + buffer_rrd_value(wb, ae->new_value);
78 + buffer_strcat(wb, ",\n");
79 +
80 + buffer_strcat(wb, "\t\t\"old_value\":");
81 + buffer_rrd_value(wb, ae->old_value);
82 + buffer_strcat(wb, "\n");
83 +
84 + buffer_strcat(wb, "\t}");
85 +}
86 +
87 +void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after) {
88 + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
89 +
90 + buffer_strcat(wb, "[");
91 +
92 + unsigned int max = host->health_log.max;
93 + unsigned int count = 0;
94 + ALARM_ENTRY *ae;
95 + for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
96 + if(ae->unique_id > after) {
97 + if(likely(count)) buffer_strcat(wb, ",");
98 + health_alarm_entry2json_nolock(wb, ae, host);
99 + }
100 + }
101 +
102 + buffer_strcat(wb, "\n]\n");
103 +
104 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
105 +}
106 +
107 +static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC *rc) {
108 + char value_string[100 + 1];
109 + format_value_and_unit(value_string, 100, rc->value, rc->units, -1);
110 +
111 + buffer_sprintf(wb,
112 + "\t\t\"%s.%s\": {\n"
113 + "\t\t\t\"id\": %lu,\n"
114 + "\t\t\t\"name\": \"%s\",\n"
115 + "\t\t\t\"chart\": \"%s\",\n"
116 + "\t\t\t\"family\": \"%s\",\n"
117 + "\t\t\t\"active\": %s,\n"
118 + "\t\t\t\"exec\": \"%s\",\n"
119 + "\t\t\t\"recipient\": \"%s\",\n"
120 + "\t\t\t\"source\": \"%s\",\n"
121 + "\t\t\t\"units\": \"%s\",\n"
122 + "\t\t\t\"info\": \"%s\",\n"
123 + "\t\t\t\"status\": \"%s\",\n"
124 + "\t\t\t\"last_status_change\": %lu,\n"
125 + "\t\t\t\"last_updated\": %lu,\n"
126 + "\t\t\t\"next_update\": %lu,\n"
127 + "\t\t\t\"update_every\": %d,\n"
128 + "\t\t\t\"delay_up_duration\": %d,\n"
129 + "\t\t\t\"delay_down_duration\": %d,\n"
130 + "\t\t\t\"delay_max_duration\": %d,\n"
131 + "\t\t\t\"delay_multiplier\": %f,\n"
132 + "\t\t\t\"delay\": %d,\n"
133 + "\t\t\t\"delay_up_to_timestamp\": %lu,\n"
134 + "\t\t\t\"value_string\": \"%s\",\n"
135 + , rc->chart, rc->name
136 + , (unsigned long)rc->id
137 + , rc->name
138 + , rc->chart
139 + , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
140 + , (rc->rrdset)?"true":"false"
141 + , rc->exec?rc->exec:host->health_default_exec
142 + , rc->recipient?rc->recipient:host->health_default_recipient
143 + , rc->source
144 + , rc->units?rc->units:""
145 + , rc->info?rc->info:""
146 + , rrdcalc_status2string(rc->status)
147 + , (unsigned long)rc->last_status_change
148 + , (unsigned long)rc->last_updated
149 + , (unsigned long)rc->next_update
150 + , rc->update_every
151 + , rc->delay_up_duration
152 + , rc->delay_down_duration
153 + , rc->delay_max_duration
154 + , rc->delay_multiplier
155 + , rc->delay_last
156 + , (unsigned long)rc->delay_up_to_timestamp
157 + , value_string
158 + );
159 +
160 + if(unlikely(rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)) {
161 + buffer_strcat(wb, "\t\t\t\"no_clear_notification\": true,\n");
162 + }
163 +
164 + if(RRDCALC_HAS_DB_LOOKUP(rc)) {
165 + if(rc->dimensions && *rc->dimensions)
166 + health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
167 +
168 + buffer_sprintf(wb,
169 + "\t\t\t\"db_after\": %lu,\n"
170 + "\t\t\t\"db_before\": %lu,\n"
171 + "\t\t\t\"lookup_method\": \"%s\",\n"
172 + "\t\t\t\"lookup_after\": %d,\n"
173 + "\t\t\t\"lookup_before\": %d,\n"
174 + "\t\t\t\"lookup_options\": \"",
175 + (unsigned long) rc->db_after,
176 + (unsigned long) rc->db_before,
177 + group_method2string(rc->group),
178 + rc->after,
179 + rc->before
180 + );
181 + buffer_data_options2string(wb, rc->options);
182 + buffer_strcat(wb, "\",\n");
183 + }
184 +
185 + if(rc->calculation) {
186 + health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
187 + health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
188 + }
189 +
190 + if(rc->warning) {
191 + health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
192 + health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
193 + }
194 +
195 + if(rc->critical) {
196 + health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
197 + health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
198 + }
199 +
200 + buffer_strcat(wb, "\t\t\t\"green\":");
201 + buffer_rrd_value(wb, rc->green);
202 + buffer_strcat(wb, ",\n");
203 +
204 + buffer_strcat(wb, "\t\t\t\"red\":");
205 + buffer_rrd_value(wb, rc->red);
206 + buffer_strcat(wb, ",\n");
207 +
208 + buffer_strcat(wb, "\t\t\t\"value\":");
209 + buffer_rrd_value(wb, rc->value);
210 + buffer_strcat(wb, "\n");
211 +
212 + buffer_strcat(wb, "\t\t}");
213 +}
214 +
215 +//void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
216 +//
217 +//}
218 +
219 +void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
220 + int i;
221 +
222 + rrdhost_rdlock(localhost);
223 + buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
224 + "\n\t\"latest_alarm_log_unique_id\": %u,"
225 + "\n\t\"status\": %s,"
226 + "\n\t\"now\": %lu,"
227 + "\n\t\"alarms\": {\n",
228 + host->hostname,
229 + (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
230 + health_enabled?"true":"false",
231 + (unsigned long)now_realtime_sec());
232 +
233 + RRDCALC *rc;
234 + for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
235 + if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
236 + continue;
237 +
238 + if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
239 + continue;
240 +
241 + if(likely(i)) buffer_strcat(wb, ",\n");
242 + health_rrdcalc2json_nolock(host, wb, rc);
243 + i++;
244 + }
245 +
246 +// buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
247 +// RRDCALCTEMPLATE *rt;
248 +// for(rt = host->templates; rt ; rt = rt->next)
249 +// health_rrdcalctemplate2json_nolock(wb, rt);
250 +
251 + buffer_strcat(wb, "\n\t}\n}\n");
252 + rrdhost_unlock(localhost);
253 +}
254 +
255 +
256 +
src/health_log.c new
+438
@@ -0,0 +1,438 @@
1 +#define NETDATA_HEALTH_INTERNALS
2 +#include "common.h"
3 +
4 +// ----------------------------------------------------------------------------
5 +// health alarm log load/save
6 +// no need for locking - only one thread is reading / writing the alarms log
7 +
8 +inline int health_alarm_log_open(RRDHOST *host) {
9 + if(host->health_log_fp)
10 + fclose(host->health_log_fp);
11 +
12 + host->health_log_fp = fopen(host->health_log_filename, "a");
13 +
14 + if(host->health_log_fp) {
15 + if (setvbuf(host->health_log_fp, NULL, _IOLBF, 0) != 0)
16 + error("Health: cannot set line buffering on health log file.");
17 + return 0;
18 + }
19 +
20 + error("Health: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", host->health_log_filename);
21 + return -1;
22 +}
23 +
24 +inline void health_alarm_log_close(RRDHOST *host) {
25 + if(host->health_log_fp) {
26 + fclose(host->health_log_fp);
27 + host->health_log_fp = NULL;
28 + }
29 +}
30 +
31 +inline void health_log_rotate(RRDHOST *host) {
32 + static size_t rotate_every = 0;
33 +
34 + if(unlikely(rotate_every == 0)) {
35 + rotate_every = (size_t)config_get_number("health", "rotate log every lines", 2000);
36 + if(rotate_every < 100) rotate_every = 100;
37 + }
38 +
39 + if(unlikely(host->health_log_entries_written > rotate_every)) {
40 + health_alarm_log_close(host);
41 +
42 + char old_filename[FILENAME_MAX + 1];
43 + snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename);
44 +
45 + if(unlink(old_filename) == -1 && errno != ENOENT)
46 + error("Health: cannot remove old alarms log file '%s'", old_filename);
47 +
48 + if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT)
49 + error("Health: cannot move file '%s' to '%s'.", host->health_log_filename, old_filename);
50 +
51 + if(unlink(host->health_log_filename) == -1 && errno != ENOENT)
52 + error("Health: cannot remove old alarms log file '%s'", host->health_log_filename);
53 +
54 + // open it with truncate
55 + host->health_log_fp = fopen(host->health_log_filename, "w");
56 +
57 + if(host->health_log_fp)
58 + fclose(host->health_log_fp);
59 + else
60 + error("Health: cannot truncate health log '%s'", host->health_log_filename);
61 +
62 + host->health_log_fp = NULL;
63 +
64 + host->health_log_entries_written = 0;
65 + health_alarm_log_open(host);
66 + }
67 +}
68 +
69 +inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
70 + health_log_rotate(host);
71 +
72 + if(likely(host->health_log_fp)) {
73 + if(unlikely(fprintf(host->health_log_fp
74 + , "%c\t%s"
75 + "\t%08x\t%08x\t%08x\t%08x\t%08x"
76 + "\t%08x\t%08x\t%08x"
77 + "\t%08x\t%08x\t%08x"
78 + "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
79 + "\t%d\t%d\t%d\t%d"
80 + "\t%Lf\t%Lf"
81 + "\n"
82 + , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
83 + , host->hostname
84 +
85 + , ae->unique_id
86 + , ae->alarm_id
87 + , ae->alarm_event_id
88 + , ae->updated_by_id
89 + , ae->updates_id
90 +
91 + , (uint32_t)ae->when
92 + , (uint32_t)ae->duration
93 + , (uint32_t)ae->non_clear_duration
94 + , (uint32_t)ae->flags
95 + , (uint32_t)ae->exec_run_timestamp
96 + , (uint32_t)ae->delay_up_to_timestamp
97 +
98 + , (ae->name)?ae->name:""
99 + , (ae->chart)?ae->chart:""
100 + , (ae->family)?ae->family:""
101 + , (ae->exec)?ae->exec:""
102 + , (ae->recipient)?ae->recipient:""
103 + , (ae->source)?ae->source:""
104 + , (ae->units)?ae->units:""
105 + , (ae->info)?ae->info:""
106 +
107 + , ae->exec_code
108 + , ae->new_status
109 + , ae->old_status
110 + , ae->delay
111 +
112 + , (long double)ae->new_value
113 + , (long double)ae->old_value
114 + ) < 0))
115 + error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
116 + else {
117 + ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
118 + host->health_log_entries_written++;
119 + }
120 + }
121 +}
122 +
123 +inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
124 + static uint32_t max_unique_id = 0, max_alarm_id = 0;
125 +
126 + errno = 0;
127 +
128 + char *s, *buf = mallocz(65536 + 1);
129 + size_t line = 0, len = 0;
130 + ssize_t loaded = 0, updated = 0, errored = 0, duplicate = 0;
131 +
132 + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
133 +
134 + while((s = fgets_trim_len(buf, 65536, fp, &len))) {
135 + host->health_log_entries_written++;
136 + line++;
137 +
138 + int max_entries = 30, entries = 0;
139 + char *pointers[max_entries];
140 +
141 + pointers[entries++] = s++;
142 + while(*s) {
143 + if(unlikely(*s == '\t')) {
144 + *s = '\0';
145 + pointers[entries++] = ++s;
146 + if(entries >= max_entries) {
147 + error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, filename, max_entries);
148 + break;
149 + }
150 + }
151 + else s++;
152 + }
153 +
154 + if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
155 + ALARM_ENTRY *ae = NULL;
156 +
157 + if(entries < 26) {
158 + error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring it.", line, filename, entries);
159 + errored++;
160 + continue;
161 + }
162 +
163 + // check that we have valid ids
164 + uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
165 + if(!unique_id) {
166 + error("Health: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", line, filename, unique_id, pointers[2]);
167 + errored++;
168 + continue;
169 + }
170 +
171 + uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
172 + if(!alarm_id) {
173 + error("Health: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", line, filename, alarm_id, pointers[3]);
174 + errored++;
175 + continue;
176 + }
177 +
178 + if(unlikely(*pointers[0] == 'A')) {
179 + // make sure it is properly numbered
180 + if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) {
181 + error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order. Ignoring it.", line, filename, unique_id);
182 + errored++;
183 + continue;
184 + }
185 +
186 + ae = callocz(1, sizeof(ALARM_ENTRY));
187 + }
188 + else if(unlikely(*pointers[0] == 'U')) {
189 + // find the original
190 + for(ae = host->health_log.alarms; ae; ae = ae->next) {
191 + if(unlikely(unique_id == ae->unique_id)) {
192 + if(unlikely(*pointers[0] == 'A')) {
193 + error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u. Using the later."
194 + , line, filename, unique_id);
195 + *pointers[0] = 'U';
196 + duplicate++;
197 + }
198 + break;
199 + }
200 + else if(unlikely(unique_id > ae->unique_id)) {
201 + // no need to continue
202 + // the linked list is sorted
203 + ae = NULL;
204 + break;
205 + }
206 + }
207 + }
208 +
209 + // if not found, skip this line
210 + if(unlikely(!ae)) {
211 + // error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, filename, unique_id);
212 + continue;
213 + }
214 +
215 + // check for a possible host missmatch
216 + //if(strcmp(pointers[1], host->hostname))
217 + // error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, filename, pointers[1], host->hostname);
218 +
219 + ae->unique_id = unique_id;
220 + ae->alarm_id = alarm_id;
221 + ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
222 + ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
223 + ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
224 +
225 + ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
226 + ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
227 + ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
228 +
229 + ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
230 + ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
231 +
232 + ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
233 + ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
234 +
235 + freez(ae->name);
236 + ae->name = strdupz(pointers[13]);
237 + ae->hash_name = simple_hash(ae->name);
238 +
239 + freez(ae->chart);
240 + ae->chart = strdupz(pointers[14]);
241 + ae->hash_chart = simple_hash(ae->chart);
242 +
243 + freez(ae->family);
244 + ae->family = strdupz(pointers[15]);
245 +
246 + freez(ae->exec);
247 + ae->exec = strdupz(pointers[16]);
248 + if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
249 +
250 + freez(ae->recipient);
251 + ae->recipient = strdupz(pointers[17]);
252 + if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
253 +
254 + freez(ae->source);
255 + ae->source = strdupz(pointers[18]);
256 + if(!*ae->source) { freez(ae->source); ae->source = NULL; }
257 +
258 + freez(ae->units);
259 + ae->units = strdupz(pointers[19]);
260 + if(!*ae->units) { freez(ae->units); ae->units = NULL; }
261 +
262 + freez(ae->info);
263 + ae->info = strdupz(pointers[20]);
264 + if(!*ae->info) { freez(ae->info); ae->info = NULL; }
265 +
266 + ae->exec_code = str2i(pointers[21]);
267 + ae->new_status = str2i(pointers[22]);
268 + ae->old_status = str2i(pointers[23]);
269 + ae->delay = str2i(pointers[24]);
270 +
271 + ae->new_value = str2l(pointers[25]);
272 + ae->old_value = str2l(pointers[26]);
273 +
274 + static char value_string[100 + 1];
275 + freez(ae->old_value_string);
276 + freez(ae->new_value_string);
277 + ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
278 + ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
279 +
280 + // add it to host if not already there
281 + if(unlikely(*pointers[0] == 'A')) {
282 + ae->next = host->health_log.alarms;
283 + host->health_log.alarms = ae;
284 + loaded++;
285 + }
286 + else updated++;
287 +
288 + if(unlikely(ae->unique_id > max_unique_id))
289 + max_unique_id = ae->unique_id;
290 +
291 + if(unlikely(ae->alarm_id >= max_alarm_id))
292 + max_alarm_id = ae->alarm_id;
293 + }
294 + else {
295 + error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, filename, pointers[0]);
296 + errored++;
297 + }
298 + }
299 +
300 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
301 +
302 + freez(buf);
303 +
304 + if(!max_unique_id) max_unique_id = (uint32_t)now_realtime_sec();
305 + if(!max_alarm_id) max_alarm_id = (uint32_t)now_realtime_sec();
306 +
307 + host->health_log.next_log_id = max_unique_id + 1;
308 + host->health_log.next_alarm_id = max_alarm_id + 1;
309 +
310 + debug(D_HEALTH, "Health: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", filename, loaded, updated, errored, duplicate);
311 + return loaded;
312 +}
313 +
314 +inline void health_alarm_log_load(RRDHOST *host) {
315 + health_alarm_log_close(host);
316 +
317 + char filename[FILENAME_MAX + 1];
318 + snprintfz(filename, FILENAME_MAX, "%s.old", host->health_log_filename);
319 + FILE *fp = fopen(filename, "r");
320 + if(!fp)
321 + error("Health: cannot open health file: %s", filename);
322 + else {
323 + health_alarm_log_read(host, fp, filename);
324 + fclose(fp);
325 + }
326 +
327 + host->health_log_entries_written = 0;
328 + fp = fopen(host->health_log_filename, "r");
329 + if(!fp)
330 + error("Health: cannot open health file: %s", host->health_log_filename);
331 + else {
332 + health_alarm_log_read(host, fp, host->health_log_filename);
333 + fclose(fp);
334 + }
335 +
336 + health_alarm_log_open(host);
337 +}
338 +
339 +
340 +// ----------------------------------------------------------------------------
341 +// health alarm log management
342 +
343 +inline void health_alarm_log(
344 + RRDHOST *host,
345 + uint32_t alarm_id,
346 + uint32_t alarm_event_id,
347 + time_t when,
348 + const char *name,
349 + const char *chart,
350 + const char *family,
351 + const char *exec,
352 + const char *recipient,
353 + time_t duration,
354 + calculated_number old_value,
355 + calculated_number new_value,
356 + int old_status,
357 + int new_status,
358 + const char *source,
359 + const char *units,
360 + const char *info,
361 + int delay,
362 + uint32_t flags
363 +) {
364 + debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
365 +
366 + ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
367 + ae->name = strdupz(name);
368 + ae->hash_name = simple_hash(ae->name);
369 +
370 + if(chart) {
371 + ae->chart = strdupz(chart);
372 + ae->hash_chart = simple_hash(ae->chart);
373 + }
374 +
375 + if(family)
376 + ae->family = strdupz(family);
377 +
378 + if(exec) ae->exec = strdupz(exec);
379 + if(recipient) ae->recipient = strdupz(recipient);
380 + if(source) ae->source = strdupz(source);
381 + if(units) ae->units = strdupz(units);
382 + if(info) ae->info = strdupz(info);
383 +
384 + ae->unique_id = host->health_log.next_log_id++;
385 + ae->alarm_id = alarm_id;
386 + ae->alarm_event_id = alarm_event_id;
387 + ae->when = when;
388 + ae->old_value = old_value;
389 + ae->new_value = new_value;
390 +
391 + static char value_string[100 + 1];
392 + ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
393 + ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
394 +
395 + ae->old_status = old_status;
396 + ae->new_status = new_status;
397 + ae->duration = duration;
398 + ae->delay = delay;
399 + ae->delay_up_to_timestamp = when + delay;
400 +
401 + ae->flags |= flags;
402 +
403 + if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
404 + ae->non_clear_duration += ae->duration;
405 +
406 + // link it
407 + pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
408 + ae->next = host->health_log.alarms;
409 + host->health_log.alarms = ae;
410 + host->health_log.count++;
411 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
412 +
413 + // match previous alarms
414 + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
415 + ALARM_ENTRY *t;
416 + for(t = host->health_log.alarms ; t ; t = t->next) {
417 + if(t != ae && t->alarm_id == ae->alarm_id) {
418 + if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) {
419 + t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
420 + t->updated_by_id = ae->unique_id;
421 + ae->updates_id = t->unique_id;
422 +
423 + if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
424 + (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
425 + ae->non_clear_duration += t->non_clear_duration;
426 +
427 + health_alarm_log_save(host, t);
428 + }
429 +
430 + // no need to continue
431 + break;
432 + }
433 + }
434 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
435 +
436 + health_alarm_log_save(host, ae);
437 +}
438 +
src/main.c
+5 -5
@@ -795,6 +795,11 @@ int main(int argc, char **argv) {
795 debug(D_SYSTEM, "Successfully set pthread stacksize to %zu bytes", wanted_stacksize);
796 }
797
798 + // ------------------------------------------------------------------------
799 + // initialize health monitoring
800 +
801 + health_init();
802 +
803 // ------------------------------------------------------------------------
804 // initialize the registry
805
@@ -805,11 +810,6 @@ int main(int argc, char **argv) {
810
811 rrd_init(hostname);
812
808 - // ------------------------------------------------------------------------
809 - // initialize health monitoring
810 -
811 - health_init();
812 -
813 if(check_config)
814 exit(1);
815
src/rrd.h
+8 -1
@@ -302,6 +302,8 @@ struct rrdhost {
302 char machine_guid[GUID_LEN + 1];
303 uint32_t hash_machine_guid;
304
305 + uint32_t flags;
306 +
307 RRDSET *rrdset_root;
308 pthread_rwlock_t rrdset_root_rwlock;
309
@@ -324,7 +326,12 @@ struct rrdhost {
326 // are created or renamed, that match them
327 RRDCALCTEMPLATE *templates;
328
327 - uint32_t flags;
329 + // health / alarm settings
330 + char *health_default_exec;
331 + char *health_default_recipient;
332 + char *health_log_filename;
333 + size_t health_log_entries_written;
334 + FILE *health_log_fp;
335
336 struct rrdhost *next;
337 };
src/rrdhost.c
+53 -10
@@ -1,6 +1,8 @@
1 #define NETDATA_RRD_INTERNALS 1
2 #include "common.h"
3
4 +RRDHOST *localhost = NULL;
5 +
6 // ----------------------------------------------------------------------------
7 // RRDHOST index
8
@@ -62,13 +64,58 @@ RRDHOST *rrdhost_create(const char *hostname, const char *guid) {
64 avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);
65 avl_init_lock(&(host->variables_root_index), rrdvar_compare);
66
67 +
68 + // ------------------------------------------------------------------------
69 + // initialize health variables
70 +
71 host->health_log.next_log_id = 1;
72 host->health_log.next_alarm_id = 1;
73 host->health_log.max = 1000;
74 host->health_log.next_log_id =
75 host->health_log.next_alarm_id = (uint32_t)now_realtime_sec();
76 +
77 + long n = config_get_number("health", "in memory max health log entries", host->health_log.max);
78 + if(n < 10) {
79 + error("Health configuration has invalid max log entries %ld. Using default %u", n, host->health_log.max);
80 + config_set_number("health", "in memory max health log entries", (long)host->health_log.max);
81 + }
82 + else
83 + host->health_log.max = (unsigned int)n;
84 +
85 pthread_rwlock_init(&(host->health_log.alarm_log_rwlock), NULL);
86
87 + char filename[FILENAME_MAX + 1];
88 +
89 + if(!localhost) {
90 + // this is localhost
91 + snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", netdata_configured_varlib_dir);
92 + host->health_log_filename = strdupz(config_get("health", "health db file", filename));
93 + }
94 + else {
95 + // this is not localhost - append our GUID to localhost path
96 + snprintfz(filename, FILENAME_MAX, "%s.%s", localhost->health_log_filename, host->machine_guid);
97 + host->health_log_filename = strdupz(filename);
98 + }
99 +
100 + snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_plugins_dir);
101 + host->health_default_exec = strdupz(config_get("health", "script to execute on alarm", filename));
102 + host->health_default_recipient = strdup("root");
103 +
104 +
105 + // ------------------------------------------------------------------------
106 + // load health configuration
107 +
108 + health_alarm_log_load(host);
109 + health_alarm_log_open(host);
110 +
111 + rrdhost_rwlock(host);
112 + health_readdir(host, health_config_dir());
113 + rrdhost_unlock(host);
114 +
115 +
116 + // ------------------------------------------------------------------------
117 + // add it to the index
118 +
119 if(rrdhost_index_add(host) != host)
120 fatal("Cannot add host '%s' to index. It already exists.", hostname);
121
@@ -89,8 +136,6 @@ RRDHOST *rrdhost_find_or_create(const char *hostname, const char *guid) {
136 // ----------------------------------------------------------------------------
137 // RRDHOST global / startup initialization
138
92 -RRDHOST *localhost = NULL;
93 -
139 void rrd_init(char *hostname) {
140 debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
141 localhost = rrdhost_create(hostname, registry_get_this_machine_guid());
@@ -143,14 +188,9 @@ void rrdhost_free(RRDHOST *host) {
188
189 pthread_rwlock_wrlock(&st->rwlock);
190
146 - while(st->variables)
147 - rrdsetvar_free(st->variables);
148 -
149 - while(st->alarms)
150 - rrdsetcalc_unlink(st->alarms);
151 -
152 - while(st->dimensions)
153 - rrddim_free(st, st->dimensions);
191 + while(st->variables) rrdsetvar_free(st->variables);
192 + while(st->alarms) rrdsetcalc_unlink(st->alarms);
193 + while(st->dimensions) rrddim_free(st, st->dimensions);
194
195 if(unlikely(rrdset_index_del(host, st) != st))
196 error("RRDSET: INTERNAL ERROR: attempt to remove from index chart '%s', removed a different chart.", st->id);
@@ -174,6 +214,9 @@ void rrdhost_free(RRDHOST *host) {
214 }
215 host->rrdset_root = NULL;
216
217 + freez(host->health_default_exec);
218 + freez(host->health_default_recipient);
219 + freez(host->health_log_filename);
220 freez(host->hostname);
221 rrdhost_unlock(host);
222 freez(host);