@cryptotaxi247 / netdata-1 / commits / 1396369d4

netdata can reload health configuration, at runtime, with SIGUSR2

Costa Tsaousis committed Aug 16, 2016 at 16:51 UTC 1396369d4addaae481f831a2ef6068dfc5eabb3a
8 files changed +140 -74
src/daemon.c
+9
@@ -30,6 +30,15 @@ void sig_handler_save(int signo)
30 }
31 }
32
33 +void sig_handler_reload_health(int signo)
34 +{
35 + if(signo) {
36 + error_log_limit_reset();
37 + info("Received signal %d to reload health configuration...", signo);
38 + health_reload();
39 + }
40 +}
41 +
42 static void chown_open_file(int fd, uid_t uid, gid_t gid) {
43 if(fd == -1) return;
44
src/daemon.h
+1
@@ -4,6 +4,7 @@
4 extern void sig_handler_exit(int signo);
5 extern void sig_handler_save(int signo);
6 extern void sig_handler_logrotate(int signo);
7 +extern void sig_handler_reload_health(int signo);
8
9 extern int become_user(const char *username, int pid_fd);
10
src/health.c
+85 -37
@@ -442,7 +442,7 @@ inline void rrdsetcalc_link_matching(RRDSET *st) {
442 // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);
443
444 RRDCALC *rc;
445 - for(rc = st->rrdhost->calculations; rc ; rc = rc->next) {
445 + for(rc = st->rrdhost->alarms; rc ; rc = rc->next) {
446 if(rc->rrdset) continue;
447
448 if(rrdcalc_is_matching_this_rrdset(rc, st))
@@ -470,8 +470,8 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
470 if(rc->rrdset_next)
471 rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
472
473 - if(st->calculations == rc)
474 - st->calculations = rc->rrdset_next;
473 + if(st->alarms == rc)
474 + st->alarms = rc->rrdset_next;
475
476 rc->rrdset_prev = rc->rrdset_next = NULL;
477
@@ -498,7 +498,7 @@ static inline int rrdcalc_exists(RRDHOST *host, const char *name, uint32_t hash)
498 RRDCALC *rc;
499
500 // make sure it does not already exist
501 - for(rc = host->calculations; rc ; rc = rc->next) {
501 + for(rc = host->alarms; rc ; rc = rc->next) {
502 if (rc->hash == hash && !strcmp(name, rc->name)) {
503 error("Health alarm '%s' already exists in host '%s'.", name, host->hostname);
504 return 1;
@@ -527,8 +527,8 @@ static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
527 }
528
529 // link it to the host
530 - rc->next = host->calculations;
531 - host->calculations = rc;
530 + rc->next = host->alarms;
531 + host->alarms = rc;
532
533 // link it to its chart
534 RRDSET *st;
@@ -627,11 +627,11 @@ void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
627 if(rc->rrdset) rrdsetcalc_unlink(rc);
628
629 // unlink it from RRDHOST
630 - if(rc == host->calculations)
631 - host->calculations = rc->next;
630 + if(rc == host->alarms)
631 + host->alarms = rc->next;
632
633 - else if(host->calculations) {
634 - RRDCALC *t, *last = host->calculations;
633 + else if(host->alarms) {
634 + RRDCALC *t, *last = host->alarms;
635
636 for(t = last->next; t && t != rc; last = t, t = t->next) ;
637 if(last && last->next == rc)
@@ -642,15 +642,15 @@ void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
642 else
643 error("Cannot unlink unlink '%s.%s' from host '%s': This host does not have any calculations", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
644
645 - if(rc->warning) expression_free(rc->warning);
646 - if(rc->critical) expression_free(rc->critical);
645 + expression_free(rc->calculation);
646 + expression_free(rc->warning);
647 + expression_free(rc->critical);
648
649 freez(rc->source);
650 freez(rc->name);
651 freez(rc->chart);
652 freez(rc->dimensions);
653 freez(rc->exec);
653 -
654 freez(rc);
655 }
656
@@ -702,8 +702,9 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
702 }
703 }
704
705 - if(rt->warning) expression_free(rt->warning);
706 - if(rt->critical) expression_free(rt->critical);
705 + expression_free(rt->calculation);
706 + expression_free(rt->warning);
707 + expression_free(rt->critical);
708
709 freez(rt->dimensions);
710 freez(rt->context);
@@ -771,7 +772,7 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
772 return 1;
773 }
774
774 -static inline int rrdcalctemplate_add(RRDHOST *host, RRDCALCTEMPLATE *rt) {
775 +static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
776 if(!rt->context) {
777 error("Health configuration for template '%s' does not have a context", rt->name);
778 return 0;
@@ -813,7 +814,7 @@ static inline int rrdcalctemplate_add(RRDHOST *host, RRDCALCTEMPLATE *rt) {
814 return 1;
815 }
816
816 -static inline int health_parse_time(char *string, int *result) {
817 +static inline int health_parse_duration(char *string, int *result) {
818 // make sure it is a number
819 if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
820 *result = 0;
@@ -855,7 +856,7 @@ static inline int health_parse_time(char *string, int *result) {
856 return 1;
857 }
858
858 -static inline int health_parse_lookup(
859 +static inline int health_parse_db_lookup(
860 size_t line, const char *path, const char *file, char *string,
861 int *group_method, int *after, int *before, int *every,
862 uint32_t *options, char **dimensions
@@ -892,7 +893,7 @@ static inline int health_parse_lookup(
893 while(*s && !isspace(*s)) s++;
894 while(*s && isspace(*s)) *s++ = '\0';
895
895 - if(!health_parse_time(key, after)) {
896 + if(!health_parse_duration(key, after)) {
897 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
898 line, path, file, key);
899 return 0;
@@ -913,7 +914,7 @@ static inline int health_parse_lookup(
914 while(*s && !isspace(*s)) s++;
915 while(*s && isspace(*s)) *s++ = '\0';
916
916 - if (!health_parse_time(value, before)) {
917 + if (!health_parse_duration(value, before)) {
918 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
919 line, path, file, value, key);
920 }
@@ -923,7 +924,7 @@ static inline int health_parse_lookup(
924 while(*s && !isspace(*s)) s++;
925 while(*s && isspace(*s)) *s++ = '\0';
926
926 - if (!health_parse_time(value, every)) {
927 + if (!health_parse_duration(value, every)) {
928 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
929 line, path, file, value, key);
930 }
@@ -1044,7 +1045,7 @@ int health_readfile(const char *path, const char *filename) {
1045 rrdcalc_free(&localhost, rc);
1046
1047 if(rt) {
1047 - if (!rrdcalctemplate_add(&localhost, rt))
1048 + if (!rrdcalctemplate_add_template_from_config(&localhost, rt))
1049 rrdcalctemplate_free(&localhost, rt);
1050 rt = NULL;
1051 }
@@ -1064,7 +1065,7 @@ int health_readfile(const char *path, const char *filename) {
1065 rc = NULL;
1066 }
1067
1067 - if(rt && !rrdcalctemplate_add(&localhost, rt))
1068 + if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1069 rrdcalctemplate_free(&localhost, rt);
1070
1071 rt = callocz(1, sizeof(RRDCALCTEMPLATE));
@@ -1088,11 +1089,12 @@ int health_readfile(const char *path, const char *filename) {
1089 rc->hash_chart = simple_hash(rc->chart);
1090 }
1091 else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1091 - health_parse_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before, &rc->update_every,
1092 - &rc->options, &rc->dimensions);
1092 + health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
1093 + &rc->update_every,
1094 + &rc->options, &rc->dimensions);
1095 }
1096 else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1095 - if(!health_parse_time(value, &rc->update_every))
1097 + if(!health_parse_duration(value, &rc->update_every))
1098 info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
1099 line, path, filename, rc->name, key, value);
1100 }
@@ -1167,11 +1169,12 @@ int health_readfile(const char *path, const char *filename) {
1169 rt->hash_context = simple_hash(rt->context);
1170 }
1171 else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1170 - health_parse_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before, &rt->update_every,
1171 - &rt->options, &rt->dimensions);
1172 + health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
1173 + &rt->update_every,
1174 + &rt->options, &rt->dimensions);
1175 }
1176 else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1174 - if(!health_parse_time(value, &rt->update_every))
1177 + if(!health_parse_duration(value, &rt->update_every))
1178 info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
1179 line, path, filename, rt->name, key, value);
1180 }
@@ -1242,7 +1245,7 @@ int health_readfile(const char *path, const char *filename) {
1245 if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1246 rrdcalc_free(&localhost, rc);
1247
1245 - if(rt && !rrdcalctemplate_add(&localhost, rt))
1248 + if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1249 rrdcalctemplate_free(&localhost, rt);
1250
1251 fclose(fp);
@@ -1290,21 +1293,24 @@ void health_readdir(const char *path) {
1293 closedir(dir);
1294 }
1295
1296 +static inline char *health_config_dir(void) {
1297 + char buffer[FILENAME_MAX + 1];
1298 + snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
1299 + return config_get("health", "health configuration directory", buffer);
1300 +}
1301 +
1302 void health_init(void) {
1303 debug(D_HEALTH, "Health configuration initializing");
1304
1296 - char *path;
1297 -
1305 if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
1306 debug(D_HEALTH, "Health is disabled.");
1307 return;
1308 }
1309
1310 + char *path = health_config_dir();
1311 +
1312 {
1313 char buffer[FILENAME_MAX + 1];
1305 - snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
1306 - path = config_get("health", "health configuration directory", buffer);
1307 -
1314 snprintfz(buffer, FILENAME_MAX, "%s/alarm.sh", config_get("global", "plugins directory", PLUGINS_DIR));
1315 health_default_exec = config_get("health", "script to execute on alarm", buffer);
1316 }
@@ -1314,6 +1320,48 @@ void health_init(void) {
1320 rrdhost_unlock(&localhost);
1321 }
1322
1323 +// ----------------------------------------------------------------------------
1324 +// re-load health configuration
1325 +
1326 +static inline void health_free_all_nolock(RRDHOST *host) {
1327 + while(host->templates)
1328 + rrdcalctemplate_free(host, host->templates);
1329 +
1330 + while(host->alarms)
1331 + rrdcalc_free(host, host->alarms);
1332 +}
1333 +
1334 +void health_reload(void) {
1335 + if(!health_enabled) {
1336 + error("Health reload is requested, but health is not enabled.");
1337 + return;
1338 + }
1339 +
1340 + char *path = health_config_dir();
1341 +
1342 + rrdhost_rwlock(&localhost);
1343 + health_free_all_nolock(&localhost);
1344 + rrdhost_unlock(&localhost);
1345 +
1346 + rrdhost_rwlock(&localhost);
1347 + health_readdir(path);
1348 + rrdhost_unlock(&localhost);
1349 +
1350 + RRDSET *st;
1351 + for(st = localhost.rrdset_root; st ; st = st->next) {
1352 + rrdhost_rwlock(&localhost);
1353 +
1354 + rrdsetcalc_link_matching(st);
1355 + rrdcalctemplate_link_matching(st);
1356 +
1357 + rrdhost_unlock(&localhost);
1358 + }
1359 +}
1360 +
1361 +
1362 +// ----------------------------------------------------------------------------
1363 +// health main thread and friends
1364 +
1365 static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
1366 if (unlikely(!rc->rrdset)) {
1367 debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
@@ -1427,7 +1475,7 @@ void *health_main(void *ptr) {
1475 rrdhost_rdlock(&localhost);
1476
1477 // the first loop is to lookup values from the db
1430 - for (rc = localhost.calculations; rc; rc = rc->next) {
1478 + for (rc = localhost.alarms; rc; rc = rc->next) {
1479 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
1480 continue;
1481
@@ -1519,7 +1567,7 @@ void *health_main(void *ptr) {
1567 if (runnable) {
1568 rrdhost_rdlock(&localhost);
1569
1522 - for (rc = localhost.calculations; rc; rc = rc->next) {
1570 + for (rc = localhost.alarms; rc; rc = rc->next) {
1571 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
1572 continue;
1573
src/health.h
+2
@@ -219,6 +219,8 @@ extern void rrdcalctemplate_link_matching(RRDSET *st);
219 extern void health_init(void);
220 extern void *health_main(void *ptr);
221
222 +extern void health_reload(void);
223 +
224 extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result);
225
226 #endif //NETDATA_HEALTH_H
src/main.c
+24 -29
@@ -536,50 +536,45 @@ int main(int argc, char **argv)
536 // this causes the threads to block signals.
537 sigset_t sigset;
538 sigfillset(&sigset);
539 -
540 - if(pthread_sigmask(SIG_BLOCK, &sigset, NULL) == -1) {
539 + if(pthread_sigmask(SIG_BLOCK, &sigset, NULL) == -1)
540 error("Could not block signals for threads");
542 - }
541
542 // Catch signals which we want to use
543 struct sigaction sa;
546 - sigemptyset(&sa.sa_mask);
547 - sigaddset(&sa.sa_mask, SIGINT);
548 - sigaddset(&sa.sa_mask, SIGTERM);
549 - sa.sa_handler = sig_handler_exit;
544 sa.sa_flags = 0;
551 - if(sigaction(SIGINT, &sa, NULL) == -1) {
545 +
546 + // ingore all signals while we run in a signal handler
547 + sigfillset(&sa.sa_mask);
548 +
549 + // INFO: If we add signals here we have to unblock them
550 + // at popen.c when running a external plugin.
551 +
552 + // Ignore SIGPIPE completely.
553 + sa.sa_handler = SIG_IGN;
554 + if(sigaction(SIGPIPE, &sa, NULL) == -1)
555 + error("Failed to change signal handler for SIGPIPE");
556 +
557 + sa.sa_handler = sig_handler_exit;
558 + if(sigaction(SIGINT, &sa, NULL) == -1)
559 error("Failed to change signal handler for SIGINT");
553 - }
554 - if(sigaction(SIGTERM, &sa, NULL) == -1) {
560 +
561 + sa.sa_handler = sig_handler_exit;
562 + if(sigaction(SIGTERM, &sa, NULL) == -1)
563 error("Failed to change signal handler for SIGTERM");
556 - }
564
558 - sigemptyset(&sa.sa_mask);
559 - sigaddset(&sa.sa_mask, SIGHUP);
565 sa.sa_handler = sig_handler_logrotate;
561 - sa.sa_flags = 0;
562 - if(sigaction(SIGHUP, &sa, NULL) == -1) {
566 + if(sigaction(SIGHUP, &sa, NULL) == -1)
567 error("Failed to change signal handler for SIGHUP");
564 - }
568
569 // save database on SIGUSR1
567 - sigemptyset(&sa.sa_mask);
568 - sigaddset(&sa.sa_mask, SIGUSR1);
570 sa.sa_handler = sig_handler_save;
570 - if(sigaction(SIGUSR1, &sa, NULL) == -1) {
571 + if(sigaction(SIGUSR1, &sa, NULL) == -1)
572 error("Failed to change signal handler for SIGUSR1");
572 - }
573
574 - // Ignore SIGPIPE completely.
575 - // INFO: If we add signals here we have to unblock them
576 - // at popen.c when running a external plugin.
577 - sigemptyset(&sa.sa_mask);
578 - sigaddset(&sa.sa_mask, SIGPIPE);
579 - sa.sa_handler = SIG_IGN;
580 - if(sigaction(SIGPIPE, &sa, NULL) == -1) {
581 - error("Failed to change signal handler for SIGPIPE");
582 - }
574 + // reload health configuration on SIGUSR2
575 + sa.sa_handler = sig_handler_reload_health;
576 + if(sigaction(SIGUSR2, &sa, NULL) == -1)
577 + error("Failed to change signal handler for SIGUSR2");
578
579 // --------------------------------------------------------------------
580
src/popen.c
+15 -4
@@ -118,14 +118,25 @@ FILE *mypopen(const char *command, pid_t *pidptr)
118 sigemptyset(&sa.sa_mask);
119 sa.sa_handler = SIG_DFL;
120 sa.sa_flags = 0;
121 -
122 - if(sigaction(SIGUSR1, &sa, NULL) == -1)
123 - error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGUSR1.", command, getpid());
121 +
122 + if(sigaction(SIGINT, &sa, NULL) == -1)
123 + error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGINT.", command, getpid());
124 +
125 + if(sigaction(SIGTERM, &sa, NULL) == -1)
126 + error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGTERM.", command, getpid());
127
128 if(sigaction(SIGPIPE, &sa, NULL) == -1)
129 error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGPIPE.", command, getpid());
127 - }
130
131 + if(sigaction(SIGHUP, &sa, NULL) == -1)
132 + error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGHUP.", command, getpid());
133 +
134 + if(sigaction(SIGUSR1, &sa, NULL) == -1)
135 + error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGUSR1.", command, getpid());
136 +
137 + if(sigaction(SIGUSR2, &sa, NULL) == -1)
138 + error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGUSR2.", command, getpid());
139 + }
140
141 info("executing command: '%s' on pid %d.", command, getpid());
142 execl("/bin/sh", "sh", "-c", command, NULL);
src/rrd.c
+2 -2
@@ -768,8 +768,8 @@ void rrdset_free_all(void)
768 while(st->variables)
769 rrdsetvar_free(st->variables);
770
771 - while(st->calculations)
772 - rrdsetcalc_unlink(st->calculations);
771 + while(st->alarms)
772 + rrdsetcalc_unlink(st->alarms);
773
774 while(st->dimensions)
775 rrddim_free(st, st->dimensions);
src/rrd.h
+2 -2
@@ -265,7 +265,7 @@ struct rrdset {
265
266 avl_tree_lock variables_root_index;
267 RRDSETVAR *variables;
268 - RRDCALC *calculations;
268 + RRDCALC *alarms;
269
270 // ------------------------------------------------------------------------
271 // members for checking the data when loading from disk
@@ -303,7 +303,7 @@ struct rrdhost {
303 // all RRDCALCs are primarily allocated and linked here
304 // RRDCALCs may be linked to charts at any point
305 // (charts may or may not exist when these are loaded)
306 - RRDCALC *calculations;
306 + RRDCALC *alarms;
307
308 RRDCALCTEMPLATE *templates;
309 };