@cryptotaxi247 / netdata-1 / commits / 430affeaf

API for getting alarm log ready

Costa Tsaousis committed Aug 25, 2016 at 19:39 UTC 430affeaf53da33214f4e6eec7db83f8239d1357
5 files changed +146 -34
src/health.c
+115 -31
@@ -5,13 +5,6 @@
5 static const char *health_default_exec = PLUGINS_DIR "/alarm-email.sh";
6 int health_enabled = 1;
7
8 -ALARM_LOG health_log = {
9 - .nextid = 0,
10 - .count = 0,
11 - .max = 1000,
12 - .alarms = NULL
13 -};
14 -
8 // ----------------------------------------------------------------------------
9 // RRDVAR management
10
@@ -443,6 +436,7 @@ static inline const char *rrdcalc_status2string(int status) {
436 static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
437 debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, st->rrdhost->hostname);
438
439 + rc->last_status_change = time(NULL);
440 rc->rrdset = st;
441
442 rc->rrdset_next = st->alarms;
@@ -1399,12 +1393,12 @@ void health_init(void) {
1393 health_default_exec = config_get("health", "script to execute on alarm", buffer);
1394 }
1395
1402 - long n = config_get_number("health", "in memory max health log entries", (long)health_log.max);
1396 + long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1397 if(n < 2) {
1404 - error("Health configuration has invalid max log entries %ld. Using default %u", n, health_log.max);
1405 - config_set_number("health", "in memory max health log entries", (long)health_log.max);
1398 + error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
1399 + config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1400 }
1407 - else health_log.max = (unsigned int)n;
1401 + else localhost.health_log.max = (unsigned int)n;
1402
1403 rrdhost_rwlock(&localhost);
1404 health_readdir(path);
@@ -1421,12 +1415,78 @@ static inline void health_string2json(BUFFER *wb, const char *prefix, const char
1415 buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
1416 }
1417
1424 -static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1418 +static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae) {
1419 + buffer_sprintf(wb, "\n\t{\n"
1420 + "\t\t\"id\":%u,\n"
1421 + "\t\t\"name\":\"%s\",\n"
1422 + "\t\t\"chart\":\"%s\",\n"
1423 + "\t\t\"family\":\"%s\",\n"
1424 + "\t\t\"processed\":%s,\n"
1425 + "\t\t\"updated\":%s,\n"
1426 + "\t\t\"exec_run\":%s,\n"
1427 + "\t\t\"exec_failed\":%s,\n"
1428 + "\t\t\"exec\":\"%s\",\n"
1429 + "\t\t\"exec_code\":%d,\n"
1430 + "\t\t\"source\":\"%s\",\n"
1431 + "\t\t\"when\":%lu,\n"
1432 + "\t\t\"duration\":%lu,\n"
1433 + "\t\t\"non_clear_duration\":%lu,\n"
1434 + "\t\t\"status\":\"%s\",\n"
1435 + "\t\t\"old_status\":\"%s\",\n",
1436 + ae->id,
1437 + ae->name,
1438 + ae->chart,
1439 + ae->family,
1440 + (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)?"true":"false",
1441 + (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false",
1442 + (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN)?"true":"false",
1443 + (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
1444 + ae->exec?ae->exec:health_default_exec,
1445 + ae->exec_code,
1446 + ae->source,
1447 + (unsigned long)ae->when,
1448 + (unsigned long)ae->duration,
1449 + (unsigned long)ae->non_clear_duration,
1450 + rrdcalc_status2string(ae->new_status),
1451 + rrdcalc_status2string(ae->old_status)
1452 + );
1453 +
1454 + buffer_strcat(wb, "\t\t\"value\":");
1455 + buffer_rrd_value(wb, ae->new_value);
1456 + buffer_strcat(wb, ",\n");
1457 +
1458 + buffer_strcat(wb, "\t\t\"old_value\":");
1459 + buffer_rrd_value(wb, ae->old_value);
1460 + buffer_strcat(wb, "\n");
1461 +
1462 + buffer_strcat(wb, "\t}");
1463 +}
1464 +
1465 +void health_alarm_log2json(RRDHOST *host, BUFFER *wb) {
1466 + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1467 +
1468 + buffer_strcat(wb, "[");
1469 +
1470 + unsigned int max = host->health_log.max;
1471 + unsigned int count = 0;
1472 + ALARM_ENTRY *ae;
1473 + for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
1474 + if(likely(count)) buffer_strcat(wb, ",");
1475 + health_alarm_entry2json_nolock(wb, ae);
1476 + }
1477
1478 + buffer_strcat(wb, "\n]\n");
1479 +
1480 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1481 +}
1482 +
1483 +static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1484 buffer_sprintf(wb,
1485 "\t\t\"%s.%s\": {\n"
1486 "\t\t\t\"name\": \"%s\",\n"
1487 "\t\t\t\"chart\": \"%s\",\n"
1488 + "\t\t\t\"family\": \"%s\",\n"
1489 + "\t\t\t\"active\": %s,\n"
1490 "\t\t\t\"exec\": \"%s\",\n"
1491 "\t\t\t\"source\": \"%s\",\n"
1492 "\t\t\t\"status\": \"%s\",\n"
@@ -1437,6 +1497,8 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1497 , rc->chart, rc->name
1498 , rc->name
1499 , rc->chart
1500 + , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
1501 + , (rc->rrdset)?"true":"false"
1502 , rc->exec?rc->exec:health_default_exec
1503 , rc->source
1504 , rrdcalc_status2string(rc->status)
@@ -1508,6 +1570,8 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb) {
1570 buffer_strcat(wb, "{\n\t\"alarms\": {\n");
1571 RRDCALC *rc;
1572 for(i = 0, rc = host->alarms; rc ; rc = rc->next, i++) {
1573 + if(!rc->rrdset) continue;
1574 +
1575 if(likely(i)) buffer_strcat(wb, ",\n");
1576 health_rrdcalc2json_nolock(wb, rc);
1577 }
@@ -1624,6 +1688,8 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1688 (uint32_t)ae->non_clear_duration
1689 );
1690
1691 + ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN;
1692 +
1693 debug(D_HEALTH, "executing command '%s'", buffer);
1694 FILE *fp = mypopen(buffer, &command_pid);
1695 if(!fp) {
@@ -1634,8 +1700,11 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1700 char *s = fgets(buffer, FILENAME_MAX, fp);
1701 (void)s;
1702 debug(D_HEALTH, "HEALTH closing command");
1637 - mypclose(fp, command_pid);
1638 - debug(D_HEALTH, "closed command");
1703 + ae->exec_code = mypclose(fp, command_pid);
1704 + debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
1705 +
1706 + if(ae->exec_code != 0)
1707 + ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED;
1708 }
1709
1710 static inline void health_process_notifications(ALARM_ENTRY *ae) {
@@ -1649,7 +1718,7 @@ static inline void health_process_notifications(ALARM_ENTRY *ae) {
1718 health_alarm_execute(ae);
1719 }
1720
1652 -static inline void health_alarm_log(time_t when,
1721 +static inline void health_alarm_log(RRDHOST *host, time_t when,
1722 const char *name, const char *chart, const char *family,
1723 const char *exec, time_t duration,
1724 calculated_number old_value, calculated_number new_value,
@@ -1671,7 +1740,7 @@ static inline void health_alarm_log(time_t when,
1740 if(exec) ae->exec = strdupz(exec);
1741 if(source) ae->source = strdupz(source);
1742
1674 - ae->id = health_log.nextid++;
1743 + ae->id = host->health_log.nextid++;
1744 ae->when = when;
1745 ae->old_value = old_value;
1746 ae->new_value = new_value;
@@ -1683,13 +1752,16 @@ static inline void health_alarm_log(time_t when,
1752 ae->non_clear_duration += ae->duration;
1753
1754 // link it
1686 - ae->next = health_log.alarms;
1687 - health_log.alarms = ae;
1688 - health_log.count++;
1755 + pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1756 + ae->next = host->health_log.alarms;
1757 + host->health_log.alarms = ae;
1758 + host->health_log.count++;
1759 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1760
1761 // match previous alarms
1762 + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1763 ALARM_ENTRY *t;
1692 - for(t = health_log.alarms ; t ; t = t->next) {
1764 + for(t = host->health_log.alarms ; t ; t = t->next) {
1765 if(t != ae &&
1766 t->hash_name == ae->hash_name &&
1767 t->hash_chart == ae->hash_chart &&
@@ -1710,13 +1782,16 @@ static inline void health_alarm_log(time_t when,
1782 }
1783 }
1784 }
1785 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1786 }
1787
1715 -static inline void health_alarm_log_process(void) {
1788 +static inline void health_alarm_log_process(RRDHOST *host) {
1789 static uint32_t last_processed = 0;
1790 ALARM_ENTRY *ae;
1791
1719 - for(ae = health_log.alarms; ae ;ae = ae->next) {
1792 + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1793 +
1794 + for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1795 if(last_processed >= ae->id) break;
1796
1797 if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
@@ -1726,18 +1801,25 @@ static inline void health_alarm_log_process(void) {
1801 }
1802 }
1803
1729 - if(health_log.alarms)
1730 - last_processed = health_log.alarms->id;
1804 + if(host->health_log.alarms)
1805 + last_processed = host->health_log.alarms->id;
1806 +
1807 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1808
1732 - if(health_log.count <= health_log.max)
1809 + if(host->health_log.count <= host->health_log.max)
1810 return;
1811
1812 // cleanup excess entries in the log
1813 + pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1814 +
1815 ALARM_ENTRY *last = NULL;
1737 - unsigned int count = health_log.max;
1738 - for(ae = health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
1739 - if(!ae || !last || last->next != ae) return;
1740 - last->next = NULL;
1816 + unsigned int count = host->health_log.max;
1817 + for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
1818 +
1819 + if(ae && last && last->next == ae)
1820 + last->next = NULL;
1821 + else
1822 + ae = NULL;
1823
1824 while(ae) {
1825 ALARM_ENTRY *t = ae->next;
@@ -1750,6 +1832,8 @@ static inline void health_alarm_log_process(void) {
1832
1833 ae = t;
1834 }
1835 +
1836 + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1837 }
1838
1839 void *health_main(void *ptr) {
@@ -1984,7 +2068,7 @@ void *health_main(void *ptr) {
2068 }
2069
2070 if(status != rc->status) {
1987 - health_alarm_log(time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source);
2071 + health_alarm_log(&localhost, time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source);
2072 rc->last_status_change = now;
2073 rc->status = status;
2074 }
@@ -2004,7 +2088,7 @@ void *health_main(void *ptr) {
2088
2089 // execute notifications
2090 // and cleanup
2007 - health_alarm_log_process();
2091 + health_alarm_log_process(&localhost);
2092
2093 now = time(NULL);
2094 if(now < next_run) {
src/health.h
+8 -2
@@ -206,8 +206,10 @@ typedef struct rrdcalctemplate {
206
207 #define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
208
209 -#define HEALTH_ENTRY_NOTIFICATIONS_PROCESSED 0x00000001
210 -#define HEALTH_ENTRY_NOTIFICATIONS_UPDATED 0x00000002
209 +#define HEALTH_ENTRY_NOTIFICATIONS_PROCESSED 0x00000001
210 +#define HEALTH_ENTRY_NOTIFICATIONS_UPDATED 0x00000002
211 +#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN 0x00000004
212 +#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED 0x00000008
213
214 typedef struct alarm_entry {
215 uint32_t id;
@@ -225,6 +227,8 @@ typedef struct alarm_entry {
227 char *family;
228
229 char *exec;
230 + int exec_code;
231 +
232 char *source;
233 calculated_number old_value;
234 calculated_number new_value;
@@ -242,6 +246,7 @@ typedef struct alarm_log {
246 unsigned int count;
247 unsigned int max;
248 ALARM_ENTRY *alarms;
249 + pthread_rwlock_t alarm_log_rwlock;
250 } ALARM_LOG;
251
252 #include "rrd.h"
@@ -266,5 +271,6 @@ extern void health_reload(void);
271
272 extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result);
273 extern void health_alarms2json(RRDHOST *host, BUFFER *wb);
274 +extern void health_alarm_log2json(RRDHOST *host, BUFFER *wb);
275
276 #endif //NETDATA_HEALTH_H
src/rrd.c
+7
@@ -40,6 +40,13 @@ RRDHOST localhost = {
40 .variables_root_index = {
41 { NULL, rrdvar_compare },
42 AVL_LOCK_INITIALIZER
43 + },
44 + .health_log = {
45 + .nextid = 1,
46 + .count = 0,
47 + .max = 1000,
48 + .alarms = NULL,
49 + .alarm_log_rwlock = PTHREAD_RWLOCK_INITIALIZER
50 }
51 };
52
src/rrd.h
+1
@@ -304,6 +304,7 @@ struct rrdhost {
304 // RRDCALCs may be linked to charts at any point
305 // (charts may or may not exist when these are loaded)
306 RRDCALC *alarms;
307 + ALARM_LOG health_log;
308
309 RRDCALCTEMPLATE *templates;
310 };
src/web_client.c
+15 -1
@@ -670,6 +670,16 @@ int web_client_api_request_v1_alarms(struct web_client *w, char *url)
670 return 200;
671 }
672
673 +int web_client_api_request_v1_alarm_log(struct web_client *w, char *url)
674 +{
675 + (void)url;
676 +
677 + buffer_flush(w->response.data);
678 + w->response.data->contenttype = CT_APPLICATION_JSON;
679 + health_alarm_log2json(&localhost, w->response.data);
680 + return 200;
681 +}
682 +
683 int web_client_api_request_v1_charts(struct web_client *w, char *url)
684 {
685 if(url) { ; }
@@ -1324,7 +1334,7 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url)
1334 }
1335
1336 int web_client_api_request_v1(struct web_client *w, char *url) {
1327 - static uint32_t hash_data = 0, hash_chart = 0, hash_charts = 0, hash_registry = 0, hash_badge = 0, hash_alarms = 0;
1337 + static uint32_t hash_data = 0, hash_chart = 0, hash_charts = 0, hash_registry = 0, hash_badge = 0, hash_alarms = 0, hash_alarm_log = 0;
1338
1339 if(unlikely(hash_data == 0)) {
1340 hash_data = simple_hash("data");
@@ -1333,6 +1343,7 @@ int web_client_api_request_v1(struct web_client *w, char *url) {
1343 hash_registry = simple_hash("registry");
1344 hash_badge = simple_hash("badge.svg");
1345 hash_alarms = simple_hash("alarms");
1346 + hash_alarm_log = simple_hash("alarm_log");
1347 }
1348
1349 // get the command
@@ -1359,6 +1370,9 @@ int web_client_api_request_v1(struct web_client *w, char *url) {
1370 else if(hash == hash_alarms && !strcmp(tok, "alarms"))
1371 return web_client_api_request_v1_alarms(w, url);
1372
1373 + else if(hash == hash_alarm_log && !strcmp(tok, "alarm_log"))
1374 + return web_client_api_request_v1_alarm_log(w, url);
1375 +
1376 else {
1377 buffer_flush(w->response.data);
1378 buffer_sprintf(w->response.data, "Unsupported v1 API command: %s", tok);