@cryptotaxi247 / netdata-1 / commits / 079422f82

added the ability to execute scripts on alarms

Costa Tsaousis committed Aug 16, 2016 at 23:36 UTC 079422f822a3b5f198388637ed0b5c766db24cfe
4 files changed +214 -26
conf.d/health.d/disks.conf
+4 -3
@@ -5,11 +5,12 @@
5 # raise an alarm if the disk is low on
6 # available disk space
7
8 -template: low_disk_space
8 +template: disk_full_percentage
9 on: disk.space
10 + calc: $used * 100 / ($avail + $used)
11 every: 1m
11 - warn: $avail * 100 / ($avail + $used) < 80
12 - crit: $avail * 100 / ($avail + $used) < 95
12 + warn: $this > 80
13 + crit: $used > 95
14
15
16 # -----------------------------------------------------------------------------
src/health.c
+181 -22
@@ -5,6 +5,13 @@
5 static const char *health_default_exec = PLUGINS_DIR "/alarm.sh";
6 int health_enabled = 1;
7
8 +ALARM_LOG health_log = {
9 + .nextid = 0,
10 + .count = 0,
11 + .max = 1000,
12 + .alarms = NULL
13 +};
14 +
15 // ----------------------------------------------------------------------------
16 // RRDVAR management
17
@@ -568,6 +575,9 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
575
576 if(dimensions) rc->dimensions = strdupz(dimensions);
577
578 + rc->value = NAN;
579 + rc->old_value = NAN;
580 +
581 rc->group = group_method;
582 rc->after = after;
583 rc->before = before;
@@ -958,9 +968,9 @@ static inline int health_parse_db_lookup(
968 return 1;
969 }
970
961 -static inline char *health_source_file(int line, const char *path, const char *filename) {
971 +static inline char *health_source_file(size_t line, const char *path, const char *filename) {
972 char buffer[FILENAME_MAX + 1];
963 - snprintfz(buffer, FILENAME_MAX, "%d@%s/%s", line, path, filename);
973 + snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
974 return strdupz(buffer);
975 }
976
@@ -1054,6 +1064,8 @@ int health_readfile(const char *path, const char *filename) {
1064 rc->name = strdupz(value);
1065 rc->hash = simple_hash(rc->name);
1066 rc->source = health_source_file(line, path, filename);
1067 + rc->value = NAN;
1068 + rc->old_value = NAN;
1069
1070 if(rrdvar_fix_name(rc->name))
1071 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
@@ -1315,6 +1327,13 @@ void health_init(void) {
1327 health_default_exec = config_get("health", "script to execute on alarm", buffer);
1328 }
1329
1330 + long n = config_get_number("health", "in memory max health log entries", (long)health_log.max);
1331 + if(n < 2) {
1332 + error("Health configuration has invalid max log entries %ld. Using default %u", n, health_log.max);
1333 + config_set_number("health", "in memory max health log entries", (long)health_log.max);
1334 + }
1335 + else health_log.max = (unsigned int)n;
1336 +
1337 rrdhost_rwlock(&localhost);
1338 health_readdir(path);
1339 rrdhost_unlock(&localhost);
@@ -1409,38 +1428,151 @@ static inline const char *rrdcalc_status2string(int status) {
1428 }
1429 }
1430
1412 -void rrdcalc_check_critical_event(RRDCALC *rc) {
1431 +static inline const char *alarm_entry_type2string(int type) {
1432 + switch(type) {
1433 + case ALARM_ENTRY_TYPE_WARNING:
1434 + return "WARNING";
1435 +
1436 + case ALARM_ENTRY_TYPE_CRITICAL:
1437 + return "WARNING";
1438 +
1439 + default:
1440 + return "UNKNOWN";
1441 + }
1442 +}
1443 +
1444 +static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1445 + if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_OFF)
1446 + return;
1447 +
1448 + char buffer[FILENAME_MAX + 1];
1449 + pid_t command_pid;
1450 +
1451 + const char *exec = ae->exec;
1452 + if(!exec) exec = health_default_exec;
1453 +
1454 + snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf', '%s'",
1455 + exec,
1456 + alarm_entry_type2string(ae->type),
1457 + ae->name,
1458 + ae->chart?ae->chart:"NOCAHRT",
1459 + rrdcalc_status2string(ae->new_status),
1460 + rrdcalc_status2string(ae->old_status),
1461 + ae->new_value,
1462 + ae->old_value,
1463 + ae->source?ae->source:"UNKNOWN"
1464 + );
1465 +
1466 + debug(D_HEALTH, "executing command '%s'", buffer);
1467 + FILE *fp = mypopen(buffer, &command_pid);
1468 + if(!fp) {
1469 + error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer);
1470 + return;
1471 + }
1472 + debug(D_HEALTH, "HEALTH reading from command");
1473 + char *s = fgets(buffer, FILENAME_MAX, fp);
1474 + (void)s;
1475 + debug(D_HEALTH, "HEALTH closing command");
1476 + mypclose(fp, command_pid);
1477 + debug(D_HEALTH, "closed command");
1478 +}
1479 +
1480 +static inline void health_process_notifications(ALARM_ENTRY *ae) {
1481 + info("Health alarm '%s.%s' = %0.2Lf - %s changed status from %s to %s",
1482 + ae->chart?ae->chart:"NOCHART", ae->name,
1483 + ae->new_value,
1484 + (ae->type == ALARM_ENTRY_TYPE_WARNING)?"WARNING":"CRITICAL",
1485 + rrdcalc_status2string(ae->old_status),
1486 + rrdcalc_status2string(ae->new_status)
1487 + );
1488 +
1489 + health_alarm_execute(ae);
1490 +}
1491 +
1492 +static inline void health_alarm_log(time_t when, int type,
1493 + const char *name, const char *chart, const char *exec,
1494 + calculated_number old_value, calculated_number new_value,
1495 + int old_status, int new_status,
1496 + const char *source
1497 +) {
1498 + ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
1499 + ae->name = strdupz(name);
1500 + if(chart) ae->chart = strdupz(chart);
1501 + if(exec) ae->exec = strdupz(exec);
1502 + if(source) ae->source = strdupz(source);
1503 +
1504 + ae->id = health_log.nextid++;
1505 + ae->when = when;
1506 + ae->type = type;
1507 + ae->old_value = old_value;
1508 + ae->new_value = new_value;
1509 + ae->old_status = old_status;
1510 + ae->new_status = new_status;
1511 +
1512 + // link it
1513 + ae->next = health_log.alarms;
1514 + health_log.alarms = ae;
1515 + health_log.count++;
1516 +}
1517 +
1518 +static inline void health_alarm_log_process(void) {
1519 + static uint32_t last_processed = 0;
1520 + ALARM_ENTRY *ae;
1521 +
1522 + for(ae = health_log.alarms; ae ;ae = ae->next) {
1523 + if(last_processed >= ae->id) break;
1524 +
1525 + if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)) {
1526 + ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
1527 + health_process_notifications(ae);
1528 + }
1529 + }
1530 +
1531 + if(health_log.alarms)
1532 + last_processed = health_log.alarms->id;
1533 +
1534 + if(health_log.count <= health_log.max)
1535 + return;
1536 +
1537 + // cleanup excess entries in the log
1538 + ALARM_ENTRY *last = NULL;
1539 + unsigned int count = health_log.max;
1540 + for(ae = health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
1541 + if(!ae || !last || last->next != ae) return;
1542 + last->next = NULL;
1543 +
1544 + while(ae) {
1545 + ALARM_ENTRY *t = ae->next;
1546 +
1547 + freez(ae->chart);
1548 + freez(ae->name);
1549 + freez(ae->exec);
1550 + freez(ae);
1551 +
1552 + ae = t;
1553 + }
1554 +}
1555 +
1556 +static inline void rrdcalc_check_critical_event(RRDCALC *rc) {
1557 calculated_number n = rc->critical->result;
1558
1559 int old_status = rc->critical_status;
1560 int new_status = rrdcalc_value2status(n);
1561
1562 if(new_status != old_status) {
1419 - info("Health alarm '%s.%s' = %0.2Lf - CRITICAL condition changed status from %s to %s",
1420 - rc->chart?rc->chart:"NOCHART", rc->name,
1421 - rc->value,
1422 - rrdcalc_status2string(old_status),
1423 - rrdcalc_status2string(new_status)
1424 - );
1425 -
1563 + health_alarm_log(time(NULL), ALARM_ENTRY_TYPE_WARNING, rc->name, rc->rrdset->id, rc->exec, rc->old_value, rc->value, old_status, new_status, rc->source);
1564 rc->critical_status = new_status;
1565 }
1566 }
1567
1430 -void rrdcalc_check_warning_event(RRDCALC *rc) {
1568 +static inline void rrdcalc_check_warning_event(RRDCALC *rc) {
1569 calculated_number n = rc->warning->result;
1570
1571 int old_status = rc->warning_status;
1572 int new_status = rrdcalc_value2status(n);
1573
1574 if(new_status != old_status) {
1437 - info("Health alarm '%s.%s' = %0.2Lf - WARNING condition changed status from %s to %s",
1438 - rc->chart?rc->chart:"NOCHART", rc->name,
1439 - rc->value,
1440 - rrdcalc_status2string(old_status),
1441 - rrdcalc_status2string(new_status)
1442 - );
1443 -
1575 + health_alarm_log(time(NULL), ALARM_ENTRY_TYPE_CRITICAL, rc->name, rc->rrdset->id, rc->exec, rc->old_value, rc->value, old_status, new_status, rc->source);
1576 rc->warning_status = new_status;
1577 }
1578 }
@@ -1482,6 +1614,7 @@ void *health_main(void *ptr) {
1614 continue;
1615
1616 runnable++;
1617 + rc->old_value = rc->value;
1618
1619 // 1. if there is database lookup, do it
1620 // 2. if there is calculation expression, run it
@@ -1498,6 +1631,8 @@ void *health_main(void *ptr) {
1631 // database lookup failed
1632 rc->value = NAN;
1633
1634 + debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
1635 +
1636 if (unlikely(!(rc->rrdcalc_options & RRDCALC_OPTION_DB_ERROR))) {
1637 rc->rrdcalc_options |= RRDCALC_OPTION_DB_ERROR;
1638 error("Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
@@ -1509,6 +1644,8 @@ void *health_main(void *ptr) {
1644 if (unlikely(old_db_timestamp == rc->db_timestamp)) {
1645 // database is stale
1646
1647 + debug(D_HEALTH, "Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
1648 +
1649 if (unlikely(!(rc->rrdcalc_options & RRDCALC_OPTION_DB_STALE))) {
1650 rc->rrdcalc_options |= RRDCALC_OPTION_DB_STALE;
1651 error("Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
@@ -1522,6 +1659,9 @@ void *health_main(void *ptr) {
1659
1660 rc->value = NAN;
1661
1662 + debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
1663 + rc->chart?rc->chart:"NOCHART", rc->name);
1664 +
1665 if (unlikely(!(rc->rrdcalc_options & RRDCALC_OPTION_DB_NAN))) {
1666 rc->rrdcalc_options |= RRDCALC_OPTION_DB_NAN;
1667 error("Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
@@ -1541,6 +1681,9 @@ void *health_main(void *ptr) {
1681
1682 rc->value = NAN;
1683
1684 + debug(D_HEALTH, "Health alarm '%s.%s': failed to evaluate calculation with error: %s",
1685 + rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
1686 +
1687 if (unlikely(!(rc->rrdcalc_options & RRDCALC_OPTION_CALC_ERROR))) {
1688 rc->rrdcalc_options |= RRDCALC_OPTION_CALC_ERROR;
1689 error("Health alarm '%s.%s': failed to evaluate calculation with error: %s",
@@ -1576,6 +1719,10 @@ void *health_main(void *ptr) {
1719 if(unlikely(rc->warning)) {
1720 if(unlikely(!expression_evaluate(rc->warning))) {
1721 // calculation failed
1722 +
1723 + debug(D_HEALTH, "Health alarm '%s.%s': warning expression failed with error: %s",
1724 + rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
1725 +
1726 if (unlikely(!(rc->rrdcalc_options & RRDCALC_OPTION_WARN_ERROR))) {
1727 rc->rrdcalc_options |= RRDCALC_OPTION_WARN_ERROR;
1728 error("Health alarm '%s.%s': warning expression failed with error: %s",
@@ -1602,6 +1749,10 @@ void *health_main(void *ptr) {
1749 if(unlikely(rc->critical)) {
1750 if(unlikely(!expression_evaluate(rc->critical))) {
1751 // calculation failed
1752 +
1753 + debug(D_HEALTH, "Health alarm '%s.%s': critical expression failed with error: %s",
1754 + rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
1755 +
1756 if (unlikely(!(rc->rrdcalc_options & RRDCALC_OPTION_CRIT_ERROR))) {
1757 rc->rrdcalc_options |= RRDCALC_OPTION_CRIT_ERROR;
1758 error("Health alarm '%s.%s': critical expression failed with error: %s",
@@ -1635,14 +1786,22 @@ void *health_main(void *ptr) {
1786 rrdhost_unlock(&localhost);
1787 }
1788
1638 -
1789 if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
1790 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1791
1642 - debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
1643 - loop, (int) (next_run - now));
1792 + // execute notifications
1793 + // and cleanup
1794 + health_alarm_log_process();
1795
1645 - sleep_usec(1000000 * (unsigned long long) (next_run - now));
1796 + now = time(NULL);
1797 + if(now < next_run) {
1798 + debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
1799 + loop, (int) (next_run - now));
1800 + sleep_usec(1000000 * (unsigned long long) (next_run - now));
1801 + }
1802 + else {
1803 + debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
1804 + }
1805 }
1806
1807 buffer_free(wb);
src/health.h
+28
@@ -149,6 +149,7 @@ typedef struct rrdcalc {
149 time_t db_timestamp;
150
151 calculated_number value;
152 + calculated_number old_value;
153
154 calculated_number green;
155 calculated_number red;
@@ -201,6 +202,33 @@ typedef struct rrdcalctemplate {
202
203 #define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
204
205 +#define ALARM_ENTRY_TYPE_WARNING 1
206 +#define ALARM_ENTRY_TYPE_CRITICAL 2
207 +
208 +#define HEALTH_ENTRY_NOTIFICATIONS_PROCESSED 0x00000001
209 +
210 +typedef struct alarm_entry {
211 + uint32_t id;
212 + time_t when;
213 + int type;
214 + char *name;
215 + char *chart;
216 + char *exec;
217 + char *source;
218 + calculated_number old_value;
219 + calculated_number new_value;
220 + int old_status;
221 + int new_status;
222 + uint32_t notifications;
223 + struct alarm_entry *next;
224 +} ALARM_ENTRY;
225 +
226 +typedef struct alarm_log {
227 + uint32_t nextid;
228 + unsigned int count;
229 + unsigned int max;
230 + ALARM_ENTRY *alarms;
231 +} ALARM_LOG;
232
233 #include "rrd.h"
234
src/rrd.h
+1 -1
@@ -13,7 +13,7 @@ extern int rrd_default_history_entries;
13 // set to zero to disable this feature
14 extern int rrd_delete_unupdated_dimensions;
15
16 -#define RRD_ID_LENGTH_MAX 1024
16 +#define RRD_ID_LENGTH_MAX 400
17
18 #define RRDSET_MAGIC "NETDATA RRD SET FILE V018"
19 #define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V018"