each alarm now has one status with the following possible values: UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
Costa Tsaousis committed
Aug 18, 2016 at 01:50 UTC
228554b027330e9806cf0e1e0eeff1409b2ed9d6
3 files changed
+132
-107
plugins.d/alarm.sh
+66
-49
@@ -1,5 +1,7 @@
1
#!/usr/bin/env bash
2
3
+me="${0}"
4
+
5
sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
6
if [ -z "${sendmail}" ]
7
then
@@ -8,20 +10,25 @@ fi
10
11
sendmail_from_pipe() {
12
"${sendmail}" -t
11
-}
13
13
-type="${1}" # WARNING or CRITICAL
14
-name="${2}" # the name of the alarm, as given in netdata health.d entries
15
-chart="${3}" # the name of the chart (type.id)
16
-status="${4}" # the current status
17
-old_status="${5}" # the previous status
18
-value="${6}" # the current value
19
-old_value="${7}" # the previous value
20
-src="${8}" # the line number and file the alarm has been configured
21
-duration="${9}" # the duration in seconds the previous state took
14
+ if [ $? -eq 0 ]
15
+ then
16
+ echo >&2 "${me}: Sent notification email for ${status} on '${chart}.${name}'"
17
+ return 0
18
+ else
19
+ echo >&2 "${me}: FAILED to send notification email for ${status} on '${chart}.${name}'"
20
+ return 1
21
+ fi
22
+}
23
23
-# don't do anything if this is not RAISED or OFF
24
-[ "${status}" != "RAISED" -a "${status}" != "OFF" ] && exit 0
24
+name="${1}" # the name of the alarm, as given in netdata health.d entries
25
+chart="${2}" # the name of the chart (type.id)
26
+status="${3}" # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
27
+old_status="${4}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
28
+value="${5}" # the current value
29
+old_value="${6}" # the previous value
30
+src="${7}" # the line number and file the alarm has been configured
31
+duration="${8}" # the duration in seconds the previous state took
32
33
# get the system hostname
34
hostname="$(hostname)"
@@ -76,54 +83,64 @@ duration4human() {
83
fi
84
}
85
79
-if [ "${old_status}" = "RAISED" -a "${status}" = "OFF" ]
80
-then
81
- # an alarm that is now OFF
82
- severity="${type} recovered"
83
- raised_for="<br/>(was in ${type,,} state for $(duration4human ${duration}))"
84
-else
85
- # an alarm that is now RAISED
86
- severity="${type}"
87
- raised_for=""
88
-fi
86
+severity="${status}"
87
+raised_for="<br/>(was in ${old_status,,} for $(duration4human ${duration}))"
88
+status_message="status unknown"
89
+color="grey"
90
+alarm="${name} = ${value}"
91
90
-# prepare the title
92
+# prepare the title based on status
93
case "${status}" in
92
- RAISED)
93
- if [ "${type}" = "CRITICAL" ]
94
- then
95
- # CRITICAL - red
96
- status_message="is critical"
97
- color="#ca414b"
98
- else
99
- # WARNING - yellow
100
- status_message="needs attention"
101
- color="#caca4b"
102
- fi
94
+ CRITICAL)
95
+ status_message="is critical"
96
+ color="#ca414b"
97
+ ;;
98
+
99
+ WARNING)
100
+ status_message="needs attention"
101
+ color="#caca4b"
102
;;
103
105
- OFF)
106
- if [ "${type}" = "CRITICAL" ]
107
- then
108
- # CRITICAL
109
- status_message="recovered"
110
- else
111
- # WARNING
112
- status_message="back to normal"
113
- fi
104
+ CLEAR)
105
+ status_message="recovered"
106
color="#77ca6d"
115
- ;;
107
117
- *)
118
- status_message="status unknown"
119
- color="grey"
108
+ # don't show the value when the status is CLEAR
109
+ # for certain alarms, this value might not have any meaning
110
+ alarm="${name}"
111
;;
112
esac
113
114
+if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
115
+then
116
+ # don't do anything if this is not WARNING, CRITICAL or CLEAR
117
+ echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}'"
118
+ exit 0
119
+elif [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
120
+then
121
+ # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
122
+ echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}' (last status was ${old_status})"
123
+ exit 0
124
+elif [ "${status}" = "CLEAR" ]
125
+then
126
+ severity="Recovered from ${old_status}"
127
+
128
+elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
129
+then
130
+ severity="Escalated to ${status}"
131
+
132
+elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
133
+then
134
+ severity="Demoted to ${status}"
135
+
136
+else
137
+ raised_for=
138
+fi
139
+
140
# send the email
141
cat <<EOF | sendmail_from_pipe
142
To: root
126
-Subject: ${type} ${hostname} ${status_message} - ${chart}.${name}
143
+Subject: ${hostname} ${status_message} - ${chart}.${name}
144
Content-Type: text/html
145
146
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -165,7 +182,7 @@ Content-Type: text/html
182
<tr style="margin:0;padding:0">
183
<td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px"
184
align="left" valign="top">
168
- <span>${name} = ${value}</span>
185
+ <span>${alarm}</span>
186
<span style="display:block;color:#666666;font-size:12px;font-weight:300;line-height:1;text-transform:uppercase">Alarm</span>
187
</td>
188
</tr>
src/health.c
+52
-53
@@ -1406,7 +1406,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
1406
static inline int rrdcalc_value2status(calculated_number n) {
1407
if(isnan(n)) return RRDCALC_STATUS_UNDEFINED;
1408
if(n) return RRDCALC_STATUS_RAISED;
1409
- return RRDCALC_STATUS_OFF;
1409
+ return RRDCALC_STATUS_CLEAR;
1410
}
1411
1412
static inline const char *rrdcalc_status2string(int status) {
@@ -1417,23 +1417,16 @@ static inline const char *rrdcalc_status2string(int status) {
1417
case RRDCALC_STATUS_UNDEFINED:
1418
return "UNDEFINED";
1419
1420
+ case RRDCALC_STATUS_CLEAR:
1421
+ return "CLEAR";
1422
+
1423
case RRDCALC_STATUS_RAISED:
1424
return "RAISED";
1425
1423
- case RRDCALC_STATUS_OFF:
1424
- return "OFF";
1425
-
1426
- default:
1427
- return "UNKNOWN";
1428
- }
1429
-}
1430
-
1431
-static inline const char *alarm_entry_type2string(int type) {
1432
- switch(type) {
1433
- case ALARM_ENTRY_TYPE_WARNING:
1426
+ case RRDCALC_STATUS_WARNING:
1427
return "WARNING";
1428
1436
- case ALARM_ENTRY_TYPE_CRITICAL:
1429
+ case RRDCALC_STATUS_CRITICAL:
1430
return "CRITICAL";
1431
1432
default:
@@ -1442,7 +1435,7 @@ static inline const char *alarm_entry_type2string(int type) {
1435
}
1436
1437
static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1445
- if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_OFF)
1438
+ if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)
1439
return;
1440
1441
char buffer[FILENAME_MAX + 1];
@@ -1451,9 +1444,8 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1444
const char *exec = ae->exec;
1445
if(!exec) exec = health_default_exec;
1446
1454
- snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u'",
1447
+ snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u'",
1448
exec,
1456
- alarm_entry_type2string(ae->type),
1449
ae->name,
1450
ae->chart?ae->chart:"NOCAHRT",
1451
rrdcalc_status2string(ae->new_status),
@@ -1479,10 +1471,9 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1471
}
1472
1473
static inline void health_process_notifications(ALARM_ENTRY *ae) {
1482
- info("Health alarm '%s.%s' = %0.2Lf - %s changed status from %s to %s",
1474
+ info("Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s",
1475
ae->chart?ae->chart:"NOCHART", ae->name,
1476
ae->new_value,
1485
- alarm_entry_type2string(ae->type),
1477
rrdcalc_status2string(ae->old_status),
1478
rrdcalc_status2string(ae->new_status)
1479
);
@@ -1490,7 +1481,7 @@ static inline void health_process_notifications(ALARM_ENTRY *ae) {
1481
health_alarm_execute(ae);
1482
}
1483
1493
-static inline void health_alarm_log(time_t when, int type,
1484
+static inline void health_alarm_log(time_t when,
1485
const char *name, const char *chart, const char *exec,
1486
time_t duration,
1487
calculated_number old_value, calculated_number new_value,
@@ -1499,22 +1490,41 @@ static inline void health_alarm_log(time_t when, int type,
1490
) {
1491
ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
1492
ae->name = strdupz(name);
1502
- if(chart) ae->chart = strdupz(chart);
1493
+ ae->hash_name = simple_hash(ae->name);
1494
+
1495
+ if(chart) {
1496
+ ae->chart = strdupz(chart);
1497
+ ae->hash_chart = simple_hash(ae->chart);
1498
+ }
1499
+
1500
if(exec) ae->exec = strdupz(exec);
1501
if(source) ae->source = strdupz(source);
1502
1503
ae->id = health_log.nextid++;
1504
ae->when = when;
1508
- ae->type = type;
1505
ae->old_value = old_value;
1506
ae->new_value = new_value;
1507
ae->old_status = old_status;
1508
ae->new_status = new_status;
1509
ae->duration = duration;
1510
+
1511
// link it
1512
ae->next = health_log.alarms;
1513
health_log.alarms = ae;
1514
health_log.count++;
1515
+
1516
+ // match previous alarms
1517
+ ALARM_ENTRY *t;
1518
+ for(t = health_log.alarms ; t ; t = t->next) {
1519
+ if(t != ae &&
1520
+ t->hash_name == ae->hash_name &&
1521
+ t->hash_chart == ae->hash_chart &&
1522
+ !strcmp(t->name, ae->name) &&
1523
+ t->chart && ae->chart && !strcmp(t->chart, ae->chart)) {
1524
+ t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1525
+ t->updated_by = ae;
1526
+ }
1527
+ }
1528
}
1529
1530
static inline void health_alarm_log_process(void) {
@@ -1524,7 +1534,8 @@ static inline void health_alarm_log_process(void) {
1534
for(ae = health_log.alarms; ae ;ae = ae->next) {
1535
if(last_processed >= ae->id) break;
1536
1527
- if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)) {
1537
+ if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
1538
+ !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)) {
1539
ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
1540
health_process_notifications(ae);
1541
}
@@ -1555,34 +1566,6 @@ static inline void health_alarm_log_process(void) {
1566
}
1567
}
1568
1558
-static inline void rrdcalc_check_warning_event(RRDCALC *rc) {
1559
- calculated_number n = rc->warning->result;
1560
-
1561
- int old_status = rc->warning_status;
1562
- int new_status = rrdcalc_value2status(n);
1563
-
1564
- if(new_status != old_status) {
1565
- time_t now = time(NULL);
1566
- health_alarm_log(time(NULL), ALARM_ENTRY_TYPE_WARNING, rc->name, rc->rrdset->id, rc->exec, now - rc->last_status_change, rc->old_value, rc->value, old_status, new_status, rc->source);
1567
- rc->last_status_change = now;
1568
- rc->warning_status = new_status;
1569
- }
1570
-}
1571
-
1572
-static inline void rrdcalc_check_critical_event(RRDCALC *rc) {
1573
- calculated_number n = rc->critical->result;
1574
-
1575
- int old_status = rc->critical_status;
1576
- int new_status = rrdcalc_value2status(n);
1577
-
1578
- if(new_status != old_status) {
1579
- time_t now = time(NULL);
1580
- health_alarm_log(time(NULL), ALARM_ENTRY_TYPE_CRITICAL, rc->name, rc->rrdset->id, rc->exec, now - rc->last_status_change, rc->old_value, rc->value, old_status, new_status, rc->source);
1581
- rc->last_status_change = now;
1582
- rc->critical_status = new_status;
1583
- }
1584
-}
1585
-
1569
void *health_main(void *ptr) {
1570
(void)ptr;
1571
@@ -1722,6 +1705,9 @@ void *health_main(void *ptr) {
1705
if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
1706
continue;
1707
1708
+ int warning_status = RRDCALC_STATUS_UNDEFINED;
1709
+ int critical_status = RRDCALC_STATUS_UNDEFINED;
1710
+
1711
if(unlikely(rc->warning)) {
1712
if(unlikely(!expression_evaluate(rc->warning))) {
1713
// calculation failed
@@ -1747,9 +1733,8 @@ void *health_main(void *ptr) {
1733
buffer_tostring(rc->warning->error_msg),
1734
rc->source
1735
);
1736
+ warning_status = rrdcalc_value2status(rc->warning->result);
1737
}
1751
-
1752
- rrdcalc_check_warning_event(rc);
1738
}
1739
1740
if(unlikely(rc->critical)) {
@@ -1777,9 +1762,23 @@ void *health_main(void *ptr) {
1762
buffer_tostring(rc->critical->error_msg),
1763
rc->source
1764
);
1765
+
1766
+ critical_status = rrdcalc_value2status(rc->critical->result);
1767
}
1768
+ }
1769
+
1770
+ int status = RRDCALC_STATUS_UNDEFINED;
1771
+
1772
+ if(warning_status == RRDCALC_STATUS_RAISED)
1773
+ status = RRDCALC_STATUS_WARNING;
1774
+
1775
+ if(critical_status == RRDCALC_STATUS_RAISED)
1776
+ status = RRDCALC_STATUS_CRITICAL;
1777
1782
- rrdcalc_check_critical_event(rc);
1778
+ if(status != rc->status) {
1779
+ health_alarm_log(time(NULL), rc->name, rc->rrdset->id, rc->exec, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source);
1780
+ rc->last_status_change = now;
1781
+ rc->status = status;
1782
}
1783
1784
rc->last_updated = now;
src/health.h
+14
-5
@@ -106,8 +106,10 @@ typedef struct rrddimvar {
106
107
#define RRDCALC_STATUS_UNINITIALIZED 0
108
#define RRDCALC_STATUS_UNDEFINED -1
109
-#define RRDCALC_STATUS_OFF 1
109
+#define RRDCALC_STATUS_CLEAR 1
110
#define RRDCALC_STATUS_RAISED 2
111
+#define RRDCALC_STATUS_WARNING 3
112
+#define RRDCALC_STATUS_CRITICAL 4
113
114
#define RRDCALC_OPTION_DB_ERROR 0x00000001
115
#define RRDCALC_OPTION_DB_NAN 0x00000002
@@ -143,6 +145,7 @@ typedef struct rrdcalc {
145
EVAL_EXPRESSION *critical;
146
147
uint32_t rrdcalc_options;
148
+ int status;
149
int warning_status;
150
int critical_status;
151
@@ -203,25 +206,31 @@ typedef struct rrdcalctemplate {
206
207
#define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
208
206
-#define ALARM_ENTRY_TYPE_WARNING 1
207
-#define ALARM_ENTRY_TYPE_CRITICAL 2
208
-
209
#define HEALTH_ENTRY_NOTIFICATIONS_PROCESSED 0x00000001
210
+#define HEALTH_ENTRY_NOTIFICATIONS_UPDATED 0x00000002
211
212
typedef struct alarm_entry {
213
uint32_t id;
214
+
215
time_t when;
216
time_t duration;
215
- int type;
217
+
218
char *name;
219
+ uint32_t hash_name;
220
+
221
char *chart;
222
+ uint32_t hash_chart;
223
+
224
char *exec;
225
char *source;
226
calculated_number old_value;
227
calculated_number new_value;
228
int old_status;
229
int new_status;
230
+
231
uint32_t notifications;
232
+
233
+ struct alarm_entry *updated_by;
234
struct alarm_entry *next;
235
} ALARM_ENTRY;
236