alarm-email.sh renamed to alarm-notify.sh
Costa Tsaousis (ktsaou) committed
Sep 9, 2016 at 00:26 UTC
466978fb46b5fc3d976e41fa0425199e031a2665
5 files changed
+197
-46
conf.d/Makefile.am
+1
@@ -7,6 +7,7 @@ dist_config_DATA = \
7
apps_groups.conf \
8
charts.d.conf \
9
python.d.conf \
10
+ health_alarm_notify.conf \
11
health_email_recipients.conf \
12
$(NULL)
13
conf.d/health_alarm_notify.conf
renamed
plugins.d/Makefile.am
+1
@@ -9,6 +9,7 @@ dist_plugins_DATA = \
9
10
dist_plugins_SCRIPTS = \
11
alarm-email.sh \
12
+ alarm-notify.sh \
13
cgroup-name.sh \
14
charts.d.dryrun-helper.sh \
15
charts.d.plugin \
plugins.d/alarm-notify.sh
renamed
+194
-45
@@ -1,33 +1,12 @@
1
#!/usr/bin/env bash
2
3
-me="${0}"
4
-
5
-sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
6
-if [ -z "${sendmail}" ]
7
-then
8
- echo >&2 "I cannot send emails - there is no sendmail command available."
9
- exit 1
10
-fi
11
-
12
-default_recipient_for_all_roles="root"
13
-declare -A recipients=()
14
-if [ -f "${NETDATA_CONFIG_DIR}/health_email_recipients.conf" ]
15
- then
16
- source "${NETDATA_CONFIG_DIR}/health_email_recipients.conf"
17
-fi
3
+# (C) Costa Tsaousis
4
+# pushover support by Jan Arnold
5
19
-sendmail_from_pipe() {
20
- "${sendmail}" -t
6
+me="${0}"
7
22
- if [ $? -eq 0 ]
23
- then
24
- echo >&2 "${me}: Sent notification email for ${status} on '${chart}.${name}'"
25
- return 0
26
- else
27
- echo >&2 "${me}: FAILED to send notification email for ${status} on '${chart}.${name}'"
28
- return 1
29
- fi
30
-}
8
+# -----------------------------------------------------------------------------
9
+# parse command line parameters
10
11
recipient="${1}" # the recepient of the email
12
hostname="${2}" # the hostname this event refers to
@@ -48,24 +27,101 @@ non_clear_duration="${16}" # the total duration in seconds this is non-clear
27
units="${17}" # the units of the value
28
info="${18}" # a short description of the alarm
29
51
-to="${recipients[${recipient}]}"
52
-[ -z "${to}" ] && to="${default_recipient_for_all_roles}"
53
-[ -z "${to}" ] && to="root"
30
+# -----------------------------------------------------------------------------
31
+# screen statuses we don't need to send a notification
32
55
-[ ! -z "${info}" ] && info=" <small><br/>${info}</small>"
33
+# don't do anything if this is not WARNING, CRITICAL or CLEAR
34
+if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
35
+then
36
+ echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}'"
37
+ exit 1
38
+fi
39
+
40
+# don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
41
+if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
42
+then
43
+ echo >&2 "${me}: not sending notification for ${status} on '${chart}.${name}' (last status was ${old_status})"
44
+ exit 1
45
+fi
46
+
47
+# -----------------------------------------------------------------------------
48
+# load configuration
49
+
50
+# needed commands
51
+# if empty they will be searched in the system path
52
+curl=
53
+sendmail=
54
55
+# enable / disable features
56
+SEND_EMAIL="YES"
57
+SEND_PUSHOVER="YES"
58
+
59
+# pushover configs
60
+PUSHOVER_APP_TOKEN=
61
+DEFAULT_RECIPIENT_PUSHOVER=
62
+declare -A role_recipients_pushover=()
63
+
64
+# email configs
65
+DEFAULT_RECIPIENT_EMAIL="root"
66
+declare -A role_recipients_email=()
67
+
68
+if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
69
+ then
70
+ source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
71
+fi
72
+
73
+# -----------------------------------------------------------------------------
74
+# find the exact recipient per method
75
+
76
+to_email="${role_recipients_email[${recipient}]}"
77
+[ -z "${to_email}" ] && to_email="${DEFAULT_RECIPIENT_EMAIL}"
78
+[ -z "${to_email}" ] && to_email="root"
79
+
80
+to_pushover="${role_recipients_pushover[${recipient}]}"
81
+[ -z "${to_pushover}" ] && to_pushover="${DEFAULT_RECIPIENT_EMAIL}"
82
+[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
83
+
84
+# -----------------------------------------------------------------------------
85
+# verify the delivery methods supported
86
+
87
+[ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
88
+
89
+if [ "${SEND_PUSHOVER}" = "YES" -a -z "${curl}" ]
90
+ then
91
+ curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
92
+ [ -z "${curl}" ] && SEND_PUSHOVER="NO"
93
+fi
94
+
95
+if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
96
+ then
97
+ sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
98
+ [ -z "${sendmail}" ] && SEND_EMAIL="NO"
99
+fi
100
+
101
+# check that we have at least a method enabled
102
+if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" ]
103
+ then
104
+ echo >&2 "I don't have a means to send a notification. Sorry!"
105
+ exit 1
106
+fi
107
+
108
+# -----------------------------------------------------------------------------
109
# get the system hostname
110
+
111
[ -z "${hostname}" ] && hostname="${NETDATA_HOSTNAME}"
112
[ -z "${hostname}" ] && hostname="${NETDATA_REGISTRY_HOSTNAME}"
113
[ -z "${hostname}" ] && hostname="$(hostname 2>/dev/null)"
114
62
-goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"
115
+# -----------------------------------------------------------------------------
116
+# get the date the alarm happened
117
118
date="$(date --date=@${when} 2>/dev/null)"
119
[ -z "${date}" ] && date="$(date 2>/dev/null)"
120
121
+# -----------------------------------------------------------------------------
122
# convert a duration in seconds, to a human readable duration
123
# using DAYS, MINUTES, SECONDS
124
+
125
duration4human() {
126
local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second"
127
d=$(( s / 86400 ))
@@ -113,11 +169,82 @@ duration4human() {
169
fi
170
}
171
172
+# -----------------------------------------------------------------------------
173
+# email sender
174
+
175
+send_email() {
176
+ if [ "${SEND_EMAIL}" = "YES" ]
177
+ then
178
+
179
+ "${sendmail}" -t
180
+
181
+ if [ $? -eq 0 ]
182
+ then
183
+ echo >&2 "${me}: Sent notification email for ${status} on '${chart}.${name}'"
184
+ return 0
185
+ else
186
+ echo >&2 "${me}: FAILED to send notification email for ${status} on '${chart}.${name}'"
187
+ return 1
188
+ fi
189
+ fi
190
+
191
+ return 1
192
+}
193
+
194
+# -----------------------------------------------------------------------------
195
+# pushover sender
196
+
197
+send_pushover() {
198
+ local apptoken="${1}" usertoken="${2}" title="${3}" message="${4}" httpcode
199
+
200
+ if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertoken}" -a ! -z "${title}" -a ! -z "${message}" ]
201
+ then
202
+
203
+ httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \
204
+ --form-string "token=${apptoken}" \
205
+ --form-string "user=${usertoken}" \
206
+ --form-string "html=1" \
207
+ --form-string "title=${title}" \
208
+ --form-string "message=${message}" \
209
+ https://api.pushover.net/1/messages.json)
210
+
211
+ if [ "${httpcode}" == "200" ]
212
+ then
213
+ echo >&2 "${me}: Sent notification push for ${status} on '${chart}.${name}'"
214
+ return 0
215
+ else
216
+ echo >&2 "${me}: FAILED to send notification push for ${status} on '${chart}.${name}' with HTTP error code ${httpcode}."
217
+ return 1
218
+ fi
219
+ fi
220
+
221
+ return 1
222
+}
223
+
224
+
225
+# -----------------------------------------------------------------------------
226
+# prepare the content of the notification
227
+
228
+# description of the alarm
229
+[ ! -z "${info}" ] && info=" <small><br/>${info}</small>"
230
+
231
+# the url to send the user on click
232
+goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"
233
+
234
+# the severity of the alarm
235
severity="${status}"
236
+
237
+# the time the alarm was raised
238
raised_for="<br/><small>(was ${old_status,,} for $(duration4human ${duration}))</small>"
239
+
240
+# the key status message
241
status_message="status unknown"
242
+
243
+# the color of the alarm
244
color="grey"
120
-alarm="${name} = ${value} ${units}"
245
+
246
+# the alarm value
247
+alarm="${name//_/ } = ${value} ${units}"
248
249
# prepare the title based on status
250
case "${status}" in
@@ -141,17 +268,7 @@ case "${status}" in
268
;;
269
esac
270
144
-if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
145
-then
146
- # don't do anything if this is not WARNING, CRITICAL or CLEAR
147
- echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}'"
148
- exit 0
149
-elif [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
150
-then
151
- # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
152
- echo >&2 "${me}: not sending notification email for ${status} on '${chart}.${name}' (last status was ${old_status})"
153
- exit 0
154
-elif [ "${status}" = "CLEAR" ]
271
+if [ "${status}" = "CLEAR" ]
272
then
273
severity="Recovered from ${old_status}"
274
if [ $non_clear_duration -gt $duration ]
@@ -179,9 +296,30 @@ else
296
raised_for=
297
fi
298
299
+
300
+# -----------------------------------------------------------------------------
301
+# send the pushover
302
+
303
+send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${hostname} ${status_message} - ${chart}.${name}" "<font size="5" color=\"${color}\"><b>${hostname} ${status_message}</b></font>
304
+
305
+<b>${alarm}</b>${info}
306
+
307
+Chart: ${chart}
308
+Family: ${family}
309
+Severity: ${severity}
310
+Time: ${date}
311
+${raised_for}
312
+<a href=\"${goto_url}\">View Netdata</a>
313
+
314
+<small>The source of this alarm is line ${src}</small>"
315
+
316
+SENT_PUSHOVER=$?
317
+
318
+# -----------------------------------------------------------------------------
319
# send the email
183
-cat <<EOF | sendmail_from_pipe
184
-To: ${to}
320
+
321
+cat <<EOF | send_email
322
+To: ${to_email}
323
Subject: ${hostname} ${status_message} - ${chart}.${name}
324
Content-Type: text/html
325
@@ -282,3 +420,14 @@ Content-Type: text/html
420
</body>
421
</html>
422
EOF
423
+
424
+SENT_EMAIL=$?
425
+
426
+# -----------------------------------------------------------------------------
427
+# let netdata know
428
+
429
+# we did send somehting
430
+[ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 ] && exit 0
431
+
432
+# we did not send anything
433
+exit 1
src/health.c
+1
-1
@@ -2,7 +2,7 @@
2
3
#define RRDVAR_MAX_LENGTH 1024
4
5
-static const char *health_default_exec = PLUGINS_DIR "/alarm-email.sh";
5
+static const char *health_default_exec = PLUGINS_DIR "/alarm-notify.sh";
6
static const char *health_default_recipient = "root";
7
int health_enabled = 1;
8