added push alarm notification via Pushover
Jan Arnold committed
Sep 8, 2016 at 14:44 UTC
7c1af9474008209239f251b932f2cf33670fe23b
2 files changed
+246
conf.d/health_pushover_tokens.conf
new
+16
@@ -0,0 +1,16 @@
1
+# Configuration for pushover alarms
2
+# by Jan Arnold
3
+# v0.1
4
+# 2016-09-12
5
+#
6
+# This is the configuration file for alarm-pushover.sh, a shell script for using the Pushover (https://pushover.net)
7
+# push service with netdata. Please copy this to the netdata config directory, which by default is "/etc/netdata/".
8
+# Once Pushover is installed on your phone or computer, you need to create an account to receive a user token.
9
+# This is needed so the push reaches your device.
10
+#
11
+# Further help for pushover: https://pushover.net/faq
12
+#
13
+# Generate an apptoken and a usertoken at https://pushover.net and insert them here
14
+
15
+APPTOKEN=""
16
+USERTOKEN=""
plugins.d/alarm-pushover.sh
new
+230
@@ -0,0 +1,230 @@
1
+#!/usr/bin/env bash
2
+# based on alarm-email.sh by netdata
3
+# by Jan Arnold
4
+# v0.1
5
+# 2016-09-12
6
+#
7
+# This is a shell script for using the Pushover (https://pushover.net) push service.
8
+# Copy this script to the netdata plugin folder which by default is "/usr/libexec/netdata/plugins.d/".
9
+# In the config file for the alarm you want to use it for add:
10
+# "exec: /usr/libexec/netdata/plugins.d/alarm-pushover.sh"
11
+# Once Pushover client is installed on your phone or computer, you need to create an account to receive a
12
+# user token. This is needed so the push reaches your device.
13
+#
14
+# Generate an apptoken at https://pushover.net and insert that and the usertoken into the
15
+# "healt_pushover_tokens.conf" file which has to reside in the netdata config dir which by
16
+# default is "/etc/netdata/".
17
+#
18
+# Further help for pushover: https://pushover.net/faq
19
+
20
+me="${0}"
21
+
22
+curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
23
+if [ -z "${curl}" ]
24
+then
25
+ echo >&2 "I cannot send push notifications - there is no curl command available."
26
+ exit 1
27
+fi
28
+
29
+if [ -f "${NETDATA_CONFIG_DIR}/health_pushover_tokens.conf" ]
30
+ then
31
+ source "${NETDATA_CONFIG_DIR}/health_pushover_tokens.conf"
32
+else
33
+ echo >&2 "${me}: FAILED to source ${NETDATA_CONFIG_DIR}/health_pushover_tokens.conf for push token"
34
+ exit 1
35
+fi
36
+
37
+if ! [ "$APPTOKEN" ]
38
+ then
39
+ echo >&2 "${me}: FAILED - Please set pushover APPTOKEN in ${NETDATA_CONFIG_DIR}/health_pushover_tokens.conf"
40
+ exit 1
41
+fi
42
+
43
+if ! [ "$USERTOKEN" ]
44
+ then
45
+ echo >&2 "${me}: FAILED - Please set pushover USERTOKEN in ${NETDATA_CONFIG_DIR}/health_pushover_tokens.conf"
46
+ exit 1
47
+fi
48
+
49
+curl_pushover() {
50
+ httpcode=$(curl --write-out %{http_code} --silent --output /dev/null \
51
+ --form-string "token=$APPTOKEN" \
52
+ --form-string "user=$USERTOKEN" \
53
+ --form-string "html=1" \
54
+ --form-string "title=$TITLE" \
55
+ --form-string "message=$MESSAGE" \
56
+ https://api.pushover.net/1/messages.json)
57
+
58
+ if [ "$httpcode" == "200" ]
59
+ then
60
+ echo >&2 "${me}: Sent notification push for ${status} on '${chart}.${name}'"
61
+ return 0
62
+ else
63
+ echo >&2 "${me}: FAILED to send notification push for ${status} on '${chart}.${name}'"
64
+ return 1
65
+ fi
66
+}
67
+
68
+recipient="${1}" # the recepient of the push
69
+hostname="${2}" # the hostname this event refers to
70
+unique_id="${3}" # the unique id of this event
71
+alarm_id="${4}" # the unique id of the alarm that generated this event
72
+event_id="${5}" # the incremental id of the event, for this alarm
73
+when="${6}" # the timestamp this event occured
74
+name="${7}" # the name of the alarm, as given in netdata health.d entries
75
+chart="${8}" # the name of the chart (type.id)
76
+family="${9}" # the family of the chart
77
+status="${10}" # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
78
+old_status="${11}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
79
+value="${12}" # the current value
80
+old_value="${13}" # the previous value
81
+src="${14}" # the line number and file the alarm has been configured
82
+duration="${15}" # the duration in seconds the previous state took
83
+non_clear_duration="${16}" # the total duration in seconds this is non-clear
84
+units="${17}" # the units of the value
85
+info="${18}" # a short description of the alarm
86
+
87
+[ ! -z "${info}" ] && info=" <small><br/>${info}</small>"
88
+
89
+# get the system hostname
90
+[ -z "${hostname}" ] && hostname="${NETDATA_HOSTNAME}"
91
+[ -z "${hostname}" ] && hostname="${NETDATA_REGISTRY_HOSTNAME}"
92
+[ -z "${hostname}" ] && hostname="$(hostname 2>/dev/null)"
93
+
94
+goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"
95
+
96
+date="$(date --date=@${when} 2>/dev/null)"
97
+[ -z "${date}" ] && date="$(date 2>/dev/null)"
98
+
99
+# convert a duration in seconds, to a human readable duration
100
+# using DAYS, MINUTES, SECONDS
101
+duration4human() {
102
+ local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second"
103
+ d=$(( s / 86400 ))
104
+ s=$(( s - (d * 86400) ))
105
+ h=$(( s / 3600 ))
106
+ s=$(( s - (h * 3600) ))
107
+ m=$(( s / 60 ))
108
+ s=$(( s - (m * 60) ))
109
+
110
+ if [ ${d} -gt 0 ]
111
+ then
112
+ [ ${m} -ge 30 ] && h=$(( h + 1 ))
113
+ [ ${d} -gt 1 ] && ds="days"
114
+ [ ${h} -gt 1 ] && hs="hours"
115
+ if [ ${h} -gt 0 ]
116
+ then
117
+ echo "${d} ${ds} and ${h} ${hs}"
118
+ else
119
+ echo "${d} ${ds}"
120
+ fi
121
+ elif [ ${h} -gt 0 ]
122
+ then
123
+ [ ${s} -ge 30 ] && m=$(( m + 1 ))
124
+ [ ${h} -gt 1 ] && hs="hours"
125
+ [ ${m} -gt 1 ] && ms="minutes"
126
+ if [ ${m} -gt 0 ]
127
+ then
128
+ echo "${h} ${hs} and ${m} ${ms}"
129
+ else
130
+ echo "${h} ${hs}"
131
+ fi
132
+ elif [ ${m} -gt 0 ]
133
+ then
134
+ [ ${m} -gt 1 ] && ms="minutes"
135
+ [ ${s} -gt 1 ] && ss="seconds"
136
+ if [ ${s} -gt 0 ]
137
+ then
138
+ echo "${m} ${ms} and ${s} ${ss}"
139
+ else
140
+ echo "${m} ${ms}"
141
+ fi
142
+ else
143
+ [ ${s} -gt 1 ] && ss="seconds"
144
+ echo "${s} ${ss}"
145
+ fi
146
+}
147
+
148
+severity="${status}"
149
+raised_for="<br/><small>(was ${old_status,,} for $(duration4human ${duration}))</small>"
150
+status_message="status unknown"
151
+color="grey"
152
+alarm="${name} = ${value} ${units}"
153
+
154
+# prepare the title based on status
155
+case "${status}" in
156
+ CRITICAL)
157
+ status_message="is critical"
158
+ color="#ca414b"
159
+ ;;
160
+
161
+ WARNING)
162
+ status_message="needs attention"
163
+ color="#caca4b"
164
+ ;;
165
+
166
+ CLEAR)
167
+ status_message="recovered"
168
+ color="#77ca6d"
169
+
170
+ # don't show the value when the status is CLEAR
171
+ # for certain alarms, this value might not have any meaning
172
+ alarm="${name}"
173
+ ;;
174
+esac
175
+
176
+if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
177
+then
178
+ # don't do anything if this is not WARNING, CRITICAL or CLEAR
179
+ echo >&2 "${me}: not sending notification push for ${status} on '${chart}.${name}'"
180
+ exit 0
181
+elif [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
182
+then
183
+ # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
184
+ echo >&2 "${me}: not sending notification push for ${status} on '${chart}.${name}' (last status was ${old_status})"
185
+ exit 0
186
+elif [ "${status}" = "CLEAR" ]
187
+then
188
+ severity="Recovered from ${old_status}"
189
+ if [ $non_clear_duration -gt $duration ]
190
+ then
191
+ raised_for="<br/><small>(had issues for $(duration4human ${non_clear_duration}))</small>"
192
+ fi
193
+
194
+elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
195
+then
196
+ severity="Escalated to ${status}"
197
+ if [ $non_clear_duration -gt $duration ]
198
+ then
199
+ raised_for="<br/><small>(has issues for $(duration4human ${non_clear_duration}))</small>"
200
+ fi
201
+
202
+elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
203
+then
204
+ severity="Demoted to ${status}"
205
+ if [ $non_clear_duration -gt $duration ]
206
+ then
207
+ raised_for="<br/><small>(has issues for $(duration4human ${non_clear_duration}))</small>"
208
+ fi
209
+
210
+else
211
+ raised_for=
212
+fi
213
+
214
+# generate push message
215
+TITLE="${hostname} ${status_message} - ${chart}.${name}"
216
+MESSAGE="<font size="5" color=\"${color}\"><b>${hostname} ${status_message}</b></font>
217
+
218
+<b>${alarm}</b>${info}
219
+
220
+Chart: ${chart}
221
+Family: ${family}
222
+Severity: ${severity}
223
+Time: ${date}
224
+${raised_for}
225
+<a href=\"${goto_url}\">View Netdata</a>
226
+
227
+<small>The source of this alarm is line ${src}</small>"
228
+
229
+# send the push
230
+curl_pushover