Alerta.io notification improvements (#4576)
* Enhance support of Alerta notifications * use 'indeterminate' severity as catch-all * move $family to "resource" * set "service" to 'Netdata' not $host * set "group" to "Performance' * set "value" to $value_sting not $alarm * add $alarm_id as a "tag" * add $roles, $name, $chart, $familty and $src as "attributes" * don't set auth header unless $ALERTA_API_KEY is set * log suppressed alerts (202 Accepted) as info not error * use $chart as resource for httpcheck alarms * Update alerta wiki page
Nick Satterly committed
Nov 10, 2018 at 02:28 UTC
521136c74cd1e3a7aff45099155335ac51fc9a41
2 files changed
+84
-211
health/notifications/alarm-notify.sh.in
+44
-15
@@ -1592,39 +1592,68 @@ EOF
1592
# alerta sender
1593
1594
send_alerta() {
1595
- local webhook="${1}" channels="${2}" httpcode sent=0 channel severity content
1595
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
1596
1597
[ "${SEND_ALERTA}" != "YES" ] && return 1
1598
1599
case "${status}" in
1600
- WARNING) severity="warning" ;;
1600
CRITICAL) severity="critical" ;;
1601
+ WARNING) severity="warning" ;;
1602
CLEAR) severity="cleared" ;;
1603
- *) severity="unknown" ;;
1603
+ *) severity="indeterminate" ;;
1604
esac
1605
1606
- info=$( echo -n ${info})
1606
+ if [[ "${chart}" == httpcheck* ]]
1607
+ then
1608
+ resource=$chart
1609
+ event=$name
1610
+ else
1611
+ resource="${host}:${family}"
1612
+ event="${chart}.${name}"
1613
+ fi
1614
1608
- # the "event" property must be unique and repetible between states to let alerta do automatic correlation using severity value
1615
for channel in ${channels}
1616
do
1611
- content="{"
1612
- content="$content \"environment\": \"${channel}\","
1613
- content="$content \"service\": [\"${host}\"],"
1614
- content="$content \"resource\": \"${host}\","
1615
- content="$content \"event\": \"${name}.${chart} (${family})\","
1616
- content="$content \"severity\": \"${severity}\","
1617
- content="$content \"value\": \"${alarm}\","
1618
- content="$content \"text\": \"${info}\""
1619
- content="$content }"
1617
+ payload="$(cat <<EOF
1618
+ {
1619
+ "resource": "${resource}",
1620
+ "event": "${event}",
1621
+ "environment": "${channel}",
1622
+ "severity": "${severity}",
1623
+ "service": ["Netdata"],
1624
+ "group": "Performance",
1625
+ "value": "${value_string}",
1626
+ "text": "${info}",
1627
+ "tags": ["alarm_id:${alarm_id}"],
1628
+ "attributes": {
1629
+ "roles": "${roles}",
1630
+ "name": "${name}",
1631
+ "chart": "${chart}",
1632
+ "family": "${family}",
1633
+ "source": "${src}",
1634
+ "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
1635
+ },
1636
+ "origin": "netdata/${this_host}",
1637
+ "type": "netdataAlarm",
1638
+ "rawData": "${BASH_ARGV[@]}"
1639
+ }
1640
+EOF
1641
+ )"
1642
1643
+ if [[ -n "${ALERTA_API_KEY}" ]]
1644
+ then
1645
+ auth="Key ${ALERTA_API_KEY}"
1646
+ fi
1647
1622
- httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: Key $ALERTA_API_KEY" -d "$content" )
1648
+ httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
1649
1650
if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
1651
then
1652
info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
1653
sent=$((sent + 1))
1654
+ elif [[ "${httpcode}" = "202" ]]
1655
+ then
1656
+ info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
1657
else
1658
error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
1659
fi
health/notifications/alerta/README.md
+40
-196
@@ -1,207 +1,50 @@
1
# alerta.io notifications
2
3
-The alerta monitoring system is a tool used to consolidate and de-duplicate alerts from multiple sources for quick ‘at-a-glance’ visualisation. With just one system you can monitor alerts from many other monitoring tools on a single screen.
3
+The [Alerta](https://alerta.io) monitoring system is a tool used to
4
+consolidate and de-duplicate alerts from multiple sources for quick
5
+‘at-a-glance’ visualisation. With just one system you can monitor
6
+alerts from many other monitoring tools on a single screen.
7
8
9
7
-When receiving alerts from multiple sources you can quickly become overwhelmed. With Alerta any alert with the same environment and resource is considered a duplicate if it has the same severity. If it has a different severity it is correlated so that you only see the most recent one. Awesome.
10
+Netadata alarms can be sent to Alerta so you can see in one place
11
+alerts coming from many Netdata hosts or also from a multi-host
12
+Netadata configuration. The big advantage over other notifications
13
+systems is that there is a main view of all active alarms with
14
+the most recent state, and it is also possible to view alarm history.
15
9
-main site http://www.alerta.io
16
+## Deploying Alerta
17
11
-We can send Netadata alarms to Alerta so yo can see in one place alerts coming from many Netdata hosts or also from a multihost Netadata configuration.\
12
-The big advantage over other notifications method is that you have in a main view all active alarms with only las state, but you can also search history.
18
+It is recommended to set up the server in a separated server, VM or
19
+container. If you have other Nginx or Apache server in your organization,
20
+it is recommended to proxy to this new server.
21
14
-## Setting up an Alerta server with Ubuntu 16.04
22
+The easiest way to install Alerta is to use the Docker image available
23
+on [Docker hub][1]. Alternatively, follow the ["getting started"][2]
24
+tutorial to deploy Alerta to an Ubuntu server. More advanced
25
+configurations are out os scope of this tutorial but information
26
+about different deployment scenaries can be found in the [docs][3].
27
16
-Here we will set a basic Alerta server to test it with Netdata alerts.\
17
-More advanced configurations are out os scope of this tutorial.
28
+[1]: https://hub.docker.com/r/alerta/alerta-web/
29
+[2]: http://alerta.readthedocs.io/en/latest/gettingstarted/tutorial-1-deploy-alerta.html
30
+[3]: http://docs.alerta.io/en/latest/deployment.html
31
19
-source: http://alerta.readthedocs.io/en/latest/gettingstarted/tutorial-1-deploy-alerta.html
32
+## Send alarms to Alerta
33
21
-I recommend to set up the server in a separated server, VM or container.\
22
-If you have other Nginx or Apache server in your organization, I recommend to proxy to this new server.
34
+Step 1. Create an API key (if authentication is enabled)
35
24
-Set us as root for easiest working
25
-```
26
-sudo su
27
-cd
28
-```
29
-
30
-Install Mongodb https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
31
-```
32
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
33
-echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list
34
-apt-get update
35
-apt-get install -y mongodb-org
36
-systemctl enable mongod
37
-systemctl start mongod
38
-systemctl status mongod
39
-```
40
-
41
-Install Nginx and Alerta uwsgi
42
-```
43
-apt-get install -y python-pip python-dev nginx
44
-pip install alerta-server uwsgi
45
-```
36
+You will need an API key to send messages from any source, if
37
+Alerta is configured to use authentication (recommended). To
38
+create an API key go to "Configuration -> API Keys" and create
39
+a new API key called "netdata" with `write:alerts` permission.
40
47
-Install web console
48
-```
49
-cd /var/www/html
50
-mkdir alerta
51
-cd alerta
52
-wget -q -O - https://github.com/alerta/angular-alerta-webui/tarball/master | tar zxf -
53
-mv alerta*/app/* .
54
-cd
55
-```
56
-## Services configuration
57
-
58
-Create a wsgi python file
59
-```
60
-nano /var/www/wsgi.py
61
-```
62
-fill with
63
-```
64
-from alerta import app
65
-```
66
-Create uWsgi configuration file
67
-```
68
-nano /etc/uwsgi.ini
69
-```
70
-fill with
71
-```
72
-[uwsgi]
73
-chdir = /var/www
74
-mount = /alerta/api=wsgi.py
75
-callable = app
76
-manage-script-name = true
77
-
78
-master = true
79
-processes = 5
80
-logger = syslog:alertad
81
-
82
-socket = /tmp/uwsgi.sock
83
-chmod-socket = 664
84
-uid = www-data
85
-gid = www-data
86
-vacuum = true
87
-
88
-die-on-term = true
89
-```
90
-Create a systemd configuration file
91
-```
92
-nano /etc/systemd/system/uwsgi.service
93
-```
94
-fill with
95
-```
96
-[Unit]
97
-Description=uWSGI service
98
-
99
-[Service]
100
-ExecStart=/usr/local/bin/uwsgi --ini /etc/uwsgi.ini
101
-
102
-[Install]
103
-WantedBy=multi-user.target
104
-```
105
-enable service
106
-```
107
-systemctl start uwsgi
108
-systemctl status uwsgi
109
-systemctl enable uwsgi
110
-```
111
-Configure nginx to serve Alerta as a uWsgi application on /alerta/api
112
-```
113
-nano /etc/nginx/sites-enabled/default
114
-```
115
-fill with
116
-```
117
-server {
118
- listen 80 default_server;
119
- listen [::]:80 default_server;
120
-
121
- location /alerta/api { try_files $uri @alerta/api; }
122
- location @alerta/api {
123
- include uwsgi_params;
124
- uwsgi_pass unix:/tmp/uwsgi.sock;
125
- proxy_set_header Host $host:$server_port;
126
- proxy_set_header X-Real-IP $remote_addr;
127
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
128
- }
129
-
130
- location / {
131
- root /var/www/html;
132
- }
133
-}
134
-```
135
-restart nginx
136
-```
137
-service nginx restart
138
-```
139
-## Config web console
140
-```
141
-nano /var/www/html/config.js
142
-```
143
-fill with
144
-```
145
-'use strict';
146
-
147
-angular.module('config', [])
148
- .constant('config', {
149
- 'endpoint' : "/alerta/api",
150
- 'provider' : "basic",
151
- 'colors' : {},
152
- 'severity' : {},
153
- 'audio' : {}
154
- });
155
-```
156
-
157
-## Config Alerta server
158
-
159
-source: http://alerta.readthedocs.io/en/latest/configuration.html
160
-
161
-Create a random string to use as SECRET_KEY
162
-```
163
-cat /dev/urandom | tr -dc A-Za-z0-9_\!\@\#\$\%\^\&\*\(\)-+= | head -c 32 && echo
164
-```
165
-will output something like
166
-```
167
-0pv8Bw7VKfW6avDAz_TqzYPme_fYV%7g
168
-```
169
-Edit alertad.conf
170
-```
171
-nano /etc/alertad.conf
172
-```
173
-fill with (take care about all single quotes)
174
-```
175
-BASE_URL='/alerta/api'
176
-AUTH_REQUIRED=True
177
-SECRET_KEY='0pv8Bw7VKfW6avDAz_TqzYPme_fYV%7g'
178
-ADMIN_USERS=['<here put you email for future login>']
179
-```
180
-
181
-restart
182
-```
183
-systemctl restart uwsgi
184
-```
185
-
186
-* go to console to http://yourserver/alerta/
187
-* go to Login -> Create an account
188
-* use your email for login so and administrative account will be created
189
-
190
-## create an API KEY
191
-
192
-You need an API KEY to send messages from any source.\
193
-To create an API KEY go to Configuration -> Api Keys\
194
-Then create a API KEY with write permisions.
195
-
196
-## configure Netdata to send alarms to Alerta
41
+Step 2. configure Netdata to send alarms to Alerta
42
43
On your system run:
44
200
-```
201
-/etc/netdata/edit-config health_alarm_notify.conf
202
-```
45
+ $ /etc/netdata/edit-config health_alarm_notify.conf
46
204
-and set
47
+and modify the file as below:
48
49
```
50
# enable/disable sending alerta notifications
@@ -214,7 +57,7 @@ ALERTA_WEBHOOK_URL="http://yourserver/alerta/api"
57
58
# Login with an administrative user to you Alerta server and create an API KEY
59
# with write permissions.
217
-ALERTA_API_KEY="you last created API KEY"
60
+ALERTA_API_KEY="INSERT_YOUR_API_KEY_HERE"
61
62
# you can define environments in /etc/alertad.conf option ALLOWED_ENVIRONMENTS
63
# standard environments are Production and Development
@@ -225,12 +68,13 @@ DEFAULT_RECIPIENT_ALERTA="Production"
68
69
## Test alarms
70
228
-We can test alarms with standard
229
-```
230
-sudo su -s /bin/bash netdata
231
-/opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test
232
-exit
233
-```
234
-But the problem is that Netdata will send 3 alarms, and because last alarm is "CLEAR" you will not se them in main Alerta page, you need to select to see "closed" alarma in top-right lookup.
71
+We can test alarms using the standard approach:
72
+
73
+ $ /opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test
74
+
75
+Note: Netdata will send 3 alarms, and because last alarm is "CLEAR"
76
+you will not se them in main Alerta page, you need to select to see
77
+"closed" alarma in top-right lookup. A little change in `alarm-notify.sh`
78
+that let us test each state one by one will be useful.
79
236
-A little change in alarm-notify.sh that let us test each state one by one will be useful.
\ No newline at end of file
80
+For more information see [https://docs.alerta.io](https://docs.alerta.io)