Health Alarm to Dynatrace Event implementation (#8476)
* Health Alarm to Dynatrace Event implementation * Update health/notifications/dynatrace/README.md Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com> * Removing unwanted " * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Update health/notifications/dynatrace/README.md Co-Authored-By: Joel Hans <joel.g.hans@gmail.com> * Implementation of https://github.com/netdata/netdata/pull/8476/checks?check_run_id=533657899 * Implemented https://github.com/netdata/netdata/pull/8476/checks?check_run_id=534740578 Co-authored-by: Patti Short <35278231+shortpatti@users.noreply.github.com> Co-authored-by: Joel Hans <joel.g.hans@gmail.com>
Illumine IT Consulting committed
Apr 6, 2020 at 19:22 UTC
3c4cb607d6b77b9fd9076dd6662d3bc13975c871
4 files changed
+170
-4
health/notifications/alarm-notify.sh.in
+79
-3
@@ -35,6 +35,7 @@
35
# - Microsoft Team notification by @tioumen
36
# - RocketChat notifications by @Hermsi1337 #3777
37
# - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
38
+# - Dynatrace Event by @illumine
39
40
# -----------------------------------------------------------------------------
41
# testing notifications
@@ -170,6 +171,7 @@ awssns
171
rocketchat
172
sms
173
hangouts
174
+dynatrace
175
"
176
177
# -----------------------------------------------------------------------------
@@ -366,6 +368,16 @@ IRC_NETWORK=
368
# hangouts configs
369
declare -A HANGOUTS_WEBHOOK_URI
370
371
+# dynatrace configs
372
+DYNATRACE_SPACE=
373
+DYNATRACE_SERVER=
374
+DYNATRACE_TOKEN=
375
+DYNATRACE_TAG_VALUE=
376
+DYNATRACE_ANNOTATION_TYPE=
377
+DYNATRACE_EVENT=
378
+SEND_DYNATRACE=
379
+
380
+
381
# load the stock and user configuration files
382
# these will overwrite the variables above
383
@@ -503,6 +515,14 @@ filter_recipient_by_criticality() {
515
#shellcheck disable=SC2153
516
{ [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
517
518
+# check dynatrace
519
+{ [ -z "${DYNATRACE_SPACE}" ] ||
520
+ [ -z "${DYNATRACE_SERVER}" ] ||
521
+ [ -z "${DYNATRACE_TOKEN}" ] ||
522
+ [ -z "${DYNATRACE_TAG_VALUE}" ] ||
523
+ [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
524
+
525
+
526
if [ "${SEND_PUSHOVER}" = "YES" ] ||
527
[ "${SEND_SLACK}" = "YES" ] ||
528
[ "${SEND_ROCKETCHAT}" = "YES" ] ||
@@ -521,7 +541,8 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
541
[ "${SEND_PROWL}" = "YES" ] ||
542
[ "${SEND_HANGOUTS}" = "YES" ] ||
543
[ "${SEND_CUSTOM}" = "YES" ] ||
524
- [ "${SEND_MSTEAM}" = "YES" ]; then
544
+ [ "${SEND_MSTEAM}" = "YES" ] ||
545
+ [ "${SEND_DYNATRACE}" = "YES" ]; then
546
# if we need curl, check for the curl command
547
if [ -z "${curl}" ]; then
548
curl="$(command -v curl 2>/dev/null)"
@@ -547,6 +568,7 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
568
SEND_PROWL="NO"
569
SEND_HANGOUTS="NO"
570
SEND_CUSTOM="NO"
571
+ SEND_DYNATRACE="NO"
572
fi
573
fi
574
@@ -673,7 +695,9 @@ for method in "${SEND_EMAIL}" \
695
"${SEND_AWSSNS}" \
696
"${SEND_SYSLOG}" \
697
"${SEND_SMS}" \
676
- "${SEND_MSTEAM}"; do
698
+ "${SEND_MSTEAM}" \
699
+ "${SEND_DYNATRACE}"; do
700
+
701
if [ "${method}" == "YES" ]; then
702
proceed=1
703
break
@@ -1887,6 +1911,51 @@ EOF
1911
1912
return 1
1913
}
1914
+# -----------------------------------------------------------------------------
1915
+# Dynatrace sender
1916
+send_dynatrace() {
1917
+ local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
1918
+ local description="NetData Notification for: ${host} ${chart}.${name} is ${status}"
1919
+ local payload=""
1920
+
1921
+ payload=$(cat <<EOF
1922
+{
1923
+ "title": "NetData Alarm from ${host}",
1924
+ "source" : "${DYNATRACE_ANNOTATION_TYPE}",
1925
+ "description" : "${description}",
1926
+ "eventType": "${DYNATRACE_EVENT}",
1927
+ "attachRules":{
1928
+ "tagRule":[{
1929
+ "meTypes":["HOST"],
1930
+ "tags":["${DYNATRACE_TAG_VALUE}"]
1931
+ }]
1932
+ },
1933
+ "customProperties":{
1934
+ "description": "${description}"
1935
+ }
1936
+}
1937
+EOF
1938
+)
1939
+
1940
+ # echo ${payload}
1941
+
1942
+ httpcode=$(docurl -X POST -H "Authorization: Api-token ${DYNATRACE_TOKEN}" -H "Content-Type: application/json" -d "${payload}" ${dynatrace_url})
1943
+ ret=$?
1944
+
1945
+
1946
+ if [ ${ret} -eq 0 ]; then
1947
+ if [ "${httpcode}" = "200" ]; then
1948
+ info "sent ${DYNATRACE_EVENT} to ${DYNATRACE_SERVER}"
1949
+ return 0
1950
+ else
1951
+ warning "Dynatrace ${DYNATRACE_SERVER} responded ${httpcode} notification for: ${host} ${chart}.${name} is ${status} was not sent!"
1952
+ return 1
1953
+ fi
1954
+ else
1955
+ error "failed to sent ${DYNATRACE_EVENT} notification for: ${host} ${chart}.${name} is ${status} to ${DYNATRACE_SERVER} with error code ${ret}."
1956
+ return 1
1957
+ fi
1958
+}
1959
1960
# -----------------------------------------------------------------------------
1961
# prepare the content of the notification
@@ -2391,6 +2460,12 @@ fi
2460
2461
SENT_EMAIL=$?
2462
2463
+# -----------------------------------------------------------------------------
2464
+# send the EVENT to Dynatrace
2465
+send_dynatrace "${host}" "${chart}" "${name}" "${status}"
2466
+SENT_DYNATRACE=$?
2467
+
2468
+
2469
# -----------------------------------------------------------------------------
2470
# let netdata know
2471
for state in "${SENT_EMAIL}" \
@@ -2416,7 +2491,8 @@ for state in "${SENT_EMAIL}" \
2491
"${SENT_AWSSNS}" \
2492
"${SENT_SYSLOG}" \
2493
"${SENT_SMS}" \
2419
- "${SENT_MSTEAM}"; do
2494
+ "${SENT_MSTEAM}" \
2495
+ "${SENT_DYNATRACE}"; do
2496
if [ "${state}" -eq 0 ]; then
2497
# we sent something
2498
exit 0
health/notifications/dynatrace/Makefile.inc
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+# THIS IS NOT A COMPLETE Makefile
4
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
5
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
6
+
7
+# install these files
8
+dist_noinst_DATA += \
9
+ dynatrace/README.md \
10
+ dynatrace/Makefile.inc \
11
+ $(NULL)
12
+
health/notifications/dynatrace/README.md
new
+36
@@ -0,0 +1,36 @@
1
+<!--
2
+---
3
+title: "Dynatrace"
4
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/dynatrace/README.md
5
+---
6
+-->
7
+
8
+# Dynatrace
9
+
10
+Dynatrace allows you to receive notifications using their Events REST API.
11
+
12
+See [the Dynatrace documentation](https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/environment-api/events/post-event/) about POSTing an event in the Events API for more details.
13
+
14
+
15
+
16
+You need:
17
+
18
+1. Dynatrace Server. You can use the same on all your Netdata servers but make sure the server is network visible from your Netdata hosts.
19
+The Dynatrace server should be with protocol prefixed (`http://` or `https://`). For example: `https://monitor.example.com`
20
+This is a required parameter.
21
+2. API Token. Generate a secure access API token that enables access to your Dynatrace monitoring data via the REST-based API.
22
+Generate a Dynatrace API authentication token. On your Dynatrace server, go to **Settings** --> **Integration** --> **Dynatrace API** --> **Generate token**.
23
+See [Dynatrace API - Authentication](https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/basics/dynatrace-api-authentication/) for more details.
24
+This is a required parameter.
25
+3. API Space. This is the URL part of the page you have access in order to generate the API Token. For example, for my generated API Token the URL is:
26
+https://monitor.illumineit.com/e/2a93fe0e-4cd5-469a-9d0d-1a064235cfce/#settings/integration/apikeys;gf=all
27
+In that case, my space is _2a93fe0e-4cd5-469a-9d0d-1a064235cfce_
28
+This is a required parameter.
29
+4. Generate a Server Tag. On your Dynatrace Server, go to **Settings** --> **Tags** --> **Manually applied tags** and create the Tag.
30
+The Netdata alarm is sent as a Dynatrace Event to be correlated with all those hosts tagged with this Tag you have created.
31
+This is a required parameter.
32
+5. Specify the Dynatrace event. This can be one of `CUSTOM_INFO`, `CUSTOM_ANNOTATION`, `CUSTOM_CONFIGURATION`, and `CUSTOM_DEPLOYMENT`.
33
+The default value is `CUSTOM_INFO`.
34
+This is a required parameter.
35
+6. Specify the annotation type. This is the source of the Dynatrace event. Put whatever it fits you, for example,
36
+_Netdata Alarm_, which is also the default value.
health/notifications/health_alarm_notify.conf
+43
-1
@@ -229,6 +229,43 @@ DEFAULT_RECIPIENT_EMAIL="root"
229
# to not send HTML but Plain Text only emails.
230
#EMAIL_PLAINTEXT_ONLY="YES"
231
232
+#------------------------------------------------------------------------------
233
+# Dynatrace global notification options
234
+#------------------------------------------------------------------------------
235
+# enable/disable sending Dynatrace notifications
236
+SEND_DYNATRACE="YES"
237
+
238
+# The Dynatrace server with protocol prefix (http:// or https://), example https://monitor.illumineit.com
239
+# Required
240
+DYNATRACE_SERVER=""
241
+
242
+# Generate a Dynatrace API authentication token
243
+# Read https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/basics/dynatrace-api-authentication/
244
+# On Dynatrace server goto Settings --> Integration --> Dynatrace API --> Generate token
245
+# Required
246
+DYNATRACE_TOKEN=""
247
+
248
+# Beware: Space is taken from dynatrace URL from browser when you create the TOKEN
249
+# Required
250
+DYNATRACE_SPACE=""
251
+
252
+# Generate a Server Tag. On the Dynatrace Server go to Settings --> Tags --> Manually applied tags create the Tag
253
+# The NetData alarm will be sent as a Dynatrace Event to be correlated with all those hosts tagged with this Tag
254
+# you created.
255
+# Required
256
+DYNATRACE_TAG_VALUE=""
257
+
258
+# Change this to what you want
259
+DYNATRACE_ANNOTATION_TYPE="NetData Alarm"
260
+
261
+# This can be CUSTOM_INFO, CUSTOM_ANNOTATION, CUSTOM_CONFIGURATION, CUSTOM_DEPLOYMENT
262
+# Applying default value
263
+# Required
264
+DYNATRACE_EVENT="CUSTOM_INFO"
265
+
266
+
267
+DEFAULT_RECIPIENT_DYNATRACE=""
268
+
269
#------------------------------------------------------------------------------
270
# hangouts (google hangouts chat) global notification options
271
@@ -873,6 +910,7 @@ role_recipients_msteam[sysadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
910
911
role_recipients_rocketchat[sysadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
912
913
+role_recipients_dynatrace[sysadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
914
# -----------------------------------------------------------------------------
915
# DNS related alarms
916
@@ -922,6 +960,7 @@ role_recipients_rocketchat[domainadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
960
961
role_recipients_sms[domainadmin]="${DEFAULT_RECIPIENT_SMS}"
962
963
+role_recipients_dynatrace[domainadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
964
# -----------------------------------------------------------------------------
965
# database servers alarms
966
# mysql, redis, memcached, postgres, etc
@@ -972,6 +1011,7 @@ role_recipients_rocketchat[dba]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
1011
1012
role_recipients_sms[dba]="${DEFAULT_RECIPIENT_SMS}"
1013
1014
+role_recipients_dynatrace[dba]="${DEFAULT_RECIPIENT_DYNATRACE}"
1015
# -----------------------------------------------------------------------------
1016
# web servers alarms
1017
# apache, nginx, lighttpd, etc
@@ -1022,6 +1062,7 @@ role_recipients_rocketchat[webmaster]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
1062
1063
role_recipients_sms[webmaster]="${DEFAULT_RECIPIENT_SMS}"
1064
1065
+role_recipients_dynatrace[webmaster]="${DEFAULT_RECIPIENT_DYNATRACE}"
1066
# -----------------------------------------------------------------------------
1067
# proxy servers alarms
1068
# squid, etc
@@ -1072,7 +1113,7 @@ role_recipients_rocketchat[proxyadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
1113
1114
role_recipients_sms[proxyadmin]="${DEFAULT_RECIPIENT_SMS}"
1115
1075
-
1116
+role_recipients_dynatrace[proxyadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
1117
# -----------------------------------------------------------------------------
1118
# peripheral devices
1119
# UPS, photovoltaics, etc
@@ -1121,3 +1162,4 @@ role_recipients_rocketchat[sitemgr]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
1162
1163
role_recipients_sms[sitemgr]="${DEFAULT_RECIPIENT_SMS}"
1164
1165
+role_recipients_dynatrace[sitemgr]="${DEFAULT_RECIPIENT_DYNATRACE}"
\ No newline at end of file