Opsgenie integration (#9879)
Bring full integration with Opsgenie.
thiagoftsm committed
Oct 28, 2020 at 13:48 UTC
1a50899551674e0f0fa684daeadd043503be949b
5 files changed
+168
-6
health/notifications/Makefile.am
+1
@@ -35,6 +35,7 @@ include hangouts/Makefile.inc
35
include irc/Makefile.inc
36
include kavenegar/Makefile.inc
37
include messagebird/Makefile.inc
38
+include opsgenie/Makefile.inc
39
include pagerduty/Makefile.inc
40
include pushbullet/Makefile.inc
41
include pushover/Makefile.inc
health/notifications/alarm-notify.sh.in
+75
-6
@@ -37,6 +37,7 @@
37
# - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
38
# - Dynatrace Event by @illumine
39
# - Stackpulse Event by @thiagoftsm
40
+# - Opsgenie by @thiaoftsm #9858
41
42
# -----------------------------------------------------------------------------
43
# testing notifications
@@ -386,6 +387,9 @@ SEND_DYNATRACE=
387
# stackpulse configs
388
STACKPULSE_WEBHOOK=
389
390
+# opsgenie configs
391
+OPSGENIE_API_KEY=
392
+
393
# load the stock and user configuration files
394
# these will overwrite the variables above
395
@@ -532,6 +536,9 @@ filter_recipient_by_criticality() {
536
[ -z "${DYNATRACE_TAG_VALUE}" ] ||
537
[ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
538
539
+# check opsgenie
540
+[ -z "${OPSGENIE_API_KEY}" ] && SEND_OPSGENIE="NO"
541
+
542
# check matrix
543
{ [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
544
@@ -559,7 +566,8 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
566
[ "${SEND_CUSTOM}" = "YES" ] ||
567
[ "${SEND_MSTEAM}" = "YES" ] ||
568
[ "${SEND_DYNATRACE}" = "YES" ] ||
562
- [ "${SEND_STACKPULSE}" = "YES" ]; then
569
+ [ "${SEND_STACKPULSE}" = "YES" ] ||
570
+ [ "${SEND_OPSGENIE}" = "YES" ]; then
571
# if we need curl, check for the curl command
572
if [ -z "${curl}" ]; then
573
curl="$(command -v curl 2>/dev/null)"
@@ -588,6 +596,7 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
596
SEND_CUSTOM="NO"
597
SEND_DYNATRACE="NO"
598
SEND_STACKPULSE="NO"
599
+ SEND_OPSGENIE="NO"
600
fi
601
fi
602
@@ -717,8 +726,9 @@ for method in "${SEND_EMAIL}" \
726
"${SEND_SMS}" \
727
"${SEND_MSTEAM}" \
728
"${SEND_DYNATRACE}" \
720
- "${SEND_STACKPULSE}" ; do
721
-
729
+ "${SEND_STACKPULSE}" \
730
+ "${SEND_OPSGENIE}" ; do
731
+
732
if [ "${method}" == "YES" ]; then
733
proceed=1
734
break
@@ -2072,6 +2082,60 @@ EOF
2082
2083
return 0
2084
}
2085
+# -----------------------------------------------------------------------------
2086
+# Opsgenie sender
2087
+
2088
+send_opsgenie() {
2089
+ local payload httpcode oldv currv
2090
+ [ "${SEND_OPSGENIE}" != "YES" ] && return 1
2091
+
2092
+ if [ -z "${OPSGENIE_API_KEY}" ] ; then
2093
+ info "Can't send Opsgenie notification, because OPSGENIE_API_KEY is not defined"
2094
+ return 1
2095
+ fi
2096
+
2097
+ # We are sending null when values are nan to avoid errors while JSON message is parsed
2098
+ [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
2099
+ [ "${value}" != "nan" ] && currv="${value}" || currv="null"
2100
+
2101
+ payload=$(cat <<EOF
2102
+ {
2103
+ "host" : "${host}",
2104
+ "unique_id" : "${unique_id}",
2105
+ "alarmId" : ${alarm_id},
2106
+ "eventId" : ${event_id},
2107
+ "chart" : "${chart}",
2108
+ "when": ${when},
2109
+ "name" : "${name}",
2110
+ "family" : "${family}",
2111
+ "status" : "${status}",
2112
+ "old_status" : "${old_status}",
2113
+ "value" : ${currv},
2114
+ "old_value" : ${oldv},
2115
+ "duration": ${duration},
2116
+ "non_clear_duration": ${non_clear_duration},
2117
+ "units" : "${units}",
2118
+ "info" : "${status_message}, ${info}",
2119
+ "calc_expression" : "${calc_expression}",
2120
+ "total_warnings" : "${total_warnings}",
2121
+ "total_critical" : "${total_critical}",
2122
+ "src" : "${src}"
2123
+ }
2124
+EOF
2125
+)
2126
+
2127
+ httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "https://api.opsgenie.com/v1/json/integrations/webhooks/netdata?apiKey=${OPSGENIE_API_KEY}")
2128
+ # https://docs.opsgenie.com/docs/alert-api#create-alert
2129
+ if [ "${httpcode}" = "200" ]; then
2130
+ info "sent opsgenie notification for: ${host} ${chart}.${name} is ${status}"
2131
+ else
2132
+ error "failed to send opsgenie notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
2133
+ return 1
2134
+ fi
2135
+
2136
+ return 0
2137
+}
2138
+
2139
# -----------------------------------------------------------------------------
2140
# prepare the content of the notification
2141
@@ -2589,12 +2653,16 @@ SENT_EMAIL=$?
2653
send_dynatrace "${host}" "${chart}" "${name}" "${status}"
2654
SENT_DYNATRACE=$?
2655
2592
-
2656
# -----------------------------------------------------------------------------
2594
-# send the EVENT to Dynatrace
2657
+# send the EVENT to Stackpulse
2658
send_stackpulse
2659
SENT_STACKPULSE=$?
2660
2661
+# -----------------------------------------------------------------------------
2662
+# send messages to Opsgenie
2663
+send_opsgenie
2664
+SENT_OPSGENIE=$?
2665
+
2666
# -----------------------------------------------------------------------------
2667
# let netdata know
2668
for state in "${SENT_EMAIL}" \
@@ -2623,7 +2691,8 @@ for state in "${SENT_EMAIL}" \
2691
"${SENT_SMS}" \
2692
"${SENT_MSTEAM}" \
2693
"${SENT_DYNATRACE}" \
2626
- "${SENT_STACKPULSE}" ; do
2694
+ "${SENT_STACKPULSE}" \
2695
+ "${SENT_OPSGENIE}"; do
2696
if [ "${state}" -eq 0 ]; then
2697
# we sent something
2698
exit 0
health/notifications/health_alarm_notify.conf
+21
@@ -278,6 +278,15 @@ STACKPULSE_WEBHOOK=""
278
279
DEFAULT_RECIPIENT_STACKPULSE=""
280
281
+#------------------------------------------------------------------------------
282
+# opsgenie global notification options
283
+SEND_OPSGENIE="YES"
284
+
285
+# Api key
286
+OPSGENIE_API_KEY=""
287
+
288
+DEFAULT_RECIPIENT_OPSGENIE=""
289
+
290
#------------------------------------------------------------------------------
291
# hangouts (google hangouts chat) global notification options
292
@@ -944,6 +953,8 @@ role_recipients_rocketchat[sysadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
953
954
role_recipients_dynatrace[sysadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
955
956
+role_recipients_opsgenie[sysadmin]="${DEFAULT_RECIPIENT_OPSGENIE}"
957
+
958
role_recipients_matrix[sysadmin]="${DEFAULT_RECIPIENT_MATRIX}"
959
960
role_recipients_stackpulse[sysadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
@@ -999,6 +1010,8 @@ role_recipients_sms[domainadmin]="${DEFAULT_RECIPIENT_SMS}"
1010
1011
role_recipients_dynatrace[domainadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
1012
1013
+role_recipients_opsgenie[domainadmin]="${DEFAULT_RECIPIENT_OPSGENIE}"
1014
+
1015
role_recipients_matrix[domainadmin]="${DEFAULT_RECIPIENT_MATRIX}"
1016
1017
role_recipients_stackpulse[domainadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
@@ -1055,6 +1068,8 @@ role_recipients_sms[dba]="${DEFAULT_RECIPIENT_SMS}"
1068
1069
role_recipients_dynatrace[dba]="${DEFAULT_RECIPIENT_DYNATRACE}"
1070
1071
+role_recipients_opsgenie[dba]="${DEFAULT_RECIPIENT_OPSGENIE}"
1072
+
1073
role_recipients_matrix[dba]="${DEFAULT_RECIPIENT_MATRIX}"
1074
1075
role_recipients_stackpulse[dba]="${DEFAULT_RECIPIENT_STACKPULSE}"
@@ -1111,6 +1126,8 @@ role_recipients_sms[webmaster]="${DEFAULT_RECIPIENT_SMS}"
1126
1127
role_recipients_dynatrace[webmaster]="${DEFAULT_RECIPIENT_DYNATRACE}"
1128
1129
+role_recipients_opsgenie[webmaster]="${DEFAULT_RECIPIENT_OPSGENIE}"
1130
+
1131
role_recipients_matrix[webmaster]="${DEFAULT_RECIPIENT_MATRIX}"
1132
1133
role_recipients_stackpulse[webmaster]="${DEFAULT_RECIPIENT_STACKPULSE}"
@@ -1167,6 +1184,8 @@ role_recipients_sms[proxyadmin]="${DEFAULT_RECIPIENT_SMS}"
1184
1185
role_recipients_dynatrace[proxyadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
1186
1187
+role_recipients_opsgenie[proxyadmin]="${DEFAULT_RECIPIENT_OPSGENIE}"
1188
+
1189
role_recipients_matrix[proxyadmin]="${DEFAULT_RECIPIENT_MATRIX}"
1190
1191
role_recipients_stackpulse[proxyadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
@@ -1221,6 +1240,8 @@ role_recipients_sms[sitemgr]="${DEFAULT_RECIPIENT_SMS}"
1240
1241
role_recipients_dynatrace[sitemgr]="${DEFAULT_RECIPIENT_DYNATRACE}"
1242
1243
+role_recipients_opsgenie[sitemgr]="${DEFAULT_RECIPIENT_OPSGENIE}"
1244
+
1245
role_recipients_matrix[sitemgr]="${DEFAULT_RECIPIENT_MATRIX}"
1246
1247
role_recipients_stackpulse[sitemgr]="${DEFAULT_RECIPIENT_STACKPULSE}"
health/notifications/opsgenie/Makefile.inc
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+# THIS IS NOT A COMPLETE Makefile
4
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
5
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
6
+
7
+# install these files
8
+dist_noinst_DATA += \
9
+ opsgenie/README.md \
10
+ opsgenie/Makefile.inc \
11
+ $(NULL)
12
+
health/notifications/opsgenie/README.md
new
+59
@@ -0,0 +1,59 @@
1
+<!--
2
+title: "Send notifications to Opsgenie"
3
+description: "Send alerts to your Opsgenie incident response account any time an anomaly or performance issue strikes a node in your infrastructure."
4
+sidebar_label: "Opsgenie"
5
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/opsgenie/README.md
6
+-->
7
+
8
+# Send notifications to Opsgenie
9
+
10
+[Opsgenie](https://www.atlassian.com/software/opsgenie) is an alerting and incident response tool. It is designed to
11
+group and filter alarms, build custom routing rules for on-call teams, and correlate deployments and commits to
12
+incidents.
13
+
14
+The first step is to create a [Netdata integration](https://docs.opsgenie.com/docs/api-integration) in the
15
+[Opsgenie](https://www.atlassian.com/software/opsgenie) dashboard. After this, you need to edit
16
+`health_alarm_notify.conf` on your system, by running the following from your [config
17
+directory](/docs/configure/nodes.md):
18
+
19
+```bash
20
+./edit-config health_alarm_notify.conf
21
+```
22
+
23
+Change the variable `OPSGENIE_API_KEY` with the API key you got from Opsgenie.
24
+
25
+```
26
+SEND_OPSGENIE="YES"
27
+
28
+# Api key
29
+# Default Opsgenie APi
30
+OPSGENIE_API_KEY="11111111-2222-3333-4444-555555555555"
31
+```
32
+
33
+Changes to `health_alarm_notify.conf` do not require a Netdata restart. You can test your Opsgenie notifications
34
+configuration by issuing the commands, replacing `ROLE` with your preferred role:
35
+
36
+```sh
37
+# become user netdata
38
+sudo su -s /bin/bash netdata
39
+
40
+# send a test alarm
41
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test ROLE
42
+```
43
+
44
+If everything works, you'll see alarms in your Opsgenie platform:
45
+
46
+
48
+
49
+If sending the test notifications fails, you can look in `/var/log/netdata/error.log` to find the relevant error
50
+message:
51
+
52
+```log
53
+2020-09-03 23:07:00: alarm-notify.sh: ERROR: failed to send opsgenie notification for: hades test.chart.test_alarm is CRITICAL, with HTTP error code 401.
54
+```
55
+
56
+You can find more details about the Opsgenie error codes in their [response
57
+docs](https://docs.opsgenie.com/docs/response).
58
+
59
+[](<>)