Stackpulse integration (#9965)
Add integration with Stackpulse.
thiagoftsm committed
Sep 21, 2020 at 13:19 UTC
5156fb013fc024d7f62611bed50c4c154033f01a
5 files changed
+180
-3
health/notifications/Makefile.am
+1
@@ -41,6 +41,7 @@ include pushover/Makefile.inc
41
include rocketchat/Makefile.inc
42
include slack/Makefile.inc
43
include smstools3/Makefile.inc
44
+include stackpulse/Makefile.inc
45
include syslog/Makefile.inc
46
include telegram/Makefile.inc
47
include twilio/Makefile.inc
health/notifications/alarm-notify.sh.in
+65
-3
@@ -36,6 +36,7 @@
36
# - RocketChat notifications by @Hermsi1337 #3777
37
# - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
38
# - Dynatrace Event by @illumine
39
+# - Stackpulse Event by @thiagoftsm
40
41
# -----------------------------------------------------------------------------
42
# testing notifications
@@ -382,6 +383,8 @@ DYNATRACE_ANNOTATION_TYPE=
383
DYNATRACE_EVENT=
384
SEND_DYNATRACE=
385
386
+# stackpulse configs
387
+STACKPULSE_WEBHOOK=
388
389
# load the stock and user configuration files
390
# these will overwrite the variables above
@@ -532,6 +535,9 @@ filter_recipient_by_criticality() {
535
# check matrix
536
{ [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
537
538
+# check stackpulse
539
+[ -z "${STACKPULSE_WEBHOOK}" ] && SEND_STACKPULSE="NO"
540
+
541
if [ "${SEND_PUSHOVER}" = "YES" ] ||
542
[ "${SEND_SLACK}" = "YES" ] ||
543
[ "${SEND_ROCKETCHAT}" = "YES" ] ||
@@ -552,7 +558,8 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
558
[ "${SEND_MATRIX}" = "YES" ] ||
559
[ "${SEND_CUSTOM}" = "YES" ] ||
560
[ "${SEND_MSTEAM}" = "YES" ] ||
555
- [ "${SEND_DYNATRACE}" = "YES" ]; then
561
+ [ "${SEND_DYNATRACE}" = "YES" ] ||
562
+ [ "${SEND_STACKPULSE}" = "YES" ]; then
563
# if we need curl, check for the curl command
564
if [ -z "${curl}" ]; then
565
curl="$(command -v curl 2>/dev/null)"
@@ -580,6 +587,7 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
587
SEND_MATRIX="NO"
588
SEND_CUSTOM="NO"
589
SEND_DYNATRACE="NO"
590
+ SEND_STACKPULSE="NO"
591
fi
592
fi
593
@@ -708,7 +716,8 @@ for method in "${SEND_EMAIL}" \
716
"${SEND_SYSLOG}" \
717
"${SEND_SMS}" \
718
"${SEND_MSTEAM}" \
711
- "${SEND_DYNATRACE}"; do
719
+ "${SEND_DYNATRACE}" \
720
+ "${SEND_STACKPULSE}" ; do
721
722
if [ "${method}" == "YES" ]; then
723
proceed=1
@@ -1968,8 +1977,10 @@ EOF
1977
1978
return 1
1979
}
1980
+
1981
# -----------------------------------------------------------------------------
1982
# Dynatrace sender
1983
+
1984
send_dynatrace() {
1985
[ "${SEND_DYNATRACE}" != "YES" ] && return 1
1986
@@ -2016,6 +2027,51 @@ EOF
2027
fi
2028
}
2029
2030
+
2031
+# -----------------------------------------------------------------------------
2032
+# Stackpulse sender
2033
+
2034
+send_stackpulse() {
2035
+ local payload httpcode oldv currv
2036
+ [ "${SEND_STACKPULSE}" != "YES" ] && return 1
2037
+
2038
+ # We are sending null when values are nan to avoid errors while JSON message is parsed
2039
+ [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
2040
+ [ "${value}" != "nan" ] && currv="${value}" || currv="null"
2041
+
2042
+ payload=$(cat <<EOF
2043
+ {
2044
+ "Node" : "${host}",
2045
+ "Chart" : "${chart}",
2046
+ "OldValue" : ${oldv},
2047
+ "Value" : ${currv},
2048
+ "Units" : "${units}",
2049
+ "OldStatus" : "${old_status}",
2050
+ "Status" : "${status}",
2051
+ "Alarm" : "${name}",
2052
+ "Date": ${when},
2053
+ "Duration": ${duration},
2054
+ "NonClearDuration": ${non_clear_duration},
2055
+ "Description" : "${status_message}, ${info}",
2056
+ "CalcExpression" : "${calc_expression}",
2057
+ "CalcParamValues" : "${calc_param_values}",
2058
+ "TotalWarnings" : "${total_warnings}",
2059
+ "TotalCritical" : "${total_critical}",
2060
+ "ID" : ${alarm_id}
2061
+ }
2062
+EOF
2063
+)
2064
+
2065
+ httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" ${STACKPULSE_WEBHOOK})
2066
+ if [ "${httpcode}" = "200" ]; then
2067
+ info "sent stackpulse notification for: ${host} ${chart}.${name} is ${status}"
2068
+ else
2069
+ error "failed to send stackpulse notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
2070
+ return 1
2071
+ fi
2072
+
2073
+ return 0
2074
+}
2075
# -----------------------------------------------------------------------------
2076
# prepare the content of the notification
2077
@@ -2534,6 +2590,11 @@ send_dynatrace "${host}" "${chart}" "${name}" "${status}"
2590
SENT_DYNATRACE=$?
2591
2592
2593
+# -----------------------------------------------------------------------------
2594
+# send the EVENT to Dynatrace
2595
+send_stackpulse
2596
+SENT_STACKPULSE=$?
2597
+
2598
# -----------------------------------------------------------------------------
2599
# let netdata know
2600
for state in "${SENT_EMAIL}" \
@@ -2561,7 +2622,8 @@ for state in "${SENT_EMAIL}" \
2622
"${SENT_SYSLOG}" \
2623
"${SENT_SMS}" \
2624
"${SENT_MSTEAM}" \
2564
- "${SENT_DYNATRACE}"; do
2625
+ "${SENT_DYNATRACE}" \
2626
+ "${SENT_STACKPULSE}" ; do
2627
if [ "${state}" -eq 0 ]; then
2628
# we sent something
2629
exit 0
health/notifications/health_alarm_notify.conf
+21
@@ -269,6 +269,15 @@ DYNATRACE_EVENT="CUSTOM_INFO"
269
270
DEFAULT_RECIPIENT_DYNATRACE=""
271
272
+#------------------------------------------------------------------------------
273
+# Stackpulse global notification options
274
+SEND_STACKPULSE="YES"
275
+
276
+# Webhook
277
+STACKPULSE_WEBHOOK=""
278
+
279
+DEFAULT_RECIPIENT_STACKPULSE=""
280
+
281
#------------------------------------------------------------------------------
282
# hangouts (google hangouts chat) global notification options
283
@@ -937,6 +946,8 @@ role_recipients_dynatrace[sysadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
946
947
role_recipients_matrix[sysadmin]="${DEFAULT_RECIPIENT_MATRIX}"
948
949
+role_recipients_stackpulse[sysadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
950
+
951
# -----------------------------------------------------------------------------
952
# DNS related alarms
953
@@ -990,6 +1001,8 @@ role_recipients_dynatrace[domainadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
1001
1002
role_recipients_matrix[domainadmin]="${DEFAULT_RECIPIENT_MATRIX}"
1003
1004
+role_recipients_stackpulse[domainadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
1005
+
1006
# -----------------------------------------------------------------------------
1007
# database servers alarms
1008
# mysql, redis, memcached, postgres, etc
@@ -1044,6 +1057,8 @@ role_recipients_dynatrace[dba]="${DEFAULT_RECIPIENT_DYNATRACE}"
1057
1058
role_recipients_matrix[dba]="${DEFAULT_RECIPIENT_MATRIX}"
1059
1060
+role_recipients_stackpulse[dba]="${DEFAULT_RECIPIENT_STACKPULSE}"
1061
+
1062
# -----------------------------------------------------------------------------
1063
# web servers alarms
1064
# apache, nginx, lighttpd, etc
@@ -1098,6 +1113,8 @@ role_recipients_dynatrace[webmaster]="${DEFAULT_RECIPIENT_DYNATRACE}"
1113
1114
role_recipients_matrix[webmaster]="${DEFAULT_RECIPIENT_MATRIX}"
1115
1116
+role_recipients_stackpulse[webmaster]="${DEFAULT_RECIPIENT_STACKPULSE}"
1117
+
1118
# -----------------------------------------------------------------------------
1119
# proxy servers alarms
1120
# squid, etc
@@ -1152,6 +1169,8 @@ role_recipients_dynatrace[proxyadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
1169
1170
role_recipients_matrix[proxyadmin]="${DEFAULT_RECIPIENT_MATRIX}"
1171
1172
+role_recipients_stackpulse[proxyadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
1173
+
1174
# -----------------------------------------------------------------------------
1175
# peripheral devices
1176
# UPS, photovoltaics, etc
@@ -1203,3 +1222,5 @@ role_recipients_sms[sitemgr]="${DEFAULT_RECIPIENT_SMS}"
1222
role_recipients_dynatrace[sitemgr]="${DEFAULT_RECIPIENT_DYNATRACE}"
1223
1224
role_recipients_matrix[sitemgr]="${DEFAULT_RECIPIENT_MATRIX}"
1225
+
1226
+role_recipients_stackpulse[sitemgr]="${DEFAULT_RECIPIENT_STACKPULSE}"
health/notifications/stackpulse/Makefile.inc
new
+12
@@ -0,0 +1,12 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+# THIS IS NOT A COMPLETE Makefile
4
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
5
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
6
+
7
+# install these files
8
+dist_noinst_DATA += \
9
+ stackpulse/README.md \
10
+ stackpulse/Makefile.inc \
11
+ $(NULL)
12
+
health/notifications/stackpulse/README.md
new
+81
@@ -0,0 +1,81 @@
1
+<!--
2
+title: "Send notifications to StackPulse"
3
+description: "Send alerts to your StackPulse Netdata integration any time an anomaly or performance issue strikes a node in your infrastructure."
4
+sidebar_label: "StackPulse"
5
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/stackpulse/README.md
6
+-->
7
+
8
+# Send notifications to StackPulse
9
+
10
+[StackPulse](https://stackpulse.com/) is a software-as-a-service platform for site reliablility engineering.
11
+StackPulse is a Software-as-a-Service platform for Site Reliablility Engineering. It helps SREs, DevOps Engineers and
12
+Software Developers reduce toil and alert fatigue while improving reliability of software services by managing,
13
+analyzing and automating incident response activities.
14
+
15
+Sending Netdata alarm notifications to StackPulse allows you to create smart automated response workflows
16
+(StackPulse playbooks) that will help you drive down your MTTD and MTTR by performing any of the following:
17
+
18
+- Enriching the incident with data from multiple sources
19
+- Performing triage actions and analyzing their results
20
+- Orchestrating incident management and notification flows
21
+- Performing automatic and semi-automatic remediation actions
22
+- Analzying incident data and remediation patterns to improve reliability of your services
23
+
24
+To send the notification you need:
25
+
26
+1. Create a Netdata integration in the `StackPulse Administration Portal`, and copy the `Endpoint` URL.
27
+
28
+
29
+
30
+2. On your node, navigate to `/etc/netdata/` and run the following command:
31
+
32
+```sh
33
+$ ./edit-config health_alarm_notify.conf
34
+```
35
+
36
+3. Set the `STACKPULSE_WEBHOOK` variable to `Endpoint` URL you copied earlier:
37
+
38
+```
39
+SEND_STACKPULSE="YES"
40
+STACKPULSE_WEBHOOK="https://hooks.stackpulse.io/v1/webhooks/YOUR_UNIQUE_ID"
41
+```
42
+
43
+4. Now [restart Netdata](/docs/getting-started.md#start-stop-and-restart-netdata). When your node creates an alarm, you
44
+ can see the associated notification on your StackPulse Administration Portal
45
+
46
+## React to alarms with playbooks
47
+
48
+StackPulse allow users to create `Playbooks` giving additional information about events that happen in specific
49
+scenarios. For example, you could create a Playbook that responds to a "low disk space" alarm by compressing and
50
+cleaning up storage partitions with dynamic data.
51
+
52
+
53
+
54
+
56
+### Create Playbooks for Netdata alarms
57
+
58
+To create a Playbook, you need to access the StackPulse Administration Portal. After the initial setup, you need to
59
+access the **TRIGGER** tab to define the scenarios used to trigger the event. The following variables are available:
60
+
61
+- `Hostname`: The host that generated the event.
62
+- `Chart`: The name of the chart.
63
+- `OldValue` : The previous value of the alarm.
64
+- `Value`: The current value of the alarm.
65
+- `Units` : The units of the value.
66
+- `OldStatus` : The previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL.
67
+- `State`: The current alarm status, the acceptable values are the same of `OldStatus`.
68
+- `Alarm` : The name of the alarm, as given in Netdata's health.d entries.
69
+- `Date` : The timestamp this event occurred.
70
+- `Duration` : The duration in seconds of the previous alarm state.
71
+- `NonClearDuration` : The total duration in seconds this is/was non-clear.
72
+- `Description` : A short description of the alarm copied from the alarm definition.
73
+- `CalcExpression` : The expression that was evaluated to trigger the alarm.
74
+- `CalcParamValues` : The values of the parameters in the expression, at the time of the evaluation.
75
+- `TotalWarnings` : Total number of alarms in WARNING state.
76
+- `TotalCritical` : Total number of alarms in CRITICAL state.
77
+- `ID` : The unique id of the alarm that generated this event.
78
+
79
+For more details how to create a scenario, take a look at the [StackPulse documentation](https://docs.stackpulse.io).
80
+
81
+[](<>)