ilert Integration (#18447)
Tim Nguyen Van committed
Sep 1, 2024 at 18:20 UTC
d5bbf77506a0a3e3f19a2ac9201b82a0efefd561
3 files changed
+135
-3
src/health/notifications/alarm-notify.sh.in
+59
-3
@@ -769,6 +769,9 @@ fi
769
# check custom
770
[ -z "${DEFAULT_RECIPIENT_CUSTOM}" ] && SEND_CUSTOM="NO"
771
772
+# check ilert
773
+[ -z "${ILERT_ALERT_SOURCE_URL}" ] && SEND_ILERT="NO"
774
+
775
# -----------------------------------------------------------------------------
776
# check the availability of targets
777
@@ -798,7 +801,8 @@ check_supported_targets() {
801
[ "${SEND_DYNATRACE}" = "YES" ] ||
802
[ "${SEND_OPSGENIE}" = "YES" ] ||
803
[ "${SEND_GOTIFY}" = "YES" ] ||
801
- [ "${SEND_NTFY}" = "YES" ]; then
804
+ [ "${SEND_NTFY}" = "YES" ] ||
805
+ [ "${SEND_ILERT}" = "YES" ]; then
806
# if we need curl, check for the curl command
807
if [ -z "${curl}" ]; then
808
curl="$(command -v curl 2>/dev/null)"
@@ -828,6 +832,7 @@ check_supported_targets() {
832
SEND_OPSGENIE="NO"
833
SEND_GOTIFY="NO"
834
SEND_NTFY="NO"
835
+ SEND_ILERT="NO"
836
fi
837
fi
838
@@ -983,7 +988,8 @@ for method in "${SEND_EMAIL}" \
988
"${SEND_DYNATRACE}" \
989
"${SEND_OPSGENIE}" \
990
"${SEND_GOTIFY}" \
986
- "${SEND_NTFY}" ; do
991
+ "${SEND_NTFY}" \
992
+ "${SEND_ILERT}" ; do
993
994
if [ "${method}" == "YES" ]; then
995
proceed=1
@@ -2431,6 +2437,50 @@ send_ntfy() {
2437
return 1
2438
}
2439
2440
+# -----------------------------------------------------------------------------
2441
+# ilert sender
2442
+
2443
+send_ilert() {
2444
+ local payload httpcode
2445
+ [ "${SEND_ILERT}" != "YES" ] && return 1
2446
+
2447
+ if [ -z "${ILERT_ALERT_SOURCE_URL}" ] ; then
2448
+ info "Can't send ilert notification, because ILERT_ALERT_SOURCE_URL is not defined"
2449
+ return 1
2450
+ fi
2451
+
2452
+ payload=$(cat <<EOF
2453
+ {
2454
+ "alert" : "${name}",
2455
+ "alert_url" : "${goto_url}",
2456
+ "alarm_id" : ${alarm_id},
2457
+ "chart" : "${chart}",
2458
+ "date" : "${when}",
2459
+ "duration" : "${duration_txt}",
2460
+ "host" : "${host}",
2461
+ "info" : "${info}",
2462
+ "message" : "${status_message}",
2463
+ "severity": ${status},
2464
+ "total_critical" : "${total_critical}",
2465
+ "total_warnings" : "${total_warnings}",
2466
+ "value" : "${value_string}",
2467
+ "image_url": "${image}",
2468
+ "src" : "${src}"
2469
+ }
2470
+EOF
2471
+)
2472
+
2473
+ httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${ILERT_ALERT_SOURCE_URL}")
2474
+ if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "202" ]; then
2475
+ info "sent ilert event for ${notification_description}"
2476
+ else
2477
+ error "failed to send ilert event for ${notification_description}, with HTTP response status code ${httpcode}."
2478
+ return 1
2479
+ fi
2480
+
2481
+ return 0
2482
+}
2483
+
2484
# -----------------------------------------------------------------------------
2485
# prepare the content of the notification
2486
@@ -3603,6 +3653,11 @@ SENT_GOTIFY=$?
3653
send_ntfy "${DEFAULT_RECIPIENT_NTFY}"
3654
SENT_NTFY=$?
3655
3656
+# -----------------------------------------------------------------------------
3657
+# send messages to ilert
3658
+send_ilert
3659
+SENT_ILERT=$?
3660
+
3661
# -----------------------------------------------------------------------------
3662
# let netdata know
3663
for state in "${SENT_EMAIL}" \
@@ -3632,7 +3687,8 @@ for state in "${SENT_EMAIL}" \
3687
"${SENT_DYNATRACE}" \
3688
"${SENT_OPSGENIE}" \
3689
"${SENT_GOTIFY}" \
3635
- "${SENT_NTFY}"; do
3690
+ "${SENT_NTFY}" \
3691
+ "${SENT_ILERT}"; do
3692
if [ "${state}" -eq 0 ]; then
3693
# we sent something
3694
exit 0
src/health/notifications/health_alarm_notify.conf
+21
@@ -844,6 +844,15 @@ NTFY_ACCESS_TOKEN=""
844
# Multiple recipients can be given like this: "https://SERVER1/TOPIC1 https://SERVER2/TOPIC2 ..."
845
DEFAULT_RECIPIENT_NTFY=""
846
847
+#------------------------------------------------------------------------------
848
+# ilert global notification options
849
+SEND_ILERT="YES"
850
+
851
+# Api key
852
+ILERT_ALERT_SOURCE_URL=""
853
+
854
+DEFAULT_RECIPIENT_ILERT=""
855
+
856
#------------------------------------------------------------------------------
857
# custom notifications
858
#
@@ -984,6 +993,8 @@ custom_sender() {
993
994
# role_recipients_ntfy[sysadmin]="${DEFAULT_RECIPIENT_NTFY}"
995
996
+# role_recipients_ilert[sysadmin]="${DEFAULT_RECIPIENT_ILERT}"
997
+
998
# -----------------------------------------------------------------------------
999
# DNS related alarms
1000
@@ -1041,6 +1052,8 @@ custom_sender() {
1052
1053
# role_recipients_ntfy[domainadmin]="${DEFAULT_RECIPIENT_NTFY}"
1054
1055
+# role_recipients_ilert[domainadmin]="${DEFAULT_RECIPIENT_ILERT}"
1056
+
1057
# -----------------------------------------------------------------------------
1058
# database servers alarms
1059
# mysql, redis, memcached, postgres, etc
@@ -1099,6 +1112,8 @@ custom_sender() {
1112
1113
# role_recipients_ntfy[dba]="${DEFAULT_RECIPIENT_NTFY}"
1114
1115
+# role_recipients_ilert[dba]="databases ${DEFAULT_RECIPIENT_ILERT}"
1116
+
1117
# -----------------------------------------------------------------------------
1118
# web servers alarms
1119
# apache, nginx, lighttpd, etc
@@ -1157,6 +1172,8 @@ custom_sender() {
1172
1173
# role_recipients_ntfy[webmaster]="${DEFAULT_RECIPIENT_NTFY}"
1174
1175
+# role_recipients_ilert[webmaster]="${DEFAULT_RECIPIENT_ILERT}"
1176
+
1177
# -----------------------------------------------------------------------------
1178
# proxy servers alarms
1179
# squid, etc
@@ -1215,6 +1232,8 @@ custom_sender() {
1232
1233
# role_recipients_ntfy[proxyadmin]="${DEFAULT_RECIPIENT_NTFY}"
1234
1235
+# role_recipients_ilert[proxyadmin]="${DEFAULT_RECIPIENT_ILERT}"
1236
+
1237
# -----------------------------------------------------------------------------
1238
# peripheral devices
1239
# UPS, photovoltaics, etc
@@ -1270,3 +1289,5 @@ custom_sender() {
1289
# role_recipients_gotify[sitemgr]="${DEFAULT_RECIPIENT_GOTIFY}"
1290
1291
# role_recipients_ntfy[sitemgr]="${DEFAULT_RECIPIENT_NTFY}"
1292
+
1293
+# role_recipients_ilert[sitemgr]="${DEFAULT_RECIPIENT_ILERT}"
src/health/notifications/ilert/metadata.yaml
new
+55
@@ -0,0 +1,55 @@
1
+# yamllint disable rule:line-length
2
+---
3
+- id: "notify-ilert"
4
+ meta:
5
+ name: "ilert"
6
+ link: "https://www.ilert.com/"
7
+ categories:
8
+ - notify.agent
9
+ icon_filename: "ilert.svg"
10
+ keywords:
11
+ - ilert
12
+ overview:
13
+ notification_description: |
14
+ ilert is an alerting and incident management tool. It helps teams reduce response times by enhancing monitoring and ticketing tools with reliable alerts, automatic escalations, on-call schedules, and features for incident response, communication, and status updates.
15
+ Sending notification to ilert via Netdata's Agent alert notification feature includes links, images and resolving of corresponding alerts.
16
+ notification_limitations: ""
17
+ setup:
18
+ prerequisites:
19
+ list:
20
+ - title: ""
21
+ description: |
22
+ - A Netdata alert source in ilert. You can create a [Netdata alert source](https://docs.ilert.com/inbound-integrations/netdata) in [ilert](https://www.ilert.com/).
23
+ - Access to the terminal where Netdata Agent is running
24
+ configuration:
25
+ file:
26
+ name: "health_alarm_notify.conf"
27
+ options:
28
+ description: "The following options can be defined for this notification"
29
+ folding:
30
+ title: "Config Options"
31
+ enabled: true
32
+ list:
33
+ - name: "SEND_ILERT"
34
+ default_value: "YES"
35
+ description: "Set `SEND_ILERT` to YES"
36
+ required: true
37
+ - name: "ILERT_ALERT_SOURCE_URL"
38
+ default_value: ""
39
+ description: "Set `ILERT_ALERT_SOURCE_URL` to your Netdata alert source url in ilert."
40
+ required: true
41
+ examples:
42
+ folding:
43
+ enabled: true
44
+ title: ""
45
+ list:
46
+ - name: "Basic Configuration"
47
+ folding:
48
+ enabled: false
49
+ description: ""
50
+ config: |
51
+ SEND_ILERT="YES"
52
+ ILERT_ALERT_SOURCE_URL="https://api.ilert.com/api/v1/events/netdata/{API-KEY}"
53
+ troubleshooting:
54
+ problems:
55
+ list: []