@cryptotaxi247 / netdata-1
Code Commits Issues Pulls Sessions CI/CD
master
master
in 3,851 lines 142 KB
Raw
1 #!/usr/bin/env bash
2 # SPDX-License-Identifier: GPL-3.0-or-later
3 #
4 # Script to send alarm notifications for netdata
5 #
6 # Features:
7 # - multiple notification methods
8 # - multiple roles per alarm
9 # - multiple recipients per role
10 # - severity filtering per recipient
11 #
12 # Supported notification methods:
13 # - emails by @ktsaou
14 # - slack.com notifications by @ktsaou
15 # - alerta.io notifications by @kattunga
16 # - discord.com notifications by @lowfive
17 # - pushover.net notifications by @ktsaou
18 # - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
19 # - telegram.org notifications by @hashworks #1002
20 # - twilio.com notifications by Levi Blaney @shadycuz #1211
21 # - kafka notifications by @ktsaou #1342
22 # - pagerduty.com notifications by Jim Cooley @jimcooley #1373
23 # - messagebird.com notifications by @tech_no_logical #1453
24 # - smseagle notificaitons by @smseagle
25 # - hipchat notifications by @ktsaou #1561
26 # - fleep notifications by @Ferroin
27 # - prowlapp.com notifications by @Ferroin
28 # - irc notifications by @manosf
29 # - custom notifications by @ktsaou
30 # - syslog messages by @Ferroin
31 # - Microsoft Team notification by @tioumen
32 # - RocketChat notifications by @Hermsi1337 #3777
33 # - Dynatrace Event by @illumine
34 # - Opsgenie by @thiaoftsm #9858
35 # - Gotify by @coffeegrind123
36 # - ntfy.sh by @Dim-P
37 #shellcheck source=/dev/null disable=SC2086,SC2154
38
39 # -----------------------------------------------------------------------------
40 # testing notifications
41
42 cmd_line="'${0}' $(printf "'%s' " "${@}")"
43
44 if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
45 if [ "${2}" = "test" ]; then
46 recipient="${1}"
47 else
48 recipient="${2}"
49 fi
50
51 [ -z "${recipient}" ] && recipient="sysadmin"
52
53 id=1
54 last="CLEAR"
55 test_res=0
56 for x in "WARNING" "CRITICAL" "CLEAR"; do
57 echo >&2
58 echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
59
60 "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0 "" "" "Test" "command to edit the alarm=0=$(hostname)" "" "" "a test alarm"
61 #shellcheck disable=SC2181
62 if [ $? -ne 0 ]; then
63 echo >&2 "# FAILED"
64 test_res=1
65 else
66 echo >&2 "# OK"
67 fi
68
69 last="${x}"
70 id=$((id + 1))
71 done
72
73 exit $test_res
74 fi
75
76 export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin:@sbindir_POST@"
77 export LC_ALL=C
78
79 # -----------------------------------------------------------------------------
80 # logging
81
82 PROGRAM_NAME="$(basename "${0}")"
83
84 # these should be the same with syslog() priorities
85 NDLP_EMERG=0 # system is unusable
86 NDLP_ALERT=1 # action must be taken immediately
87 NDLP_CRIT=2 # critical conditions
88 NDLP_ERR=3 # error conditions
89 NDLP_WARN=4 # warning conditions
90 NDLP_NOTICE=5 # normal but significant condition
91 NDLP_INFO=6 # informational
92 NDLP_DEBUG=7 # debug-level messages
93
94 # the max (numerically) log level we will log
95 LOG_LEVEL=$NDLP_INFO
96
97 set_log_min_priority() {
98 case "${NETDATA_LOG_LEVEL,,}" in
99 "emerg" | "emergency")
100 LOG_LEVEL=$NDLP_EMERG
101 ;;
102
103 "alert")
104 LOG_LEVEL=$NDLP_ALERT
105 ;;
106
107 "crit" | "critical")
108 LOG_LEVEL=$NDLP_CRIT
109 ;;
110
111 "err" | "error")
112 LOG_LEVEL=$NDLP_ERR
113 ;;
114
115 "warn" | "warning")
116 LOG_LEVEL=$NDLP_WARN
117 ;;
118
119 "notice")
120 LOG_LEVEL=$NDLP_NOTICE
121 ;;
122
123 "info")
124 LOG_LEVEL=$NDLP_INFO
125 ;;
126
127 "debug")
128 LOG_LEVEL=$NDLP_DEBUG
129 ;;
130 esac
131 }
132
133 set_log_min_priority
134
135 log() {
136 local level="${1}"
137 shift 1
138
139 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return
140
141 systemd-cat-native --log-as-netdata <<EOFLOG
142 INVOCATION_ID=${NETDATA_INVOCATION_ID}
143 SYSLOG_IDENTIFIER=${PROGRAM_NAME}
144 PRIORITY=${level}
145 THREAD_TAG=alarm-notify
146 ND_LOG_SOURCE=health
147 ND_NIDL_NODE=${host}
148 ND_NIDL_INSTANCE=${chart}
149 ND_NIDL_CONTEXT=${context}
150 ND_ALERT_NAME=${name}
151 ND_ALERT_ID=${alarm_id}
152 ND_ALERT_UNIQUE_ID=${unique_id}
153 ND_ALERT_EVENT_ID=${alarm_event_id}
154 ND_ALERT_TRANSITION_ID=${transition_id//-/}
155 ND_ALERT_CLASS=${classification}
156 ND_ALERT_COMPONENT=${component}
157 ND_ALERT_TYPE=${type}
158 ND_ALERT_RECIPIENT=${roles}
159 ND_ALERT_VALUE=${value}
160 ND_ALERT_VALUE_OLD=${old_value}
161 ND_ALERT_STATUS=${status}
162 ND_ALERT_STATUS_OLD=${old_status}
163 ND_ALERT_UNITS=${units}
164 ND_ALERT_SUMMARY=${summary}
165 ND_ALERT_INFO=${info}
166 ND_ALERT_DURATION=${duration}
167 ND_REQUEST=${cmd_line}
168 MESSAGE_ID=6db0018e83e34320ae2a659d78019fb7
169 MESSAGE=[ALERT NOTIFICATION]: ${*//$'\n'/\\n}
170
171 EOFLOG
172 # AN EMPTY LINE IS NEEDED ABOVE
173 }
174
175 info() {
176 log "$NDLP_INFO" "${@}"
177 }
178
179 warning() {
180 log "$NDLP_WARN" "${@}"
181 }
182
183 error() {
184 log "$NDLP_ERR" "${@}"
185 }
186
187 fatal() {
188 log "$NDLP_ALERT" "${@}"
189 exit 1
190 }
191
192 debug() {
193 log "$NDLP_DEBUG" "${@}"
194 }
195
196 debug=0
197 if [ "${NETDATA_ALARM_NOTIFY_DEBUG-0}" = "1" ]; then
198 debug=1
199 LOG_LEVEL=$NDLP_DEBUG
200 fi
201
202 # -----------------------------------------------------------------------------
203 # check for BASH v4+ (required for associative arrays)
204
205 if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
206 echo >&2 "BASH version 4 or later is required (this is ${BASH_VERSION})."
207 exit 1
208 fi
209
210
211 # -----------------------------------------------------------------------------
212
213 docurl() {
214 if [ -z "${curl}" ]; then
215 error "${curl} is unset."
216 return 1
217 fi
218
219 if [ "${debug}" = "1" ]; then
220 echo >&2 "--- BEGIN curl command ---"
221 printf >&2 "%q " ${curl} "${@}"
222 echo >&2
223 echo >&2 "--- END curl command ---"
224
225 local out code ret
226 out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
227 code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
228 ret=$?
229 echo >&2 "--- BEGIN received response ---"
230 cat >&2 "${out}"
231 echo >&2
232 echo >&2 "--- END received response ---"
233 echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
234 rm "${out}"
235 echo "${code}"
236 return ${ret}
237 fi
238
239 ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
240 return $?
241 }
242
243 # -----------------------------------------------------------------------------
244 # List of all the notification mechanisms we support.
245 # Used in a couple of places to write more compact code.
246
247 method_names="
248 alerta
249 awssns
250 custom
251 discord
252 dynatrace
253 email
254 fleep
255 flock
256 gotify
257 hipchat
258 irc
259 kavenegar
260 matrix
261 messagebird
262 msteams
263 ntfy
264 pd
265 prowl
266 pushbullet
267 pushover
268 rocketchat
269 slack
270 sms
271 syslog
272 telegram
273 twilio
274 smseagle
275 "
276
277 # -----------------------------------------------------------------------------
278 # this is to be overwritten by the config file
279
280 custom_sender() {
281 info "custom notification mechanism is not configured; not sending ${notification_description}"
282 }
283
284 # -----------------------------------------------------------------------------
285 # defaults to allow running this script by hand
286
287 [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
288 [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
289 [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
290 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
291 [ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"
292
293 # -----------------------------------------------------------------------------
294 # parse command line parameters
295
296 if [[ ${1} = "unittest" ]]; then
297 unittest=1 # enable unit testing mode
298 roles="${2}" # the role that should be used for unit testing
299 cfgfile="${3}" # the location of the config file to use for unit testing
300 status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
301 old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
302 elif [[ ${1} = "dump_methods" ]]; then
303 dump_methods=1
304 status="WARNING"
305 else
306 roles="${1}" # the roles that should be notified for this event
307 args_host="${2}" # the host generated this event
308 unique_id="${3}" # the unique id of this event
309 alarm_id="${4}" # the unique id of the alarm that generated this event
310 event_id="${5}" # the incremental id of the event, for this alarm id
311 when="${6}" # the timestamp this event occurred
312 name="${7}" # the name of the alarm, as given in netdata health.d entries
313 chart="${8}" # the name of the chart (type.id)
314 status="${9}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
315 old_status="${10}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
316 value="${11}" # the current value of the alarm
317 old_value="${12}" # the previous value of the alarm
318 src="${13}" # the line number and file the alarm has been configured
319 duration="${14}" # the duration in seconds of the previous alarm state
320 non_clear_duration="${15}" # the total duration in seconds this is/was non-clear
321 units="${16}" # the units of the value
322 info="${17}" # a short description of the alarm
323 value_string="${18}" # friendly value (with units)
324 # shellcheck disable=SC2034
325 # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
326 old_value_string="${19}" # friendly old value (with units), previously named "old_value_string"
327 calc_expression="${20}" # contains the expression that was evaluated to trigger the alarm
328 calc_param_values="${21}" # the values of the parameters in the expression, at the time of the evaluation
329 total_warnings="${22}" # Total number of alarms in WARNING state
330 total_critical="${23}" # Total number of alarms in CRITICAL state
331 total_warn_alarms="${24}" # List of alarms in warning state
332 total_crit_alarms="${25}" # List of alarms in critical state
333 classification="${26}" # The class field from .conf files
334 edit_command_line="${27}" # The command to edit the alarm, with the line number
335 child_machine_guid="${28}" # the machine_guid of the child
336 transition_id="${29}" # the transition_id of the alert
337 summary="${30}" # the summary text field of the alert
338 context="${31}" # the context of the chart
339 component="${32}"
340 type="${33}"
341 fi
342
343 # -----------------------------------------------------------------------------
344 # find a suitable hostname to use, if netdata did not supply a hostname
345
346 if [ -z "${args_host}" ]; then
347 this_host=$(hostname -s 2>/dev/null)
348 host="${this_host}"
349 args_host="${this_host}"
350 else
351 host="${args_host}"
352 fi
353
354 notification_description="notification to '${roles}' for transition from ${old_status} to ${status}, of alert '${name}' = '${value_string}', of instance '${chart}', context '${context}' on host '${host}'"
355
356 # -----------------------------------------------------------------------------
357 # screen statuses we don't need to send a notification
358
359 # don't do anything if this is not WARNING, CRITICAL or CLEAR
360 if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
361 debug "not sending ${notification_description}"
362 exit 1
363 fi
364
365 # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
366 if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
367 debug "not sending ${notification_description}"
368 exit 1
369 fi
370
371 # -----------------------------------------------------------------------------
372 # load configuration
373
374 # By default fetch images from the global public registry.
375 # This is required by default, since all notification methods need to download
376 # images via the Internet, and private registries might not be reachable.
377 # This can be overwritten at the configuration file.
378 images_base_url="https://registry.my-netdata.io"
379
380 # curl options to use
381 curl_options=""
382
383 # hostname handling
384 use_fqdn="NO"
385
386 # needed commands
387 # if empty they will be searched in the system path
388 curl=
389 sendmail=
390
391 # enable / disable features
392 for method_name in ${method_names^^}; do
393 declare SEND_${method_name}="YES"
394 declare DEFAULT_RECIPIENT_${method_name}
395 done
396
397 for method_name in ${method_names}; do
398 declare -A role_recipients_${method_name}
399 done
400
401 # slack configs
402 SLACK_WEBHOOK_URL=
403
404 # Microsoft Teams configs
405 MSTEAMS_WEBHOOK_URL=
406
407 # Legacy Microsoft Teams configs for backwards compatibility:
408 declare -A role_recipients_msteam
409
410 # rocketchat configs
411 ROCKETCHAT_WEBHOOK_URL=
412
413 # alerta configs
414 ALERTA_WEBHOOK_URL=
415 ALERTA_API_KEY=
416
417 # flock configs
418 FLOCK_WEBHOOK_URL=
419
420 # discord configs
421 DISCORD_WEBHOOK_URL=
422
423 # pushover configs
424 PUSHOVER_APP_TOKEN=
425
426 # pushbullet configs
427 PUSHBULLET_ACCESS_TOKEN=
428 PUSHBULLET_SOURCE_DEVICE=
429
430 # twilio configs
431 TWILIO_ACCOUNT_SID=
432 TWILIO_ACCOUNT_TOKEN=
433 TWILIO_NUMBER=
434
435 # hipchat configs
436 HIPCHAT_SERVER=
437 HIPCHAT_AUTH_TOKEN=
438
439 # messagebird configs
440 MESSAGEBIRD_ACCESS_KEY=
441 MESSAGEBIRD_NUMBER=
442
443 #smseagle configs
444 SMSEAGLE_API_URL=
445 SMSEAGLE_API_ACCESSTOKEN=
446 SMSEAGLE_MSG_TYPE="sms"
447 SMSEAGLE_CALL_DURATION="10"
448 SMSEAGLE_VOICE_ID="1"
449
450 # kavenegar configs
451 KAVENEGAR_API_KEY=
452 KAVENEGAR_SENDER=
453
454 # telegram configs
455 TELEGRAM_BOT_TOKEN=
456
457 # kafka configs
458 SEND_KAFKA="YES"
459 KAFKA_URL=
460 KAFKA_SENDER_IP=
461
462 # pagerduty.com configs
463 PD_SERVICE_KEY=
464 USE_PD_VERSION=
465
466 # fleep.io configs
467 FLEEP_SENDER="${host}"
468
469 # Amazon SNS configs
470 AWSSNS_MESSAGE_FORMAT=
471
472 # Matrix configs
473 MATRIX_HOMESERVER=
474 MATRIX_ACCESSTOKEN=
475
476 # syslog configs
477 SYSLOG_FACILITY=
478
479 # email configs
480 EMAIL_SENDER=
481 EMAIL_CHARSET=$(locale charmap 2>/dev/null)
482 EMAIL_THREADING=
483 EMAIL_PLAINTEXT_ONLY=
484
485 # irc configs
486 IRC_NICKNAME=
487 IRC_REALNAME=
488 IRC_NETWORK=
489 IRC_PORT=6667
490
491 # dynatrace configs
492 DYNATRACE_SPACE=
493 DYNATRACE_SERVER=
494 DYNATRACE_TOKEN=
495 DYNATRACE_TAG_VALUE=
496 DYNATRACE_ANNOTATION_TYPE=
497 DYNATRACE_EVENT=
498 SEND_DYNATRACE=
499
500 # gotify configs
501 GOTIFY_APP_URL=
502 GOTIFY_APP_TOKEN=
503
504 # opsgenie configs
505 OPSGENIE_API_KEY=
506
507 # signl4 configs
508 SIGNL4_WEBHOOK_URL=
509
510 # load the stock and user configuration files
511 # these will overwrite the variables above
512
513 if [ ${unittest} ]; then
514 if source "${cfgfile}"; then
515 error "Failed to load requested config file."
516 exit 1
517 fi
518 else
519 for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
520 if [ -f "${CONFIG}" ]; then
521 debug "Loading config file '${CONFIG}'..."
522 source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
523 else
524 debug "Cannot find file '${CONFIG}'."
525 fi
526 done
527 fi
528
529 if [[ ! $curl_options =~ .*\--connect-timeout ]]; then
530 curl_options+=" --connect-timeout 5"
531 fi
532
533 OPSGENIE_API_URL=${OPSGENIE_API_URL:-"https://api.opsgenie.com"}
534 TELEGRAM_API_URL=${TELEGRAM_API_URL:-"https://api.telegram.org"}
535
536 # If we didn't autodetect the character set for e-mail and it wasn't
537 # set by the user, we need to set it to a reasonable default. UTF-8
538 # should be correct for almost all modern UNIX systems.
539 if [ -z ${EMAIL_CHARSET} ]; then
540 EMAIL_CHARSET="UTF-8"
541 fi
542
543 # If we've been asked to use FQDN's for the URL's in the alarm, do so,
544 # unless we're sending an alarm for a child system which we can't get the
545 # FQDN of easily.
546 if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
547 host="$(hostname -f 2>/dev/null)"
548 fi
549
550
551 # -----------------------------------------------------------------------------
552 # migrate old Microsoft Teams configuration keys after loading configuration
553
554 msteams_migration() {
555 SEND_MSTEAMS=${SEND_MSTEAM:-$SEND_MSTEAMS}
556 unset -v SEND_MSTEAM
557 DEFAULT_RECIPIENT_MSTEAMS=${DEFAULT_RECIPIENT_MSTEAM:-$DEFAULT_RECIPIENT_MSTEAMS}
558 MSTEAMS_WEBHOOK_URL=${MSTEAM_WEBHOOK_URL:-$MSTEAMS_WEBHOOK_URL}
559 MSTEAMS_ICON_DEFAULT=${MSTEAM_ICON_DEFAULT:-$MSTEAMS_ICON_DEFAULT}
560 MSTEAMS_ICON_CLEAR=${MSTEAM_ICON_CLEAR:-$MSTEAMS_ICON_CLEAR}
561 MSTEAMS_ICON_WARNING=${MSTEAM_ICON_WARNING:-$MSTEAMS_ICON_WARNING}
562 MSTEAMS_ICON_CRITICAL=${MSTEAM_ICON_CRITICAL:-$MSTEAMS_ICON_CRITICAL}
563 MSTEAMS_COLOR_DEFAULT=${MSTEAM_COLOR_DEFAULT:-$MSTEAMS_COLOR_DEFAULT}
564 MSTEAMS_COLOR_CLEAR=${MSTEAM_COLOR_CLEAR:-$MSTEAMS_COLOR_CLEAR}
565 MSTEAMS_COLOR_WARNING=${MSTEAM_COLOR_WARNING:-$MSTEAMS_COLOR_WARNING}
566 MSTEAMS_COLOR_CRITICAL=${MSTEAM_COLOR_CRITICAL:-$MSTEAMS_COLOR_CRITICAL}
567
568 # migrate role specific recipients:
569 for key in "${!role_recipients_msteam[@]}"; do
570 # Disable check, if role_recipients_msteams is ever used:
571 # The role_recipients_$method are created and used programmatically
572 # by iterating over $methods. shellcheck therefore doesn't realize
573 # that role_recipients_msteams is actually used in the block
574 # "find the recipients' addresses per method".
575 # shellcheck disable=SC2034
576 role_recipients_msteams["$key"]="${role_recipients_msteam["$key"]}"
577 done
578 }
579
580 msteams_migration
581
582 # -----------------------------------------------------------------------------
583 # filter a recipient based on alarm event severity
584
585 filter_recipient_by_criticality() {
586 local method="${1}" recipient_arg="${2}"
587 local tracking_dir tracking_file modifier modifiers recipient="${recipient_arg/|*/}"
588 local mod_critical=0 mod_noclear=0 mod_nowarn=0
589
590 # no severity filtering for this person
591 [ "${recipient}" = "${recipient_arg}" ] && return 0
592
593 # find out which modifiers are set
594 modifiers="${recipient_arg#*|}"
595 modifiers="${modifiers//|/ }" # replace pipes with spaces
596 modifiers="${modifiers,,}" # lowercase
597 for modifier in ${modifiers}; do
598 case "${modifier}" in
599 critical) mod_critical=1 ;;
600 noclear) mod_noclear=1 ;;
601 nowarn) mod_nowarn=1 ;;
602
603 *)
604 error "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: invalid modifier '${modifier}'."
605 # invalid modifier, always send notification
606 return 0
607 ;;
608 esac
609 done
610
611 # set status tracking directory/file var
612 tracking_dir="${NETDATA_CACHE_DIR}/alarm-notify/${method}/${recipient}"
613 tracking_file="${tracking_dir}/${alarm_id}"
614
615 # create the status tracking directory for this user if "critical" modifier is set
616 [ "${mod_critical}" == "1" ] && [ ! -d "${tracking_dir}" ] && mkdir -p "${tracking_dir}"
617
618 case "${status}" in
619 CRITICAL)
620 # "critical" modifier set, create tracking file for future status changes
621 if [ "${mod_critical}" == "1" ]; then
622 touch "${tracking_file}"
623 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
624 return 0
625 fi
626
627 # always send CRITICAL notification
628 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is CRITICAL"
629 return 0
630 ;;
631
632 WARNING)
633 # "nowarn" modifier set, block notification
634 if [ "${mod_nowarn}" == "1" ]; then
635 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: BLOCK: recipient should not receive this notification (nowarn modifier set)"
636 return 1
637 fi
638
639 # "critical" modifier not set, send notification
640 if [ "${mod_critical}" == "0" ]; then
641 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is WARNING"
642 return 0
643 fi
644
645 # "critical" modifier set, send notification if tracking file exists
646 if [ "${mod_critical}" == "1" ] && [ -f "${tracking_file}" ]; then
647 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
648 return 0
649 fi
650 ;;
651
652 CLEAR)
653 if [ -f "${tracking_file}" ]; then
654 tracking_file_existed="yes"
655 rm "${tracking_file}"
656 else
657 tracking_file_existed=""
658 fi
659
660 # "noclear" modifier set, block notification
661 if [ "${mod_noclear}" == "1" ]; then
662 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: BLOCK: recipient should not receive this notification (noclear modifier set)"
663 return 1
664 fi
665
666 # "critical" modifier not set, send notification
667 if [ "${mod_critical}" == "0" ]; then
668 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: the alarm is CLEAR"
669 return 0
670 fi
671
672 # "critical" modifier set, send notification if tracking file exists
673 if [ "${mod_critical}" == "1" ] && [ -n "${tracking_file_existed}" ]; then
674 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (no status change will be sent from now)"
675 return 0
676 fi
677 ;;
678
679 *)
680 # "critical" modifier set, send notification if tracking file exists
681 if [ "${mod_critical}" == "1" ] && [ -f "${tracking_file}" ]; then
682 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
683 return 0
684 fi
685 ;;
686 esac
687
688 debug "SEVERITY FILTERING for ${recipient_arg} VIA ${method}: BLOCK: recipient should not receive this notification"
689 return 1
690 }
691
692 # -----------------------------------------------------------------------------
693 # check the configured targets
694
695 # check email
696 if [ "${SEND_EMAIL}" = "AUTO" ]; then
697 if command -v curl >/dev/null 2>&1; then
698 SEND_EMAIL="YES"
699 else
700 SEND_EMAIL="NO"
701 fi
702 fi
703
704 # check slack
705 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
706
707 # check rocketchat
708 [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
709
710 # check alerta
711 [ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
712
713 # check flock
714 [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
715
716 # check discord
717 [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
718
719 # check pushover
720 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
721
722 # check pushbullet
723 [ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
724
725 # check twilio
726 { [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
727
728 # check hipchat
729 [ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
730
731 # check messagebird
732 { [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
733
734 # check smseagle
735 { [ -z "${SMSEAGLE_API_URL}" ] || [ -z "${SMSEAGLE_API_ACCESSTOKEN}" ] || [ -z "${SMSEAGLE_MSG_TYPE}" ]; } && SEND_SMSEAGLE="NO"
736
737 # check kavenegar
738 { [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
739
740 # check telegram
741 [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
742
743 # check kafka
744 { [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
745
746 # check irc
747 [ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
748
749 # check fleep
750 #shellcheck disable=SC2153
751 { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
752
753 # check dynatrace
754 { [ -z "${DYNATRACE_SPACE}" ] ||
755 [ -z "${DYNATRACE_SERVER}" ] ||
756 [ -z "${DYNATRACE_TOKEN}" ] ||
757 [ -z "${DYNATRACE_TAG_VALUE}" ] ||
758 [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
759
760 # check opsgenie
761 [ -z "${OPSGENIE_API_KEY}" ] && SEND_OPSGENIE="NO"
762
763 # check matrix
764 { [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
765
766 # check gotify
767 { [ -z "${GOTIFY_APP_TOKEN}" ] || [ -z "${GOTIFY_APP_URL}" ]; } && SEND_GOTIFY="NO"
768
769 # check ntfy
770 [ -z "${DEFAULT_RECIPIENT_NTFY}" ] && SEND_NTFY="NO"
771
772 # check msteams
773 [ -z "${MSTEAMS_WEBHOOK_URL}" ] && SEND_MSTEAMS="NO"
774
775 # check pd
776 [ -z "${DEFAULT_RECIPIENT_PD}" ] && SEND_PD="NO"
777
778 # check prowl
779 [ -z "${DEFAULT_RECIPIENT_PROWL}" ] && SEND_PROWL="NO"
780
781 # check custom
782 [ -z "${DEFAULT_RECIPIENT_CUSTOM}" ] && SEND_CUSTOM="NO"
783
784 # check ilert
785 [ -z "${ILERT_ALERT_SOURCE_URL}" ] && SEND_ILERT="NO"
786
787 # check signl4
788 [ -z "${SIGNL4_WEBHOOK_URL}" ] && SEND_SIGNL4="NO"
789
790 # -----------------------------------------------------------------------------
791 # check the availability of targets
792
793 check_supported_targets() {
794 local log=${1}
795 shift
796
797 if [ "${SEND_PUSHOVER}" = "YES" ] ||
798 [ "${SEND_SLACK}" = "YES" ] ||
799 [ "${SEND_ROCKETCHAT}" = "YES" ] ||
800 [ "${SEND_ALERTA}" = "YES" ] ||
801 [ "${SEND_PD}" = "YES" ] ||
802 [ "${SEND_FLOCK}" = "YES" ] ||
803 [ "${SEND_DISCORD}" = "YES" ] ||
804 [ "${SEND_HIPCHAT}" = "YES" ] ||
805 [ "${SEND_TWILIO}" = "YES" ] ||
806 [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
807 [ "${SEND_SMSEAGLE}" = "YES" ] ||
808 [ "${SEND_KAVENEGAR}" = "YES" ] ||
809 [ "${SEND_TELEGRAM}" = "YES" ] ||
810 [ "${SEND_PUSHBULLET}" = "YES" ] ||
811 [ "${SEND_KAFKA}" = "YES" ] ||
812 [ "${SEND_FLEEP}" = "YES" ] ||
813 [ "${SEND_PROWL}" = "YES" ] ||
814 [ "${SEND_MATRIX}" = "YES" ] ||
815 [ "${SEND_CUSTOM}" = "YES" ] ||
816 [ "${SEND_MSTEAMS}" = "YES" ] ||
817 [ "${SEND_DYNATRACE}" = "YES" ] ||
818 [ "${SEND_OPSGENIE}" = "YES" ] ||
819 [ "${SEND_GOTIFY}" = "YES" ] ||
820 [ "${SEND_NTFY}" = "YES" ] ||
821 [ "${SEND_ILERT}" = "YES" ] ||
822 [ "${SEND_SIGNL4}" = "YES" ]; then
823 # if we need curl, check for the curl command
824 if [ -z "${curl}" ]; then
825 curl="$(command -v curl 2>/dev/null)"
826 fi
827 if [ -z "${curl}" ]; then
828 $log "Cannot find curl command in the system path. Disabling all curl based notifications."
829 SEND_PUSHOVER="NO"
830 SEND_PUSHBULLET="NO"
831 SEND_TELEGRAM="NO"
832 SEND_SLACK="NO"
833 SEND_MSTEAMS="NO"
834 SEND_ROCKETCHAT="NO"
835 SEND_ALERTA="NO"
836 SEND_PD="NO"
837 SEND_FLOCK="NO"
838 SEND_DISCORD="NO"
839 SEND_TWILIO="NO"
840 SEND_HIPCHAT="NO"
841 SEND_MESSAGEBIRD="NO"
842 SEND_SMSEAGLE="NO"
843 SEND_KAVENEGAR="NO"
844 SEND_KAFKA="NO"
845 SEND_FLEEP="NO"
846 SEND_PROWL="NO"
847 SEND_MATRIX="NO"
848 SEND_CUSTOM="NO"
849 SEND_DYNATRACE="NO"
850 SEND_OPSGENIE="NO"
851 SEND_GOTIFY="NO"
852 SEND_NTFY="NO"
853 SEND_ILERT="NO"
854 SEND_SIGNL4="NO"
855 fi
856 fi
857
858 if [ "${SEND_SMS}" = "YES" ]; then
859 if [ -z "${sendsms}" ]; then
860 sendsms="$(command -v sendsms 2>/dev/null)"
861 fi
862 if [ -z "${sendsms}" ]; then
863 SEND_SMS="NO"
864 fi
865 fi
866 # if we need sendmail, check for the sendmail command
867 if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
868 sendmail="$(command -v sendmail 2>/dev/null)"
869 if [ -z "${sendmail}" ]; then
870 $log "Cannot find sendmail command in the system path. Disabling email notifications."
871 SEND_EMAIL="NO"
872 fi
873 fi
874
875 # if we need logger, check for the logger command
876 if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
877 logger="$(command -v logger 2>/dev/null)"
878 if [ -z "${logger}" ]; then
879 $log "Cannot find logger command in the system path. Disabling syslog notifications."
880 SEND_SYSLOG="NO"
881 fi
882 fi
883
884 # if we need aws, check for the aws command
885 if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
886 aws="$(command -v aws 2>/dev/null)"
887 if [ -z "${aws}" ]; then
888 $log "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
889 SEND_AWSSNS="NO"
890 fi
891 fi
892
893 # if we need nc, check for the nc command
894 if [ "${SEND_IRC}" = "YES" ] && [ -z "${nc}" ]; then
895 nc="$(command -v nc 2>/dev/null)"
896 if [ -z "${nc}" ]; then
897 $log "Cannot find nc command in the system path. Disabling IRC notifications."
898 SEND_IRC="NO"
899 fi
900 fi
901 }
902
903 if [ ${dump_methods} ]; then
904 check_supported_targets debug
905 for name in "${!SEND_@}"; do
906 if [ "${!name}" = "YES" ]; then
907 echo "$name"
908 fi
909 done
910 exit 0
911 fi
912
913 # -----------------------------------------------------------------------------
914 # find the recipients' addresses per method
915
916 # netdata may call us with multiple roles, and roles may have multiple but
917 # overlapping recipients - so, here we find the unique recipients.
918 have_to_send_something="NO"
919 for method_name in ${method_names}; do
920 send_var="SEND_${method_name^^}"
921 if [ "${!send_var}" = "NO" ]; then
922 continue
923 fi
924
925 declare -A arr_var=()
926
927 for x in ${roles//,/ }; do
928 # the roles 'silent' and 'disabled' mean:
929 # don't send a notification for this role
930 if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
931 continue
932 fi
933
934 role_recipients="role_recipients_${method_name}[$x]"
935 default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
936
937 a="${!role_recipients}"
938 [ -z "${a}" ] && a="${!default_recipient_var}"
939 for r in ${a//,/ }; do
940 [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
941 done
942 done
943
944 # build the list of recipients
945 to_var="to_${method_name}"
946 declare to_${method_name}="${!arr_var[*]}"
947
948 if [ -z "${!to_var}" ]; then
949 declare ${send_var}="NO"
950 else
951 have_to_send_something="YES"
952 fi
953 done
954
955 # -----------------------------------------------------------------------------
956 # handle fixup of the email recipient list.
957
958 fix_to_email() {
959 to_email=
960 while [ -n "${1}" ]; do
961 [ -n "${to_email}" ] && to_email="${to_email}, "
962 to_email="${to_email}${1}"
963 shift 1
964 done
965 }
966
967 # ${to_email} without quotes here
968 fix_to_email ${to_email}
969
970 # -----------------------------------------------------------------------------
971 # handle output if we're running in unit test mode
972 if [ ${unittest} ]; then
973 for method_name in ${method_names}; do
974 to_var="to_${method_name}"
975 echo "results: ${method_name}: ${!to_var}"
976 done
977 exit 0
978 fi
979
980 # -----------------------------------------------------------------------------
981 # check that we have at least a method enabled
982 proceed=0
983 for method in "${SEND_EMAIL}" \
984 "${SEND_PUSHOVER}" \
985 "${SEND_TELEGRAM}" \
986 "${SEND_SLACK}" \
987 "${SEND_ROCKETCHAT}" \
988 "${SEND_ALERTA}" \
989 "${SEND_FLOCK}" \
990 "${SEND_DISCORD}" \
991 "${SEND_TWILIO}" \
992 "${SEND_HIPCHAT}" \
993 "${SEND_MESSAGEBIRD}" \
994 "${SEND_SMSEAGLE}" \
995 "${SEND_KAVENEGAR}" \
996 "${SEND_PUSHBULLET}" \
997 "${SEND_KAFKA}" \
998 "${SEND_PD}" \
999 "${SEND_FLEEP}" \
1000 "${SEND_PROWL}" \
1001 "${SEND_MATRIX}" \
1002 "${SEND_CUSTOM}" \
1003 "${SEND_IRC}" \
1004 "${SEND_AWSSNS}" \
1005 "${SEND_SYSLOG}" \
1006 "${SEND_SMS}" \
1007 "${SEND_MSTEAMS}" \
1008 "${SEND_DYNATRACE}" \
1009 "${SEND_OPSGENIE}" \
1010 "${SEND_GOTIFY}" \
1011 "${SEND_NTFY}" \
1012 "${SEND_ILERT}" \
1013 "${SEND_SIGNL4}" ; do
1014
1015 if [ "${method}" == "YES" ]; then
1016 proceed=1
1017 break
1018 fi
1019 done
1020
1021 if [ "$proceed" -eq 0 ]; then
1022 if [ "${have_to_send_something}" = "NO" ]; then
1023 debug "All notification methods are disabled; not sending ${notification_description}."
1024 exit 0
1025 else
1026 fatal "All notification methods are disabled; not sending ${notification_description}."
1027 fi
1028 fi
1029
1030 check_supported_targets error
1031
1032 # -----------------------------------------------------------------------------
1033 # get the date the alarm happened
1034
1035 date=$(date --date=@${when} "${date_format}" 2>/dev/null)
1036 [ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
1037 [ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
1038 [ -z "${date}" ] && date=$(date 2>/dev/null)
1039
1040 # -----------------------------------------------------------------------------
1041 # get the date in utc the alarm happened
1042
1043 date_utc=$(date --date=@${when} "${date_format}" -u 2>/dev/null)
1044 [ -z "${date_utc}" ] && date_utc=$(date -u "${date_format}" 2>/dev/null)
1045 [ -z "${date_utc}" ] && date_utc=$(date -u --date=@${when} 2>/dev/null)
1046 [ -z "${date_utc}" ] && date_utc=$(date -u 2>/dev/null)
1047
1048 # ----------------------------------------------------------------------------
1049 # prepare some extra headers if we've been asked to thread e-mails
1050 if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
1051 email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
1052 else
1053 email_thread_headers=
1054 fi
1055
1056 # -----------------------------------------------------------------------------
1057 # function to URL encode a string
1058
1059 urlencode() {
1060 local string="${1}" strlen encoded pos c o
1061
1062 strlen=${#string}
1063 for ((pos = 0; pos < strlen; pos++)); do
1064 c=${string:pos:1}
1065 case "${c}" in
1066 [-_.~a-zA-Z0-9])
1067 o="${c}"
1068 ;;
1069
1070 *)
1071 printf -v o '%%%02x' "'${c}"
1072 ;;
1073 esac
1074 encoded+="${o}"
1075 done
1076
1077 REPLY="${encoded}"
1078 echo "${REPLY}"
1079 }
1080
1081 # -----------------------------------------------------------------------------
1082 # function to convert a duration in seconds, to a human readable duration
1083 # using DAYS, MINUTES, SECONDS
1084
1085 duration4human() {
1086 local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
1087 d=$((s / 86400))
1088 s=$((s - (d * 86400)))
1089 h=$((s / 3600))
1090 s=$((s - (h * 3600)))
1091 m=$((s / 60))
1092 s=$((s - (m * 60)))
1093
1094 if [ ${d} -gt 0 ]; then
1095 [ ${m} -ge 30 ] && h=$((h + 1))
1096 [ ${d} -gt 1 ] && ds="days"
1097 [ ${h} -gt 1 ] && hs="hours"
1098 if [ ${h} -gt 0 ]; then
1099 ret="${d} ${ds} and ${h} ${hs}"
1100 else
1101 ret="${d} ${ds}"
1102 fi
1103 elif [ ${h} -gt 0 ]; then
1104 [ ${s} -ge 30 ] && m=$((m + 1))
1105 [ ${h} -gt 1 ] && hs="hours"
1106 [ ${m} -gt 1 ] && ms="minutes"
1107 if [ ${m} -gt 0 ]; then
1108 ret="${h} ${hs} and ${m} ${ms}"
1109 else
1110 ret="${h} ${hs}"
1111 fi
1112 elif [ ${m} -gt 0 ]; then
1113 [ ${m} -gt 1 ] && ms="minutes"
1114 [ ${s} -gt 1 ] && ss="seconds"
1115 if [ ${s} -gt 0 ]; then
1116 ret="${m} ${ms} and ${s} ${ss}"
1117 else
1118 ret="${m} ${ms}"
1119 fi
1120 else
1121 [ ${s} -gt 1 ] && ss="seconds"
1122 ret="${s} ${ss}"
1123 fi
1124
1125 REPLY="${ret}"
1126 echo "${REPLY}"
1127 }
1128
1129 # -----------------------------------------------------------------------------
1130 # email sender
1131
1132 send_email() {
1133 local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
1134 if [ "${SEND_EMAIL}" = "YES" ]; then
1135
1136 if [ -n "${EMAIL_SENDER}" ]; then
1137 if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
1138 # the name includes double quotes
1139 sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
1140 sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
1141 elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
1142 # the name includes single quotes
1143 sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
1144 sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
1145 elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
1146 # the name does not have any quotes
1147 sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
1148 sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
1149 fi
1150 fi
1151
1152 [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
1153 [ -n "${sender_name}" ] && ${sendmail} -F 2>&1 | head -1 | grep -qv "sendmail: unrecognized option: F" && opts+=(-F "${sender_name}")
1154
1155 if [ "${debug}" = "1" ]; then
1156 echo >&2 "--- BEGIN sendmail command ---"
1157 printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
1158 echo >&2
1159 echo >&2 "--- END sendmail command ---"
1160 fi
1161
1162 local cmd_output
1163 cmd_output=$("${sendmail}" -t "${opts[@]}" 2>&1)
1164 ret=$?
1165
1166 if [ ${ret} -eq 0 ]; then
1167 info "sent email to '${to_email}' for ${notification_description}"
1168 return 0
1169 else
1170 error "failed to send email to '${to_email}' for ${notification_description}, with error code ${ret} (${cmd_output})."
1171 return 1
1172 fi
1173 fi
1174
1175 return 1
1176 }
1177
1178 # -----------------------------------------------------------------------------
1179 # pushover sender
1180
1181 send_pushover() {
1182 local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
1183
1184 if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
1185
1186 # https://pushover.net/api
1187 priority=-2
1188 case "${status}" in
1189 CLEAR) priority=-1 ;; # low priority: no sound or vibration
1190 WARNING) priority=0 ;; # normal priority: respect quiet hours
1191 CRITICAL) priority=1 ;; # high priority: bypass quiet hours
1192 *) priority=-2 ;; # lowest priority: no notification at all
1193 esac
1194
1195 # Pushover API limits: title=250, message=1024, url=512, url_title=100
1196 # Use UTF-8 locale for truncation to avoid splitting multibyte characters
1197 local _lc="${LC_ALL}"; LC_ALL=C.UTF-8
1198 [ ${#title} -gt 250 ] && title="${title:0:247}..."
1199 [ ${#message} -gt 1024 ] && message="${message:0:1021}..."
1200 [ ${#url} -gt 512 ] && url=""
1201 LC_ALL="${_lc}"
1202
1203 for user in ${usertokens}; do
1204 httpcode=$(docurl \
1205 --form-string "token=${apptoken}" \
1206 --form-string "user=${user}" \
1207 --form-string "html=1" \
1208 --form-string "title=${title}" \
1209 --form-string "message=${message}" \
1210 --form-string "timestamp=${when}" \
1211 --form-string "url=${url}" \
1212 --form-string "url_title=Open netdata dashboard to view the alarm" \
1213 --form-string "priority=${priority}" \
1214 https://api.pushover.net/1/messages.json)
1215
1216 if [ "${httpcode}" = "200" ]; then
1217 info "sent pushover notification to '${user}' for ${notification_description}"
1218 sent=$((sent + 1))
1219 else
1220 error "failed to send pushover notification to '${user}' for ${notification_description}, with HTTP response status code ${httpcode}."
1221 fi
1222 done
1223
1224 [ ${sent} -gt 0 ] && return 0
1225 fi
1226
1227 return 1
1228 }
1229
1230 # -----------------------------------------------------------------------------
1231 # pushbullet sender
1232
1233 send_pushbullet() {
1234 local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 userOrChannelTag
1235 if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
1236
1237 # https://docs.pushbullet.com/#create-push
1238 # Accept specification of user(s) (PushBullet account email address) and/or channel tag(s), separated by spaces.
1239 # If recipient begins with a "#" then send to channel tag, otherwise send to email recipient.
1240
1241 for userOrChannelTag in ${recipients}; do
1242 if [ "${userOrChannelTag::1}" = "#" ]; then
1243 userOrChannelTag_type="channel_tag"
1244 userOrChannelTag="${userOrChannelTag:1}" # Remove hash from start of channel tag (required by pushbullet API)
1245 else
1246 userOrChannelTag_type="email"
1247 fi
1248
1249 httpcode=$(docurl \
1250 --header 'Access-Token: '${userapikey}'' \
1251 --header 'Content-Type: application/json' \
1252 --data-binary @<(
1253 cat <<EOF
1254 {"title": "${title}",
1255 "type": "link",
1256 "${userOrChannelTag_type}": "${userOrChannelTag}",
1257 "body": "$(echo -n ${message})",
1258 "url": "${url}",
1259 "source_device_iden": "${source_device}"}
1260 EOF
1261 ) "https://api.pushbullet.com/v2/pushes" -X POST)
1262
1263 if [ "${httpcode}" = "200" ]; then
1264 info "sent pushbullet notification to '${userOrChannelTag}' for ${notification_description}"
1265 sent=$((sent + 1))
1266 else
1267 error "failed to send pushbullet notification to '${userOrChannelTag}' for ${notification_description}, with HTTP response status code ${httpcode}."
1268 fi
1269 done
1270
1271 [ ${sent} -gt 0 ] && return 0
1272 fi
1273
1274 return 1
1275 }
1276
1277 # -----------------------------------------------------------------------------
1278 # kafka sender
1279
1280 send_kafka() {
1281 local httpcode sent=0
1282 if [ "${SEND_KAFKA}" = "YES" ]; then
1283 httpcode=$(docurl -X POST \
1284 --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
1285 "${KAFKA_URL}")
1286
1287 if [ "${httpcode}" = "204" ]; then
1288 info "sent kafka data to '${KAFKA_SENDER_IP}' for ${notification_description}"
1289 sent=$((sent + 1))
1290 else
1291 error "failed to send kafka data to '${KAFKA_SENDER_IP}' for ${notification_description}, with HTTP response status code ${httpcode}."
1292 fi
1293
1294 [ ${sent} -gt 0 ] && return 0
1295 fi
1296
1297 return 1
1298 }
1299
1300 # -----------------------------------------------------------------------------
1301 # pagerduty.com sender
1302
1303 send_pd() {
1304 local recipients="${1}" sent=0 severity current_time payload url response_code
1305 unset t
1306 case ${status} in
1307 CLEAR) t='resolve' ; severity='info' ;;
1308 WARNING) t='trigger' ; severity='warning' ;;
1309 CRITICAL) t='trigger' ; severity='critical' ;;
1310 esac
1311
1312 if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
1313 if [ "$(uname)" == "Linux" ]; then
1314 current_time=$(date -d @${when} +'%Y-%m-%dT%H:%M:%S.000')
1315 else
1316 current_time=$(date -r ${when} +'%Y-%m-%dT%H:%M:%S.000')
1317 fi
1318 for PD_SERVICE_KEY in ${recipients}; do
1319 d="${status} ${name} = ${value_string} - ${host}"
1320 if [ ${USE_PD_VERSION} = "2" ]; then
1321 payload="$(
1322 cat <<EOF
1323 {
1324 "payload" : {
1325 "summary": "${info:0:1024}",
1326 "source" : "${args_host}",
1327 "severity" : "${severity}",
1328 "timestamp" : "${current_time}",
1329 "class" : "${chart}",
1330 "custom_details": {
1331 "value_w_units": "${value_string}",
1332 "when": "${when}",
1333 "duration" : "${duration}",
1334 "roles": "${roles}",
1335 "alarm_id" : "${alarm_id}",
1336 "name" : "${name}",
1337 "chart" : "${chart}",
1338 "status" : "${status}",
1339 "old_status" : "${old_status}",
1340 "value" : "${value}",
1341 "old_value" : "${old_value}",
1342 "src" : "${src}",
1343 "non_clear_duration" : "${non_clear_duration}",
1344 "units" : "${units}",
1345 "info" : "${info}"
1346 }
1347 },
1348 "routing_key": "${PD_SERVICE_KEY}",
1349 "event_action": "${t}",
1350 "dedup_key": "${unique_id}"
1351 }
1352 EOF
1353 )"
1354 url="https://events.pagerduty.com/v2/enqueue"
1355 response_code="202"
1356 else
1357 payload="$(
1358 cat <<EOF
1359 {
1360 "service_key": "${PD_SERVICE_KEY}",
1361 "event_type": "${t}",
1362 "incident_key" : "${alarm_id}",
1363 "description": "${d}",
1364 "details": {
1365 "value_w_units": "${value_string}",
1366 "when": "${when}",
1367 "duration" : "${duration}",
1368 "roles": "${roles}",
1369 "alarm_id" : "${alarm_id}",
1370 "name" : "${name}",
1371 "chart" : "${chart}",
1372 "status" : "${status}",
1373 "old_status" : "${old_status}",
1374 "value" : "${value}",
1375 "old_value" : "${old_value}",
1376 "src" : "${src}",
1377 "non_clear_duration" : "${non_clear_duration}",
1378 "units" : "${units}",
1379 "info" : "${info}"
1380 }
1381 }
1382 EOF
1383 )"
1384 url="https://events.pagerduty.com/generic/2010-04-15/create_event.json"
1385 response_code="200"
1386 fi
1387 httpcode=$(docurl -X POST --data "${payload}" ${url})
1388 if [ "${httpcode}" = "${response_code}" ]; then
1389 info "sent pagerduty event for ${notification_description}"
1390 sent=$((sent + 1))
1391 else
1392 error "failed to send pagerduty event for ${notification_description}, with HTTP response status code ${httpcode}."
1393 fi
1394 done
1395
1396 [ ${sent} -gt 0 ] && return 0
1397 fi
1398
1399 return 1
1400 }
1401
1402 # -----------------------------------------------------------------------------
1403 # twilio sender
1404
1405 send_twilio() {
1406 local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
1407 if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
1408 #https://www.twilio.com/packages/labs/code/bash/twilio-sms
1409 for user in ${recipients}; do
1410 httpcode=$(docurl -X POST \
1411 --data-urlencode "From=${twilionumber}" \
1412 --data-urlencode "To=${user}" \
1413 --data-urlencode "Body=${title} ${message}" \
1414 -u "${accountsid}:${accounttoken}" \
1415 "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
1416
1417 if [ "${httpcode}" = "201" ]; then
1418 info "sent Twilio SMS to '${user}' for ${notification_description}"
1419 sent=$((sent + 1))
1420 else
1421 error "failed to send Twilio SMS to '${user}' for ${notification_description}, with HTTP response status code ${httpcode}."
1422 fi
1423 done
1424
1425 [ ${sent} -gt 0 ] && return 0
1426 fi
1427
1428 return 1
1429 }
1430
1431 # -----------------------------------------------------------------------------
1432 # hipchat sender
1433
1434 send_hipchat() {
1435 local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
1436
1437 # remove <small></small> from the message
1438 message="${message//<small>/}"
1439 message="${message//<\/small>/}"
1440
1441 if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
1442 # Valid values: html, text.
1443 # Defaults to 'html'.
1444 msg_format="html"
1445
1446 # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
1447 case "${status}" in
1448 WARNING) color="yellow" ;;
1449 CRITICAL) color="red" ;;
1450 CLEAR) color="green" ;;
1451 *) color="gray" ;;
1452 esac
1453
1454 # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
1455 # Each recipient's notification preferences are taken into account.
1456 # Defaults to false.
1457 notify="true"
1458
1459 for room in ${recipients}; do
1460 httpcode=$(docurl -X POST \
1461 -H "Content-type: application/json" \
1462 -H "Authorization: Bearer ${authtoken}" \
1463 -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
1464 "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
1465
1466 if [ "${httpcode}" = "204" ]; then
1467 info "sent HipChat notification to '${room}' for ${notification_description}"
1468 sent=$((sent + 1))
1469 else
1470 error "failed to send HipChat notification to '${room}' for ${notification_description}, with HTTP response status code ${httpcode}."
1471 fi
1472 done
1473
1474 [ ${sent} -gt 0 ] && return 0
1475 fi
1476
1477 return 1
1478 }
1479
1480 # -----------------------------------------------------------------------------
1481 # messagebird sender
1482
1483 send_messagebird() {
1484 local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
1485 if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
1486 #https://developers.messagebird.com/docs/messaging
1487 for user in ${recipients}; do
1488 httpcode=$(docurl -X POST \
1489 --data-urlencode "originator=${messagebirdnumber}" \
1490 --data-urlencode "recipients=${user}" \
1491 --data-urlencode "body=${title} ${message}" \
1492 --data-urlencode "datacoding=auto" \
1493 -H "Authorization: AccessKey ${accesskey}" \
1494 "https://rest.messagebird.com/messages")
1495
1496 if [ "${httpcode}" = "201" ]; then
1497 info "sent Messagebird SMS to '${user}' for ${notification_description}"
1498 sent=$((sent + 1))
1499 else
1500 error "failed to send Messagebird SMS to '${user}' for ${notification_description}, with HTTP response status code ${httpcode}."
1501 fi
1502 done
1503
1504 [ ${sent} -gt 0 ] && return 0
1505 fi
1506
1507 return 1
1508 }
1509
1510 # -----------------------------------------------------------------------------
1511 # smseagle sender
1512
1513 send_smseagle() {
1514 local address="${1}" access_token="${2}" msg_type="${3}" recipients="${4}" duration="${5}" voice_id="${6}" msg="${host} ${status_message}: ${chart}, ${alarm}" httpcode sent=0 user
1515 if [ "${SEND_SMSEAGLE}" = "YES" ] && [ -n "${address}" ] && [ -n "${access_token}" ] && [ -n "${msg_type}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
1516
1517 if [ -z "${duration}" ] && { [ "${msg_type}" = "ring" ] || [ "${msg_type}" = "tts" ] || [ "${msg_type}" = "tts_advanced" ]; }; then
1518 duration=10
1519 fi
1520
1521 if [ -z "${voice_id}" ] && [ "${msg_type}" = "tts_advanced" ]; then
1522 voice_id=1
1523 fi
1524
1525 IFS=' ' read -r -a recipientsArray <<< "${recipients}"
1526 for ((i=0; i<${#recipientsArray[@]}; i++)); do
1527 if [ $i -gt 0 ]; then
1528 to+=","
1529 fi
1530 to+="\"${recipientsArray[$i]}\""
1531 done
1532
1533 payload=$(cat <<EOF
1534 {
1535 "to": [${to}],
1536 "text": "${msg}",
1537 "duration": ${duration},
1538 "voice_id": ${voice_id}
1539 }
1540 EOF
1541 )
1542 msg_type_endpoint="messages/sms"
1543
1544 case ${msg_type} in
1545 "sms") msg_type_endpoint="messages/sms" ;;
1546 "mms") msg_type_endpoint="messages/mms" ;;
1547 "ring") msg_type_endpoint="calls/ring" ;;
1548 "tts") msg_type_endpoint="calls/tts" ;;
1549 "tts_advanced") msg_type_endpoint="calls/tts_advanced" ;;
1550 *)
1551 esac
1552
1553 httpcode=$(docurl -X POST \
1554 --data "${payload}" \
1555 -H "access-token: ${access_token}" \
1556 -H "Accept: application/json" \
1557 -H "Content-Type: application/json" \
1558 "${address}/api/v2/${msg_type_endpoint}"
1559 )
1560
1561 if [ "${httpcode}" = "200" ]; then
1562 info "Sending successful for ${notification_description}"
1563 return 0
1564 fi
1565
1566 error "Sending failed for ${notification_description}, with response code ${httpcode}."
1567 return 1
1568 fi
1569 return 1
1570 }
1571
1572 # -----------------------------------------------------------------------------
1573 # kavenegar sender
1574
1575 send_kavenegar() {
1576 local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
1577 if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
1578 # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
1579 for user in ${recipients}; do
1580 httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
1581 --data-urlencode "sender=${kavenegarsender}" \
1582 --data-urlencode "receptor=${user}" \
1583 --data-urlencode "message=${title} ${message}")
1584
1585 if [ "${httpcode}" = "200" ]; then
1586 info "sent Kavenegar SMS to '${user}' for ${notification_description}"
1587 sent=$((sent + 1))
1588 else
1589 error "failed to send Kavenegar SMS to '${user}' for ${notification_description}, with HTTP response status code ${httpcode}."
1590 fi
1591 done
1592
1593 [ ${sent} -gt 0 ] && return 0
1594 fi
1595
1596 return 1
1597 }
1598
1599 # -----------------------------------------------------------------------------
1600 # telegram sender
1601
1602 send_telegram() {
1603 local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
1604
1605 if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
1606
1607 case "${status}" in
1608 WARNING) emoji="⚠️" ;;
1609 CRITICAL) emoji="🔴" ;;
1610 CLEAR) emoji="✅" ;;
1611 *) emoji="⚪️" ;;
1612 esac
1613
1614 if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
1615 for chatid in ${chatids}; do
1616 notify_telegram=1
1617 notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0}
1618
1619 IFS=":" read -r chatID threadID <<< "${chatid}"
1620
1621 # https://core.telegram.org/bots/api#sendmessage
1622 api_url="${TELEGRAM_API_URL}/bot${bottoken}/sendMessage?chat_id=${chatID}"
1623 if [ -n "${threadID}" ]; then
1624 api_url+="&message_thread_id=${threadID}"
1625 fi
1626
1627 while [ ${notify_telegram} -eq 1 ]; do
1628 httpcode=$(docurl ${disableNotification} \
1629 --data-urlencode "parse_mode=HTML" \
1630 --data-urlencode "disable_web_page_preview=true" \
1631 --data-urlencode "text=${emoji} ${message}" \
1632 "${api_url}")
1633
1634 notify_telegram=0
1635
1636 if [ "${httpcode}" = "200" ]; then
1637 info "sent telegram notification to '${chatid}' for ${notification_description}"
1638 sent=$((sent + 1))
1639 elif [ "${httpcode}" = "401" ]; then
1640 error "failed to send telegram notification to '${chatid}' for ${notification_description}, wrong bot token."
1641 elif [ "${httpcode}" = "429" ]; then
1642 if [ "$notify_retries" -gt 0 ]; then
1643 error "failed to send telegram notification to '${chatid}' for ${notification_description}, rate limit exceeded, retrying after 1s."
1644 notify_retries=$((notify_retries - 1))
1645 notify_telegram=1
1646 sleep 1
1647 else
1648 error "failed to send telegram notification to '${chatid}' for ${notification_description}, rate limit exceeded."
1649 fi
1650 else
1651 error "failed to send telegram notification to '${chatid}' for ${notification_description}, with HTTP response status code ${httpcode}."
1652 fi
1653 done
1654 done
1655
1656 [ ${sent} -gt 0 ] && return 0
1657 fi
1658
1659 return 1
1660 }
1661
1662 # -----------------------------------------------------------------------------
1663 # Microsoft Team sender
1664
1665 send_msteams() {
1666
1667 local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
1668
1669 [ "${SEND_MSTEAMS}" != "YES" ] && return 1
1670
1671 case "${status}" in
1672 WARNING) icon="${MSTEAMS_ICON_WARNING}" && color="${MSTEAMS_COLOR_WARNING}" ;;
1673 CRITICAL) icon="${MSTEAMS_ICON_CRITICAL}" && color="${MSTEAMS_COLOR_CRITICAL}" ;;
1674 CLEAR) icon="${MSTEAMS_ICON_CLEAR}" && color="${MSTEAMS_COLOR_CLEAR}" ;;
1675 *) icon="${MSTEAMS_ICON_DEFAULT}" && color="${MSTEAMS_COLOR_DEFAULT}" ;;
1676 esac
1677
1678 for channel in ${channels}; do
1679 ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
1680 ## Online designer : https://adaptivecards.io/designer/
1681 payload="$(
1682 cat <<EOF
1683 {
1684 "@context": "http://schema.org/extensions",
1685 "@type": "MessageCard",
1686 "themeColor": "${color}",
1687 "title": "$icon Alert ${status} from netdata for ${host}",
1688 "text": "${host} ${status_message}, ${chart}, *${alarm}*",
1689 "potentialAction": [
1690 {
1691 "@type": "OpenUri",
1692 "name": "Netdata",
1693 "targets": [
1694 { "os": "default", "uri": "${goto_url}" }
1695 ]
1696 }
1697 ]
1698 }
1699 EOF
1700 )"
1701
1702 # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
1703 cur_webhook="${webhook//CHANNEL/${channel}}"
1704
1705 httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${cur_webhook}")
1706
1707 if [ "${httpcode}" = "200" ]; then
1708 info "sent Microsoft team notification to '${cur_webhook}' for ${notification_description}"
1709 sent=$((sent + 1))
1710 else
1711 error "failed to send Microsoft team to '${cur_webhook}' for ${notification_description}, with HTTP response status code ${httpcode}."
1712 fi
1713 done
1714
1715 [ ${sent} -gt 0 ] && return 0
1716
1717 return 1
1718 }
1719
1720 # slack sender
1721
1722 send_slack() {
1723 local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
1724
1725 [ "${SEND_SLACK}" != "YES" ] && return 1
1726
1727 case "${status}" in
1728 WARNING) color="warning" ;;
1729 CRITICAL) color="danger" ;;
1730 CLEAR) color="good" ;;
1731 *) color="#777777" ;;
1732 esac
1733
1734 for channel in ${channels}; do
1735 # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
1736 if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
1737
1738 # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
1739 if [ "${channel}" = "#" ]; then
1740 ch=""
1741 chstr="without specifying a channel"
1742 else
1743 ch="\"channel\": \"${channel}\","
1744 chstr="to '${channel}'"
1745 fi
1746
1747 payload="$(
1748 cat <<EOF
1749 {
1750 $ch
1751 "username": "netdata on ${host}",
1752 "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
1753 "text": "${host} ${status_message}, \`${chart}\`, *${alarm}*",
1754 "attachments": [
1755 {
1756 "fallback": "${alarm} - ${chart} - ${info}",
1757 "color": "${color}",
1758 "title": "${alarm}",
1759 "title_link": "${goto_url}",
1760 "text": "${info}",
1761 "fields": [
1762 {
1763 "title": "${chart}",
1764 "value": "chart",
1765 "short": true
1766 }
1767 ],
1768 "thumb_url": "${image}",
1769 "footer": "by ${host}",
1770 "ts": ${when}
1771 }
1772 ]
1773 }
1774 EOF
1775 )"
1776
1777 httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
1778 if [ "${httpcode}" = "200" ]; then
1779 info "sent slack notification ${chstr} for ${notification_description}"
1780 sent=$((sent + 1))
1781 else
1782 error "failed to send slack notification ${chstr} for ${notification_description}, with HTTP response status code ${httpcode}."
1783 fi
1784 done
1785
1786 [ ${sent} -gt 0 ] && return 0
1787
1788 return 1
1789 }
1790
1791 # -----------------------------------------------------------------------------
1792 # rocketchat sender
1793
1794 send_rocketchat() {
1795 local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
1796
1797 [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
1798
1799 case "${status}" in
1800 WARNING) color="warning" ;;
1801 CRITICAL) color="danger" ;;
1802 CLEAR) color="good" ;;
1803 *) color="#777777" ;;
1804 esac
1805
1806 for channel in ${channels}; do
1807 payload="$(
1808 cat <<EOF
1809 {
1810 "channel": "#${channel}",
1811 "alias": "netdata on ${host}",
1812 "avatar": "${images_base_url}/images/banner-icon-144x144.png",
1813 "text": "${host} ${status_message}, \`${chart}\`, *${alarm}*",
1814 "attachments": [
1815 {
1816 "color": "${color}",
1817 "title": "${alarm}",
1818 "title_link": "${goto_url}",
1819 "text": "${info}",
1820 "fields": [
1821 {
1822 "title": "${chart}",
1823 "short": true,
1824 "value": "chart"
1825 }
1826 ],
1827 "thumb_url": "${image}",
1828 "ts": "${when}"
1829 }
1830 ]
1831 }
1832 EOF
1833 )"
1834
1835 httpcode=$(docurl -X POST -H "Content-Type: application/json" --data "${payload}" "${webhook}")
1836 if [ "${httpcode}" = "200" ]; then
1837 info "sent rocketchat notification to '${channel}' for ${notification_description}"
1838 sent=$((sent + 1))
1839 else
1840 error "failed to send rocketchat notification to '${channel}' for ${notification_description}, with HTTP response status code ${httpcode}."
1841 fi
1842 done
1843
1844 [ ${sent} -gt 0 ] && return 0
1845
1846 return 1
1847 }
1848
1849 # -----------------------------------------------------------------------------
1850 # alerta sender
1851
1852 send_alerta() {
1853 local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
1854
1855 [ "${SEND_ALERTA}" != "YES" ] && return 1
1856
1857 case "${status}" in
1858 CRITICAL) severity="critical" ;;
1859 WARNING) severity="warning" ;;
1860 CLEAR) severity="cleared" ;;
1861 *) severity="indeterminate" ;;
1862 esac
1863
1864 if [[ ${chart} == httpcheck* ]]; then
1865 resource=$chart
1866 event=$name
1867 else
1868 resource="${host}"
1869 event="${chart}.${name}"
1870 fi
1871
1872 for channel in ${channels}; do
1873 payload="$(
1874 cat <<EOF
1875 {
1876 "resource": "${resource}",
1877 "event": "${event}",
1878 "environment": "${channel}",
1879 "severity": "${severity}",
1880 "service": ["Netdata"],
1881 "group": "Performance",
1882 "value": "${value_string}",
1883 "text": "${info}",
1884 "tags": ["alarm_id:${alarm_id}"],
1885 "attributes": {
1886 "roles": "${roles}",
1887 "name": "${name}",
1888 "chart": "${chart}",
1889 "source": "${src}",
1890 "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
1891 },
1892 "origin": "netdata/${host}",
1893 "type": "netdataAlarm",
1894 "rawData": "${BASH_ARGV[@]}"
1895 }
1896 EOF
1897 )"
1898
1899 if [ -n "${ALERTA_API_KEY}" ]; then
1900 auth="Key ${ALERTA_API_KEY}"
1901 fi
1902
1903 httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
1904
1905 if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
1906 info "sent alerta notification to '${channel}' for ${notification_description}"
1907 sent=$((sent + 1))
1908 elif [ "${httpcode}" = "202" ]; then
1909 info "suppressed alerta notification to '${channel}' for ${notification_description}"
1910 else
1911 error "failed to send alerta notification to '${channel}' for ${notification_description}, with HTTP response status code ${httpcode}."
1912 fi
1913 done
1914
1915 [ ${sent} -gt 0 ] && return 0
1916
1917 return 1
1918 }
1919
1920 # -----------------------------------------------------------------------------
1921 # flock sender
1922
1923 send_flock() {
1924 local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
1925
1926 [ "${SEND_FLOCK}" != "YES" ] && return 1
1927
1928 case "${status}" in
1929 WARNING) color="warning" ;;
1930 CRITICAL) color="danger" ;;
1931 CLEAR) color="good" ;;
1932 *) color="#777777" ;;
1933 esac
1934
1935 for channel in ${channels}; do
1936 httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
1937 \"sendAs\": {
1938 \"name\" : \"netdata on ${host}\",
1939 \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
1940 },
1941 \"text\": \"${host} *${status_message}*\",
1942 \"timestamp\": \"${when}\",
1943 \"attachments\": [
1944 {
1945 \"description\": \"${chart} - ${info}\",
1946 \"color\": \"${color}\",
1947 \"title\": \"${alarm}\",
1948 \"url\": \"${goto_url}\",
1949 \"text\": \"${info}\",
1950 \"views\": {
1951 \"image\": {
1952 \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
1953 \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
1954 \"filename\": \"${image}\"
1955 }
1956 }
1957 }
1958 ]
1959 }")
1960 if [ "${httpcode}" = "200" ]; then
1961 info "sent flock notification to '${channel}' for ${notification_description}"
1962 sent=$((sent + 1))
1963 else
1964 error "failed to send flock notification to '${channel}' for ${notification_description}, with HTTP response status code ${httpcode}."
1965 fi
1966 done
1967
1968 [ ${sent} -gt 0 ] && return 0
1969
1970 return 1
1971 }
1972
1973 # -----------------------------------------------------------------------------
1974 # discord sender
1975
1976 send_discord() {
1977 local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
1978
1979 [ "${SEND_DISCORD}" != "YES" ] && return 1
1980
1981 case "${status}" in
1982 WARNING) color="warning" ;;
1983 CRITICAL) color="danger" ;;
1984 CLEAR) color="good" ;;
1985 *) color="#777777" ;;
1986 esac
1987
1988 for channel in ${channels}; do
1989 username="netdata on ${host}"
1990 [ ${#username} -gt 32 ] && username="${username:0:29}..."
1991
1992 payload="$(
1993 cat <<EOF
1994 {
1995 "channel": "#${channel}",
1996 "username": "${username}",
1997 "text": "${host} ${status_message}, \`${chart}\`, *${alarm}*",
1998 "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
1999 "attachments": [
2000 {
2001 "color": "${color}",
2002 "title": "${alarm}",
2003 "title_link": "${goto_url}",
2004 "text": "${info}",
2005 "fields": [
2006 {
2007 "title": "${chart}"
2008 }
2009 ],
2010 "thumb_url": "${image}",
2011 "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
2012 "footer": "${host}",
2013 "ts": ${when}
2014 }
2015 ]
2016 }
2017 EOF
2018 )"
2019
2020 httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
2021 if [ "${httpcode}" = "200" ]; then
2022 info "sent discord notification to '${channel}' for ${notification_description}"
2023 sent=$((sent + 1))
2024 else
2025 error "failed to send discord notification to '${channel}' for ${notification_description}, with HTTP response status code ${httpcode}."
2026 fi
2027 done
2028
2029 [ ${sent} -gt 0 ] && return 0
2030
2031 return 1
2032 }
2033
2034 # -----------------------------------------------------------------------------
2035 # fleep sender
2036
2037 send_fleep() {
2038 local httpcode sent=0 webhooks="${1}" data message
2039 if [ "${SEND_FLEEP}" = "YES" ]; then
2040 message="${host} ${status_message}, \`${chart}\`, *${alarm}*\\n${info}"
2041
2042 for hook in ${webhooks}; do
2043 data="{ "
2044 data="${data} 'message': '${message}', "
2045 data="${data} 'user': '${FLEEP_SENDER}' "
2046 data="${data} }"
2047
2048 httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
2049
2050 if [ "${httpcode}" = "200" ]; then
2051 info "sent fleep data to user '${FLEEP_SENDER}' for ${notification_description}"
2052 sent=$((sent + 1))
2053 else
2054 error "failed to send fleep data to user '${FLEEP_SENDER}' for ${notification_description}, with HTTP response status code ${httpcode}."
2055 fi
2056 done
2057
2058 [ ${sent} -gt 0 ] && return 0
2059 fi
2060
2061 return 1
2062 }
2063
2064 # -----------------------------------------------------------------------------
2065 # Prowl sender
2066
2067 send_prowl() {
2068 local httpcode sent=0 data message keys prio=0 alarm_url event
2069 if [ "${SEND_PROWL}" = "YES" ]; then
2070 message="$(urlencode "${host} ${status_message}, \`${chart}\`, *${alarm}*\\n${info}")"
2071 message="description=${message}"
2072 keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
2073 keys="apikey=${keys}"
2074 app="application=Netdata"
2075
2076 case "${status}" in
2077 CRITICAL)
2078 prio=2
2079 ;;
2080 WARNING)
2081 prio=1
2082 ;;
2083 esac
2084 prio="priority=${prio}"
2085
2086 alarm_url="$(urlencode ${goto_url})"
2087 alarm_url="url=${alarm_url}"
2088 event="$(urlencode "${host} ${status_message}")"
2089 event="event=${event}"
2090
2091 data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
2092
2093 httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
2094
2095 if [ "${httpcode}" = "200" ]; then
2096 info "sent prowl event for ${notification_description}"
2097 sent=1
2098 else
2099 error "failed to send prowl event for ${notification_description}, with HTTP response status code ${httpcode}."
2100 fi
2101
2102 [ ${sent} -gt 0 ] && return 0
2103 fi
2104
2105 return 1
2106 }
2107
2108 # -----------------------------------------------------------------------------
2109 # irc sender
2110
2111 send_irc() {
2112 local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" PORT="${5}" SERVERNAME="${6}" MESSAGE="${7}" sent=0 channel color send_alarm reply_codes error
2113
2114 if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ] && [ -n "${PORT}" ]; then
2115 case "${status}" in
2116 WARNING) color="warning" ;;
2117 CRITICAL) color="danger" ;;
2118 CLEAR) color="good" ;;
2119 *) color="#777777" ;;
2120 esac
2121
2122 SNDMESSAGE="${MESSAGE//$'\n'/", "}"
2123 for CHANNEL in ${CHANNELS}; do
2124 error=0
2125 send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${SNDMESSAGE}\\nQUIT\\n" \ | ${nc} "${NETWORK}" "${PORT}")
2126 reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
2127 for code in ${reply_codes}; do
2128 if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
2129 error=1
2130 break
2131 fi
2132 done
2133
2134 if [ "${error}" -eq 0 ]; then
2135 info "sent irc notification to '${CHANNEL}' for ${notification_description}"
2136 sent=$((sent + 1))
2137 else
2138 error "failed to send irc notification to '${CHANNEL}' for ${notification_description}, with error code ${code}."
2139 fi
2140 done
2141 fi
2142
2143 [ ${sent} -gt 0 ] && return 0
2144
2145 return 1
2146 }
2147
2148 # -----------------------------------------------------------------------------
2149 # Amazon SNS sender
2150
2151 send_awssns() {
2152 local targets="${1}" message='' sent=0 region=''
2153 local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
2154
2155 [ "${SEND_AWSSNS}" = "YES" ] || return 1
2156
2157 message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
2158
2159 for target in ${targets}; do
2160 # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
2161 region="$(echo ${target} | cut -f 4 -d ':')"
2162 if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
2163 info "sent Amazon SNS notification to '${target}' for ${notification_description}"
2164 sent=$((sent + 1))
2165 else
2166 error "failed to send Amazon SNS notification to '${target}' for ${notification_description}"
2167 fi
2168 done
2169
2170 [ ${sent} -gt 0 ] && return 0
2171
2172 return 1
2173 }
2174
2175 # -----------------------------------------------------------------------------
2176 # Matrix sender
2177
2178 send_matrix() {
2179 local homeserver="${1}" webhook accesstoken rooms="${2}" httpcode sent=0 payload txnid
2180
2181 [ "${SEND_MATRIX}" != "YES" ] && return 1
2182 [ -z "${MATRIX_ACCESSTOKEN}" ] && return 1
2183
2184 accesstoken="${MATRIX_ACCESSTOKEN}"
2185
2186 case "${status}" in
2187 WARNING) emoji="⚠️" ;;
2188 CRITICAL) emoji="🔴" ;;
2189 CLEAR) emoji="✅" ;;
2190 *) emoji="⚪️" ;;
2191 esac
2192
2193 for room in ${rooms}; do
2194 txnid="nd_$(date +%s)_$$_$RANDOM"
2195 webhook="$homeserver/_matrix/client/r0/rooms/$(urlencode $room)/send/m.room.message/$txnid"
2196 payload="$(
2197 cat <<EOF
2198 {
2199 "msgtype": "m.notice",
2200 "format": "org.matrix.custom.html",
2201 "formatted_body": "${emoji} ${host} ${status_message} - <b>${name//_/ }</b><br>${chart}<br><a href=\"${goto_url}\">${alarm}</a><br><i>${info}</i>",
2202 "body": "${emoji} ${host} ${status_message} - ${name//_/ } ${chart} ${goto_url} ${alarm} ${info}"
2203 }
2204 EOF
2205 )"
2206
2207 httpcode=$(docurl -X PUT -H "Authorization: Bearer ${accesstoken}" -H "Content-Type: application/json" --data "${payload}" "${webhook}")
2208 if [ "${httpcode}" == "200" ]; then
2209 info "sent Matrix notification to '${room}' for ${notification_description}"
2210 sent=$((sent + 1))
2211 else
2212 error "failed to send Matrix notification to '${room}' for ${notification_description}, with HTTP response status code ${httpcode}."
2213 fi
2214 done
2215
2216 [ ${sent} -gt 0 ] && return 0
2217
2218 return 1
2219 }
2220
2221 # -----------------------------------------------------------------------------
2222 # syslog sender
2223
2224 send_syslog() {
2225 local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
2226 local priority='' message='' server='' port='' prefix=''
2227 local temp1='' temp2=''
2228
2229 [ "${SEND_SYSLOG}" = "YES" ] || return 1
2230
2231 if [ "${status}" = "CRITICAL" ]; then
2232 level='crit'
2233 elif [ "${status}" = "WARNING" ]; then
2234 level='warning'
2235 fi
2236
2237 for target in ${targets}; do
2238 priority="${facility}.${level}"
2239 message=''
2240 server=''
2241 port=''
2242 prefix=''
2243 temp1=''
2244 temp2=''
2245
2246 prefix=$(echo ${target} | cut -d '/' -f 2)
2247 temp1=$(echo ${target} | cut -d '/' -f 1)
2248
2249 if [ ${prefix} != ${temp1} ]; then
2250 if (echo ${temp1} | grep -q '@'); then
2251 temp2=$(echo ${temp1} | cut -d '@' -f 1)
2252 server=$(echo ${temp1} | cut -d '@' -f 2)
2253
2254 if [ ${temp2} != ${server} ]; then
2255 priority=${temp2}
2256 fi
2257
2258 port=$(echo ${server} | rev | cut -d ':' -f 1 | rev)
2259
2260 if (echo ${server} | grep -E -q '\[.*\]'); then
2261 if (echo ${port} | grep -q ']'); then
2262 port=''
2263 else
2264 server=$(echo ${server} | rev | cut -d ':' -f 2- | rev)
2265 fi
2266 else
2267 if [ ${port} = ${server} ]; then
2268 port=''
2269 else
2270 server=$(echo ${server} | cut -d ':' -f 1)
2271 fi
2272 fi
2273 else
2274 priority=${temp1}
2275 fi
2276 fi
2277
2278 message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
2279
2280 if [ ${server} ]; then
2281 logger_options="${logger_options} -n ${server}"
2282 if [ ${port} ]; then
2283 logger_options="${logger_options} -P ${port}"
2284 fi
2285 fi
2286
2287 ${logger} -p ${priority} ${logger_options} "${message}"
2288 done
2289
2290 return $?
2291 }
2292
2293 # -----------------------------------------------------------------------------
2294 # SMS sender
2295
2296 send_sms() {
2297 local recipients="${1}" errcode errmessage sent=0
2298
2299 # Human readable SMS
2300 local msg="${host} ${status_message}: ${chart}, ${alarm}"
2301
2302 # limit it to 160 characters
2303 msg="${msg:0:160}"
2304
2305 if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
2306 # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
2307 for phone in ${recipients}; do
2308 errmessage=$($sendsms $phone "$msg" 2>&1)
2309 errcode=$?
2310 if [ ${errcode} -eq 0 ]; then
2311 info "sent smstools3 SMS to '${user}' for ${notification_description}"
2312 sent=$((sent + 1))
2313 else
2314 error "failed to send smstools3 SMS to '${user}' for ${notification_description}, with error code ${errcode}: ${errmessage}."
2315 fi
2316 done
2317
2318 [ ${sent} -gt 0 ] && return 0
2319 fi
2320
2321 return 1
2322 }
2323
2324 # -----------------------------------------------------------------------------
2325 # Dynatrace sender
2326
2327 send_dynatrace() {
2328 [ "${SEND_DYNATRACE}" != "YES" ] && return 1
2329
2330 local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
2331 local description="Netdata Notification for: ${host} ${chart}.${name} is ${status}"
2332 local payload=""
2333
2334 payload=$(cat <<EOF
2335 {
2336 "title": "Netdata Alarm from ${host}",
2337 "source" : "${DYNATRACE_ANNOTATION_TYPE}",
2338 "description" : "${description}",
2339 "eventType": "${DYNATRACE_EVENT}",
2340 "attachRules":{
2341 "tagRule":[{
2342 "meTypes":["HOST"],
2343 "tags":["${DYNATRACE_TAG_VALUE}"]
2344 }]
2345 },
2346 "customProperties":{
2347 "description": "${description}"
2348 }
2349 }
2350 EOF
2351 )
2352
2353 # echo ${payload}
2354
2355 httpcode=$(docurl -X POST -H "Authorization: Api-token ${DYNATRACE_TOKEN}" -H "Content-Type: application/json" -d "${payload}" ${dynatrace_url})
2356 ret=$?
2357
2358
2359 if [ ${ret} -eq 0 ]; then
2360 if [ "${httpcode}" = "200" ]; then
2361 info "sent Dynatrace event '${DYNATRACE_EVENT}' to '${DYNATRACE_SERVER}' for ${notification_description}"
2362 return 0
2363 else
2364 warning "failed to send Dynatrace event to '${DYNATRACE_SERVER}' for ${notification_description}, with HTTP response status code ${httpcode}"
2365 return 1
2366 fi
2367 else
2368 error "failed to sent Dynatrace '${DYNATRACE_EVENT}' to '${DYNATRACE_SERVER}' for ${notification_description}, with code ${ret}."
2369 return 1
2370 fi
2371 }
2372
2373 # -----------------------------------------------------------------------------
2374 # Opsgenie sender
2375
2376 send_opsgenie() {
2377 local payload httpcode oldv currv priority
2378 [ "${SEND_OPSGENIE}" != "YES" ] && return 1
2379
2380 if [ -z "${OPSGENIE_API_KEY}" ] ; then
2381 info "Can't send Opsgenie notification, because OPSGENIE_API_KEY is not defined"
2382 return 1
2383 fi
2384
2385 # Priority for OpsGenie alert (https://support.atlassian.com/opsgenie/docs/update-alert-priority-level/)
2386 case "${status}" in
2387 CRITICAL) priority="P1" ;; # Critical is P1
2388 WARNING) priority="P3" ;; # Warning is P3
2389 CLEAR) priority="P5" ;; # Clear is P5
2390 *) priority="P3" ;; # OpsGenie's default alert level is P3
2391 esac
2392
2393 # We are sending null when values are nan to avoid errors while JSON message is parsed
2394 [ "${old_value}" != "nan" ] && oldv="${old_value}" || oldv="null"
2395 [ "${value}" != "nan" ] && currv="${value}" || currv="null"
2396
2397 payload=$(cat <<EOF
2398 {
2399 "host" : "${host}",
2400 "unique_id" : "${unique_id}",
2401 "alarmId" : ${alarm_id},
2402 "eventId" : ${event_id},
2403 "chart" : "${chart}",
2404 "when": ${when},
2405 "name" : "${name}",
2406 "priority" : "${priority}",
2407 "status" : "${status}",
2408 "old_status" : "${old_status}",
2409 "value" : ${currv},
2410 "old_value" : ${oldv},
2411 "duration": ${duration},
2412 "non_clear_duration": ${non_clear_duration},
2413 "units" : "${units}",
2414 "info" : "${status_message}, ${info}",
2415 "calc_expression" : "${calc_expression}",
2416 "total_warnings" : "${total_warnings}",
2417 "total_critical" : "${total_critical}",
2418 "src" : "${src}"
2419 }
2420 EOF
2421 )
2422
2423 httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${OPSGENIE_API_URL}/v1/json/integrations/webhooks/netdata?apiKey=${OPSGENIE_API_KEY}")
2424 # https://docs.opsgenie.com/docs/alert-api#create-alert
2425 if [ "${httpcode}" = "200" ]; then
2426 info "sent opsgenie event for ${notification_description}"
2427 else
2428 error "failed to send opsgenie event for ${notification_description}, with HTTP response status code ${httpcode}."
2429 return 1
2430 fi
2431
2432 return 0
2433 }
2434
2435 # -----------------------------------------------------------------------------
2436 # Gotify sender
2437
2438 send_gotify() {
2439 local payload httpcode priority
2440 [ "${SEND_GOTIFY}" != "YES" ] && return 1
2441
2442 if [ -z "${GOTIFY_APP_TOKEN}" ] ; then
2443 info "Can't send Gotify notification, because GOTIFY_APP_TOKEN is not defined"
2444 return 1
2445 fi
2446
2447 # priority for Gotify Android app
2448 case "${status}" in
2449 CRITICAL) priority=10 ;; # sound + vibration
2450 WARNING) priority=4 ;; # sound
2451 *) priority=1 ;; # notification only
2452 esac
2453
2454 payload=$(cat <<EOF
2455 {
2456 "title" : "${status}, ${name} = ${value_string}, on ${host}",
2457 "message" : "${date}: ${chart} ${value_string}",
2458 "priority" : ${priority}
2459 }
2460 EOF
2461 )
2462
2463 httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${GOTIFY_APP_URL}/message?token=${GOTIFY_APP_TOKEN}")
2464 if [ "${httpcode}" = "200" ]; then
2465 info "sent gotify event for ${notification_description}"
2466 else
2467 error "failed to send gotify event for ${notification_description}, with HTTP response status code ${httpcode}."
2468 return 1
2469 fi
2470
2471 return 0
2472 }
2473
2474 # -----------------------------------------------------------------------------
2475 # ntfy sender
2476
2477 send_ntfy() {
2478 local httpcode priority recipients=${1} sent=0 msg
2479
2480 [ "${SEND_NTFY}" != "YES" ] && return 1
2481
2482 case "${status}" in
2483 WARNING) emoji="warning" ;;
2484 CRITICAL) emoji="red_circle" ;;
2485 CLEAR) emoji="white_check_mark" ;;
2486 *) emoji="white_circle" ;;
2487 esac
2488
2489 case ${status} in
2490 WARNING) priority="high";;
2491 CRITICAL) priority="urgent";;
2492 *) priority="default" ;;
2493 esac
2494
2495 # Adding ntfy header generation logic
2496 # Heavily inspired by https://github.com/nickexyz/ntfy-shellscripts/blob/main/sabnzbd.sh
2497 tmp_header=""
2498 if [[ -n "${NTFY_USERNAME}" ]] && [[ -n "${NTFY_PASSWORD}" ]]; then
2499 ntfy_base64=$( echo -n "$NTFY_USERNAME:$NTFY_PASSWORD" | base64 )
2500 tmp_header="Authorization: Basic ${ntfy_base64}"
2501 elif [ -n "${NTFY_ACCESS_TOKEN}" ]; then
2502 tmp_header="Authorization: Bearer ${NTFY_ACCESS_TOKEN}"
2503 fi
2504 ntfy_auth_header=()
2505 if [ -n "${tmp_header}" ]; then
2506 ntfy_auth_header=("-H" "${tmp_header}")
2507 fi
2508 for recipient in ${recipients}; do
2509 msg="${host} ${status_message}: ${alarm} - ${info}"
2510 httpcode=$(docurl -X POST \
2511 "${ntfy_auth_header[@]}" \
2512 -H "Title: ${host}: ${name//_/ }" \
2513 -H "Tags: ${emoji}" \
2514 -H "Priority: ${priority}" \
2515 -H "Actions: view, View node, ${goto_url}, clear=true;" \
2516 -d "${msg}" \
2517 ${recipient})
2518 if [ "${httpcode}" == "200" ]; then
2519 info "sent ntfy notification to '${recipient}' for ${notification_description}"
2520 sent=$((sent + 1))
2521 else
2522 error "failed to send ntfy notification to '${recipient}' for ${notification_description}, with HTTP response status code ${httpcode}."
2523 fi
2524 done
2525
2526 [ ${sent} -gt 0 ] && return 0
2527
2528 return 1
2529 }
2530
2531 # -----------------------------------------------------------------------------
2532 # ilert sender
2533
2534 send_ilert() {
2535 local payload httpcode
2536 [ "${SEND_ILERT}" != "YES" ] && return 1
2537
2538 if [ -z "${ILERT_ALERT_SOURCE_URL}" ] ; then
2539 info "Can't send ilert notification, because ILERT_ALERT_SOURCE_URL is not defined"
2540 return 1
2541 fi
2542
2543 payload=$(cat <<EOF
2544 {
2545 "alert" : "${name}",
2546 "alert_url" : "${goto_url}",
2547 "alarm_id" : ${alarm_id},
2548 "chart" : "${chart}",
2549 "date" : "${when}",
2550 "duration" : "${duration_txt}",
2551 "host" : "${host}",
2552 "info" : "${info}",
2553 "message" : "${status_message}",
2554 "severity": ${status},
2555 "total_critical" : "${total_critical}",
2556 "total_warnings" : "${total_warnings}",
2557 "value" : "${value_string}",
2558 "image_url": "${image}",
2559 "src" : "${src}"
2560 }
2561 EOF
2562 )
2563
2564 httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${ILERT_ALERT_SOURCE_URL}")
2565 if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "202" ]; then
2566 info "sent ilert event for ${notification_description}"
2567 else
2568 error "failed to send ilert event for ${notification_description}, with HTTP response status code ${httpcode}."
2569 return 1
2570 fi
2571
2572 return 0
2573 }
2574
2575 # -----------------------------------------------------------------------------
2576 # signl4 sender
2577
2578 send_signl4() {
2579 local payload httpcode signl4_status
2580 [ "${SEND_SIGNL4}" != "YES" ] && return 1
2581
2582 signl4_status="new"
2583 if [ "${status}" = "CLEAR" ]; then signl4_status="resolved"; fi
2584
2585 payload=$(cat <<EOF
2586 {
2587 "title" : "${name}",
2588 "message" : "${status_message}",
2589 "alert_url" : "${goto_url}",
2590 "alarm_id" : "${alarm_id}",
2591 "chart" : "${chart}",
2592 "date" : "${when}",
2593 "duration" : "${duration_txt}",
2594 "host" : "${host}",
2595 "info" : "${info}",
2596 "severity": "${status}",
2597 "total_critical" : "${total_critical}",
2598 "total_warnings" : "${total_warnings}",
2599 "value" : "${value_string}",
2600 "image_url": "${image}",
2601 "src" : "${src}",
2602 "X-S4-ExternalID" : "${unique_id}",
2603 "X-S4-Status" : "${signl4_status}",
2604 "X-S4-SourceSystem" : "Netdata"
2605 }
2606 EOF
2607 )
2608
2609 httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${SIGNL4_WEBHOOK_URL}")
2610 if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ] || [ "${httpcode}" = "202" ]; then
2611 info "sent signl4 event for ${notification_description}"
2612 else
2613 error "failed to send signl4 event for ${notification_description}, with HTTP response status code ${httpcode}."
2614 return 1
2615 fi
2616
2617 return 0
2618 }
2619
2620 # -----------------------------------------------------------------------------
2621 # prepare the content of the notification
2622
2623 # the url to send the user on click
2624 urlencode "${args_host}" >/dev/null
2625 url_host="${REPLY}"
2626 urlencode "${chart}" >/dev/null
2627 url_chart="${REPLY}"
2628 urlencode "${name}" >/dev/null
2629 url_name="${REPLY}"
2630 urlencode "${value_string}" >/dev/null
2631 url_value_string="${REPLY}"
2632
2633 redirect_params="host=${url_host}&chart=${url_chart}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}&alarm_when=${when}&alarm_status=${status}&alarm_chart=${chart}&alarm_value=${url_value_string}"
2634
2635 if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
2636 if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
2637 NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
2638 else
2639 error "failed to identify this agent via its NETDATA_REGISTRY_UNIQUE_ID."
2640 fi
2641 fi
2642
2643 goto_url="${NETDATA_REGISTRY_URL}/registry-alert-redirect.html?agent_machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&host_machine_guid=${child_machine_guid}&transition_id=${transition_id}&${redirect_params}"
2644
2645 # the severity of the alarm
2646 severity="${status}"
2647
2648 # the time the alarm was raised
2649 duration4human ${duration} >/dev/null
2650 duration_txt="${REPLY}"
2651 duration4human ${non_clear_duration} >/dev/null
2652 non_clear_duration_txt="${REPLY}"
2653 raised_for="(was ${old_status,,} for ${duration_txt})"
2654
2655 # the key status message
2656 status_message="status unknown"
2657
2658 # the color of the alarm
2659 color="grey"
2660
2661 # the alarm value
2662 alarm="${summary//_/ } = ${value_string}"
2663
2664 # the image of the alarm
2665 image="${images_base_url}/images/banner-icon-144x144.png"
2666
2667 # have a default email status, in case the following case does not catch it
2668 status_email_subject="${status}"
2669
2670 # prepare the title based on status
2671 case "${status}" in
2672 CRITICAL)
2673 image="${images_base_url}/images/alert-128-red.png"
2674 alarm_badge="https://app.netdata.cloud/static/email/img/label_critical.png"
2675 status_message="is critical"
2676 status_email_subject="Critical"
2677 color="#ca414b"
2678 rich_status_raised_for="Raised to critical, for ${non_clear_duration_txt}"
2679 background_color="#FFEBEF"
2680 border_color="#FF4136"
2681 text_color="#FF4136"
2682 action_text_color="#FFFFFF"
2683 ;;
2684
2685 WARNING)
2686 image="${images_base_url}/images/alert-128-orange.png"
2687 alarm_badge="https://app.netdata.cloud/static/email/img/label_warning.png"
2688 status_message="needs attention"
2689 status_email_subject="Warning"
2690 color="#ffc107"
2691 rich_status_raised_for="Raised to warning, for ${non_clear_duration_txt}"
2692 background_color="#FFF8E1"
2693 border_color="#FFC300"
2694 text_color="#536775"
2695 action_text_color="#35414A"
2696 ;;
2697
2698 CLEAR)
2699 image="${images_base_url}/images/check-mark-2-128-green.png"
2700 alarm_badge="https://app.netdata.cloud/static/email/img/label_recovered.png"
2701 status_message="recovered"
2702 status_email_subject="Clear"
2703 color="#77ca6d"
2704 rich_status_raised_for=
2705 background_color="#E5F5E8"
2706 border_color="#68C47D"
2707 text_color="#00AB44"
2708 action_text_color="#FFFFFF"
2709 ;;
2710 esac
2711
2712 # the html email subject
2713 html_email_subject="${status_email_subject}, ${summary} = ${value_string}, on ${host}"
2714
2715 if [ "${status}" = "CLEAR" ]; then
2716 severity="Recovered from ${old_status}"
2717 if [ ${non_clear_duration} -gt ${duration} ]; then
2718 raised_for="(alarm was raised for ${non_clear_duration_txt})"
2719 fi
2720 rich_status_raised_for="Recovered from ${old_status,,}, ${raised_for}"
2721
2722 # don't show the value when the status is CLEAR
2723 # for certain alarms, this value might not have any meaning
2724 alarm="${summary//_/ } ${raised_for}"
2725 html_email_subject="${status_email_subject}, ${summary} ${raised_for}, on ${host}"
2726
2727 elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
2728 severity="Escalated to ${status}"
2729 if [ ${non_clear_duration} -gt ${duration} ]; then
2730 raised_for="(alarm is raised for ${non_clear_duration_txt})"
2731 fi
2732 rich_status_raised_for="Escalated to critical, ${raised_for}"
2733
2734 elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
2735 severity="Demoted to ${status}"
2736 if [ ${non_clear_duration} -gt ${duration} ]; then
2737 raised_for="(alarm is raised for ${non_clear_duration_txt})"
2738 fi
2739 rich_status_raised_for="Demoted to warning, ${raised_for}"
2740
2741 else
2742 raised_for=
2743 fi
2744
2745 # prepare HTML versions of elements
2746 info_html=
2747 [ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
2748
2749 raised_for_html=
2750 [ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
2751
2752 # -----------------------------------------------------------------------------
2753 # send the slack notification
2754
2755 # slack aggregates posts from the same username
2756 # so we use "${host} ${status}" as the bot username, to make them diff
2757
2758 send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
2759 SENT_SLACK=$?
2760
2761 # -----------------------------------------------------------------------------
2762 # send the Microsoft Teams notification
2763
2764 # Microsoft teams aggregates posts from the same username
2765 # so we use "${host} ${status}" as the bot username, to make them diff
2766
2767 send_msteams "${MSTEAMS_WEBHOOK_URL}" "${to_msteams}"
2768 SENT_MSTEAMS=$?
2769
2770 # -----------------------------------------------------------------------------
2771 # send the rocketchat notification
2772
2773 # rocketchat aggregates posts from the same username
2774 # so we use "${host} ${status}" as the bot username, to make them diff
2775
2776 send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
2777 SENT_ROCKETCHAT=$?
2778
2779 # -----------------------------------------------------------------------------
2780 # send the alerta notification
2781
2782 # alerta aggregates posts from the same username
2783 # so we use "${host} ${status}" as the bot username, to make them diff
2784
2785 send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
2786 SENT_ALERTA=$?
2787
2788 # -----------------------------------------------------------------------------
2789 # send the flock notification
2790
2791 # flock aggregates posts from the same username
2792 # so we use "${host} ${status}" as the bot username, to make them diff
2793
2794 send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
2795 SENT_FLOCK=$?
2796
2797 # -----------------------------------------------------------------------------
2798 # send the discord notification
2799
2800 # discord aggregates posts from the same username
2801 # so we use "${host} ${status}" as the bot username, to make them diff
2802
2803 send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
2804 SENT_DISCORD=$?
2805
2806 # -----------------------------------------------------------------------------
2807 # send the pushover notification
2808
2809 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
2810 <font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
2811 <small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
2812 <small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
2813 <small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
2814 <a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
2815 <small><small>The source of this alarm is line ${src}</small></small>
2816 "
2817
2818 SENT_PUSHOVER=$?
2819
2820 # -----------------------------------------------------------------------------
2821 # send the pushbullet notification
2822
2823 send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
2824 Severity: ${severity}\\n
2825 Chart: ${chart}\\n
2826 ${date}\\n
2827 The source of this alarm is line ${src}"
2828
2829 SENT_PUSHBULLET=$?
2830
2831 # -----------------------------------------------------------------------------
2832 # send the twilio SMS
2833
2834 send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
2835 Severity: ${severity}
2836 Chart: ${chart}
2837 ${info}"
2838
2839 SENT_TWILIO=$?
2840
2841 # -----------------------------------------------------------------------------
2842 # send the messagebird SMS
2843
2844 send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
2845 Severity: ${severity}
2846 Chart: ${chart}
2847 ${info}"
2848
2849 SENT_MESSAGEBIRD=$?
2850
2851 # -----------------------------------------------------------------------------
2852 # send the SMSEAGLE SMS
2853
2854 send_smseagle "${SMSEAGLE_API_URL}" "${SMSEAGLE_API_ACCESSTOKEN}" "${SMSEAGLE_MSG_TYPE}" "${to_smseagle}" "${SMSEAGLE_CALL_DURATION}" "${SMSEAGLE_VOICE_ID}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
2855 Severity: ${severity}
2856 Chart: ${chart}
2857 ${info}"
2858
2859 SENT_SMSEAGLE=$?
2860
2861 # -----------------------------------------------------------------------------
2862 # send the kavenegar SMS
2863
2864 send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
2865 Severity: ${severity}
2866 Chart: ${chart}
2867 ${info}"
2868
2869 SENT_KAVENEGAR=$?
2870
2871 # -----------------------------------------------------------------------------
2872 # send the telegram.org message
2873
2874 # https://core.telegram.org/bots/api#formatting-options
2875 send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
2876 ${chart}
2877 <a href=\"${goto_url}\">${alarm}</a>
2878 <i>${info}</i>"
2879
2880 SENT_TELEGRAM=$?
2881
2882 # -----------------------------------------------------------------------------
2883 # send the kafka message
2884
2885 send_kafka
2886 SENT_KAFKA=$?
2887
2888 # -----------------------------------------------------------------------------
2889 # send the pagerduty.com message
2890
2891 send_pd "${to_pd}"
2892 SENT_PD=$?
2893
2894 # -----------------------------------------------------------------------------
2895 # send the fleep message
2896
2897 send_fleep "${to_fleep}"
2898 SENT_FLEEP=$?
2899
2900 # -----------------------------------------------------------------------------
2901 # send the Prowl message
2902
2903 send_prowl "${to_prowl}"
2904 SENT_PROWL=$?
2905
2906 # -----------------------------------------------------------------------------
2907 # send the irc message
2908
2909 send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${IRC_PORT}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
2910 Severity: ${severity}
2911 Chart: ${chart}
2912 ${info}"
2913
2914 SENT_IRC=$?
2915
2916 # -----------------------------------------------------------------------------
2917 # send the SMS message with smstools3
2918
2919 send_sms "${to_sms}"
2920
2921 SENT_SMS=$?
2922
2923 # -----------------------------------------------------------------------------
2924 # send the custom message
2925
2926 send_custom() {
2927 # is it enabled?
2928 [ "${SEND_CUSTOM}" != "YES" ] && return 1
2929
2930 # do we have any sender?
2931 [ -z "${1}" ] && return 1
2932
2933 # call the custom_sender function
2934 custom_sender "${@}"
2935 }
2936
2937 send_custom "${to_custom}"
2938 SENT_CUSTOM=$?
2939
2940 # -----------------------------------------------------------------------------
2941 # send hipchat message
2942
2943 send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
2944 ${host} ${status_message}<br/> \
2945 <b>${alarm}</b> ${info_html}<br/> \
2946 <b>${chart}</b><br/> \
2947 <b>${date}${raised_for_html}</b><br/> \
2948 <a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
2949 (source of alarm ${src}) \
2950 "
2951
2952 SENT_HIPCHAT=$?
2953
2954 # -----------------------------------------------------------------------------
2955 # send the Amazon SNS message
2956
2957 send_awssns "${to_awssns}"
2958
2959 SENT_AWSSNS=$?
2960
2961 # -----------------------------------------------------------------------------
2962 # send the Matrix message
2963 send_matrix "${MATRIX_HOMESERVER}" "${to_matrix}"
2964
2965 SENT_MATRIX=$?
2966
2967
2968 # -----------------------------------------------------------------------------
2969 # send the syslog message
2970
2971 send_syslog "${to_syslog}"
2972
2973 SENT_SYSLOG=$?
2974
2975 # -----------------------------------------------------------------------------
2976 # send the email
2977
2978 IFS='' read -r -d '' email_plaintext_part <<EOF
2979 Content-Type: text/plain; encoding=${EMAIL_CHARSET}
2980 Content-Disposition: inline
2981 Content-Transfer-Encoding: 8bit
2982
2983 ${host} ${status_message}
2984
2985 ${alarm} ${info}
2986 ${raised_for}
2987
2988 Chart : ${chart}
2989 Severity: ${severity}
2990 URL : ${goto_url}
2991 Source : ${src}
2992 Date : ${date}
2993 Notification generated on ${host}
2994
2995 Evaluated Expression : ${calc_expression}
2996 Expression Variables : ${calc_param_values}
2997
2998 The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
2999 EOF
3000
3001 if [[ "${EMAIL_PLAINTEXT_ONLY}" == "YES" ]]; then
3002
3003 send_email <<EOF
3004 To: ${to_email}
3005 Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
3006 MIME-Version: 1.0
3007 Content-Type: multipart/alternative; boundary="multipart-boundary"
3008 ${email_thread_headers}
3009 X-Netdata-Severity: ${status,,}
3010 X-Netdata-Alert-Name: $name
3011 X-Netdata-Chart: $chart
3012 X-Netdata-Classification: $classification
3013 X-Netdata-Host: $host
3014 X-Netdata-Role: $roles
3015
3016 This is a MIME-encoded multipart message
3017
3018 --multipart-boundary
3019 ${email_plaintext_part}
3020 --multipart-boundary--
3021 EOF
3022
3023 else
3024
3025 now=$(date "+%s")
3026
3027 if [ -n "$total_warn_alarms" ]; then
3028 while read -d, -r pair; do
3029 IFS='=' read -r key val <<<"$pair"
3030
3031 date_w=$(date --date=@${val} "${date_format}" 2>/dev/null)
3032 [ -z "${date_w}" ] && date_w=$(date "${date_format}" 2>/dev/null)
3033 [ -z "${date_w}" ] && date_w=$(date --date=@${val} 2>/dev/null)
3034 [ -z "${date_w}" ] && date_w=$(date 2>/dev/null)
3035
3036 elapsed=$((now - val))
3037
3038 duration4human ${elapsed} >/dev/null
3039 elapsed_txt="${REPLY}"
3040
3041 WARN_ALARMS+="
3042 <div class=\"set-font\" style=\"font-family: 'IBM Plex Sans', sans-serif; background: #FFFFFF; background-color: #FFFFFF; margin: 0px auto; max-width: 600px;\">
3043 <table align=\"center\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"background:#FFFFFF;background-color:#FFFFFF;width:100%;\">
3044 <tbody>
3045 <tr>
3046 <td style=\"border-top:8px solid #F7F8F8;direction:ltr;font-size:0px;padding:20px 0;text-align:center;\">
3047 <!--[if mso | IE]><table role=\"presentation\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\"><tr><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
3048 <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
3049 <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"vertical-align:top;\" width=\"100%\">
3050 <tbody>
3051 <tr>
3052 <td align=\"left\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
3053 <div style=\"font-family:Open Sans, sans-serif;font-size:14px;font-weight:600;line-height:1;text-align:left;color:#35414A;\">${key}</div>
3054 </td>
3055 </tr>
3056 <tr>
3057 <td align=\"left\" style=\"font-size:0px;padding:10px 25px;padding-top:2px;word-break:break-word;\">
3058 <div style=\"font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:left;color:#35414A;\">${date_w}</div>
3059 </td>
3060 </tr>
3061 </tbody>
3062 </table>
3063 </div>
3064 <!--[if mso | IE]></td><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
3065 <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
3066 <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" width=\"100%\">
3067 <tbody>
3068 <tr>
3069 <td style=\"vertical-align:top;padding-top:13px;\">
3070 <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style width=\"100%\">
3071 <tbody>
3072 <tr>
3073 <td align=\"right\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
3074 <div style=\"font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:right;color:#555555;\"><span style=\"background-color:#FFF8E1; border: 1px solid #FFC300; border-radius:36px; padding: 2px 12px; margin-top: 20px; white-space: nowrap\">
3075 Warning for ${elapsed_txt}
3076 </span></div>
3077 </td>
3078 </tr>
3079 </tbody>
3080 </table>
3081 </td>
3082 </tr>
3083 </tbody>
3084 </table>
3085 </div>
3086 <!--[if mso | IE]></td></tr></table><![endif]-->
3087 </td>
3088 </tr>
3089 </tbody>
3090 </table>
3091 </div>
3092 "
3093
3094 done <<<"$total_warn_alarms,"
3095 fi
3096
3097 if [ -n "$total_crit_alarms" ]; then
3098 while read -d, -r pair; do
3099 IFS='=' read -r key val <<<"$pair"
3100
3101 date_c=$(date --date=@${val} "${date_format}" 2>/dev/null)
3102 [ -z "${date_c}" ] && date_c=$(date "${date_format}" 2>/dev/null)
3103 [ -z "${date_c}" ] && date_c=$(date --date=@${val} 2>/dev/null)
3104 [ -z "${date_c}" ] && date_c=$(date 2>/dev/null)
3105
3106 elapsed=$((now - val))
3107
3108 duration4human ${elapsed} >/dev/null
3109 elapsed_txt="${REPLY}"
3110
3111 CRIT_ALARMS+="
3112 <div class=\"set-font\" style=\"font-family: 'IBM Plex Sans', sans-serif; background: #FFFFFF; background-color: #FFFFFF; margin: 0px auto; max-width: 600px;\">
3113 <table align=\"center\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"background:#FFFFFF;background-color:#FFFFFF;width:100%;\">
3114 <tbody>
3115 <tr>
3116 <td style=\"border-top:8px solid #F7F8F8;direction:ltr;font-size:0px;padding:20px 0;text-align:center;\">
3117 <!--[if mso | IE]><table role=\"presentation\" border=\"0\" cellpadding=\"0\" cellspacing=\"0\"><tr><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
3118 <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
3119 <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style=\"vertical-align:top;\" width=\"100%\">
3120 <tbody>
3121 <tr>
3122 <td align=\"left\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
3123 <div style=\"font-family:Open Sans, sans-serif;font-size:14px;font-weight:600;line-height:1;text-align:left;color:#35414A;\">${key}</div>
3124 </td>
3125 </tr>
3126 <tr>
3127 <td align=\"left\" style=\"font-size:0px;padding:10px 25px;padding-top:2px;word-break:break-word;\">
3128 <div style=\"font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:left;color:#35414A;\">${date_c}</div>
3129 </td>
3130 </tr>
3131 </tbody>
3132 </table>
3133 </div>
3134 <!--[if mso | IE]></td><td class=\"\" style=\"vertical-align:top;width:300px;\" ><![endif]-->
3135 <div class=\"mj-column-per-50 mj-outlook-group-fix\" style=\"font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;\">
3136 <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" width=\"100%\">
3137 <tbody>
3138 <tr>
3139 <td style=\"vertical-align:top;padding-top:13px;\">
3140 <table border=\"0\" cellpadding=\"0\" cellspacing=\"0\" role=\"presentation\" style width=\"100%\">
3141 <tbody>
3142 <tr>
3143 <td align=\"right\" style=\"font-size:0px;padding:10px 25px;word-break:break-word;\">
3144 <div style=\"font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:right;color:#35414A;\"><span style=\"background-color:#FFEBEF; border: 1px solid #FF4136; border-radius:36px; padding: 2px 12px; margin-top: 20px; white-space: nowrap\">
3145 Critical for ${elapsed_txt}
3146 </span></div>
3147 </td>
3148 </tr>
3149 </tbody>
3150 </table>
3151 </td>
3152 </tr>
3153 </tbody>
3154 </table>
3155 </div>
3156 <!--[if mso | IE]></td></tr></table><![endif]-->
3157 </td>
3158 </tr>
3159 </tbody>
3160 </table>
3161 </div>
3162 "
3163
3164 done <<<"$total_crit_alarms,"
3165 fi
3166
3167 if (( total_warnings + total_critical > 15 )); then
3168 EXTRA_ALARMS_LIST_TEXT="(Showing latest 15 alerts)"
3169 fi
3170
3171 if [ -n "$edit_command_line" ]; then
3172 IFS='=' read -r edit_command line s_host <<<"$edit_command_line"
3173 fi
3174
3175 IFS='' read -r -d '' email_html_part <<EOF
3176 Content-Type: text/html; encoding=${EMAIL_CHARSET}
3177 Content-Disposition: inline
3178 Content-Transfer-Encoding: 8bit
3179
3180 <!doctype html>
3181 <html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office">
3182 <head>
3183 <title>
3184 </title>
3185 <!--[if !mso]><!-->
3186 <meta http-equiv="X-UA-Compatible" content="IE=edge">
3187 <!--<![endif]-->
3188 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
3189 <meta name="viewport" content="width=device-width, initial-scale=1">
3190 <style type="text/css">
3191 #outlook a { padding:0; }
3192 body { margin:0;padding:0;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%; }
3193 table, td { border-collapse:collapse;mso-table-lspace:0pt;mso-table-rspace:0pt; }
3194 img { border:0;height:auto;line-height:100%; outline:none;text-decoration:none;-ms-interpolation-mode:bicubic; }
3195 p { display:block;margin:13px 0; }
3196 </style>
3197 <!--[if mso]>
3198 <xml>
3199 <o:OfficeDocumentSettings>
3200 <o:AllowPNG/>
3201 <o:PixelsPerInch>96</o:PixelsPerInch>
3202 </o:OfficeDocumentSettings>
3203 </xml>
3204 <![endif]-->
3205 <!--[if lte mso 11]>
3206 <style type="text/css">
3207 .mj-outlook-group-fix { width:100% !important; }
3208 </style>
3209 <![endif]-->
3210 <!--[if !mso]><!-->
3211 <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;500;600;700&display=swap" rel="stylesheet" type="text/css">
3212 <link href="https://fonts.googleapis.com/css?family=Ubuntu:300,400,500,700" rel="stylesheet" type="text/css">
3213 <style type="text/css">
3214 @import url(https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;500;600;700&display=swap);
3215 @import url(https://fonts.googleapis.com/css?family=Ubuntu:300,400,500,700);
3216 </style>
3217 <!--<![endif]-->
3218 <style type="text/css">
3219 @media only screen and (min-width:100px) {
3220 .mj-column-px-130 { width:130px !important; max-width: 130px; }
3221 .mj-column-per-50 { width:50% !important; max-width: 50%; }
3222 .mj-column-per-70 { width:70% !important; max-width: 70%; }
3223 .mj-column-per-30 { width:30% !important; max-width: 30%; }
3224 .mj-column-per-100 { width:100% !important; max-width: 100%; }
3225 .mj-column-px-66 { width:66px !important; max-width: 66px; }
3226 .mj-column-px-400 { width:400px !important; max-width: 400px; }
3227 }
3228 </style>
3229 <style type="text/css">
3230 @media only screen and (max-width:100px) {
3231 table.mj-full-width-mobile { width: 100% !important; }
3232 td.mj-full-width-mobile { width: auto !important; }
3233 }
3234 </style>
3235 </head>
3236 <body style="word-spacing:normal;">
3237 <div class="svgbg" style="background-image: url('https://staging.netdata.cloud/static/email/img/isotype_600.png'); background-repeat: no-repeat; background-position: top center; background-size: 600px 192px;">
3238 <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3239 <div style="margin:0px auto;max-width:600px;">
3240 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3241 <tbody>
3242 <tr>
3243 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:50px;padding-left:0;text-align:left;">
3244 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:130px;" ><![endif]-->
3245 <div class="mj-column-px-130 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:130px;">
3246 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3247 <tbody>
3248 <tr>
3249 <td align="center" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
3250 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
3251 <tbody>
3252 <tr>
3253 <td style="width:130px;">
3254 <img alt="Netdata Logo" height="auto" src="https://app.netdata.cloud/static/email/img/full_logo.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="130">
3255 </td>
3256 </tr>
3257 </tbody>
3258 </table>
3259 </td>
3260 </tr>
3261 </tbody>
3262 </table>
3263 </div>
3264 <!--[if mso | IE]></td><td class="" style="vertical-align:top;width:300px;" ><![endif]-->
3265 <div class="mj-column-per-50 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:50%;">
3266 <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
3267 <tbody>
3268 <tr>
3269 <td style="vertical-align:top;padding-top:4px;">
3270 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
3271 <tbody>
3272 <tr>
3273 <td align="left" style="font-size:0px;padding:10px 25px;padding-left:10px;word-break:break-word;">
3274 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Notification</div>
3275 </td>
3276 </tr>
3277 </tbody>
3278 </table>
3279 </td>
3280 </tr>
3281 </tbody>
3282 </table>
3283 </div>
3284 <!--[if mso | IE]></td></tr></table><![endif]-->
3285 </td>
3286 </tr>
3287 </tbody>
3288 </table>
3289 </div>
3290 <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="no-collapse-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3291 <div class="no-collapse" style="border-collapse: initial; margin: 0px auto; border-radius: 4px; max-width: 600px;">
3292 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;border-radius:4px;">
3293 <tbody>
3294 <tr>
3295 <td style="border:1px solid ${border_color};direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
3296 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3297 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
3298 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3299 <tbody>
3300 <tr>
3301 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
3302 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:418.6px;" ><![endif]-->
3303 <div class="mj-column-per-70 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:70%;">
3304 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3305 <tbody>
3306 <tr>
3307 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:15px;word-break:break-word;">
3308 <div style="font-family:Open Sans, sans-serif;font-size:20px;font-weight:700;line-height:1;text-align:left;color:#35414A;">${summary}</div>
3309 </td>
3310 </tr>
3311 </tbody>
3312 </table>
3313 </div>
3314 <!--[if mso | IE]></td><td class="" style="vertical-align:top;width:179.4px;" ><![endif]-->
3315 <div class="mj-column-per-30 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:30%;">
3316 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3317 <tbody>
3318 <tr>
3319 <td align="right" style="font-size:0px;padding:10px 25px;padding-right:25px;word-break:break-word;">
3320 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
3321 <tbody>
3322 <tr>
3323 <td style="width:100px;">
3324 <img height="auto" src="${alarm_badge}" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="100"/>
3325 </td>
3326 </tr>
3327 </tbody>
3328 </table>
3329 </td>
3330 </tr>
3331 </tbody>
3332 </table>
3333 </div>
3334 <!--[if mso | IE]></td></tr></table><![endif]-->
3335 </td>
3336 </tr>
3337 </tbody>
3338 </table>
3339 </div>
3340 <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3341 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
3342 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3343 <tbody>
3344 <tr>
3345 <td style="direction:ltr;font-size:0px;padding:0;text-align:center;">
3346 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
3347 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3348 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3349 <tbody>
3350 <tr>
3351 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3352 <div style="font-family:IBM Plex Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">on ${host}</div>
3353 </td>
3354 </tr>
3355 </tbody>
3356 </table>
3357 </div>
3358 <!--[if mso | IE]></td></tr></table><![endif]-->
3359 </td>
3360 </tr>
3361 </tbody>
3362 </table>
3363 </div>
3364 <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3365 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
3366 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3367 <tbody>
3368 <tr>
3369 <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
3370 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
3371 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3372 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3373 <tbody>
3374 <tr>
3375 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3376 <div style="font-family:Open Sans, sans-serif;font-size:26px;font-weight:700;line-height:1;text-align:left;color:#35414A;"><span style="color: ${text_color}; font-size:26px; background: ${background_color}; padding:4px 24px; border-radius: 36px">${value_string}
3377 </span></div>
3378 </td>
3379 </tr>
3380 </tbody>
3381 </table>
3382 </div>
3383 <!--[if mso | IE]></td></tr></table><![endif]-->
3384 </td>
3385 </tr>
3386 </tbody>
3387 </table>
3388 </div>
3389 <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3390 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
3391 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3392 <tbody>
3393 <tr>
3394 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
3395 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
3396 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3397 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3398 <tbody>
3399 <tr>
3400 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3401 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:21px;text-align:left;color:#35414A;">Details: ${info}</div>
3402 </td>
3403 </tr>
3404 </tbody>
3405 </table>
3406 </div>
3407 <!--[if mso | IE]></td></tr></table><![endif]-->
3408 </td>
3409 </tr>
3410 </tbody>
3411 </table>
3412 </div>
3413 <!--[if mso | IE]></td></tr></table></td></tr><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:598px;" width="598" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3414 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 598px;">
3415 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3416 <tbody>
3417 <tr>
3418 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:0;padding-top:0;text-align:center;">
3419 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:598px;" ><![endif]-->
3420 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3421 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3422 <tbody>
3423 <tr>
3424 <td align="center" vertical-align="middle" style="font-size:0px;padding:10px 25px;word-break:break-word;">
3425 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:separate;width:100%;line-height:100%;">
3426 <tr>
3427 <td
3428 align="center" bgcolor="${border_color}" role="presentation" style="border:none;border-radius:3px;cursor:auto;height:44px;background:${border_color};" valign="middle">
3429 <p style="display:block;background:${border_color};color:#ffffff;font-size:13px;font-weight:600;line-height:44px;margin:0;text-decoration:none;text-transform:none;mso-padding-alt:0px;border-radius:3px;">
3430 <a href="${goto_url}" style="color: ${action_text_color}; text-decoration: none; width: 100%; display: inline-block">GO TO CHART</a>
3431 </p>
3432 </td>
3433 </tr>
3434 </table>
3435 </td>
3436 </tr>
3437 </tbody>
3438 </table>
3439 </div>
3440 <!--[if mso | IE]></td></tr></table><![endif]-->
3441 </td>
3442 </tr>
3443 </tbody>
3444 </table>
3445 </div>
3446 <!--[if mso | IE]></td></tr></table></td></tr></table><![endif]-->
3447 </td>
3448 </tr>
3449 </tbody>
3450 </table>
3451 </div>
3452 <!--[if mso | IE]></td></tr></table><![endif]-->
3453 <div style="height:32px;line-height:32px;">&#8202;</div>
3454 <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3455 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; background: ${background_color}; background-color: ${background_color}; margin: 0px auto; border-radius: 4px; max-width: 600px;">
3456 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="background:${background_color};background-color:${background_color};width:100%;border-radius:4px;">
3457 <tbody>
3458 <tr>
3459 <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
3460 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
3461 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3462 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3463 <tbody>
3464 <tr>
3465 <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
3466 <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Alert:
3467 <span style="font-weight:700; font-size:20px">${name}</span></div>
3468 </td>
3469 </tr>
3470 <tr>
3471 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3472 <div style="font-family:Open Sans, sans-serif;font-size:18px;line-height:1;text-align:left;color:#35414A;">Chart:
3473 <span style="font-weight:700; font-size:20px">${chart}</span></div>
3474 </td>
3475 </tr>
3476 <tr>
3477 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:4px;word-break:break-word;">
3478 <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1;text-align:left;color:#35414A;">${rich_status_raised_for}</div>
3479 </td>
3480 </tr>
3481 <tr>
3482 <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
3483 <p style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:100%;">
3484 </p>
3485 <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:550px;" role="presentation" width="550px" ><tr><td style="height:0;line-height:0;"> &nbsp;
3486 </td></tr></table><![endif]-->
3487 </td>
3488 </tr>
3489 <tr>
3490 <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
3491 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">On
3492 <span style="font-weight:600">${date}</span></div>
3493 </td>
3494 </tr>
3495 <tr>
3496 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3497 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">By:
3498 <span style="font-weight:600">${host}</span></div>
3499 </td>
3500 </tr>
3501 <tr>
3502 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:4px;word-break:break-word;">
3503 <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1;text-align:left;color:#35414A;">Global time:
3504 <span style="font-weight:600">${date_utc}</span></div>
3505 </td>
3506 </tr>
3507 <tr>
3508 <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
3509 <p style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:100%;">
3510 </p>
3511 <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" style="border-top:solid 1px lightgrey;font-size:1px;margin:0px auto;width:550px;" role="presentation" width="550px" ><tr><td style="height:0;line-height:0;"> &nbsp;
3512 </td></tr></table><![endif]-->
3513 </td>
3514 </tr>
3515 <tr>
3516 <td align="left" style="font-size:0px;padding:10px 25px;padding-bottom:6px;word-break:break-word;">
3517 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Classification:
3518 <span style="font-weight:600">${classification}</span></div>
3519 </td>
3520 </tr>
3521 <tr>
3522 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;word-break:break-word;">
3523 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:left;color:#35414A;">Role:
3524 <span style="font-weight:600">${roles}</span></div>
3525 </td>
3526 </tr>
3527 </tbody>
3528 </table>
3529 </div>
3530 <!--[if mso | IE]></td></tr></table><![endif]-->
3531 </td>
3532 </tr>
3533 </tbody>
3534 </table>
3535 </div>
3536 <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3537 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
3538 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3539 <tbody>
3540 <tr>
3541 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-left:25px;text-align:left;">
3542 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:66px;" ><![endif]-->
3543 <div class="mj-column-px-66 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:66px;">
3544 <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
3545 <tbody>
3546 <tr>
3547 <td style="vertical-align:top;padding:0;">
3548 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
3549 <tbody>
3550 <tr>
3551 <td align="left" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
3552 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
3553 <tbody>
3554 <tr>
3555 <td style="width:48px;">
3556 <img height="auto" src="https://app.netdata.cloud/static/email/img/community_icon.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="48">
3557 </td>
3558 </tr>
3559 </tbody>
3560 </table>
3561 </td>
3562 </tr>
3563 </tbody>
3564 </table>
3565 </td>
3566 </tr>
3567 </tbody>
3568 </table>
3569 </div>
3570 <!--[if mso | IE]></td><td align="left" class="" style="vertical-align:top;width:400px;" ><![endif]-->
3571 <div class="mj-column-px-400 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:400px;">
3572 <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
3573 <tbody>
3574 <tr>
3575 <td style="vertical-align:top;padding-left:0;">
3576 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
3577 <tbody>
3578 <tr>
3579 <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
3580 <div style="font-family:Open Sans, sans-serif;font-size:16px;font-weight:700;line-height:1;text-align:left;color:#35414A;">Want to know more about this alert?</div>
3581 </td>
3582 </tr>
3583 <tr>
3584 <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
3585 <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1.3;text-align:left;color:#35414A;">Join the troubleshooting discussion for this alert on our <a href="https://community.netdata.cloud/t/${name//[._]/-}" class="link" style="color: #00AB44; text-decoration: none;">community forums</a>.</div>
3586 </td>
3587 </tr>
3588 </tbody>
3589 </table>
3590 </td>
3591 </tr>
3592 </tbody>
3593 </table>
3594 </div>
3595 <!--[if mso | IE]></td></tr></table><![endif]-->
3596 </td>
3597 </tr>
3598 </tbody>
3599 </table>
3600 </div>
3601 <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3602 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
3603 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3604 <tbody>
3605 <tr>
3606 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-left:25px;text-align:left;">
3607 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:66px;" ><![endif]-->
3608 <div class="mj-column-px-66 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:66px;">
3609 <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
3610 <tbody>
3611 <tr>
3612 <td style="vertical-align:top;padding:0;">
3613 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
3614 <tbody>
3615 <tr>
3616 <td align="left" style="font-size:0px;padding:10px 25px;padding-right:0;padding-left:0;word-break:break-word;">
3617 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="border-collapse:collapse;border-spacing:0px;">
3618 <tbody>
3619 <tr>
3620 <td style="width:48px;">
3621 <img height="auto" src="https://app.netdata.cloud/static/email/img/configure_icon.png" style="border:0;display:block;outline:none;text-decoration:none;height:auto;width:100%;font-size:13px;" width="48">
3622 </td>
3623 </tr>
3624 </tbody>
3625 </table>
3626 </td>
3627 </tr>
3628 </tbody>
3629 </table>
3630 </td>
3631 </tr>
3632 </tbody>
3633 </table>
3634 </div>
3635 <!--[if mso | IE]></td><td align="left" class="" style="vertical-align:top;width:400px;" ><![endif]-->
3636 <div class="mj-column-px-400 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:400px;">
3637 <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
3638 <tbody>
3639 <tr>
3640 <td style="vertical-align:top;padding-left:0;">
3641 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
3642 <tbody>
3643 <tr>
3644 <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
3645 <div style="font-family:Open Sans, sans-serif;font-size:16px;font-weight:700;line-height:1;text-align:left;color:#35414A;">Need to configure this alert?</div>
3646 </td>
3647 </tr>
3648 <tr>
3649 <td align="left" style="font-size:0px;padding:10px 25px;padding-left:0;word-break:break-word;">
3650 <div style="font-family:Open Sans, sans-serif;font-size:14px;line-height:1.3;text-align:left;color:#35414A;"><span style="color: #00AB44"><a href="https://learn.netdata.cloud/docs/agent/health/notifications#:~:text=To%20edit%20it%20on%20your,have%20one%20or%20more%20destinations" class="link" style="color: #00AB44; text-decoration: none;">Edit</a></span> this alert's configuration file by logging into $s_host and running the following command:</div>
3651 </td>
3652 </tr>
3653 <tr>
3654 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:8px;padding-left:0;word-break:break-word;">
3655 <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1.3;text-align:left;color:#35414A;">${edit_command} <br>
3656 <br>The alarm to edit is at line ${line}</div>
3657 </td>
3658 </tr>
3659 </tbody>
3660 </table>
3661 </td>
3662 </tr>
3663 </tbody>
3664 </table>
3665 </div>
3666 <!--[if mso | IE]></td></tr></table><![endif]-->
3667 </td>
3668 </tr>
3669 </tbody>
3670 </table>
3671 </div>
3672 <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="history-wrapper-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3673 <div class="history-wrapper" style="background: #F7F8F8; background-color: #F7F8F8; margin: 0px auto; max-width: 100%;">
3674 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="background:#F7F8F8;background-color:#F7F8F8;width:100%;">
3675 <tbody>
3676 <tr>
3677 <td style="direction:ltr;font-size:0px;padding:0;padding-bottom:24px;text-align:center;">
3678 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="set-font-outlook" width="600px" ><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3679 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
3680 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3681 <tbody>
3682 <tr>
3683 <td style="direction:ltr;font-size:0px;padding:20px 0;padding-bottom:12px;text-align:center;">
3684 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
3685 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3686 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
3687 <tbody>
3688 <tr>
3689 <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
3690 <div style="font-family:Open Sans, sans-serif;font-size:16px;line-height:1;text-align:center;color:#35414A;">The node has
3691 <span style="font-weight:600">${total_warnings} warning</span>
3692 and
3693 <span style="font-weight:600">${total_critical} critical</span>
3694 additional active alert(s)</div>
3695 </td>
3696 </tr>
3697 <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
3698 <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:center;color:#35414A;">${EXTRA_ALARMS_LIST_TEXT}</div>
3699 </td>
3700 </tr>
3701 </tbody>
3702 </table>
3703 </div>
3704 <!--[if mso | IE]></td></tr></table><![endif]-->
3705 </td>
3706 </tr>
3707 </tbody>
3708 </table>
3709 </div>
3710 ${CRIT_ALARMS}
3711 ${WARN_ALARMS}
3712 <!--[if mso | IE]></td></tr></table></td></tr></table><![endif]-->
3713 </td>
3714 </tr>
3715 </tbody>
3716 </table>
3717 </div>
3718 <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="set-font-outlook" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]-->
3719 <div class="set-font" style="font-family: 'IBM Plex Sans', sans-serif; margin: 0px auto; max-width: 600px;">
3720 <table align="center" border="0" cellpadding="0" cellspacing="0" role="presentation" style="width:100%;">
3721 <tbody>
3722 <tr>
3723 <td style="direction:ltr;font-size:0px;padding:20px 0;text-align:center;">
3724 <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]-->
3725 <div class="mj-column-per-100 mj-outlook-group-fix" style="font-size:0px;text-align:left;direction:ltr;display:inline-block;vertical-align:top;width:100%;">
3726 <table border="0" cellpadding="0" cellspacing="0" role="presentation" width="100%">
3727 <tbody>
3728 <tr>
3729 <td style="vertical-align:top;padding-top:44px;padding-bottom:12px;">
3730 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style width="100%">
3731 <tbody>
3732 <tr>
3733 <td align="left" style="font-size:0px;padding:10px 25px;padding-top:0;padding-bottom:0;word-break:break-word;">
3734 <div style="font-family:Open Sans, sans-serif;font-size:13px;line-height:1;text-align:center;color:#35414A;">© Netdata $(date +'%Y') - X-Ray Vision for your infrastructure</div>
3735 </td>
3736 </tr>
3737 </tbody>
3738 </table>
3739 </td>
3740 </tr>
3741 </tbody>
3742 </table>
3743 </div>
3744 <!--[if mso | IE]></td></tr></table><![endif]-->
3745 </td>
3746 </tr>
3747 </tbody>
3748 </table>
3749 </div>
3750 <!--[if mso | IE]></td></tr></table><![endif]-->
3751 </div>
3752 </body>
3753 </html>
3754 EOF
3755
3756 send_email <<EOF
3757 To: ${to_email}
3758 Subject: ${html_email_subject}
3759 MIME-Version: 1.0
3760 Content-Type: multipart/alternative; boundary="multipart-boundary"
3761 ${email_thread_headers}
3762 X-Netdata-Severity: ${status,,}
3763 X-Netdata-Alert-Name: $name
3764 X-Netdata-Chart: $chart
3765 X-Netdata-Classification: $classification
3766 X-Netdata-Host: $host
3767 X-Netdata-Role: $roles
3768
3769 This is a MIME-encoded multipart message
3770
3771 --multipart-boundary
3772 ${email_plaintext_part}
3773 --multipart-boundary
3774 ${email_html_part}
3775 --multipart-boundary--
3776 EOF
3777
3778 fi
3779
3780 SENT_EMAIL=$?
3781
3782 # -----------------------------------------------------------------------------
3783 # send the EVENT to Dynatrace
3784 send_dynatrace "${host}" "${chart}" "${name}" "${status}"
3785 SENT_DYNATRACE=$?
3786
3787 # -----------------------------------------------------------------------------
3788 # send messages to Opsgenie
3789 send_opsgenie
3790 SENT_OPSGENIE=$?
3791
3792 # -----------------------------------------------------------------------------
3793 # send messages to Gotify
3794 send_gotify
3795 SENT_GOTIFY=$?
3796
3797 # -----------------------------------------------------------------------------
3798 # send messages to ntfy
3799 send_ntfy "${DEFAULT_RECIPIENT_NTFY}"
3800 SENT_NTFY=$?
3801
3802 # -----------------------------------------------------------------------------
3803 # send messages to ilert
3804 send_ilert
3805 SENT_ILERT=$?
3806
3807 # -----------------------------------------------------------------------------
3808 # send messages to signl4
3809 send_signl4
3810 SENT_SIGNL4=$?
3811
3812 # -----------------------------------------------------------------------------
3813 # let netdata know
3814 for state in "${SENT_EMAIL}" \
3815 "${SENT_PUSHOVER}" \
3816 "${SENT_TELEGRAM}" \
3817 "${SENT_SLACK}" \
3818 "${SENT_ROCKETCHAT}" \
3819 "${SENT_ALERTA}" \
3820 "${SENT_FLOCK}" \
3821 "${SENT_DISCORD}" \
3822 "${SENT_TWILIO}" \
3823 "${SENT_HIPCHAT}" \
3824 "${SENT_MESSAGEBIRD}" \
3825 "${SENT_SMSEAGLE}" \
3826 "${SENT_KAVENEGAR}" \
3827 "${SENT_PUSHBULLET}" \
3828 "${SENT_KAFKA}" \
3829 "${SENT_PD}" \
3830 "${SENT_FLEEP}" \
3831 "${SENT_PROWL}" \
3832 "${SENT_CUSTOM}" \
3833 "${SENT_IRC}" \
3834 "${SENT_AWSSNS}" \
3835 "${SENT_MATRIX}" \
3836 "${SENT_SYSLOG}" \
3837 "${SENT_SMS}" \
3838 "${SENT_MSTEAMS}" \
3839 "${SENT_DYNATRACE}" \
3840 "${SENT_OPSGENIE}" \
3841 "${SENT_GOTIFY}" \
3842 "${SENT_NTFY}" \
3843 "${SENT_ILERT}" \
3844 "${SENT_SIGNL4}"; do
3845 if [ "${state}" -eq 0 ]; then
3846 # we sent something
3847 exit 0
3848 fi
3849 done
3850 # we did not send anything
3851 exit 1