@cryptotaxi247 / netdata-1 / commits / 512266bf5

added slack notifications support

Costa Tsaousis (ktsaou) committed Sep 10, 2016 at 01:42 UTC 512266bf55bf13f076a73dba2ace264e9f81163a

2 files changed +223 -66

conf.d/health_alarm_notify.conf

+35 -5

@@ -11,10 +11,12 @@

12 # The full path to the sendmail command.

13 # If empty, the system $PATH will be searched for it.

14 +# If not found, email notifications will be disabled.

15 sendmail=""

17 # The full path of the curl command.

18 # If empty, the system $PATH will be searched for it.

19 +# If not found, pushover and slack notifications will be disabled.

20 curl=""

@@ -22,13 +24,13 @@ curl=""

24 # sending emails

26 # note: multiple recipients can be given like this:

25 -# "admin1@example.com, admin2@example.com, ..."

27 +# "admin1@example.com admin2@example.com ..."

28 # note it needs a comma!

30 # enable/disable sending emails

31 SEND_EMAIL="YES"

31 -# if a role recipient is not configured, an email will be send to

33 +# if a role recipient is not configured, an email will be send to:

34 DEFAULT_RECIPIENT_EMAIL="root"

@@ -36,21 +38,40 @@ DEFAULT_RECIPIENT_EMAIL="root"

38 # sending pushover notifications (pushover.net)

40 # note: multiple recipients can be given like this:

39 -# "USERTOKEN1 USERTOKEN2 USERTOKEN3 ..."

41 +# "USERTOKEN1 USERTOKEN2 ..."

43 # enable/disable sending pushover notifications

44 SEND_PUSHOVER="YES"

44 -# Login to pushover.com to get your pushover app token

46 +# Login to pushover.net to get your pushover app token

47 # You need only one for all your netdata servers.

48 # Without it, netdata cannot send pushover notifications.

49 PUSHOVER_APP_TOKEN=""

51 # if a role recipient is not configured, a notification will be send to

50 -# this pushover user token

52 +# this pushover user token:

53 DEFAULT_RECIPIENT_PUSHOVER=""

56 +###############################################################################

57 +# sending slack notifications

58 +

59 +# note: multiple recipients can be given like this:

60 +# "CHANNEL1 CHANNEL2 ..."

61 +

62 +# enable/disable sending pushover notifications

63 +SEND_SLACK="YES"

64 +

65 +# Login to slack.com and create an incoming webhook.

66 +# You need only one for all your netdata servers.

67 +# Without it, netdata cannot send slack notifications.

68 +SLACK_WEBHOOK_URL=""

69 +

70 +# if a role recipient is not configured, a notification will be send to

71 +# this slack channel:

72 +DEFAULT_RECIPIENT_SLACK=""

73 +

74 +

75 ###############################################################################

76 # RECIPIENTS PER ROLE

@@ -62,6 +83,8 @@ role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}"

84 role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

86 +role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"

87 +

89 # -----------------------------------------------------------------------------

90 # DNS related alarms

@@ -70,6 +93,8 @@ role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}"

94 role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

96 +role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"

97 +

99 # -----------------------------------------------------------------------------

100 # database servers alarms

@@ -79,6 +104,8 @@ role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"

104

105 role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}"

106

107 +role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"

108 +

109

110 # -----------------------------------------------------------------------------

111 # web servers alarms

@@ -88,6 +115,8 @@ role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"

115

116 role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}"

117

118 +role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"

119 +

120

121 # -----------------------------------------------------------------------------

122 # proxy servers alarms

@@ -97,3 +126,4 @@ role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"

126

127 role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

128

129 +role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"

plugins.d/alarm-notify.sh

+188 -61

@@ -53,8 +53,14 @@ curl=

53 sendmail=

55 # enable / disable features

56 -SEND_EMAIL="YES"

56 +SEND_SLACK="YES"

57 SEND_PUSHOVER="YES"

58 +SEND_EMAIL="YES"

59 +

60 +# slack configs

61 +SLACK_WEBHOOK_URL=

62 +DEFAULT_RECIPIENT_SLACK=

63 +declare -A role_recipients_slack=()

65 # pushover configs

66 PUSHOVER_APP_TOKEN=

@@ -65,6 +71,8 @@ declare -A role_recipients_pushover=()

71 DEFAULT_RECIPIENT_EMAIL="root"

72 declare -A role_recipients_email=()

74 +# load the user configuration

75 +# this will overwrite the variables above

76 if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]

77 then

78 source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"

@@ -73,23 +81,65 @@ fi

81 # -----------------------------------------------------------------------------

82 # find the exact recipient per method

76 -to_email="${role_recipients_email[${recipient}]}"

77 -[ -z "${to_email}" ] && to_email="${DEFAULT_RECIPIENT_EMAIL}"

84 +declare -A arr_slack=()

85 +declare -A arr_pushover=()

86 +declare -A arr_email=()

87 +

88 +# netdata may call us with multiple recipients

89 +# so, here we find the unique ones

90 +for x in ${recipient}

91 +do

92 + # email

93 + a="${role_recipients_email[${recipient}]}"

94 + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"

95 + for r in ${a//,/ }; do arr_email[${r}]="1"; done

96 +

97 + # pushover

98 + a="${role_recipients_pushover[${recipient}]}"

99 + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"

100 + for r in ${a//,/ }; do arr_pushover[${r}]="1"; done

101 +

102 + # slack

103 + a="${role_recipients_slack[${recipient}]}"

104 + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"

105 + for r in ${a//,/ }; do arr_slack[${r}]="1"; done

106 +done

107 +

108 +# build the list of slack recipients (channels)

109 +to_slack="${!arr_slack[*]}"

110 +[ -z "${to_slack}" ] && SEND_SLACK="NO"

111 +

112 +# build the list of pushover recipients (user tokens)

113 +to_pushover="${!arr_pushover[*]}"

114 +[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"

115 +

116 +# build the list of email recipients (email addresses)

117 +to_email=

118 +for x in "${!arr_email[*]}"

119 +do

120 + [ ! -z "${to_email}" ] && to_email="${to_email}, "

121 + to_email="${to_email}${x}"

122 +done

123 [ -z "${to_email}" ] && to_email="root"

124

80 -to_pushover="${role_recipients_pushover[${recipient}]}"

81 -[ -z "${to_pushover}" ] && to_pushover="${DEFAULT_RECIPIENT_EMAIL}"

82 -[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"

125

126 # -----------------------------------------------------------------------------

127 # verify the delivery methods supported

128

129 +# check slack

130 +[ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"

131 +

132 +# check pushover

133 [ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"

134

89 -if [ "${SEND_PUSHOVER}" = "YES" -a -z "${curl}" ]

135 +if [ $ "${SEND_PUSHOVER}" = "YES" -o "${SEND_SLACK}" = "YES" $ -a -z "${curl}" ]

136 then

137 curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"

92 - [ -z "${curl}" ] && SEND_PUSHOVER="NO"

138 + if [ -z "${curl}" ]

139 + then

140 + SEND_PUSHOVER="NO"

141 + SEND_SLACK="NO"

142 + fi

143 fi

144

145 if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]

@@ -99,7 +149,7 @@ if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]

149 fi

150

151 # check that we have at least a method enabled

102 -if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" ]

152 +if [ "${SEND_EMAIL}" != "YES" -a "${SEND_PUSHOVER}" != "YES" -a "${SEND_SLACK}" != "YES" ]

153 then

154 echo >&2 "I don't have a means to send a notification. Sorry!"

155 exit 1

@@ -118,6 +168,30 @@ fi

168 date="$(date --date=@${when} 2>/dev/null)"

169 [ -z "${date}" ] && date="$(date 2>/dev/null)"

170

171 +# -----------------------------------------------------------------------------

172 +# URL encode a string

173 +

174 +urlencode() {

175 + local string="${1}" strlen encoded pos c o

176 +

177 + strlen=${#string}

178 + for (( pos=0 ; pos<strlen ; pos++ ))

179 + do

180 + c=${string:$pos:1}

181 + case "$c" in

182 + [-_.~a-zA-Z0-9])

183 + o="${c}"

184 + ;;

185 +

186 + *)

187 + printf -v o '%%%02x' "'$c"

188 + ;;

189 + esac

190 + encoded+="${o}"

191 + done

192 + echo "${encoded}"

193 +}

194 +

195 # -----------------------------------------------------------------------------

196 # convert a duration in seconds, to a human readable duration

197 # using DAYS, MINUTES, SECONDS

@@ -195,15 +269,15 @@ send_email() {

269 # pushover sender

270

271 send_pushover() {

198 - local apptoken="${1}" usertoken="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority

272 + local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority

273

200 - if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertoken}" -a ! -z "${title}" -a ! -z "${message}" ]

274 + if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]

275 then

276

277 priority=0

278 [ "${status}" = "CRITICAL" ] && priority=1

279

206 - for user in ${usertoken//,/ }

280 + for user in ${usertokens}

281 do

282 httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null \

283 --form-string "token=${apptoken}" \

@@ -219,10 +293,47 @@ send_pushover() {

293

294 if [ "${httpcode}" == "200" ]

295 then

222 - echo >&2 "${me}: Sent notification push for ${status} on '${chart}.${name}' to '${user}'"

296 + echo >&2 "${me}: Sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"

297 + sent=$((sent + 1))

298 + else

299 + echo >&2 "${me}: Failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."

300 + fi

301 + done

302 +

303 + [ ${sent} -gt 0 ] && return 0

304 + fi

305 +

306 + return 1

307 +}

308 +

309 +# -----------------------------------------------------------------------------

310 +# slack sender

311 +

312 +send_slack() {

313 + local webhook="${1}" channels="${2}" when="${3}" username="${4}" image="${5}" author="${6}" httpcode sent=0 channel color

314 +

315 + if [ "${SEND_SLACK}" = "YES" -a ! -z "${webhook}" -a ! -z "${channels}" -a ! -z "${username}" -a ! -z "${image}" -a ! -z "${author}" ]

316 + then

317 +

318 + case "${status}" in

319 + WARNING) color="warning" ;;

320 + CRITICAL) color="danger" ;;

321 + CLEAR) color="good" ;;

322 + *) color="#777777" ;;

323 + esac

324 +

325 + for channel in ${channels}

326 + do

327 + httpcode=$(${curl} --write-out %{http_code} --silent --output /dev/null -X POST --data-urlencode \

328 + "payload={\"channel\": \"#${channel}\", \"username\": \"${username}\", \"text\": \"${hostname} ${status_message} about ${author} ${raised_for} - click <${goto_url}|here> to view the netdata dashboard.\", \"icon_url\": \"${image}\", \"attachments\": [{\"fallback\": \"${alarm} - ${info}\", \"color\": \"${color}\", \"title\": \"${alarm}\", \"title_link\": \"${goto_url}\", \"text\": \"${info}\", \"footer\": \"netdata\", \"footer_icon\": \"${NETDATA_REGISTRY_URL}/images/seo-performance-128.png\", \"ts\": ${when}}]}" \

329 + "${webhook}")

330 +

331 + if [ "${httpcode}" == "200" ]

332 + then

333 + echo >&2 "${me}: Sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"

334 sent=$((sent + 1))

335 else

225 - echo >&2 "${me}: FAILED to send notification push for ${status} on '${chart}.${name}' to '${user}' with HTTP error code ${httpcode}."

336 + echo >&2 "${me}: Failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."

337 fi

338 done

339

@@ -236,17 +347,14 @@ send_pushover() {

347 # -----------------------------------------------------------------------------

348 # prepare the content of the notification

349

239 -# description of the alarm

240 -[ ! -z "${info}" ] && info=" ${info}"

241 -

350 # the url to send the user on click

243 -goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"

351 +goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=$(urlencode "${NETDATA_REGISTRY_HOSTNAME}")&chart=${chart}&family=${family}"

352

353 # the severity of the alarm

354 severity="${status}"

355

356 # the time the alarm was raised

249 -raised_for=" (was ${old_status,,} for $(duration4human ${duration}))"

357 +raised_for="(was ${old_status,,} for $(duration4human ${duration}))"

358

359 # the key status message

360 status_message="status unknown"

@@ -257,19 +365,25 @@ color="grey"

365 # the alarm value

366 alarm="${name//_/ } = ${value} ${units}"

367

368 +# the image of the alarm

369 +image="${NETDATA_REGISTRY_URL}/images/seo-performance-128.png"

370 +

371 # prepare the title based on status

372 case "${status}" in

373 CRITICAL)

374 + image="${NETDATA_REGISTRY_URL}/images/alert-128-red.png"

375 status_message="is critical"

376 color="#ca414b"

377 ;;

378

379 WARNING)

380 + image="${NETDATA_REGISTRY_URL}/images/alert-128-orange.png"

381 status_message="needs attention"

382 color="#caca4b"

383 ;;

384

385 CLEAR)

386 + image="${NETDATA_REGISTRY_URL}/images/check-mark-2-128-green.png"

387 status_message="recovered"

388 color="#77ca6d"

389

@@ -284,7 +398,7 @@ then

398 severity="Recovered from ${old_status}"

399 if [ $non_clear_duration -gt $duration ]

400 then

287 - raised_for=" (had issues for $(duration4human ${non_clear_duration}))"

401 + raised_for="(had issues for $(duration4human ${non_clear_duration}))"

402 fi

403

404 elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]

@@ -292,7 +406,7 @@ then

406 severity="Escalated to ${status}"

407 if [ $non_clear_duration -gt $duration ]

408 then

295 - raised_for=" (has issues for $(duration4human ${non_clear_duration}))"

409 + raised_for="(has issues for $(duration4human ${non_clear_duration}))"

410 fi

411

412 elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]

@@ -300,23 +414,38 @@ then

414 severity="Demoted to ${status}"

415 if [ $non_clear_duration -gt $duration ]

416 then

303 - raised_for=" (has issues for $(duration4human ${non_clear_duration}))"

417 + raised_for="(has issues for $(duration4human ${non_clear_duration}))"

418 fi

419

420 else

421 raised_for=

422 fi

423

424 +# prepare HTML versions of elements

425 +info_html=

426 +[ ! -z "${info}" ] && info_html=" ${info}"

427 +

428 +raised_for_html=

429 +[ ! -z "${raised_for}" ] && raised_for_html=" ${raised_for}"

430 +

431 +# -----------------------------------------------------------------------------

432 +# send the slack notification

433 +

434 +# slack aggregates posts from the same username

435 +# so we use "${hostname} ${status}" as the bot username, to make them diff

436 +

437 +send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}" "${when}" "${hostname} ${status}" "${image}" "${chart} (${family})"

438 +SENT_SLACK=$?

439

440 # -----------------------------------------------------------------------------

312 -# send the pushover

441 +# send the pushover notification

442

443 send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${hostname} ${status_message} - ${name//_/ } - ${chart}" "

315 -${alarm}${info}

444 +${alarm}${info_html}

445 ${chart} Chart

446 ${family} Family

447 ${severity} Severity

319 -${date}${raised_for} Time

448 +${date}${raised_for_html} Time

449 <a href=\"${goto_url}\">View Netdata</a>

450 The source of this alarm is line ${src}

451 "

@@ -326,79 +455,77 @@ SENT_PUSHOVER=$?

455 # -----------------------------------------------------------------------------

456 # send the email

457

329 -cat <<EOF | send_email

458 +send_email <<EOF

459 To: ${to_email}

460 Subject: ${hostname} ${status_message} - ${name//_/ } - ${chart}

461 Content-Type: text/html

462

463 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

464 <html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">

336 -<body style="font-family:'Helvetica Neue','Helvetica',Helvetica,Arial,sans-serif;font-size:14px;width:100%!important;min-height:100%;line-height:1.6;background:#f6f6f6;margin:0;padding:0">

465 +<body style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">

466 <table>

467 <tbody>

468 <tr>

340 - <td style="vertical-align:top;" valign="top"></td>

341 - <td width="700" style="vertical-align:top;display:block!important;max-width:700px!important;clear:both!important;margin:0 auto;padding:0" valign="top">

342 - <div style="max-width:700px;display:block;margin:0 auto;padding:20px">

343 - <table width="100%" cellpadding="0" cellspacing="0" style="background:#fff;border:1px solid #e9e9e9">

469 + <td style="vertical-align: top;" valign="top"></td>

470 + <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">

471 + <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">

472 + <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">

473 <tbody>

474 <tr>

346 - <td bgcolor="#eee" style="padding: 5px 20px 5px 20px;background-color:#eee;">

347 - <div style="font-size:20px;color:#777;font-weight: bold;">netdata notification</div>

475 + <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">

476 + <div style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>

477 </td>

478 </tr>

479 <tr>

351 - <td bgcolor="${color}" style="font-size:16px;vertical-align:top;font-weight:400;text-align:center;margin:0;padding:10px;color:#ffffff;background:${color}!important;border:1px solid ${color};border-top-color:${color}" align="center" valign="top">

352 - <h1 style="font-weight:400;margin:0">${hostname} ${status_message}</h1>

480 + <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">

481 + <h1 style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${hostname} ${status_message}</h1>

482 </td>

483 </tr>

484 <tr>

356 - <td style="vertical-align:top" valign="top">

357 - <div style="margin:0;padding:20px;max-width:700px">

485 + <td style="vertical-align: top;" valign="top">

486 + <div style="margin: 0; padding: 20px; max-width: 700px;">

487 <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">

488 <tbody>

489 <tr>

361 - <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px" align="left" valign="top">

490 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">

491 ${chart}

363 - Chart

492 + Chart

493 </td>

494 </tr>

366 - <tr style="margin:0;padding:0">

367 - <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px" align="left" valign="top">

368 - ${alarm}${info}

369 - Alarm

495 + <tr style="margin: 0; padding: 0;">

496 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">

497 + ${alarm}${info_html}

498 + Alarm

499 </td>

500 </tr>

501 <tr>

373 - <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px" align="left" valign="top">

502 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">

503 ${family}

375 - Family

504 + Family

505 </td>

506 </tr>

378 - <tr style="margin:0;padding:0">

379 - <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px" align="left" valign="top">

507 + <tr style="margin: 0; padding: 0;">

508 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">

509 ${severity}

381 - Severity

510 + Severity

511 </td>

512 </tr>

384 - <tr style="margin:0;padding:0">

385 - <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px" align="left" valign="top">${date}

386 - ${raised_for} Time

513 + <tr style="margin: 0; padding: 0;">

514 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">${date}

515 + ${raised_for_html} Time

516 </td>

517 </tr>

389 - <!--

390 - <tr style="margin:0;padding:0">

391 - <td style="font-size:18px;vertical-align:top;margin:0;padding:0 0 20px">

392 - <a href="${goto_url}" style="font-size:14px;color:#ffffff;text-decoration:none;line-height:1.5;font-weight:bold;text-align:center;display:inline-block;text-transform:capitalize;background:#35568d;border-width:1px;border-style:solid;border-color:#2b4c86;margin:0;padding:10px 15px" target="_blank">View Netdata</a>

518 + <tr style="margin: 0; padding: 0;">

519 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">

520 + <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>

521 </td>

522 </tr>

395 - -->

396 - <tr style="text-align:center;margin:0;padding:0">

397 - <td style="font-size:11px;vertical-align:top;margin:0;padding:10px 0 0 0;color:#666666" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>

523 + <tr style="text-align: center; margin: 0; padding: 0;">

524 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code>

525 </td>

526 </tr>

400 - <tr style="text-align:center;margin:0;padding:0">

401 - <td style="font-size:12px;vertical-align:top;margin:0;padding:20px 0 0 0;color:#666666;border-top:1px solid #f0f0f0" align="center" valign="bottom">Sent by

527 + <tr style="text-align: center; margin: 0; padding: 0;">

528 + <td style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by

529 <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance monitoring.

530 </td>

531 </tr>

@@ -424,7 +551,7 @@ SENT_EMAIL=$?

551 # let netdata know

552

553 # we did send somehting

427 -[ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 ] && exit 0

554 +[ ${SENT_EMAIL} -eq 0 -o ${SENT_PUSHOVER} -eq 0 -o ${SENT_SLACK} -eq 0 ] && exit 0

555

556 # we did not send anything

557 exit 1