Fix telegram-bot rate limit (#13119)
Co-authored-by: ilyam8 <ilya@netdata.cloud>
МАН69К committed
Aug 19, 2022 at 19:21 UTC
1cbd7ab42d8eedd12dc0f14d29f24e4c84dc3cac
2 files changed
+36
-15
health/notifications/alarm-notify.sh.in
+31
-15
@@ -1339,21 +1339,37 @@ send_telegram() {
1339
1340
if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
1341
for chatid in ${chatids}; do
1342
- # https://core.telegram.org/bots/api#sendmessage
1343
- httpcode=$(docurl ${disableNotification} \
1344
- --data-urlencode "parse_mode=HTML" \
1345
- --data-urlencode "disable_web_page_preview=true" \
1346
- --data-urlencode "text=${emoji} ${message}" \
1347
- "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
1348
-
1349
- if [ "${httpcode}" = "200" ]; then
1350
- info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
1351
- sent=$((sent + 1))
1352
- elif [ "${httpcode}" = "401" ]; then
1353
- error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
1354
- else
1355
- error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
1356
- fi
1342
+ notify_telegram=1
1343
+ notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0}
1344
+
1345
+ while [ ${notify_telegram} -eq 1 ]; do
1346
+ # https://core.telegram.org/bots/api#sendmessage
1347
+ httpcode=$(docurl ${disableNotification} \
1348
+ --data-urlencode "parse_mode=HTML" \
1349
+ --data-urlencode "disable_web_page_preview=true" \
1350
+ --data-urlencode "text=${emoji} ${message}" \
1351
+ "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
1352
+
1353
+ notify_telegram=0
1354
+
1355
+ if [ "${httpcode}" = "200" ]; then
1356
+ info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
1357
+ sent=$((sent + 1))
1358
+ elif [ "${httpcode}" = "401" ]; then
1359
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
1360
+ elif [ "${httpcode}" = "429" ]; then
1361
+ if [ "$notify_retries" -gt 0 ]; then
1362
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded, retrying after 1s."
1363
+ notify_retries=$((notify_retries - 1))
1364
+ notify_telegram=1
1365
+ sleep 1
1366
+ else
1367
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded."
1368
+ fi
1369
+ else
1370
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
1371
+ fi
1372
+ done
1373
done
1374
1375
[ ${sent} -gt 0 ] && return 0
health/notifications/health_alarm_notify.conf
+5
@@ -443,6 +443,11 @@ SEND_TELEGRAM="YES"
443
# Without it, netdata cannot send telegram messages.
444
TELEGRAM_BOT_TOKEN=""
445
446
+# If an API limit error is returned on sending a message, Netdata will retry this number of times before giving up.
447
+# Setting the number to 0 makes Netdata do no retries (which is the default).
448
+# See https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this
449
+TELEGRAM_RETRIES_ON_LIMIT="0"
450
+
451
# To get your chat ID send the command /getid to telegram bot @myidbot
452
# (https://t.me/myidbot). Each user also needs to open a conversation with the
453
# bot that will be sending notifications.