Introduce a random sleep in the Netdata updater (#9079)
* Introduce a random sleep in the Netdata updater * Only sleep if we're not a tty (e.g: cron) and use a random interval between 30m-60m * Set lower bound to 1s * Disable random sleep / netdata-updater splay in lifecycle tests
James Mills committed
May 26, 2020 at 15:58 UTC
cea8a3fcbb3b48fce545d19bce2b5cf920c6794d
2 files changed
+13
-2
packaging/installer/netdata-updater.sh
+10
-2
@@ -185,8 +185,8 @@ update() {
185
do_not_start="--dont-start-it"
186
fi
187
188
- if [ -n "${NETDATA_SELECTED_DASHBOARD}" ] ; then
189
- env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}"
188
+ if [ -n "${NETDATA_SELECTED_DASHBOARD}" ]; then
189
+ env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}"
190
fi
191
192
info "Re-installing netdata..."
@@ -210,6 +210,14 @@ tmpdir=
210
211
trap cleanup EXIT
212
213
+# Random sleep to aileviate stampede effect of Agents upgrading
214
+# and disconnecting/reconnecting at the same time (or near to).
215
+# But only we're not a controlling terminal (tty)
216
+# Randomly sleep between 1s and 60m
217
+if [ ! -t 1 ]; then
218
+ sleep $(((RANDOM % 3600) + 1))s
219
+fi
220
+
221
# Usually stored in /etc/netdata/.environment
222
: "${ENVIRONMENT_FILE:=THIS_SHOULD_BE_REPLACED_BY_INSTALLER_SCRIPT}"
223
tests/updater_checks.bats
+3
@@ -56,6 +56,9 @@ setup() {
56
# Run the updater, with the override so that it uses the local repo we have at hand
57
# Try to run the installed, if any, otherwise just run the one from the repo
58
export NETDATA_LOCAL_TARBAL_OVERRIDE="${PWD}"
59
+ # Disable random sleep / splay for netdata-updater to avoid sampede effect
60
+ # of many agents (dis|re)connecting too quickly all at onace to Netdata Cloud
61
+ unset RANDOM; export RANDOM=0
62
/etc/cron.daily/netdata-updater || ./packaging/installer/netdata-updater.sh
63
! grep "new_installation" "${ENV}"
64
}