Update handling of shutdown of the Netdata agent on update and uninstall. (#7595)
* Make the agent killing logic more reliable. This adjusts how we handle terminating the agent during an upgrade so that it does a better job of actually ensuring it's dead. The new logic works as follows: * After each signal is sent, we wait 5 seconds before checking for the agent again. * If it's been 60 seconds (12 cycles) since we started trying to kill the PID, we switch from SIGTERM to sending SIGKILL. * If it's been 120 seconds (24 cycles) since we started trying to kill the PID, we just give up. * Attempt to use netdatacli to shutdown Netdata. This is far more reliable than any other approach we have if it works. * Update uninstaller to use same stop logic as installer. This improves it's reliability significantly and makes it work correctly on FreeBSD. * Fix codacy issues.
Austin S. Hemmelgarn committed
Jan 7, 2020 at 06:23 UTC
42a72f61dbd6f66bc9cf9fa9e273cc47d466491b
2 files changed
+105
-26
packaging/installer/functions.sh
+21
-5
@@ -424,7 +424,7 @@ install_netdata_service() {
424
pidisnetdata() {
425
if [ -d /proc/self ]; then
426
[ -z "$1" -o ! -f "/proc/$1/stat" ] && return 1
427
- [ "$(cat "/proc/$1/stat" | cut -d '(' -f 2 | cut -d ')' -f 1)" = "netdata" ] && return 0
427
+ [ "$(cut -d '(' -f 2 "/proc/$1/stat" | cut -d ')' -f 1)" = "netdata" ] && return 0
428
return 1
429
fi
430
return 0
@@ -437,17 +437,27 @@ stop_netdata_on_pid() {
437
438
printf >&2 "Stopping netdata on pid %s ..." "${pid}"
439
while [ -n "$pid" ] && [ ${ret} -eq 0 ]; do
440
- if [ ${count} -gt 45 ]; then
440
+ if [ ${count} -gt 24 ]; then
441
echo >&2 "Cannot stop the running netdata on pid ${pid}."
442
return 1
443
fi
444
445
count=$((count + 1))
446
447
- run kill "${pid}" 2>/dev/null
448
- ret=$?
447
+ pidisnetdata "${pid}" || ret=1
448
+ if [ ${ret} -eq 1 ] ; then
449
+ break
450
+ fi
451
+
452
+ if [ ${count} -lt 12 ] ; then
453
+ run kill "${pid}" 2>/dev/null
454
+ ret=$?
455
+ else
456
+ run kill -9 "${pid}" 2>/dev/null
457
+ ret=$?
458
+ fi
459
450
- test ${ret} -eq 0 && printf >&2 "." && sleep 2
460
+ test ${ret} -eq 0 && printf >&2 "." && sleep 5
461
462
done
463
@@ -480,6 +490,12 @@ netdata_pids() {
490
491
stop_all_netdata() {
492
local p
493
+
494
+ if [ -n $(netdata_pids) -a -n "$(builtin type -P netdatacli)" ] ; then
495
+ netdatacli shutdown-agent
496
+ sleep 20
497
+ fi
498
+
499
for p in $(netdata_pids); do
500
# shellcheck disable=SC2086
501
stop_netdata_on_pid ${p}
packaging/installer/netdata-uninstaller.sh
+84
-21
@@ -258,25 +258,101 @@ rm_dir() {
258
fi
259
}
260
261
+safe_pidof() {
262
+ local pidof_cmd="$(command -v pidof 2>/dev/null)"
263
+ if [ -n "${pidof_cmd}" ]; then
264
+ ${pidof_cmd} "${@}"
265
+ return $?
266
+ else
267
+ ps -acxo pid,comm |
268
+ sed "s/^ *//g" |
269
+ grep netdata |
270
+ cut -d ' ' -f 1
271
+ return $?
272
+ fi
273
+}
274
+
275
+pidisnetdata() {
276
+ if [ -d /proc/self ]; then
277
+ [ -z "$1" -o ! -f "/proc/$1/stat" ] && return 1
278
+ [ "$(cut -d '(' -f 2 "/proc/$1/stat" | cut -d ')' -f 1)" = "netdata" ] && return 0
279
+ return 1
280
+ fi
281
+ return 0
282
+}
283
+
284
+stop_netdata_on_pid() {
285
+ local pid="${1}" ret=0 count=0
286
+
287
+ pidisnetdata "${pid}" || return 0
288
+
289
+ printf >&2 "Stopping netdata on pid %s ..." "${pid}"
290
+ while [ -n "$pid" ] && [ ${ret} -eq 0 ]; do
291
+ if [ ${count} -gt 24 ]; then
292
+ echo >&2 "Cannot stop the running netdata on pid ${pid}."
293
+ return 1
294
+ fi
295
+
296
+ count=$((count + 1))
297
+
298
+ pidisnetdata "${pid}" || ret=1
299
+ if [ ${ret} -eq 1 ] ; then
300
+ break
301
+ fi
302
+
303
+ if [ ${count} -lt 12 ] ; then
304
+ run kill "${pid}" 2>/dev/null
305
+ ret=$?
306
+ else
307
+ run kill -9 "${pid}" 2>/dev/null
308
+ ret=$?
309
+ fi
310
+
311
+ test ${ret} -eq 0 && printf >&2 "." && sleep 5
312
+
313
+ done
314
+
315
+ echo >&2
316
+ if [ ${ret} -eq 0 ]; then
317
+ echo >&2 "SORRY! CANNOT STOP netdata ON PID ${pid} !"
318
+ return 1
319
+ fi
320
+
321
+ echo >&2 "netdata on pid ${pid} stopped."
322
+ return 0
323
+}
324
+
325
netdata_pids() {
326
local p myns ns
327
+
328
myns="$(readlink /proc/self/ns/pid 2>/dev/null)"
329
+
330
for p in \
331
$(cat /var/run/netdata.pid 2>/dev/null) \
332
$(cat /var/run/netdata/netdata.pid 2>/dev/null) \
267
- $(pidof netdata 2>/dev/null); do
268
-
333
+ $(safe_pidof netdata 2>/dev/null); do
334
ns="$(readlink "/proc/${p}/ns/pid" 2>/dev/null)"
270
- #shellcheck disable=SC2002
335
+
336
if [ -z "${myns}" ] || [ -z "${ns}" ] || [ "${myns}" = "${ns}" ]; then
272
- name="$(cat "/proc/${p}/stat" 2>/dev/null | cut -d '(' -f 2 | cut -d ')' -f 1)"
273
- if [ "${name}" = "netdata" ]; then
274
- echo "${p}"
275
- fi
337
+ pidisnetdata "${p}" && echo "${p}"
338
fi
339
done
340
}
341
342
+stop_all_netdata() {
343
+ local p
344
+
345
+ if [ -n $(netdata_pids) -a -n "$(builtin type -P netdatacli)" ] ; then
346
+ netdatacli shutdown-agent
347
+ sleep 20
348
+ fi
349
+
350
+ for p in $(netdata_pids); do
351
+ # shellcheck disable=SC2086
352
+ stop_netdata_on_pid ${p}
353
+ done
354
+}
355
+
356
trap quit_msg EXIT
357
358
#shellcheck source=/dev/null
@@ -284,20 +360,7 @@ source "${ENVIRONMENT_FILE}" || exit 1
360
361
#### STOP NETDATA
362
echo >&2 "Stopping a possibly running netdata..."
287
-for p in $(netdata_pids); do
288
- i=0
289
- while kill "${p}" 2>/dev/null; do
290
- if [ "$i" -gt 30 ]; then
291
- echo >&2 "Forcefully stopping netdata with pid ${p}"
292
- run kill -9 "${p}"
293
- run sleep 2
294
- break
295
- fi
296
- sleep 1
297
- i=$((i + 1))
298
- done
299
-done
300
-sleep 2
363
+stop_all_netdata
364
365
#### REMOVE NETDATA FILES
366
rm_file /etc/logrotate.d/netdata