Abort the agent if a single shutdown step takes more than 60 seconds. (#17060)
* Add timed-wait for completion. * Abort if any shutdown step takes more than 60 seconds to complete. * Timeout only on sentry builds.
vkalintiris committed
Feb 27, 2024 at 20:10 UTC
700d77b5b9939671e77b789d07ed35419f7db369
3 files changed
+67
-14
src/daemon/watcher.c
+28
-14
@@ -1,6 +1,5 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
-#include "daemon/common.h"
3
#include "watcher.h"
4
5
watcher_step_t *watcher_steps;
@@ -30,24 +29,39 @@ void *watcher_main(void *arg)
29
completion_wait_for(&shutdown_begin_completion);
30
usec_t shutdown_start_time = now_monotonic_usec();
31
33
- // TODO:
34
- // - add a version of completion_wait_for with timeout
35
- // - check the step's duration and abort when the timeout has expired.
32
+ netdata_log_error("Shutdown process started");
33
+
34
+ unsigned timeout = 60;
35
+
36
for (int step_id = 0; step_id != WATCHER_STEP_ID_MAX; step_id++) {
37
usec_t step_start_time = now_monotonic_usec();
38
+
39
+#ifdef ENABLE_SENTRY
40
+ // Wait with a timeout
41
+ bool ok = completion_timedwait_for(&watcher_steps[step_id].p, timeout);
42
+#else
43
+ // Wait indefinitely
44
+ bool ok = true;
45
completion_wait_for(&watcher_steps[step_id].p);
39
- usec_t step_end_time = now_monotonic_usec();
40
-
41
- usec_t step_duration = step_end_time - step_start_time;
42
- netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds",
43
- step_id + 1,
44
- WATCHER_STEP_ID_MAX,
45
- watcher_steps[step_id].msg,
46
- step_duration / USEC_PER_MS);
46
+#endif
47
+
48
+ usec_t step_duration = now_monotonic_usec() - step_start_time;
49
+
50
+ if (ok) {
51
+ netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds",
52
+ step_id + 1, WATCHER_STEP_ID_MAX,
53
+ watcher_steps[step_id].msg, step_duration / USEC_PER_MS);
54
+ } else {
55
+ // Do not call fatal() because it will try to execute the exit
56
+ // sequence twice.
57
+ netdata_log_error("shutdown step: [%d/%d] - '%s' took more than %u seconds (ie. %llu milliseconds)",
58
+ step_id + 1, WATCHER_STEP_ID_MAX, watcher_steps[step_id].msg,
59
+ timeout, step_duration / USEC_PER_MS);
60
+
61
+ abort();
62
+ }
63
}
64
49
- netdata_log_error("Shutdown process started");
50
-
65
completion_wait_for(&shutdown_end_completion);
66
usec_t shutdown_end_time = now_monotonic_usec();
67
src/libnetdata/completion/completion.c
+35
@@ -26,6 +26,41 @@ void completion_wait_for(struct completion *p)
26
uv_mutex_unlock(&p->mutex);
27
}
28
29
+bool completion_timedwait_for(struct completion *p, uint64_t timeout)
30
+{
31
+ timeout *= NSEC_PER_SEC;
32
+
33
+ uint64_t start_time = uv_hrtime();
34
+ bool result = true;
35
+
36
+ uv_mutex_lock(&p->mutex);
37
+ while (!p->completed) {
38
+ int rc = uv_cond_timedwait(&p->cond, &p->mutex, timeout);
39
+
40
+ if (rc == 0) {
41
+ result = true;
42
+ break;
43
+ } else if (rc == UV_ETIMEDOUT) {
44
+ result = false;
45
+ break;
46
+ }
47
+
48
+ /*
49
+ * handle spurious wakeups
50
+ */
51
+
52
+ uint64_t elapsed = uv_hrtime() - start_time;
53
+ if (elapsed >= timeout) {
54
+ result = false;
55
+ break;
56
+ }
57
+ timeout -= elapsed;
58
+ }
59
+ uv_mutex_unlock(&p->mutex);
60
+
61
+ return result;
62
+}
63
+
64
void completion_mark_complete(struct completion *p)
65
{
66
uv_mutex_lock(&p->mutex);
src/libnetdata/completion/completion.h
+4
@@ -18,6 +18,10 @@ void completion_destroy(struct completion *p);
18
19
void completion_wait_for(struct completion *p);
20
21
+// Wait for at most `timeout` seconds. Return true on success, false on
22
+// error or timeout.
23
+bool completion_timedwait_for(struct completion *p, uint64_t timeout);
24
+
25
void completion_mark_complete(struct completion *p);
26
27
unsigned completion_wait_for_a_job(struct completion *p, unsigned completed_jobs);