Make watcher thread wait for explicit steps. (#17079)
This will allow backtraces to display the specific step that might timeout.
vkalintiris committed
Feb 29, 2024 at 15:14 UTC
c42658403291d7492f14edb627768b8953aa62fd
1 file changed
+60
-31
src/daemon/watcher.c
+60
-31
@@ -20,47 +20,76 @@ void watcher_step_complete(watcher_step_id_t step_id) {
20
completion_mark_complete(&watcher_steps[step_id].p);
21
}
22
23
+static void watcher_wait_for_step(const watcher_step_id_t step_id)
24
+{
25
+ unsigned timeout = 90;
26
+
27
+ usec_t step_start_time = now_monotonic_usec();
28
+
29
+#ifdef ENABLE_SENTRY
30
+ // Wait with a timeout
31
+ bool ok = completion_timedwait_for(&watcher_steps[step_id].p, timeout);
32
+#else
33
+ // Wait indefinitely
34
+ bool ok = true;
35
+ completion_wait_for(&watcher_steps[step_id].p);
36
+#endif
37
+
38
+ usec_t step_duration = now_monotonic_usec() - step_start_time;
39
+
40
+ if (ok) {
41
+ netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds",
42
+ step_id + 1, WATCHER_STEP_ID_MAX,
43
+ watcher_steps[step_id].msg, step_duration / USEC_PER_MS);
44
+ } else {
45
+ // Do not call fatal() because it will try to execute the exit
46
+ // sequence twice.
47
+ netdata_log_error("shutdown step: [%d/%d] - '%s' took more than %u seconds (ie. %llu milliseconds)",
48
+ step_id + 1, WATCHER_STEP_ID_MAX, watcher_steps[step_id].msg,
49
+ timeout, step_duration / USEC_PER_MS);
50
+
51
+ abort();
52
+ }
53
+}
54
+
55
void *watcher_main(void *arg)
56
{
57
UNUSED(arg);
58
59
netdata_log_debug(D_SYSTEM, "Watcher thread started");
60
61
+ // wait until the agent starts the shutdown process
62
completion_wait_for(&shutdown_begin_completion);
30
- usec_t shutdown_start_time = now_monotonic_usec();
31
-
63
netdata_log_error("Shutdown process started");
64
34
- unsigned timeout = 60;
35
-
36
- for (int step_id = 0; step_id != WATCHER_STEP_ID_MAX; step_id++) {
37
- usec_t step_start_time = now_monotonic_usec();
38
-
39
-#ifdef ENABLE_SENTRY
40
- // Wait with a timeout
41
- bool ok = completion_timedwait_for(&watcher_steps[step_id].p, timeout);
42
-#else
43
- // Wait indefinitely
44
- bool ok = true;
45
- completion_wait_for(&watcher_steps[step_id].p);
46
-#endif
65
+ usec_t shutdown_start_time = now_monotonic_usec();
66
48
- usec_t step_duration = now_monotonic_usec() - step_start_time;
49
-
50
- if (ok) {
51
- netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds",
52
- step_id + 1, WATCHER_STEP_ID_MAX,
53
- watcher_steps[step_id].msg, step_duration / USEC_PER_MS);
54
- } else {
55
- // Do not call fatal() because it will try to execute the exit
56
- // sequence twice.
57
- netdata_log_error("shutdown step: [%d/%d] - '%s' took more than %u seconds (ie. %llu milliseconds)",
58
- step_id + 1, WATCHER_STEP_ID_MAX, watcher_steps[step_id].msg,
59
- timeout, step_duration / USEC_PER_MS);
60
-
61
- abort();
62
- }
63
- }
67
+ watcher_wait_for_step(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE);
68
+ watcher_wait_for_step(WATCHER_STEP_ID_DBENGINE_EXIT_MODE);
69
+ watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS);
70
+ watcher_wait_for_step(WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK);
71
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD);
72
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS);
73
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS);
74
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_REPLICATION_THREADS);
75
+ watcher_wait_for_step(WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN);
76
+ watcher_wait_for_step(WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS);
77
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_CONTEXT_THREAD);
78
+ watcher_wait_for_step(WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE);
79
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_ACLK_THREADS);
80
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS);
81
+ watcher_wait_for_step(WATCHER_STEP_ID_CANCEL_MAIN_THREADS);
82
+ watcher_wait_for_step(WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS);
83
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS);
84
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_METASYNC_THREADS);
85
+ watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
86
+ watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
87
+ watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
88
+ watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);
89
+ watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);
90
+ watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE);
91
+ watcher_wait_for_step(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);
92
+ watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);
93
94
completion_wait_for(&shutdown_end_completion);
95
usec_t shutdown_end_time = now_monotonic_usec();