| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "daemon-shutdown-watcher.h" |
| 4 | #include "status-file.h" |
| 5 | |
| 6 | #ifdef ENABLE_SENTRY |
| 7 | #include "sentry-native/sentry-native.h" |
| 8 | #endif |
| 9 | |
| 10 | watcher_step_t *watcher_steps; |
| 11 | |
| 12 | static struct completion shutdown_begin_completion; |
| 13 | static struct completion shutdown_end_completion; |
| 14 | static ND_THREAD *watcher_thread; |
| 15 | |
| 16 | static BUFFER *steps_timings = NULL; |
| 17 | |
| 18 | NEVER_INLINE |
| 19 | static void shutdown_timed_out(void) { |
| 20 | // keep this as a separate function, to have it logged like this in sentry |
| 21 | daemon_status_file_shutdown_timeout(steps_timings); |
| 22 | |
| 23 | // NOTE: We intentionally skip adding a Sentry breadcrumb here because: |
| 24 | // 1. During shutdown timeout, the status file has already been saved with timeout info |
| 25 | // 2. Sentry may crash trying to access potentially freed/corrupted strings from session_status |
| 26 | // 3. The abort() below will trigger a signal that Sentry will catch anyway |
| 27 | |
| 28 | abort(); |
| 29 | } |
| 30 | |
| 31 | void watcher_shutdown_begin(void) { |
| 32 | completion_mark_complete(&shutdown_begin_completion); |
| 33 | } |
| 34 | |
| 35 | void watcher_shutdown_end(void) { |
| 36 | completion_mark_complete(&shutdown_end_completion); |
| 37 | } |
| 38 | |
| 39 | void watcher_step_complete(watcher_step_id_t step_id) { |
| 40 | completion_mark_complete(&watcher_steps[step_id].p); |
| 41 | } |
| 42 | |
| 43 | static void watcher_wait_for_step(const watcher_step_id_t step_id, usec_t shutdown_start_time) |
| 44 | { |
| 45 | if(!steps_timings) { |
| 46 | steps_timings = buffer_create(0, NULL); |
| 47 | buffer_strcat(steps_timings, STACK_TRACE_INFO_PREFIX " shutdown steps timings"); |
| 48 | } |
| 49 | |
| 50 | usec_t step_start_time = now_monotonic_usec(); |
| 51 | usec_t step_start_duration = step_start_time - shutdown_start_time; |
| 52 | |
| 53 | char start_duration_txt[64]; |
| 54 | duration_snprintf( |
| 55 | start_duration_txt, sizeof(start_duration_txt), (int64_t)step_start_duration, "us", true); |
| 56 | |
| 57 | netdata_log_info("shutdown step: [%d/%d] - {at %s} started '%s'...", |
| 58 | (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt, |
| 59 | watcher_steps[step_id].msg); |
| 60 | |
| 61 | #if defined(FSANITIZE_ADDRESS) |
| 62 | fprintf(stdout, " > shutdown step: [%d/%d] - {at %s} started '%s'...\n", |
| 63 | (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt, |
| 64 | watcher_steps[step_id].msg); |
| 65 | #endif |
| 66 | |
| 67 | daemon_status_file_shutdown_step(watcher_steps[step_id].msg, buffer_tostring(steps_timings)); |
| 68 | |
| 69 | // Wait with a timeout |
| 70 | time_t timeout = 135; // systemd gives us 150, we timeout at 135 |
| 71 | |
| 72 | time_t remaining_seconds = timeout - (time_t)(step_start_duration / USEC_PER_SEC); |
| 73 | if(remaining_seconds < 0) |
| 74 | remaining_seconds = 0; |
| 75 | |
| 76 | #if defined(FSANITIZE_ADDRESS) |
| 77 | completion_wait_for(&watcher_steps[step_id].p); |
| 78 | bool ok = true; |
| 79 | #else |
| 80 | bool ok = completion_timedwait_for(&watcher_steps[step_id].p, remaining_seconds); |
| 81 | #endif |
| 82 | |
| 83 | usec_t step_duration = now_monotonic_usec() - step_start_time; |
| 84 | |
| 85 | char step_duration_txt[64]; |
| 86 | duration_snprintf( |
| 87 | step_duration_txt, sizeof(step_duration_txt), (int64_t)(step_duration), "us", true); |
| 88 | |
| 89 | buffer_sprintf(steps_timings, "\n#%u '%s': %s", step_id + 1, watcher_steps[step_id].msg, step_duration_txt); |
| 90 | |
| 91 | if (ok) { |
| 92 | netdata_log_info("shutdown step: [%d/%d] - {at %s} finished '%s' in %s", |
| 93 | (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt, |
| 94 | watcher_steps[step_id].msg, step_duration_txt); |
| 95 | |
| 96 | #if defined(FSANITIZE_ADDRESS) |
| 97 | fprintf(stdout, " > shutdown step: [%d/%d] - {at %s} finished '%s' in %s\n", |
| 98 | (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt, |
| 99 | watcher_steps[step_id].msg, step_duration_txt); |
| 100 | #endif |
| 101 | } else { |
| 102 | // Do not call fatal() because it will try to execute the exit |
| 103 | // sequence twice. |
| 104 | netdata_log_error("shutdown step: [%d/%d] - {at %s} timeout '%s' takes too long (%s) - giving up...", |
| 105 | (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt, |
| 106 | watcher_steps[step_id].msg, step_duration_txt); |
| 107 | |
| 108 | #if defined(FSANITIZE_ADDRESS) |
| 109 | fprintf(stdout, "shutdown step: [%d/%d] - {at %s} timeout '%s' takes too long (%s) - giving up...\n", |
| 110 | (int)step_id + 1, (int)WATCHER_STEP_ID_MAX, start_duration_txt, |
| 111 | watcher_steps[step_id].msg, step_duration_txt); |
| 112 | #endif |
| 113 | |
| 114 | shutdown_timed_out(); |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | void watcher_main(void *arg) |
| 119 | { |
| 120 | UNUSED(arg); |
| 121 | |
| 122 | netdata_log_debug(D_SYSTEM, "Watcher thread started"); |
| 123 | |
| 124 | // wait until the agent starts the shutdown process |
| 125 | completion_wait_for(&shutdown_begin_completion); |
| 126 | netdata_log_info("Shutdown process started"); |
| 127 | |
| 128 | usec_t shutdown_start_time = now_monotonic_usec(); |
| 129 | |
| 130 | watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS, shutdown_start_time); |
| 131 | watcher_wait_for_step( |
| 132 | WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS, shutdown_start_time); |
| 133 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD, shutdown_start_time); |
| 134 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS, shutdown_start_time); |
| 135 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS, shutdown_start_time); |
| 136 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS, shutdown_start_time); |
| 137 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_REPLICATION_THREADS, shutdown_start_time); |
| 138 | watcher_wait_for_step(WATCHER_STEP_ID_DISABLE_ML_DETEC_AND_TRAIN_THREADS, shutdown_start_time); |
| 139 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_CONTEXT_THREAD, shutdown_start_time); |
| 140 | watcher_wait_for_step(WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE, shutdown_start_time); |
| 141 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_ACLK_SYNC_THREAD, shutdown_start_time); |
| 142 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_ACLK_MQTT_THREAD, shutdown_start_time); |
| 143 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS, shutdown_start_time); |
| 144 | watcher_wait_for_step(WATCHER_STEP_ID_CANCEL_MAIN_THREADS, shutdown_start_time); |
| 145 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS, shutdown_start_time); |
| 146 | watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH, shutdown_start_time); |
| 147 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS, shutdown_start_time); |
| 148 | watcher_wait_for_step(WATCHER_STEP_ID_STOP_METASYNC_THREADS, shutdown_start_time); |
| 149 | watcher_wait_for_step(WATCHER_STEP_ID_JOIN_STATIC_THREADS, shutdown_start_time); |
| 150 | watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_DATABASES, shutdown_start_time); |
| 151 | watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE, shutdown_start_time); |
| 152 | watcher_wait_for_step(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES, shutdown_start_time); |
| 153 | |
| 154 | completion_wait_for(&shutdown_end_completion); |
| 155 | usec_t shutdown_end_time = now_monotonic_usec(); |
| 156 | |
| 157 | usec_t shutdown_duration = shutdown_end_time - shutdown_start_time; |
| 158 | |
| 159 | char shutdown_timing[64]; |
| 160 | duration_snprintf(shutdown_timing, sizeof(shutdown_timing), (int64_t)shutdown_duration, "us", 1); |
| 161 | netdata_log_info("Shutdown process ended in %s", shutdown_timing); |
| 162 | |
| 163 | daemon_status_file_shutdown_step(NULL, buffer_tostring(steps_timings)); |
| 164 | daemon_status_file_update_status(DAEMON_STATUS_EXITED); |
| 165 | } |
| 166 | |
| 167 | void watcher_thread_start() { |
| 168 | watcher_steps = callocz(WATCHER_STEP_ID_MAX, sizeof(watcher_step_t)); |
| 169 | |
| 170 | watcher_steps[WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS].msg = "close webrtc connections"; |
| 171 | watcher_steps[WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS] |
| 172 | .msg = "disable maintenance, new queries, new web requests, new streaming connections and aclk"; |
| 173 | watcher_steps[WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD].msg = "stop maintenance thread"; |
| 174 | watcher_steps[WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS].msg = |
| 175 | "stop exporters, health and web servers threads"; |
| 176 | watcher_steps[WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS].msg = "stop collectors and streaming threads"; |
| 177 | watcher_steps[WATCHER_STEP_ID_STOP_REPLICATION_THREADS].msg = "stop replication threads"; |
| 178 | watcher_steps[WATCHER_STEP_ID_DISABLE_ML_DETEC_AND_TRAIN_THREADS].msg = "disable ML detection and training threads"; |
| 179 | watcher_steps[WATCHER_STEP_ID_STOP_CONTEXT_THREAD].msg = "stop context thread"; |
| 180 | watcher_steps[WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE].msg = "clear web client cache"; |
| 181 | watcher_steps[WATCHER_STEP_ID_STOP_ACLK_SYNC_THREAD].msg = "stop ACLK sync thread"; |
| 182 | watcher_steps[WATCHER_STEP_ID_STOP_ACLK_MQTT_THREAD].msg = "stop ACLK MQTT connection thread"; |
| 183 | watcher_steps[WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS].msg = "stop all remaining worker threads"; |
| 184 | watcher_steps[WATCHER_STEP_ID_CANCEL_MAIN_THREADS].msg = "cancel main threads"; |
| 185 | watcher_steps[WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS].msg = "stop collection for all hosts"; |
| 186 | watcher_steps[WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH].msg = |
| 187 | "wait for dbengine collectors to finish"; |
| 188 | watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg = "stop dbengine tiers"; |
| 189 | watcher_steps[WATCHER_STEP_ID_STOP_METASYNC_THREADS].msg = "stop metasync threads"; |
| 190 | watcher_steps[WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS].msg = "stop websocket threads"; |
| 191 | watcher_steps[WATCHER_STEP_ID_JOIN_STATIC_THREADS].msg = "join static threads"; |
| 192 | watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_DATABASES].msg = "close SQL databases"; |
| 193 | watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg = "remove pid file"; |
| 194 | watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg = "free openssl structures"; |
| 195 | |
| 196 | for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) { |
| 197 | completion_init(&watcher_steps[i].p); |
| 198 | } |
| 199 | |
| 200 | completion_init(&shutdown_begin_completion); |
| 201 | completion_init(&shutdown_end_completion); |
| 202 | |
| 203 | watcher_thread = nd_thread_create("EXIT_WATCHER", NETDATA_THREAD_OPTION_DEFAULT, watcher_main, NULL); |
| 204 | } |
| 205 | |
| 206 | void watcher_thread_stop() { |
| 207 | nd_thread_join(watcher_thread); |
| 208 | |
| 209 | for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) { |
| 210 | completion_destroy(&watcher_steps[i].p); |
| 211 | } |
| 212 | |
| 213 | completion_destroy(&shutdown_begin_completion); |
| 214 | completion_destroy(&shutdown_end_completion); |
| 215 | |
| 216 | freez(watcher_steps); |
| 217 | } |