@cryptotaxi247 / netdata-1 / commits / 147fae356

Adjust MCP shutdown sequence (#22452)

fix(shutdown): drain websockets before tearing down dbengine/metasync/workers MCP message dispatch is synchronous on the websocket thread, so an in-flight handler blocks the loop from observing its cancel flag. Joining late means the handler waits on torn-down substrates and the 135s watchdog aborts. Move the join right after STOP_EXPORTERS.

Stelios Fragkakis committed May 9, 2026 at 20:35 UTC 147fae3569041c8446f62c7456296fe4b1764b96

3 files changed +10 -5

src/daemon/daemon-shutdown-watcher.c

+1 -1

@@ -132,6 +132,7 @@ void watcher_main(void *arg)

132 WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS, shutdown_start_time);

133 watcher_wait_for_step(WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD, shutdown_start_time);

134 watcher_wait_for_step(WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS, shutdown_start_time);

135 + watcher_wait_for_step(WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS, shutdown_start_time);

136 watcher_wait_for_step(WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS, shutdown_start_time);

137 watcher_wait_for_step(WATCHER_STEP_ID_STOP_REPLICATION_THREADS, shutdown_start_time);

138 watcher_wait_for_step(WATCHER_STEP_ID_DISABLE_ML_DETEC_AND_TRAIN_THREADS, shutdown_start_time);

@@ -145,7 +146,6 @@ void watcher_main(void *arg)

146 watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH, shutdown_start_time);

147 watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS, shutdown_start_time);

148 watcher_wait_for_step(WATCHER_STEP_ID_STOP_METASYNC_THREADS, shutdown_start_time);

148 - watcher_wait_for_step(WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS, shutdown_start_time);

149 watcher_wait_for_step(WATCHER_STEP_ID_JOIN_STATIC_THREADS, shutdown_start_time);

150 watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_DATABASES, shutdown_start_time);

151 watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE, shutdown_start_time);

src/daemon/daemon-shutdown-watcher.h

+1 -1

@@ -10,6 +10,7 @@ typedef enum {

10 WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS,

11 WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD,

12 WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS,

13 + WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS,

14 WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS,

15 WATCHER_STEP_ID_STOP_REPLICATION_THREADS,

16 WATCHER_STEP_ID_DISABLE_ML_DETEC_AND_TRAIN_THREADS,

@@ -23,7 +24,6 @@ typedef enum {

24 WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH,

25 WATCHER_STEP_ID_STOP_DBENGINE_TIERS,

26 WATCHER_STEP_ID_STOP_METASYNC_THREADS,

26 - WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS,

27 WATCHER_STEP_ID_JOIN_STATIC_THREADS,

28 WATCHER_STEP_ID_CLOSE_SQL_DATABASES,

29 WATCHER_STEP_ID_REMOVE_PID_FILE,

src/daemon/daemon-shutdown.c

+8 -3

@@ -216,6 +216,14 @@ static void netdata_cleanup_and_exit(EXIT_REASON reason, bool abnormal, bool exi

216 service_wait_exit(SERVICE_EXPORTERS | SERVICE_HEALTH | SERVICE_WEB_SERVER | SERVICE_HTTPD, 3 * USEC_PER_SEC);

217 watcher_step_complete(WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS);

218

219 + // Drain websocket threads while data substrates are still alive. Websocket

220 + // message dispatch (e.g. MCP) is synchronous on the websocket thread; an

221 + // in-flight handler will not let the loop observe its cancel flag until it

222 + // returns. If we wait until after dbengine/metasync/workers are gone, an

223 + // MCP request that needs them never returns and the watchdog aborts.

224 + websocket_threads_join();

225 + watcher_step_complete(WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS);

226 +

227 stream_threads_cancel();

228 service_wait_exit(SERVICE_COLLECTORS | SERVICE_STREAMING, 20 * USEC_PER_SEC);

229 service_signal_exit(SERVICE_STREAMING_CONNECTOR);

@@ -306,9 +314,6 @@ static void netdata_cleanup_and_exit(EXIT_REASON reason, bool abnormal, bool exi

314 if (!abnormal)

315 add_agent_event(EVENT_AGENT_SHUTDOWN_TIME, (int64_t)(now_monotonic_usec() - shutdown_start_time));

316

309 - websocket_threads_join();

310 - watcher_step_complete(WATCHER_STEP_ID_STOP_WEBSOCKET_THREADS);

311 -

317 nd_thread_join_threads();

318 watcher_step_complete(WATCHER_STEP_ID_JOIN_STATIC_THREADS);

319