Add watcher thread to report shutdown steps. (#17010)
* Add watcher thread to report shutdown steps. * Add license header
vkalintiris committed
Feb 26, 2024 at 11:16 UTC
5f68469ee56bff4f9f10d9a94c4461e76a7b2c52
5 files changed
+261
-105
CMakeLists.txt
+2
@@ -824,6 +824,8 @@ set(DAEMON_FILES
824
src/daemon/signals.c
825
src/daemon/signals.h
826
src/daemon/service.c
827
+ src/daemon/watcher.c
828
+ src/daemon/watcher.h
829
src/daemon/static_threads.c
830
src/daemon/static_threads.h
831
src/daemon/commands.c
src/daemon/main.c
+65
-104
@@ -2,6 +2,7 @@
2
3
#include "common.h"
4
#include "buildinfo.h"
5
+#include "daemon/watcher.h"
6
#include "static_threads.h"
7
8
#include "database/engine/page_test.h"
@@ -299,25 +300,10 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
300
return (running == 0);
301
}
302
302
-#define delta_shutdown_time(msg) \
303
- do { \
304
- usec_t now_ut = now_monotonic_usec(); \
305
- if(prev_msg) \
306
- netdata_log_info("NETDATA SHUTDOWN: in %llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \
307
- else \
308
- netdata_log_info("NETDATA SHUTDOWN: next: %s", msg); \
309
- last_ut = now_ut; \
310
- prev_msg = msg; \
311
- timeout = false; \
312
- } while(0)
313
-
303
void web_client_cache_destroy(void);
304
305
void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) {
317
- usec_t started_ut = now_monotonic_usec();
318
- usec_t last_ut = started_ut;
319
- const char *prev_msg = NULL;
320
- bool timeout = false;
306
+ watcher_shutdown_begin();
307
308
nd_log_limits_unlimited();
309
netdata_log_info("NETDATA SHUTDOWN: initializing shutdown with code %d...", ret);
@@ -330,104 +316,78 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
316
statistic = (analytics_statistic_t) {"EXIT", ret?"ERROR":"OK","-"};
317
analytics_statistic_send(&statistic);
318
333
- delta_shutdown_time("create shutdown file");
334
-
319
char agent_crash_file[FILENAME_MAX + 1];
320
char agent_incomplete_shutdown_file[FILENAME_MAX + 1];
321
snprintfz(agent_crash_file, FILENAME_MAX, "%s/.agent_crash", netdata_configured_varlib_dir);
322
snprintfz(agent_incomplete_shutdown_file, FILENAME_MAX, "%s/.agent_incomplete_shutdown", netdata_configured_varlib_dir);
323
(void) rename(agent_crash_file, agent_incomplete_shutdown_file);
324
+ watcher_step_complete(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE);
325
326
#ifdef ENABLE_DBENGINE
327
if(dbengine_enabled) {
343
- delta_shutdown_time("dbengine exit mode");
328
for (size_t tier = 0; tier < storage_tiers; tier++)
329
rrdeng_exit_mode(multidb_ctx[tier]);
330
}
331
#endif
348
-
349
- delta_shutdown_time("close webrtc connections");
332
+ watcher_step_complete(WATCHER_STEP_ID_DBENGINE_EXIT_MODE);
333
334
webrtc_close_all_connections();
335
+ watcher_step_complete(WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS);
336
353
- delta_shutdown_time("disable maintenance, new queries, new web requests, new streaming connections and aclk");
337
+ service_signal_exit(SERVICE_MAINTENANCE | ABILITY_DATA_QUERIES | ABILITY_WEB_REQUESTS |
338
+ ABILITY_STREAMING_CONNECTIONS | SERVICE_ACLK | SERVICE_ACLKSYNC);
339
+ watcher_step_complete(WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK);
340
355
- service_signal_exit(
356
- SERVICE_MAINTENANCE
357
- | ABILITY_DATA_QUERIES
358
- | ABILITY_WEB_REQUESTS
359
- | ABILITY_STREAMING_CONNECTIONS
360
- | SERVICE_ACLK
361
- | SERVICE_ACLKSYNC
362
- );
341
+ service_wait_exit(SERVICE_MAINTENANCE, 3 * USEC_PER_SEC);
342
+ watcher_step_complete(WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD);
343
364
- delta_shutdown_time("stop maintenance thread");
344
+ service_wait_exit(SERVICE_EXPORTERS | SERVICE_HEALTH | SERVICE_WEB_SERVER | SERVICE_HTTPD, 3 * USEC_PER_SEC);
345
+ watcher_step_complete(WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS);
346
366
- timeout = !service_wait_exit(
367
- SERVICE_MAINTENANCE
368
- , 3 * USEC_PER_SEC);
347
+ service_wait_exit(SERVICE_COLLECTORS | SERVICE_STREAMING, 3 * USEC_PER_SEC);
348
+ watcher_step_complete(WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS);
349
370
- delta_shutdown_time("stop replication, exporters, health and web servers threads");
371
-
372
- timeout = !service_wait_exit(
373
- SERVICE_EXPORTERS
374
- | SERVICE_HEALTH
375
- | SERVICE_WEB_SERVER
376
- | SERVICE_HTTPD
377
- , 3 * USEC_PER_SEC);
378
-
379
- delta_shutdown_time("stop collectors and streaming threads");
380
-
381
- timeout = !service_wait_exit(
382
- SERVICE_COLLECTORS
383
- | SERVICE_STREAMING
384
- , 3 * USEC_PER_SEC);
385
-
386
- delta_shutdown_time("stop replication threads");
387
-
388
- timeout = !service_wait_exit(
389
- SERVICE_REPLICATION // replication has to be stopped after STREAMING, because it cleans up ARAL
390
- , 3 * USEC_PER_SEC);
391
-
392
- delta_shutdown_time("prepare metasync shutdown");
350
+ service_wait_exit(SERVICE_REPLICATION, 3 * USEC_PER_SEC);
351
+ watcher_step_complete(WATCHER_STEP_ID_STOP_REPLICATION_THREADS);
352
353
metadata_sync_shutdown_prepare();
395
-
396
- delta_shutdown_time("disable ML detection and training threads");
354
+ watcher_step_complete(WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN);
355
356
ml_stop_threads();
357
ml_fini();
358
+ watcher_step_complete(WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS);
359
401
- delta_shutdown_time("stop context thread");
402
-
403
- timeout = !service_wait_exit(
404
- SERVICE_CONTEXT
405
- , 3 * USEC_PER_SEC);
406
-
407
- delta_shutdown_time("clear web client cache");
360
+ service_wait_exit(SERVICE_CONTEXT, 3 * USEC_PER_SEC);
361
+ watcher_step_complete(WATCHER_STEP_ID_STOP_CONTEXT_THREAD);
362
363
web_client_cache_destroy();
364
+ watcher_step_complete(WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE);
365
411
- delta_shutdown_time("stop aclk threads");
412
-
413
- timeout = !service_wait_exit(
414
- SERVICE_ACLK
415
- , 3 * USEC_PER_SEC);
416
-
417
- delta_shutdown_time("stop all remaining worker threads");
366
+ service_wait_exit(SERVICE_ACLK, 3 * USEC_PER_SEC);
367
+ watcher_step_complete(WATCHER_STEP_ID_STOP_ACLK_THREADS);
368
419
- timeout = !service_wait_exit(~0, 10 * USEC_PER_SEC);
420
-
421
- delta_shutdown_time("cancel main threads");
369
+ service_wait_exit(~0, 10 * USEC_PER_SEC);
370
+ watcher_step_complete(WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS);
371
372
cancel_main_threads();
373
+ watcher_step_complete(WATCHER_STEP_ID_CANCEL_MAIN_THREADS);
374
+
375
+ if (ret)
376
+ {
377
+ watcher_step_complete(WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS);
378
+ watcher_step_complete(WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS);
379
+ watcher_step_complete(WATCHER_STEP_ID_STOP_METASYNC_THREADS);
380
425
- if(!ret) {
381
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
382
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
383
+ watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
384
+ }
385
+ else
386
+ {
387
// exit cleanly
388
389
#ifdef ENABLE_DBENGINE
390
if(dbengine_enabled) {
430
- delta_shutdown_time("flush dbengine tiers");
391
for (size_t tier = 0; tier < storage_tiers; tier++)
392
rrdeng_prepare_exit(multidb_ctx[tier]);
393
@@ -439,21 +399,16 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
399
}
400
}
401
#endif
402
+ watcher_step_complete(WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS);
403
443
- // free the database
444
- delta_shutdown_time("stop collection for all hosts");
445
-
446
- // rrdhost_free_all();
404
rrd_finalize_collection_for_all_hosts();
448
-
449
- delta_shutdown_time("stop metasync threads");
405
+ watcher_step_complete(WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS);
406
407
metadata_sync_shutdown();
408
+ watcher_step_complete(WATCHER_STEP_ID_STOP_METASYNC_THREADS);
409
410
#ifdef ENABLE_DBENGINE
411
if(dbengine_enabled) {
455
- delta_shutdown_time("wait for dbengine collectors to finish");
456
-
412
size_t running = 1;
413
size_t count = 10;
414
while(running && count) {
@@ -467,52 +422,55 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
422
}
423
count--;
424
}
470
-
471
- delta_shutdown_time("wait for dbengine main cache to finish flushing");
425
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
426
427
while (pgc_hot_and_dirty_entries(main_cache)) {
428
pgc_flush_all_hot_and_dirty_pages(main_cache, PGC_SECTION_ALL);
429
sleep_usec(100 * USEC_PER_MS);
430
}
431
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
432
478
- delta_shutdown_time("stop dbengine tiers");
433
for (size_t tier = 0; tier < storage_tiers; tier++)
434
rrdeng_exit(multidb_ctx[tier]);
481
-
435
rrdeng_enq_cmd(NULL, RRDENG_OPCODE_SHUTDOWN_EVLOOP, NULL, NULL, STORAGE_PRIORITY_BEST_EFFORT, NULL, NULL);
436
+ watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
437
+ } else {
438
+ // Skip these steps
439
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
440
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
441
+ watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
442
}
443
+#else
444
+ // Skip these steps
445
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
446
+ watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
447
+ watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
448
#endif
449
}
450
487
- delta_shutdown_time("close SQL context db");
488
-
451
sql_close_context_database();
490
-
491
- delta_shutdown_time("closed SQL main db");
452
+ watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);
453
454
sql_close_database();
455
+ watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);
456
457
// unlink the pid
458
if(pidfile[0]) {
497
- delta_shutdown_time("remove pid file");
498
-
459
if(unlink(pidfile) != 0)
460
netdata_log_error("EXIT: cannot unlink pidfile '%s'.", pidfile);
461
}
462
+ watcher_step_complete(WATCHER_STEP_ID_REMOVE_PID_FILE);
463
464
#ifdef ENABLE_HTTPS
504
- delta_shutdown_time("free openssl structures");
465
netdata_ssl_cleanup();
466
#endif
507
-
508
- delta_shutdown_time("remove incomplete shutdown file");
467
+ watcher_step_complete(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);
468
469
(void) unlink(agent_incomplete_shutdown_file);
511
-
512
- delta_shutdown_time("exit");
513
-
514
- usec_t ended_ut = now_monotonic_usec();
515
- netdata_log_info("NETDATA SHUTDOWN: completed in %llu ms - netdata is now exiting - bye bye...", (ended_ut - started_ut) / USEC_PER_MS);
470
+ watcher_step_complete(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);
471
+
472
+ watcher_shutdown_end();
473
+ watcher_thread_stop();
474
475
#ifdef ENABLE_SENTRY
476
if (ret)
@@ -1917,6 +1875,9 @@ int main(int argc, char **argv) {
1875
load_cloud_conf(0);
1876
}
1877
1878
+ // @stelfrag: Where is the right place to call this?
1879
+ watcher_thread_start();
1880
+
1881
// ------------------------------------------------------------------------
1882
// initialize netdata
1883
{
src/daemon/static_threads.c
+1
-1
@@ -14,7 +14,7 @@ void *service_main(void *ptr);
14
void *statsd_main(void *ptr);
15
void *timex_main(void *ptr);
16
void *profile_main(void *ptr);
17
-void *replication_thread_main(void *ptr __maybe_unused);
17
+void *replication_thread_main(void *ptr);
18
19
extern bool global_statistics_enabled;
20
src/daemon/watcher.c
new
+138
@@ -0,0 +1,138 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "daemon/common.h"
4
+#include "watcher.h"
5
+
6
+watcher_step_t *watcher_steps;
7
+
8
+static struct completion shutdown_begin_completion;
9
+static struct completion shutdown_end_completion;
10
+static netdata_thread_t watcher_thread;
11
+
12
+void watcher_shutdown_begin(void) {
13
+ completion_mark_complete(&shutdown_begin_completion);
14
+}
15
+
16
+void watcher_shutdown_end(void) {
17
+ completion_mark_complete(&shutdown_end_completion);
18
+}
19
+
20
+void watcher_step_complete(watcher_step_id_t step_id) {
21
+ completion_mark_complete(&watcher_steps[step_id].p);
22
+}
23
+
24
+void *watcher_main(void *arg)
25
+{
26
+ UNUSED(arg);
27
+
28
+ netdata_log_debug(D_SYSTEM, "Watcher thread started");
29
+
30
+ completion_wait_for(&shutdown_begin_completion);
31
+ usec_t shutdown_start_time = now_monotonic_usec();
32
+
33
+ // TODO:
34
+ // - add a version of completion_wait_for with timeout
35
+ // - check the step's duration and abort when the timeout has expired.
36
+ for (int step_id = 0; step_id != WATCHER_STEP_ID_MAX; step_id++) {
37
+ usec_t step_start_time = now_monotonic_usec();
38
+ completion_wait_for(&watcher_steps[step_id].p);
39
+ usec_t step_end_time = now_monotonic_usec();
40
+
41
+ usec_t step_duration = step_end_time - step_start_time;
42
+ netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds",
43
+ step_id + 1,
44
+ WATCHER_STEP_ID_MAX,
45
+ watcher_steps[step_id].msg,
46
+ step_duration / USEC_PER_MS);
47
+ }
48
+
49
+ netdata_log_error("Shutdown process started");
50
+
51
+ completion_wait_for(&shutdown_end_completion);
52
+ usec_t shutdown_end_time = now_monotonic_usec();
53
+
54
+ usec_t shutdown_duration = shutdown_end_time - shutdown_start_time;
55
+ netdata_log_error("Shutdown process ended in %llu milliseconds",
56
+ shutdown_duration / USEC_PER_MS);
57
+
58
+ return NULL;
59
+}
60
+
61
+void watcher_thread_start() {
62
+ watcher_steps = callocz(WATCHER_STEP_ID_MAX, sizeof(watcher_step_t));
63
+
64
+ watcher_steps[WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE].msg =
65
+ "create shutdown file";
66
+ watcher_steps[WATCHER_STEP_ID_DBENGINE_EXIT_MODE].msg =
67
+ "dbengine exit mode";
68
+ watcher_steps[WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS].msg =
69
+ "close webrtc connections";
70
+ watcher_steps[WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK].msg =
71
+ "disable maintenance, new queries, new web requests, new streaming connections and aclk";
72
+ watcher_steps[WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD].msg =
73
+ "stop maintenance thread";
74
+ watcher_steps[WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS].msg =
75
+ "stop exporters, health and web servers threads";
76
+ watcher_steps[WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS].msg =
77
+ "stop collectors and streaming threads";
78
+ watcher_steps[WATCHER_STEP_ID_STOP_REPLICATION_THREADS].msg =
79
+ "stop replication threads";
80
+ watcher_steps[WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN].msg =
81
+ "prepare metasync shutdown";
82
+ watcher_steps[WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS].msg =
83
+ "disable ML detection and training threads";
84
+ watcher_steps[WATCHER_STEP_ID_STOP_CONTEXT_THREAD].msg =
85
+ "stop context thread";
86
+ watcher_steps[WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE].msg =
87
+ "clear web client cache";
88
+ watcher_steps[WATCHER_STEP_ID_STOP_ACLK_THREADS].msg =
89
+ "stop aclk threads";
90
+ watcher_steps[WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS].msg =
91
+ "stop all remaining worker threads";
92
+ watcher_steps[WATCHER_STEP_ID_CANCEL_MAIN_THREADS].msg =
93
+ "cancel main threads";
94
+ watcher_steps[WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS].msg =
95
+ "flush dbengine tiers";
96
+ watcher_steps[WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS].msg =
97
+ "stop collection for all hosts";
98
+ watcher_steps[WATCHER_STEP_ID_STOP_METASYNC_THREADS].msg =
99
+ "stop metasync threads";
100
+ watcher_steps[WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH].msg =
101
+ "wait for dbengine collectors to finish";
102
+ watcher_steps[WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING].msg =
103
+ "wait for dbengine main cache to finish flushing";
104
+ watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg =
105
+ "stop dbengine tiers";
106
+ watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB].msg =
107
+ "close SQL context db";
108
+ watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB].msg =
109
+ "close SQL main db";
110
+ watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg =
111
+ "remove pid file";
112
+ watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg =
113
+ "free openssl structures";
114
+ watcher_steps[WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE].msg =
115
+ "remove incomplete shutdown file";
116
+
117
+ for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) {
118
+ completion_init(&watcher_steps[i].p);
119
+ }
120
+
121
+ completion_init(&shutdown_begin_completion);
122
+ completion_init(&shutdown_end_completion);
123
+
124
+ netdata_thread_create(&watcher_thread, "P[WATCHER]", NETDATA_THREAD_OPTION_JOINABLE, watcher_main, NULL);
125
+}
126
+
127
+void watcher_thread_stop() {
128
+ netdata_thread_join(watcher_thread, NULL);
129
+
130
+ for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) {
131
+ completion_destroy(&watcher_steps[i].p);
132
+ }
133
+
134
+ completion_destroy(&shutdown_begin_completion);
135
+ completion_destroy(&shutdown_end_completion);
136
+
137
+ freez(watcher_steps);
138
+}
src/daemon/watcher.h
new
+55
@@ -0,0 +1,55 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef DAEMON_WATCHER_H
4
+#define DAEMON_WATCHER_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+typedef enum {
9
+ WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE = 0,
10
+ WATCHER_STEP_ID_DBENGINE_EXIT_MODE,
11
+ WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS,
12
+ WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK,
13
+ WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD,
14
+ WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS,
15
+ WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS,
16
+ WATCHER_STEP_ID_STOP_REPLICATION_THREADS,
17
+ WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN,
18
+ WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS,
19
+ WATCHER_STEP_ID_STOP_CONTEXT_THREAD,
20
+ WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE,
21
+ WATCHER_STEP_ID_STOP_ACLK_THREADS,
22
+ WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS,
23
+ WATCHER_STEP_ID_CANCEL_MAIN_THREADS,
24
+ WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS,
25
+ WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS,
26
+ WATCHER_STEP_ID_STOP_METASYNC_THREADS,
27
+ WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH,
28
+ WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING,
29
+ WATCHER_STEP_ID_STOP_DBENGINE_TIERS,
30
+ WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB,
31
+ WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB,
32
+ WATCHER_STEP_ID_REMOVE_PID_FILE,
33
+ WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES,
34
+ WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE,
35
+
36
+ // Always keep this as the last enum value
37
+ WATCHER_STEP_ID_MAX
38
+} watcher_step_id_t;
39
+
40
+typedef struct {
41
+ const char *msg;
42
+ struct completion p;
43
+} watcher_step_t;
44
+
45
+extern watcher_step_t *watcher_steps;
46
+
47
+void watcher_thread_start(void);
48
+void watcher_thread_stop(void);
49
+
50
+void watcher_shutdown_begin(void);
51
+void watcher_shutdown_end(void);
52
+
53
+void watcher_step_complete(watcher_step_id_t step_id);
54
+
55
+#endif /* DAEMON_WATCHER_H */