Improve agent shutdown (#17868)
* Handle libuv threads differently * Check collectors running for freebsd plugin * Request ML threads to stop earlier in the shutdown process * Use service_running(SERVICE_COLLECTORS) to check for plugin_macos
Stelios Fragkakis committed
Jun 13, 2024 at 23:38 UTC
f564520f62b13d9f641010d96c5e9d5da0b7bf44
3 files changed
+25
-9
src/collectors/freebsd.plugin/plugin_freebsd.c
+3
-3
@@ -109,12 +109,12 @@ void *freebsd_main(void *ptr)
109
heartbeat_t hb;
110
heartbeat_init(&hb);
111
112
- while (!netdata_exit) {
112
+ while(service_running(SERVICE_COLLECTORS)) {
113
worker_is_idle();
114
115
usec_t hb_dt = heartbeat_next(&hb, step);
116
117
- if (unlikely(netdata_exit))
117
+ if (!service_running(SERVICE_COLLECTORS))
118
break;
119
120
for (i = 0; freebsd_modules[i].name; i++) {
@@ -127,7 +127,7 @@ void *freebsd_main(void *ptr)
127
worker_is_busy(i);
128
pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
129
130
- if (unlikely(netdata_exit))
130
+ if (!service_running(SERVICE_COLLECTORS))
131
break;
132
}
133
}
src/collectors/macos.plugin/plugin_macos.c
+5
-2
@@ -58,10 +58,13 @@ void *macos_main(void *ptr)
58
heartbeat_t hb;
59
heartbeat_init(&hb);
60
61
- while (!netdata_exit) {
61
+ while(service_running(SERVICE_COLLECTORS)) {
62
worker_is_idle();
63
usec_t hb_dt = heartbeat_next(&hb, step);
64
65
+ if (!service_running(SERVICE_COLLECTORS))
66
+ break;
67
+
68
for (int i = 0; macos_modules[i].name; i++) {
69
struct macos_module *pm = &macos_modules[i];
70
if (unlikely(!pm->enabled))
@@ -72,7 +75,7 @@ void *macos_main(void *ptr)
75
worker_is_busy(i);
76
pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
77
75
- if (unlikely(netdata_exit))
78
+ if (!service_running(SERVICE_COLLECTORS))
79
break;
80
}
81
}
src/daemon/main.c
+17
-4
@@ -121,7 +121,11 @@ bool service_running(SERVICE_TYPE service) {
121
122
sth->services |= service;
123
124
- return !sth->stop_immediately && !netdata_exit && !nd_thread_signaled_to_cancel();
124
+ bool cancelled = false;
125
+ if (sth->type == SERVICE_THREAD_TYPE_NETDATA)
126
+ cancelled = nd_thread_signaled_to_cancel();
127
+
128
+ return !sth->stop_immediately && !netdata_exit && !cancelled;
129
}
130
131
void service_signal_exit(SERVICE_TYPE service) {
@@ -136,8 +140,16 @@ void service_signal_exit(SERVICE_TYPE service) {
140
if((sth->services & service)) {
141
sth->stop_immediately = true;
142
139
- // this does not harm - it just raises a flag
140
- nd_thread_signal_cancel(sth->netdata_thread);
143
+ switch(sth->type) {
144
+ default:
145
+ case SERVICE_THREAD_TYPE_NETDATA:
146
+ nd_thread_signal_cancel(sth->netdata_thread);
147
+ break;
148
+
149
+ case SERVICE_THREAD_TYPE_EVENT_LOOP:
150
+ case SERVICE_THREAD_TYPE_LIBUV:
151
+ break;
152
+ }
153
154
if(sth->request_quit_callback) {
155
spinlock_unlock(&service_globals.lock);
@@ -332,6 +344,8 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
344
(void) rename(agent_crash_file, agent_incomplete_shutdown_file);
345
watcher_step_complete(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE);
346
347
+ ml_stop_threads();
348
+
349
#ifdef ENABLE_DBENGINE
350
if(dbengine_enabled) {
351
for (size_t tier = 0; tier < storage_tiers; tier++)
@@ -362,7 +376,6 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
376
metadata_sync_shutdown_prepare();
377
watcher_step_complete(WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN);
378
365
- ml_stop_threads();
379
ml_fini();
380
watcher_step_complete(WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS);
381