Terminate cgroups discovery thread faster during shutdown (#16246)
Stelios Fragkakis committed
Oct 20, 2023 at 10:55 UTC
205dc81dc691fa0e587446340086d3b511be1fb5
1 file changed
+27
-36
collectors/cgroups.plugin/sys_fs_cgroup.c
+27
-36
@@ -895,7 +895,6 @@ struct discovery_thread {
895
uv_thread_t thread;
896
uv_mutex_t mutex;
897
uv_cond_t cond_var;
898
- int start_discovery;
898
int exited;
899
} discovery_thread;
900
@@ -2781,14 +2780,6 @@ static inline void discovery_find_all_cgroups() {
2780
netdata_log_debug(D_CGROUP, "done searching for cgroups");
2781
}
2782
2784
-static void cgroup_discovery_cleanup(void *ptr) {
2785
- UNUSED(ptr);
2786
-
2787
- discovery_thread.exited = 1;
2788
- worker_unregister();
2789
- service_exits();
2790
-}
2791
-
2783
static inline char *cgroup_chart_type(char *buffer, struct cgroup *cg) {
2784
if(buffer[0]) return buffer;
2785
@@ -2806,8 +2797,6 @@ void cgroup_discovery_worker(void *ptr)
2797
{
2798
UNUSED(ptr);
2799
2809
- netdata_thread_cleanup_push(cgroup_discovery_cleanup, ptr);
2810
-
2800
worker_register("CGROUPSDISC");
2801
worker_register_job_name(WORKER_DISCOVERY_INIT, "init");
2802
worker_register_job_name(WORKER_DISCOVERY_FIND, "find");
@@ -2827,13 +2816,13 @@ void cgroup_discovery_worker(void *ptr)
2816
NULL,
2817
SIMPLE_PATTERN_EXACT, true);
2818
2819
+ service_register(SERVICE_THREAD_TYPE_LIBUV, NULL, NULL, NULL, false);
2820
+
2821
while (service_running(SERVICE_COLLECTORS)) {
2822
worker_is_idle();
2823
2824
uv_mutex_lock(&discovery_thread.mutex);
2834
- while (!discovery_thread.start_discovery && service_running(SERVICE_COLLECTORS))
2835
- uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex);
2836
- discovery_thread.start_discovery = 0;
2825
+ uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex);
2826
uv_mutex_unlock(&discovery_thread.mutex);
2827
2828
if (unlikely(!service_running(SERVICE_COLLECTORS)))
@@ -2841,8 +2830,10 @@ void cgroup_discovery_worker(void *ptr)
2830
2831
discovery_find_all_cgroups();
2832
}
2844
-
2845
- netdata_thread_cleanup_pop(1);
2833
+ collector_info("discovery thread stopped");
2834
+ worker_unregister();
2835
+ service_exits();
2836
+ __atomic_store_n(&discovery_thread.exited,1,__ATOMIC_RELAXED);
2837
}
2838
2839
// ----------------------------------------------------------------------------
@@ -4569,19 +4560,15 @@ static void cgroup_main_cleanup(void *ptr) {
4560
4561
usec_t max = 2 * USEC_PER_SEC, step = 50000;
4562
4572
- if (!discovery_thread.exited) {
4573
- collector_info("stopping discovery thread worker");
4574
- uv_mutex_lock(&discovery_thread.mutex);
4575
- discovery_thread.start_discovery = 1;
4576
- uv_cond_signal(&discovery_thread.cond_var);
4577
- uv_mutex_unlock(&discovery_thread.mutex);
4578
- }
4579
-
4580
- collector_info("waiting for discovery thread to finish...");
4581
-
4582
- while (!discovery_thread.exited && max > 0) {
4583
- max -= step;
4584
- sleep_usec(step);
4563
+ if (!__atomic_load_n(&discovery_thread.exited, __ATOMIC_RELAXED)) {
4564
+ collector_info("waiting for discovery thread to finish...");
4565
+ while (!__atomic_load_n(&discovery_thread.exited, __ATOMIC_RELAXED) && max > 0) {
4566
+ uv_mutex_lock(&discovery_thread.mutex);
4567
+ uv_cond_signal(&discovery_thread.cond_var);
4568
+ uv_mutex_unlock(&discovery_thread.mutex);
4569
+ max -= step;
4570
+ sleep_usec(step);
4571
+ }
4572
}
4573
4574
if (shm_mutex_cgroup_ebpf != SEM_FAILED) {
@@ -4621,8 +4608,6 @@ void *cgroups_main(void *ptr) {
4608
goto exit;
4609
}
4610
4624
- // dispatch a discovery worker thread
4625
- discovery_thread.start_discovery = 0;
4611
discovery_thread.exited = 0;
4612
4613
if (uv_mutex_init(&discovery_thread.mutex)) {
@@ -4646,6 +4631,7 @@ void *cgroups_main(void *ptr) {
4631
usec_t step = cgroup_update_every * USEC_PER_SEC;
4632
usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0;
4633
4634
+ netdata_thread_disable_cancelability();
4635
while(service_running(SERVICE_COLLECTORS)) {
4636
worker_is_idle();
4637
@@ -4654,8 +4640,9 @@ void *cgroups_main(void *ptr) {
4640
4641
find_dt += hb_dt;
4642
if (unlikely(find_dt >= find_every || (!is_inside_k8s && cgroups_check))) {
4643
+ uv_mutex_lock(&discovery_thread.mutex);
4644
uv_cond_signal(&discovery_thread.cond_var);
4658
- discovery_thread.start_discovery = 1;
4645
+ uv_mutex_unlock(&discovery_thread.mutex);
4646
find_dt = 0;
4647
cgroups_check = 0;
4648
}
@@ -4665,18 +4652,22 @@ void *cgroups_main(void *ptr) {
4652
4653
worker_is_busy(WORKER_CGROUPS_READ);
4654
read_all_discovered_cgroups(cgroup_root);
4668
- if(unlikely(!service_running(SERVICE_COLLECTORS))) break;
4669
-
4655
+ if (unlikely(!service_running(SERVICE_COLLECTORS))) {
4656
+ uv_mutex_unlock(&cgroup_root_mutex);
4657
+ break;
4658
+ }
4659
worker_is_busy(WORKER_CGROUPS_CHART);
4660
update_cgroup_charts(cgroup_update_every);
4672
- if(unlikely(!service_running(SERVICE_COLLECTORS))) break;
4661
+ if (unlikely(!service_running(SERVICE_COLLECTORS))) {
4662
+ uv_mutex_unlock(&cgroup_root_mutex);
4663
+ break;
4664
+ }
4665
4666
worker_is_idle();
4667
uv_mutex_unlock(&cgroup_root_mutex);
4668
}
4669
4670
exit:
4679
- worker_unregister();
4671
netdata_thread_cleanup_pop(1);
4672
return NULL;
4673
}