Improve event loop thread creation (#19951)
* Retry uv_thread_create if there is a temporary error (EAGAIN) * Mark thread slot as free on thread creation failure
Stelios Fragkakis committed
Mar 24, 2025 at 21:39 UTC
98e77bafd693bf11f43c378b830b3db5d20d312d
5 files changed
+54
-4
src/daemon/libuv_workers.c
+19
@@ -109,3 +109,22 @@ void register_libuv_worker_jobs() {
109
registered = true;
110
register_libuv_worker_jobs_internal();
111
}
112
+
113
+// utils
114
+#define MAX_THREAD_CREATE_RETRIES (10)
115
+#define MAX_THREAD_CREATE_WAIT_MS (1000)
116
+
117
+int create_uv_thread(uv_thread_t *thread, uv_thread_cb thread_func, void *arg, int *retries)
118
+{
119
+ int err;
120
+
121
+ do {
122
+ err = uv_thread_create(thread, thread_func, arg);
123
+ if (err == 0)
124
+ break;
125
+
126
+ uv_sleep(MAX_THREAD_CREATE_WAIT_MS);
127
+ } while (err == UV_EAGAIN && ++(*retries) < MAX_THREAD_CREATE_RETRIES);
128
+
129
+ return err;
130
+}
src/daemon/libuv_workers.h
+1
@@ -88,5 +88,6 @@ enum event_loop_job {
88
};
89
90
void register_libuv_worker_jobs();
91
+int create_uv_thread(uv_thread_t *thread, uv_thread_cb thread_func, void *arg, int *retries);
92
93
#endif //NETDATA_EVENT_LOOP_H
src/database/engine/rrdengine.c
+10
-1
@@ -1804,7 +1804,16 @@ bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {
1804
1805
dbengine_initialize_structures();
1806
1807
- fatal_assert(0 == uv_thread_create(&rrdeng_main.thread, dbengine_event_loop, &rrdeng_main));
1807
+ int retries = 0;
1808
+ int create_uv_thread_rc = create_uv_thread(&rrdeng_main.thread, dbengine_event_loop, &rrdeng_main, &retries);
1809
+ if (create_uv_thread_rc)
1810
+ nd_log_daemon(NDLP_ERR, "Failed to create DBENGINE thread, error %s, after %d retries", uv_err_name(create_uv_thread_rc), retries);
1811
+
1812
+ fatal_assert(0 == create_uv_thread_rc);
1813
+
1814
+ if (retries)
1815
+ nd_log_daemon(NDLP_WARNING, "DBENGINE thread was created after %d attempts", retries);
1816
+
1817
spawned = true;
1818
}
1819
src/database/sqlite/sqlite_aclk.c
+11
-1
@@ -890,7 +890,17 @@ static void aclk_synchronization_init(void)
890
{
891
memset(&aclk_sync_config, 0, sizeof(aclk_sync_config));
892
completion_init(&aclk_sync_config.start_stop_complete);
893
- fatal_assert(0 == uv_thread_create(&aclk_sync_config.thread, aclk_synchronization, &aclk_sync_config));
893
+
894
+ int retries = 0;
895
+ int create_uv_thread_rc = create_uv_thread(&aclk_sync_config.thread, aclk_synchronization, &aclk_sync_config, &retries);
896
+ if (create_uv_thread_rc)
897
+ nd_log_daemon(NDLP_ERR, "Failed to create ACLK synchronization thread, error %s, after %d retries", uv_err_name(create_uv_thread_rc), retries);
898
+
899
+ fatal_assert(0 == create_uv_thread_rc);
900
+
901
+ if (retries)
902
+ nd_log_daemon(NDLP_WARNING, "ACLK synchronization thread was created after %d attempts", retries);
903
+
904
completion_wait_for(&aclk_sync_config.start_stop_complete);
905
completion_destroy(&aclk_sync_config.start_stop_complete);
906
}
src/database/sqlite/sqlite_metadata.c
+13
-2
@@ -1984,8 +1984,11 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)
1984
hclt[thread_index].host = host;
1985
rc = uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]);
1986
async_exec += (rc == 0);
1987
+ // if it failed, mark the thread slot as free
1988
+ if (rc)
1989
+ __atomic_store_n(&hclt[thread_index].busy, false, __ATOMIC_RELAXED);
1990
}
1988
- // if single thread, thread creation failure or failure to find slot
1991
+ // if single thread, thread creation failure or failure tofind slot
1992
if (rc || !thread_found) {
1993
sync_exec++;
1994
struct host_context_load_thread hclt_sync = {.host = host};
@@ -2762,7 +2765,15 @@ void metadata_sync_init(void)
2765
memset(&metasync_worker, 0, sizeof(metasync_worker));
2766
completion_init(&metasync_worker.start_stop_complete);
2767
2765
- fatal_assert(0 == uv_thread_create(&metasync_worker.thread, metadata_event_loop, &metasync_worker));
2768
+ int retries = 0;
2769
+ int create_uv_thread_rc = create_uv_thread(&metasync_worker.thread, metadata_event_loop, &metasync_worker, &retries);
2770
+ if (create_uv_thread_rc)
2771
+ nd_log_daemon(NDLP_ERR, "Failed to create SQLite metadata sync thread, error %s, after %d retries", uv_err_name(create_uv_thread_rc), retries);
2772
+
2773
+ fatal_assert(0 == create_uv_thread_rc);
2774
+
2775
+ if (retries)
2776
+ nd_log_daemon(NDLP_WARNING, "SQLite metadata sync thread was created after %d attempts", retries);
2777
2778
completion_wait_for(&metasync_worker.start_stop_complete);
2779
completion_destroy(&metasync_worker.start_stop_complete);