Fix occasional shutdown deadlock (#16495)
* Wait for RRDENG_OPCODE_CTX_QUIESCE to complete before attempting rrd_finalize_collection_for_all_hosts * Submit RRDENG_OPCODE_CTX_QUIESCE for all tiers and then wait for completion
Stelios Fragkakis committed
Nov 28, 2023 at 20:41 UTC
d6a81a99279f9a2caccdc7d88bb6e50382c55f1b
2 files changed
+7
-3
daemon/main.c
+7
@@ -424,6 +424,13 @@ void netdata_cleanup_and_exit(int ret) {
424
delta_shutdown_time("flush dbengine tiers");
425
for (size_t tier = 0; tier < storage_tiers; tier++)
426
rrdeng_prepare_exit(multidb_ctx[tier]);
427
+
428
+ for (size_t tier = 0; tier < storage_tiers; tier++) {
429
+ if (!multidb_ctx[tier])
430
+ continue;
431
+ completion_wait_for(&multidb_ctx[tier]->quiesce.completion);
432
+ completion_destroy(&multidb_ctx[tier]->quiesce.completion);
433
+ }
434
}
435
#endif
436
database/engine/rrdengine.c
-3
@@ -1345,9 +1345,6 @@ static void after_ctx_shutdown(struct rrdengine_instance *ctx __maybe_unused, vo
1345
static void *ctx_shutdown_tp_worker(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t *uv_work_req __maybe_unused) {
1346
worker_is_busy(UV_EVENT_DBENGINE_SHUTDOWN);
1347
1348
- completion_wait_for(&ctx->quiesce.completion);
1349
- completion_destroy(&ctx->quiesce.completion);
1350
-
1348
bool logged = false;
1349
while(__atomic_load_n(&ctx->atomic.extents_currently_being_flushed, __ATOMIC_RELAXED) ||
1350
__atomic_load_n(&ctx->atomic.inflight_queries, __ATOMIC_RELAXED)) {