Improve shutdown when collectors are active (#16315)
* Collectors should not be running at this point, but allow shutdown to continue after several retries (workaround) * Proceed with shutdown after 10 attempts
Stelios Fragkakis committed
Nov 1, 2023 at 16:22 UTC
ca592a96308915bd939fc20faa7abaed91d619cc
2 files changed
+6
-2
daemon/main.c
+3
-1
@@ -440,7 +440,8 @@ void netdata_cleanup_and_exit(int ret) {
440
delta_shutdown_time("wait for dbengine collectors to finish");
441
442
size_t running = 1;
443
- while(running) {
443
+ size_t count = 10;
444
+ while(running && count) {
445
running = 0;
446
for (size_t tier = 0; tier < storage_tiers; tier++)
447
running += rrdeng_collectors_running(multidb_ctx[tier]);
@@ -451,6 +452,7 @@ void netdata_cleanup_and_exit(int ret) {
452
// sleep_usec(100 * USEC_PER_MS);
453
cleanup_destroyed_dictionaries();
454
}
455
+ count--;
456
}
457
458
delta_shutdown_time("wait for dbengine main cache to finish flushing");
database/engine/rrdengineapi.c
+3
-1
@@ -1241,12 +1241,14 @@ int rrdeng_exit(struct rrdengine_instance *ctx) {
1241
// 4. then wait for completion
1242
1243
bool logged = false;
1244
- while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && !unittest_running) {
1244
+ size_t count = 10;
1245
+ while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && count && !unittest_running) {
1246
if(!logged) {
1247
netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", (ctx->config.legacy) ? -1 : ctx->config.tier);
1248
logged = true;
1249
}
1250
sleep_usec(100 * USEC_PER_MS);
1251
+ count--;
1252
}
1253
1254
netdata_log_info("DBENGINE: flushing main cache for tier %d", (ctx->config.legacy) ? -1 : ctx->config.tier);