Improve agent shutdown time (#18434)
Modify datafile acquire for delete timeout if we are shutting down
Stelios Fragkakis committed
Aug 29, 2024 at 17:04 UTC
ceeba615afe7096e97f9e478c381c63ea1f8f340
3 files changed
+13
-7
src/database/engine/datafile.c
+5
-4
@@ -66,7 +66,8 @@ void datafile_release(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS re
66
spinlock_unlock(&df->users.spinlock);
67
}
68
69
-bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
69
+bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdown)
70
+{
71
bool can_be_deleted = false;
72
73
spinlock_lock(&df->users.spinlock);
@@ -107,7 +108,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
108
109
if(!df->users.time_to_evict) {
110
// first time we did the above
110
- df->users.time_to_evict = now_s + 120;
111
+ df->users.time_to_evict = now_s + is_shutdown ? DATAFILE_DELETE_TIMEOUT_SHORT : DATAFILE_DELETE_TIMEOUT_LONG;
112
internal_error(true, "DBENGINE: datafile %u of tier %d is not used by any open cache pages, "
113
"but it has %u lockers (oc:%u, pd:%u), "
114
"%zu clean and %zu hot open cache pages "
@@ -572,8 +573,8 @@ void finalize_data_files(struct rrdengine_instance *ctx)
573
struct rrdengine_journalfile *journalfile = datafile->journalfile;
574
575
logged = false;
575
- size_t iterations = 100;
576
- while(!datafile_acquire_for_deletion(datafile) && datafile != ctx->datafiles.first->prev && --iterations > 0) {
576
+ size_t iterations = 10;
577
+ while(!datafile_acquire_for_deletion(datafile, true) && datafile != ctx->datafiles.first->prev && --iterations > 0) {
578
if(!logged) {
579
netdata_log_info("Waiting to acquire data file %u of tier %d to close it...", datafile->fileno, ctx->config.tier);
580
logged = true;
src/database/engine/datafile.h
+6
-1
@@ -24,6 +24,11 @@ struct rrdengine_instance;
24
#define MAX_DATAFILES (65536 * 4) /* Supports up to 64TiB for now */
25
#define TARGET_DATAFILES (50)
26
27
+// When trying to acquire a datafile for deletion and an attempt to evict pages is completed
28
+// the acquire for deletion will return true after this timeout
29
+#define DATAFILE_DELETE_TIMEOUT_SHORT (1)
30
+#define DATAFILE_DELETE_TIMEOUT_LONG (120)
31
+
32
typedef enum __attribute__ ((__packed__)) {
33
DATAFILE_ACQUIRE_OPEN_CACHE = 0,
34
DATAFILE_ACQUIRE_PAGE_DETAILS,
@@ -72,7 +77,7 @@ struct rrdengine_datafile {
77
78
bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason);
79
void datafile_release(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason);
75
-bool datafile_acquire_for_deletion(struct rrdengine_datafile *df);
80
+bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdown);
81
82
void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool having_lock);
83
void datafile_list_delete_unsafe(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile);
src/database/engine/rrdengine.c
+2
-2
@@ -1218,7 +1218,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *
1218
if(worker)
1219
worker_is_busy(UV_EVENT_DBENGINE_DATAFILE_DELETE_WAIT);
1220
1221
- bool datafile_got_for_deletion = datafile_acquire_for_deletion(datafile);
1221
+ bool datafile_got_for_deletion = datafile_acquire_for_deletion(datafile, false);
1222
1223
if (update_retention)
1224
update_metrics_first_time_s(ctx, datafile, datafile->next, worker);
@@ -1227,7 +1227,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *
1227
if(worker)
1228
worker_is_busy(UV_EVENT_DBENGINE_DATAFILE_DELETE_WAIT);
1229
1230
- datafile_got_for_deletion = datafile_acquire_for_deletion(datafile);
1230
+ datafile_got_for_deletion = datafile_acquire_for_deletion(datafile, false);
1231
1232
if (!datafile_got_for_deletion) {
1233
netdata_log_info("DBENGINE: waiting for data file '%s/"