Improve forced new datafile on startup (#21282)
* Force a new datafile on startup when the last datafile contains data more than a day old * Make sure we have a valid last_time_S
Stelios Fragkakis committed
Nov 10, 2025 at 22:32 UTC
bc1ae857129ce6db04039d498ba2bf36836a74cc
1 file changed
+9
-2
src/database/engine/journalfile.c
+9
-2
@@ -655,6 +655,7 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
655
656
time_t now_s = max_acceptable_collected_time();
657
time_t extent_first_time_s = journalfile->v2.first_time_s ? journalfile->v2.first_time_s : LONG_MAX;
658
+ time_t extent_last_time_s = journalfile->v2.last_time_s ? journalfile->v2.last_time_s : 0;
659
for (i = 0; i < count ; ++i) {
660
nd_uuid_t *temp_id;
661
uint8_t page_type = jf_metric_data->descr[i].type;
@@ -719,11 +720,13 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
720
jf_metric_data->extent_size);
721
722
extent_first_time_s = MIN(extent_first_time_s, vd.start_time_s);
723
+ extent_last_time_s = MAX(extent_last_time_s, vd.end_time_s);
724
725
mrg_metric_release(main_mrg, metric);
726
}
727
728
journalfile->v2.first_time_s = extent_first_time_s;
729
+ journalfile->v2.last_time_s = extent_last_time_s;
730
731
time_t old = __atomic_load_n(&ctx->atomic.first_time_s, __ATOMIC_RELAXED);;
732
do {
@@ -828,7 +831,7 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,
831
832
max_size = pos + size_bytes - pos_i;
833
ret = journalfile_replay_transaction(ctx, journalfile, buf + pos_i, &id, max_size);
831
- if (!ret) /* TODO: support transactions bigger than 4K */
834
+ if (!ret)
835
/* unknown transaction size, move on to the next block */
836
pos_i = ALIGN_BYTES_FLOOR(pos_i + RRDENG_BLOCK_SIZE);
837
else
@@ -1602,7 +1605,11 @@ int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfil
1605
nd_log_daemon(NDLP_DEBUG, "DBENGINE: journal file \"%s\" loaded (size:%" PRIu64 ").", path, file_size);
1606
1607
bool is_last_file = (ctx_last_fileno_get(ctx) == journalfile->datafile->fileno);
1605
- if (is_last_file && journalfile->datafile->pos <= rrdeng_target_data_file_size(ctx) / 3) {
1608
+ bool has_old_data = false;
1609
+ if (ctx->config.tier == 0 && journalfile->v2.last_time_s > 0)
1610
+ has_old_data = (now_realtime_sec() - journalfile->v2.last_time_s) > 86400;
1611
+
1612
+ if (is_last_file && journalfile->datafile->pos <= rrdeng_target_data_file_size(ctx) / 3 && !has_old_data) {
1613
ctx->loading.create_new_datafile_pair = false;
1614
return 0;
1615
}