Improve estimated disk space usage for data file rotation (#20019)
* Add locks to protect the datafile disk space calculation * Avoid queueing RRDENG_OPCODE_DATABASE_ROTATE operations needlessly
Stelios Fragkakis committed
Apr 2, 2025 at 12:13 UTC
9019b5603e8ed27dc7c330692fbb4781e017bf06
3 files changed
+21
-10
src/daemon/pulse/pulse-db-dbengine-retention.c
+1
-1
@@ -56,7 +56,7 @@ void dbengine_retention_statistics(bool extended __maybe_unused) {
56
// get_used_disk_space is used to determine if database cleanup (file rotation should happen)
57
// and adds to the disk space used the desired file size of the active
58
// datafile
59
- uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier]);
59
+ uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier], false);
60
//uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
61
62
uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
src/database/engine/rrdengine.c
+19
-8
@@ -913,7 +913,7 @@ static void *extent_write_tp_worker(
913
914
extent_flush_to_open(ctx, xt_io_descr, df_write_error);
915
916
- if(ctx_is_available_for_queries(ctx))
916
+ if(ctx_is_available_for_queries(ctx) && rrdeng_ctx_tier_cap_exceeded(ctx))
917
rrdeng_enq_cmd(ctx, RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);
918
done:
919
if(completion)
@@ -1655,13 +1655,22 @@ static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, vo
1655
return data;
1656
}
1657
1658
-uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx)
1658
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx, bool having_lock)
1659
{
1660
uint64_t active_space = 0;
1661
1662
+ if (!having_lock)
1663
+ uv_rwlock_rdlock(&ctx->datafiles.rwlock);
1664
+
1665
if (ctx->datafiles.first && ctx->datafiles.first->prev)
1666
active_space = ctx->datafiles.first->prev->pos;
1667
1668
+ if (!having_lock)
1669
+ uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
1670
+
1671
+ // calculate the estimated disk space based on the expected final size of the datafile
1672
+ // We cant know the final v1/v2 journal size -- we let the current v1 size be part of the calculation by not
1673
+ // including it in the active_space
1674
uint64_t estimated_disk_space = ctx_current_disk_space_get(ctx) + rrdeng_target_data_file_size(ctx) - active_space;
1675
1676
uint64_t database_space = get_total_database_space();
@@ -1688,15 +1697,17 @@ static time_t get_tier_retention(struct rrdengine_instance *ctx)
1697
// Check if disk or retention time cap reached
1698
bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx)
1699
{
1691
- if(!ctx->datafiles.first)
1692
- // no datafiles available
1693
- return false;
1700
1695
- if(!ctx->datafiles.first->next)
1696
- // only 1 datafile available
1701
+ uv_rwlock_rdlock(&ctx->datafiles.rwlock);
1702
+ if (!ctx->datafiles.first || !ctx->datafiles.first->next) {
1703
+ uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
1704
return false;
1705
+ }
1706
+
1707
+ uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx, true);
1708
+
1709
+ uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
1710
1699
- uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx);
1711
time_t retention = get_tier_retention(ctx);
1712
1713
if (ctx->config.max_retention_s && retention > ctx->config.max_retention_s)
src/database/engine/rrdengine.h
+1
-1
@@ -546,7 +546,7 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri
546
}
547
548
// --------------------------------------------------------------------------------------------------------------------
549
-uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx);
549
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx, bool having_lock);
550
void rrdeng_calculate_tier_disk_space_percentage(void);
551
uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx);
552
void dbengine_shutdown();