@cryptotaxi247 / netdata-1 / commits / 1aa8a3bb1

Support time based retention (#17413)

* Add charts Rename options Global backfill Metadata calculation (percentage) Retention timer Adjust time Calculate iterations Update every is less than 60 and divisor of 60 * Add option to specify time retention dbengine tier x retention days Switch to dbengine tier x disk space MB Fix grouping iterations * Allow disk space to be 0 (unlimited) Assume retention to be the one specified with time Sane value for maximum datafile target size if max disk space is 0 * Test configured disk space to be total free space - 10% Rework human readable retention and expected retention in nodes_instances api * Further fix on human readable retention If no time restriction is specified, use disk space calculated one * Remove dbengine parallel initialization option Remove duplicate, commented out code * Allow tier disk space to be 0 in which case the current disk space available will be used for calculations * Proper calculation of iterations Support seconds in human duration representation Remove commented out code Create tiers as needed * Function to return sqlite database space Compile with disable-ml properly Do proper time retention check Temporary additional info in node_instances api * Do not account for metadata size if old settings are detected * Consider regacy tier multihost disk space MB settings as well * Adjust time retention calculation * Cleanup / allow 5% free disk space when all disk is to be used just to be safe * Update some defaults * Use default 1024 MB for each tier Add new dbengine tier 0 multihost disk space MB Time based retention defaults to disabled if all parameters are default * Update src/database/engine/rrdengine.c Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud> * Switch retention days to integer * Respect "dbengine multihost disk space MB" setting for tier 0 if "dbengine tier 0 multihost disk space MB" is not specified * Add dbengine_tier label * Fix retention_percentage check * Change to tier * Time % calculation includes first datafile * Add disk space used by metadata in node_instances Calculated disk space matches the calculated used for datafile rotation Fix percentage contribution calculation of metadata to each tier * Revent changes to node_instances API * Do not take into account metadata diskspace usage for now * Add note for the disk space usage calculation in dbengine_tier_retention charts vs space reported in /api/v2/node_instances * Restore update every checks RRDENG_MIN_DISK_SPACE_MB 256 MB (from 64) dbengine multihost disk space MB maps to dbengine tier 0 disk space MB Disk space for tiers "dbengine tier X disk space MB" * Check backfill option before tier disk space config to reorder it in netdata.conf output * Check tier iteration option before the disk space config per tier * Check if legacy dbengine multihost disk space MB option is set before migration of options Rename options * Fix warning * Allow dbengine tier 0 disk space MB to be zero and not default to 256 MB. If the value is non zero for tier 0, it must be at least 256 MB * Check for options in global section --------- Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>

Stelios Fragkakis committed Jun 7, 2024 at 16:12 UTC 1aa8a3bb14e80720fdd186cea74d7574f5f62ac1

18 files changed +493 -159

src/daemon/main.c

+42 -14

@@ -1043,6 +1043,24 @@ static void backwards_compatible_config() {

1043 config_move(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space",

1044 CONFIG_SECTION_DB, "dbengine multihost disk space MB");

1045

1046 + config_move(CONFIG_SECTION_DB, "dbengine disk space MB",

1047 + CONFIG_SECTION_DB, "dbengine multihost disk space MB");

1048 +

1049 + config_move(CONFIG_SECTION_DB, "dbengine multihost disk space MB",

1050 + CONFIG_SECTION_DB, "dbengine tier 0 disk space MB");

1051 +

1052 + config_move(CONFIG_SECTION_DB, "dbengine tier 1 multihost disk space MB",

1053 + CONFIG_SECTION_DB, "dbengine tier 1 disk space MB");

1054 +

1055 + config_move(CONFIG_SECTION_DB, "dbengine tier 2 multihost disk space MB",

1056 + CONFIG_SECTION_DB, "dbengine tier 2 disk space MB");

1057 +

1058 + config_move(CONFIG_SECTION_DB, "dbengine tier 3 multihost disk space MB",

1059 + CONFIG_SECTION_DB, "dbengine tier 3 disk space MB");

1060 +

1061 + config_move(CONFIG_SECTION_DB, "dbengine tier 4 multihost disk space MB",

1062 + CONFIG_SECTION_DB, "dbengine tier 4 disk space MB");

1063 +

1064 config_move(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)",

1065 CONFIG_SECTION_DB, "memory deduplication (ksm)");

1066

@@ -1099,7 +1117,14 @@ static int get_hostname(char *buf, size_t buf_size) {

1117 return gethostname(buf, buf_size);

1118 }

1119

1102 -static void get_netdata_configured_variables() {

1120 +static void get_netdata_configured_variables()

1121 +{

1122 + legacy_multihost_db_space = config_exists(CONFIG_SECTION_DB, "dbengine multihost disk space MB");

1123 + if (!legacy_multihost_db_space)

1124 + legacy_multihost_db_space = config_exists(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space");

1125 + if (!legacy_multihost_db_space)

1126 + legacy_multihost_db_space = config_exists(CONFIG_SECTION_GLOBAL, "dbengine disk space");

1127 +

1128 backwards_compatible_config();

1129

1130 // ------------------------------------------------------------------------

@@ -1201,20 +1226,23 @@ static void get_netdata_configured_variables() {

1226

1227 // ------------------------------------------------------------------------

1228 // get default Database Engine disk space quota in MiB

1229 +//

1230 +// // if (!config_exists(CONFIG_SECTION_DB, "dbengine disk space MB") && !config_exists(CONFIG_SECTION_DB, "dbengine multihost disk space MB"))

1231 +//

1232 +// default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);

1233 +// if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {

1234 +// netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);

1235 +// default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;

1236 +// config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);

1237 +// }

1238 +//

1239 +// default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());

1240 +// if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {

1241 +// netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);

1242 +// default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;

1243 +// config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);

1244 +// }

1245

1205 - default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);

1206 - if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {

1207 - netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);

1208 - default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;

1209 - config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);

1210 - }

1211 -

1212 - default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());

1213 - if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {

1214 - netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);

1215 - default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;

1216 - config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);

1217 - }

1246 #else

1247 if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

1248 error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");

src/daemon/service.c

@@ -314,6 +314,8 @@ void *service_main(void *ptr)

314 }

315 real_step = USEC_PER_SEC;

316

317 + dbengine_retention_statistics();

318 +

319 svc_rrd_cleanup_obsolete_charts_from_all_hosts();

320

321 if (service_running(SERVICE_MAINTENANCE))

src/database/contexts/api_v2.c

+53 -4

@@ -1124,6 +1124,30 @@ void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings

1124

1125 void build_info_to_json_object(BUFFER *b);

1126

1127 +static void convert_seconds_to_dhms(time_t seconds, char *result, int result_size) {

1128 + int days, hours, minutes;

1129 +

1130 + days = (int) (seconds / (24 * 3600));

1131 + seconds = (int) (seconds % (24 * 3600));

1132 + hours = (int) (seconds / 3600);

1133 + seconds %= 3600;

1134 + minutes = (int) (seconds / 60);

1135 + seconds %= 60;

1136 +

1137 + // Format the result into the provided string buffer

1138 + BUFFER *buf = buffer_create(128, NULL);

1139 + if (days)

1140 + buffer_sprintf(buf,"%d day%s%s", days, days==1 ? "" : "s", hours || minutes ? ", " : "");

1141 + if (hours)

1142 + buffer_sprintf(buf,"%d hour%s%s", hours, hours==1 ? "" : "s", minutes ? ", " : "");

1143 + if (minutes)

1144 + buffer_sprintf(buf,"%d minute%s%s", minutes, minutes==1 ? "" : "s", seconds ? ", " : "");

1145 + if (seconds)

1146 + buffer_sprintf(buf,"%d second%s", (int) seconds, seconds==1 ? "" : "s");

1147 + strncpyz(result, buffer_tostring(buf), result_size);

1148 + buffer_free(buf);

1149 +}

1150 +

1151 void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now_s, bool info, bool array) {

1152 if(!now_s)

1153 now_s = now_realtime_sec();

@@ -1151,11 +1175,15 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now

1175 buffer_json_cloud_status(wb, now_s);

1176

1177 buffer_json_member_add_array(wb, "db_size");

1178 + size_t group_seconds = localhost->rrd_update_every;

1179 for (size_t tier = 0; tier < storage_tiers; tier++) {

1180 STORAGE_ENGINE *eng = localhost->db[tier].eng;

1181 if (!eng) continue;

1182

1183 + group_seconds *= storage_tiers_grouping_iterations[tier];

1184 uint64_t max = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);

1185 + if (!max)

1186 + max = get_directory_free_bytes_space(multidb_ctx[tier]);

1187 uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);

1188 time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);

1189 size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si);

@@ -1168,6 +1196,10 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now

1196

1197 buffer_json_add_array_item_object(wb);

1198 buffer_json_member_add_uint64(wb, "tier", tier);

1199 + char human_retention[128];

1200 + convert_seconds_to_dhms((time_t) group_seconds, human_retention, sizeof(human_retention) - 1);

1201 + buffer_json_member_add_string(wb, "point_every", human_retention);

1202 +

1203 buffer_json_member_add_uint64(wb, "metrics", storage_engine_metrics(eng->seb, localhost->db[tier].si));

1204 buffer_json_member_add_uint64(wb, "samples", storage_engine_samples(eng->seb, localhost->db[tier].si));

1205

@@ -1178,13 +1210,30 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now

1210 }

1211

1212 if(first_time_s) {

1213 + time_t retention = now_s - first_time_s;

1214 +

1215 buffer_json_member_add_time_t(wb, "from", first_time_s);

1216 buffer_json_member_add_time_t(wb, "to", now_s);

1183 - buffer_json_member_add_time_t(wb, "retention", now_s - first_time_s);

1217 + buffer_json_member_add_time_t(wb, "retention", retention);

1218 +

1219 + convert_seconds_to_dhms(retention, human_retention, sizeof(human_retention) - 1);

1220 + buffer_json_member_add_string(wb, "retention_human", human_retention);

1221

1185 - if(used || max) // we have disk space information

1186 - buffer_json_member_add_time_t(wb, "expected_retention",

1187 - (time_t) ((NETDATA_DOUBLE) (now_s - first_time_s) * 100.0 / percent));

1222 + if(used || max) { // we have disk space information

1223 + time_t time_retention = multidb_ctx[tier]->config.max_retention_s;

1224 + time_t space_retention = (time_t)((NETDATA_DOUBLE)(now_s - first_time_s) * 100.0 / percent);

1225 + time_t actual_retention = MIN(space_retention, time_retention ? time_retention : space_retention);

1226 +

1227 + if (time_retention) {

1228 + convert_seconds_to_dhms(time_retention, human_retention, sizeof(human_retention) - 1);

1229 + buffer_json_member_add_time_t(wb, "requested_retention", time_retention);

1230 + buffer_json_member_add_string(wb, "requested_retention_human", human_retention);

1231 + }

1232 +

1233 + convert_seconds_to_dhms(actual_retention, human_retention, sizeof(human_retention) - 1);

1234 + buffer_json_member_add_time_t(wb, "expected_retention", actual_retention);

1235 + buffer_json_member_add_string(wb, "expected_retention_human", human_retention);

1236 + }

1237 }

1238

1239 if(currently_collected_metrics)

src/database/engine/datafile.c

+1 -1

@@ -543,7 +543,7 @@ int init_data_files(struct rrdengine_instance *ctx)

543 if (ctx->loading.create_new_datafile_pair)

544 create_new_datafile_pair(ctx, false);

545

546 - while(rrdeng_ctx_exceeded_disk_quota(ctx))

546 + while(rrdeng_ctx_tier_cap_exceeded(ctx))

547 datafile_delete(ctx, ctx->datafiles.first, false, false);

548 }

549

src/database/engine/journalfile.c

+11

@@ -669,6 +669,7 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,

669 }

670

671 time_t now_s = max_acceptable_collected_time();

672 + time_t extent_first_time_s = journalfile->v2.first_time_s ? journalfile->v2.first_time_s : LONG_MAX;

673 for (i = 0; i < count ; ++i) {

674 nd_uuid_t *temp_id;

675 uint8_t page_type = jf_metric_data->descr[i].type;

@@ -728,8 +729,18 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,

729 journalfile->datafile,

730 jf_metric_data->extent_offset, jf_metric_data->extent_size, jf_metric_data->descr[i].page_length);

731

732 + extent_first_time_s = MIN(extent_first_time_s, vd.start_time_s);

733 +

734 mrg_metric_release(main_mrg, metric);

735 }

736 +

737 + journalfile->v2.first_time_s = extent_first_time_s;

738 +

739 + time_t old = __atomic_load_n(&ctx->atomic.first_time_s, __ATOMIC_RELAXED);;

740 + do {

741 + if(old <= extent_first_time_s)

742 + break;

743 + } while(!__atomic_compare_exchange_n(&ctx->atomic.first_time_s, &old, extent_first_time_s, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));

744 }

745

746 /*

src/database/engine/pagecache.h

-9

@@ -33,15 +33,6 @@ struct page_descr_with_data {

33 } link;

34 };

36 -#define PAGE_INFO_SCRATCH_SZ (8)

37 -struct rrdeng_page_info {

38 - uint8_t scratch[PAGE_INFO_SCRATCH_SZ]; /* scratch area to be used by page-cache users */

39 -

40 - usec_t start_time_ut;

41 - usec_t end_time_ut;

42 - uint32_t page_length;

43 -};

44 -

36 struct pg_alignment {

37 uint32_t refcount;

38 };

src/database/engine/rrdengine.c

+198 -19

@@ -40,6 +40,7 @@ struct rrdeng_main {

40 uv_loop_t loop;

41 uv_async_t async;

42 uv_timer_t timer;

43 + uv_timer_t retention_timer;

44 pid_t tid;

45 bool shutdown;

@@ -115,12 +116,6 @@ static void sanity_check(void)

116

117 /* page count must fit in 8 bits */

118 BUILD_BUG_ON(MAX_PAGES_PER_EXTENT > 255);

118 -

119 - /* extent cache count must fit in 32 bits */

120 -// BUILD_BUG_ON(MAX_CACHED_EXTENTS > 32);

121 -

122 - /* page info scratch space must be able to hold 2 32-bit integers */

123 - BUILD_BUG_ON(sizeof(((struct rrdeng_page_info *)0)->scratch) < 2 * sizeof(uint32_t));

119 }

120

121 // ----------------------------------------------------------------------------

@@ -1294,7 +1289,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *

1289 static void *database_rotate_tp_worker(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t *uv_work_req __maybe_unused) {

1290 datafile_delete(ctx, ctx->datafiles.first, ctx_is_available_for_queries(ctx), true);

1291

1297 - if (rrdeng_ctx_exceeded_disk_quota(ctx))

1292 + if (rrdeng_ctx_tier_cap_exceeded(ctx))

1293 rrdeng_enq_cmd(ctx, RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);

1294

1295 rrdcontext_db_rotation();

@@ -1403,26 +1398,27 @@ static void *query_prep_tp_worker(struct rrdengine_instance *ctx __maybe_unused,

1398 }

1399

1400 uint64_t rrdeng_target_data_file_size(struct rrdengine_instance *ctx) {

1406 - uint64_t target_size = ctx->config.max_disk_space / TARGET_DATAFILES;

1401 + uint64_t target_size = ctx->config.max_disk_space ? ctx->config.max_disk_space / TARGET_DATAFILES : MAX_DATAFILE_SIZE;

1402 target_size = MIN(target_size, MAX_DATAFILE_SIZE);

1403 target_size = MAX(target_size, MIN_DATAFILE_SIZE);

1404 return target_size;

1405 }

1406

1412 -bool rrdeng_ctx_exceeded_disk_quota(struct rrdengine_instance *ctx)

1407 +time_t get_datafile_end_time(struct rrdengine_instance *ctx)

1408 {

1414 - if(!ctx->datafiles.first)

1415 - // no datafiles available

1416 - return false;

1409 + time_t last_time_s = 0;

1410

1418 - if(!ctx->datafiles.first->next)

1419 - // only 1 datafile available

1420 - return false;

1411 + uv_rwlock_rdlock(&ctx->datafiles.rwlock);

1412 + struct rrdengine_datafile *datafile = ctx->datafiles.first;

1413

1422 - uint64_t estimated_disk_space = ctx_current_disk_space_get(ctx) + rrdeng_target_data_file_size(ctx) -

1423 - (ctx->datafiles.first->prev ? ctx->datafiles.first->prev->pos : 0);

1414 + if (datafile) {

1415 + last_time_s = datafile->journalfile->v2.last_time_s;

1416 + if (!last_time_s)

1417 + last_time_s = datafile->journalfile->v2.first_time_s;

1418 + }

1419

1425 - return estimated_disk_space > ctx->config.max_disk_space;

1420 + uv_rwlock_rdunlock(&ctx->datafiles.rwlock);

1421 + return last_time_s;

1422 }

1423

1424 /* return 0 on success */

@@ -1593,6 +1589,74 @@ static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, vo

1589 return data;

1590 }

1591

1592 +uint64_t get_used_disk_space(struct rrdengine_instance *ctx)

1593 +{

1594 + uint64_t active_space = 0;

1595 +

1596 + if (ctx->datafiles.first && ctx->datafiles.first->prev)

1597 + active_space = ctx->datafiles.first->prev->pos;

1598 +

1599 + uint64_t estimated_disk_space = ctx_current_disk_space_get(ctx) + rrdeng_target_data_file_size(ctx) - active_space;

1600 +

1601 + uint64_t database_space = get_total_database_space();

1602 + uint64_t adjusted_database_space = database_space * ctx->config.disk_percentage / 100 ;

1603 + estimated_disk_space += adjusted_database_space;

1604 +

1605 + return estimated_disk_space;

1606 +}

1607 +

1608 +static time_t get_tier_retention(struct rrdengine_instance *ctx)

1609 +{

1610 + time_t retention = 0;

1611 + if (localhost) {

1612 + STORAGE_ENGINE *eng = localhost->db[ctx->config.tier].eng;

1613 + if (eng) {

1614 + time_t first_time_s = get_datafile_end_time(ctx);

1615 + if (first_time_s)

1616 + retention = now_realtime_sec() - first_time_s;

1617 + }

1618 + }

1619 + return retention;

1620 +}

1621 +

1622 +// Check if disk or retention time cap reached

1623 +bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx)

1624 +{

1625 + if(!ctx->datafiles.first)

1626 + // no datafiles available

1627 + return false;

1628 +

1629 + if(!ctx->datafiles.first->next)

1630 + // only 1 datafile available

1631 + return false;

1632 +

1633 + uint64_t estimated_disk_space = get_used_disk_space(ctx);

1634 + time_t retention = get_tier_retention(ctx);

1635 +

1636 + if (ctx->config.max_retention_s && retention > ctx->config.max_retention_s)

1637 + return true;

1638 +

1639 + if (ctx->config.max_disk_space && estimated_disk_space > ctx->config.max_disk_space)

1640 + return true;

1641 +

1642 + return false;

1643 +}

1644 +

1645 +void retention_timer_cb(uv_timer_t *handle)

1646 +{

1647 + worker_is_busy(RRDENG_TIMER_CB);

1648 + uv_stop(handle->loop);

1649 + uv_update_time(handle->loop);

1650 +

1651 + for (size_t tier = 0; tier < storage_tiers; tier++) {

1652 + bool cleanup = rrdeng_ctx_tier_cap_exceeded(multidb_ctx[tier]);

1653 + if (cleanup)

1654 + rrdeng_enq_cmd(multidb_ctx[tier], RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);

1655 + }

1656 +

1657 + worker_is_idle();

1658 +}

1659 +

1660 void timer_cb(uv_timer_t* handle) {

1661 worker_is_busy(RRDENG_TIMER_CB);

1662 uv_stop(handle->loop);

@@ -1656,7 +1720,17 @@ bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {

1720 fatal_assert(0 == uv_loop_close(&rrdeng_main.loop));

1721 return false;

1722 }

1723 +

1724 + ret = uv_timer_init(&rrdeng_main.loop, &rrdeng_main.retention_timer);

1725 + if (ret) {

1726 + netdata_log_error("DBENGINE: uv_timer_init(): %s", uv_strerror(ret));

1727 + uv_close((uv_handle_t *)&rrdeng_main.async, NULL);

1728 + fatal_assert(0 == uv_loop_close(&rrdeng_main.loop));

1729 + return false;

1730 + }

1731 +

1732 rrdeng_main.timer.data = &rrdeng_main;

1733 + rrdeng_main.retention_timer.data = &rrdeng_main;

1734

1735 dbengine_initialize_structures();

1736

@@ -1688,6 +1762,106 @@ static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_w

1762 work_dispatch(ctx, pdc, NULL, cmd.opcode, query_prep_tp_worker, NULL);

1763 }

1764

1765 +uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)

1766 +{

1767 + uint64_t free_bytes = 0;

1768 + struct statvfs buff_statvfs;

1769 + if (statvfs(ctx->config.dbfiles_path, &buff_statvfs) == 0)

1770 + free_bytes = buff_statvfs.f_bavail * buff_statvfs.f_bsize;

1771 +

1772 + return (free_bytes - (free_bytes * 5 / 100));

1773 +}

1774 +

1775 +void calculate_tier_disk_space_percentage(void)

1776 +{

1777 + static uint64_t tier_space[RRD_STORAGE_TIERS];

1778 +

1779 + uint64_t total_diskspace = 0;

1780 + for(size_t tier = 0; tier < storage_tiers ;tier++) {

1781 + uint64_t tier_disk_space = multidb_ctx[tier]->config.max_disk_space ?

1782 + multidb_ctx[tier]->config.max_disk_space :

1783 + get_directory_free_bytes_space(multidb_ctx[tier]);

1784 + total_diskspace += tier_disk_space;

1785 + tier_space[tier] = tier_disk_space;

1786 + }

1787 +

1788 + if (total_diskspace) {

1789 + for (size_t tier = 0; tier < storage_tiers; tier++) {

1790 + multidb_ctx[tier]->config.disk_percentage = (100 * tier_space[tier] / total_diskspace);

1791 + }

1792 + }

1793 +}

1794 +

1795 +void dbengine_retention_statistics(void)

1796 +{

1797 + static bool init = false;

1798 + static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];

1799 +

1800 + calculate_tier_disk_space_percentage();

1801 +

1802 + for (size_t tier = 0; tier < storage_tiers; tier++) {

1803 + if (init == false) {

1804 + char id[200];

1805 + snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);

1806 + stats[tier].st = rrdset_create_localhost(

1807 + "netdata",

1808 + id,

1809 + NULL,

1810 + "dbengine",

1811 + "netdata.dbengine_tier_retention",

1812 + "dbengine space and time retention",

1813 + "%",

1814 + "netdata",

1815 + "stats",

1816 + 200000,

1817 + 10,

1818 + RRDSET_TYPE_LINE);

1819 +

1820 + stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

1821 + stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

1822 +

1823 + char tier_str[5];

1824 + snprintfz(tier_str, 4, "%zu", tier);

1825 + rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);

1826 +

1827 + rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);

1828 + rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);

1829 + rrdset_metadata_updated(stats[tier].st);

1830 + }

1831 +

1832 + STORAGE_ENGINE *eng = localhost->db[tier].eng;

1833 + time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);

1834 + time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;

1835 +

1836 + //

1837 + // Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances

1838 + // get_used_disk_space is used to determine if database cleanup (file rotation should happen)

1839 + // and adds to the disk space used the desired file size of the active

1840 + // datafile

1841 + uint64_t disk_space = get_used_disk_space(multidb_ctx[tier]);

1842 + //uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);

1843 +

1844 + uint64_t config_disk_space = multidb_ctx[tier]->config.max_disk_space;

1845 + if (!config_disk_space)

1846 + config_disk_space = get_directory_free_bytes_space(multidb_ctx[tier]);

1847 +

1848 + collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);

1849 +

1850 + collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?

1851 + 100 * retention / multidb_ctx[tier]->config.max_retention_s :

1852 + 0;

1853 +

1854 + if (retention_percentage > 100)

1855 + retention_percentage = 100;

1856 +

1857 + rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);

1858 + rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);

1859 +

1860 + rrdset_done(stats[tier].st);

1861 + }

1862 + init = true;

1863 +}

1864 +

1865 void dbengine_event_loop(void* arg) {

1866 sanity_check();

1867 uv_thread_set_name_np("DBENGINE");

@@ -1737,6 +1911,7 @@ void dbengine_event_loop(void* arg) {

1911 main->tid = gettid_cached();

1912

1913 fatal_assert(0 == uv_timer_start(&main->timer, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));

1914 + fatal_assert(0 == uv_timer_start(&main->retention_timer, retention_timer_cb, TIMER_PERIOD_MS * 60, TIMER_PERIOD_MS * 60));

1915

1916 bool shutdown = false;

1917 while (likely(!shutdown)) {

@@ -1817,7 +1992,7 @@ void dbengine_event_loop(void* arg) {

1992 if (!__atomic_load_n(&ctx->atomic.now_deleting_files, __ATOMIC_RELAXED) &&

1993 ctx->datafiles.first->next != NULL &&

1994 ctx->datafiles.first->next->next != NULL &&

1820 - rrdeng_ctx_exceeded_disk_quota(ctx)) {

1995 + rrdeng_ctx_tier_cap_exceeded(ctx)) {

1996

1997 __atomic_store_n(&ctx->atomic.now_deleting_files, true, __ATOMIC_RELAXED);

1998 work_dispatch(ctx, NULL, NULL, opcode, database_rotate_tp_worker, after_database_rotate);

@@ -1854,7 +2029,11 @@ void dbengine_event_loop(void* arg) {

2029 uv_close((uv_handle_t *)&main->async, NULL);

2030 (void) uv_timer_stop(&main->timer);

2031 uv_close((uv_handle_t *)&main->timer, NULL);

2032 +

2033 + (void) uv_timer_stop(&main->retention_timer);

2034 + uv_close((uv_handle_t *)&main->retention_timer, NULL);

2035 shutdown = true;

2036 + break;

2037 }

2038

2039 case RRDENG_OPCODE_NOOP: {

src/database/engine/rrdengine.h

+28 -9

@@ -33,6 +33,12 @@ struct rrdeng_cmd;

33 #define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u"

34 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"

36 +typedef struct dbengine_tier_stats {

37 + RRDSET *st;

38 + RRDDIM *rd_space;

39 + RRDDIM *rd_time;

40 +} DBENGINE_TIER_STATS;

41 +

42 typedef enum __attribute__ ((__packed__)) {

43 // final status for all pages

44 // if a page does not have one of these, it is considered unroutable

@@ -346,16 +352,25 @@ extern rrdeng_stats_t rrdeng_reserved_file_descriptors;

352 extern rrdeng_stats_t global_pg_cache_over_half_dirty_events;

353 extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */

354

349 -struct rrdengine_instance {

350 - struct {

351 - int tier; // the tier of this ctx

352 - uint8_t page_type; // default page type for this context

355 +typedef struct tier_config_prototype {

356 + int tier; // the tier of this ctx

357 + uint8_t page_type; // default page type for this context

358 + uint64_t max_disk_space; // the max disk space this ctx is allowed to use

359 + time_t max_retention_s; // The max retention in seconds

360 + uint8_t disk_percentage; // percentage of metadata that contribute towards tier space used

361 + uint8_t global_compress_alg; // the wanted compression algorithm

362 + char dbfiles_path[FILENAME_MAX + 1];

363

354 - uint64_t max_disk_space; // the max disk space this ctx is allowed to use

355 - uint8_t global_compress_alg; // the wanted compression algorithm

364 + struct {

365 + uint32_t uses;

366 + bool enabled;

367 + bool is_on_disk;

368 + SPINLOCK spinlock;

369 + } _internal;

370 +} TIER_CONFIG_PROTOTYPE;

371

357 - char dbfiles_path[FILENAME_MAX + 1];

358 - } config;

372 +struct rrdengine_instance {

373 + TIER_CONFIG_PROTOTYPE config;

374

375 struct {

376 uv_rwlock_t rwlock; // the linked list of datafiles is protected by this lock

@@ -448,7 +463,7 @@ static inline void ctx_last_flush_fileno_set(struct rrdengine_instance *ctx, uns

463 void *dbengine_extent_alloc(size_t size);

464 void dbengine_extent_free(void *extent, size_t size);

465

451 -bool rrdeng_ctx_exceeded_disk_quota(struct rrdengine_instance *ctx);

466 +bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx);

467 int init_rrd_files(struct rrdengine_instance *ctx);

468 void finalize_rrd_files(struct rrdengine_instance *ctx);

469 bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx);

@@ -536,5 +551,9 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri

551 }

552

553 // --------------------------------------------------------------------------------------------------------------------

554 +uint64_t get_used_disk_space(struct rrdengine_instance *ctx);

555 +void calculate_tier_disk_space_percentage(void);

556 +void dbengine_retention_statistics(void);

557 +uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx);

558

559 #endif /* NETDATA_RRDENGINE_H */

src/database/engine/rrdengineapi.c

+22 -7

@@ -26,8 +26,10 @@ uint8_t tier_page_type[RRD_STORAGE_TIERS] = {

27 #if defined(ENV32BIT)

28 size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192};

29 +size_t tier_quota_mb[RRD_STORAGE_TIERS] = {512, 512, 512, 0, 0};

30 #else

31 size_t tier_page_size[RRD_STORAGE_TIERS] = {4096, 2048, 384, 384, 384};

32 +size_t tier_quota_mb[RRD_STORAGE_TIERS] = {1024, 1024, 1024, 128, 64};

33 #endif

35 #if RRDENG_PAGE_TYPE_MAX != 2

@@ -58,8 +60,11 @@ __attribute__((constructor)) void initialize_multidb_ctx(void) {

60 }

62 int db_engine_journal_check = 0;

61 -int default_rrdeng_disk_quota_mb = 256;

62 -int default_multidb_disk_quota_mb = 256;

63 +bool new_dbengine_defaults = false;

64 +bool legacy_multihost_db_space = false;

65 +int default_rrdeng_disk_quota_mb = RRDENG_DEFAULT_TIER_DISK_SPACE_MB;

66 +int default_multidb_disk_quota_mb = RRDENG_DEFAULT_TIER_DISK_SPACE_MB;

67 +RRD_BACKFILL default_backfill = RRD_BACKFILL_NEW;

69 #if defined(ENV32BIT)

70 int default_rrdeng_page_cache_mb = 16;

@@ -1129,8 +1134,13 @@ void rrdeng_exit_mode(struct rrdengine_instance *ctx) {

1134 /*

1135 * Returns 0 on success, negative on error

1136 */

1132 -int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,

1133 - unsigned disk_space_mb, size_t tier) {

1137 +int rrdeng_init(

1138 + struct rrdengine_instance **ctxp,

1139 + const char *dbfiles_path,

1140 + unsigned disk_space_mb,

1141 + size_t tier,

1142 + time_t max_retention_s)

1143 +{

1144 struct rrdengine_instance *ctx;

1145 uint32_t max_open_files;

1146

@@ -1159,12 +1169,17 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,

1169 ctx->config.tier = (int)tier;

1170 ctx->config.page_type = tier_page_type[tier];

1171 ctx->config.global_compress_alg = dbengine_default_compression();

1162 - if (disk_space_mb < RRDENG_MIN_DISK_SPACE_MB)

1163 - disk_space_mb = RRDENG_MIN_DISK_SPACE_MB;

1164 - ctx->config.max_disk_space = disk_space_mb * 1048576LLU;

1172 +

1173 strncpyz(ctx->config.dbfiles_path, dbfiles_path, sizeof(ctx->config.dbfiles_path) - 1);

1174 ctx->config.dbfiles_path[sizeof(ctx->config.dbfiles_path) - 1] = '\0';

1175

1176 + if (disk_space_mb && disk_space_mb < RRDENG_MIN_DISK_SPACE_MB)

1177 + disk_space_mb = RRDENG_MIN_DISK_SPACE_MB;

1178 +

1179 + ctx->config.max_disk_space = disk_space_mb * 1048576LLU;

1180 +

1181 + ctx->config.max_retention_s = max_retention_s;

1182 +

1183 ctx->atomic.transaction_id = 1;

1184 ctx->quiesce.enabled = false;

1185

src/database/engine/rrdengineapi.h

+13 -3

@@ -6,7 +6,8 @@

6 #include "rrdengine.h"

8 #define RRDENG_MIN_PAGE_CACHE_SIZE_MB (8)

9 -#define RRDENG_MIN_DISK_SPACE_MB (64)

9 +#define RRDENG_MIN_DISK_SPACE_MB (256)

10 +#define RRDENG_DEFAULT_TIER_DISK_SPACE_MB (1024)

12 #define RRDENG_NR_STATS (38)

@@ -17,9 +18,14 @@ extern int default_rrdeng_extent_cache_mb;

18 extern int db_engine_journal_check;

19 extern int default_rrdeng_disk_quota_mb;

20 extern int default_multidb_disk_quota_mb;

21 +extern bool new_dbengine_defaults;

22 +extern bool legacy_multihost_db_space;

23 +extern RRD_BACKFILL default_backfill;

24 +

25 extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];

26 extern size_t page_type_size[];

27 extern size_t tier_page_size[];

28 +extern size_t tier_quota_mb[];

29 extern uint8_t tier_page_type[];

31 #define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type]

@@ -54,8 +60,12 @@ time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *s

60 void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);

62 /* must call once before using anything */

57 -int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,

58 - unsigned disk_space_mb, size_t tier);

63 +int rrdeng_init(

64 + struct rrdengine_instance **ctxp,

65 + const char *dbfiles_path,

66 + unsigned disk_space_mb,

67 + size_t tier,

68 + time_t max_retention_s);

70 void rrdeng_readiness_wait(struct rrdengine_instance *ctx);

71 void rrdeng_exit_mode(struct rrdengine_instance *ctx);

src/database/engine/rrdenginelib.c

-23

@@ -76,26 +76,3 @@ int open_file_for_io(char *path, int flags, uv_file *file, int direct)

76 return fd;

77 }

79 -int compute_multidb_diskspace()

80 -{

81 - char multidb_disk_space_file[FILENAME_MAX + 1];

82 - FILE *fp;

83 - int computed_multidb_disk_quota_mb = -1;

84 -

85 - snprintfz(multidb_disk_space_file, FILENAME_MAX, "%s/dbengine_multihost_size", netdata_configured_varlib_dir);

86 - fp = fopen(multidb_disk_space_file, "r");

87 - if (likely(fp)) {

88 - int rc = fscanf(fp, "%d", &computed_multidb_disk_quota_mb);

89 - fclose(fp);

90 - if (unlikely(rc != 1 || computed_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB)) {

91 - errno = 0;

92 - netdata_log_error("File '%s' contains invalid input, it will be rebuild", multidb_disk_space_file);

93 - computed_multidb_disk_quota_mb = -1;

94 - }

95 - }

96 -

97 - if (computed_multidb_disk_quota_mb == -1)

98 - computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;

99 -

100 - return computed_multidb_disk_quota_mb;

101 -}

src/database/engine/rrdenginelib.h

-1

@@ -88,6 +88,5 @@ static inline int open_file_buffered_io(char *path, int flags, uv_file *file)

88 {

89 return open_file_for_io(path, flags, file, 0);

90 }

91 -int compute_multidb_diskspace();

92 #endif /* NETDATA_RRDENGINELIB_H */

src/database/rrd.h

-2

@@ -155,8 +155,6 @@ typedef enum __attribute__ ((__packed__)) {

155 RRD_BACKFILL_NEW

156 } RRD_BACKFILL;

157

158 -extern RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];

159 -

158 #define UPDATE_EVERY 1

159 #define UPDATE_EVERY_MAX 3600

160

src/database/rrdhost.c

+100 -65

@@ -3,17 +3,18 @@

3 #define NETDATA_RRD_INTERNALS

4 #include "rrd.h"

6 +#if RRD_STORAGE_TIERS != 5

7 +#error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier

8 +#endif

9 +

10 static void rrdhost_streaming_sender_structures_init(RRDHOST *host);

12 bool dbengine_enabled = false; // will become true if and when dbengine is initialized

13 size_t storage_tiers = 3;

14 bool use_direct_io = true;

11 -size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };

12 -RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };

13 -

14 -#if RRD_STORAGE_TIERS != 5

15 -#error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier

16 -#endif

15 +size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = {1, 60, 60, 60, 60};

16 +size_t storage_tiers_collection_per_sec[RRD_STORAGE_TIERS] = {1, 60, 3600, 8 * 3600, 24 * 3600};

17 +double storage_tiers_retention_days[RRD_STORAGE_TIERS] = {14, 90, 2 * 365, 2 * 365, 2 * 365};

19 size_t get_tier_grouping(size_t tier) {

20 if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1;

@@ -299,6 +300,7 @@ static RRDHOST *prepare_host_for_unittest(RRDHOST *host)

300 (struct rrdengine_instance **)&host->db[0].si,

301 dbenginepath,

302 default_rrdeng_disk_quota_mb,

303 + 0,

304 0); // may fail here for legacy dbengine initialization

305

306 initialized = (ret == 0);

@@ -682,7 +684,7 @@ static void rrdhost_update(RRDHOST *host

684 host->rrdpush_replication_step = rrdpush_replication_step;

685

686 ml_host_new(host);

685 -

687 +

688 rrdhost_load_rrdcontext_data(host);

689 nd_log(NDLS_DAEMON, NDLP_DEBUG,

690 "Host %s is not in archived mode anymore",

@@ -814,19 +816,58 @@ struct dbengine_initialization {

816 ND_THREAD *thread;

817 char path[FILENAME_MAX + 1];

818 int disk_space_mb;

819 + size_t retention_seconds;

820 size_t tier;

821 int ret;

822 };

823

824 +typedef struct rrd_alert_prototype {

825 + struct rrd_alert_match match;

826 + struct rrd_alert_config config;

827 +

828 + struct {

829 + uint32_t uses;

830 + bool enabled;

831 + bool is_on_disk;

832 + SPINLOCK spinlock;

833 + struct rrd_alert_prototype *prev, *next;

834 + } _internal;

835 +} RRD_ALERT_PROTOTYPE;

836 +

837 void *dbengine_tier_init(void *ptr) {

838 struct dbengine_initialization *dbi = ptr;

823 - dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);

839 + dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier, dbi->retention_seconds);

840 return ptr;

841 }

842 +

843 +RRD_BACKFILL get_dbengine_backfill(RRD_BACKFILL backfill)

844 +{

845 + const char *bf = config_get(

846 + CONFIG_SECTION_DB,

847 + "dbengine tier backfill",

848 + backfill == RRD_BACKFILL_NEW ? "new" :

849 + backfill == RRD_BACKFILL_FULL ? "full" :

850 + "none");

851 +

852 + if (strcmp(bf, "new") == 0)

853 + backfill = RRD_BACKFILL_NEW;

854 + else if (strcmp(bf, "full") == 0)

855 + backfill = RRD_BACKFILL_FULL;

856 + else if (strcmp(bf, "none") == 0)

857 + backfill = RRD_BACKFILL_NONE;

858 + else {

859 + nd_log(NDLS_DAEMON, NDLP_WARNING, "DBENGINE: unknown backfill value '%s', assuming 'new'", bf);

860 + config_set(CONFIG_SECTION_DB, "dbengine tier backfill", "new");

861 + backfill = RRD_BACKFILL_NEW;

862 + }

863 + return backfill;

864 +}

865 +

866 #endif

867

868 void dbengine_init(char *hostname) {

869 #ifdef ENABLE_DBENGINE

870 +

871 use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", use_direct_io);

872

873 unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", DEFAULT_PAGES_PER_EXTENT);

@@ -842,8 +883,7 @@ void dbengine_init(char *hostname) {

883

884 storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);

885 if(storage_tiers < 1) {

845 - nd_log(NDLS_DAEMON, NDLP_WARNING,

846 - "At least 1 storage tier is required. Assuming 1.");

886 + nd_log(NDLS_DAEMON, NDLP_WARNING, "At least 1 storage tier is required. Assuming 1.");

887

888 storage_tiers = 1;

889 config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);

@@ -857,66 +897,54 @@ void dbengine_init(char *hostname) {

897 config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);

898 }

899

860 - bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;

861 - parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);

900 + new_dbengine_defaults =

901 + (!legacy_multihost_db_space &&

902 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 1 update every iterations") &&

903 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 2 update every iterations") &&

904 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 3 update every iterations") &&

905 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 4 update every iterations") &&

906 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 1 disk space MB") &&

907 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 2 disk space MB") &&

908 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 3 disk space MB") &&

909 + !config_exists(CONFIG_SECTION_DB, "dbengine tier 4 disk space MB"));

910

863 - struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};

864 -

865 - bool tiers_adjusted = false;

866 - size_t created_tiers = 0;

867 - char dbenginepath[FILENAME_MAX + 1];

911 + default_backfill = get_dbengine_backfill(RRD_BACKFILL_NEW);

912 char dbengineconfig[200 + 1];

869 - int divisor = 1;

870 - for(size_t tier = 0; tier < storage_tiers ;tier++) {

913

872 - if(tier > 0)

873 - divisor *= 2;

914 + size_t grouping_iterations = default_rrd_update_every;

915 + storage_tiers_grouping_iterations[0] = default_rrd_update_every;

916

875 - int disk_space_mb = default_multidb_disk_quota_mb / divisor;

876 - size_t grouping_iterations = storage_tiers_grouping_iterations[tier];

877 - RRD_BACKFILL backfill = storage_tiers_backfill[tier];

917 + for (size_t tier = 1; tier < storage_tiers; tier++) {

918 + grouping_iterations = storage_tiers_grouping_iterations[tier];

919 + snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu update every iterations", tier);

920 + grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);

921 + if(grouping_iterations < 2) {

922 + grouping_iterations = 2;

923 + config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);

924 + nd_log(NDLS_DAEMON, NDLP_WARNING,

925 + "DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",

926 + hostname, tier);

927 + }

928 + storage_tiers_grouping_iterations[tier] = grouping_iterations;

929 + }

930

879 - if(tier > 0) {

880 - snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu multihost disk space MB", tier);

881 - disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);

931 + default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine tier 0 disk space MB", RRDENG_DEFAULT_TIER_DISK_SPACE_MB);

932 + if(default_multidb_disk_quota_mb && default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {

933 + netdata_log_error("Invalid disk space %d for tier 0 given. Defaulting to %d.", default_multidb_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);

934 + default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;

935 + config_set_number(CONFIG_SECTION_DB, "dbengine tier 0 disk space MB", default_multidb_disk_quota_mb);

936 + }

937

883 - snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu update every iterations", tier);

884 - grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);

885 - if(grouping_iterations < 2) {

886 - grouping_iterations = 2;

887 - config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);

888 - nd_log(NDLS_DAEMON, NDLP_WARNING,

889 - "DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",

890 - hostname, tier);

891 - }

938 + bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;

939

893 - snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu backfill", tier);

894 - const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");

895 - if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;

896 - else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;

897 - else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;

898 - else {

899 - nd_log(NDLS_DAEMON, NDLP_WARNING, "DBENGINE: unknown backfill value '%s', assuming 'new'", bf);

900 - config_set(CONFIG_SECTION_DB, dbengineconfig, "new");

901 - backfill = RRD_BACKFILL_NEW;

902 - }

903 - }

940 + struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};

941

905 - storage_tiers_grouping_iterations[tier] = grouping_iterations;

906 - storage_tiers_backfill[tier] = backfill;

942 + size_t created_tiers = 0;

943 + char dbenginepath[FILENAME_MAX + 1];

944

908 - if(tier > 0 && get_tier_grouping(tier) > 65535) {

909 - storage_tiers_grouping_iterations[tier] = 1;

910 - nd_log(NDLS_DAEMON, NDLP_WARNING,

911 - "DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. "

912 - "Disabling tiers %zu and above",

913 - hostname, tier, tier);

914 - storage_tiers = tier;

915 - tiers_adjusted = true;

916 - break;

917 - }

945 + for (size_t tier = 0; tier < storage_tiers; tier++) {

946

919 - if(tier == 0)

947 + if (tier == 0)

948 snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);

949 else

950 snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);

@@ -924,13 +952,20 @@ void dbengine_init(char *hostname) {

952 int ret = mkdir(dbenginepath, 0775);

953 if (ret != 0 && errno != EEXIST) {

954 nd_log(NDLS_DAEMON, NDLP_CRIT, "DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);

927 - break;

955 + continue;

956 }

957

930 - internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);

958 + int disk_space_mb = tier ? RRDENG_DEFAULT_TIER_DISK_SPACE_MB : default_multidb_disk_quota_mb;

959 + snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu disk space MB", tier);

960 + disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);

961 +

962 + snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu retention days", tier);

963 + storage_tiers_retention_days[tier] = config_get_number(

964 + CONFIG_SECTION_DB, dbengineconfig, new_dbengine_defaults ? storage_tiers_retention_days[tier] : 0);

965

932 - tiers_init[tier].disk_space_mb = disk_space_mb;

966 + tiers_init[tier].disk_space_mb = (int) disk_space_mb;

967 tiers_init[tier].tier = tier;

968 + tiers_init[tier].retention_seconds = (size_t) (86400.0 * storage_tiers_retention_days[tier]);

969 strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);

970 tiers_init[tier].ret = 0;

971

@@ -942,8 +977,6 @@ void dbengine_init(char *hostname) {

977 else

978 dbengine_tier_init(&tiers_init[tier]);

979 }

945 - if (tiers_adjusted)

946 - config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);

980

981 for(size_t tier = 0; tier < storage_tiers ;tier++) {

982 if(parallel_initialization)

@@ -971,6 +1004,8 @@ void dbengine_init(char *hostname) {

1004 for(size_t tier = 0; tier < storage_tiers ;tier++)

1005 rrdeng_readiness_wait(multidb_ctx[tier]);

1006

1007 + calculate_tier_disk_space_percentage();

1008 +

1009 dbengine_enabled = true;

1010 #else

1011 storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);

src/database/sqlite/sqlite_functions.c

+16

@@ -372,6 +372,22 @@ void sqlite_close_databases(void)

372 sql_close_database(db_meta, "METADATA");

373 }

374

375 +uint64_t get_total_database_space(void)

376 +{

377 + return 0;

378 +

379 +/*

380 + if (!new_dbengine_defaults)

381 + return 0;

382 +

383 + uint64_t database_space = sqlite_get_meta_space() + sqlite_get_context_space();

384 +#ifdef ENABLE_ML

385 + database_space += sqlite_get_ml_space();

386 +#endif

387 + return database_space;

388 +*/

389 +}

390 +

391 int sqlite_library_init(void)

392 {

393 initialize_thread_key_pool();

src/database/sqlite/sqlite_functions.h

@@ -117,4 +117,5 @@ void sqlite_library_shutdown(void);

117

118 void sql_close_database(sqlite3 *database, const char *database_name);

119 void sqlite_close_databases(void);

120 +uint64_t get_total_database_space(void);

121 #endif //NETDATA_SQLITE_FUNCTIONS_H

src/database/sqlite/sqlite_metadata.c

@@ -2306,6 +2306,10 @@ void metadata_delete_host_chart_labels(char *machine_guid)

2306 nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command delete chart labels for host %s", machine_guid);

2307 }

2308

2309 +uint64_t sqlite_get_meta_space(void)

2310 +{

2311 + return sqlite_get_db_space(db_meta);

2312 +}

2313

2314 //

2315 // unitests

src/web/api/queries/query.c

+2 -2

@@ -1966,7 +1966,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG

1966

1967 void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {

1968 if(unlikely(tier >= storage_tiers)) return;

1969 - if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;

1969 + if(default_backfill == RRD_BACKFILL_NONE) return;

1970

1971 struct rrddim_tier *t = &rd->tiers[tier];

1972 if(unlikely(!t)) return;

@@ -1976,7 +1976,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s

1976 time_t time_diff = now_s - latest_time_s;

1977

1978 // if the user wants only NEW backfilling, and we don't have any data

1979 - if(storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_s <= 0) return;

1979 + if(default_backfill == RRD_BACKFILL_NEW && latest_time_s <= 0) return;

1980

1981 // there is really nothing we can do

1982 if(now_s <= latest_time_s || time_diff < granularity) return;