enable dbengine tiering by default (#13914)
enable tiering by default
Costa Tsaousis committed
Nov 15, 2022 at 23:13 UTC
2de2e45656dc9875326d7d0e0f76de61d1377e2a
5 files changed
+21
-11
daemon/main.c
+2
-2
@@ -679,7 +679,7 @@ static void get_netdata_configured_variables() {
679
// get default Database Engine page cache size in MiB
680
681
db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "dbengine page cache with malloc", CONFIG_BOOLEAN_NO);
682
- default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
682
+ default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", storage_tiers_cache_quota_mb[0]);
683
db_engine_journal_indexing = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal indexing", CONFIG_BOOLEAN_YES);
684
db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO);
685
@@ -692,7 +692,7 @@ static void get_netdata_configured_variables() {
692
// ------------------------------------------------------------------------
693
// get default Database Engine disk space quota in MiB
694
695
- default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
695
+ default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", storage_tiers_disk_quota_mb[0]);
696
if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
697
error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
698
default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
database/engine/rrdengineapi.c
+4
-3
@@ -30,11 +30,12 @@ __attribute__((constructor)) void initialize_multidb_ctx(void) {
30
int db_engine_use_malloc = 0;
31
int default_rrdeng_page_fetch_timeout = 3;
32
int default_rrdeng_page_fetch_retries = 3;
33
-int default_rrdeng_page_cache_mb = 32;
33
+int default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB * 4; // not used for default - use storage_tiers_cache_quota_mb[0]
34
+int default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB * 4; // not used for default - use storage_tiers_disk_quota_mb[0]
35
+int default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB * 4; // not used for default - use storage_tiers_disk_quota_mb[0]
36
int db_engine_journal_indexing = 1;
37
int db_engine_journal_check = 0;
36
-int default_rrdeng_disk_quota_mb = 256;
37
-int default_multidb_disk_quota_mb = 256;
38
+
39
/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
40
uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
41
database/engine/rrdenginelib.c
+2
-2
@@ -292,7 +292,7 @@ int compute_multidb_diskspace()
292
if (computed_multidb_disk_quota_mb == -1) {
293
int rc = count_legacy_children(netdata_configured_cache_dir);
294
if (likely(rc >= 0)) {
295
- computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
295
+ computed_multidb_disk_quota_mb = (rc + 1) * (int)storage_tiers_disk_quota_mb[0];
296
info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
297
298
fp = fopen(multidb_disk_space_file, "w");
@@ -304,7 +304,7 @@ int compute_multidb_diskspace()
304
error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
305
}
306
else
307
- computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
307
+ computed_multidb_disk_quota_mb = (int)storage_tiers_disk_quota_mb[0];
308
}
309
310
return computed_multidb_disk_quota_mb;
database/rrd.h
+2
@@ -58,6 +58,8 @@ struct pg_cache_page_index;
58
extern bool dbengine_enabled;
59
extern size_t storage_tiers;
60
extern size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
61
+extern size_t storage_tiers_disk_quota_mb[RRD_STORAGE_TIERS];
62
+extern size_t storage_tiers_cache_quota_mb[RRD_STORAGE_TIERS];
63
64
typedef enum {
65
RRD_BACKFILL_NONE,
database/rrdhost.c
+11
-4
@@ -4,8 +4,10 @@
4
#include "rrd.h"
5
6
bool dbengine_enabled = false; // will become true if and when dbengine is initialized
7
-size_t storage_tiers = 1;
7
+size_t storage_tiers = 1; // this default is not used - 3 when dbengine is used, 1 otherwise
8
size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
9
+size_t storage_tiers_disk_quota_mb[RRD_STORAGE_TIERS] = { RRDENG_MIN_DISK_SPACE_MB * 4, RRDENG_MIN_DISK_SPACE_MB * 2, RRDENG_MIN_DISK_SPACE_MB, RRDENG_MIN_DISK_SPACE_MB, RRDENG_MIN_DISK_SPACE_MB };
10
+size_t storage_tiers_cache_quota_mb[RRD_STORAGE_TIERS] = { RRDENG_MIN_PAGE_CACHE_SIZE_MB * 4, RRDENG_MIN_PAGE_CACHE_SIZE_MB * 2, RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_PAGE_CACHE_SIZE_MB };
11
RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
12
13
#if RRD_STORAGE_TIERS != 5
@@ -756,7 +758,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim
758
759
void dbengine_init(char *hostname) {
760
#ifdef ENABLE_DBENGINE
759
- storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
761
+ storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) ? 3 : 1);
762
if(storage_tiers < 1) {
763
error("At least 1 storage tier is required. Assuming 1.");
764
storage_tiers = 1;
@@ -767,6 +769,11 @@ void dbengine_init(char *hostname) {
769
storage_tiers = RRD_STORAGE_TIERS;
770
config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
771
}
772
+ if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && storage_tiers != 1) {
773
+ error("Host '%s': only 1 database tier can be supported without dbengine.", hostname);
774
+ storage_tiers = 1;
775
+ config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
776
+ }
777
778
default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
779
if (default_rrdeng_page_fetch_timeout < 1) {
@@ -802,8 +809,8 @@ void dbengine_init(char *hostname) {
809
break;
810
}
811
805
- int page_cache_mb = default_rrdeng_page_cache_mb;
806
- int disk_space_mb = default_multidb_disk_quota_mb;
812
+ int page_cache_mb = (int)storage_tiers_cache_quota_mb[tier];
813
+ int disk_space_mb = (int)storage_tiers_disk_quota_mb[tier];
814
size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
815
RRD_BACKFILL backfill = storage_tiers_backfill[tier];
816