| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "netdata-conf-db.h" |
| 4 | #include "daemon/common.h" |
| 5 | |
| 6 | #define DAYS 86400 |
| 7 | int default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; |
| 8 | |
| 9 | bool dbengine_enabled = false; // will become true if and when dbengine is initialized |
| 10 | bool dbengine_use_direct_io = true; |
| 11 | static size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = {1, 60, 60, 60, 60}; |
| 12 | static time_t storage_tiers_retention_time_s[RRD_STORAGE_TIERS] = {14 * DAYS, 90 * DAYS, 2 * 365 * DAYS, 2 * 365 * DAYS, 2 * 365 * DAYS}; |
| 13 | |
| 14 | time_t rrdset_free_obsolete_time_s = 3600; |
| 15 | time_t rrdhost_cleanup_orphan_to_archive_time_s = 3600; |
| 16 | time_t rrdhost_free_ephemeral_time_s = 0; |
| 17 | |
| 18 | extern time_t dbengine_journal_v2_unmount_time; |
| 19 | |
| 20 | size_t get_tier_grouping(size_t tier) { |
| 21 | if(unlikely(tier >= nd_profile.storage_tiers)) tier = nd_profile.storage_tiers - 1; |
| 22 | |
| 23 | size_t grouping = 1; |
| 24 | // first tier is always 1 iteration of whatever update every the chart has |
| 25 | for(size_t i = 1; i <= tier ;i++) |
| 26 | grouping *= storage_tiers_grouping_iterations[i]; |
| 27 | |
| 28 | return grouping; |
| 29 | } |
| 30 | |
| 31 | static void netdata_conf_dbengine_pre_logs(void) { |
| 32 | FUNCTION_RUN_ONCE(); |
| 33 | |
| 34 | errno_clear(); |
| 35 | |
| 36 | #ifdef ENABLE_DBENGINE |
| 37 | // this is required for dbegnine to work, so call it here (it is ok, it won't run twice) |
| 38 | netdata_conf_section_directories(); |
| 39 | |
| 40 | // ------------------------------------------------------------------------ |
| 41 | // get default Database Engine page type |
| 42 | |
| 43 | const char *page_type = inicfg_get(&netdata_config, CONFIG_SECTION_DB, "dbengine page type", "gorilla"); |
| 44 | if (strcmp(page_type, "gorilla") == 0) |
| 45 | tier_page_type[0] = RRDENG_PAGE_TYPE_GORILLA_32BIT; |
| 46 | else if (strcmp(page_type, "raw") == 0) |
| 47 | tier_page_type[0] = RRDENG_PAGE_TYPE_ARRAY_32BIT; |
| 48 | else { |
| 49 | tier_page_type[0] = RRDENG_PAGE_TYPE_ARRAY_32BIT; |
| 50 | netdata_log_error("Invalid dbengine page type ''%s' given. Defaulting to 'raw'.", page_type); |
| 51 | } |
| 52 | |
| 53 | // ------------------------------------------------------------------------ |
| 54 | // get default Database Engine page cache size in MiB |
| 55 | |
| 56 | default_rrdeng_page_cache_mb = (int) inicfg_get_size_mb(&netdata_config, CONFIG_SECTION_DB, "dbengine page cache size", default_rrdeng_page_cache_mb); |
| 57 | default_rrdeng_extent_cache_mb = (int) inicfg_get_size_mb(&netdata_config, CONFIG_SECTION_DB, "dbengine extent cache size", default_rrdeng_extent_cache_mb); |
| 58 | db_engine_journal_check = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO); |
| 59 | |
| 60 | if(default_rrdeng_extent_cache_mb < 0) { |
| 61 | default_rrdeng_extent_cache_mb = 0; |
| 62 | inicfg_set_size_mb(&netdata_config, CONFIG_SECTION_DB, "dbengine extent cache size", default_rrdeng_extent_cache_mb); |
| 63 | } |
| 64 | |
| 65 | if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) { |
| 66 | netdata_log_error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB); |
| 67 | default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB; |
| 68 | inicfg_set_size_mb(&netdata_config, CONFIG_SECTION_DB, "dbengine page cache size", default_rrdeng_page_cache_mb); |
| 69 | } |
| 70 | |
| 71 | // ------------------------------------------------------------------------ |
| 72 | // get default Database Engine disk space quota in MiB |
| 73 | // |
| 74 | // // if (!config_exists(CONFIG_SECTION_DB, "dbengine disk space MB") && !config_exists(CONFIG_SECTION_DB, "dbengine multihost disk space MB")) |
| 75 | // |
| 76 | // default_rrdeng_disk_quota_mb = (int) inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb); |
| 77 | // if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { |
| 78 | // netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB); |
| 79 | // default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB; |
| 80 | // inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb); |
| 81 | // } |
| 82 | // |
| 83 | // default_multidb_disk_quota_mb = (int) inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace()); |
| 84 | // if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { |
| 85 | // netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb); |
| 86 | // default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; |
| 87 | // inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb); |
| 88 | // } |
| 89 | |
| 90 | #else |
| 91 | if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) { |
| 92 | error_report("RRD_DB_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead."); |
| 93 | default_rrd_memory_mode = RRD_DB_MODE_RAM; |
| 94 | } |
| 95 | #endif |
| 96 | } |
| 97 | |
| 98 | #ifdef ENABLE_DBENGINE |
| 99 | struct dbengine_initialization { |
| 100 | ND_THREAD *thread; |
| 101 | char path[FILENAME_MAX + 1]; |
| 102 | int disk_space_mb; |
| 103 | size_t retention_seconds; |
| 104 | size_t tier; |
| 105 | int ret; |
| 106 | }; |
| 107 | |
| 108 | void dbengine_tier_init(void *ptr) { |
| 109 | struct dbengine_initialization *dbi = ptr; |
| 110 | dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier, dbi->retention_seconds); |
| 111 | } |
| 112 | |
| 113 | RRD_BACKFILL get_dbengine_backfill(RRD_BACKFILL backfill) |
| 114 | { |
| 115 | const char *bf = inicfg_get(&netdata_config, |
| 116 | CONFIG_SECTION_DB, |
| 117 | "dbengine tier backfill", |
| 118 | backfill == RRD_BACKFILL_NEW ? "new" : |
| 119 | backfill == RRD_BACKFILL_FULL ? "full" : |
| 120 | "none"); |
| 121 | |
| 122 | if (strcmp(bf, "new") == 0) |
| 123 | backfill = RRD_BACKFILL_NEW; |
| 124 | else if (strcmp(bf, "full") == 0) |
| 125 | backfill = RRD_BACKFILL_FULL; |
| 126 | else if (strcmp(bf, "none") == 0) |
| 127 | backfill = RRD_BACKFILL_NONE; |
| 128 | else { |
| 129 | nd_log(NDLS_DAEMON, NDLP_WARNING, "DBENGINE: unknown backfill value '%s', assuming 'new'", bf); |
| 130 | inicfg_set(&netdata_config, CONFIG_SECTION_DB, "dbengine tier backfill", "new"); |
| 131 | backfill = RRD_BACKFILL_NEW; |
| 132 | } |
| 133 | return backfill; |
| 134 | } |
| 135 | #endif |
| 136 | |
| 137 | void netdata_conf_dbengine_init(const char *hostname) { |
| 138 | #ifdef ENABLE_DBENGINE |
| 139 | |
| 140 | // ---------------------------------------------------------------------------------------------------------------- |
| 141 | // out of memory protection and use all ram for caches |
| 142 | |
| 143 | dbengine_out_of_memory_protection = 0; // will be calculated below |
| 144 | OS_SYSTEM_MEMORY sm = os_system_memory(true); |
| 145 | if(OS_SYSTEM_MEMORY_OK(sm) && sm.ram_total_bytes > sm.ram_available_bytes) { |
| 146 | // calculate the default out of memory protection size |
| 147 | uint64_t keep_free = sm.ram_total_bytes / 10; |
| 148 | if(keep_free > 5ULL * 1024 * 1024 * 1024) |
| 149 | keep_free = 5ULL * 1024 * 1024 * 1024; |
| 150 | char buf[64]; |
| 151 | size_snprintf(buf, sizeof(buf), keep_free, "B", false); |
| 152 | size_parse(buf, &dbengine_out_of_memory_protection, "B"); |
| 153 | } |
| 154 | |
| 155 | if(dbengine_out_of_memory_protection) { |
| 156 | dbengine_use_all_ram_for_caches = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_DB, "dbengine use all ram for caches", dbengine_use_all_ram_for_caches); |
| 157 | dbengine_out_of_memory_protection = inicfg_get_size_bytes(&netdata_config, CONFIG_SECTION_DB, "dbengine out of memory protection", dbengine_out_of_memory_protection); |
| 158 | |
| 159 | char buf_total[64], buf_avail[64], buf_oom[64]; |
| 160 | size_snprintf(buf_total, sizeof(buf_total), sm.ram_total_bytes, "B", false); |
| 161 | size_snprintf(buf_avail, sizeof(buf_avail), sm.ram_available_bytes, "B", false); |
| 162 | size_snprintf(buf_oom, sizeof(buf_oom), dbengine_out_of_memory_protection, "B", false); |
| 163 | |
| 164 | nd_log(NDLS_DAEMON, NDLP_NOTICE, |
| 165 | "DBENGINE memory protection enabled. " |
| 166 | "Netdata will limit DBENGINE memory usage to help keep at least %s of system RAM available when possible and reduce OOM risk. " |
| 167 | "System memory total: %s, currently available: %s, use all RAM for caches: %s", |
| 168 | buf_oom, buf_total, buf_avail, dbengine_use_all_ram_for_caches ? "enabled" : "disabled"); |
| 169 | } |
| 170 | else { |
| 171 | dbengine_out_of_memory_protection = 0; |
| 172 | dbengine_use_all_ram_for_caches = false; |
| 173 | |
| 174 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 175 | "DBENGINE memory protection is disabled because Netdata could not detect system memory size. " |
| 176 | "\"use all RAM for caches\" is also disabled."); |
| 177 | } |
| 178 | |
| 179 | // ---------------------------------------------------------------------------------------------------------------- |
| 180 | |
| 181 | dbengine_use_direct_io = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_DB, "dbengine use direct io", dbengine_use_direct_io); |
| 182 | dbengine_journal_v2_unmount_time = inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "dbengine journal v2 unmount time", nd_profile.dbengine_journal_v2_unmount_time); |
| 183 | |
| 184 | unsigned read_num = (unsigned)inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "dbengine pages per extent", DEFAULT_PAGES_PER_EXTENT); |
| 185 | if (read_num > 0 && read_num <= DEFAULT_PAGES_PER_EXTENT) |
| 186 | rrdeng_pages_per_extent = read_num; |
| 187 | else { |
| 188 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 189 | "Invalid dbengine pages per extent %u given. Using %u.", |
| 190 | read_num, rrdeng_pages_per_extent); |
| 191 | |
| 192 | inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent); |
| 193 | } |
| 194 | |
| 195 | nd_profile.storage_tiers = inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "storage tiers", nd_profile.storage_tiers); |
| 196 | if(nd_profile.storage_tiers < 1) { |
| 197 | nd_log(NDLS_DAEMON, NDLP_WARNING, "At least 1 storage tier is required. Assuming 1."); |
| 198 | |
| 199 | nd_profile.storage_tiers = 1; |
| 200 | inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "storage tiers", nd_profile.storage_tiers); |
| 201 | } |
| 202 | if(nd_profile.storage_tiers > RRD_STORAGE_TIERS) { |
| 203 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 204 | "Up to %d storage tier are supported. Assuming %d.", |
| 205 | RRD_STORAGE_TIERS, RRD_STORAGE_TIERS); |
| 206 | |
| 207 | nd_profile.storage_tiers = RRD_STORAGE_TIERS; |
| 208 | inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "storage tiers", nd_profile.storage_tiers); |
| 209 | } |
| 210 | |
| 211 | new_dbengine_defaults = |
| 212 | (!legacy_multihost_db_space && |
| 213 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 1 update every iterations") && |
| 214 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 2 update every iterations") && |
| 215 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 3 update every iterations") && |
| 216 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 4 update every iterations") && |
| 217 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 1 retention size") && |
| 218 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 2 retention size") && |
| 219 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 3 retention size") && |
| 220 | !inicfg_exists(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 4 retention size")); |
| 221 | |
| 222 | default_backfill = get_dbengine_backfill(RRD_BACKFILL_NEW); |
| 223 | char dbengineconfig[200 + 1]; |
| 224 | |
| 225 | size_t grouping_iterations = nd_profile.update_every; |
| 226 | storage_tiers_grouping_iterations[0] = nd_profile.update_every; |
| 227 | |
| 228 | for (size_t tier = 1; tier < nd_profile.storage_tiers; tier++) { |
| 229 | grouping_iterations = storage_tiers_grouping_iterations[tier]; |
| 230 | snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu update every iterations", tier); |
| 231 | grouping_iterations = inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, dbengineconfig, grouping_iterations); |
| 232 | if(grouping_iterations < 2) { |
| 233 | grouping_iterations = 2; |
| 234 | inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, dbengineconfig, grouping_iterations); |
| 235 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 236 | "DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.", |
| 237 | hostname, tier); |
| 238 | } |
| 239 | storage_tiers_grouping_iterations[tier] = grouping_iterations; |
| 240 | } |
| 241 | |
| 242 | default_multidb_disk_quota_mb = (int) inicfg_get_size_mb(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 0 retention size", RRDENG_DEFAULT_TIER_DISK_SPACE_MB); |
| 243 | if(default_multidb_disk_quota_mb && default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { |
| 244 | netdata_log_error("Invalid disk space %d for tier 0 given. Defaulting to %d.", default_multidb_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB); |
| 245 | default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB; |
| 246 | inicfg_set_size_mb(&netdata_config, CONFIG_SECTION_DB, "dbengine tier 0 retention size", default_multidb_disk_quota_mb); |
| 247 | } |
| 248 | |
| 249 | #ifdef OS_WINDOWS |
| 250 | // FIXME: for whatever reason joining the initialization threads |
| 251 | // fails on Windows. |
| 252 | bool parallel_initialization = false; |
| 253 | #else |
| 254 | bool parallel_initialization = (nd_profile.storage_tiers <= netdata_conf_cpus()) ? true : false; |
| 255 | #endif |
| 256 | |
| 257 | struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {}; |
| 258 | |
| 259 | size_t created_tiers = 0; |
| 260 | char dbenginepath[FILENAME_MAX + 1]; |
| 261 | |
| 262 | for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) { |
| 263 | |
| 264 | if (tier == 0) |
| 265 | snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir); |
| 266 | else |
| 267 | snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier); |
| 268 | |
| 269 | int ret = mkdir(dbenginepath, 0775); |
| 270 | if (ret != 0 && errno != EEXIST) { |
| 271 | nd_log(NDLS_DAEMON, NDLP_CRIT, "DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath); |
| 272 | continue; |
| 273 | } |
| 274 | |
| 275 | int disk_space_mb = tier ? RRDENG_DEFAULT_TIER_DISK_SPACE_MB : default_multidb_disk_quota_mb; |
| 276 | snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu retention size", tier); |
| 277 | disk_space_mb = inicfg_get_size_mb(&netdata_config, CONFIG_SECTION_DB, dbengineconfig, disk_space_mb); |
| 278 | |
| 279 | snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu retention time", tier); |
| 280 | storage_tiers_retention_time_s[tier] = inicfg_get_duration_days_to_seconds( |
| 281 | &netdata_config, CONFIG_SECTION_DB, |
| 282 | dbengineconfig, new_dbengine_defaults ? storage_tiers_retention_time_s[tier] : 0); |
| 283 | |
| 284 | tiers_init[tier].disk_space_mb = (int) disk_space_mb; |
| 285 | tiers_init[tier].tier = tier; |
| 286 | tiers_init[tier].retention_seconds = (size_t) storage_tiers_retention_time_s[tier]; |
| 287 | strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX); |
| 288 | tiers_init[tier].ret = 0; |
| 289 | |
| 290 | if(parallel_initialization) { |
| 291 | char tag[NETDATA_THREAD_TAG_MAX + 1]; |
| 292 | snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier); |
| 293 | tiers_init[tier].thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, dbengine_tier_init, &tiers_init[tier]); |
| 294 | } |
| 295 | else |
| 296 | dbengine_tier_init(&tiers_init[tier]); |
| 297 | } |
| 298 | |
| 299 | for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++) { |
| 300 | if(parallel_initialization) |
| 301 | nd_thread_join(tiers_init[tier].thread); |
| 302 | |
| 303 | if(tiers_init[tier].ret != 0) { |
| 304 | nd_log(NDLS_DAEMON, NDLP_ERR, |
| 305 | "DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'", |
| 306 | hostname, tiers_init[tier].tier, tiers_init[tier].path); |
| 307 | } |
| 308 | else if(created_tiers == tier) |
| 309 | created_tiers++; |
| 310 | } |
| 311 | |
| 312 | if(created_tiers && created_tiers < nd_profile.storage_tiers) { |
| 313 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 314 | "DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.", |
| 315 | hostname, created_tiers, |
| 316 | nd_profile.storage_tiers, created_tiers); |
| 317 | |
| 318 | nd_profile.storage_tiers = created_tiers; |
| 319 | } |
| 320 | else if(!created_tiers) |
| 321 | fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir); |
| 322 | |
| 323 | for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++) |
| 324 | rrdeng_readiness_wait(multidb_ctx[tier]); |
| 325 | |
| 326 | rrdeng_calculate_tier_disk_space_percentage(); |
| 327 | |
| 328 | dbengine_enabled = true; |
| 329 | #else |
| 330 | nd_profile.storage_tiers = inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "storage tiers", 1); |
| 331 | if(nd_profile.storage_tiers != 1) { |
| 332 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 333 | "DBENGINE is not available on '%s', so only 1 database tier can be supported.", |
| 334 | hostname); |
| 335 | |
| 336 | nd_profile.storage_tiers = 1; |
| 337 | inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "storage tiers", nd_profile.storage_tiers); |
| 338 | } |
| 339 | dbengine_enabled = false; |
| 340 | #endif |
| 341 | } |
| 342 | |
| 343 | void netdata_conf_section_db(void) { |
| 344 | FUNCTION_RUN_ONCE(); |
| 345 | |
| 346 | // ------------------------------------------------------------------------ |
| 347 | // get default database update frequency |
| 348 | |
| 349 | nd_profile.update_every = (int) inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "update every", nd_profile.update_every); |
| 350 | if(nd_profile.update_every < UPDATE_EVERY_MIN) { |
| 351 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 352 | "Data collection frequency in netdata.conf ([" CONFIG_SECTION_DB "].update every), changed from %d to %d", |
| 353 | (int)nd_profile.update_every, UPDATE_EVERY_MIN); |
| 354 | nd_profile.update_every = UPDATE_EVERY_MIN; |
| 355 | inicfg_set_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "update every", nd_profile.update_every); |
| 356 | } |
| 357 | if(nd_profile.update_every > UPDATE_EVERY_MAX) { |
| 358 | nd_log(NDLS_DAEMON, NDLP_WARNING, |
| 359 | "Data collection frequency in netdata.conf ([" CONFIG_SECTION_DB "].update every), changed from %d to %d", |
| 360 | (int)nd_profile.update_every, UPDATE_EVERY_MIN); |
| 361 | nd_profile.update_every = UPDATE_EVERY_MAX; |
| 362 | inicfg_set_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "update every", nd_profile.update_every); |
| 363 | } |
| 364 | |
| 365 | // ------------------------------------------------------------------------ |
| 366 | // get the database selection |
| 367 | |
| 368 | { |
| 369 | const char *mode = inicfg_get(&netdata_config, CONFIG_SECTION_DB, "db", rrd_memory_mode_name(default_rrd_memory_mode)); |
| 370 | default_rrd_memory_mode = rrd_memory_mode_id(mode); |
| 371 | if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) { |
| 372 | netdata_log_error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode)); |
| 373 | inicfg_set(&netdata_config, CONFIG_SECTION_DB, "db", rrd_memory_mode_name(default_rrd_memory_mode)); |
| 374 | } |
| 375 | } |
| 376 | |
| 377 | // ------------------------------------------------------------------------ |
| 378 | // get default database size |
| 379 | |
| 380 | if(default_rrd_memory_mode != RRD_DB_MODE_DBENGINE && default_rrd_memory_mode != RRD_DB_MODE_NONE) { |
| 381 | default_rrd_history_entries = (int)inicfg_get_duration_seconds(&netdata_config, |
| 382 | CONFIG_SECTION_DB, "retention", |
| 383 | align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); |
| 384 | |
| 385 | long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); |
| 386 | if (h != default_rrd_history_entries) { |
| 387 | inicfg_set_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "retention", h); |
| 388 | default_rrd_history_entries = (int)h; |
| 389 | } |
| 390 | } |
| 391 | |
| 392 | // -------------------------------------------------------------------- |
| 393 | // get KSM settings |
| 394 | |
| 395 | #ifdef MADV_MERGEABLE |
| 396 | enable_ksm = inicfg_get_boolean_ondemand(&netdata_config, CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm); |
| 397 | #endif |
| 398 | |
| 399 | // -------------------------------------------------------------------- |
| 400 | |
| 401 | rrdhost_cleanup_orphan_to_archive_time_s = |
| 402 | inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "cleanup orphan hosts after", rrdhost_cleanup_orphan_to_archive_time_s); |
| 403 | if(rrdhost_cleanup_orphan_to_archive_time_s < 10) { |
| 404 | rrdhost_cleanup_orphan_to_archive_time_s = 10; |
| 405 | inicfg_set_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "cleanup orphan hosts after", rrdhost_cleanup_orphan_to_archive_time_s); |
| 406 | } |
| 407 | |
| 408 | rrdhost_free_ephemeral_time_s = |
| 409 | inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "cleanup ephemeral hosts after", rrdhost_free_ephemeral_time_s); |
| 410 | if(rrdhost_free_ephemeral_time_s && rrdhost_free_ephemeral_time_s < rrdhost_cleanup_orphan_to_archive_time_s) { |
| 411 | // the free ephemeral time cannot be less than the cleanup orphan time |
| 412 | rrdhost_free_ephemeral_time_s = rrdhost_cleanup_orphan_to_archive_time_s; |
| 413 | inicfg_set_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "cleanup ephemeral hosts after", rrdhost_free_ephemeral_time_s); |
| 414 | } |
| 415 | |
| 416 | rrdset_free_obsolete_time_s = |
| 417 | inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "cleanup obsolete charts after", rrdset_free_obsolete_time_s); |
| 418 | if (rrdset_free_obsolete_time_s < 10) { |
| 419 | // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short |
| 420 | // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at |
| 421 | // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information. |
| 422 | rrdset_free_obsolete_time_s = 10; |
| 423 | netdata_log_info("The \"cleanup obsolete charts after\" option was set to 10 seconds."); |
| 424 | inicfg_set_duration_seconds(&netdata_config, CONFIG_SECTION_DB, "cleanup obsolete charts after", rrdset_free_obsolete_time_s); |
| 425 | } |
| 426 | |
| 427 | gap_when_lost_iterations_above = (int)inicfg_get_number(&netdata_config, CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above); |
| 428 | if (gap_when_lost_iterations_above < 1) { |
| 429 | gap_when_lost_iterations_above = 1; |
| 430 | inicfg_set_number(&netdata_config, CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above); |
| 431 | } |
| 432 | gap_when_lost_iterations_above += 2; |
| 433 | |
| 434 | // ------------------------------------------------------------------------ |
| 435 | |
| 436 | netdata_conf_dbengine_pre_logs(); |
| 437 | } |