Multi-Tier database backend for long term metrics storage (#13263)
* Tier part 1 * Tier part 2 * Tier part 3 * Tier part 4 * Tier part 5 * Fix some ML compilation errors * fix more conflicts * pass proper tier * move metric_uuid from state to RRDDIM * move aclk_live_status from state to RRDDIM * move ml_dimension from state to RRDDIM * abstracted the data collection interface * support flushing for mem db too * abstracted the query api * abstracted latest/oldest time per metric * cleanup * store_metric for tier1 * fix for store_metric * allow multiple tiers, more than 2 * state to tier * Change storage type in db. Query param to request min, max, sum or average * Store tier data correctly * Fix skipping tier page type * Add tier grouping in the tier * Fix to handle archived charts (part 1) * Temp fix for query granularity when requesting tier1 data * Fix parameters in the correct order and calculate the anomaly based on the anomaly count * Proper tiering grouping * Anomaly calculation based on anomaly count * force type checking on storage handles * update cmocka tests * fully dynamic number of storage tiers * fix static allocation * configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode * use default page dt using the tiering info * automatic selection of tier * fix for automatic selection of tier * working prototype of dynamic tier selection * automatic selection of tier done right (I hope) * ask for the proper tier value, based on the grouping function * fixes for unittests and load_metric_next() * fixes for lgtm findings * minor renames * add dbengine to page cache size setting * add dbengine to page cache with malloc * query engine optimized to loop as little are required based on the view_update_every * query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA * report db points per tier in jsonwrap * query planer that switches database tiers on the fly to satisfy the query for the entire timeframe * dbegnine statistics and documentation (in progress) * calculate average point duration in db * handle single point pages the best we can * handle single point pages even better * Keep page type in the rrdeng_page_descr * updated doc * handle future backwards compatibility - improved statistics * support &tier=X in queries * enfore increasing iterations on tiers * tier 1 is always 1 iteration * backfilling higher tiers on first data collection * reversed anomaly bit * set up to 5 tiers * natural points should only be offered on tier 0, except a specific tier is selected * do not allow more than 65535 points of tier0 to be aggregated on any tier * Work only on actually activated tiers * fix query interpolation * fix query interpolation again * fix lgtm finding * Activate one tier for now * backfilling of higher tiers using raw metrics from lower tiers * fix for crash on start when storage tiers is increased from the default * more statistics on exit * fix bug that prevented higher tiers to get any values; added backfilling options * fixed the statistics log line * removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle * fixed division by zero on zero points_wanted * removed dead code * Decide on the descr->type for the type of metric * dont store metrics on unknown page types * free db_metric_handle on sql based context queries * Disable STORAGE_POINT value check in the exporting engine unit tests * fix for db modes other than dbengine * fix for aclk archived chart queries destroying db_metric_handles of valid rrddims * fix left-over freez() instead of OWA freez on median queries Co-authored-by: Costa Tsaousis <costa@netdata.cloud> Co-authored-by: Vladimir Kobal <vlad@prokk.net>
Stelios Fragkakis committed
Jul 6, 2022 at 14:01 UTC
49234f23de3a32682daff07ca229b6b62f24c090
62 files changed
+2356
-1279
collectors/plugins.d/pluginsd_parser.c
+2
-2
@@ -146,13 +146,13 @@ PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name
146
if (likely(unhide_dimension)) {
147
rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
148
if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
149
- (void)sql_set_dimension_option(&rd->state->metric_uuid, NULL);
149
+ (void)sql_set_dimension_option(&rd->metric_uuid, NULL);
150
rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
151
}
152
} else {
153
rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
154
if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
155
- (void)sql_set_dimension_option(&rd->state->metric_uuid, "hidden");
155
+ (void)sql_set_dimension_option(&rd->metric_uuid, "hidden");
156
rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
157
}
158
}
collectors/statsd.plugin/statsd.c
+5
-20
@@ -271,9 +271,7 @@ static struct statsd {
271
size_t tcp_idle_timeout;
272
collected_number decimal_detail;
273
size_t private_charts;
274
- size_t max_private_charts;
274
size_t max_private_charts_hard;
276
- RRD_MEMORY_MODE private_charts_memory_mode;
275
long private_charts_rrd_history_entries;
276
unsigned int private_charts_hidden:1;
277
@@ -290,7 +288,6 @@ static struct statsd {
288
LISTEN_SOCKETS sockets;
289
} statsd = {
290
.enabled = 1,
293
- .max_private_charts = 200,
291
.max_private_charts_hard = 1000,
292
.private_charts_hidden = 0,
293
.recvmmsg_size = 10,
@@ -1591,7 +1588,7 @@ static inline void statsd_get_metric_type_and_id(STATSD_METRIC *m, char *type, c
1588
}
1589
1590
static inline RRDSET *statsd_private_rrdset_create(
1594
- STATSD_METRIC *m
1591
+ STATSD_METRIC *m __maybe_unused
1592
, const char *type
1593
, const char *id
1594
, const char *name
@@ -1603,16 +1600,6 @@ static inline RRDSET *statsd_private_rrdset_create(
1600
, int update_every
1601
, RRDSET_TYPE chart_type
1602
) {
1606
- RRD_MEMORY_MODE memory_mode = statsd.private_charts_memory_mode;
1607
- long history = statsd.private_charts_rrd_history_entries;
1608
-
1609
- if(unlikely(statsd.private_charts >= statsd.max_private_charts)) {
1610
- debug(D_STATSD, "STATSD: metric '%s' will be charted with memory mode = none, because the maximum number of charts has been reached.", m->name);
1611
- info("STATSD: metric '%s' will be charted with memory mode = none, because the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts);
1612
- memory_mode = RRD_MEMORY_MODE_NONE;
1613
- history = 5;
1614
- }
1615
-
1603
statsd.private_charts++;
1604
RRDSET *st = rrdset_create_custom(
1605
localhost // host
@@ -1628,8 +1615,8 @@ static inline RRDSET *statsd_private_rrdset_create(
1615
, priority // priority
1616
, update_every // update every
1617
, chart_type // chart type
1631
- , memory_mode // memory mode
1632
- , history // history
1618
+ , default_rrd_memory_mode // memory mode
1619
+ , default_rrd_history_entries // history
1620
);
1621
rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
1622
@@ -2300,7 +2287,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
2287
if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {
2288
if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) {
2289
debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);
2303
- info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts);
2290
+ info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts_hard);
2291
m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
2292
}
2293
else {
@@ -2446,9 +2433,7 @@ void *statsd_main(void *ptr) {
2433
#endif
2434
2435
statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
2449
- statsd.max_private_charts = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts allowed", (long long)statsd.max_private_charts);
2450
- statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5);
2451
- statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
2436
+ statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);
2437
statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
2438
statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
2439
statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);
daemon/config/README.md
+15
-15
@@ -82,21 +82,21 @@ Please note that your data history will be lost if you have modified `history` p
82
83
### [db] section options
84
85
-| setting | default | info |
86
-|:----------------------------------:|:----------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
87
-| mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size MB` and `dbengine disk space MB`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. |
88
-| retention | `3600` | Used with `mode = save/map/ram/alloc`, not the default `mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. Check [Memory Requirements](/database/README.md) for more information. |
89
-| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). |
90
-| page cache size MB | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. |
91
-| dbengine disk space MB | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. |
92
-| dbengine multihost disk space MB | 256 | Same functionality as `dbengine disk space MB`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. |
93
-| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database and the dbengine page cache to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) |
94
-| cleanup obsolete charts after secs | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions |
95
-| gap when lost iterations above | `1` | |
96
-| cleanup orphan hosts after secs | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. |
97
-| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions |
98
-| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. |
99
-| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. |
85
+| setting | default | info |
86
+|:----------------------------------:|:----------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
87
+| mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `dbengine page cache size MB` and `dbengine disk space MB`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. |
88
+| retention | `3600` | Used with `mode = save/map/ram/alloc`, not the default `mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. Check [Memory Requirements](/database/README.md) for more information. |
89
+| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). |
90
+| dbengine page cache size MB | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. |
91
+| dbengine disk space MB | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. |
92
+| dbengine multihost disk space MB | 256 | Same functionality as `dbengine disk space MB`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. |
93
+| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database and the dbengine page cache to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) |
94
+| cleanup obsolete charts after secs | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions |
95
+| gap when lost iterations above | `1` | |
96
+| cleanup orphan hosts after secs | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. |
97
+| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions |
98
+| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. |
99
+| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. |
100
101
### [directories] section options
102
daemon/global_statistics.c
+19
-12
@@ -451,21 +451,28 @@ static void dbengine_statistics_charts(void) {
451
RRDHOST *host;
452
unsigned long long stats_array[RRDENG_NR_STATS] = {0};
453
unsigned long long local_stats_array[RRDENG_NR_STATS];
454
- unsigned dbengine_contexts = 0, counted_multihost_db = 0, i;
454
+ unsigned dbengine_contexts = 0, counted_multihost_db[RRD_STORAGE_TIERS] = { 0 }, i;
455
456
rrdhost_foreach_read(host) {
457
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
458
- if (&multidb_ctx == host->rrdeng_ctx) {
459
- if (counted_multihost_db)
460
- continue; /* Only count multi-host DB once */
461
- counted_multihost_db = 1;
462
- }
463
- ++dbengine_contexts;
464
- /* get localhost's DB engine's statistics */
465
- rrdeng_get_37_statistics(host->rrdeng_ctx, local_stats_array);
466
- for (i = 0; i < RRDENG_NR_STATS; ++i) {
467
- /* aggregate statistics across hosts */
468
- stats_array[i] += local_stats_array[i];
458
+
459
+ /* get localhost's DB engine's statistics for each tier */
460
+ for(int tier = 0; tier < storage_tiers ;tier++) {
461
+ if(!host->storage_instance[tier]) continue;
462
+
463
+ if(is_storage_engine_shared(host->storage_instance[tier])) {
464
+ if(counted_multihost_db[tier])
465
+ continue;
466
+ else
467
+ counted_multihost_db[tier] = 1;
468
+ }
469
+
470
+ ++dbengine_contexts;
471
+ rrdeng_get_37_statistics((struct rrdengine_instance *)host->storage_instance[tier], local_stats_array);
472
+ for (i = 0; i < RRDENG_NR_STATS; ++i) {
473
+ /* aggregate statistics across hosts */
474
+ stats_array[i] += local_stats_array[i];
475
+ }
476
}
477
}
478
}
daemon/main.c
+16
-23
@@ -55,11 +55,13 @@ void netdata_cleanup_and_exit(int ret) {
55
// free the database
56
info("EXIT: freeing database memory...");
57
#ifdef ENABLE_DBENGINE
58
- rrdeng_prepare_exit(&multidb_ctx);
58
+ for(int tier = 0; tier < storage_tiers ; tier++)
59
+ rrdeng_prepare_exit(multidb_ctx[tier]);
60
#endif
61
rrdhost_free_all();
62
#ifdef ENABLE_DBENGINE
62
- rrdeng_exit(&multidb_ctx);
63
+ for(int tier = 0; tier < storage_tiers ; tier++)
64
+ rrdeng_exit(multidb_ctx[tier]);
65
#endif
66
}
67
sql_close_database();
@@ -533,10 +535,16 @@ static void backwards_compatible_config() {
535
CONFIG_SECTION_DB, "update every");
536
537
config_move(CONFIG_SECTION_GLOBAL, "page cache size",
536
- CONFIG_SECTION_DB, "page cache size MB");
538
+ CONFIG_SECTION_DB, "dbengine page cache size MB");
539
+
540
+ config_move(CONFIG_SECTION_DB, "page cache size",
541
+ CONFIG_SECTION_DB, "dbengine page cache size MB");
542
543
config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc",
539
- CONFIG_SECTION_DB, "page cache with malloc");
544
+ CONFIG_SECTION_DB, "dbengine page cache with malloc");
545
+
546
+ config_move(CONFIG_SECTION_DB, "page cache with malloc",
547
+ CONFIG_SECTION_DB, "dbengine page cache with malloc");
548
549
config_move(CONFIG_SECTION_GLOBAL, "dbengine disk space",
550
CONFIG_SECTION_DB, "dbengine disk space MB");
@@ -650,12 +658,12 @@ static void get_netdata_configured_variables() {
658
// ------------------------------------------------------------------------
659
// get default Database Engine page cache size in MiB
660
653
- db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "page cache with malloc", CONFIG_BOOLEAN_NO);
654
- default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb);
661
+ db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "dbengine page cache with malloc", CONFIG_BOOLEAN_NO);
662
+ default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
663
if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
664
error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
665
default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
658
- config_set_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb);
666
+ config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
667
}
668
669
// ------------------------------------------------------------------------
@@ -946,6 +954,7 @@ int main(int argc, char **argv) {
954
default_rrd_update_every = 1;
955
default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
956
default_health_enabled = 0;
957
+ storage_tiers = 1;
958
registry_init();
959
if(rrd_init("unittest", NULL)) {
960
fprintf(stderr, "rrd_init failed for unittest\n");
@@ -1303,22 +1312,6 @@ int main(int argc, char **argv) {
1312
// initialize the log files
1313
open_all_log_files();
1314
1306
-#ifdef ENABLE_DBENGINE
1307
- default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
1308
- if (default_rrdeng_page_fetch_timeout < 1) {
1309
- info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
1310
- default_rrdeng_page_fetch_timeout = 1;
1311
- config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout);
1312
- }
1313
-
1314
- default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES);
1315
- if (default_rrdeng_page_fetch_retries < 1) {
1316
- info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries);
1317
- default_rrdeng_page_fetch_retries = 1;
1318
- config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries);
1319
- }
1320
-#endif
1321
-
1315
get_system_timezone();
1316
1317
// --------------------------------------------------------------------
daemon/unit_test.c
+27
-23
@@ -1704,7 +1704,7 @@ static void test_dbengine_create_charts(RRDHOST *host, RRDSET *st[CHARTS], RRDDI
1704
// Fluh pages for subsequent real values
1705
for (i = 0 ; i < CHARTS ; ++i) {
1706
for (j = 0; j < DIMS; ++j) {
1707
- rrdeng_store_metric_flush_current_page(rd[i][j]);
1707
+ rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle);
1708
}
1709
}
1710
}
@@ -1751,11 +1751,10 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1751
{
1752
fprintf(stderr, "%s() running...\n", __FUNCTION__ );
1753
uint8_t same;
1754
- time_t time_now, time_retrieved;
1754
+ time_t time_now, time_retrieved, end_time;
1755
int i, j, k, c, errors, update_every;
1756
collected_number last;
1757
NETDATA_DOUBLE value, expected;
1758
- SN_FLAGS nflags;
1758
struct rrddim_query_handle handle;
1759
size_t value_errors = 0, time_errors = 0;
1760
@@ -1767,14 +1766,16 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1766
time_now = time_start + (c + 1) * update_every;
1767
for (i = 0 ; i < CHARTS ; ++i) {
1768
for (j = 0; j < DIMS; ++j) {
1770
- rd[i][j]->state->query_ops.init(rd[i][j], &handle, time_now, time_now + QUERY_BATCH * update_every);
1769
+ rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, TIER_QUERY_FETCH_SUM);
1770
for (k = 0; k < QUERY_BATCH; ++k) {
1771
last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
1772
j * REGION_POINTS[current_region] + c + k;
1773
expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
1774
1776
- time_t end_time;
1777
- value = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);
1775
+ STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops.next_metric(&handle);
1776
+ value = sp.sum;
1777
+ time_retrieved = sp.start_time;
1778
+ end_time = sp.end_time;
1779
1780
same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
1781
if(!same) {
@@ -1793,7 +1794,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1794
errors++;
1795
}
1796
}
1796
- rd[i][j]->state->query_ops.finalize(&handle);
1797
+ rd[i][j]->tiers[0]->query_ops.finalize(&handle);
1798
}
1799
}
1800
}
@@ -1826,7 +1827,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1827
ONEWAYALLOC *owa = onewayalloc_create(0);
1828
RRDR *r = rrd2rrdr(owa, st[i], points, time_start, time_end,
1829
RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS,
1829
- NULL, NULL, NULL, 0);
1830
+ NULL, NULL, NULL, 0, 0);
1831
1832
if (!r) {
1833
fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", st[i]->name, current_region);
@@ -1913,7 +1914,7 @@ int test_dbengine(void)
1914
for (i = 0 ; i < CHARTS ; ++i) {
1915
st[i]->update_every = update_every;
1916
for (j = 0; j < DIMS; ++j) {
1916
- rrdeng_store_metric_flush_current_page(rd[i][j]);
1917
+ rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle);
1918
}
1919
}
1920
@@ -1932,7 +1933,7 @@ int test_dbengine(void)
1933
for (i = 0 ; i < CHARTS ; ++i) {
1934
st[i]->update_every = update_every;
1935
for (j = 0; j < DIMS; ++j) {
1935
- rrdeng_store_metric_flush_current_page(rd[i][j]);
1936
+ rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle);
1937
}
1938
}
1939
@@ -1960,7 +1961,7 @@ int test_dbengine(void)
1961
ONEWAYALLOC *owa = onewayalloc_create(0);
1962
RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every,
1963
time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
1963
- RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0);
1964
+ RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0, 0);
1965
if (!r) {
1966
fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
1967
++errors;
@@ -2005,9 +2006,9 @@ int test_dbengine(void)
2006
}
2007
error_out:
2008
rrd_wrlock();
2008
- rrdeng_prepare_exit(host->rrdeng_ctx);
2009
+ rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[0]);
2010
rrdhost_delete_charts(host);
2010
- rrdeng_exit(host->rrdeng_ctx);
2011
+ rrdeng_exit((struct rrdengine_instance *)host->storage_instance[0]);
2012
rrd_unlock();
2013
2014
return errors + value_errors + time_errors;
@@ -2092,7 +2093,7 @@ static void generate_dbengine_chart(void *arg)
2093
thread_info->time_max = time_current;
2094
}
2095
for (j = 0; j < DSET_DIMS; ++j) {
2095
- rrdeng_store_metric_finalize(rd[j]);
2096
+ rrdeng_store_metric_finalize((rd[j])->tiers[0]->db_collection_handle);
2097
}
2098
}
2099
@@ -2182,10 +2183,9 @@ static void query_dbengine_chart(void *arg)
2183
RRDSET *st;
2184
RRDDIM *rd;
2185
uint8_t same;
2185
- time_t time_now, time_retrieved;
2186
+ time_t time_now, time_retrieved, end_time;
2187
collected_number generatedv;
2188
NETDATA_DOUBLE value, expected;
2188
- SN_FLAGS nflags;
2189
struct rrddim_query_handle handle;
2190
size_t value_errors = 0, time_errors = 0;
2191
@@ -2213,13 +2213,13 @@ static void query_dbengine_chart(void *arg)
2213
time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
2214
}
2215
2216
- rd->state->query_ops.init(rd, &handle, time_after, time_before);
2216
+ rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, TIER_QUERY_FETCH_SUM);
2217
++thread_info->queries_nr;
2218
for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
2219
generatedv = generate_dbengine_chart_value(i, j, time_now);
2220
expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
2221
2222
- if (unlikely(rd->state->query_ops.is_finished(&handle))) {
2222
+ if (unlikely(rd->tiers[0]->query_ops.is_finished(&handle))) {
2223
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
2224
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
2225
", found data gap, ### E R R O R ###\n",
@@ -2228,8 +2228,12 @@ static void query_dbengine_chart(void *arg)
2228
}
2229
break;
2230
}
2231
- time_t end_time;
2232
- value = rd->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);
2231
+
2232
+ STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
2233
+ value = sp.sum;
2234
+ time_retrieved = sp.start_time;
2235
+ end_time = sp.end_time;
2236
+
2237
if (!netdata_double_isnumber(value)) {
2238
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
2239
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
@@ -2263,7 +2267,7 @@ static void query_dbengine_chart(void *arg)
2267
}
2268
}
2269
}
2266
- rd->state->query_ops.finalize(&handle);
2270
+ rd->tiers[0]->query_ops.finalize(&handle);
2271
} while(!thread_info->done);
2272
2273
if(value_errors)
@@ -2411,9 +2415,9 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2415
}
2416
freez(query_threads);
2417
rrd_wrlock();
2414
- rrdeng_prepare_exit(host->rrdeng_ctx);
2418
+ rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[0]);
2419
rrdhost_delete_charts(host);
2416
- rrdeng_exit(host->rrdeng_ctx);
2420
+ rrdeng_exit((struct rrdengine_instance *)host->storage_instance[0]);
2421
rrd_unlock();
2422
}
2423
database/engine/README.md
+6
-6
@@ -26,18 +26,18 @@ To use the database engine, open `netdata.conf` and set `[db].mode` to `dbengine
26
mode = dbengine
27
```
28
29
-To configure the database engine, look for the `page cache size MB` and `dbengine multihost disk space MB` settings in the
29
+To configure the database engine, look for the `dbengine page cache size MB` and `dbengine multihost disk space MB` settings in the
30
`[db]` section of your `netdata.conf`. The Agent ignores the `[db].retention` setting when using the dbengine.
31
32
```conf
33
[db]
34
- page cache size MB = 32
34
+ dbengine page cache size MB = 32
35
dbengine multihost disk space MB = 256
36
```
37
38
The above values are the default values for Page Cache size and DB engine disk space quota.
39
40
-The `page cache size MB` option determines the amount of RAM dedicated to caching Netdata metric values. The
40
+The `dbengine page cache size MB` option determines the amount of RAM dedicated to caching Netdata metric values. The
41
actual page cache size will be slightly larger than this figure—see the [memory requirements](#memory-requirements)
42
section for details.
43
@@ -59,10 +59,10 @@ Netdata metric values per legacy database engine instance (see [details on the l
59
60
### Streaming metrics to the database engine
61
62
-When using the multihost database engine, all parent and child nodes share the same `page cache size MB` and `dbengine
62
+When using the multihost database engine, all parent and child nodes share the same `dbengine page cache size MB` and `dbengine
63
multihost disk space MB` in a single dbengine instance. The [**database engine
64
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
65
-helps you properly set `page cache size MB` and `dbengine multihost disk space MB` on your parent node to allocate enough
65
+helps you properly set `dbengine page cache size MB` and `dbengine multihost disk space MB` on your parent node to allocate enough
66
resources based on your metrics retention policy and how many child nodes you have.
67
68
#### Legacy mode
@@ -98,7 +98,7 @@ available memory.
98
There are explicit memory requirements **per** DB engine **instance**:
99
100
- The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what
101
- the user configured with `page cache size MB`.
101
+ the user configured with `dbengine page cache size MB`.
102
103
- an additional `#pages-on-disk x 4096 x 0.03` bytes of RAM are allocated for metadata.
104
database/engine/datafile.c
+27
-1
@@ -444,18 +444,44 @@ void finalize_data_files(struct rrdengine_instance *ctx)
444
struct rrdengine_journalfile *journalfile;
445
struct extent_info *extent, *next_extent;
446
447
+ size_t extents_number = 0;
448
+ size_t extents_bytes = 0;
449
+ size_t page_compressed_sizes = 0;
450
+
451
+ size_t files_number = 0;
452
+ size_t files_bytes = 0;
453
+
454
for (datafile = ctx->datafiles.first ; datafile != NULL ; datafile = next_datafile) {
455
journalfile = datafile->journalfile;
456
next_datafile = datafile->next;
457
458
for (extent = datafile->extents.first ; extent != NULL ; extent = next_extent) {
459
+ extents_number++;
460
+ extents_bytes += sizeof(*extent) + sizeof(struct rrdeng_page_descr *) * extent->number_of_pages;
461
+ page_compressed_sizes += extent->size;
462
+
463
next_extent = extent->next;
464
freez(extent);
465
}
466
close_journal_file(journalfile, datafile);
467
close_data_file(datafile);
468
+
469
+ files_number++;
470
+ files_bytes += sizeof(*journalfile) + sizeof(*datafile);
471
+
472
freez(journalfile);
473
freez(datafile);
459
-
474
}
475
+
476
+ if(!files_number) files_number = 1;
477
+ if(!extents_number) extents_number = 1;
478
+
479
+ info("DBENGINE STATISTICS ON DATAFILES:"
480
+ " Files %zu, structures %zu bytes, %0.2f bytes per file."
481
+ " Extents %zu, structures %zu bytes, %0.2f bytes per extent."
482
+ " Compressed size of all pages: %zu bytes."
483
+ , files_number, files_bytes, (double)files_bytes/files_number
484
+ , extents_number, extents_bytes, (double)extents_bytes/extents_number
485
+ , page_compressed_sizes
486
+ );
487
}
database/engine/journalfile.c
+3
-2
@@ -302,8 +302,8 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
302
Pvoid_t *PValue;
303
struct pg_cache_page_index *page_index = NULL;
304
305
- if (PAGE_METRICS != jf_metric_data->descr[i].type) {
306
- error("Unknown page type encountered.");
305
+ if (jf_metric_data->descr[i].type > PAGE_TYPE_MAX) {
306
+ error("Unknown page type %d encountered.", jf_metric_data->descr[i].type );
307
continue;
308
}
309
temp_id = (uuid_t *)jf_metric_data->descr[i].uuid;
@@ -331,6 +331,7 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
331
descr->end_time = jf_metric_data->descr[i].end_time;
332
descr->id = &page_index->id;
333
descr->extent = extent;
334
+ descr->type = jf_metric_data->descr[i].type;
335
extent->pages[valid_pages++] = descr;
336
pg_cache_insert(ctx, page_index, descr);
337
}
database/engine/pagecache.c
+133
-12
@@ -1194,24 +1194,66 @@ void init_page_cache(struct rrdengine_instance *ctx)
1194
init_committed_page_index(ctx);
1195
}
1196
1197
+
1198
+
1199
+/*
1200
+ * METRIC # number
1201
+ * 1. INDEX: JudyHS # bytes
1202
+ * 2. DATA: page_index # bytes
1203
+ *
1204
+ * PAGE (1 page of 1 metric) # number
1205
+ * 1. INDEX AT METRIC: page_index->JudyL_array # bytes
1206
+ * 2. DATA: descr # bytes
1207
+ *
1208
+ * PAGE CACHE (1 page of 1 metric at the cache) # number
1209
+ * 1. pg_cache_descr (if PG_CACHE_DESCR_ALLOCATED) # bytes
1210
+ * 2. data (if RRD_PAGE_POPULATED) # bytes
1211
+ *
1212
+ */
1213
+
1214
+
1215
void free_page_cache(struct rrdengine_instance *ctx)
1216
{
1217
struct page_cache *pg_cache = &ctx->pg_cache;
1200
- Word_t ret_Judy, bytes_freed = 0;
1218
Pvoid_t *PValue;
1219
struct pg_cache_page_index *page_index, *prev_page_index;
1220
Word_t Index;
1221
struct rrdeng_page_descr *descr;
1222
struct page_cache_descr *pg_cache_descr;
1223
1224
+ Word_t metrics_number = 0,
1225
+ metrics_bytes = 0,
1226
+ metrics_index_bytes = 0,
1227
+ metrics_duration = 0;
1228
+
1229
+ Word_t pages_number = 0,
1230
+ pages_bytes = 0,
1231
+ pages_index_bytes = 0;
1232
+
1233
+ Word_t pages_size_per_type[256] = { 0 },
1234
+ pages_count_per_type[256] = { 0 };
1235
+
1236
+ Word_t cache_pages_number = 0,
1237
+ cache_pages_bytes = 0,
1238
+ cache_pages_data_bytes = 0;
1239
+
1240
+ size_t points_in_db = 0,
1241
+ uncompressed_points_size = 0,
1242
+ seconds_in_db = 0,
1243
+ single_point_pages = 0;
1244
+
1245
+ Word_t pages_dirty_index_bytes = 0;
1246
+
1247
+ usec_t oldest_time_ut = LONG_MAX, latest_time_ut = 0;
1248
+
1249
/* Free committed page index */
1208
- ret_Judy = JudyLFreeArray(&pg_cache->committed_page_index.JudyL_array, PJE0);
1250
+ pages_dirty_index_bytes = JudyLFreeArray(&pg_cache->committed_page_index.JudyL_array, PJE0);
1251
fatal_assert(NULL == pg_cache->committed_page_index.JudyL_array);
1210
- bytes_freed += ret_Judy;
1252
1253
for (page_index = pg_cache->metrics_index.last_page_index ;
1254
page_index != NULL ;
1255
page_index = prev_page_index) {
1256
+
1257
prev_page_index = page_index->prev;
1258
1259
/* Find first page in range */
@@ -1219,37 +1261,116 @@ void free_page_cache(struct rrdengine_instance *ctx)
1261
PValue = JudyLFirst(page_index->JudyL_array, &Index, PJE0);
1262
descr = unlikely(NULL == PValue) ? NULL : *PValue;
1263
1264
+ size_t metric_duration = 0;
1265
+ size_t metric_update_every = 0;
1266
+ size_t metric_single_point_pages = 0;
1267
+
1268
while (descr != NULL) {
1269
/* Iterate all page descriptors of this metric */
1270
1271
if (descr->pg_cache_descr_state & PG_CACHE_DESCR_ALLOCATED) {
1272
+ cache_pages_number++;
1273
+
1274
/* Check rrdenglocking.c */
1275
pg_cache_descr = descr->pg_cache_descr;
1276
if (pg_cache_descr->flags & RRD_PAGE_POPULATED) {
1277
dbengine_page_free(pg_cache_descr->page);
1230
- bytes_freed += RRDENG_BLOCK_SIZE;
1278
+ cache_pages_data_bytes += RRDENG_BLOCK_SIZE;
1279
}
1280
rrdeng_destroy_pg_cache_descr(ctx, pg_cache_descr);
1233
- bytes_freed += sizeof(*pg_cache_descr);
1281
+ cache_pages_bytes += sizeof(*pg_cache_descr);
1282
+ }
1283
+
1284
+ if(descr->start_time < oldest_time_ut)
1285
+ oldest_time_ut = descr->start_time;
1286
+
1287
+ if(descr->end_time > latest_time_ut)
1288
+ latest_time_ut = descr->end_time;
1289
+
1290
+ pages_size_per_type[descr->type] += descr->page_length;
1291
+ pages_count_per_type[descr->type]++;
1292
+
1293
+ size_t points_in_page = (descr->page_length / ctx->storage_size);
1294
+ size_t page_duration = ((descr->end_time - descr->start_time) / USEC_PER_SEC);
1295
+ size_t update_every = (page_duration == 0) ? 1 : page_duration / (points_in_page - 1);
1296
+
1297
+ if (!page_duration && metric_update_every) {
1298
+ page_duration = metric_update_every;
1299
+ update_every = metric_update_every;
1300
+ }
1301
+ else if(page_duration)
1302
+ metric_update_every = update_every;
1303
+
1304
+ uncompressed_points_size += descr->page_length;
1305
+
1306
+ if(page_duration > 0) {
1307
+ page_duration = update_every * points_in_page;
1308
+ metric_duration += page_duration;
1309
+ seconds_in_db += page_duration;
1310
+ points_in_db += descr->page_length / ctx->storage_size;
1311
}
1312
+ else
1313
+ metric_single_point_pages++;
1314
+
1315
freez(descr);
1236
- bytes_freed += sizeof(*descr);
1316
+ pages_bytes += sizeof(*descr);
1317
+ pages_number++;
1318
1319
PValue = JudyLNext(page_index->JudyL_array, &Index, PJE0);
1320
descr = unlikely(NULL == PValue) ? NULL : *PValue;
1321
}
1322
1323
+ if(metric_single_point_pages && metric_update_every) {
1324
+ points_in_db += metric_single_point_pages;
1325
+ seconds_in_db += metric_update_every * metric_single_point_pages;
1326
+ metric_duration += metric_update_every * metric_single_point_pages;
1327
+ }
1328
+ else
1329
+ single_point_pages += metric_single_point_pages;
1330
+
1331
/* Free page index */
1243
- ret_Judy = JudyLFreeArray(&page_index->JudyL_array, PJE0);
1332
+ pages_index_bytes += JudyLFreeArray(&page_index->JudyL_array, PJE0);
1333
fatal_assert(NULL == page_index->JudyL_array);
1245
- bytes_freed += ret_Judy;
1334
freez(page_index);
1247
- bytes_freed += sizeof(*page_index);
1335
+
1336
+ metrics_number++;
1337
+ metrics_bytes += sizeof(*page_index);
1338
+ metrics_duration += metric_duration;
1339
}
1340
/* Free metrics index */
1250
- ret_Judy = JudyHSFreeArray(&pg_cache->metrics_index.JudyHS_array, PJE0);
1341
+ metrics_index_bytes = JudyHSFreeArray(&pg_cache->metrics_index.JudyHS_array, PJE0);
1342
fatal_assert(NULL == pg_cache->metrics_index.JudyHS_array);
1252
- bytes_freed += ret_Judy;
1343
1254
- info("Freed %lu bytes of memory from page cache.", bytes_freed);
1344
+ if(!metrics_number) metrics_number = 1;
1345
+ if(!pages_number) pages_number = 1;
1346
+ if(!cache_pages_number) cache_pages_number = 1;
1347
+ if(!points_in_db) points_in_db = 1;
1348
+ if(latest_time_ut == oldest_time_ut) oldest_time_ut -= USEC_PER_SEC;
1349
+
1350
+ if(single_point_pages) {
1351
+ long double avg_duration = (long double)seconds_in_db / points_in_db;
1352
+ points_in_db += single_point_pages;
1353
+ seconds_in_db += (size_t)(avg_duration * single_point_pages);
1354
+ }
1355
+
1356
+ info("DBENGINE STATISTICS ON METRICS:"
1357
+ " Metrics: %lu (structures %lu bytes - per metric %0.2f, index (HS) %lu bytes - per metric %0.2f bytes - duration %zu secs) |"
1358
+ " Page descriptors: %lu (structures %lu bytes - per page %0.2f bytes, index (L) %lu bytes - per page %0.2f, dirty index %lu bytes). |"
1359
+ " Page cache: %lu pages (structures %lu bytes - per page %0.2f bytes, data %lu bytes). |"
1360
+ " Points in db %zu, uncompressed size of points database %zu bytes. |"
1361
+ " Duration of all points %zu seconds, average point duration %0.2f seconds."
1362
+ " Duration of the database %llu seconds, average metric duration %0.2f seconds, average metric lifetime %0.2f%%."
1363
+ , metrics_number, metrics_bytes, (double)metrics_bytes/metrics_number, metrics_index_bytes, (double)metrics_index_bytes/metrics_number, metrics_duration
1364
+ , pages_number, pages_bytes, (double)pages_bytes/pages_number, pages_index_bytes, (double)pages_index_bytes/pages_number, pages_dirty_index_bytes
1365
+ , cache_pages_number, cache_pages_bytes, (double)cache_pages_bytes/cache_pages_number, cache_pages_data_bytes
1366
+ , points_in_db, uncompressed_points_size
1367
+ , seconds_in_db, (double)seconds_in_db/points_in_db
1368
+ , (latest_time_ut - oldest_time_ut) / USEC_PER_SEC, (double)metrics_duration/metrics_number
1369
+ , (double)metrics_duration/metrics_number * 100.0 / ((latest_time_ut - oldest_time_ut) / USEC_PER_SEC)
1370
+ );
1371
+
1372
+ for(int i = 0; i < 256 ;i++) {
1373
+ if(pages_count_per_type[i])
1374
+ info("DBENGINE STATISTICS ON PAGE TYPES: page type %d total pages %lu, average page size %0.2f bytes", i, pages_count_per_type[i], (double)pages_size_per_type[i]/pages_count_per_type[i]);
1375
+ }
1376
}
database/engine/pagecache.h
+1
@@ -63,6 +63,7 @@ struct rrdeng_page_descr {
63
usec_t start_time;
64
usec_t end_time;
65
uint32_t page_length;
66
+ uint8_t type;
67
};
68
69
#define PAGE_INFO_SCRATCH_SZ (8)
database/engine/rrddiskprotocol.h
+2
-1
@@ -35,7 +35,8 @@ struct rrdeng_df_sb {
35
* Page types
36
*/
37
#define PAGE_METRICS (0)
38
-#define PAGE_LOGS (1) /* reserved */
38
+#define PAGE_TIER (1)
39
+#define PAGE_TYPE_MAX (1)
40
41
/*
42
* Data file page descriptor
database/engine/rrdengine.c
+2
-2
@@ -781,7 +781,7 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
781
xt_io_descr->descr_commit_idx_array[i] = descr_commit_idx_array[i];
782
783
descr = xt_io_descr->descr_array[i];
784
- header->descr[i].type = PAGE_METRICS;
784
+ header->descr[i].type = descr->type;
785
uuid_copy(*(uuid_t *)header->descr[i].uuid, *descr->id);
786
header->descr[i].page_length = descr->page_length;
787
header->descr[i].start_time = descr->start_time;
@@ -1339,7 +1339,7 @@ void rrdengine_main(void)
1339
struct rrdengine_instance *ctx;
1340
1341
sanity_check();
1342
- ret = rrdeng_init(NULL, &ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
1342
+ ret = rrdeng_init(NULL, &ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB, 0);
1343
if (ret) {
1344
exit(ret);
1345
}
database/engine/rrdengine.h
+6
@@ -35,6 +35,7 @@ struct rrdengine_instance;
35
#define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
36
37
struct rrdeng_collect_handle {
38
+ struct rrdeng_metric_handle *metric_handle;
39
struct rrdeng_page_descr *descr;
40
unsigned long page_correlation_id;
41
struct rrdengine_instance *ctx;
@@ -43,6 +44,7 @@ struct rrdeng_collect_handle {
44
};
45
46
struct rrdeng_query_handle {
47
+ struct rrdeng_metric_handle *metric_handle;
48
struct rrdeng_page_descr *descr;
49
struct rrdengine_instance *ctx;
50
struct pg_cache_page_index *page_index;
@@ -50,6 +52,7 @@ struct rrdeng_query_handle {
52
time_t now;
53
unsigned position;
54
unsigned entries;
55
+ TIER_QUERY_FETCH tier_query_fetch_type;
56
storage_number *page;
57
usec_t page_end_time;
58
uint32_t page_length;
@@ -239,12 +242,15 @@ struct rrdengine_instance {
242
char machine_guid[GUID_LEN + 1]; /* the unique ID of the corresponding host, or localhost for multihost DB */
243
uint64_t disk_space;
244
uint64_t max_disk_space;
245
+ int tier;
246
+ size_t storage_size;
247
unsigned last_fileno; /* newest index of datafile and journalfile */
248
unsigned long max_cache_pages;
249
unsigned long cache_pages_low_watermark;
250
unsigned long metric_API_max_producers;
251
252
uint8_t quiesce; /* set to SET_QUIESCE before shutdown of the engine */
253
+ uint8_t page_type; /* set to SET_QUIESCE before shutdown of the engine */
254
255
struct rrdengine_statistics stats;
256
};
database/engine/rrdengineapi.c
+224
-326
@@ -2,7 +2,23 @@
2
#include "rrdengine.h"
3
4
/* Default global database instance */
5
-struct rrdengine_instance multidb_ctx;
5
+struct rrdengine_instance multidb_ctx_storage_tier0;
6
+struct rrdengine_instance multidb_ctx_storage_tier1;
7
+struct rrdengine_instance multidb_ctx_storage_tier2;
8
+struct rrdengine_instance multidb_ctx_storage_tier3;
9
+struct rrdengine_instance multidb_ctx_storage_tier4;
10
+#if RRD_STORAGE_TIERS != 5
11
+#error RRD_STORAGE_TIERS is not 5 - you need to add allocations here
12
+#endif
13
+struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
14
+
15
+__attribute__((constructor)) void initialize_multidb_ctx(void) {
16
+ multidb_ctx[0] = &multidb_ctx_storage_tier0;
17
+ multidb_ctx[1] = &multidb_ctx_storage_tier1;
18
+ multidb_ctx[2] = &multidb_ctx_storage_tier2;
19
+ multidb_ctx[3] = &multidb_ctx_storage_tier3;
20
+ multidb_ctx[4] = &multidb_ctx_storage_tier4;
21
+}
22
23
int db_engine_use_malloc = 0;
24
int default_rrdeng_page_fetch_timeout = 3;
@@ -13,9 +29,10 @@ int default_multidb_disk_quota_mb = 256;
29
/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
30
uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
31
16
-static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host)
17
-{
18
- return host->rrdeng_ctx;
32
+static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host, int tier) {
33
+ if(tier < 0 || tier >= RRD_STORAGE_TIERS) tier = 0;
34
+ if(!host->storage_instance[tier]) tier = 0;
35
+ return (struct rrdengine_instance *)host->storage_instance[tier];
36
}
37
38
/* This UUID is not unique across hosts */
@@ -52,10 +69,20 @@ void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uu
69
memcpy(ret_uuid, hash_value, sizeof(uuid_t));
70
}
71
55
-void rrdeng_metric_init(RRDDIM *rd)
56
-{
57
- struct page_cache *pg_cache;
72
+struct rrdeng_metric_handle {
73
+ RRDDIM *rd;
74
struct rrdengine_instance *ctx;
75
+ uuid_t *rrdeng_uuid; // database engine metric UUID
76
+ struct pg_cache_page_index *page_index;
77
+};
78
+
79
+void rrdeng_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle) {
80
+ freez(db_metric_handle);
81
+}
82
+
83
+STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance) {
84
+ struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
85
+ struct page_cache *pg_cache;
86
uuid_t legacy_uuid;
87
uuid_t multihost_legacy_uuid;
88
Pvoid_t *PValue;
@@ -63,15 +90,10 @@ void rrdeng_metric_init(RRDDIM *rd)
90
int is_multihost_child = 0;
91
RRDHOST *host = rd->rrdset->rrdhost;
92
66
- ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);
67
- if (unlikely(!ctx)) {
68
- error("Failed to fetch multidb context");
69
- return;
70
- }
93
pg_cache = &ctx->pg_cache;
94
95
rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);
74
- if (host != localhost && host->rrdeng_ctx == &multidb_ctx)
96
+ if (host != localhost && is_storage_engine_shared((STORAGE_INSTANCE *)ctx))
97
is_multihost_child = 1;
98
99
uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
@@ -85,16 +107,16 @@ void rrdeng_metric_init(RRDDIM *rd)
107
* Drop legacy support, normal path */
108
109
uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
88
- PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &rd->state->metric_uuid, sizeof(uuid_t));
110
+ PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &rd->metric_uuid, sizeof(uuid_t));
111
if (likely(NULL != PValue)) {
112
page_index = *PValue;
113
}
114
uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
115
if (NULL == PValue) {
116
uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
95
- PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &rd->state->metric_uuid, sizeof(uuid_t), PJE0);
117
+ PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
118
fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
97
- *PValue = page_index = create_page_index(&rd->state->metric_uuid);
119
+ *PValue = page_index = create_page_index(&rd->metric_uuid);
120
page_index->prev = pg_cache->metrics_index.last_page_index;
121
pg_cache->metrics_index.last_page_index = page_index;
122
uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
@@ -105,52 +127,56 @@ void rrdeng_metric_init(RRDDIM *rd)
127
rrdeng_convert_legacy_uuid_to_multihost(rd->rrdset->rrdhost->machine_guid, &legacy_uuid,
128
&multihost_legacy_uuid);
129
108
- int need_to_store = uuid_compare(rd->state->metric_uuid, multihost_legacy_uuid);
130
+ int need_to_store = uuid_compare(rd->metric_uuid, multihost_legacy_uuid);
131
110
- uuid_copy(rd->state->metric_uuid, multihost_legacy_uuid);
132
+ uuid_copy(rd->metric_uuid, multihost_legacy_uuid);
133
112
- if (unlikely(need_to_store))
113
- (void)sql_store_dimension(&rd->state->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
134
+ if (unlikely(need_to_store && !ctx->tier))
135
+ (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
136
rd->algorithm);
115
-
137
}
117
- rd->state->rrdeng_uuid = &page_index->id;
118
- rd->state->page_index = page_index;
138
+
139
+ struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle));
140
+ mh->rd = rd;
141
+ mh->ctx = ctx;
142
+ mh->rrdeng_uuid = &page_index->id;
143
+ mh->page_index = page_index;
144
+ return (STORAGE_METRIC_HANDLE *)mh;
145
}
146
147
/*
148
* Gets a handle for storing metrics to the database.
149
* The handle must be released with rrdeng_store_metric_final().
150
*/
125
-void rrdeng_store_metric_init(RRDDIM *rd)
126
-{
151
+STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
152
+ struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
153
+
154
struct rrdeng_collect_handle *handle;
128
- struct rrdengine_instance *ctx;
155
struct pg_cache_page_index *page_index;
156
131
- ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);
132
-
157
handle = callocz(1, sizeof(struct rrdeng_collect_handle));
134
- handle->ctx = ctx;
158
+ handle->metric_handle = metric_handle;
159
+ handle->ctx = metric_handle->ctx;
160
handle->descr = NULL;
161
handle->unaligned_page = 0;
137
- rd->state->handle = (STORAGE_COLLECT_HANDLE *)handle;
162
139
- page_index = rd->state->page_index;
163
+ page_index = metric_handle->page_index;
164
uv_rwlock_wrlock(&page_index->lock);
165
++page_index->writers;
166
uv_rwlock_wrunlock(&page_index->lock);
167
+
168
+ return (STORAGE_COLLECT_HANDLE *)handle;
169
}
170
171
/* The page must be populated and referenced */
146
-static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr)
172
+static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr, size_t storage_size)
173
{
174
unsigned i;
175
uint8_t has_only_empty_metrics = 1;
176
storage_number *page;
177
178
page = descr->pg_cache_descr->page;
153
- for (i = 0 ; i < descr->page_length / sizeof(storage_number); ++i) {
179
+ for (i = 0 ; i < descr->page_length / storage_size; ++i) {
180
if (SN_EMPTY_SLOT != page[i]) {
181
has_only_empty_metrics = 0;
182
break;
@@ -159,26 +185,21 @@ static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr)
185
return has_only_empty_metrics;
186
}
187
162
-void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
163
-{
164
- struct rrdeng_collect_handle *handle;
165
- struct rrdengine_instance *ctx;
166
- struct rrdeng_page_descr *descr;
188
+void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle) {
189
+ struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
190
+ // struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
191
+ struct rrdengine_instance *ctx = handle->ctx;
192
+ struct rrdeng_page_descr *descr = handle->descr;
193
+
194
+ if (unlikely(!ctx)) return;
195
+ if (unlikely(!descr)) return;
196
168
- handle = (struct rrdeng_collect_handle *)rd->state->handle;
169
- ctx = handle->ctx;
170
- if (unlikely(!ctx))
171
- return;
172
- descr = handle->descr;
173
- if (unlikely(NULL == descr)) {
174
- return;
175
- }
197
if (likely(descr->page_length)) {
198
int page_is_empty;
199
200
rrd_stat_atomic_add(&ctx->stats.metric_API_producers, -1);
201
181
- page_is_empty = page_has_only_empty_metrics(descr);
202
+ page_is_empty = page_has_only_empty_metrics(descr, ctx->storage_size);
203
if (page_is_empty) {
204
debug(D_RRDENGINE, "Page has empty metrics only, deleting:");
205
if (unlikely(debug_flags & D_RRDENGINE))
@@ -195,20 +216,23 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
216
handle->descr = NULL;
217
}
218
198
-void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n, SN_FLAGS flags)
219
+void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE n,
220
+ NETDATA_DOUBLE min_value,
221
+ NETDATA_DOUBLE max_value,
222
+ uint16_t count,
223
+ uint16_t anomaly_count,
224
+ SN_FLAGS flags)
225
{
200
- storage_number number = pack_storage_number(n, flags);
226
+ struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
227
+ struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
228
+ struct rrdengine_instance *ctx = handle->ctx;
229
+ struct page_cache *pg_cache = &ctx->pg_cache;
230
+ struct rrdeng_page_descr *descr = handle->descr;
231
+ RRDDIM *rd = metric_handle->rd;
232
202
- struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)rd->state->handle;
203
- struct rrdengine_instance *ctx;
204
- struct page_cache *pg_cache;
205
- struct rrdeng_page_descr *descr;
206
- storage_number *page;
233
+ void *page;
234
uint8_t must_flush_unaligned_page = 0, perfect_page_alignment = 0;
208
-
209
- ctx = handle->ctx;
210
- pg_cache = &ctx->pg_cache;
211
- descr = handle->descr;
235
+ size_t storage_size = ctx->storage_size;
236
237
if (descr) {
238
/* Make alignment decisions */
@@ -218,7 +242,7 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n
242
perfect_page_alignment = 1;
243
}
244
/* is the metric far enough out of alignment with the others? */
221
- if (unlikely(descr->page_length + sizeof(number) < rd->rrdset->rrddim_page_alignment)) {
245
+ if (unlikely(descr->page_length + storage_size < rd->rrdset->rrddim_page_alignment)) {
246
handle->unaligned_page = 1;
247
debug(D_RRDENGINE, "Metric page is not aligned with chart:");
248
if (unlikely(debug_flags & D_RRDENGINE))
@@ -226,18 +250,18 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n
250
}
251
if (unlikely(handle->unaligned_page &&
252
/* did the other metrics change page? */
229
- rd->rrdset->rrddim_page_alignment <= sizeof(number))) {
253
+ rd->rrdset->rrddim_page_alignment <= storage_size)) {
254
debug(D_RRDENGINE, "Flushing unaligned metric page.");
255
must_flush_unaligned_page = 1;
256
handle->unaligned_page = 0;
257
}
258
}
259
if (unlikely(NULL == descr ||
236
- descr->page_length + sizeof(number) > RRDENG_BLOCK_SIZE ||
260
+ descr->page_length + storage_size > RRDENG_BLOCK_SIZE ||
261
must_flush_unaligned_page)) {
238
- rrdeng_store_metric_flush_current_page(rd);
262
+ rrdeng_store_metric_flush_current_page(collection_handle);
263
240
- page = rrdeng_create_page(ctx, &rd->state->page_index->id, &descr);
264
+ page = rrdeng_create_page(ctx, &metric_handle->page_index->id, &descr);
265
fatal_assert(page);
266
267
handle->descr = descr;
@@ -249,9 +273,37 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n
273
perfect_page_alignment = 1;
274
}
275
}
276
+
277
page = descr->pg_cache_descr->page;
253
- page[descr->page_length / sizeof(number)] = number;
254
- pg_cache_atomic_set_pg_info(descr, point_in_time, descr->page_length + sizeof(number));
278
+
279
+ switch (descr->type) {
280
+ case PAGE_METRICS: {
281
+ ((storage_number *)page)[descr->page_length / storage_size] = pack_storage_number(n, flags);
282
+ }
283
+ break;
284
+
285
+ case PAGE_TIER: {
286
+ storage_number_tier1_t number_tier1;
287
+ number_tier1.sum_value = (float)n;
288
+ number_tier1.min_value = (float)min_value;
289
+ number_tier1.max_value = (float)max_value;
290
+ number_tier1.anomaly_count = anomaly_count;
291
+ number_tier1.count = count;
292
+ ((storage_number_tier1_t *)page)[descr->page_length / storage_size] = number_tier1;
293
+ }
294
+ break;
295
+
296
+ default: {
297
+ static bool logged = false;
298
+ if(!logged) {
299
+ error("DBENGINE: cannot store metric on unknown page type id %d", descr->type);
300
+ logged = true;
301
+ }
302
+ }
303
+ break;
304
+ }
305
+
306
+ pg_cache_atomic_set_pg_info(descr, point_in_time, descr->page_length + storage_size);
307
308
if (perfect_page_alignment)
309
rd->rrdset->rrddim_page_alignment = descr->page_length;
@@ -271,9 +323,9 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n
323
}
324
}
325
274
- pg_cache_insert(ctx, rd->state->page_index, descr);
326
+ pg_cache_insert(ctx, metric_handle->page_index, descr);
327
} else {
276
- pg_cache_add_new_metric_time(rd->state->page_index, descr);
328
+ pg_cache_add_new_metric_time(metric_handle->page_index, descr);
329
}
330
}
331
@@ -281,15 +333,14 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n
333
* Releases the database reference from the handle for storing metrics.
334
* Returns 1 if it's safe to delete the dimension.
335
*/
284
-int rrdeng_store_metric_finalize(RRDDIM *rd)
285
-{
286
- struct rrdeng_collect_handle *handle;
287
- struct pg_cache_page_index *page_index;
336
+int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
337
+ struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
338
+ struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
339
+ struct pg_cache_page_index *page_index = metric_handle->page_index;
340
+
341
uint8_t can_delete_metric = 0;
342
290
- handle = (struct rrdeng_collect_handle *)rd->state->handle;
291
- page_index = rd->state->page_index;
292
- rrdeng_store_metric_flush_current_page(rd);
343
+ rrdeng_store_metric_flush_current_page(collection_handle);
344
uv_rwlock_wrlock(&page_index->lock);
345
if (!--page_index->writers && !page_index->page_count) {
346
can_delete_metric = 1;
@@ -297,242 +348,52 @@ int rrdeng_store_metric_finalize(RRDDIM *rd)
348
uv_rwlock_wrunlock(&page_index->lock);
349
freez(handle);
350
300
- return can_delete_metric;
301
-}
302
-
303
-/* Returns 1 if the data collection interval is well defined, 0 otherwise */
304
-static int metrics_with_known_interval(struct rrdeng_page_descr *descr)
305
-{
306
- unsigned page_entries;
307
-
308
- if (unlikely(INVALID_TIME == descr->start_time || INVALID_TIME == descr->end_time))
309
- return 0;
310
- page_entries = descr->page_length / sizeof(storage_number);
311
- if (likely(page_entries > 1)) {
312
- return 1;
313
- }
314
- return 0;
315
-}
316
-
317
-static inline uint32_t *pginfo_to_dt(struct rrdeng_page_info *page_info)
318
-{
319
- return (uint32_t *)&page_info->scratch[0];
320
-}
321
-
322
-static inline uint32_t *pginfo_to_points(struct rrdeng_page_info *page_info)
323
-{
324
- return (uint32_t *)&page_info->scratch[sizeof(uint32_t)];
325
-}
326
-
327
-/**
328
- * Calculates the regions of different data collection intervals in a netdata chart in the time range
329
- * [start_time,end_time]. This call takes the netdata chart read lock.
330
- * @param st the netdata chart whose data collection interval boundaries are calculated.
331
- * @param start_time inclusive starting time in usec
332
- * @param end_time inclusive ending time in usec
333
- * @param region_info_arrayp It allocates (*region_info_arrayp) and populates it with information of regions of a
334
- * reference dimension that that have different data collection intervals and overlap with the time range
335
- * [start_time,end_time]. The caller must free (*region_info_arrayp) with freez(). If region_info_arrayp is set
336
- * to NULL nothing was allocated.
337
- * @param max_intervalp is dereferenced and set to be the largest data collection interval of all regions.
338
- * @return number of regions with different data collection intervals.
339
- */
340
-unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
341
- struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list)
342
-{
343
- struct pg_cache_page_index *page_index;
344
- struct rrdengine_instance *ctx;
345
- unsigned pages_nr;
346
- RRDDIM *rd_iter, *rd;
347
- struct rrdeng_page_info *page_info_array, *curr, *prev, *old_prev;
348
- unsigned i, j, page_entries, region_points, page_points, regions, max_interval;
349
- time_t now;
350
- usec_t dt, current_position_time, max_time = 0, min_time, curr_time, first_valid_time_in_page;
351
- struct rrdeng_region_info *region_info_array;
352
- uint8_t is_first_region_initialized;
353
-
354
- ctx = get_rrdeng_ctx_from_host(st->rrdhost);
355
- regions = 1;
356
- *max_intervalp = max_interval = 0;
357
- region_info_array = NULL;
358
- *region_info_arrayp = NULL;
359
- page_info_array = NULL;
360
-
361
- RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
362
- rrdset_rdlock(st);
363
- for(rd_iter = temp_rd?temp_rd:st->dimensions, rd = NULL, min_time = (usec_t)-1 ; rd_iter ; rd_iter = rd_iter->next) {
364
- /*
365
- * Choose oldest dimension as reference. This is not equivalent to the union of all dimensions
366
- * but it is a best effort approximation with a bias towards older metrics in a chart. It
367
- * matches netdata behaviour in the sense that dimensions are generally aligned in a chart
368
- * and older dimensions contain more information about the time range. It does not work well
369
- * for metrics that have recently stopped being collected.
370
- */
371
- curr_time = pg_cache_oldest_time_in_range(ctx, rd_iter->state->rrdeng_uuid,
372
- start_time * USEC_PER_SEC, end_time * USEC_PER_SEC);
373
- if (INVALID_TIME != curr_time && curr_time < min_time) {
374
- rd = rd_iter;
375
- min_time = curr_time;
376
- }
377
- }
378
- rrdset_unlock(st);
379
- if (NULL == rd) {
380
- return 1;
381
- }
382
- pages_nr = pg_cache_preload(ctx, rd->state->rrdeng_uuid, start_time * USEC_PER_SEC, end_time * USEC_PER_SEC,
383
- &page_info_array, &page_index);
384
- if (pages_nr) {
385
- /* conservative allocation, will reduce the size later if necessary */
386
- region_info_array = mallocz(sizeof(*region_info_array) * pages_nr);
387
- }
388
- is_first_region_initialized = 0;
389
- region_points = 0;
390
-
391
- int is_out_of_order_reported = 0;
392
- /* pages loop */
393
- for (i = 0, curr = NULL, prev = NULL ; i < pages_nr ; ++i) {
394
- old_prev = prev;
395
- prev = curr;
396
- curr = &page_info_array[i];
397
- *pginfo_to_points(curr) = 0; /* initialize to invalid page */
398
- *pginfo_to_dt(curr) = 0; /* no known data collection interval yet */
399
- if (unlikely(INVALID_TIME == curr->start_time || INVALID_TIME == curr->end_time ||
400
- curr->end_time < curr->start_time)) {
401
- info("Ignoring page with invalid timestamps.");
402
- prev = old_prev;
403
- continue;
404
- }
405
- page_entries = curr->page_length / sizeof(storage_number);
406
- fatal_assert(0 != page_entries);
407
- if (likely(1 != page_entries)) {
408
- dt = (curr->end_time - curr->start_time) / (page_entries - 1);
409
- *pginfo_to_dt(curr) = ROUND_USEC_TO_SEC(dt);
410
- if (unlikely(0 == *pginfo_to_dt(curr)))
411
- *pginfo_to_dt(curr) = 1;
412
- } else {
413
- dt = 0;
414
- }
415
- for (j = 0, page_points = 0 ; j < page_entries ; ++j) {
416
- uint8_t is_metric_out_of_order, is_metric_earlier_than_range;
417
-
418
- is_metric_earlier_than_range = 0;
419
- is_metric_out_of_order = 0;
420
-
421
- current_position_time = curr->start_time + j * dt;
422
- now = current_position_time / USEC_PER_SEC;
423
- if (now > end_time) { /* there will be no more pages in the time range */
424
- break;
425
- }
426
- if (now < start_time)
427
- is_metric_earlier_than_range = 1;
428
- if (unlikely(current_position_time < max_time)) /* just went back in time */
429
- is_metric_out_of_order = 1;
430
- if (is_metric_earlier_than_range || unlikely(is_metric_out_of_order)) {
431
- if (unlikely(is_metric_out_of_order))
432
- is_out_of_order_reported++;
433
- continue; /* next entry */
434
- }
435
- /* here is a valid metric */
436
- ++page_points;
437
- region_info_array[regions - 1].points = ++region_points;
438
- max_time = current_position_time;
439
- if (1 == page_points)
440
- first_valid_time_in_page = current_position_time;
441
- if (unlikely(!is_first_region_initialized)) {
442
- fatal_assert(1 == regions);
443
- /* this is the first region */
444
- region_info_array[0].start_time = current_position_time;
445
- is_first_region_initialized = 1;
446
- }
447
- }
448
- *pginfo_to_points(curr) = page_points;
449
- if (0 == page_points) {
450
- prev = old_prev;
451
- continue;
452
- }
453
-
454
- if (unlikely(0 == *pginfo_to_dt(curr))) { /* unknown data collection interval */
455
- fatal_assert(1 == page_points);
456
-
457
- if (likely(NULL != prev)) { /* get interval from previous page */
458
- *pginfo_to_dt(curr) = *pginfo_to_dt(prev);
459
- } else { /* there is no previous page in the query */
460
- struct rrdeng_page_info db_page_info;
461
-
462
- /* go to database */
463
- pg_cache_get_filtered_info_prev(ctx, page_index, curr->start_time,
464
- metrics_with_known_interval, &db_page_info);
465
- if (unlikely(db_page_info.start_time == INVALID_TIME || db_page_info.end_time == INVALID_TIME ||
466
- 0 == db_page_info.page_length)) { /* nothing in the database, default to update_every */
467
- *pginfo_to_dt(curr) = rd->update_every;
468
- } else {
469
- unsigned db_entries;
470
- usec_t db_dt;
471
-
472
- db_entries = db_page_info.page_length / sizeof(storage_number);
473
- db_dt = (db_page_info.end_time - db_page_info.start_time) / (db_entries - 1);
474
- *pginfo_to_dt(curr) = ROUND_USEC_TO_SEC(db_dt);
475
- if (unlikely(0 == *pginfo_to_dt(curr)))
476
- *pginfo_to_dt(curr) = 1;
477
-
478
- }
479
- }
480
- }
481
- if (likely(prev) && unlikely(*pginfo_to_dt(curr) != *pginfo_to_dt(prev))) {
482
- info("Data collection interval change detected in query: %"PRIu32" -> %"PRIu32,
483
- *pginfo_to_dt(prev), *pginfo_to_dt(curr));
484
- region_info_array[regions++ - 1].points -= page_points;
485
- region_info_array[regions - 1].points = region_points = page_points;
486
- region_info_array[regions - 1].start_time = first_valid_time_in_page;
487
- }
488
- if (*pginfo_to_dt(curr) > max_interval)
489
- max_interval = *pginfo_to_dt(curr);
490
- region_info_array[regions - 1].update_every = *pginfo_to_dt(curr);
491
- }
492
- if (page_info_array)
493
- freez(page_info_array);
494
- if (region_info_array) {
495
- if (likely(is_first_region_initialized)) {
496
- /* free unnecessary memory */
497
- region_info_array = reallocz(region_info_array, sizeof(*region_info_array) * regions);
498
- *region_info_arrayp = region_info_array;
499
- *max_intervalp = max_interval;
500
- } else {
501
- /* empty result */
502
- freez(region_info_array);
503
- }
504
- }
505
- if (is_out_of_order_reported)
506
- info("Ignored %d metrics with out of order timestamp in %u regions.", is_out_of_order_reported, regions);
507
- return regions;
351
+ return can_delete_metric;
352
}
353
354
+//static inline uint32_t *pginfo_to_dt(struct rrdeng_page_info *page_info)
355
+//{
356
+// return (uint32_t *)&page_info->scratch[0];
357
+//}
358
+//
359
+//static inline uint32_t *pginfo_to_points(struct rrdeng_page_info *page_info)
360
+//{
361
+// return (uint32_t *)&page_info->scratch[sizeof(uint32_t)];
362
+//}
363
+//
364
/*
365
* Gets a handle for loading metrics from the database.
366
* The handle must be released with rrdeng_load_metric_final().
367
*/
514
-void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle, time_t start_time, time_t end_time)
368
+void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type)
369
{
370
+ struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
371
+ struct rrdengine_instance *ctx = metric_handle->ctx;
372
+ RRDDIM *rd = metric_handle->rd;
373
+
374
// fprintf(stderr, "%s: %s/%s start time %ld, end time %ld\n", __FUNCTION__ , rd->rrdset->name, rd->name, start_time, end_time);
375
376
struct rrdeng_query_handle *handle;
519
- struct rrdengine_instance *ctx;
377
unsigned pages_nr;
378
522
- ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);
379
rrdimm_handle->start_time = start_time;
380
rrdimm_handle->end_time = end_time;
381
382
handle = callocz(1, sizeof(struct rrdeng_query_handle));
383
handle->next_page_time = start_time;
384
handle->now = start_time;
529
- handle->dt = rd->update_every * USEC_PER_SEC;
530
- handle->dt_sec = rd->update_every;
385
+ handle->tier_query_fetch_type = tier_query_fetch_type;
386
+ // TODO we should store the dt of each page in each page
387
+ // this will produce wrong values for dt in case the user changes
388
+ // the update every of the charts or the tier grouping iterations
389
+ handle->dt_sec = get_tier_grouping(ctx->tier) * (time_t)rd->update_every;
390
+ handle->dt = handle->dt_sec * USEC_PER_SEC;
391
handle->position = 0;
392
handle->ctx = ctx;
393
+ handle->metric_handle = metric_handle;
394
handle->descr = NULL;
395
rrdimm_handle->handle = (STORAGE_QUERY_HANDLE *)handle;
535
- pages_nr = pg_cache_preload(ctx, rd->state->rrdeng_uuid, start_time * USEC_PER_SEC, end_time * USEC_PER_SEC,
396
+ pages_nr = pg_cache_preload(ctx, metric_handle->rrdeng_uuid, start_time * USEC_PER_SEC, end_time * USEC_PER_SEC,
397
NULL, &handle->page_index);
398
if (unlikely(NULL == handle->page_index || 0 == pages_nr))
399
// there are no metrics to load
@@ -580,7 +441,7 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
441
442
if (unlikely(descr->start_time != page_end_time && next_page_time > descr->start_time)) {
443
// we're in the middle of the page somewhere
583
- unsigned entries = page_length / sizeof(storage_number);
444
+ unsigned entries = page_length / ctx->storage_size;
445
position = ((uint64_t)(next_page_time - descr->start_time)) * (entries - 1) /
446
(page_end_time - descr->start_time);
447
}
@@ -590,9 +451,14 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
451
handle->page_end_time = page_end_time;
452
handle->page_length = page_length;
453
handle->page = descr->pg_cache_descr->page;
593
- usec_t entries = handle->entries = page_length / sizeof(storage_number);
454
+ usec_t entries = handle->entries = page_length / ctx->storage_size;
455
if (likely(entries > 1))
456
handle->dt = (page_end_time - descr->start_time) / (entries - 1);
457
+ else {
458
+ // TODO we should store the dt of each page in each page
459
+ // now we keep the dt of whatever was before
460
+ ;
461
+ }
462
463
handle->dt_sec = (time_t)(handle->dt / USEC_PER_SEC);
464
handle->position = position;
@@ -603,21 +469,21 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
469
// Returns the metric and sets its timestamp into current_time
470
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
471
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
606
-NETDATA_DOUBLE
607
-rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
472
+STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle) {
473
struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
474
+ // struct rrdeng_metric_handle *metric_handle = handle->metric_handle;
475
476
+ STORAGE_POINT sp;
477
struct rrdeng_page_descr *descr = handle->descr;
478
unsigned position = handle->position + 1;
479
time_t now = handle->now + handle->dt_sec;
480
+ storage_number_tier1_t tier1_value;
481
482
if (unlikely(INVALID_TIME == handle->next_page_time)) {
483
handle->next_page_time = INVALID_TIME;
484
handle->now = now;
617
- *start_time = now - handle->dt_sec;
618
- *end_time = now;
619
- *flags = SN_EMPTY_SLOT;
620
- return NAN;
485
+ storage_point_empty(sp, now - handle->dt_sec, now);
486
+ return sp;
487
}
488
489
if (unlikely(!descr || position >= handle->entries)) {
@@ -626,10 +492,8 @@ rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start
492
// next calls will not load any more metrics
493
handle->next_page_time = INVALID_TIME;
494
handle->now = now;
629
- *start_time = now - handle->dt_sec;
630
- *end_time = now;
631
- *flags = SN_EMPTY_SLOT;
632
- return NAN;
495
+ storage_point_empty(sp, now - handle->dt_sec, now);
496
+ return sp;
497
}
498
499
descr = handle->descr;
@@ -637,19 +501,51 @@ rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start
501
now = (time_t)((descr->start_time + position * handle->dt) / USEC_PER_SEC);
502
}
503
640
- storage_number n = handle->page[position];
504
+ sp.start_time = now - handle->dt_sec;
505
+ sp.end_time = now;
506
+
507
handle->position = position;
508
handle->now = now;
509
510
+ switch(descr->type) {
511
+ case PAGE_METRICS: {
512
+ storage_number n = handle->page[position];
513
+ sp.min = sp.max = sp.sum = unpack_storage_number(n);
514
+ sp.flags = n & SN_ALL_FLAGS;
515
+ sp.count = 1;
516
+ sp.anomaly_count = (n & SN_ANOMALY_BIT) ? 0 : 1;
517
+ }
518
+ break;
519
+
520
+ case PAGE_TIER: {
521
+ tier1_value = ((storage_number_tier1_t *)handle->page)[position];
522
+ sp.flags = tier1_value.anomaly_count ? 0 : SN_ANOMALY_BIT;
523
+ sp.count = tier1_value.count;
524
+ sp.anomaly_count = tier1_value.anomaly_count;
525
+ sp.min = tier1_value.min_value;
526
+ sp.max = tier1_value.max_value;
527
+ sp.sum = tier1_value.sum_value;
528
+ }
529
+ break;
530
+
531
+ // we don't know this page type
532
+ default: {
533
+ static bool logged = false;
534
+ if(!logged) {
535
+ error("DBENGINE: unknown page type %d found. Cannot decode it. Ignoring its metrics.", descr->type);
536
+ logged = true;
537
+ }
538
+ storage_point_empty(sp, sp.start_time, sp.end_time);
539
+ }
540
+ break;
541
+ }
542
+
543
if (unlikely(now >= rrdimm_handle->end_time)) {
544
// next calls will not load any more metrics
545
handle->next_page_time = INVALID_TIME;
546
}
547
649
- *flags = n & SN_ALL_FLAGS;
650
- *start_time = now - handle->dt_sec;
651
- *end_time = now;
652
- return unpack_storage_number(n);
548
+ return sp;
549
}
550
551
int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle)
@@ -679,31 +575,27 @@ void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
575
rrdimm_handle->handle = NULL;
576
}
577
682
-time_t rrdeng_metric_latest_time(RRDDIM *rd)
683
-{
684
- struct pg_cache_page_index *page_index;
685
-
686
- page_index = rd->state->page_index;
578
+time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
579
+ struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
580
581
+ struct pg_cache_page_index *page_index = metric_handle->page_index;
582
return page_index->latest_time / USEC_PER_SEC;
583
}
690
-time_t rrdeng_metric_oldest_time(RRDDIM *rd)
691
-{
692
- struct pg_cache_page_index *page_index;
693
-
694
- page_index = rd->state->page_index;
584
+time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
585
+ struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
586
587
+ struct pg_cache_page_index *page_index = metric_handle->page_index;
588
return page_index->oldest_time / USEC_PER_SEC;
589
}
590
699
-int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t)
591
+int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier)
592
{
593
struct page_cache *pg_cache;
594
struct rrdengine_instance *ctx;
595
Pvoid_t *PValue;
596
struct pg_cache_page_index *page_index = NULL;
597
706
- ctx = get_rrdeng_ctx_from_host(localhost);
598
+ ctx = get_rrdeng_ctx_from_host(localhost, tier);
599
if (unlikely(!ctx)) {
600
error("Failed to fetch multidb context");
601
return 1;
@@ -736,6 +628,7 @@ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrde
628
629
descr = pg_cache_create_descr();
630
descr->id = id; /* TODO: add page type: metric, log, something? */
631
+ descr->type = ctx->page_type;
632
page = dbengine_page_alloc(); /*TODO: add page size */
633
rrdeng_page_descr_mutex_lock(ctx, descr);
634
pg_cache_descr = descr->pg_cache_descr;
@@ -899,8 +792,7 @@ void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle)
792
* Returns 0 on success, negative on error
793
*/
794
int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
902
- unsigned disk_space_mb)
903
-{
795
+ unsigned disk_space_mb, int tier) {
796
struct rrdengine_instance *ctx;
797
int error;
798
uint32_t max_open_files;
@@ -912,18 +804,24 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
804
if (rrdeng_reserved_file_descriptors > max_open_files) {
805
error(
806
"Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
915
- (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
807
+ (unsigned)rrdeng_reserved_file_descriptors,
808
+ (unsigned)max_open_files);
809
810
rrd_stat_atomic_add(&global_fs_errors, 1);
811
rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
812
return UV_EMFILE;
813
}
814
922
- if (NULL == ctxp) {
923
- ctx = &multidb_ctx;
815
+ if(NULL == ctxp) {
816
+ ctx = multidb_ctx[tier];
817
memset(ctx, 0, sizeof(*ctx));
925
- } else {
818
+ ctx->storage_size = (tier == 0) ? sizeof(storage_number) : sizeof(storage_number_tier1_t);
819
+ ctx->tier = tier;
820
+ ctx->page_type = !tier ? PAGE_METRICS : PAGE_TIER; // TODO: In the future it can be different page type per tier
821
+ }
822
+ else {
823
*ctxp = ctx = callocz(1, sizeof(*ctx));
824
+ ctx->storage_size = sizeof(storage_number);
825
}
826
ctx->global_compress_alg = RRD_LZ4;
827
if (page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB)
@@ -977,7 +875,7 @@ error_after_rrdeng_worker:
875
finalize_rrd_files(ctx);
876
error_after_init_rrd_files:
877
free_page_cache(ctx);
980
- if (ctx != &multidb_ctx) {
878
+ if (!is_storage_engine_shared((STORAGE_INSTANCE *)ctx)) {
879
freez(ctx);
880
*ctxp = NULL;
881
}
@@ -1006,9 +904,9 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
904
//metalog_exit(ctx->metalog_ctx);
905
free_page_cache(ctx);
906
1009
- if (ctx != &multidb_ctx) {
907
+ if(!is_storage_engine_shared((STORAGE_INSTANCE *)ctx))
908
freez(ctx);
1011
- }
909
+
910
rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
911
return 0;
912
}
database/engine/rrdengineapi.h
+25
-15
@@ -19,7 +19,7 @@ extern int default_rrdeng_page_cache_mb;
19
extern int default_rrdeng_disk_quota_mb;
20
extern int default_multidb_disk_quota_mb;
21
extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;
22
-extern struct rrdengine_instance multidb_ctx;
22
+extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
23
24
struct rrdeng_region_info {
25
time_t start_time;
@@ -39,29 +39,39 @@ extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN +
39
uuid_t *ret_uuid);
40
41
42
-extern void rrdeng_metric_init(RRDDIM *rd);
43
-extern void rrdeng_store_metric_init(RRDDIM *rd);
44
-extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);
45
-extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
46
-extern int rrdeng_store_metric_finalize(RRDDIM *rd);
47
-extern unsigned
48
- rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
42
+extern STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
43
+extern void rrdeng_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle);
44
+
45
+extern STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle);
46
+extern void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
47
+extern void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE n,
48
+ NETDATA_DOUBLE min_value,
49
+ NETDATA_DOUBLE max_value,
50
+ uint16_t count,
51
+ uint16_t anomaly_count,
52
+ SN_FLAGS flags);
53
+extern int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
54
+
55
+extern unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
56
struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
50
-extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,
51
- time_t start_time, time_t end_time);
52
-extern NETDATA_DOUBLE rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
57
+
58
+extern void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle,
59
+ time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
60
+extern STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle);
61
+
62
extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);
63
extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
55
-extern time_t rrdeng_metric_latest_time(RRDDIM *rd);
56
-extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
64
+extern time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
65
+extern time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
66
+
67
extern void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
68
69
/* must call once before using anything */
70
extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
61
- unsigned disk_space_mb);
71
+ unsigned disk_space_mb, int tier);
72
73
extern int rrdeng_exit(struct rrdengine_instance *ctx);
74
extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
65
-extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
75
+extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier);
76
77
#endif /* NETDATA_RRDENGINEAPI_H */
database/metric_correlations.c
+7
-7
@@ -401,7 +401,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
401
high_rrdr = rrd2rrdr(owa, st, points,
402
after, before, group,
403
group_time, options, NULL, context_param_list, group_options,
404
- timeout);
404
+ timeout, 0);
405
if(!high_rrdr) {
406
info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name);
407
goto cleanup;
@@ -427,7 +427,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
427
base_rrdr = rrd2rrdr(owa, st,high_points << shifts,
428
baseline_after, baseline_before, group,
429
group_time, options, NULL, context_param_list, group_options,
430
- (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)));
430
+ (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), 0);
431
if(!base_rrdr) {
432
info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name);
433
goto cleanup;
@@ -549,7 +549,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
549
group, group_options, group_time, options,
550
NULL, NULL,
551
&stats->db_points, &stats->result_points,
552
- &value_is_null, &base_anomaly_rate, 0);
552
+ &value_is_null, &base_anomaly_rate, 0, 0);
553
554
if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) {
555
// this means no data for the baseline window, but we may have data for the highlighted one - assume zero
@@ -565,7 +565,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
565
group, group_options, group_time, options,
566
NULL, NULL,
567
&stats->db_points, &stats->result_points,
568
- &value_is_null, &high_anomaly_rate, 0);
568
+ &value_is_null, &high_anomaly_rate, 0, 0);
569
570
if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) {
571
// this means no data for the highlighted duration - so skip it
@@ -590,7 +590,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
590
group_time, options,
591
NULL, NULL,
592
&stats->db_points, &stats->result_points,
593
- &value_is_null, NULL, 0);
593
+ &value_is_null, NULL, 0, 0);
594
595
if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) {
596
info("MC: highlighted countif query failed, but highlighted average worked - strange...");
@@ -733,12 +733,12 @@ int metric_correlations(RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD me
733
734
if(!points) points = 500;
735
736
- rrdr_relative_window_to_absolute(&after, &before, default_rrd_update_every, points);
736
+ rrdr_relative_window_to_absolute(&after, &before);
737
738
if(baseline_before <= API_RELATIVE_TIME_MAX)
739
baseline_before += after;
740
741
- rrdr_relative_window_to_absolute(&baseline_after, &baseline_before, default_rrd_update_every, points * 4);
741
+ rrdr_relative_window_to_absolute(&baseline_after, &baseline_before);
742
743
if (before <= after || baseline_before <= baseline_after) {
744
buffer_strcat(wb, "{\"error\": \"Invalid baseline or highlight ranges.\" }");
database/ram/rrddim_mem.c
+150
-26
@@ -5,34 +5,151 @@
5
// ----------------------------------------------------------------------------
6
// RRDDIM legacy data collection functions
7
8
-void rrddim_collect_init(RRDDIM *rd) {
8
+STORAGE_METRIC_HANDLE *rrddim_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance __maybe_unused) {
9
+ return (STORAGE_METRIC_HANDLE *)rd;
10
+}
11
+
12
+void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) {
13
+ ;
14
+}
15
+
16
+STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
17
+ RRDDIM *rd = (RRDDIM *)db_metric_handle;
18
rd->db[rd->rrdset->current_entry] = SN_EMPTY_SLOT;
10
- rd->state->handle = calloc(1, sizeof(struct mem_collect_handle));
19
+ struct mem_collect_handle *ch = calloc(1, sizeof(struct mem_collect_handle));
20
+ ch->rd = rd;
21
+ return (STORAGE_COLLECT_HANDLE *)ch;
22
}
12
-void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags) {
13
- (void)point_in_time;
23
+
24
+void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number,
25
+ NETDATA_DOUBLE min_value,
26
+ NETDATA_DOUBLE max_value,
27
+ uint16_t count,
28
+ uint16_t anomaly_count,
29
+ SN_FLAGS flags)
30
+{
31
+ UNUSED(point_in_time);
32
+ UNUSED(min_value);
33
+ UNUSED(max_value);
34
+ UNUSED(count);
35
+ UNUSED(anomaly_count);
36
+
37
+ struct mem_collect_handle *ch = (struct mem_collect_handle *)collection_handle;
38
+ RRDDIM *rd = ch->rd;
39
rd->db[rd->rrdset->current_entry] = pack_storage_number(number, flags);
40
}
16
-int rrddim_collect_finalize(RRDDIM *rd) {
17
- free((struct mem_collect_handle*)rd->state->handle);
41
+
42
+void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
43
+ struct mem_collect_handle *ch = (struct mem_collect_handle *)collection_handle;
44
+ RRDDIM *rd = ch->rd;
45
+ memset(rd->db, 0, rd->entries * sizeof(storage_number));
46
+}
47
+
48
+int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
49
+ free(collection_handle);
50
return 0;
51
}
52
53
+// ----------------------------------------------------------------------------
54
+
55
+// get the total duration in seconds of the round robin database
56
+#define rrddim_duration(st) (( (time_t)(rd)->rrdset->counter >= (time_t)(rd)->rrdset->entries ? (time_t)(rd)->rrdset->entries : (time_t)(rd)->rrdset->counter ) * (time_t)(rd)->rrdset->update_every)
57
+
58
+// get the last slot updated in the round robin database
59
+#define rrddim_last_slot(rd) ((size_t)(((rd)->rrdset->current_entry == 0) ? (rd)->rrdset->entries - 1 : (rd)->rrdset->current_entry - 1))
60
+
61
+// return the slot that has the oldest value
62
+#define rrddim_first_slot(rd) ((size_t)((rd)->rrdset->counter >= (size_t)(rd)->rrdset->entries ? (rd)->rrdset->current_entry : 0))
63
+
64
+// get the slot of the round robin database, for the given timestamp (t)
65
+// it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database
66
+// only valid when not using dbengine
67
+static inline size_t rrddim_time2slot(RRDDIM *rd, time_t t) {
68
+ size_t ret = 0;
69
+ time_t last_entry_t = rrddim_query_latest_time((STORAGE_METRIC_HANDLE *)rd);
70
+ time_t first_entry_t = rrddim_query_oldest_time((STORAGE_METRIC_HANDLE *)rd);
71
+ size_t entries = rd->rrdset->entries;
72
+ size_t first_slot = rrddim_first_slot(rd);
73
+ size_t last_slot = rrddim_last_slot(rd);
74
+ size_t update_every = rd->rrdset->update_every;
75
+
76
+ if(t >= last_entry_t) {
77
+ // the requested time is after the last entry we have
78
+ ret = last_slot;
79
+ }
80
+ else {
81
+ if(t <= first_entry_t) {
82
+ // the requested time is before the first entry we have
83
+ ret = first_slot;
84
+ }
85
+ else {
86
+ if(last_slot >= (size_t)((last_entry_t - t) / update_every))
87
+ ret = last_slot - ((last_entry_t - t) / update_every);
88
+ else
89
+ ret = last_slot - ((last_entry_t - t) / update_every) + entries;
90
+ }
91
+ }
92
+
93
+ if(unlikely(ret >= entries)) {
94
+ error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rd->name);
95
+ ret = entries - 1;
96
+ }
97
+
98
+ return ret;
99
+}
100
+
101
+// get the timestamp of a specific slot in the round robin database
102
+// only valid when not using dbengine
103
+static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) {
104
+ time_t ret;
105
+ time_t last_entry_t = rrddim_query_latest_time((STORAGE_METRIC_HANDLE *)rd);
106
+ time_t first_entry_t = rrddim_query_oldest_time((STORAGE_METRIC_HANDLE *)rd);
107
+ size_t entries = rd->rrdset->entries;
108
+ size_t last_slot = rrddim_last_slot(rd);
109
+ size_t update_every = rd->rrdset->update_every;
110
+
111
+ if(slot >= entries) {
112
+ error("INTERNAL ERROR: caller of rrddim_slot2time() gives invalid slot %zu", slot);
113
+ slot = entries - 1;
114
+ }
115
+
116
+ if(slot > last_slot)
117
+ ret = last_entry_t - (time_t)(update_every * (last_slot - slot + entries));
118
+ else
119
+ ret = last_entry_t - (time_t)(update_every * (last_slot - slot));
120
+
121
+ if(unlikely(ret < first_entry_t)) {
122
+ error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rd->name);
123
+ ret = first_entry_t;
124
+ }
125
+
126
+ if(unlikely(ret > last_entry_t)) {
127
+ error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rd->name);
128
+ ret = last_entry_t;
129
+ }
130
+
131
+ return ret;
132
+}
133
+
134
// ----------------------------------------------------------------------------
135
// RRDDIM legacy database query functions
136
24
-void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time) {
137
+void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type) {
138
+ UNUSED(tier_query_fetch_type);
139
+
140
+ RRDDIM *rd = (RRDDIM *)db_metric_handle;
141
+
142
handle->rd = rd;
143
handle->start_time = start_time;
144
handle->end_time = end_time;
145
struct mem_query_handle* h = calloc(1, sizeof(struct mem_query_handle));
29
- h->slot = rrdset_time2slot(rd->rrdset, start_time);
30
- h->last_slot = rrdset_time2slot(rd->rrdset, end_time);
31
- h->dt = rd->update_every;
146
+ h->slot = rrddim_time2slot(rd, start_time);
147
+ h->last_slot = rrddim_time2slot(rd, end_time);
148
+ h->dt = rd->rrdset->update_every;
149
150
h->next_timestamp = start_time;
34
- h->slot_timestamp = rrdset_slot2time(rd->rrdset, h->slot);
35
- h->last_timestamp = rrdset_slot2time(rd->rrdset, h->last_slot);
151
+ h->slot_timestamp = rrddim_slot2time(rd, h->slot);
152
+ h->last_timestamp = rrddim_slot2time(rd, h->last_slot);
153
154
// info("RRDDIM QUERY INIT: start %ld, end %ld, next %ld, first %ld, last %ld, dt %ld", start_time, end_time, h->next_timestamp, h->slot_timestamp, h->last_timestamp, h->dt);
155
@@ -42,28 +159,30 @@ void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t st
159
// Returns the metric and sets its timestamp into current_time
160
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
161
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
45
-NETDATA_DOUBLE
46
-rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
162
+STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle) {
163
RRDDIM *rd = handle->rd;
164
struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
165
size_t entries = rd->rrdset->entries;
166
size_t slot = h->slot;
167
168
+ STORAGE_POINT sp;
169
+ sp.count = 1;
170
+
171
time_t this_timestamp = h->next_timestamp;
172
h->next_timestamp += h->dt;
173
174
// set this timestamp for our caller
56
- *start_time = this_timestamp - h->dt;
57
- *end_time = this_timestamp;
175
+ sp.start_time = this_timestamp - h->dt;
176
+ sp.end_time = this_timestamp;
177
178
if(unlikely(this_timestamp < h->slot_timestamp)) {
60
- *flags = SN_EMPTY_SLOT;
61
- return NAN;
179
+ storage_point_empty(sp, sp.start_time, sp.end_time);
180
+ return sp;
181
}
182
183
if(unlikely(this_timestamp > h->last_timestamp)) {
65
- *flags = SN_EMPTY_SLOT;
66
- return NAN;
184
+ storage_point_empty(sp, sp.start_time, sp.end_time);
185
+ return sp;
186
}
187
188
storage_number n = rd->db[slot++];
@@ -72,8 +191,11 @@ rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time,
191
h->slot = slot;
192
h->slot_timestamp += h->dt;
193
75
- *flags = (n & SN_ALL_FLAGS);
76
- return unpack_storage_number(n);
194
+ sp.anomaly_count = (n & SN_ANOMALY_BIT) ? 0 : 1;
195
+ sp.flags = (n & SN_ALL_FLAGS);
196
+ sp.min = sp.max = sp.sum = unpack_storage_number(n);
197
+
198
+ return sp;
199
}
200
201
int rrddim_query_is_finished(struct rrddim_query_handle *handle) {
@@ -89,10 +211,12 @@ void rrddim_query_finalize(struct rrddim_query_handle *handle) {
211
freez(handle->handle);
212
}
213
92
-time_t rrddim_query_latest_time(RRDDIM *rd) {
93
- return rrdset_last_entry_t_nolock(rd->rrdset);
214
+time_t rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
215
+ RRDDIM *rd = (RRDDIM *)db_metric_handle;
216
+ return rd->rrdset->last_updated.tv_sec;
217
}
218
96
-time_t rrddim_query_oldest_time(RRDDIM *rd) {
97
- return rrdset_first_entry_t_nolock(rd->rrdset);
219
+time_t rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
220
+ RRDDIM *rd = (RRDDIM *)db_metric_handle;
221
+ return (time_t)(rd->rrdset->last_updated.tv_sec - rrddim_duration(rd));
222
}
database/ram/rrddim_mem.h
+17
-8
@@ -6,6 +6,7 @@
6
#include "database/rrd.h"
7
8
struct mem_collect_handle {
9
+ RRDDIM *rd;
10
long slot;
11
long entries;
12
};
@@ -19,16 +20,24 @@ struct mem_query_handle {
20
size_t last_slot;
21
};
22
22
-extern void rrddim_collect_init(RRDDIM *rd);
23
-extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
24
-extern int rrddim_collect_finalize(RRDDIM *rd);
23
+extern STORAGE_METRIC_HANDLE *rrddim_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
24
+extern void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle);
25
26
-extern void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
27
-extern NETDATA_DOUBLE
28
-rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
26
+extern STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle);
27
+extern void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number,
28
+ NETDATA_DOUBLE min_value,
29
+ NETDATA_DOUBLE max_value,
30
+ uint16_t count,
31
+ uint16_t anomaly_count,
32
+ SN_FLAGS flags);
33
+extern void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
34
+extern int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
35
+
36
+extern void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
37
+extern STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle);
38
extern int rrddim_query_is_finished(struct rrddim_query_handle *handle);
39
extern void rrddim_query_finalize(struct rrddim_query_handle *handle);
31
-extern time_t rrddim_query_latest_time(RRDDIM *rd);
32
-extern time_t rrddim_query_oldest_time(RRDDIM *rd);
40
+extern time_t rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
41
+extern time_t rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
42
43
#endif
database/rrd.h
+146
-154
@@ -7,6 +7,11 @@
7
extern "C" {
8
#endif
9
10
+// non-existing structs instead of voids
11
+// to enable type checking at compile time
12
+typedef struct storage_instance STORAGE_INSTANCE;
13
+typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
14
+
15
// forward typedefs
16
typedef struct rrdhost RRDHOST;
17
typedef struct rrddim RRDDIM;
@@ -23,9 +28,10 @@ typedef void *ml_host_t;
28
typedef void *ml_dimension_t;
29
30
// forward declarations
26
-struct rrddim_volatile;
31
+struct rrddim_tier;
32
struct rrdset_volatile;
33
struct context_param;
34
+
35
#ifdef ENABLE_DBENGINE
36
struct rrdeng_page_descr;
37
struct rrdengine_instance;
@@ -34,6 +40,7 @@ struct pg_cache_page_index;
40
41
#include "daemon/common.h"
42
#include "web/api/queries/query.h"
43
+#include "web/api/queries/rrdr.h"
44
#include "rrdvar.h"
45
#include "rrdsetvar.h"
46
#include "rrddimvar.h"
@@ -43,6 +50,17 @@ struct pg_cache_page_index;
50
#include "aclk/aclk_rrdhost_state.h"
51
#include "sqlite/sqlite_health.h"
52
53
+extern int storage_tiers;
54
+extern int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
55
+
56
+typedef enum {
57
+ RRD_BACKFILL_NONE,
58
+ RRD_BACKFILL_FULL,
59
+ RRD_BACKFILL_NEW
60
+} RRD_BACKFILL;
61
+
62
+extern RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
63
+
64
enum {
65
CONTEXT_FLAGS_ARCHIVE = 0x01,
66
CONTEXT_FLAGS_CHART = 0x02,
@@ -226,6 +244,8 @@ struct rrddim {
244
245
avl_t avl; // the binary index - this has to be first member!
246
247
+ uuid_t metric_uuid; // global UUID for this metric (unique_across hosts)
248
+
249
// ------------------------------------------------------------------------
250
// the dimension definition
251
@@ -257,7 +277,13 @@ struct rrddim {
277
// this is actual date time we updated the last_collected_value
278
// THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
279
260
- struct rrddim_volatile *state; // volatile state that is not persistently stored
280
+#ifdef ENABLE_ACLK
281
+ int aclk_live_status;
282
+#endif
283
+ ml_dimension_t ml_dimension;
284
+
285
+ struct rrddim_tier *tiers[RRD_STORAGE_TIERS]; // our tiers of databases
286
+
287
size_t collections_counter; // the number of times we added values to this rrdim
288
collected_number collected_value_max; // the absolute maximum of the collected value
289
@@ -326,30 +352,71 @@ struct rrddim_query_handle {
352
RRDDIM *rd;
353
time_t start_time;
354
time_t end_time;
355
+ TIER_QUERY_FETCH tier_query_fetch_type;
356
STORAGE_QUERY_HANDLE* handle;
357
};
358
359
+typedef struct storage_point {
360
+ NETDATA_DOUBLE min; // when count > 1, this is the minimum among them
361
+ NETDATA_DOUBLE max; // when count > 1, this is the maximum among them
362
+ NETDATA_DOUBLE sum; // the point sum - divided by count gives the average
363
+
364
+ // end_time - start_time = point duration
365
+ time_t start_time; // the time the point starts
366
+ time_t end_time; // the time the point ends
367
+
368
+ unsigned count; // the number of original points aggregated
369
+ unsigned anomaly_count; // the number of original points found anomalous
370
+
371
+ SN_FLAGS flags; // flags stored with the point
372
+} STORAGE_POINT;
373
+
374
+#define storage_point_unset(x) do { \
375
+ (x).min = (x).max = (x).sum = NAN; \
376
+ (x).count = 0; \
377
+ (x).anomaly_count = 0; \
378
+ (x).flags = SN_EMPTY_SLOT; \
379
+ (x).start_time = 0; \
380
+ (x).end_time = 0; \
381
+ } while(0)
382
+
383
+#define storage_point_empty(x, start_t, end_t) do { \
384
+ (x).min = (x).max = (x).sum = NAN; \
385
+ (x).count = 1; \
386
+ (x).anomaly_count = 0; \
387
+ (x).flags = SN_EMPTY_SLOT; \
388
+ (x).start_time = start_t; \
389
+ (x).end_time = end_t; \
390
+ } while(0)
391
+
392
+#define storage_point_is_unset(x) (!(x).count)
393
+#define storage_point_is_empty(x) (!netdata_double_isnumber((x).sum))
394
+
395
// ------------------------------------------------------------------------
396
// function pointers that handle data collection
397
struct rrddim_collect_ops {
398
// an initialization function to run before starting collection
336
- void (*init)(RRDDIM *rd);
399
+ STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle);
400
401
// run this to store each metric into the database
339
- void (*store_metric)(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
402
+ void (*store_metric)(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number, NETDATA_DOUBLE min_value,
403
+ NETDATA_DOUBLE max_value, uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
404
+
405
+ // run this to flush / reset the current data collection sequence
406
+ void (*flush)(STORAGE_COLLECT_HANDLE *collection_handle);
407
408
// an finalization function to run after collection is over
409
// returns 1 if it's safe to delete the dimension
343
- int (*finalize)(RRDDIM *rd);
410
+ int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle);
411
};
412
413
// function pointers that handle database queries
414
struct rrddim_query_ops {
415
// run this before starting a series of next_metric() database queries
349
- void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
416
+ void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
417
418
// run this to load each metric number from the database
352
- NETDATA_DOUBLE (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags);
419
+ STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
420
421
// run this to test if the series of next_metric() database queries is finished
422
int (*is_finished)(struct rrddim_query_handle *handle);
@@ -358,29 +425,31 @@ struct rrddim_query_ops {
425
void (*finalize)(struct rrddim_query_handle *handle);
426
427
// get the timestamp of the last entry of this metric
361
- time_t (*latest_time)(RRDDIM *rd);
428
+ time_t (*latest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
429
430
// get the timestamp of the first entry of this metric
364
- time_t (*oldest_time)(RRDDIM *rd);
431
+ time_t (*oldest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
432
};
433
434
+
435
// ----------------------------------------------------------------------------
368
-// volatile state per RRD dimension
369
-struct rrddim_volatile {
370
-#ifdef ENABLE_DBENGINE
371
- uuid_t *rrdeng_uuid; // database engine metric UUID
372
- struct pg_cache_page_index *page_index;
373
-#endif
374
-#ifdef ENABLE_ACLK
375
- int aclk_live_status;
376
-#endif
377
- uuid_t metric_uuid; // global UUID for this metric (unique_across hosts)
378
- STORAGE_COLLECT_HANDLE* handle;
436
+// Storage tier data for every dimension
437
+
438
+struct rrddim_tier {
439
+ int tier_grouping;
440
+ RRD_MEMORY_MODE mode; // the memory mode of this tier
441
+ RRD_BACKFILL backfill; // backfilling configuration
442
+ STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database
443
+ STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle
444
+ STORAGE_POINT virtual_point;
445
+ time_t next_point_time;
446
+ usec_t last_collected_ut;
447
struct rrddim_collect_ops collect_ops;
448
struct rrddim_query_ops query_ops;
381
- ml_dimension_t ml_dimension;
449
};
450
451
+extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now);
452
+
453
// ----------------------------------------------------------------------------
454
// volatile state per chart
455
struct rrdset_volatile {
@@ -854,9 +923,8 @@ struct rrdhost {
923
avl_tree_lock rrdfamily_root_index; // the host's chart families index
924
avl_tree_lock rrdvar_root_index; // the host's chart variables index
925
857
-#ifdef ENABLE_DBENGINE
858
- struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host
859
-#endif
926
+ STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers
927
+
928
uuid_t host_uuid; // Global GUID for this host
929
uuid_t *node_id; // Cloud node_id
930
@@ -900,6 +968,10 @@ extern netdata_rwlock_t rrd_rwlock;
968
969
// ----------------------------------------------------------------------------
970
971
+extern bool is_storage_engine_shared(STORAGE_INSTANCE *engine);
972
+
973
+// ----------------------------------------------------------------------------
974
+
975
extern size_t rrd_hosts_available;
976
extern time_t rrdhost_free_orphan_time;
977
@@ -1067,28 +1139,49 @@ extern void rrdset_isnot_obsolete(RRDSET *st);
1139
#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
1140
#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
1141
1070
-// get the total duration in seconds of the round robin database
1071
-#define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
1072
-
1142
// get the timestamp of the last entry in the round robin database
1074
-static inline time_t rrdset_last_entry_t_nolock(RRDSET *st)
1075
-{
1076
- if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1077
- RRDDIM *rd;
1078
- time_t last_entry_t = 0;
1143
+static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
1144
+ time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
1145
1080
- rrddim_foreach_read(rd, st) {
1081
- last_entry_t = MAX(last_entry_t, rd->state->query_ops.latest_time(rd));
1082
- }
1146
+ for(int tier = 1; tier < storage_tiers ;tier++) {
1147
+ if(unlikely(!rd->tiers[tier])) continue;
1148
1084
- return last_entry_t;
1085
- } else {
1086
- return (time_t)st->last_updated.tv_sec;
1149
+ time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
1150
+ if(t > latest)
1151
+ latest = t;
1152
}
1153
+
1154
+ return latest;
1155
}
1156
1090
-static inline time_t rrdset_last_entry_t(RRDSET *st)
1091
-{
1157
+static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
1158
+ time_t oldest = 0;
1159
+
1160
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1161
+ if(unlikely(!rd->tiers[tier])) continue;
1162
+
1163
+ time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
1164
+ if(t != 0 && (oldest == 0 || t < oldest))
1165
+ oldest = t;
1166
+ }
1167
+
1168
+ return oldest;
1169
+}
1170
+
1171
+// get the timestamp of the last entry in the round robin database
1172
+static inline time_t rrdset_last_entry_t_nolock(RRDSET *st) {
1173
+ RRDDIM *rd;
1174
+ time_t last_entry_t = 0;
1175
+
1176
+ rrddim_foreach_read(rd, st) {
1177
+ time_t t = rrddim_last_entry_t(rd);
1178
+ if(t > last_entry_t) last_entry_t = t;
1179
+ }
1180
+
1181
+ return last_entry_t;
1182
+}
1183
+
1184
+static inline time_t rrdset_last_entry_t(RRDSET *st) {
1185
time_t last_entry_t;
1186
1187
netdata_rwlock_rdlock(&st->rrdset_rwlock);
@@ -1099,24 +1192,18 @@ static inline time_t rrdset_last_entry_t(RRDSET *st)
1192
}
1193
1194
// get the timestamp of first entry in the round robin database
1102
-static inline time_t rrdset_first_entry_t_nolock(RRDSET *st)
1103
-{
1104
- if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1105
- RRDDIM *rd;
1106
- time_t first_entry_t = LONG_MAX;
1107
-
1108
- rrddim_foreach_read(rd, st) {
1109
- first_entry_t =
1110
- MIN(first_entry_t,
1111
- rd->state->query_ops.oldest_time(rd) > st->update_every ?
1112
- rd->state->query_ops.oldest_time(rd) - st->update_every : 0);
1113
- }
1114
-
1115
- if (unlikely(LONG_MAX == first_entry_t)) return 0;
1116
- return first_entry_t;
1117
- } else {
1118
- return (time_t)(rrdset_last_entry_t_nolock(st) - rrdset_duration(st));
1195
+static inline time_t rrdset_first_entry_t_nolock(RRDSET *st) {
1196
+ RRDDIM *rd;
1197
+ time_t first_entry_t = LONG_MAX;
1198
+
1199
+ rrddim_foreach_read(rd, st) {
1200
+ time_t t = rrddim_first_entry_t(rd);
1201
+ if(t < first_entry_t)
1202
+ first_entry_t = t;
1203
}
1204
+
1205
+ if (unlikely(LONG_MAX == first_entry_t)) return 0;
1206
+ return first_entry_t;
1207
}
1208
1209
static inline time_t rrdset_first_entry_t(RRDSET *st)
@@ -1130,105 +1217,8 @@ static inline time_t rrdset_first_entry_t(RRDSET *st)
1217
return first_entry_t;
1218
}
1219
1133
-// get the timestamp of the last entry in the round robin database
1134
-static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
1135
- if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
1136
- return rd->state->query_ops.latest_time(rd);
1137
- return (time_t)rd->rrdset->last_updated.tv_sec;
1138
-}
1139
-
1140
-static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
1141
- if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
1142
- return rd->state->query_ops.oldest_time(rd);
1143
- return (time_t)(rd->rrdset->last_updated.tv_sec - rrdset_duration(rd->rrdset));
1144
-}
1145
-
1220
time_t rrdhost_last_entry_t(RRDHOST *h);
1221
1148
-// get the last slot updated in the round robin database
1149
-#define rrdset_last_slot(st) ((size_t)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
1150
-
1151
-// get the first / oldest slot updated in the round robin database
1152
-// #define rrdset_first_slot(st) ((size_t)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
1153
-
1154
-// return the slot that has the oldest value
1155
-
1156
-static inline size_t rrdset_first_slot(RRDSET *st) {
1157
- if(st->counter >= (size_t)st->entries) {
1158
- // the database has been rotated at least once
1159
- // the oldest entry is the one that will be next
1160
- // overwritten by data collection
1161
- return (size_t)st->current_entry;
1162
- }
1163
-
1164
- // we do not have rotated the db yet
1165
- // so 0 is the first entry
1166
- return 0;
1167
-}
1168
-
1169
-// get the slot of the round robin database, for the given timestamp (t)
1170
-// it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database
1171
-// only valid when not using dbengine
1172
-static inline size_t rrdset_time2slot(RRDSET *st, time_t t) {
1173
- size_t ret = 0;
1174
- time_t last_entry_t = rrdset_last_entry_t_nolock(st);
1175
- time_t first_entry_t = rrdset_first_entry_t_nolock(st);
1176
-
1177
- if(t >= last_entry_t) {
1178
- // the requested time is after the last entry we have
1179
- ret = rrdset_last_slot(st);
1180
- }
1181
- else {
1182
- if(t <= first_entry_t) {
1183
- // the requested time is before the first entry we have
1184
- ret = rrdset_first_slot(st);
1185
- }
1186
- else {
1187
- if(rrdset_last_slot(st) >= (size_t)((last_entry_t - t) / st->update_every))
1188
- ret = rrdset_last_slot(st) - ((last_entry_t - t) / st->update_every);
1189
- else
1190
- ret = rrdset_last_slot(st) - ((last_entry_t - t) / st->update_every) + st->entries;
1191
- }
1192
- }
1193
-
1194
- if(unlikely(ret >= (size_t)st->entries)) {
1195
- error("INTERNAL ERROR: rrdset_time2slot() on %s returns values outside entries", st->name);
1196
- ret = (size_t)(st->entries - 1);
1197
- }
1198
-
1199
- return ret;
1200
-}
1201
-
1202
-// get the timestamp of a specific slot in the round robin database
1203
-// only valid when not using dbengine
1204
-static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
1205
- time_t ret;
1206
- time_t last_entry_t = rrdset_last_entry_t_nolock(st);
1207
- time_t first_entry_t = rrdset_first_entry_t_nolock(st);
1208
-
1209
- if(slot >= (size_t)st->entries) {
1210
- error("INTERNAL ERROR: caller of rrdset_slot2time() gives invalid slot %zu", slot);
1211
- slot = (size_t)st->entries - 1;
1212
- }
1213
-
1214
- if(slot > rrdset_last_slot(st))
1215
- ret = last_entry_t - (time_t)(st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries));
1216
- else
1217
- ret = last_entry_t - (time_t)(st->update_every * (rrdset_last_slot(st) - slot));
1218
-
1219
- if(unlikely(ret < first_entry_t)) {
1220
- error("INTERNAL ERROR: rrdset_slot2time() on %s returns time too far in the past", st->name);
1221
- ret = first_entry_t;
1222
- }
1223
-
1224
- if(unlikely(ret > last_entry_t)) {
1225
- error("INTERNAL ERROR: rrdset_slot2time() on %s returns time into the future", st->name);
1226
- ret = last_entry_t;
1227
- }
1228
-
1229
- return ret;
1230
-}
1231
-
1222
// ----------------------------------------------------------------------------
1223
// RRD DIMENSION functions
1224
@@ -1318,6 +1308,8 @@ extern void set_host_properties(
1308
const char *guid, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
1309
const char *program_name, const char *program_version);
1310
1311
+extern int get_tier_grouping(int tier);
1312
+
1313
// ----------------------------------------------------------------------------
1314
// RRD DB engine declarations
1315
database/rrddim.c
+85
-25
@@ -144,7 +144,7 @@ time_t calc_dimension_liveness(RRDDIM *rd, time_t now)
144
{
145
time_t last_updated = rd->last_collected_time.tv_sec;
146
int live;
147
- if (rd->state->aclk_live_status == 1)
147
+ if (rd->aclk_live_status == 1)
148
live =
149
((now - last_updated) <
150
MIN(rrdset_free_obsolete_time, RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER * rd->update_every));
@@ -168,9 +168,16 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
168
rc += rrddim_set_algorithm(st, rd, algorithm);
169
rc += rrddim_set_multiplier(st, rd, multiplier);
170
rc += rrddim_set_divisor(st, rd, divisor);
171
+
172
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
172
- store_active_dimension(&rd->state->metric_uuid);
173
- rd->state->collect_ops.init(rd);
173
+ store_active_dimension(&rd->metric_uuid);
174
+
175
+ for(int tier = 0; tier < storage_tiers ;tier++) {
176
+ if (rd->tiers[tier])
177
+ rd->tiers[tier]->db_collection_handle =
178
+ rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
179
+ }
180
+
181
rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
182
rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
183
rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT);
@@ -180,9 +187,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
187
rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
188
rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
189
}
190
+
191
if (unlikely(rc)) {
192
debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
185
- (void)sql_store_dimension(&rd->state->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
193
+ (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
194
rd->algorithm);
195
#ifdef ENABLE_ACLK
196
queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec()));
@@ -246,23 +254,54 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
254
255
rd->rrd_memory_mode = memory_mode;
256
249
- rd->state = callocz(1, sizeof(*rd->state));
257
#ifdef ENABLE_ACLK
251
- rd->state->aclk_live_status = -1;
258
+ rd->aclk_live_status = -1;
259
#endif
253
- (void) find_dimension_uuid(st, rd, &(rd->state->metric_uuid));
260
+ (void) find_dimension_uuid(st, rd, &(rd->metric_uuid));
261
+
262
+ // initialize the db tiers
263
+ {
264
+ size_t initialized = 0;
265
+ RRD_MEMORY_MODE wanted_mode = memory_mode;
266
+ for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
267
+ STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
268
+ if(!eng) continue;
269
+
270
+ rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier));
271
+ rd->tiers[tier]->tier_grouping = get_tier_grouping(tier);
272
+ rd->tiers[tier]->mode = eng->id;
273
+ rd->tiers[tier]->collect_ops = eng->api.collect_ops;
274
+ rd->tiers[tier]->query_ops = eng->api.query_ops;
275
+ rd->tiers[tier]->db_metric_handle = eng->api.init(rd, host->storage_instance[tier]);
276
+ storage_point_unset(rd->tiers[tier]->virtual_point);
277
+ initialized++;
278
+
279
+ // internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping);
280
+ }
281
255
- STORAGE_ENGINE* eng = storage_engine_get(memory_mode);
256
- rd->state->collect_ops = eng->api.collect_ops;
257
- rd->state->query_ops = eng->api.query_ops;
282
+ if(!initialized)
283
+ error("Failed to initialize all db tiers for chart '%s', dimension '%s", st->name, rd->name);
284
259
-#ifdef ENABLE_DBENGINE
260
- if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
261
- rrdeng_metric_init(rd);
285
+ if(!rd->tiers[0])
286
+ error("Failed to initialize the first db tier for chart '%s', dimension '%s", st->name, rd->name);
287
}
263
-#endif
264
- store_active_dimension(&rd->state->metric_uuid);
265
- rd->state->collect_ops.init(rd);
288
+
289
+ store_active_dimension(&rd->metric_uuid);
290
+
291
+ // initialize data collection for all tiers
292
+ {
293
+ size_t initialized = 0;
294
+ for (int tier = 0; tier < storage_tiers; tier++) {
295
+ if (rd->tiers[tier]) {
296
+ rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
297
+ initialized++;
298
+ }
299
+ }
300
+
301
+ if(!initialized)
302
+ error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", st->name, rd->name);
303
+ }
304
+
305
// append this dimension
306
if(!st->dimensions)
307
st->dimensions = rd;
@@ -318,10 +357,22 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
357
debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
358
359
if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
321
- uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
322
- if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
360
+
361
+ size_t tiers_available = 0, tiers_said_yes = 0;
362
+ for(int tier = 0; tier < storage_tiers ;tier++) {
363
+ if(rd->tiers[tier]) {
364
+ tiers_available++;
365
+
366
+ if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle))
367
+ tiers_said_yes++;
368
+
369
+ rd->tiers[tier]->db_collection_handle = NULL;
370
+ }
371
+ }
372
+
373
+ if (tiers_available == tiers_said_yes && tiers_said_yes && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
374
/* This metric has no data and no references */
324
- delete_dimension_uuid(&rd->state->metric_uuid);
375
+ delete_dimension_uuid(&rd->metric_uuid);
376
}
377
}
378
@@ -350,13 +401,20 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
401
// aclk_send_dimension_update(rd);
402
//#endif
403
353
- freez((void *)rd->id);
354
- freez((void *)rd->name);
355
- freez(rd->state);
356
-
404
// this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
405
rrddim_memory_file_free(rd);
406
407
+ for(int tier = 0; tier < storage_tiers ;tier++) {
408
+ if(!rd->tiers[tier]) continue;
409
+
410
+ STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode);
411
+ if(eng)
412
+ eng->api.free(rd->tiers[tier]->db_metric_handle);
413
+
414
+ freez(rd->tiers[tier]);
415
+ rd->tiers[tier] = NULL;
416
+ }
417
+
418
if(rd->db) {
419
if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
420
munmap(rd->db, rd->memsize);
@@ -364,6 +422,8 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
422
freez(rd->db);
423
}
424
425
+ freez((void *)rd->id);
426
+ freez((void *)rd->name);
427
freez(rd);
428
}
429
@@ -382,7 +442,7 @@ int rrddim_hide(RRDSET *st, const char *id) {
442
return 1;
443
}
444
if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
385
- (void)sql_set_dimension_option(&rd->state->metric_uuid, "hidden");
445
+ (void)sql_set_dimension_option(&rd->metric_uuid, "hidden");
446
447
rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
448
rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
@@ -399,7 +459,7 @@ int rrddim_unhide(RRDSET *st, const char *id) {
459
return 1;
460
}
461
if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
402
- (void)sql_set_dimension_option(&rd->state->metric_uuid, NULL);
462
+ (void)sql_set_dimension_option(&rd->metric_uuid, NULL);
463
464
rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
465
rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
database/rrdhost.c
+199
-29
@@ -3,6 +3,26 @@
3
#define NETDATA_RRD_INTERNALS
4
#include "rrd.h"
5
6
+int storage_tiers = 1;
7
+int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
8
+RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
9
+
10
+#if RRD_STORAGE_TIERS != 5
11
+#error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier
12
+#endif
13
+
14
+int get_tier_grouping(int tier) {
15
+ if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1;
16
+ if(unlikely(tier < 0)) tier = 0;
17
+
18
+ int grouping = 1;
19
+ // first tier is always 1 iteration of whatever update every the chart has
20
+ for(int i = 1; i <= tier ;i++)
21
+ grouping *= storage_tiers_grouping_iterations[i];
22
+
23
+ return grouping;
24
+}
25
+
26
RRDHOST *localhost = NULL;
27
size_t rrd_hosts_available = 0;
28
netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER;
@@ -10,6 +30,18 @@ netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER;
30
time_t rrdset_free_obsolete_time = 3600;
31
time_t rrdhost_free_orphan_time = 3600;
32
33
+bool is_storage_engine_shared(STORAGE_INSTANCE *engine) {
34
+#ifdef ENABLE_DBENGINE
35
+ for(int tier = 0; tier < storage_tiers ;tier++) {
36
+ if (engine == (STORAGE_INSTANCE *)multidb_ctx[tier])
37
+ return true;
38
+ }
39
+#endif
40
+
41
+ return false;
42
+}
43
+
44
+
45
// ----------------------------------------------------------------------------
46
// RRDHOST index
47
@@ -344,11 +376,28 @@ RRDHOST *rrdhost_create(const char *hostname,
376
if (ret != 0 && errno != EEXIST)
377
error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);
378
else ret = 0; // succeed
347
- if (is_legacy) // initialize legacy dbengine instance as needed
348
- ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, default_rrdeng_page_cache_mb,
349
- default_rrdeng_disk_quota_mb); // may fail here for legacy dbengine initialization
350
- else
351
- host->rrdeng_ctx = &multidb_ctx;
379
+ if (is_legacy) {
380
+ // initialize legacy dbengine instance as needed
381
+
382
+ ret = rrdeng_init(
383
+ host,
384
+ (struct rrdengine_instance **)&host->storage_instance[0],
385
+ dbenginepath,
386
+ default_rrdeng_page_cache_mb,
387
+ default_rrdeng_disk_quota_mb,
388
+ 0); // may fail here for legacy dbengine initialization
389
+
390
+ if(ret == 0) {
391
+ // assign the rest of the shared storage instances to it
392
+ // to allow them collect its metrics too
393
+ for(int tier = 1; tier < storage_tiers ; tier++)
394
+ host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
395
+ }
396
+ }
397
+ else {
398
+ for(int tier = 0; tier < storage_tiers ; tier++)
399
+ host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
400
+ }
401
if (ret) { // check legacy or multihost initialization success
402
error(
403
"Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
@@ -366,7 +415,9 @@ RRDHOST *rrdhost_create(const char *hostname,
415
}
416
else {
417
#ifdef ENABLE_DBENGINE
369
- host->rrdeng_ctx = &multidb_ctx;
418
+ // the first tier is reserved for the non-dbengine modes
419
+ for(int tier = 1; tier < storage_tiers ; tier++)
420
+ host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
421
#endif
422
}
423
@@ -673,7 +724,7 @@ restart_after_removal:
724
if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST)
725
#ifdef ENABLE_DBENGINE
726
/* don't delete multi-host DB host files */
676
- && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx)
727
+ && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0]))
728
#endif
729
)
730
rrdhost_delete_charts(host);
@@ -690,6 +741,35 @@ restart_after_removal:
741
// RRDHOST global / startup initialization
742
743
int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
744
+
745
+#ifdef ENABLE_DBENGINE
746
+ storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
747
+ if(storage_tiers < 1) {
748
+ error("At least 1 storage tier is required. Assuming 1.");
749
+ storage_tiers = 1;
750
+ config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
751
+ }
752
+ if(storage_tiers > RRD_STORAGE_TIERS) {
753
+ error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS);
754
+ storage_tiers = RRD_STORAGE_TIERS;
755
+ config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
756
+ }
757
+
758
+ default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
759
+ if (default_rrdeng_page_fetch_timeout < 1) {
760
+ info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
761
+ default_rrdeng_page_fetch_timeout = 1;
762
+ config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout);
763
+ }
764
+
765
+ default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES);
766
+ if (default_rrdeng_page_fetch_retries < 1) {
767
+ info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries);
768
+ default_rrdeng_page_fetch_retries = 1;
769
+ config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries);
770
+ }
771
+#endif
772
+
773
rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
774
// Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
775
// cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
@@ -713,7 +793,6 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
793
}
794
795
health_init();
716
-
796
rrdpush_init();
797
798
debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
@@ -746,24 +825,95 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
825
}
826
827
#ifdef ENABLE_DBENGINE
828
+ int created_tiers = 0;
829
char dbenginepath[FILENAME_MAX + 1];
750
- int ret;
751
- snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", localhost->cache_dir);
752
- ret = mkdir(dbenginepath, 0775);
753
- if (ret != 0 && errno != EEXIST)
754
- error("Host '%s': cannot create directory '%s'", localhost->hostname, dbenginepath);
755
- else // Unconditionally create multihost db to support on demand host creation
756
- ret = rrdeng_init(NULL, NULL, dbenginepath, default_rrdeng_page_cache_mb, default_multidb_disk_quota_mb);
757
- if (ret) {
758
- error(
759
- "Host '%s' with machine guid '%s' failed to initialize multi-host DB engine instance at '%s'.",
760
- localhost->hostname, localhost->machine_guid, localhost->cache_dir);
830
+ char dbengineconfig[200 + 1];
831
+ for(int tier = 0; tier < storage_tiers ;tier++) {
832
+ if(tier == 0)
833
+ snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", localhost->cache_dir);
834
+ else
835
+ snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%d", localhost->cache_dir, tier);
836
+
837
+ int ret = mkdir(dbenginepath, 0775);
838
+ if (ret != 0 && errno != EEXIST) {
839
+ error("DBENGINE on '%s': cannot create directory '%s'", localhost->hostname, dbenginepath);
840
+ break;
841
+ }
842
+
843
+ int page_cache_mb = default_rrdeng_page_cache_mb;
844
+ int disk_space_mb = default_multidb_disk_quota_mb;
845
+ int grouping_iterations = storage_tiers_grouping_iterations[tier];
846
+ RRD_BACKFILL backfill = storage_tiers_backfill[tier];
847
+
848
+ if(tier > 0) {
849
+ snprintfz(dbengineconfig, 200, "dbengine tier %d page cache size MB", tier);
850
+ page_cache_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, page_cache_mb);
851
+
852
+ snprintfz(dbengineconfig, 200, "dbengine tier %d multihost disk space MB", tier);
853
+ disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
854
+
855
+ snprintfz(dbengineconfig, 200, "dbengine tier %d update every iterations", tier);
856
+ grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
857
+ if(grouping_iterations < 2) {
858
+ grouping_iterations = 2;
859
+ config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
860
+ error("DBENGINE on '%s': 'dbegnine tier %d update every iterations' cannot be less than 2. Assuming 2.", localhost->hostname, tier);
861
+ }
862
+
863
+ snprintfz(dbengineconfig, 200, "dbengine tier %d backfill", tier);
864
+ const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
865
+ if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
866
+ else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
867
+ else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;
868
+ else {
869
+ error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf);
870
+ config_set(CONFIG_SECTION_DB, dbengineconfig, "new");
871
+ backfill = RRD_BACKFILL_NEW;
872
+ }
873
+ }
874
+
875
+ storage_tiers_grouping_iterations[tier] = grouping_iterations;
876
+ storage_tiers_backfill[tier] = backfill;
877
+
878
+ if(tier > 0 && get_tier_grouping(tier) > 65535) {
879
+ storage_tiers_grouping_iterations[tier] = 1;
880
+ error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", localhost->hostname, tier, tier);
881
+ break;
882
+ }
883
+
884
+ internal_error(true, "DBENGINE tier %d grouping iterations is set to %d", tier, storage_tiers_grouping_iterations[tier]);
885
+ ret = rrdeng_init(NULL, NULL, dbenginepath, page_cache_mb, disk_space_mb, tier);
886
+ if(ret != 0) {
887
+ error("DBENGINE on '%s': Failed to initialize multi-host database tier %d on path '%s'",
888
+ localhost->hostname, tier, dbenginepath);
889
+ break;
890
+ }
891
+ else
892
+ created_tiers++;
893
+ }
894
+
895
+ if(created_tiers && created_tiers < storage_tiers) {
896
+ error("DBENGINE on '%s': Managed to create %d tiers instead of %d. Continuing with %d available.",
897
+ localhost->hostname, created_tiers, storage_tiers, created_tiers);
898
+ storage_tiers = created_tiers;
899
+ }
900
+ else if(!created_tiers) {
901
+ error("DBENGINE on '%s', with machine guid '%s', failed to initialize databases at '%s'.",
902
+ localhost->hostname, localhost->machine_guid, localhost->cache_dir);
903
rrdhost_free(localhost);
904
localhost = NULL;
905
rrd_unlock();
764
- fatal("Failed to initialize dbengine");
906
+ fatal("DBENGINE: Failed to be initialized.");
907
+ }
908
+#else
909
+ storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
910
+ if(storage_tiers != 1) {
911
+ error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", localhost->hostname);
912
+ storage_tiers = 1;
913
+ config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
914
}
915
#endif
916
+
917
if (likely(system_info))
918
migrate_localhost(&localhost->host_uuid);
919
sql_aclk_sync_init();
@@ -911,11 +1061,14 @@ void rrdhost_free(RRDHOST *host) {
1061
// release its children resources
1062
1063
#ifdef ENABLE_DBENGINE
914
- if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
915
- if (host->rrdeng_ctx != &multidb_ctx)
916
- rrdeng_prepare_exit(host->rrdeng_ctx);
1064
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1065
+ if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
1066
+ host->storage_instance[tier] &&
1067
+ !is_storage_engine_shared(host->storage_instance[tier]))
1068
+ rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[tier]);
1069
}
1070
#endif
1071
+
1072
while(host->rrdset_root)
1073
rrdset_free(host->rrdset_root);
1074
@@ -947,8 +1100,12 @@ void rrdhost_free(RRDHOST *host) {
1100
health_alarm_log_free(host);
1101
1102
#ifdef ENABLE_DBENGINE
950
- if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx != &multidb_ctx)
951
- rrdeng_exit(host->rrdeng_ctx);
1103
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1104
+ if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
1105
+ host->storage_instance[tier] &&
1106
+ !is_storage_engine_shared(host->storage_instance[tier]))
1107
+ rrdeng_exit((struct rrdengine_instance *)host->storage_instance[tier]);
1108
+ }
1109
#endif
1110
1111
// ------------------------------------------------------------------------
@@ -1267,7 +1424,7 @@ void rrdhost_cleanup_all(void) {
1424
if (host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) && !host->receiver
1425
#ifdef ENABLE_DBENGINE
1426
/* don't delete multi-host DB host files */
1270
- && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx)
1427
+ && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0]))
1428
#endif
1429
)
1430
rrdhost_delete_charts(host);
@@ -1323,11 +1480,24 @@ restart_after_removal:
1480
1481
if (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
1482
rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
1483
+
1484
/* only a collector can mark a chart as obsolete, so we must remove the reference */
1327
- uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
1328
- if (can_delete_metric) {
1485
+
1486
+ size_t tiers_available = 0, tiers_said_yes = 0;
1487
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1488
+ if(rd->tiers[tier]) {
1489
+ tiers_available++;
1490
+
1491
+ if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle))
1492
+ tiers_said_yes++;
1493
+
1494
+ rd->tiers[tier]->db_collection_handle = NULL;
1495
+ }
1496
+ }
1497
+
1498
+ if (tiers_available == tiers_said_yes && tiers_said_yes) {
1499
/* This metric has no data and no references */
1330
- delete_dimension_uuid(&rd->state->metric_uuid);
1500
+ delete_dimension_uuid(&rd->metric_uuid);
1501
rrddim_free(st, rd);
1502
if (unlikely(!last)) {
1503
rd = st->dimensions;
database/rrdset.c
+128
-14
@@ -273,12 +273,13 @@ void rrdset_reset(RRDSET *st) {
273
rd->last_collected_time.tv_sec = 0;
274
rd->last_collected_time.tv_usec = 0;
275
rd->collections_counter = 0;
276
- // memset(rd->values, 0, rd->entries * sizeof(storage_number));
277
-#ifdef ENABLE_DBENGINE
278
- if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode && !rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
279
- rrdeng_store_metric_flush_current_page(rd);
276
+
277
+ if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
278
+ for(int tier = 0; tier < storage_tiers ;tier++) {
279
+ if(rd->tiers[tier])
280
+ rd->tiers[tier]->collect_ops.flush(rd->tiers[tier]->db_collection_handle);
281
+ }
282
}
281
-#endif
283
}
284
}
285
@@ -963,6 +964,105 @@ static inline usec_t rrdset_init_last_updated_time(RRDSET *st) {
964
return last_updated_ut;
965
}
966
967
+static inline time_t tier_next_point_time(RRDDIM *rd, struct rrddim_tier *t, time_t now) {
968
+ time_t loop = (time_t)rd->update_every * (time_t)t->tier_grouping;
969
+ return now + loop - ((now + loop) % loop);
970
+}
971
+
972
+void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut) {
973
+ if (unlikely(!t->next_point_time))
974
+ t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
975
+
976
+ // merge the dates into our virtual point
977
+ if (unlikely(sp.start_time < t->virtual_point.start_time))
978
+ t->virtual_point.start_time = sp.start_time;
979
+
980
+ if (likely(sp.end_time > t->virtual_point.end_time))
981
+ t->virtual_point.end_time = sp.end_time;
982
+
983
+ // merge the values into our virtual point
984
+ if (likely(!storage_point_is_empty(sp))) {
985
+ // we aggregate only non NULLs into higher tiers
986
+
987
+ if (likely(!storage_point_is_unset(t->virtual_point))) {
988
+ // merge the collected point to our virtual one
989
+ t->virtual_point.sum += sp.sum;
990
+ t->virtual_point.min = MIN(t->virtual_point.min, sp.min);
991
+ t->virtual_point.max = MAX(t->virtual_point.max, sp.max);
992
+ t->virtual_point.count += sp.count;
993
+ t->virtual_point.anomaly_count += sp.anomaly_count;
994
+ t->virtual_point.flags |= sp.flags;
995
+ }
996
+ else {
997
+ // reset our virtual point to this one
998
+ t->virtual_point = sp;
999
+ }
1000
+ }
1001
+
1002
+ if(unlikely(sp.end_time >= t->next_point_time)) {
1003
+ if (likely(!storage_point_is_unset(t->virtual_point))) {
1004
+
1005
+ t->collect_ops.store_metric(
1006
+ t->db_collection_handle,
1007
+ now_ut,
1008
+ t->virtual_point.sum,
1009
+ t->virtual_point.min,
1010
+ t->virtual_point.max,
1011
+ t->virtual_point.count,
1012
+ t->virtual_point.anomaly_count,
1013
+ t->virtual_point.flags);
1014
+ }
1015
+ else {
1016
+ t->collect_ops.store_metric(
1017
+ t->db_collection_handle,
1018
+ now_ut,
1019
+ NAN,
1020
+ NAN,
1021
+ NAN,
1022
+ 0,
1023
+ 0,
1024
+ SN_EMPTY_SLOT);
1025
+ }
1026
+
1027
+ t->virtual_point.count = 0;
1028
+ t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
1029
+ }
1030
+}
1031
+
1032
+static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
1033
+
1034
+ // store the metric on tier 0
1035
+ rd->tiers[0]->collect_ops.store_metric(rd->tiers[0]->db_collection_handle, point_end_time_ut, n, 0, 0, 1, 0, flags);
1036
+
1037
+ for(int tier = 1; tier < storage_tiers ;tier++) {
1038
+ if(unlikely(!rd->tiers[tier])) continue;
1039
+
1040
+ struct rrddim_tier *t = rd->tiers[tier];
1041
+
1042
+ time_t now = (time_t)(point_end_time_ut / USEC_PER_SEC);
1043
+
1044
+ if(!t->last_collected_ut) {
1045
+ // we have not collected this tier before
1046
+ // let's fill any gap that may exist
1047
+ rrdr_fill_tier_gap_from_smaller_tiers(rd, tier, now);
1048
+ }
1049
+
1050
+ STORAGE_POINT sp = {
1051
+ .start_time = now - rd->update_every,
1052
+ .end_time = now,
1053
+ .min = n,
1054
+ .max = n,
1055
+ .sum = n,
1056
+ .count = 1,
1057
+ .anomaly_count = (flags & SN_ANOMALY_BIT) ? 0 : 1,
1058
+ .flags = flags
1059
+ };
1060
+
1061
+ t->last_collected_ut = point_end_time_ut;
1062
+ store_metric_at_tier(rd, t, sp, point_end_time_ut);
1063
+ }
1064
+}
1065
+
1066
static inline size_t rrdset_done_interpolate(
1067
RRDSET *st
1068
, usec_t update_every_ut
@@ -1086,8 +1186,8 @@ static inline size_t rrdset_done_interpolate(
1186
1187
if(unlikely(!store_this_entry)) {
1188
(void) ml_is_anomalous(rd, 0, false);
1089
-
1090
- rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT);
1189
+// rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, 0, 0, 1, SN_EMPTY_SLOT, 0);
1190
+ store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT);
1191
continue;
1192
}
1193
@@ -1099,7 +1199,8 @@ static inline size_t rrdset_done_interpolate(
1199
dim_storage_flags &= ~ ((uint32_t) SN_ANOMALY_BIT);
1200
}
1201
1102
- rd->state->collect_ops.store_metric(rd, next_store_ut, new_value, dim_storage_flags);
1202
+// rd->state->collect_ops.store_metric(rd, next_store_ut, new_value, 0, 0, 1, dim_storage_flags, 0);
1203
+ store_metric(rd, next_store_ut, new_value, dim_storage_flags);
1204
rd->last_stored_value = new_value;
1205
}
1206
else {
@@ -1112,7 +1213,8 @@ static inline size_t rrdset_done_interpolate(
1213
);
1214
#endif
1215
1115
- rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT);
1216
+// rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, 0, 0, 1, SN_EMPTY_SLOT, 0);
1217
+ store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT);
1218
rd->last_stored_value = NAN;
1219
}
1220
@@ -1597,10 +1699,10 @@ after_first_database_work:
1699
// it is now time to interpolate values on a second boundary
1700
1701
#ifdef NETDATA_INTERNAL_CHECKS
1600
- if(unlikely(now_collect_ut < next_store_ut)) {
1702
+ if(unlikely(now_collect_ut < next_store_ut && st->counter_done > 1)) {
1703
// this is collected in the same interpolation point
1704
rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
1603
- info("INTERNAL CHECK: host '%s', chart '%s' is collected in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, next_store_ut - now_collect_ut);
1705
+ info("INTERNAL CHECK: host '%s', chart '%s' collection %zu is in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, st->counter_done, next_store_ut - now_collect_ut);
1706
}
1707
#endif
1708
@@ -1734,10 +1836,22 @@ after_second_database_work:
1836
1837
rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
1838
/* only a collector can mark a chart as obsolete, so we must remove the reference */
1737
- uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
1738
- if (can_delete_metric) {
1839
+
1840
+ size_t tiers_available = 0, tiers_said_yes = 0;
1841
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1842
+ if(rd->tiers[tier]) {
1843
+ tiers_available++;
1844
+
1845
+ if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle))
1846
+ tiers_said_yes++;
1847
+
1848
+ rd->tiers[tier]->db_collection_handle = NULL;
1849
+ }
1850
+ }
1851
+
1852
+ if (tiers_available == tiers_said_yes && tiers_said_yes) {
1853
/* This metric has no data and no references */
1740
- delete_dimension_uuid(&rd->state->metric_uuid);
1854
+ delete_dimension_uuid(&rd->metric_uuid);
1855
} else {
1856
/* Do not delete this dimension */
1857
#ifdef ENABLE_ACLK
database/sqlite/sqlite_aclk_chart.c
+10
-10
@@ -611,7 +611,7 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
611
rrddim_foreach_read(rd, st)
612
{
613
rrddim_flag_clear(rd, RRDDIM_FLAG_ACLK);
614
- rd->state->aclk_live_status = (rd->state->aclk_live_status == 0);
614
+ rd->aclk_live_status = (rd->aclk_live_status == 0);
615
}
616
rrdset_unlock(st);
617
}
@@ -927,7 +927,7 @@ void aclk_update_retention(struct aclk_database_worker_config *wc)
927
#ifdef ENABLE_DBENGINE
928
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
929
rc =
930
- rrdeng_metric_latest_time_by_uuid((uuid_t *)sqlite3_column_blob(res, 0), &first_entry_t, &last_entry_t);
930
+ rrdeng_metric_latest_time_by_uuid((uuid_t *)sqlite3_column_blob(res, 0), &first_entry_t, &last_entry_t, 0);
931
else
932
#endif
933
{
@@ -1088,15 +1088,15 @@ void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated)
1088
{
1089
int live = !last_updated;
1090
1091
- if (likely(rd->state->aclk_live_status == live))
1091
+ if (likely(rd->aclk_live_status == live))
1092
return;
1093
1094
- time_t created_at = rd->state->query_ops.oldest_time(rd);
1094
+ time_t created_at = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle);
1095
1096
if (unlikely(!created_at && rd->updated))
1097
created_at = rd->last_collected_time.tv_sec;
1098
1099
- rd->state->aclk_live_status = live;
1099
+ rd->aclk_live_status = live;
1100
1101
struct aclk_database_worker_config *wc = rd->rrdset->rrdhost->dbsync_worker;
1102
if (unlikely(!wc))
@@ -1124,7 +1124,7 @@ void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated)
1124
return;
1125
1126
struct aclk_chart_dimension_data *aclk_cd_data = mallocz(sizeof(*aclk_cd_data));
1127
- uuid_copy(aclk_cd_data->uuid, rd->state->metric_uuid);
1127
+ uuid_copy(aclk_cd_data->uuid, rd->metric_uuid);
1128
aclk_cd_data->payload = payload;
1129
aclk_cd_data->payload_size = size;
1130
aclk_cd_data->check_payload = 1;
@@ -1139,7 +1139,7 @@ void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated)
1139
if (unlikely(rc)) {
1140
freez(aclk_cd_data->payload);
1141
freez(aclk_cd_data);
1142
- rd->state->aclk_live_status = !live;
1142
+ rd->aclk_live_status = !live;
1143
}
1144
return;
1145
}
@@ -1156,11 +1156,11 @@ void aclk_send_dimension_update(RRDDIM *rd)
1156
time_t now = now_realtime_sec();
1157
int live = ((now - rd->last_collected_time.tv_sec) < (RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER * rd->update_every));
1158
1159
- if (!live || rd->state->aclk_live_status != live || !first_entry_t) {
1159
+ if (!live || rd->aclk_live_status != live || !first_entry_t) {
1160
(void)aclk_upd_dimension_event(
1161
rd->rrdset->rrdhost->dbsync_worker,
1162
claim_id,
1163
- &rd->state->metric_uuid,
1163
+ &rd->metric_uuid,
1164
rd->id,
1165
rd->name,
1166
rd->rrdset->id,
@@ -1189,7 +1189,7 @@ void aclk_send_dimension_update(RRDDIM *rd)
1189
first_entry_t,
1190
last_entry_t,
1191
now - last_entry_t);
1192
- rd->state->aclk_live_status = live;
1192
+ rd->aclk_live_status = live;
1193
}
1194
1195
freez(claim_id);
database/sqlite/sqlite_functions.c
+28
-18
@@ -1369,8 +1369,10 @@ RRDHOST *sql_create_host_by_uuid(char *hostname)
1369
1370
host->system_info = callocz(1, sizeof(*host->system_info));;
1371
rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
1372
+
1373
#ifdef ENABLE_DBENGINE
1373
- host->rrdeng_ctx = &multidb_ctx;
1374
+ for(int tier = 0; tier < storage_tiers ; tier++)
1375
+ host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
1376
#endif
1377
1378
failed:
@@ -1538,7 +1540,7 @@ failed:
1540
}
1541
1542
int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id,
1541
- uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid)
1543
+ uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid, int tier)
1544
{
1545
#ifdef ENABLE_DBENGINE
1546
int rc;
@@ -1546,13 +1548,13 @@ int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id
1548
uuid_t multihost_legacy_uuid;
1549
time_t dim_first_entry_t, dim_last_entry_t;
1550
1549
- rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t);
1551
+ rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
1552
if (unlikely(rc)) {
1553
rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid);
1552
- rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t);
1554
+ rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
1555
if (likely(rc)) {
1556
rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid);
1555
- rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t);
1557
+ rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
1558
if (likely(!rc))
1559
uuid_copy(*rrdeng_uuid, multihost_legacy_uuid);
1560
}
@@ -1578,27 +1580,35 @@ int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id
1580
return 1;
1581
#endif
1582
}
1581
-
1583
+#include "../storage_engine.h"
1584
#ifdef ENABLE_DBENGINE
1585
static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
1586
{
1587
RRDDIM *rd = onewayalloc_callocz(owa, 1, sizeof(*rd));
1588
rd->rrdset = st;
1589
+ rd->update_every = st->update_every;
1590
rd->last_stored_value = NAN;
1591
rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
1589
- rd->state = onewayalloc_mallocz(owa, sizeof(*rd->state));
1590
- rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
1591
- rd->state->query_ops.init = rrdeng_load_metric_init;
1592
- rd->state->query_ops.next_metric = rrdeng_load_metric_next;
1593
- rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
1594
- rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
1595
- rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
1596
- rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
1597
- rd->state->rrdeng_uuid = onewayalloc_mallocz(owa, sizeof(uuid_t));
1598
- uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid);
1599
- uuid_copy(rd->state->metric_uuid, *metric_uuid);
1592
+ STORAGE_ENGINE *eng = storage_engine_get(RRD_MEMORY_MODE_DBENGINE);
1593
+
1594
+ uuid_copy(rd->metric_uuid, *metric_uuid);
1595
rd->id = onewayalloc_strdupz(owa, id);
1596
rd->name = onewayalloc_strdupz(owa, name);
1597
+
1598
+ for(int tier = 0; tier < storage_tiers ;tier++) {
1599
+ rd->tiers[tier] = onewayalloc_callocz(owa, 1, sizeof(*rd->tiers[tier]));
1600
+ rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
1601
+ rd->tiers[tier]->tier_grouping = get_tier_grouping(tier);
1602
+ rd->tiers[tier]->mode = RRD_MEMORY_MODE_DBENGINE;
1603
+ rd->tiers[tier]->query_ops.init = rrdeng_load_metric_init;
1604
+ rd->tiers[tier]->query_ops.next_metric = rrdeng_load_metric_next;
1605
+ rd->tiers[tier]->query_ops.is_finished = rrdeng_load_metric_is_finished;
1606
+ rd->tiers[tier]->query_ops.finalize = rrdeng_load_metric_finalize;
1607
+ rd->tiers[tier]->query_ops.latest_time = rrdeng_metric_latest_time;
1608
+ rd->tiers[tier]->query_ops.oldest_time = rrdeng_metric_oldest_time;
1609
+ rd->tiers[tier]->db_metric_handle = eng->api.init(rd, st->rrdhost->storage_instance[tier]);
1610
+ }
1611
+
1612
return rd;
1613
}
1614
#endif
@@ -1697,7 +1707,7 @@ void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **para
1707
1708
if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1),
1709
(uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t,
1700
- &rrdeng_uuid)))
1710
+ &rrdeng_uuid, 0)))
1711
continue;
1712
1713
st->counter++;
database/storage_engine.c
+14
@@ -9,6 +9,7 @@
9
#define im_collect_ops { \
10
.init = rrddim_collect_init,\
11
.store_metric = rrddim_collect_store_metric,\
12
+ .flush = rrddim_store_metric_flush,\
13
.finalize = rrddim_collect_finalize\
14
}
15
@@ -26,6 +27,8 @@ static STORAGE_ENGINE engines[] = {
27
.id = RRD_MEMORY_MODE_NONE,
28
.name = RRD_MEMORY_MODE_NONE_NAME,
29
.api = {
30
+ .init = rrddim_metric_init,
31
+ .free = rrddim_metric_free,
32
.collect_ops = im_collect_ops,
33
.query_ops = im_query_ops
34
}
@@ -34,6 +37,8 @@ static STORAGE_ENGINE engines[] = {
37
.id = RRD_MEMORY_MODE_RAM,
38
.name = RRD_MEMORY_MODE_RAM_NAME,
39
.api = {
40
+ .init = rrddim_metric_init,
41
+ .free = rrddim_metric_free,
42
.collect_ops = im_collect_ops,
43
.query_ops = im_query_ops
44
}
@@ -42,6 +47,8 @@ static STORAGE_ENGINE engines[] = {
47
.id = RRD_MEMORY_MODE_MAP,
48
.name = RRD_MEMORY_MODE_MAP_NAME,
49
.api = {
50
+ .init = rrddim_metric_init,
51
+ .free = rrddim_metric_free,
52
.collect_ops = im_collect_ops,
53
.query_ops = im_query_ops
54
}
@@ -50,6 +57,8 @@ static STORAGE_ENGINE engines[] = {
57
.id = RRD_MEMORY_MODE_SAVE,
58
.name = RRD_MEMORY_MODE_SAVE_NAME,
59
.api = {
60
+ .init = rrddim_metric_init,
61
+ .free = rrddim_metric_free,
62
.collect_ops = im_collect_ops,
63
.query_ops = im_query_ops
64
}
@@ -58,6 +67,8 @@ static STORAGE_ENGINE engines[] = {
67
.id = RRD_MEMORY_MODE_ALLOC,
68
.name = RRD_MEMORY_MODE_ALLOC_NAME,
69
.api = {
70
+ .init = rrddim_metric_init,
71
+ .free = rrddim_metric_free,
72
.collect_ops = im_collect_ops,
73
.query_ops = im_query_ops
74
}
@@ -67,9 +78,12 @@ static STORAGE_ENGINE engines[] = {
78
.id = RRD_MEMORY_MODE_DBENGINE,
79
.name = RRD_MEMORY_MODE_DBENGINE_NAME,
80
.api = {
81
+ .init = rrdeng_metric_init,
82
+ .free = rrdeng_metric_free,
83
.collect_ops = {
84
.init = rrdeng_store_metric_init,
85
.store_metric = rrdeng_store_metric_next,
86
+ .flush = rrdeng_store_metric_flush_current_page,
87
.finalize = rrdeng_store_metric_finalize
88
},
89
.query_ops = {
database/storage_engine.h
+2
@@ -10,6 +10,8 @@ typedef struct storage_engine STORAGE_ENGINE;
10
// ------------------------------------------------------------------------
11
// function pointers for all APIs provided by a storge engine
12
typedef struct storage_engine_api {
13
+ STORAGE_METRIC_HANDLE *(*init)(RRDDIM *rd, STORAGE_INSTANCE *instance);
14
+ void (*free)(STORAGE_METRIC_HANDLE *);
15
struct rrddim_collect_ops collect_ops;
16
struct rrddim_query_ops query_ops;
17
} STORAGE_ENGINE_API;
docs/guides/longer-metrics-storage.md
+109
-101
@@ -1,150 +1,158 @@
1
<!--
2
-title: "Change how long Netdata stores metrics"
3
-description: "With a single configuration change, the Netdata Agent can store days, weeks, or months of metrics at its famous per-second granularity."
2
+title: "Netdata Longer Metrics Retention"
3
+description: ""
4
custom_edit_url: https://github.com/netdata/netdata/edit/master/docs/guides/longer-metrics-storage.md
5
-->
6
7
-# Change how long Netdata stores metrics
7
+# Netdata Longer Metrics Retention
8
9
-Netdata helps you collect thousands of system and application metrics every second, but what about storing them for the
10
-long term?
9
+Metrics retention affects 3 parameters on the operation of a Netdata Agent:
10
12
-Many people think Netdata can only store about an hour's worth of real-time metrics, but that's simply not true any
13
-more. With the right settings, Netdata is quite capable of efficiently storing hours or days worth of historical,
14
-per-second metrics without having to rely on an [exporting engine](/docs/export/external-databases.md).
11
+1. The disk space required to store the metrics.
12
+2. The memory the Netdata Agent will require to have that retention available for queries.
13
+3. The CPU resources that will be required to query longer time-frames.
14
16
-This guide gives two options for configuring Netdata to store more metrics. **We recommend the default [database
17
-engine](#using-the-database-engine)**, but you can stick with or switch to the round-robin database if you prefer.
15
+As retention increases, the resources required to support that retention increase too.
16
19
-Let's get started.
17
+Since Netdata Agents usually run at the edge, inside production systems, Netdata Agent **parents** should be considered. When having a **parent - child** setup, the child (the Netdata Agent running on a production system) delegates all its functions, including longer metrics retention and querying, to the parent node that can dedicate more resources to this task. A single Netdata Agent parent can centralize multiple children Netdata Agents (dozens, hundreds, or even thousands depending on its available resources).
18
21
-## Using the database engine
19
23
-The database engine uses RAM to store recent metrics while also using a "spill to disk" feature that takes advantage of
24
-available disk space for long-term metrics storage. This feature of the database engine allows you to store a much
25
-larger dataset than your system's available RAM.
20
+## Ephemerality of metrics
21
27
-The database engine is currently the default method of storing metrics, but if you're not sure which database you're
28
-using, check out your `netdata.conf` file and look for the `[db].mode` setting:
22
+The ephemerality of metrics plays an important role in retention. In environments where metrics stop being collected and new metrics are constantly being generated, we are interested about 2 parameters:
23
30
-```conf
31
-[db]
32
- mode = dbengine
33
-```
34
-
35
-If `[db].mode` is set to anything but `dbengine`, change it and restart Netdata using the standard command for
36
-restarting services on your system. You're now using the database engine!
24
+1. The **expected concurrent number of metrics** as an average for the lifetime of the database.
25
+ This affects mainly the storage requirements.
26
38
-What makes the database engine efficient? While it's structured like a traditional database, the database engine splits
39
-data between RAM and disk. The database engine caches and indexes data on RAM to keep memory usage low, and then
40
-compresses older metrics onto disk for long-term storage.
27
+2. The **expected total number of unique metrics** for the lifetime of the database.
28
+ This affects mainly the memory requirements for having all these metrics indexed and available to be queried.
29
42
-When the Netdata dashboard queries for historical metrics, the database engine will use its cache, stored in RAM, to
43
-return relevant metrics for visualization in charts.
30
+## Granularity of metrics
31
45
-Now, given that the database engine uses _both_ RAM and disk, there are two other settings to consider: `page cache
46
-size MB` and `dbengine multihost disk space MB`.
32
+The granularity of metrics (the frequency they are collected and stored, i.e. their resolution) is significantly affecting retention.
33
48
-```conf
49
-[db]
50
- page cache size MB = 32
51
- dbengine multihost disk space MB = 256
52
-```
34
+Lowering the granularity from per second to every two seconds, will double their retention and half the CPU requirements of the Netdata Agent, without affecting disk space or memory requirements.
35
54
-`[db].page cache size MB` sets the maximum amount of RAM the database engine will use for caching and indexing.
55
-`[db].dbengine multihost disk space MB` sets the maximum disk space the database engine will use for storing
56
-compressed metrics. The default settings retain about four day's worth of metrics on a system collecting 2,000 metrics
57
-every second.
36
+## Which database mode to use
37
59
-[**See our database engine
60
-calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
61
-to help you correctly set `[db].dbengine multihost disk space MB` based on your needs. The calculator gives an accurate estimate
62
-based on how many child nodes you have, how many metrics your Agent collects, and more.
38
+Netdata Agents support multiple database modes.
39
64
-With the database engine active, you can back up your `/var/cache/netdata/dbengine/` folder to another location for
65
-redundancy.
40
+The default mode `[db].mode = dbengine` has been designed to scale for longer retentions.
41
67
-Now that you know how to switch to the database engine, let's cover the default round-robin database for those who
68
-aren't ready to make the move.
42
+The other available database modes are designed to minimize resource utilization and should usually be considered on **parent - child** setups at the children side.
43
70
-## Using the round-robin database
44
+So,
45
72
-In previous versions, Netdata used a round-robin database to store 1 hour of per-second metrics.
46
+* On a single node setup, use `[db].mode = dbengine` to increase retention.
47
+* On a **parent - child** setup, use `[db].mode = dbengine` on the parent to increase retention and a more resource efficient mode (like `save`, `ram` or `none`) for the child to minimize resources utilization.
48
74
-To see if you're still using this database, or if you would like to switch to it, open your `netdata.conf` file and see
75
-if `[db].mode` option is set to `save`.
49
+To use `dbengine`, set this in `netdata.conf` (it is the default):
50
77
-```conf
51
+```
52
[db]
79
- mode = save
53
+ mode = dbengine
54
```
55
82
-If `[db].mode` is set to `save`, then you're using the round-robin database. If so, the `[db].retention` option is set to
83
-`3600`, which is the equivalent to 3,600 seconds, or one hour.
56
+## Tiering
57
85
-To increase your historical metrics, you can increase `[db].retention` to the number of seconds you'd like to store:
58
+`dbengine` supports tiering. Tiering allows having up to 3 versions of the data:
59
87
-```conf
60
+1. Tier 0 is the high resolution data.
61
+2. Tier 1 is the first tier that samples data every 60 data collections of Tier 0.
62
+3. Tier 2 is the second tier that samples data every 3600 data collections of Tier 0 (60 of Tier 1).
63
+
64
+To enable tiering set `[db].storage tiers` in `netdata.conf` (the default is 1, to enable only Tier 0):
65
+
66
+```
67
[db]
89
- # 2 hours = 2 * 60 * 60 = 7200 seconds
90
- retention = 7200
91
- # 4 hours = 4 * 60 * 60 = 14440 seconds
92
- retention = 14440
93
- # 24 hours = 24 * 60 * 60 = 86400 seconds
94
- retention = 86400
68
+ mode = dbengine
69
+ storage tiers = 3
70
```
71
97
-And so on.
72
+## Disk space requirements
73
99
-Next, check to see how many metrics Netdata collects on your system, and how much RAM that uses. Visit the Netdata
100
-dashboard and look at the bottom-right corner of the interface. You'll find a sentence similar to the following:
74
+Netdata Agents require about 1 bytes on disk per database point on Tier 0 and 4 times more on higher tiers (Tier 1 and 2). They require 4 times more storage per point compared to Tier 0, because for every point higher tiers store `min`, `max`, `sum`, `count` and `anomaly rate` (the values are 5, but they require 4 times the storage because `count` and `anomaly rate` are 16-bit integers). The `average` is calculated on the fly at query time using `sum / count`.
75
102
-> Every second, Netdata collects 1,938 metrics, presents them in 299 charts and monitors them with 81 alarms. Netdata is
103
-> using 25 MB of memory on **netdata-linux** for 1 hour, 6 minutes and 36 seconds of real-time history.
76
+### Tier 0 - per second for a week
77
105
-On this desktop system, using a Ryzen 5 1600 and 16GB of RAM, the round-robin databases uses 25 MB of RAM to store just
106
-over an hour's worth of data for nearly 2,000 metrics.
78
+For 2000 metrics, collected every second and retained for a week, Tier 0 needs: 1 byte x 2000 metrics x 3600 secs per hour x 24 hours per day x 7 days per week = 1100MB.
79
108
-You should base this number on two things: How much history you need for your use case, and how much RAM you're willing
109
-to dedicate to Netdata.
80
+The setting to control this is in `netdata.conf`:
81
111
-How much RAM will a longer retention use? Let's use a little math.
82
+```
83
+[db]
84
+ mode = dbengine
85
+
86
+ # per second data collection
87
+ update every = 1
88
+
89
+ # enable only Tier 0
90
+ storage tiers = 1
91
+
92
+ # Tier 0, per second data for a week
93
+ dbengine multihost disk space MB = 1100
94
+```
95
113
-The round-robin database needs 4 bytes for every value Netdata collects. If Netdata collects metrics every second,
114
-that's 4 bytes, per second, per metric.
96
+By setting it to `1100` and restarting the Netdata Agent, this node will start maintaining about a week of data. But pay attention to the number of metrics. If you have more than 2000 metrics on a node, or you need more that a week of high resolution metrics, you may need to adjust this setting accordingly.
97
116
-```text
117
-4 bytes * X seconds * Y metrics = RAM usage in bytes
118
-```
98
+### Tier 1 - per minute for a month
99
120
-Let's assume your system collects 1,000 metrics per second.
100
+Tier 1 is by default sampling the data every 60 points of Tier 0. If Tier 0 is per second, then Tier 1 is per minute.
101
122
-```text
123
-4 bytes * 3600 seconds * 1,000 metrics = 14400000 bytes = 14.4 MB RAM
124
-```
102
+Tier 1 needs 4 times more storage per point compared to Tier 0. So, for 2000 metrics, with per minute resolution, retained for a month, Tier 1 needs: 4 bytes x 2000 metrics x 60 minutes per hour x 24 hours per day x 30 days per month = 330MB.
103
126
-With that formula, you can calculate the RAM usage for much larger history settings.
127
-
128
-```conf
129
-# 2 hours at 1,000 metrics per second
130
-4 bytes * 7200 seconds * 1,000 metrics = 28800000 bytes = 28.8 MB RAM
131
-# 2 hours at 2,000 metrics per second
132
-4 bytes * 7200 seconds * 2,000 metrics = 57600000 bytes = 57.6 MB RAM
133
-# 4 hours at 2,000 metrics per second
134
-4 bytes * 14440 seconds * 2,000 metrics = 115520000 bytes = 115.52 MB RAM
135
-# 24 hours at 1,000 metrics per second
136
-4 bytes * 86400 seconds * 1,000 metrics = 345600000 bytes = 345.6 MB RAM
104
+Do this in `netdata.conf`:
105
+
106
+```
107
+[db]
108
+ mode = dbengine
109
+
110
+ # per second data collection
111
+ update every = 1
112
+
113
+ # enable only Tier 0 and Tier 1
114
+ storage tiers = 2
115
+
116
+ # Tier 0, per second data for a week
117
+ dbengine multihost disk space MB = 1100
118
+
119
+ # Tier 1, per minute data for a month
120
+ dbengine tier 1 multihost disk space MB = 330
121
```
122
139
-## What's next?
123
+Once `netdata.conf` is edited, the Netdata Agent needs to be restarted for the changes to take effect.
124
+
125
+### Tier 2 - per hour for a year
126
+
127
+Tier 2 is by default sampling data every 3600 points of Tier 0 (60 of Tier 1). If Tier 0 is per second, then Tier 2 is per hour.
128
141
-Now that you have either configured database engine or round-robin database engine to store more metrics, you'll
142
-probably want to see it in action!
129
+The storage requirements are the same to Tier 1.
130
+
131
+For 2000 metrics, with per hour resolution, retained for a year, Tier 2 needs: 4 bytes x 2000 metrics x 24 hours per day x 365 days per year = 67MB.
132
+
133
+Do this in `netdata.conf`:
134
+
135
+```
136
+[db]
137
+ mode = dbengine
138
+
139
+ # per second data collection
140
+ update every = 1
141
+
142
+ # enable only Tier 0 and Tier 1
143
+ storage tiers = 3
144
+
145
+ # Tier 0, per second data for a week
146
+ dbengine multihost disk space MB = 1100
147
+
148
+ # Tier 1, per minute data for a month
149
+ dbengine tier 1 multihost disk space MB = 330
150
+
151
+ # Tier 2, per hour data for a year
152
+ dbengine tier 2 multihost disk space MB = 67
153
+```
154
144
-For more information about how to pan charts to view historical metrics, see our documentation on [using
145
-charts](/web/README.md#using-charts).
155
+Once `netdata.conf` is edited, the Netdata Agent needs to be restarted for the changes to take effect.
156
147
-And if you'd now like to reduce Netdata's resource usage, view our [performance
148
-guide](/docs/guides/configure/performance.md) for our best practices on optimization.
157
158
exporting/process_data.c
+9
-12
@@ -77,8 +77,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
77
time_t before = instance->before;
78
79
// find the edges of the rrd database for this chart
80
- time_t first_t = rd->state->query_ops.oldest_time(rd);
81
- time_t last_t = rd->state->query_ops.latest_time(rd);
80
+ time_t first_t = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle);
81
+ time_t last_t = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
82
time_t update_every = st->update_every;
83
struct rrddim_query_handle handle;
84
@@ -124,23 +124,20 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
124
125
size_t counter = 0;
126
NETDATA_DOUBLE sum = 0;
127
- NETDATA_DOUBLE value;
127
129
- for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
130
- time_t curr_t, end_t;
131
- SN_FLAGS flags;
132
- value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);
128
+ for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) {
129
+ STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
130
134
- if (unlikely(!netdata_double_isnumber(value))) {
131
+ if (unlikely(storage_point_is_empty(sp))) {
132
// not collected
133
continue;
134
}
135
139
- sum += value;
140
-
141
- counter++;
136
+ sum += sp.sum;
137
+ counter += sp.count;
138
}
143
- rd->state->query_ops.finalize(&handle);
139
+ rd->tiers[0]->query_ops.finalize(&handle);
140
+
141
if (unlikely(!counter)) {
142
debug(
143
D_EXPORTING,
exporting/tests/exporting_fixtures.c
+8
-8
@@ -63,13 +63,13 @@ int setup_rrdhost()
63
rd->collections_counter++;
64
rd->next = NULL;
65
66
- rd->state = calloc(1, sizeof(*rd->state));
67
- rd->state->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
68
- rd->state->query_ops.latest_time = __mock_rrddim_query_latest_time;
69
- rd->state->query_ops.init = __mock_rrddim_query_init;
70
- rd->state->query_ops.is_finished = __mock_rrddim_query_is_finished;
71
- rd->state->query_ops.next_metric = __mock_rrddim_query_next_metric;
72
- rd->state->query_ops.finalize = __mock_rrddim_query_finalize;
66
+ rd->tiers[0] = calloc(1, sizeof(struct rrddim_tier));
67
+ rd->tiers[0]->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
68
+ rd->tiers[0]->query_ops.latest_time = __mock_rrddim_query_latest_time;
69
+ rd->tiers[0]->query_ops.init = __mock_rrddim_query_init;
70
+ rd->tiers[0]->query_ops.is_finished = __mock_rrddim_query_is_finished;
71
+ rd->tiers[0]->query_ops.next_metric = __mock_rrddim_query_next_metric;
72
+ rd->tiers[0]->query_ops.finalize = __mock_rrddim_query_finalize;
73
74
return 0;
75
}
@@ -79,7 +79,7 @@ int teardown_rrdhost()
79
RRDDIM *rd = localhost->rrdset_root->dimensions;
80
free((void *)rd->name);
81
free((void *)rd->id);
82
- free(rd->state);
82
+ free(rd->tiers[0]);
83
free(rd);
84
85
RRDSET *st = localhost->rrdset_root;
exporting/tests/netdata_doubles.c
+11
-12
@@ -196,26 +196,27 @@ void rrdset_update_heterogeneous_flag(RRDSET *st)
196
(void)st;
197
}
198
199
-time_t __mock_rrddim_query_oldest_time(RRDDIM *rd)
199
+time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle)
200
{
201
- (void)rd;
201
+ (void)db_metric_handle;
202
203
function_called();
204
return mock_type(time_t);
205
}
206
207
-time_t __mock_rrddim_query_latest_time(RRDDIM *rd)
207
+time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle)
208
{
209
- (void)rd;
209
+ (void)db_metric_handle;
210
211
function_called();
212
return mock_type(time_t);
213
}
214
215
-void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time)
215
+void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type)
216
{
217
- (void)rd;
217
+ (void)db_metric_handle;
218
(void)handle;
219
+ (void)tier_query_fetch_type;
220
221
function_called();
222
check_expected(start_time);
@@ -230,16 +231,14 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
231
return mock_type(int);
232
}
233
233
-NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
234
+STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle)
235
{
236
(void)handle;
236
- (void)start_time;
237
- (void)end_time;
238
- (void) flags;
239
-
237
238
function_called();
242
- return mock_type(NETDATA_DOUBLE);
239
+
240
+ STORAGE_POINT sp = {};
241
+ return sp;
242
}
243
244
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
exporting/tests/test_exporting_engine.c
-2
@@ -307,12 +307,10 @@ static void test_exporting_calculate_value_from_stored_data(void **state)
307
expect_function_call(__mock_rrddim_query_is_finished);
308
will_return(__mock_rrddim_query_is_finished, 0);
309
expect_function_call(__mock_rrddim_query_next_metric);
310
- will_return(__mock_rrddim_query_next_metric, 27);
310
311
expect_function_call(__mock_rrddim_query_is_finished);
312
will_return(__mock_rrddim_query_is_finished, 0);
313
expect_function_call(__mock_rrddim_query_next_metric);
315
- will_return(__mock_rrddim_query_next_metric, 45);
314
315
expect_function_call(__mock_rrddim_query_is_finished);
316
will_return(__mock_rrddim_query_is_finished, 1);
exporting/tests/test_exporting_engine.h
+4
-4
@@ -57,11 +57,11 @@ int __wrap_connect_to_one_of(
57
void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);
58
void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line);
59
void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line);
60
-time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
61
-time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
62
-void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
60
+time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
61
+time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
62
+void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
63
int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
64
-NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
64
+STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle);
65
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
66
67
// -----------------------------------------------------------------------
health/health.c
+1
-1
@@ -859,7 +859,7 @@ void *health_main(void *ptr) {
859
0, rc->options,
860
&rc->db_after,&rc->db_before,
861
NULL, NULL,
862
- &value_is_null, NULL, 0);
862
+ &value_is_null, NULL, 0, 0);
863
864
if (unlikely(ret != 200)) {
865
// database lookup failed
libnetdata/libnetdata.h
+2
@@ -351,6 +351,8 @@ extern char *netdata_configured_host_prefix;
351
// BEWARE: Outside of the C code this also exists in alarm-notify.sh
352
#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
353
354
+#define RRD_STORAGE_TIERS 5
355
+
356
# ifdef __cplusplus
357
}
358
# endif
libnetdata/log/log.h
+1
-1
@@ -82,7 +82,7 @@ static inline void debug_dummy(void) {}
82
83
#ifdef NETDATA_INTERNAL_CHECKS
84
#define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
85
-#define internal_error(condition, args...) do { if(unlikely(condition)) error_int("INTERNAL ERROR", __FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
85
+#define internal_error(condition, args...) do { if(unlikely(condition)) error_int("IERR", __FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
86
#else
87
#define debug(type, args...) debug_dummy()
88
#define internal_error(args...) debug_dummy()
libnetdata/onewayalloc/onewayalloc.c
+8
@@ -166,6 +166,14 @@ void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_
166
return;
167
}
168
169
+void *onewayalloc_doublesize(ONEWAYALLOC *owa, const void *src, size_t oldsize) {
170
+ size_t newsize = oldsize * 2;
171
+ void *dst = onewayalloc_mallocz(owa, newsize);
172
+ memcpy(dst, src, oldsize);
173
+ onewayalloc_freez(owa, src);
174
+ return dst;
175
+}
176
+
177
void onewayalloc_destroy(ONEWAYALLOC *owa) {
178
if(!owa) return;
179
libnetdata/onewayalloc/onewayalloc.h
+2
@@ -14,4 +14,6 @@ extern char *onewayalloc_strdupz(ONEWAYALLOC *owa, const char *s);
14
extern void *onewayalloc_memdupz(ONEWAYALLOC *owa, const void *src, size_t size);
15
extern void onewayalloc_freez(ONEWAYALLOC *owa, const void *ptr);
16
17
+extern void *onewayalloc_doublesize(ONEWAYALLOC *owa, const void *src, size_t oldsize);
18
+
19
#endif // ONEWAYALLOC_H
libnetdata/storage_number/storage_number.h
+9
@@ -63,6 +63,15 @@ typedef long long collected_number;
63
#endif
64
65
typedef uint32_t storage_number;
66
+
67
+typedef struct storage_number_tier1 {
68
+ float sum_value;
69
+ float min_value;
70
+ float max_value;
71
+ uint16_t count;
72
+ uint16_t anomaly_count;
73
+} storage_number_tier1_t;
74
+
75
#define STORAGE_NUMBER_FORMAT "%u"
76
77
typedef enum {
ml/Dimension.h
+3
-3
@@ -12,13 +12,13 @@ namespace ml {
12
13
class RrdDimension {
14
public:
15
- RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->state->query_ops) { }
15
+ RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->tiers[0]->query_ops) { }
16
17
RRDDIM *getRD() const { return RD; }
18
19
- time_t latestTime() { return Ops->latest_time(RD); }
19
+ time_t latestTime() { return Ops->latest_time(RD->tiers[0]->db_metric_handle); }
20
21
- time_t oldestTime() { return Ops->oldest_time(RD); }
21
+ time_t oldestTime() { return Ops->oldest_time(RD->tiers[0]->db_metric_handle); }
22
23
unsigned updateEvery() const { return RD->update_every; }
24
ml/Query.h
+6
-8
@@ -8,19 +8,19 @@ namespace ml {
8
class Query {
9
public:
10
Query(RRDDIM *RD) : RD(RD) {
11
- Ops = &RD->state->query_ops;
11
+ Ops = &RD->tiers[0]->query_ops;
12
}
13
14
time_t latestTime() {
15
- return Ops->latest_time(RD);
15
+ return Ops->latest_time(RD->tiers[0]->db_metric_handle);
16
}
17
18
time_t oldestTime() {
19
- return Ops->oldest_time(RD);
19
+ return Ops->oldest_time(RD->tiers[0]->db_metric_handle);
20
}
21
22
void init(time_t AfterT, time_t BeforeT) {
23
- Ops->init(RD, &Handle, AfterT, BeforeT);
23
+ Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT, TIER_QUERY_FETCH_SUM);
24
}
25
26
bool isFinished() {
@@ -28,10 +28,8 @@ public:
28
}
29
30
std::pair<time_t, CalculatedNumber> nextMetric() {
31
- time_t CurrT, EndT;
32
- SN_FLAGS Flags;
33
- auto Value = (CalculatedNumber)Ops->next_metric(&Handle, &CurrT, &EndT, &Flags);
34
- return { CurrT, Value };
31
+ STORAGE_POINT sp = Ops->next_metric(&Handle);
32
+ return { sp.start_time, sp.sum / sp.count };
33
}
34
35
~Query() {
ml/ml.cc
+5
-5
@@ -80,12 +80,12 @@ void ml_new_dimension(RRDDIM *RD) {
80
return;
81
82
Dimension *D = new Dimension(RD);
83
- RD->state->ml_dimension = static_cast<ml_dimension_t>(D);
83
+ RD->ml_dimension = static_cast<ml_dimension_t>(D);
84
H->addDimension(D);
85
}
86
87
void ml_delete_dimension(RRDDIM *RD) {
88
- Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
88
+ Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
89
if (!D)
90
return;
91
@@ -95,7 +95,7 @@ void ml_delete_dimension(RRDDIM *RD) {
95
else
96
H->removeDimension(D);
97
98
- RD->state->ml_dimension = nullptr;
98
+ RD->ml_dimension = nullptr;
99
}
100
101
char *ml_get_host_info(RRDHOST *RH) {
@@ -125,7 +125,7 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
125
}
126
127
bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
128
- Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
128
+ Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
129
if (!D)
130
return false;
131
@@ -210,7 +210,7 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
210
void ml_dimension_update_name(RRDSET *RS, RRDDIM *RD, const char *Name) {
211
(void) RS;
212
213
- Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
213
+ Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
214
if (!D)
215
return;
216
web/api/badges/web_buffer_svg.c
+1
-1
@@ -1111,7 +1111,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
1111
points, after, before, group, group_options, 0, options,
1112
NULL, &latest_timestamp,
1113
NULL, NULL,
1114
- &value_is_null, NULL, 0);
1114
+ &value_is_null, NULL, 0, 0);
1115
1116
// if the value cannot be calculated, show empty badge
1117
if (ret != HTTP_RESP_OK) {
web/api/formatters/json_wrapper.c
+13
-4
@@ -90,7 +90,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
90
, kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
91
, kq, kq, sq);
92
93
- web_client_api_request_v1_data_options_to_string(wb, options);
93
+ web_client_api_request_v1_data_options_to_string(wb, r->internal.query_options);
94
95
buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq);
96
@@ -343,12 +343,21 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
343
344
rrdr_buffer_print_format(wb, format);
345
346
+ buffer_sprintf(wb, "%s,\n"
347
+ " %sdb_points_per_tier%s: [ "
348
+ , sq
349
+ , kq, kq
350
+ );
351
+
352
+ for(int tier = 0; tier < storage_tiers ; tier++)
353
+ buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]);
354
+
355
+ buffer_strcat(wb, " ]");
356
+
357
if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) {
347
- buffer_sprintf(wb, "%s,\n %schart_variables%s: ", sq, kq, kq);
358
+ buffer_sprintf(wb, ",\n %schart_variables%s: ", kq, kq);
359
health_api_v1_chart_custom_variables2json(r->st, wb);
360
}
350
- else
351
- buffer_sprintf(wb, "%s", sq);
361
362
buffer_sprintf(wb, ",\n %sresult%s: ", kq, kq);
363
web/api/formatters/rrd2json.c
+23
-4
@@ -1,6 +1,7 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "web/api/web_api_v1.h"
4
+#include "database/storage_engine.h"
5
6
static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
7
{
@@ -18,7 +19,18 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc
19
}
20
}
21
21
- onewayalloc_freez(owa, temp_rd->state);
22
+ for(int tier = 0; tier < storage_tiers ;tier++) {
23
+ if(!temp_rd->tiers[tier]) continue;
24
+
25
+ if(archive_mode) {
26
+ STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
27
+ if (eng)
28
+ eng->api.free(temp_rd->tiers[tier]->db_metric_handle);
29
+ }
30
+
31
+ onewayalloc_freez(owa, temp_rd->tiers[tier]);
32
+ }
33
+
34
onewayalloc_freez(owa, temp_rd);
35
}
36
@@ -89,7 +101,12 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis
101
RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
102
rd->id = onewayalloc_strdupz(owa, rd1->id);
103
rd->name = onewayalloc_strdupz(owa, rd1->name);
92
- rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state));
104
+ for(int tier = 0; tier < storage_tiers ;tier++) {
105
+ if(rd1->tiers[tier])
106
+ rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier]));
107
+ else
108
+ rd->tiers[tier] = NULL;
109
+ }
110
rd->next = (*param_list)->rd;
111
(*param_list)->rd = rd;
112
}
@@ -168,6 +185,7 @@ int rrdset2value_api_v1(
185
, int *value_is_null
186
, uint8_t *anomaly_rate
187
, int timeout
188
+ , int tier
189
) {
190
int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
191
@@ -175,7 +193,7 @@ int rrdset2value_api_v1(
193
194
RRDR *r = rrd2rrdr(owa, st, points, after, before,
195
group_method, group_time, options, dimensions, NULL,
178
- group_options, timeout);
196
+ group_options, timeout, tier);
197
198
if(!r) {
199
if(value_is_null) *value_is_null = 1;
@@ -232,6 +250,7 @@ int rrdset2anything_api_v1(
250
, long group_time
251
, uint32_t options
252
, time_t *latest_timestamp
253
+ , int tier
254
)
255
{
256
BUFFER *wb = query_params->wb;
@@ -250,7 +269,7 @@ int rrdset2anything_api_v1(
269
dimensions ? buffer_tostring(dimensions) : NULL,
270
query_params->context_param_list,
271
group_options,
253
- query_params->timeout);
272
+ query_params->timeout, tier);
273
if(!r) {
274
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
275
return HTTP_RESP_INTERNAL_SERVER_ERROR;
web/api/formatters/rrd2json.h
+2
@@ -78,6 +78,7 @@ extern int rrdset2anything_api_v1(
78
, long group_time
79
, uint32_t options
80
, time_t *latest_timestamp
81
+ , int tier
82
);
83
84
extern int rrdset2value_api_v1(
@@ -99,6 +100,7 @@ extern int rrdset2value_api_v1(
100
, int *value_is_null
101
, uint8_t *anomaly_rate
102
, int timeout
103
+ , int tier
104
);
105
106
extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st);
web/api/formatters/value/value.c
+1
-5
@@ -3,11 +3,7 @@
3
#include "value.h"
4
5
6
-inline NETDATA_DOUBLE
7
-rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {
8
- if (r->st_needs_lock)
9
- rrdset_check_rdlock(r->st);
10
-
6
+inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {
7
long c;
8
RRDDIM *d;
9
web/api/queries/average/average.c
+2
-2
@@ -11,7 +11,7 @@ struct grouping_average {
11
};
12
13
void grouping_create_average(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = callocz(1, sizeof(struct grouping_average));
14
+ r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average));
15
}
16
17
// resets when switches dimensions
@@ -23,7 +23,7 @@ void grouping_reset_average(RRDR *r) {
23
}
24
25
void grouping_free_average(RRDR *r) {
26
- freez(r->internal.grouping_data);
26
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
r->internal.grouping_data = NULL;
28
}
29
web/api/queries/countif/countif.c
+2
-2
@@ -37,7 +37,7 @@ static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
37
}
38
39
void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {
40
- struct grouping_countif *g = callocz(1, sizeof(struct grouping_countif));
40
+ struct grouping_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_countif));
41
r->internal.grouping_data = g;
42
43
if(options && *options) {
@@ -106,7 +106,7 @@ void grouping_reset_countif(RRDR *r) {
106
}
107
108
void grouping_free_countif(RRDR *r) {
109
- freez(r->internal.grouping_data);
109
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
110
r->internal.grouping_data = NULL;
111
}
112
web/api/queries/des/des.c
+5
-5
@@ -37,16 +37,16 @@ static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) {
37
NETDATA_DOUBLE points;
38
if(r->group == 1) {
39
// provide a running DES
40
- points = r->internal.points_wanted;
40
+ points = (NETDATA_DOUBLE)r->internal.points_wanted;
41
}
42
else {
43
// provide a SES with flush points
44
- points = r->group;
44
+ points = (NETDATA_DOUBLE)r->group;
45
}
46
47
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
48
// A commonly used value for alpha is 2 / (N + 1)
49
- return (points > max_window_size) ? max_window_size : points;
49
+ return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points;
50
}
51
52
static inline void set_alpha(RRDR *r, struct grouping_des *g) {
@@ -70,7 +70,7 @@ static inline void set_beta(RRDR *r, struct grouping_des *g) {
70
}
71
72
void grouping_create_des(RRDR *r, const char *options __maybe_unused) {
73
- struct grouping_des *g = (struct grouping_des *)mallocz(sizeof(struct grouping_des));
73
+ struct grouping_des *g = (struct grouping_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct grouping_des));
74
set_alpha(r, g);
75
set_beta(r, g);
76
g->level = 0.0;
@@ -92,7 +92,7 @@ void grouping_reset_des(RRDR *r) {
92
}
93
94
void grouping_free_des(RRDR *r) {
95
- freez(r->internal.grouping_data);
95
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
96
r->internal.grouping_data = NULL;
97
}
98
web/api/queries/incremental_sum/incremental_sum.c
+2
-2
@@ -12,7 +12,7 @@ struct grouping_incremental_sum {
12
};
13
14
void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) {
15
- r->internal.grouping_data = callocz(1, sizeof(struct grouping_incremental_sum));
15
+ r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum));
16
}
17
18
// resets when switches dimensions
@@ -25,7 +25,7 @@ void grouping_reset_incremental_sum(RRDR *r) {
25
}
26
27
void grouping_free_incremental_sum(RRDR *r) {
28
- freez(r->internal.grouping_data);
28
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
29
r->internal.grouping_data = NULL;
30
}
31
web/api/queries/max/max.c
+2
-2
@@ -11,7 +11,7 @@ struct grouping_max {
11
};
12
13
void grouping_create_max(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = callocz(1, sizeof(struct grouping_max));
14
+ r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max));
15
}
16
17
// resets when switches dimensions
@@ -23,7 +23,7 @@ void grouping_reset_max(RRDR *r) {
23
}
24
25
void grouping_free_max(RRDR *r) {
26
- freez(r->internal.grouping_data);
26
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
r->internal.grouping_data = NULL;
28
}
29
web/api/queries/median/median.c
+9
-4
@@ -10,14 +10,15 @@ struct grouping_median {
10
size_t series_size;
11
size_t next_pos;
12
13
- NETDATA_DOUBLE series[];
13
+ NETDATA_DOUBLE *series;
14
};
15
16
void grouping_create_median(RRDR *r, const char *options __maybe_unused) {
17
long entries = r->group;
18
if(entries < 0) entries = 0;
19
20
- struct grouping_median *g = (struct grouping_median *)callocz(1, sizeof(struct grouping_median) + entries * sizeof(NETDATA_DOUBLE));
20
+ struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median));
21
+ g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22
g->series_size = (size_t)entries;
23
24
r->internal.grouping_data = g;
@@ -31,7 +32,10 @@ void grouping_reset_median(RRDR *r) {
32
}
33
34
void grouping_free_median(RRDR *r) {
34
- freez(r->internal.grouping_data);
35
+ struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
36
+ if(g) onewayalloc_freez(r->internal.owa, g->series);
37
+
38
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
39
r->internal.grouping_data = NULL;
40
}
41
@@ -39,7 +43,8 @@ void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) {
43
struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
44
45
if(unlikely(g->next_pos >= g->series_size)) {
42
- error("INTERNAL ERROR: median buffer overflow on chart '%s' - next_pos = %zu, series_size = %zu, r->group = %ld.", r->st->name, g->next_pos, g->series_size, r->group);
46
+ g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
47
+ g->series_size *= 2;
48
}
49
else
50
g->series[g->next_pos++] = (NETDATA_DOUBLE)value;
web/api/queries/min/min.c
+2
-2
@@ -11,7 +11,7 @@ struct grouping_min {
11
};
12
13
void grouping_create_min(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = callocz(1, sizeof(struct grouping_min));
14
+ r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min));
15
}
16
17
// resets when switches dimensions
@@ -23,7 +23,7 @@ void grouping_reset_min(RRDR *r) {
23
}
24
25
void grouping_free_min(RRDR *r) {
26
- freez(r->internal.grouping_data);
26
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
r->internal.grouping_data = NULL;
28
}
29
web/api/queries/query.c
+722
-268
@@ -3,7 +3,6 @@
3
#include "query.h"
4
#include "web/api/formatters/rrd2json.h"
5
#include "rrdr.h"
6
-#include "database/ram/rrddim_mem.h"
6
7
#include "average/average.h"
8
#include "countif/countif.h"
@@ -50,6 +49,8 @@ static struct {
49
// continue after a flush as if nothing changed, for others a
50
// cleanup of the internal structures may be required).
51
NETDATA_DOUBLE (*flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
52
+
53
+ TIER_QUERY_FETCH tier_query_fetch;
54
} api_v1_data_groups[] = {
55
{.name = "average",
56
.hash = 0,
@@ -59,7 +60,8 @@ static struct {
60
.reset = grouping_reset_average,
61
.free = grouping_free_average,
62
.add = grouping_add_average,
62
- .flush = grouping_flush_average
63
+ .flush = grouping_flush_average,
64
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
65
},
66
{.name = "mean", // alias on 'average'
67
.hash = 0,
@@ -69,7 +71,8 @@ static struct {
71
.reset = grouping_reset_average,
72
.free = grouping_free_average,
73
.add = grouping_add_average,
72
- .flush = grouping_flush_average
74
+ .flush = grouping_flush_average,
75
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
76
},
77
{.name = "incremental_sum",
78
.hash = 0,
@@ -79,7 +82,8 @@ static struct {
82
.reset = grouping_reset_incremental_sum,
83
.free = grouping_free_incremental_sum,
84
.add = grouping_add_incremental_sum,
82
- .flush = grouping_flush_incremental_sum
85
+ .flush = grouping_flush_incremental_sum,
86
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
87
},
88
{.name = "incremental-sum",
89
.hash = 0,
@@ -89,7 +93,8 @@ static struct {
93
.reset = grouping_reset_incremental_sum,
94
.free = grouping_free_incremental_sum,
95
.add = grouping_add_incremental_sum,
92
- .flush = grouping_flush_incremental_sum
96
+ .flush = grouping_flush_incremental_sum,
97
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
98
},
99
{.name = "median",
100
.hash = 0,
@@ -99,7 +104,8 @@ static struct {
104
.reset = grouping_reset_median,
105
.free = grouping_free_median,
106
.add = grouping_add_median,
102
- .flush = grouping_flush_median
107
+ .flush = grouping_flush_median,
108
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
109
},
110
{.name = "min",
111
.hash = 0,
@@ -109,7 +115,8 @@ static struct {
115
.reset = grouping_reset_min,
116
.free = grouping_free_min,
117
.add = grouping_add_min,
112
- .flush = grouping_flush_min
118
+ .flush = grouping_flush_min,
119
+ .tier_query_fetch = TIER_QUERY_FETCH_MIN
120
},
121
{.name = "max",
122
.hash = 0,
@@ -119,7 +126,8 @@ static struct {
126
.reset = grouping_reset_max,
127
.free = grouping_free_max,
128
.add = grouping_add_max,
122
- .flush = grouping_flush_max
129
+ .flush = grouping_flush_max,
130
+ .tier_query_fetch = TIER_QUERY_FETCH_MAX
131
},
132
{.name = "sum",
133
.hash = 0,
@@ -129,7 +137,8 @@ static struct {
137
.reset = grouping_reset_sum,
138
.free = grouping_free_sum,
139
.add = grouping_add_sum,
132
- .flush = grouping_flush_sum
140
+ .flush = grouping_flush_sum,
141
+ .tier_query_fetch = TIER_QUERY_FETCH_SUM
142
},
143
144
// standard deviation
@@ -141,7 +150,8 @@ static struct {
150
.reset = grouping_reset_stddev,
151
.free = grouping_free_stddev,
152
.add = grouping_add_stddev,
144
- .flush = grouping_flush_stddev
153
+ .flush = grouping_flush_stddev,
154
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
155
},
156
{.name = "cv", // coefficient variation is calculated by stddev
157
.hash = 0,
@@ -151,7 +161,8 @@ static struct {
161
.reset = grouping_reset_stddev, // not an error, stddev calculates this too
162
.free = grouping_free_stddev, // not an error, stddev calculates this too
163
.add = grouping_add_stddev, // not an error, stddev calculates this too
154
- .flush = grouping_flush_coefficient_of_variation
164
+ .flush = grouping_flush_coefficient_of_variation,
165
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
166
},
167
{.name = "rsd", // alias of 'cv'
168
.hash = 0,
@@ -161,7 +172,8 @@ static struct {
172
.reset = grouping_reset_stddev, // not an error, stddev calculates this too
173
.free = grouping_free_stddev, // not an error, stddev calculates this too
174
.add = grouping_add_stddev, // not an error, stddev calculates this too
164
- .flush = grouping_flush_coefficient_of_variation
175
+ .flush = grouping_flush_coefficient_of_variation,
176
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
177
},
178
179
/*
@@ -173,7 +185,8 @@ static struct {
185
.reset = grouping_reset_stddev,
186
.free = grouping_free_stddev,
187
.add = grouping_add_stddev,
176
- .flush = grouping_flush_mean
188
+ .flush = grouping_flush_mean,
189
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
190
},
191
*/
192
@@ -186,7 +199,8 @@ static struct {
199
.reset = grouping_reset_stddev,
200
.free = grouping_free_stddev,
201
.add = grouping_add_stddev,
189
- .flush = grouping_flush_variance
202
+ .flush = grouping_flush_variance,
203
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
204
},
205
*/
206
@@ -194,44 +208,48 @@ static struct {
208
{.name = "ses",
209
.hash = 0,
210
.value = RRDR_GROUPING_SES,
197
- .init = grouping_init_ses,
211
+ .init = grouping_init_ses,
212
.create= grouping_create_ses,
213
.reset = grouping_reset_ses,
214
.free = grouping_free_ses,
215
.add = grouping_add_ses,
202
- .flush = grouping_flush_ses
216
+ .flush = grouping_flush_ses,
217
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
218
},
219
{.name = "ema", // alias for 'ses'
220
.hash = 0,
221
.value = RRDR_GROUPING_SES,
207
- .init = NULL,
222
+ .init = NULL,
223
.create= grouping_create_ses,
224
.reset = grouping_reset_ses,
225
.free = grouping_free_ses,
226
.add = grouping_add_ses,
212
- .flush = grouping_flush_ses
227
+ .flush = grouping_flush_ses,
228
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
229
},
230
{.name = "ewma", // alias for ses
231
.hash = 0,
232
.value = RRDR_GROUPING_SES,
217
- .init = NULL,
233
+ .init = NULL,
234
.create= grouping_create_ses,
235
.reset = grouping_reset_ses,
236
.free = grouping_free_ses,
237
.add = grouping_add_ses,
222
- .flush = grouping_flush_ses
238
+ .flush = grouping_flush_ses,
239
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
240
},
241
242
// double exponential smoothing
243
{.name = "des",
244
.hash = 0,
245
.value = RRDR_GROUPING_DES,
229
- .init = grouping_init_des,
246
+ .init = grouping_init_des,
247
.create= grouping_create_des,
248
.reset = grouping_reset_des,
249
.free = grouping_free_des,
250
.add = grouping_add_des,
234
- .flush = grouping_flush_des
251
+ .flush = grouping_flush_des,
252
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
253
},
254
255
{.name = "countif",
@@ -242,7 +260,8 @@ static struct {
260
.reset = grouping_reset_countif,
261
.free = grouping_free_countif,
262
.add = grouping_add_countif,
245
- .flush = grouping_flush_countif
263
+ .flush = grouping_flush_countif,
264
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
265
},
266
267
// terminator
@@ -254,7 +273,8 @@ static struct {
273
.reset = grouping_reset_average,
274
.free = grouping_free_average,
275
.add = grouping_add_average,
257
- .flush = grouping_flush_average
276
+ .flush = grouping_flush_average,
277
+ .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
278
}
279
};
280
@@ -306,22 +326,24 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) {
326
int i, found = 0;
327
for(i = 0; !found && api_v1_data_groups[i].name ;i++) {
328
if(api_v1_data_groups[i].value == group_method) {
309
- r->internal.grouping_create= api_v1_data_groups[i].create;
310
- r->internal.grouping_reset = api_v1_data_groups[i].reset;
311
- r->internal.grouping_free = api_v1_data_groups[i].free;
312
- r->internal.grouping_add = api_v1_data_groups[i].add;
313
- r->internal.grouping_flush = api_v1_data_groups[i].flush;
329
+ r->internal.grouping_create = api_v1_data_groups[i].create;
330
+ r->internal.grouping_reset = api_v1_data_groups[i].reset;
331
+ r->internal.grouping_free = api_v1_data_groups[i].free;
332
+ r->internal.grouping_add = api_v1_data_groups[i].add;
333
+ r->internal.grouping_flush = api_v1_data_groups[i].flush;
334
+ r->internal.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch;
335
found = 1;
336
}
337
}
338
if(!found) {
339
errno = 0;
340
internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method);
320
- r->internal.grouping_create = grouping_create_average;
321
- r->internal.grouping_reset = grouping_reset_average;
322
- r->internal.grouping_free = grouping_free_average;
323
- r->internal.grouping_add = grouping_add_average;
324
- r->internal.grouping_flush = grouping_flush_average;
341
+ r->internal.grouping_create = grouping_create_average;
342
+ r->internal.grouping_reset = grouping_reset_average;
343
+ r->internal.grouping_free = grouping_free_average;
344
+ r->internal.grouping_add = grouping_add_average;
345
+ r->internal.grouping_flush = grouping_flush_average;
346
+ r->internal.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE;
347
}
348
}
349
@@ -380,7 +402,7 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options,
402
403
// check if all dimensions are hidden
404
if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
383
- // there are a few selected dimensions
405
+ // there are a few selected dimensions,
406
// but they are all zero
407
// enable the selected ones
408
// to avoid returning an empty chart
@@ -424,281 +446,671 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {
446
447
448
// ----------------------------------------------------------------------------
427
-// fill RRDR for a single dimension
449
+// tier management
450
+
451
+static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, time_t before_wanted, long points_wanted) {
452
+ if(unlikely(storage_tiers < 2))
453
+ return 0;
454
+
455
+ if(unlikely(after_wanted == before_wanted || points_wanted <= 0 || !rd || !rd->rrdset)) {
456
+
457
+ if(!rd)
458
+ internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation");
459
+ else
460
+ internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation",
461
+ (rd->rrdset)?rd->rrdset->name:"unknown", rd->name);
462
+
463
+ return 0;
464
+ }
465
+
466
+ //BUFFER *wb = buffer_create(1000);
467
+ //buffer_sprintf(wb, "Best tier for chart '%s', dim '%s', from %ld to %ld (dur %ld, every %d), points %ld",
468
+ // rd->rrdset->name, rd->name, after_wanted, before_wanted, before_wanted - after_wanted, rd->update_every, points_wanted);
469
+
470
+ long weight[storage_tiers];
471
+
472
+ for(int tier = 0; tier < storage_tiers ; tier++) {
473
+ if(unlikely(!rd->tiers[tier])) {
474
+ internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized",
475
+ tier, rd->rrdset->name, rd->name);
476
+ // buffer_free(wb);
477
+ return 0;
478
+ }
479
+
480
+ time_t first_t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
481
+ time_t last_t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
482
+
483
+ time_t common_after = MAX(first_t, after_wanted);
484
+ time_t common_before = MIN(last_t, before_wanted);
485
+
486
+ long time_coverage = (common_before - common_after) * 1000 / (before_wanted - after_wanted);
487
+ if(time_coverage < 0) time_coverage = 0;
488
+
489
+ int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every;
490
+ if(unlikely(update_every == 0)) {
491
+ internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d",
492
+ tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every);
493
+ // buffer_free(wb);
494
+ return 0;
495
+ }
496
+
497
+ long points_available = (before_wanted - after_wanted) / update_every;
498
+ long points_delta = points_available - points_wanted;
499
+ long points_coverage = (points_delta < 0) ? points_available * 1000 / points_wanted: 1000;
500
+
501
+ if(points_available <= 0)
502
+ weight[tier] = -LONG_MAX;
503
+ else
504
+ weight[tier] = points_coverage;
505
+
506
+ // buffer_sprintf(wb, ": tier %d, first %ld, last %ld (dur %ld, tg %d, every %d), points %ld, tcoverage %ld, pcoverage %ld, weight %ld",
507
+ // tier, first_t, last_t, last_t - first_t, rd->tiers[tier]->tier_grouping, update_every,
508
+ // points_available, time_coverage, points_coverage, weight[tier]);
509
+ }
510
+
511
+ int best_tier = 0;
512
+ for(int tier = 1; tier < storage_tiers ; tier++) {
513
+ if(weight[tier] >= weight[best_tier])
514
+ best_tier = tier;
515
+ }
516
429
-static inline NETDATA_DOUBLE interpolate_value(NETDATA_DOUBLE this_value, NETDATA_DOUBLE last_value, time_t last_value_end_t, time_t this_value_start_t, time_t now, time_t this_value_end_t) {
430
- if(unlikely(
431
- this_value_start_t + 1 == this_value_end_t ||
432
- !netdata_double_isnumber(this_value) ||
433
- !netdata_double_isnumber(last_value) ||
434
- last_value_end_t != this_value_start_t))
435
- return this_value;
517
+ if(weight[best_tier] == -LONG_MAX)
518
+ best_tier = 0;
519
437
- return last_value + (this_value - last_value) * ( 1.0 - (NETDATA_DOUBLE)(this_value_end_t - now) / (NETDATA_DOUBLE)(this_value_end_t - this_value_start_t) );
520
+ //buffer_sprintf(wb, ": final best tier %d", best_tier);
521
+ //internal_error(true, "%s", buffer_tostring(wb));
522
+ //buffer_free(wb);
523
+
524
+ return best_tier;
525
}
526
440
-static inline void rrd2rrdr_do_dimension(
441
- RRDR *r
442
- , long points_wanted
443
- , RRDDIM *rd
444
- , long dim_id_in_rrdr
445
- , time_t after_wanted
446
- , time_t before_wanted
447
- , RRDR_OPTIONS options
448
-){
449
- time_t now = after_wanted,
450
- query_granularity = r->update_every / r->group,
451
- max_date = 0,
452
- min_date = 0;
527
+static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) {
528
+ int ret = st->update_every;
529
+
530
+ if(unlikely(!st->dimensions))
531
+ return ret;
532
454
- bool interpolate = query_granularity < rd->update_every;
533
+ rrdset_rdlock(st);
534
+ int best_tier;
535
456
- long group_points_wanted = r->group,
457
- points_added = 0, group_points_added = 0, group_points_non_zero = 0,
458
- rrdr_line = -1;
536
+ if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers)
537
+ best_tier = tier;
538
+ else
539
+ best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted);
540
+
541
+ if(!st->dimensions->tiers[best_tier]) {
542
+ internal_error(
543
+ true,
544
+ "QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name);
545
+ }
546
+ else {
547
+ ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every;
548
+ if(unlikely(!ret)) {
549
+ internal_error(
550
+ true,
551
+ "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d",
552
+ st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every);
553
+
554
+ ret = st->update_every;
555
+ }
556
+ }
557
+
558
+ rrdset_unlock(st);
559
460
- size_t group_anomaly_rate = 0;
560
+ return ret;
561
+}
562
+
563
+// ----------------------------------------------------------------------------
564
+// query ops
565
+
566
+typedef struct query_point {
567
+ NETDATA_DOUBLE value;
568
+ SN_FLAGS flags;
569
+ size_t anomaly;
570
+ time_t start_time;
571
+ time_t end_time;
572
+} QUERY_POINT;
573
+
574
+QUERY_POINT QUERY_POINT_EMPTY = {
575
+ .value = NAN,
576
+ .flags = SN_EMPTY_SLOT,
577
+ .anomaly = 0,
578
+ .start_time = 0,
579
+ .end_time = 0
580
+};
581
+
582
+typedef struct query_plan_entry {
583
+ size_t tier;
584
+ time_t after;
585
+ time_t before;
586
+} QUERY_PLAN_ENTRY;
587
462
- RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING;
588
+typedef struct query_plan {
589
+ size_t entries;
590
+ QUERY_PLAN_ENTRY data[RRD_STORAGE_TIERS*2];
591
+} QUERY_PLAN;
592
593
+typedef struct query_engine_ops {
594
+ // configuration
595
+ RRDR *r;
596
+ RRDDIM *rd;
597
+ time_t view_update_every;
598
+ time_t query_granularity;
599
+ TIER_QUERY_FETCH tier_query_fetch;
600
+
601
+ // query planer
602
+ QUERY_PLAN plan;
603
+ size_t current_plan;
604
+ time_t current_plan_expire_time;
605
+
606
+ // storage queries
607
+ size_t tier;
608
+ struct rrddim_tier *tier_ptr;
609
struct rrddim_query_handle handle;
610
+ STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
611
+ int (*is_finished)(struct rrddim_query_handle *handle);
612
+ void (*finalize)(struct rrddim_query_handle *handle);
613
466
- NETDATA_DOUBLE min = r->min, max = r->max;
467
- size_t db_points_read = 0;
468
-
469
- // cache the function pointers we need in the loop
470
- NETDATA_DOUBLE (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags) = rd->state->query_ops.next_metric;
471
- void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value) = r->internal.grouping_add;
472
- NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush;
473
-
474
- NETDATA_DOUBLE last2_point_value;
475
- //SN_FLAGS last2_point_flags;
476
- //size_t last2_point_anomaly;
477
- //time_t last2_point_start_time;
478
- time_t last2_point_end_time;
479
-
480
- NETDATA_DOUBLE last1_point_value = NAN;
481
- SN_FLAGS last1_point_flags = SN_EMPTY_SLOT;
482
- size_t last1_point_anomaly = 0;
483
- time_t last1_point_start_time = 0;
484
- time_t last1_point_end_time = 0;
485
-
486
- NETDATA_DOUBLE new_point_value = NAN;
487
- SN_FLAGS new_point_flags = SN_EMPTY_SLOT;
488
- size_t new_point_anomaly = 0;
489
- time_t new_point_start_time = 0;
490
- time_t new_point_end_time = 0;
491
-
492
- for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += query_granularity) {
493
-
494
- if(unlikely(now > before_wanted))
495
- break;
614
+ // aggregating points over time
615
+ void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
616
+ NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
617
+ size_t group_points_non_zero;
618
+ size_t group_points_added;
619
+ size_t group_anomaly_rate;
620
+ RRDR_VALUE_FLAGS group_value_flags;
621
497
- last2_point_value = last1_point_value;
498
- //last2_point_flags = last1_point_flags;
499
- //last2_point_anomaly = last1_point_anomaly;
500
- //last2_point_start_time = last1_point_start_time;
501
- last2_point_end_time = last1_point_end_time;
622
+ // statistics
623
+ size_t db_total_points_read;
624
+ size_t db_points_read_per_tier[RRD_STORAGE_TIERS];
625
+} QUERY_ENGINE_OPS;
626
503
- last1_point_value = new_point_value;
504
- last1_point_flags = new_point_flags;
505
- last1_point_anomaly = new_point_anomaly;
506
- last1_point_start_time = new_point_start_time;
507
- last1_point_end_time = new_point_end_time;
627
509
- if(likely(!rd->state->query_ops.is_finished(&handle))) {
510
- // fetch the new point
511
- new_point_value = next_metric(&handle, &new_point_start_time, &new_point_end_time, &new_point_flags);
512
- db_points_read++;
513
-
514
- // dbengine does not take into account the starting time of points
515
- // and depending on the data collection frequency it may return
516
- // a point that is just before the wanted one.
517
- // So, here we fetch the next one.
518
- if(unlikely(new_point_end_time < now)) {
519
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before now (now = %ld, after_wanted = %ld, before_wanted = %ld, dt = %ld). Fetching the next one.",
520
- rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time, now, after_wanted, before_wanted, query_granularity);
521
-
522
- new_point_value = next_metric(&handle, &new_point_start_time, &new_point_end_time, &new_point_flags);
523
- db_points_read++;
524
- }
628
+// ----------------------------------------------------------------------------
629
+// query planer
630
+
631
+#define query_plan_should_switch_plan(ops, now) ((now) >= (ops).current_plan_expire_time)
632
+
633
+static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, time_t overwrite_after) {
634
+ if(unlikely(plan_id >= ops->plan.entries))
635
+ plan_id = ops->plan.entries - 1;
636
+
637
+ time_t after = ops->plan.data[plan_id].after;
638
+ time_t before = ops->plan.data[plan_id].before;
639
+
640
+ if(overwrite_after > after && overwrite_after < before)
641
+ after = overwrite_after;
642
+
643
+ ops->tier = ops->plan.data[plan_id].tier;
644
+ ops->tier_ptr = ops->rd->tiers[ops->tier];
645
+ ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch);
646
+ ops->next_metric = ops->tier_ptr->query_ops.next_metric;
647
+ ops->is_finished = ops->tier_ptr->query_ops.is_finished;
648
+ ops->finalize = ops->tier_ptr->query_ops.finalize;
649
+ ops->current_plan = plan_id;
650
+ ops->current_plan_expire_time = ops->plan.data[plan_id].before;
651
+}
652
+
653
+static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) {
654
+ internal_error(now < ops->current_plan_expire_time && now < ops->plan.data[ops->current_plan].before,
655
+ "QUERY: switching query plan too early!");
656
+
657
+ time_t next_plan_before_time;
658
+ do {
659
+ ops->current_plan++;
660
+
661
+ if (ops->current_plan >= ops->plan.entries) {
662
+ ops->current_plan = ops->plan.entries - 1;
663
+ return;
664
+ }
665
+
666
+ next_plan_before_time = ops->plan.data[ops->current_plan].before;
667
+ } while(now >= next_plan_before_time || last_point_end_time >= next_plan_before_time);
668
+
669
+ if(ops->finalize) {
670
+ ops->finalize(&ops->handle);
671
+ ops->finalize = NULL;
672
+ }
673
+
674
+ query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
675
+
676
+ // internal_error(true, "QUERY: switched plan to %zu (all is %zu), previous expiration was %ld, this starts at %ld, now is %ld, last_point_end_time %ld", ops->current_plan, ops->plan.entries, ops->plan.data[ops->current_plan-1].before, ops->plan.data[ops->current_plan].after, now, last_point_end_time);
677
+}
678
+
679
+static int compare_query_plan_entries_on_start_time(const void *a, const void *b) {
680
+ QUERY_PLAN_ENTRY *p1 = (QUERY_PLAN_ENTRY *)a;
681
+ QUERY_PLAN_ENTRY *p2 = (QUERY_PLAN_ENTRY *)b;
682
+ return (p1->after < p2->after)?-1:1;
683
+}
684
+
685
+static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, long points_wanted) {
686
+ RRDDIM *rd = ops->rd;
687
526
- if(likely(netdata_double_isnumber(new_point_value))) {
527
- new_point_anomaly = (new_point_flags & SN_ANOMALY_BIT) ? 0 : 100;
688
+ //BUFFER *wb = buffer_create(1000);
689
+ //buffer_sprintf(wb, "QUERY PLAN for chart '%s' dimension '%s', from %ld to %ld:", rd->rrdset->name, rd->name, after_wanted, before_wanted);
690
529
- if(unlikely(options & RRDR_OPTION_ANOMALY_BIT))
530
- new_point_value = (NETDATA_DOUBLE)new_point_anomaly;
691
+ // put our selected tier as the first plan
692
+ size_t selected_tier;
693
+
694
+ if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER && ops->r->internal.query_tier >= 0 && ops->r->internal.query_tier < storage_tiers) {
695
+ selected_tier = ops->r->internal.query_tier;
696
+ }
697
+ else {
698
+
699
+ selected_tier = rrddim_find_best_tier_for_timeframe(rd, after_wanted, before_wanted, points_wanted);
700
+
701
+ if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)
702
+ ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER;
703
+ }
704
+
705
+ ops->plan.entries = 1;
706
+ ops->plan.data[0].tier = selected_tier;
707
+ ops->plan.data[0].after = rd->tiers[selected_tier]->query_ops.oldest_time(rd->tiers[selected_tier]->db_metric_handle);
708
+ ops->plan.data[0].before = rd->tiers[selected_tier]->query_ops.latest_time(rd->tiers[selected_tier]->db_metric_handle);
709
+
710
+ if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) {
711
+ // the selected tier
712
+ time_t selected_tier_first_time_t = ops->plan.data[0].after;
713
+ time_t selected_tier_last_time_t = ops->plan.data[0].before;
714
+
715
+ //buffer_sprintf(wb, ": SELECTED tier %zu, from %ld to %ld", selected_tier, ops->plan.data[0].after, ops->plan.data[0].before);
716
+
717
+ // check if our selected tier can start the query
718
+ if (selected_tier_first_time_t > after_wanted) {
719
+ // we need some help from other tiers
720
+ for (int tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
721
+ // find the first time of this tier
722
+ time_t first_time_t = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
723
+
724
+ //buffer_sprintf(wb, ": EVAL AFTER tier %d, %ld", tier, first_time_t);
725
+
726
+ // can it help?
727
+ if (first_time_t < selected_tier_first_time_t) {
728
+ // it can help us add detail at the beginning of the query
729
+ QUERY_PLAN_ENTRY t = {
730
+ .tier = tr,
731
+ .after = (first_time_t < after_wanted) ? after_wanted : first_time_t,
732
+ .before = selected_tier_first_time_t};
733
+ ops->plan.data[ops->plan.entries++] = t;
734
+
735
+ // prepare for the tier
736
+ selected_tier_first_time_t = t.after;
737
+
738
+ if (t.after <= after_wanted)
739
+ break;
740
+ }
741
}
532
- else {
533
- new_point_flags = SN_EMPTY_SLOT;
534
- new_point_value = NAN;
535
- new_point_anomaly = 0;
742
+ }
743
+
744
+ // check if our selected tier can finish the query
745
+ if (selected_tier_last_time_t < before_wanted) {
746
+ // we need some help from other tiers
747
+ for (int tr = (int)selected_tier - 1; tr >= 0; tr--) {
748
+ // find the last time of this tier
749
+ time_t last_time_t = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
750
+
751
+ //buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_t);
752
+
753
+ // can it help?
754
+ if (last_time_t > selected_tier_last_time_t) {
755
+ // it can help us add detail at the end of the query
756
+ QUERY_PLAN_ENTRY t = {
757
+ .tier = tr,
758
+ .after = selected_tier_last_time_t,
759
+ .before = (last_time_t > before_wanted) ? before_wanted : last_time_t};
760
+ ops->plan.data[ops->plan.entries++] = t;
761
+
762
+ // prepare for the tier
763
+ selected_tier_last_time_t = t.before;
764
+
765
+ if (t.before >= before_wanted)
766
+ break;
767
+ }
768
}
769
+ }
770
+ }
771
538
- if(unlikely(new_point_start_time == new_point_end_time)) {
539
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
540
- rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time);
772
+ // sort the query plan
773
+ if(ops->plan.entries > 1)
774
+ qsort(&ops->plan.data, ops->plan.entries, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time);
775
+
776
+ // make sure it has the whole timeframe we need
777
+ ops->plan.data[0].after = after_wanted;
778
+ ops->plan.data[ops->plan.entries - 1].before = before_wanted;
779
+
780
+ //buffer_sprintf(wb, ": FINAL STEPS %zu", ops->plan.entries);
781
+
782
+ //for(size_t i = 0; i < ops->plan.entries ;i++)
783
+ // buffer_sprintf(wb, ": STEP %zu = use tier %zu from %ld to %ld", i+1, ops->plan.data[i].tier, ops->plan.data[i].after, ops->plan.data[i].before);
784
+
785
+ //internal_error(true, "%s", buffer_tostring(wb));
786
+
787
+ query_planer_activate_plan(ops, 0, 0);
788
+}
789
+
790
+
791
+// ----------------------------------------------------------------------------
792
+// dimension level query engine
793
+
794
+#define query_interpolate_point(this_point, last_point, now) do { \
795
+ if(likely( \
796
+ /* the point to interpolate is more than 1s wide */ \
797
+ (this_point).end_time - (this_point).start_time > 1 \
798
+ \
799
+ /* the two points are exactly next to each other */ \
800
+ && (last_point).end_time == (this_point).start_time \
801
+ \
802
+ /* both points are valid numbers */ \
803
+ && netdata_double_isnumber((this_point).value) \
804
+ && netdata_double_isnumber((last_point).value) \
805
+ \
806
+ )) { \
807
+ (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).end_time - (now)) / (NETDATA_DOUBLE)((this_point).end_time - (this_point).start_time)); \
808
+ (this_point).end_time = now; \
809
+ } \
810
+} while(0)
811
+
812
+#define query_add_point_to_group(r, point, ops) do { \
813
+ if(likely(netdata_double_isnumber((point).value))) { \
814
+ if(likely((point).value != 0.0)) \
815
+ (ops).group_points_non_zero++; \
816
+ \
817
+ if(unlikely((point).flags & SN_EXISTS_RESET)) \
818
+ (ops).group_value_flags |= RRDR_VALUE_RESET; \
819
+ \
820
+ (ops).grouping_add(r, (point).value); \
821
+ } \
822
+ \
823
+ (ops).group_points_added++; \
824
+ (ops).group_anomaly_rate += (point).anomaly; \
825
+} while(0)
826
542
- new_point_start_time = new_point_end_time - rd->update_every;
827
+static inline void rrd2rrdr_do_dimension(
828
+ RRDR *r
829
+ , long points_wanted
830
+ , RRDDIM *rd
831
+ , long dim_id_in_rrdr
832
+ , time_t after_wanted
833
+ , time_t before_wanted
834
+){
835
+ time_t max_date = 0,
836
+ min_date = 0;
837
+
838
+ size_t points_added = 0;
839
+
840
+ QUERY_ENGINE_OPS ops = {
841
+ .r = r,
842
+ .rd = rd,
843
+ .grouping_add = r->internal.grouping_add,
844
+ .grouping_flush = r->internal.grouping_flush,
845
+ .tier_query_fetch = r->internal.tier_query_fetch,
846
+ .view_update_every = r->update_every,
847
+ .query_granularity = r->update_every / r->group,
848
+ .group_value_flags = RRDR_VALUE_NOTHING
849
+ };
850
+
851
+ long rrdr_line = -1;
852
+ bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
853
+
854
+ query_plan(&ops, after_wanted, before_wanted, points_wanted);
855
+
856
+ NETDATA_DOUBLE min = r->min, max = r->max;
857
+
858
+ QUERY_POINT last2_point = QUERY_POINT_EMPTY;
859
+ QUERY_POINT last1_point = QUERY_POINT_EMPTY;
860
+ QUERY_POINT new_point = QUERY_POINT_EMPTY;
861
+
862
+ // The main loop, based on the query granularity we need
863
+ for(time_t now = after_wanted + ops.view_update_every - ops.query_granularity; (long)points_added < points_wanted ; now += ops.view_update_every) {
864
+
865
+ if(query_plan_should_switch_plan(ops, now))
866
+ query_planer_next_plan(&ops, now, new_point.end_time);
867
+
868
+ // real all the points of the db, prior to the time we need (now)
869
+ size_t count_same_end_time = 0;
870
+ while(count_same_end_time < 100) {
871
+ if(likely(count_same_end_time == 0)) {
872
+ last2_point = last1_point;
873
+ last1_point = new_point;
874
+ }
875
+
876
+ if(unlikely(ops.is_finished(&ops.handle))) {
877
+ if(count_same_end_time != 0) {
878
+ last2_point = last1_point;
879
+ last1_point = new_point;
880
+ }
881
+ new_point = QUERY_POINT_EMPTY;
882
+ new_point.start_time = last1_point.end_time;
883
+ new_point.end_time = now;
884
+ break;
885
}
886
545
- if(unlikely(new_point_start_time < last1_point_start_time && new_point_end_time < last1_point_end_time)) {
546
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, before the last point start time %ld, end time %ld",
547
- rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time,
548
- last1_point_start_time,
549
- last1_point_end_time);
887
+ // fetch the new point
888
+ {
889
+ STORAGE_POINT sp = ops.next_metric(&ops.handle);
890
+
891
+ ops.db_points_read_per_tier[ops.tier]++;
892
+ ops.db_total_points_read++;
893
+
894
+ new_point.start_time = sp.start_time;
895
+ new_point.end_time = sp.end_time;
896
+ new_point.anomaly = sp.count ? sp.anomaly_count * 100 / sp.count : 0;
897
+
898
+ if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
899
+
900
+ if(unlikely(use_anomaly_bit_as_value))
901
+ new_point.value = (NETDATA_DOUBLE)new_point.anomaly;
902
+
903
+ else {
904
+ switch (ops.tier_query_fetch) {
905
+ default:
906
+ case TIER_QUERY_FETCH_AVERAGE:
907
+ new_point.value = sp.sum / sp.count;
908
+ break;
909
+
910
+ case TIER_QUERY_FETCH_MIN:
911
+ new_point.value = sp.min;
912
+ break;
913
+
914
+ case TIER_QUERY_FETCH_MAX:
915
+ new_point.value = sp.max;
916
+ break;
917
551
- new_point_value = last1_point_value;
552
- new_point_flags = last1_point_flags;
553
- new_point_start_time = last1_point_start_time;
554
- new_point_end_time = last1_point_end_time;
918
+ case TIER_QUERY_FETCH_SUM:
919
+ new_point.value = sp.sum;
920
+ break;
921
+ };
922
+ }
923
+ }
924
+ else {
925
+ new_point.value = NAN;
926
+ new_point.flags = SN_EMPTY_SLOT;
927
+ }
928
}
929
557
- if(unlikely(new_point_end_time < last1_point_end_time)) {
558
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu end time %ld, before the last point end time %ld",
559
- rd->rrdset->name, rd->name, db_points_read, new_point_end_time,
560
- last1_point_end_time);
930
+ if(unlikely(new_point.start_time == new_point.end_time)) {
931
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
932
+ rd->rrdset->name, rd->name, ops.db_total_points_read, new_point.start_time, new_point.end_time);
933
562
- new_point_value = last1_point_value;
563
- new_point_flags = last1_point_flags;
564
- new_point_start_time = last1_point_start_time;
565
- new_point_end_time = last1_point_end_time;
934
+ new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every);
935
}
936
568
- if(unlikely(new_point_end_time < now)) {
569
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before now (now = %ld, after_wanted = %ld, before_wanted = %ld, dt = %ld)",
570
- rd->rrdset->name, rd->name, db_points_read, new_point_start_time, new_point_end_time,
571
- now, after_wanted, before_wanted, query_granularity);
937
+ if(unlikely(new_point.end_time <= last1_point.end_time)) {
938
+ internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point end time %ld, now is %ld",
939
+ rd->rrdset->name, rd->name, ops.db_total_points_read, new_point.start_time, new_point.end_time, last1_point.end_time, now);
940
573
- new_point_end_time = now;
941
+ count_same_end_time++;
942
+ continue;
943
}
944
+
945
+ count_same_end_time = 0;
946
+
947
+ if(new_point.end_time < now)
948
+ query_add_point_to_group(r, new_point, ops);
949
+ else
950
+ break;
951
}
576
- else {
577
- new_point_value = NAN;
578
- new_point_flags = SN_EMPTY_SLOT;
579
- new_point_start_time = last1_point_end_time;
580
- new_point_end_time = now;
952
+
953
+ if(count_same_end_time) {
954
+ internal_error(true,
955
+ "QUERY: the database does not advance the query, it returned an end time less or equal to %ld, %zu times",
956
+ last1_point.end_time, count_same_end_time);
957
+
958
+ new_point.end_time = now;
959
}
960
961
// the inner loop
584
- // we have 3 points in memory: last, new, next
962
+ // we have 3 points in memory: last2, last1, new
963
// we select the one to use based on their timestamps
964
965
size_t iterations = 0;
588
- for ( ; now <= new_point_end_time && points_added < points_wanted; now += query_granularity, iterations++) {
966
+ for ( ; now <= new_point.end_time && (long)points_added < points_wanted; now += ops.view_update_every, iterations++) {
967
+ QUERY_POINT current_point;
968
590
- NETDATA_DOUBLE current_point_value;
591
- SN_FLAGS current_point_flags;
592
- size_t current_point_anomaly;
593
- //time_t current_point_start_time;
594
- //time_t current_point_end_time;
595
-
596
- if(likely(now > new_point_start_time)) {
969
+ if(likely(now > new_point.start_time)) {
970
// it is time for our NEW point to be used
598
- current_point_value = interpolate ? interpolate_value(new_point_value, last1_point_value, last1_point_end_time, new_point_start_time, now, new_point_end_time) : new_point_value;
599
- current_point_flags = new_point_flags;
600
- current_point_anomaly = new_point_anomaly;
601
- //current_point_start_time = new_point_start_time;
602
- //current_point_end_time = new_point_end_time;
971
+ current_point = new_point;
972
+ query_interpolate_point(current_point, last1_point, now);
973
}
604
- else if(likely(now <= last1_point_end_time)) {
974
+ else if(likely(now <= last1_point.end_time)) {
975
// our LAST point is still valid
606
- current_point_value = interpolate ? interpolate_value(last1_point_value, last2_point_value, last2_point_end_time, last1_point_start_time, now, last1_point_end_time) : last1_point_value;
607
- current_point_flags = last1_point_flags;
608
- current_point_anomaly = last1_point_anomaly;
609
- //current_point_start_time = last_point_start_time;
610
- //current_point_end_time = last_point_end_time;
976
+ current_point = last1_point;
977
+ query_interpolate_point(current_point, last2_point, now);
978
}
979
else {
980
// a GAP, we don't have a value this time
614
- current_point_value = NAN;
615
- current_point_flags = SN_EMPTY_SLOT;
616
- current_point_anomaly = 0;
617
- //current_point_start_time = now - dt;
618
- //current_point_end_time = now;
981
+ current_point = QUERY_POINT_EMPTY;
982
}
983
621
- if(likely(netdata_double_isnumber(current_point_value))) {
622
- if(likely(current_point_value != 0.0))
623
- group_points_non_zero++;
624
-
625
- if(unlikely(current_point_flags & SN_EXISTS_RESET))
626
- group_value_flags |= RRDR_VALUE_RESET;
984
+ query_add_point_to_group(r, current_point, ops);
985
628
- grouping_add(r, current_point_value);
629
- }
986
+ rrdr_line = rrdr_line_init(r, now, rrdr_line);
987
+ size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
988
631
- // add this value for grouping
632
- group_points_added++;
633
- group_anomaly_rate += current_point_anomaly;
989
+ if(unlikely(!min_date)) min_date = now;
990
+ max_date = now;
991
635
- if(unlikely(group_points_added == group_points_wanted)) {
636
- rrdr_line = rrdr_line_init(r, now, rrdr_line);
637
- size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
992
+ // find the place to store our values
993
+ RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];
994
639
- if(unlikely(!min_date)) min_date = now;
640
- max_date = now;
995
+ // update the dimension options
996
+ if(likely(ops.group_points_non_zero))
997
+ r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
998
642
- // find the place to store our values
643
- RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];
999
+ // store the specific point options
1000
+ *rrdr_value_options_ptr = ops.group_value_flags;
1001
645
- // update the dimension options
646
- if(likely(group_points_non_zero))
647
- r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
1002
+ // store the group value
1003
+ NETDATA_DOUBLE group_value = ops.grouping_flush(r, rrdr_value_options_ptr);
1004
+ r->v[rrdr_o_v_index] = group_value;
1005
649
- // store the specific point options
650
- *rrdr_value_options_ptr = group_value_flags;
1006
+ // we only store uint8_t anomaly rates,
1007
+ // so let's get double precision by storing
1008
+ // anomaly rates in the range 0 - 200
1009
+ ops.group_anomaly_rate = (ops.group_anomaly_rate << 1) / ops.group_points_added;
1010
+ r->ar[rrdr_o_v_index] = (uint8_t)ops.group_anomaly_rate;
1011
652
- // store the group value
653
- NETDATA_DOUBLE group_value = grouping_flush(r, rrdr_value_options_ptr);
654
- r->v[rrdr_o_v_index] = group_value;
1012
+ if(likely(points_added || dim_id_in_rrdr)) {
1013
+ // find the min/max across all dimensions
1014
656
- // we only store uint8_t anomaly rates,
657
- // so let's get double precision by storing
658
- // anomaly rates in the range 0 - 200
659
- group_anomaly_rate = (group_anomaly_rate << 1) / group_points_added;
660
- r->ar[rrdr_o_v_index] = (uint8_t)group_anomaly_rate;
1015
+ if(unlikely(group_value < min)) min = group_value;
1016
+ if(unlikely(group_value > max)) max = group_value;
1017
662
- if(likely(points_added || dim_id_in_rrdr)) {
663
- // find the min/max across all dimensions
664
-
665
- if(unlikely(group_value < min)) min = group_value;
666
- if(unlikely(group_value > max)) max = group_value;
667
-
668
- }
669
- else {
670
- // runs only when dim_id_in_rrdr == 0 && points_added == 0
671
- // so, on the first point added for the query.
672
- min = max = group_value;
673
- }
674
-
675
- points_added++;
676
- group_points_added = 0;
677
- group_value_flags = RRDR_VALUE_NOTHING;
678
- group_points_non_zero = 0;
679
- group_anomaly_rate = 0;
1018
}
1019
+ else {
1020
+ // runs only when dim_id_in_rrdr == 0 && points_added == 0
1021
+ // so, on the first point added for the query.
1022
+ min = max = group_value;
1023
+ }
1024
+
1025
+ points_added++;
1026
+ ops.group_points_added = 0;
1027
+ ops.group_value_flags = RRDR_VALUE_NOTHING;
1028
+ ops.group_points_non_zero = 0;
1029
+ ops.group_anomaly_rate = 0;
1030
}
682
- // the loop above increased "now" by dt,
683
- // but the main loop will increase it,
1031
+ // the loop above increased "now" by query_granularity,
1032
+ // but the main loop will increase it too,
1033
// so, let's undo the last iteration of this loop
1034
if(iterations)
686
- now -= query_granularity;
1035
+ now -= ops.view_update_every;
1036
}
688
- rd->state->query_ops.finalize(&handle);
1037
+ ops.finalize(&ops.handle);
1038
690
- r->internal.db_points_read += db_points_read;
1039
r->internal.result_points_generated += points_added;
1040
+ r->internal.db_points_read += ops.db_total_points_read;
1041
+ for(int tr = 0; tr < storage_tiers ; tr++)
1042
+ r->internal.tier_points_read[tr] += ops.db_points_read_per_tier[tr];
1043
1044
r->min = min;
1045
r->max = max;
1046
r->before = max_date;
696
- r->after = min_date - (r->group - 1) * query_granularity;
1047
+ r->after = min_date - ops.view_update_every + ops.query_granularity;
1048
rrdr_done(r, rrdr_line);
1049
699
- internal_error(points_wanted != points_added,
1050
+ internal_error((long)points_added != points_wanted,
1051
"QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).",
701
- r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, db_points_read);
1052
+ r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
1053
+}
1054
+
1055
+// ----------------------------------------------------------------------------
1056
+// fill the gap of a tier
1057
+
1058
+extern void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
1059
+
1060
+void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
1061
+ if(unlikely(tier < 0 || tier >= storage_tiers)) return;
1062
+ if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
1063
+
1064
+ struct rrddim_tier *t = rd->tiers[tier];
1065
+ if(unlikely(!t)) return;
1066
+
1067
+ time_t latest_time_t = t->query_ops.latest_time(t->db_metric_handle);
1068
+ time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every;
1069
+ time_t time_diff = now - latest_time_t;
1070
+
1071
+ // if the user wants only NEW backfilling, and we don't have any data
1072
+ if(storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_t <= 0) return;
1073
+
1074
+ // there is really nothing we can do
1075
+ if(now <= latest_time_t || time_diff < granularity) return;
1076
+
1077
+ struct rrddim_query_handle handle;
1078
+
1079
+ size_t all_points_read = 0;
1080
+
1081
+ // for each lower tier
1082
+ for(int tr = tier - 1; tr >= 0 ;tr--){
1083
+ time_t smaller_tier_first_time = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
1084
+ time_t smaller_tier_last_time = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
1085
+ if(smaller_tier_last_time <= latest_time_t) continue; // it is as bad as we are
1086
+
1087
+ long after_wanted = (latest_time_t < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_t;
1088
+ long before_wanted = smaller_tier_last_time;
1089
+
1090
+ struct rrddim_tier *tmp = rd->tiers[tr];
1091
+ tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE);
1092
+
1093
+ size_t points = 0;
1094
+
1095
+ while(!tmp->query_ops.is_finished(&handle)) {
1096
+
1097
+ STORAGE_POINT sp = tmp->query_ops.next_metric(&handle);
1098
+
1099
+ if(sp.end_time > latest_time_t) {
1100
+ latest_time_t = sp.end_time;
1101
+ store_metric_at_tier(rd, t, sp, sp.end_time * USEC_PER_SEC);
1102
+ points++;
1103
+ }
1104
+ }
1105
+
1106
+ all_points_read += points;
1107
+ tmp->query_ops.finalize(&handle);
1108
+
1109
+ internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d",
1110
+ rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr);
1111
+ }
1112
+
1113
+ rrdr_query_completed(all_points_read, all_points_read);
1114
}
1115
1116
// ----------------------------------------------------------------------------
@@ -706,6 +1118,7 @@ static inline void rrd2rrdr_do_dimension(
1118
1119
#ifdef NETDATA_INTERNAL_CHECKS
1120
static void rrd2rrdr_log_request_response_metadata(RRDR *r
1121
+ , RRDR_OPTIONS options __maybe_unused
1122
, RRDR_GROUPING group_method
1123
, bool aligned
1124
, long group
@@ -723,9 +1136,9 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1136
) {
1137
netdata_rwlock_rdlock(&r->st->rrdset_rwlock);
1138
info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), "
726
- "after (got: %zu, want: %zu, req: %zu, db: %zu), "
727
- "before (got: %zu, want: %zu, req: %zu, db: %zu), "
728
- "duration (got: %zu, want: %zu, req: %zu, db: %zu), "
1139
+ "after (got: %zu, want: %zu, req: %ld, db: %zu), "
1140
+ "before (got: %zu, want: %zu, req: %ld, db: %zu), "
1141
+ "duration (got: %zu, want: %zu, req: %ld, db: %zu), "
1142
//"slot (after: %zu, before: %zu, delta: %zu), "
1143
"points (got: %ld, want: %ld, req: %ld, db: %ld), "
1144
"%s"
@@ -742,19 +1155,19 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1155
// after
1156
, (size_t)r->after
1157
, (size_t)after_wanted
745
- , (size_t)after_requested
1158
+ , after_requested
1159
, (size_t)rrdset_first_entry_t_nolock(r->st)
1160
1161
// before
1162
, (size_t)r->before
1163
, (size_t)before_wanted
751
- , (size_t)before_requested
1164
+ , before_requested
1165
, (size_t)rrdset_last_entry_t_nolock(r->st)
1166
1167
// duration
1168
, (size_t)(r->before - r->after + r->st->update_every)
1169
, (size_t)(before_wanted - after_wanted + r->st->update_every)
757
- , (size_t)(before_requested - after_requested)
1170
+ , before_requested - after_requested
1171
, (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every)
1172
1173
// slot
@@ -778,7 +1191,7 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1191
#endif // NETDATA_INTERNAL_CHECKS
1192
1193
// Returns 1 if an absolute period was requested or 0 if it was a relative period
781
-int rrdr_relative_window_to_absolute(long long *after, long long *before, int update_every, long points) {
1194
+int rrdr_relative_window_to_absolute(long long *after, long long *before) {
1195
time_t now = now_realtime_sec() - 1;
1196
1197
int absolute_period_requested = -1;
@@ -806,7 +1219,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before, int up
1219
// if the user didn't give an after, use the number of points
1220
// to give a sane default
1221
if(after_requested == 0)
809
- after_requested = -(points * update_every);
1222
+ after_requested = -600;
1223
1224
// since the query engine now returns inclusive timestamps
1225
// it is awkward to return 6 points when after=-5 is given
@@ -851,10 +1264,12 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before, int up
1264
buffer_free(debug_log); \
1265
debug_log = NULL; \
1266
}
1267
+#define query_debug_log_free() do { buffer_free(debug_log); } while(0)
1268
#else
1269
#define query_debug_log_init() debug_dummy()
1270
#define query_debug_log(args...) debug_dummy()
1271
#define query_debug_log_fin() debug_dummy()
1272
+#define query_debug_log_free() debug_dummy()
1273
#endif
1274
1275
RRDR *rrd2rrdr(
@@ -870,6 +1285,7 @@ RRDR *rrd2rrdr(
1285
, struct context_param *context_param_list
1286
, const char *group_options
1287
, int timeout
1288
+ , int tier
1289
) {
1290
// RULES
1291
// points_requested = 0
@@ -910,10 +1326,23 @@ RRDR *rrd2rrdr(
1326
query_debug_log(":relative+natural");
1327
}
1328
913
- // this is the update_every of the query
914
- // it may be different to the update_every of the database
915
- time_t query_granularity = (natural_points)?update_every:1;
916
- query_debug_log(":query_granularity %ld", query_granularity);
1329
+ // if the user wants virtual points, make sure we do it
1330
+ if(options & RRDR_OPTION_VIRTUAL_POINTS)
1331
+ natural_points = false;
1332
+
1333
+ // set the right flag about natural and virtual points
1334
+ if(natural_points) {
1335
+ options |= RRDR_OPTION_NATURAL_POINTS;
1336
+
1337
+ if(options & RRDR_OPTION_VIRTUAL_POINTS)
1338
+ options &= ~RRDR_OPTION_VIRTUAL_POINTS;
1339
+ }
1340
+ else {
1341
+ options |= RRDR_OPTION_VIRTUAL_POINTS;
1342
+
1343
+ if(options & RRDR_OPTION_NATURAL_POINTS)
1344
+ options &= ~RRDR_OPTION_NATURAL_POINTS;
1345
+ }
1346
1347
if(after_wanted == 0 || before_wanted == 0) {
1348
// for non-context queries we have to find the duration of the database
@@ -927,6 +1356,12 @@ RRDR *rrd2rrdr(
1356
time_t last_entry_t = rrdset_last_entry_t_nolock(st);
1357
rrdset_unlock(st);
1358
1359
+ if(first_entry_t == 0 || last_entry_t == 0) {
1360
+ internal_error(true, "QUERY: chart without data detected on '%s'", st->name);
1361
+ query_debug_log_free();
1362
+ return NULL;
1363
+ }
1364
+
1365
query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
1366
1367
if (after_wanted == 0) {
@@ -952,17 +1387,29 @@ RRDR *rrd2rrdr(
1387
after_wanted = -600;
1388
query_debug_log(":zero600 after_wanted %lld", after_wanted);
1389
}
1390
+ }
1391
956
- if(points_wanted == 0) {
957
- points_wanted = 600;
958
- query_debug_log(":zero600 points_wanted %ld", points_wanted);
959
- }
1392
+ if(points_wanted == 0) {
1393
+ points_wanted = 600;
1394
+ query_debug_log(":zero600 points_wanted %ld", points_wanted);
1395
}
1396
1397
// convert our before_wanted and after_wanted to absolute
963
- rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, (int)query_granularity, points_wanted);
1398
+ rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
1399
query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted);
1400
1401
+ if(natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
1402
+ update_every = rrdset_find_natural_update_every_for_timeframe(st, after_wanted, before_wanted, points_wanted, options, tier);
1403
+ if(update_every <= 0) update_every = st->update_every;
1404
+ query_debug_log(":natural update every %d", update_every);
1405
+ }
1406
+
1407
+ // this is the update_every of the query
1408
+ // it may be different to the update_every of the database
1409
+ time_t query_granularity = (natural_points)?update_every:1;
1410
+ if(query_granularity <= 0) query_granularity = 1;
1411
+ query_debug_log(":query_granularity %ld", query_granularity);
1412
+
1413
// align before_wanted and after_wanted to query_granularity
1414
if (before_wanted % query_granularity) {
1415
before_wanted -= before_wanted % query_granularity;
@@ -977,6 +1424,7 @@ RRDR *rrd2rrdr(
1424
// automatic_natural_points is set when the user wants all the points available in the database
1425
if(automatic_natural_points) {
1426
points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
1427
+ if(unlikely(points_wanted <= 0)) points_wanted = 1;
1428
query_debug_log(":auto natural points_wanted %ld", points_wanted);
1429
}
1430
@@ -1003,6 +1451,7 @@ RRDR *rrd2rrdr(
1451
1452
// the available points of the query
1453
long points_available = (duration + 1) / query_granularity;
1454
+ if(unlikely(points_available <= 0)) points_available = 1;
1455
query_debug_log(":points_available %ld", points_available);
1456
1457
if(points_wanted > points_available) {
@@ -1032,6 +1481,9 @@ RRDR *rrd2rrdr(
1481
if(points_wanted * group < points_available)
1482
points_wanted++;
1483
1484
+ if(unlikely(points_wanted <= 0))
1485
+ points_wanted = 1;
1486
+
1487
query_debug_log(":optimal points %ld", points_wanted);
1488
}
1489
@@ -1132,7 +1584,8 @@ RRDR *rrd2rrdr(
1584
r->internal.points_wanted = points_wanted;
1585
r->internal.resampling_group = resampling_group;
1586
r->internal.resampling_divisor = resampling_divisor;
1135
-
1587
+ r->internal.query_options = options;
1588
+ r->internal.query_tier = tier;
1589
1590
// -------------------------------------------------------------------------
1591
// assign the processor functions
@@ -1179,7 +1632,7 @@ RRDR *rrd2rrdr(
1632
// reset the grouping for the new dimension
1633
r->internal.grouping_reset(r);
1634
1182
- rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted, options);
1635
+ rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted);
1636
if (timeout)
1637
now_realtime_timeval(&query_current_time);
1638
@@ -1216,9 +1669,9 @@ RRDR *rrd2rrdr(
1669
}
1670
1671
dimensions_used++;
1219
- if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
1672
+ if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
1673
log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
1221
- dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
1674
+ (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
1675
r->result_options |= RRDR_RESULT_OPTION_CANCEL;
1676
break;
1677
}
@@ -1227,45 +1680,46 @@ RRDR *rrd2rrdr(
1680
#ifdef NETDATA_INTERNAL_CHECKS
1681
if (dimensions_used) {
1682
if(r->internal.log)
1230
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group,
1683
+ rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
1684
after_wanted, after_requested, before_wanted, before_requested,
1685
points_requested, points_wanted, /*after_slot, before_slot,*/
1686
r->internal.log);
1687
1688
if(r->rows != points_wanted)
1236
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group,
1689
+ rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
1690
after_wanted, after_requested, before_wanted, before_requested,
1691
points_requested, points_wanted, /*after_slot, before_slot,*/
1692
"got 'points' is not wanted 'points'");
1693
1694
if(aligned && (r->before % (group * query_granularity)) != 0)
1242
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group,
1695
+ rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
1696
after_wanted, after_requested, before_wanted,before_wanted,
1697
points_requested, points_wanted, /*after_slot, before_slot,*/
1698
"'before' is not aligned but alignment is required");
1699
1700
// 'after' should not be aligned, since we start inside the first group
1701
//if(aligned && (r->after % group) != 0)
1249
- // rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
1702
+ // rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
1703
1704
if(r->before != before_wanted)
1252
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group,
1705
+ rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
1706
after_wanted, after_requested, before_wanted, before_requested,
1707
points_requested, points_wanted, /*after_slot, before_slot,*/
1708
"chart is not aligned to requested 'before'");
1709
1710
if(r->before != before_wanted)
1258
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group,
1711
+ rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
1712
after_wanted, after_requested, before_wanted, before_requested,
1713
points_requested, points_wanted, /*after_slot, before_slot,*/
1714
"got 'before' is not wanted 'before'");
1715
1716
// reported 'after' varies, depending on group
1717
if(r->after != after_wanted)
1265
- rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group,
1718
+ rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
1719
after_wanted, after_requested, before_wanted, before_requested,
1720
points_requested, points_wanted, /*after_slot, before_slot,*/
1721
"got 'after' is not wanted 'after'");
1722
+
1723
}
1724
#endif
1725
web/api/queries/query.h
+8
@@ -3,6 +3,10 @@
3
#ifndef NETDATA_API_DATA_QUERY_H
4
#define NETDATA_API_DATA_QUERY_H
5
6
+#ifdef __cplusplus
7
+extern "C" {
8
+#endif
9
+
10
typedef enum rrdr_grouping {
11
RRDR_GROUPING_UNDEFINED = 0,
12
RRDR_GROUPING_AVERAGE,
@@ -23,4 +27,8 @@ extern void web_client_api_v1_init_grouping(void);
27
extern RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def);
28
extern const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group);
29
30
+#ifdef __cplusplus
31
+}
32
+#endif
33
+
34
#endif //NETDATA_API_DATA_QUERY_H
web/api/queries/rrdr.c
+34
-48
@@ -58,35 +58,12 @@ static void rrdr_dump(RRDR *r)
58
}
59
*/
60
61
-
62
-
63
-
64
-inline static void rrdr_lock_rrdset(RRDR *r) {
65
- if(unlikely(!r)) {
66
- error("NULL value given!");
67
- return;
68
- }
69
-
70
- rrdset_rdlock(r->st);
71
- r->has_st_lock = 1;
72
-}
73
-
74
-inline static void rrdr_unlock_rrdset(RRDR *r) {
75
- if(unlikely(!r)) {
76
- error("NULL value given!");
77
- return;
78
- }
79
-
80
- if(likely(r->has_st_lock)) {
81
- r->has_st_lock = 0;
82
- rrdset_unlock(r->st);
83
- }
84
-}
85
-
61
inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
62
if(unlikely(!r)) return;
63
89
- rrdr_unlock_rrdset(r);
64
+ if(likely(r->st_locked_by_rrdr_create))
65
+ rrdset_unlock(r->st);
66
+
67
onewayalloc_freez(owa, r->t);
68
onewayalloc_freez(owa, r->v);
69
onewayalloc_freez(owa, r->o);
@@ -95,39 +72,51 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
72
onewayalloc_freez(owa, r);
73
}
74
98
-RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list)
99
-{
100
- if (unlikely(!st)) {
101
- error("NULL value given!");
102
- return NULL;
103
- }
104
-
75
+RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) {
76
RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
106
- r->st = st;
77
+ r->internal.owa = owa;
78
+
79
+ r->d = dimensions;
80
+ r->n = points;
81
+
82
+ r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
83
+ r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
84
+ r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS));
85
+ r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(uint8_t));
86
+ r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS));
87
88
+ r->group = 1;
89
+ r->update_every = 1;
90
+
91
+ return r;
92
+}
93
+
94
+RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) {
95
+ if (unlikely(!st)) return NULL;
96
+
97
+ bool st_locked_by_rrdr_create = false;
98
if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
109
- rrdr_lock_rrdset(r);
110
- r->st_needs_lock = 1;
99
+ rrdset_rdlock(st);
100
+ st_locked_by_rrdr_create = true;
101
}
102
103
+ // count the number of dimensions
104
+ int dimensions = 0;
105
RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
106
RRDDIM *rd;
107
if (temp_rd) {
108
RRDDIM *t = temp_rd;
109
while (t) {
118
- r->d++;
110
+ dimensions++;
111
t = t->next;
112
}
113
} else
122
- rrddim_foreach_read(rd, st) r->d++;
123
-
124
- r->n = n;
114
+ rrddim_foreach_read(rd, st) dimensions++;
115
126
- r->t = onewayalloc_callocz(owa, (size_t)n, sizeof(time_t));
127
- r->v = onewayalloc_mallocz(owa, n * r->d * sizeof(NETDATA_DOUBLE));
128
- r->o = onewayalloc_mallocz(owa, n * r->d * sizeof(RRDR_VALUE_FLAGS));
129
- r->ar = onewayalloc_mallocz(owa, n * r->d * sizeof(uint8_t));
130
- r->od = onewayalloc_mallocz(owa, r->d * sizeof(RRDR_DIMENSION_FLAGS));
116
+ // create the rrdr
117
+ RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n);
118
+ r->st = st;
119
+ r->st_locked_by_rrdr_create = st_locked_by_rrdr_create;
120
121
// set the hidden flag on hidden dimensions
122
int c;
@@ -138,8 +127,5 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa
127
r->od[c] = RRDR_DIMENSION_DEFAULT;
128
}
129
141
- r->group = 1;
142
- r->update_every = 1;
143
-
130
return r;
131
}
web/api/queries/rrdr.h
+32
-10
@@ -4,6 +4,18 @@
4
#define NETDATA_QUERIES_RRDR_H
5
6
#include "libnetdata/libnetdata.h"
7
+#include "web/api/queries/query.h"
8
+
9
+#ifdef __cplusplus
10
+extern "C" {
11
+#endif
12
+
13
+typedef enum tier_query_fetch {
14
+ TIER_QUERY_FETCH_SUM,
15
+ TIER_QUERY_FETCH_MIN,
16
+ TIER_QUERY_FETCH_MAX,
17
+ TIER_QUERY_FETCH_AVERAGE
18
+} TIER_QUERY_FETCH;
19
20
typedef enum rrdr_options {
21
RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
@@ -24,12 +36,14 @@ typedef enum rrdr_options {
36
RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
37
RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response
38
RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
27
- RRDR_OPTION_ANOMALY_BIT = 0x00040000, // Return the anomaly bit stored in each collected_number
28
- RRDR_OPTION_RETURN_RAW = 0x00080000, // Return raw data for aggregating across multiple nodes
29
- RRDR_OPTION_RETURN_JWAR = 0x00100000, // Return anomaly rates in jsonwrap
39
+ RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
40
+ RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number
41
+ RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes
42
+ RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap
43
+ RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
44
45
// internal ones - not to be exposed to the API
32
- RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
46
+ RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
47
} RRDR_OPTIONS;
48
49
typedef enum rrdr_value_flag {
@@ -67,7 +81,7 @@ typedef struct rrdresult {
81
RRDR_DIMENSION_FLAGS *od; // the options for the dimensions
82
83
time_t *t; // array of n timestamps
70
- NETDATA_DOUBLE *v; // array n x d values
84
+ NETDATA_DOUBLE *v; // array n x d values
85
RRDR_VALUE_FLAGS *o; // array n x d options for each value returned
86
uint8_t *ar; // array n x d of anomaly rates (0 - 200)
87
@@ -80,11 +94,13 @@ typedef struct rrdresult {
94
time_t before;
95
time_t after;
96
83
- int has_st_lock; // if st is read locked by us
84
- uint8_t st_needs_lock; // if ST should be locked
97
+ bool st_locked_by_rrdr_create; // if st is read locked by us
98
99
// internal rrd2rrdr() members below this point
100
struct {
101
+ int query_tier; // the selected tier
102
+ RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query)
103
+
104
long points_wanted;
105
long resampling_group;
106
NETDATA_DOUBLE resampling_divisor;
@@ -96,12 +112,15 @@ typedef struct rrdresult {
112
NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
113
void *grouping_data;
114
115
+ TIER_QUERY_FETCH tier_query_fetch;
116
#ifdef NETDATA_INTERNAL_CHECKS
117
const char *log;
118
#endif
119
120
size_t db_points_read;
121
size_t result_points_generated;
122
+ size_t tier_points_read[RRD_STORAGE_TIERS];
123
+ ONEWAYALLOC *owa;
124
} internal;
125
} RRDR;
126
@@ -110,6 +129,7 @@ typedef struct rrdresult {
129
#include "database/rrd.h"
130
extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
131
extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list);
132
+extern RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points);
133
134
#include "../web_api_v1.h"
135
#include "web/api/queries/query.h"
@@ -118,10 +138,12 @@ extern RRDR *rrd2rrdr(
138
ONEWAYALLOC *owa,
139
RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted,
140
RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
121
- struct context_param *context_param_list, const char *group_options, int timeout);
141
+ struct context_param *context_param_list, const char *group_options, int timeout, int tier);
142
123
-extern int rrdr_relative_window_to_absolute(long long *after, long long *before, int update_every, long points);
143
+extern int rrdr_relative_window_to_absolute(long long *after, long long *before);
144
125
-#include "query.h"
145
+#ifdef __cplusplus
146
+}
147
+#endif
148
149
#endif //NETDATA_QUERIES_RRDR_H
web/api/queries/ses/ses.c
+5
-5
@@ -31,14 +31,14 @@ static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_ses *g) {
31
NETDATA_DOUBLE points;
32
if(r->group == 1) {
33
// provide a running DES
34
- points = r->internal.points_wanted;
34
+ points = (NETDATA_DOUBLE)r->internal.points_wanted;
35
}
36
else {
37
// provide a SES with flush points
38
- points = r->group;
38
+ points = (NETDATA_DOUBLE)r->group;
39
}
40
41
- return (points > max_window_size) ? max_window_size : points;
41
+ return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points;
42
}
43
44
static inline void set_alpha(RRDR *r, struct grouping_ses *g) {
@@ -49,7 +49,7 @@ static inline void set_alpha(RRDR *r, struct grouping_ses *g) {
49
}
50
51
void grouping_create_ses(RRDR *r, const char *options __maybe_unused) {
52
- struct grouping_ses *g = (struct grouping_ses *)callocz(1, sizeof(struct grouping_ses));
52
+ struct grouping_ses *g = (struct grouping_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_ses));
53
set_alpha(r, g);
54
g->level = 0.0;
55
r->internal.grouping_data = g;
@@ -64,7 +64,7 @@ void grouping_reset_ses(RRDR *r) {
64
}
65
66
void grouping_free_ses(RRDR *r) {
67
- freez(r->internal.grouping_data);
67
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
68
r->internal.grouping_data = NULL;
69
}
70
web/api/queries/stddev/stddev.c
+3
-3
@@ -15,7 +15,7 @@ struct grouping_stddev {
15
};
16
17
void grouping_create_stddev(RRDR *r, const char *options __maybe_unused) {
18
- r->internal.grouping_data = callocz(1, sizeof(struct grouping_stddev));
18
+ r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_stddev));
19
}
20
21
// resets when switches dimensions
@@ -26,7 +26,7 @@ void grouping_reset_stddev(RRDR *r) {
26
}
27
28
void grouping_free_stddev(RRDR *r) {
29
- freez(r->internal.grouping_data);
29
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
30
r->internal.grouping_data = NULL;
31
}
32
@@ -55,7 +55,7 @@ static inline NETDATA_DOUBLE mean(struct grouping_stddev *g) {
55
}
56
57
static inline NETDATA_DOUBLE variance(struct grouping_stddev *g) {
58
- return ( (g->count > 1) ? g->m_newS/(g->count - 1) : 0.0 );
58
+ return ( (g->count > 1) ? g->m_newS/(NETDATA_DOUBLE)(g->count - 1) : 0.0 );
59
}
60
static inline NETDATA_DOUBLE stddev(struct grouping_stddev *g) {
61
return sqrtndd(variance(g));
web/api/queries/sum/sum.c
+2
-2
@@ -11,7 +11,7 @@ struct grouping_sum {
11
};
12
13
void grouping_create_sum(RRDR *r, const char *options __maybe_unused) {
14
- r->internal.grouping_data = callocz(1, sizeof(struct grouping_sum));
14
+ r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_sum));
15
}
16
17
// resets when switches dimensions
@@ -23,7 +23,7 @@ void grouping_reset_sum(RRDR *r) {
23
}
24
25
void grouping_free_sum(RRDR *r) {
26
- freez(r->internal.grouping_data);
26
+ onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
27
r->internal.grouping_data = NULL;
28
}
29
web/api/web_api_v1.c
+9
-2
@@ -36,9 +36,11 @@ static struct {
36
, {"match-names" , 0 , RRDR_OPTION_MATCH_NAMES}
37
, {"showcustomvars" , 0 , RRDR_OPTION_CUSTOM_VARS}
38
, {"anomaly-bit" , 0 , RRDR_OPTION_ANOMALY_BIT}
39
+ , {"selected-tier" , 0 , RRDR_OPTION_SELECTED_TIER}
40
, {"raw" , 0 , RRDR_OPTION_RETURN_RAW}
41
, {"jw-anomaly-rates" , 0 , RRDR_OPTION_RETURN_JWAR}
42
, {"natural-points" , 0 , RRDR_OPTION_NATURAL_POINTS}
43
+ , {"virtual-points" , 0 , RRDR_OPTION_VIRTUAL_POINTS}
44
, {NULL , 0 , 0}
45
};
46
@@ -436,7 +438,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
438
char *chart_label_key = NULL;
439
char *chart_labels_filter = NULL;
440
char *group_options = NULL;
439
-
441
+ int tier = 0;
442
int group = RRDR_GROUPING_AVERAGE;
443
int show_dimensions = 0;
444
uint32_t format = DATASOURCE_JSON;
@@ -520,6 +522,11 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
522
else if(!strcmp(name, "max_anomaly_rates")) {
523
max_anomaly_rates_str = value;
524
}
525
+ else if(!strcmp(name, "tier")) {
526
+ tier = str2i(value);
527
+ if(tier >= 0 && tier < storage_tiers)
528
+ options |= RRDR_OPTION_SELECTED_TIER;
529
+ }
530
}
531
532
// validate the google parameters given
@@ -678,7 +685,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
685
.wb = w->response.data};
686
687
ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
681
- points, after, before, group, group_options, group_time, options, &last_timestamp_in_data);
688
+ points, after, before, group, group_options, group_time, options, &last_timestamp_in_data, tier);
689
690
free_context_param_list(owa, &context_param_list);
691