@cryptotaxi247 / netdata-1 / commits / 00712b351

QUERY_TARGET: new query engine for Netdata Agent (#13697)

* initial implementation of QUERY_TARGET * rrd2rrdr() interface * rrddim_find_best_tier_for_timeframe() ported * added dimension filtering * added db object in query target * rrd2rrdr() ported * working on formatters * working on jsonwrapper * finally, it compiles... * 1st run without crashes * query planer working * cleanup old code * review changes * fix also changing data collection frequency * fix signess * fix rrdlabels and dimension ordering * fixes * remove unused variable * ml should accept NULL response from rrd2rrdr() * number formatting fixes * more number formatting fixes * more number formatting fixes * support mc parallel queries * formatting and cleanup * added rrd2rrdr_legacy() as a simplified interface to run a query * make sure rrdset_find_natural_update_every_for_timeframe() returns a value * make signed comparisons * weights endpoint using rrdcontexts * fix for legacy db modes and cleanup * fix for chart_ids and remove AR chart from weights endpoint * Ignore command if not initialized yet * remove unused members * properly initialize window * code cleanup - rrddim linked list is gone; rrdset rwlock is gone too * reviewed RRDR.internal members * eliminate unnecessary members of QUERY_TARGET * more complete query ids; more detailed information on aborted queries * properly terminate option strings * query id contains group_options which is controlled by users, so escaping is necessary * tense in query id * tense in query id - again * added the remaining query options to the query id * Expose hidden option to the dimension * use the hidden flag when loading context dimensions * Specify table alias for option * dont update chart last access time, unless at least a dimension of the chart will be queried Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

Costa Tsaousis committed Oct 23, 2022 at 23:46 UTC 00712b351b3c83a54a147ca23365458acbef3105
50 files changed +2448 -1840
daemon/global_statistics.c
+6 -5
@@ -695,13 +695,14 @@ static void dbengine_statistics_charts(void) {
695 unsigned dbengine_contexts = 0, counted_multihost_db[RRD_STORAGE_TIERS] = { 0 }, i;
696
697 rrdhost_foreach_read(host) {
698 - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
698 + if (!rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
699
700 /* get localhost's DB engine's statistics for each tier */
701 - for(int tier = 0; tier < storage_tiers ;tier++) {
702 - if(!host->storage_instance[tier]) continue;
701 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
702 + if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue;
703 + if(!host->db[tier].instance) continue;
704
704 - if(is_storage_engine_shared(host->storage_instance[tier])) {
705 + if(is_storage_engine_shared(host->db[tier].instance)) {
706 if(counted_multihost_db[tier])
707 continue;
708 else
@@ -709,7 +710,7 @@ static void dbengine_statistics_charts(void) {
710 }
711
712 ++dbengine_contexts;
712 - rrdeng_get_37_statistics((struct rrdengine_instance *)host->storage_instance[tier], local_stats_array);
713 + rrdeng_get_37_statistics((struct rrdengine_instance *)host->db[tier].instance, local_stats_array);
714 for (i = 0; i < RRDENG_NR_STATS; ++i) {
715 /* aggregate statistics across hosts */
716 stats_array[i] += local_stats_array[i];
daemon/main.c
+2 -2
@@ -56,7 +56,7 @@ void netdata_cleanup_and_exit(int ret) {
56 info("EXIT: freeing database memory...");
57 #ifdef ENABLE_DBENGINE
58 if(dbengine_enabled) {
59 - for (int tier = 0; tier < storage_tiers; tier++)
59 + for (size_t tier = 0; tier < storage_tiers; tier++)
60 rrdeng_prepare_exit(multidb_ctx[tier]);
61 }
62 #endif
@@ -65,7 +65,7 @@ void netdata_cleanup_and_exit(int ret) {
65 metadata_sync_shutdown();
66 #ifdef ENABLE_DBENGINE
67 if(dbengine_enabled) {
68 - for (int tier = 0; tier < storage_tiers; tier++)
68 + for (size_t tier = 0; tier < storage_tiers; tier++)
69 rrdeng_exit(multidb_ctx[tier]);
70 }
71 #endif
daemon/service.c
+3 -3
@@ -47,11 +47,11 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
47 /* only a collector can mark a chart as obsolete, so we must remove the reference */
48
49 size_t tiers_available = 0, tiers_said_yes = 0;
50 - for(int tier = 0; tier < storage_tiers ;tier++) {
50 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
51 if(rd->tiers[tier]) {
52 tiers_available++;
53
54 - if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle))
54 + if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
55 tiers_said_yes++;
56
57 rd->tiers[tier]->db_collection_handle = NULL;
@@ -217,7 +217,7 @@ restart_after_removal:
217
218 if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST)
219 /* don't delete multi-host DB host files */
220 - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0]))
220 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
221 ) {
222 worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS);
223 rrdhost_delete_charts(host);
daemon/unit_test.c
+29 -29
@@ -1818,7 +1818,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
1818 // feed it with the test data
1819 for (i = 0 ; i < CHARTS ; ++i) {
1820 for (j = 0 ; j < DIMS ; ++j) {
1821 - rd[i][j]->tiers[0]->collect_ops.change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
1821 + rd[i][j]->tiers[0]->collect_ops->change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
1822
1823 rd[i][j]->last_collected_time.tv_sec =
1824 st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
@@ -1852,7 +1852,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1852 int i, j, k, c, errors, update_every;
1853 collected_number last;
1854 NETDATA_DOUBLE value, expected;
1855 - struct rrddim_query_handle handle;
1855 + struct storage_engine_query_handle handle;
1856 size_t value_errors = 0, time_errors = 0;
1857
1858 update_every = REGION_UPDATE_EVERY[current_region];
@@ -1863,13 +1863,13 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1863 time_now = time_start + (c + 1) * update_every;
1864 for (i = 0 ; i < CHARTS ; ++i) {
1865 for (j = 0; j < DIMS; ++j) {
1866 - rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every);
1866 + rd[i][j]->tiers[0]->query_ops->init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every);
1867 for (k = 0; k < QUERY_BATCH; ++k) {
1868 last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
1869 j * REGION_POINTS[current_region] + c + k;
1870 expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
1871
1872 - STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops.next_metric(&handle);
1872 + STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops->next_metric(&handle);
1873 value = sp.sum;
1874 time_retrieved = sp.start_time;
1875 end_time = sp.end_time;
@@ -1891,7 +1891,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1891 errors++;
1892 }
1893 }
1894 - rd[i][j]->tiers[0]->query_ops.finalize(&handle);
1894 + rd[i][j]->tiers[0]->query_ops->finalize(&handle);
1895 }
1896 }
1897 }
@@ -1922,23 +1922,22 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
1922 long points = (time_end - time_start) / update_every;
1923 for (i = 0 ; i < CHARTS ; ++i) {
1924 ONEWAYALLOC *owa = onewayalloc_create(0);
1925 - RRDR *r = rrd2rrdr(owa, st[i], points, time_start, time_end,
1926 - RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS,
1927 - NULL, NULL, NULL, 0, 0);
1928 -
1925 + RRDR *r = rrd2rrdr_legacy(owa, st[i], points, time_start, time_end,
1926 + RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS,
1927 + NULL, NULL, 0, 0);
1928 if (!r) {
1929 fprintf(stderr, " DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", rrdset_name(st[i]), current_region);
1930 return ++errors;
1931 } else {
1933 - assert(r->st == st[i]);
1934 - for (c = 0; c != rrdr_rows(r) ; ++c) {
1932 + assert(r->internal.qt->request.st == st[i]);
1933 + for (c = 0; c != (long)rrdr_rows(r) ; ++c) {
1934 RRDDIM *d;
1935 time_now = time_start + (c + 1) * update_every;
1936 time_retrieved = r->t[c];
1937
1938 // for each dimension
1940 - rrddim_foreach_read(d, r->st) {
1941 - if(unlikely((int)d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe
1939 + rrddim_foreach_read(d, r->internal.qt->request.st) {
1940 + if(unlikely(d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe
1941
1942 j = (int)d_dfe.counter;
1943
@@ -2061,25 +2060,26 @@ int test_dbengine(void)
2060 long point_offset = (time_start[current_region] - time_start[0]) / update_every;
2061 for (i = 0 ; i < CHARTS ; ++i) {
2062 ONEWAYALLOC *owa = onewayalloc_create(0);
2064 - RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every,
2065 - time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
2066 - RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0, 0);
2063 + RRDR *r = rrd2rrdr_legacy(owa, st[i], points, time_start[0] + update_every,
2064 + time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
2065 + RRDR_OPTION_NATURAL_POINTS, NULL, NULL, 0, 0);
2066 +
2067 if (!r) {
2068 fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", rrdset_name(st[i]));
2069 ++errors;
2070 } else {
2071 long c;
2072
2073 - assert(r->st == st[i]);
2073 + assert(r->internal.qt->request.st == st[i]);
2074 // test current region values only, since they must be left unchanged
2075 - for (c = point_offset ; c < point_offset + rrdr_rows(r) / REGIONS / 2 ; ++c) {
2075 + for (c = point_offset ; c < (long)(point_offset + rrdr_rows(r) / REGIONS / 2) ; ++c) {
2076 RRDDIM *d;
2077 time_t time_now = time_start[current_region] + (c - point_offset + 2) * update_every;
2078 time_t time_retrieved = r->t[c];
2079
2080 // for each dimension
2081 - rrddim_foreach_read(d, r->st) {
2082 - if(unlikely((int)d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe
2081 + rrddim_foreach_read(d, r->internal.qt->request.st) {
2082 + if(unlikely(d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe
2083
2084 j = (int)d_dfe.counter;
2085
@@ -2113,9 +2113,9 @@ int test_dbengine(void)
2113 }
2114 error_out:
2115 rrd_wrlock();
2116 - rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[0]);
2116 + rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
2117 rrdhost_delete_charts(host);
2118 - rrdeng_exit((struct rrdengine_instance *)host->storage_instance[0]);
2118 + rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
2119 rrd_unlock();
2120
2121 return errors + value_errors + time_errors;
@@ -2293,7 +2293,7 @@ static void query_dbengine_chart(void *arg)
2293 time_t time_now, time_retrieved, end_time;
2294 collected_number generatedv;
2295 NETDATA_DOUBLE value, expected;
2296 - struct rrddim_query_handle handle;
2296 + struct storage_engine_query_handle handle;
2297 size_t value_errors = 0, time_errors = 0;
2298
2299 do {
@@ -2320,13 +2320,13 @@ static void query_dbengine_chart(void *arg)
2320 time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
2321 }
2322
2323 - rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before);
2323 + rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before);
2324 ++thread_info->queries_nr;
2325 for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
2326 generatedv = generate_dbengine_chart_value(i, j, time_now);
2327 expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
2328
2329 - if (unlikely(rd->tiers[0]->query_ops.is_finished(&handle))) {
2329 + if (unlikely(rd->tiers[0]->query_ops->is_finished(&handle))) {
2330 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
2331 fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
2332 ", found data gap, ### E R R O R ###\n",
@@ -2336,7 +2336,7 @@ static void query_dbengine_chart(void *arg)
2336 break;
2337 }
2338
2339 - STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
2339 + STORAGE_POINT sp = rd->tiers[0]->query_ops->next_metric(&handle);
2340 value = sp.sum;
2341 time_retrieved = sp.start_time;
2342 end_time = sp.end_time;
@@ -2374,7 +2374,7 @@ static void query_dbengine_chart(void *arg)
2374 }
2375 }
2376 }
2377 - rd->tiers[0]->query_ops.finalize(&handle);
2377 + rd->tiers[0]->query_ops->finalize(&handle);
2378 } while(!thread_info->done);
2379
2380 if(value_errors)
@@ -2522,9 +2522,9 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2522 }
2523 freez(query_threads);
2524 rrd_wrlock();
2525 - rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[0]);
2525 + rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
2526 rrdhost_delete_charts(host);
2527 - rrdeng_exit((struct rrdengine_instance *)host->storage_instance[0]);
2527 + rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
2528 rrd_unlock();
2529 }
2530
database/engine/rrdengineapi.c
+21 -27
@@ -36,16 +36,6 @@ int default_multidb_disk_quota_mb = 256;
36 /* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
37 uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
38
39 -
40 -// ----------------------------------------------------------------------------
41 -// helpers
42 -
43 -static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host, int tier) {
44 - if(tier < 0 || tier >= RRD_STORAGE_TIERS) tier = 0;
45 - if(!host->storage_instance[tier]) tier = 0;
46 - return (struct rrdengine_instance *)host->storage_instance[tier];
47 -}
48 -
39 // ----------------------------------------------------------------------------
40 // metrics groups
41
@@ -114,16 +104,18 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get_legacy(STORAGE_INSTANCE *db_instance, c
104
105 void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle) {
106 struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
117 - struct rrdengine_instance *ctx = page_index->ctx;
118 - struct page_cache *pg_cache = &ctx->pg_cache;
107
120 - uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
121 - page_index->refcount--;
122 - if(page_index->alignment && page_index->refcount == 0) {
123 - page_index->alignment->refcount--;
108 + unsigned short refcount = __atomic_sub_fetch(&page_index->refcount, 1, __ATOMIC_SEQ_CST);
109 + if(refcount == 0 && page_index->alignment) {
110 + __atomic_sub_fetch(&page_index->alignment->refcount, 1, __ATOMIC_SEQ_CST);
111 page_index->alignment = NULL;
112 }
126 - uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
113 +}
114 +
115 +STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle) {
116 + struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
117 + __atomic_add_fetch(&page_index->refcount, 1, __ATOMIC_SEQ_CST);
118 + return db_metric_handle;
119 }
120
121 STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg) {
@@ -134,9 +126,12 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *
126
127 uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
128 Pvoid_t *PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, uuid, sizeof(uuid_t));
137 - if (likely(NULL != PValue)) {
129 + if (likely(NULL != PValue))
130 page_index = *PValue;
139 - page_index->refcount++;
131 + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
132 +
133 + if (likely(page_index)) {
134 + __atomic_add_fetch(&page_index->refcount, 1, __ATOMIC_SEQ_CST);
135
136 if(pa) {
137 if(page_index->alignment && page_index->alignment != pa)
@@ -144,11 +139,10 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *
139
140 if(!page_index->alignment) {
141 page_index->alignment = pa;
147 - pa->refcount++;
142 + __atomic_add_fetch(&pa->refcount, 1, __ATOMIC_SEQ_CST);
143 }
144 }
145 }
151 - uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
146
147 return (STORAGE_METRIC_HANDLE *)page_index;
148 }
@@ -572,7 +566,7 @@ void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *col
566 * Gets a handle for loading metrics from the database.
567 * The handle must be released with rrdeng_load_metric_final().
568 */
575 -void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle, time_t start_time_s, time_t end_time_s)
569 +void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrdimm_handle, time_t start_time_s, time_t end_time_s)
570 {
571 struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
572 struct rrdengine_instance *ctx = page_index->ctx;
@@ -603,7 +597,7 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrd
597 handle->wanted_start_time_s = INVALID_TIME;
598 }
599
606 -static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle, bool debug_this __maybe_unused) {
600 +static int rrdeng_load_page_next(struct storage_engine_query_handle *rrdimm_handle, bool debug_this __maybe_unused) {
601 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
602
603 struct rrdengine_instance *ctx = handle->ctx;
@@ -682,7 +676,7 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle, bool
676 // Returns the metric and sets its timestamp into current_time
677 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
678 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
685 -STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle) {
679 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle) {
680 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrddim_handle->handle;
681 // struct rrdeng_metric_handle *metric_handle = handle->metric_handle;
682
@@ -799,7 +793,7 @@ STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle)
793 return sp;
794 }
795
802 -int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle)
796 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrdimm_handle)
797 {
798 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
799 return (INVALID_TIME == handle->wanted_start_time_s);
@@ -808,7 +802,7 @@ int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle)
802 /*
803 * Releases the database reference from the handle for loading metrics.
804 */
811 -void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
805 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrdimm_handle)
806 {
807 struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
808 struct rrdengine_instance *ctx = handle->ctx;
@@ -1039,7 +1033,7 @@ void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle)
1033 * Returns 0 on success, negative on error
1034 */
1035 int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
1042 - unsigned disk_space_mb, int tier) {
1036 + unsigned disk_space_mb, size_t tier) {
1037 struct rrdengine_instance *ctx;
1038 int error;
1039 uint32_t max_open_files;
database/engine/rrdengineapi.h
+6 -8
@@ -47,6 +47,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *
47 STORAGE_METRIC_HANDLE *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
48 STORAGE_METRIC_HANDLE *rrdeng_metric_get_legacy(STORAGE_INSTANCE *db_instance, const char *rd_id, const char *st_id, STORAGE_METRICS_GROUP *smg);
49 void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
50 +STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);
51
52 STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
53 void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
@@ -59,16 +60,13 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
60 SN_FLAGS flags);
61 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
62
62 -unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time_s, time_t end_time_s,
63 - struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
64 -
65 -void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle,
63 +void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrdimm_handle,
64 time_t start_time_s, time_t end_time_s);
67 -STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle);
65 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
66
67
70 -int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);
71 -void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
68 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrdimm_handle);
69 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrdimm_handle);
70 time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
71 time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
72
@@ -76,7 +74,7 @@ void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long
74
75 /* must call once before using anything */
76 int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
79 - unsigned disk_space_mb, int tier);
77 + unsigned disk_space_mb, size_t tier);
78
79 int rrdeng_exit(struct rrdengine_instance *ctx);
80 void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
database/ram/rrddim_mem.c
+10 -6
@@ -5,7 +5,6 @@
5 static Pvoid_t rrddim_JudyHS_array = NULL;
6 static netdata_rwlock_t rrddim_JudyHS_rwlock = NETDATA_RWLOCK_INITIALIZER;
7
8 -
8 // ----------------------------------------------------------------------------
9 // metrics groups
10
@@ -39,7 +38,8 @@ rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance __maybe_un
38 return (STORAGE_METRIC_HANDLE *)rd;
39 }
40
42 -STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid, STORAGE_METRICS_GROUP *smg __maybe_unused) {
41 +STORAGE_METRIC_HANDLE *
42 +rrddim_metric_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid, STORAGE_METRICS_GROUP *smg __maybe_unused) {
43 RRDDIM *rd = NULL;
44 netdata_rwlock_rdlock(&rrddim_JudyHS_rwlock);
45 Pvoid_t *PValue = JudyHSGet(rrddim_JudyHS_array, uuid, sizeof(uuid_t));
@@ -50,6 +50,10 @@ STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance __maybe_u
50 return (STORAGE_METRIC_HANDLE *)rd;
51 }
52
53 +STORAGE_METRIC_HANDLE *rrddim_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle) {
54 + return db_metric_handle;
55 +}
56 +
57 void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) {
58 RRDDIM *rd = (RRDDIM *)db_metric_handle;
59
@@ -186,7 +190,7 @@ static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) {
190 // ----------------------------------------------------------------------------
191 // RRDDIM legacy database query functions
192
189 -void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time) {
193 +void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle, time_t start_time, time_t end_time) {
194 RRDDIM *rd = (RRDDIM *)db_metric_handle;
195
196 handle->rd = rd;
@@ -209,7 +213,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_qu
213 // Returns the metric and sets its timestamp into current_time
214 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
215 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
212 -STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle) {
216 +STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle) {
217 RRDDIM *rd = handle->rd;
218 struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
219 size_t entries = rd->rrdset->entries;
@@ -248,12 +252,12 @@ STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle) {
252 return sp;
253 }
254
251 -int rrddim_query_is_finished(struct rrddim_query_handle *handle) {
255 +int rrddim_query_is_finished(struct storage_engine_query_handle *handle) {
256 struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
257 return (h->next_timestamp > handle->end_time_s);
258 }
259
256 -void rrddim_query_finalize(struct rrddim_query_handle *handle) {
260 +void rrddim_query_finalize(struct storage_engine_query_handle *handle) {
261 #ifdef NETDATA_INTERNAL_CHECKS
262 if(!rrddim_query_is_finished(handle))
263 error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", rrdset_id(handle->rd->rrdset), rrddim_name(handle->rd));
database/ram/rrddim_mem.h
+5 -4
@@ -22,6 +22,7 @@ struct mem_query_handle {
22
23 STORAGE_METRIC_HANDLE *rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
24 STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
25 +STORAGE_METRIC_HANDLE *rrddim_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);
26 void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
27
28 STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
@@ -38,10 +39,10 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
39 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
40 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
41
41 -void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
42 -STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle);
43 -int rrddim_query_is_finished(struct rrddim_query_handle *handle);
44 -void rrddim_query_finalize(struct rrddim_query_handle *handle);
42 +void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle, time_t start_time, time_t end_time);
43 +STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle);
44 +int rrddim_query_is_finished(struct storage_engine_query_handle *handle);
45 +void rrddim_query_finalize(struct storage_engine_query_handle *handle);
46 time_t rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
47 time_t rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
48
database/rrd.h
+74 -67
@@ -20,19 +20,21 @@ typedef struct rrdset RRDSET;
20 typedef struct rrdcalc RRDCALC;
21 typedef struct rrdcalctemplate RRDCALCTEMPLATE;
22 typedef struct alarm_entry ALARM_ENTRY;
23 -typedef struct context_param CONTEXT_PARAM;
23
24 typedef struct rrdfamily_acquired RRDFAMILY_ACQUIRED;
25 typedef struct rrdvar_acquired RRDVAR_ACQUIRED;
26 typedef struct rrdsetvar_acquired RRDSETVAR_ACQUIRED;
27 typedef struct rrdcalc_acquired RRDCALC_ACQUIRED;
28
29 +typedef struct rrdhost_acquired RRDHOST_ACQUIRED;
30 +typedef struct rrdset_acquired RRDSET_ACQUIRED;
31 +typedef struct rrddim_acquired RRDDIM_ACQUIRED;
32 +
33 typedef void *ml_host_t;
34 typedef void *ml_dimension_t;
35
36 // forward declarations
37 struct rrddim_tier;
35 -struct context_param;
38
39 #ifdef ENABLE_DBENGINE
40 struct rrdeng_page_descr;
@@ -54,8 +56,8 @@ struct pg_cache_page_index;
56 #include "rrdcontext.h"
57
58 extern bool dbengine_enabled;
57 -extern int storage_tiers;
58 -extern int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
59 +extern size_t storage_tiers;
60 +extern size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
61
62 typedef enum {
63 RRD_BACKFILL_NONE,
@@ -174,8 +176,9 @@ DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rf);
176 // options are permanent configuration options (no atomics to alter/access them)
177 typedef enum rrddim_options {
178 RRDDIM_OPTION_NONE = 0,
177 - RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
178 - RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
179 + RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
180 + RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
181 + RRDDIM_OPTION_BACKFILLED_HIGH_TIERS = (1 << 2), // when set, we have backfilled higher tiers
182
183 // this is 8-bit
184 } RRDDIM_OPTIONS;
@@ -227,6 +230,7 @@ void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLAB
230 void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
231 void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
232 void rrdlabels_get_value_to_char_or_null(DICTIONARY *labels, char **value, const char *key);
233 +void rrdlabels_flush(DICTIONARY *labels_dict);
234
235 void rrdlabels_unmark_all(DICTIONARY *labels);
236 void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
@@ -281,18 +285,11 @@ struct rrddim {
285 // ------------------------------------------------------------------------
286 // operational state members
287
284 -#ifdef ENABLE_ACLK
285 - int aclk_live_status;
286 -#endif
287 -
288 ml_dimension_t ml_dimension; // machine learning data about this dimension
289
290 // ------------------------------------------------------------------------
291 // linking to siblings and parents
292
293 - struct rrddim *next; // linking of dimensions within the same data set
294 - struct rrddim *prev; // linking of dimensions within the same data set
295 -
293 struct rrdset *rrdset;
294
295 RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
@@ -346,21 +343,6 @@ size_t rrddim_memory_file_header_size(void);
343 void rrddim_memory_file_save(RRDDIM *rd);
344
345 // ----------------------------------------------------------------------------
349 -// engine-specific iterator state for dimension data collection
350 -typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
351 -
352 -// ----------------------------------------------------------------------------
353 -// engine-specific iterator state for dimension data queries
354 -typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
355 -
356 -// ----------------------------------------------------------------------------
357 -// iterator state for RRD dimension data queries
358 -struct rrddim_query_handle {
359 - RRDDIM *rd;
360 - time_t start_time_s;
361 - time_t end_time_s;
362 - STORAGE_QUERY_HANDLE* handle;
363 -};
346
347 typedef struct storage_point {
348 NETDATA_DOUBLE min; // when count > 1, this is the minimum among them
@@ -398,9 +380,17 @@ typedef struct storage_point {
380 #define storage_point_is_unset(x) (!(x).count)
381 #define storage_point_is_empty(x) (!netdata_double_isnumber((x).sum))
382
383 +// ----------------------------------------------------------------------------
384 +// engine-specific iterator state for dimension data collection
385 +typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
386 +
387 +// ----------------------------------------------------------------------------
388 +// engine-specific iterator state for dimension data queries
389 +typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
390 +
391 // ------------------------------------------------------------------------
392 // function pointers that handle data collection
403 -struct rrddim_collect_ops {
393 +struct storage_engine_collect_ops {
394 // an initialization function to run before starting collection
395 STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
396
@@ -421,19 +411,28 @@ struct rrddim_collect_ops {
411 void (*metrics_group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *sa);
412 };
413
414 +// ----------------------------------------------------------------------------
415 +// iterator state for RRD dimension data queries
416 +struct storage_engine_query_handle {
417 + RRDDIM *rd;
418 + time_t start_time_s;
419 + time_t end_time_s;
420 + STORAGE_QUERY_HANDLE* handle;
421 +};
422 +
423 // function pointers that handle database queries
425 -struct rrddim_query_ops {
424 +struct storage_engine_query_ops {
425 // run this before starting a series of next_metric() database queries
427 - void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
426 + void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle, time_t start_time, time_t end_time);
427
428 // run this to load each metric number from the database
430 - STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
429 + STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
430
431 // run this to test if the series of next_metric() database queries is finished
433 - int (*is_finished)(struct rrddim_query_handle *handle);
432 + int (*is_finished)(struct storage_engine_query_handle *handle);
433
434 // run this after finishing a series of load_metric() database queries
436 - void (*finalize)(struct rrddim_query_handle *handle);
435 + void (*finalize)(struct storage_engine_query_handle *handle);
436
437 // get the timestamp of the last entry of this metric
438 time_t (*latest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
@@ -442,23 +441,45 @@ struct rrddim_query_ops {
441 time_t (*oldest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
442 };
443
444 +typedef struct storage_engine STORAGE_ENGINE;
445 +
446 +// ------------------------------------------------------------------------
447 +// function pointers for all APIs provided by a storage engine
448 +typedef struct storage_engine_api {
449 + // metric management
450 + STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
451 + STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *instance, STORAGE_METRICS_GROUP *smg);
452 + void (*metric_release)(STORAGE_METRIC_HANDLE *);
453 + STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
454 +
455 + // operations
456 + struct storage_engine_collect_ops collect_ops;
457 + struct storage_engine_query_ops query_ops;
458 +} STORAGE_ENGINE_API;
459 +
460 +struct storage_engine {
461 + RRD_MEMORY_MODE id;
462 + const char* name;
463 + STORAGE_ENGINE_API api;
464 +};
465 +
466 +STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
467 +STORAGE_ENGINE* storage_engine_find(const char* name);
468
469 // ----------------------------------------------------------------------------
470 // Storage tier data for every dimension
471
472 struct rrddim_tier {
450 - int tier_grouping;
451 - RRD_MEMORY_MODE mode; // the memory mode of this tier
473 + size_t tier_grouping;
474 STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database
475 STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle
476 STORAGE_POINT virtual_point;
477 time_t next_point_time;
456 - usec_t last_collected_ut;
457 - struct rrddim_collect_ops collect_ops;
458 - struct rrddim_query_ops query_ops;
478 + struct storage_engine_collect_ops *collect_ops;
479 + struct storage_engine_query_ops *query_ops;
480 };
481
461 -void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now);
482 +void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now);
483
484 // ----------------------------------------------------------------------------
485 // these loop macros make sure the linked list is accessed with the right lock
@@ -549,10 +570,6 @@ struct rrdset {
570 DICTIONARY *rrddimvar_root_index; // dimension variables
571 // we use this dictionary to manage their allocation
572
552 - // TODO - dimensions linked list and lock to be removed
553 - netdata_rwlock_t rrdset_rwlock; // protects the dimensions linked list
554 - RRDDIM *dimensions; // chart metrics
555 -
573 // ------------------------------------------------------------------------
574 // operational state members
575
@@ -600,19 +617,6 @@ struct rrdset {
617
618 time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream
619
603 - // ------------------------------------------------------------------------
604 - // context queries temp variables
605 - // TODO - eliminate these
606 -
607 - time_t last_entry_t; // the last_entry_t computed for transient RRDSET
608 -
609 - // ------------------------------------------------------------------------
610 - // dbengine specifics
611 - // TODO - they should be managed by storage engine
612 - // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
613 -
614 - size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
615 -
620 // ------------------------------------------------------------------------
621 // db mode SAVE, MAP specifics
622 // TODO - they should be managed by storage engine
@@ -666,10 +670,6 @@ struct rrdset {
670 #define rrdset_name(st) string2str((st)->name)
671 #define rrdset_id(st) string2str((st)->id)
672
669 -#define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
670 -#define rrdset_wrlock(st) netdata_rwlock_wrlock(&((st)->rrdset_rwlock))
671 -#define rrdset_unlock(st) netdata_rwlock_unlock(&((st)->rrdset_rwlock))
672 -
673 STRING *rrd_string_strdupz(const char *s);
674
675 // ----------------------------------------------------------------------------
@@ -910,11 +910,20 @@ struct rrdhost {
910
911 int rrd_update_every; // the update frequency of the host
912 long rrd_history_entries; // the number of history entries for the host's charts
913 - RRD_MEMORY_MODE rrd_memory_mode; // the memory more for the charts of this host
913 +
914 + RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
915 + // the actual per tier is at .db[tier].mode
916
917 char *cache_dir; // the directory to save RRD cache files
918 char *varlib_dir; // the directory to save health log
919
920 + struct {
921 + RRD_MEMORY_MODE mode; // the db mode for this tier
922 + STORAGE_ENGINE *eng; // the storage engine API for this tier
923 + STORAGE_INSTANCE *instance; // the db instance for this tier
924 + size_t tier_grouping; // tier 0 iterations aggregated on this tier
925 + } db[RRD_STORAGE_TIERS];
926 +
927 struct rrdhost_system_info *system_info; // information collected from the host environment
928
929 // ------------------------------------------------------------------------
@@ -999,8 +1008,6 @@ struct rrdhost {
1008 DICTIONARY *rrdvars; // the host's chart variables index
1009 // this includes custom host variables
1010
1002 - STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers
1003 -
1011 RRDCONTEXTS *rrdctx_hub_queue;
1012 RRDCONTEXTS *rrdctx_post_processing_queue;
1013 RRDCONTEXTS *rrdctx;
@@ -1229,8 +1236,8 @@ void rrdset_isnot_obsolete(RRDSET *st);
1236
1237 // checks if the RRDSET should be offered to viewers
1238 #define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
1232 -#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
1233 -#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
1239 +#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
1240 +#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
1241
1242 time_t rrddim_first_entry_t(RRDDIM *rd);
1243 time_t rrddim_last_entry_t(RRDDIM *rd);
@@ -1313,7 +1320,7 @@ void set_host_properties(
1320 const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
1321 const char *program_name, const char *program_version);
1322
1316 -int get_tier_grouping(int tier);
1323 +size_t get_tier_grouping(size_t tier);
1324
1325 // ----------------------------------------------------------------------------
1326 // RRD DB engine declarations
database/rrdcontext.c
+756 -32
@@ -5,6 +5,7 @@
5 #include "aclk/schema-wrappers/context.h"
6 #include "aclk/aclk_contexts_api.h"
7 #include "aclk/aclk.h"
8 +#include "storage_engine.h"
9
10 #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
11 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
@@ -115,7 +116,7 @@ typedef enum {
116 // check if ANY of the given flags (bits) is set
117 #define rrd_flag_check(obj, flag) (rrd_flags_get(obj) & (flag))
118
118 -// check if ALL of the given flags (bits) are set
119 +// check if ALL the given flags (bits) are set
120 #define rrd_flag_check_all(obj, flag) (rrd_flag_check(obj, flag) == (flag))
121
122 // set one or more flags (bits)
@@ -320,15 +321,41 @@ typedef struct rrdcontext {
321 // ----------------------------------------------------------------------------
322 // helper one-liners for RRDMETRIC
323
324 +static void rrdmetric_update_retention(RRDMETRIC *rm);
325 +
326 static inline RRDMETRIC *rrdmetric_acquired_value(RRDMETRIC_ACQUIRED *rma) {
327 return dictionary_acquired_item_value((DICTIONARY_ITEM *)rma);
328 }
329
330 +static inline RRDMETRIC_ACQUIRED *rrdmetric_acquired_dup(RRDMETRIC_ACQUIRED *rma) {
331 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
332 + return (RRDMETRIC_ACQUIRED *)dictionary_acquired_item_dup(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma);
333 +}
334 +
335 static inline void rrdmetric_release(RRDMETRIC_ACQUIRED *rma) {
336 RRDMETRIC *rm = rrdmetric_acquired_value(rma);
337 dictionary_acquired_item_release(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma);
338 }
339
340 +const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) {
341 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
342 + return string2str(rm->id);
343 +}
344 +
345 +const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma) {
346 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
347 + return string2str(rm->name);
348 +}
349 +
350 +NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma) {
351 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
352 +
353 + if(rm->rrddim)
354 + return rm->rrddim->last_stored_value;
355 +
356 + return NAN;
357 +}
358 +
359 // ----------------------------------------------------------------------------
360 // helper one-liners for RRDINSTANCE
361
@@ -336,11 +363,37 @@ static inline RRDINSTANCE *rrdinstance_acquired_value(RRDINSTANCE_ACQUIRED *ria)
363 return dictionary_acquired_item_value((DICTIONARY_ITEM *)ria);
364 }
365
366 +static inline RRDINSTANCE_ACQUIRED *rrdinstance_acquired_dup(RRDINSTANCE_ACQUIRED *ria) {
367 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
368 + return (RRDINSTANCE_ACQUIRED *)dictionary_acquired_item_dup(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
369 +}
370 +
371 static inline void rrdinstance_release(RRDINSTANCE_ACQUIRED *ria) {
372 RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
373 dictionary_acquired_item_release(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
374 }
375
376 +const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria) {
377 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
378 + return string2str(ri->id);
379 +}
380 +
381 +const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) {
382 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
383 + return string2str(ri->name);
384 +}
385 +
386 +DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) {
387 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
388 + return ri->rrdlabels;
389 +}
390 +
391 +DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria) {
392 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
393 + if(!ri->rrdset) return NULL;
394 + return ri->rrdset->functions_view;
395 +}
396 +
397 // ----------------------------------------------------------------------------
398 // helper one-liners for RRDCONTEXT
399
@@ -348,6 +401,16 @@ static inline RRDCONTEXT *rrdcontext_acquired_value(RRDCONTEXT_ACQUIRED *rca) {
401 return dictionary_acquired_item_value((DICTIONARY_ITEM *)rca);
402 }
403
404 +const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca) {
405 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
406 + return string2str(rc->id);
407 +}
408 +
409 +static inline RRDCONTEXT_ACQUIRED *rrdcontext_acquired_dup(RRDCONTEXT_ACQUIRED *rca) {
410 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
411 + return (RRDCONTEXT_ACQUIRED *)dictionary_acquired_item_dup((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
412 +}
413 +
414 static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) {
415 RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
416 dictionary_acquired_item_release((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
@@ -467,9 +530,9 @@ static void rrdmetric_delete_callback(const DICTIONARY_ITEM *item __maybe_unused
530
531 // called when the same rrdmetric is inserted again to the rrdmetrics dictionary of a rrdinstance
532 // while this is called, the dictionary is write locked, but there may be other users of the object
470 -static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldv, void *newv, void *rrdinstance __maybe_unused) {
471 - RRDMETRIC *rm = oldv;
472 - RRDMETRIC *rm_new = newv;
533 +static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdinstance __maybe_unused) {
534 + RRDMETRIC *rm = old_value;
535 + RRDMETRIC *rm_new = new_value;
536
537 internal_error(rm->id != rm_new->id,
538 "RRDMETRIC: '%s' cannot change id to '%s'",
@@ -729,9 +792,9 @@ static void rrdinstance_delete_callback(const DICTIONARY_ITEM *item __maybe_unus
792 rrdinstance_free(ri);
793 }
794
732 -static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldv, void *newv, void *rrdcontext __maybe_unused) {
733 - RRDINSTANCE *ri = (RRDINSTANCE *)oldv;
734 - RRDINSTANCE *ri_new = (RRDINSTANCE *)newv;
795 +static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdcontext __maybe_unused) {
796 + RRDINSTANCE *ri = (RRDINSTANCE *)old_value;
797 + RRDINSTANCE *ri_new = (RRDINSTANCE *)new_value;
798
799 internal_error(ri->id != ri_new->id,
800 "RRDINSTANCE: '%s' cannot change id to '%s'",
@@ -934,7 +997,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
997 }
998
999 if(rca_old && ria_old) {
937 - // Ooops! The chart changed context!
1000 + // Oops! The chart changed context!
1001
1002 // RRDCONTEXT *rc_old = rrdcontext_acquired_value(rca_old);
1003 RRDINSTANCE *ri_old = rrdinstance_acquired_value(ria_old);
@@ -1145,8 +1208,8 @@ static void rrdcontext_insert_callback(const DICTIONARY_ITEM *item __maybe_unuse
1208
1209 rc->version = rc->hub.version;
1210 rc->priority = rc->hub.priority;
1148 - rc->first_time_t = rc->hub.first_time_t;
1149 - rc->last_time_t = rc->hub.last_time_t;
1211 + rc->first_time_t = (time_t)rc->hub.first_time_t;
1212 + rc->last_time_t = (time_t)rc->hub.last_time_t;
1213
1214 if(rc->hub.deleted || !rc->hub.first_time_t)
1215 rrd_flag_set_deleted(rc, RRD_FLAG_NONE);
@@ -1180,11 +1243,11 @@ static void rrdcontext_delete_callback(const DICTIONARY_ITEM *item __maybe_unuse
1243 rrdcontext_freez(rc);
1244 }
1245
1183 -static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldv, void *newv, void *rrdhost __maybe_unused) {
1184 - RRDCONTEXT *rc = (RRDCONTEXT *)oldv;
1185 - RRDCONTEXT *rc_new = (RRDCONTEXT *)newv;
1246 +static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdhost __maybe_unused) {
1247 + RRDCONTEXT *rc = (RRDCONTEXT *)old_value;
1248 + RRDCONTEXT *rc_new = (RRDCONTEXT *)new_value;
1249
1187 - //current rc is not archived, new_rc is archived, dont merge
1250 + //current rc is not archived, new_rc is archived, don't merge
1251 if (!rrd_flag_is_archived(rc) && rrd_flag_is_archived(rc_new)) {
1252 rrdcontext_freez(rc_new);
1253 return false;
@@ -1972,13 +2035,8 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
2035
2036 RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2037
1975 - if(after != 0 && before != 0) {
1976 - long long after_wanted = after;
1977 - long long before_wanted = before;
1978 - rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
1979 - after = after_wanted;
1980 - before = before_wanted;
1981 - }
2038 + if(after != 0 && before != 0)
2039 + rrdr_relative_window_to_absolute(&after, &before);
2040
2041 struct rrdcontext_to_json t_contexts = {
2042 .wb = wb,
@@ -2012,13 +2070,8 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
2070 if(host->node_id)
2071 uuid_unparse(*host->node_id, node_uuid);
2072
2015 - if(after != 0 && before != 0) {
2016 - long long after_wanted = after;
2017 - long long before_wanted = before;
2018 - rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
2019 - after = after_wanted;
2020 - before = before_wanted;
2021 - }
2073 + if(after != 0 && before != 0)
2074 + rrdr_relative_window_to_absolute(&after, &before);
2075
2076 buffer_sprintf(wb, "{\n"
2077 "\t\"hostname\": \"%s\""
@@ -2057,6 +2110,675 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
2110 return HTTP_RESP_OK;
2111 }
2112
2113 +// ----------------------------------------------------------------------------
2114 +// weights API
2115 +
2116 +static void metric_entry_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
2117 + struct metric_entry *t = value;
2118 + t->rca = rrdcontext_acquired_dup(t->rca);
2119 + t->ria = rrdinstance_acquired_dup(t->ria);
2120 + t->rma = rrdmetric_acquired_dup(t->rma);
2121 +}
2122 +static void metric_entry_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
2123 + struct metric_entry *t = value;
2124 + rrdcontext_release(t->rca);
2125 + rrdinstance_release(t->ria);
2126 + rrdmetric_release(t->rma);
2127 +}
2128 +static bool metric_entry_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value __maybe_unused, void *new_value __maybe_unused, void *data __maybe_unused) {
2129 + fatal("RRDCONTEXT: %s() detected a conflict on a metric pointer!", __FUNCTION__);
2130 + return false;
2131 +}
2132 +
2133 +DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contexts) {
2134 + if(!host || !host->rrdctx)
2135 + return NULL;
2136 +
2137 + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
2138 + dictionary_register_insert_callback(dict, metric_entry_insert_callback, NULL);
2139 + dictionary_register_delete_callback(dict, metric_entry_delete_callback, NULL);
2140 + dictionary_register_conflict_callback(dict, metric_entry_conflict_callback, NULL);
2141 +
2142 + RRDCONTEXT *rc;
2143 + dfe_start_reentrant((DICTIONARY *)host->rrdctx, rc) {
2144 + if(rrd_flag_is_deleted(rc))
2145 + continue;
2146 +
2147 + if(contexts && !simple_pattern_matches(contexts, string2str(rc->id)))
2148 + continue;
2149 +
2150 + RRDINSTANCE *ri;
2151 + dfe_start_read(rc->rrdinstances, ri) {
2152 + if(rrd_flag_is_deleted(ri))
2153 + continue;
2154 +
2155 + if(ri->rrdset && rrdset_is_ar_chart(ri->rrdset))
2156 + continue;
2157 +
2158 + RRDMETRIC *rm;
2159 + dfe_start_read(ri->rrdmetrics, rm) {
2160 + if(rrd_flag_is_deleted(rm))
2161 + continue;
2162 +
2163 + struct metric_entry tmp = {
2164 + .rca = (RRDCONTEXT_ACQUIRED *)rc_dfe.item,
2165 + .ria = (RRDINSTANCE_ACQUIRED *)ri_dfe.item,
2166 + .rma = (RRDMETRIC_ACQUIRED *)rm_dfe.item,
2167 + };
2168 +
2169 + char buffer[20 + 1];
2170 + ssize_t len = snprintfz(buffer, 20, "%p", rm);
2171 + dictionary_set_advanced(dict, buffer, len + 1, &tmp, sizeof(struct metric_entry), NULL);
2172 + }
2173 + dfe_done(rm);
2174 + }
2175 + dfe_done(ri);
2176 + }
2177 + dfe_done(rc);
2178 +
2179 + return dict;
2180 +}
2181 +
2182 +// ----------------------------------------------------------------------------
2183 +// query API
2184 +
2185 +typedef struct query_target_locals {
2186 + time_t start_s;
2187 +
2188 + QUERY_TARGET *qt;
2189 +
2190 + RRDSET *st;
2191 +
2192 + const char *hosts;
2193 + const char *contexts;
2194 + const char *charts;
2195 + const char *dimensions;
2196 + const char *chart_label_key;
2197 + const char *charts_labels_filter;
2198 +
2199 + long long after;
2200 + long long before;
2201 + bool match_ids;
2202 + bool match_names;
2203 +
2204 + RRDHOST *host;
2205 + RRDCONTEXT_ACQUIRED *rca;
2206 + RRDINSTANCE_ACQUIRED *ria;
2207 +
2208 + size_t metrics_skipped_due_to_not_matching_timeframe;
2209 +} QUERY_TARGET_LOCALS;
2210 +
2211 +static __thread QUERY_TARGET thread_query_target = {};
2212 +void query_target_release(QUERY_TARGET *qt) {
2213 + if(unlikely(!qt)) return;
2214 +
2215 + simple_pattern_free(qt->hosts.pattern);
2216 + qt->hosts.pattern = NULL;
2217 +
2218 + simple_pattern_free(qt->contexts.pattern);
2219 + qt->contexts.pattern = NULL;
2220 +
2221 + simple_pattern_free(qt->instances.pattern);
2222 + qt->instances.pattern = NULL;
2223 +
2224 + simple_pattern_free(qt->instances.chart_label_key_pattern);
2225 + qt->instances.chart_label_key_pattern = NULL;
2226 +
2227 + simple_pattern_free(qt->instances.charts_labels_filter_pattern);
2228 + qt->instances.charts_labels_filter_pattern = NULL;
2229 +
2230 + simple_pattern_free(qt->query.pattern);
2231 + qt->query.pattern = NULL;
2232 +
2233 + // release the query
2234 + for(size_t i = 0, used = qt->query.used; i < used ;i++) {
2235 + string_freez(qt->query.array[i].dimension.id);
2236 + qt->query.array[i].dimension.id = NULL;
2237 +
2238 + string_freez(qt->query.array[i].dimension.name);
2239 + qt->query.array[i].dimension.name = NULL;
2240 +
2241 + string_freez(qt->query.array[i].chart.id);
2242 + qt->query.array[i].chart.id = NULL;
2243 +
2244 + string_freez(qt->query.array[i].chart.name);
2245 + qt->query.array[i].chart.name = NULL;
2246 +
2247 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
2248 + if(qt->query.array[i].tiers[tier].db_metric_handle) {
2249 + STORAGE_ENGINE *eng = qt->query.array[i].tiers[tier].eng;
2250 + eng->api.metric_release(qt->query.array[i].tiers[tier].db_metric_handle);
2251 + qt->query.array[i].tiers[tier].db_metric_handle = NULL;
2252 + }
2253 + }
2254 + }
2255 +
2256 + // release the metrics
2257 + for(size_t i = 0, used = qt->metrics.used; i < used ;i++) {
2258 + rrdmetric_release(qt->metrics.array[i]);
2259 + qt->metrics.array[i] = NULL;
2260 + }
2261 +
2262 + // release the instances
2263 + for(size_t i = 0, used = qt->instances.used; i < used ;i++) {
2264 + rrdinstance_release(qt->instances.array[i]);
2265 + qt->instances.array[i] = NULL;
2266 + }
2267 +
2268 + // release the contexts
2269 + for(size_t i = 0, used = qt->contexts.used; i < used ;i++) {
2270 + rrdcontext_release(qt->contexts.array[i]);
2271 + qt->contexts.array[i] = NULL;
2272 + }
2273 +
2274 + // release the hosts
2275 + for(size_t i = 0, used = qt->hosts.used; i < used ;i++) {
2276 + qt->hosts.array[i] = NULL;
2277 + }
2278 +
2279 + qt->query.used = 0;
2280 + qt->metrics.used = 0;
2281 + qt->instances.used = 0;
2282 + qt->contexts.used = 0;
2283 + qt->hosts.used = 0;
2284 +
2285 + qt->db.minimum_latest_update_every = 0;
2286 + qt->db.first_time_t = 0;
2287 + qt->db.last_time_t = 0;
2288 +
2289 + qt->id[0] = '\0';
2290 +
2291 + qt->used = false;
2292 +}
2293 +void query_target_free(void) {
2294 + if(thread_query_target.used)
2295 + query_target_release(&thread_query_target);
2296 +
2297 + freez(thread_query_target.query.array);
2298 + thread_query_target.query.array = NULL;
2299 + thread_query_target.query.size = 0;
2300 +
2301 + freez(thread_query_target.metrics.array);
2302 + thread_query_target.metrics.array = NULL;
2303 + thread_query_target.metrics.size = 0;
2304 +
2305 + freez(thread_query_target.instances.array);
2306 + thread_query_target.instances.array = NULL;
2307 + thread_query_target.instances.size = 0;
2308 +
2309 + freez(thread_query_target.contexts.array);
2310 + thread_query_target.contexts.array = NULL;
2311 + thread_query_target.contexts.size = 0;
2312 +
2313 + freez(thread_query_target.hosts.array);
2314 + thread_query_target.hosts.array = NULL;
2315 + thread_query_target.hosts.size = 0;
2316 +}
2317 +
2318 +static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED *rma, RRDINSTANCE *ri, bool instance_matches_label_filters) {
2319 + QUERY_TARGET *qt = qtl->qt;
2320 +
2321 + RRDMETRIC *rm = rrdmetric_acquired_value(rma);
2322 + if(rrd_flag_is_deleted(rm))
2323 + return;
2324 +
2325 + if(qt->metrics.used == qt->metrics.size) {
2326 + qt->metrics.size = (qt->metrics.size) ? qt->metrics.size * 2 : 1;
2327 + qt->metrics.array = reallocz(qt->metrics.array, qt->metrics.size * sizeof(RRDMETRIC_ACQUIRED *));
2328 + }
2329 + qt->metrics.array[qt->metrics.used++] = rrdmetric_acquired_dup(rma);
2330 +
2331 + if(!instance_matches_label_filters)
2332 + return;
2333 +
2334 + time_t common_first_time_t = 0;
2335 + time_t common_last_time_t = 0;
2336 + time_t common_update_every = 0;
2337 + size_t tiers_added = 0;
2338 + struct {
2339 + STORAGE_ENGINE *eng;
2340 + STORAGE_METRIC_HANDLE *db_metric_handle;
2341 + time_t db_first_time_t;
2342 + time_t db_last_time_t;
2343 + time_t db_update_every;
2344 + } tier_retention[storage_tiers];
2345 +
2346 + for (size_t tier = 0; tier < storage_tiers; tier++) {
2347 + STORAGE_ENGINE *eng = qtl->host->db[tier].eng;
2348 + tier_retention[tier].eng = eng;
2349 + tier_retention[tier].db_update_every = (time_t) (qtl->host->db[tier].tier_grouping * ri->update_every);
2350 +
2351 + if(rm->rrddim && rm->rrddim->tiers[tier]->db_metric_handle)
2352 + tier_retention[tier].db_metric_handle = eng->api.metric_dup(rm->rrddim->tiers[tier]->db_metric_handle);
2353 + else
2354 + tier_retention[tier].db_metric_handle = eng->api.metric_get(qtl->host->db[tier].instance, &rm->uuid, NULL);
2355 +
2356 + if(tier_retention[tier].db_metric_handle) {
2357 + tier_retention[tier].db_first_time_t = tier_retention[tier].eng->api.query_ops.oldest_time(tier_retention[tier].db_metric_handle);
2358 + tier_retention[tier].db_last_time_t = tier_retention[tier].eng->api.query_ops.latest_time(tier_retention[tier].db_metric_handle);
2359 +
2360 + if(!common_first_time_t)
2361 + common_first_time_t = tier_retention[tier].db_first_time_t;
2362 + else
2363 + common_first_time_t = MIN(common_first_time_t, tier_retention[tier].db_first_time_t);
2364 +
2365 + if(!common_last_time_t)
2366 + common_last_time_t = tier_retention[tier].db_last_time_t;
2367 + else
2368 + common_last_time_t = MAX(common_last_time_t, tier_retention[tier].db_last_time_t);
2369 +
2370 + if(!common_update_every)
2371 + common_update_every = tier_retention[tier].db_update_every;
2372 + else
2373 + common_update_every = MIN(common_update_every, tier_retention[tier].db_update_every);
2374 +
2375 + tiers_added++;
2376 + }
2377 + else {
2378 + tier_retention[tier].db_first_time_t = 0;
2379 + tier_retention[tier].db_last_time_t = 0;
2380 + tier_retention[tier].db_update_every = 0;
2381 + }
2382 + }
2383 +
2384 + bool timeframe_matches = (tiers_added && common_first_time_t <= qt->window.before && common_last_time_t >= qt->window.after) ? true : false;
2385 +
2386 + if(timeframe_matches) {
2387 + RRDR_DIMENSION_FLAGS options = RRDR_DIMENSION_DEFAULT;
2388 +
2389 + if (rrd_flag_check(rm, RRD_FLAG_HIDDEN)
2390 + || (rm->rrddim && rrddim_option_check(rm->rrddim, RRDDIM_OPTION_HIDDEN))) {
2391 + options |= RRDR_DIMENSION_HIDDEN;
2392 + options &= ~RRDR_DIMENSION_SELECTED;
2393 + }
2394 +
2395 + if (qt->query.pattern) {
2396 + // we have a dimensions pattern
2397 + // lets see if this dimension is selected
2398 +
2399 + if ((qtl->match_ids && simple_pattern_matches(qt->query.pattern, string2str(rm->id)))
2400 + || (qtl->match_names && simple_pattern_matches(qt->query.pattern, string2str(rm->name)))
2401 + ) {
2402 + // it matches the pattern
2403 + options |= (RRDR_DIMENSION_SELECTED | RRDR_DIMENSION_NONZERO);
2404 + options &= ~RRDR_DIMENSION_HIDDEN;
2405 + }
2406 + else {
2407 + // it does not match the pattern
2408 + options |= RRDR_DIMENSION_HIDDEN;
2409 + options &= ~RRDR_DIMENSION_SELECTED;
2410 + }
2411 + }
2412 + else {
2413 + // we don't have a dimensions pattern
2414 + // so this is a selected dimension
2415 + // if it is not hidden
2416 + if(!(options & RRDR_DIMENSION_HIDDEN))
2417 + options |= RRDR_DIMENSION_SELECTED;
2418 + }
2419 +
2420 + if((options & RRDR_DIMENSION_HIDDEN) && (options & RRDR_DIMENSION_SELECTED))
2421 + options &= ~RRDR_DIMENSION_HIDDEN;
2422 +
2423 + if(!(options & RRDR_DIMENSION_HIDDEN) || (qt->request.options & RRDR_OPTION_PERCENTAGE)) {
2424 + // we have a non-hidden dimension
2425 + // let's add it to the query metrics
2426 +
2427 + if(ri->rrdset)
2428 + ri->rrdset->last_accessed_time = qtl->start_s;
2429 +
2430 + if (qt->query.used == qt->query.size) {
2431 + qt->query.size = (qt->query.size) ? qt->query.size * 2 : 1;
2432 + qt->query.array = reallocz(qt->query.array, qt->query.size * sizeof(QUERY_METRIC));
2433 + }
2434 + QUERY_METRIC *qm = &qt->query.array[qt->query.used++];
2435 +
2436 + qm->dimension.options = options;
2437 +
2438 + qm->link.host = qtl->host;
2439 + qm->link.rca = qtl->rca;
2440 + qm->link.ria = qtl->ria;
2441 + qm->link.rma = rma;
2442 +
2443 + qm->chart.id = string_dup(ri->id);
2444 + qm->chart.name = string_dup(ri->name);
2445 +
2446 + qm->dimension.id = string_dup(rm->id);
2447 + qm->dimension.name = string_dup(rm->name);
2448 +
2449 + if (!qt->db.first_time_t || common_first_time_t < qt->db.first_time_t)
2450 + qt->db.first_time_t = common_first_time_t;
2451 +
2452 + if (!qt->db.last_time_t || common_last_time_t > qt->db.last_time_t)
2453 + qt->db.last_time_t = common_last_time_t;
2454 +
2455 + for (size_t tier = 0; tier < storage_tiers; tier++) {
2456 + qm->tiers[tier].eng = tier_retention[tier].eng;
2457 + qm->tiers[tier].db_metric_handle = tier_retention[tier].db_metric_handle;
2458 + qm->tiers[tier].db_first_time_t = tier_retention[tier].db_first_time_t;
2459 + qm->tiers[tier].db_last_time_t = tier_retention[tier].db_last_time_t;
2460 + qm->tiers[tier].db_update_every = tier_retention[tier].db_update_every;
2461 + }
2462 + }
2463 + }
2464 + else {
2465 + qtl->metrics_skipped_due_to_not_matching_timeframe++;
2466 +
2467 + // cleanup anything we allocated to the retention we will not use
2468 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
2469 + if (tier_retention[tier].db_metric_handle)
2470 + tier_retention[tier].eng->api.metric_release(tier_retention[tier].db_metric_handle);
2471 + }
2472 + }
2473 +}
2474 +
2475 +static void query_target_add_instance(QUERY_TARGET_LOCALS *qtl, RRDINSTANCE_ACQUIRED *ria) {
2476 + QUERY_TARGET *qt = qtl->qt;
2477 +
2478 + RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
2479 + if(rrd_flag_is_deleted(ri))
2480 + return;
2481 +
2482 + if(qt->instances.used == qt->instances.size) {
2483 + qt->instances.size = (qt->instances.size) ? qt->instances.size * 2 : 1;
2484 + qt->instances.array = reallocz(qt->instances.array, qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *));
2485 + }
2486 +
2487 + qtl->ria = qt->instances.array[qt->instances.used++] = rrdinstance_acquired_dup(ria);
2488 +
2489 + if(qt->db.minimum_latest_update_every == 0 || ri->update_every < qt->db.minimum_latest_update_every)
2490 + qt->db.minimum_latest_update_every = ri->update_every;
2491 +
2492 + bool instance_matches_label_filters = true;
2493 + if ((qt->instances.chart_label_key_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.chart_label_key_pattern, ':')) ||
2494 + (qt->instances.charts_labels_filter_pattern && !rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, qt->instances.charts_labels_filter_pattern, ':')))
2495 + instance_matches_label_filters = false;
2496 +
2497 + size_t added = 0;
2498 +
2499 + if(unlikely(qt->request.rma)) {
2500 + query_target_add_metric(qtl, qt->request.rma, ri, instance_matches_label_filters);
2501 + added++;
2502 + }
2503 + else {
2504 + RRDMETRIC *rm;
2505 + dfe_start_read(ri->rrdmetrics, rm){
2506 + query_target_add_metric(qtl, (RRDMETRIC_ACQUIRED *) rm_dfe.item, ri,
2507 + instance_matches_label_filters);
2508 + added++;
2509 + }
2510 + dfe_done(rm);
2511 + }
2512 +
2513 + if(!added) {
2514 + qt->instances.used--;
2515 + rrdinstance_release(ria);
2516 + }
2517 +}
2518 +
2519 +static void query_target_add_context(QUERY_TARGET_LOCALS *qtl, RRDCONTEXT_ACQUIRED *rca) {
2520 + QUERY_TARGET *qt = qtl->qt;
2521 +
2522 + RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
2523 + if(rrd_flag_is_deleted(rc))
2524 + return;
2525 +
2526 + if(qt->contexts.used == qt->contexts.size) {
2527 + qt->contexts.size = (qt->contexts.size) ? qt->contexts.size * 2 : 1;
2528 + qt->contexts.array = reallocz(qt->contexts.array, qt->contexts.size * sizeof(RRDCONTEXT_ACQUIRED *));
2529 + }
2530 + qtl->rca = qt->contexts.array[qt->contexts.used++] = rrdcontext_acquired_dup(rca);
2531 +
2532 + size_t added = 0;
2533 + if(unlikely(qt->request.ria)) {
2534 + query_target_add_instance(qtl, qt->request.ria);
2535 + added++;
2536 + }
2537 + else if(unlikely(qtl->st && qtl->st->rrdcontext == rca && qtl->st->rrdinstance)) {
2538 + query_target_add_instance(qtl, qtl->st->rrdinstance);
2539 + added++;
2540 + }
2541 + else {
2542 + RRDINSTANCE *ri;
2543 + dfe_start_read(rc->rrdinstances, ri) {
2544 + if(!qt->instances.pattern
2545 + || (qtl->match_ids && simple_pattern_matches(qt->instances.pattern, string2str(ri->id)))
2546 + || (qtl->match_names && simple_pattern_matches(qt->instances.pattern, string2str(ri->name)))
2547 + ) {
2548 + query_target_add_instance(qtl, (RRDINSTANCE_ACQUIRED *)ri_dfe.item);
2549 + added++;
2550 + }
2551 + }
2552 + dfe_done(ri);
2553 + }
2554 +
2555 + if(!added) {
2556 + qt->contexts.used--;
2557 + rrdcontext_release(rca);
2558 + }
2559 +}
2560 +
2561 +static void query_target_add_host(QUERY_TARGET_LOCALS *qtl, RRDHOST *host) {
2562 + QUERY_TARGET *qt = qtl->qt;
2563 +
2564 + if(qt->hosts.used == qt->hosts.size) {
2565 + qt->hosts.size = (qt->hosts.size) ? qt->hosts.size * 2 : 1;
2566 + qt->hosts.array = reallocz(qt->hosts.array, qt->hosts.size * sizeof(RRDHOST *));
2567 + }
2568 + qtl->host = qt->hosts.array[qt->hosts.used++] = host;
2569 +
2570 + // is the chart given valid?
2571 + if(unlikely(qtl->st && (!qtl->st->rrdinstance || !qtl->st->rrdcontext))) {
2572 + error("QUERY TARGET: RRDSET '%s' given, because it is not linked to rrdcontext structures. Switching to context query.", rrdset_name(qtl->st));
2573 +
2574 + if(!is_valid_sp(qtl->charts))
2575 + qtl->charts = rrdset_name(qtl->st);
2576 +
2577 + qtl->st = NULL;
2578 + }
2579 +
2580 + size_t added = 0;
2581 + if(unlikely(qt->request.rca)) {
2582 + query_target_add_context(qtl, qt->request.rca);
2583 + added++;
2584 + }
2585 + else if(unlikely(qtl->st)) {
2586 + // single chart data queries
2587 + query_target_add_context(qtl, qtl->st->rrdcontext);
2588 + added++;
2589 + }
2590 + else {
2591 + // context pattern queries
2592 + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)qtl->host->rrdctx, qtl->contexts);
2593 + if(likely(rca)) {
2594 + // we found it!
2595 + query_target_add_context(qtl, rca);
2596 + rrdcontext_release(rca);
2597 + added++;
2598 + }
2599 + else {
2600 + // Probably it is a pattern, we need to search for it...
2601 + RRDCONTEXT *rc;
2602 + dfe_start_read((DICTIONARY *)qtl->host->rrdctx, rc) {
2603 + if(qt->contexts.pattern && !simple_pattern_matches(qt->contexts.pattern, string2str(rc->id)))
2604 + continue;
2605 +
2606 + query_target_add_context(qtl, (RRDCONTEXT_ACQUIRED *)rc_dfe.item);
2607 + added++;
2608 + }
2609 + dfe_done(rc);
2610 + }
2611 + }
2612 +
2613 + if(!added) {
2614 + qt->hosts.used--;
2615 + }
2616 +}
2617 +
2618 +void query_target_generate_name(QUERY_TARGET *qt) {
2619 + char options_buffer[100 + 1];
2620 + web_client_api_request_v1_data_options_to_string(options_buffer, 100, qt->request.options);
2621 +
2622 + char resampling_buffer[20 + 1] = "";
2623 + if(qt->request.resampling_time > 1)
2624 + snprintfz(resampling_buffer, 20, "/resampling:%ld", qt->request.resampling_time);
2625 +
2626 + char tier_buffer[20 + 1] = "";
2627 + if(qt->request.options & RRDR_OPTION_SELECTED_TIER)
2628 + snprintfz(tier_buffer, 20, "/tier:%zu", qt->request.tier);
2629 +
2630 + if(qt->request.st)
2631 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "chart://host:%s/instance:%s/dimensions:%s/after:%ld/before:%ld/points:%zu/group:%s%s/options:%s%s%s"
2632 + , rrdhost_hostname(qt->request.st->rrdhost)
2633 + , rrdset_name(qt->request.st)
2634 + , (qt->request.dimensions) ? qt->request.dimensions : "*"
2635 + , qt->request.after
2636 + , qt->request.before
2637 + , qt->request.points
2638 + , web_client_api_request_v1_data_group_to_string(qt->request.group_method)
2639 + , qt->request.group_options?qt->request.group_options:""
2640 + , options_buffer
2641 + , resampling_buffer
2642 + , tier_buffer
2643 + );
2644 + else if(qt->request.host && qt->request.rca && qt->request.ria && qt->request.rma)
2645 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "metric://host:%s/context:%s/instance:%s/dimension:%s/after:%ld/before:%ld/points:%zu/group:%s%s/options:%s%s%s"
2646 + , rrdhost_hostname(qt->request.host)
2647 + , rrdcontext_acquired_id(qt->request.rca)
2648 + , rrdinstance_acquired_id(qt->request.ria)
2649 + , rrdmetric_acquired_id(qt->request.rma)
2650 + , qt->request.after
2651 + , qt->request.before
2652 + , qt->request.points
2653 + , web_client_api_request_v1_data_group_to_string(qt->request.group_method)
2654 + , qt->request.group_options?qt->request.group_options:""
2655 + , options_buffer
2656 + , resampling_buffer
2657 + , tier_buffer
2658 + );
2659 + else
2660 + snprintfz(qt->id, MAX_QUERY_TARGET_ID_LENGTH, "context://host:%s/contexts:%s/instances:%s/dimensions:%s/after:%ld/before:%ld/points:%zu/group:%s%s/options:%s%s%s"
2661 + , (qt->request.host) ? rrdhost_hostname(qt->request.host) : ((qt->request.hosts) ? qt->request.hosts : "*")
2662 + , (qt->request.contexts) ? qt->request.contexts : "*"
2663 + , (qt->request.charts) ? qt->request.charts : "*"
2664 + , (qt->request.dimensions) ? qt->request.dimensions : "*"
2665 + , qt->request.after
2666 + , qt->request.before
2667 + , qt->request.points
2668 + , web_client_api_request_v1_data_group_to_string(qt->request.group_method)
2669 + , qt->request.group_options?qt->request.group_options:""
2670 + , options_buffer
2671 + , resampling_buffer
2672 + , tier_buffer
2673 + );
2674 +
2675 + json_fix_string(qt->id);
2676 +}
2677 +
2678 +QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
2679 + QUERY_TARGET *qt = &thread_query_target;
2680 +
2681 + if(qt->used)
2682 + fatal("QUERY TARGET: this query target is already used.");
2683 +
2684 + qt->used = true;
2685 +
2686 + // copy the request into query_thread_target
2687 + qt->request = *qtr;
2688 +
2689 + query_target_generate_name(qt);
2690 + qt->window.after = qt->request.after;
2691 + qt->window.before = qt->request.before;
2692 + rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before);
2693 +
2694 + // prepare our local variables - we need these across all these functions
2695 + QUERY_TARGET_LOCALS qtl = {
2696 + .qt = qt,
2697 + .start_s = now_realtime_sec(),
2698 + .host = qt->request.host,
2699 + .st = qt->request.st,
2700 + .hosts = qt->request.hosts,
2701 + .contexts = qt->request.contexts,
2702 + .charts = qt->request.charts,
2703 + .dimensions = qt->request.dimensions,
2704 + .chart_label_key = qt->request.chart_label_key,
2705 + .charts_labels_filter = qt->request.charts_labels_filter,
2706 + };
2707 +
2708 + qt->db.minimum_latest_update_every = 0; // it will be updated by query_target_add_query()
2709 +
2710 + // prepare all the patterns
2711 + qt->hosts.pattern = is_valid_sp(qtl.hosts) ? simple_pattern_create(qtl.hosts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2712 + qt->contexts.pattern = is_valid_sp(qtl.contexts) ? simple_pattern_create(qtl.contexts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2713 + qt->instances.pattern = is_valid_sp(qtl.charts) ? simple_pattern_create(qtl.charts, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2714 + qt->query.pattern = is_valid_sp(qtl.dimensions) ? simple_pattern_create(qtl.dimensions, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2715 + qt->instances.chart_label_key_pattern = is_valid_sp(qtl.chart_label_key) ? simple_pattern_create(qtl.chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2716 + qt->instances.charts_labels_filter_pattern = is_valid_sp(qtl.charts_labels_filter) ? simple_pattern_create(qtl.charts_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT) : NULL;
2717 +
2718 + qtl.match_ids = qt->request.options & RRDR_OPTION_MATCH_IDS;
2719 + qtl.match_names = qt->request.options & RRDR_OPTION_MATCH_NAMES;
2720 + if(likely(!qtl.match_ids && !qtl.match_names))
2721 + qtl.match_ids = qtl.match_names = true;
2722 +
2723 + // verify that the chart belongs to the host we are interested
2724 + if(qtl.st) {
2725 + if (!qtl.host) {
2726 + // It is NULL, set it ourselves.
2727 + qtl.host = qtl.st->rrdhost;
2728 + }
2729 + else if (unlikely(qtl.host != qtl.st->rrdhost)) {
2730 + // Oops! A different host!
2731 + error("QUERY TARGET: RRDSET '%s' given does not belong to host '%s'. Switching query host to '%s'",
2732 + rrdset_name(qtl.st), rrdhost_hostname(qtl.host), rrdhost_hostname(qtl.st->rrdhost));
2733 + qtl.host = qtl.st->rrdhost;
2734 + }
2735 + }
2736 +
2737 + if(qtl.host) {
2738 + // single host query
2739 + query_target_add_host(&qtl, qtl.host);
2740 + qtl.hosts = rrdhost_hostname(qtl.host);
2741 + }
2742 + else {
2743 + // multi host query
2744 + rrd_rdlock();
2745 + rrdhost_foreach_read(qtl.host) {
2746 + if(!qt->hosts.pattern || simple_pattern_matches(qt->hosts.pattern, rrdhost_hostname(qtl.host)))
2747 + query_target_add_host(&qtl, qtl.host);
2748 + }
2749 + rrd_unlock();
2750 + }
2751 +
2752 + // make sure everything is good
2753 + if(!qt->query.used || !qt->metrics.used || !qt->instances.used || !qt->contexts.used || !qt->hosts.used) {
2754 + internal_error(
2755 + true
2756 + , "QUERY TARGET: query '%s' does not have all the data required. "
2757 + "Matched %u hosts, %u contexts, %u instances, %u dimensions, %u metrics to query, "
2758 + "%zu metrics skipped because they don't have data in the desired time-frame. "
2759 + "Aborting it."
2760 + , qt->id
2761 + , qt->hosts.used
2762 + , qt->contexts.used
2763 + , qt->instances.used
2764 + , qt->metrics.used
2765 + , qt->query.used
2766 + , qtl.metrics_skipped_due_to_not_matching_timeframe
2767 + );
2768 +
2769 + query_target_release(qt);
2770 + return NULL;
2771 + }
2772 +
2773 + if(!query_target_calculate_window(qt)) {
2774 + query_target_release(qt);
2775 + return NULL;
2776 + }
2777 +
2778 + return qt;
2779 +}
2780 +
2781 +
2782 // ----------------------------------------------------------------------------
2783 // load from SQL
2784
@@ -2073,6 +2795,8 @@ static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
2795 .name = string_strdupz(sd->name),
2796 .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomic
2797 };
2798 + if(sd->hidden) trm.flags |= RRD_FLAG_HIDDEN;
2799 +
2800 uuid_copy(trm.uuid, sd->dim_id);
2801
2802 dictionary_set(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
@@ -2244,11 +2968,11 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) {
2968 #ifdef ENABLE_DBENGINE
2969 else if (dbengine_enabled) {
2970 RRDHOST *rrdhost = rm->ri->rc->rrdhost;
2247 - for (int tier = 0; tier < storage_tiers; tier++) {
2248 - if(!rrdhost->storage_instance[tier]) continue;
2971 + for (size_t tier = 0; tier < storage_tiers; tier++) {
2972 + if(!rrdhost->db[tier].instance) continue;
2973
2974 time_t first_time_t, last_time_t;
2251 - if (rrdeng_metric_retention_by_uuid(rrdhost->storage_instance[tier], &rm->uuid, &first_time_t, &last_time_t) == 0) {
2975 + if (rrdeng_metric_retention_by_uuid(rrdhost->db[tier].instance, &rm->uuid, &first_time_t, &last_time_t) == 0) {
2976 if (first_time_t < min_first_time_t)
2977 min_first_time_t = first_time_t;
2978
@@ -3045,7 +3769,7 @@ void *rrdcontext_main(void *ptr) {
3769 worker_register_job_name(WORKER_JOB_HOSTS, "hosts");
3770 worker_register_job_name(WORKER_JOB_CHECK, "dedup checks");
3771 worker_register_job_name(WORKER_JOB_SEND, "sent contexts");
3048 - worker_register_job_name(WORKER_JOB_DEQUEUE, "deduped contexts");
3772 + worker_register_job_name(WORKER_JOB_DEQUEUE, "deduplicated contexts");
3773 worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention");
3774 worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts");
3775 worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups");
database/rrdcontext.h
+155 -1
@@ -8,7 +8,6 @@
8
9 typedef struct rrdmetric_acquired RRDMETRIC_ACQUIRED;
10
11 -
11 // ----------------------------------------------------------------------------
12 // RRDINSTANCE
13
@@ -24,6 +23,15 @@ typedef struct rrdcontext_acquired RRDCONTEXT_ACQUIRED;
23
24 #include "rrd.h"
25
26 +const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma);
27 +const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma);
28 +NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma);
29 +
30 +const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria);
31 +const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria);
32 +DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria);
33 +DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria);
34 +
35 // ----------------------------------------------------------------------------
36 // public API for rrdhost
37
@@ -55,6 +63,11 @@ typedef enum {
63 int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions);
64 int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions);
65
66 +// ----------------------------------------------------------------------------
67 +// public API for rrdcontexts
68 +
69 +const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca);
70 +
71 // ----------------------------------------------------------------------------
72 // public API for rrddims
73
@@ -88,5 +101,146 @@ void rrdcontext_hub_stop_streaming_command(void *cmd);
101 void rrdcontext_db_rotation(void);
102 void *rrdcontext_main(void *);
103
104 +// ----------------------------------------------------------------------------
105 +// public API for weights
106 +
107 +struct metric_entry {
108 + RRDCONTEXT_ACQUIRED *rca;
109 + RRDINSTANCE_ACQUIRED *ria;
110 + RRDMETRIC_ACQUIRED *rma;
111 +};
112 +
113 +DICTIONARY *rrdcontext_all_metrics_to_dict(RRDHOST *host, SIMPLE_PATTERN *contexts);
114 +
115 +// ----------------------------------------------------------------------------
116 +// public API for queries
117 +
118 +typedef struct query_metric {
119 + struct query_metric_tier {
120 + struct storage_engine *eng;
121 + STORAGE_METRIC_HANDLE *db_metric_handle;
122 + time_t db_first_time_t; // the oldest timestamp available for this tier
123 + time_t db_last_time_t; // the latest timestamp available for this tier
124 + time_t db_update_every; // latest update every for this tier
125 + } tiers[RRD_STORAGE_TIERS];
126 +
127 + struct {
128 + RRDHOST *host;
129 + RRDCONTEXT_ACQUIRED *rca;
130 + RRDINSTANCE_ACQUIRED *ria;
131 + RRDMETRIC_ACQUIRED *rma;
132 + } link;
133 +
134 + struct {
135 + STRING *id;
136 + STRING *name;
137 + RRDR_DIMENSION_FLAGS options;
138 + } dimension;
139 +
140 + struct {
141 + STRING *id;
142 + STRING *name;
143 + } chart;
144 +
145 +} QUERY_METRIC;
146 +
147 +#define MAX_QUERY_TARGET_ID_LENGTH 255
148 +
149 +typedef struct query_target_request {
150 + RRDHOST *host; // the host to be queried (can be NULL, hosts will be used)
151 + RRDCONTEXT_ACQUIRED *rca; // the context to be queried (can be NULL)
152 + RRDINSTANCE_ACQUIRED *ria; // the instance to be queried (can be NULL)
153 + RRDMETRIC_ACQUIRED *rma; // the metric to be queried (can be NULL)
154 + RRDSET *st; // the chart to be queried (NULL, for context queries)
155 + const char *hosts; // hosts simple pattern
156 + const char *contexts; // contexts simple pattern (context queries)
157 + const char *charts; // charts simple pattern (for context queries)
158 + const char *dimensions; // dimensions simple pattern
159 + const char *chart_label_key; // select only the chart having this label key
160 + const char *charts_labels_filter; // select only the charts having this combo of label key:value
161 + time_t after; // the requested timeframe
162 + time_t before; // the requested timeframe
163 + size_t points; // the requested number of points
164 + time_t timeout; // the timeout of the query in seconds
165 + int max_anomaly_rates; // it only applies to anomaly rates chart - TODO - remove it
166 + uint32_t format; // DATASOURCE_FORMAT
167 + RRDR_OPTIONS options;
168 + RRDR_GROUPING group_method;
169 + const char *group_options;
170 + time_t resampling_time;
171 + size_t tier;
172 +} QUERY_TARGET_REQUEST;
173 +
174 +typedef struct query_target {
175 + char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging)
176 + QUERY_TARGET_REQUEST request;
177 +
178 + bool used; // when true, this query is currently being used
179 +
180 + struct {
181 + bool relative; // true when the request made with relative timestamps, true if it was absolute
182 + bool aligned;
183 + time_t after; // the absolute timestamp this query is about
184 + time_t before; // the absolute timestamp this query is about
185 + time_t query_granularity;
186 + size_t points; // the number of points the query will return (maybe different from the request)
187 + size_t group;
188 + RRDR_GROUPING group_method;
189 + const char *group_options;
190 + size_t resampling_group;
191 + NETDATA_DOUBLE resampling_divisor;
192 + RRDR_OPTIONS options;
193 + size_t tier;
194 + } window;
195 +
196 + struct {
197 + time_t first_time_t; // the combined first_time_t of all metrics in the query, across all tiers
198 + time_t last_time_t; // the combined last_time_T of all metrics in the query, across all tiers
199 + time_t minimum_latest_update_every; // the min update every of the metrics in the query
200 + } db;
201 +
202 + struct {
203 + QUERY_METRIC *array; // the metrics to be queried (all of them should be queried, no exceptions)
204 + uint32_t used; // how many items of the array are used
205 + uint32_t size; // the size of the array
206 + SIMPLE_PATTERN *pattern;
207 + } query;
208 +
209 + struct {
210 + RRDMETRIC_ACQUIRED **array;
211 + uint32_t used; // how many items of the array are used
212 + uint32_t size; // the size of the array
213 + } metrics;
214 +
215 + struct {
216 + RRDINSTANCE_ACQUIRED **array;
217 + uint32_t used; // how many items of the array are used
218 + uint32_t size; // the size of the array
219 + SIMPLE_PATTERN *pattern;
220 + SIMPLE_PATTERN *chart_label_key_pattern;
221 + SIMPLE_PATTERN *charts_labels_filter_pattern;
222 + } instances;
223 +
224 + struct {
225 + RRDCONTEXT_ACQUIRED **array;
226 + uint32_t used; // how many items of the array are used
227 + uint32_t size; // the size of the array
228 + SIMPLE_PATTERN *pattern;
229 + } contexts;
230 +
231 + struct {
232 + RRDHOST **array;
233 + uint32_t used; // how many items of the array are used
234 + uint32_t size; // the size of the array
235 + SIMPLE_PATTERN *pattern;
236 + } hosts;
237 +
238 +} QUERY_TARGET;
239 +
240 +void query_target_free(void);
241 +void query_target_release(QUERY_TARGET *qt);
242 +
243 +QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr);
244 +
245 #endif // NETDATA_RRDCONTEXT_H
246
database/rrddim.c
+30 -46
@@ -82,10 +82,6 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
82
83 rd->rrd_memory_mode = ctr->memory_mode;
84
85 -#ifdef ENABLE_ACLK
86 - rd->aclk_live_status = -1;
87 -#endif
88 -
85 if (unlikely(find_dimension_uuid(st, rd, &(rd->metric_uuid)))) {
86 uuid_generate(rd->metric_uuid);
87 }
@@ -93,17 +89,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
89 // initialize the db tiers
90 {
91 size_t initialized = 0;
96 - RRD_MEMORY_MODE wanted_mode = ctr->memory_mode;
97 - for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
98 - STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
99 - if(!eng) continue;
100 -
92 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
93 + STORAGE_ENGINE *eng = host->db[tier].eng;
94 rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier));
102 - rd->tiers[tier]->tier_grouping = get_tier_grouping(tier);
103 - rd->tiers[tier]->mode = eng->id;
104 - rd->tiers[tier]->collect_ops = eng->api.collect_ops;
105 - rd->tiers[tier]->query_ops = eng->api.query_ops;
106 - rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->storage_instance[tier], rd->rrdset->storage_metrics_groups[tier]);
95 + rd->tiers[tier]->tier_grouping = host->db[tier].tier_grouping;
96 + rd->tiers[tier]->collect_ops = &eng->api.collect_ops;
97 + rd->tiers[tier]->query_ops = &eng->api.query_ops;
98 + rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance, rd->rrdset->storage_metrics_groups[tier]);
99 storage_point_unset(rd->tiers[tier]->virtual_point);
100 initialized++;
101
@@ -120,9 +112,9 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
112 // initialize data collection for all tiers
113 {
114 size_t initialized = 0;
123 - for (int tier = 0; tier < storage_tiers; tier++) {
115 + for (size_t tier = 0; tier < storage_tiers; tier++) {
116 if (rd->tiers[tier]) {
125 - rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle, get_tier_grouping(tier) * st->update_every);
117 + rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every);
118 initialized++;
119 }
120 }
@@ -131,10 +123,14 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
123 error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
124 }
125
134 - if(st->dimensions) {
135 - RRDDIM *td = st->dimensions;
126 + if(rrdset_number_of_dimensions(st) != 0) {
127 + RRDDIM *td;
128 + dfe_start_write(st->rrddim_root_index, td) {
129 + if(!td) break;
130 + }
131 + dfe_done(td);
132
137 - if(td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor)) {
133 + if(td && (td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor))) {
134 if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
135 #ifdef NETDATA_INTERNAL_CHECKS
136 info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",
@@ -172,10 +168,11 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
168
169 static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *rrdset) {
170 RRDDIM *rd = rrddim;
175 - RRDSET *st = rrdset; (void)st;
171 + RRDSET *st = rrdset;
172 + RRDHOST *host = st->rrdhost;
173
174 internal_error(false, "RRDDIM: deleting dimension '%s' of chart '%s' of host '%s'",
178 - rrddim_name(rd), rrdset_name(st), rrdhost_hostname(st->rrdhost));
175 + rrddim_name(rd), rrdset_name(st), rrdhost_hostname(host));
176
177 rrdcontext_removed_rrddim(rd);
178
@@ -186,11 +183,11 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
183 if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
184
185 size_t tiers_available = 0, tiers_said_yes = 0;
189 - for(int tier = 0; tier < storage_tiers ;tier++) {
186 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
187 if(rd->tiers[tier]) {
188 tiers_available++;
189
193 - if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle))
190 + if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
191 tiers_said_yes++;
192
193 rd->tiers[tier]->db_collection_handle = NULL;
@@ -214,12 +211,11 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
211 // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
212 rrddim_memory_file_free(rd);
213
217 - for(int tier = 0; tier < storage_tiers ;tier++) {
214 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
215 if(!rd->tiers[tier]) continue;
216
220 - STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode);
221 - if(eng)
222 - eng->api.metric_release(rd->tiers[tier]->db_metric_handle);
217 + STORAGE_ENGINE* eng = host->db[tier].eng;
218 + eng->api.metric_release(rd->tiers[tier]->db_metric_handle);
219
220 freez(rd->tiers[tier]);
221 rd->tiers[tier] = NULL;
@@ -252,10 +248,10 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
248
249 if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
250
255 - for(int tier = 0; tier < storage_tiers ;tier++) {
251 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
252 if (rd->tiers[tier])
253 rd->tiers[tier]->db_collection_handle =
258 - rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle, get_tier_grouping(tier) * st->update_every);
254 + rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every);
255 }
256
257 rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
@@ -399,12 +395,12 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
395
396 // get the timestamp of the last entry in the round-robin database
397 time_t rrddim_last_entry_t(RRDDIM *rd) {
402 - time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
398 + time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
399
404 - for(int tier = 1; tier < storage_tiers ;tier++) {
400 + for(size_t tier = 1; tier < storage_tiers ;tier++) {
401 if(unlikely(!rd->tiers[tier])) continue;
402
407 - time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
403 + time_t t = rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle);
404 if(t > latest)
405 latest = t;
406 }
@@ -415,10 +411,10 @@ time_t rrddim_last_entry_t(RRDDIM *rd) {
411 time_t rrddim_first_entry_t(RRDDIM *rd) {
412 time_t oldest = 0;
413
418 - for(int tier = 0; tier < storage_tiers ;tier++) {
414 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
415 if(unlikely(!rd->tiers[tier])) continue;
416
421 - time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
417 + time_t t = rd->tiers[tier]->query_ops->oldest_time(rd->tiers[tier]->db_metric_handle);
418 if(t != 0 && (oldest == 0 || t < oldest))
419 oldest = t;
420 }
@@ -445,14 +441,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st
441 };
442
443 RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, sizeof(RRDDIM), &tmp);
448 -
449 - if(tmp.react_action == RRDDIM_REACT_NEW) {
450 - // append this dimension
451 - rrdset_wrlock(st);
452 - DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->dimensions, rd, prev, next);
453 - rrdset_unlock(st);
454 - }
455 -
444 return(rd);
445 }
446
@@ -460,10 +448,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st
448 // RRDDIM remove / free a dimension
449
450 void rrddim_free(RRDSET *st, RRDDIM *rd) {
463 - rrdset_wrlock(st);
464 - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->dimensions, rd, prev, next);
465 - rrdset_unlock(st);
466 -
451 dictionary_del(st->rrddim_root_index, string2str(rd->id));
452 }
453
database/rrdfunctions.c
+4 -4
@@ -745,14 +745,14 @@ void host_functions2json(RRDHOST *host, BUFFER *wb, int tabs, const char *kq, co
745 functions2json(host->functions, wb, ident, kq, sq);
746 }
747
748 -void chart_functions_to_dict(RRDSET *st, DICTIONARY *dict) {
749 - if(!st || !st->functions_view) return;
748 +void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst) {
749 + if(!rrdset_functions_view || !dst) return;
750
751 struct rrd_collector_function *t;
752 - dfe_start_read(st->functions_view, t) {
752 + dfe_start_read(rrdset_functions_view, t) {
753 if(!t->collector->running) continue;
754
755 - dictionary_set(dict, t_dfe.name, NULL, 0);
755 + dictionary_set(dst, t_dfe.name, NULL, 0);
756 }
757 dfe_done(t);
758 }
database/rrdfunctions.h
+1 -1
@@ -25,7 +25,7 @@ int rrd_call_function_async(RRDHOST *host, BUFFER *wb, int timeout, const char *
25 void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb);
26
27 void chart_functions2json(RRDSET *st, BUFFER *wb, int tabs, const char *kq, const char *sq);
28 -void chart_functions_to_dict(RRDSET *st, DICTIONARY *dict);
28 +void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst);
29 void host_functions2json(RRDHOST *host, BUFFER *wb, int tabs, const char *kq, const char *sq);
30
31 uint8_t functions_format_to_content_type(const char *format);
database/rrdhost.c
+59 -39
@@ -4,21 +4,20 @@
4 #include "rrd.h"
5
6 bool dbengine_enabled = false; // will become true if and when dbengine is initialized
7 -int storage_tiers = 1;
8 -int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
7 +size_t storage_tiers = 1;
8 +size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
9 RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
10
11 #if RRD_STORAGE_TIERS != 5
12 #error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier
13 #endif
14
15 -int get_tier_grouping(int tier) {
15 +size_t get_tier_grouping(size_t tier) {
16 if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1;
17 - if(unlikely(tier < 0)) tier = 0;
17
19 - int grouping = 1;
18 + size_t grouping = 1;
19 // first tier is always 1 iteration of whatever update every the chart has
21 - for(int i = 1; i <= tier ;i++)
20 + for(size_t i = 1; i <= tier ;i++)
21 grouping *= storage_tiers_grouping_iterations[i];
22
23 return grouping;
@@ -33,7 +32,7 @@ time_t rrdhost_free_orphan_time = 3600;
32
33 bool is_storage_engine_shared(STORAGE_INSTANCE *engine) {
34 #ifdef ENABLE_DBENGINE
36 - for(int tier = 0; tier < storage_tiers ;tier++) {
35 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
36 if (engine == (STORAGE_INSTANCE *)multidb_ctx[tier])
37 return true;
38 }
@@ -367,9 +366,13 @@ int is_legacy = 1;
366 if (is_legacy) {
367 // initialize legacy dbengine instance as needed
368
369 + host->db[0].mode = RRD_MEMORY_MODE_DBENGINE;
370 + host->db[0].eng = storage_engine_get(host->db[0].mode);
371 + host->db[0].tier_grouping = get_tier_grouping(0);
372 +
373 ret = rrdeng_init(
374 host,
372 - (struct rrdengine_instance **)&host->storage_instance[0],
375 + (struct rrdengine_instance **)&host->db[0].instance,
376 dbenginepath,
377 default_rrdeng_page_cache_mb,
378 default_rrdeng_disk_quota_mb,
@@ -378,13 +381,21 @@ int is_legacy = 1;
381 if(ret == 0) {
382 // assign the rest of the shared storage instances to it
383 // to allow them collect its metrics too
381 - for(int tier = 1; tier < storage_tiers ; tier++)
382 - host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
384 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
385 + host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
386 + host->db[tier].eng = storage_engine_get(host->db[tier].mode);
387 + host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
388 + host->db[tier].tier_grouping = get_tier_grouping(tier);
389 + }
390 }
391 }
392 else {
386 - for(int tier = 0; tier < storage_tiers ; tier++)
387 - host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
393 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
394 + host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
395 + host->db[tier].eng = storage_engine_get(host->db[tier].mode);
396 + host->db[tier].instance = (STORAGE_INSTANCE *)multidb_ctx[tier];
397 + host->db[tier].tier_grouping = get_tier_grouping(tier);
398 + }
399 }
400 if (ret) { // check legacy or multihost initialization success
401 error(
@@ -402,10 +413,19 @@ int is_legacy = 1;
413 #endif
414 }
415 else {
416 + host->db[0].mode = host->rrd_memory_mode;
417 + host->db[0].eng = storage_engine_get(host->db[0].mode);
418 + host->db[0].instance = NULL;
419 + host->db[0].tier_grouping = get_tier_grouping(0);
420 +
421 #ifdef ENABLE_DBENGINE
422 // the first tier is reserved for the non-dbengine modes
407 - for(int tier = 1; tier < storage_tiers ; tier++)
408 - host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
423 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
424 + host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
425 + host->db[tier].eng = storage_engine_get(host->db[tier].mode);
426 + host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
427 + host->db[tier].tier_grouping = get_tier_grouping(tier);
428 + }
429 #endif
430 }
431
@@ -725,14 +745,14 @@ void dbengine_init(char *hostname) {
745 else
746 rrdeng_page_descr_use_malloc();
747
728 - int created_tiers = 0;
748 + size_t created_tiers = 0;
749 char dbenginepath[FILENAME_MAX + 1];
750 char dbengineconfig[200 + 1];
731 - for(int tier = 0; tier < storage_tiers ;tier++) {
751 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
752 if(tier == 0)
753 snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
754 else
735 - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%d", netdata_configured_cache_dir, tier);
755 + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
756
757 int ret = mkdir(dbenginepath, 0775);
758 if (ret != 0 && errno != EEXIST) {
@@ -742,25 +762,25 @@ void dbengine_init(char *hostname) {
762
763 int page_cache_mb = default_rrdeng_page_cache_mb;
764 int disk_space_mb = default_multidb_disk_quota_mb;
745 - int grouping_iterations = storage_tiers_grouping_iterations[tier];
765 + size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
766 RRD_BACKFILL backfill = storage_tiers_backfill[tier];
767
768 if(tier > 0) {
749 - snprintfz(dbengineconfig, 200, "dbengine tier %d page cache size MB", tier);
769 + snprintfz(dbengineconfig, 200, "dbengine tier %zu page cache size MB", tier);
770 page_cache_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, page_cache_mb);
771
752 - snprintfz(dbengineconfig, 200, "dbengine tier %d multihost disk space MB", tier);
772 + snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
773 disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
774
755 - snprintfz(dbengineconfig, 200, "dbengine tier %d update every iterations", tier);
775 + snprintfz(dbengineconfig, 200, "dbengine tier %zu update every iterations", tier);
776 grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
777 if(grouping_iterations < 2) {
778 grouping_iterations = 2;
779 config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
760 - error("DBENGINE on '%s': 'dbegnine tier %d update every iterations' cannot be less than 2. Assuming 2.", hostname, tier);
780 + error("DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.", hostname, tier);
781 }
782
763 - snprintfz(dbengineconfig, 200, "dbengine tier %d backfill", tier);
783 + snprintfz(dbengineconfig, 200, "dbengine tier %zu backfill", tier);
784 const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
785 if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
786 else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
@@ -777,14 +797,14 @@ void dbengine_init(char *hostname) {
797
798 if(tier > 0 && get_tier_grouping(tier) > 65535) {
799 storage_tiers_grouping_iterations[tier] = 1;
780 - error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", hostname, tier, tier);
800 + error("DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. Disabling tiers above %zu", hostname, tier, tier);
801 break;
802 }
803
784 - internal_error(true, "DBENGINE tier %d grouping iterations is set to %d", tier, storage_tiers_grouping_iterations[tier]);
804 + internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);
805 ret = rrdeng_init(NULL, NULL, dbenginepath, page_cache_mb, disk_space_mb, tier);
806 if(ret != 0) {
787 - error("DBENGINE on '%s': Failed to initialize multi-host database tier %d on path '%s'",
807 + error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
808 hostname, tier, dbenginepath);
809 break;
810 }
@@ -793,7 +813,7 @@ void dbengine_init(char *hostname) {
813 }
814
815 if(created_tiers && created_tiers < storage_tiers) {
796 - error("DBENGINE on '%s': Managed to create %d tiers instead of %d. Continuing with %d available.",
816 + error("DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.",
817 hostname, created_tiers, storage_tiers, created_tiers);
818 storage_tiers = created_tiers;
819 }
@@ -840,7 +860,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
860
861 if(!dbengine_enabled) {
862 if (storage_tiers > 1) {
843 - error("dbengine is not enabled, but %d tiers have been requested. Resetting tiers to 1", storage_tiers);
863 + error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1", storage_tiers);
864 storage_tiers = 1;
865 }
866
@@ -1036,11 +1056,11 @@ void rrdhost_free(RRDHOST *host, bool force) {
1056 // release its children resources
1057
1058 #ifdef ENABLE_DBENGINE
1039 - for(int tier = 0; tier < storage_tiers ;tier++) {
1040 - if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
1041 - host->storage_instance[tier] &&
1042 - !is_storage_engine_shared(host->storage_instance[tier]))
1043 - rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[tier]);
1059 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
1060 + if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
1061 + && host->db[tier].instance
1062 + && !is_storage_engine_shared(host->db[tier].instance))
1063 + rrdeng_prepare_exit((struct rrdengine_instance *)host->db[tier].instance);
1064 }
1065 #endif
1066
@@ -1055,11 +1075,11 @@ void rrdhost_free(RRDHOST *host, bool force) {
1075 health_alarm_log_free(host);
1076
1077 #ifdef ENABLE_DBENGINE
1058 - for(int tier = 0; tier < storage_tiers ;tier++) {
1059 - if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
1060 - host->storage_instance[tier] &&
1061 - !is_storage_engine_shared(host->storage_instance[tier]))
1062 - rrdeng_exit((struct rrdengine_instance *)host->storage_instance[tier]);
1078 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
1079 + if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
1080 + && host->db[tier].instance
1081 + && !is_storage_engine_shared(host->db[tier].instance))
1082 + rrdeng_exit((struct rrdengine_instance *)host->db[tier].instance);
1083 }
1084 #endif
1085
@@ -1400,7 +1420,7 @@ void rrdhost_cleanup_all(void) {
1420 rrdhost_foreach_read(host) {
1421 if (host != localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver
1422 /* don't delete multi-host DB host files */
1403 - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0]))
1423 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))
1424 )
1425 rrdhost_delete_charts(host);
1426 else
database/rrdlabels.c
+5
@@ -548,6 +548,9 @@ void rrdlabels_destroy(DICTIONARY *labels_dict) {
548 dictionary_destroy(labels_dict);
549 }
550
551 +void rrdlabels_flush(DICTIONARY *labels_dict) {
552 + dictionary_flush(labels_dict);
553 +}
554
555 // ----------------------------------------------------------------------------
556 // rrdlabels_add()
@@ -633,6 +636,8 @@ void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) {
636 // rrdlabels_get_value_to_buffer_or_null()
637
638 void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null) {
639 + if(!labels) return;
640 +
641 const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
642 RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
643
database/rrdset.c
+16 -40
@@ -5,25 +5,6 @@
5 #include <sched.h>
6 #include "storage_engine.h"
7
8 -// ----------------------------------------------------------------------------
9 -
10 -void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
11 - debug(D_RRD_CALLS, "Checking read lock on chart '%s'", rrdset_id(st));
12 -
13 - int ret = netdata_rwlock_trywrlock(&st->rrdset_rwlock);
14 - if(ret == 0)
15 - fatal("RRDSET '%s' should be read-locked, but it is not, at function %s() at line %lu of file '%s'", rrdset_id(st), function, line, file);
16 -}
17 -
18 -void __rrdset_check_wrlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
19 - debug(D_RRD_CALLS, "Checking write lock on chart '%s'", rrdset_id(st));
20 -
21 - int ret = netdata_rwlock_tryrdlock(&st->rrdset_rwlock);
22 - if(ret == 0)
23 - fatal("RRDSET '%s' should be write-locked, but it is not, at function %s() at line %lu of file '%s'", rrdset_id(st), function, line, file);
24 -}
25 -
26 -
8 // ----------------------------------------------------------------------------
9 // RRDSET name index
10
@@ -163,7 +144,6 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
144 if(unlikely(st->id == anomaly_rates_chart))
145 st->flags |= RRDSET_FLAG_ANOMALY_RATE_CHART;
146
166 - netdata_rwlock_init(&st->rrdset_rwlock);
147 netdata_rwlock_init(&st->alerts.rwlock);
148
149 if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
@@ -175,12 +155,11 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
155
156 // initialize the db tiers
157 {
178 - RRD_MEMORY_MODE wanted_mode = ctr->memory_mode;
179 - for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
180 - STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
158 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
159 + STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
160 if(!eng) continue;
161
183 - st->storage_metrics_groups[tier] = eng->api.collect_ops.metrics_group_get(host->storage_instance[tier], &st->chart_uuid);
162 + st->storage_metrics_groups[tier] = eng->api.collect_ops.metrics_group_get(host->db[tier].instance, &st->chart_uuid);
163 }
164 }
165
@@ -213,12 +192,11 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
192
193 // cleanup storage engines
194 {
216 - RRD_MEMORY_MODE wanted_mode = st->rrd_memory_mode;
217 - for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
218 - STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
195 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
196 + STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
197 if(!eng) continue;
198
221 - eng->api.collect_ops.metrics_group_release(host->storage_instance[tier], st->storage_metrics_groups[tier]);
199 + eng->api.collect_ops.metrics_group_release(host->db[tier].instance, st->storage_metrics_groups[tier]);
200 }
201 }
202
@@ -262,7 +240,6 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
240 // ------------------------------------------------------------------------
241 // free it
242
265 - netdata_rwlock_destroy(&st->rrdset_rwlock);
243 netdata_rwlock_destroy(&st->alerts.rwlock);
244
245 string_freez(st->id);
@@ -313,9 +290,9 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
290 // switch update every to the storage engine
291 RRDDIM *rd;
292 rrddim_foreach_read(rd, st) {
316 - for (int tier = 0; tier < storage_tiers; tier++) {
293 + for (size_t tier = 0; tier < storage_tiers; tier++) {
294 if (rd->tiers[tier] && rd->tiers[tier]->db_collection_handle)
318 - rd->tiers[tier]->collect_ops.change_collection_frequency(rd->tiers[tier]->db_collection_handle, get_tier_grouping(tier) * st->update_every);
295 + rd->tiers[tier]->collect_ops->change_collection_frequency(rd->tiers[tier]->db_collection_handle, (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
296 }
297 }
298 rrddim_foreach_done(rd);
@@ -652,7 +629,6 @@ void rrdset_reset(RRDSET *st) {
629 st->current_entry = 0;
630 st->counter = 0;
631 st->counter_done = 0;
655 - st->rrddim_page_alignment = 0;
632
633 RRDDIM *rd;
634 rrddim_foreach_read(rd, st) {
@@ -661,9 +637,9 @@ void rrdset_reset(RRDSET *st) {
637 rd->collections_counter = 0;
638
639 if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
664 - for(int tier = 0; tier < storage_tiers ;tier++) {
640 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
641 if(rd->tiers[tier])
666 - rd->tiers[tier]->collect_ops.flush(rd->tiers[tier]->db_collection_handle);
642 + rd->tiers[tier]->collect_ops->flush(rd->tiers[tier]->db_collection_handle);
643 }
644 }
645 }
@@ -1043,7 +1019,7 @@ void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, u
1019 if(unlikely(sp.start_time > t->next_point_time)) {
1020 if (likely(!storage_point_is_unset(t->virtual_point))) {
1021
1046 - t->collect_ops.store_metric(
1022 + t->collect_ops->store_metric(
1023 t->db_collection_handle,
1024 t->next_point_time * USEC_PER_SEC,
1025 t->virtual_point.sum,
@@ -1054,7 +1030,7 @@ void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, u
1030 t->virtual_point.flags);
1031 }
1032 else {
1057 - t->collect_ops.store_metric(
1033 + t->collect_ops->store_metric(
1034 t->db_collection_handle,
1035 t->next_point_time * USEC_PER_SEC,
1036 NAN,
@@ -1097,7 +1073,7 @@ void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, u
1073
1074 void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
1075 // store the metric on tier 0
1100 - rd->tiers[0]->collect_ops.store_metric(rd->tiers[0]->db_collection_handle, point_end_time_ut, n, 0, 0, 1, 0, flags);
1076 + rd->tiers[0]->collect_ops->store_metric(rd->tiers[0]->db_collection_handle, point_end_time_ut, n, 0, 0, 1, 0, flags);
1077
1078 time_t now = (time_t)(point_end_time_ut / USEC_PER_SEC);
1079
@@ -1112,18 +1088,18 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1088 .flags = flags
1089 };
1090
1115 - for(int tier = 1; tier < storage_tiers ;tier++) {
1091 + for(size_t tier = 1; tier < storage_tiers ;tier++) {
1092 if(unlikely(!rd->tiers[tier])) continue;
1093
1094 struct rrddim_tier *t = rd->tiers[tier];
1095
1120 - if(!t->last_collected_ut) {
1096 + if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) {
1097 // we have not collected this tier before
1098 // let's fill any gap that may exist
1099 rrdr_fill_tier_gap_from_smaller_tiers(rd, tier, now);
1100 + rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS);
1101 }
1102
1126 - t->last_collected_ut = point_end_time_ut;
1103 store_metric_at_tier(rd, t, sp, point_end_time_ut);
1104 }
1105
database/sqlite/sqlite_context.c
+3 -1
@@ -172,7 +172,8 @@ skip_load:
172 }
173
174 // Dimension list
175 -#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name FROM meta.dimension d WHERE d.chart_id = @id and d.dim_id is not null;"
175 +#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \
176 + "FROM meta.dimension d WHERE d.chart_id = @id and d.dim_id is not null ORDER BY d.rowid ASC;"
177 void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)
178 {
179 int rc;
@@ -198,6 +199,7 @@ void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DA
199 uuid_copy(dimension_data.dim_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
200 dimension_data.id = (char *) sqlite3_column_text(res, 1);
201 dimension_data.name = (char *) sqlite3_column_text(res, 2);
202 + dimension_data.hidden = sqlite3_column_int(res, 3);
203 dict_cb(&dimension_data, data);
204 }
205
database/sqlite/sqlite_context.h
+1
@@ -24,6 +24,7 @@ typedef struct ctx_dimension {
24 uuid_t dim_id;
25 char *id;
26 char *name;
27 + bool hidden;
28 } SQL_DIMENSION_DATA;
29
30 typedef struct ctx_label {
database/sqlite/sqlite_metadata.c
+4 -2
@@ -664,7 +664,7 @@ static bool dimension_can_be_deleted(uuid_t *dim_uuid)
664 {
665 #ifdef ENABLE_DBENGINE
666 bool no_retention = true;
667 - for (int tier = 0; tier < storage_tiers; tier++) {
667 + for (size_t tier = 0; tier < storage_tiers; tier++) {
668 if (!multidb_ctx[tier])
669 continue;
670 time_t first_time_t = 0, last_time_t = 0;
@@ -1320,7 +1320,7 @@ error_after_loop_init:
1320 worker_unregister();
1321 }
1322
1323 -struct metadata_wc metasync_worker;
1323 +struct metadata_wc metasync_worker = {.loop = NULL};
1324
1325 void metadata_sync_shutdown(void)
1326 {
@@ -1447,6 +1447,8 @@ void metaqueue_host_update_info(const char *machine_guid)
1447
1448 void metaqueue_delete_dimension_uuid(uuid_t *uuid)
1449 {
1450 + if (unlikely(!metasync_worker.loop))
1451 + return;
1452 uuid_t *use_uuid = mallocz(sizeof(*uuid));
1453 uuid_copy(*use_uuid, *uuid);
1454 queue_metadata_cmd(METADATA_DEL_DIMENSION, use_uuid, NULL);
database/storage_engine.c
+6 -12
@@ -32,9 +32,8 @@ static STORAGE_ENGINE engines[] = {
32 .api = {
33 .metric_get = rrddim_metric_get,
34 .metric_get_or_create = rrddim_metric_get_or_create,
35 + .metric_dup = rrddim_metric_dup,
36 .metric_release = rrddim_metric_release,
36 - .group_get = rrddim_metrics_group_get,
37 - .group_release = rrddim_metrics_group_release,
37 .collect_ops = im_collect_ops,
38 .query_ops = im_query_ops
39 }
@@ -45,9 +44,8 @@ static STORAGE_ENGINE engines[] = {
44 .api = {
45 .metric_get = rrddim_metric_get,
46 .metric_get_or_create = rrddim_metric_get_or_create,
47 + .metric_dup = rrddim_metric_dup,
48 .metric_release = rrddim_metric_release,
49 - .group_get = rrddim_metrics_group_get,
50 - .group_release = rrddim_metrics_group_release,
49 .collect_ops = im_collect_ops,
50 .query_ops = im_query_ops
51 }
@@ -58,9 +56,8 @@ static STORAGE_ENGINE engines[] = {
56 .api = {
57 .metric_get = rrddim_metric_get,
58 .metric_get_or_create = rrddim_metric_get_or_create,
59 + .metric_dup = rrddim_metric_dup,
60 .metric_release = rrddim_metric_release,
62 - .group_get = rrddim_metrics_group_get,
63 - .group_release = rrddim_metrics_group_release,
61 .collect_ops = im_collect_ops,
62 .query_ops = im_query_ops
63 }
@@ -71,9 +68,8 @@ static STORAGE_ENGINE engines[] = {
68 .api = {
69 .metric_get = rrddim_metric_get,
70 .metric_get_or_create = rrddim_metric_get_or_create,
71 + .metric_dup = rrddim_metric_dup,
72 .metric_release = rrddim_metric_release,
75 - .group_get = rrddim_metrics_group_get,
76 - .group_release = rrddim_metrics_group_release,
73 .collect_ops = im_collect_ops,
74 .query_ops = im_query_ops
75 }
@@ -84,9 +80,8 @@ static STORAGE_ENGINE engines[] = {
80 .api = {
81 .metric_get = rrddim_metric_get,
82 .metric_get_or_create = rrddim_metric_get_or_create,
83 + .metric_dup = rrddim_metric_dup,
84 .metric_release = rrddim_metric_release,
88 - .group_get = rrddim_metrics_group_get,
89 - .group_release = rrddim_metrics_group_release,
85 .collect_ops = im_collect_ops,
86 .query_ops = im_query_ops
87 }
@@ -98,9 +93,8 @@ static STORAGE_ENGINE engines[] = {
93 .api = {
94 .metric_get = rrdeng_metric_get,
95 .metric_get_or_create = rrdeng_metric_get_or_create,
96 + .metric_dup = rrdeng_metric_dup,
97 .metric_release = rrdeng_metric_release,
102 - .group_get = rrdeng_metrics_group_get,
103 - .group_release = rrdeng_metrics_group_release,
98 .collect_ops = {
99 .init = rrdeng_store_metric_init,
100 .store_metric = rrdeng_store_metric_next,
database/storage_engine.h
-28
@@ -5,34 +5,6 @@
5
6 #include "rrd.h"
7
8 -typedef struct storage_engine STORAGE_ENGINE;
9 -
10 -// ------------------------------------------------------------------------
11 -// function pointers for all APIs provided by a storge engine
12 -typedef struct storage_engine_api {
13 - // metric management
14 - STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
15 - STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *instance, STORAGE_METRICS_GROUP *smg);
16 - void (*metric_release)(STORAGE_METRIC_HANDLE *);
17 -
18 - // metrics groups management
19 - STORAGE_METRICS_GROUP *(*group_get)(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
20 - void (*group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
21 -
22 - // operations
23 - struct rrddim_collect_ops collect_ops;
24 - struct rrddim_query_ops query_ops;
25 -} STORAGE_ENGINE_API;
26 -
27 -struct storage_engine {
28 - RRD_MEMORY_MODE id;
29 - const char* name;
30 - STORAGE_ENGINE_API api;
31 -};
32 -
33 -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
34 -STORAGE_ENGINE* storage_engine_find(const char* name);
35 -
8 // Iterator over existing engines
9 STORAGE_ENGINE* storage_engine_foreach_init();
10 STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);
exporting/process_data.c
+6 -6
@@ -77,10 +77,10 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
77 time_t before = instance->before;
78
79 // find the edges of the rrd database for this chart
80 - time_t first_t = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle);
81 - time_t last_t = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
80 + time_t first_t = rd->tiers[0]->query_ops->oldest_time(rd->tiers[0]->db_metric_handle);
81 + time_t last_t = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
82 time_t update_every = st->update_every;
83 - struct rrddim_query_handle handle;
83 + struct storage_engine_query_handle handle;
84
85 // step back a little, to make sure we have complete data collection
86 // for all metrics
@@ -125,8 +125,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
125 size_t counter = 0;
126 NETDATA_DOUBLE sum = 0;
127
128 - for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before); !rd->tiers[0]->query_ops.is_finished(&handle);) {
129 - STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
128 + for (rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, after, before); !rd->tiers[0]->query_ops->is_finished(&handle);) {
129 + STORAGE_POINT sp = rd->tiers[0]->query_ops->next_metric(&handle);
130
131 if (unlikely(storage_point_is_empty(sp))) {
132 // not collected
@@ -136,7 +136,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
136 sum += sp.sum;
137 counter += sp.count;
138 }
139 - rd->tiers[0]->query_ops.finalize(&handle);
139 + rd->tiers[0]->query_ops->finalize(&handle);
140
141 if (unlikely(!counter)) {
142 debug(
libnetdata/required_dummies.h
+1
@@ -35,6 +35,7 @@ int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE
35
36 void rrdset_thread_rda_free(void){};
37 void sender_thread_buffer_free(void){};
38 +void query_target_free(void){};
39
40 // required by get_system_cpus()
41 char *netdata_configured_host_prefix = "";
libnetdata/simple_pattern/simple_pattern.h
+3 -1
@@ -34,7 +34,9 @@ void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) ;
34 int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) ;
35 char *simple_pattern_iterate(SIMPLE_PATTERN **p);
36
37 -//Auxiliary function to create a pattern
37 +// Auxiliary function to create a pattern
38 char *simple_pattern_trim_around_equal(char *src);
39
40 +#define is_valid_sp(x) ((x) && *(x) && !((x)[0] == '*' && (x)[1] == '\0'))
41 +
42 #endif //NETDATA_SIMPLE_PATTERN_H
libnetdata/threads/threads.c
+2
@@ -108,6 +108,7 @@ void netdata_threads_init_after_fork(size_t stacksize) {
108
109 extern void rrdset_thread_rda_free(void);
110 extern void sender_thread_buffer_free(void);
111 +extern void query_target_free(void);
112
113 static void thread_cleanup(void *ptr) {
114 if(netdata_thread != ptr) {
@@ -120,6 +121,7 @@ static void thread_cleanup(void *ptr) {
121
122 sender_thread_buffer_free();
123 rrdset_thread_rda_free();
124 + query_target_free();
125 thread_cache_destroy();
126
127 freez((void *)netdata_thread->tag);
ml/ADCharts.cc
+22 -21
@@ -133,31 +133,32 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
133 time_t After = Before - Cfg.AnomalyDetectionQueryDuration;
134 RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
135
136 - RRDR *R = rrd2rrdr(
137 - OWA, HostRateRS,
138 - 1 /* points wanted */,
139 - After,
140 - Before,
141 - Cfg.AnomalyDetectionGroupingMethod,
142 - 0 /* resampling time */,
143 - Options, "anomaly_rate",
144 - NULL /* context param list */,
145 - NULL /* group options */,
146 - 0, /* timeout */
147 - 0 /* tier */
136 + RRDR *R = rrd2rrdr_legacy(
137 + OWA, HostRateRS,
138 + 1 /* points wanted */,
139 + After,
140 + Before,
141 + Cfg.AnomalyDetectionGroupingMethod,
142 + 0 /* resampling time */,
143 + Options, "anomaly_rate",
144 + NULL /* group options */,
145 + 0, /* timeout */
146 + 0 /* tier */
147 );
149 - assert(R->d == 1 && R->n == 1 && R->rows == 1);
148 + if(R) {
149 + assert(R->d == 1 && R->n == 1 && R->rows == 1);
150
151 - static thread_local bool PrevAboveThreshold = false;
152 - bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
153 - bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
154 - PrevAboveThreshold = AboveThreshold;
151 + static thread_local bool PrevAboveThreshold = false;
152 + bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
153 + bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
154 + PrevAboveThreshold = AboveThreshold;
155
156 - rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
157 - rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
158 - rrdset_done(AnomalyDetectionRS);
156 + rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
157 + rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
158 + rrdset_done(AnomalyDetectionRS);
159
160 - rrdr_free(OWA, R);
160 + rrdr_free(OWA, R);
161 + }
162 onewayalloc_destroy(OWA);
163 }
164
ml/Query.h
+3 -3
@@ -8,7 +8,7 @@ namespace ml {
8 class Query {
9 public:
10 Query(RRDDIM *RD) : RD(RD), Initialized(false) {
11 - Ops = &RD->tiers[0]->query_ops;
11 + Ops = RD->tiers[0]->query_ops;
12 }
13
14 time_t latestTime() {
@@ -42,8 +42,8 @@ private:
42 RRDDIM *RD;
43 bool Initialized;
44
45 - struct rrddim_query_ops *Ops;
46 - struct rrddim_query_handle Handle;
45 + struct storage_engine_query_ops *Ops;
46 + struct storage_engine_query_handle Handle;
47 };
48
49 } // namespace ml
ml/ml.cc
+3 -3
@@ -151,7 +151,7 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
151 if (R->rows != 1)
152 return;
153
154 - if (MaxAnomalyRates < 1 || MaxAnomalyRates >= R->d)
154 + if (MaxAnomalyRates < 1 || MaxAnomalyRates >= (int)R->d)
155 return;
156
157 NETDATA_DOUBLE *CNs = R->v;
@@ -160,12 +160,12 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
160 std::vector<std::pair<NETDATA_DOUBLE, int>> V;
161
162 V.reserve(R->d);
163 - for (int Idx = 0; Idx != R->d; Idx++)
163 + for (int Idx = 0; Idx != (int)R->d; Idx++)
164 V.emplace_back(CNs[Idx], Idx);
165
166 std::sort(V.rbegin(), V.rend());
167
168 - for (int Idx = MaxAnomalyRates; Idx != R->d; Idx++) {
168 + for (int Idx = MaxAnomalyRates; Idx != (int)R->d; Idx++) {
169 int UnsortedIdx = V[Idx].second;
170
171 int OldFlags = static_cast<int>(DimFlags[UnsortedIdx]);
web/api/formatters/csv/csv.c
+8 -9
@@ -3,15 +3,14 @@
3 #include "libnetdata/libnetdata.h"
4 #include "csv.h"
5
6 -void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd) {
7 - rrdset_check_rdlock(r->st);
8 -
6 +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
7 //info("RRD2CSV(): %s: BEGIN", r->st->id);
8 + QUERY_TARGET *qt = r->internal.qt;
9 long c, i;
11 - RRDDIM *d;
10 + const long used = qt->query.used;
11
12 // print the csv header
14 - for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
13 + for(c = 0, i = 0; c < used ; c++) {
14 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
15 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
16
@@ -23,7 +22,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
22 }
23 buffer_strcat(wb, separator);
24 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
26 - buffer_strcat(wb, rrddim_name(d));
25 + buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
26 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
27 i++;
28 }
@@ -31,7 +30,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
30
31 if(format == DATASOURCE_CSV_MARKDOWN) {
32 // print the --- line after header
34 - for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
33 + for(c = 0, i = 0; c < used ;c++) {
34 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
35 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
36
@@ -89,7 +88,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
88 int set_min_max = 0;
89 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
90 total = 0;
92 - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
91 + for(c = 0; c < used ;c++) {
92 NETDATA_DOUBLE n = cn[c];
93
94 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -103,7 +102,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
102 }
103
104 // for each dimension
106 - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
105 + for(c = 0; c < used ;c++) {
106 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
107 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
108
web/api/formatters/csv/csv.h
+1 -1
@@ -5,7 +5,7 @@
5
6 #include "web/api/queries/rrdr.h"
7
8 -void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd);
8 +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines);
9
10 #include "../rrd2json.h"
11
web/api/formatters/json/json.c
+11 -19
@@ -5,15 +5,7 @@
5 #define JSON_DATES_JS 1
6 #define JSON_DATES_TIMESTAMP 2
7
8 -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list)
9 -{
10 - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
11 -
12 - int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
13 -
14 - if (should_lock)
15 - rrdset_check_rdlock(r->st);
16 -
8 +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
9 //info("RRD2JSON(): %s: BEGIN", r->st->id);
10 int row_annotations = 0, dates, dates_with_new = 0;
11 char kq[2] = "", // key quote
@@ -112,21 +104,21 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
104 // -------------------------------------------------------------------------
105 // print the JSON header
106
107 + QUERY_TARGET *qt = r->internal.qt;
108 long c, i;
116 - RRDDIM *rd;
109 + const long used = qt->query.used;
110
111 // print the header lines
119 - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
112 + for(c = 0, i = 0; c < used ; c++) {
113 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
114 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
115
116 buffer_fast_strcat(wb, pre_label, pre_label_len);
124 - buffer_strcat(wb, rrddim_name(rd));
125 -// buffer_strcat(wb, ".");
126 -// buffer_strcat(wb, rd->rrdset->name);
117 + buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
118 buffer_fast_strcat(wb, post_label, post_label_len);
119 i++;
120 }
121 +
122 if(!i) {
123 buffer_fast_strcat(wb, pre_label, pre_label_len);
124 buffer_fast_strcat(wb, "no data", 7);
@@ -134,7 +126,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
126 }
127 size_t total_number_of_dimensions = i;
128
137 - // print the begin of row data
129 + // print the beginning of row data
130 buffer_strcat(wb, data_begin);
131
132 // if all dimensions are hidden, print a null
@@ -187,7 +179,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
179 if(unlikely(row_annotations)) {
180 // google supports one annotation per row
181 int annotation_found = 0;
190 - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) {
182 + for(c = 0; c < used ; c++) {
183 if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue;
184
185 if(unlikely(co[c] & RRDR_VALUE_RESET)) {
@@ -222,7 +214,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
214 int set_min_max = 0;
215 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
216 total = 0;
225 - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
217 + for(c = 0; c < used ;c++) {
218 NETDATA_DOUBLE n;
219 if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
220 n = ar[c];
@@ -240,7 +232,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
232 }
233
234 // for each dimension
243 - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
235 + for(c = 0; c < used ;c++) {
236 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
237 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
238
@@ -253,7 +245,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
245 buffer_fast_strcat(wb, pre_value, pre_value_len);
246
247 if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
256 - buffer_sprintf(wb, "%s%s%s: ", kq, rrddim_name(rd), kq);
248 + buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq);
249
250 if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {
251 if(unlikely(options & RRDR_OPTION_NULL2ZERO))
web/api/formatters/json/json.h
+1 -1
@@ -5,6 +5,6 @@
5
6 #include "../rrd2json.h"
7
8 -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list);
8 +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable);
9
10 #endif //NETDATA_API_FORMATTER_JSON_H
web/api/formatters/json_wrapper.c
+80 -80
@@ -33,25 +33,17 @@ static int fill_formatted_callback(const char *name, const char *value, RRDLABEL
33 }
34
35 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
36 - RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data)
36 + RRDR_GROUPING group_method)
37 {
38 - struct context_param *context_param_list = rrdset_query_data->context_param_list;
39 - char *chart_label_key = rrdset_query_data->chart_label_key;
40 -
41 - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
42 - int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
43 - uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT));
44 -
45 - if (should_lock)
46 - rrdset_check_rdlock(r->st);
38 + QUERY_TARGET *qt = r->internal.qt;
39
40 long rows = rrdr_rows(r);
41 long c, i;
50 - RRDDIM *rd;
42 + const long query_used = qt->query.used;
43
44 //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
45 char kq[2] = "", // key quote
54 - sq[2] = ""; // string quote
46 + sq[2] = ""; // string quote
47
48 if( options & RRDR_OPTION_GOOGLE_JSON ) {
49 kq[0] = '\0';
@@ -66,43 +58,43 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
58 " %sapi%s: 1,\n"
59 " %sid%s: %s%s%s,\n"
60 " %sname%s: %s%s%s,\n"
69 - " %sview_update_every%s: %d,\n"
70 - " %supdate_every%s: %d,\n"
71 - " %sfirst_entry%s: %u,\n"
72 - " %slast_entry%s: %u,\n"
73 - " %sbefore%s: %u,\n"
74 - " %safter%s: %u,\n"
61 + " %sview_update_every%s: %ld,\n"
62 + " %supdate_every%s: %ld,\n"
63 + " %sfirst_entry%s: %ld,\n"
64 + " %slast_entry%s: %ld,\n"
65 + " %sbefore%s: %ld,\n"
66 + " %safter%s: %ld,\n"
67 " %sgroup%s: %s%s%s,\n"
68 " %soptions%s: %s"
69 , kq, kq
78 - , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_id(r->st), sq
79 - , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_name(r->st), sq
70 + , kq, kq, sq, qt->id, sq
71 + , kq, kq, sq, qt->id, sq
72 , kq, kq, r->update_every
81 - , kq, kq, r->st->update_every
82 - , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st))
83 - , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st))
84 - , kq, kq, (uint32_t)r->before
85 - , kq, kq, (uint32_t)r->after
73 + , kq, kq, qt->db.minimum_latest_update_every
74 + , kq, kq, qt->db.first_time_t
75 + , kq, kq, qt->db.last_time_t
76 + , kq, kq, r->before
77 + , kq, kq, r->after
78 , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
79 , kq, kq, sq);
80
89 - web_client_api_request_v1_data_options_to_string(wb, r->internal.query_options);
81 + web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options);
82
83 buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq);
84
93 - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
85 + for(c = 0, i = 0; c < query_used ; c++) {
86 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
87 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
88
89 if(i) buffer_strcat(wb, ", ");
90 buffer_strcat(wb, sq);
99 - buffer_strcat(wb, rrddim_name(rd));
91 + buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
92 buffer_strcat(wb, sq);
93 i++;
94 }
95 if(!i) {
96 #ifdef NETDATA_INTERNAL_CHECKS
105 - error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", rrdset_id(r->st), r->d, options);
97 + error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options);
98 #endif
99 rows = 0;
100 buffer_strcat(wb, sq);
@@ -114,13 +106,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
106 " %sdimension_ids%s: ["
107 , kq, kq);
108
117 - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
109 + for(c = 0, i = 0; c < query_used ; c++) {
110 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
111 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
112
113 if(i) buffer_strcat(wb, ", ");
114 buffer_strcat(wb, sq);
123 - buffer_strcat(wb, rrddim_id(rd));
115 + buffer_strcat(wb, string2str(qt->query.array[c].dimension.id));
116 buffer_strcat(wb, sq);
117 i++;
118 }
@@ -132,7 +124,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
124 }
125 buffer_strcat(wb, "],\n");
126
135 - if (rrdset_query_data->show_dimensions) {
127 + if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) {
128 buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq);
129
130 char name[RRD_ID_LENGTH_MAX * 2 + 2];
@@ -140,35 +132,46 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
132
133 struct value_output co = {.c = 0, .wb = wb};
134
143 - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
144 - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
145 - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrddim_id(rd), rrddim_name(rd));
146 - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrddim_id(rd), rrddim_name(rd));
147 - dictionary_set(dict, name, output, len+1);
135 + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
136 + for (c = 0; c < (long)qt->metrics.used ;c++) {
137 + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
138 + rrdmetric_acquired_id(qt->metrics.array[c]),
139 + rrdmetric_acquired_name(qt->metrics.array[c]));
140 +
141 + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
142 + rrdmetric_acquired_id(qt->metrics.array[c]),
143 + rrdmetric_acquired_name(qt->metrics.array[c]));
144 +
145 + dictionary_set(dict, name, output, len + 1);
146 }
147 dictionary_walkthrough_read(dict, value_list_output_callback, &co);
148 dictionary_destroy(dict);
149
150 co.c = 0;
151 buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq);
154 - dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
155 - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
156 - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset));
157 - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset));
152 + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
153 + for (c = 0; c < (long)qt->instances.used ; c++) {
154 + RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
155 +
156 + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
157 + rrdinstance_acquired_id(ria),
158 + rrdinstance_acquired_name(ria));
159 +
160 + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
161 + rrdinstance_acquired_id(ria),
162 + rrdinstance_acquired_name(ria));
163 +
164 dictionary_set(dict, name, output, len + 1);
165 }
160 -
166 dictionary_walkthrough_read(dict, value_list_output_callback, &co);
167 dictionary_destroy(dict);
168
164 - RRDSET *st;
169 co.c = 0;
170 buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq);
167 - dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
168 - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
169 - st = rd->rrdset;
170 - if (st->rrdlabels)
171 - rrdlabels_walkthrough_read(st->rrdlabels, fill_formatted_callback, dict);
171 + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
172 + for (c = 0; c < (long)qt->instances.used ; c++) {
173 + RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
174 + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict);
175 }
176 dictionary_walkthrough_read(dict, value_list_output_callback, &co);
177 dictionary_destroy(dict);
@@ -178,8 +181,14 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
181 // functions
182 {
183 DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
181 - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
182 - chart_functions_to_dict(rd->rrdset, funcs);
184 + RRDINSTANCE_ACQUIRED *ria = NULL;
185 + for (c = 0; c < query_used ; c++) {
186 + QUERY_METRIC *qm = &qt->query.array[c];
187 + if(qm->link.ria == ria)
188 + continue;
189 +
190 + ria = qm->link.ria;
191 + chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs);
192 }
193
194 buffer_sprintf(wb, " %sfunctions%s: [", kq, kq);
@@ -194,23 +203,26 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
203 buffer_strcat(wb, "],\n");
204 }
205
197 - // Composite charts
198 - if (context_mode && temp_rd) {
206 + // context query
207 + if (!qt->request.st) {
208 buffer_sprintf(
209 wb,
210 " %schart_ids%s: [",
211 kq, kq);
212
204 - for (c = 0, i = 0, rd = temp_rd ; rd && c < r->d; c++, rd = rd->next) {
213 + for (c = 0, i = 0; c < query_used; c++) {
214 + QUERY_METRIC *qm = &qt->query.array[c];
215 +
216 if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
217 continue;
218 +
219 if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
220 continue;
221
222 if (i)
223 buffer_strcat(wb, ", ");
224 buffer_strcat(wb, sq);
213 - buffer_strcat(wb, rrdset_id(rd->rrdset));
225 + buffer_strcat(wb, string2str(qm->chart.id));
226 buffer_strcat(wb, sq);
227 i++;
228 }
@@ -221,29 +233,29 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
233 buffer_strcat(wb, sq);
234 }
235 buffer_strcat(wb, "],\n");
224 - if (chart_label_key) {
236 + if (qt->instances.chart_label_key_pattern) {
237 buffer_sprintf(wb, " %schart_labels%s: { ", kq, kq);
238
227 - SIMPLE_PATTERN *pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
228 - SIMPLE_PATTERN *original_pattern = pattern;
239 + SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern;
240 char *label_key = NULL;
241 int keys = 0;
242 while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
232 -
243 if (keys)
244 buffer_strcat(wb, ", ");
245 buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq);
246 keys++;
247
238 - for (c = 0, i = 0, rd = temp_rd; rd && c < r->d; c++, rd = rd->next) {
248 + for (c = 0, i = 0; c < query_used; c++) {
249 + QUERY_METRIC *qm = &qt->query.array[c];
250 +
251 if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
252 continue;
253 if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
254 continue;
255 +
256 if (i)
257 buffer_strcat(wb, ", ");
245 -
246 - rrdlabels_get_value_to_buffer_or_null(rd->rrdset->rrdlabels, wb, label_key, sq, "null");
258 + rrdlabels_get_value_to_buffer_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key, sq, "null");
259 i++;
260 }
261 if (!i) {
@@ -255,33 +267,26 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
267 buffer_strcat(wb, "]");
268 }
269 buffer_strcat(wb, "},\n");
258 - simple_pattern_free(original_pattern);
270 }
271 }
272
273 buffer_sprintf(wb, " %slatest_values%s: ["
274 , kq, kq);
275
265 - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
276 + for(c = 0, i = 0; c < query_used ;c++) {
277 + QUERY_METRIC *qm = &qt->query.array[c];
278 +
279 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
280 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
281
282 if(i) buffer_strcat(wb, ", ");
283 i++;
284
272 - NETDATA_DOUBLE value = rd->last_stored_value;
285 + NETDATA_DOUBLE value = rrdmetric_acquired_last_stored_value(qm->link.rma);
286 if (NAN == value)
287 buffer_strcat(wb, "null");
288 else
289 buffer_rrd_value(wb, value);
277 - /*
278 - storage_number n = rd->values[rrdset_last_slot(r->st)];
279 -
280 - if(!does_storage_number_exist(n))
281 - buffer_strcat(wb, "null");
282 - else
283 - buffer_rrd_value(wb, unpack_storage_number(n));
284 - */
290 }
291 if(!i) {
292 rows = 0;
@@ -298,7 +303,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
303
304 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
305 total = 0;
301 - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
306 + for(c = 0; c < query_used ;c++) {
307 NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
308 NETDATA_DOUBLE n = cn[c];
309
@@ -311,7 +316,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
316 if(total == 0) total = 1;
317 }
318
314 - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
319 + for(c = 0, i = 0; c < query_used ;c++) {
320 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
321 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
322
@@ -361,16 +366,11 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
366 , kq, kq
367 );
368
364 - for(int tier = 0; tier < storage_tiers ; tier++)
369 + for(size_t tier = 0; tier < storage_tiers ; tier++)
370 buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]);
371
372 buffer_strcat(wb, " ]");
373
369 - if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) {
370 - buffer_sprintf(wb, ",\n %schart_variables%s: ", kq, kq);
371 - health_api_v1_chart_custom_variables2json(r->st, wb);
372 - }
373 -
374 buffer_sprintf(wb, ",\n %sresult%s: ", kq, kq);
375
376 if(string_value) buffer_strcat(wb, sq);
web/api/formatters/json_wrapper.h
+1 -1
@@ -8,7 +8,7 @@
8
9
10 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
11 - RRDR_GROUPING group_method, QUERY_PARAMS *query_params);
11 + RRDR_GROUPING group_method);
12 void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
13 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
14
web/api/formatters/rrd2json.c
+66 -208
@@ -3,128 +3,6 @@
3 #include "web/api/web_api_v1.h"
4 #include "database/storage_engine.h"
5
6 -static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
7 -{
8 - if (unlikely(!temp_rd))
9 - return;
10 -
11 - string_freez(temp_rd->id);
12 - string_freez(temp_rd->name);
13 -
14 - if (unlikely(archive_mode)) {
15 - temp_rd->rrdset->counter--;
16 - if (!temp_rd->rrdset->counter) {
17 - string_freez(temp_rd->rrdset->id);
18 - string_freez(temp_rd->rrdset->name);
19 - string_freez(temp_rd->rrdset->context);
20 - onewayalloc_freez(owa, temp_rd->rrdset);
21 - }
22 - }
23 -
24 - for(int tier = 0; tier < storage_tiers ;tier++) {
25 - if(!temp_rd->tiers[tier]) continue;
26 -
27 - if(archive_mode) {
28 - STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
29 - if (eng)
30 - eng->api.metric_release(temp_rd->tiers[tier]->db_metric_handle);
31 - }
32 -
33 - onewayalloc_freez(owa, temp_rd->tiers[tier]);
34 - }
35 -
36 - onewayalloc_freez(owa, temp_rd);
37 -}
38 -
39 -static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
40 -{
41 - if (unlikely(!temp_rd))
42 - return;
43 -
44 - RRDDIM *t;
45 - while (temp_rd) {
46 - t = temp_rd->next;
47 - free_single_rrdrim(owa, temp_rd, archive_mode);
48 - temp_rd = t;
49 - }
50 -}
51 -
52 -void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list)
53 -{
54 - if (unlikely(!param_list || !*param_list))
55 - return;
56 -
57 - free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE);
58 - onewayalloc_freez(owa, (*param_list));
59 - *param_list = NULL;
60 -}
61 -
62 -void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested)
63 -{
64 - RRDDIM *temp_rd = context_param_list->rd;
65 - RRDDIM *new_rd_list = NULL, *t;
66 - int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE);
67 -
68 - RRDSET *st = temp_rd->rrdset;
69 - RRDSET *last_st = st;
70 - time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
71 - time_t last_last_entry_t = last_entry_t;
72 - while (temp_rd) {
73 - t = temp_rd->next;
74 -
75 - st = temp_rd->rrdset;
76 - if (st == last_st) {
77 - last_entry_t = last_last_entry_t;
78 - }else {
79 - last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
80 - last_last_entry_t = last_entry_t;
81 - last_st = st;
82 - }
83 -
84 - if (last_entry_t >= after_requested) {
85 - temp_rd->next = new_rd_list;
86 - new_rd_list = temp_rd;
87 - } else
88 - free_single_rrdrim(owa, temp_rd, is_archived);
89 - temp_rd = t;
90 - }
91 - context_param_list->rd = new_rd_list;
92 -};
93 -
94 -void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st)
95 -{
96 - if (unlikely(!param_list || !st))
97 - return;
98 -
99 - if (unlikely(!(*param_list))) {
100 - *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
101 - (*param_list)->first_entry_t = LONG_MAX;
102 - (*param_list)->last_entry_t = 0;
103 - (*param_list)->flags = CONTEXT_FLAGS_CONTEXT;
104 - (*param_list)->rd = NULL;
105 - }
106 -
107 - st->last_accessed_time = now_realtime_sec();
108 - (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t(st));
109 - (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t(st));
110 -
111 - RRDDIM *rd1;
112 - rrddim_foreach_read(rd1, st) {
113 - RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
114 - rd->id = string_dup(rd1->id);
115 - rd->name = string_dup(rd1->name);
116 - for(int tier = 0; tier < storage_tiers ;tier++) {
117 - if(rd1->tiers[tier])
118 - rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier]));
119 - else
120 - rd->tiers[tier] = NULL;
121 - }
122 - rd->next = (*param_list)->rd;
123 - (*param_list)->rd = rd;
124 - }
125 - rrddim_foreach_done(rd1);
126 -}
127 -
6 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
7 rrdset2json(st, wb, NULL, NULL, 0);
8 }
@@ -182,12 +60,12 @@ int rrdset2value_api_v1(
60 , BUFFER *wb
61 , NETDATA_DOUBLE *n
62 , const char *dimensions
185 - , long points
186 - , long long after
187 - , long long before
188 - , int group_method
63 + , size_t points
64 + , time_t after
65 + , time_t before
66 + , RRDR_GROUPING group_method
67 , const char *group_options
190 - , long group_time
68 + , time_t resampling_time
69 , uint32_t options
70 , time_t *db_after
71 , time_t *db_before
@@ -196,16 +74,25 @@ int rrdset2value_api_v1(
74 , size_t *result_points_generated
75 , int *value_is_null
76 , NETDATA_DOUBLE *anomaly_rate
199 - , int timeout
200 - , int tier
77 + , time_t timeout
78 + , size_t tier
79 ) {
80 int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
81
82 ONEWAYALLOC *owa = onewayalloc_create(0);
205 -
206 - RRDR *r = rrd2rrdr(owa, st, points, after, before,
207 - group_method, group_time, options, dimensions, NULL,
208 - group_options, timeout, tier);
83 + RRDR *r = rrd2rrdr_legacy(
84 + owa,
85 + st,
86 + points,
87 + after,
88 + before,
89 + group_method,
90 + resampling_time,
91 + options,
92 + dimensions,
93 + group_options,
94 + timeout,
95 + tier);
96
97 if(!r) {
98 if(value_is_null) *value_is_null = 1;
@@ -217,7 +104,7 @@ int rrdset2value_api_v1(
104 *db_points_read += r->internal.db_points_read;
105
106 if(db_points_per_tier) {
220 - for(int t = 0; t < storage_tiers ;t++)
107 + for(size_t t = 0; t < storage_tiers ;t++)
108 db_points_per_tier[t] += r->internal.tier_points_read[t];
109 }
110
@@ -243,50 +130,19 @@ int rrdset2value_api_v1(
130 if(db_after) *db_after = r->after;
131 if(db_before) *db_before = r->before;
132
246 - long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
247 - *n = rrdr2value(r, i, options, value_is_null, anomaly_rate, NULL);
133 + long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
134 + *n = rrdr2value(r, i, options, value_is_null, anomaly_rate);
135 ret = HTTP_RESP_OK;
136
137 cleanup:
251 - if(r) rrdr_free(owa, r);
138 + rrdr_free(owa, r);
139 onewayalloc_destroy(owa);
140 return ret;
141 }
142
256 -int rrdset2anything_api_v1(
257 - ONEWAYALLOC *owa
258 - , RRDSET *st
259 - , QUERY_PARAMS *query_params
260 - , BUFFER *dimensions
261 - , uint32_t format
262 - , long points
263 - , long long after
264 - , long long before
265 - , int group_method
266 - , const char *group_options
267 - , long group_time
268 - , uint32_t options
269 - , time_t *latest_timestamp
270 - , int tier
271 -)
272 -{
273 - BUFFER *wb = query_params->wb;
274 - if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
275 - st->last_accessed_time = now_realtime_sec();
276 -
277 - RRDR *r = rrd2rrdr(
278 - owa,
279 - st,
280 - points,
281 - after,
282 - before,
283 - group_method,
284 - group_time,
285 - options,
286 - dimensions ? buffer_tostring(dimensions) : NULL,
287 - query_params->context_param_list,
288 - group_options,
289 - query_params->timeout, tier);
143 +int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) {
144 +
145 + RRDR *r = rrd2rrdr(owa, qt);
146 if(!r) {
147 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
148 return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -297,10 +153,8 @@ int rrdset2anything_api_v1(
153 return HTTP_RESP_BACKEND_FETCH_FAILED;
154 }
155
300 - if (rrdset_is_ar_chart(st))
301 - ml_process_rrdr(r, query_params->max_anomaly_rates);
302 -
303 - RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;
156 + if (qt->request.st && rrdset_is_ar_chart(qt->request.st))
157 + ml_process_rrdr(r, qt->request.max_anomaly_rates);
158
159 if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
160 buffer_no_cacheable(wb);
@@ -310,85 +164,89 @@ int rrdset2anything_api_v1(
164 if(latest_timestamp && rrdr_rows(r) > 0)
165 *latest_timestamp = r->before;
166
167 + DATASOURCE_FORMAT format = qt->request.format;
168 + RRDR_OPTIONS options = qt->request.options;
169 + RRDR_GROUPING group_method = qt->request.group_method;
170 +
171 switch(format) {
172 case DATASOURCE_SSV:
173 if(options & RRDR_OPTION_JSON_WRAP) {
174 wb->contenttype = CT_APPLICATION_JSON;
317 - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
318 - rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
175 + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
176 + rrdr2ssv(r, wb, options, "", " ", "");
177 rrdr_json_wrapper_end(r, wb, format, options, 1);
178 }
179 else {
180 wb->contenttype = CT_TEXT_PLAIN;
323 - rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
181 + rrdr2ssv(r, wb, options, "", " ", "");
182 }
183 break;
184
185 case DATASOURCE_SSV_COMMA:
186 if(options & RRDR_OPTION_JSON_WRAP) {
187 wb->contenttype = CT_APPLICATION_JSON;
330 - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
331 - rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
188 + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
189 + rrdr2ssv(r, wb, options, "", ",", "");
190 rrdr_json_wrapper_end(r, wb, format, options, 1);
191 }
192 else {
193 wb->contenttype = CT_TEXT_PLAIN;
336 - rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
194 + rrdr2ssv(r, wb, options, "", ",", "");
195 }
196 break;
197
198 case DATASOURCE_JS_ARRAY:
199 if(options & RRDR_OPTION_JSON_WRAP) {
200 wb->contenttype = CT_APPLICATION_JSON;
343 - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
344 - rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
201 + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
202 + rrdr2ssv(r, wb, options, "[", ",", "]");
203 rrdr_json_wrapper_end(r, wb, format, options, 0);
204 }
205 else {
206 wb->contenttype = CT_APPLICATION_JSON;
349 - rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
207 + rrdr2ssv(r, wb, options, "[", ",", "]");
208 }
209 break;
210
211 case DATASOURCE_CSV:
212 if(options & RRDR_OPTION_JSON_WRAP) {
213 wb->contenttype = CT_APPLICATION_JSON;
356 - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
357 - rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
214 + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
215 + rrdr2csv(r, wb, format, options, "", ",", "\\n", "");
216 rrdr_json_wrapper_end(r, wb, format, options, 1);
217 }
218 else {
219 wb->contenttype = CT_TEXT_PLAIN;
362 - rrdr2csv(r, wb, format, options, "", ",", "\r\n", "", temp_rd);
220 + rrdr2csv(r, wb, format, options, "", ",", "\r\n", "");
221 }
222 break;
223
224 case DATASOURCE_CSV_MARKDOWN:
225 if(options & RRDR_OPTION_JSON_WRAP) {
226 wb->contenttype = CT_APPLICATION_JSON;
369 - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
370 - rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
227 + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
228 + rrdr2csv(r, wb, format, options, "", "|", "\\n", "");
229 rrdr_json_wrapper_end(r, wb, format, options, 1);
230 }
231 else {
232 wb->contenttype = CT_TEXT_PLAIN;
375 - rrdr2csv(r, wb, format, options, "", "|", "\r\n", "", temp_rd);
233 + rrdr2csv(r, wb, format, options, "", "|", "\r\n", "");
234 }
235 break;
236
237 case DATASOURCE_CSV_JSON_ARRAY:
238 wb->contenttype = CT_APPLICATION_JSON;
239 if(options & RRDR_OPTION_JSON_WRAP) {
382 - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
240 + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
241 buffer_strcat(wb, "[\n");
384 - rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
242 + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
243 buffer_strcat(wb, "\n]");
244 rrdr_json_wrapper_end(r, wb, format, options, 0);
245 }
246 else {
247 wb->contenttype = CT_APPLICATION_JSON;
248 buffer_strcat(wb, "[\n");
391 - rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
249 + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
250 buffer_strcat(wb, "\n]");
251 }
252 break;
@@ -396,29 +254,29 @@ int rrdset2anything_api_v1(
254 case DATASOURCE_TSV:
255 if(options & RRDR_OPTION_JSON_WRAP) {
256 wb->contenttype = CT_APPLICATION_JSON;
399 - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
400 - rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
257 + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
258 + rrdr2csv(r, wb, format, options, "", "\t", "\\n", "");
259 rrdr_json_wrapper_end(r, wb, format, options, 1);
260 }
261 else {
262 wb->contenttype = CT_TEXT_PLAIN;
405 - rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "", temp_rd);
263 + rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "");
264 }
265 break;
266
267 case DATASOURCE_HTML:
268 if(options & RRDR_OPTION_JSON_WRAP) {
269 wb->contenttype = CT_APPLICATION_JSON;
412 - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
270 + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
271 buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
414 - rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
272 + rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
273 buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
274 rrdr_json_wrapper_end(r, wb, format, options, 1);
275 }
276 else {
277 wb->contenttype = CT_TEXT_HTML;
278 buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
421 - rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "", temp_rd);
279 + rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
280 buffer_strcat(wb, "</table>\n</center>\n</html>\n");
281 }
282 break;
@@ -427,9 +285,9 @@ int rrdset2anything_api_v1(
285 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
286
287 if(options & RRDR_OPTION_JSON_WRAP)
430 - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
288 + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
289
432 - rrdr2json(r, wb, options, 1, query_params->context_param_list);
290 + rrdr2json(r, wb, options, 1);
291
292 if(options & RRDR_OPTION_JSON_WRAP)
293 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -439,9 +297,9 @@ int rrdset2anything_api_v1(
297 wb->contenttype = CT_APPLICATION_JSON;
298
299 if(options & RRDR_OPTION_JSON_WRAP)
442 - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
300 + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
301
444 - rrdr2json(r, wb, options, 1, query_params->context_param_list);
302 + rrdr2json(r, wb, options, 1);
303
304 if(options & RRDR_OPTION_JSON_WRAP)
305 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -450,9 +308,9 @@ int rrdset2anything_api_v1(
308 case DATASOURCE_JSONP:
309 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
310 if(options & RRDR_OPTION_JSON_WRAP)
453 - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
311 + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
312
455 - rrdr2json(r, wb, options, 0, query_params->context_param_list);
313 + rrdr2json(r, wb, options, 0);
314
315 if(options & RRDR_OPTION_JSON_WRAP)
316 rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -463,14 +321,14 @@ int rrdset2anything_api_v1(
321 wb->contenttype = CT_APPLICATION_JSON;
322
323 if(options & RRDR_OPTION_JSON_WRAP)
466 - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
324 + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
325
468 - rrdr2json(r, wb, options, 0, query_params->context_param_list);
326 + rrdr2json(r, wb, options, 0);
327
328 if(options & RRDR_OPTION_JSON_WRAP) {
329 if(options & RRDR_OPTION_RETURN_JWAR) {
330 rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0);
473 - rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0, query_params->context_param_list);
331 + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0);
332 }
333 rrdr_json_wrapper_end(r, wb, format, options, 0);
334 }
web/api/formatters/rrd2json.h
+22 -50
@@ -5,16 +5,6 @@
5
6 #include "web/api/web_api_v1.h"
7
8 -typedef struct query_params {
9 - struct context_param *context_param_list;
10 - BUFFER *wb;
11 - char *chart_label_key;
12 - int max_anomaly_rates;
13 - int timeout;
14 - int show_dimensions;
15 -} QUERY_PARAMS;
16 -
17 -
8 #include "web/api/exporters/allmetrics.h"
9 #include "web/api/queries/rrdr.h"
10
@@ -34,19 +24,20 @@ typedef struct query_params {
24 #define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
25
26 // type of JSON generations
37 -#define DATASOURCE_INVALID (-1)
38 -#define DATASOURCE_JSON 0
39 -#define DATASOURCE_DATATABLE_JSON 1
40 -#define DATASOURCE_DATATABLE_JSONP 2
41 -#define DATASOURCE_SSV 3
42 -#define DATASOURCE_CSV 4
43 -#define DATASOURCE_JSONP 5
44 -#define DATASOURCE_TSV 6
45 -#define DATASOURCE_HTML 7
46 -#define DATASOURCE_JS_ARRAY 8
47 -#define DATASOURCE_SSV_COMMA 9
48 -#define DATASOURCE_CSV_JSON_ARRAY 10
49 -#define DATASOURCE_CSV_MARKDOWN 11
27 +typedef enum {
28 + DATASOURCE_JSON = 0,
29 + DATASOURCE_DATATABLE_JSON = 1,
30 + DATASOURCE_DATATABLE_JSONP = 2,
31 + DATASOURCE_SSV = 3,
32 + DATASOURCE_CSV = 4,
33 + DATASOURCE_JSONP = 5,
34 + DATASOURCE_TSV = 6,
35 + DATASOURCE_HTML = 7,
36 + DATASOURCE_JS_ARRAY = 8,
37 + DATASOURCE_SSV_COMMA = 9,
38 + DATASOURCE_CSV_JSON_ARRAY = 10,
39 + DATASOURCE_CSV_MARKDOWN = 11,
40 +} DATASOURCE_FORMAT;
41
42 #define DATASOURCE_FORMAT_JSON "json"
43 #define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
@@ -64,34 +55,19 @@ typedef struct query_params {
55 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
56 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
57
67 -int rrdset2anything_api_v1(
68 - ONEWAYALLOC *owa
69 - , RRDSET *st
70 - , QUERY_PARAMS *query_params
71 - , BUFFER *dimensions
72 - , uint32_t format
73 - , long points
74 - , long long after
75 - , long long before
76 - , int group_method
77 - , const char *group_options
78 - , long group_time
79 - , uint32_t options
80 - , time_t *latest_timestamp
81 - , int tier
82 -);
58 +int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp);
59
60 int rrdset2value_api_v1(
61 RRDSET *st
62 , BUFFER *wb
63 , NETDATA_DOUBLE *n
64 , const char *dimensions
89 - , long points
90 - , long long after
91 - , long long before
92 - , int group_method
65 + , size_t points
66 + , time_t after
67 + , time_t before
68 + , RRDR_GROUPING group_method
69 , const char *group_options
94 - , long group_time
70 + , time_t resampling_time
71 , uint32_t options
72 , time_t *db_after
73 , time_t *db_before
@@ -100,12 +76,8 @@ int rrdset2value_api_v1(
76 , size_t *result_points_generated
77 , int *value_is_null
78 , NETDATA_DOUBLE *anomaly_rate
103 - , int timeout
104 - , int tier
79 + , time_t timeout
80 + , size_t tier
81 );
82
107 -void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st);
108 -void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested);
109 -void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list);
110 -
83 #endif /* NETDATA_RRD2JSON_H */
web/api/formatters/ssv/ssv.c
+2 -2
@@ -2,7 +2,7 @@
2
3 #include "ssv.h"
4
5 -void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd) {
5 +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) {
6 //info("RRD2SSV(): %s: BEGIN", r->st->id);
7 long i;
8
@@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con
17 // for each line in the array
18 for(i = start; i != end ;i += step) {
19 int all_values_are_null = 0;
20 - NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd);
20 + NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL);
21
22 if(likely(i != start)) {
23 if(r->min > v) r->min = v;
web/api/formatters/ssv/ssv.h
+1 -1
@@ -5,6 +5,6 @@
5
6 #include "../rrd2json.h"
7
8 -void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd);
8 +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix);
9
10 #endif //NETDATA_API_FORMATTER_SSV_H
web/api/formatters/value/value.c
+64 -5
@@ -3,9 +3,10 @@
3 #include "value.h"
4
5
6 -inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd) {
6 +inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) {
7 + QUERY_TARGET *qt = r->internal.qt;
8 long c;
8 - RRDDIM *d;
9 + const long used = qt->query.used;
10
11 NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
12 RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
@@ -20,7 +21,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
21 int set_min_max = 0;
22 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
23 total = 0;
23 - for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) {
24 + for (c = 0; c < used; c++) {
25 NETDATA_DOUBLE n = cn[c];
26
27 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -34,7 +35,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
35 }
36
37 // for each dimension
37 - for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) {
38 + for (c = 0; c < used; c++) {
39 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
40 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
41
@@ -80,7 +81,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
81
82 if(anomaly_rate) {
83 if(!r->d) *anomaly_rate = 0;
83 - else *anomaly_rate = total_anomaly_rate / r->d;
84 + else *anomaly_rate = total_anomaly_rate / (NETDATA_DOUBLE)r->d;
85 }
86
87 if(unlikely(all_null)) {
@@ -100,3 +101,61 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
101
102 return v;
103 }
104 +
105 +QUERY_VALUE rrdmetric2value(RRDHOST *host,
106 + struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,
107 + time_t after, time_t before,
108 + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
109 + size_t tier, time_t timeout
110 +) {
111 + QUERY_TARGET_REQUEST qtr = {
112 + .host = host,
113 + .rca = rca,
114 + .ria = ria,
115 + .rma = rma,
116 + .after = after,
117 + .before = before,
118 + .points = 1,
119 + .options = options,
120 + .group_method = group_method,
121 + .group_options = group_options,
122 + .tier = tier,
123 + .timeout = timeout,
124 + };
125 +
126 + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
127 + RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
128 +
129 + QUERY_VALUE qv;
130 +
131 + if(!r || rrdr_rows(r) == 0) {
132 + qv = (QUERY_VALUE) {
133 + .value = NAN,
134 + .anomaly_rate = NAN,
135 + };
136 + }
137 + else {
138 + qv = (QUERY_VALUE) {
139 + .after = r->after,
140 + .before = r->before,
141 + .points_read = r->internal.db_points_read,
142 + .result_points = r->internal.result_points_generated,
143 + };
144 +
145 + for(size_t t = 0; t < storage_tiers ;t++)
146 + qv.storage_points_per_tier[t] = r->internal.tier_points_read[t];
147 +
148 + long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
149 + int all_values_are_null = 0;
150 + qv.value = rrdr2value(r, i, options, &all_values_are_null, &qv.anomaly_rate);
151 + if(all_values_are_null) {
152 + qv.value = NAN;
153 + qv.anomaly_rate = NAN;
154 + }
155 + }
156 +
157 + rrdr_free(owa, r);
158 + onewayalloc_destroy(owa);
159 +
160 + return qv;
161 +}
web/api/formatters/value/value.h
+22 -1
@@ -5,6 +5,27 @@
5
6 #include "../rrd2json.h"
7
8 -NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd);
8 +typedef struct storage_value {
9 + NETDATA_DOUBLE value;
10 + NETDATA_DOUBLE anomaly_rate;
11 + time_t after;
12 + time_t before;
13 + size_t points_read;
14 + size_t storage_points_per_tier[RRD_STORAGE_TIERS];
15 + size_t result_points;
16 +} QUERY_VALUE;
17 +
18 +struct rrdmetric_acquired;
19 +struct rrdinstance_acquired;
20 +struct rrdcontext_acquired;
21 +
22 +QUERY_VALUE rrdmetric2value(RRDHOST *host,
23 + struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,
24 + time_t after, time_t before,
25 + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
26 + size_t tier, time_t timeout
27 +);
28 +
29 +NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate);
30
31 #endif //NETDATA_API_FORMATTER_VALUE_H
web/api/netdata-swagger.json
-1
@@ -573,7 +573,6 @@
573 "unaligned",
574 "match-ids",
575 "match-names",
576 - "showcustomvars",
576 "allow_past",
577 "anomaly-bit"
578 ]
web/api/netdata-swagger.yaml
-1
@@ -471,7 +471,6 @@ paths:
471 - unaligned
472 - match-ids
473 - match-names
474 - - showcustomvars
474 - allow_past
475 - anomaly-bit
476 default:
web/api/queries/query.c
+442 -431
@@ -657,71 +657,6 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) {
657 }
658 }
659
660 -// ----------------------------------------------------------------------------
661 -
662 -static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
663 - struct context_param *context_param_list)
664 -{
665 - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
666 - int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
667 -
668 - if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
669 -
670 - if (should_lock)
671 - rrdset_check_rdlock(r->st);
672 -
673 - int match_ids = 0, match_names = 0;
674 -
675 - if(unlikely(options & RRDR_OPTION_MATCH_IDS))
676 - match_ids = 1;
677 - if(unlikely(options & RRDR_OPTION_MATCH_NAMES))
678 - match_names = 1;
679 -
680 - if(likely(!match_ids && !match_names))
681 - match_ids = match_names = 1;
682 -
683 - SIMPLE_PATTERN *pattern = simple_pattern_create(dims, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
684 -
685 - RRDDIM *d;
686 - long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
687 - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) {
688 - if( (match_ids && simple_pattern_matches(pattern, rrddim_id(d)))
689 - || (match_names && simple_pattern_matches(pattern, rrddim_name(d)))
690 - ) {
691 - r->od[c] |= RRDR_DIMENSION_SELECTED;
692 - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) r->od[c] &= ~RRDR_DIMENSION_HIDDEN;
693 - dims_selected++;
694 -
695 - // since the user needs this dimension
696 - // make it appear as NONZERO, to return it
697 - // even if the dimension has only zeros
698 - // unless option non_zero is set
699 - if(unlikely(!(options & RRDR_OPTION_NONZERO)))
700 - r->od[c] |= RRDR_DIMENSION_NONZERO;
701 -
702 - // count the visible dimensions
703 - if(likely(r->od[c] & RRDR_DIMENSION_NONZERO))
704 - dims_not_hidden_not_zero++;
705 - }
706 - else {
707 - r->od[c] |= RRDR_DIMENSION_HIDDEN;
708 - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
709 - }
710 - }
711 - simple_pattern_free(pattern);
712 -
713 - // check if all dimensions are hidden
714 - if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
715 - // there are a few selected dimensions,
716 - // but they are all zero
717 - // enable the selected ones
718 - // to avoid returning an empty chart
719 - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next)
720 - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED))
721 - r->od[c] |= RRDR_DIMENSION_NONZERO;
722 - }
723 -}
724 -
660 // ----------------------------------------------------------------------------
661 // helpers to find our way in RRDR
662
@@ -736,13 +671,13 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr
671 static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {
672 rrdr_line++;
673
739 - internal_error(rrdr_line >= r->n,
740 - "QUERY: requested to step above RRDR size for chart '%s'",
741 - rrdset_name(r->st));
674 + internal_error(rrdr_line >= (long)r->n,
675 + "QUERY: requested to step above RRDR size for query '%s'",
676 + r->internal.qt->id);
677
678 internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t,
744 - "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'",
745 - (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, rrdset_name(r->st));
679 + "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of query '%s'",
680 + (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->internal.qt->id);
681
682 // save the time
683 r->t[rrdr_line] = t;
@@ -758,68 +693,131 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {
693 // ----------------------------------------------------------------------------
694 // tier management
695
761 -static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, time_t before_wanted, long points_wanted) {
696 +static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) {
697 + if(!qm->tiers[tier].db_metric_handle || !qm->tiers[tier].db_first_time_t || !qm->tiers[tier].db_last_time_t || !qm->tiers[tier].db_update_every)
698 + return false;
699 +
700 + return true;
701 +}
702 +
703 +static size_t query_metric_first_working_tier(QUERY_METRIC *qm) {
704 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
705 +
706 + // find the db time-range for this tier for all metrics
707 + STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
708 + time_t first_t = qm->tiers[tier].db_first_time_t;
709 + time_t last_t = qm->tiers[tier].db_last_time_t;
710 + time_t update_every = qm->tiers[tier].db_update_every;
711 +
712 + if(!db_metric_handle || !first_t || !last_t || !update_every)
713 + continue;
714 +
715 + return tier;
716 + }
717 +
718 + return 0;
719 +}
720 +
721 +static long query_plan_points_coverage_weight(time_t db_first_t, time_t db_last_t, time_t db_update_every, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
722 + if(db_first_t == 0 || db_last_t == 0 || db_update_every == 0)
723 + return -LONG_MAX;
724 +
725 + time_t common_first_t = MAX(db_first_t, after_wanted);
726 + time_t common_last_t = MIN(db_last_t, before_wanted);
727 +
728 + long points_available = (common_last_t - common_first_t) / db_update_every;
729 + long points_delta = (long)(points_available - points_wanted);
730 + long points_coverage = (points_delta < 0) ? (long)(points_available * 1000 / points_wanted): 1000;
731 +
732 + if(points_available <= 0)
733 + return -LONG_MAX;
734 +
735 + return points_coverage;
736 +}
737 +
738 +static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
739 if(unlikely(storage_tiers < 2))
740 return 0;
741
765 - if(unlikely(after_wanted == before_wanted || points_wanted <= 0 || !rd || !rd->rrdset)) {
742 + if(unlikely(after_wanted == before_wanted || points_wanted <= 0))
743 + return query_metric_first_working_tier(qm);
744
767 - if(!rd)
768 - internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation");
769 - else
770 - internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation",
771 - (rd->rrdset)?rrdset_name(rd->rrdset):"unknown", rrddim_name(rd));
745 + long weight[storage_tiers];
746
773 - return 0;
747 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
748 +
749 + // find the db time-range for this tier for all metrics
750 + STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
751 + time_t first_t = qm->tiers[tier].db_first_time_t;
752 + time_t last_t = qm->tiers[tier].db_last_time_t;
753 + time_t update_every = qm->tiers[tier].db_update_every;
754 +
755 + if(!db_metric_handle || !first_t || !last_t || !update_every) {
756 + weight[tier] = -LONG_MAX;
757 + continue;
758 + }
759 +
760 + weight[tier] = query_plan_points_coverage_weight(first_t, last_t, update_every, after_wanted, before_wanted, points_wanted);
761 }
762
776 - //BUFFER *wb = buffer_create(1000);
777 - //buffer_sprintf(wb, "Best tier for chart '%s', dim '%s', from %ld to %ld (dur %ld, every %d), points %ld",
778 - // rd->rrdset->name, rd->name, after_wanted, before_wanted, before_wanted - after_wanted, rd->update_every, points_wanted);
763 + size_t best_tier = 0;
764 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
765 + if(weight[tier] >= weight[best_tier])
766 + best_tier = tier;
767 + }
768 +
769 + return best_tier;
770 +}
771 +
772 +static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
773 + if(unlikely(storage_tiers < 2))
774 + return 0;
775 +
776 + if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) {
777 + internal_error(true, "QUERY: '%s' has invalid params to tier calculation", qt->id);
778 + return 0;
779 + }
780
781 long weight[storage_tiers];
782
782 - for(int tier = 0; tier < storage_tiers ; tier++) {
783 - if(unlikely(!rd->tiers[tier])) {
784 - internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized",
785 - tier, rrdset_name(rd->rrdset), rrddim_name(rd));
786 - // buffer_free(wb);
787 - return 0;
788 - }
783 + for(size_t tier = 0; tier < storage_tiers ; tier++) {
784
790 - time_t first_t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
791 - time_t last_t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
785 + time_t common_first_t = 0;
786 + time_t common_last_t = 0;
787 + time_t common_update_every = 0;
788
793 - time_t common_after = MAX(first_t, after_wanted);
794 - time_t common_before = MIN(last_t, before_wanted);
789 + // find the db time-range for this tier for all metrics
790 + for(size_t i = 0, used = qt->query.used; i < used ; i++) {
791 + QUERY_METRIC *qm = &qt->query.array[i];
792
796 - long time_coverage = (common_before - common_after) * 1000 / (before_wanted - after_wanted);
797 - if(time_coverage < 0) time_coverage = 0;
793 + time_t first_t = qm->tiers[tier].db_first_time_t;
794 + time_t last_t = qm->tiers[tier].db_last_time_t;
795 + time_t update_every = qm->tiers[tier].db_update_every;
796
799 - int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every;
800 - if(unlikely(update_every == 0)) {
801 - internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d",
802 - tier, rrdset_name(rd->rrdset), rrddim_name(rd), rd->tiers[tier]->tier_grouping, rd->update_every);
803 - // buffer_free(wb);
804 - return 0;
805 - }
797 + if(!first_t || !last_t || !update_every)
798 + continue;
799
807 - long points_available = (before_wanted - after_wanted) / update_every;
808 - long points_delta = points_available - points_wanted;
809 - long points_coverage = (points_delta < 0) ? points_available * 1000 / points_wanted: 1000;
800 + if(!common_first_t)
801 + common_first_t = first_t;
802 + else
803 + common_first_t = MIN(first_t, common_first_t);
804
811 - if(points_available <= 0)
812 - weight[tier] = -LONG_MAX;
813 - else
814 - weight[tier] = points_coverage;
805 + if(!common_last_t)
806 + common_last_t = last_t;
807 + else
808 + common_last_t = MAX(last_t, common_last_t);
809
816 - // buffer_sprintf(wb, ": tier %d, first %ld, last %ld (dur %ld, tg %d, every %d), points %ld, tcoverage %ld, pcoverage %ld, weight %ld",
817 - // tier, first_t, last_t, last_t - first_t, rd->tiers[tier]->tier_grouping, update_every,
818 - // points_available, time_coverage, points_coverage, weight[tier]);
810 + if(!common_update_every)
811 + common_update_every = update_every;
812 + else
813 + common_update_every = MIN(update_every, common_update_every);
814 + }
815 +
816 + weight[tier] = query_plan_points_coverage_weight(common_first_t, common_last_t, common_update_every, after_wanted, before_wanted, points_wanted);
817 }
818
821 - int best_tier = 0;
822 - for(int tier = 1; tier < storage_tiers ; tier++) {
819 + size_t best_tier = 0;
820 + for(size_t tier = 1; tier < storage_tiers ; tier++) {
821 if(weight[tier] >= weight[best_tier])
822 best_tier = tier;
823 }
@@ -827,49 +825,30 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted,
825 if(weight[best_tier] == -LONG_MAX)
826 best_tier = 0;
827
830 - //buffer_sprintf(wb, ": final best tier %d", best_tier);
831 - //internal_error(true, "%s", buffer_tostring(wb));
832 - //buffer_free(wb);
833 -
828 return best_tier;
829 }
830
837 -static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) {
838 - int ret = st->update_every;
831 +static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier) {
832 + size_t best_tier;
833 + if((options & RRDR_OPTION_SELECTED_TIER) && tier < storage_tiers)
834 + best_tier = tier;
835 + else
836 + best_tier = rrddim_find_best_tier_for_timeframe(qt, after_wanted, before_wanted, points_wanted);
837
840 - if(unlikely(!rrdset_number_of_dimensions(st)))
841 - return ret;
838 + // find the db minimum update every for this tier for all metrics
839 + time_t common_update_every = default_rrd_update_every;
840 + for(size_t i = 0, used = qt->query.used; i < used ; i++) {
841 + QUERY_METRIC *qm = &qt->query.array[i];
842
843 - RRDDIM *first_rd = NULL;
844 - rrddim_foreach_read(first_rd, st) break; rrddim_foreach_done(first_rd);
845 - if(!first_rd)
846 - return ret;
843 + time_t update_every = qm->tiers[best_tier].db_update_every;
844
848 - int best_tier;
849 - if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers)
850 - best_tier = tier;
851 - else {
852 - best_tier = rrddim_find_best_tier_for_timeframe(first_rd, after_wanted, before_wanted, points_wanted);
853 - }
854 -
855 - if(!first_rd->tiers[best_tier]) {
856 - internal_error(
857 - true,
858 - "QUERY: tier %d on chart '%s', is not initialized", best_tier, rrdset_name(st));
859 - }
860 - else {
861 - ret = (int)first_rd->tiers[best_tier]->tier_grouping * (int)st->update_every;
862 - if(unlikely(!ret)) {
863 - internal_error(
864 - true,
865 - "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d",
866 - rrdset_name(st), first_rd->tiers[best_tier]->tier_grouping, st->update_every);
867 -
868 - ret = st->update_every;
869 - }
845 + if(!i)
846 + common_update_every = update_every;
847 + else
848 + common_update_every = MIN(update_every, common_update_every);
849 }
850
872 - return ret;
851 + return common_update_every;
852 }
853
854 // ----------------------------------------------------------------------------
@@ -917,7 +896,7 @@ typedef struct query_plan {
896 typedef struct query_engine_ops {
897 // configuration
898 RRDR *r;
920 - RRDDIM *rd;
899 + QUERY_METRIC *qm;
900 time_t view_update_every;
901 time_t query_granularity;
902 TIER_QUERY_FETCH tier_query_fetch;
@@ -929,11 +908,11 @@ typedef struct query_engine_ops {
908
909 // storage queries
910 size_t tier;
932 - struct rrddim_tier *tier_ptr;
933 - struct rrddim_query_handle handle;
934 - STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
935 - int (*is_finished)(struct rrddim_query_handle *handle);
936 - void (*finalize)(struct rrddim_query_handle *handle);
911 + struct query_metric_tier *tier_ptr;
912 + struct storage_engine_query_handle handle;
913 + STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
914 + int (*is_finished)(struct storage_engine_query_handle *handle);
915 + void (*finalize)(struct storage_engine_query_handle *handle);
916
917 // aggregating points over time
918 void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
@@ -954,10 +933,15 @@ typedef struct query_engine_ops {
933
934 #define query_plan_should_switch_plan(ops, now) ((now) >= (ops).current_plan_expire_time)
935
957 -static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, time_t overwrite_after) {
936 +static bool query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, time_t overwrite_after) {
937 if(unlikely(plan_id >= ops->plan.entries))
938 plan_id = ops->plan.entries - 1;
939
940 + if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[plan_id].tier)) {
941 + ops->current_plan_expire_time = ops->plan.data[plan_id].before;
942 + return false;
943 + }
944 +
945 time_t after = ops->plan.data[plan_id].after;
946 time_t before = ops->plan.data[plan_id].before;
947
@@ -965,13 +949,15 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
949 after = overwrite_after;
950
951 ops->tier = ops->plan.data[plan_id].tier;
968 - ops->tier_ptr = ops->rd->tiers[ops->tier];
969 - ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
970 - ops->next_metric = ops->tier_ptr->query_ops.next_metric;
971 - ops->is_finished = ops->tier_ptr->query_ops.is_finished;
972 - ops->finalize = ops->tier_ptr->query_ops.finalize;
952 + ops->tier_ptr = &ops->qm->tiers[ops->tier];
953 + ops->tier_ptr->eng->api.query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
954 + ops->next_metric = ops->tier_ptr->eng->api.query_ops.next_metric;
955 + ops->is_finished = ops->tier_ptr->eng->api.query_ops.is_finished;
956 + ops->finalize = ops->tier_ptr->eng->api.query_ops.finalize;
957 ops->current_plan = plan_id;
958 ops->current_plan_expire_time = ops->plan.data[plan_id].before;
959 +
960 + return true;
961 }
962
963 static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) {
@@ -984,6 +970,8 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las
970
971 if (ops->current_plan >= ops->plan.entries) {
972 ops->current_plan = ops->plan.entries - 1;
973 + // let the query run with current plan
974 + // we will not switch it
975 return;
976 }
977
@@ -995,9 +983,9 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las
983 ops->finalize = NULL;
984 }
985
998 - query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
999 -
986 // internal_error(true, "QUERY: switched plan to %zu (all is %zu), previous expiration was %ld, this starts at %ld, now is %ld, last_point_end_time %ld", ops->current_plan, ops->plan.entries, ops->plan.data[ops->current_plan-1].before, ops->plan.data[ops->current_plan].after, now, last_point_end_time);
987 +
988 + query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
989 }
990
991 static int compare_query_plan_entries_on_start_time(const void *a, const void *b) {
@@ -1006,21 +994,20 @@ static int compare_query_plan_entries_on_start_time(const void *a, const void *b
994 return (p1->after < p2->after)?-1:1;
995 }
996
1009 -static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, long points_wanted) {
1010 - RRDDIM *rd = ops->rd;
1011 -
997 +static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
998 //BUFFER *wb = buffer_create(1000);
999 //buffer_sprintf(wb, "QUERY PLAN for chart '%s' dimension '%s', from %ld to %ld:", rd->rrdset->name, rd->name, after_wanted, before_wanted);
1000
1001 // put our selected tier as the first plan
1002 size_t selected_tier;
1003
1018 - if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER && ops->r->internal.query_tier >= 0 && ops->r->internal.query_tier < storage_tiers) {
1019 - selected_tier = ops->r->internal.query_tier;
1004 + if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER
1005 + && ops->r->internal.qt->window.tier < storage_tiers
1006 + && query_metric_is_valid_tier(ops->qm, ops->r->internal.qt->window.tier)) {
1007 + selected_tier = ops->r->internal.qt->window.tier;
1008 }
1009 else {
1022 -
1023 - selected_tier = rrddim_find_best_tier_for_timeframe(rd, after_wanted, before_wanted, points_wanted);
1010 + selected_tier = query_metric_best_tier_for_timeframe(ops->qm, after_wanted, before_wanted, points_wanted);
1011
1012 if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)
1013 ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER;
@@ -1028,8 +1015,8 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1015
1016 ops->plan.entries = 1;
1017 ops->plan.data[0].tier = selected_tier;
1031 - ops->plan.data[0].after = rd->tiers[selected_tier]->query_ops.oldest_time(rd->tiers[selected_tier]->db_metric_handle);
1032 - ops->plan.data[0].before = rd->tiers[selected_tier]->query_ops.latest_time(rd->tiers[selected_tier]->db_metric_handle);
1018 + ops->plan.data[0].after = ops->qm->tiers[selected_tier].db_first_time_t;
1019 + ops->plan.data[0].before = ops->qm->tiers[selected_tier].db_last_time_t;
1020
1021 if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) {
1022 // the selected tier
@@ -1041,9 +1028,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1028 // check if our selected tier can start the query
1029 if (selected_tier_first_time_t > after_wanted) {
1030 // we need some help from other tiers
1044 - for (int tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
1031 + for (size_t tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
1032 + if(!query_metric_is_valid_tier(ops->qm, tr))
1033 + continue;
1034 +
1035 // find the first time of this tier
1046 - time_t first_time_t = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
1036 + time_t first_time_t = ops->qm->tiers[tr].db_first_time_t;
1037
1038 //buffer_sprintf(wb, ": EVAL AFTER tier %d, %ld", tier, first_time_t);
1039
@@ -1069,8 +1059,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1059 if (selected_tier_last_time_t < before_wanted) {
1060 // we need some help from other tiers
1061 for (int tr = (int)selected_tier - 1; tr >= 0; tr--) {
1062 + if(!query_metric_is_valid_tier(ops->qm, tr))
1063 + continue;
1064 +
1065 // find the last time of this tier
1073 - time_t last_time_t = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
1066 + time_t last_time_t = ops->qm->tiers[tr].db_last_time_t;
1067
1068 //buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_t);
1069
@@ -1108,7 +1101,7 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1101
1102 //internal_error(true, "%s", buffer_tostring(wb));
1103
1111 - query_planer_activate_plan(ops, 0, 0);
1104 + return query_planer_activate_plan(ops, 0, 0);
1105 }
1106
1107
@@ -1148,14 +1141,13 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1141 (ops).group_anomaly_rate += (point).anomaly; \
1142 } while(0)
1143
1151 -static inline void rrd2rrdr_do_dimension(
1152 - RRDR *r
1153 - , long points_wanted
1154 - , RRDDIM *rd
1155 - , long dim_id_in_rrdr
1156 - , time_t after_wanted
1157 - , time_t before_wanted
1158 -){
1144 +static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
1145 + QUERY_TARGET *qt = r->internal.qt;
1146 + QUERY_METRIC *qm = &qt->query.array[dim_id_in_rrdr];
1147 + size_t points_wanted = qt->window.points;
1148 + time_t after_wanted = qt->window.after;
1149 + time_t before_wanted = qt->window.before;
1150 +
1151 // bool debug_this = false;
1152 // if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
1153 // debug_this = true;
@@ -1167,19 +1159,20 @@ static inline void rrd2rrdr_do_dimension(
1159
1160 QUERY_ENGINE_OPS ops = {
1161 .r = r,
1170 - .rd = rd,
1162 + .qm = qm,
1163 .grouping_add = r->internal.grouping_add,
1164 .grouping_flush = r->internal.grouping_flush,
1165 .tier_query_fetch = r->internal.tier_query_fetch,
1166 .view_update_every = r->update_every,
1175 - .query_granularity = r->update_every / r->group,
1167 + .query_granularity = (time_t)(r->update_every / r->group),
1168 .group_value_flags = RRDR_VALUE_NOTHING
1169 };
1170
1171 long rrdr_line = -1;
1172 bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
1173
1182 - query_plan(&ops, after_wanted, before_wanted, points_wanted);
1174 + if(!query_plan(&ops, after_wanted, before_wanted, points_wanted))
1175 + return;
1176
1177 NETDATA_DOUBLE min = r->min, max = r->max;
1178
@@ -1191,7 +1184,7 @@ static inline void rrd2rrdr_do_dimension(
1184 time_t now_end_time = after_wanted + ops.view_update_every - ops.query_granularity;
1185
1186 // The main loop, based on the query granularity we need
1194 - for( ; (long)points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
1187 + for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
1188
1189 if(query_plan_should_switch_plan(ops, now_end_time))
1190 query_planer_next_plan(&ops, now_end_time, new_point.end_time);
@@ -1271,16 +1264,16 @@ static inline void rrd2rrdr_do_dimension(
1264
1265 // check if the db is giving us zero duration points
1266 if(unlikely(new_point.start_time == new_point.end_time)) {
1274 - internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
1275 - rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time);
1267 + internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu start time %ld, end time %ld, that are both equal",
1268 + qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time);
1269
1277 - new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every);
1270 + new_point.start_time = new_point.end_time - ops.tier_ptr->db_update_every;
1271 }
1272
1273 // check if the db is advancing the query
1274 if(unlikely(new_point.end_time <= last1_point.end_time)) {
1282 - internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld",
1283 - rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time,
1275 + internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld",
1276 + qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time,
1277 last1_point.id, last1_point.end_time, now_start_time, now_end_time);
1278
1279 count_same_end_time++;
@@ -1307,8 +1300,8 @@ static inline void rrd2rrdr_do_dimension(
1300
1301 // we only log if this is not point 1
1302 internal_error(new_point.end_time < after_wanted && new_point.id > 1,
1310 - "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
1311 - rrdset_name(rd->rrdset), rrddim_name(rd),
1303 + "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
1304 + qt->id, string2str(qm->dimension.id),
1305 new_point.id, new_point.start_time, new_point.end_time,
1306 now_start_time, now_end_time,
1307 after_wanted, before_wanted);
@@ -1324,8 +1317,8 @@ static inline void rrd2rrdr_do_dimension(
1317
1318 if(unlikely(count_same_end_time)) {
1319 internal_error(true,
1327 - "QUERY: the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
1328 - last1_point.end_time, count_same_end_time);
1320 + "QUERY: '%s', dimension '%s', the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
1321 + qt->id, string2str(qm->dimension.id), last1_point.end_time, count_same_end_time);
1322
1323 if(unlikely(new_point.end_time <= last1_point.end_time))
1324 new_point.end_time = now_end_time;
@@ -1336,7 +1329,7 @@ static inline void rrd2rrdr_do_dimension(
1329 // we select the one to use based on their timestamps
1330
1331 size_t iterations = 0;
1339 - for ( ; now_end_time <= new_point.end_time && (long)points_added < points_wanted ;
1332 + for ( ; now_end_time <= new_point.end_time && points_added < points_wanted ;
1333 now_end_time += ops.view_update_every, iterations++) {
1334
1335 // now_start_time is wrong in this loop
@@ -1349,22 +1342,35 @@ static inline void rrd2rrdr_do_dimension(
1342 current_point = new_point;
1343 query_interpolate_point(current_point, last1_point, now_end_time);
1344
1352 - internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
1353 - "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
1354 - " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.",
1355 - rrdset_name(rd->rrdset), rrddim_name(rd), after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
1356 - current_point.id, current_point.start_time, current_point.end_time, now_end_time);
1345 + internal_error(current_point.id > 0
1346 + && last1_point.id == 0
1347 + && current_point.end_time > after_wanted
1348 + && current_point.end_time > now_end_time,
1349 + "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
1350 + " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
1351 + " but we could really favor by having last_point1 in this query.",
1352 + qt->id, string2str(qm->dimension.id),
1353 + after_wanted, before_wanted,
1354 + ops.view_update_every, ops.query_granularity,
1355 + current_point.id, current_point.start_time, current_point.end_time,
1356 + now_end_time);
1357 }
1358 else if(likely(now_end_time <= last1_point.end_time)) {
1359 // our LAST point is still valid
1360 current_point = last1_point;
1361 query_interpolate_point(current_point, last2_point, now_end_time);
1362
1363 - internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
1364 - "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
1365 - " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.",
1366 - rrdset_name(rd->rrdset), rrddim_name(rd), after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
1367 - current_point.id, current_point.start_time, current_point.end_time, now_end_time);
1363 + internal_error(current_point.id > 0
1364 + && last2_point.id == 0
1365 + && current_point.end_time > after_wanted
1366 + && current_point.end_time > now_end_time,
1367 + "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
1368 + " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
1369 + " but we could really favor by having last_point2 in this query.",
1370 + qt->id, string2str(qm->dimension.id),
1371 + after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
1372 + current_point.id, current_point.start_time, current_point.end_time,
1373 + now_end_time);
1374 }
1375 else {
1376 // a GAP, we don't have a value this time
@@ -1427,7 +1433,7 @@ static inline void rrd2rrdr_do_dimension(
1433
1434 r->internal.result_points_generated += points_added;
1435 r->internal.db_points_read += ops.db_total_points_read;
1430 - for(int tr = 0; tr < storage_tiers ; tr++)
1436 + for(size_t tr = 0; tr < storage_tiers ; tr++)
1437 r->internal.tier_points_read[tr] += ops.db_points_read_per_tier[tr];
1438
1439 r->min = min;
@@ -1436,9 +1442,10 @@ static inline void rrd2rrdr_do_dimension(
1442 r->after = min_date - ops.view_update_every + ops.query_granularity;
1443 rrdr_done(r, rrdr_line);
1444
1439 - internal_error((long)points_added != points_wanted,
1440 - "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).",
1441 - rrdset_name(r->st), rrddim_name(rd), (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
1445 + internal_error(points_added != points_wanted,
1446 + "QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).",
1447 + qt->id, string2str(qm->dimension.id),
1448 + (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
1449 }
1450
1451 // ----------------------------------------------------------------------------
@@ -1446,14 +1453,14 @@ static inline void rrd2rrdr_do_dimension(
1453
1454 extern void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
1455
1449 -void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
1450 - if(unlikely(tier < 0 || tier >= storage_tiers)) return;
1456 +void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now) {
1457 + if(unlikely(tier >= storage_tiers)) return;
1458 if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
1459
1460 struct rrddim_tier *t = rd->tiers[tier];
1461 if(unlikely(!t)) return;
1462
1456 - time_t latest_time_t = t->query_ops.latest_time(t->db_metric_handle);
1463 + time_t latest_time_t = t->query_ops->latest_time(t->db_metric_handle);
1464 time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every;
1465 time_t time_diff = now - latest_time_t;
1466
@@ -1463,27 +1470,27 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
1470 // there is really nothing we can do
1471 if(now <= latest_time_t || time_diff < granularity) return;
1472
1466 - struct rrddim_query_handle handle;
1473 + struct storage_engine_query_handle handle;
1474
1475 size_t all_points_read = 0;
1476
1477 // for each lower tier
1471 - for(int tr = tier - 1; tr >= 0 ;tr--){
1472 - time_t smaller_tier_first_time = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
1473 - time_t smaller_tier_last_time = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
1478 + for(int tr = (int)tier - 1; tr >= 0 ;tr--){
1479 + time_t smaller_tier_first_time = rd->tiers[tr]->query_ops->oldest_time(rd->tiers[tr]->db_metric_handle);
1480 + time_t smaller_tier_last_time = rd->tiers[tr]->query_ops->latest_time(rd->tiers[tr]->db_metric_handle);
1481 if(smaller_tier_last_time <= latest_time_t) continue; // it is as bad as we are
1482
1483 long after_wanted = (latest_time_t < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_t;
1484 long before_wanted = smaller_tier_last_time;
1485
1486 struct rrddim_tier *tmp = rd->tiers[tr];
1480 - tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
1487 + tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
1488
1489 size_t points = 0;
1490
1484 - while(!tmp->query_ops.is_finished(&handle)) {
1491 + while(!tmp->query_ops->is_finished(&handle)) {
1492
1486 - STORAGE_POINT sp = tmp->query_ops.next_metric(&handle);
1493 + STORAGE_POINT sp = tmp->query_ops->next_metric(&handle);
1494
1495 if(sp.end_time > latest_time_t) {
1496 latest_time_t = sp.end_time;
@@ -1493,7 +1500,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
1500 }
1501
1502 all_points_read += points;
1496 - tmp->query_ops.finalize(&handle);
1503 + tmp->query_ops->finalize(&handle);
1504
1505 //internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d",
1506 // rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr);
@@ -1510,32 +1517,33 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1517 , RRDR_OPTIONS options __maybe_unused
1518 , RRDR_GROUPING group_method
1519 , bool aligned
1513 - , long group
1514 - , long resampling_time
1515 - , long resampling_group
1520 + , size_t group
1521 + , time_t resampling_time
1522 + , size_t resampling_group
1523 , time_t after_wanted
1524 , time_t after_requested
1525 , time_t before_wanted
1526 , time_t before_requested
1520 - , long points_requested
1521 - , long points_wanted
1527 + , size_t points_requested
1528 + , size_t points_wanted
1529 //, size_t after_slot
1530 //, size_t before_slot
1531 , const char *msg
1532 ) {
1533
1527 - time_t first_entry_t = rrdset_first_entry_t(r->st);
1528 - time_t last_entry_t = rrdset_last_entry_t(r->st);
1534 + time_t first_entry_t = r->internal.qt->db.first_time_t;
1535 + time_t last_entry_t = r->internal.qt->db.last_time_t;
1536
1530 - info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), "
1531 - "after (got: %zu, want: %zu, req: %ld, db: %zu), "
1532 - "before (got: %zu, want: %zu, req: %ld, db: %zu), "
1533 - "duration (got: %zu, want: %zu, req: %ld, db: %zu), "
1534 - //"slot (after: %zu, before: %zu, delta: %zu), "
1535 - "points (got: %ld, want: %ld, req: %ld, db: %ld), "
1537 + internal_error(
1538 + true,
1539 + "rrd2rrdr() on %s update every %ld with %s grouping %s (group: %zu, resampling_time: %ld, resampling_group: %zu), "
1540 + "after (got: %ld, want: %ld, req: %ld, db: %ld), "
1541 + "before (got: %ld, want: %ld, req: %ld, db: %ld), "
1542 + "duration (got: %ld, want: %ld, req: %ld, db: %ld), "
1543 + "points (got: %zu, want: %zu, req: %zu), "
1544 "%s"
1537 - , rrdset_name(r->st)
1538 - , r->st->update_every
1545 + , r->internal.qt->id
1546 + , r->internal.qt->window.query_granularity
1547
1548 // grouping
1549 , (aligned) ? "aligned" : "unaligned"
@@ -1545,35 +1553,27 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1553 , resampling_group
1554
1555 // after
1548 - , (size_t)r->after
1549 - , (size_t)after_wanted
1556 + , r->after
1557 + , after_wanted
1558 , after_requested
1551 - , (size_t)first_entry_t
1559 + , first_entry_t
1560
1561 // before
1554 - , (size_t)r->before
1555 - , (size_t)before_wanted
1562 + , r->before
1563 + , before_wanted
1564 , before_requested
1557 - , (size_t)last_entry_t
1565 + , last_entry_t
1566
1567 // duration
1560 - , (size_t)(r->before - r->after + r->st->update_every)
1561 - , (size_t)(before_wanted - after_wanted + r->st->update_every)
1562 - , before_requested - after_requested
1563 - , (size_t)((last_entry_t - first_entry_t) + r->st->update_every)
1564 -
1565 - // slot
1566 - /*
1567 - , after_slot
1568 - , before_slot
1569 - , (after_slot > before_slot) ? (r->st->entries - after_slot + before_slot) : (before_slot - after_slot)
1570 - */
1568 + , (long)(r->before - r->after + r->internal.qt->window.query_granularity)
1569 + , (long)(before_wanted - after_wanted + r->internal.qt->window.query_granularity)
1570 + , (long)before_requested - after_requested
1571 + , (long)((last_entry_t - first_entry_t) + r->internal.qt->window.query_granularity)
1572
1573 // points
1574 , r->rows
1575 , points_wanted
1576 , points_requested
1576 - , r->st->entries
1577
1578 // message
1579 , msg
@@ -1582,7 +1582,7 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
1582 #endif // NETDATA_INTERNAL_CHECKS
1583
1584 // Returns 1 if an absolute period was requested or 0 if it was a relative period
1585 -int rrdr_relative_window_to_absolute(long long *after, long long *before) {
1585 +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) {
1586 time_t now = now_realtime_sec() - 1;
1587
1588 int absolute_period_requested = -1;
@@ -1642,7 +1642,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
1642 *before = before_requested;
1643 *after = after_requested;
1644
1645 - return absolute_period_requested;
1645 + return (absolute_period_requested != 1);
1646 }
1647
1648 // #define DEBUG_QUERY_LOGIC 1
@@ -1651,7 +1651,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
1651 #define query_debug_log_init() BUFFER *debug_log = buffer_create(1000)
1652 #define query_debug_log(args...) buffer_sprintf(debug_log, ##args)
1653 #define query_debug_log_fin() { \
1654 - info("QUERY: chart '%s', after:%lld, before:%lld, duration:%lld, points:%ld, res:%ld - wanted => after:%lld, before:%lld, points:%ld, group:%ld, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", st->name, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
1654 + info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
1655 buffer_free(debug_log); \
1656 debug_log = NULL; \
1657 }
@@ -1663,21 +1663,18 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
1663 #define query_debug_log_free() debug_dummy()
1664 #endif
1665
1666 -RRDR *rrd2rrdr(
1667 - ONEWAYALLOC *owa
1668 - , RRDSET *st
1669 - , long points_requested
1670 - , long long after_requested
1671 - , long long before_requested
1672 - , RRDR_GROUPING group_method
1673 - , long resampling_time_requested
1674 - , RRDR_OPTIONS options
1675 - , const char *dimensions
1676 - , struct context_param *context_param_list
1677 - , const char *group_options
1678 - , int timeout
1679 - , int tier
1680 -) {
1666 +bool query_target_calculate_window(QUERY_TARGET *qt) {
1667 + if (unlikely(!qt)) return false;
1668 +
1669 + size_t points_requested = (long)qt->request.points;
1670 + time_t after_requested = qt->request.after;
1671 + time_t before_requested = qt->request.before;
1672 + RRDR_GROUPING group_method = qt->request.group_method;
1673 + time_t resampling_time_requested = qt->request.resampling_time;
1674 + RRDR_OPTIONS options = qt->request.options;
1675 + size_t tier = qt->request.tier;
1676 + time_t update_every = qt->db.minimum_latest_update_every;
1677 +
1678 // RULES
1679 // points_requested = 0
1680 // the user wants all the natural points the database has
@@ -1691,10 +1688,9 @@ RRDR *rrd2rrdr(
1688 // when natural points are wanted, the query has to be aligned to the update_every
1689 // of the database
1690
1694 - long points_wanted = points_requested;
1695 - long long after_wanted = after_requested;
1696 - long long before_wanted = before_requested;
1697 - int update_every = st->update_every;
1691 + size_t points_wanted = points_requested;
1692 + time_t after_wanted = after_requested;
1693 + time_t before_wanted = before_requested;
1694
1695 bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
1696 bool automatic_natural_points = (points_wanted == 0);
@@ -1704,13 +1700,7 @@ RRDR *rrd2rrdr(
1700
1701 query_debug_log_init();
1702
1707 - // make sure points_wanted is positive
1708 - if(points_wanted < 0) {
1709 - points_wanted = -points_wanted;
1710 - query_debug_log(":-points_wanted %ld", points_wanted);
1711 - }
1712 -
1713 - if(ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
1703 + if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
1704 relative_period_requested = true;
1705 natural_points = true;
1706 options |= RRDR_OPTION_NATURAL_POINTS;
@@ -1718,103 +1708,93 @@ RRDR *rrd2rrdr(
1708 }
1709
1710 // if the user wants virtual points, make sure we do it
1721 - if(options & RRDR_OPTION_VIRTUAL_POINTS)
1711 + if (options & RRDR_OPTION_VIRTUAL_POINTS)
1712 natural_points = false;
1713
1714 // set the right flag about natural and virtual points
1725 - if(natural_points) {
1715 + if (natural_points) {
1716 options |= RRDR_OPTION_NATURAL_POINTS;
1717
1728 - if(options & RRDR_OPTION_VIRTUAL_POINTS)
1718 + if (options & RRDR_OPTION_VIRTUAL_POINTS)
1719 options &= ~RRDR_OPTION_VIRTUAL_POINTS;
1720 }
1721 else {
1722 options |= RRDR_OPTION_VIRTUAL_POINTS;
1723
1734 - if(options & RRDR_OPTION_NATURAL_POINTS)
1724 + if (options & RRDR_OPTION_NATURAL_POINTS)
1725 options &= ~RRDR_OPTION_NATURAL_POINTS;
1726 }
1727
1738 - if(after_wanted == 0 || before_wanted == 0) {
1739 - // for non-context queries we have to find the duration of the database
1740 - // for context queries we will assume 600 seconds duration
1741 -
1742 - if(!context_param_list) {
1743 - relative_period_requested = true;
1744 -
1745 - time_t first_entry_t = rrdset_first_entry_t(st);
1746 - time_t last_entry_t = rrdset_last_entry_t(st);
1747 -
1748 - if(first_entry_t == 0 || last_entry_t == 0) {
1749 - internal_error(true, "QUERY: chart without data detected on '%s'", rrdset_name(st));
1750 - query_debug_log_free();
1751 - return NULL;
1752 - }
1728 + if (after_wanted == 0 || before_wanted == 0) {
1729 + relative_period_requested = true;
1730
1754 - query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
1731 + time_t first_entry_t = qt->db.first_time_t;
1732 + time_t last_entry_t = qt->db.last_time_t;
1733
1756 - if (after_wanted == 0) {
1757 - after_wanted = first_entry_t;
1758 - query_debug_log(":zero after_wanted %lld", after_wanted);
1759 - }
1734 + if (first_entry_t == 0 || last_entry_t == 0) {
1735 + internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld", qt->id, first_entry_t, last_entry_t);
1736 + query_debug_log_free();
1737 + return false;
1738 + }
1739
1761 - if (before_wanted == 0) {
1762 - before_wanted = last_entry_t;
1763 - before_is_aligned_to_db_end = true;
1764 - query_debug_log(":zero before_wanted %lld", before_wanted);
1765 - }
1740 + query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
1741
1767 - if(points_wanted == 0) {
1768 - points_wanted = (last_entry_t - first_entry_t) / update_every;
1769 - query_debug_log(":zero points_wanted %ld", points_wanted);
1770 - }
1742 + if (after_wanted == 0) {
1743 + after_wanted = first_entry_t;
1744 + query_debug_log(":zero after_wanted %ld", after_wanted);
1745 }
1746
1773 - // if they are still zero, assume 600
1747 + if (before_wanted == 0) {
1748 + before_wanted = last_entry_t;
1749 + before_is_aligned_to_db_end = true;
1750 + query_debug_log(":zero before_wanted %ld", before_wanted);
1751 + }
1752
1775 - if(after_wanted == 0) {
1776 - after_wanted = -600;
1777 - query_debug_log(":zero600 after_wanted %lld", after_wanted);
1753 + if (points_wanted == 0) {
1754 + points_wanted = (last_entry_t - first_entry_t) / update_every;
1755 + query_debug_log(":zero points_wanted %zu", points_wanted);
1756 }
1757 }
1758
1781 - if(points_wanted == 0) {
1759 + if (points_wanted == 0) {
1760 points_wanted = 600;
1783 - query_debug_log(":zero600 points_wanted %ld", points_wanted);
1761 + query_debug_log(":zero600 points_wanted %zu", points_wanted);
1762 }
1763
1764 // convert our before_wanted and after_wanted to absolute
1765 rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
1788 - query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted);
1766 + query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
1767
1790 - if(natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
1791 - update_every = rrdset_find_natural_update_every_for_timeframe(st, after_wanted, before_wanted, points_wanted, options, tier);
1792 - if(update_every <= 0) update_every = st->update_every;
1793 - query_debug_log(":natural update every %d", update_every);
1768 + if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
1769 + update_every = rrdset_find_natural_update_every_for_timeframe(
1770 + qt, after_wanted, before_wanted, points_wanted, options, tier);
1771 +
1772 + if (update_every <= 0) update_every = qt->db.minimum_latest_update_every;
1773 + query_debug_log(":natural update every %ld", update_every);
1774 }
1775
1776 // this is the update_every of the query
1777 // it may be different to the update_every of the database
1798 - time_t query_granularity = (natural_points)?update_every:1;
1799 - if(query_granularity <= 0) query_granularity = 1;
1778 + time_t query_granularity = (natural_points) ? update_every : 1;
1779 + if (query_granularity <= 0) query_granularity = 1;
1780 query_debug_log(":query_granularity %ld", query_granularity);
1781
1782 // align before_wanted and after_wanted to query_granularity
1783 if (before_wanted % query_granularity) {
1784 before_wanted -= before_wanted % query_granularity;
1805 - query_debug_log(":granularity align before_wanted %lld", before_wanted);
1785 + query_debug_log(":granularity align before_wanted %ld", before_wanted);
1786 }
1787
1788 if (after_wanted % query_granularity) {
1789 after_wanted -= after_wanted % query_granularity;
1810 - query_debug_log(":granularity align after_wanted %lld", after_wanted);
1790 + query_debug_log(":granularity align after_wanted %ld", after_wanted);
1791 }
1792
1793 // automatic_natural_points is set when the user wants all the points available in the database
1814 - if(automatic_natural_points) {
1794 + if (automatic_natural_points) {
1795 points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
1816 - if(unlikely(points_wanted <= 0)) points_wanted = 1;
1817 - query_debug_log(":auto natural points_wanted %ld", points_wanted);
1796 + if (unlikely(points_wanted <= 0)) points_wanted = 1;
1797 + query_debug_log(":auto natural points_wanted %zu", points_wanted);
1798 }
1799
1800 time_t duration = before_wanted - after_wanted;
@@ -1823,42 +1803,42 @@ RRDR *rrd2rrdr(
1803 if (unlikely(resampling_time_requested > duration)) {
1804 after_wanted = before_wanted - resampling_time_requested;
1805 duration = before_wanted - after_wanted;
1826 - query_debug_log(":resampling after_wanted %lld", after_wanted);
1806 + query_debug_log(":resampling after_wanted %ld", after_wanted);
1807 }
1808
1809 // if the duration is not aligned to resampling time
1810 // extend the duration to the past, to avoid a gap at the chart
1811 // only when the missing duration is above 1/10th of a point
1832 - if(resampling_time_requested > query_granularity && duration % resampling_time_requested) {
1812 + if (resampling_time_requested > query_granularity && duration % resampling_time_requested) {
1813 time_t delta = duration % resampling_time_requested;
1834 - if(delta > resampling_time_requested / 10) {
1814 + if (delta > resampling_time_requested / 10) {
1815 after_wanted -= resampling_time_requested - delta;
1816 duration = before_wanted - after_wanted;
1837 - query_debug_log(":resampling2 after_wanted %lld", after_wanted);
1817 + query_debug_log(":resampling2 after_wanted %ld", after_wanted);
1818 }
1819 }
1820
1821 // the available points of the query
1842 - long points_available = (duration + 1) / query_granularity;
1843 - if(unlikely(points_available <= 0)) points_available = 1;
1844 - query_debug_log(":points_available %ld", points_available);
1822 + size_t points_available = (duration + 1) / query_granularity;
1823 + if (unlikely(points_available <= 0)) points_available = 1;
1824 + query_debug_log(":points_available %zu", points_available);
1825
1846 - if(points_wanted > points_available) {
1826 + if (points_wanted > points_available) {
1827 points_wanted = points_available;
1848 - query_debug_log(":max points_wanted %ld", points_wanted);
1828 + query_debug_log(":max points_wanted %zu", points_wanted);
1829 }
1830
1831 // calculate the desired grouping of source data points
1852 - long group = points_available / points_wanted;
1853 - if(group <= 0) group = 1;
1832 + size_t group = points_available / points_wanted;
1833 + if (group == 0) group = 1;
1834
1835 // round "group" to the closest integer
1856 - if(points_available % points_wanted > points_wanted / 2)
1836 + if (points_available % points_wanted > points_wanted / 2)
1837 group++;
1838
1859 - query_debug_log(":group %ld", group);
1839 + query_debug_log(":group %zu", group);
1840
1861 - if(points_wanted * group * query_granularity < duration) {
1841 + if (points_wanted * group * query_granularity < (size_t)duration) {
1842 // the grouping we are going to do, is not enough
1843 // to cover the entire duration requested, so
1844 // we have to change the number of points, to make sure we will
@@ -1867,91 +1847,143 @@ RRDR *rrd2rrdr(
1847 // let's see how many points are the optimal
1848 points_wanted = points_available / group;
1849
1870 - if(points_wanted * group < points_available)
1850 + if (points_wanted * group < points_available)
1851 points_wanted++;
1852
1873 - if(unlikely(points_wanted <= 0))
1853 + if (unlikely(points_wanted == 0))
1854 points_wanted = 1;
1855
1876 - query_debug_log(":optimal points %ld", points_wanted);
1856 + query_debug_log(":optimal points %zu", points_wanted);
1857 }
1858
1859 // resampling_time_requested enforces a certain grouping multiple
1860 NETDATA_DOUBLE resampling_divisor = 1.0;
1881 - long resampling_group = 1;
1882 - if(unlikely(resampling_time_requested > query_granularity)) {
1861 + size_t resampling_group = 1;
1862 + if (unlikely(resampling_time_requested > query_granularity)) {
1863 // the points we should group to satisfy gtime
1864 resampling_group = resampling_time_requested / query_granularity;
1885 - if(unlikely(resampling_time_requested % query_granularity))
1865 + if (unlikely(resampling_time_requested % query_granularity))
1866 resampling_group++;
1867
1888 - query_debug_log(":resampling group %ld", resampling_group);
1868 + query_debug_log(":resampling group %zu", resampling_group);
1869
1870 // adapt group according to resampling_group
1891 - if(unlikely(group < resampling_group)) {
1892 - group = resampling_group; // do not allow grouping below the desired one
1893 - query_debug_log(":group less res %ld", group);
1871 + if (unlikely(group < resampling_group)) {
1872 + group = resampling_group; // do not allow grouping below the desired one
1873 + query_debug_log(":group less res %zu", group);
1874 }
1895 - if(unlikely(group % resampling_group)) {
1875 + if (unlikely(group % resampling_group)) {
1876 group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
1897 - query_debug_log(":group mod res %ld", group);
1877 + query_debug_log(":group mod res %zu", group);
1878 }
1879
1880 // resampling_divisor = group / resampling_group;
1901 - resampling_divisor = (NETDATA_DOUBLE)(group * query_granularity) / (NETDATA_DOUBLE)resampling_time_requested;
1881 + resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested;
1882 query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor);
1883 }
1884
1885 // now that we have group, align the requested timeframe to fit it.
1906 - if(aligned && before_wanted % (group * query_granularity)) {
1907 - if(before_is_aligned_to_db_end)
1908 - before_wanted -= before_wanted % (group * query_granularity);
1886 + if (aligned && before_wanted % (group * query_granularity)) {
1887 + if (before_is_aligned_to_db_end)
1888 + before_wanted -= before_wanted % (time_t)(group * query_granularity);
1889 else
1910 - before_wanted += (group * query_granularity) - before_wanted % (group * query_granularity);
1911 - query_debug_log(":align before_wanted %lld", before_wanted);
1890 + before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity);
1891 + query_debug_log(":align before_wanted %ld", before_wanted);
1892 }
1893
1914 - after_wanted = before_wanted - (points_wanted * group * query_granularity) + query_granularity;
1915 - query_debug_log(":final after_wanted %lld", after_wanted);
1894 + after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity;
1895 + query_debug_log(":final after_wanted %ld", after_wanted);
1896
1897 duration = before_wanted - after_wanted;
1898 query_debug_log(":final duration %ld", duration + 1);
1899
1920 - // check the context query based on the starting time of the query
1921 - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
1922 - rebuild_context_param_list(owa, context_param_list, after_wanted);
1923 - st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
1924 -
1925 - if(unlikely(!st))
1926 - return NULL;
1927 - }
1900 + query_debug_log_fin();
1901
1902 internal_error(points_wanted != duration / (query_granularity * group) + 1,
1930 - "QUERY: points_wanted %ld is not points %ld",
1931 - points_wanted, duration / (query_granularity * group) + 1);
1903 + "QUERY: points_wanted %zu is not points %zu",
1904 + points_wanted, (size_t)(duration / (query_granularity * group) + 1));
1905
1906 internal_error(group < resampling_group,
1934 - "QUERY: group %ld is less than the desired group points %ld",
1907 + "QUERY: group %zu is less than the desired group points %zu",
1908 group, resampling_group);
1909
1910 internal_error(group > resampling_group && group % resampling_group,
1938 - "QUERY: group %ld is not a multiple of the desired group points %ld",
1911 + "QUERY: group %zu is not a multiple of the desired group points %zu",
1912 group, resampling_group);
1913
1914 // -------------------------------------------------------------------------
1942 - // initialize our result set
1943 - // this also locks the chart for us
1915 + // update QUERY_TARGET with our calculations
1916 +
1917 + qt->window.after = after_wanted;
1918 + qt->window.before = before_wanted;
1919 + qt->window.relative = relative_period_requested;
1920 + qt->window.points = points_wanted;
1921 + qt->window.group = group;
1922 + qt->window.group_method = group_method;
1923 + qt->window.group_options = qt->request.group_options;
1924 + qt->window.query_granularity = query_granularity;
1925 + qt->window.resampling_group = resampling_group;
1926 + qt->window.resampling_divisor = resampling_divisor;
1927 + qt->window.options = options;
1928 + qt->window.tier = tier;
1929 + qt->window.aligned = aligned;
1930 +
1931 + return true;
1932 +}
1933
1945 - RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
1934 +RRDR *rrd2rrdr_legacy(
1935 + ONEWAYALLOC *owa,
1936 + RRDSET *st, size_t points, time_t after, time_t before,
1937 + RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,
1938 + const char *group_options, time_t timeout, size_t tier) {
1939 +
1940 + QUERY_TARGET_REQUEST qtr = {
1941 + .st = st,
1942 + .points = points,
1943 + .after = after,
1944 + .before = before,
1945 + .group_method = group_method,
1946 + .resampling_time = resampling_time,
1947 + .options = options,
1948 + .dimensions = dimensions,
1949 + .group_options = group_options,
1950 + .timeout = timeout,
1951 + .tier = tier,
1952 + };
1953 +
1954 + return rrd2rrdr(owa, query_target_create(&qtr));
1955 +}
1956 +
1957 +RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
1958 + if(!qt || !owa) return NULL;
1959 +
1960 + time_t timeout = qt->request.timeout;
1961 + time_t resampling_time_requested = qt->request.resampling_time;
1962 + time_t after_requested = qt->request.after;
1963 + time_t before_requested = qt->request.before;
1964 + size_t points_requested = qt->request.points;
1965 +
1966 + RRDR_OPTIONS options = qt->window.options;
1967 + size_t points_wanted = qt->window.points;
1968 + time_t after_wanted = qt->window.after;
1969 + time_t before_wanted = qt->window.before;
1970 + bool relative_period_requested = qt->window.relative;
1971 + bool aligned = qt->window.aligned;
1972 + RRDR_GROUPING group_method = qt->window.group_method;
1973 + size_t group = qt->window.group;
1974 + size_t resampling_group = qt->window.resampling_group;
1975 + time_t query_granularity = qt->window.query_granularity;
1976 +
1977 + RRDR *r = rrdr_create(owa, qt);
1978 if(unlikely(!r)) {
1947 - internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld",
1948 - rrdset_id(st), (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
1979 + internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu",
1980 + qt->id, after_wanted, before_wanted, points_wanted);
1981 return NULL;
1982 }
1983
1984 if(unlikely(!r->d || !points_wanted)) {
1953 - internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld",
1954 - rrdset_id(st), (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
1985 + internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu",
1986 + qt->id, after_wanted, before_wanted, points_wanted);
1987 return r;
1988 }
1989
@@ -1960,67 +1992,46 @@ RRDR *rrd2rrdr(
1992 else
1993 r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1994
1963 - // find how many dimensions we have
1964 - long dimensions_count = r->d;
1965 -
1995 // -------------------------------------------------------------------------
1996 // initialize RRDR
1997
1969 - r->group = group;
1970 - r->update_every = (int)(group * query_granularity);
1971 - r->before = before_wanted;
1972 - r->after = after_wanted;
1973 - r->internal.points_wanted = points_wanted;
1974 - r->internal.resampling_group = resampling_group;
1975 - r->internal.resampling_divisor = resampling_divisor;
1976 - r->internal.query_options = options;
1977 - r->internal.query_tier = tier;
1998 + r->group = qt->window.group;
1999 + r->update_every = (int) (qt->window.group * qt->window.query_granularity);
2000 + r->before = qt->window.before;
2001 + r->after = qt->window.after;
2002 + r->internal.points_wanted = qt->window.points;
2003 + r->internal.resampling_group = qt->window.resampling_group;
2004 + r->internal.resampling_divisor = qt->window.resampling_divisor;
2005 + r->internal.query_options = qt->window.options;
2006
2007 // -------------------------------------------------------------------------
2008 // assign the processor functions
1981 - rrdr_set_grouping_function(r, group_method);
2009 + rrdr_set_grouping_function(r, qt->window.group_method);
2010
2011 // allocate any memory required by the grouping method
1984 - r->internal.grouping_create(r, group_options);
1985 -
1986 -
1987 - // -------------------------------------------------------------------------
1988 - // disable the not-wanted dimensions
1989 -
1990 - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
1991 - rrdset_check_rdlock(st);
1992 -
1993 - if(dimensions && *dimensions)
1994 - rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
1995 -
1996 - query_debug_log_fin();
2012 + r->internal.grouping_create(r, qt->window.group_options);
2013
2014 // -------------------------------------------------------------------------
2015 // do the work for each dimension
2016
2017 time_t max_after = 0, min_before = 0;
2002 - long max_rows = 0;
2018 + size_t max_rows = 0;
2019
2004 - RRDDIM *first_rd = context_param_list ? context_param_list->rd : st->dimensions;
2005 - RRDDIM *rd;
2006 - long c, dimensions_used = 0, dimensions_nonzero = 0;
2020 + long dimensions_used = 0, dimensions_nonzero = 0;
2021 struct timeval query_start_time;
2022 struct timeval query_current_time;
2023 if (timeout) now_realtime_timeval(&query_start_time);
2024
2011 - for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
2025 + for(size_t c = 0, max = qt->query.used; c < max ; c++) {
2026 + // set the query target dimension options to rrdr
2027 + r->od[c] = qt->query.array[c].dimension.options;
2028
2013 - // if we need a percentage, we need to calculate all dimensions
2014 - if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) {
2015 - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
2016 - continue;
2017 - }
2029 r->od[c] |= RRDR_DIMENSION_SELECTED;
2030
2031 // reset the grouping for the new dimension
2032 r->internal.grouping_reset(r);
2033
2023 - rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted);
2034 + rrd2rrdr_do_dimension(r, c);
2035 if (timeout)
2036 now_realtime_timeval(&query_current_time);
2037
@@ -2036,29 +2047,29 @@ RRDR *rrd2rrdr(
2047 else {
2048 if(r->after != max_after) {
2049 internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
2039 - rrdset_name(st), (size_t)max_after, rrddim_name(rd), (size_t)r->after);
2050 + string2str(qt->query.array[c].dimension.id), (size_t)max_after, string2str(qt->query.array[c].dimension.name), (size_t)r->after);
2051
2052 r->after = (r->after > max_after) ? r->after : max_after;
2053 }
2054
2055 if(r->before != min_before) {
2056 internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
2046 - rrdset_name(st), (size_t)min_before, rrddim_name(rd), (size_t)r->before);
2057 + string2str(qt->query.array[c].dimension.id), (size_t)min_before, string2str(qt->query.array[c].dimension.name), (size_t)r->before);
2058
2059 r->before = (r->before < min_before) ? r->before : min_before;
2060 }
2061
2062 if(r->rows != max_rows) {
2063 internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
2053 - rrdset_name(st), (size_t)max_rows, rrddim_name(rd), (size_t)r->rows);
2064 + string2str(qt->query.array[c].dimension.id), (size_t)max_rows, string2str(qt->query.array[c].dimension.name), (size_t)r->rows);
2065
2066 r->rows = (r->rows > max_rows) ? r->rows : max_rows;
2067 }
2068 }
2069
2070 dimensions_used++;
2060 - if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
2061 - log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
2071 + if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)timeout) {
2072 + log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %ld ms)",
2073 (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
2074 r->result_options |= RRDR_RESULT_OPTION_CANCEL;
2075 break;
@@ -2118,7 +2129,7 @@ RRDR *rrd2rrdr(
2129 if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
2130 // all the dimensions are zero
2131 // mark them as NONZERO to send them all
2121 - for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
2132 + for(size_t c = 0, max = qt->query.used; c < max ; c++) {
2133 if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
2134 r->od[c] |= RRDR_DIMENSION_NONZERO;
2135 }
web/api/queries/rrdr.c
+20 -45
@@ -61,9 +61,7 @@ static void rrdr_dump(RRDR *r)
61 inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
62 if(unlikely(!r)) return;
63
64 - if(likely(r->st_locked_by_rrdr_create))
65 - rrdset_unlock(r->st);
66 -
64 + query_target_release(r->internal.qt);
65 onewayalloc_freez(owa, r->t);
66 onewayalloc_freez(owa, r->v);
67 onewayalloc_freez(owa, r->o);
@@ -72,12 +70,28 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
70 onewayalloc_freez(owa, r);
71 }
72
75 -RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) {
73 +RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
74 + if(unlikely(!qt))
75 + return NULL;
76 +
77 + if(unlikely(!qt->query.used || !qt->window.points)) {
78 + query_target_release(qt);
79 + return NULL;
80 + }
81 +
82 + size_t dimensions = qt->query.used;
83 + size_t points = qt->window.points;
84 +
85 + // create the rrdr
86 RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
87 r->internal.owa = owa;
88 + r->internal.qt = qt;
89
79 - r->d = dimensions;
80 - r->n = points;
90 + r->before = qt->window.before;
91 + r->after = qt->window.after;
92 + r->internal.points_wanted = qt->window.points;
93 + r->d = (int)dimensions;
94 + r->n = (int)points;
95
96 r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
97 r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
@@ -90,42 +104,3 @@ RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points
104
105 return r;
106 }
93 -
94 -RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) {
95 - if (unlikely(!st)) return NULL;
96 -
97 - bool st_locked_by_rrdr_create = false;
98 - if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
99 - rrdset_rdlock(st);
100 - st_locked_by_rrdr_create = true;
101 - }
102 -
103 - // count the number of dimensions
104 - int dimensions = 0;
105 - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
106 - RRDDIM *rd;
107 - if (temp_rd) {
108 - RRDDIM *t = temp_rd;
109 - while (t) {
110 - dimensions++;
111 - t = t->next;
112 - }
113 - } else
114 - dimensions = rrdset_number_of_dimensions(st);
115 -
116 - // create the rrdr
117 - RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n);
118 - r->st = st;
119 - r->st_locked_by_rrdr_create = st_locked_by_rrdr_create;
120 -
121 - // set the hidden flag on hidden dimensions
122 - int c;
123 - for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) {
124 - if (unlikely(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)))
125 - r->od[c] = RRDR_DIMENSION_HIDDEN;
126 - else
127 - r->od[c] = RRDR_DIMENSION_DEFAULT;
128 - }
129 -
130 - return r;
131 -}
web/api/queries/rrdr.h
+56 -54
@@ -18,33 +18,33 @@ typedef enum tier_query_fetch {
18 } TIER_QUERY_FETCH;
19
20 typedef enum rrdr_options {
21 - RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
22 - RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
23 - RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
24 - RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
25 - RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
26 - RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
27 - RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
28 - RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
29 - RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
30 - RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
31 - RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
32 - RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
33 - RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
34 - RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
35 - RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
36 - RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
37 - RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response
38 - RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
39 - RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
40 - RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number
41 - RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes
42 - RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap
43 - RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
21 + RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
22 + RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
23 + RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
24 + RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
25 + RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
26 + RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
27 + RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
28 + RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
29 + RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
30 + RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
31 + RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
32 + RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
33 + RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
34 + RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
35 + RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
36 + RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
37 + RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
38 + RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
39 + RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number
40 + RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes
41 + RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap
42 + RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
43 + RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list
44
45 // internal ones - not to be exposed to the API
46 - RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
47 - RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
46 + RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
47 + RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
48 } RRDR_OPTIONS;
49
50 typedef enum rrdr_value_flag {
@@ -68,16 +68,14 @@ typedef enum rrdr_result_flags {
68 // (should not to be cached by browsers and proxies)
69 RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames
70 RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled
71 -} RRDR_RESULT_FLAGS;
71 +} RRDR_RESULT_OPTIONS;
72
73 typedef struct rrdresult {
74 - struct rrdset *st; // the chart this result refers to
74 + RRDR_RESULT_OPTIONS result_options; // RRDR_RESULT_OPTION_*
75
76 - RRDR_RESULT_FLAGS result_options; // RRDR_RESULT_OPTION_*
77 -
78 - int d; // the number of dimensions
79 - long n; // the number of values in the arrays
80 - long rows; // the number of rows used
76 + size_t d; // the number of dimensions
77 + size_t n; // the number of values in the arrays
78 + size_t rows; // the number of rows used
79
80 RRDR_DIMENSION_FLAGS *od; // the options for the dimensions
81
@@ -86,8 +84,8 @@ typedef struct rrdresult {
84 RRDR_VALUE_FLAGS *o; // array n x d options for each value returned
85 NETDATA_DOUBLE *ar; // array n x d of anomaly rates (0 - 100)
86
89 - long group; // how many collected values were grouped for each row
90 - int update_every; // what is the suggested update frequency in seconds
87 + size_t group; // how many collected values were grouped for each row
88 + time_t update_every; // what is the suggested update frequency in seconds
89
90 NETDATA_DOUBLE min;
91 NETDATA_DOUBLE max;
@@ -95,33 +93,35 @@ typedef struct rrdresult {
93 time_t before;
94 time_t after;
95
98 - bool st_locked_by_rrdr_create; // if st is read locked by us
99 -
96 // internal rrd2rrdr() members below this point
97 struct {
102 - int query_tier; // the selected tier
103 - RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query)
98 + ONEWAYALLOC *owa; // the allocator used
99 + struct query_target *qt; // the QUERY_TARGET
100 +
101 + RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query)
102
105 - long points_wanted;
106 - long resampling_group;
107 - NETDATA_DOUBLE resampling_divisor;
103 + size_t points_wanted; // used by SES and DES
104 + size_t resampling_group; // used by AVERAGE
105 + NETDATA_DOUBLE resampling_divisor; // used by AVERAGE
106
107 + // grouping function pointers
108 void (*grouping_create)(struct rrdresult *r, const char *options);
109 void (*grouping_reset)(struct rrdresult *r);
110 void (*grouping_free)(struct rrdresult *r);
111 void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
112 NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
114 - void *grouping_data;
113
116 - TIER_QUERY_FETCH tier_query_fetch;
117 - #ifdef NETDATA_INTERNAL_CHECKS
114 + TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT
115 + void *grouping_data; // the internal data of the grouping function
116 +
117 +#ifdef NETDATA_INTERNAL_CHECKS
118 const char *log;
119 - #endif
119 +#endif
120
121 + // statistics
122 size_t db_points_read;
123 size_t result_points_generated;
124 size_t tier_points_read[RRD_STORAGE_TIERS];
124 - ONEWAYALLOC *owa;
125 } internal;
126 } RRDR;
127
@@ -129,19 +129,21 @@ typedef struct rrdresult {
129
130 #include "database/rrd.h"
131 void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
132 -RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list);
133 -RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points);
132 +RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt);
133
134 #include "../web_api_v1.h"
135 #include "web/api/queries/query.h"
136
138 -RRDR *rrd2rrdr(
139 - ONEWAYALLOC *owa,
140 - RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted,
141 - RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
142 - struct context_param *context_param_list, const char *group_options, int timeout, int tier);
137 +RRDR *rrd2rrdr_legacy(
138 + ONEWAYALLOC *owa,
139 + RRDSET *st, size_t points, time_t after, time_t before,
140 + RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,
141 + const char *group_options, time_t timeout, size_t tier);
142 +
143 +RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
144 +bool query_target_calculate_window(struct query_target *qt);
145
144 -int rrdr_relative_window_to_absolute(long long *after, long long *before);
146 +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before);
147
148 #ifdef __cplusplus
149 }
web/api/queries/weights.c
+322 -428
@@ -56,35 +56,14 @@ typedef enum {
56
57 struct register_result {
58 RESULT_FLAGS flags;
59 - RRDSET *st;
60 - const char *chart_id;
61 - const char *context;
62 - const char *dim_name;
59 + RRDCONTEXT_ACQUIRED *rca;
60 + RRDINSTANCE_ACQUIRED *ria;
61 + RRDMETRIC_ACQUIRED *rma;
62 NETDATA_DOUBLE value;
64 -
65 - struct register_result *next; // used to link contexts together
63 };
64
68 -static void register_result_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
69 - struct register_result *t = (struct register_result *)value;
70 -
71 - if(t->chart_id) t->chart_id = strdupz(t->chart_id);
72 - if(t->context) t->context = strdupz(t->context);
73 - if(t->dim_name) t->dim_name = strdupz(t->dim_name);
74 -}
75 -
76 -static void register_result_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
77 - struct register_result *t = (struct register_result *)value;
78 -
79 - freez((void *)t->chart_id);
80 - freez((void *)t->context);
81 - freez((void *)t->dim_name);
82 -}
83 -
65 static DICTIONARY *register_result_init() {
66 DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED);
86 - dictionary_register_insert_callback(results, register_result_insert_callback, results);
87 - dictionary_register_delete_callback(results, register_result_delete_callback, results);
67 return results;
68 }
69
@@ -93,8 +72,9 @@ static void register_result_destroy(DICTIONARY *results) {
72 }
73
74 static void register_result(DICTIONARY *results,
96 - RRDSET *st,
97 - RRDDIM *d,
75 + RRDCONTEXT_ACQUIRED *rca,
76 + RRDINSTANCE_ACQUIRED *ria,
77 + RRDMETRIC_ACQUIRED *rma,
78 NETDATA_DOUBLE value,
79 RESULT_FLAGS flags,
80 WEIGHTS_STATS *stats,
@@ -115,34 +95,34 @@ static void register_result(DICTIONARY *results,
95
96 struct register_result t = {
97 .flags = flags,
118 - .st = st,
119 - .chart_id = rrdset_id(st),
120 - .context = rrdset_context(st),
121 - .dim_name = rrddim_name(d),
98 + .rca = rca,
99 + .ria = ria,
100 + .rma = rma,
101 .value = v
102 };
103
125 - char buf[5000 + 1];
126 - snprintfz(buf, 5000, "%s:%s", rrdset_id(st), rrddim_name(d));
127 - dictionary_set(results, buf, &t, sizeof(struct register_result));
104 + // we can use the pointer address or RMA as a unique key for each metric
105 + char buf[20 + 1];
106 + ssize_t len = snprintfz(buf, 20, "%p", rma);
107 + dictionary_set_advanced(results, buf, len + 1, &t, sizeof(struct register_result), NULL);
108 }
109
110 // ----------------------------------------------------------------------------
111 // Generation of JSON output for the results
112
113 static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *wb,
134 - long long after, long long before,
135 - long long baseline_after, long long baseline_before,
136 - long points, WEIGHTS_METHOD method,
114 + time_t after, time_t before,
115 + time_t baseline_after, time_t baseline_before,
116 + size_t points, WEIGHTS_METHOD method,
117 RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
118 size_t examined_dimensions __maybe_unused, usec_t duration,
119 WEIGHTS_STATS *stats) {
120
121 buffer_sprintf(wb, "{\n"
142 - "\t\"after\": %lld,\n"
143 - "\t\"before\": %lld,\n"
144 - "\t\"duration\": %lld,\n"
145 - "\t\"points\": %ld,\n",
122 + "\t\"after\": %ld,\n"
123 + "\t\"before\": %ld,\n"
124 + "\t\"duration\": %ld,\n"
125 + "\t\"points\": %zu,\n",
126 after,
127 before,
128 before - after,
@@ -151,10 +131,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
131
132 if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME)
133 buffer_sprintf(wb, ""
154 - "\t\"baseline_after\": %lld,\n"
155 - "\t\"baseline_before\": %lld,\n"
156 - "\t\"baseline_duration\": %lld,\n"
157 - "\t\"baseline_points\": %ld,\n",
134 + "\t\"baseline_after\": %ld,\n"
135 + "\t\"baseline_before\": %ld,\n"
136 + "\t\"baseline_duration\": %ld,\n"
137 + "\t\"baseline_points\": %zu,\n",
138 baseline_after,
139 baseline_before,
140 baseline_before - baseline_after,
@@ -176,7 +156,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
156 stats->db_points
157 );
158
179 - for(int tier = 0; tier < storage_tiers ;tier++)
159 + for(size_t tier = 0; tier < storage_tiers ;tier++)
160 buffer_sprintf(wb, "%s%zu", tier?", ":"", stats->db_points_per_tier[tier]);
161
162 buffer_sprintf(wb, " ]\n"
@@ -188,13 +168,13 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
168 weights_method_to_string(method)
169 );
170
191 - web_client_api_request_v1_data_options_to_string(wb, options);
171 + web_client_api_request_v1_data_options_to_buffer(wb, options);
172 }
173
174 static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
195 - long long after, long long before,
196 - long long baseline_after, long long baseline_before,
197 - long points, WEIGHTS_METHOD method,
175 + time_t after, time_t before,
176 + time_t baseline_after, time_t baseline_before,
177 + size_t points, WEIGHTS_METHOD method,
178 RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
179 size_t examined_dimensions, usec_t duration,
180 WEIGHTS_STATS *stats) {
@@ -206,23 +186,23 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
186
187 size_t charts = 0, chart_dims = 0, total_dimensions = 0;
188 struct register_result *t;
209 - RRDSET *last_st = NULL; // never access this - we use it only for comparison
189 + RRDINSTANCE_ACQUIRED *last_ria = NULL; // never access this - we use it only for comparison
190 dfe_start_read(results, t) {
211 - if(!last_st || t->st != last_st) {
212 - last_st = t->st;
191 + if(t->ria != last_ria) {
192 + last_ria = t->ria;
193
194 if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n");
195 buffer_strcat(wb, "\t\t\"");
216 - buffer_strcat(wb, t->chart_id);
196 + buffer_strcat(wb, rrdinstance_acquired_id(t->ria));
197 buffer_strcat(wb, "\": {\n");
198 buffer_strcat(wb, "\t\t\t\"context\": \"");
219 - buffer_strcat(wb, t->context);
199 + buffer_strcat(wb, rrdcontext_acquired_id(t->rca));
200 buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
201 charts++;
202 chart_dims = 0;
203 }
204 if (chart_dims) buffer_sprintf(wb, ",\n");
225 - buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value);
205 + buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value);
206 chart_dims++;
207 total_dimensions++;
208 }
@@ -245,9 +225,9 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
225 }
226
227 static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *wb,
248 - long long after, long long before,
249 - long long baseline_after, long long baseline_before,
250 - long points, WEIGHTS_METHOD method,
228 + time_t after, time_t before,
229 + time_t baseline_after, time_t baseline_before,
230 + size_t points, WEIGHTS_METHOD method,
231 RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
232 size_t examined_dimensions, usec_t duration,
233 WEIGHTS_STATS *stats) {
@@ -255,75 +235,80 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
235 results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
236 points, method, group, options, shifts, examined_dimensions, duration, stats);
237
258 - DICTIONARY *context_results = dictionary_create(
259 - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_NAME_LINK_DONT_CLONE |
260 - DICT_OPTION_DONT_OVERWRITE_VALUE);
238 + buffer_strcat(wb, "\",\n\t\"contexts\": {\n");
239
240 + size_t contexts = 0, charts = 0, total_dimensions = 0, context_dims = 0, chart_dims = 0;
241 + NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0;
242 struct register_result *t;
243 + RRDCONTEXT_ACQUIRED *last_rca = NULL;
244 + RRDINSTANCE_ACQUIRED *last_ria = NULL;
245 dfe_start_read(results, t) {
264 - struct register_result *tc = dictionary_set(context_results, t->context, t, sizeof(*t));
265 - if(tc == t)
266 - t->next = NULL;
267 - else {
268 - t->next = tc->next;
269 - tc->next = t;
246 +
247 + if(t->rca != last_rca) {
248 + last_rca = t->rca;
249 +
250 + if(contexts)
251 + buffer_sprintf(wb, "\n"
252 + "\t\t\t\t\t},\n"
253 + "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
254 + "\t\t\t\t}\n\t\t\t},\n"
255 + "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n"
256 + , charts_total_weight / (double)chart_dims
257 + , contexts_total_weight / (double)context_dims);
258 +
259 + buffer_strcat(wb, "\t\t\"");
260 + buffer_strcat(wb, rrdcontext_acquired_id(t->rca));
261 + buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n");
262 +
263 + contexts++;
264 + charts = 0;
265 + context_dims = 0;
266 + contexts_total_weight = 0.0;
267 +
268 + last_ria = NULL;
269 }
271 - }
272 - dfe_done(t);
270
274 - buffer_strcat(wb, "\",\n\t\"contexts\": {\n");
271 + if(t->ria != last_ria) {
272 + last_ria = t->ria;
273
276 - size_t contexts = 0, total_dimensions = 0, charts = 0, context_dims = 0, chart_dims = 0;
277 - NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0;
278 - RRDSET *last_st = NULL; // never access this - we use it only for comparison
279 - dfe_start_read(context_results, t) {
280 -
281 - if(contexts)
282 - buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims);
283 -
284 - contexts++;
285 - context_dims = 0;
286 - contexts_total_weight = 0.0;
287 -
288 - buffer_strcat(wb, "\t\t\"");
289 - buffer_strcat(wb, t->context);
290 - buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n");
291 -
292 - charts = 0;
293 - chart_dims = 0;
294 - struct register_result *tt;
295 - for(tt = t; tt ; tt = tt->next) {
296 - if(!last_st || tt->st != last_st) {
297 - last_st = tt->st;
298 -
299 - if(charts)
300 - buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t},\n", charts_total_weight / chart_dims);
301 -
302 - buffer_strcat(wb, "\t\t\t\t\"");
303 - buffer_strcat(wb, tt->chart_id);
304 - buffer_strcat(wb, "\": {\n");
305 - buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n");
306 - charts++;
307 - chart_dims = 0;
308 - charts_total_weight = 0.0;
309 - }
310 -
311 - if (chart_dims) buffer_sprintf(wb, ",\n");
312 - buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, tt->dim_name, tt->value);
313 - charts_total_weight += tt->value;
314 - contexts_total_weight += tt->value;
315 - chart_dims++;
316 - context_dims++;
317 - total_dimensions++;
274 + if(charts)
275 + buffer_sprintf(wb, "\n"
276 + "\t\t\t\t\t},\n"
277 + "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
278 + "\t\t\t\t},\n"
279 + , charts_total_weight / (double)chart_dims);
280 +
281 + buffer_strcat(wb, "\t\t\t\t\"");
282 + buffer_strcat(wb, rrdinstance_acquired_id(t->ria));
283 + buffer_strcat(wb, "\": {\n");
284 + buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n");
285 +
286 + charts++;
287 + chart_dims = 0;
288 + charts_total_weight = 0.0;
289 }
290 +
291 + if (chart_dims) buffer_sprintf(wb, ",\n");
292 + buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value);
293 + charts_total_weight += t->value;
294 + contexts_total_weight += t->value;
295 + chart_dims++;
296 + context_dims++;
297 + total_dimensions++;
298 }
299 dfe_done(t);
300
322 - dictionary_destroy(context_results);
323 -
301 // close dimensions and chart
302 if (total_dimensions)
326 - buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t}\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims);
303 + buffer_sprintf(wb, "\n"
304 + "\t\t\t\t\t},\n"
305 + "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
306 + "\t\t\t\t}\n"
307 + "\t\t\t},\n"
308 + "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
309 + "\t\t}\n"
310 + , charts_total_weight / (double)chart_dims
311 + , contexts_total_weight / (double)context_dims);
312
313 // close correlated_charts
314 buffer_sprintf(wb, "\t},\n"
@@ -383,7 +368,10 @@ static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, si
368 return added;
369 }
370
386 -static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMBERS highlight_diffs[], int high_size, uint32_t base_shifts) {
371 +static double ks_2samp(
372 + DIFFS_NUMBERS baseline_diffs[], int base_size,
373 + DIFFS_NUMBERS highlight_diffs[], int high_size,
374 + uint32_t base_shifts) {
375
376 qsort(baseline_diffs, base_size, sizeof(DIFFS_NUMBERS), compare_diffs);
377 qsort(highlight_diffs, high_size, sizeof(DIFFS_NUMBERS), compare_diffs);
@@ -406,7 +394,7 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
394 // This would require a lot of multiplications and divisions.
395 //
396 // To speed it up, we do the binary search to find the index of each number
409 - // but then we divide the base index by the power of two number (shifts) it
397 + // but, then we divide the base index by the power of two number (shifts) it
398 // is bigger than high index. So the 2 indexes are now comparable.
399 // We also keep track of the original indexes with min and max, to properly
400 // calculate their percentages once the loops finish.
@@ -487,7 +475,9 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
475
476 static double kstwo(
477 NETDATA_DOUBLE baseline[], int baseline_points,
490 - NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) {
478 + NETDATA_DOUBLE highlight[], int highlight_points,
479 + uint32_t base_shifts) {
480 +
481 // -1 in size, since the calculate_pairs_diffs() returns one less point
482 DIFFS_NUMBERS baseline_diffs[baseline_points - 1];
483 DIFFS_NUMBERS highlight_diffs[highlight_points - 1];
@@ -506,313 +496,214 @@ static double kstwo(
496 return ks_2samp(baseline_diffs, base_size, highlight_diffs, high_size, base_shifts);
497 }
498
499 +NETDATA_DOUBLE *rrd2rrdr_ks2(
500 + ONEWAYALLOC *owa, RRDHOST *host,
501 + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
502 + time_t after, time_t before, size_t points, RRDR_OPTIONS options,
503 + RRDR_GROUPING group_method, const char *group_options, size_t tier,
504 + WEIGHTS_STATS *stats,
505 + size_t *entries
506 + ) {
507 +
508 + NETDATA_DOUBLE *ret = NULL;
509 +
510 + QUERY_TARGET_REQUEST qtr = {
511 + .host = host,
512 + .rca = rca,
513 + .ria = ria,
514 + .rma = rma,
515 + .after = after,
516 + .before = before,
517 + .points = points,
518 + .options = options,
519 + .group_method = group_method,
520 + .group_options = group_options,
521 + .tier = tier
522 + };
523
510 -static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
511 - long long baseline_after, long long baseline_before,
512 - long long after, long long before,
513 - long long points, RRDR_OPTIONS options,
514 - RRDR_GROUPING group, const char *group_options, int tier,
515 - uint32_t shifts, int timeout,
516 - WEIGHTS_STATS *stats, bool register_zero) {
517 - options |= RRDR_OPTION_NATURAL_POINTS;
518 -
519 - long group_time = 0;
520 - struct context_param *context_param_list = NULL;
521 -
522 - int examined_dimensions = 0;
523 -
524 - RRDR *high_rrdr = NULL;
525 - RRDR *base_rrdr = NULL;
526 -
527 - // get first the highlight to find the number of points available
528 - stats->db_queries++;
529 - usec_t started_usec = now_realtime_usec();
530 - ONEWAYALLOC *owa = onewayalloc_create(0);
531 - high_rrdr = rrd2rrdr(owa, st, points,
532 - after, before, group,
533 - group_time, options, NULL, context_param_list, group_options,
534 - timeout, tier);
535 - if(!high_rrdr) {
536 - info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", rrdset_name(st));
524 + RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
525 + if(!r)
526 goto cleanup;
538 - }
527
540 - for(int i = 0; i < storage_tiers ;i++)
541 - stats->db_points_per_tier[i] += high_rrdr->internal.tier_points_read[i];
528 + stats->db_queries++;
529 + stats->result_points += r->internal.result_points_generated;
530 + stats->db_points += r->internal.db_points_read;
531 + for(size_t tr = 0; tr < storage_tiers ; tr++)
532 + stats->db_points_per_tier[tr] += r->internal.tier_points_read[tr];
533
543 - stats->db_points += high_rrdr->internal.db_points_read;
544 - stats->result_points += high_rrdr->internal.result_points_generated;
545 - if(!high_rrdr->d) {
546 - info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", rrdset_name(st));
534 + if(r->d != 1) {
535 + error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d);
536 goto cleanup;
537 }
549 - if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
550 - info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", rrdset_name(st));
538 +
539 + if(unlikely(r->od[0] & RRDR_DIMENSION_HIDDEN))
540 goto cleanup;
552 - }
553 - int high_points = rrdr_rows(high_rrdr);
541
555 - usec_t now_usec = now_realtime_usec();
556 - if(now_usec - started_usec > timeout * USEC_PER_MS)
542 + if(unlikely(!(r->od[0] & RRDR_DIMENSION_NONZERO)))
543 goto cleanup;
544
559 - // get the baseline, requesting the same number of points as the highlight
560 - stats->db_queries++;
561 - base_rrdr = rrd2rrdr(owa, st,high_points << shifts,
562 - baseline_after, baseline_before, group,
563 - group_time, options, NULL, context_param_list, group_options,
564 - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier);
565 - if(!base_rrdr) {
566 - info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", rrdset_name(st));
545 + if(rrdr_rows(r) < 2)
546 goto cleanup;
568 - }
547
570 - for(int i = 0; i < storage_tiers ;i++)
571 - stats->db_points_per_tier[i] += base_rrdr->internal.tier_points_read[i];
548 + *entries = rrdr_rows(r);
549 + ret = onewayalloc_mallocz(owa, sizeof(NETDATA_DOUBLE) * rrdr_rows(r));
550
573 - stats->db_points += base_rrdr->internal.db_points_read;
574 - stats->result_points += base_rrdr->internal.result_points_generated;
575 - if(!base_rrdr->d) {
576 - info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", rrdset_name(st));
577 - goto cleanup;
578 - }
579 - if (base_rrdr->d != high_rrdr->d) {
580 - info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", rrdset_name(st));
581 - goto cleanup;
582 - }
583 - if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
584 - info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", rrdset_name(st));
585 - goto cleanup;
586 - }
587 - int base_points = rrdr_rows(base_rrdr);
551 + // copy the points of the dimension to a contiguous array
552 + // there is no need to check for empty values, since empty values are already zero
553 + // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
554 + memcpy(ret, r->v, rrdr_rows(r) * sizeof(NETDATA_DOUBLE));
555
589 - now_usec = now_realtime_usec();
590 - if(now_usec - started_usec > timeout * USEC_PER_MS)
591 - goto cleanup;
556 +cleanup:
557 + rrdr_free(owa, r);
558 + return ret;
559 +}
560 +
561 +static void rrdset_metric_correlations_ks2(
562 + RRDHOST *host,
563 + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
564 + DICTIONARY *results,
565 + time_t baseline_after, time_t baseline_before,
566 + time_t after, time_t before,
567 + size_t points, RRDR_OPTIONS options,
568 + RRDR_GROUPING group_method, const char *group_options, size_t tier,
569 + uint32_t shifts,
570 + WEIGHTS_STATS *stats, bool register_zero
571 + ) {
572
593 - // we need at least 2 points to do the job
594 - if(base_points < 2 || high_points < 2)
573 + options |= RRDR_OPTION_NATURAL_POINTS;
574 +
575 + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
576 +
577 + size_t high_points = 0;
578 + NETDATA_DOUBLE *highlight = rrd2rrdr_ks2(
579 + owa, host, rca, ria, rma, after, before, points,
580 + options, group_method, group_options, tier, stats, &high_points);
581 +
582 + if(!highlight)
583 goto cleanup;
584
597 - // for each dimension
598 - RRDDIM *d;
599 - int i;
600 - rrddim_foreach_read(d, base_rrdr->st) {
601 - if(unlikely((int)d_dfe.counter >= base_rrdr->d)) break;
602 - i = (int)d_dfe.counter; // d_counter is provided by the dictionary
585 + size_t base_points = 0;
586 + NETDATA_DOUBLE *baseline = rrd2rrdr_ks2(
587 + owa, host, rca, ria, rma, baseline_after, baseline_before, high_points << shifts,
588 + options, group_method, group_options, tier, stats, &base_points);
589
604 - // skip the not evaluated ones
605 - if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN))
606 - continue;
590 + if(!baseline)
591 + goto cleanup;
592
608 - examined_dimensions++;
593 + stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
594 +
595 + double prob = kstwo(baseline, (int)base_points, highlight, (int)high_points, shifts);
596 + if(!isnan(prob) && !isinf(prob)) {
597
610 - // skip the dimensions that are just zero for both the baseline and the highlight
611 - if(unlikely(!(base_rrdr->od[i] & RRDR_DIMENSION_NONZERO) && !(high_rrdr->od[i] & RRDR_DIMENSION_NONZERO)))
612 - continue;
613 -
614 - // copy the baseline points of the dimension to a contiguous array
615 - // there is no need to check for empty values, since empty are already zero
616 - NETDATA_DOUBLE baseline[base_points];
617 - for(int c = 0; c < base_points; c++)
618 - baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ];
619 -
620 - // copy the highlight points of the dimension to a contiguous array
621 - // there is no need to check for empty values, since empty values are already zero
622 - // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
623 - NETDATA_DOUBLE highlight[high_points];
624 - for(int c = 0; c < high_points; c++)
625 - highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ];
626 -
627 - stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
628 -
629 - double prob = kstwo(baseline, base_points, highlight, high_points, shifts);
630 - if(!isnan(prob) && !isinf(prob)) {
631 -
632 - // these conditions should never happen, but still let's check
633 - if(unlikely(prob < 0.0)) {
634 - error("Metric correlations: kstwo() returned a negative number: %f", prob);
635 - prob = -prob;
636 - }
637 - if(unlikely(prob > 1.0)) {
638 - error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
639 - prob = 1.0;
640 - }
641 -
642 - // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
643 - // so we flip the result of kstwo()
644 - register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);
598 + // these conditions should never happen, but still let's check
599 + if(unlikely(prob < 0.0)) {
600 + error("Metric correlations: kstwo() returned a negative number: %f", prob);
601 + prob = -prob;
602 }
603 + if(unlikely(prob > 1.0)) {
604 + error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
605 + prob = 1.0;
606 + }
607 +
608 + // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
609 + // so, we flip the result of kstwo()
610 + register_result(results, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);
611 }
647 - rrddim_foreach_done(d);
612
613 cleanup:
650 - rrdr_free(owa, high_rrdr);
651 - rrdr_free(owa, base_rrdr);
614 onewayalloc_destroy(owa);
653 - return examined_dimensions;
615 }
616
617 // ----------------------------------------------------------------------------
618 // VOLUME algorithm functions
619
659 -static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
660 - long long baseline_after, long long baseline_before,
661 - long long after, long long before,
662 - RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
663 - int tier, int timeout,
664 - WEIGHTS_STATS *stats, bool register_zero) {
620 +static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats) {
621 + stats->db_queries++;
622 + stats->result_points += qv->result_points;
623 + stats->db_points += qv->points_read;
624 + for(size_t tier = 0; tier < storage_tiers ; tier++)
625 + stats->db_points_per_tier[tier] += qv->storage_points_per_tier[tier];
626 +}
627
666 - options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS;
667 - long group_time = 0;
628 +static void rrdset_metric_correlations_volume(
629 + RRDHOST *host,
630 + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
631 + DICTIONARY *results,
632 + time_t baseline_after, time_t baseline_before,
633 + time_t after, time_t before,
634 + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
635 + size_t tier,
636 + WEIGHTS_STATS *stats, bool register_zero) {
637
669 - int examined_dimensions = 0;
670 - int ret, value_is_null;
671 - usec_t started_usec = now_realtime_usec();
638 + options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS;
639
673 - RRDDIM *d;
674 - rrddim_foreach_read(d, st) {
675 - usec_t now_usec = now_realtime_usec();
676 - if(now_usec - started_usec > timeout * USEC_PER_MS)
677 - return examined_dimensions;
640 + QUERY_VALUE baseline_average = rrdmetric2value(host, rca, ria, rma, baseline_after, baseline_before, options, group_method, group_options, tier, 0);
641 + merge_query_value_to_stats(&baseline_average, stats);
642
679 - // we count how many metrics we evaluated
680 - examined_dimensions++;
643 + if(!netdata_double_isnumber(baseline_average.value)) {
644 + // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
645 + baseline_average.value = 0.0;
646 + }
647
682 - // there is no point to pass a timeout to these queries
683 - // since the query engine checks for a timeout between
684 - // dimensions, and we query a single dimension at a time.
685 -
686 - stats->db_queries++;
687 - NETDATA_DOUBLE baseline_average = NAN;
688 - NETDATA_DOUBLE base_anomaly_rate = 0;
689 - value_is_null = 1;
690 - ret = rrdset2value_api_v1(st, NULL, &baseline_average, rrddim_id(d), 1,
691 - baseline_after, baseline_before,
692 - group, group_options, group_time, options,
693 - NULL, NULL,
694 - &stats->db_points, stats->db_points_per_tier,
695 - &stats->result_points,
696 - &value_is_null, &base_anomaly_rate, 0, tier);
697 -
698 - if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) {
699 - // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
700 - baseline_average = 0.0;
701 - }
648 + QUERY_VALUE highlight_average = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0);
649 + merge_query_value_to_stats(&highlight_average, stats);
650
703 - stats->db_queries++;
704 - NETDATA_DOUBLE highlight_average = NAN;
705 - NETDATA_DOUBLE high_anomaly_rate = 0;
706 - value_is_null = 1;
707 - ret = rrdset2value_api_v1(st, NULL, &highlight_average, rrddim_id(d), 1,
708 - after, before,
709 - group, group_options, group_time, options,
710 - NULL, NULL,
711 - &stats->db_points, stats->db_points_per_tier,
712 - &stats->result_points,
713 - &value_is_null, &high_anomaly_rate, 0, tier);
714 -
715 - if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) {
716 - // this means no data for the highlighted duration - so skip it
717 - continue;
718 - }
651 + if(!netdata_double_isnumber(highlight_average.value))
652 + return;
653
720 - if(baseline_average == highlight_average) {
721 - // they are the same - let's move on
722 - continue;
723 - }
654 + if(baseline_average.value == highlight_average.value) {
655 + // they are the same - let's move on
656 + return;
657 + }
658
725 - stats->db_queries++;
726 - NETDATA_DOUBLE highlight_countif = NAN;
727 - value_is_null = 1;
728 -
729 - char highlighted_countif_options[50 + 1];
730 - snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
731 -
732 - ret = rrdset2value_api_v1(st, NULL, &highlight_countif, rrddim_id(d), 1,
733 - after, before,
734 - RRDR_GROUPING_COUNTIF,highlighted_countif_options,
735 - group_time, options,
736 - NULL, NULL,
737 - &stats->db_points, stats->db_points_per_tier,
738 - &stats->result_points,
739 - &value_is_null, NULL, 0, tier);
740 -
741 - if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) {
742 - info("MC: highlighted countif query failed, but highlighted average worked - strange...");
743 - continue;
744 - }
659 + char highlight_countif_options[50 + 1];
660 + snprintfz(highlight_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average.value < baseline_average.value ? "<" : ">", baseline_average.value);
661 + QUERY_VALUE highlight_countif = rrdmetric2value(host, rca, ria, rma, after, before, options, RRDR_GROUPING_COUNTIF, highlight_countif_options, tier, 0);
662 + merge_query_value_to_stats(&highlight_countif, stats);
663
746 - // this represents the percentage of time
747 - // the highlighted window was above/below the baseline window
748 - // (above or below depending on their averages)
749 - highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0
664 + if(!netdata_double_isnumber(highlight_countif.value)) {
665 + info("WEIGHTS: highlighted countif query failed, but highlighted average worked - strange...");
666 + return;
667 + }
668
751 - RESULT_FLAGS flags;
752 - NETDATA_DOUBLE pcent = NAN;
753 - if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) {
754 - flags = RESULT_IS_BASE_HIGH_RATIO;
755 - pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif;
756 - }
757 - else {
758 - flags = RESULT_IS_PERCENTAGE_OF_TIME;
759 - pcent = highlight_countif;
760 - }
669 + // this represents the percentage of time
670 + // the highlighted window was above/below the baseline window
671 + // (above or below depending on their averages)
672 + highlight_countif.value = highlight_countif.value / 100.0; // countif returns 0 - 100.0
673
762 - register_result(results, st, d, pcent, flags, stats, register_zero);
674 + RESULT_FLAGS flags;
675 + NETDATA_DOUBLE pcent = NAN;
676 + if(isgreater(baseline_average.value, 0.0) || isless(baseline_average.value, 0.0)) {
677 + flags = RESULT_IS_BASE_HIGH_RATIO;
678 + pcent = (highlight_average.value - baseline_average.value) / baseline_average.value * highlight_countif.value;
679 + }
680 + else {
681 + flags = RESULT_IS_PERCENTAGE_OF_TIME;
682 + pcent = highlight_countif.value;
683 }
764 - rrddim_foreach_done(d);
684
766 - return examined_dimensions;
685 + register_result(results, rca, ria, rma, pcent, flags, stats, register_zero);
686 }
687
688 // ----------------------------------------------------------------------------
689 // ANOMALY RATE algorithm functions
690
772 -static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results,
773 - long long after, long long before,
774 - RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
775 - int tier, int timeout,
776 - WEIGHTS_STATS *stats, bool register_zero) {
691 +static void rrdset_weights_anomaly_rate(
692 + RRDHOST *host,
693 + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
694 + DICTIONARY *results,
695 + time_t after, time_t before,
696 + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
697 + size_t tier,
698 + WEIGHTS_STATS *stats, bool register_zero) {
699
700 options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS;
779 - long group_time = 0;
780 -
781 - int examined_dimensions = 0;
782 - int ret, value_is_null;
783 - usec_t started_usec = now_realtime_usec();
784 -
785 - RRDDIM *d;
786 - rrddim_foreach_read(d, st) {
787 - usec_t now_usec = now_realtime_usec();
788 - if(now_usec - started_usec > timeout * USEC_PER_MS)
789 - return examined_dimensions;
701
791 - // we count how many metrics we evaluated
792 - examined_dimensions++;
793 -
794 - // there is no point to pass a timeout to these queries
795 - // since the query engine checks for a timeout between
796 - // dimensions, and we query a single dimension at a time.
797 -
798 - stats->db_queries++;
799 - NETDATA_DOUBLE average = NAN;
800 - NETDATA_DOUBLE anomaly_rate = 0;
801 - value_is_null = 1;
802 - ret = rrdset2value_api_v1(st, NULL, &average, rrddim_id(d), 1,
803 - after, before,
804 - group, group_options, group_time, options,
805 - NULL, NULL,
806 - &stats->db_points, stats->db_points_per_tier,
807 - &stats->result_points,
808 - &value_is_null, &anomaly_rate, 0, tier);
809 -
810 - if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average))
811 - register_result(results, st, d, average, 0, stats, register_zero);
812 - }
813 - rrddim_foreach_done(d);
702 + QUERY_VALUE qv = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0);
703 + merge_query_value_to_stats(&qv, stats);
704
815 - return examined_dimensions;
705 + if(netdata_double_isnumber(qv.value))
706 + register_result(results, rca, ria, rma, qv.value, 0, stats, register_zero);
707 }
708
709 // ----------------------------------------------------------------------------
@@ -900,15 +791,17 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) {
791 // ----------------------------------------------------------------------------
792 // The main function
793
903 -int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
904 - RRDR_GROUPING group, const char *group_options,
905 - long long baseline_after, long long baseline_before,
906 - long long after, long long before,
907 - long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout) {
794 +int web_api_v1_weights(
795 + RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
796 + RRDR_GROUPING group, const char *group_options,
797 + time_t baseline_after, time_t baseline_before,
798 + time_t after, time_t before,
799 + size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout) {
800 +
801 WEIGHTS_STATS stats = {};
802
803 DICTIONARY *results = register_result_init();
911 - DICTIONARY *charts = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE);;
804 + DICTIONARY *metrics = NULL;
805 char *error = NULL;
806 int resp = HTTP_RESP_OK;
807
@@ -997,19 +890,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
890 baseline_after = baseline_before - (high_delta << shifts);
891 }
892
1000 - // dont lock here and wait for results
1001 - // get the charts and run mc after
1002 - RRDSET *st;
1003 - rrdset_foreach_read(st, host) {
1004 - if (rrdset_is_available_for_viewers(st)) {
1005 - if(!contexts || simple_pattern_matches(contexts, rrdset_context(st)))
1006 - dictionary_set(charts, rrdset_name(st), NULL, 0);
1007 - }
1008 - }
1009 - rrdset_foreach_done(st);
1010 -
893 size_t examined_dimensions = 0;
1012 - void *ptr;
894
895 bool register_zero = true;
896 if(options & RRDR_OPTION_NONZERO) {
@@ -1017,8 +898,11 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
898 options &= ~RRDR_OPTION_NONZERO;
899 }
900
1020 - // for every chart in the dictionary
1021 - dfe_start_read(charts, ptr) {
901 + metrics = rrdcontext_all_metrics_to_dict(host, contexts);
902 + struct metric_entry *me;
903 +
904 + // for every metric_entry in the dictionary
905 + dfe_start_read(metrics, me) {
906 usec_t now_usec = now_realtime_usec();
907 if(now_usec - started_usec > timeout_usec) {
908 error = "timed out";
@@ -1026,42 +910,48 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
910 goto cleanup;
911 }
912
1029 - st = rrdset_find_byname(host, ptr_dfe.name); // ptr_dfe.name is provided by dictionary
1030 - if(!st) continue;
913 + examined_dimensions++;
914
915 switch(method) {
916 case WEIGHTS_METHOD_ANOMALY_RATE:
917 options |= RRDR_OPTION_ANOMALY_BIT;
1035 - points = 1;
1036 - examined_dimensions += rrdset_weights_anomaly_rate(st, results,
1037 - after, before,
1038 - options, group, group_options, tier,
1039 - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
1040 - &stats, register_zero);
918 + rrdset_weights_anomaly_rate(
919 + host,
920 + me->rca, me->ria, me->rma,
921 + results,
922 + after, before,
923 + options, group, group_options, tier,
924 + &stats, register_zero
925 + );
926 break;
927
928 case WEIGHTS_METHOD_MC_VOLUME:
1044 - points = 1;
1045 - examined_dimensions += rrdset_metric_correlations_volume(st, results,
1046 - baseline_after, baseline_before,
1047 - after, before,
1048 - options, group, group_options, tier,
1049 - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
1050 - &stats, register_zero);
929 + rrdset_metric_correlations_volume(
930 + host,
931 + me->rca, me->ria, me->rma,
932 + results,
933 + baseline_after, baseline_before,
934 + after, before,
935 + options, group, group_options, tier,
936 + &stats, register_zero
937 + );
938 break;
939
940 default:
941 case WEIGHTS_METHOD_MC_KS2:
1055 - examined_dimensions += rrdset_metric_correlations_ks2(st, results,
1056 - baseline_after, baseline_before,
1057 - after, before,
1058 - points, options, group, group_options, tier, shifts,
1059 - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
1060 - &stats, register_zero);
942 + rrdset_metric_correlations_ks2(
943 + host,
944 + me->rca, me->ria, me->rma,
945 + results,
946 + baseline_after, baseline_before,
947 + after, before, points,
948 + options, group, group_options, tier, shifts,
949 + &stats, register_zero
950 + );
951 break;
952 }
953 }
1064 - dfe_done(ptr);
954 + dfe_done(me);
955
956 if(!register_zero)
957 options |= RRDR_OPTION_NONZERO;
@@ -1077,22 +967,26 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
967 size_t added_dimensions = 0;
968 switch(format) {
969 case WEIGHTS_FORMAT_CHARTS:
1080 - added_dimensions = registered_results_to_json_charts(results, wb,
1081 - after, before,
1082 - baseline_after, baseline_before,
1083 - points, method, group, options, shifts,
1084 - examined_dimensions,
1085 - ended_usec - started_usec, &stats);
970 + added_dimensions =
971 + registered_results_to_json_charts(
972 + results, wb,
973 + after, before,
974 + baseline_after, baseline_before,
975 + points, method, group, options, shifts,
976 + examined_dimensions,
977 + ended_usec - started_usec, &stats);
978 break;
979
980 default:
981 case WEIGHTS_FORMAT_CONTEXTS:
1090 - added_dimensions = registered_results_to_json_contexts(results, wb,
1091 - after, before,
1092 - baseline_after, baseline_before,
1093 - points, method, group, options, shifts,
1094 - examined_dimensions,
1095 - ended_usec - started_usec, &stats);
982 + added_dimensions =
983 + registered_results_to_json_contexts(
984 + results, wb,
985 + after, before,
986 + baseline_after, baseline_before,
987 + points, method, group, options, shifts,
988 + examined_dimensions,
989 + ended_usec - started_usec, &stats);
990 break;
991 }
992
@@ -1102,7 +996,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
996 }
997
998 cleanup:
1105 - if(charts) dictionary_destroy(charts);
999 + if(metrics) dictionary_destroy(metrics);
1000 if(results) register_result_destroy(results);
1001
1002 if(error) {
web/api/queries/weights.h
+3 -3
@@ -22,9 +22,9 @@ extern WEIGHTS_METHOD default_metric_correlations_method;
22
23 int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
24 RRDR_GROUPING group, const char *group_options,
25 - long long baseline_after, long long baseline_before,
26 - long long after, long long before,
27 - long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout);
25 + time_t baseline_after, time_t baseline_before,
26 + time_t after, time_t before,
27 + size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout);
28
29 WEIGHTS_METHOD weights_string_to_method(const char *method);
30 const char *weights_method_to_string(WEIGHTS_METHOD method);
web/api/web_api_v1.c
+88 -107
@@ -34,20 +34,20 @@ static struct {
34 , {"match-ids" , 0 , RRDR_OPTION_MATCH_IDS}
35 , {"match_names" , 0 , RRDR_OPTION_MATCH_NAMES}
36 , {"match-names" , 0 , RRDR_OPTION_MATCH_NAMES}
37 - , {"showcustomvars" , 0 , RRDR_OPTION_CUSTOM_VARS}
37 , {"anomaly-bit" , 0 , RRDR_OPTION_ANOMALY_BIT}
38 , {"selected-tier" , 0 , RRDR_OPTION_SELECTED_TIER}
39 , {"raw" , 0 , RRDR_OPTION_RETURN_RAW}
40 , {"jw-anomaly-rates" , 0 , RRDR_OPTION_RETURN_JWAR}
41 , {"natural-points" , 0 , RRDR_OPTION_NATURAL_POINTS}
42 , {"virtual-points" , 0 , RRDR_OPTION_VIRTUAL_POINTS}
43 + , {"all-dimensions" , 0 , RRDR_OPTION_ALL_DIMENSIONS}
44 , {NULL , 0 , 0}
45 };
46
47 static struct {
48 const char *name;
49 uint32_t hash;
50 - uint32_t value;
50 + DATASOURCE_FORMAT value;
51 } api_v1_data_formats[] = {
52 { DATASOURCE_FORMAT_DATATABLE_JSON , 0 , DATASOURCE_DATATABLE_JSON}
53 , {DATASOURCE_FORMAT_DATATABLE_JSONP, 0 , DATASOURCE_DATATABLE_JSONP}
@@ -68,7 +68,7 @@ static struct {
68 static struct {
69 const char *name;
70 uint32_t hash;
71 - uint32_t value;
71 + DATASOURCE_FORMAT value;
72 } api_v1_data_google_formats[] = {
73 // this is not error - when google requests json, it expects javascript
74 // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source#responseformat
@@ -185,20 +185,46 @@ inline RRDR_OPTIONS web_client_api_request_v1_data_options(char *o) {
185 return ret;
186 }
187
188 -void web_client_api_request_v1_data_options_to_string(BUFFER *wb, RRDR_OPTIONS options) {
188 +void web_client_api_request_v1_data_options_to_buffer(BUFFER *wb, RRDR_OPTIONS options) {
189 RRDR_OPTIONS used = 0; // to prevent adding duplicates
190 int added = 0;
191 for(int i = 0; api_v1_data_options[i].name ; i++) {
192 if (unlikely((api_v1_data_options[i].value & options) && !(api_v1_data_options[i].value & used))) {
193 + const char *name = api_v1_data_options[i].name;
194 + used |= api_v1_data_options[i].value;
195 +
196 if(added) buffer_strcat(wb, ",");
194 - buffer_strcat(wb, api_v1_data_options[i].name);
197 + buffer_strcat(wb, name);
198 +
199 + added++;
200 + }
201 + }
202 +}
203 +
204 +void web_client_api_request_v1_data_options_to_string(char *buf, size_t size, RRDR_OPTIONS options) {
205 + char *write = buf;
206 + char *end = &buf[size - 1];
207 +
208 + RRDR_OPTIONS used = 0; // to prevent adding duplicates
209 + int added = 0;
210 + for(int i = 0; api_v1_data_options[i].name ; i++) {
211 + if (unlikely((api_v1_data_options[i].value & options) && !(api_v1_data_options[i].value & used))) {
212 + const char *name = api_v1_data_options[i].name;
213 used |= api_v1_data_options[i].value;
214 +
215 + if(added && write < end)
216 + *write++ = ',';
217 +
218 + while(*name && write < end)
219 + *write++ = *name++;
220 +
221 added++;
222 }
223 }
224 + *write = *end = '\0';
225 }
226
201 -inline uint32_t web_client_api_request_v1_data_format(char *name) {
227 +inline DATASOURCE_FORMAT web_client_api_request_v1_data_format(char *name) {
228 uint32_t hash = simple_hash(name);
229 int i;
230
@@ -589,11 +615,10 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
615 char *chart_label_key = NULL;
616 char *chart_labels_filter = NULL;
617 char *group_options = NULL;
592 - int tier = 0;
593 - int group = RRDR_GROUPING_AVERAGE;
594 - int show_dimensions = 0;
595 - uint32_t format = DATASOURCE_JSON;
596 - uint32_t options = 0x00000000;
618 + size_t tier = 0;
619 + RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;
620 + DATASOURCE_FORMAT format = DATASOURCE_JSON;
621 + RRDR_OPTIONS options = 0;
622
623 while(url) {
624 char *value = mystrsep(&url, "&");
@@ -617,7 +642,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
642 buffer_strcat(dimensions, "|");
643 buffer_strcat(dimensions, value);
644 }
620 - else if(!strcmp(name, "show_dimensions")) show_dimensions = 1;
645 + else if(!strcmp(name, "show_dimensions")) options |= RRDR_OPTION_ALL_DIMENSIONS;
646 else if(!strcmp(name, "after")) after_str = value;
647 else if(!strcmp(name, "before")) before_str = value;
648 else if(!strcmp(name, "points")) points_str = value;
@@ -674,9 +699,11 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
699 max_anomaly_rates_str = value;
700 }
701 else if(!strcmp(name, "tier")) {
677 - tier = str2i(value);
678 - if(tier >= 0 && tier < storage_tiers)
702 + tier = str2ul(value);
703 + if(tier < storage_tiers)
704 options |= RRDR_OPTION_SELECTED_TIER;
705 + else
706 + tier = 0;
707 }
708 }
709
@@ -691,73 +718,15 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
718 RRDSET *st = NULL;
719 ONEWAYALLOC *owa = onewayalloc_create(0);
720
694 - if((!chart || !*chart) && (!context)) {
695 - buffer_sprintf(w->response.data, "No chart id is given at the request.");
721 + if(!is_valid_sp(chart) && !is_valid_sp(context)) {
722 + buffer_sprintf(w->response.data, "No chart or context is given.");
723 goto cleanup;
724 }
725
699 - struct context_param *context_param_list = NULL;
700 -
701 - if (context && !chart) {
702 - RRDSET *st1;
703 -
704 - SIMPLE_PATTERN *chart_label_key_pattern = NULL;
705 - if(chart_label_key)
706 - chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
707 -
708 - SIMPLE_PATTERN *chart_labels_filter_pattern = NULL;
709 - if(chart_labels_filter)
710 - chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
711 -
712 - STRING *context_string = string_strdupz(context);
713 - rrdset_foreach_read(st1, host) {
714 - if (st1->context == context_string &&
715 - (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->rrdlabels, chart_label_key_pattern, ':')) &&
716 - (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->rrdlabels, chart_labels_filter_pattern, ':')))
717 - build_context_param_list(owa, &context_param_list, st1);
718 - }
719 - rrdset_foreach_done(st1);
720 - string_freez(context_string);
721 -
722 - if (likely(context_param_list && context_param_list->rd)) // Just set the first one
723 - st = context_param_list->rd->rrdset;
724 - }
725 - else {
726 + if(chart) {
727 + // check if this is a specific chart
728 st = rrdset_find(host, chart);
727 - if (!st)
728 - st = rrdset_find_byname(host, chart);
729 - if (likely(st))
730 - st->last_accessed_time = now_realtime_sec();
731 - }
732 -
733 - if (!st) {
734 - if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset))
735 - st = context_param_list->rd->rrdset;
736 - else {
737 - free_context_param_list(owa, &context_param_list);
738 - context_param_list = NULL;
739 - }
740 - }
741 -
742 - if (!st && !context_param_list) {
743 - if (context && !chart) {
744 - if (!chart_label_key) {
745 - buffer_strcat(w->response.data, "Context is not found: ");
746 - buffer_strcat_htmlescape(w->response.data, context);
747 - } else {
748 - buffer_strcat(w->response.data, "Context: ");
749 - buffer_strcat_htmlescape(w->response.data, context);
750 - buffer_strcat(w->response.data, " or chart label key: ");
751 - buffer_strcat_htmlescape(w->response.data, chart_label_key);
752 - buffer_strcat(w->response.data, " not found");
753 - }
754 - }
755 - else {
756 - buffer_strcat(w->response.data, "Chart is not found: ");
757 - buffer_strcat_htmlescape(w->response.data, chart);
758 - }
759 - ret = HTTP_RESP_NOT_FOUND;
760 - goto cleanup;
729 + if (!st) st = rrdset_find_byname(host, chart);
730 }
731
732 long long before = (before_str && *before_str)?str2l(before_str):0;
@@ -767,6 +736,35 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
736 long group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0;
737 int max_anomaly_rates = (max_anomaly_rates_str && *max_anomaly_rates_str) ? str2i(max_anomaly_rates_str) : 0;
738
739 + QUERY_TARGET_REQUEST qtr = {
740 + .after = after,
741 + .before = before,
742 + .host = host,
743 + .st = st,
744 + .hosts = NULL,
745 + .contexts = context,
746 + .charts = chart,
747 + .dimensions = (dimensions)?buffer_tostring(dimensions):NULL,
748 + .timeout = timeout,
749 + .max_anomaly_rates = max_anomaly_rates,
750 + .points = points,
751 + .format = format,
752 + .options = options,
753 + .group_method = group,
754 + .group_options = group_options,
755 + .resampling_time = group_time,
756 + .tier = tier,
757 + .chart_label_key = chart_label_key,
758 + .charts_labels_filter = chart_labels_filter,
759 + };
760 + QUERY_TARGET *qt = query_target_create(&qtr);
761 +
762 + if(!qt || !qt->query.used) {
763 + buffer_sprintf(w->response.data, "No metrics where matched to query.");
764 + ret = HTTP_RESP_NOT_FOUND;
765 + goto cleanup;
766 + }
767 +
768 if (timeout) {
769 struct timeval now;
770 now_realtime_timeval(&now);
@@ -775,21 +773,13 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
773 if (timeout <= 0) {
774 buffer_flush(w->response.data);
775 buffer_strcat(w->response.data, "Query timeout exceeded");
778 - return HTTP_RESP_BACKEND_FETCH_FAILED;
776 + ret = HTTP_RESP_BACKEND_FETCH_FAILED;
777 + goto cleanup;
778 }
779 }
780
782 - debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', format '%u', options '0x%08x'"
783 - , w->id
784 - , chart
785 - , (dimensions)?buffer_tostring(dimensions):""
786 - , after
787 - , before
788 - , points
789 - , group
790 - , format
791 - , options
792 - );
781 + debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%u', format '%u', options '0x%08x'"
782 + , w->id, chart, (dimensions)?buffer_tostring(dimensions):"", after, before , points, group, format, options);
783
784 if(outFileName && *outFileName) {
785 buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", outFileName);
@@ -820,18 +810,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
810 buffer_strcat(w->response.data, "(");
811 }
812
823 - QUERY_PARAMS query_params = {
824 - .context_param_list = context_param_list,
825 - .timeout = timeout,
826 - .max_anomaly_rates = max_anomaly_rates,
827 - .show_dimensions = show_dimensions,
828 - .chart_label_key = chart_label_key,
829 - .wb = w->response.data};
830 -
831 - ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
832 - points, after, before, group, group_options, group_time, options, &last_timestamp_in_data, tier);
833 -
834 - free_context_param_list(owa, &context_param_list);
813 + ret = data_query_execute(owa, w->response.data, qt, &last_timestamp_in_data);
814
815 if(format == DATASOURCE_DATATABLE_JSONP) {
816 if(google_timestamp < last_timestamp_in_data)
@@ -1407,7 +1386,7 @@ static int web_client_api_request_v1_weights_internal(RRDHOST *host, struct web_
1386 int options_count = 0;
1387 RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;
1388 int timeout = 0;
1410 - int tier = 0;
1389 + size_t tier = 0;
1390 const char *group_options = NULL, *contexts_str = NULL;
1391
1392 while (url) {
@@ -1455,9 +1434,11 @@ static int web_client_api_request_v1_weights_internal(RRDHOST *host, struct web_
1434 contexts_str = value;
1435
1436 else if(!strcmp(name, "tier")) {
1458 - tier = str2i(value);
1459 - if(tier >= 0 && tier < storage_tiers)
1437 + tier = str2ul(value);
1438 + if(tier < storage_tiers)
1439 options |= RRDR_OPTION_SELECTED_TIER;
1440 + else
1441 + tier = 0;
1442 }
1443 }
1444
@@ -1533,7 +1514,7 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host, struct web_client *w
1514 return HTTP_RESP_NOT_FOUND;
1515 }
1516 #else
1536 -static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, int tier) {
1517 +static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, size_t tier) {
1518 RRDENG_SIZE_STATS stats = rrdeng_size_statistics(multidb_ctx[tier]);
1519
1520 buffer_sprintf(wb,
@@ -1624,8 +1605,8 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1605 wb->contenttype = CT_APPLICATION_JSON;
1606 buffer_no_cacheable(wb);
1607 buffer_strcat(wb, "{");
1627 - for(int tier = 0; tier < storage_tiers ;tier++) {
1628 - buffer_sprintf(wb, "%s\n\t\"tier%d\": {", tier?",":"", tier);
1608 + for(size_t tier = 0; tier < storage_tiers ;tier++) {
1609 + buffer_sprintf(wb, "%s\n\t\"tier%zu\": {", tier?",":"", tier);
1610 web_client_api_v1_dbengine_stats_for_tier(wb, tier);
1611 buffer_strcat(wb, "\n\t}");
1612 }
web/api/web_api_v1.h
+2 -1
@@ -11,7 +11,8 @@
11
12 #define MAX_CHART_LABELS_FILTER (32)
13 RRDR_OPTIONS web_client_api_request_v1_data_options(char *o);
14 -void web_client_api_request_v1_data_options_to_string(BUFFER *wb, RRDR_OPTIONS options);
14 +void web_client_api_request_v1_data_options_to_buffer(BUFFER *wb, RRDR_OPTIONS options);
15 +void web_client_api_request_v1_data_options_to_string(char *buf, size_t size, RRDR_OPTIONS options);
16
17 uint32_t web_client_api_request_v1_data_format(char *name);
18 uint32_t web_client_api_request_v1_data_google_format(char *name);