@cryptotaxi247 / netdata-1 / commits / 204dd9ae2

Boost dbengine (#14832)

* configure extent cache size * workers can now execute up to 10 jobs in a run, boosting query prep and extent reads * fix dispatched and executing counters * boost to the max * increase libuv worker threads * query prep always get more prio than extent reads; stop processing in batch when dbengine is queue is critical * fix accounting of query prep * inlining of time-grouping functions, to speed up queries with billions of points * make switching based on a local const variable * print one pending contexts loading message per iteration * inlined store engine query API * inlined storage engine data collection api * inlined all storage engine query ops * eliminate and inline data collection ops * simplified query group-by * more error handling * optimized partial trimming of group-by queries * preparative work to support multiple passes of group-by * more preparative work to support multiple passes of group-by (accepts multiple group-by params) * unified query timings * unified query timings - weights endpoint * query target is no longer a static thread variable - there is a list of cached query targets, each of which of freed every 1000 queries * fix query memory accounting * added summary.dimension[].pri and sorted summary.dimensions based on priority and then name * limit max ACLK WEB response size to 30MB * the response type should be text/plain * more preparative work for multiple group-by passes * create functions for generating group by keys, ids and names * multiple group-by passes are now supported * parse group-by options array also with an index * implemented percentage-of-instance group by function * family is now merged in multi-node contexts * prevent uninitialized use

Costa Tsaousis committed Apr 7, 2023 at 21:25 UTC 204dd9ae272445d13f308badb07e99675fa34892
59 files changed +3112 -2417
aclk/aclk_query.c
+8
@@ -5,6 +5,7 @@
5 #include "aclk_tx_msgs.h"
6
7 #define WEB_HDR_ACCEPT_ENC "Accept-Encoding:"
8 +#define ACLK_MAX_WEB_RESPONSE_SIZE (30 * 1024 * 1024)
9
10 pthread_cond_t query_cond_wait = PTHREAD_COND_INITIALIZER;
11 pthread_mutex_t query_lock_wait = PTHREAD_MUTEX_INITIALIZER;
@@ -22,6 +23,13 @@ static usec_t aclk_web_api_request(RRDHOST *host, struct web_client *w, char *ur
23 else
24 w->response.code = web_client_api_request_v1(host, w, url);
25
26 + if(buffer_strlen(w->response.data) > ACLK_MAX_WEB_RESPONSE_SIZE) {
27 + buffer_flush(w->response.data);
28 + buffer_strcat(w->response.data, "response is too big");
29 + w->response.data->content_type = CT_TEXT_PLAIN;
30 + w->response.code = HTTP_RESP_CONTENT_TOO_LONG;
31 + }
32 +
33 t = now_monotonic_high_precision_usec() - t;
34
35 if (aclk_stats_enabled) {
daemon/main.c
+1 -1
@@ -1834,7 +1834,7 @@ int main(int argc, char **argv) {
1834 #endif
1835
1836 // set libuv worker threads
1837 - libuv_worker_threads = (int)get_netdata_cpus() * 2;
1837 + libuv_worker_threads = (int)get_netdata_cpus() * 6;
1838
1839 if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS)
1840 libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
daemon/service.c
+1 -1
@@ -55,7 +55,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
55 if(rd->tiers[tier].db_collection_handle) {
56 tiers_available++;
57
58 - if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle))
58 + if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle))
59 tiers_said_no_retention++;
60
61 rd->tiers[tier].db_collection_handle = NULL;
daemon/unit_test.c
+8 -8
@@ -1937,7 +1937,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
1937 // feed it with the test data
1938 for (i = 0 ; i < CHARTS ; ++i) {
1939 for (j = 0 ; j < DIMS ; ++j) {
1940 - rd[i][j]->tiers[0].collect_ops->change_collection_frequency(rd[i][j]->tiers[0].db_collection_handle, update_every);
1940 + storage_engine_store_change_collection_frequency(rd[i][j]->tiers[0].db_collection_handle, update_every);
1941
1942 rd[i][j]->last_collected_time.tv_sec =
1943 st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
@@ -1988,13 +1988,13 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
1988 time_now = time_start + (c + 1) * update_every;
1989 for (i = 0 ; i < CHARTS ; ++i) {
1990 for (j = 0; j < DIMS; ++j) {
1991 - rd[i][j]->tiers[0].query_ops->init(rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
1991 + storage_engine_query_init(rd[i][j]->tiers[0].backend, rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
1992 for (k = 0; k < QUERY_BATCH; ++k) {
1993 last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
1994 j * REGION_POINTS[current_region] + c + k;
1995 expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
1996
1997 - STORAGE_POINT sp = rd[i][j]->tiers[0].query_ops->next_metric(&handle);
1997 + STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
1998 value = sp.sum;
1999 time_retrieved = sp.start_time_s;
2000 end_time = sp.end_time_s;
@@ -2016,7 +2016,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
2016 errors++;
2017 }
2018 }
2019 - rd[i][j]->tiers[0].query_ops->finalize(&handle);
2019 + storage_engine_query_finalize(&handle);
2020 }
2021 }
2022 }
@@ -2444,13 +2444,13 @@ static void query_dbengine_chart(void *arg)
2444 time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
2445 }
2446
2447 - rd->tiers[0].query_ops->init(rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
2447 + storage_engine_query_init(rd->tiers[0].backend, rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
2448 ++thread_info->queries_nr;
2449 for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
2450 generatedv = generate_dbengine_chart_value(i, j, time_now);
2451 expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
2452
2453 - if (unlikely(rd->tiers[0].query_ops->is_finished(&handle))) {
2453 + if (unlikely(storage_engine_query_is_finished(&handle))) {
2454 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
2455 fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
2456 ", found data gap, ### E R R O R ###\n",
@@ -2460,7 +2460,7 @@ static void query_dbengine_chart(void *arg)
2460 break;
2461 }
2462
2463 - STORAGE_POINT sp = rd->tiers[0].query_ops->next_metric(&handle);
2463 + STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
2464 value = sp.sum;
2465 time_retrieved = sp.start_time_s;
2466 end_time = sp.end_time_s;
@@ -2498,7 +2498,7 @@ static void query_dbengine_chart(void *arg)
2498 }
2499 }
2500 }
2501 - rd->tiers[0].query_ops->finalize(&handle);
2501 + storage_engine_query_finalize(&handle);
2502 } while(!thread_info->done);
2503
2504 if(value_errors)
database/contexts/api_v2.c
+54 -16
@@ -112,6 +112,8 @@ struct rrdcontext_to_json_v2_data {
112 SIMPLE_PATTERN *pattern;
113 FTS_INDEX fts;
114 } q;
115 +
116 + struct query_timings timings;
117 };
118
119 static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc, SIMPLE_PATTERN *q) {
@@ -194,7 +196,7 @@ static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED
196 struct rrdcontext_to_json_v2_entry t = {
197 .count = 0,
198 .id = rc->id,
197 - .family = rc->family,
199 + .family = string_dup(rc->family),
200 .priority = rc->priority,
201 .first_time_s = rc->first_time_s,
202 .last_time_s = rc->last_time_s,
@@ -219,6 +221,10 @@ static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED
221
222 if(z->last_time_s < rc->last_time_s)
223 z->last_time_s = rc->last_time_s;
224 +
225 + if(z->family != rc->family) {
226 + z->family = string_2way_merge(z->family, rc->family);
227 + }
228 }
229
230 return 1;
@@ -248,7 +254,7 @@ static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool qu
254 struct rrdcontext_to_json_v2_data *ctl = data;
255 BUFFER *wb = ctl->wb;
256
251 - if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->request->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS)
257 + if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS)
258 // timed out
259 return -2;
260
@@ -384,7 +390,22 @@ static void buffer_json_contexts_v2_options_to_array(BUFFER *wb, CONTEXTS_V2_OPT
390 buffer_json_add_array_item_string(wb, "search");
391 }
392
387 -void buffer_json_agents_array_v2(BUFFER *wb, time_t now_s) {
393 +void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings) {
394 + timings->finished_ut = now_monotonic_usec();
395 + if(!timings->executed_ut)
396 + timings->executed_ut = timings->finished_ut;
397 + if(!timings->preprocessed_ut)
398 + timings->preprocessed_ut = timings->received_ut;
399 + buffer_json_member_add_object(wb, key);
400 + buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(timings->preprocessed_ut - timings->received_ut) / USEC_PER_MS);
401 + buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(timings->executed_ut - timings->preprocessed_ut) / USEC_PER_MS);
402 + buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->executed_ut) / USEC_PER_MS);
403 + buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS);
404 + buffer_json_member_add_double(wb, "cloud_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS);
405 + buffer_json_object_close(wb);
406 +}
407 +
408 +void buffer_json_agents_array_v2(BUFFER *wb, struct query_timings *timings, time_t now_s) {
409 if(!now_s)
410 now_s = now_realtime_sec();
411
@@ -395,15 +416,30 @@ void buffer_json_agents_array_v2(BUFFER *wb, time_t now_s) {
416 buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost));
417 buffer_json_member_add_time_t(wb, "now", now_s);
418 buffer_json_member_add_uint64(wb, "ai", 0);
419 +
420 + if(timings)
421 + buffer_json_query_timings(wb, "timings", timings);
422 +
423 buffer_json_object_close(wb);
424 buffer_json_array_close(wb);
425 }
426
427 +void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings) {
428 + buffer_json_member_add_object(wb, key);
429 + buffer_json_member_add_double(wb, "routing_ms", 0.0);
430 + buffer_json_member_add_double(wb, "node_max_ms", 0.0);
431 + buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS);
432 + buffer_json_object_close(wb);
433 +}
434 +
435 +void contexts_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
436 + struct rrdcontext_to_json_v2_entry *z = value;
437 + string_freez(z->family);
438 +}
439 +
440 int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options) {
441 int resp = HTTP_RESP_OK;
442
405 - req->timings.processing_ut = now_monotonic_usec();
406 -
443 if(options & CONTEXTS_V2_SEARCH)
444 options |= CONTEXTS_V2_CONTEXTS;
445
@@ -418,15 +454,22 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
454 .contexts.pattern = string_to_simple_pattern(req->contexts),
455 .contexts.scope_pattern = string_to_simple_pattern(req->scope_contexts),
456 .q.pattern = string_to_simple_pattern_nocase(req->q),
457 + .timings = {
458 + .received_ut = now_monotonic_usec(),
459 + }
460 };
461
423 - if(options & CONTEXTS_V2_CONTEXTS)
424 - ctl.ctx = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct rrdcontext_to_json_v2_entry));
462 + if(options & CONTEXTS_V2_CONTEXTS) {
463 + ctl.ctx = dictionary_create_advanced(
464 + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL,
465 + sizeof(struct rrdcontext_to_json_v2_entry));
466 +
467 + dictionary_register_delete_callback(ctl.ctx, contexts_delete_callback, NULL);
468 + }
469
470 time_t now_s = now_realtime_sec();
471 buffer_json_initialize(wb, "\"", "\"", 0, true, false);
472 buffer_json_member_add_uint64(wb, "api", 2);
429 - buffer_json_agents_array_v2(wb, now_s);
473
474 if(options & CONTEXTS_V2_DEBUG) {
475 buffer_json_member_add_object(wb, "request");
@@ -473,7 +516,7 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
516 if(options & (CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED | CONTEXTS_V2_DEBUG))
517 buffer_json_array_close(wb);
518
476 - req->timings.output_ut = now_monotonic_usec();
519 + ctl.timings.executed_ut = now_monotonic_usec();
520 version_hashes_api_v2(wb, &ctl.versions);
521
522 if(options & CONTEXTS_V2_CONTEXTS) {
@@ -506,13 +549,8 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
549 buffer_json_object_close(wb);
550 }
551
509 - req->timings.finished_ut = now_monotonic_usec();
510 - buffer_json_member_add_object(wb, "timings");
511 - buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(req->timings.processing_ut - req->timings.received_ut) / USEC_PER_MS);
512 - buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(req->timings.output_ut - req->timings.processing_ut) / USEC_PER_MS);
513 - buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(req->timings.finished_ut - req->timings.output_ut) / USEC_PER_MS);
514 - buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(req->timings.finished_ut - req->timings.received_ut) / USEC_PER_MS);
515 - buffer_json_object_close(wb);
552 + buffer_json_agents_array_v2(wb, &ctl.timings, now_s);
553 + buffer_json_cloud_timings(wb, "timings", &ctl.timings);
554 buffer_json_finalize(wb);
555
556 cleanup:
database/contexts/query_target.c
+130 -71
@@ -12,42 +12,55 @@ static void query_instance_release(QUERY_INSTANCE *qi);
12 static void query_context_release(QUERY_CONTEXT *qc);
13 static void query_node_release(QUERY_NODE *qn);
14
15 -static __thread QUERY_TARGET thread_query_target = {};
16 -
17 -// ----------------------------------------------------------------------------
18 -// query API
19 -
20 -typedef struct query_target_locals {
21 - time_t start_s;
22 -
23 - QUERY_TARGET *qt;
15 +static __thread QUERY_TARGET *thread_qt = NULL;
16 +static struct {
17 + struct {
18 + SPINLOCK spinlock;
19 + size_t count;
20 + QUERY_TARGET *base;
21 + } available;
22
25 - RRDSET *st;
23 + struct {
24 + SPINLOCK spinlock;
25 + size_t count;
26 + QUERY_TARGET *base;
27 + } used;
28 +} query_target_base = {
29 + .available = {
30 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
31 + .base = NULL,
32 + .count = 0,
33 + },
34 + .used = {
35 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
36 + .base = NULL,
37 + .count = 0,
38 + },
39 +};
40 +
41 +static void query_target_destroy(QUERY_TARGET *qt) {
42 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->query.size * sizeof(*qt->query.array), __ATOMIC_RELAXED);
43 + freez(qt->query.array);
44
27 - const char *scope_nodes;
28 - const char *scope_contexts;
45 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->dimensions.size * sizeof(*qt->dimensions.array), __ATOMIC_RELAXED);
46 + freez(qt->dimensions.array);
47
30 - const char *nodes;
31 - const char *contexts;
32 - const char *instances;
33 - const char *dimensions;
34 - const char *chart_label_key;
35 - const char *labels;
36 - const char *alerts;
48 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->instances.size * sizeof(*qt->instances.array), __ATOMIC_RELAXED);
49 + freez(qt->instances.array);
50
38 - long long after;
39 - long long before;
40 - bool match_ids;
41 - bool match_names;
51 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->contexts.size * sizeof(*qt->contexts.array), __ATOMIC_RELAXED);
52 + freez(qt->contexts.array);
53
43 - size_t metrics_skipped_due_to_not_matching_timeframe;
54 + __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->nodes.size * sizeof(*qt->nodes.array), __ATOMIC_RELAXED);
55 + freez(qt->nodes.array);
56
45 - char host_uuid_buffer[UUID_STR_LEN];
46 - QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
47 -} QUERY_TARGET_LOCALS;
57 + freez(qt);
58 +}
59
60 void query_target_release(QUERY_TARGET *qt) {
50 - if(unlikely(!qt || !qt->used)) return;
61 + if(unlikely(!qt)) return;
62 +
63 + internal_fatal(!qt->internal.used, "QUERY TARGET: qt to be released is not used");
64
65 simple_pattern_free(qt->nodes.scope_pattern);
66 qt->nodes.scope_pattern = NULL;
@@ -113,44 +126,91 @@ void query_target_release(QUERY_TARGET *qt) {
126 qt->db.first_time_s = 0;
127 qt->db.last_time_s = 0;
128
116 - qt->group_by.used = 0;
129 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
130 + qt->group_by[g].used = 0;
131
132 qt->id[0] = '\0';
133
120 - qt->used = false;
134 + netdata_spinlock_lock(&query_target_base.used.spinlock);
135 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(query_target_base.used.base, qt, internal.prev, internal.next);
136 + query_target_base.used.count--;
137 + netdata_spinlock_unlock(&query_target_base.used.spinlock);
138 +
139 + qt->internal.used = false;
140 + thread_qt = NULL;
141 +
142 + if (qt->internal.queries > 1000) {
143 + query_target_destroy(qt);
144 + }
145 + else {
146 + netdata_spinlock_lock(&query_target_base.available.spinlock);
147 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(query_target_base.available.base, qt, internal.prev, internal.next);
148 + query_target_base.available.count++;
149 + netdata_spinlock_unlock(&query_target_base.available.spinlock);
150 + }
151 }
122 -void query_target_free(void) {
123 - QUERY_TARGET *qt = &thread_query_target;
152
125 - if(qt->used)
126 - query_target_release(qt);
153 +static QUERY_TARGET *query_target_get(void) {
154 + netdata_spinlock_lock(&query_target_base.available.spinlock);
155 + QUERY_TARGET *qt = query_target_base.available.base;
156 + if (qt) {
157 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(query_target_base.available.base, qt, internal.prev, internal.next);
158 + query_target_base.available.count--;
159 + }
160 + netdata_spinlock_unlock(&query_target_base.available.spinlock);
161
128 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->query.size * sizeof(QUERY_METRIC), __ATOMIC_RELAXED);
129 - freez(qt->query.array);
130 - qt->query.array = NULL;
131 - qt->query.size = 0;
162 + if(unlikely(!qt))
163 + qt = callocz(1, sizeof(*qt));
164
133 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->dimensions.size * sizeof(RRDMETRIC_ACQUIRED *), __ATOMIC_RELAXED);
134 - freez(qt->dimensions.array);
135 - qt->dimensions.array = NULL;
136 - qt->dimensions.size = 0;
165 + netdata_spinlock_lock(&query_target_base.used.spinlock);
166 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(query_target_base.used.base, qt, internal.prev, internal.next);
167 + query_target_base.used.count++;
168 + netdata_spinlock_unlock(&query_target_base.used.spinlock);
169
138 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->instances.size * sizeof(RRDINSTANCE_ACQUIRED *), __ATOMIC_RELAXED);
139 - freez(qt->instances.array);
140 - qt->instances.array = NULL;
141 - qt->instances.size = 0;
170 + qt->internal.used = true;
171 + qt->internal.queries++;
172 + thread_qt = qt;
173
143 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->contexts.size * sizeof(RRDCONTEXT_ACQUIRED *), __ATOMIC_RELAXED);
144 - freez(qt->contexts.array);
145 - qt->contexts.array = NULL;
146 - qt->contexts.size = 0;
174 + return qt;
175 +}
176
148 - __atomic_sub_fetch(&netdata_buffers_statistics.query_targets_size, qt->nodes.size * sizeof(RRDHOST *), __ATOMIC_RELAXED);
149 - freez(qt->nodes.array);
150 - qt->nodes.array = NULL;
151 - qt->nodes.size = 0;
177 +// this is used to release a query target from a cancelled thread
178 +void query_target_free(void) {
179 + query_target_release(thread_qt);
180 }
181
182 +// ----------------------------------------------------------------------------
183 +// query API
184 +
185 +typedef struct query_target_locals {
186 + time_t start_s;
187 +
188 + QUERY_TARGET *qt;
189 +
190 + RRDSET *st;
191 +
192 + const char *scope_nodes;
193 + const char *scope_contexts;
194 +
195 + const char *nodes;
196 + const char *contexts;
197 + const char *instances;
198 + const char *dimensions;
199 + const char *chart_label_key;
200 + const char *labels;
201 + const char *alerts;
202 +
203 + long long after;
204 + long long before;
205 + bool match_ids;
206 + bool match_names;
207 +
208 + size_t metrics_skipped_due_to_not_matching_timeframe;
209 +
210 + char host_uuid_buffer[UUID_STR_LEN];
211 + QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
212 +} QUERY_TARGET_LOCALS;
213 +
214 struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
215 QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
216 return qn->rrdhost->db[tier].eng;
@@ -198,8 +258,8 @@ static bool query_metric_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CON
258 tier_retention[tier].db_metric_handle = eng->api.metric_get(qn->rrdhost->db[tier].instance, &rm->uuid);
259
260 if(tier_retention[tier].db_metric_handle) {
201 - tier_retention[tier].db_first_time_s = tier_retention[tier].eng->api.query_ops.oldest_time_s(tier_retention[tier].db_metric_handle);
202 - tier_retention[tier].db_last_time_s = tier_retention[tier].eng->api.query_ops.latest_time_s(tier_retention[tier].db_metric_handle);
261 + tier_retention[tier].db_first_time_s = storage_engine_oldest_time_s(tier_retention[tier].eng->backend, tier_retention[tier].db_metric_handle);
262 + tier_retention[tier].db_last_time_s = storage_engine_latest_time_s(tier_retention[tier].eng->backend, tier_retention[tier].db_metric_handle);
263
264 if(!common_first_time_s)
265 common_first_time_s = tier_retention[tier].db_first_time_s;
@@ -301,7 +361,7 @@ static inline void query_dimension_release(QUERY_DIMENSION *qd) {
361 qd->rma = NULL;
362 }
363
304 -static QUERY_DIMENSION *query_dimension_allocate(QUERY_TARGET *qt, RRDMETRIC_ACQUIRED *rma, QUERY_STATUS status) {
364 +static QUERY_DIMENSION *query_dimension_allocate(QUERY_TARGET *qt, RRDMETRIC_ACQUIRED *rma, QUERY_STATUS status, size_t priority) {
365 if(qt->dimensions.used == qt->dimensions.size) {
366 size_t old_mem = qt->dimensions.size * sizeof(*qt->dimensions.array);
367 qt->dimensions.size = query_target_realloc_size(qt->dimensions.size, 4);
@@ -316,12 +376,13 @@ static QUERY_DIMENSION *query_dimension_allocate(QUERY_TARGET *qt, RRDMETRIC_ACQ
376 qd->slot = qt->dimensions.used++;
377 qd->rma = rrdmetric_acquired_dup(rma);
378 qd->status = status;
379 + qd->priority = priority;
380
381 return qd;
382 }
383
384 static bool query_dimension_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi,
324 - RRDMETRIC_ACQUIRED *rma, bool queryable_instance, size_t *metrics_added) {
385 + RRDMETRIC_ACQUIRED *rma, bool queryable_instance, size_t *metrics_added, size_t priority) {
386 QUERY_TARGET *qt = qtl->qt;
387
388 RRDMETRIC *rm = rrdmetric_acquired_value(rma);
@@ -364,7 +425,7 @@ static bool query_dimension_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_
425 // the user selection does not match this dimension
426 // but, we may still need to query it
427
367 - if (qt->request.options & RRDR_OPTION_PERCENTAGE) {
428 + if (query_target_needs_all_dimensions(qt)) {
429 // this is percentage calculation
430 // so, we need this dimension to calculate the percentage
431 needed = true;
@@ -389,7 +450,7 @@ static bool query_dimension_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_
450 status |= QUERY_STATUS_DIMENSION_HIDDEN;
451 options |= RRDR_DIMENSION_HIDDEN;
452
392 - if (qt->request.options & RRDR_OPTION_PERCENTAGE) {
453 + if (query_target_needs_all_dimensions(qt)) {
454 // this is percentage calculation
455 // so, we need this dimension to calculate the percentage
456 needed = true;
@@ -430,7 +491,7 @@ static bool query_dimension_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_
491 if(undo)
492 return false;
493
433 - query_dimension_allocate(qt, rma, status);
494 + query_dimension_allocate(qt, rma, status, priority);
495 return true;
496 }
497
@@ -694,17 +755,17 @@ static bool query_instance_add(QUERY_TARGET_LOCALS *qtl, QUERY_NODE *qn, QUERY_C
755 if(queryable_instance && qt->request.version >= 2)
756 query_target_eval_instance_rrdcalc(qtl, qn, qc, qi);
757
697 - size_t dimensions_added = 0, metrics_added = 0;
758 + size_t dimensions_added = 0, metrics_added = 0, priority = 0;
759
760 if(unlikely(qt->request.rma)) {
700 - if(query_dimension_add(qtl, qn, qc, qi, qt->request.rma, queryable_instance, &metrics_added))
761 + if(query_dimension_add(qtl, qn, qc, qi, qt->request.rma, queryable_instance, &metrics_added, priority++))
762 dimensions_added++;
763 }
764 else {
765 RRDMETRIC *rm;
766 dfe_start_read(ri->rrdmetrics, rm) {
767 if(query_dimension_add(qtl, qn, qc, qi, (RRDMETRIC_ACQUIRED *) rm_dfe.item,
707 - queryable_instance, &metrics_added))
768 + queryable_instance, &metrics_added, priority++))
769 dimensions_added++;
770 }
771 dfe_done(rm);
@@ -957,13 +1018,7 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
1018 if(!service_running(ABILITY_DATA_QUERIES))
1019 return NULL;
1020
960 - QUERY_TARGET *qt = &thread_query_target;
961 -
962 - if(qt->used)
963 - fatal("QUERY TARGET: this query target is already used (%zu queries made with this QUERY_TARGET so far).", qt->queries);
964 -
965 - qt->used = true;
966 - qt->queries++;
1021 + QUERY_TARGET *qt = query_target_get();
1022
1023 if(!qtr->received_ut)
1024 qtr->received_ut = now_monotonic_usec();
@@ -985,7 +1040,11 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
1040 query_target_generate_name(qt);
1041 qt->window.after = qt->request.after;
1042 qt->window.before = qt->request.before;
1043 +
1044 qt->window.options = qt->request.options;
1045 + if(query_target_has_percentage_of_instance(qt))
1046 + qt->window.options &= ~RRDR_OPTION_PERCENTAGE;
1047 +
1048 rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
1049
1050 // prepare our local variables - we need these across all these functions
database/contexts/rrdcontext.h
+54 -23
@@ -201,6 +201,7 @@ typedef struct query_instance {
201
202 typedef struct query_dimension {
203 uint32_t slot;
204 + uint32_t priority;
205 RRDMETRIC_ACQUIRED *rma;
206 QUERY_STATUS status;
207 } QUERY_DIMENSION;
@@ -231,7 +232,8 @@ typedef struct query_metric {
232 STORAGE_POINT query_points;
233
234 struct {
234 - size_t slot;
235 + uint32_t slot;
236 + uint32_t first_slot;
237 STRING *id;
238 STRING *name;
239 STRING *units;
@@ -241,9 +243,16 @@ typedef struct query_metric {
243 } QUERY_METRIC;
244
245 #define MAX_QUERY_TARGET_ID_LENGTH 255
246 +#define MAX_QUERY_GROUP_BY_PASSES 2
247
248 typedef bool (*qt_interrupt_callback_t)(void *data);
249
250 +struct group_by_pass {
251 + RRDR_GROUP_BY group_by;
252 + char *group_by_label;
253 + RRDR_GROUP_BY_FUNCTION aggregation;
254 +};
255 +
256 typedef struct query_target_request {
257 size_t version;
258
@@ -284,9 +293,7 @@ typedef struct query_target_request {
293 const char *time_group_options;
294
295 // group by across multiple time-series
287 - RRDR_GROUP_BY group_by;
288 - char *group_by_label;
289 - RRDR_GROUP_BY_FUNCTION group_by_aggregate_function;
296 + struct group_by_pass group_by[MAX_QUERY_GROUP_BY_PASSES];
297
298 usec_t received_ut;
299
@@ -313,15 +320,19 @@ struct query_versions {
320 uint64_t alerts_soft_hash;
321 };
322
323 +struct query_timings {
324 + usec_t received_ut;
325 + usec_t preprocessed_ut;
326 + usec_t executed_ut;
327 + usec_t finished_ut;
328 +};
329 +
330 #define query_view_update_every(qt) ((qt)->window.group * (qt)->window.query_granularity)
331
332 typedef struct query_target {
333 char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging)
334 QUERY_TARGET_REQUEST request;
335
322 - bool used; // when true, this query is currently being used
323 - size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries
324 -
336 struct {
337 time_t now; // the current timestamp, the absolute max for any query timestamp
338 bool relative; // true when the request made with relative timestamps, true if it was absolute
@@ -388,19 +399,20 @@ typedef struct query_target {
399
400 struct {
401 size_t used;
391 - char *label_keys[GROUP_BY_MAX_LABEL_KEYS];
392 - } group_by;
402 + char *label_keys[GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES];
403 + } group_by[MAX_QUERY_GROUP_BY_PASSES];
404
405 STORAGE_POINT query_points;
395 -
406 struct query_versions versions;
407 + struct query_timings timings;
408
409 struct {
399 - usec_t received_ut;
400 - usec_t preprocessed_ut;
401 - usec_t executed_ut;
402 - usec_t finished_ut;
403 - } timings;
410 + SPINLOCK spinlock;
411 + bool used; // when true, this query is currently being used
412 + size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries
413 + struct query_target *prev;
414 + struct query_target *next;
415 + } internal;
416 } QUERY_TARGET;
417
418 static inline NEVERNULL QUERY_NODE *query_node(QUERY_TARGET *qt, size_t id) {
@@ -455,13 +467,6 @@ struct api_v2_contexts_request {
467 char *contexts;
468 char *q;
469
458 - struct {
459 - usec_t received_ut;
460 - usec_t processing_ut;
461 - usec_t output_ut;
462 - usec_t finished_ut;
463 - } timings;
464 -
470 time_t timeout_ms;
471
472 qt_interrupt_callback_t interrupt_callback;
@@ -479,8 +484,10 @@ typedef enum __attribute__ ((__packed__)) {
484 int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options);
485
486 RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o);
482 -void buffer_json_agents_array_v2(BUFFER *wb, time_t now_s);
487 +void buffer_json_agents_array_v2(BUFFER *wb, struct query_timings *timings, time_t now_s);
488 void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut);
489 +void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings);
490 +void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings);
491
492 // ----------------------------------------------------------------------------
493 // scope
@@ -515,5 +522,29 @@ bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t b
522 (((first_entry_s) - ((update_every_s) * 2) <= (before)) && \
523 ((last_entry_s) + ((update_every_s) * 2) >= (after)))
524
525 +#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW)
526 +
527 +static inline bool query_target_has_percentage_of_instance(QUERY_TARGET *qt) {
528 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
529 + if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
530 + return true;
531 +
532 + return false;
533 +}
534 +
535 +static inline bool query_target_needs_all_dimensions(QUERY_TARGET *qt) {
536 + if(qt->request.options & RRDR_OPTION_PERCENTAGE)
537 + return true;
538 +
539 + return query_target_has_percentage_of_instance(qt);
540 +}
541 +
542 +static inline bool query_target_has_percentage_units(QUERY_TARGET *qt) {
543 + if(qt->window.time_group_method == RRDR_GROUPING_CV || query_target_needs_all_dimensions(qt))
544 + return true;
545 +
546 + return false;
547 +}
548 +
549 #endif // NETDATA_RRDCONTEXT_H
550
database/engine/pagecache.c
+8 -2
@@ -769,7 +769,10 @@ inline void rrdeng_prep_wait(PDC *pdc) {
769 }
770 }
771
772 -void rrdeng_prep_query(PDC *pdc) {
772 +void rrdeng_prep_query(struct page_details_control *pdc, bool worker) {
773 + if(worker)
774 + worker_is_busy(UV_EVENT_DBENGINE_QUERY);
775 +
776 size_t pages_to_load = 0;
777 pdc->page_list_JudyL = get_page_list(pdc->ctx, pdc->metric,
778 pdc->start_time_s * USEC_PER_SEC,
@@ -792,6 +795,9 @@ void rrdeng_prep_query(PDC *pdc) {
795 completion_mark_complete(&pdc->prep_completion);
796
797 pdc_release_and_destroy_if_unreferenced(pdc, true, true);
798 +
799 + if(worker)
800 + worker_is_idle();
801 }
802
803 /**
@@ -824,7 +830,7 @@ void pg_cache_preload(struct rrdeng_query_handle *handle) {
830 handle->pdc->refcount++; // we get 1 for the query thread and 1 for the prep thread
831
832 if(unlikely(handle->pdc->priority == STORAGE_PRIORITY_SYNCHRONOUS))
827 - rrdeng_prep_query(handle->pdc);
833 + rrdeng_prep_query(handle->pdc, false);
834 else
835 rrdeng_enq_cmd(handle->ctx, RRDENG_OPCODE_QUERY, handle->pdc, NULL, handle->priority, NULL, NULL);
836 }
database/engine/pagecache.h
+1 -1
@@ -52,7 +52,7 @@ struct rrdeng_query_handle;
52 struct page_details_control;
53
54 void rrdeng_prep_wait(struct page_details_control *pdc);
55 -void rrdeng_prep_query(struct page_details_control *pdc);
55 +void rrdeng_prep_query(struct page_details_control *pdc, bool worker);
56 void pg_cache_preload(struct rrdeng_query_handle *handle);
57 struct pgc_page *pg_cache_lookup_next(struct rrdengine_instance *ctx, struct page_details_control *pdc, time_t now_s, time_t last_update_every_s, size_t *entries);
58 void pgc_and_mrg_initialize(void);
database/engine/pdc.c
+3 -3
@@ -1151,6 +1151,9 @@ static inline void datafile_extent_read_free(void *buffer) {
1151 }
1152
1153 void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *epdl, bool worker) {
1154 + if(worker)
1155 + worker_is_busy(UV_EVENT_DBENGINE_EXTENT_CACHE_LOOKUP);
1156 +
1157 size_t *statistics_counter = NULL;
1158 PDC_PAGE_STATUS not_loaded_pages_tag = 0, loaded_pages_tag = 0;
1159
@@ -1173,9 +1176,6 @@ void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *e
1176 goto cleanup;
1177 }
1178
1176 - if(worker)
1177 - worker_is_busy(UV_EVENT_DBENGINE_EXTENT_CACHE_LOOKUP);
1178 -
1179 bool extent_found_in_cache = false;
1180
1181 void *extent_compressed_data = NULL;
database/engine/rrdengine.c
+110 -50
@@ -16,6 +16,24 @@ unsigned rrdeng_pages_per_extent = MAX_PAGES_PER_EXTENT;
16 #error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least (RRDENG_MAX_OPCODE + 2)
17 #endif
18
19 +struct rrdeng_cmd {
20 + struct rrdengine_instance *ctx;
21 + enum rrdeng_opcode opcode;
22 + void *data;
23 + struct completion *completion;
24 + enum storage_priority priority;
25 + dequeue_callback_t dequeue_cb;
26 +
27 + struct {
28 + struct rrdeng_cmd *prev;
29 + struct rrdeng_cmd *next;
30 + } queue;
31 +};
32 +
33 +static inline struct rrdeng_cmd rrdeng_deq_cmd(bool from_worker);
34 +static inline void worker_dispatch_extent_read(struct rrdeng_cmd cmd, bool from_worker);
35 +static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_worker);
36 +
37 struct rrdeng_main {
38 uv_thread_t thread;
39 uv_loop_t loop;
@@ -45,7 +63,6 @@ struct rrdeng_main {
63 struct {
64 size_t dispatched;
65 size_t executing;
48 - size_t pending_cb;
66 } atomics;
67 } work_cmd;
68
@@ -132,8 +149,22 @@ static void work_request_init(void) {
149 );
150 }
151
135 -static inline bool work_request_full(void) {
136 - return __atomic_load_n(&rrdeng_main.work_cmd.atomics.dispatched, __ATOMIC_RELAXED) >= (size_t)(libuv_worker_threads - RESERVED_LIBUV_WORKER_THREADS);
152 +enum LIBUV_WORKERS_STATUS {
153 + LIBUV_WORKERS_RELAXED,
154 + LIBUV_WORKERS_STRESSED,
155 + LIBUV_WORKERS_CRITICAL,
156 +};
157 +
158 +static inline enum LIBUV_WORKERS_STATUS work_request_full(void) {
159 + size_t dispatched = __atomic_load_n(&rrdeng_main.work_cmd.atomics.dispatched, __ATOMIC_RELAXED);
160 +
161 + if(dispatched >= (size_t)(libuv_worker_threads))
162 + return LIBUV_WORKERS_CRITICAL;
163 +
164 + else if(dispatched >= (size_t)(libuv_worker_threads - RESERVED_LIBUV_WORKER_THREADS))
165 + return LIBUV_WORKERS_STRESSED;
166 +
167 + return LIBUV_WORKERS_RELAXED;
168 }
169
170 static inline void work_done(struct rrdeng_work *work_request) {
@@ -147,12 +178,38 @@ static void work_standard_worker(uv_work_t *req) {
178 worker_is_busy(UV_EVENT_WORKER_INIT);
179
180 struct rrdeng_work *work_request = req->data;
181 +
182 work_request->data = work_request->work_cb(work_request->ctx, work_request->data, work_request->completion, req);
183 worker_is_idle();
184
185 + if(work_request->opcode == RRDENG_OPCODE_EXTENT_READ || work_request->opcode == RRDENG_OPCODE_QUERY) {
186 + internal_fatal(work_request->after_work_cb != NULL, "DBENGINE: opcodes with a callback should not boosted");
187 +
188 + while(1) {
189 + struct rrdeng_cmd cmd = rrdeng_deq_cmd(true);
190 + if (cmd.opcode == RRDENG_OPCODE_NOOP)
191 + break;
192 +
193 + worker_is_busy(UV_EVENT_WORKER_INIT);
194 + switch (cmd.opcode) {
195 + case RRDENG_OPCODE_EXTENT_READ:
196 + worker_dispatch_extent_read(cmd, true);
197 + break;
198 +
199 + case RRDENG_OPCODE_QUERY:
200 + worker_dispatch_query_prep(cmd, true);
201 + break;
202 +
203 + default:
204 + fatal("DBENGINE: Opcode should not be executed synchronously");
205 + break;
206 + }
207 + worker_is_idle();
208 + }
209 + }
210 +
211 __atomic_sub_fetch(&rrdeng_main.work_cmd.atomics.dispatched, 1, __ATOMIC_RELAXED);
212 __atomic_sub_fetch(&rrdeng_main.work_cmd.atomics.executing, 1, __ATOMIC_RELAXED);
155 - __atomic_add_fetch(&rrdeng_main.work_cmd.atomics.pending_cb, 1, __ATOMIC_RELAXED);
213
214 // signal the event loop a worker is available
215 fatal_assert(0 == uv_async_send(&rrdeng_main.async));
@@ -167,7 +224,6 @@ static void after_work_standard_callback(uv_work_t* req, int status) {
224 work_request->after_work_cb(work_request->ctx, work_request->data, work_request->completion, req, status);
225
226 work_done(work_request);
170 - __atomic_sub_fetch(&rrdeng_main.work_cmd.atomics.pending_cb, 1, __ATOMIC_RELAXED);
227
228 worker_is_idle();
229 }
@@ -369,20 +425,6 @@ void wal_release(WAL *wal) {
425 // ----------------------------------------------------------------------------
426 // command queue cache
427
372 -struct rrdeng_cmd {
373 - struct rrdengine_instance *ctx;
374 - enum rrdeng_opcode opcode;
375 - void *data;
376 - struct completion *completion;
377 - enum storage_priority priority;
378 - dequeue_callback_t dequeue_cb;
379 -
380 - struct {
381 - struct rrdeng_cmd *prev;
382 - struct rrdeng_cmd *next;
383 - } queue;
384 -};
385 -
428 static void rrdeng_cmd_queue_init(void) {
429 rrdeng_main.cmd_queue.ar = aral_create("dbengine-opcodes",
430 sizeof(struct rrdeng_cmd),
@@ -465,14 +507,33 @@ static inline bool rrdeng_cmd_has_waiting_opcodes_in_lower_priorities(STORAGE_PR
507 return false;
508 }
509
468 -static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
510 +#define opcode_empty (struct rrdeng_cmd) { \
511 + .ctx = NULL, \
512 + .opcode = RRDENG_OPCODE_NOOP, \
513 + .priority = STORAGE_PRIORITY_BEST_EFFORT, \
514 + .completion = NULL, \
515 + .data = NULL, \
516 +}
517 +
518 +static inline struct rrdeng_cmd rrdeng_deq_cmd(bool from_worker) {
519 struct rrdeng_cmd *cmd = NULL;
520 + enum LIBUV_WORKERS_STATUS status = work_request_full();
521
471 - STORAGE_PRIORITY max_priority = work_request_full() ? STORAGE_PRIORITY_INTERNAL_DBENGINE : STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE - 1;
522 + STORAGE_PRIORITY min_priority, max_priority;
523 + min_priority = STORAGE_PRIORITY_INTERNAL_DBENGINE;
524 + max_priority = (status != LIBUV_WORKERS_RELAXED) ? STORAGE_PRIORITY_INTERNAL_DBENGINE : STORAGE_PRIORITY_INTERNAL_MAX_DONT_USE - 1;
525 +
526 + if(from_worker) {
527 + if(status == LIBUV_WORKERS_CRITICAL)
528 + return opcode_empty;
529 +
530 + min_priority = STORAGE_PRIORITY_INTERNAL_QUERY_PREP;
531 + max_priority = STORAGE_PRIORITY_BEST_EFFORT;
532 + }
533
534 // find an opcode to execute from the queue
535 netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);
475 - for(STORAGE_PRIORITY priority = STORAGE_PRIORITY_INTERNAL_DBENGINE; priority <= max_priority ; priority++) {
536 + for(STORAGE_PRIORITY priority = min_priority; priority <= max_priority ; priority++) {
537 cmd = rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority];
538 if(cmd) {
539
@@ -508,13 +569,7 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(void) {
569 aral_freez(rrdeng_main.cmd_queue.ar, cmd);
570 }
571 else
511 - ret = (struct rrdeng_cmd) {
512 - .ctx = NULL,
513 - .opcode = RRDENG_OPCODE_NOOP,
514 - .priority = STORAGE_PRIORITY_BEST_EFFORT,
515 - .completion = NULL,
516 - .data = NULL,
517 - };
572 + ret = opcode_empty;
573
574 return ret;
575 }
@@ -1353,14 +1408,9 @@ static void *cache_evict_tp_worker(struct rrdengine_instance *ctx __maybe_unused
1408 return data;
1409 }
1410
1356 -static void after_prep_query(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t* req __maybe_unused, int status __maybe_unused) {
1357 - ;
1358 -}
1359 -
1411 static void *query_prep_tp_worker(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t *req __maybe_unused) {
1361 - worker_is_busy(UV_EVENT_DBENGINE_QUERY);
1412 PDC *pdc = data;
1363 - rrdeng_prep_query(pdc);
1413 + rrdeng_prep_query(pdc, true);
1414 return data;
1415 }
1416
@@ -1484,10 +1534,6 @@ static void after_do_cache_evict(struct rrdengine_instance *ctx __maybe_unused,
1534 rrdeng_main.evictions_running--;
1535 }
1536
1487 -static void after_extent_read(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t* req __maybe_unused, int status __maybe_unused) {
1488 - ;
1489 -}
1490 -
1537 static void after_journal_v2_indexing(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t* req __maybe_unused, int status __maybe_unused) {
1538 __atomic_store_n(&ctx->atomic.migration_to_v2_running, false, __ATOMIC_RELAXED);
1539 rrdeng_enq_cmd(ctx, RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);
@@ -1616,6 +1662,26 @@ bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {
1662 return true;
1663 }
1664
1665 +static inline void worker_dispatch_extent_read(struct rrdeng_cmd cmd, bool from_worker) {
1666 + struct rrdengine_instance *ctx = cmd.ctx;
1667 + EPDL *epdl = cmd.data;
1668 +
1669 + if(from_worker)
1670 + epdl_find_extent_and_populate_pages(ctx, epdl, true);
1671 + else
1672 + work_dispatch(ctx, epdl, NULL, cmd.opcode, extent_read_tp_worker, NULL);
1673 +}
1674 +
1675 +static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_worker) {
1676 + struct rrdengine_instance *ctx = cmd.ctx;
1677 + PDC *pdc = cmd.data;
1678 +
1679 + if(from_worker)
1680 + rrdeng_prep_query(pdc, true);
1681 + else
1682 + work_dispatch(ctx, pdc, NULL, cmd.opcode, query_prep_tp_worker, NULL);
1683 +}
1684 +
1685 void dbengine_event_loop(void* arg) {
1686 sanity_check();
1687 uv_thread_set_name_np(pthread_self(), "DBENGINE");
@@ -1673,25 +1739,19 @@ void dbengine_event_loop(void* arg) {
1739 /* wait for commands */
1740 do {
1741 worker_is_busy(RRDENG_OPCODE_MAX);
1676 - cmd = rrdeng_deq_cmd();
1742 + cmd = rrdeng_deq_cmd(RRDENG_OPCODE_NOOP);
1743 opcode = cmd.opcode;
1744
1745 worker_is_busy(opcode);
1746
1747 switch (opcode) {
1682 - case RRDENG_OPCODE_EXTENT_READ: {
1683 - struct rrdengine_instance *ctx = cmd.ctx;
1684 - EPDL *epdl = cmd.data;
1685 - work_dispatch(ctx, epdl, NULL, opcode, extent_read_tp_worker, after_extent_read);
1748 + case RRDENG_OPCODE_EXTENT_READ:
1749 + worker_dispatch_extent_read(cmd, false);
1750 break;
1687 - }
1751
1689 - case RRDENG_OPCODE_QUERY: {
1690 - struct rrdengine_instance *ctx = cmd.ctx;
1691 - PDC *pdc = cmd.data;
1692 - work_dispatch(ctx, pdc, NULL, opcode, query_prep_tp_worker, after_prep_query);
1752 + case RRDENG_OPCODE_QUERY:
1753 + worker_dispatch_query_prep(cmd, false);
1754 break;
1694 - }
1755
1756 case RRDENG_OPCODE_EXTENT_WRITE: {
1757 struct rrdengine_instance *ctx = cmd.ctx;
database/engine/rrdengine.h
+6 -3
@@ -181,14 +181,17 @@ typedef enum __attribute__ ((__packed__)) {
181 } RRDENG_COLLECT_PAGE_FLAGS;
182
183 struct rrdeng_collect_handle {
184 + struct storage_collect_handle common; // has to be first item
185 +
186 + RRDENG_COLLECT_PAGE_FLAGS page_flags;
187 + RRDENG_COLLECT_HANDLE_OPTIONS options;
188 + uint8_t type;
189 +
190 struct metric *metric;
191 struct pgc_page *page;
192 void *data;
193 size_t data_size;
194 struct pg_alignment *alignment;
189 - RRDENG_COLLECT_HANDLE_OPTIONS options;
190 - uint8_t type;
191 - RRDENG_COLLECT_PAGE_FLAGS page_flags;
195 uint32_t page_entries_max;
196 uint32_t page_position; // keep track of the current page size, to make sure we don't exceed it
197 usec_t page_start_time_ut;
database/engine/rrdengineapi.c
+3
@@ -255,6 +255,7 @@ STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metri
255 struct rrdeng_collect_handle *handle;
256
257 handle = callocz(1, sizeof(struct rrdeng_collect_handle));
258 + handle->common.backend = STORAGE_ENGINE_BACKEND_DBENGINE;
259 handle->metric = metric;
260 handle->page = NULL;
261 handle->data = NULL;
@@ -774,6 +775,7 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle,
775 rrddim_handle->start_time_s = handle->start_time_s;
776 rrddim_handle->end_time_s = handle->end_time_s;
777 rrddim_handle->priority = priority;
778 + rrddim_handle->backend = STORAGE_ENGINE_BACKEND_DBENGINE;
779
780 pg_cache_preload(handle);
781
@@ -789,6 +791,7 @@ void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle,
791 rrddim_handle->start_time_s = handle->start_time_s;
792 rrddim_handle->end_time_s = 0;
793 rrddim_handle->priority = priority;
794 + rrddim_handle->backend = STORAGE_ENGINE_BACKEND_DBENGINE;
795 }
796 }
797
database/ram/rrddim_mem.c
+4 -2
@@ -143,6 +143,7 @@ STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_han
143 internal_fatal((uint32_t)mh->update_every_s != update_every, "RRDDIM: update requested does not match the dimension");
144
145 struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
146 + ch->common.backend = STORAGE_ENGINE_BACKEND_RRDDIM;
147 ch->rd = rd;
148 ch->db_metric_handle = db_metric_handle;
149
@@ -204,7 +205,7 @@ static inline void rrddim_fill_the_gap(STORAGE_COLLECT_HANDLE *collection_handle
205
206 void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle,
207 usec_t point_in_time_ut,
207 - NETDATA_DOUBLE number,
208 + NETDATA_DOUBLE n,
209 NETDATA_DOUBLE min_value __maybe_unused,
210 NETDATA_DOUBLE max_value __maybe_unused,
211 uint16_t count __maybe_unused,
@@ -226,7 +227,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle,
227 if(unlikely(mh->last_updated_s && point_in_time_s - mh->update_every_s > mh->last_updated_s))
228 rrddim_fill_the_gap(collection_handle, point_in_time_s);
229
229 - rd->db[mh->current_entry] = pack_storage_number(number, flags);
230 + rd->db[mh->current_entry] = pack_storage_number(n, flags);
231 mh->counter++;
232 mh->current_entry = (mh->current_entry + 1) >= mh->entries ? 0 : mh->current_entry + 1;
233 mh->last_updated_s = point_in_time_s;
@@ -340,6 +341,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_e
341 handle->start_time_s = start_time_s;
342 handle->end_time_s = end_time_s;
343 handle->priority = priority;
344 + handle->backend = STORAGE_ENGINE_BACKEND_RRDDIM;
345 struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
346 h->db_metric_handle = db_metric_handle;
347
database/ram/rrddim_mem.h
+3 -1
@@ -6,6 +6,8 @@
6 #include "database/rrd.h"
7
8 struct mem_collect_handle {
9 + struct storage_collect_handle common; // has to be first item
10 +
11 STORAGE_METRIC_HANDLE *db_metric_handle;
12 RRDDIM *rd;
13 };
@@ -32,7 +34,7 @@ void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS
34
35 STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
36 void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
35 -void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE number,
37 +void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
38 NETDATA_DOUBLE min_value,
39 NETDATA_DOUBLE max_value,
40 uint16_t count,
database/rrd.h
+211 -47
@@ -109,12 +109,20 @@ RRD_MEMORY_MODE rrd_memory_mode_id(const char *name);
109
110 typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
111
112 +typedef enum __attribute__ ((__packed__)) {
113 + STORAGE_ENGINE_BACKEND_RRDDIM = 1,
114 + STORAGE_ENGINE_BACKEND_DBENGINE = 2,
115 +} STORAGE_ENGINE_BACKEND;
116 +
117 +#define is_valid_backend(backend) ((backend) >= STORAGE_ENGINE_BACKEND_RRDDIM && (backend) <= STORAGE_ENGINE_BACKEND_DBENGINE)
118 +
119 // iterator state for RRD dimension data queries
120 struct storage_engine_query_handle {
121 time_t start_time_s;
122 time_t end_time_s;
123 STORAGE_PRIORITY priority;
117 - STORAGE_QUERY_HANDLE* handle;
124 + STORAGE_ENGINE_BACKEND backend;
125 + STORAGE_QUERY_HANDLE *handle;
126 };
127
128 // ----------------------------------------------------------------------------
@@ -162,11 +170,11 @@ extern time_t rrdset_free_obsolete_time_s;
170
171 #if defined(ENV32BIT)
172 #define MIN_LIBUV_WORKER_THREADS 8
165 -#define MAX_LIBUV_WORKER_THREADS 64
173 +#define MAX_LIBUV_WORKER_THREADS 128
174 #define RESERVED_LIBUV_WORKER_THREADS 3
175 #else
176 #define MIN_LIBUV_WORKER_THREADS 16
169 -#define MAX_LIBUV_WORKER_THREADS 128
177 +#define MAX_LIBUV_WORKER_THREADS 1024
178 #define RESERVED_LIBUV_WORKER_THREADS 6
179 #endif
180
@@ -301,19 +309,20 @@ bool exporting_labels_filter_callback(const char *name, const char *value, RRDLA
309
310 // ----------------------------------------------------------------------------
311 // engine-specific iterator state for dimension data collection
304 -typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
312 +typedef struct storage_collect_handle {
313 + STORAGE_ENGINE_BACKEND backend;
314 +} STORAGE_COLLECT_HANDLE;
315
316 // ----------------------------------------------------------------------------
317 // Storage tier data for every dimension
318
319 struct rrddim_tier {
320 STORAGE_POINT virtual_point;
321 + STORAGE_ENGINE_BACKEND backend;
322 size_t tier_grouping;
323 time_t next_point_end_time_s;
324 STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database
325 STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle
315 - struct storage_engine_collect_ops *collect_ops;
316 - struct storage_engine_query_ops *query_ops;
326 };
327
328 void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s);
@@ -412,56 +421,214 @@ size_t rrddim_memory_file_header_size(void);
421 void rrddim_memory_file_save(RRDDIM *rd);
422
423 // ------------------------------------------------------------------------
415 -// function pointers that handle data collection
416 -struct storage_engine_collect_ops {
417 - // an initialization function to run before starting collection
418 - STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
424 +// DATA COLLECTION STORAGE OPS
425
420 - // run this to store each metric into the database
421 - void (*store_metric)(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number, NETDATA_DOUBLE min_value,
422 - NETDATA_DOUBLE max_value, uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
426 +STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
427 +STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
428 +static inline STORAGE_METRICS_GROUP *storage_engine_metrics_group_get(STORAGE_ENGINE_BACKEND backend, STORAGE_INSTANCE *db_instance, uuid_t *uuid) {
429 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
430
424 - // run this to flush / reset the current data collection sequence
425 - void (*flush)(STORAGE_COLLECT_HANDLE *collection_handle);
431 +#ifdef ENABLE_DBENGINE
432 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
433 + return rrdeng_metrics_group_get(db_instance, uuid);
434 +#endif
435 + return rrddim_metrics_group_get(db_instance, uuid);
436 +}
437
427 - // a finalization function to run after collection is over
428 - // returns 1 if it's safe to delete the dimension
429 - int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle);
438 +void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
439 +void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
440 +static inline void storage_engine_metrics_group_release(STORAGE_ENGINE_BACKEND backend, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
441 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
442
431 - void (*change_collection_frequency)(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
443 +#ifdef ENABLE_DBENGINE
444 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
445 + rrdeng_metrics_group_release(db_instance, smg);
446 + else
447 +#endif
448 + rrddim_metrics_group_release(db_instance, smg);
449 +}
450 +
451 +STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
452 +STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
453 +static inline STORAGE_COLLECT_HANDLE *storage_metric_store_init(STORAGE_ENGINE_BACKEND backend, STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg) {
454 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
455 +
456 +#ifdef ENABLE_DBENGINE
457 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
458 + return rrdeng_store_metric_init(db_metric_handle, update_every, smg);
459 +#endif
460 + return rrddim_collect_init(db_metric_handle, update_every, smg);
461 +}
462 +
463 +void rrdeng_store_metric_next(
464 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
465 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
466 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
467 +
468 +void rrddim_collect_store_metric(
469 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
470 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
471 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
472 +
473 +static inline void storage_engine_store_metric(
474 + STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut,
475 + NETDATA_DOUBLE n, NETDATA_DOUBLE min_value, NETDATA_DOUBLE max_value,
476 + uint16_t count, uint16_t anomaly_count, SN_FLAGS flags) {
477 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
478 +
479 +#ifdef ENABLE_DBENGINE
480 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
481 + return rrdeng_store_metric_next(collection_handle, point_in_time_ut,
482 + n, min_value, max_value,
483 + count, anomaly_count, flags);
484 +#endif
485 + return rrddim_collect_store_metric(collection_handle, point_in_time_ut,
486 + n, min_value, max_value,
487 + count, anomaly_count, flags);
488 +}
489 +
490 +void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
491 +void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
492 +static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
493 + if(unlikely(!collection_handle))
494 + return;
495 +
496 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
497 +
498 +#ifdef ENABLE_DBENGINE
499 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
500 + rrdeng_store_metric_flush_current_page(collection_handle);
501 + else
502 +#endif
503 + rrddim_store_metric_flush(collection_handle);
504 +}
505 +
506 +int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
507 +int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
508 +// a finalization function to run after collection is over
509 +// returns 1 if it's safe to delete the dimension
510 +static inline int storage_engine_store_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
511 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
512 +
513 +#ifdef ENABLE_DBENGINE
514 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
515 + return rrdeng_store_metric_finalize(collection_handle);
516 +#endif
517 +
518 + return rrddim_collect_finalize(collection_handle);
519 +}
520 +
521 +void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
522 +void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
523 +static inline void storage_engine_store_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every) {
524 + internal_fatal(!is_valid_backend(collection_handle->backend), "STORAGE: invalid backend");
525 +
526 +#ifdef ENABLE_DBENGINE
527 + if(likely(collection_handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
528 + rrdeng_store_metric_change_collection_frequency(collection_handle, update_every);
529 + else
530 +#endif
531 + rrddim_store_metric_change_collection_frequency(collection_handle, update_every);
532 +}
533
433 - STORAGE_METRICS_GROUP *(*metrics_group_get)(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
434 - void (*metrics_group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *sa);
435 -};
534
535 // ----------------------------------------------------------------------------
536 +// STORAGE ENGINE QUERY OPS
537 +
538 +time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
539 +time_t rrddim_query_oldest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
540 +static inline time_t storage_engine_oldest_time_s(STORAGE_ENGINE_BACKEND backend, STORAGE_METRIC_HANDLE *db_metric_handle) {
541 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
542 +
543 +#ifdef ENABLE_DBENGINE
544 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
545 + return rrdeng_metric_oldest_time(db_metric_handle);
546 +#endif
547 + return rrddim_query_oldest_time_s(db_metric_handle);
548 +}
549
439 -// function pointers that handle database queries
440 -struct storage_engine_query_ops {
441 - // run this before starting a series of next_metric() database queries
442 - void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle, time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
550 +time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
551 +time_t rrddim_query_latest_time_s(STORAGE_METRIC_HANDLE *db_metric_handle);
552 +static inline time_t storage_engine_latest_time_s(STORAGE_ENGINE_BACKEND backend, STORAGE_METRIC_HANDLE *db_metric_handle) {
553 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
554
444 - // run this to load each metric number from the database
445 - STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
555 +#ifdef ENABLE_DBENGINE
556 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
557 + return rrdeng_metric_latest_time(db_metric_handle);
558 +#endif
559 + return rrddim_query_latest_time_s(db_metric_handle);
560 +}
561
447 - // run this to test if the series of next_metric() database queries is finished
448 - int (*is_finished)(struct storage_engine_query_handle *handle);
562 +void rrdeng_load_metric_init(
563 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle,
564 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
565
450 - // run this after finishing a series of load_metric() database queries
451 - void (*finalize)(struct storage_engine_query_handle *handle);
566 +void rrddim_query_init(
567 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
568 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
569
453 - // get the timestamp of the last entry of this metric
454 - time_t (*latest_time_s)(STORAGE_METRIC_HANDLE *db_metric_handle);
570 +static inline void storage_engine_query_init(
571 + STORAGE_ENGINE_BACKEND backend,
572 + STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *handle,
573 + time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority) {
574 + internal_fatal(!is_valid_backend(backend), "STORAGE: invalid backend");
575
456 - // get the timestamp of the first entry of this metric
457 - time_t (*oldest_time_s)(STORAGE_METRIC_HANDLE *db_metric_handle);
576 +#ifdef ENABLE_DBENGINE
577 + if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
578 + rrdeng_load_metric_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
579 + else
580 +#endif
581 + rrddim_query_init(db_metric_handle, handle, start_time_s, end_time_s, priority);
582 +}
583
459 - // adapt 'before' timestamp to the optimal for the query
460 - // can only move 'before' ahead (to the future)
461 - time_t (*align_to_optimal_before)(struct storage_engine_query_handle *handle);
462 -};
584 +STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);
585 +STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handle);
586 +static inline STORAGE_POINT storage_engine_query_next_metric(struct storage_engine_query_handle *handle) {
587 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
588 +
589 +#ifdef ENABLE_DBENGINE
590 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
591 + return rrdeng_load_metric_next(handle);
592 +#endif
593 + return rrddim_query_next_metric(handle);
594 +}
595 +
596 +int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
597 +int rrddim_query_is_finished(struct storage_engine_query_handle *handle);
598 +static inline int storage_engine_query_is_finished(struct storage_engine_query_handle *handle) {
599 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
600 +
601 +#ifdef ENABLE_DBENGINE
602 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
603 + return rrdeng_load_metric_is_finished(handle);
604 +#endif
605 + return rrddim_query_is_finished(handle);
606 +}
607 +
608 +void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
609 +void rrddim_query_finalize(struct storage_engine_query_handle *handle);
610 +static inline void storage_engine_query_finalize(struct storage_engine_query_handle *handle) {
611 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
612 +
613 +#ifdef ENABLE_DBENGINE
614 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
615 + rrdeng_load_metric_finalize(handle);
616 + else
617 +#endif
618 + rrddim_query_finalize(handle);
619 +}
620
464 -typedef struct storage_engine STORAGE_ENGINE;
621 +time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
622 +time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);
623 +static inline time_t storage_engine_align_to_optimal_before(struct storage_engine_query_handle *handle) {
624 + internal_fatal(!is_valid_backend(handle->backend), "STORAGE: invalid backend");
625 +
626 +#ifdef ENABLE_DBENGINE
627 + if(likely(handle->backend == STORAGE_ENGINE_BACKEND_DBENGINE))
628 + return rrdeng_load_align_to_optimal_before(handle);
629 +#endif
630 + return rrddim_query_align_to_optimal_before(handle);
631 +}
632
633 // ------------------------------------------------------------------------
634 // function pointers for all APIs provided by a storage engine
@@ -472,17 +639,14 @@ typedef struct storage_engine_api {
639 void (*metric_release)(STORAGE_METRIC_HANDLE *);
640 STORAGE_METRIC_HANDLE *(*metric_dup)(STORAGE_METRIC_HANDLE *);
641 bool (*metric_retention_by_uuid)(STORAGE_INSTANCE *db_instance, uuid_t *uuid, time_t *first_entry_s, time_t *last_entry_s);
475 -
476 - // operations
477 - struct storage_engine_collect_ops collect_ops;
478 - struct storage_engine_query_ops query_ops;
642 } STORAGE_ENGINE_API;
643
481 -struct storage_engine {
644 +typedef struct storage_engine {
645 + STORAGE_ENGINE_BACKEND backend;
646 RRD_MEMORY_MODE id;
647 const char* name;
648 STORAGE_ENGINE_API api;
485 -};
649 +} STORAGE_ENGINE;
650
651 STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);
652 STORAGE_ENGINE* storage_engine_find(const char* name);
database/rrddim.c
+7 -7
@@ -94,9 +94,8 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
94 size_t initialized = 0;
95 for(size_t tier = 0; tier < storage_tiers ; tier++) {
96 STORAGE_ENGINE *eng = host->db[tier].eng;
97 + rd->tiers[tier].backend = eng->backend;
98 rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
98 - rd->tiers[tier].collect_ops = &eng->api.collect_ops;
99 - rd->tiers[tier].query_ops = &eng->api.query_ops;
99 rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
100 storage_point_unset(rd->tiers[tier].virtual_point);
101 initialized++;
@@ -116,7 +115,8 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
115 size_t initialized = 0;
116 for (size_t tier = 0; tier < storage_tiers; tier++) {
117 if (rd->tiers[tier].db_metric_handle) {
119 - rd->tiers[tier].db_collection_handle = rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
118 + rd->tiers[tier].db_collection_handle =
119 + storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
120 initialized++;
121 }
122 }
@@ -175,7 +175,7 @@ bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
175
176 tiers_available++;
177
178 - if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle))
178 + if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle))
179 tiers_said_no_retention++;
180
181 rd->tiers[tier].db_collection_handle = NULL;
@@ -253,7 +253,7 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
253 for(size_t tier = 0; tier < storage_tiers ;tier++) {
254 if (!rd->tiers[tier].db_collection_handle)
255 rd->tiers[tier].db_collection_handle =
256 - rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
256 + storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
257 }
258
259 if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@@ -416,7 +416,7 @@ time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) {
416 if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
417 return 0;
418
419 - return rd->tiers[tier].query_ops->latest_time_s(rd->tiers[tier].db_metric_handle);
419 + return storage_engine_latest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle);
420 }
421
422 // get the timestamp of the last entry in the round-robin database
@@ -438,7 +438,7 @@ time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) {
438 if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
439 return 0;
440
441 - return rd->tiers[tier].query_ops->oldest_time_s(rd->tiers[tier].db_metric_handle);
441 + return storage_engine_oldest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle);
442 }
443
444 time_t rrddim_first_entry_s(RRDDIM *rd) {
database/rrdset.c
+13 -10
@@ -158,7 +158,7 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
158 STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
159 if(!eng) continue;
160
161 - st->storage_metrics_groups[tier] = eng->api.collect_ops.metrics_group_get(host->db[tier].instance, &st->chart_uuid);
161 + st->storage_metrics_groups[tier] = storage_engine_metrics_group_get(eng->backend, host->db[tier].instance, &st->chart_uuid);
162 }
163 }
164
@@ -203,7 +203,7 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
203 if(!eng) continue;
204
205 if(st->storage_metrics_groups[tier]) {
206 - eng->api.collect_ops.metrics_group_release(host->db[tier].instance, st->storage_metrics_groups[tier]);
206 + storage_engine_metrics_group_release(eng->backend, host->db[tier].instance, st->storage_metrics_groups[tier]);
207 st->storage_metrics_groups[tier] = NULL;
208 }
209 }
@@ -741,10 +741,8 @@ void rrdset_reset(RRDSET *st) {
741 rd->collections_counter = 0;
742
743 if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
744 - for(size_t tier = 0; tier < storage_tiers ;tier++) {
745 - if(rd->tiers[tier].db_collection_handle)
746 - rd->tiers[tier].collect_ops->flush(rd->tiers[tier].db_collection_handle);
747 - }
744 + for(size_t tier = 0; tier < storage_tiers ;tier++)
745 + storage_engine_store_flush(rd->tiers[tier].db_collection_handle);
746 }
747 }
748 rrddim_foreach_done(rd);
@@ -1120,7 +1118,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1118
1119 if (likely(!storage_point_is_unset(t->virtual_point))) {
1120
1123 - t->collect_ops->store_metric(
1121 + storage_engine_store_metric(
1122 t->db_collection_handle,
1123 t->next_point_end_time_s * USEC_PER_SEC,
1124 t->virtual_point.sum,
@@ -1131,7 +1129,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
1129 t->virtual_point.flags);
1130 }
1131 else {
1134 - t->collect_ops->store_metric(
1132 + storage_engine_store_metric(
1133 t->db_collection_handle,
1134 t->next_point_end_time_s * USEC_PER_SEC,
1135 NAN,
@@ -1203,7 +1201,10 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1201 #endif // NETDATA_LOG_COLLECTION_ERRORS
1202
1203 // store the metric on tier 0
1206 - rd->tiers[0].collect_ops->store_metric(rd->tiers[0].db_collection_handle, point_end_time_ut, n, 0, 0, 1, 0, flags);
1204 + storage_engine_store_metric(rd->tiers[0].db_collection_handle, point_end_time_ut,
1205 + n, 0, 0,
1206 + 1, 0, flags);
1207 +
1208 rrdset_done_statistics_points_stored_per_tier[0]++;
1209
1210 time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC);
@@ -1981,7 +1982,9 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
1982 rrddim_foreach_read(rd, st) {
1983 for (size_t tier = 0; tier < storage_tiers; tier++) {
1984 if (rd->tiers[tier].db_collection_handle)
1984 - rd->tiers[tier].collect_ops->change_collection_frequency(rd->tiers[tier].db_collection_handle, (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
1985 + storage_engine_store_change_collection_frequency(
1986 + rd->tiers[tier].db_collection_handle,
1987 + (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
1988 }
1989
1990 assert(rd->update_every == (int) prev_update_every_s &&
database/sqlite/sqlite_aclk_node.c
+6 -1
@@ -144,6 +144,7 @@ void aclk_check_node_info_and_collectors(void)
144 if (unlikely(!aclk_connected))
145 return;
146
147 + size_t pending = 0;
148 dfe_start_reentrant(rrdhost_root_index, host) {
149
150 struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
@@ -151,7 +152,8 @@ void aclk_check_node_info_and_collectors(void)
152 continue;
153
154 if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))) {
154 - info("ACLK: 'host:%s' not sending node info, context load is pending", rrdhost_hostname(host));
155 + internal_error(true, "ACLK SYNC: Context still pending for %s", rrdhost_hostname(host));
156 + pending++;
157 continue;
158 }
159
@@ -168,6 +170,9 @@ void aclk_check_node_info_and_collectors(void)
170 }
171 }
172 dfe_done(host);
173 +
174 + if(pending)
175 + info("ACLK: %zu nodes are pending for contexts to load, skipped sending node info for them", pending);
176 }
177
178 #endif
database/storage_engine.c
+6 -48
@@ -6,120 +6,78 @@
6 #include "engine/rrdengineapi.h"
7 #endif
8
9 -#define im_collect_ops { \
10 - .init = rrddim_collect_init, \
11 - .store_metric = rrddim_collect_store_metric, \
12 - .flush = rrddim_store_metric_flush, \
13 - .finalize = rrddim_collect_finalize, \
14 - .change_collection_frequency = rrddim_store_metric_change_collection_frequency, \
15 - .metrics_group_get = rrddim_metrics_group_get, \
16 - .metrics_group_release = rrddim_metrics_group_release, \
17 -}
18 -
19 -#define im_query_ops { \
20 - .init = rrddim_query_init, \
21 - .next_metric = rrddim_query_next_metric, \
22 - .is_finished = rrddim_query_is_finished, \
23 - .finalize = rrddim_query_finalize, \
24 - .latest_time_s = rrddim_query_latest_time_s, \
25 - .oldest_time_s = rrddim_query_oldest_time_s, \
26 - .align_to_optimal_before = rrddim_query_align_to_optimal_before, \
27 -}
28 -
9 static STORAGE_ENGINE engines[] = {
10 {
11 .id = RRD_MEMORY_MODE_NONE,
12 .name = RRD_MEMORY_MODE_NONE_NAME,
13 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
14 .api = {
15 .metric_get = rrddim_metric_get,
16 .metric_get_or_create = rrddim_metric_get_or_create,
17 .metric_dup = rrddim_metric_dup,
18 .metric_release = rrddim_metric_release,
19 .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
39 - .collect_ops = im_collect_ops,
40 - .query_ops = im_query_ops,
20 }
21 },
22 {
23 .id = RRD_MEMORY_MODE_RAM,
24 .name = RRD_MEMORY_MODE_RAM_NAME,
25 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
26 .api = {
27 .metric_get = rrddim_metric_get,
28 .metric_get_or_create = rrddim_metric_get_or_create,
29 .metric_dup = rrddim_metric_dup,
30 .metric_release = rrddim_metric_release,
31 .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
52 - .collect_ops = im_collect_ops,
53 - .query_ops = im_query_ops,
32 }
33 },
34 {
35 .id = RRD_MEMORY_MODE_MAP,
36 .name = RRD_MEMORY_MODE_MAP_NAME,
37 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
38 .api = {
39 .metric_get = rrddim_metric_get,
40 .metric_get_or_create = rrddim_metric_get_or_create,
41 .metric_dup = rrddim_metric_dup,
42 .metric_release = rrddim_metric_release,
43 .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
65 - .collect_ops = im_collect_ops,
66 - .query_ops = im_query_ops,
44 }
45 },
46 {
47 .id = RRD_MEMORY_MODE_SAVE,
48 .name = RRD_MEMORY_MODE_SAVE_NAME,
49 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
50 .api = {
51 .metric_get = rrddim_metric_get,
52 .metric_get_or_create = rrddim_metric_get_or_create,
53 .metric_dup = rrddim_metric_dup,
54 .metric_release = rrddim_metric_release,
55 .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
78 - .collect_ops = im_collect_ops,
79 - .query_ops = im_query_ops,
56 }
57 },
58 {
59 .id = RRD_MEMORY_MODE_ALLOC,
60 .name = RRD_MEMORY_MODE_ALLOC_NAME,
61 + .backend = STORAGE_ENGINE_BACKEND_RRDDIM,
62 .api = {
63 .metric_get = rrddim_metric_get,
64 .metric_get_or_create = rrddim_metric_get_or_create,
65 .metric_dup = rrddim_metric_dup,
66 .metric_release = rrddim_metric_release,
67 .metric_retention_by_uuid = rrddim_metric_retention_by_uuid,
91 - .collect_ops = im_collect_ops,
92 - .query_ops = im_query_ops,
68 }
69 },
70 #ifdef ENABLE_DBENGINE
71 {
72 .id = RRD_MEMORY_MODE_DBENGINE,
73 .name = RRD_MEMORY_MODE_DBENGINE_NAME,
74 + .backend = STORAGE_ENGINE_BACKEND_DBENGINE,
75 .api = {
76 .metric_get = rrdeng_metric_get,
77 .metric_get_or_create = rrdeng_metric_get_or_create,
78 .metric_dup = rrdeng_metric_dup,
79 .metric_release = rrdeng_metric_release,
80 .metric_retention_by_uuid = rrdeng_metric_retention_by_uuid,
105 - .collect_ops = {
106 - .init = rrdeng_store_metric_init,
107 - .store_metric = rrdeng_store_metric_next,
108 - .flush = rrdeng_store_metric_flush_current_page,
109 - .finalize = rrdeng_store_metric_finalize,
110 - .change_collection_frequency = rrdeng_store_metric_change_collection_frequency,
111 - .metrics_group_get = rrdeng_metrics_group_get,
112 - .metrics_group_release = rrdeng_metrics_group_release,
113 - },
114 - .query_ops = {
115 - .init = rrdeng_load_metric_init,
116 - .next_metric = rrdeng_load_metric_next,
117 - .is_finished = rrdeng_load_metric_is_finished,
118 - .finalize = rrdeng_load_metric_finalize,
119 - .latest_time_s = rrdeng_metric_latest_time,
120 - .oldest_time_s = rrdeng_metric_oldest_time,
121 - .align_to_optimal_before = rrdeng_load_align_to_optimal_before,
122 - }
81 }
82 },
83 #endif
exporting/process_data.c
+5 -5
@@ -77,8 +77,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
77 time_t before = instance->before;
78
79 // find the edges of the rrd database for this chart
80 - time_t first_t = rd->tiers[0].query_ops->oldest_time_s(rd->tiers[0].db_metric_handle);
81 - time_t last_t = rd->tiers[0].query_ops->latest_time_s(rd->tiers[0].db_metric_handle);
80 + time_t first_t = storage_engine_oldest_time_s(rd->tiers[0].backend, rd->tiers[0].db_metric_handle);
81 + time_t last_t = storage_engine_latest_time_s(rd->tiers[0].backend, rd->tiers[0].db_metric_handle);
82 time_t update_every = st->update_every;
83 struct storage_engine_query_handle handle;
84
@@ -126,8 +126,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
126 size_t counter = 0;
127 NETDATA_DOUBLE sum = 0;
128
129 - for (rd->tiers[0].query_ops->init(rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_LOW); !rd->tiers[0].query_ops->is_finished(&handle);) {
130 - STORAGE_POINT sp = rd->tiers[0].query_ops->next_metric(&handle);
129 + for (storage_engine_query_init(rd->tiers[0].backend, rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_LOW); !storage_engine_query_is_finished(&handle);) {
130 + STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
131 points_read++;
132
133 if (unlikely(storage_point_is_gap(sp))) {
@@ -138,7 +138,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
138 sum += sp.sum;
139 counter += sp.count;
140 }
141 - rd->tiers[0].query_ops->finalize(&handle);
141 + storage_engine_query_finalize(&handle);
142 global_statistics_exporters_query_completed(points_read);
143
144 if (unlikely(!counter)) {
ml/ml.cc
+5 -8
@@ -341,24 +341,21 @@ ml_dimension_calculated_numbers(ml_dimension_t *dim, const ml_training_request_t
341 /*
342 * Execute the query
343 */
344 - struct storage_engine_query_ops *ops = dim->rd->tiers[0].query_ops;
344 struct storage_engine_query_handle handle;
345
347 - ops->init(dim->rd->tiers[0].db_metric_handle,
348 - &handle,
349 - training_response.query_after_t,
350 - training_response.query_before_t,
346 + storage_engine_query_init(dim->rd->tiers[0].backend, dim->rd->tiers[0].db_metric_handle, &handle,
347 + training_response.query_after_t, training_response.query_before_t,
348 STORAGE_PRIORITY_BEST_EFFORT);
349
350 size_t idx = 0;
351 memset(tls_data.training_cns, 0, sizeof(calculated_number_t) * max_n * (Cfg.lag_n + 1));
352 calculated_number_t last_value = std::numeric_limits<calculated_number_t>::quiet_NaN();
353
357 - while (!ops->is_finished(&handle)) {
354 + while (!storage_engine_query_is_finished(&handle)) {
355 if (idx == max_n)
356 break;
357
361 - STORAGE_POINT sp = ops->next_metric(&handle);
358 + STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
359
360 time_t timestamp = sp.end_time_s;
361 calculated_number_t value = sp.sum / sp.count;
@@ -376,7 +373,7 @@ ml_dimension_calculated_numbers(ml_dimension_t *dim, const ml_training_request_t
373
374 idx++;
375 }
379 - ops->finalize(&handle);
376 + storage_engine_query_finalize(&handle);
377
378 global_statistics_ml_query_completed(/* points_read */ idx);
379
streaming/replication.c
+7 -8
@@ -96,7 +96,7 @@ struct replication_query {
96 size_t points_read;
97 size_t points_generated;
98
99 - struct storage_engine_query_ops *ops;
99 + STORAGE_ENGINE_BACKEND backend;
100 struct replication_request *rq;
101
102 size_t dimensions;
@@ -162,7 +162,7 @@ static struct replication_query *replication_query_prepare(
162 }
163 }
164
165 - q->ops = &st->rrdhost->db[0].eng->api.query_ops;
165 + q->backend = st->rrdhost->db[0].eng->backend;
166
167 // prepare our array of dimensions
168 size_t count = 0;
@@ -184,7 +184,7 @@ static struct replication_query *replication_query_prepare(
184 d->rda = dictionary_acquired_item_dup(rd_dfe.dict, rd_dfe.item);
185 d->rd = rd;
186
187 - q->ops->init(rd->tiers[0].db_metric_handle, &d->handle, q->query.after, q->query.before,
187 + storage_engine_query_init(q->backend, rd->tiers[0].db_metric_handle, &d->handle, q->query.after, q->query.before,
188 q->query.locked_data_collection ? STORAGE_PRIORITY_HIGH : STORAGE_PRIORITY_LOW);
189 d->enabled = true;
190 d->skip = false;
@@ -260,7 +260,7 @@ static void replication_query_finalize(BUFFER *wb, struct replication_query *q,
260 struct replication_dimension *d = &q->data[i];
261 if (unlikely(!d->enabled)) continue;
262
263 - q->ops->finalize(&d->handle);
263 + storage_engine_query_finalize(&d->handle);
264
265 dictionary_acquired_item_release(d->dict, d->rda);
266
@@ -292,7 +292,7 @@ static void replication_query_align_to_optimal_before(struct replication_query *
292 struct replication_dimension *d = &q->data[i];
293 if(unlikely(!d->enabled)) continue;
294
295 - time_t new_before = q->ops->align_to_optimal_before(&d->handle);
295 + time_t new_before = rrdeng_load_align_to_optimal_before(&d->handle);
296 if (!expanded_before || new_before < expanded_before)
297 expanded_before = new_before;
298 }
@@ -311,7 +311,6 @@ static bool replication_query_execute(BUFFER *wb, struct replication_query *q, s
311 time_t after = q->query.after;
312 time_t before = q->query.before;
313 size_t dimensions = q->dimensions;
314 - struct storage_engine_query_ops *ops = q->ops;
314 time_t wall_clock_time = q->wall_clock_time;
315
316 bool finished_with_gap = false;
@@ -331,8 +330,8 @@ static bool replication_query_execute(BUFFER *wb, struct replication_query *q, s
330
331 // fetch the first valid point for the dimension
332 int max_skip = 1000;
334 - while(d->sp.end_time_s < now && !ops->is_finished(&d->handle) && max_skip-- >= 0) {
335 - d->sp = ops->next_metric(&d->handle);
333 + while(d->sp.end_time_s < now && !storage_engine_query_is_finished(&d->handle) && max_skip-- >= 0) {
334 + d->sp = storage_engine_query_next_metric(&d->handle);
335 points_read++;
336 }
337
web/api/formatters/json_wrapper.c
+134 -74
@@ -368,17 +368,60 @@ static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, cons
368 buffer_json_array_close(wb);
369 }
370
371 +struct dimensions_sorted_walkthrough_data {
372 + BUFFER *wb;
373 + struct summary_total_counts *totals;
374 + QUERY_TARGET *qt;
375 +};
376 +
377 +struct dimensions_sorted_entry {
378 + const char *id;
379 + const char *name;
380 + STORAGE_POINT query_points;
381 + QUERY_METRICS_COUNTS metrics;
382 + uint32_t priority;
383 +};
384 +
385 +static int dimensions_sorted_walktrhough_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
386 + struct dimensions_sorted_walkthrough_data *sdwd = data;
387 + BUFFER *wb = sdwd->wb;
388 + struct summary_total_counts *totals = sdwd->totals;
389 + QUERY_TARGET *qt = sdwd->qt;
390 + struct dimensions_sorted_entry *z = value;
391 +
392 + buffer_json_add_array_item_object(wb);
393 + buffer_json_member_add_string(wb, "id", z->id);
394 + if (z->id != z->name && z->name)
395 + buffer_json_member_add_string(wb, "nm", z->name);
396 +
397 + query_target_metric_counts(wb, &z->metrics);
398 + query_target_points_statistics(wb, qt, &z->query_points);
399 + buffer_json_member_add_uint64(wb, "pri", z->priority);
400 + buffer_json_object_close(wb);
401 +
402 + aggregate_into_summary_totals(totals, &z->metrics);
403 +
404 + return 1;
405 +}
406 +
407 +int dimensions_sorted_compar(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2) {
408 + struct dimensions_sorted_entry *z1 = dictionary_acquired_item_value(*item1);
409 + struct dimensions_sorted_entry *z2 = dictionary_acquired_item_value(*item2);
410 +
411 + if(z1->priority == z2->priority)
412 + return strcmp(dictionary_acquired_item_name(*item1), dictionary_acquired_item_name(*item2));
413 + else if(z1->priority < z2->priority)
414 + return -1;
415 + else
416 + return 1;
417 +}
418 +
419 static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) {
372 - char name[RRD_ID_LENGTH_MAX * 2 + 2];
420 + char buf[RRD_ID_LENGTH_MAX * 2 + 2];
421
422 buffer_json_member_add_array(wb, key);
423 DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
376 - struct {
377 - const char *id;
378 - const char *name;
379 - STORAGE_POINT query_points;
380 - QUERY_METRICS_COUNTS metrics;
381 - } *z;
424 + struct dimensions_sorted_entry *z;
425 size_t q = 0;
426 for (long c = 0; c < (long) qt->dimensions.used; c++) {
427 QUERY_DIMENSION * qd = query_dimension(qt, c);
@@ -392,23 +435,31 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
435 qm = tqm;
436 }
437
438 + const char *key, *id, *name;
439 +
440 if(v2) {
396 - z = dictionary_set(dict, rrdmetric_acquired_name(rma), NULL, sizeof(*z));
397 - if(!z->id)
398 - z->id = rrdmetric_acquired_name(rma);
399 - if(!z->name)
400 - z->name = rrdmetric_acquired_name(rma);
441 + key = rrdmetric_acquired_name(rma);
442 + id = key;
443 + name = key;
444 }
445 else {
403 - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
446 + snprintfz(buf, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
447 rrdmetric_acquired_id(rma),
448 rrdmetric_acquired_name(rma));
449 + key = buf;
450 + id = rrdmetric_acquired_id(rma);
451 + name = rrdmetric_acquired_name(rma);
452 + }
453
407 - z = dictionary_set(dict, name, NULL, sizeof(*z));
408 - if (!z->id)
409 - z->id = rrdmetric_acquired_id(rma);
410 - if (!z->name)
411 - z->name = rrdmetric_acquired_name(rma);
454 + z = dictionary_set(dict, key, NULL, sizeof(*z));
455 + if(!z->id) {
456 + z->id = id;
457 + z->name = name;
458 + z->priority = qd->priority;
459 + }
460 + else {
461 + if(qd->priority < z->priority)
462 + z->priority = qd->priority;
463 }
464
465 if(qm) {
@@ -423,27 +474,26 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
474 else
475 z->metrics.excluded++;
476 }
426 - dfe_start_read(dict, z) {
427 - if(v2) {
428 - buffer_json_add_array_item_object(wb);
429 - buffer_json_member_add_string(wb, "id", z->id);
430 - if(z->id != z->name)
431 - buffer_json_member_add_string(wb, "nm", z->name);
432 -
433 - query_target_metric_counts(wb, &z->metrics);
434 - query_target_points_statistics(wb, qt, &z->query_points);
435 - buffer_json_object_close(wb);
477
437 - aggregate_into_summary_totals(totals, &z->metrics);
438 - }
439 - else {
440 - buffer_json_add_array_item_array(wb);
441 - buffer_json_add_array_item_string(wb, z->id);
442 - buffer_json_add_array_item_string(wb, z->name);
443 - buffer_json_array_close(wb);
444 - }
445 - }
446 - dfe_done(z);
478 + if(v2) {
479 + struct dimensions_sorted_walkthrough_data t = {
480 + .wb = wb,
481 + .totals = totals,
482 + .qt = qt,
483 + };
484 + dictionary_sorted_walkthrough_rw(dict, DICTIONARY_LOCK_READ, dimensions_sorted_walktrhough_cb,
485 + &t, dimensions_sorted_compar);
486 + }
487 + else {
488 + // v1
489 + dfe_start_read(dict, z) {
490 + buffer_json_add_array_item_array(wb);
491 + buffer_json_add_array_item_string(wb, z->id);
492 + buffer_json_add_array_item_string(wb, z->name);
493 + buffer_json_array_close(wb);
494 + }
495 + dfe_done(z);
496 + }
497 dictionary_destroy(dict);
498 buffer_json_array_close(wb);
499 }
@@ -805,18 +855,6 @@ static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char
855 buffer_json_object_close(wb);
856 }
857
808 -static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
809 - QUERY_TARGET *qt = r->internal.qt;
810 -
811 - qt->timings.finished_ut = now_monotonic_usec();
812 - buffer_json_member_add_object(wb, key);
813 - buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(qt->timings.preprocessed_ut - qt->timings.received_ut) / USEC_PER_MS);
814 - buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(qt->timings.executed_ut - qt->timings.preprocessed_ut) / USEC_PER_MS);
815 - buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.executed_ut) / USEC_PER_MS);
816 - buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.received_ut) / USEC_PER_MS);
817 - buffer_json_object_close(wb);
818 -}
819 -
858 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
859 QUERY_TARGET *qt = r->internal.qt;
860 DATASOURCE_FORMAT format = qt->request.format;
@@ -948,35 +986,50 @@ static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_
986
987 buffer_json_member_add_array(wb, key);
988
951 - if(qt->request.group_by & RRDR_GROUP_BY_SELECTED)
989 + // find the deeper group-by
990 + ssize_t g = 0;
991 + for(g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
992 + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
993 + break;
994 + }
995 +
996 + if(g > 0)
997 + g--;
998 +
999 + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by;
1000 +
1001 + if(group_by & RRDR_GROUP_BY_SELECTED)
1002 buffer_json_add_array_item_string(wb, "selected");
1003
1004 + else if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
1005 + buffer_json_add_array_item_string(wb, "percentage-of-instance");
1006 +
1007 else {
1008
956 - if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION)
1009 + if(group_by & RRDR_GROUP_BY_DIMENSION)
1010 buffer_json_add_array_item_string(wb, "dimension");
1011
959 - if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE)
1012 + if(group_by & RRDR_GROUP_BY_INSTANCE)
1013 buffer_json_add_array_item_string(wb, "instance");
1014
962 - if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
1015 + if(group_by & RRDR_GROUP_BY_LABEL) {
1016 BUFFER *b = buffer_create(0, NULL);
964 - for (size_t l = 0; l < qt->group_by.used; l++) {
1017 + for (size_t l = 0; l < qt->group_by[g].used; l++) {
1018 buffer_flush(b);
1019 buffer_fast_strcat(b, "label:", 6);
967 - buffer_strcat(b, qt->group_by.label_keys[l]);
1020 + buffer_strcat(b, qt->group_by[g].label_keys[l]);
1021 buffer_json_add_array_item_string(wb, buffer_tostring(b));
1022 }
1023 buffer_free(b);
1024 }
1025
973 - if(qt->request.group_by & RRDR_GROUP_BY_NODE)
1026 + if(group_by & RRDR_GROUP_BY_NODE)
1027 buffer_json_add_array_item_string(wb, "node");
1028
976 - if(qt->request.group_by & RRDR_GROUP_BY_CONTEXT)
1029 + if(group_by & RRDR_GROUP_BY_CONTEXT)
1030 buffer_json_add_array_item_string(wb, "context");
1031
979 - if(qt->request.group_by & RRDR_GROUP_BY_UNITS)
1032 + if(group_by & RRDR_GROUP_BY_UNITS)
1033 buffer_json_add_array_item_string(wb, "units");
1034 }
1035
@@ -1237,7 +1290,6 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
1290
1291 buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
1292 buffer_json_member_add_uint64(wb, "api", 2);
1240 - buffer_json_agents_array_v2(wb, 0);
1293
1294 if(options & RRDR_OPTION_DEBUG) {
1295 buffer_json_member_add_string(wb, "id", qt->id);
@@ -1284,21 +1336,28 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
1336 buffer_json_member_add_string(wb, "time_resampling", NULL);
1337 buffer_json_object_close(wb); // time
1338
1287 - buffer_json_member_add_object(wb, "metrics");
1339 + buffer_json_member_add_array(wb, "metrics");
1340 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
1341 + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
1342 + break;
1343
1289 - buffer_json_member_add_array(wb, "group_by");
1290 - buffer_json_group_by_to_array(wb, qt->request.group_by);
1291 - buffer_json_array_close(wb);
1344 + buffer_json_add_array_item_object(wb);
1345 + {
1346 + buffer_json_member_add_array(wb, "group_by");
1347 + buffer_json_group_by_to_array(wb, qt->request.group_by[g].group_by);
1348 + buffer_json_array_close(wb);
1349
1293 - buffer_json_member_add_array(wb, "group_by_label");
1294 - for(size_t l = 0; l < qt->group_by.used ;l++)
1295 - buffer_json_add_array_item_string(wb, qt->group_by.label_keys[l]);
1296 - buffer_json_array_close(wb);
1350 + buffer_json_member_add_array(wb, "group_by_label");
1351 + for (size_t l = 0; l < qt->group_by[g].used; l++)
1352 + buffer_json_add_array_item_string(wb, qt->group_by[g].label_keys[l]);
1353 + buffer_json_array_close(wb);
1354
1298 - buffer_json_member_add_string(wb, "aggregation",
1299 - group_by_aggregate_function_to_string(
1300 - qt->request.group_by_aggregate_function));
1301 - buffer_json_object_close(wb); // dimensions
1355 + buffer_json_member_add_string(
1356 + wb, "aggregation",group_by_aggregate_function_to_string(qt->request.group_by[g].aggregation));
1357 + }
1358 + buffer_json_object_close(wb);
1359 + }
1360 + buffer_json_array_close(wb); // group_by
1361 }
1362 buffer_json_object_close(wb); // aggregations
1363
@@ -1444,7 +1503,7 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) {
1503 buffer_json_member_add_double(wb, "min", r->view.min);
1504 buffer_json_member_add_double(wb, "max", r->view.max);
1505
1447 - rrdr_timings_v12(wb, "timings", r);
1506 + buffer_json_query_timings(wb, "timings", &r->internal.qt->timings);
1507 buffer_json_finalize(wb);
1508 }
1509
@@ -1497,6 +1556,7 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
1556 }
1557 buffer_json_object_close(wb); // view
1558
1500 - rrdr_timings_v12(wb, "timings", r);
1559 + buffer_json_agents_array_v2(wb, &r->internal.qt->timings, 0);
1560 + buffer_json_cloud_timings(wb, "timings", &r->internal.qt->timings);
1561 buffer_json_finalize(wb);
1562 }
web/api/formatters/rrd2json.c
-8
@@ -3,14 +3,6 @@
3 #include "web/api/web_api_v1.h"
4 #include "database/storage_engine.h"
5
6 -inline bool query_target_has_percentage_units(struct query_target *qt) {
7 - if(qt->window.options & RRDR_OPTION_PERCENTAGE ||
8 - qt->window.time_group_method == RRDR_GROUPING_CV)
9 - return true;
10 -
11 - return false;
12 -}
13 -
6 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
7 rrdset2json(st, wb, NULL, NULL, 0);
8 }
web/api/formatters/rrd2json.h
-4
@@ -61,10 +61,6 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, ti
61
62 void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options);
63
64 -struct query_target;
65 -bool query_target_has_percentage_units(struct query_target *qt);
66 -#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW)
67 -
64 int rrdset2value_api_v1(
65 RRDSET *st
66 , BUFFER *wb
web/api/formatters/value/value.c
+3 -1
@@ -93,7 +93,8 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
93 };
94
95 ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
96 - RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
96 + QUERY_TARGET *qt = query_target_create(&qtr);
97 + RRDR *r = rrd2rrdr(owa, qt);
98
99 QUERY_VALUE qv;
100
@@ -143,6 +144,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
144 }
145
146 rrdr_free(owa, r);
147 + query_target_release(qt);
148 onewayalloc_destroy(owa);
149
150 return qv;
web/api/netdata-swagger.yaml
+1
@@ -249,6 +249,7 @@ paths:
249 enum:
250 - dimension
251 - instance
252 + - percentage-of-instance
253 - label
254 - node
255 - context
web/api/queries/average/average.c
-52
@@ -2,55 +2,3 @@
2
3 #include "average.h"
4
5 -// ----------------------------------------------------------------------------
6 -// average
7 -
8 -struct grouping_average {
9 - NETDATA_DOUBLE sum;
10 - size_t count;
11 -};
12 -
13 -void grouping_create_average(RRDR *r, const char *options __maybe_unused) {
14 - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average));
15 -}
16 -
17 -// resets when switches dimensions
18 -// so, clear everything to restart
19 -void grouping_reset_average(RRDR *r) {
20 - struct grouping_average *g = (struct grouping_average *)r->time_grouping.data;
21 - g->sum = 0;
22 - g->count = 0;
23 -}
24 -
25 -void grouping_free_average(RRDR *r) {
26 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
27 - r->time_grouping.data = NULL;
28 -}
29 -
30 -void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) {
31 - struct grouping_average *g = (struct grouping_average *)r->time_grouping.data;
32 - g->sum += value;
33 - g->count++;
34 -}
35 -
36 -NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
37 - struct grouping_average *g = (struct grouping_average *)r->time_grouping.data;
38 -
39 - NETDATA_DOUBLE value;
40 -
41 - if(unlikely(!g->count)) {
42 - value = 0.0;
43 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
44 - }
45 - else {
46 - if(unlikely(r->time_grouping.resampling_group != 1))
47 - value = g->sum / r->time_grouping.resampling_divisor;
48 - else
49 - value = g->sum / g->count;
50 - }
51 -
52 - g->sum = 0.0;
53 - g->count = 0;
54 -
55 - return value;
56 -}
web/api/queries/average/average.h
+52 -5
@@ -6,10 +6,57 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_average(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_average(RRDR *r);
11 -void grouping_free_average(RRDR *r);
12 -void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +// ----------------------------------------------------------------------------
10 +// average
11 +
12 +struct tg_average {
13 + NETDATA_DOUBLE sum;
14 + size_t count;
15 +};
16 +
17 +static inline void tg_average_create(RRDR *r, const char *options __maybe_unused) {
18 + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_average));
19 +}
20 +
21 +// resets when switches dimensions
22 +// so, clear everything to restart
23 +static inline void tg_average_reset(RRDR *r) {
24 + struct tg_average *g = (struct tg_average *)r->time_grouping.data;
25 + g->sum = 0;
26 + g->count = 0;
27 +}
28 +
29 +static inline void tg_average_free(RRDR *r) {
30 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
31 + r->time_grouping.data = NULL;
32 +}
33 +
34 +static inline void tg_average_add(RRDR *r, NETDATA_DOUBLE value) {
35 + struct tg_average *g = (struct tg_average *)r->time_grouping.data;
36 + g->sum += value;
37 + g->count++;
38 +}
39 +
40 +static inline NETDATA_DOUBLE tg_average_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
41 + struct tg_average *g = (struct tg_average *)r->time_grouping.data;
42 +
43 + NETDATA_DOUBLE value;
44 +
45 + if(unlikely(!g->count)) {
46 + value = 0.0;
47 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
48 + }
49 + else {
50 + if(unlikely(r->time_grouping.resampling_group != 1))
51 + value = g->sum / r->time_grouping.resampling_divisor;
52 + else
53 + value = g->sum / g->count;
54 + }
55 +
56 + g->sum = 0.0;
57 + g->count = 0;
58 +
59 + return value;
60 +}
61
62 #endif //NETDATA_API_QUERY_AVERAGE_H
web/api/queries/countif/countif.c
-129
@@ -5,132 +5,3 @@
5 // ----------------------------------------------------------------------------
6 // countif
7
8 -struct grouping_countif {
9 - size_t (*comparison)(NETDATA_DOUBLE, NETDATA_DOUBLE);
10 - NETDATA_DOUBLE target;
11 - size_t count;
12 - size_t matched;
13 -};
14 -
15 -static size_t countif_equal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
16 - return (v == target);
17 -}
18 -
19 -static size_t countif_notequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
20 - return (v != target);
21 -}
22 -
23 -static size_t countif_less(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
24 - return (v < target);
25 -}
26 -
27 -static size_t countif_lessequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
28 - return (v <= target);
29 -}
30 -
31 -static size_t countif_greater(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
32 - return (v > target);
33 -}
34 -
35 -static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
36 - return (v >= target);
37 -}
38 -
39 -void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {
40 - struct grouping_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_countif));
41 - r->time_grouping.data = g;
42 -
43 - if(options && *options) {
44 - // skip any leading spaces
45 - while(isspace(*options)) options++;
46 -
47 - // find the comparison function
48 - switch(*options) {
49 - case '!':
50 - options++;
51 - if(*options != '=' && *options != ':')
52 - options--;
53 - g->comparison = countif_notequal;
54 - break;
55 -
56 - case '>':
57 - options++;
58 - if(*options == '=' || *options == ':') {
59 - g->comparison = countif_greaterequal;
60 - }
61 - else {
62 - options--;
63 - g->comparison = countif_greater;
64 - }
65 - break;
66 -
67 - case '<':
68 - options++;
69 - if(*options == '>') {
70 - g->comparison = countif_notequal;
71 - }
72 - else if(*options == '=' || *options == ':') {
73 - g->comparison = countif_lessequal;
74 - }
75 - else {
76 - options--;
77 - g->comparison = countif_less;
78 - }
79 - break;
80 -
81 - default:
82 - case '=':
83 - case ':':
84 - g->comparison = countif_equal;
85 - break;
86 - }
87 - if(*options) options++;
88 -
89 - // skip everything up to the first digit
90 - while(isspace(*options)) options++;
91 -
92 - g->target = str2ndd(options, NULL);
93 - }
94 - else {
95 - g->target = 0.0;
96 - g->comparison = countif_equal;
97 - }
98 -}
99 -
100 -// resets when switches dimensions
101 -// so, clear everything to restart
102 -void grouping_reset_countif(RRDR *r) {
103 - struct grouping_countif *g = (struct grouping_countif *)r->time_grouping.data;
104 - g->matched = 0;
105 - g->count = 0;
106 -}
107 -
108 -void grouping_free_countif(RRDR *r) {
109 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
110 - r->time_grouping.data = NULL;
111 -}
112 -
113 -void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) {
114 - struct grouping_countif *g = (struct grouping_countif *)r->time_grouping.data;
115 - g->matched += g->comparison(value, g->target);
116 - g->count++;
117 -}
118 -
119 -NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
120 - struct grouping_countif *g = (struct grouping_countif *)r->time_grouping.data;
121 -
122 - NETDATA_DOUBLE value;
123 -
124 - if(unlikely(!g->count)) {
125 - value = 0.0;
126 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
127 - }
128 - else {
129 - value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count;
130 - }
131 -
132 - g->matched = 0;
133 - g->count = 0;
134 -
135 - return value;
136 -}
web/api/queries/countif/countif.h
+138 -5
@@ -6,10 +6,143 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_countif(RRDR *r);
11 -void grouping_free_countif(RRDR *r);
12 -void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +enum tg_countif_cmp {
10 + TG_COUNTIF_EQUAL,
11 + TG_COUNTIF_NOTEQUAL,
12 + TG_COUNTIF_LESS,
13 + TG_COUNTIF_LESSEQUAL,
14 + TG_COUNTIF_GREATER,
15 + TG_COUNTIF_GREATEREQUAL,
16 +};
17 +
18 +struct tg_countif {
19 + enum tg_countif_cmp comparison;
20 + NETDATA_DOUBLE target;
21 + size_t count;
22 + size_t matched;
23 +};
24 +
25 +static inline void tg_countif_create(RRDR *r, const char *options __maybe_unused) {
26 + struct tg_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_countif));
27 + r->time_grouping.data = g;
28 +
29 + if(options && *options) {
30 + // skip any leading spaces
31 + while(isspace(*options)) options++;
32 +
33 + // find the comparison function
34 + switch(*options) {
35 + case '!':
36 + options++;
37 + if(*options != '=' && *options != ':')
38 + options--;
39 + g->comparison = TG_COUNTIF_NOTEQUAL;
40 + break;
41 +
42 + case '>':
43 + options++;
44 + if(*options == '=' || *options == ':') {
45 + g->comparison = TG_COUNTIF_GREATEREQUAL;
46 + }
47 + else {
48 + options--;
49 + g->comparison = TG_COUNTIF_GREATER;
50 + }
51 + break;
52 +
53 + case '<':
54 + options++;
55 + if(*options == '>') {
56 + g->comparison = TG_COUNTIF_NOTEQUAL;
57 + }
58 + else if(*options == '=' || *options == ':') {
59 + g->comparison = TG_COUNTIF_LESSEQUAL;
60 + }
61 + else {
62 + options--;
63 + g->comparison = TG_COUNTIF_LESS;
64 + }
65 + break;
66 +
67 + default:
68 + case '=':
69 + case ':':
70 + g->comparison = TG_COUNTIF_EQUAL;
71 + break;
72 + }
73 + if(*options) options++;
74 +
75 + // skip everything up to the first digit
76 + while(isspace(*options)) options++;
77 +
78 + g->target = str2ndd(options, NULL);
79 + }
80 + else {
81 + g->target = 0.0;
82 + g->comparison = TG_COUNTIF_EQUAL;
83 + }
84 +}
85 +
86 +// resets when switches dimensions
87 +// so, clear everything to restart
88 +static inline void tg_countif_reset(RRDR *r) {
89 + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data;
90 + g->matched = 0;
91 + g->count = 0;
92 +}
93 +
94 +static inline void tg_countif_free(RRDR *r) {
95 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
96 + r->time_grouping.data = NULL;
97 +}
98 +
99 +static inline void tg_countif_add(RRDR *r, NETDATA_DOUBLE value) {
100 + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data;
101 + switch(g->comparison) {
102 + case TG_COUNTIF_GREATER:
103 + if(value > g->target) g->matched++;
104 + break;
105 +
106 + case TG_COUNTIF_GREATEREQUAL:
107 + if(value >= g->target) g->matched++;
108 + break;
109 +
110 + case TG_COUNTIF_LESS:
111 + if(value < g->target) g->matched++;
112 + break;
113 +
114 + case TG_COUNTIF_LESSEQUAL:
115 + if(value <= g->target) g->matched++;
116 + break;
117 +
118 + case TG_COUNTIF_EQUAL:
119 + if(value == g->target) g->matched++;
120 + break;
121 +
122 + case TG_COUNTIF_NOTEQUAL:
123 + if(value != g->target) g->matched++;
124 + break;
125 + }
126 + g->count++;
127 +}
128 +
129 +static inline NETDATA_DOUBLE tg_countif_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
130 + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data;
131 +
132 + NETDATA_DOUBLE value;
133 +
134 + if(unlikely(!g->count)) {
135 + value = 0.0;
136 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
137 + }
138 + else {
139 + value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count;
140 + }
141 +
142 + g->matched = 0;
143 + g->count = 0;
144 +
145 + return value;
146 +}
147
148 #endif //NETDATA_API_QUERY_COUNTIF_H
web/api/queries/des/des.c
-129
@@ -6,132 +6,3 @@
6
7 // ----------------------------------------------------------------------------
8 // single exponential smoothing
9 -
10 -struct grouping_des {
11 - NETDATA_DOUBLE alpha;
12 - NETDATA_DOUBLE alpha_other;
13 - NETDATA_DOUBLE beta;
14 - NETDATA_DOUBLE beta_other;
15 -
16 - NETDATA_DOUBLE level;
17 - NETDATA_DOUBLE trend;
18 -
19 - size_t count;
20 -};
21 -
22 -static size_t max_window_size = 15;
23 -
24 -void grouping_init_des(void) {
25 - long long ret = config_get_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size);
26 - if(ret <= 1) {
27 - config_set_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size);
28 - }
29 - else {
30 - max_window_size = (size_t) ret;
31 - }
32 -}
33 -
34 -static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) {
35 - (void)g;
36 -
37 - NETDATA_DOUBLE points;
38 - if(r->view.group == 1) {
39 - // provide a running DES
40 - points = (NETDATA_DOUBLE)r->time_grouping.points_wanted;
41 - }
42 - else {
43 - // provide a SES with flush points
44 - points = (NETDATA_DOUBLE)r->view.group;
45 - }
46 -
47 - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
48 - // A commonly used value for alpha is 2 / (N + 1)
49 - return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points;
50 -}
51 -
52 -static inline void set_alpha(RRDR *r, struct grouping_des *g) {
53 - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
54 - // A commonly used value for alpha is 2 / (N + 1)
55 -
56 - g->alpha = 2.0 / (window(r, g) + 1.0);
57 - g->alpha_other = 1.0 - g->alpha;
58 -
59 - //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha);
60 -}
61 -
62 -static inline void set_beta(RRDR *r, struct grouping_des *g) {
63 - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
64 - // A commonly used value for alpha is 2 / (N + 1)
65 -
66 - g->beta = 2.0 / (window(r, g) + 1.0);
67 - g->beta_other = 1.0 - g->beta;
68 -
69 - //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta);
70 -}
71 -
72 -void grouping_create_des(RRDR *r, const char *options __maybe_unused) {
73 - struct grouping_des *g = (struct grouping_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct grouping_des));
74 - set_alpha(r, g);
75 - set_beta(r, g);
76 - g->level = 0.0;
77 - g->trend = 0.0;
78 - g->count = 0;
79 - r->time_grouping.data = g;
80 -}
81 -
82 -// resets when switches dimensions
83 -// so, clear everything to restart
84 -void grouping_reset_des(RRDR *r) {
85 - struct grouping_des *g = (struct grouping_des *)r->time_grouping.data;
86 - g->level = 0.0;
87 - g->trend = 0.0;
88 - g->count = 0;
89 -
90 - // fprintf(stderr, "\nDES: ");
91 -
92 -}
93 -
94 -void grouping_free_des(RRDR *r) {
95 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
96 - r->time_grouping.data = NULL;
97 -}
98 -
99 -void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) {
100 - struct grouping_des *g = (struct grouping_des *)r->time_grouping.data;
101 -
102 - if(likely(g->count > 0)) {
103 - // we have at least a number so far
104 -
105 - if(unlikely(g->count == 1)) {
106 - // the second value we got
107 - g->trend = value - g->trend;
108 - g->level = value;
109 - }
110 -
111 - // for the values, except the first
112 - NETDATA_DOUBLE last_level = g->level;
113 - g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend));
114 - g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend);
115 - }
116 - else {
117 - // the first value we got
118 - g->level = g->trend = value;
119 - }
120 -
121 - g->count++;
122 -
123 - //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend);
124 -}
125 -
126 -NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
127 - struct grouping_des *g = (struct grouping_des *)r->time_grouping.data;
128 -
129 - if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
130 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
131 - return 0.0;
132 - }
133 -
134 - //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level);
135 -
136 - return g->level;
137 -}
web/api/queries/des/des.h
+127 -6
@@ -6,12 +6,133 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_init_des(void);
9 +struct tg_des {
10 + NETDATA_DOUBLE alpha;
11 + NETDATA_DOUBLE alpha_other;
12 + NETDATA_DOUBLE beta;
13 + NETDATA_DOUBLE beta_other;
14
11 -void grouping_create_des(RRDR *r, const char *options __maybe_unused);
12 -void grouping_reset_des(RRDR *r);
13 -void grouping_free_des(RRDR *r);
14 -void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
15 -NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
15 + NETDATA_DOUBLE level;
16 + NETDATA_DOUBLE trend;
17 +
18 + size_t count;
19 +};
20 +
21 +static size_t tg_des_max_window_size = 15;
22 +
23 +static inline void tg_des_init(void) {
24 + long long ret = config_get_number(CONFIG_SECTION_WEB, "des max tg_des_window", (long long)tg_des_max_window_size);
25 + if(ret <= 1) {
26 + config_set_number(CONFIG_SECTION_WEB, "des max tg_des_window", (long long)tg_des_max_window_size);
27 + }
28 + else {
29 + tg_des_max_window_size = (size_t) ret;
30 + }
31 +}
32 +
33 +static inline NETDATA_DOUBLE tg_des_window(RRDR *r, struct tg_des *g) {
34 + (void)g;
35 +
36 + NETDATA_DOUBLE points;
37 + if(r->view.group == 1) {
38 + // provide a running DES
39 + points = (NETDATA_DOUBLE)r->time_grouping.points_wanted;
40 + }
41 + else {
42 + // provide a SES with flush points
43 + points = (NETDATA_DOUBLE)r->view.group;
44 + }
45 +
46 + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
47 + // A commonly used value for alpha is 2 / (N + 1)
48 + return (points > (NETDATA_DOUBLE)tg_des_max_window_size) ? (NETDATA_DOUBLE)tg_des_max_window_size : points;
49 +}
50 +
51 +static inline void tg_des_set_alpha(RRDR *r, struct tg_des *g) {
52 + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
53 + // A commonly used value for alpha is 2 / (N + 1)
54 +
55 + g->alpha = 2.0 / (tg_des_window(r, g) + 1.0);
56 + g->alpha_other = 1.0 - g->alpha;
57 +
58 + //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha);
59 +}
60 +
61 +static inline void tg_des_set_beta(RRDR *r, struct tg_des *g) {
62 + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
63 + // A commonly used value for alpha is 2 / (N + 1)
64 +
65 + g->beta = 2.0 / (tg_des_window(r, g) + 1.0);
66 + g->beta_other = 1.0 - g->beta;
67 +
68 + //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta);
69 +}
70 +
71 +static inline void tg_des_create(RRDR *r, const char *options __maybe_unused) {
72 + struct tg_des *g = (struct tg_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct tg_des));
73 + tg_des_set_alpha(r, g);
74 + tg_des_set_beta(r, g);
75 + g->level = 0.0;
76 + g->trend = 0.0;
77 + g->count = 0;
78 + r->time_grouping.data = g;
79 +}
80 +
81 +// resets when switches dimensions
82 +// so, clear everything to restart
83 +static inline void tg_des_reset(RRDR *r) {
84 + struct tg_des *g = (struct tg_des *)r->time_grouping.data;
85 + g->level = 0.0;
86 + g->trend = 0.0;
87 + g->count = 0;
88 +
89 + // fprintf(stderr, "\nDES: ");
90 +
91 +}
92 +
93 +static inline void tg_des_free(RRDR *r) {
94 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
95 + r->time_grouping.data = NULL;
96 +}
97 +
98 +static inline void tg_des_add(RRDR *r, NETDATA_DOUBLE value) {
99 + struct tg_des *g = (struct tg_des *)r->time_grouping.data;
100 +
101 + if(likely(g->count > 0)) {
102 + // we have at least a number so far
103 +
104 + if(unlikely(g->count == 1)) {
105 + // the second value we got
106 + g->trend = value - g->trend;
107 + g->level = value;
108 + }
109 +
110 + // for the values, except the first
111 + NETDATA_DOUBLE last_level = g->level;
112 + g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend));
113 + g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend);
114 + }
115 + else {
116 + // the first value we got
117 + g->level = g->trend = value;
118 + }
119 +
120 + g->count++;
121 +
122 + //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend);
123 +}
124 +
125 +static inline NETDATA_DOUBLE tg_des_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
126 + struct tg_des *g = (struct tg_des *)r->time_grouping.data;
127 +
128 + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
129 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
130 + return 0.0;
131 + }
132 +
133 + //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level);
134 +
135 + return g->level;
136 +}
137
138 #endif //NETDATA_API_QUERIES_DES_H
web/api/queries/incremental_sum/incremental_sum.c
-59
@@ -5,62 +5,3 @@
5 // ----------------------------------------------------------------------------
6 // incremental sum
7
8 -struct grouping_incremental_sum {
9 - NETDATA_DOUBLE first;
10 - NETDATA_DOUBLE last;
11 - size_t count;
12 -};
13 -
14 -void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) {
15 - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum));
16 -}
17 -
18 -// resets when switches dimensions
19 -// so, clear everything to restart
20 -void grouping_reset_incremental_sum(RRDR *r) {
21 - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->time_grouping.data;
22 - g->first = 0;
23 - g->last = 0;
24 - g->count = 0;
25 -}
26 -
27 -void grouping_free_incremental_sum(RRDR *r) {
28 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
29 - r->time_grouping.data = NULL;
30 -}
31 -
32 -void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) {
33 - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->time_grouping.data;
34 -
35 - if(unlikely(!g->count)) {
36 - g->first = value;
37 - g->count++;
38 - }
39 - else {
40 - g->last = value;
41 - g->count++;
42 - }
43 -}
44 -
45 -NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
46 - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->time_grouping.data;
47 -
48 - NETDATA_DOUBLE value;
49 -
50 - if(unlikely(!g->count)) {
51 - value = 0.0;
52 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
53 - }
54 - else if(unlikely(g->count == 1)) {
55 - value = 0.0;
56 - }
57 - else {
58 - value = g->last - g->first;
59 - }
60 -
61 - g->first = 0.0;
62 - g->last = 0.0;
63 - g->count = 0;
64 -
65 - return value;
66 -}
web/api/queries/incremental_sum/incremental_sum.h
+59 -5
@@ -6,10 +6,64 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_incremental_sum(RRDR *r);
11 -void grouping_free_incremental_sum(RRDR *r);
12 -void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_incremental_sum {
10 + NETDATA_DOUBLE first;
11 + NETDATA_DOUBLE last;
12 + size_t count;
13 +};
14 +
15 +static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) {
16 + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_incremental_sum));
17 +}
18 +
19 +// resets when switches dimensions
20 +// so, clear everything to restart
21 +static inline void tg_incremental_sum_reset(RRDR *r) {
22 + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data;
23 + g->first = 0;
24 + g->last = 0;
25 + g->count = 0;
26 +}
27 +
28 +static inline void tg_incremental_sum_free(RRDR *r) {
29 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
30 + r->time_grouping.data = NULL;
31 +}
32 +
33 +static inline void tg_incremental_sum_add(RRDR *r, NETDATA_DOUBLE value) {
34 + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data;
35 +
36 + if(unlikely(!g->count)) {
37 + g->first = value;
38 + g->count++;
39 + }
40 + else {
41 + g->last = value;
42 + g->count++;
43 + }
44 +}
45 +
46 +static inline NETDATA_DOUBLE tg_incremental_sum_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
47 + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data;
48 +
49 + NETDATA_DOUBLE value;
50 +
51 + if(unlikely(!g->count)) {
52 + value = 0.0;
53 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
54 + }
55 + else if(unlikely(g->count == 1)) {
56 + value = 0.0;
57 + }
58 + else {
59 + value = g->last - g->first;
60 + }
61 +
62 + g->first = 0.0;
63 + g->last = 0.0;
64 + g->count = 0;
65 +
66 + return value;
67 +}
68
69 #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H
web/api/queries/max/max.c
-50
@@ -5,53 +5,3 @@
5 // ----------------------------------------------------------------------------
6 // max
7
8 -struct grouping_max {
9 - NETDATA_DOUBLE max;
10 - size_t count;
11 -};
12 -
13 -void grouping_create_max(RRDR *r, const char *options __maybe_unused) {
14 - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max));
15 -}
16 -
17 -// resets when switches dimensions
18 -// so, clear everything to restart
19 -void grouping_reset_max(RRDR *r) {
20 - struct grouping_max *g = (struct grouping_max *)r->time_grouping.data;
21 - g->max = 0;
22 - g->count = 0;
23 -}
24 -
25 -void grouping_free_max(RRDR *r) {
26 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
27 - r->time_grouping.data = NULL;
28 -}
29 -
30 -void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) {
31 - struct grouping_max *g = (struct grouping_max *)r->time_grouping.data;
32 -
33 - if(!g->count || fabsndd(value) > fabsndd(g->max)) {
34 - g->max = value;
35 - g->count++;
36 - }
37 -}
38 -
39 -NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
40 - struct grouping_max *g = (struct grouping_max *)r->time_grouping.data;
41 -
42 - NETDATA_DOUBLE value;
43 -
44 - if(unlikely(!g->count)) {
45 - value = 0.0;
46 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
47 - }
48 - else {
49 - value = g->max;
50 - }
51 -
52 - g->max = 0.0;
53 - g->count = 0;
54 -
55 - return value;
56 -}
57 -
web/api/queries/max/max.h
+49 -5
@@ -6,10 +6,54 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_max(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_max(RRDR *r);
11 -void grouping_free_max(RRDR *r);
12 -void grouping_add_max(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_max {
10 + NETDATA_DOUBLE max;
11 + size_t count;
12 +};
13 +
14 +static inline void tg_max_create(RRDR *r, const char *options __maybe_unused) {
15 + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_max));
16 +}
17 +
18 +// resets when switches dimensions
19 +// so, clear everything to restart
20 +static inline void tg_max_reset(RRDR *r) {
21 + struct tg_max *g = (struct tg_max *)r->time_grouping.data;
22 + g->max = 0;
23 + g->count = 0;
24 +}
25 +
26 +static inline void tg_max_free(RRDR *r) {
27 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
28 + r->time_grouping.data = NULL;
29 +}
30 +
31 +static inline void tg_max_add(RRDR *r, NETDATA_DOUBLE value) {
32 + struct tg_max *g = (struct tg_max *)r->time_grouping.data;
33 +
34 + if(!g->count || fabsndd(value) > fabsndd(g->max)) {
35 + g->max = value;
36 + g->count++;
37 + }
38 +}
39 +
40 +static inline NETDATA_DOUBLE tg_max_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
41 + struct tg_max *g = (struct tg_max *)r->time_grouping.data;
42 +
43 + NETDATA_DOUBLE value;
44 +
45 + if(unlikely(!g->count)) {
46 + value = 0.0;
47 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
48 + }
49 + else {
50 + value = g->max;
51 + }
52 +
53 + g->max = 0.0;
54 + g->count = 0;
55 +
56 + return value;
57 +}
58
59 #endif //NETDATA_API_QUERY_MAX_H
web/api/queries/median/median.c
-134
@@ -4,137 +4,3 @@
4
5 // ----------------------------------------------------------------------------
6 // median
7 -
8 -struct grouping_median {
9 - size_t series_size;
10 - size_t next_pos;
11 - NETDATA_DOUBLE percent;
12 -
13 - NETDATA_DOUBLE *series;
14 -};
15 -
16 -void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17 - long entries = r->view.group;
18 - if(entries < 10) entries = 10;
19 -
20 - struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median));
21 - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22 - g->series_size = (size_t)entries;
23 -
24 - g->percent = def;
25 - if(options && *options) {
26 - g->percent = str2ndd(options, NULL);
27 - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
28 - if(g->percent < 0.0) g->percent = 0.0;
29 - if(g->percent > 50.0) g->percent = 50.0;
30 - }
31 -
32 - g->percent = g->percent / 100.0;
33 - r->time_grouping.data = g;
34 -}
35 -
36 -void grouping_create_median(RRDR *r, const char *options) {
37 - grouping_create_median_internal(r, options, 0.0);
38 -}
39 -void grouping_create_trimmed_median1(RRDR *r, const char *options) {
40 - grouping_create_median_internal(r, options, 1.0);
41 -}
42 -void grouping_create_trimmed_median2(RRDR *r, const char *options) {
43 - grouping_create_median_internal(r, options, 2.0);
44 -}
45 -void grouping_create_trimmed_median3(RRDR *r, const char *options) {
46 - grouping_create_median_internal(r, options, 3.0);
47 -}
48 -void grouping_create_trimmed_median5(RRDR *r, const char *options) {
49 - grouping_create_median_internal(r, options, 5.0);
50 -}
51 -void grouping_create_trimmed_median10(RRDR *r, const char *options) {
52 - grouping_create_median_internal(r, options, 10.0);
53 -}
54 -void grouping_create_trimmed_median15(RRDR *r, const char *options) {
55 - grouping_create_median_internal(r, options, 15.0);
56 -}
57 -void grouping_create_trimmed_median20(RRDR *r, const char *options) {
58 - grouping_create_median_internal(r, options, 20.0);
59 -}
60 -void grouping_create_trimmed_median25(RRDR *r, const char *options) {
61 - grouping_create_median_internal(r, options, 25.0);
62 -}
63 -
64 -// resets when switches dimensions
65 -// so, clear everything to restart
66 -void grouping_reset_median(RRDR *r) {
67 - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data;
68 - g->next_pos = 0;
69 -}
70 -
71 -void grouping_free_median(RRDR *r) {
72 - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data;
73 - if(g) onewayalloc_freez(r->internal.owa, g->series);
74 -
75 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
76 - r->time_grouping.data = NULL;
77 -}
78 -
79 -void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) {
80 - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data;
81 -
82 - if(unlikely(g->next_pos >= g->series_size)) {
83 - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
84 - g->series_size *= 2;
85 - }
86 -
87 - g->series[g->next_pos++] = value;
88 -}
89 -
90 -NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
91 - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data;
92 -
93 - size_t available_slots = g->next_pos;
94 - NETDATA_DOUBLE value;
95 -
96 - if(unlikely(!available_slots)) {
97 - value = 0.0;
98 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
99 - }
100 - else if(available_slots == 1) {
101 - value = g->series[0];
102 - }
103 - else {
104 - sort_series(g->series, available_slots);
105 -
106 - size_t start_slot = 0;
107 - size_t end_slot = available_slots - 1;
108 -
109 - if(g->percent > 0.0) {
110 - NETDATA_DOUBLE min = g->series[0];
111 - NETDATA_DOUBLE max = g->series[available_slots - 1];
112 - NETDATA_DOUBLE delta = (max - min) * g->percent;
113 -
114 - NETDATA_DOUBLE wanted_min = min + delta;
115 - NETDATA_DOUBLE wanted_max = max - delta;
116 -
117 - for (start_slot = 0; start_slot < available_slots; start_slot++)
118 - if (g->series[start_slot] >= wanted_min) break;
119 -
120 - for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--)
121 - if (g->series[end_slot] <= wanted_max) break;
122 - }
123 -
124 - if(start_slot == end_slot)
125 - value = g->series[start_slot];
126 - else
127 - value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1);
128 - }
129 -
130 - if(unlikely(!netdata_double_isnumber(value))) {
131 - value = 0.0;
132 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
133 - }
134 -
135 - //log_series_to_stderr(g->series, g->next_pos, value, "median");
136 -
137 - g->next_pos = 0;
138 -
139 - return value;
140 -}
web/api/queries/median/median.h
+133 -13
@@ -6,18 +6,138 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_median(RRDR *r, const char *options);
10 -void grouping_create_trimmed_median1(RRDR *r, const char *options);
11 -void grouping_create_trimmed_median2(RRDR *r, const char *options);
12 -void grouping_create_trimmed_median3(RRDR *r, const char *options);
13 -void grouping_create_trimmed_median5(RRDR *r, const char *options);
14 -void grouping_create_trimmed_median10(RRDR *r, const char *options);
15 -void grouping_create_trimmed_median15(RRDR *r, const char *options);
16 -void grouping_create_trimmed_median20(RRDR *r, const char *options);
17 -void grouping_create_trimmed_median25(RRDR *r, const char *options);
18 -void grouping_reset_median(RRDR *r);
19 -void grouping_free_median(RRDR *r);
20 -void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);
21 -NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_median {
10 + size_t series_size;
11 + size_t next_pos;
12 + NETDATA_DOUBLE percent;
13 +
14 + NETDATA_DOUBLE *series;
15 +};
16 +
17 +static inline void tg_median_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
18 + long entries = r->view.group;
19 + if(entries < 10) entries = 10;
20 +
21 + struct tg_median *g = (struct tg_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_median));
22 + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
23 + g->series_size = (size_t)entries;
24 +
25 + g->percent = def;
26 + if(options && *options) {
27 + g->percent = str2ndd(options, NULL);
28 + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
29 + if(g->percent < 0.0) g->percent = 0.0;
30 + if(g->percent > 50.0) g->percent = 50.0;
31 + }
32 +
33 + g->percent = g->percent / 100.0;
34 + r->time_grouping.data = g;
35 +}
36 +
37 +static inline void tg_median_create(RRDR *r, const char *options) {
38 + tg_median_create_internal(r, options, 0.0);
39 +}
40 +static inline void tg_median_create_trimmed_1(RRDR *r, const char *options) {
41 + tg_median_create_internal(r, options, 1.0);
42 +}
43 +static inline void tg_median_create_trimmed_2(RRDR *r, const char *options) {
44 + tg_median_create_internal(r, options, 2.0);
45 +}
46 +static inline void tg_median_create_trimmed_3(RRDR *r, const char *options) {
47 + tg_median_create_internal(r, options, 3.0);
48 +}
49 +static inline void tg_median_create_trimmed_5(RRDR *r, const char *options) {
50 + tg_median_create_internal(r, options, 5.0);
51 +}
52 +static inline void tg_median_create_trimmed_10(RRDR *r, const char *options) {
53 + tg_median_create_internal(r, options, 10.0);
54 +}
55 +static inline void tg_median_create_trimmed_15(RRDR *r, const char *options) {
56 + tg_median_create_internal(r, options, 15.0);
57 +}
58 +static inline void tg_median_create_trimmed_20(RRDR *r, const char *options) {
59 + tg_median_create_internal(r, options, 20.0);
60 +}
61 +static inline void tg_median_create_trimmed_25(RRDR *r, const char *options) {
62 + tg_median_create_internal(r, options, 25.0);
63 +}
64 +
65 +// resets when switches dimensions
66 +// so, clear everything to restart
67 +static inline void tg_median_reset(RRDR *r) {
68 + struct tg_median *g = (struct tg_median *)r->time_grouping.data;
69 + g->next_pos = 0;
70 +}
71 +
72 +static inline void tg_median_free(RRDR *r) {
73 + struct tg_median *g = (struct tg_median *)r->time_grouping.data;
74 + if(g) onewayalloc_freez(r->internal.owa, g->series);
75 +
76 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
77 + r->time_grouping.data = NULL;
78 +}
79 +
80 +static inline void tg_median_add(RRDR *r, NETDATA_DOUBLE value) {
81 + struct tg_median *g = (struct tg_median *)r->time_grouping.data;
82 +
83 + if(unlikely(g->next_pos >= g->series_size)) {
84 + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
85 + g->series_size *= 2;
86 + }
87 +
88 + g->series[g->next_pos++] = value;
89 +}
90 +
91 +static inline NETDATA_DOUBLE tg_median_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
92 + struct tg_median *g = (struct tg_median *)r->time_grouping.data;
93 +
94 + size_t available_slots = g->next_pos;
95 + NETDATA_DOUBLE value;
96 +
97 + if(unlikely(!available_slots)) {
98 + value = 0.0;
99 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
100 + }
101 + else if(available_slots == 1) {
102 + value = g->series[0];
103 + }
104 + else {
105 + sort_series(g->series, available_slots);
106 +
107 + size_t start_slot = 0;
108 + size_t end_slot = available_slots - 1;
109 +
110 + if(g->percent > 0.0) {
111 + NETDATA_DOUBLE min = g->series[0];
112 + NETDATA_DOUBLE max = g->series[available_slots - 1];
113 + NETDATA_DOUBLE delta = (max - min) * g->percent;
114 +
115 + NETDATA_DOUBLE wanted_min = min + delta;
116 + NETDATA_DOUBLE wanted_max = max - delta;
117 +
118 + for (start_slot = 0; start_slot < available_slots; start_slot++)
119 + if (g->series[start_slot] >= wanted_min) break;
120 +
121 + for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--)
122 + if (g->series[end_slot] <= wanted_max) break;
123 + }
124 +
125 + if(start_slot == end_slot)
126 + value = g->series[start_slot];
127 + else
128 + value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1);
129 + }
130 +
131 + if(unlikely(!netdata_double_isnumber(value))) {
132 + value = 0.0;
133 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
134 + }
135 +
136 + //log_series_to_stderr(g->series, g->next_pos, value, "median");
137 +
138 + g->next_pos = 0;
139 +
140 + return value;
141 +}
142
143 #endif //NETDATA_API_QUERIES_MEDIAN_H
web/api/queries/min/min.c
-50
@@ -5,53 +5,3 @@
5 // ----------------------------------------------------------------------------
6 // min
7
8 -struct grouping_min {
9 - NETDATA_DOUBLE min;
10 - size_t count;
11 -};
12 -
13 -void grouping_create_min(RRDR *r, const char *options __maybe_unused) {
14 - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min));
15 -}
16 -
17 -// resets when switches dimensions
18 -// so, clear everything to restart
19 -void grouping_reset_min(RRDR *r) {
20 - struct grouping_min *g = (struct grouping_min *)r->time_grouping.data;
21 - g->min = 0;
22 - g->count = 0;
23 -}
24 -
25 -void grouping_free_min(RRDR *r) {
26 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
27 - r->time_grouping.data = NULL;
28 -}
29 -
30 -void grouping_add_min(RRDR *r, NETDATA_DOUBLE value) {
31 - struct grouping_min *g = (struct grouping_min *)r->time_grouping.data;
32 -
33 - if(!g->count || fabsndd(value) < fabsndd(g->min)) {
34 - g->min = value;
35 - g->count++;
36 - }
37 -}
38 -
39 -NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
40 - struct grouping_min *g = (struct grouping_min *)r->time_grouping.data;
41 -
42 - NETDATA_DOUBLE value;
43 -
44 - if(unlikely(!g->count)) {
45 - value = 0.0;
46 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
47 - }
48 - else {
49 - value = g->min;
50 - }
51 -
52 - g->min = 0.0;
53 - g->count = 0;
54 -
55 - return value;
56 -}
57 -
web/api/queries/min/min.h
+49 -5
@@ -6,10 +6,54 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_min(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_min(RRDR *r);
11 -void grouping_free_min(RRDR *r);
12 -void grouping_add_min(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_min {
10 + NETDATA_DOUBLE min;
11 + size_t count;
12 +};
13 +
14 +static inline void tg_min_create(RRDR *r, const char *options __maybe_unused) {
15 + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_min));
16 +}
17 +
18 +// resets when switches dimensions
19 +// so, clear everything to restart
20 +static inline void tg_min_reset(RRDR *r) {
21 + struct tg_min *g = (struct tg_min *)r->time_grouping.data;
22 + g->min = 0;
23 + g->count = 0;
24 +}
25 +
26 +static inline void tg_min_free(RRDR *r) {
27 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
28 + r->time_grouping.data = NULL;
29 +}
30 +
31 +static inline void tg_min_add(RRDR *r, NETDATA_DOUBLE value) {
32 + struct tg_min *g = (struct tg_min *)r->time_grouping.data;
33 +
34 + if(!g->count || fabsndd(value) < fabsndd(g->min)) {
35 + g->min = value;
36 + g->count++;
37 + }
38 +}
39 +
40 +static inline NETDATA_DOUBLE tg_min_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
41 + struct tg_min *g = (struct tg_min *)r->time_grouping.data;
42 +
43 + NETDATA_DOUBLE value;
44 +
45 + if(unlikely(!g->count)) {
46 + value = 0.0;
47 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
48 + }
49 + else {
50 + value = g->min;
51 + }
52 +
53 + g->min = 0.0;
54 + g->count = 0;
55 +
56 + return value;
57 +}
58
59 #endif //NETDATA_API_QUERY_MIN_H
web/api/queries/percentile/percentile.c
-163
@@ -4,166 +4,3 @@
4
5 // ----------------------------------------------------------------------------
6 // median
7 -
8 -struct grouping_percentile {
9 - size_t series_size;
10 - size_t next_pos;
11 - NETDATA_DOUBLE percent;
12 -
13 - NETDATA_DOUBLE *series;
14 -};
15 -
16 -static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17 - long entries = r->view.group;
18 - if(entries < 10) entries = 10;
19 -
20 - struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile));
21 - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22 - g->series_size = (size_t)entries;
23 -
24 - g->percent = def;
25 - if(options && *options) {
26 - g->percent = str2ndd(options, NULL);
27 - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
28 - if(g->percent < 0.0) g->percent = 0.0;
29 - if(g->percent > 100.0) g->percent = 100.0;
30 - }
31 -
32 - g->percent = g->percent / 100.0;
33 - r->time_grouping.data = g;
34 -}
35 -
36 -void grouping_create_percentile25(RRDR *r, const char *options) {
37 - grouping_create_percentile_internal(r, options, 25.0);
38 -}
39 -void grouping_create_percentile50(RRDR *r, const char *options) {
40 - grouping_create_percentile_internal(r, options, 50.0);
41 -}
42 -void grouping_create_percentile75(RRDR *r, const char *options) {
43 - grouping_create_percentile_internal(r, options, 75.0);
44 -}
45 -void grouping_create_percentile80(RRDR *r, const char *options) {
46 - grouping_create_percentile_internal(r, options, 80.0);
47 -}
48 -void grouping_create_percentile90(RRDR *r, const char *options) {
49 - grouping_create_percentile_internal(r, options, 90.0);
50 -}
51 -void grouping_create_percentile95(RRDR *r, const char *options) {
52 - grouping_create_percentile_internal(r, options, 95.0);
53 -}
54 -void grouping_create_percentile97(RRDR *r, const char *options) {
55 - grouping_create_percentile_internal(r, options, 97.0);
56 -}
57 -void grouping_create_percentile98(RRDR *r, const char *options) {
58 - grouping_create_percentile_internal(r, options, 98.0);
59 -}
60 -void grouping_create_percentile99(RRDR *r, const char *options) {
61 - grouping_create_percentile_internal(r, options, 99.0);
62 -}
63 -
64 -// resets when switches dimensions
65 -// so, clear everything to restart
66 -void grouping_reset_percentile(RRDR *r) {
67 - struct grouping_percentile *g = (struct grouping_percentile *)r->time_grouping.data;
68 - g->next_pos = 0;
69 -}
70 -
71 -void grouping_free_percentile(RRDR *r) {
72 - struct grouping_percentile *g = (struct grouping_percentile *)r->time_grouping.data;
73 - if(g) onewayalloc_freez(r->internal.owa, g->series);
74 -
75 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
76 - r->time_grouping.data = NULL;
77 -}
78 -
79 -void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) {
80 - struct grouping_percentile *g = (struct grouping_percentile *)r->time_grouping.data;
81 -
82 - if(unlikely(g->next_pos >= g->series_size)) {
83 - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
84 - g->series_size *= 2;
85 - }
86 -
87 - g->series[g->next_pos++] = value;
88 -}
89 -
90 -NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
91 - struct grouping_percentile *g = (struct grouping_percentile *)r->time_grouping.data;
92 -
93 - NETDATA_DOUBLE value;
94 - size_t available_slots = g->next_pos;
95 -
96 - if(unlikely(!available_slots)) {
97 - value = 0.0;
98 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
99 - }
100 - else if(available_slots == 1) {
101 - value = g->series[0];
102 - }
103 - else {
104 - sort_series(g->series, available_slots);
105 -
106 - NETDATA_DOUBLE min = g->series[0];
107 - NETDATA_DOUBLE max = g->series[available_slots - 1];
108 -
109 - if (min != max) {
110 - size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
111 - if(!slots_to_use) slots_to_use = 1;
112 -
113 - NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
114 - NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
115 -
116 - NETDATA_DOUBLE percent_interpolation_slot = 0.0;
117 - NETDATA_DOUBLE percent_last_slot = 0.0;
118 - if(percent_delta > 0.0) {
119 - NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
120 - NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
121 -
122 - percent_interpolation_slot = percent_delta / percent_1slot;
123 - percent_last_slot = 1 - percent_interpolation_slot;
124 - }
125 -
126 - int start_slot, stop_slot, step, last_slot, interpolation_slot;
127 - if(min >= 0.0 && max >= 0.0) {
128 - start_slot = 0;
129 - stop_slot = start_slot + (int)slots_to_use;
130 - last_slot = stop_slot - 1;
131 - interpolation_slot = stop_slot;
132 - step = 1;
133 - }
134 - else {
135 - start_slot = (int)available_slots - 1;
136 - stop_slot = start_slot - (int)slots_to_use;
137 - last_slot = stop_slot + 1;
138 - interpolation_slot = stop_slot;
139 - step = -1;
140 - }
141 -
142 - value = 0.0;
143 - for(int slot = start_slot; slot != stop_slot ; slot += step)
144 - value += g->series[slot];
145 -
146 - size_t counted = slots_to_use;
147 - if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
148 - value += g->series[interpolation_slot] * percent_interpolation_slot;
149 - value += g->series[last_slot] * percent_last_slot;
150 - counted++;
151 - }
152 -
153 - value = value / (NETDATA_DOUBLE)counted;
154 - }
155 - else
156 - value = min;
157 - }
158 -
159 - if(unlikely(!netdata_double_isnumber(value))) {
160 - value = 0.0;
161 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
162 - }
163 -
164 - //log_series_to_stderr(g->series, g->next_pos, value, "percentile");
165 -
166 - g->next_pos = 0;
167 -
168 - return value;
169 -}
web/api/queries/percentile/percentile.h
+162 -13
@@ -6,18 +6,167 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_percentile25(RRDR *r, const char *options);
10 -void grouping_create_percentile50(RRDR *r, const char *options);
11 -void grouping_create_percentile75(RRDR *r, const char *options);
12 -void grouping_create_percentile80(RRDR *r, const char *options);
13 -void grouping_create_percentile90(RRDR *r, const char *options);
14 -void grouping_create_percentile95(RRDR *r, const char *options);
15 -void grouping_create_percentile97(RRDR *r, const char *options);
16 -void grouping_create_percentile98(RRDR *r, const char *options);
17 -void grouping_create_percentile99(RRDR *r, const char *options );
18 -void grouping_reset_percentile(RRDR *r);
19 -void grouping_free_percentile(RRDR *r);
20 -void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
21 -NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_percentile {
10 + size_t series_size;
11 + size_t next_pos;
12 + NETDATA_DOUBLE percent;
13 +
14 + NETDATA_DOUBLE *series;
15 +};
16 +
17 +static inline void tg_percentile_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
18 + long entries = r->view.group;
19 + if(entries < 10) entries = 10;
20 +
21 + struct tg_percentile *g = (struct tg_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_percentile));
22 + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
23 + g->series_size = (size_t)entries;
24 +
25 + g->percent = def;
26 + if(options && *options) {
27 + g->percent = str2ndd(options, NULL);
28 + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
29 + if(g->percent < 0.0) g->percent = 0.0;
30 + if(g->percent > 100.0) g->percent = 100.0;
31 + }
32 +
33 + g->percent = g->percent / 100.0;
34 + r->time_grouping.data = g;
35 +}
36 +
37 +static inline void tg_percentile_create_25(RRDR *r, const char *options) {
38 + tg_percentile_create_internal(r, options, 25.0);
39 +}
40 +static inline void tg_percentile_create_50(RRDR *r, const char *options) {
41 + tg_percentile_create_internal(r, options, 50.0);
42 +}
43 +static inline void tg_percentile_create_75(RRDR *r, const char *options) {
44 + tg_percentile_create_internal(r, options, 75.0);
45 +}
46 +static inline void tg_percentile_create_80(RRDR *r, const char *options) {
47 + tg_percentile_create_internal(r, options, 80.0);
48 +}
49 +static inline void tg_percentile_create_90(RRDR *r, const char *options) {
50 + tg_percentile_create_internal(r, options, 90.0);
51 +}
52 +static inline void tg_percentile_create_95(RRDR *r, const char *options) {
53 + tg_percentile_create_internal(r, options, 95.0);
54 +}
55 +static inline void tg_percentile_create_97(RRDR *r, const char *options) {
56 + tg_percentile_create_internal(r, options, 97.0);
57 +}
58 +static inline void tg_percentile_create_98(RRDR *r, const char *options) {
59 + tg_percentile_create_internal(r, options, 98.0);
60 +}
61 +static inline void tg_percentile_create_99(RRDR *r, const char *options) {
62 + tg_percentile_create_internal(r, options, 99.0);
63 +}
64 +
65 +// resets when switches dimensions
66 +// so, clear everything to restart
67 +static inline void tg_percentile_reset(RRDR *r) {
68 + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data;
69 + g->next_pos = 0;
70 +}
71 +
72 +static inline void tg_percentile_free(RRDR *r) {
73 + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data;
74 + if(g) onewayalloc_freez(r->internal.owa, g->series);
75 +
76 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
77 + r->time_grouping.data = NULL;
78 +}
79 +
80 +static inline void tg_percentile_add(RRDR *r, NETDATA_DOUBLE value) {
81 + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data;
82 +
83 + if(unlikely(g->next_pos >= g->series_size)) {
84 + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
85 + g->series_size *= 2;
86 + }
87 +
88 + g->series[g->next_pos++] = value;
89 +}
90 +
91 +static inline NETDATA_DOUBLE tg_percentile_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
92 + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data;
93 +
94 + NETDATA_DOUBLE value;
95 + size_t available_slots = g->next_pos;
96 +
97 + if(unlikely(!available_slots)) {
98 + value = 0.0;
99 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
100 + }
101 + else if(available_slots == 1) {
102 + value = g->series[0];
103 + }
104 + else {
105 + sort_series(g->series, available_slots);
106 +
107 + NETDATA_DOUBLE min = g->series[0];
108 + NETDATA_DOUBLE max = g->series[available_slots - 1];
109 +
110 + if (min != max) {
111 + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
112 + if(!slots_to_use) slots_to_use = 1;
113 +
114 + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
115 + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
116 +
117 + NETDATA_DOUBLE percent_interpolation_slot = 0.0;
118 + NETDATA_DOUBLE percent_last_slot = 0.0;
119 + if(percent_delta > 0.0) {
120 + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
121 + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
122 +
123 + percent_interpolation_slot = percent_delta / percent_1slot;
124 + percent_last_slot = 1 - percent_interpolation_slot;
125 + }
126 +
127 + int start_slot, stop_slot, step, last_slot, interpolation_slot;
128 + if(min >= 0.0 && max >= 0.0) {
129 + start_slot = 0;
130 + stop_slot = start_slot + (int)slots_to_use;
131 + last_slot = stop_slot - 1;
132 + interpolation_slot = stop_slot;
133 + step = 1;
134 + }
135 + else {
136 + start_slot = (int)available_slots - 1;
137 + stop_slot = start_slot - (int)slots_to_use;
138 + last_slot = stop_slot + 1;
139 + interpolation_slot = stop_slot;
140 + step = -1;
141 + }
142 +
143 + value = 0.0;
144 + for(int slot = start_slot; slot != stop_slot ; slot += step)
145 + value += g->series[slot];
146 +
147 + size_t counted = slots_to_use;
148 + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
149 + value += g->series[interpolation_slot] * percent_interpolation_slot;
150 + value += g->series[last_slot] * percent_last_slot;
151 + counted++;
152 + }
153 +
154 + value = value / (NETDATA_DOUBLE)counted;
155 + }
156 + else
157 + value = min;
158 + }
159 +
160 + if(unlikely(!netdata_double_isnumber(value))) {
161 + value = 0.0;
162 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
163 + }
164 +
165 + //log_series_to_stderr(g->series, g->next_pos, value, "percentile");
166 +
167 + g->next_pos = 0;
168 +
169 + return value;
170 +}
171
172 #endif //NETDATA_API_QUERIES_PERCENTILE_H
web/api/queries/query.c
+1010 -678
@@ -25,6 +25,7 @@ static struct {
25 const char *name;
26 uint32_t hash;
27 RRDR_TIME_GROUPING value;
28 + RRDR_TIME_GROUPING add_flush;
29
30 // One time initialization for the module.
31 // This is called once, when netdata starts.
@@ -59,408 +60,445 @@ static struct {
60 {.name = "average",
61 .hash = 0,
62 .value = RRDR_GROUPING_AVERAGE,
63 + .add_flush = RRDR_GROUPING_AVERAGE,
64 .init = NULL,
63 - .create= grouping_create_average,
64 - .reset = grouping_reset_average,
65 - .free = grouping_free_average,
66 - .add = grouping_add_average,
67 - .flush = grouping_flush_average,
65 + .create= tg_average_create,
66 + .reset = tg_average_reset,
67 + .free = tg_average_free,
68 + .add = tg_average_add,
69 + .flush = tg_average_flush,
70 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
71 },
72 {.name = "avg", // alias on 'average'
73 .hash = 0,
74 .value = RRDR_GROUPING_AVERAGE,
75 + .add_flush = RRDR_GROUPING_AVERAGE,
76 .init = NULL,
74 - .create= grouping_create_average,
75 - .reset = grouping_reset_average,
76 - .free = grouping_free_average,
77 - .add = grouping_add_average,
78 - .flush = grouping_flush_average,
77 + .create= tg_average_create,
78 + .reset = tg_average_reset,
79 + .free = tg_average_free,
80 + .add = tg_average_add,
81 + .flush = tg_average_flush,
82 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
83 },
84 {.name = "mean", // alias on 'average'
85 .hash = 0,
86 .value = RRDR_GROUPING_AVERAGE,
87 + .add_flush = RRDR_GROUPING_AVERAGE,
88 .init = NULL,
85 - .create= grouping_create_average,
86 - .reset = grouping_reset_average,
87 - .free = grouping_free_average,
88 - .add = grouping_add_average,
89 - .flush = grouping_flush_average,
89 + .create= tg_average_create,
90 + .reset = tg_average_reset,
91 + .free = tg_average_free,
92 + .add = tg_average_add,
93 + .flush = tg_average_flush,
94 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
95 },
96 {.name = "trimmed-mean1",
97 .hash = 0,
98 .value = RRDR_GROUPING_TRIMMED_MEAN1,
99 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
100 .init = NULL,
96 - .create= grouping_create_trimmed_mean1,
97 - .reset = grouping_reset_trimmed_mean,
98 - .free = grouping_free_trimmed_mean,
99 - .add = grouping_add_trimmed_mean,
100 - .flush = grouping_flush_trimmed_mean,
101 + .create= tg_trimmed_mean_create_1,
102 + .reset = tg_trimmed_mean_reset,
103 + .free = tg_trimmed_mean_free,
104 + .add = tg_trimmed_mean_add,
105 + .flush = tg_trimmed_mean_flush,
106 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
107 },
108 {.name = "trimmed-mean2",
109 .hash = 0,
110 .value = RRDR_GROUPING_TRIMMED_MEAN2,
111 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
112 .init = NULL,
107 - .create= grouping_create_trimmed_mean2,
108 - .reset = grouping_reset_trimmed_mean,
109 - .free = grouping_free_trimmed_mean,
110 - .add = grouping_add_trimmed_mean,
111 - .flush = grouping_flush_trimmed_mean,
113 + .create= tg_trimmed_mean_create_2,
114 + .reset = tg_trimmed_mean_reset,
115 + .free = tg_trimmed_mean_free,
116 + .add = tg_trimmed_mean_add,
117 + .flush = tg_trimmed_mean_flush,
118 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
119 },
120 {.name = "trimmed-mean3",
121 .hash = 0,
122 .value = RRDR_GROUPING_TRIMMED_MEAN3,
123 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
124 .init = NULL,
118 - .create= grouping_create_trimmed_mean3,
119 - .reset = grouping_reset_trimmed_mean,
120 - .free = grouping_free_trimmed_mean,
121 - .add = grouping_add_trimmed_mean,
122 - .flush = grouping_flush_trimmed_mean,
125 + .create= tg_trimmed_mean_create_3,
126 + .reset = tg_trimmed_mean_reset,
127 + .free = tg_trimmed_mean_free,
128 + .add = tg_trimmed_mean_add,
129 + .flush = tg_trimmed_mean_flush,
130 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
131 },
132 {.name = "trimmed-mean5",
133 .hash = 0,
127 - .value = RRDR_GROUPING_TRIMMED_MEAN5,
134 + .value = RRDR_GROUPING_TRIMMED_MEAN,
135 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
136 .init = NULL,
129 - .create= grouping_create_trimmed_mean5,
130 - .reset = grouping_reset_trimmed_mean,
131 - .free = grouping_free_trimmed_mean,
132 - .add = grouping_add_trimmed_mean,
133 - .flush = grouping_flush_trimmed_mean,
137 + .create= tg_trimmed_mean_create_5,
138 + .reset = tg_trimmed_mean_reset,
139 + .free = tg_trimmed_mean_free,
140 + .add = tg_trimmed_mean_add,
141 + .flush = tg_trimmed_mean_flush,
142 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
143 },
144 {.name = "trimmed-mean10",
145 .hash = 0,
146 .value = RRDR_GROUPING_TRIMMED_MEAN10,
147 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
148 .init = NULL,
140 - .create= grouping_create_trimmed_mean10,
141 - .reset = grouping_reset_trimmed_mean,
142 - .free = grouping_free_trimmed_mean,
143 - .add = grouping_add_trimmed_mean,
144 - .flush = grouping_flush_trimmed_mean,
149 + .create= tg_trimmed_mean_create_10,
150 + .reset = tg_trimmed_mean_reset,
151 + .free = tg_trimmed_mean_free,
152 + .add = tg_trimmed_mean_add,
153 + .flush = tg_trimmed_mean_flush,
154 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
155 },
156 {.name = "trimmed-mean15",
157 .hash = 0,
158 .value = RRDR_GROUPING_TRIMMED_MEAN15,
159 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
160 .init = NULL,
151 - .create= grouping_create_trimmed_mean15,
152 - .reset = grouping_reset_trimmed_mean,
153 - .free = grouping_free_trimmed_mean,
154 - .add = grouping_add_trimmed_mean,
155 - .flush = grouping_flush_trimmed_mean,
161 + .create= tg_trimmed_mean_create_15,
162 + .reset = tg_trimmed_mean_reset,
163 + .free = tg_trimmed_mean_free,
164 + .add = tg_trimmed_mean_add,
165 + .flush = tg_trimmed_mean_flush,
166 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
167 },
168 {.name = "trimmed-mean20",
169 .hash = 0,
170 .value = RRDR_GROUPING_TRIMMED_MEAN20,
171 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
172 .init = NULL,
162 - .create= grouping_create_trimmed_mean20,
163 - .reset = grouping_reset_trimmed_mean,
164 - .free = grouping_free_trimmed_mean,
165 - .add = grouping_add_trimmed_mean,
166 - .flush = grouping_flush_trimmed_mean,
173 + .create= tg_trimmed_mean_create_20,
174 + .reset = tg_trimmed_mean_reset,
175 + .free = tg_trimmed_mean_free,
176 + .add = tg_trimmed_mean_add,
177 + .flush = tg_trimmed_mean_flush,
178 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
179 },
180 {.name = "trimmed-mean25",
181 .hash = 0,
182 .value = RRDR_GROUPING_TRIMMED_MEAN25,
183 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
184 .init = NULL,
173 - .create= grouping_create_trimmed_mean25,
174 - .reset = grouping_reset_trimmed_mean,
175 - .free = grouping_free_trimmed_mean,
176 - .add = grouping_add_trimmed_mean,
177 - .flush = grouping_flush_trimmed_mean,
185 + .create= tg_trimmed_mean_create_25,
186 + .reset = tg_trimmed_mean_reset,
187 + .free = tg_trimmed_mean_free,
188 + .add = tg_trimmed_mean_add,
189 + .flush = tg_trimmed_mean_flush,
190 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
191 },
192 {.name = "trimmed-mean",
193 .hash = 0,
182 - .value = RRDR_GROUPING_TRIMMED_MEAN5,
194 + .value = RRDR_GROUPING_TRIMMED_MEAN,
195 + .add_flush = RRDR_GROUPING_TRIMMED_MEAN,
196 .init = NULL,
184 - .create= grouping_create_trimmed_mean5,
185 - .reset = grouping_reset_trimmed_mean,
186 - .free = grouping_free_trimmed_mean,
187 - .add = grouping_add_trimmed_mean,
188 - .flush = grouping_flush_trimmed_mean,
197 + .create= tg_trimmed_mean_create_5,
198 + .reset = tg_trimmed_mean_reset,
199 + .free = tg_trimmed_mean_free,
200 + .add = tg_trimmed_mean_add,
201 + .flush = tg_trimmed_mean_flush,
202 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
203 },
204 {.name = "incremental_sum",
205 .hash = 0,
206 .value = RRDR_GROUPING_INCREMENTAL_SUM,
207 + .add_flush = RRDR_GROUPING_INCREMENTAL_SUM,
208 .init = NULL,
195 - .create= grouping_create_incremental_sum,
196 - .reset = grouping_reset_incremental_sum,
197 - .free = grouping_free_incremental_sum,
198 - .add = grouping_add_incremental_sum,
199 - .flush = grouping_flush_incremental_sum,
209 + .create= tg_incremental_sum_create,
210 + .reset = tg_incremental_sum_reset,
211 + .free = tg_incremental_sum_free,
212 + .add = tg_incremental_sum_add,
213 + .flush = tg_incremental_sum_flush,
214 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
215 },
216 {.name = "incremental-sum",
217 .hash = 0,
218 .value = RRDR_GROUPING_INCREMENTAL_SUM,
219 + .add_flush = RRDR_GROUPING_INCREMENTAL_SUM,
220 .init = NULL,
206 - .create= grouping_create_incremental_sum,
207 - .reset = grouping_reset_incremental_sum,
208 - .free = grouping_free_incremental_sum,
209 - .add = grouping_add_incremental_sum,
210 - .flush = grouping_flush_incremental_sum,
221 + .create= tg_incremental_sum_create,
222 + .reset = tg_incremental_sum_reset,
223 + .free = tg_incremental_sum_free,
224 + .add = tg_incremental_sum_add,
225 + .flush = tg_incremental_sum_flush,
226 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
227 },
228 {.name = "median",
229 .hash = 0,
230 .value = RRDR_GROUPING_MEDIAN,
231 + .add_flush = RRDR_GROUPING_MEDIAN,
232 .init = NULL,
217 - .create= grouping_create_median,
218 - .reset = grouping_reset_median,
219 - .free = grouping_free_median,
220 - .add = grouping_add_median,
221 - .flush = grouping_flush_median,
233 + .create= tg_median_create,
234 + .reset = tg_median_reset,
235 + .free = tg_median_free,
236 + .add = tg_median_add,
237 + .flush = tg_median_flush,
238 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
239 },
240 {.name = "trimmed-median1",
241 .hash = 0,
242 .value = RRDR_GROUPING_TRIMMED_MEDIAN1,
243 + .add_flush = RRDR_GROUPING_MEDIAN,
244 .init = NULL,
228 - .create= grouping_create_trimmed_median1,
229 - .reset = grouping_reset_median,
230 - .free = grouping_free_median,
231 - .add = grouping_add_median,
232 - .flush = grouping_flush_median,
245 + .create= tg_median_create_trimmed_1,
246 + .reset = tg_median_reset,
247 + .free = tg_median_free,
248 + .add = tg_median_add,
249 + .flush = tg_median_flush,
250 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
251 },
252 {.name = "trimmed-median2",
253 .hash = 0,
254 .value = RRDR_GROUPING_TRIMMED_MEDIAN2,
255 + .add_flush = RRDR_GROUPING_MEDIAN,
256 .init = NULL,
239 - .create= grouping_create_trimmed_median2,
240 - .reset = grouping_reset_median,
241 - .free = grouping_free_median,
242 - .add = grouping_add_median,
243 - .flush = grouping_flush_median,
257 + .create= tg_median_create_trimmed_2,
258 + .reset = tg_median_reset,
259 + .free = tg_median_free,
260 + .add = tg_median_add,
261 + .flush = tg_median_flush,
262 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
263 },
264 {.name = "trimmed-median3",
265 .hash = 0,
266 .value = RRDR_GROUPING_TRIMMED_MEDIAN3,
267 + .add_flush = RRDR_GROUPING_MEDIAN,
268 .init = NULL,
250 - .create= grouping_create_trimmed_median3,
251 - .reset = grouping_reset_median,
252 - .free = grouping_free_median,
253 - .add = grouping_add_median,
254 - .flush = grouping_flush_median,
269 + .create= tg_median_create_trimmed_3,
270 + .reset = tg_median_reset,
271 + .free = tg_median_free,
272 + .add = tg_median_add,
273 + .flush = tg_median_flush,
274 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
275 },
276 {.name = "trimmed-median5",
277 .hash = 0,
278 .value = RRDR_GROUPING_TRIMMED_MEDIAN5,
279 + .add_flush = RRDR_GROUPING_MEDIAN,
280 .init = NULL,
261 - .create= grouping_create_trimmed_median5,
262 - .reset = grouping_reset_median,
263 - .free = grouping_free_median,
264 - .add = grouping_add_median,
265 - .flush = grouping_flush_median,
281 + .create= tg_median_create_trimmed_5,
282 + .reset = tg_median_reset,
283 + .free = tg_median_free,
284 + .add = tg_median_add,
285 + .flush = tg_median_flush,
286 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
287 },
288 {.name = "trimmed-median10",
289 .hash = 0,
290 .value = RRDR_GROUPING_TRIMMED_MEDIAN10,
291 + .add_flush = RRDR_GROUPING_MEDIAN,
292 .init = NULL,
272 - .create= grouping_create_trimmed_median10,
273 - .reset = grouping_reset_median,
274 - .free = grouping_free_median,
275 - .add = grouping_add_median,
276 - .flush = grouping_flush_median,
293 + .create= tg_median_create_trimmed_10,
294 + .reset = tg_median_reset,
295 + .free = tg_median_free,
296 + .add = tg_median_add,
297 + .flush = tg_median_flush,
298 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
299 },
300 {.name = "trimmed-median15",
301 .hash = 0,
302 .value = RRDR_GROUPING_TRIMMED_MEDIAN15,
303 + .add_flush = RRDR_GROUPING_MEDIAN,
304 .init = NULL,
283 - .create= grouping_create_trimmed_median15,
284 - .reset = grouping_reset_median,
285 - .free = grouping_free_median,
286 - .add = grouping_add_median,
287 - .flush = grouping_flush_median,
305 + .create= tg_median_create_trimmed_15,
306 + .reset = tg_median_reset,
307 + .free = tg_median_free,
308 + .add = tg_median_add,
309 + .flush = tg_median_flush,
310 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
311 },
312 {.name = "trimmed-median20",
313 .hash = 0,
314 .value = RRDR_GROUPING_TRIMMED_MEDIAN20,
315 + .add_flush = RRDR_GROUPING_MEDIAN,
316 .init = NULL,
294 - .create= grouping_create_trimmed_median20,
295 - .reset = grouping_reset_median,
296 - .free = grouping_free_median,
297 - .add = grouping_add_median,
298 - .flush = grouping_flush_median,
317 + .create= tg_median_create_trimmed_20,
318 + .reset = tg_median_reset,
319 + .free = tg_median_free,
320 + .add = tg_median_add,
321 + .flush = tg_median_flush,
322 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
323 },
324 {.name = "trimmed-median25",
325 .hash = 0,
326 .value = RRDR_GROUPING_TRIMMED_MEDIAN25,
327 + .add_flush = RRDR_GROUPING_MEDIAN,
328 .init = NULL,
305 - .create= grouping_create_trimmed_median25,
306 - .reset = grouping_reset_median,
307 - .free = grouping_free_median,
308 - .add = grouping_add_median,
309 - .flush = grouping_flush_median,
329 + .create= tg_median_create_trimmed_25,
330 + .reset = tg_median_reset,
331 + .free = tg_median_free,
332 + .add = tg_median_add,
333 + .flush = tg_median_flush,
334 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
335 },
336 {.name = "trimmed-median",
337 .hash = 0,
338 .value = RRDR_GROUPING_TRIMMED_MEDIAN5,
339 + .add_flush = RRDR_GROUPING_MEDIAN,
340 .init = NULL,
316 - .create= grouping_create_trimmed_median5,
317 - .reset = grouping_reset_median,
318 - .free = grouping_free_median,
319 - .add = grouping_add_median,
320 - .flush = grouping_flush_median,
341 + .create= tg_median_create_trimmed_5,
342 + .reset = tg_median_reset,
343 + .free = tg_median_free,
344 + .add = tg_median_add,
345 + .flush = tg_median_flush,
346 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
347 },
348 {.name = "percentile25",
349 .hash = 0,
350 .value = RRDR_GROUPING_PERCENTILE25,
351 + .add_flush = RRDR_GROUPING_PERCENTILE,
352 .init = NULL,
327 - .create= grouping_create_percentile25,
328 - .reset = grouping_reset_percentile,
329 - .free = grouping_free_percentile,
330 - .add = grouping_add_percentile,
331 - .flush = grouping_flush_percentile,
353 + .create= tg_percentile_create_25,
354 + .reset = tg_percentile_reset,
355 + .free = tg_percentile_free,
356 + .add = tg_percentile_add,
357 + .flush = tg_percentile_flush,
358 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
359 },
360 {.name = "percentile50",
361 .hash = 0,
362 .value = RRDR_GROUPING_PERCENTILE50,
363 + .add_flush = RRDR_GROUPING_PERCENTILE,
364 .init = NULL,
338 - .create= grouping_create_percentile50,
339 - .reset = grouping_reset_percentile,
340 - .free = grouping_free_percentile,
341 - .add = grouping_add_percentile,
342 - .flush = grouping_flush_percentile,
365 + .create= tg_percentile_create_50,
366 + .reset = tg_percentile_reset,
367 + .free = tg_percentile_free,
368 + .add = tg_percentile_add,
369 + .flush = tg_percentile_flush,
370 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
371 },
372 {.name = "percentile75",
373 .hash = 0,
374 .value = RRDR_GROUPING_PERCENTILE75,
375 + .add_flush = RRDR_GROUPING_PERCENTILE,
376 .init = NULL,
349 - .create= grouping_create_percentile75,
350 - .reset = grouping_reset_percentile,
351 - .free = grouping_free_percentile,
352 - .add = grouping_add_percentile,
353 - .flush = grouping_flush_percentile,
377 + .create= tg_percentile_create_75,
378 + .reset = tg_percentile_reset,
379 + .free = tg_percentile_free,
380 + .add = tg_percentile_add,
381 + .flush = tg_percentile_flush,
382 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
383 },
384 {.name = "percentile80",
385 .hash = 0,
386 .value = RRDR_GROUPING_PERCENTILE80,
387 + .add_flush = RRDR_GROUPING_PERCENTILE,
388 .init = NULL,
360 - .create= grouping_create_percentile80,
361 - .reset = grouping_reset_percentile,
362 - .free = grouping_free_percentile,
363 - .add = grouping_add_percentile,
364 - .flush = grouping_flush_percentile,
389 + .create= tg_percentile_create_80,
390 + .reset = tg_percentile_reset,
391 + .free = tg_percentile_free,
392 + .add = tg_percentile_add,
393 + .flush = tg_percentile_flush,
394 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
395 },
396 {.name = "percentile90",
397 .hash = 0,
398 .value = RRDR_GROUPING_PERCENTILE90,
399 + .add_flush = RRDR_GROUPING_PERCENTILE,
400 .init = NULL,
371 - .create= grouping_create_percentile90,
372 - .reset = grouping_reset_percentile,
373 - .free = grouping_free_percentile,
374 - .add = grouping_add_percentile,
375 - .flush = grouping_flush_percentile,
401 + .create= tg_percentile_create_90,
402 + .reset = tg_percentile_reset,
403 + .free = tg_percentile_free,
404 + .add = tg_percentile_add,
405 + .flush = tg_percentile_flush,
406 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
407 },
408 {.name = "percentile95",
409 .hash = 0,
380 - .value = RRDR_GROUPING_PERCENTILE95,
410 + .value = RRDR_GROUPING_PERCENTILE,
411 + .add_flush = RRDR_GROUPING_PERCENTILE,
412 .init = NULL,
382 - .create= grouping_create_percentile95,
383 - .reset = grouping_reset_percentile,
384 - .free = grouping_free_percentile,
385 - .add = grouping_add_percentile,
386 - .flush = grouping_flush_percentile,
413 + .create= tg_percentile_create_95,
414 + .reset = tg_percentile_reset,
415 + .free = tg_percentile_free,
416 + .add = tg_percentile_add,
417 + .flush = tg_percentile_flush,
418 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
419 },
420 {.name = "percentile97",
421 .hash = 0,
422 .value = RRDR_GROUPING_PERCENTILE97,
423 + .add_flush = RRDR_GROUPING_PERCENTILE,
424 .init = NULL,
393 - .create= grouping_create_percentile97,
394 - .reset = grouping_reset_percentile,
395 - .free = grouping_free_percentile,
396 - .add = grouping_add_percentile,
397 - .flush = grouping_flush_percentile,
425 + .create= tg_percentile_create_97,
426 + .reset = tg_percentile_reset,
427 + .free = tg_percentile_free,
428 + .add = tg_percentile_add,
429 + .flush = tg_percentile_flush,
430 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
431 },
432 {.name = "percentile98",
433 .hash = 0,
434 .value = RRDR_GROUPING_PERCENTILE98,
435 + .add_flush = RRDR_GROUPING_PERCENTILE,
436 .init = NULL,
404 - .create= grouping_create_percentile98,
405 - .reset = grouping_reset_percentile,
406 - .free = grouping_free_percentile,
407 - .add = grouping_add_percentile,
408 - .flush = grouping_flush_percentile,
437 + .create= tg_percentile_create_98,
438 + .reset = tg_percentile_reset,
439 + .free = tg_percentile_free,
440 + .add = tg_percentile_add,
441 + .flush = tg_percentile_flush,
442 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
443 },
444 {.name = "percentile99",
445 .hash = 0,
446 .value = RRDR_GROUPING_PERCENTILE99,
447 + .add_flush = RRDR_GROUPING_PERCENTILE,
448 .init = NULL,
415 - .create= grouping_create_percentile99,
416 - .reset = grouping_reset_percentile,
417 - .free = grouping_free_percentile,
418 - .add = grouping_add_percentile,
419 - .flush = grouping_flush_percentile,
449 + .create= tg_percentile_create_99,
450 + .reset = tg_percentile_reset,
451 + .free = tg_percentile_free,
452 + .add = tg_percentile_add,
453 + .flush = tg_percentile_flush,
454 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
455 },
456 {.name = "percentile",
457 .hash = 0,
424 - .value = RRDR_GROUPING_PERCENTILE95,
458 + .value = RRDR_GROUPING_PERCENTILE,
459 + .add_flush = RRDR_GROUPING_PERCENTILE,
460 .init = NULL,
426 - .create= grouping_create_percentile95,
427 - .reset = grouping_reset_percentile,
428 - .free = grouping_free_percentile,
429 - .add = grouping_add_percentile,
430 - .flush = grouping_flush_percentile,
461 + .create= tg_percentile_create_95,
462 + .reset = tg_percentile_reset,
463 + .free = tg_percentile_free,
464 + .add = tg_percentile_add,
465 + .flush = tg_percentile_flush,
466 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
467 },
468 {.name = "min",
469 .hash = 0,
470 .value = RRDR_GROUPING_MIN,
471 + .add_flush = RRDR_GROUPING_MIN,
472 .init = NULL,
437 - .create= grouping_create_min,
438 - .reset = grouping_reset_min,
439 - .free = grouping_free_min,
440 - .add = grouping_add_min,
441 - .flush = grouping_flush_min,
473 + .create= tg_min_create,
474 + .reset = tg_min_reset,
475 + .free = tg_min_free,
476 + .add = tg_min_add,
477 + .flush = tg_min_flush,
478 .tier_query_fetch = TIER_QUERY_FETCH_MIN
479 },
480 {.name = "max",
481 .hash = 0,
482 .value = RRDR_GROUPING_MAX,
483 + .add_flush = RRDR_GROUPING_MAX,
484 .init = NULL,
448 - .create= grouping_create_max,
449 - .reset = grouping_reset_max,
450 - .free = grouping_free_max,
451 - .add = grouping_add_max,
452 - .flush = grouping_flush_max,
485 + .create= tg_max_create,
486 + .reset = tg_max_reset,
487 + .free = tg_max_free,
488 + .add = tg_max_add,
489 + .flush = tg_max_flush,
490 .tier_query_fetch = TIER_QUERY_FETCH_MAX
491 },
492 {.name = "sum",
493 .hash = 0,
494 .value = RRDR_GROUPING_SUM,
495 + .add_flush = RRDR_GROUPING_SUM,
496 .init = NULL,
459 - .create= grouping_create_sum,
460 - .reset = grouping_reset_sum,
461 - .free = grouping_free_sum,
462 - .add = grouping_add_sum,
463 - .flush = grouping_flush_sum,
497 + .create= tg_sum_create,
498 + .reset = tg_sum_reset,
499 + .free = tg_sum_free,
500 + .add = tg_sum_add,
501 + .flush = tg_sum_flush,
502 .tier_query_fetch = TIER_QUERY_FETCH_SUM
503 },
504
@@ -468,97 +506,75 @@ static struct {
506 {.name = "stddev",
507 .hash = 0,
508 .value = RRDR_GROUPING_STDDEV,
509 + .add_flush = RRDR_GROUPING_STDDEV,
510 .init = NULL,
472 - .create= grouping_create_stddev,
473 - .reset = grouping_reset_stddev,
474 - .free = grouping_free_stddev,
475 - .add = grouping_add_stddev,
476 - .flush = grouping_flush_stddev,
511 + .create= tg_stddev_create,
512 + .reset = tg_stddev_reset,
513 + .free = tg_stddev_free,
514 + .add = tg_stddev_add,
515 + .flush = tg_stddev_flush,
516 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
517 },
518 {.name = "cv", // coefficient variation is calculated by stddev
519 .hash = 0,
520 .value = RRDR_GROUPING_CV,
521 + .add_flush = RRDR_GROUPING_CV,
522 .init = NULL,
483 - .create= grouping_create_stddev, // not an error, stddev calculates this too
484 - .reset = grouping_reset_stddev, // not an error, stddev calculates this too
485 - .free = grouping_free_stddev, // not an error, stddev calculates this too
486 - .add = grouping_add_stddev, // not an error, stddev calculates this too
487 - .flush = grouping_flush_coefficient_of_variation,
523 + .create= tg_stddev_create, // not an error, stddev calculates this too
524 + .reset = tg_stddev_reset, // not an error, stddev calculates this too
525 + .free = tg_stddev_free, // not an error, stddev calculates this too
526 + .add = tg_stddev_add, // not an error, stddev calculates this too
527 + .flush = tg_stddev_coefficient_of_variation_flush,
528 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
529 },
530 {.name = "rsd", // alias of 'cv'
531 .hash = 0,
532 .value = RRDR_GROUPING_CV,
533 + .add_flush = RRDR_GROUPING_CV,
534 .init = NULL,
494 - .create= grouping_create_stddev, // not an error, stddev calculates this too
495 - .reset = grouping_reset_stddev, // not an error, stddev calculates this too
496 - .free = grouping_free_stddev, // not an error, stddev calculates this too
497 - .add = grouping_add_stddev, // not an error, stddev calculates this too
498 - .flush = grouping_flush_coefficient_of_variation,
535 + .create= tg_stddev_create, // not an error, stddev calculates this too
536 + .reset = tg_stddev_reset, // not an error, stddev calculates this too
537 + .free = tg_stddev_free, // not an error, stddev calculates this too
538 + .add = tg_stddev_add, // not an error, stddev calculates this too
539 + .flush = tg_stddev_coefficient_of_variation_flush,
540 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
541 },
542
502 - /*
503 - {.name = "mean", // same as average, no need to define it again
504 - .hash = 0,
505 - .value = RRDR_GROUPING_MEAN,
506 - .setup = NULL,
507 - .create= grouping_create_stddev,
508 - .reset = grouping_reset_stddev,
509 - .free = grouping_free_stddev,
510 - .add = grouping_add_stddev,
511 - .flush = grouping_flush_mean,
512 - .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
513 - },
514 - */
515 -
516 - /*
517 - {.name = "variance", // meaningless to offer
518 - .hash = 0,
519 - .value = RRDR_GROUPING_VARIANCE,
520 - .setup = NULL,
521 - .create= grouping_create_stddev,
522 - .reset = grouping_reset_stddev,
523 - .free = grouping_free_stddev,
524 - .add = grouping_add_stddev,
525 - .flush = grouping_flush_variance,
526 - .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
527 - },
528 - */
529 -
543 // single exponential smoothing
544 {.name = "ses",
545 .hash = 0,
546 .value = RRDR_GROUPING_SES,
534 - .init = grouping_init_ses,
535 - .create= grouping_create_ses,
536 - .reset = grouping_reset_ses,
537 - .free = grouping_free_ses,
538 - .add = grouping_add_ses,
539 - .flush = grouping_flush_ses,
547 + .add_flush = RRDR_GROUPING_SES,
548 + .init = tg_ses_init,
549 + .create= tg_ses_create,
550 + .reset = tg_ses_reset,
551 + .free = tg_ses_free,
552 + .add = tg_ses_add,
553 + .flush = tg_ses_flush,
554 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
555 },
556 {.name = "ema", // alias for 'ses'
557 .hash = 0,
558 .value = RRDR_GROUPING_SES,
559 + .add_flush = RRDR_GROUPING_SES,
560 .init = NULL,
546 - .create= grouping_create_ses,
547 - .reset = grouping_reset_ses,
548 - .free = grouping_free_ses,
549 - .add = grouping_add_ses,
550 - .flush = grouping_flush_ses,
561 + .create= tg_ses_create,
562 + .reset = tg_ses_reset,
563 + .free = tg_ses_free,
564 + .add = tg_ses_add,
565 + .flush = tg_ses_flush,
566 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
567 },
568 {.name = "ewma", // alias for ses
569 .hash = 0,
570 .value = RRDR_GROUPING_SES,
571 + .add_flush = RRDR_GROUPING_SES,
572 .init = NULL,
557 - .create= grouping_create_ses,
558 - .reset = grouping_reset_ses,
559 - .free = grouping_free_ses,
560 - .add = grouping_add_ses,
561 - .flush = grouping_flush_ses,
573 + .create= tg_ses_create,
574 + .reset = tg_ses_reset,
575 + .free = tg_ses_free,
576 + .add = tg_ses_add,
577 + .flush = tg_ses_flush,
578 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
579 },
580
@@ -566,24 +582,26 @@ static struct {
582 {.name = "des",
583 .hash = 0,
584 .value = RRDR_GROUPING_DES,
569 - .init = grouping_init_des,
570 - .create= grouping_create_des,
571 - .reset = grouping_reset_des,
572 - .free = grouping_free_des,
573 - .add = grouping_add_des,
574 - .flush = grouping_flush_des,
585 + .add_flush = RRDR_GROUPING_DES,
586 + .init = tg_des_init,
587 + .create= tg_des_create,
588 + .reset = tg_des_reset,
589 + .free = tg_des_free,
590 + .add = tg_des_add,
591 + .flush = tg_des_flush,
592 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
593 },
594
595 {.name = "countif",
596 .hash = 0,
597 .value = RRDR_GROUPING_COUNTIF,
598 + .add_flush = RRDR_GROUPING_COUNTIF,
599 .init = NULL,
582 - .create= grouping_create_countif,
583 - .reset = grouping_reset_countif,
584 - .free = grouping_free_countif,
585 - .add = grouping_add_countif,
586 - .flush = grouping_flush_countif,
600 + .create= tg_countif_create,
601 + .reset = tg_countif_reset,
602 + .free = tg_countif_free,
603 + .add = tg_countif_add,
604 + .flush = tg_countif_flush,
605 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
606 },
607
@@ -591,12 +609,13 @@ static struct {
609 {.name = NULL,
610 .hash = 0,
611 .value = RRDR_GROUPING_UNDEFINED,
612 + .add_flush = RRDR_GROUPING_AVERAGE,
613 .init = NULL,
595 - .create= grouping_create_average,
596 - .reset = grouping_reset_average,
597 - .free = grouping_free_average,
598 - .add = grouping_add_average,
599 - .flush = grouping_flush_average,
614 + .create= tg_average_create,
615 + .reset = tg_average_reset,
616 + .free = tg_average_free,
617 + .add = tg_average_add,
618 + .flush = tg_average_flush,
619 .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
620 }
621 };
@@ -655,18 +674,123 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method)
674 r->time_grouping.add = api_v1_data_groups[i].add;
675 r->time_grouping.flush = api_v1_data_groups[i].flush;
676 r->time_grouping.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch;
677 + r->time_grouping.add_flush = api_v1_data_groups[i].add_flush;
678 found = 1;
679 }
680 }
681 if(!found) {
682 errno = 0;
683 internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method);
664 - r->time_grouping.create = grouping_create_average;
665 - r->time_grouping.reset = grouping_reset_average;
666 - r->time_grouping.free = grouping_free_average;
667 - r->time_grouping.add = grouping_add_average;
668 - r->time_grouping.flush = grouping_flush_average;
684 + r->time_grouping.create = tg_average_create;
685 + r->time_grouping.reset = tg_average_reset;
686 + r->time_grouping.free = tg_average_free;
687 + r->time_grouping.add = tg_average_add;
688 + r->time_grouping.flush = tg_average_flush;
689 r->time_grouping.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE;
690 + r->time_grouping.add_flush = RRDR_GROUPING_AVERAGE;
691 + }
692 +}
693 +
694 +static inline void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush) {
695 + switch(add_flush) {
696 + case RRDR_GROUPING_AVERAGE:
697 + tg_average_add(r, value);
698 + break;
699 +
700 + case RRDR_GROUPING_MAX:
701 + tg_max_add(r, value);
702 + break;
703 +
704 + case RRDR_GROUPING_MIN:
705 + tg_min_add(r, value);
706 + break;
707 +
708 + case RRDR_GROUPING_MEDIAN:
709 + tg_median_add(r, value);
710 + break;
711 +
712 + case RRDR_GROUPING_STDDEV:
713 + case RRDR_GROUPING_CV:
714 + tg_stddev_add(r, value);
715 + break;
716 +
717 + case RRDR_GROUPING_SUM:
718 + tg_sum_add(r, value);
719 + break;
720 +
721 + case RRDR_GROUPING_COUNTIF:
722 + tg_countif_add(r, value);
723 + break;
724 +
725 + case RRDR_GROUPING_TRIMMED_MEAN:
726 + tg_trimmed_mean_add(r, value);
727 + break;
728 +
729 + case RRDR_GROUPING_PERCENTILE:
730 + tg_percentile_add(r, value);
731 + break;
732 +
733 + case RRDR_GROUPING_SES:
734 + tg_ses_add(r, value);
735 + break;
736 +
737 + case RRDR_GROUPING_DES:
738 + tg_des_add(r, value);
739 + break;
740 +
741 + case RRDR_GROUPING_INCREMENTAL_SUM:
742 + tg_incremental_sum_add(r, value);
743 + break;
744 +
745 + default:
746 + r->time_grouping.add(r, value);
747 + break;
748 + }
749 +}
750 +
751 +static inline NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush) {
752 + switch(add_flush) {
753 + case RRDR_GROUPING_AVERAGE:
754 + return tg_average_flush(r, rrdr_value_options_ptr);
755 +
756 + case RRDR_GROUPING_MAX:
757 + return tg_max_flush(r, rrdr_value_options_ptr);
758 +
759 + case RRDR_GROUPING_MIN:
760 + return tg_min_flush(r, rrdr_value_options_ptr);
761 +
762 + case RRDR_GROUPING_MEDIAN:
763 + return tg_median_flush(r, rrdr_value_options_ptr);
764 +
765 + case RRDR_GROUPING_STDDEV:
766 + return tg_stddev_flush(r, rrdr_value_options_ptr);
767 +
768 + case RRDR_GROUPING_CV:
769 + return tg_stddev_coefficient_of_variation_flush(r, rrdr_value_options_ptr);
770 +
771 + case RRDR_GROUPING_SUM:
772 + return tg_sum_flush(r, rrdr_value_options_ptr);
773 +
774 + case RRDR_GROUPING_COUNTIF:
775 + return tg_countif_flush(r, rrdr_value_options_ptr);
776 +
777 + case RRDR_GROUPING_TRIMMED_MEAN:
778 + return tg_trimmed_mean_flush(r, rrdr_value_options_ptr);
779 +
780 + case RRDR_GROUPING_PERCENTILE:
781 + return tg_percentile_flush(r, rrdr_value_options_ptr);
782 +
783 + case RRDR_GROUPING_SES:
784 + return tg_ses_flush(r, rrdr_value_options_ptr);
785 +
786 + case RRDR_GROUPING_DES:
787 + return tg_des_flush(r, rrdr_value_options_ptr);
788 +
789 + case RRDR_GROUPING_INCREMENTAL_SUM:
790 + return tg_incremental_sum_flush(r, rrdr_value_options_ptr);
791 +
792 + default:
793 + return r->time_grouping.flush(r, rrdr_value_options_ptr);
794 }
795 }
796
@@ -686,6 +810,9 @@ RRDR_GROUP_BY group_by_parse(char *s) {
810 if (strcmp(key, "instance") == 0)
811 group_by |= RRDR_GROUP_BY_INSTANCE;
812
813 + if (strcmp(key, "percentage-of-instance") == 0)
814 + group_by |= RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE;
815 +
816 if (strcmp(key, "label") == 0)
817 group_by |= RRDR_GROUP_BY_LABEL;
818
@@ -699,30 +826,45 @@ RRDR_GROUP_BY group_by_parse(char *s) {
826 group_by |= RRDR_GROUP_BY_UNITS;
827 }
828
829 + if((group_by & RRDR_GROUP_BY_SELECTED) && (group_by & ~RRDR_GROUP_BY_SELECTED)) {
830 + internal_error(true, "group-by given by query has 'selected' together with more groupings");
831 + group_by = RRDR_GROUP_BY_SELECTED; // remove all other groupings
832 + }
833 +
834 + if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
835 + group_by = RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE; // remove all other groupings
836 +
837 return group_by;
838 }
839
840 void buffer_json_group_by_to_array(BUFFER *wb, RRDR_GROUP_BY group_by) {
706 - if(group_by & RRDR_GROUP_BY_SELECTED)
707 - buffer_json_add_array_item_string(wb, "selected");
841 + if(group_by == RRDR_GROUP_BY_NONE)
842 + buffer_json_add_array_item_string(wb, "none");
843 + else {
844 + if (group_by & RRDR_GROUP_BY_DIMENSION)
845 + buffer_json_add_array_item_string(wb, "dimension");
846 +
847 + if (group_by & RRDR_GROUP_BY_INSTANCE)
848 + buffer_json_add_array_item_string(wb, "instance");
849
709 - if(group_by & RRDR_GROUP_BY_DIMENSION)
710 - buffer_json_add_array_item_string(wb, "dimension");
850 + if (group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
851 + buffer_json_add_array_item_string(wb, "percentage-of-instance");
852
712 - if(group_by & RRDR_GROUP_BY_INSTANCE)
713 - buffer_json_add_array_item_string(wb, "instance");
853 + if (group_by & RRDR_GROUP_BY_LABEL)
854 + buffer_json_add_array_item_string(wb, "label");
855
715 - if(group_by & RRDR_GROUP_BY_LABEL)
716 - buffer_json_add_array_item_string(wb, "label");
856 + if (group_by & RRDR_GROUP_BY_NODE)
857 + buffer_json_add_array_item_string(wb, "node");
858
718 - if(group_by & RRDR_GROUP_BY_NODE)
719 - buffer_json_add_array_item_string(wb, "node");
859 + if (group_by & RRDR_GROUP_BY_CONTEXT)
860 + buffer_json_add_array_item_string(wb, "context");
861
721 - if(group_by & RRDR_GROUP_BY_CONTEXT)
722 - buffer_json_add_array_item_string(wb, "context");
862 + if (group_by & RRDR_GROUP_BY_UNITS)
863 + buffer_json_add_array_item_string(wb, "units");
864
724 - if(group_by & RRDR_GROUP_BY_UNITS)
725 - buffer_json_add_array_item_string(wb, "units");
865 + if (group_by & RRDR_GROUP_BY_SELECTED)
866 + buffer_json_add_array_item_string(wb, "selected");
867 + }
868 }
869
870 RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s) {
@@ -1024,13 +1166,8 @@ typedef struct query_engine_ops {
1166 size_t tier;
1167 struct query_metric_tier *tier_ptr;
1168 struct storage_engine_query_handle *handle;
1027 - STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
1028 - int (*is_finished)(struct storage_engine_query_handle *handle);
1029 - void (*finalize)(struct storage_engine_query_handle *handle);
1169
1170 // aggregating points over time
1032 - void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
1033 - NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
1171 size_t group_points_non_zero;
1172 size_t group_points_added;
1173 STORAGE_POINT group_point; // aggregates min, max, sum, count, anomaly count for each group point
@@ -1045,9 +1182,6 @@ typedef struct query_engine_ops {
1182 time_t expanded_after;
1183 time_t expanded_before;
1184 struct storage_engine_query_handle handle;
1048 - STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
1049 - int (*is_finished)(struct storage_engine_query_handle *handle);
1050 - void (*finalize)(struct storage_engine_query_handle *handle);
1185 bool initialized;
1186 bool finalized;
1187 } plans[QUERY_PLANS_MAX];
@@ -1116,15 +1250,9 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) {
1250
1251 struct query_metric_tier *tier_ptr = &qm->tiers[tier];
1252 STORAGE_ENGINE *eng = query_metric_storage_engine(ops->r->internal.qt, qm, tier);
1119 - eng->api.query_ops.init(
1120 - tier_ptr->db_metric_handle,
1121 - &ops->plans[p].handle,
1122 - after, before,
1123 - ops->r->internal.qt->request.priority);
1124 -
1125 - ops->plans[p].next_metric = eng->api.query_ops.next_metric;
1126 - ops->plans[p].is_finished = eng->api.query_ops.is_finished;
1127 - ops->plans[p].finalize = eng->api.query_ops.finalize;
1253 + storage_engine_query_init(eng->backend, tier_ptr->db_metric_handle, &ops->plans[p].handle,
1254 + after, before, ops->r->internal.qt->request.priority);
1255 +
1256 ops->plans[p].initialized = true;
1257 ops->plans[p].finalized = false;
1258 }
@@ -1134,18 +1262,9 @@ static void query_planer_finalize_plan(QUERY_ENGINE_OPS *ops, size_t plan_id) {
1262 // QUERY_METRIC *qm = ops->qm;
1263
1264 if(ops->plans[plan_id].initialized && !ops->plans[plan_id].finalized) {
1137 - ops->plans[plan_id].finalize(&ops->plans[plan_id].handle);
1265 + storage_engine_query_finalize(&ops->plans[plan_id].handle);
1266 ops->plans[plan_id].initialized = false;
1267 ops->plans[plan_id].finalized = true;
1140 - ops->plans[plan_id].next_metric = NULL;
1141 - ops->plans[plan_id].is_finished = NULL;
1142 - ops->plans[plan_id].finalize = NULL;
1143 -
1144 - if(ops->current_plan == plan_id) {
1145 - ops->next_metric = NULL;
1146 - ops->is_finished = NULL;
1147 - ops->finalize = NULL;
1148 - }
1268 }
1269 }
1270
@@ -1168,9 +1287,6 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
1287 ops->tier = qm->plan.array[plan_id].tier;
1288 ops->tier_ptr = &qm->tiers[ops->tier];
1289 ops->handle = &ops->plans[plan_id].handle;
1171 - ops->next_metric = ops->plans[plan_id].next_metric;
1172 - ops->is_finished = ops->plans[plan_id].is_finished;
1173 - ops->finalize = ops->plans[plan_id].finalize;
1290 ops->current_plan = plan_id;
1291
1292 if(plan_id + 1 < qm->plan.used && qm->plan.array[plan_id + 1].after < qm->plan.array[plan_id].before)
@@ -1365,7 +1481,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1481 } \
1482 } while(0)
1483
1368 -#define query_add_point_to_group(r, point, ops) do { \
1484 +#define query_add_point_to_group(r, point, ops, add_flush) do { \
1485 if(likely(netdata_double_isnumber((point).value))) { \
1486 if(likely(fpclassify((point).value) != FP_ZERO)) \
1487 (ops)->group_points_non_zero++; \
@@ -1373,7 +1489,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1489 if(unlikely((point).sp.flags & SN_FLAG_RESET)) \
1490 (ops)->group_value_flags |= RRDR_VALUE_RESET; \
1491 \
1376 - (ops)->grouping_add(r, (point).value); \
1492 + time_grouping_add(r, (point).value, add_flush); \
1493 \
1494 storage_point_merge_to((ops)->group_point, (point).sp); \
1495 if(!(point).added) \
@@ -1422,8 +1538,6 @@ static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_prep(RRDR *r, size_t query_metric_id
1538 *ops = (QUERY_ENGINE_OPS) {
1539 .r = r,
1540 .qm = query_metric(qt, query_metric_id),
1425 - .grouping_add = r->time_grouping.add,
1426 - .grouping_flush = r->time_grouping.flush,
1541 .tier_query_fetch = r->time_grouping.tier_query_fetch,
1542 .view_update_every = r->view.update_every,
1543 .query_granularity = (time_t)(r->view.update_every / r->view.group),
@@ -1442,6 +1556,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1556 QUERY_TARGET *qt = r->internal.qt;
1557 QUERY_METRIC *qm = ops->qm;
1558
1559 + const RRDR_TIME_GROUPING add_flush = r->time_grouping.add_flush;
1560 +
1561 ops->group_point = STORAGE_POINT_UNSET;
1562 ops->query_point = STORAGE_POINT_UNSET;
1563
@@ -1494,7 +1610,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1610 last1_point = new_point;
1611 }
1612
1497 - if(unlikely(ops->is_finished(ops->handle))) {
1613 + if(unlikely(storage_engine_query_is_finished(ops->handle))) {
1614 query_is_finished_counter++;
1615
1616 if(count_same_end_time != 0) {
@@ -1517,7 +1633,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1633 STORAGE_POINT sp;
1634 if(likely(storage_point_is_unset(next1_point))) {
1635 db_points_read_since_plan_switch++;
1520 - sp = ops->next_metric(ops->handle);
1636 + sp = storage_engine_query_next_metric(ops->handle);
1637 ops->db_points_read_per_tier[ops->tier]++;
1638 ops->db_total_points_read++;
1639
@@ -1543,7 +1659,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1659 // A. the entire point of the previous plan is to the future of point from the next plan
1660 // B. part of the point of the previous plan overlaps with the point from the next plan
1661
1546 - STORAGE_POINT sp2 = ops->next_metric(ops->handle);
1662 + STORAGE_POINT sp2 = storage_engine_query_next_metric(ops->handle);
1663 ops->db_points_read_per_tier[ops->tier]++;
1664 ops->db_total_points_read++;
1665
@@ -1637,7 +1753,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1753 if(likely(new_point.sp.end_time_s >= now_start_time)) { // likely to favor tier0
1754 // this db point ends after our now_start time
1755
1640 - query_add_point_to_group(r, new_point, ops);
1756 + query_add_point_to_group(r, new_point, ops, add_flush);
1757 new_point.added = true;
1758 }
1759 else {
@@ -1745,7 +1861,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1861 current_point = QUERY_POINT_EMPTY;
1862 }
1863
1748 - query_add_point_to_group(r, current_point, ops);
1864 + query_add_point_to_group(r, current_point, ops, add_flush);
1865
1866 rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line);
1867 size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
@@ -1761,7 +1877,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
1877 *rrdr_value_options_ptr = ops->group_value_flags;
1878
1879 // store the group value
1764 - NETDATA_DOUBLE group_value = ops->grouping_flush(r, rrdr_value_options_ptr);
1880 + NETDATA_DOUBLE group_value = time_grouping_flush(r, rrdr_value_options_ptr, add_flush);
1881 r->v[rrdr_o_v_index] = group_value;
1882
1883 r->ar[rrdr_o_v_index] = storage_point_anomaly_rate(ops->group_point);
@@ -1829,7 +1945,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
1945 struct rrddim_tier *t = &rd->tiers[tier];
1946 if(unlikely(!t)) return;
1947
1832 - time_t latest_time_s = t->query_ops->latest_time_s(t->db_metric_handle);
1948 + time_t latest_time_s = storage_engine_latest_time_s(t->backend, t->db_metric_handle);
1949 time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every;
1950 time_t time_diff = now_s - latest_time_s;
1951
@@ -1843,21 +1959,21 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
1959
1960 // for each lower tier
1961 for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){
1846 - time_t smaller_tier_first_time = rd->tiers[read_tier].query_ops->oldest_time_s(rd->tiers[read_tier].db_metric_handle);
1847 - time_t smaller_tier_last_time = rd->tiers[read_tier].query_ops->latest_time_s(rd->tiers[read_tier].db_metric_handle);
1962 + time_t smaller_tier_first_time = storage_engine_oldest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle);
1963 + time_t smaller_tier_last_time = storage_engine_latest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle);
1964 if(smaller_tier_last_time <= latest_time_s) continue; // it is as bad as we are
1965
1966 long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s;
1967 long before_wanted = smaller_tier_last_time;
1968
1969 struct rrddim_tier *tmp = &rd->tiers[read_tier];
1854 - tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
1970 + storage_engine_query_init(tmp->backend, tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
1971
1972 size_t points_read = 0;
1973
1858 - while(!tmp->query_ops->is_finished(&handle)) {
1974 + while(!storage_engine_query_is_finished(&handle)) {
1975
1860 - STORAGE_POINT sp = tmp->query_ops->next_metric(&handle);
1976 + STORAGE_POINT sp = storage_engine_query_next_metric(&handle);
1977 points_read++;
1978
1979 if(sp.end_time_s > latest_time_s) {
@@ -1866,7 +1982,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
1982 }
1983 }
1984
1869 - tmp->query_ops->finalize(&handle);
1985 + storage_engine_query_finalize(&handle);
1986 store_metric_collection_completed();
1987 global_statistics_backfill_query_completed(points_read);
1988
@@ -2439,15 +2555,179 @@ static void rrd2rrdr_set_timestamps(RRDR *r) {
2555 before_wanted, r->t[points_wanted - 1]);
2556 }
2557
2558 +static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) {
2559 + buffer_flush(key);
2560 + if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) {
2561 + buffer_strcat(key, "__hidden_dimensions__");
2562 + }
2563 + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
2564 + buffer_strcat(key, "selected");
2565 + }
2566 + else {
2567 + if (group_by & RRDR_GROUP_BY_DIMENSION) {
2568 + buffer_fast_strcat(key, "|", 1);
2569 + buffer_strcat(key, query_metric_name(qt, qm));
2570 + }
2571 +
2572 + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
2573 + buffer_fast_strcat(key, "|", 1);
2574 + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
2575 + }
2576 +
2577 + if (group_by & RRDR_GROUP_BY_LABEL) {
2578 + DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
2579 + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
2580 + buffer_fast_strcat(key, "|", 1);
2581 + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
2582 + }
2583 + }
2584 +
2585 + if (group_by & RRDR_GROUP_BY_NODE) {
2586 + buffer_fast_strcat(key, "|", 1);
2587 + buffer_strcat(key, qn->rrdhost->machine_guid);
2588 + }
2589 +
2590 + if (group_by & RRDR_GROUP_BY_CONTEXT) {
2591 + buffer_fast_strcat(key, "|", 1);
2592 + buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
2593 + }
2594 +
2595 + if (group_by & RRDR_GROUP_BY_UNITS) {
2596 + buffer_fast_strcat(key, "|", 1);
2597 + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
2598 + }
2599 + }
2600 +}
2601 +
2602 +static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) {
2603 + buffer_flush(key);
2604 + if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) {
2605 + buffer_strcat(key, "__hidden_dimensions__");
2606 + }
2607 + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
2608 + buffer_strcat(key, "selected");
2609 + }
2610 + else {
2611 + if (group_by & RRDR_GROUP_BY_DIMENSION) {
2612 + buffer_strcat(key, query_metric_name(qt, qm));
2613 + }
2614 +
2615 + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
2616 + if (buffer_strlen(key) != 0)
2617 + buffer_fast_strcat(key, ",", 1);
2618 +
2619 + if (group_by & RRDR_GROUP_BY_NODE)
2620 + buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
2621 + else
2622 + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
2623 + }
2624 +
2625 + if (group_by & RRDR_GROUP_BY_LABEL) {
2626 + DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
2627 + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
2628 + if (buffer_strlen(key) != 0)
2629 + buffer_fast_strcat(key, ",", 1);
2630 + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
2631 + }
2632 + }
2633 +
2634 + if (group_by & RRDR_GROUP_BY_NODE) {
2635 + if (buffer_strlen(key) != 0)
2636 + buffer_fast_strcat(key, ",", 1);
2637 +
2638 + buffer_strcat(key, qn->rrdhost->machine_guid);
2639 + }
2640 +
2641 + if (group_by & RRDR_GROUP_BY_CONTEXT) {
2642 + if (buffer_strlen(key) != 0)
2643 + buffer_fast_strcat(key, ",", 1);
2644 +
2645 + buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
2646 + }
2647 +
2648 + if (group_by & RRDR_GROUP_BY_UNITS) {
2649 + if (buffer_strlen(key) != 0)
2650 + buffer_fast_strcat(key, ",", 1);
2651 +
2652 + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
2653 + }
2654 + }
2655 +}
2656 +
2657 +static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) {
2658 + buffer_flush(key);
2659 + if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) {
2660 + buffer_strcat(key, "__hidden_dimensions__");
2661 + }
2662 + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
2663 + buffer_strcat(key, "selected");
2664 + }
2665 + else {
2666 + if (group_by & RRDR_GROUP_BY_DIMENSION) {
2667 + buffer_strcat(key, query_metric_name(qt, qm));
2668 + }
2669 +
2670 + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
2671 + if (buffer_strlen(key) != 0)
2672 + buffer_fast_strcat(key, ",", 1);
2673 +
2674 + if (group_by & RRDR_GROUP_BY_NODE)
2675 + buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
2676 + else
2677 + buffer_strcat(key, string2str(query_instance_name_fqdn(qi, qt->request.version)));
2678 + }
2679 +
2680 + if (group_by & RRDR_GROUP_BY_LABEL) {
2681 + DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
2682 + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
2683 + if (buffer_strlen(key) != 0)
2684 + buffer_fast_strcat(key, ",", 1);
2685 + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
2686 + }
2687 + }
2688 +
2689 + if (group_by & RRDR_GROUP_BY_NODE) {
2690 + if (buffer_strlen(key) != 0)
2691 + buffer_fast_strcat(key, ",", 1);
2692 +
2693 + buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
2694 + }
2695 +
2696 + if (group_by & RRDR_GROUP_BY_CONTEXT) {
2697 + if (buffer_strlen(key) != 0)
2698 + buffer_fast_strcat(key, ",", 1);
2699 +
2700 + buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
2701 + }
2702 +
2703 + if (group_by & RRDR_GROUP_BY_UNITS) {
2704 + if (buffer_strlen(key) != 0)
2705 + buffer_fast_strcat(key, ",", 1);
2706 +
2707 + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
2708 + }
2709 + }
2710 +}
2711 +
2712 +struct rrdr_group_by_entry {
2713 + size_t priority;
2714 + size_t count;
2715 + STRING *id;
2716 + STRING *name;
2717 + STRING *units;
2718 + RRDR_DIMENSION_FLAGS od;
2719 + DICTIONARY *dl;
2720 +};
2721 +
2722 static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2723 RRDR_OPTIONS options = qt->window.options;
2724
2445 - if(qt->request.group_by == RRDR_GROUP_BY_NONE) {
2725 + if(qt->request.version < 2) {
2726 + // v1 query
2727 RRDR *r = rrdr_create(owa, qt, qt->query.used, qt->window.points);
2728 if(unlikely(!r)) {
2729 internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, dimensions=%u, points=%zu",
2730 qt->id, qt->window.after, qt->window.before, qt->query.used, qt->window.points);
2450 - query_target_release(qt);
2731 return NULL;
2732 }
2733 r->group_by.r = NULL;
@@ -2462,420 +2742,405 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2742 rrd2rrdr_set_timestamps(r);
2743 return r;
2744 }
2745 + // v2 query
2746
2466 - struct rrdr_group_by_entry *entries = onewayalloc_callocz(owa, qt->query.used, sizeof(struct rrdr_group_by_entry));
2467 - DICTIONARY *groups = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
2468 -
2469 - if(qt->request.group_by & RRDR_GROUP_BY_LABEL && qt->request.group_by_label && *qt->request.group_by_label)
2470 - qt->group_by.used = quoted_strings_splitter(qt->request.group_by_label, qt->group_by.label_keys, GROUP_BY_MAX_LABEL_KEYS, group_by_label_is_space);
2471 -
2472 - if(!qt->group_by.used)
2473 - qt->request.group_by &= ~RRDR_GROUP_BY_LABEL;
2474 -
2475 - if(!(qt->request.group_by & (RRDR_GROUP_BY_SELECTED | RRDR_GROUP_BY_DIMENSION | RRDR_GROUP_BY_INSTANCE | RRDR_GROUP_BY_LABEL | RRDR_GROUP_BY_NODE | RRDR_GROUP_BY_CONTEXT)))
2476 - qt->request.group_by = RRDR_GROUP_BY_DIMENSION;
2477 -
2478 - DICTIONARY *label_keys = NULL;
2479 - if(options & RRDR_OPTION_GROUP_BY_LABELS)
2480 - label_keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0);
2747 + // parse all the group-by label keys
2748 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
2749 + if (qt->request.group_by[g].group_by & RRDR_GROUP_BY_LABEL &&
2750 + qt->request.group_by[g].group_by_label && *qt->request.group_by[g].group_by_label)
2751 + qt->group_by[g].used = quoted_strings_splitter(
2752 + qt->request.group_by[g].group_by_label, qt->group_by[g].label_keys,
2753 + GROUP_BY_MAX_LABEL_KEYS, group_by_label_is_space);
2754
2482 - int added = 0;
2483 - BUFFER *key = buffer_create(0, NULL);
2484 - QUERY_INSTANCE *last_qi = NULL;
2485 - size_t priority = 0;
2486 - time_t update_every_max = 0;
2487 - for(size_t d = 0; d < qt->query.used ; d++) {
2488 - QUERY_METRIC *qm = query_metric(qt, d);
2489 - QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
2490 - QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
2491 - QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
2755 + if (!qt->group_by[g].used)
2756 + qt->request.group_by[g].group_by &= ~RRDR_GROUP_BY_LABEL;
2757 + }
2758
2493 - if(qi != last_qi) {
2494 - priority = 0;
2495 - last_qi = qi;
2759 + // make sure there are valid group-by methods
2760 + bool query_has_percentage_of_instance = false;
2761 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) {
2762 + if(!(qt->request.group_by[g].group_by & SUPPORTED_GROUP_BY_METHODS))
2763 + qt->request.group_by[g].group_by = (g == 0) ? RRDR_GROUP_BY_DIMENSION : RRDR_GROUP_BY_NONE;
2764
2497 - time_t update_every = rrdinstance_acquired_update_every(qi->ria);
2498 - if(update_every > update_every_max)
2499 - update_every_max = update_every;
2500 - }
2501 - else
2502 - priority++;
2765 + if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
2766 + query_has_percentage_of_instance = true;
2767 + }
2768
2504 - // --------------------------------------------------------------------
2505 - // generate the group by key
2769 + // merge all group-by options to upper levels
2770 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) {
2771 + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
2772 + continue;
2773
2507 - buffer_flush(key);
2508 - if(unlikely(qm->status & RRDR_DIMENSION_HIDDEN)) {
2509 - buffer_strcat(key, "__hidden_dimensions__");
2510 - }
2511 - else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
2512 - buffer_strcat(key, "selected");
2774 + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_SELECTED) {
2775 + for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++)
2776 + qt->request.group_by[r].group_by = RRDR_GROUP_BY_NONE;
2777 }
2778 else {
2515 - if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
2516 - buffer_fast_strcat(key, "|", 1);
2517 - buffer_strcat(key, query_metric_name(qt, qm));
2518 - }
2779 + for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++) {
2780 + if (qt->request.group_by[r].group_by == RRDR_GROUP_BY_NONE)
2781 + continue;
2782
2520 - if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
2521 - buffer_fast_strcat(key, "|", 1);
2522 - buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
2523 - }
2783 + if (qt->request.group_by[r].group_by != RRDR_GROUP_BY_SELECTED) {
2784 + if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
2785 + qt->request.group_by[g].group_by |= RRDR_GROUP_BY_INSTANCE;
2786 + else
2787 + qt->request.group_by[g].group_by |= qt->request.group_by[r].group_by;
2788 +
2789 + if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_LABEL) {
2790 + for (size_t lr = 0; lr < qt->group_by[r].used; lr++) {
2791 + bool found = false;
2792 + for (size_t lg = 0; lg < qt->group_by[g].used; lg++) {
2793 + if (strcmp(qt->group_by[g].label_keys[lg], qt->group_by[r].label_keys[lr]) == 0) {
2794 + found = true;
2795 + break;
2796 + }
2797 + }
2798
2525 - if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
2526 - DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
2527 - for (size_t l = 0; l < qt->group_by.used; l++) {
2528 - buffer_fast_strcat(key, "|", 1);
2529 - rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
2799 + if (!found && qt->group_by[g].used < GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES)
2800 + qt->group_by[g].label_keys[qt->group_by[g].used++] = qt->group_by[r].label_keys[lr];
2801 + }
2802 + }
2803 }
2804 }
2532 -
2533 - if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
2534 - buffer_fast_strcat(key, "|", 1);
2535 - buffer_strcat(key, qn->rrdhost->machine_guid);
2536 - }
2537 -
2538 - if (qt->request.group_by & RRDR_GROUP_BY_CONTEXT) {
2539 - buffer_fast_strcat(key, "|", 1);
2540 - buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
2541 - }
2542 -
2543 - if (qt->request.group_by & RRDR_GROUP_BY_UNITS) {
2544 - buffer_fast_strcat(key, "|", 1);
2545 - buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
2546 - }
2805 }
2806 + }
2807
2549 - // lookup the key in the dictionary
2808 + int added = 0;
2809 + RRDR *first_r = NULL, *last_r = NULL;
2810 + BUFFER *key = buffer_create(0, NULL);
2811 + struct rrdr_group_by_entry *entries = onewayalloc_mallocz(owa, qt->query.used * sizeof(struct rrdr_group_by_entry));
2812 + DICTIONARY *groups = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
2813 + DICTIONARY *label_keys = NULL;
2814
2551 - int pos = -1;
2552 - int *set = dictionary_set(groups, buffer_tostring(key), &pos, sizeof(pos));
2553 - if(*set == -1) {
2554 - // the key just added to the dictionary
2815 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
2816 + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by;
2817
2556 - *set = pos = added++;
2818 + if(group_by == RRDR_GROUP_BY_NONE)
2819 + break;
2820
2558 - // ----------------------------------------------------------------
2559 - // generate the dimension id
2821 + memset(entries, 0, qt->query.used * sizeof(struct rrdr_group_by_entry));
2822 + dictionary_flush(groups);
2823 + added = 0;
2824
2561 - buffer_flush(key);
2562 - if(unlikely(qm->status & RRDR_DIMENSION_HIDDEN)) {
2563 - buffer_strcat(key, "__hidden_dimensions__");
2564 - }
2565 - else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
2566 - buffer_strcat(key, "selected");
2567 - }
2568 - else {
2569 - if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
2570 - buffer_strcat(key, query_metric_name(qt, qm));
2571 - }
2825 + size_t hidden_dimensions = 0;
2826 + bool final_grouping = (g == MAX_QUERY_GROUP_BY_PASSES - 1 || qt->request.group_by[g + 1].group_by == RRDR_GROUP_BY_NONE) ? true : false;
2827
2573 - if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
2574 - if (buffer_strlen(key) != 0)
2575 - buffer_fast_strcat(key, ",", 1);
2828 + if (final_grouping && (options & RRDR_OPTION_GROUP_BY_LABELS))
2829 + label_keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0);
2830
2577 - if (qt->request.group_by & RRDR_GROUP_BY_NODE)
2578 - buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
2579 - else
2580 - buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
2581 - }
2831 + QUERY_INSTANCE *last_qi = NULL;
2832 + size_t priority = 0;
2833 + time_t update_every_max = 0;
2834 + for (size_t d = 0; d < qt->query.used; d++) {
2835 + QUERY_METRIC *qm = query_metric(qt, d);
2836 + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
2837 + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
2838 + QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
2839 + QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
2840
2583 - if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
2584 - DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
2585 - for (size_t l = 0; l < qt->group_by.used; l++) {
2586 - if (buffer_strlen(key) != 0)
2587 - buffer_fast_strcat(key, ",", 1);
2588 - rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
2589 - }
2590 - }
2841 + if (qi != last_qi) {
2842 + last_qi = qi;
2843
2592 - if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
2593 - if (buffer_strlen(key) != 0)
2594 - buffer_fast_strcat(key, ",", 1);
2844 + time_t update_every = rrdinstance_acquired_update_every(qi->ria);
2845 + if (update_every > update_every_max)
2846 + update_every_max = update_every;
2847 + }
2848
2596 - buffer_strcat(key, qn->rrdhost->machine_guid);
2597 - }
2849 + priority = qd->priority;
2850
2599 - if (qt->request.group_by & RRDR_GROUP_BY_CONTEXT) {
2600 - if (buffer_strlen(key) != 0)
2601 - buffer_fast_strcat(key, ",", 1);
2851 + if(qm->status & RRDR_DIMENSION_HIDDEN)
2852 + hidden_dimensions++;
2853
2603 - buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
2604 - }
2854 + // --------------------------------------------------------------------
2855 + // generate the group by key
2856
2606 - if (qt->request.group_by & RRDR_GROUP_BY_UNITS) {
2607 - if (buffer_strlen(key) != 0)
2608 - buffer_fast_strcat(key, ",", 1);
2857 + query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance);
2858
2610 - buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
2611 - }
2612 - }
2859 + // lookup the key in the dictionary
2860
2614 - entries[pos].id = string_strdupz(buffer_tostring(key));
2861 + int pos = -1;
2862 + int *set = dictionary_set(groups, buffer_tostring(key), &pos, sizeof(pos));
2863 + if (*set == -1) {
2864 + // the key just added to the dictionary
2865
2616 - // ----------------------------------------------------------------
2617 - // generate the dimension name
2866 + *set = pos = added++;
2867
2619 - buffer_flush(key);
2620 - if(unlikely(qm->status & RRDR_DIMENSION_HIDDEN)) {
2621 - buffer_strcat(key, "__hidden_dimensions__");
2622 - }
2623 - else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
2624 - buffer_strcat(key, "selected");
2625 - }
2626 - else {
2627 - if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
2628 - buffer_strcat(key, query_metric_name(qt, qm));
2629 - }
2868 + // ----------------------------------------------------------------
2869 + // generate the dimension id
2870
2631 - if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
2632 - if (buffer_strlen(key) != 0)
2633 - buffer_fast_strcat(key, ",", 1);
2871 + query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance);
2872 + entries[pos].id = string_strdupz(buffer_tostring(key));
2873
2635 - if (qt->request.group_by & RRDR_GROUP_BY_NODE)
2636 - buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
2637 - else
2638 - buffer_strcat(key, string2str(query_instance_name_fqdn(qi, qt->request.version)));
2639 - }
2874 + // ----------------------------------------------------------------
2875 + // generate the dimension name
2876
2641 - if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
2642 - DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
2643 - for (size_t l = 0; l < qt->group_by.used; l++) {
2644 - if (buffer_strlen(key) != 0)
2645 - buffer_fast_strcat(key, ",", 1);
2646 - rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
2647 - }
2648 - }
2877 + query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance);
2878 + entries[pos].name = string_strdupz(buffer_tostring(key));
2879
2650 - if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
2651 - if (buffer_strlen(key) != 0)
2652 - buffer_fast_strcat(key, ",", 1);
2880 + // add the rest of the info
2881 + entries[pos].units = rrdinstance_acquired_units_dup(qi->ria);
2882 + entries[pos].priority = priority;
2883
2654 - buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
2884 + if (label_keys) {
2885 + entries[pos].dl = dictionary_create_advanced(
2886 + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
2887 + NULL, sizeof(struct group_by_label_key));
2888 + dictionary_register_insert_callback(entries[pos].dl, group_by_label_key_insert_cb, label_keys);
2889 + dictionary_register_delete_callback(entries[pos].dl, group_by_label_key_delete_cb, label_keys);
2890 }
2891 + } else {
2892 + // the key found in the dictionary
2893 + pos = *set;
2894 + }
2895
2657 - if (qt->request.group_by & RRDR_GROUP_BY_CONTEXT) {
2658 - if (buffer_strlen(key) != 0)
2659 - buffer_fast_strcat(key, ",", 1);
2896 + entries[pos].count++;
2897
2661 - buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
2662 - }
2898 + if (unlikely(priority < entries[pos].priority))
2899 + entries[pos].priority = priority;
2900
2664 - if (qt->request.group_by & RRDR_GROUP_BY_UNITS) {
2665 - if (buffer_strlen(key) != 0)
2666 - buffer_fast_strcat(key, ",", 1);
2901 + if(g > 0)
2902 + last_r->dgbs[qm->grouped_as.slot] = pos;
2903 + else
2904 + qm->grouped_as.first_slot = pos;
2905 +
2906 + qm->grouped_as.slot = pos;
2907 + qm->grouped_as.id = entries[pos].id;
2908 + qm->grouped_as.name = entries[pos].name;
2909 + qm->grouped_as.units = entries[pos].units;
2910 +
2911 + // copy the dimension flags decided by the query target
2912 + // we need this, because if a dimension is explicitly selected
2913 + // the query target adds to it the non-zero flag
2914 + qm->status |= RRDR_DIMENSION_GROUPED;
2915 +
2916 + if(query_has_percentage_of_instance)
2917 + // when the query has percentage of instance
2918 + // there will be no hidden dimensions in the final query
2919 + // so we have to remove the hidden flag from all dimensions
2920 + entries[pos].od |= qm->status & ~RRDR_DIMENSION_HIDDEN;
2921 + else
2922 + entries[pos].od |= qm->status;
2923
2668 - buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
2669 - }
2670 - }
2924 + if (entries[pos].dl)
2925 + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(qi->ria),
2926 + rrdlabels_traversal_cb_to_group_by_label_key, entries[pos].dl);
2927 + }
2928
2672 - entries[pos].name = string_strdupz(buffer_tostring(key));
2929 + RRDR *r = rrdr_create(owa, qt, added, qt->window.points);
2930 + if (!r) {
2931 + internal_error(true,
2932 + "QUERY: cannot create group by RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
2933 + qt->id, qt->window.after, qt->window.before, added, qt->window.points);
2934 + goto cleanup;
2935 + }
2936
2674 - // add the rest of the info
2675 - entries[pos].units = rrdinstance_acquired_units_dup(qi->ria);
2676 - entries[pos].priority = priority;
2937 + bool hidden_dimension_on_percentage_of_instance = hidden_dimensions && (group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE);
2938
2678 - if(options & RRDR_OPTION_GROUP_BY_LABELS) {
2679 - entries[pos].dl = dictionary_create_advanced(
2680 - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
2681 - NULL, sizeof(struct group_by_label_key));
2682 - dictionary_register_insert_callback(entries[pos].dl, group_by_label_key_insert_cb, label_keys);
2683 - dictionary_register_delete_callback(entries[pos].dl, group_by_label_key_delete_cb, label_keys);
2684 - }
2685 - }
2686 - else {
2687 - // the key found in the dictionary
2688 - pos = *set;
2689 - }
2939 + // prevent double cleanup in case of error
2940 + added = 0;
2941
2691 - entries[pos].count++;
2942 + if(!last_r)
2943 + first_r = last_r = r;
2944 + else
2945 + last_r->group_by.r = r;
2946 +
2947 + last_r = r;
2948 +
2949 + rrd2rrdr_set_timestamps(r);
2950 + r->dp = onewayalloc_callocz(owa, r->d, sizeof(*r->dp));
2951 + r->dview = onewayalloc_callocz(owa, r->d, sizeof(*r->dview));
2952 + r->dgbc = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbc));
2953 + r->gbc = onewayalloc_callocz(owa, r->n * r->d, sizeof(*r->gbc));
2954 + r->dqp = onewayalloc_callocz(owa, r->d, sizeof(STORAGE_POINT));
2955 +
2956 + if(hidden_dimension_on_percentage_of_instance)
2957 + // this is where we are going to group the hidden dimensions
2958 + r->vh = onewayalloc_mallocz(owa, r->n * r->d * sizeof(*r->vh));
2959 +
2960 + if(!final_grouping)
2961 + // this is where we are going to store the slot in the next RRDR
2962 + // that we are going to group by the dimension of this RRDR
2963 + r->dgbs = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbs));
2964 +
2965 + if (label_keys) {
2966 + r->dl = onewayalloc_callocz(owa, r->d, sizeof(DICTIONARY *));
2967 + r->label_keys = label_keys;
2968 + label_keys = NULL;
2969 + }
2970
2693 - if(unlikely(priority < entries[pos].priority))
2694 - entries[pos].priority = priority;
2971 + // zero r (dimension options, names, and ids)
2972 + // this is required, because group-by may lead to empty dimensions
2973 + for (size_t d = 0; d < r->d; d++) {
2974 + r->di[d] = entries[d].id;
2975 + r->dn[d] = entries[d].name;
2976
2696 - qm->grouped_as.slot = pos;
2697 - qm->grouped_as.id = entries[pos].id;
2698 - qm->grouped_as.name = entries[pos].name;
2699 - qm->grouped_as.units = entries[pos].units;
2977 + r->od[d] = entries[d].od;
2978 + r->du[d] = entries[d].units;
2979 + r->dp[d] = entries[d].priority;
2980 + r->dgbc[d] = entries[d].count;
2981
2701 - // copy the dimension flags decided by the query target
2702 - // we need this, because if a dimension is explicitly selected
2703 - // the query target adds to it the non-zero flag
2704 - qm->status |= RRDR_DIMENSION_GROUPED;
2705 - entries[pos].od |= qm->status;
2982 + if (r->dl)
2983 + r->dl[d] = entries[d].dl;
2984 + }
2985
2707 - if(entries[pos].dl)
2708 - rrdlabels_walkthrough_read(rrdinstance_acquired_labels(qi->ria),
2709 - rrdlabels_traversal_cb_to_group_by_label_key, entries[pos].dl);
2986 + // initialize partial trimming
2987 + r->partial_data_trimming.max_update_every = update_every_max;
2988 + r->partial_data_trimming.expected_after =
2989 + (!(qt->window.options & RRDR_OPTION_RETURN_RAW) &&
2990 + qt->window.before >= qt->window.now - update_every_max) ?
2991 + qt->window.before - update_every_max :
2992 + qt->window.before;
2993 + r->partial_data_trimming.trimmed_after = qt->window.before;
2994 +
2995 + // make all values empty
2996 + for (size_t i = 0; i != r->n; i++) {
2997 + NETDATA_DOUBLE *cn = &r->v[i * r->d];
2998 + RRDR_VALUE_FLAGS *co = &r->o[i * r->d];
2999 + NETDATA_DOUBLE *ar = &r->ar[i * r->d];
3000 + NETDATA_DOUBLE *vh = r->vh ? &r->vh[i * r->d] : NULL;
3001 +
3002 + for (size_t d = 0; d < r->d; d++) {
3003 + cn[d] = NAN;
3004 + ar[d] = 0.0;
3005 + co[d] = RRDR_VALUE_EMPTY;
3006 +
3007 + if(vh)
3008 + *vh = NAN;
3009 + }
3010 + }
3011 }
3012
2712 - RRDR *r = rrdr_create(owa, qt, added, qt->window.points);
2713 - if(!r) {
2714 - internal_error(true, "QUERY: cannot create group by RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
2715 - qt->id, qt->window.after, qt->window.before, added, qt->window.points);
3013 + if(!first_r || !last_r)
3014 goto cleanup;
2717 - }
3015
2719 - r->group_by.r = rrdr_create(owa, qt, 1, qt->window.points);
2720 - if(!r->group_by.r) {
2721 - internal_error(true, "QUERY: cannot create group by temporary RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
3016 + RRDR *r_tmp = rrdr_create(owa, qt, 1, qt->window.points);
3017 + if (!r_tmp) {
3018 + internal_error(true,
3019 + "QUERY: cannot create group by temporary RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
3020 qt->id, qt->window.after, qt->window.before, 1, qt->window.points);
3021 goto cleanup;
3022 }
3023 + rrd2rrdr_set_timestamps(r_tmp);
3024 + r_tmp->group_by.r = first_r;
3025
2726 - rrd2rrdr_set_timestamps(r);
2727 - rrd2rrdr_set_timestamps(r->group_by.r);
2728 -
2729 - r->dp = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dp));
2730 - r->dview = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dview));
2731 - r->dgbc = onewayalloc_callocz(r->internal.owa, r->d, sizeof(*r->dgbc));
2732 - r->gbc = onewayalloc_callocz(r->internal.owa, r->n * r->d, sizeof(*r->gbc));
2733 - r->dqp = onewayalloc_callocz(r->internal.owa, r->d, sizeof(STORAGE_POINT));
2734 -
2735 - if(options & RRDR_OPTION_GROUP_BY_LABELS) {
2736 - r->dl = onewayalloc_callocz(r->internal.owa, r->d, sizeof(DICTIONARY *));
2737 - r->label_keys = label_keys;
2738 - }
2739 -
2740 - // zero r (dimension options, names, and ids)
2741 - // this is required, because group-by may lead to empty dimensions
2742 - for(size_t d = 0; d < r->d ; d++) {
2743 - r->di[d] = entries[d].id;
2744 - r->dn[d] = entries[d].name;
2745 -
2746 - r->od[d] = entries[d].od;
2747 - r->du[d] = entries[d].units;
2748 - r->dp[d] = entries[d].priority;
2749 - r->dgbc[d] = entries[d].count;
2750 -
2751 - if(r->dl)
2752 - r->dl[d] = entries[d].dl;
2753 - }
2754 -
2755 - // initialize partial trimming
2756 - r->partial_data_trimming.max_update_every = update_every_max;
2757 - r->partial_data_trimming.expected_after =
2758 - (!(qt->window.options & RRDR_OPTION_RETURN_RAW) && qt->window.before >= qt->window.now - update_every_max) ?
2759 - qt->window.before - update_every_max :
2760 - qt->window.before;
2761 - r->partial_data_trimming.trimmed_after = qt->window.before;
2762 -
2763 - // make all values empty
2764 - for(size_t i = 0; i != r->n ;i++) {
2765 - NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
2766 - RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
2767 - NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
2768 - for (size_t d = 0; d < r->d; d++) {
2769 - cn[d] = 0.0;
2770 - ar[d] = 0.0;
2771 - co[d] = RRDR_VALUE_EMPTY;
3026 +cleanup:
3027 + if(!first_r || !last_r || !r_tmp) {
3028 + if(r_tmp) {
3029 + r_tmp->group_by.r = NULL;
3030 + rrdr_free(owa, r_tmp);
3031 }
2773 - }
3032
2775 -cleanup:
2776 - buffer_free(key);
3033 + if(first_r) {
3034 + RRDR *r = first_r;
3035 + while (r) {
3036 + r_tmp = r->group_by.r;
3037 + r->group_by.r = NULL;
3038 + rrdr_free(owa, r);
3039 + r = r_tmp;
3040 + }
3041 + }
3042
2778 - if(!r) {
2779 - if(entries) {
2780 - for (int d2 = 0; d2 < added; d2++) {
2781 - string_freez(entries[d2].id);
2782 - string_freez(entries[d2].name);
2783 - dictionary_destroy(entries[d2].dl);
3043 + if(entries && added) {
3044 + for (int d = 0; d < added; d++) {
3045 + string_freez(entries[d].id);
3046 + string_freez(entries[d].name);
3047 + string_freez(entries[d].units);
3048 + dictionary_destroy(entries[d].dl);
3049 }
3050 }
3051 dictionary_destroy(label_keys);
2787 - query_target_release(qt);
2788 - }
2789 - else if(!r->group_by.r) {
2790 - rrdr_free(owa, r);
2791 - r = NULL;
3052 +
3053 + first_r = last_r = r_tmp = NULL;
3054 }
3055
3056 + buffer_free(key);
3057 onewayalloc_freez(owa, entries);
3058 dictionary_destroy(groups);
3059
2797 - return r;
3060 + return r_tmp;
3061 }
3062
2800 -static void rrd2rrdr_group_by_add_metric(RRDR *r, size_t query_metric_id) {
2801 - if(!r->group_by.r)
3063 +static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, size_t d_tmp,
3064 + RRDR_GROUP_BY_FUNCTION group_by_aggregate_function,
3065 + STORAGE_POINT *query_points, size_t pass __maybe_unused) {
3066 + if(!r_tmp || r_dst == r_tmp || !(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED))
3067 return;
3068
2804 - QUERY_TARGET *qt = r->internal.qt;
2805 - RRDR_OPTIONS options = qt->window.options;
2806 - RRDR *r_tmp = r->group_by.r;
3069 + internal_fatal(r_dst->n != r_tmp->n, "QUERY: group-by source and destination do not have the same number of rows");
3070 + internal_fatal(d_dst >= r_dst->d, "QUERY: group-by destination dimension number exceeds destination RRDR size");
3071 + internal_fatal(d_tmp >= r_tmp->d, "QUERY: group-by source dimension number exceeds source RRDR size");
3072 + internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)");
3073 + internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)");
3074 +
3075 + bool hidden_dimension_on_percentage_of_instance = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh;
3076
2808 - QUERY_METRIC *qm = query_metric(qt, query_metric_id);
2809 - size_t d = qm->grouped_as.slot;
3077 + if(!hidden_dimension_on_percentage_of_instance) {
3078 + r_dst->od[d_dst] |= r_tmp->od[d_tmp];
3079 + storage_point_merge_to(r_dst->dqp[d_dst], *query_points);
3080 + }
3081
3082 // do the group_by
3083 for(size_t i = 0; i != rrdr_rows(r_tmp) ; i++) {
3084
2814 - size_t idx_tmp = i * r_tmp->d;
2815 - NETDATA_DOUBLE *cn_tmp_base = &r_tmp->v[ idx_tmp ];
2816 - RRDR_VALUE_FLAGS *co_tmp_base = &r_tmp->o[ idx_tmp ];
2817 - NETDATA_DOUBLE *ar_tmp_base = &r_tmp->ar[ idx_tmp ];
3085 + size_t idx_tmp = i * r_tmp->d + d_tmp;
3086 + NETDATA_DOUBLE n_tmp = r_tmp->v[ idx_tmp ];
3087 + RRDR_VALUE_FLAGS o_tmp = r_tmp->o[ idx_tmp ];
3088 + NETDATA_DOUBLE ar_tmp = r_tmp->ar[ idx_tmp ];
3089
2819 - size_t idx = i * r->d;
2820 - NETDATA_DOUBLE *cn_base = &r->v[ idx ];
2821 - RRDR_VALUE_FLAGS *co_base = &r->o[ idx ];
2822 - NETDATA_DOUBLE *ar_base = &r->ar[ idx ];
2823 - uint32_t *gbc_base = &r->gbc[ idx ];
2824 -
2825 - for(size_t d_tmp = 0; d_tmp < r_tmp->d ; d_tmp++) {
2826 - if(unlikely(!(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED)))
2827 - continue;
2828 -
2829 - NETDATA_DOUBLE n_tmp = cn_tmp_base[d_tmp];
2830 - RRDR_VALUE_FLAGS o_tmp = co_tmp_base[d_tmp];
2831 - NETDATA_DOUBLE ar_tmp = ar_tmp_base[d_tmp];
3090 + if(o_tmp & RRDR_VALUE_EMPTY)
3091 + continue;
3092
2833 - if(o_tmp & RRDR_VALUE_EMPTY) {
2834 - if(options & RRDR_OPTION_NULL2ZERO)
2835 - n_tmp = 0.0;
3093 + size_t idx_dst = i * r_dst->d + d_dst;
3094 + NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_instance) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ];
3095 + RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ];
3096 + NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ];
3097 + uint32_t *gbc = &r_dst->gbc[ idx_dst ];
3098 +
3099 + switch(group_by_aggregate_function) {
3100 + default:
3101 + case RRDR_GROUP_BY_FUNCTION_AVERAGE:
3102 + case RRDR_GROUP_BY_FUNCTION_SUM:
3103 + if(isnan(*cn))
3104 + *cn = n_tmp;
3105 else
2837 - continue;
2838 - }
2839 -
2840 - r->od[d] |= RRDR_DIMENSION_QUERIED;
2841 -
2842 - NETDATA_DOUBLE *cn = &cn_base[d];
2843 - RRDR_VALUE_FLAGS *co = &co_base[d];
2844 - NETDATA_DOUBLE *ar = &ar_base[d];
2845 - uint32_t *gbc = &gbc_base[d];
2846 -
2847 - switch(qt->request.group_by_aggregate_function) {
2848 - default:
2849 - case RRDR_GROUP_BY_FUNCTION_AVERAGE:
2850 - case RRDR_GROUP_BY_FUNCTION_SUM:
3106 *cn += n_tmp;
2852 - break;
3107 + break;
3108
2854 - case RRDR_GROUP_BY_FUNCTION_MIN:
2855 - if(!*gbc || n_tmp < *cn)
2856 - *cn = n_tmp;
2857 - break;
3109 + case RRDR_GROUP_BY_FUNCTION_MIN:
3110 + if(isnan(*cn) || n_tmp < *cn)
3111 + *cn = n_tmp;
3112 + break;
3113
2859 - case RRDR_GROUP_BY_FUNCTION_MAX:
2860 - if(!*gbc || n_tmp > *cn)
2861 - *cn = n_tmp;
2862 - break;
2863 - }
3114 + case RRDR_GROUP_BY_FUNCTION_MAX:
3115 + if(isnan(*cn) || n_tmp > *cn)
3116 + *cn = n_tmp;
3117 + break;
3118 + }
3119
3120 + if(!hidden_dimension_on_percentage_of_instance) {
3121 + *co &= ~RRDR_VALUE_EMPTY;
3122 *co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL));
3123 *ar += ar_tmp;
3124 (*gbc)++;
3125 }
3126 }
2870 -
2871 - storage_point_merge_to(r->dqp[d], qm->query_points);
3127 }
3128
3129 static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
2875 - // FIXME - this is not optimal, we should not traverse the entire array to go to the end of it
3130 + time_t trimmable_after = r->partial_data_trimming.expected_after;
3131 +
3132 + // find the point just before the trimmable ones
3133 + ssize_t i = (ssize_t)r->n - 1;
3134 + for( ; i >= 0 ;i--) {
3135 + if (r->t[i] < trimmable_after)
3136 + break;
3137 + }
3138 +
3139 + if(unlikely(i < 0))
3140 + return;
3141
3142 size_t last_row_gbc = 0;
2878 - for (size_t i = 0; i != r->n; i++) {
3143 + for (; i < (ssize_t)r->n; i++) {
3144 size_t row_gbc = 0;
3145 for (size_t d = 0; d < r->d; d++) {
3146 if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
@@ -2884,7 +3149,7 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
3149 row_gbc += r->gbc[ i * r->d + d ];
3150 }
3151
2887 - if (unlikely(r->t[i] > r->partial_data_trimming.expected_after && row_gbc < last_row_gbc)) {
3152 + if (unlikely(r->t[i] >= trimmable_after && row_gbc < last_row_gbc)) {
3153 // discard the rest of the points
3154 r->partial_data_trimming.trimmed_after = r->t[i];
3155 r->rows = i;
@@ -2895,6 +3160,30 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
3160 }
3161 }
3162
3163 +static void rrdr2rrdr_group_by_calculate_percentage_of_instance(RRDR *r) {
3164 + if(!r->vh)
3165 + return;
3166 +
3167 + for(size_t i = 0; i < r->n ;i++) {
3168 + NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
3169 + NETDATA_DOUBLE *ch = &r->vh[ i * r->d ];
3170 +
3171 + for(size_t d = 0; d < r->d ;d++) {
3172 + NETDATA_DOUBLE n = cn[d];
3173 + NETDATA_DOUBLE h = ch[d];
3174 +
3175 + if(isnan(n))
3176 + cn[d] = 0.0;
3177 +
3178 + else if(isnan(h))
3179 + cn[d] = 100.0;
3180 +
3181 + else
3182 + cn[d] = n * 100.0 / (n + h);
3183 + }
3184 + }
3185 +}
3186 +
3187 static void rrd2rrdr_convert_to_percentage(RRDR *r) {
3188 size_t global_min_max_values = 0;
3189 NETDATA_DOUBLE global_min = NAN, global_max = NAN;
@@ -2987,22 +3276,52 @@ static void rrd2rrdr_convert_to_percentage(RRDR *r) {
3276 }
3277 }
3278
2990 -static void rrd2rrdr_group_by_finalize(RRDR *r) {
2991 - QUERY_TARGET *qt = r->internal.qt;
3279 +static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3280 + QUERY_TARGET *qt = r_tmp->internal.qt;
3281 RRDR_OPTIONS options = qt->window.options;
3282
2994 - if(!r->group_by.r) {
3283 + if(!r_tmp->group_by.r) {
3284 // v1 query
3285 if(options & RRDR_OPTION_PERCENTAGE)
2997 - rrd2rrdr_convert_to_percentage(r);
2998 - return;
3286 + rrd2rrdr_convert_to_percentage(r_tmp);
3287 + return r_tmp;
3288 }
3289 // v2 query
3290
3002 - // copy the timestamps
3003 - for(size_t i = 0; i != r->n ;i++) {
3004 - r->t[i] = r->group_by.r->t[i];
3291 + // do the additional passes on RRDRs
3292 + RRDR *last_r = r_tmp->group_by.r;
3293 + rrdr2rrdr_group_by_calculate_percentage_of_instance(last_r);
3294 +
3295 + RRDR *r = last_r->group_by.r;
3296 + size_t pass = 0;
3297 + while(r) {
3298 + pass++;
3299 + for(size_t d = 0; d < last_r->d ;d++) {
3300 + rrd2rrdr_group_by_add_metric(r, last_r->dgbs[d], last_r, d,
3301 + qt->request.group_by[pass].aggregation,
3302 + &last_r->dqp[d], pass);
3303 + }
3304 + rrdr2rrdr_group_by_calculate_percentage_of_instance(r);
3305 +
3306 + last_r = r;
3307 + r = last_r->group_by.r;
3308 + }
3309 +
3310 + // free all RRDRs except the last one
3311 + r = r_tmp;
3312 + while(r != last_r) {
3313 + r_tmp = r->group_by.r;
3314 + r->group_by.r = NULL;
3315 + rrdr_free(r->internal.owa, r);
3316 + r = r_tmp;
3317 }
3318 + r = last_r;
3319 +
3320 + // find the final aggregation
3321 + RRDR_GROUP_BY_FUNCTION aggregation = qt->request.group_by[0].aggregation;
3322 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
3323 + if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE)
3324 + aggregation = qt->request.group_by[g].aggregation;
3325
3326 if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before)
3327 rrdr2rrdr_group_by_partial_trimming(r);
@@ -3038,7 +3357,7 @@ static void rrd2rrdr_group_by_finalize(RRDR *r) {
3357 sum += *cn;
3358 ars += *ar;
3359
3041 - if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt))
3360 + if(aggregation == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt))
3361 n = (*cn /= gbc);
3362 else
3363 n = *cn;
@@ -3129,6 +3448,8 @@ static void rrd2rrdr_group_by_finalize(RRDR *r) {
3448 }
3449 }
3450 }
3451 +
3452 + return r;
3453 }
3454
3455 // ----------------------------------------------------------------------------
@@ -3158,31 +3479,40 @@ RRDR *rrd2rrdr_legacy(
3479 .priority = priority,
3480 };
3481
3161 - return rrd2rrdr(owa, query_target_create(&qtr));
3482 + QUERY_TARGET *qt = query_target_create(&qtr);
3483 + RRDR *r = rrd2rrdr(owa, qt);
3484 + if(!r) {
3485 + query_target_release(qt);
3486 + return NULL;
3487 + }
3488 +
3489 + r->internal.release_with_rrdr_qt = qt;
3490 + return r;
3491 }
3492
3493 RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3165 - if(!qt)
3494 + if(!qt || !owa)
3495 return NULL;
3496
3168 - if(!owa) {
3169 - query_target_release(qt);
3170 - return NULL;
3171 - }
3172 -
3497 // qt.window members are the WANTED ones.
3498 // qt.request members are the REQUESTED ones.
3499
3176 - RRDR *r = rrd2rrdr_group_by_initialize(owa, qt);
3177 - if(!r)
3500 + RRDR *r_tmp = rrd2rrdr_group_by_initialize(owa, qt);
3501 + if(!r_tmp)
3502 return NULL;
3503
3504 + // the RRDR we group-by at
3505 + RRDR *r = (r_tmp->group_by.r) ? r_tmp->group_by.r : r_tmp;
3506 +
3507 + // the final RRDR to return to callers
3508 + RRDR *last_r = r_tmp;
3509 + while(last_r->group_by.r)
3510 + last_r = last_r->group_by.r;
3511 +
3512 if(qt->window.relative)
3181 - r->view.flags |= RRDR_RESULT_FLAG_RELATIVE;
3513 + last_r->view.flags |= RRDR_RESULT_FLAG_RELATIVE;
3514 else
3183 - r->view.flags |= RRDR_RESULT_FLAG_ABSOLUTE;
3184 -
3185 - RRDR *r_tmp = r->group_by.r ? r->group_by.r : r;
3515 + last_r->view.flags |= RRDR_RESULT_FLAG_ABSOLUTE;
3516
3517 // -------------------------------------------------------------------------
3518 // assign the processor functions
@@ -3271,7 +3601,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3601 r->view.before = r_tmp->view.before;
3602 r->rows = r_tmp->rows;
3603
3274 - rrd2rrdr_group_by_add_metric(r, d);
3604 + rrd2rrdr_group_by_add_metric(r, qm->grouped_as.first_slot, r_tmp, dim_in_rrdr_tmp,
3605 + qt->request.group_by[0].aggregation, &qm->query_points, 0);
3606 }
3607
3608 rrd2rrdr_query_ops_release(ops[d]); // reuse this ops allocation
@@ -3378,7 +3709,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3709 // free all resources used by the grouping method
3710 r_tmp->time_grouping.free(r_tmp);
3711
3381 - rrd2rrdr_group_by_finalize(r);
3712 + // get the final RRDR to send to the caller
3713 + r = rrd2rrdr_group_by_finalize(r_tmp);
3714
3715 #ifdef NETDATA_INTERNAL_CHECKS
3716 if (dimensions_used && !(r->view.flags & RRDR_RESULT_FLAG_CANCEL)) {
web/api/queries/query.h
+18 -6
@@ -17,7 +17,7 @@ typedef enum rrdr_time_grouping {
17 RRDR_GROUPING_TRIMMED_MEAN1,
18 RRDR_GROUPING_TRIMMED_MEAN2,
19 RRDR_GROUPING_TRIMMED_MEAN3,
20 - RRDR_GROUPING_TRIMMED_MEAN5,
20 + RRDR_GROUPING_TRIMMED_MEAN,
21 RRDR_GROUPING_TRIMMED_MEAN10,
22 RRDR_GROUPING_TRIMMED_MEAN15,
23 RRDR_GROUPING_TRIMMED_MEAN20,
@@ -36,7 +36,7 @@ typedef enum rrdr_time_grouping {
36 RRDR_GROUPING_PERCENTILE75,
37 RRDR_GROUPING_PERCENTILE80,
38 RRDR_GROUPING_PERCENTILE90,
39 - RRDR_GROUPING_PERCENTILE95,
39 + RRDR_GROUPING_PERCENTILE,
40 RRDR_GROUPING_PERCENTILE97,
41 RRDR_GROUPING_PERCENTILE98,
42 RRDR_GROUPING_PERCENTILE99,
@@ -56,20 +56,32 @@ typedef enum rrdr_group_by {
56 RRDR_GROUP_BY_NONE = 0,
57 RRDR_GROUP_BY_SELECTED = (1 << 0),
58 RRDR_GROUP_BY_DIMENSION = (1 << 1),
59 - RRDR_GROUP_BY_NODE = (1 << 2),
60 - RRDR_GROUP_BY_INSTANCE = (1 << 3),
61 - RRDR_GROUP_BY_LABEL = (1 << 4),
59 + RRDR_GROUP_BY_INSTANCE = (1 << 2),
60 + RRDR_GROUP_BY_LABEL = (1 << 3),
61 + RRDR_GROUP_BY_NODE = (1 << 4),
62 RRDR_GROUP_BY_CONTEXT = (1 << 5),
63 RRDR_GROUP_BY_UNITS = (1 << 6),
64 + RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE = (1 << 7),
65 } RRDR_GROUP_BY;
66
67 +#define SUPPORTED_GROUP_BY_METHODS (\
68 + RRDR_GROUP_BY_SELECTED |\
69 + RRDR_GROUP_BY_DIMENSION |\
70 + RRDR_GROUP_BY_INSTANCE |\
71 + RRDR_GROUP_BY_LABEL |\
72 + RRDR_GROUP_BY_NODE |\
73 + RRDR_GROUP_BY_CONTEXT |\
74 + RRDR_GROUP_BY_UNITS |\
75 + RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE \
76 +)
77 +
78 struct web_buffer;
79
80 RRDR_GROUP_BY group_by_parse(char *s);
81 void buffer_json_group_by_to_array(struct web_buffer *wb, RRDR_GROUP_BY group_by);
82
83 typedef enum rrdr_group_by_function {
72 - RRDR_GROUP_BY_FUNCTION_AVERAGE,
84 + RRDR_GROUP_BY_FUNCTION_AVERAGE = 0,
85 RRDR_GROUP_BY_FUNCTION_MIN,
86 RRDR_GROUP_BY_FUNCTION_MAX,
87 RRDR_GROUP_BY_FUNCTION_SUM,
web/api/queries/rrdr.c
+3 -1
@@ -67,10 +67,11 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
67 string_freez(r->du[d]);
68 }
69
70 - query_target_release(r->internal.qt);
70 + query_target_release(r->internal.release_with_rrdr_qt);
71
72 onewayalloc_freez(owa, r->t);
73 onewayalloc_freez(owa, r->v);
74 + onewayalloc_freez(owa, r->vh);
75 onewayalloc_freez(owa, r->o);
76 onewayalloc_freez(owa, r->od);
77 onewayalloc_freez(owa, r->di);
@@ -82,6 +83,7 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
83 onewayalloc_freez(owa, r->ar);
84 onewayalloc_freez(owa, r->gbc);
85 onewayalloc_freez(owa, r->dgbc);
86 + onewayalloc_freez(owa, r->dgbs);
87
88 if(r->dl) {
89 for(size_t d = 0; d < r->d ;d++)
web/api/queries/rrdr.h
+6 -11
@@ -82,16 +82,6 @@ typedef enum __attribute__ ((__packed__)) rrdr_result_flags {
82 RRDR_RESULT_FLAG_CANCEL = (1 << 2), // the query needs to be cancelled
83 } RRDR_RESULT_FLAGS;
84
85 -struct rrdr_group_by_entry {
86 - size_t priority;
87 - size_t count;
88 - STRING *id;
89 - STRING *name;
90 - STRING *units;
91 - RRDR_DIMENSION_FLAGS od;
92 - DICTIONARY *dl;
93 -};
94 -
85 #define RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER 1000.0
86
87 typedef struct rrdresult {
@@ -104,11 +94,13 @@ typedef struct rrdresult {
94 STRING **di; // array of d dimension ids
95 STRING **dn; // array of d dimension names
96 STRING **du; // array of d dimension units
107 - uint32_t *dgbc; // array of d dimension units - NOT ALLOCATED when RRDR is created
97 + uint32_t *dgbs; // array of d dimension group by slots - NOT ALLOCATED when RRDR is created
98 + uint32_t *dgbc; // array of d dimension group by counts - NOT ALLOCATED when RRDR is created
99 uint32_t *dp; // array of d dimension priority - NOT ALLOCATED when RRDR is created
100 DICTIONARY **dl; // array of d dimension labels - NOT ALLOCATED when RRDR is created
101 STORAGE_POINT *dqp; // array of d dimensions query points - NOT ALLOCATED when RRDR is created
102 STORAGE_POINT *dview; // array of d dimensions group by view - NOT ALLOCATED when RRDR is created
103 + NETDATA_DOUBLE *vh; // array of n x d hidden values, while grouping - NOT ALLOCATED when RRDR is created
104
105 DICTIONARY *label_keys;
106
@@ -137,6 +129,7 @@ typedef struct rrdresult {
129 void *data; // the internal data of the grouping function
130
131 // grouping function pointers
132 + RRDR_TIME_GROUPING add_flush;
133 void (*create)(struct rrdresult *r, const char *options);
134 void (*reset)(struct rrdresult *r);
135 void (*free)(struct rrdresult *r);
@@ -169,6 +162,8 @@ typedef struct rrdresult {
162 #ifdef NETDATA_INTERNAL_CHECKS
163 const char *log;
164 #endif
165 +
166 + struct query_target *release_with_rrdr_qt;
167 } internal;
168 } RRDR;
169
web/api/queries/ses/ses.c
-82
@@ -6,85 +6,3 @@
6 // ----------------------------------------------------------------------------
7 // single exponential smoothing
8
9 -struct grouping_ses {
10 - NETDATA_DOUBLE alpha;
11 - NETDATA_DOUBLE alpha_other;
12 - NETDATA_DOUBLE level;
13 - size_t count;
14 -};
15 -
16 -static size_t max_window_size = 15;
17 -
18 -void grouping_init_ses(void) {
19 - long long ret = config_get_number(CONFIG_SECTION_WEB, "ses max window", (long long)max_window_size);
20 - if(ret <= 1) {
21 - config_set_number(CONFIG_SECTION_WEB, "ses max window", (long long)max_window_size);
22 - }
23 - else {
24 - max_window_size = (size_t) ret;
25 - }
26 -}
27 -
28 -static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_ses *g) {
29 - (void)g;
30 -
31 - NETDATA_DOUBLE points;
32 - if(r->view.group == 1) {
33 - // provide a running DES
34 - points = (NETDATA_DOUBLE)r->time_grouping.points_wanted;
35 - }
36 - else {
37 - // provide a SES with flush points
38 - points = (NETDATA_DOUBLE)r->view.group;
39 - }
40 -
41 - return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points;
42 -}
43 -
44 -static inline void set_alpha(RRDR *r, struct grouping_ses *g) {
45 - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
46 - // A commonly used value for alpha is 2 / (N + 1)
47 - g->alpha = 2.0 / (window(r, g) + 1.0);
48 - g->alpha_other = 1.0 - g->alpha;
49 -}
50 -
51 -void grouping_create_ses(RRDR *r, const char *options __maybe_unused) {
52 - struct grouping_ses *g = (struct grouping_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_ses));
53 - set_alpha(r, g);
54 - g->level = 0.0;
55 - r->time_grouping.data = g;
56 -}
57 -
58 -// resets when switches dimensions
59 -// so, clear everything to restart
60 -void grouping_reset_ses(RRDR *r) {
61 - struct grouping_ses *g = (struct grouping_ses *)r->time_grouping.data;
62 - g->level = 0.0;
63 - g->count = 0;
64 -}
65 -
66 -void grouping_free_ses(RRDR *r) {
67 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
68 - r->time_grouping.data = NULL;
69 -}
70 -
71 -void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value) {
72 - struct grouping_ses *g = (struct grouping_ses *)r->time_grouping.data;
73 -
74 - if(unlikely(!g->count))
75 - g->level = value;
76 -
77 - g->level = g->alpha * value + g->alpha_other * g->level;
78 - g->count++;
79 -}
80 -
81 -NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
82 - struct grouping_ses *g = (struct grouping_ses *)r->time_grouping.data;
83 -
84 - if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
85 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
86 - return 0.0;
87 - }
88 -
89 - return g->level;
90 -}
web/api/queries/ses/ses.h
+81 -6
@@ -6,12 +6,87 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_init_ses(void);
9 +struct tg_ses {
10 + NETDATA_DOUBLE alpha;
11 + NETDATA_DOUBLE alpha_other;
12 + NETDATA_DOUBLE level;
13 + size_t count;
14 +};
15
11 -void grouping_create_ses(RRDR *r, const char *options __maybe_unused);
12 -void grouping_reset_ses(RRDR *r);
13 -void grouping_free_ses(RRDR *r);
14 -void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value);
15 -NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
16 +static size_t tg_ses_max_window_size = 15;
17 +
18 +static inline void tg_ses_init(void) {
19 + long long ret = config_get_number(CONFIG_SECTION_WEB, "ses max tg_des_window", (long long)tg_ses_max_window_size);
20 + if(ret <= 1) {
21 + config_set_number(CONFIG_SECTION_WEB, "ses max tg_des_window", (long long)tg_ses_max_window_size);
22 + }
23 + else {
24 + tg_ses_max_window_size = (size_t) ret;
25 + }
26 +}
27 +
28 +static inline NETDATA_DOUBLE tg_ses_window(RRDR *r, struct tg_ses *g) {
29 + (void)g;
30 +
31 + NETDATA_DOUBLE points;
32 + if(r->view.group == 1) {
33 + // provide a running DES
34 + points = (NETDATA_DOUBLE)r->time_grouping.points_wanted;
35 + }
36 + else {
37 + // provide a SES with flush points
38 + points = (NETDATA_DOUBLE)r->view.group;
39 + }
40 +
41 + return (points > (NETDATA_DOUBLE)tg_ses_max_window_size) ? (NETDATA_DOUBLE)tg_ses_max_window_size : points;
42 +}
43 +
44 +static inline void tg_ses_set_alpha(RRDR *r, struct tg_ses *g) {
45 + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
46 + // A commonly used value for alpha is 2 / (N + 1)
47 + g->alpha = 2.0 / (tg_ses_window(r, g) + 1.0);
48 + g->alpha_other = 1.0 - g->alpha;
49 +}
50 +
51 +static inline void tg_ses_create(RRDR *r, const char *options __maybe_unused) {
52 + struct tg_ses *g = (struct tg_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_ses));
53 + tg_ses_set_alpha(r, g);
54 + g->level = 0.0;
55 + r->time_grouping.data = g;
56 +}
57 +
58 +// resets when switches dimensions
59 +// so, clear everything to restart
60 +static inline void tg_ses_reset(RRDR *r) {
61 + struct tg_ses *g = (struct tg_ses *)r->time_grouping.data;
62 + g->level = 0.0;
63 + g->count = 0;
64 +}
65 +
66 +static inline void tg_ses_free(RRDR *r) {
67 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
68 + r->time_grouping.data = NULL;
69 +}
70 +
71 +static inline void tg_ses_add(RRDR *r, NETDATA_DOUBLE value) {
72 + struct tg_ses *g = (struct tg_ses *)r->time_grouping.data;
73 +
74 + if(unlikely(!g->count))
75 + g->level = value;
76 +
77 + g->level = g->alpha * value + g->alpha_other * g->level;
78 + g->count++;
79 +}
80 +
81 +static inline NETDATA_DOUBLE tg_ses_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
82 + struct tg_ses *g = (struct tg_ses *)r->time_grouping.data;
83 +
84 + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
85 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
86 + return 0.0;
87 + }
88 +
89 + return g->level;
90 +}
91
92 #endif //NETDATA_API_QUERIES_SES_H
web/api/queries/stddev/stddev.c
-112
@@ -6,118 +6,6 @@
6 // ----------------------------------------------------------------------------
7 // stddev
8
9 -// this implementation comes from:
10 -// https://www.johndcook.com/blog/standard_deviation/
11 -
12 -struct grouping_stddev {
13 - long count;
14 - NETDATA_DOUBLE m_oldM, m_newM, m_oldS, m_newS;
15 -};
16 -
17 -void grouping_create_stddev(RRDR *r, const char *options __maybe_unused) {
18 - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_stddev));
19 -}
20 -
21 -// resets when switches dimensions
22 -// so, clear everything to restart
23 -void grouping_reset_stddev(RRDR *r) {
24 - struct grouping_stddev *g = (struct grouping_stddev *)r->time_grouping.data;
25 - g->count = 0;
26 -}
27 -
28 -void grouping_free_stddev(RRDR *r) {
29 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
30 - r->time_grouping.data = NULL;
31 -}
32 -
33 -void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value) {
34 - struct grouping_stddev *g = (struct grouping_stddev *)r->time_grouping.data;
35 -
36 - g->count++;
37 -
38 - // See Knuth TAOCP vol 2, 3rd edition, page 232
39 - if (g->count == 1) {
40 - g->m_oldM = g->m_newM = value;
41 - g->m_oldS = 0.0;
42 - }
43 - else {
44 - g->m_newM = g->m_oldM + (value - g->m_oldM) / g->count;
45 - g->m_newS = g->m_oldS + (value - g->m_oldM) * (value - g->m_newM);
46 -
47 - // set up for next iteration
48 - g->m_oldM = g->m_newM;
49 - g->m_oldS = g->m_newS;
50 - }
51 -}
52 -
53 -static inline NETDATA_DOUBLE mean(struct grouping_stddev *g) {
54 - return (g->count > 0) ? g->m_newM : 0.0;
55 -}
56 -
57 -static inline NETDATA_DOUBLE variance(struct grouping_stddev *g) {
58 - return ( (g->count > 1) ? g->m_newS/(NETDATA_DOUBLE)(g->count - 1) : 0.0 );
59 -}
60 -static inline NETDATA_DOUBLE stddev(struct grouping_stddev *g) {
61 - return sqrtndd(variance(g));
62 -}
63 -
64 -NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
65 - struct grouping_stddev *g = (struct grouping_stddev *)r->time_grouping.data;
66 -
67 - NETDATA_DOUBLE value;
68 -
69 - if(likely(g->count > 1)) {
70 - value = stddev(g);
71 -
72 - if(!netdata_double_isnumber(value)) {
73 - value = 0.0;
74 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
75 - }
76 - }
77 - else if(g->count == 1) {
78 - value = 0.0;
79 - }
80 - else {
81 - value = 0.0;
82 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
83 - }
84 -
85 - grouping_reset_stddev(r);
86 -
87 - return value;
88 -}
89 -
90 -// https://en.wikipedia.org/wiki/Coefficient_of_variation
91 -NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
92 - struct grouping_stddev *g = (struct grouping_stddev *)r->time_grouping.data;
93 -
94 - NETDATA_DOUBLE value;
95 -
96 - if(likely(g->count > 1)) {
97 - NETDATA_DOUBLE m = mean(g);
98 - value = 100.0 * stddev(g) / ((m < 0)? -m : m);
99 -
100 - if(unlikely(!netdata_double_isnumber(value))) {
101 - value = 0.0;
102 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
103 - }
104 - }
105 - else if(g->count == 1) {
106 - // one value collected
107 - value = 0.0;
108 - }
109 - else {
110 - // no values collected
111 - value = 0.0;
112 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
113 - }
114 -
115 - grouping_reset_stddev(r);
116 -
117 - return value;
118 -}
119 -
120 -
9 /*
10 * Mean = average
11 *
web/api/queries/stddev/stddev.h
+110 -8
@@ -6,13 +6,115 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_stddev(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_stddev(RRDR *r);
11 -void grouping_free_stddev(RRDR *r);
12 -void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
14 -NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
15 -// NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
16 -// NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +// this implementation comes from:
10 +// https://www.johndcook.com/blog/standard_deviation/
11 +
12 +struct tg_stddev {
13 + long count;
14 + NETDATA_DOUBLE m_oldM, m_newM, m_oldS, m_newS;
15 +};
16 +
17 +static inline void tg_stddev_create(RRDR *r, const char *options __maybe_unused) {
18 + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_stddev));
19 +}
20 +
21 +// resets when switches dimensions
22 +// so, clear everything to restart
23 +static inline void tg_stddev_reset(RRDR *r) {
24 + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data;
25 + g->count = 0;
26 +}
27 +
28 +static inline void tg_stddev_free(RRDR *r) {
29 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
30 + r->time_grouping.data = NULL;
31 +}
32 +
33 +static inline void tg_stddev_add(RRDR *r, NETDATA_DOUBLE value) {
34 + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data;
35 +
36 + g->count++;
37 +
38 + // See Knuth TAOCP vol 2, 3rd edition, page 232
39 + if (g->count == 1) {
40 + g->m_oldM = g->m_newM = value;
41 + g->m_oldS = 0.0;
42 + }
43 + else {
44 + g->m_newM = g->m_oldM + (value - g->m_oldM) / g->count;
45 + g->m_newS = g->m_oldS + (value - g->m_oldM) * (value - g->m_newM);
46 +
47 + // set up for next iteration
48 + g->m_oldM = g->m_newM;
49 + g->m_oldS = g->m_newS;
50 + }
51 +}
52 +
53 +static inline NETDATA_DOUBLE tg_stddev_mean(struct tg_stddev *g) {
54 + return (g->count > 0) ? g->m_newM : 0.0;
55 +}
56 +
57 +static inline NETDATA_DOUBLE tg_stddev_variance(struct tg_stddev *g) {
58 + return ( (g->count > 1) ? g->m_newS/(NETDATA_DOUBLE)(g->count - 1) : 0.0 );
59 +}
60 +static inline NETDATA_DOUBLE tg_stddev_stddev(struct tg_stddev *g) {
61 + return sqrtndd(tg_stddev_variance(g));
62 +}
63 +
64 +static inline NETDATA_DOUBLE tg_stddev_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
65 + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data;
66 +
67 + NETDATA_DOUBLE value;
68 +
69 + if(likely(g->count > 1)) {
70 + value = tg_stddev_stddev(g);
71 +
72 + if(!netdata_double_isnumber(value)) {
73 + value = 0.0;
74 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
75 + }
76 + }
77 + else if(g->count == 1) {
78 + value = 0.0;
79 + }
80 + else {
81 + value = 0.0;
82 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
83 + }
84 +
85 + tg_stddev_reset(r);
86 +
87 + return value;
88 +}
89 +
90 +// https://en.wikipedia.org/wiki/Coefficient_of_variation
91 +static inline NETDATA_DOUBLE tg_stddev_coefficient_of_variation_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
92 + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data;
93 +
94 + NETDATA_DOUBLE value;
95 +
96 + if(likely(g->count > 1)) {
97 + NETDATA_DOUBLE m = tg_stddev_mean(g);
98 + value = 100.0 * tg_stddev_stddev(g) / ((m < 0)? -m : m);
99 +
100 + if(unlikely(!netdata_double_isnumber(value))) {
101 + value = 0.0;
102 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
103 + }
104 + }
105 + else if(g->count == 1) {
106 + // one value collected
107 + value = 0.0;
108 + }
109 + else {
110 + // no values collected
111 + value = 0.0;
112 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
113 + }
114 +
115 + tg_stddev_reset(r);
116 +
117 + return value;
118 +}
119
120 #endif //NETDATA_API_QUERIES_STDDEV_H
web/api/queries/sum/sum.c
-46
@@ -5,51 +5,5 @@
5 // ----------------------------------------------------------------------------
6 // sum
7
8 -struct grouping_sum {
9 - NETDATA_DOUBLE sum;
10 - size_t count;
11 -};
12 -
13 -void grouping_create_sum(RRDR *r, const char *options __maybe_unused) {
14 - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_sum));
15 -}
16 -
17 -// resets when switches dimensions
18 -// so, clear everything to restart
19 -void grouping_reset_sum(RRDR *r) {
20 - struct grouping_sum *g = (struct grouping_sum *)r->time_grouping.data;
21 - g->sum = 0;
22 - g->count = 0;
23 -}
24 -
25 -void grouping_free_sum(RRDR *r) {
26 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
27 - r->time_grouping.data = NULL;
28 -}
29 -
30 -void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value) {
31 - struct grouping_sum *g = (struct grouping_sum *)r->time_grouping.data;
32 - g->sum += value;
33 - g->count++;
34 -}
35 -
36 -NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
37 - struct grouping_sum *g = (struct grouping_sum *)r->time_grouping.data;
38 -
39 - NETDATA_DOUBLE value;
40 -
41 - if(unlikely(!g->count)) {
42 - value = 0.0;
43 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
44 - }
45 - else {
46 - value = g->sum;
47 - }
48 -
49 - g->sum = 0.0;
50 - g->count = 0;
51 -
52 - return value;
53 -}
8
9
web/api/queries/sum/sum.h
+46 -5
@@ -6,10 +6,51 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_sum(RRDR *r, const char *options __maybe_unused);
10 -void grouping_reset_sum(RRDR *r);
11 -void grouping_free_sum(RRDR *r);
12 -void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value);
13 -NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_sum {
10 + NETDATA_DOUBLE sum;
11 + size_t count;
12 +};
13 +
14 +static inline void tg_sum_create(RRDR *r, const char *options __maybe_unused) {
15 + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_sum));
16 +}
17 +
18 +// resets when switches dimensions
19 +// so, clear everything to restart
20 +static inline void tg_sum_reset(RRDR *r) {
21 + struct tg_sum *g = (struct tg_sum *)r->time_grouping.data;
22 + g->sum = 0;
23 + g->count = 0;
24 +}
25 +
26 +static inline void tg_sum_free(RRDR *r) {
27 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
28 + r->time_grouping.data = NULL;
29 +}
30 +
31 +static inline void tg_sum_add(RRDR *r, NETDATA_DOUBLE value) {
32 + struct tg_sum *g = (struct tg_sum *)r->time_grouping.data;
33 + g->sum += value;
34 + g->count++;
35 +}
36 +
37 +static inline NETDATA_DOUBLE tg_sum_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
38 + struct tg_sum *g = (struct tg_sum *)r->time_grouping.data;
39 +
40 + NETDATA_DOUBLE value;
41 +
42 + if(unlikely(!g->count)) {
43 + value = 0.0;
44 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
45 + }
46 + else {
47 + value = g->sum;
48 + }
49 +
50 + g->sum = 0.0;
51 + g->count = 0;
52 +
53 + return value;
54 +}
55
56 #endif //NETDATA_API_QUERY_SUM_H
web/api/queries/trimmed_mean/trimmed_mean.c
-159
@@ -5,162 +5,3 @@
5 // ----------------------------------------------------------------------------
6 // median
7
8 -struct grouping_trimmed_mean {
9 - size_t series_size;
10 - size_t next_pos;
11 - NETDATA_DOUBLE percent;
12 -
13 - NETDATA_DOUBLE *series;
14 -};
15 -
16 -static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
17 - long entries = r->view.group;
18 - if(entries < 10) entries = 10;
19 -
20 - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_trimmed_mean));
21 - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
22 - g->series_size = (size_t)entries;
23 -
24 - g->percent = def;
25 - if(options && *options) {
26 - g->percent = str2ndd(options, NULL);
27 - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
28 - if(g->percent < 0.0) g->percent = 0.0;
29 - if(g->percent > 50.0) g->percent = 50.0;
30 - }
31 -
32 - g->percent = 1.0 - ((g->percent / 100.0) * 2.0);
33 - r->time_grouping.data = g;
34 -}
35 -
36 -void grouping_create_trimmed_mean1(RRDR *r, const char *options) {
37 - grouping_create_trimmed_mean_internal(r, options, 1.0);
38 -}
39 -void grouping_create_trimmed_mean2(RRDR *r, const char *options) {
40 - grouping_create_trimmed_mean_internal(r, options, 2.0);
41 -}
42 -void grouping_create_trimmed_mean3(RRDR *r, const char *options) {
43 - grouping_create_trimmed_mean_internal(r, options, 3.0);
44 -}
45 -void grouping_create_trimmed_mean5(RRDR *r, const char *options) {
46 - grouping_create_trimmed_mean_internal(r, options, 5.0);
47 -}
48 -void grouping_create_trimmed_mean10(RRDR *r, const char *options) {
49 - grouping_create_trimmed_mean_internal(r, options, 10.0);
50 -}
51 -void grouping_create_trimmed_mean15(RRDR *r, const char *options) {
52 - grouping_create_trimmed_mean_internal(r, options, 15.0);
53 -}
54 -void grouping_create_trimmed_mean20(RRDR *r, const char *options) {
55 - grouping_create_trimmed_mean_internal(r, options, 20.0);
56 -}
57 -void grouping_create_trimmed_mean25(RRDR *r, const char *options) {
58 - grouping_create_trimmed_mean_internal(r, options, 25.0);
59 -}
60 -
61 -// resets when switches dimensions
62 -// so, clear everything to restart
63 -void grouping_reset_trimmed_mean(RRDR *r) {
64 - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->time_grouping.data;
65 - g->next_pos = 0;
66 -}
67 -
68 -void grouping_free_trimmed_mean(RRDR *r) {
69 - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->time_grouping.data;
70 - if(g) onewayalloc_freez(r->internal.owa, g->series);
71 -
72 - onewayalloc_freez(r->internal.owa, r->time_grouping.data);
73 - r->time_grouping.data = NULL;
74 -}
75 -
76 -void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) {
77 - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->time_grouping.data;
78 -
79 - if(unlikely(g->next_pos >= g->series_size)) {
80 - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
81 - g->series_size *= 2;
82 - }
83 -
84 - g->series[g->next_pos++] = value;
85 -}
86 -
87 -NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
88 - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->time_grouping.data;
89 -
90 - NETDATA_DOUBLE value;
91 - size_t available_slots = g->next_pos;
92 -
93 - if(unlikely(!available_slots)) {
94 - value = 0.0;
95 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
96 - }
97 - else if(available_slots == 1) {
98 - value = g->series[0];
99 - }
100 - else {
101 - sort_series(g->series, available_slots);
102 -
103 - NETDATA_DOUBLE min = g->series[0];
104 - NETDATA_DOUBLE max = g->series[available_slots - 1];
105 -
106 - if (min != max) {
107 - size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
108 - if(!slots_to_use) slots_to_use = 1;
109 -
110 - NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
111 - NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
112 -
113 - NETDATA_DOUBLE percent_interpolation_slot = 0.0;
114 - NETDATA_DOUBLE percent_last_slot = 0.0;
115 - if(percent_delta > 0.0) {
116 - NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
117 - NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
118 -
119 - percent_interpolation_slot = percent_delta / percent_1slot;
120 - percent_last_slot = 1 - percent_interpolation_slot;
121 - }
122 -
123 - int start_slot, stop_slot, step, last_slot, interpolation_slot;
124 - if(min >= 0.0 && max >= 0.0) {
125 - start_slot = (int)((available_slots - slots_to_use) / 2);
126 - stop_slot = start_slot + (int)slots_to_use;
127 - last_slot = stop_slot - 1;
128 - interpolation_slot = stop_slot;
129 - step = 1;
130 - }
131 - else {
132 - start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2);
133 - stop_slot = start_slot - (int)slots_to_use;
134 - last_slot = stop_slot + 1;
135 - interpolation_slot = stop_slot;
136 - step = -1;
137 - }
138 -
139 - value = 0.0;
140 - for(int slot = start_slot; slot != stop_slot ; slot += step)
141 - value += g->series[slot];
142 -
143 - size_t counted = slots_to_use;
144 - if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
145 - value += g->series[interpolation_slot] * percent_interpolation_slot;
146 - value += g->series[last_slot] * percent_last_slot;
147 - counted++;
148 - }
149 -
150 - value = value / (NETDATA_DOUBLE)counted;
151 - }
152 - else
153 - value = min;
154 - }
155 -
156 - if(unlikely(!netdata_double_isnumber(value))) {
157 - value = 0.0;
158 - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
159 - }
160 -
161 - //log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean");
162 -
163 - g->next_pos = 0;
164 -
165 - return value;
166 -}
web/api/queries/trimmed_mean/trimmed_mean.h
+159 -12
@@ -6,17 +6,164 @@
6 #include "../query.h"
7 #include "../rrdr.h"
8
9 -void grouping_create_trimmed_mean1(RRDR *r, const char *options);
10 -void grouping_create_trimmed_mean2(RRDR *r, const char *options);
11 -void grouping_create_trimmed_mean3(RRDR *r, const char *options);
12 -void grouping_create_trimmed_mean5(RRDR *r, const char *options);
13 -void grouping_create_trimmed_mean10(RRDR *r, const char *options);
14 -void grouping_create_trimmed_mean15(RRDR *r, const char *options);
15 -void grouping_create_trimmed_mean20(RRDR *r, const char *options);
16 -void grouping_create_trimmed_mean25(RRDR *r, const char *options);
17 -void grouping_reset_trimmed_mean(RRDR *r);
18 -void grouping_free_trimmed_mean(RRDR *r);
19 -void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
20 -NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
9 +struct tg_trimmed_mean {
10 + size_t series_size;
11 + size_t next_pos;
12 + NETDATA_DOUBLE percent;
13 +
14 + NETDATA_DOUBLE *series;
15 +};
16 +
17 +static inline void tg_trimmed_mean_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
18 + long entries = r->view.group;
19 + if(entries < 10) entries = 10;
20 +
21 + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_trimmed_mean));
22 + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
23 + g->series_size = (size_t)entries;
24 +
25 + g->percent = def;
26 + if(options && *options) {
27 + g->percent = str2ndd(options, NULL);
28 + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
29 + if(g->percent < 0.0) g->percent = 0.0;
30 + if(g->percent > 50.0) g->percent = 50.0;
31 + }
32 +
33 + g->percent = 1.0 - ((g->percent / 100.0) * 2.0);
34 + r->time_grouping.data = g;
35 +}
36 +
37 +static inline void tg_trimmed_mean_create_1(RRDR *r, const char *options) {
38 + tg_trimmed_mean_create_internal(r, options, 1.0);
39 +}
40 +static inline void tg_trimmed_mean_create_2(RRDR *r, const char *options) {
41 + tg_trimmed_mean_create_internal(r, options, 2.0);
42 +}
43 +static inline void tg_trimmed_mean_create_3(RRDR *r, const char *options) {
44 + tg_trimmed_mean_create_internal(r, options, 3.0);
45 +}
46 +static inline void tg_trimmed_mean_create_5(RRDR *r, const char *options) {
47 + tg_trimmed_mean_create_internal(r, options, 5.0);
48 +}
49 +static inline void tg_trimmed_mean_create_10(RRDR *r, const char *options) {
50 + tg_trimmed_mean_create_internal(r, options, 10.0);
51 +}
52 +static inline void tg_trimmed_mean_create_15(RRDR *r, const char *options) {
53 + tg_trimmed_mean_create_internal(r, options, 15.0);
54 +}
55 +static inline void tg_trimmed_mean_create_20(RRDR *r, const char *options) {
56 + tg_trimmed_mean_create_internal(r, options, 20.0);
57 +}
58 +static inline void tg_trimmed_mean_create_25(RRDR *r, const char *options) {
59 + tg_trimmed_mean_create_internal(r, options, 25.0);
60 +}
61 +
62 +// resets when switches dimensions
63 +// so, clear everything to restart
64 +static inline void tg_trimmed_mean_reset(RRDR *r) {
65 + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data;
66 + g->next_pos = 0;
67 +}
68 +
69 +static inline void tg_trimmed_mean_free(RRDR *r) {
70 + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data;
71 + if(g) onewayalloc_freez(r->internal.owa, g->series);
72 +
73 + onewayalloc_freez(r->internal.owa, r->time_grouping.data);
74 + r->time_grouping.data = NULL;
75 +}
76 +
77 +static inline void tg_trimmed_mean_add(RRDR *r, NETDATA_DOUBLE value) {
78 + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data;
79 +
80 + if(unlikely(g->next_pos >= g->series_size)) {
81 + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
82 + g->series_size *= 2;
83 + }
84 +
85 + g->series[g->next_pos++] = value;
86 +}
87 +
88 +static inline NETDATA_DOUBLE tg_trimmed_mean_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
89 + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data;
90 +
91 + NETDATA_DOUBLE value;
92 + size_t available_slots = g->next_pos;
93 +
94 + if(unlikely(!available_slots)) {
95 + value = 0.0;
96 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
97 + }
98 + else if(available_slots == 1) {
99 + value = g->series[0];
100 + }
101 + else {
102 + sort_series(g->series, available_slots);
103 +
104 + NETDATA_DOUBLE min = g->series[0];
105 + NETDATA_DOUBLE max = g->series[available_slots - 1];
106 +
107 + if (min != max) {
108 + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
109 + if(!slots_to_use) slots_to_use = 1;
110 +
111 + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
112 + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
113 +
114 + NETDATA_DOUBLE percent_interpolation_slot = 0.0;
115 + NETDATA_DOUBLE percent_last_slot = 0.0;
116 + if(percent_delta > 0.0) {
117 + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
118 + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
119 +
120 + percent_interpolation_slot = percent_delta / percent_1slot;
121 + percent_last_slot = 1 - percent_interpolation_slot;
122 + }
123 +
124 + int start_slot, stop_slot, step, last_slot, interpolation_slot;
125 + if(min >= 0.0 && max >= 0.0) {
126 + start_slot = (int)((available_slots - slots_to_use) / 2);
127 + stop_slot = start_slot + (int)slots_to_use;
128 + last_slot = stop_slot - 1;
129 + interpolation_slot = stop_slot;
130 + step = 1;
131 + }
132 + else {
133 + start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2);
134 + stop_slot = start_slot - (int)slots_to_use;
135 + last_slot = stop_slot + 1;
136 + interpolation_slot = stop_slot;
137 + step = -1;
138 + }
139 +
140 + value = 0.0;
141 + for(int slot = start_slot; slot != stop_slot ; slot += step)
142 + value += g->series[slot];
143 +
144 + size_t counted = slots_to_use;
145 + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
146 + value += g->series[interpolation_slot] * percent_interpolation_slot;
147 + value += g->series[last_slot] * percent_last_slot;
148 + counted++;
149 + }
150 +
151 + value = value / (NETDATA_DOUBLE)counted;
152 + }
153 + else
154 + value = min;
155 + }
156 +
157 + if(unlikely(!netdata_double_isnumber(value))) {
158 + value = 0.0;
159 + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
160 + }
161 +
162 + //log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean");
163 +
164 + g->next_pos = 0;
165 +
166 + return value;
167 +}
168
169 #endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H
web/api/queries/weights.c
+51 -46
@@ -518,21 +518,48 @@ static inline ssize_t dict_unique_id_name_add(DICTIONARY *dict, const char *id,
518
519 return (ssize_t)dun->i;
520 }
521 +struct query_weights_data {
522 + QUERY_WEIGHTS_REQUEST *qwr;
523 +
524 + SIMPLE_PATTERN *scope_nodes_sp;
525 + SIMPLE_PATTERN *scope_contexts_sp;
526 + SIMPLE_PATTERN *nodes_sp;
527 + SIMPLE_PATTERN *contexts_sp;
528 + SIMPLE_PATTERN *instances_sp;
529 + SIMPLE_PATTERN *dimensions_sp;
530 + SIMPLE_PATTERN *labels_sp;
531 + SIMPLE_PATTERN *alerts_sp;
532 +
533 + usec_t timeout_us;
534 + bool timed_out;
535 + bool interrupted;
536 +
537 + struct query_timings timings;
538 +
539 + size_t examined_dimensions;
540 + bool register_zero;
541 +
542 + DICTIONARY *results;
543 + WEIGHTS_STATS stats;
544 +
545 + uint32_t shifts;
546 +
547 + struct query_versions versions;
548 +};
549
550 static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *wb,
551 time_t after, time_t before,
552 time_t baseline_after, time_t baseline_before,
553 size_t points, WEIGHTS_METHOD method,
554 RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
527 - size_t examined_dimensions, usec_t duration,
555 + size_t examined_dimensions, struct query_weights_data *qwd,
556 WEIGHTS_STATS *stats,
557 struct query_versions *versions) {
558 buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
559 buffer_json_member_add_uint64(wb, "api", 2);
532 - buffer_json_agents_array_v2(wb, 0);
560
561 results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
535 - points, method, group, options, shifts, examined_dimensions, duration, stats);
562 + points, method, group, options, shifts, examined_dimensions, qwd->timings.executed_ut - qwd->timings.received_ut, stats);
563
564 version_hashes_api_v2(wb, versions);
565
@@ -706,6 +733,7 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *
733
734 buffer_json_object_close(wb); //dictionaries
735
736 + buffer_json_agents_array_v2(wb, &qwd->timings, 0);
737 buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions);
738 buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions);
739 buffer_json_finalize(wb);
@@ -921,7 +949,8 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
949 .priority = STORAGE_PRIORITY_SYNCHRONOUS,
950 };
951
924 - RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
952 + QUERY_TARGET *qt = query_target_create(&qtr);
953 + RRDR *r = rrd2rrdr(owa, qt);
954 if(!r)
955 goto cleanup;
956
@@ -962,6 +991,7 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
991
992 cleanup:
993 rrdr_free(owa, r);
994 + query_target_release(qt);
995 return ret;
996 }
997
@@ -1129,35 +1159,6 @@ static void rrdset_weights_value(
1159 register_result(results, host, rca, ria, rma, qv.value, 0, &qv.sp, NULL, stats, register_zero, qv.duration_ut);
1160 }
1161
1132 -struct query_weights_data {
1133 - QUERY_WEIGHTS_REQUEST *qwr;
1134 -
1135 - SIMPLE_PATTERN *scope_nodes_sp;
1136 - SIMPLE_PATTERN *scope_contexts_sp;
1137 - SIMPLE_PATTERN *nodes_sp;
1138 - SIMPLE_PATTERN *contexts_sp;
1139 - SIMPLE_PATTERN *instances_sp;
1140 - SIMPLE_PATTERN *dimensions_sp;
1141 - SIMPLE_PATTERN *labels_sp;
1142 - SIMPLE_PATTERN *alerts_sp;
1143 -
1144 - usec_t now_us;
1145 - usec_t started_us;
1146 - usec_t timeout_us;
1147 - bool timed_out;
1148 - bool interrupted;
1149 -
1150 - size_t examined_dimensions;
1151 - bool register_zero;
1152 -
1153 - DICTIONARY *results;
1154 - WEIGHTS_STATS stats;
1155 -
1156 - uint32_t shifts;
1157 -
1158 - struct query_versions versions;
1159 -};
1160 -
1162 static void rrdset_weights_multi_dimensional_value(struct query_weights_data *qwd) {
1163 QUERY_TARGET_REQUEST qtr = {
1164 .version = 1,
@@ -1182,9 +1183,10 @@ static void rrdset_weights_multi_dimensional_value(struct query_weights_data *qw
1183 };
1184
1185 ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
1185 - RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
1186 + QUERY_TARGET *qt = query_target_create(&qtr);
1187 + RRDR *r = rrd2rrdr(owa, qt);
1188
1187 - if(rrdr_rows(r) != 1 || !r->d || r->d != r->internal.qt->query.used)
1189 + if(!r || rrdr_rows(r) != 1 || !r->d || r->d != r->internal.qt->query.used)
1190 goto cleanup;
1191
1192 QUERY_VALUE qv = {
@@ -1225,6 +1227,7 @@ static void rrdset_weights_multi_dimensional_value(struct query_weights_data *qw
1227
1228 cleanup:
1229 rrdr_free(owa, r);
1230 + query_target_release(qt);
1231 onewayalloc_destroy(owa);
1232 }
1233
@@ -1319,12 +1322,6 @@ static ssize_t weights_for_rrdmetric(void *data, RRDHOST *host, RRDCONTEXT_ACQUI
1322 struct query_weights_data *qwd = data;
1323 QUERY_WEIGHTS_REQUEST *qwr = qwd->qwr;
1324
1322 - qwd->now_us = now_monotonic_usec();
1323 - if(qwd->now_us - qwd->started_us > qwd->timeout_us) {
1324 - qwd->timed_out = true;
1325 - return -1;
1326 - }
1327 -
1325 if(qwd->qwr->interrupt_callback && qwd->qwr->interrupt_callback(qwd->qwr->interrupt_callback_data)) {
1326 qwd->interrupted = true;
1327 return -1;
@@ -1378,6 +1375,12 @@ static ssize_t weights_for_rrdmetric(void *data, RRDHOST *host, RRDCONTEXT_ACQUI
1375 break;
1376 }
1377
1378 + qwd->timings.executed_ut = now_monotonic_usec();
1379 + if(qwd->timings.executed_ut - qwd->timings.received_ut > qwd->timeout_us) {
1380 + qwd->timed_out = true;
1381 + return -1;
1382 + }
1383 +
1384 return 1;
1385 }
1386
@@ -1456,13 +1459,15 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1459 .labels_sp = string_to_simple_pattern(qwr->labels),
1460 .alerts_sp = string_to_simple_pattern(qwr->alerts),
1461 .timeout_us = qwr->timeout_ms * USEC_PER_MS,
1459 - .started_us = now_monotonic_usec(),
1462 .timed_out = false,
1463 .examined_dimensions = 0,
1464 .register_zero = true,
1465 .results = register_result_init(),
1466 .stats = {},
1467 .shifts = 0,
1468 + .timings = {
1469 + .received_ut = now_monotonic_usec(),
1470 + }
1471 };
1472
1473 if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL))
@@ -1580,7 +1585,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1585 if(!(qwr->options & RRDR_OPTION_RETURN_RAW) && qwr->method != WEIGHTS_METHOD_VALUE)
1586 spread_results_evenly(qwd.results, &qwd.stats);
1587
1583 - usec_t ended_usec = now_monotonic_usec();
1588 + usec_t ended_usec = qwd.timings.executed_ut = now_monotonic_usec();
1589
1590 // generate the json output we need
1591 buffer_flush(wb);
@@ -1595,7 +1600,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1600 qwr->baseline_after, qwr->baseline_before,
1601 qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,
1602 qwd.examined_dimensions,
1598 - ended_usec - qwd.started_us, &qwd.stats);
1603 + ended_usec - qwd.timings.received_ut, &qwd.stats);
1604 break;
1605
1606 case WEIGHTS_FORMAT_CONTEXTS:
@@ -1606,7 +1611,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1611 qwr->baseline_after, qwr->baseline_before,
1612 qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,
1613 qwd.examined_dimensions,
1609 - ended_usec - qwd.started_us, &qwd.stats);
1614 + ended_usec - qwd.timings.received_ut, &qwd.stats);
1615 break;
1616
1617 default:
@@ -1618,7 +1623,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1623 qwr->baseline_after, qwr->baseline_before,
1624 qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts,
1625 qwd.examined_dimensions,
1621 - ended_usec - qwd.started_us, &qwd.stats, &qwd.versions);
1626 + &qwd, &qwd.stats, &qwd.versions);
1627 break;
1628 }
1629
web/api/web_api_v2.c
+66 -19
@@ -4,7 +4,6 @@
4
5 static int web_client_api_request_v2_contexts_internal(RRDHOST *host __maybe_unused, struct web_client *w, char *url, CONTEXTS_V2_OPTIONS options) {
6 struct api_v2_contexts_request req = { 0 };
7 - req.timings.received_ut = now_monotonic_usec();
7
8 while(url) {
9 char *value = mystrsep(&url, "&");
@@ -49,6 +48,21 @@ static int web_client_api_request_v2_weights(RRDHOST *host __maybe_unused, struc
48 WEIGHTS_FORMAT_MULTINODE, 2);
49 }
50
51 +#define GROUP_BY_KEY_MAX_LENGTH 30
52 +static struct {
53 + char group_by[GROUP_BY_KEY_MAX_LENGTH + 1];
54 + char aggregation[GROUP_BY_KEY_MAX_LENGTH + 1];
55 + char group_by_label[GROUP_BY_KEY_MAX_LENGTH + 1];
56 +} group_by_keys[MAX_QUERY_GROUP_BY_PASSES];
57 +
58 +__attribute__((constructor)) void initialize_group_by_keys(void) {
59 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
60 + snprintfz(group_by_keys[g].group_by, GROUP_BY_KEY_MAX_LENGTH, "group_by[%zu]", g);
61 + snprintfz(group_by_keys[g].aggregation, GROUP_BY_KEY_MAX_LENGTH, "aggregation[%zu]", g);
62 + snprintfz(group_by_keys[g].group_by_label, GROUP_BY_KEY_MAX_LENGTH, "group_by_label[%zu]", g);
63 + }
64 +}
65 +
66 static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
67 usec_t received_ut = now_monotonic_usec();
68
@@ -80,14 +94,21 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
94 char *alerts = NULL;
95 char *time_group_options = NULL;
96 char *tier_str = NULL;
83 - char *group_by_label = NULL;
97 size_t tier = 0;
98 RRDR_TIME_GROUPING time_group = RRDR_GROUPING_AVERAGE;
86 - RRDR_GROUP_BY group_by = RRDR_GROUP_BY_DIMENSION;
87 - RRDR_GROUP_BY_FUNCTION group_by_aggregate = RRDR_GROUP_BY_FUNCTION_AVERAGE;
99 DATASOURCE_FORMAT format = DATASOURCE_JSON2;
100 RRDR_OPTIONS options = RRDR_OPTION_VIRTUAL_POINTS | RRDR_OPTION_JSON_WRAP | RRDR_OPTION_RETURN_JWAR;
101
102 + struct group_by_pass group_by[MAX_QUERY_GROUP_BY_PASSES] = {
103 + {
104 + .group_by = RRDR_GROUP_BY_DIMENSION,
105 + .group_by_label = NULL,
106 + .aggregation = RRDR_GROUP_BY_FUNCTION_AVERAGE,
107 + },
108 + };
109 +
110 + size_t group_by_idx = 0, group_by_label_idx = 0, aggregation_idx = 0;
111 +
112 while(url) {
113 char *value = mystrsep(&url, "&");
114 if(!value || !*value) continue;
@@ -111,9 +132,21 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
132 else if(!strcmp(name, "before")) before_str = value;
133 else if(!strcmp(name, "points")) points_str = value;
134 else if(!strcmp(name, "timeout")) timeout_str = value;
114 - else if(!strcmp(name, "group_by")) group_by = group_by_parse(value);
115 - else if(!strcmp(name, "group_by_label")) group_by_label = value;
116 - else if(!strcmp(name, "aggregation")) group_by_aggregate = group_by_aggregate_function_parse(value);
135 + else if(!strcmp(name, "group_by")) {
136 + group_by[group_by_idx++].group_by = group_by_parse(value);
137 + if(group_by_idx >= MAX_QUERY_GROUP_BY_PASSES)
138 + group_by_idx = MAX_QUERY_GROUP_BY_PASSES - 1;
139 + }
140 + else if(!strcmp(name, "group_by_label")) {
141 + group_by[group_by_label_idx++].group_by_label = value;
142 + if(group_by_label_idx >= MAX_QUERY_GROUP_BY_PASSES)
143 + group_by_label_idx = MAX_QUERY_GROUP_BY_PASSES - 1;
144 + }
145 + else if(!strcmp(name, "aggregation")) {
146 + group_by[aggregation_idx++].aggregation = group_by_aggregate_function_parse(value);
147 + if(aggregation_idx >= MAX_QUERY_GROUP_BY_PASSES)
148 + aggregation_idx = MAX_QUERY_GROUP_BY_PASSES - 1;
149 + }
150 else if(!strcmp(name, "format")) format = web_client_api_request_v1_data_format(value);
151 else if(!strcmp(name, "options")) options |= web_client_api_request_v1_data_options(value);
152 else if(!strcmp(name, "time_group")) time_group = time_grouping_parse(value, RRDR_GROUPING_AVERAGE);
@@ -153,6 +186,16 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
186 outFileName = tqx_value;
187 }
188 }
189 + else {
190 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
191 + if(!strcmp(name, group_by_keys[g].group_by))
192 + group_by[g].group_by = group_by_parse(value);
193 + else if(!strcmp(name, group_by_keys[g].group_by_label))
194 + group_by[g].group_by_label = value;
195 + else if(!strcmp(name, group_by_keys[g].aggregation))
196 + group_by[g].aggregation = group_by_aggregate_function_parse(value);
197 + }
198 + }
199 }
200
201 // validate the google parameters given
@@ -163,17 +206,20 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
206 fix_google_param(responseHandler);
207 fix_google_param(outFileName);
208
166 - if(group_by_label && *group_by_label)
167 - group_by |= RRDR_GROUP_BY_LABEL;
168 -
169 - if(group_by == RRDR_GROUP_BY_NONE)
170 - group_by = RRDR_GROUP_BY_DIMENSION;
209 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
210 + if (group_by[g].group_by_label && *group_by[g].group_by_label)
211 + group_by[g].group_by |= RRDR_GROUP_BY_LABEL;
212 + }
213
172 - if(group_by & RRDR_GROUP_BY_SELECTED)
173 - group_by = RRDR_GROUP_BY_SELECTED; // remove all other groupings
214 + if(group_by[0].group_by == RRDR_GROUP_BY_NONE)
215 + group_by[0].group_by = RRDR_GROUP_BY_DIMENSION;
216
175 - if((group_by & ~(RRDR_GROUP_BY_DIMENSION)) || (options & RRDR_OPTION_PERCENTAGE))
176 - options |= RRDR_OPTION_ABSOLUTE;
217 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
218 + if ((group_by[g].group_by & ~(RRDR_GROUP_BY_DIMENSION)) || (options & RRDR_OPTION_PERCENTAGE)) {
219 + options |= RRDR_OPTION_ABSOLUTE;
220 + break;
221 + }
222 + }
223
224 if(options & RRDR_OPTION_DEBUG)
225 options &= ~RRDR_OPTION_MINIFY;
@@ -209,9 +255,6 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
255 .points = points,
256 .format = format,
257 .options = options,
212 - .group_by = group_by,
213 - .group_by_label = group_by_label,
214 - .group_by_aggregate_function = group_by_aggregate,
258 .time_group_method = time_group,
259 .time_group_options = time_group_options,
260 .resampling_time = resampling_time,
@@ -225,6 +268,10 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
268 .interrupt_callback = web_client_interrupt_callback,
269 .interrupt_callback_data = w,
270 };
271 +
272 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
273 + qtr.group_by[g] = group_by[g];
274 +
275 QUERY_TARGET *qt = query_target_create(&qtr);
276 ONEWAYALLOC *owa = NULL;
277
web/server/web_client.h
+1
@@ -24,6 +24,7 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;
24 #define HTTP_RESP_NOT_FOUND 404
25 #define HTTP_RESP_CONFLICT 409
26 #define HTTP_RESP_PRECOND_FAIL 412
27 +#define HTTP_RESP_CONTENT_TOO_LONG 413
28
29 // HTTP_CODES 5XX Server Errors
30 #define HTTP_RESP_INTERNAL_SERVER_ERROR 500