@cryptotaxi247 / netdata-1 / commits / 4ddaa0d8c

refactor(queries): split query engine into smaller files (#22187)

* refactor(queries): split query engine into smaller files Split the two largest query engine files into focused, single-concern modules to improve maintainability and prepare for upcoming fixes. query.c (1156 lines) split into: - query.c (408) - entry points: rrd2rrdr(), rrd2rrdr_legacy() - query-execute.c (451) - dimension query execution loop, interpolation - query-window.c (303) - query window calculation query-group-by.c (1383 lines) split into: - query-group-by.c (226) - parsing, JSON helpers, label utilities - query-group-by-init.c (516) - group-by initialization - query-group-by-finalize.c (411) - finalization, trimming, aggregation - query-cardinality-limit.c (235) - cardinality limiting Zero behavior changes. All files compile clean with no duplicate symbols. * fix(queries): restore exact original content in split files The initial split had trailing whitespace stripped from ~30 lines in query-cardinality-limit.c and a few missing trailing blank lines. Regenerated from the original source to ensure byte-for-byte fidelity (verified with diff against every line range of the originals).

Costa Tsaousis committed Apr 10, 2026 at 15:04 UTC 4ddaa0d8ce06bcd86b9a180cc8d463f7dc1cf114
9 files changed +1937 -1908
CMakeLists.txt
+5
@@ -1388,7 +1388,12 @@ set(API_PLUGIN_FILES
1388 src/web/api/queries/rrdr.h
1389 src/web/api/queries/query.c
1390 src/web/api/queries/query.h
1391 + src/web/api/queries/query-execute.c
1392 + src/web/api/queries/query-window.c
1393 src/web/api/queries/query-group-by.c
1394 + src/web/api/queries/query-group-by-init.c
1395 + src/web/api/queries/query-group-by-finalize.c
1396 + src/web/api/queries/query-cardinality-limit.c
1397 src/web/api/queries/query-group-over-time.c
1398 src/web/api/queries/query-internal.h
1399 src/web/api/queries/query-plan.c
src/web/api/queries/query-cardinality-limit.c new
+235
@@ -0,0 +1,235 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "query-internal.h"
4 +
5 +static int compare_contributions(const void *a, const void *b) {
6 + const struct { size_t dim_idx; NETDATA_DOUBLE contribution; } *da = a;
7 + const struct { size_t dim_idx; NETDATA_DOUBLE contribution; } *db = b;
8 +
9 + if (da->contribution > db->contribution) return -1;
10 + if (da->contribution < db->contribution) return 1;
11 + return 0;
12 +}
13 +
14 +RRDR *rrd2rrdr_cardinality_limit(RRDR *r) {
15 + QUERY_TARGET *qt = r->internal.qt;
16 +
17 + if(!qt || qt->request.cardinality_limit == 0 || r->d <= qt->request.cardinality_limit)
18 + return r;
19 +
20 + ONEWAYALLOC *owa = r->internal.owa;
21 +
22 + // Calculate contribution of each dimension using dview statistics (sum of values)
23 + NETDATA_DOUBLE *contributions = onewayalloc_mallocz(owa, r->d * sizeof(NETDATA_DOUBLE));
24 +
25 + // Count queried dimensions and get their contributions from dview
26 + size_t queried_count = 0;
27 + for (size_t d = 0; d < r->d; d++) {
28 + contributions[d] = 0.0;
29 +
30 + if (!(r->od[d] & RRDR_DIMENSION_QUERIED))
31 + continue;
32 +
33 + queried_count++;
34 +
35 + // Use the sum from dview if available, otherwise fall back to manual calculation
36 + if(r->dview && !isnan(r->dview[d].sum)) {
37 + contributions[d] = fabsndd(r->dview[d].sum);
38 + } else {
39 + // Fallback: calculate manually from values
40 + for(size_t i = 0; i < r->rows; i++) {
41 + size_t idx = i * r->d + d;
42 +
43 + if(r->o[idx] & RRDR_VALUE_EMPTY)
44 + continue;
45 +
46 + NETDATA_DOUBLE value = r->v[idx];
47 + if(!isnan(value))
48 + contributions[d] += fabsndd(value);
49 + }
50 + }
51 + }
52 +
53 + // If we don't need to reduce, return original
54 + if(queried_count <= qt->request.cardinality_limit) {
55 + onewayalloc_freez(owa, contributions);
56 + return r;
57 + }
58 +
59 + // Create array of dimension indices sorted by contribution (descending)
60 + struct {
61 + size_t dim_idx;
62 + NETDATA_DOUBLE contribution;
63 + } *sorted_dims = onewayalloc_mallocz(owa, queried_count * sizeof(*sorted_dims));
64 +
65 + size_t sorted_idx = 0;
66 + for (size_t d = 0; d < r->d; d++) {
67 + if (r->od[d] & RRDR_DIMENSION_QUERIED) {
68 + sorted_dims[sorted_idx].dim_idx = d;
69 + sorted_dims[sorted_idx].contribution = contributions[d];
70 + sorted_idx++;
71 + }
72 + }
73 +
74 + // Sort by contribution (descending)
75 + qsort(sorted_dims, queried_count, sizeof(*sorted_dims), compare_contributions);
76 +
77 + // Create new RRDR with limited dimensions
78 + size_t new_d = qt->request.cardinality_limit;
79 + size_t remaining_count = queried_count - (qt->request.cardinality_limit - 1);
80 + if(remaining_count > 0)
81 + new_d = qt->request.cardinality_limit; // Keep one slot for "remaining N dimensions"
82 + else
83 + new_d = queried_count; // No remaining dimensions needed
84 +
85 + RRDR *new_r = rrdr_create(owa, qt, new_d, r->n);
86 + if (!new_r) {
87 + internal_error(true, "QUERY: cannot create cardinality limited RRDR");
88 + onewayalloc_freez(owa, contributions);
89 + onewayalloc_freez(owa, sorted_dims);
90 + return r;
91 + }
92 +
93 + // Copy basic metadata from original RRDR
94 + new_r->view = r->view;
95 + new_r->time_grouping = r->time_grouping;
96 + new_r->partial_data_trimming = r->partial_data_trimming;
97 + new_r->rows = r->rows;
98 +
99 + // Copy timestamps
100 + memcpy(new_r->t, r->t, r->n * sizeof(time_t));
101 +
102 + // Setup arrays for new RRDR
103 + if(new_r->d) {
104 + new_r->dp = onewayalloc_callocz(owa, new_r->d, sizeof(*new_r->dp));
105 + new_r->dview = onewayalloc_callocz(owa, new_r->d, sizeof(*new_r->dview));
106 +
107 + if(new_r->n) {
108 + // Initialize all values as empty
109 + for (size_t i = 0; i < new_r->n; i++) {
110 + for (size_t d = 0; d < new_r->d; d++) {
111 + size_t idx = i * new_r->d + d;
112 + new_r->v[idx] = NAN;
113 + new_r->ar[idx] = 0.0;
114 + new_r->o[idx] = RRDR_VALUE_EMPTY;
115 + }
116 + }
117 + }
118 + }
119 +
120 + // Copy top dimensions
121 + size_t kept_dimensions = (remaining_count > 0) ? qt->request.cardinality_limit - 1 : queried_count;
122 +
123 + for (size_t i = 0; i < kept_dimensions; i++) {
124 + size_t src_d = sorted_dims[i].dim_idx;
125 +
126 + // Copy metadata
127 + new_r->di[i] = string_dup(r->di[src_d]);
128 + new_r->dn[i] = string_dup(r->dn[src_d]);
129 + new_r->od[i] = r->od[src_d];
130 + new_r->du[i] = string_dup(r->du[src_d]);
131 + new_r->dp[i] = r->dp[src_d];
132 +
133 + // Copy data
134 + for (size_t row = 0; row < r->rows; row++) {
135 + size_t src_idx = row * r->d + src_d;
136 + size_t dst_idx = row * new_r->d + i;
137 +
138 + new_r->v[dst_idx] = r->v[src_idx];
139 + new_r->ar[dst_idx] = r->ar[src_idx];
140 + new_r->o[dst_idx] = r->o[src_idx];
141 + }
142 +
143 + // Copy dview stats
144 + if(r->dview)
145 + new_r->dview[i] = r->dview[src_d];
146 + }
147 +
148 + // Create "remaining N dimensions" if needed
149 + if (remaining_count > 0) {
150 + size_t remaining_idx = kept_dimensions;
151 +
152 + char remaining_name[256];
153 + snprintfz(remaining_name, sizeof(remaining_name), "remaining %zu dimension%s",
154 + remaining_count, remaining_count == 1 ? "" : "s");
155 +
156 + new_r->di[remaining_idx] = string_strdupz(remaining_name);
157 + new_r->dn[remaining_idx] = string_strdupz(remaining_name);
158 + new_r->od[remaining_idx] = RRDR_DIMENSION_QUERIED | RRDR_DIMENSION_NONZERO;
159 +
160 + // Use the units from the first remaining dimension
161 + if(kept_dimensions < queried_count) {
162 + size_t first_remaining_d = sorted_dims[kept_dimensions].dim_idx;
163 + new_r->du[remaining_idx] = string_dup(r->du[first_remaining_d]);
164 + new_r->dp[remaining_idx] = r->dp[first_remaining_d];
165 + }
166 +
167 + // Aggregate remaining dimensions
168 + NETDATA_DOUBLE sum = 0.0, min = NAN, max = NAN, ars = 0.0;
169 + size_t count = 0;
170 +
171 + for (size_t row = 0; row < r->rows; row++) {
172 + size_t dst_idx = row * new_r->d + remaining_idx;
173 + NETDATA_DOUBLE aggregated_value = 0.0;
174 + NETDATA_DOUBLE aggregated_ar = 0.0;
175 + RRDR_VALUE_FLAGS aggregated_flags = RRDR_VALUE_NOTHING;
176 + bool has_values = false;
177 +
178 + for (size_t i = kept_dimensions; i < queried_count; i++) {
179 + size_t src_d = sorted_dims[i].dim_idx;
180 + size_t src_idx = row * r->d + src_d;
181 +
182 + if(!(r->o[src_idx] & RRDR_VALUE_EMPTY)) {
183 + NETDATA_DOUBLE value = r->v[src_idx];
184 + if(!isnan(value)) {
185 + aggregated_value += value;
186 + aggregated_ar += r->ar[src_idx];
187 + aggregated_flags |= (r->o[src_idx] & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL));
188 + has_values = true;
189 + }
190 + }
191 + }
192 +
193 + if(has_values) {
194 + new_r->v[dst_idx] = aggregated_value;
195 + new_r->ar[dst_idx] = aggregated_ar;
196 + new_r->o[dst_idx] = aggregated_flags & ~RRDR_VALUE_EMPTY;
197 +
198 + // Update statistics for dview
199 + sum += aggregated_value;
200 + ars += aggregated_ar;
201 + if(count == 0) {
202 + min = max = aggregated_value;
203 + } else {
204 + if(aggregated_value < min) min = aggregated_value;
205 + if(aggregated_value > max) max = aggregated_value;
206 + }
207 + count++;
208 + } else {
209 + new_r->v[dst_idx] = NAN;
210 + new_r->ar[dst_idx] = 0.0;
211 + new_r->o[dst_idx] = RRDR_VALUE_EMPTY;
212 + }
213 + }
214 +
215 + // Set dview for remaining dimension
216 + if(new_r->dview) {
217 + new_r->dview[remaining_idx] = (STORAGE_POINT) {
218 + .sum = sum,
219 + .count = count,
220 + .min = min,
221 + .max = max,
222 + .anomaly_count = (size_t)(ars * RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER / 100.0),
223 + };
224 + }
225 + }
226 +
227 + // Cleanup
228 + onewayalloc_freez(owa, contributions);
229 + onewayalloc_freez(owa, sorted_dims);
230 +
231 + // Free the original RRDR
232 + rrdr_free(owa, r);
233 +
234 + return new_r;
235 +}
src/web/api/queries/query-execute.c new
+451
@@ -0,0 +1,451 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "query-internal.h"
4 +
5 +// ----------------------------------------------------------------------------
6 +// helpers to find our way in RRDR
7 +
8 +ALWAYS_INLINE
9 +static RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long rrdr_line) {
10 + return &r->o[ rrdr_line * r->d ];
11 +}
12 +
13 +ALWAYS_INLINE
14 +static NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rrdr_line) {
15 + return &r->v[ rrdr_line * r->d ];
16 +}
17 +
18 +ALWAYS_INLINE
19 +static long rrdr_line_init(RRDR *r __maybe_unused, time_t t __maybe_unused, long rrdr_line) {
20 + rrdr_line++;
21 +
22 + internal_fatal(rrdr_line >= (long)r->n,
23 + "QUERY: requested to step above RRDR size for query '%s'",
24 + r->internal.qt->id);
25 +
26 + internal_fatal(r->t[rrdr_line] != t,
27 + "QUERY: wrong timestamp at RRDR line %ld, expected %ld, got %ld, of query '%s'",
28 + rrdr_line, r->t[rrdr_line], t, r->internal.qt->id);
29 +
30 + return rrdr_line;
31 +}
32 +
33 +// ----------------------------------------------------------------------------
34 +// dimension level query engine
35 +
36 +#define query_interpolate_point(this_point, last_point, now) do { \
37 + if(likely( \
38 + /* the point to interpolate is more than 1s wide */ \
39 + (this_point).sp.end_time_s - (this_point).sp.start_time_s > 1 \
40 + \
41 + /* the two points are exactly next to each other */ \
42 + && (last_point).sp.end_time_s == (this_point).sp.start_time_s \
43 + \
44 + /* both points are valid numbers */ \
45 + && netdata_double_isnumber((this_point).value) \
46 + && netdata_double_isnumber((last_point).value) \
47 + \
48 + )) { \
49 + (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).sp.end_time_s - (now)) / (NETDATA_DOUBLE)((this_point).sp.end_time_s - (this_point).sp.start_time_s)); \
50 + (this_point).sp.end_time_s = now; \
51 + } \
52 +} while(0)
53 +
54 +#define query_add_point_to_group(r, point, ops, add_flush) do { \
55 + if(likely(netdata_double_isnumber((point).value))) { \
56 + if(likely(fpclassify((point).value) != FP_ZERO)) \
57 + (ops)->group_points_non_zero++; \
58 + \
59 + if(unlikely((point).sp.flags & SN_FLAG_RESET)) \
60 + (ops)->group_value_flags |= RRDR_VALUE_RESET; \
61 + \
62 + time_grouping_add(r, (point).value, add_flush); \
63 + \
64 + storage_point_merge_to((ops)->group_point, (point).sp); \
65 + if(!(point).added) \
66 + storage_point_merge_to((ops)->query_point, (point).sp); \
67 + } \
68 + \
69 + (ops)->group_points_added++; \
70 +} while(0)
71 +
72 +NOT_INLINE_HOT void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops) {
73 + QUERY_TARGET *qt = r->internal.qt;
74 + QUERY_METRIC *qm = ops->qm;
75 +
76 + const RRDR_TIME_GROUPING add_flush = r->time_grouping.add_flush;
77 +
78 + ops->group_point = STORAGE_POINT_UNSET;
79 + ops->query_point = STORAGE_POINT_UNSET;
80 +
81 + RRDR_OPTIONS options = qt->window.options;
82 + size_t points_wanted = qt->window.points;
83 + time_t after_wanted = qt->window.after;
84 + time_t before_wanted = qt->window.before; (void)before_wanted;
85 +
86 +// bool debug_this = false;
87 +// if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
88 +// debug_this = true;
89 +
90 + size_t points_added = 0;
91 +
92 + long rrdr_line = -1;
93 + bool use_anomaly_bit_as_value = (r->internal.qt->window.options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
94 +
95 + NETDATA_DOUBLE min = r->view.min, max = r->view.max;
96 +
97 + QUERY_POINT last2_point = QUERY_POINT_EMPTY;
98 + QUERY_POINT last1_point = QUERY_POINT_EMPTY;
99 + QUERY_POINT new_point = QUERY_POINT_EMPTY;
100 +
101 + // ONE POINT READ-AHEAD
102 + // when we switch plans, we read-ahead a point from the next plan
103 + // to join them smoothly at the exact time the next plan begins
104 + STORAGE_POINT next1_point = STORAGE_POINT_UNSET;
105 +
106 + time_t now_start_time = after_wanted - ops->query_granularity;
107 + time_t now_end_time = after_wanted + ops->view_update_every - ops->query_granularity;
108 +
109 + size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch;
110 + size_t query_is_finished_counter = 0;
111 +
112 + // The main loop, based on the query granularity we need
113 + for( ; points_added < points_wanted && query_is_finished_counter <= 10 ;
114 + now_start_time = now_end_time, now_end_time += ops->view_update_every) {
115 +
116 + if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {
117 + query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s);
118 + db_points_read_since_plan_switch = 0;
119 + }
120 +
121 + // read all the points of the db, prior to the time we need (now_end_time)
122 +
123 + size_t count_same_end_time = 0;
124 + while(count_same_end_time < 100) {
125 + if(likely(count_same_end_time == 0)) {
126 + last2_point = last1_point;
127 + last1_point = new_point;
128 + }
129 +
130 + if(unlikely(storage_engine_query_is_finished(ops->seqh))) {
131 + query_is_finished_counter++;
132 +
133 + if(count_same_end_time != 0) {
134 + last2_point = last1_point;
135 + last1_point = new_point;
136 + }
137 + new_point = QUERY_POINT_EMPTY;
138 + new_point.sp.start_time_s = last1_point.sp.end_time_s;
139 + new_point.sp.end_time_s = now_end_time;
140 +//
141 +// if(debug_this) netdata_log_info("QUERY: is finished() returned true");
142 +//
143 + break;
144 + }
145 + else
146 + query_is_finished_counter = 0;
147 +
148 + // fetch the new point
149 + {
150 + STORAGE_POINT sp;
151 + if(likely(storage_point_is_unset(next1_point))) {
152 + db_points_read_since_plan_switch++;
153 + sp = storage_engine_query_next_metric(ops->seqh);
154 + ops->db_points_read_per_tier[ops->tier]++;
155 + ops->db_total_points_read++;
156 +
157 + if(unlikely(options & RRDR_OPTION_ABSOLUTE))
158 + storage_point_make_positive(sp);
159 + }
160 + else {
161 + // ONE POINT READ-AHEAD
162 + sp = next1_point;
163 + storage_point_unset(next1_point);
164 + db_points_read_since_plan_switch = 1;
165 + }
166 +
167 + // ONE POINT READ-AHEAD
168 + if(unlikely(query_plan_should_switch_plan(ops, sp.end_time_s) &&
169 + query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s))) {
170 +
171 + // The end time of the current point, crosses our plans (tiers)
172 + // so, we switched plan (tier)
173 + //
174 + // There are 2 cases now:
175 + //
176 + // A. the entire point of the previous plan is to the future of point from the next plan
177 + // B. part of the point of the previous plan overlaps with the point from the next plan
178 +
179 + STORAGE_POINT sp2 = storage_engine_query_next_metric(ops->seqh);
180 + ops->db_points_read_per_tier[ops->tier]++;
181 + ops->db_total_points_read++;
182 +
183 + if(unlikely(options & RRDR_OPTION_ABSOLUTE))
184 + storage_point_make_positive(sp);
185 +
186 + if(sp.start_time_s > sp2.start_time_s)
187 + // the point from the previous plan is useless
188 + sp = sp2;
189 + else
190 + // let the query run from the previous plan
191 + // but setting this will also cut off the interpolation
192 + // of the point from the previous plan
193 + next1_point = sp2;
194 + }
195 +
196 + new_point.sp = sp;
197 + new_point.added = false;
198 + query_point_set_id(new_point, ops->db_total_points_read);
199 +
200 +// if(debug_this)
201 +// netdata_log_info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld",
202 +// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
203 +//
204 + // get the right value from the point we got
205 + if(likely(!storage_point_is_unset(sp) && !storage_point_is_gap(sp))) {
206 +
207 + if(unlikely(use_anomaly_bit_as_value))
208 + new_point.value = storage_point_anomaly_rate(new_point.sp);
209 +
210 + else {
211 + switch (ops->tier_query_fetch) {
212 + default:
213 + case TIER_QUERY_FETCH_AVERAGE:
214 + new_point.value = sp.sum / (NETDATA_DOUBLE)sp.count;
215 + break;
216 +
217 + case TIER_QUERY_FETCH_MIN:
218 + new_point.value = sp.min;
219 + break;
220 +
221 + case TIER_QUERY_FETCH_MAX:
222 + new_point.value = sp.max;
223 + break;
224 +
225 + case TIER_QUERY_FETCH_SUM:
226 + new_point.value = sp.sum;
227 + break;
228 + }
229 + }
230 + }
231 + else
232 + new_point.value = NAN;
233 + }
234 +
235 + // check if the db is giving us zero duration points
236 + if(unlikely(db_points_read_since_plan_switch > 1 &&
237 + new_point.sp.start_time_s == new_point.sp.end_time_s)) {
238 +
239 + internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned "
240 + "point %zu from %ld to %ld, that are both equal",
241 + qt->id, query_metric_id(qt, qm),
242 + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s);
243 +
244 + new_point.sp.start_time_s = new_point.sp.end_time_s - ops->tier_ptr->db_update_every_s;
245 + }
246 +
247 + // check if the db is advancing the query
248 + if(unlikely(db_points_read_since_plan_switch > 1 &&
249 + new_point.sp.end_time_s <= last1_point.sp.end_time_s)) {
250 +
251 + internal_error(true,
252 + "QUERY: '%s', dimension '%s' next_metric() returned "
253 + "point %zu from %ld to %ld, before the "
254 + "last point %zu from %ld to %ld, "
255 + "now is %ld to %ld",
256 + qt->id, query_metric_id(qt, qm),
257 + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s,
258 + last1_point.id, last1_point.sp.start_time_s, last1_point.sp.end_time_s,
259 + now_start_time, now_end_time);
260 +
261 + count_same_end_time++;
262 + continue;
263 + }
264 + count_same_end_time = 0;
265 +
266 + // decide how to use this point
267 + if(likely(new_point.sp.end_time_s < now_end_time)) { // likely to favor tier0
268 + // this db point ends before our now_end_time
269 +
270 + if(likely(new_point.sp.end_time_s >= now_start_time)) { // likely to favor tier0
271 + // this db point ends after our now_start time
272 +
273 + query_add_point_to_group(r, new_point, ops, add_flush);
274 + new_point.added = true;
275 + }
276 + else {
277 + // we don't need this db point
278 + // it is totally outside our current time-frame
279 +
280 + // this is desirable for the first point of the query
281 + // because it allows us to interpolate the next point
282 + // at exactly the time we will want
283 +
284 + // we only log if this is not point 1
285 + internal_error(new_point.sp.end_time_s < ops->plan_expanded_after &&
286 + db_points_read_since_plan_switch > 1,
287 + "QUERY: '%s', dimension '%s' next_metric() "
288 + "returned point %zu from %ld time %ld, "
289 + "which is entirely before our current timeframe %ld to %ld "
290 + "(and before the entire query, after %ld, before %ld)",
291 + qt->id, query_metric_id(qt, qm),
292 + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s,
293 + now_start_time, now_end_time,
294 + ops->plan_expanded_after, ops->plan_expanded_before);
295 + }
296 +
297 + }
298 + else {
299 + // the point ends in the future
300 + // so, we will interpolate it below, at the inner loop
301 + break;
302 + }
303 + }
304 +
305 + if(unlikely(count_same_end_time)) {
306 + internal_error(true,
307 + "QUERY: '%s', dimension '%s', the database does not advance the query,"
308 + " it returned an end time less or equal to the end time of the last "
309 + "point we got %ld, %zu times",
310 + qt->id, query_metric_id(qt, qm),
311 + last1_point.sp.end_time_s, count_same_end_time);
312 +
313 + if(unlikely(new_point.sp.end_time_s <= last1_point.sp.end_time_s))
314 + new_point.sp.end_time_s = now_end_time;
315 + }
316 +
317 + time_t stop_time = new_point.sp.end_time_s;
318 + if(unlikely(!storage_point_is_unset(next1_point) && next1_point.start_time_s >= now_end_time)) {
319 + // ONE POINT READ-AHEAD
320 + // the point crosses the start time of the
321 + // read ahead storage point we have read
322 + stop_time = next1_point.start_time_s;
323 + }
324 +
325 + // the inner loop
326 + // we have 3 points in memory: last2, last1, new
327 + // we select the one to use based on their timestamps
328 +
329 + internal_fatal(now_end_time > stop_time || points_added >= points_wanted,
330 + "QUERY: first part of query provides invalid point to interpolate (now_end_time %ld, stop_time %ld",
331 + now_end_time, stop_time);
332 +
333 + do {
334 + // now_start_time is wrong in this loop
335 + // but, we don't need it
336 +
337 + QUERY_POINT current_point;
338 +
339 + if(likely(now_end_time > new_point.sp.start_time_s)) {
340 + // it is time for our NEW point to be used
341 + current_point = new_point;
342 + new_point.added = true; // first copy, then set it, so that new_point will not be added again
343 + query_interpolate_point(current_point, last1_point, now_end_time);
344 +
345 +// internal_error(current_point.id > 0
346 +// && last1_point.id == 0
347 +// && current_point.end_time > after_wanted
348 +// && current_point.end_time > now_end_time,
349 +// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
350 +// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
351 +// " but we could really favor by having last_point1 in this query.",
352 +// qt->id, string2str(qm->dimension.id),
353 +// after_wanted, before_wanted,
354 +// ops.view_update_every, ops.query_granularity,
355 +// current_point.id, current_point.start_time, current_point.end_time,
356 +// now_end_time);
357 + }
358 + else if(likely(now_end_time <= last1_point.sp.end_time_s)) {
359 + // our LAST point is still valid
360 + current_point = last1_point;
361 + last1_point.added = true; // first copy, then set it, so that last1_point will not be added again
362 + query_interpolate_point(current_point, last2_point, now_end_time);
363 +
364 +// internal_error(current_point.id > 0
365 +// && last2_point.id == 0
366 +// && current_point.end_time > after_wanted
367 +// && current_point.end_time > now_end_time,
368 +// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
369 +// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
370 +// " but we could really favor by having last_point2 in this query.",
371 +// qt->id, string2str(qm->dimension.id),
372 +// after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
373 +// current_point.id, current_point.start_time, current_point.end_time,
374 +// now_end_time);
375 + }
376 + else {
377 + // a GAP, we don't have a value this time
378 + current_point = QUERY_POINT_EMPTY;
379 + }
380 +
381 + query_add_point_to_group(r, current_point, ops, add_flush);
382 +
383 + rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line);
384 + size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
385 +
386 + // find the place to store our values
387 + RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];
388 +
389 + // update the dimension options
390 + if(likely(ops->group_points_non_zero))
391 + r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
392 +
393 + // store the specific point options
394 + *rrdr_value_options_ptr = ops->group_value_flags;
395 +
396 + // store the group value
397 + NETDATA_DOUBLE group_value = time_grouping_flush(r, rrdr_value_options_ptr, add_flush);
398 + r->v[rrdr_o_v_index] = group_value;
399 +
400 + r->ar[rrdr_o_v_index] = storage_point_anomaly_rate(ops->group_point);
401 +
402 + if(likely(points_added || r->internal.queries_count)) {
403 + // find the min/max across all dimensions
404 +
405 + if(unlikely(group_value < min)) min = group_value;
406 + if(unlikely(group_value > max)) max = group_value;
407 +
408 + }
409 + else {
410 + // runs only when r->internal.queries_count == 0 && points_added == 0
411 + // so, on the first point added for the query.
412 + min = max = group_value;
413 + }
414 +
415 + points_added++;
416 + ops->group_points_added = 0;
417 + ops->group_value_flags = RRDR_VALUE_NOTHING;
418 + ops->group_points_non_zero = 0;
419 + ops->group_point = STORAGE_POINT_UNSET;
420 +
421 + now_end_time += ops->view_update_every;
422 + } while(now_end_time <= stop_time && points_added < points_wanted);
423 +
424 + // the loop above increased "now" by ops->view_update_every,
425 + // but the main loop will increase it too,
426 + // so, let's undo the last iteration of this loop
427 + now_end_time -= ops->view_update_every;
428 + }
429 + query_planer_finalize_remaining_plans(ops);
430 +
431 + qm->query_points = ops->query_point;
432 +
433 + // fill the rest of the points with empty values
434 + while (points_added < points_wanted) {
435 + rrdr_line++;
436 + size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
437 + r->o[rrdr_o_v_index] = RRDR_VALUE_EMPTY;
438 + r->v[rrdr_o_v_index] = 0.0;
439 + r->ar[rrdr_o_v_index] = 0.0;
440 + points_added++;
441 + }
442 +
443 + r->internal.queries_count++;
444 + r->view.min = min;
445 + r->view.max = max;
446 +
447 + r->stats.result_points_generated += points_added;
448 + r->stats.db_points_read += ops->db_total_points_read;
449 + for(size_t tr = 0; tr < nd_profile.storage_tiers; tr++)
450 + qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
451 +}
src/web/api/queries/query-group-by-finalize.c new
+411
@@ -0,0 +1,411 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "query-internal.h"
4 +
5 +void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, size_t d_tmp,
6 + RRDR_GROUP_BY_FUNCTION group_by_aggregate_function,
7 + STORAGE_POINT *query_points, size_t pass __maybe_unused) {
8 + if(!r_tmp || r_dst == r_tmp || !(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED))
9 + return;
10 +
11 + internal_fatal(r_dst->n != r_tmp->n, "QUERY: group-by source and destination do not have the same number of rows");
12 + internal_fatal(d_dst >= r_dst->d, "QUERY: group-by destination dimension number exceeds destination RRDR size");
13 + internal_fatal(d_tmp >= r_tmp->d, "QUERY: group-by source dimension number exceeds source RRDR size");
14 + internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)");
15 + internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)");
16 +
17 + bool hidden_dimension_on_percentage_of_group = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh;
18 +
19 + if(!hidden_dimension_on_percentage_of_group) {
20 + r_dst->od[d_dst] |= r_tmp->od[d_tmp];
21 + storage_point_merge_to(r_dst->dqp[d_dst], *query_points);
22 + }
23 +
24 + // do the group_by
25 + for(size_t i = 0; i != rrdr_rows(r_tmp) ; i++) {
26 +
27 + size_t idx_tmp = i * r_tmp->d + d_tmp;
28 + NETDATA_DOUBLE n_tmp = r_tmp->v[ idx_tmp ];
29 + RRDR_VALUE_FLAGS o_tmp = r_tmp->o[ idx_tmp ];
30 + NETDATA_DOUBLE ar_tmp = r_tmp->ar[ idx_tmp ];
31 +
32 + if(o_tmp & RRDR_VALUE_EMPTY)
33 + continue;
34 +
35 + size_t idx_dst = i * r_dst->d + d_dst;
36 + NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_group) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ];
37 + RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ];
38 + NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ];
39 + uint32_t *gbc = &r_dst->gbc[ idx_dst ];
40 +
41 + switch(group_by_aggregate_function) {
42 + default:
43 + case RRDR_GROUP_BY_FUNCTION_AVERAGE:
44 + case RRDR_GROUP_BY_FUNCTION_SUM:
45 + case RRDR_GROUP_BY_FUNCTION_PERCENTAGE:
46 + if(isnan(*cn))
47 + *cn = n_tmp;
48 + else
49 + *cn += n_tmp;
50 + break;
51 +
52 + case RRDR_GROUP_BY_FUNCTION_MIN:
53 + if(isnan(*cn) || n_tmp < *cn)
54 + *cn = n_tmp;
55 + break;
56 +
57 + case RRDR_GROUP_BY_FUNCTION_MAX:
58 + if(isnan(*cn) || n_tmp > *cn)
59 + *cn = n_tmp;
60 + break;
61 +
62 + case RRDR_GROUP_BY_FUNCTION_EXTREMES:
63 + // For extremes, we need to keep track of the value with the maximum absolute value
64 + if(isnan(*cn) || fabsndd(n_tmp) > fabsndd(*cn))
65 + *cn = n_tmp;
66 + break;
67 + }
68 +
69 + if(!hidden_dimension_on_percentage_of_group) {
70 + *co &= ~RRDR_VALUE_EMPTY;
71 + *co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL));
72 + *ar += ar_tmp;
73 + (*gbc)++;
74 + }
75 + }
76 +}
77 +
78 +void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
79 + time_t trimmable_after = r->partial_data_trimming.expected_after;
80 +
81 + // find the point just before the trimmable ones
82 + ssize_t i = (ssize_t)r->n - 1;
83 + for( ; i >= 0 ;i--) {
84 + if (r->t[i] < trimmable_after)
85 + break;
86 + }
87 +
88 + if(unlikely(i < 0))
89 + return;
90 +
91 + // internal_error(true, "Found trimmable index %zd (from 0 to %zu)", i, r->n - 1);
92 +
93 + size_t last_row_gbc = 0;
94 + for (; i < (ssize_t)r->n; i++) {
95 + size_t row_gbc = 0;
96 + for (size_t d = 0; d < r->d; d++) {
97 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
98 + continue;
99 +
100 + row_gbc += r->gbc[ i * r->d + d ];
101 + }
102 +
103 + // internal_error(true, "GBC of index %zd is %zu", i, row_gbc);
104 +
105 + if (unlikely(r->t[i] >= trimmable_after && (row_gbc < last_row_gbc || !row_gbc))) {
106 + // discard the rest of the points
107 + // internal_error(true, "Discarding points %zd to %zu", i, r->n - 1);
108 + r->partial_data_trimming.trimmed_after = r->t[i];
109 + r->rows = i;
110 + break;
111 + }
112 + else
113 + last_row_gbc = row_gbc;
114 + }
115 +}
116 +
117 +void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
118 + if(!r->vh)
119 + return;
120 +
121 + if(query_target_aggregatable(r->internal.qt) && query_has_group_by_aggregation_percentage(r->internal.qt))
122 + return;
123 +
124 + for(size_t i = 0; i < r->n ;i++) {
125 + NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
126 + NETDATA_DOUBLE *ch = &r->vh[ i * r->d ];
127 +
128 + for(size_t d = 0; d < r->d ;d++) {
129 + NETDATA_DOUBLE n = cn[d];
130 + NETDATA_DOUBLE h = ch[d];
131 +
132 + if(isnan(n))
133 + cn[d] = 0.0;
134 +
135 + else if(isnan(h))
136 + cn[d] = 100.0;
137 +
138 + else
139 + cn[d] = n * 100.0 / (n + h);
140 + }
141 + }
142 +}
143 +
144 +
145 +void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) {
146 + if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt))
147 + return;
148 +
149 + size_t global_min_max_values = 0;
150 + NETDATA_DOUBLE global_min = NAN, global_max = NAN;
151 +
152 + for(size_t i = 0; i != r->n ;i++) {
153 + NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
154 + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
155 +
156 + NETDATA_DOUBLE total = 0;
157 + for (size_t d = 0; d < r->d; d++) {
158 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
159 + continue;
160 +
161 + if(co[d] & RRDR_VALUE_EMPTY)
162 + continue;
163 +
164 + total += cn[d];
165 + }
166 +
167 + if(total == 0.0)
168 + total = 1.0;
169 +
170 + for (size_t d = 0; d < r->d; d++) {
171 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
172 + continue;
173 +
174 + if(co[d] & RRDR_VALUE_EMPTY)
175 + continue;
176 +
177 + NETDATA_DOUBLE n = cn[d];
178 + n = cn[d] = n * 100.0 / total;
179 +
180 + if(unlikely(!global_min_max_values++))
181 + global_min = global_max = n;
182 + else {
183 + if(n < global_min)
184 + global_min = n;
185 + if(n > global_max)
186 + global_max = n;
187 + }
188 + }
189 + }
190 +
191 + r->view.min = global_min;
192 + r->view.max = global_max;
193 +
194 + if(!r->dview)
195 + // v1 query
196 + return;
197 +
198 + // v2 query
199 +
200 + for (size_t d = 0; d < r->d; d++) {
201 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
202 + continue;
203 +
204 + size_t count = 0;
205 + NETDATA_DOUBLE min = 0.0, max = 0.0, sum = 0.0, ars = 0.0;
206 + for(size_t i = 0; i != r->rows ;i++) { // we use r->rows to respect trimming
207 + size_t idx = i * r->d + d;
208 +
209 + RRDR_VALUE_FLAGS o = r->o[ idx ];
210 +
211 + if (o & RRDR_VALUE_EMPTY)
212 + continue;
213 +
214 + NETDATA_DOUBLE ar = r->ar[ idx ];
215 + ars += ar;
216 +
217 + NETDATA_DOUBLE n = r->v[ idx ];
218 + sum += n;
219 +
220 + if(!count++)
221 + min = max = n;
222 + else {
223 + if(n < min)
224 + min = n;
225 + if(n > max)
226 + max = n;
227 + }
228 + }
229 +
230 + r->dview[d] = (STORAGE_POINT) {
231 + .sum = sum,
232 + .count = count,
233 + .min = min,
234 + .max = max,
235 + .anomaly_count = (size_t)(ars * (NETDATA_DOUBLE)count),
236 + };
237 + }
238 +}
239 +
240 +RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
241 + QUERY_TARGET *qt = r_tmp->internal.qt;
242 +
243 + if(!r_tmp->group_by.r) {
244 + // v1 query
245 + rrd2rrdr_convert_values_to_percentage_of_total(r_tmp);
246 + return r_tmp;
247 + }
248 + // v2 query
249 +
250 + // do the additional passes on RRDRs
251 + RRDR *last_r = r_tmp->group_by.r;
252 + rrdr2rrdr_group_by_calculate_percentage_of_group(last_r);
253 +
254 + RRDR *r = last_r->group_by.r;
255 + size_t pass = 0;
256 + while(r) {
257 + pass++;
258 + for(size_t d = 0; d < last_r->d ;d++) {
259 + rrd2rrdr_group_by_add_metric(r, last_r->dgbs[d], last_r, d,
260 + qt->request.group_by[pass].aggregation,
261 + &last_r->dqp[d], pass);
262 + }
263 + rrdr2rrdr_group_by_calculate_percentage_of_group(r);
264 +
265 + last_r = r;
266 + r = last_r->group_by.r;
267 + }
268 +
269 + // free all RRDRs except the last one
270 + r = r_tmp;
271 + while(r != last_r) {
272 + r_tmp = r->group_by.r;
273 + r->group_by.r = NULL;
274 + rrdr_free(r->internal.owa, r);
275 + r = r_tmp;
276 + }
277 + r = last_r;
278 +
279 + // find the final aggregation
280 + RRDR_GROUP_BY_FUNCTION aggregation = qt->request.group_by[0].aggregation;
281 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
282 + if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE)
283 + aggregation = qt->request.group_by[g].aggregation;
284 +
285 + if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before)
286 + rrdr2rrdr_group_by_partial_trimming(r);
287 +
288 + // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
289 + size_t global_min_max_values = 0;
290 + size_t dimensions_nonzero = 0;
291 + NETDATA_DOUBLE global_min = NAN, global_max = NAN;
292 + for (size_t d = 0; d < r->d; d++) {
293 + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
294 + continue;
295 +
296 + size_t points_nonzero = 0;
297 + NETDATA_DOUBLE min = 0, max = 0, sum = 0, ars = 0;
298 + size_t count = 0;
299 +
300 + for(size_t i = 0; i != r->n ;i++) {
301 + size_t idx = i * r->d + d;
302 +
303 + NETDATA_DOUBLE *cn = &r->v[ idx ];
304 + RRDR_VALUE_FLAGS *co = &r->o[ idx ];
305 + NETDATA_DOUBLE *ar = &r->ar[ idx ];
306 + uint32_t gbc = r->gbc[ idx ];
307 +
308 + if(likely(gbc)) {
309 + *co &= ~RRDR_VALUE_EMPTY;
310 +
311 + if(gbc != r->dgbc[d])
312 + *co |= RRDR_VALUE_PARTIAL;
313 +
314 + NETDATA_DOUBLE n;
315 +
316 + sum += *cn;
317 + ars += *ar;
318 +
319 + if(aggregation == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt))
320 + n = (*cn /= gbc);
321 + else
322 + n = *cn;
323 +
324 + if(!query_target_aggregatable(qt))
325 + *ar /= gbc;
326 +
327 + if(islessgreater(n, 0.0))
328 + points_nonzero++;
329 +
330 + if(unlikely(!count))
331 + min = max = n;
332 + else {
333 + if(n < min)
334 + min = n;
335 +
336 + if(n > max)
337 + max = n;
338 + }
339 +
340 + if(unlikely(!global_min_max_values++))
341 + global_min = global_max = n;
342 + else {
343 + if(n < global_min)
344 + global_min = n;
345 +
346 + if(n > global_max)
347 + global_max = n;
348 + }
349 +
350 + count += gbc;
351 + }
352 + }
353 +
354 + if(points_nonzero) {
355 + r->od[d] |= RRDR_DIMENSION_NONZERO;
356 + dimensions_nonzero++;
357 + }
358 +
359 + r->dview[d] = (STORAGE_POINT) {
360 + .sum = sum,
361 + .count = count,
362 + .min = min,
363 + .max = max,
364 + .anomaly_count = (size_t)(ars * RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER / 100.0),
365 + };
366 + }
367 +
368 + r->view.min = global_min;
369 + r->view.max = global_max;
370 +
371 + if(!dimensions_nonzero && (qt->window.options & RRDR_OPTION_NONZERO)) {
372 + // all dimensions are zero
373 + // remove the nonzero option
374 + qt->window.options &= ~RRDR_OPTION_NONZERO;
375 + }
376 +
377 + rrd2rrdr_convert_values_to_percentage_of_total(r);
378 +
379 + // update query instance counts in query host and query context
380 + {
381 + size_t h = 0, c = 0, i = 0;
382 + for(; h < qt->nodes.used ; h++) {
383 + QUERY_NODE *qn = &qt->nodes.array[h];
384 +
385 + for(; c < qt->contexts.used ;c++) {
386 + QUERY_CONTEXT *qc = &qt->contexts.array[c];
387 +
388 + if(!rrdcontext_acquired_belongs_to_host(qc->rca, qn->rrdhost))
389 + break;
390 +
391 + for(; i < qt->instances.used ;i++) {
392 + QUERY_INSTANCE *qi = &qt->instances.array[i];
393 +
394 + if(!rrdinstance_acquired_belongs_to_context(qi->ria, qc->rca))
395 + break;
396 +
397 + if(qi->metrics.queried) {
398 + qc->instances.queried++;
399 + qn->instances.queried++;
400 + }
401 + else if(qi->metrics.failed) {
402 + qc->instances.failed++;
403 + qn->instances.failed++;
404 + }
405 + }
406 + }
407 + }
408 + }
409 +
410 + return r;
411 +}
src/web/api/queries/query-group-by-init.c new
+517
@@ -0,0 +1,517 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "query-internal.h"
4 +
5 +static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
6 + buffer_flush(key);
7 + if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
8 + buffer_strcat(key, "__hidden_dimensions__");
9 + }
10 + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
11 + buffer_strcat(key, "selected");
12 + }
13 + else {
14 + if (group_by & RRDR_GROUP_BY_DIMENSION) {
15 + buffer_fast_strcat(key, "|", 1);
16 + buffer_strcat(key, query_metric_name(qt, qm));
17 + }
18 +
19 + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
20 + buffer_fast_strcat(key, "|", 1);
21 + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
22 + }
23 +
24 + if (group_by & RRDR_GROUP_BY_LABEL) {
25 + RRDLABELS *labels = rrdinstance_acquired_labels(qi->ria);
26 + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
27 + buffer_fast_strcat(key, "|", 1);
28 + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
29 + }
30 + }
31 +
32 + if (group_by & RRDR_GROUP_BY_NODE) {
33 + buffer_fast_strcat(key, "|", 1);
34 + buffer_strcat(key, qn->rrdhost->machine_guid);
35 + }
36 +
37 + if (group_by & RRDR_GROUP_BY_CONTEXT) {
38 + buffer_fast_strcat(key, "|", 1);
39 + buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
40 + }
41 +
42 + if (group_by & RRDR_GROUP_BY_UNITS) {
43 + buffer_fast_strcat(key, "|", 1);
44 + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
45 + }
46 + }
47 +}
48 +
49 +static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
50 + buffer_flush(key);
51 + if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
52 + buffer_strcat(key, "__hidden_dimensions__");
53 + }
54 + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
55 + buffer_strcat(key, "selected");
56 + }
57 + else {
58 + if (group_by & RRDR_GROUP_BY_DIMENSION) {
59 + buffer_strcat(key, query_metric_name(qt, qm));
60 + }
61 +
62 + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
63 + if (buffer_strlen(key) != 0)
64 + buffer_fast_strcat(key, ",", 1);
65 +
66 + if (group_by & RRDR_GROUP_BY_NODE)
67 + buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
68 + else
69 + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
70 + }
71 +
72 + if (group_by & RRDR_GROUP_BY_LABEL) {
73 + RRDLABELS *labels = rrdinstance_acquired_labels(qi->ria);
74 + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
75 + if (buffer_strlen(key) != 0)
76 + buffer_fast_strcat(key, ",", 1);
77 + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
78 + }
79 + }
80 +
81 + if (group_by & RRDR_GROUP_BY_NODE) {
82 + if (buffer_strlen(key) != 0)
83 + buffer_fast_strcat(key, ",", 1);
84 +
85 + buffer_strcat(key, qn->rrdhost->machine_guid);
86 + }
87 +
88 + if (group_by & RRDR_GROUP_BY_CONTEXT) {
89 + if (buffer_strlen(key) != 0)
90 + buffer_fast_strcat(key, ",", 1);
91 +
92 + buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
93 + }
94 +
95 + if (group_by & RRDR_GROUP_BY_UNITS) {
96 + if (buffer_strlen(key) != 0)
97 + buffer_fast_strcat(key, ",", 1);
98 +
99 + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
100 + }
101 + }
102 +}
103 +
104 +static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
105 + buffer_flush(key);
106 + if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
107 + buffer_strcat(key, "__hidden_dimensions__");
108 + }
109 + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
110 + buffer_strcat(key, "selected");
111 + }
112 + else {
113 + if (group_by & RRDR_GROUP_BY_DIMENSION) {
114 + buffer_strcat(key, query_metric_name(qt, qm));
115 + }
116 +
117 + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
118 + if (buffer_strlen(key) != 0)
119 + buffer_fast_strcat(key, ",", 1);
120 +
121 + if (group_by & RRDR_GROUP_BY_NODE)
122 + buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
123 + else
124 + buffer_strcat(key, string2str(query_instance_name_fqdn(qi, qt->request.version)));
125 + }
126 +
127 + if (group_by & RRDR_GROUP_BY_LABEL) {
128 + RRDLABELS *labels = rrdinstance_acquired_labels(qi->ria);
129 + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
130 + if (buffer_strlen(key) != 0)
131 + buffer_fast_strcat(key, ",", 1);
132 + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
133 + }
134 + }
135 +
136 + if (group_by & RRDR_GROUP_BY_NODE) {
137 + if (buffer_strlen(key) != 0)
138 + buffer_fast_strcat(key, ",", 1);
139 +
140 + buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
141 + }
142 +
143 + if (group_by & RRDR_GROUP_BY_CONTEXT) {
144 + if (buffer_strlen(key) != 0)
145 + buffer_fast_strcat(key, ",", 1);
146 +
147 + buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
148 + }
149 +
150 + if (group_by & RRDR_GROUP_BY_UNITS) {
151 + if (buffer_strlen(key) != 0)
152 + buffer_fast_strcat(key, ",", 1);
153 +
154 + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
155 + }
156 + }
157 +}
158 +
159 +struct rrdr_group_by_entry {
160 + size_t priority;
161 + size_t count;
162 + STRING *id;
163 + STRING *name;
164 + STRING *units;
165 + RRDR_DIMENSION_FLAGS od;
166 + DICTIONARY *dl;
167 +};
168 +
169 +RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
170 + RRDR *r_tmp = NULL;
171 + RRDR_OPTIONS options = qt->window.options;
172 +
173 + if(qt->request.version < 2) {
174 + // v1 query
175 + RRDR *r = rrdr_create(owa, qt, qt->query.used, qt->window.points);
176 + if(unlikely(!r)) {
177 + internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, dimensions=%u, points=%zu",
178 + qt->id, qt->window.after, qt->window.before, qt->query.used, qt->window.points);
179 + return NULL;
180 + }
181 + r->group_by.r = NULL;
182 +
183 + for(size_t d = 0; d < qt->query.used ; d++) {
184 + QUERY_METRIC *qm = query_metric(qt, d);
185 + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
186 + r->di[d] = rrdmetric_acquired_id_dup(qd->rma);
187 + r->dn[d] = rrdmetric_acquired_name_dup(qd->rma);
188 + }
189 +
190 + rrd2rrdr_set_timestamps(r);
191 + return r;
192 + }
193 + // v2 query
194 +
195 + // parse all the group-by label keys
196 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
197 + if (qt->request.group_by[g].group_by & RRDR_GROUP_BY_LABEL &&
198 + qt->request.group_by[g].group_by_label && *qt->request.group_by[g].group_by_label)
199 + qt->group_by[g].used = quoted_strings_splitter_query_group_by_label(
200 + qt->request.group_by[g].group_by_label, qt->group_by[g].label_keys,
201 + GROUP_BY_MAX_LABEL_KEYS);
202 +
203 + if (!qt->group_by[g].used)
204 + qt->request.group_by[g].group_by &= ~RRDR_GROUP_BY_LABEL;
205 + }
206 +
207 + // make sure there are valid group-by methods
208 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
209 + if(!(qt->request.group_by[g].group_by & SUPPORTED_GROUP_BY_METHODS))
210 + qt->request.group_by[g].group_by = (g == 0) ? RRDR_GROUP_BY_DIMENSION : RRDR_GROUP_BY_NONE;
211 + }
212 +
213 + bool query_has_percentage_of_group = query_target_has_percentage_of_group(qt);
214 +
215 + // merge all group-by options to upper levels,
216 + // so that the top level has all the groupings of the inner levels,
217 + // and each subsequent level has all the groupings of its inner levels.
218 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) {
219 + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
220 + continue;
221 +
222 + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_SELECTED) {
223 + for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++)
224 + qt->request.group_by[r].group_by = RRDR_GROUP_BY_NONE;
225 + }
226 + else {
227 + for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++) {
228 + if (qt->request.group_by[r].group_by == RRDR_GROUP_BY_NONE)
229 + continue;
230 +
231 + if (qt->request.group_by[r].group_by != RRDR_GROUP_BY_SELECTED) {
232 + if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
233 + qt->request.group_by[g].group_by |= RRDR_GROUP_BY_INSTANCE;
234 + else
235 + qt->request.group_by[g].group_by |= qt->request.group_by[r].group_by;
236 +
237 + if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_LABEL) {
238 + for (size_t lr = 0; lr < qt->group_by[r].used; lr++) {
239 + bool found = false;
240 + for (size_t lg = 0; lg < qt->group_by[g].used; lg++) {
241 + if (strcmp(qt->group_by[g].label_keys[lg], qt->group_by[r].label_keys[lr]) == 0) {
242 + found = true;
243 + break;
244 + }
245 + }
246 +
247 + if (!found && qt->group_by[g].used < GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES)
248 + qt->group_by[g].label_keys[qt->group_by[g].used++] = qt->group_by[r].label_keys[lr];
249 + }
250 + }
251 + }
252 + }
253 + }
254 + }
255 +
256 + int added = 0;
257 + RRDR *first_r = NULL, *last_r = NULL;
258 + BUFFER *key = buffer_create(0, NULL);
259 + struct rrdr_group_by_entry *entries = onewayalloc_mallocz(owa, qt->query.used * sizeof(struct rrdr_group_by_entry));
260 + DICTIONARY *groups = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
261 + DICTIONARY *label_keys = NULL;
262 +
263 + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
264 + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by;
265 + RRDR_GROUP_BY_FUNCTION aggregation_method = qt->request.group_by[g].aggregation;
266 +
267 + if(group_by == RRDR_GROUP_BY_NONE)
268 + break;
269 +
270 + memset(entries, 0, qt->query.used * sizeof(struct rrdr_group_by_entry));
271 + dictionary_flush(groups);
272 + added = 0;
273 +
274 + size_t hidden_dimensions = 0;
275 + bool final_grouping = (g == MAX_QUERY_GROUP_BY_PASSES - 1 || qt->request.group_by[g + 1].group_by == RRDR_GROUP_BY_NONE) ? true : false;
276 +
277 + if (final_grouping && (options & RRDR_OPTION_GROUP_BY_LABELS))
278 + label_keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0);
279 +
280 + QUERY_INSTANCE *last_qi = NULL;
281 + size_t priority = 0;
282 + time_t update_every_max = 0;
283 + for (size_t d = 0; d < qt->query.used; d++) {
284 + QUERY_METRIC *qm = query_metric(qt, d);
285 + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
286 + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
287 + QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
288 + QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
289 +
290 + if (qi != last_qi) {
291 + last_qi = qi;
292 +
293 + time_t update_every = rrdinstance_acquired_update_every(qi->ria);
294 + if (update_every > update_every_max)
295 + update_every_max = update_every;
296 + }
297 +
298 + priority = qd->priority;
299 +
300 + if(qm->status & RRDR_DIMENSION_HIDDEN)
301 + hidden_dimensions++;
302 +
303 + // --------------------------------------------------------------------
304 + // generate the group by key
305 +
306 + query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
307 +
308 + // lookup the key in the dictionary
309 +
310 + int pos = -1;
311 + int *set = dictionary_set(groups, buffer_tostring(key), &pos, sizeof(pos));
312 + if (*set == -1) {
313 + // the key just added to the dictionary
314 +
315 + *set = pos = added++;
316 +
317 + // ----------------------------------------------------------------
318 + // generate the dimension id
319 +
320 + query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
321 + entries[pos].id = string_strdupz(buffer_tostring(key));
322 +
323 + // ----------------------------------------------------------------
324 + // generate the dimension name
325 +
326 + query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
327 + entries[pos].name = string_strdupz(buffer_tostring(key));
328 +
329 + // add the rest of the info
330 + entries[pos].units = rrdinstance_acquired_units_dup(qi->ria);
331 + entries[pos].priority = priority;
332 +
333 + if (label_keys) {
334 + entries[pos].dl = dictionary_create_advanced(
335 + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
336 + NULL, sizeof(struct group_by_label_key));
337 + dictionary_register_insert_callback(entries[pos].dl, group_by_label_key_insert_cb, label_keys);
338 + dictionary_register_delete_callback(entries[pos].dl, group_by_label_key_delete_cb, label_keys);
339 + }
340 + } else {
341 + // the key found in the dictionary
342 + pos = *set;
343 + }
344 +
345 + entries[pos].count++;
346 +
347 + if (unlikely(priority < entries[pos].priority))
348 + entries[pos].priority = priority;
349 +
350 + if(g > 0)
351 + last_r->dgbs[qm->grouped_as.slot] = pos;
352 + else
353 + qm->grouped_as.first_slot = pos;
354 +
355 + qm->grouped_as.slot = pos;
356 + qm->grouped_as.id = entries[pos].id;
357 + qm->grouped_as.name = entries[pos].name;
358 + qm->grouped_as.units = entries[pos].units;
359 +
360 + // copy the dimension flags decided by the query target
361 + // we need this, because if a dimension is explicitly selected
362 + // the query target adds to it the non-zero flag
363 + qm->status |= RRDR_DIMENSION_GROUPED;
364 +
365 + if(query_has_percentage_of_group)
366 + // when the query has percentage of group
367 + // there will be no hidden dimensions in the final query,
368 + // so we have to remove the hidden flag from all dimensions
369 + entries[pos].od |= qm->status & ~RRDR_DIMENSION_HIDDEN;
370 + else
371 + entries[pos].od |= qm->status;
372 +
373 + if (entries[pos].dl)
374 + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(qi->ria),
375 + rrdlabels_traversal_cb_to_group_by_label_key, entries[pos].dl);
376 + }
377 +
378 + RRDR *r = rrdr_create(owa, qt, added, qt->window.points);
379 + if (!r) {
380 + internal_error(true,
381 + "QUERY: cannot create group by RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
382 + qt->id, qt->window.after, qt->window.before, added, qt->window.points);
383 + goto cleanup;
384 + }
385 + // prevent double free at cleanup in case of error
386 + added = 0;
387 +
388 + // link this RRDR
389 + if(!last_r)
390 + first_r = last_r = r;
391 + else
392 + last_r->group_by.r = r;
393 +
394 + last_r = r;
395 +
396 + rrd2rrdr_set_timestamps(r);
397 +
398 + if(r->d) {
399 + r->dp = onewayalloc_callocz(owa, r->d, sizeof(*r->dp));
400 + r->dview = onewayalloc_callocz(owa, r->d, sizeof(*r->dview));
401 + r->dgbc = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbc));
402 + r->dqp = onewayalloc_callocz(owa, r->d, sizeof(STORAGE_POINT));
403 +
404 + if(!final_grouping)
405 + // this is where we are going to store the slot in the next RRDR
406 + // that we are going to group by the dimension of this RRDR
407 + r->dgbs = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbs));
408 +
409 + if (label_keys) {
410 + r->dl = onewayalloc_callocz(owa, r->d, sizeof(DICTIONARY *));
411 + r->label_keys = label_keys;
412 + label_keys = NULL;
413 + }
414 +
415 + if(r->n) {
416 + r->gbc = onewayalloc_callocz(owa, r->n * r->d, sizeof(*r->gbc));
417 +
418 + if(hidden_dimensions && ((group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) || (aggregation_method == RRDR_GROUP_BY_FUNCTION_PERCENTAGE)))
419 + // this is where we are going to group the hidden dimensions
420 + r->vh = onewayalloc_mallocz(owa, r->n * r->d * sizeof(*r->vh));
421 + }
422 + }
423 +
424 + // zero r (dimension options, names, and ids)
425 + // this is required, because group-by may lead to empty dimensions
426 + for (size_t d = 0; d < r->d; d++) {
427 + r->di[d] = entries[d].id;
428 + r->dn[d] = entries[d].name;
429 +
430 + r->od[d] = entries[d].od;
431 + r->du[d] = entries[d].units;
432 + r->dp[d] = entries[d].priority;
433 + r->dgbc[d] = entries[d].count;
434 +
435 + if (r->dl)
436 + r->dl[d] = entries[d].dl;
437 + }
438 +
439 + // initialize partial trimming
440 + r->partial_data_trimming.max_update_every = update_every_max * 2;
441 + r->partial_data_trimming.expected_after =
442 + (!query_target_aggregatable(qt) &&
443 + qt->window.before >= qt->window.now - r->partial_data_trimming.max_update_every) ?
444 + qt->window.before - r->partial_data_trimming.max_update_every :
445 + qt->window.before;
446 + r->partial_data_trimming.trimmed_after = qt->window.before;
447 +
448 + // make all values empty
449 + if(r->n && r->d) {
450 + for (size_t i = 0; i != r->n; i++) {
451 + NETDATA_DOUBLE *cn = &r->v[i * r->d];
452 + RRDR_VALUE_FLAGS *co = &r->o[i * r->d];
453 + NETDATA_DOUBLE *ar = &r->ar[i * r->d];
454 + NETDATA_DOUBLE *vh = r->vh ? &r->vh[i * r->d] : NULL;
455 +
456 + for (size_t d = 0; d < r->d; d++) {
457 + cn[d] = NAN;
458 + ar[d] = 0.0;
459 + co[d] = RRDR_VALUE_EMPTY;
460 +
461 + if (vh)
462 + vh[d] = NAN;
463 + }
464 + }
465 + }
466 + }
467 +
468 + if(!first_r || !last_r)
469 + goto cleanup;
470 +
471 + r_tmp = rrdr_create(owa, qt, 1, qt->window.points);
472 + if (!r_tmp) {
473 + internal_error(true,
474 + "QUERY: cannot create group by temporary RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
475 + qt->id, qt->window.after, qt->window.before, 1, qt->window.points);
476 + goto cleanup;
477 + }
478 + rrd2rrdr_set_timestamps(r_tmp);
479 + r_tmp->group_by.r = first_r;
480 +
481 +cleanup:
482 + if(!first_r || !last_r || !r_tmp) {
483 + if(r_tmp) {
484 + r_tmp->group_by.r = NULL;
485 + rrdr_free(owa, r_tmp);
486 + }
487 +
488 + if(first_r) {
489 + RRDR *r = first_r;
490 + while (r) {
491 + r_tmp = r->group_by.r;
492 + r->group_by.r = NULL;
493 + rrdr_free(owa, r);
494 + r = r_tmp;
495 + }
496 + }
497 +
498 + if(entries && added) {
499 + for (int d = 0; d < added; d++) {
500 + string_freez(entries[d].id);
501 + string_freez(entries[d].name);
502 + string_freez(entries[d].units);
503 + dictionary_destroy(entries[d].dl);
504 + }
505 + }
506 + dictionary_destroy(label_keys);
507 +
508 + first_r = last_r = r_tmp = NULL;
509 + }
510 +
511 + buffer_free(key);
512 + onewayalloc_freez(owa, entries);
513 + dictionary_destroy(groups);
514 +
515 + return r_tmp;
516 +}
517 +
src/web/api/queries/query-group-by.c
+4 -1160
@@ -130,11 +130,7 @@ const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_b
130 // ----------------------------------------------------------------------------
131 // group by
132
133 -struct group_by_label_key {
134 - DICTIONARY *values;
135 -};
136 -
137 -static void group_by_label_key_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
133 +void group_by_label_key_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
134 // add the key to our r->label_keys global keys dictionary
135 DICTIONARY *label_keys = data;
136 dictionary_set(label_keys, dictionary_acquired_item_name(item), NULL, 0);
@@ -144,12 +140,12 @@ static void group_by_label_key_insert_cb(const DICTIONARY_ITEM *item __maybe_unu
140 k->values = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0);
141 }
142
147 -static void group_by_label_key_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
143 +void group_by_label_key_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
144 struct group_by_label_key *k = value;
145 dictionary_destroy(k->values);
146 }
147
152 -static int rrdlabels_traversal_cb_to_group_by_label_key(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
148 +int rrdlabels_traversal_cb_to_group_by_label_key(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
149 DICTIONARY *dl = data;
150 struct group_by_label_key *k = dictionary_set(dl, name, NULL, sizeof(struct group_by_label_key));
151 dictionary_set(k->values, value, NULL, 0);
@@ -191,7 +187,7 @@ void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTION
187 buffer_json_object_close(wb); // key
188 }
189
194 -static void rrd2rrdr_set_timestamps(RRDR *r) {
190 +void rrd2rrdr_set_timestamps(RRDR *r) {
191 QUERY_TARGET *qt = r->internal.qt;
192
193 internal_fatal(qt->window.points != r->n, "QUERY: mismatch to the number of points in qt and r");
@@ -229,1155 +225,3 @@ static void rrd2rrdr_set_timestamps(RRDR *r) {
225 before_wanted, r->t[points_wanted - 1]);
226 }
227
232 -static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
233 - buffer_flush(key);
234 - if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
235 - buffer_strcat(key, "__hidden_dimensions__");
236 - }
237 - else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
238 - buffer_strcat(key, "selected");
239 - }
240 - else {
241 - if (group_by & RRDR_GROUP_BY_DIMENSION) {
242 - buffer_fast_strcat(key, "|", 1);
243 - buffer_strcat(key, query_metric_name(qt, qm));
244 - }
245 -
246 - if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
247 - buffer_fast_strcat(key, "|", 1);
248 - buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
249 - }
250 -
251 - if (group_by & RRDR_GROUP_BY_LABEL) {
252 - RRDLABELS *labels = rrdinstance_acquired_labels(qi->ria);
253 - for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
254 - buffer_fast_strcat(key, "|", 1);
255 - rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
256 - }
257 - }
258 -
259 - if (group_by & RRDR_GROUP_BY_NODE) {
260 - buffer_fast_strcat(key, "|", 1);
261 - buffer_strcat(key, qn->rrdhost->machine_guid);
262 - }
263 -
264 - if (group_by & RRDR_GROUP_BY_CONTEXT) {
265 - buffer_fast_strcat(key, "|", 1);
266 - buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
267 - }
268 -
269 - if (group_by & RRDR_GROUP_BY_UNITS) {
270 - buffer_fast_strcat(key, "|", 1);
271 - buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
272 - }
273 - }
274 -}
275 -
276 -static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
277 - buffer_flush(key);
278 - if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
279 - buffer_strcat(key, "__hidden_dimensions__");
280 - }
281 - else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
282 - buffer_strcat(key, "selected");
283 - }
284 - else {
285 - if (group_by & RRDR_GROUP_BY_DIMENSION) {
286 - buffer_strcat(key, query_metric_name(qt, qm));
287 - }
288 -
289 - if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
290 - if (buffer_strlen(key) != 0)
291 - buffer_fast_strcat(key, ",", 1);
292 -
293 - if (group_by & RRDR_GROUP_BY_NODE)
294 - buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
295 - else
296 - buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version)));
297 - }
298 -
299 - if (group_by & RRDR_GROUP_BY_LABEL) {
300 - RRDLABELS *labels = rrdinstance_acquired_labels(qi->ria);
301 - for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
302 - if (buffer_strlen(key) != 0)
303 - buffer_fast_strcat(key, ",", 1);
304 - rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
305 - }
306 - }
307 -
308 - if (group_by & RRDR_GROUP_BY_NODE) {
309 - if (buffer_strlen(key) != 0)
310 - buffer_fast_strcat(key, ",", 1);
311 -
312 - buffer_strcat(key, qn->rrdhost->machine_guid);
313 - }
314 -
315 - if (group_by & RRDR_GROUP_BY_CONTEXT) {
316 - if (buffer_strlen(key) != 0)
317 - buffer_fast_strcat(key, ",", 1);
318 -
319 - buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
320 - }
321 -
322 - if (group_by & RRDR_GROUP_BY_UNITS) {
323 - if (buffer_strlen(key) != 0)
324 - buffer_fast_strcat(key, ",", 1);
325 -
326 - buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
327 - }
328 - }
329 -}
330 -
331 -static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
332 - buffer_flush(key);
333 - if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
334 - buffer_strcat(key, "__hidden_dimensions__");
335 - }
336 - else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
337 - buffer_strcat(key, "selected");
338 - }
339 - else {
340 - if (group_by & RRDR_GROUP_BY_DIMENSION) {
341 - buffer_strcat(key, query_metric_name(qt, qm));
342 - }
343 -
344 - if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) {
345 - if (buffer_strlen(key) != 0)
346 - buffer_fast_strcat(key, ",", 1);
347 -
348 - if (group_by & RRDR_GROUP_BY_NODE)
349 - buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
350 - else
351 - buffer_strcat(key, string2str(query_instance_name_fqdn(qi, qt->request.version)));
352 - }
353 -
354 - if (group_by & RRDR_GROUP_BY_LABEL) {
355 - RRDLABELS *labels = rrdinstance_acquired_labels(qi->ria);
356 - for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) {
357 - if (buffer_strlen(key) != 0)
358 - buffer_fast_strcat(key, ",", 1);
359 - rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]");
360 - }
361 - }
362 -
363 - if (group_by & RRDR_GROUP_BY_NODE) {
364 - if (buffer_strlen(key) != 0)
365 - buffer_fast_strcat(key, ",", 1);
366 -
367 - buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
368 - }
369 -
370 - if (group_by & RRDR_GROUP_BY_CONTEXT) {
371 - if (buffer_strlen(key) != 0)
372 - buffer_fast_strcat(key, ",", 1);
373 -
374 - buffer_strcat(key, rrdcontext_acquired_id(qc->rca));
375 - }
376 -
377 - if (group_by & RRDR_GROUP_BY_UNITS) {
378 - if (buffer_strlen(key) != 0)
379 - buffer_fast_strcat(key, ",", 1);
380 -
381 - buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria));
382 - }
383 - }
384 -}
385 -
386 -struct rrdr_group_by_entry {
387 - size_t priority;
388 - size_t count;
389 - STRING *id;
390 - STRING *name;
391 - STRING *units;
392 - RRDR_DIMENSION_FLAGS od;
393 - DICTIONARY *dl;
394 -};
395 -
396 -RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
397 - RRDR *r_tmp = NULL;
398 - RRDR_OPTIONS options = qt->window.options;
399 -
400 - if(qt->request.version < 2) {
401 - // v1 query
402 - RRDR *r = rrdr_create(owa, qt, qt->query.used, qt->window.points);
403 - if(unlikely(!r)) {
404 - internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, dimensions=%u, points=%zu",
405 - qt->id, qt->window.after, qt->window.before, qt->query.used, qt->window.points);
406 - return NULL;
407 - }
408 - r->group_by.r = NULL;
409 -
410 - for(size_t d = 0; d < qt->query.used ; d++) {
411 - QUERY_METRIC *qm = query_metric(qt, d);
412 - QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
413 - r->di[d] = rrdmetric_acquired_id_dup(qd->rma);
414 - r->dn[d] = rrdmetric_acquired_name_dup(qd->rma);
415 - }
416 -
417 - rrd2rrdr_set_timestamps(r);
418 - return r;
419 - }
420 - // v2 query
421 -
422 - // parse all the group-by label keys
423 - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
424 - if (qt->request.group_by[g].group_by & RRDR_GROUP_BY_LABEL &&
425 - qt->request.group_by[g].group_by_label && *qt->request.group_by[g].group_by_label)
426 - qt->group_by[g].used = quoted_strings_splitter_query_group_by_label(
427 - qt->request.group_by[g].group_by_label, qt->group_by[g].label_keys,
428 - GROUP_BY_MAX_LABEL_KEYS);
429 -
430 - if (!qt->group_by[g].used)
431 - qt->request.group_by[g].group_by &= ~RRDR_GROUP_BY_LABEL;
432 - }
433 -
434 - // make sure there are valid group-by methods
435 - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
436 - if(!(qt->request.group_by[g].group_by & SUPPORTED_GROUP_BY_METHODS))
437 - qt->request.group_by[g].group_by = (g == 0) ? RRDR_GROUP_BY_DIMENSION : RRDR_GROUP_BY_NONE;
438 - }
439 -
440 - bool query_has_percentage_of_group = query_target_has_percentage_of_group(qt);
441 -
442 - // merge all group-by options to upper levels,
443 - // so that the top level has all the groupings of the inner levels,
444 - // and each subsequent level has all the groupings of its inner levels.
445 - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) {
446 - if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE)
447 - continue;
448 -
449 - if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_SELECTED) {
450 - for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++)
451 - qt->request.group_by[r].group_by = RRDR_GROUP_BY_NONE;
452 - }
453 - else {
454 - for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++) {
455 - if (qt->request.group_by[r].group_by == RRDR_GROUP_BY_NONE)
456 - continue;
457 -
458 - if (qt->request.group_by[r].group_by != RRDR_GROUP_BY_SELECTED) {
459 - if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
460 - qt->request.group_by[g].group_by |= RRDR_GROUP_BY_INSTANCE;
461 - else
462 - qt->request.group_by[g].group_by |= qt->request.group_by[r].group_by;
463 -
464 - if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_LABEL) {
465 - for (size_t lr = 0; lr < qt->group_by[r].used; lr++) {
466 - bool found = false;
467 - for (size_t lg = 0; lg < qt->group_by[g].used; lg++) {
468 - if (strcmp(qt->group_by[g].label_keys[lg], qt->group_by[r].label_keys[lr]) == 0) {
469 - found = true;
470 - break;
471 - }
472 - }
473 -
474 - if (!found && qt->group_by[g].used < GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES)
475 - qt->group_by[g].label_keys[qt->group_by[g].used++] = qt->group_by[r].label_keys[lr];
476 - }
477 - }
478 - }
479 - }
480 - }
481 - }
482 -
483 - int added = 0;
484 - RRDR *first_r = NULL, *last_r = NULL;
485 - BUFFER *key = buffer_create(0, NULL);
486 - struct rrdr_group_by_entry *entries = onewayalloc_mallocz(owa, qt->query.used * sizeof(struct rrdr_group_by_entry));
487 - DICTIONARY *groups = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
488 - DICTIONARY *label_keys = NULL;
489 -
490 - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
491 - RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by;
492 - RRDR_GROUP_BY_FUNCTION aggregation_method = qt->request.group_by[g].aggregation;
493 -
494 - if(group_by == RRDR_GROUP_BY_NONE)
495 - break;
496 -
497 - memset(entries, 0, qt->query.used * sizeof(struct rrdr_group_by_entry));
498 - dictionary_flush(groups);
499 - added = 0;
500 -
501 - size_t hidden_dimensions = 0;
502 - bool final_grouping = (g == MAX_QUERY_GROUP_BY_PASSES - 1 || qt->request.group_by[g + 1].group_by == RRDR_GROUP_BY_NONE) ? true : false;
503 -
504 - if (final_grouping && (options & RRDR_OPTION_GROUP_BY_LABELS))
505 - label_keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0);
506 -
507 - QUERY_INSTANCE *last_qi = NULL;
508 - size_t priority = 0;
509 - time_t update_every_max = 0;
510 - for (size_t d = 0; d < qt->query.used; d++) {
511 - QUERY_METRIC *qm = query_metric(qt, d);
512 - QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id);
513 - QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
514 - QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id);
515 - QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
516 -
517 - if (qi != last_qi) {
518 - last_qi = qi;
519 -
520 - time_t update_every = rrdinstance_acquired_update_every(qi->ria);
521 - if (update_every > update_every_max)
522 - update_every_max = update_every;
523 - }
524 -
525 - priority = qd->priority;
526 -
527 - if(qm->status & RRDR_DIMENSION_HIDDEN)
528 - hidden_dimensions++;
529 -
530 - // --------------------------------------------------------------------
531 - // generate the group by key
532 -
533 - query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
534 -
535 - // lookup the key in the dictionary
536 -
537 - int pos = -1;
538 - int *set = dictionary_set(groups, buffer_tostring(key), &pos, sizeof(pos));
539 - if (*set == -1) {
540 - // the key just added to the dictionary
541 -
542 - *set = pos = added++;
543 -
544 - // ----------------------------------------------------------------
545 - // generate the dimension id
546 -
547 - query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
548 - entries[pos].id = string_strdupz(buffer_tostring(key));
549 -
550 - // ----------------------------------------------------------------
551 - // generate the dimension name
552 -
553 - query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
554 - entries[pos].name = string_strdupz(buffer_tostring(key));
555 -
556 - // add the rest of the info
557 - entries[pos].units = rrdinstance_acquired_units_dup(qi->ria);
558 - entries[pos].priority = priority;
559 -
560 - if (label_keys) {
561 - entries[pos].dl = dictionary_create_advanced(
562 - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
563 - NULL, sizeof(struct group_by_label_key));
564 - dictionary_register_insert_callback(entries[pos].dl, group_by_label_key_insert_cb, label_keys);
565 - dictionary_register_delete_callback(entries[pos].dl, group_by_label_key_delete_cb, label_keys);
566 - }
567 - } else {
568 - // the key found in the dictionary
569 - pos = *set;
570 - }
571 -
572 - entries[pos].count++;
573 -
574 - if (unlikely(priority < entries[pos].priority))
575 - entries[pos].priority = priority;
576 -
577 - if(g > 0)
578 - last_r->dgbs[qm->grouped_as.slot] = pos;
579 - else
580 - qm->grouped_as.first_slot = pos;
581 -
582 - qm->grouped_as.slot = pos;
583 - qm->grouped_as.id = entries[pos].id;
584 - qm->grouped_as.name = entries[pos].name;
585 - qm->grouped_as.units = entries[pos].units;
586 -
587 - // copy the dimension flags decided by the query target
588 - // we need this, because if a dimension is explicitly selected
589 - // the query target adds to it the non-zero flag
590 - qm->status |= RRDR_DIMENSION_GROUPED;
591 -
592 - if(query_has_percentage_of_group)
593 - // when the query has percentage of group
594 - // there will be no hidden dimensions in the final query,
595 - // so we have to remove the hidden flag from all dimensions
596 - entries[pos].od |= qm->status & ~RRDR_DIMENSION_HIDDEN;
597 - else
598 - entries[pos].od |= qm->status;
599 -
600 - if (entries[pos].dl)
601 - rrdlabels_walkthrough_read(rrdinstance_acquired_labels(qi->ria),
602 - rrdlabels_traversal_cb_to_group_by_label_key, entries[pos].dl);
603 - }
604 -
605 - RRDR *r = rrdr_create(owa, qt, added, qt->window.points);
606 - if (!r) {
607 - internal_error(true,
608 - "QUERY: cannot create group by RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
609 - qt->id, qt->window.after, qt->window.before, added, qt->window.points);
610 - goto cleanup;
611 - }
612 - // prevent double free at cleanup in case of error
613 - added = 0;
614 -
615 - // link this RRDR
616 - if(!last_r)
617 - first_r = last_r = r;
618 - else
619 - last_r->group_by.r = r;
620 -
621 - last_r = r;
622 -
623 - rrd2rrdr_set_timestamps(r);
624 -
625 - if(r->d) {
626 - r->dp = onewayalloc_callocz(owa, r->d, sizeof(*r->dp));
627 - r->dview = onewayalloc_callocz(owa, r->d, sizeof(*r->dview));
628 - r->dgbc = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbc));
629 - r->dqp = onewayalloc_callocz(owa, r->d, sizeof(STORAGE_POINT));
630 -
631 - if(!final_grouping)
632 - // this is where we are going to store the slot in the next RRDR
633 - // that we are going to group by the dimension of this RRDR
634 - r->dgbs = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbs));
635 -
636 - if (label_keys) {
637 - r->dl = onewayalloc_callocz(owa, r->d, sizeof(DICTIONARY *));
638 - r->label_keys = label_keys;
639 - label_keys = NULL;
640 - }
641 -
642 - if(r->n) {
643 - r->gbc = onewayalloc_callocz(owa, r->n * r->d, sizeof(*r->gbc));
644 -
645 - if(hidden_dimensions && ((group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) || (aggregation_method == RRDR_GROUP_BY_FUNCTION_PERCENTAGE)))
646 - // this is where we are going to group the hidden dimensions
647 - r->vh = onewayalloc_mallocz(owa, r->n * r->d * sizeof(*r->vh));
648 - }
649 - }
650 -
651 - // zero r (dimension options, names, and ids)
652 - // this is required, because group-by may lead to empty dimensions
653 - for (size_t d = 0; d < r->d; d++) {
654 - r->di[d] = entries[d].id;
655 - r->dn[d] = entries[d].name;
656 -
657 - r->od[d] = entries[d].od;
658 - r->du[d] = entries[d].units;
659 - r->dp[d] = entries[d].priority;
660 - r->dgbc[d] = entries[d].count;
661 -
662 - if (r->dl)
663 - r->dl[d] = entries[d].dl;
664 - }
665 -
666 - // initialize partial trimming
667 - r->partial_data_trimming.max_update_every = update_every_max * 2;
668 - r->partial_data_trimming.expected_after =
669 - (!query_target_aggregatable(qt) &&
670 - qt->window.before >= qt->window.now - r->partial_data_trimming.max_update_every) ?
671 - qt->window.before - r->partial_data_trimming.max_update_every :
672 - qt->window.before;
673 - r->partial_data_trimming.trimmed_after = qt->window.before;
674 -
675 - // make all values empty
676 - if(r->n && r->d) {
677 - for (size_t i = 0; i != r->n; i++) {
678 - NETDATA_DOUBLE *cn = &r->v[i * r->d];
679 - RRDR_VALUE_FLAGS *co = &r->o[i * r->d];
680 - NETDATA_DOUBLE *ar = &r->ar[i * r->d];
681 - NETDATA_DOUBLE *vh = r->vh ? &r->vh[i * r->d] : NULL;
682 -
683 - for (size_t d = 0; d < r->d; d++) {
684 - cn[d] = NAN;
685 - ar[d] = 0.0;
686 - co[d] = RRDR_VALUE_EMPTY;
687 -
688 - if (vh)
689 - vh[d] = NAN;
690 - }
691 - }
692 - }
693 - }
694 -
695 - if(!first_r || !last_r)
696 - goto cleanup;
697 -
698 - r_tmp = rrdr_create(owa, qt, 1, qt->window.points);
699 - if (!r_tmp) {
700 - internal_error(true,
701 - "QUERY: cannot create group by temporary RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
702 - qt->id, qt->window.after, qt->window.before, 1, qt->window.points);
703 - goto cleanup;
704 - }
705 - rrd2rrdr_set_timestamps(r_tmp);
706 - r_tmp->group_by.r = first_r;
707 -
708 -cleanup:
709 - if(!first_r || !last_r || !r_tmp) {
710 - if(r_tmp) {
711 - r_tmp->group_by.r = NULL;
712 - rrdr_free(owa, r_tmp);
713 - }
714 -
715 - if(first_r) {
716 - RRDR *r = first_r;
717 - while (r) {
718 - r_tmp = r->group_by.r;
719 - r->group_by.r = NULL;
720 - rrdr_free(owa, r);
721 - r = r_tmp;
722 - }
723 - }
724 -
725 - if(entries && added) {
726 - for (int d = 0; d < added; d++) {
727 - string_freez(entries[d].id);
728 - string_freez(entries[d].name);
729 - string_freez(entries[d].units);
730 - dictionary_destroy(entries[d].dl);
731 - }
732 - }
733 - dictionary_destroy(label_keys);
734 -
735 - first_r = last_r = r_tmp = NULL;
736 - }
737 -
738 - buffer_free(key);
739 - onewayalloc_freez(owa, entries);
740 - dictionary_destroy(groups);
741 -
742 - return r_tmp;
743 -}
744 -
745 -void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, size_t d_tmp,
746 - RRDR_GROUP_BY_FUNCTION group_by_aggregate_function,
747 - STORAGE_POINT *query_points, size_t pass __maybe_unused) {
748 - if(!r_tmp || r_dst == r_tmp || !(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED))
749 - return;
750 -
751 - internal_fatal(r_dst->n != r_tmp->n, "QUERY: group-by source and destination do not have the same number of rows");
752 - internal_fatal(d_dst >= r_dst->d, "QUERY: group-by destination dimension number exceeds destination RRDR size");
753 - internal_fatal(d_tmp >= r_tmp->d, "QUERY: group-by source dimension number exceeds source RRDR size");
754 - internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)");
755 - internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)");
756 -
757 - bool hidden_dimension_on_percentage_of_group = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh;
758 -
759 - if(!hidden_dimension_on_percentage_of_group) {
760 - r_dst->od[d_dst] |= r_tmp->od[d_tmp];
761 - storage_point_merge_to(r_dst->dqp[d_dst], *query_points);
762 - }
763 -
764 - // do the group_by
765 - for(size_t i = 0; i != rrdr_rows(r_tmp) ; i++) {
766 -
767 - size_t idx_tmp = i * r_tmp->d + d_tmp;
768 - NETDATA_DOUBLE n_tmp = r_tmp->v[ idx_tmp ];
769 - RRDR_VALUE_FLAGS o_tmp = r_tmp->o[ idx_tmp ];
770 - NETDATA_DOUBLE ar_tmp = r_tmp->ar[ idx_tmp ];
771 -
772 - if(o_tmp & RRDR_VALUE_EMPTY)
773 - continue;
774 -
775 - size_t idx_dst = i * r_dst->d + d_dst;
776 - NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_group) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ];
777 - RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ];
778 - NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ];
779 - uint32_t *gbc = &r_dst->gbc[ idx_dst ];
780 -
781 - switch(group_by_aggregate_function) {
782 - default:
783 - case RRDR_GROUP_BY_FUNCTION_AVERAGE:
784 - case RRDR_GROUP_BY_FUNCTION_SUM:
785 - case RRDR_GROUP_BY_FUNCTION_PERCENTAGE:
786 - if(isnan(*cn))
787 - *cn = n_tmp;
788 - else
789 - *cn += n_tmp;
790 - break;
791 -
792 - case RRDR_GROUP_BY_FUNCTION_MIN:
793 - if(isnan(*cn) || n_tmp < *cn)
794 - *cn = n_tmp;
795 - break;
796 -
797 - case RRDR_GROUP_BY_FUNCTION_MAX:
798 - if(isnan(*cn) || n_tmp > *cn)
799 - *cn = n_tmp;
800 - break;
801 -
802 - case RRDR_GROUP_BY_FUNCTION_EXTREMES:
803 - // For extremes, we need to keep track of the value with the maximum absolute value
804 - if(isnan(*cn) || fabsndd(n_tmp) > fabsndd(*cn))
805 - *cn = n_tmp;
806 - break;
807 - }
808 -
809 - if(!hidden_dimension_on_percentage_of_group) {
810 - *co &= ~RRDR_VALUE_EMPTY;
811 - *co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL));
812 - *ar += ar_tmp;
813 - (*gbc)++;
814 - }
815 - }
816 -}
817 -
818 -void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
819 - time_t trimmable_after = r->partial_data_trimming.expected_after;
820 -
821 - // find the point just before the trimmable ones
822 - ssize_t i = (ssize_t)r->n - 1;
823 - for( ; i >= 0 ;i--) {
824 - if (r->t[i] < trimmable_after)
825 - break;
826 - }
827 -
828 - if(unlikely(i < 0))
829 - return;
830 -
831 - // internal_error(true, "Found trimmable index %zd (from 0 to %zu)", i, r->n - 1);
832 -
833 - size_t last_row_gbc = 0;
834 - for (; i < (ssize_t)r->n; i++) {
835 - size_t row_gbc = 0;
836 - for (size_t d = 0; d < r->d; d++) {
837 - if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
838 - continue;
839 -
840 - row_gbc += r->gbc[ i * r->d + d ];
841 - }
842 -
843 - // internal_error(true, "GBC of index %zd is %zu", i, row_gbc);
844 -
845 - if (unlikely(r->t[i] >= trimmable_after && (row_gbc < last_row_gbc || !row_gbc))) {
846 - // discard the rest of the points
847 - // internal_error(true, "Discarding points %zd to %zu", i, r->n - 1);
848 - r->partial_data_trimming.trimmed_after = r->t[i];
849 - r->rows = i;
850 - break;
851 - }
852 - else
853 - last_row_gbc = row_gbc;
854 - }
855 -}
856 -
857 -void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
858 - if(!r->vh)
859 - return;
860 -
861 - if(query_target_aggregatable(r->internal.qt) && query_has_group_by_aggregation_percentage(r->internal.qt))
862 - return;
863 -
864 - for(size_t i = 0; i < r->n ;i++) {
865 - NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
866 - NETDATA_DOUBLE *ch = &r->vh[ i * r->d ];
867 -
868 - for(size_t d = 0; d < r->d ;d++) {
869 - NETDATA_DOUBLE n = cn[d];
870 - NETDATA_DOUBLE h = ch[d];
871 -
872 - if(isnan(n))
873 - cn[d] = 0.0;
874 -
875 - else if(isnan(h))
876 - cn[d] = 100.0;
877 -
878 - else
879 - cn[d] = n * 100.0 / (n + h);
880 - }
881 - }
882 -}
883 -
884 -
885 -void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) {
886 - if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt))
887 - return;
888 -
889 - size_t global_min_max_values = 0;
890 - NETDATA_DOUBLE global_min = NAN, global_max = NAN;
891 -
892 - for(size_t i = 0; i != r->n ;i++) {
893 - NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
894 - RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
895 -
896 - NETDATA_DOUBLE total = 0;
897 - for (size_t d = 0; d < r->d; d++) {
898 - if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
899 - continue;
900 -
901 - if(co[d] & RRDR_VALUE_EMPTY)
902 - continue;
903 -
904 - total += cn[d];
905 - }
906 -
907 - if(total == 0.0)
908 - total = 1.0;
909 -
910 - for (size_t d = 0; d < r->d; d++) {
911 - if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
912 - continue;
913 -
914 - if(co[d] & RRDR_VALUE_EMPTY)
915 - continue;
916 -
917 - NETDATA_DOUBLE n = cn[d];
918 - n = cn[d] = n * 100.0 / total;
919 -
920 - if(unlikely(!global_min_max_values++))
921 - global_min = global_max = n;
922 - else {
923 - if(n < global_min)
924 - global_min = n;
925 - if(n > global_max)
926 - global_max = n;
927 - }
928 - }
929 - }
930 -
931 - r->view.min = global_min;
932 - r->view.max = global_max;
933 -
934 - if(!r->dview)
935 - // v1 query
936 - return;
937 -
938 - // v2 query
939 -
940 - for (size_t d = 0; d < r->d; d++) {
941 - if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
942 - continue;
943 -
944 - size_t count = 0;
945 - NETDATA_DOUBLE min = 0.0, max = 0.0, sum = 0.0, ars = 0.0;
946 - for(size_t i = 0; i != r->rows ;i++) { // we use r->rows to respect trimming
947 - size_t idx = i * r->d + d;
948 -
949 - RRDR_VALUE_FLAGS o = r->o[ idx ];
950 -
951 - if (o & RRDR_VALUE_EMPTY)
952 - continue;
953 -
954 - NETDATA_DOUBLE ar = r->ar[ idx ];
955 - ars += ar;
956 -
957 - NETDATA_DOUBLE n = r->v[ idx ];
958 - sum += n;
959 -
960 - if(!count++)
961 - min = max = n;
962 - else {
963 - if(n < min)
964 - min = n;
965 - if(n > max)
966 - max = n;
967 - }
968 - }
969 -
970 - r->dview[d] = (STORAGE_POINT) {
971 - .sum = sum,
972 - .count = count,
973 - .min = min,
974 - .max = max,
975 - .anomaly_count = (size_t)(ars * (NETDATA_DOUBLE)count),
976 - };
977 - }
978 -}
979 -
980 -RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
981 - QUERY_TARGET *qt = r_tmp->internal.qt;
982 -
983 - if(!r_tmp->group_by.r) {
984 - // v1 query
985 - rrd2rrdr_convert_values_to_percentage_of_total(r_tmp);
986 - return r_tmp;
987 - }
988 - // v2 query
989 -
990 - // do the additional passes on RRDRs
991 - RRDR *last_r = r_tmp->group_by.r;
992 - rrdr2rrdr_group_by_calculate_percentage_of_group(last_r);
993 -
994 - RRDR *r = last_r->group_by.r;
995 - size_t pass = 0;
996 - while(r) {
997 - pass++;
998 - for(size_t d = 0; d < last_r->d ;d++) {
999 - rrd2rrdr_group_by_add_metric(r, last_r->dgbs[d], last_r, d,
1000 - qt->request.group_by[pass].aggregation,
1001 - &last_r->dqp[d], pass);
1002 - }
1003 - rrdr2rrdr_group_by_calculate_percentage_of_group(r);
1004 -
1005 - last_r = r;
1006 - r = last_r->group_by.r;
1007 - }
1008 -
1009 - // free all RRDRs except the last one
1010 - r = r_tmp;
1011 - while(r != last_r) {
1012 - r_tmp = r->group_by.r;
1013 - r->group_by.r = NULL;
1014 - rrdr_free(r->internal.owa, r);
1015 - r = r_tmp;
1016 - }
1017 - r = last_r;
1018 -
1019 - // find the final aggregation
1020 - RRDR_GROUP_BY_FUNCTION aggregation = qt->request.group_by[0].aggregation;
1021 - for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
1022 - if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE)
1023 - aggregation = qt->request.group_by[g].aggregation;
1024 -
1025 - if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before)
1026 - rrdr2rrdr_group_by_partial_trimming(r);
1027 -
1028 - // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
1029 - size_t global_min_max_values = 0;
1030 - size_t dimensions_nonzero = 0;
1031 - NETDATA_DOUBLE global_min = NAN, global_max = NAN;
1032 - for (size_t d = 0; d < r->d; d++) {
1033 - if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
1034 - continue;
1035 -
1036 - size_t points_nonzero = 0;
1037 - NETDATA_DOUBLE min = 0, max = 0, sum = 0, ars = 0;
1038 - size_t count = 0;
1039 -
1040 - for(size_t i = 0; i != r->n ;i++) {
1041 - size_t idx = i * r->d + d;
1042 -
1043 - NETDATA_DOUBLE *cn = &r->v[ idx ];
1044 - RRDR_VALUE_FLAGS *co = &r->o[ idx ];
1045 - NETDATA_DOUBLE *ar = &r->ar[ idx ];
1046 - uint32_t gbc = r->gbc[ idx ];
1047 -
1048 - if(likely(gbc)) {
1049 - *co &= ~RRDR_VALUE_EMPTY;
1050 -
1051 - if(gbc != r->dgbc[d])
1052 - *co |= RRDR_VALUE_PARTIAL;
1053 -
1054 - NETDATA_DOUBLE n;
1055 -
1056 - sum += *cn;
1057 - ars += *ar;
1058 -
1059 - if(aggregation == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt))
1060 - n = (*cn /= gbc);
1061 - else
1062 - n = *cn;
1063 -
1064 - if(!query_target_aggregatable(qt))
1065 - *ar /= gbc;
1066 -
1067 - if(islessgreater(n, 0.0))
1068 - points_nonzero++;
1069 -
1070 - if(unlikely(!count))
1071 - min = max = n;
1072 - else {
1073 - if(n < min)
1074 - min = n;
1075 -
1076 - if(n > max)
1077 - max = n;
1078 - }
1079 -
1080 - if(unlikely(!global_min_max_values++))
1081 - global_min = global_max = n;
1082 - else {
1083 - if(n < global_min)
1084 - global_min = n;
1085 -
1086 - if(n > global_max)
1087 - global_max = n;
1088 - }
1089 -
1090 - count += gbc;
1091 - }
1092 - }
1093 -
1094 - if(points_nonzero) {
1095 - r->od[d] |= RRDR_DIMENSION_NONZERO;
1096 - dimensions_nonzero++;
1097 - }
1098 -
1099 - r->dview[d] = (STORAGE_POINT) {
1100 - .sum = sum,
1101 - .count = count,
1102 - .min = min,
1103 - .max = max,
1104 - .anomaly_count = (size_t)(ars * RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER / 100.0),
1105 - };
1106 - }
1107 -
1108 - r->view.min = global_min;
1109 - r->view.max = global_max;
1110 -
1111 - if(!dimensions_nonzero && (qt->window.options & RRDR_OPTION_NONZERO)) {
1112 - // all dimensions are zero
1113 - // remove the nonzero option
1114 - qt->window.options &= ~RRDR_OPTION_NONZERO;
1115 - }
1116 -
1117 - rrd2rrdr_convert_values_to_percentage_of_total(r);
1118 -
1119 - // update query instance counts in query host and query context
1120 - {
1121 - size_t h = 0, c = 0, i = 0;
1122 - for(; h < qt->nodes.used ; h++) {
1123 - QUERY_NODE *qn = &qt->nodes.array[h];
1124 -
1125 - for(; c < qt->contexts.used ;c++) {
1126 - QUERY_CONTEXT *qc = &qt->contexts.array[c];
1127 -
1128 - if(!rrdcontext_acquired_belongs_to_host(qc->rca, qn->rrdhost))
1129 - break;
1130 -
1131 - for(; i < qt->instances.used ;i++) {
1132 - QUERY_INSTANCE *qi = &qt->instances.array[i];
1133 -
1134 - if(!rrdinstance_acquired_belongs_to_context(qi->ria, qc->rca))
1135 - break;
1136 -
1137 - if(qi->metrics.queried) {
1138 - qc->instances.queried++;
1139 - qn->instances.queried++;
1140 - }
1141 - else if(qi->metrics.failed) {
1142 - qc->instances.failed++;
1143 - qn->instances.failed++;
1144 - }
1145 - }
1146 - }
1147 - }
1148 - }
1149 -
1150 - return r;
1151 -}
1152 -
1153 -static int compare_contributions(const void *a, const void *b) {
1154 - const struct { size_t dim_idx; NETDATA_DOUBLE contribution; } *da = a;
1155 - const struct { size_t dim_idx; NETDATA_DOUBLE contribution; } *db = b;
1156 -
1157 - if (da->contribution > db->contribution) return -1;
1158 - if (da->contribution < db->contribution) return 1;
1159 - return 0;
1160 -}
1161 -
1162 -RRDR *rrd2rrdr_cardinality_limit(RRDR *r) {
1163 - QUERY_TARGET *qt = r->internal.qt;
1164 -
1165 - if(!qt || qt->request.cardinality_limit == 0 || r->d <= qt->request.cardinality_limit)
1166 - return r;
1167 -
1168 - ONEWAYALLOC *owa = r->internal.owa;
1169 -
1170 - // Calculate contribution of each dimension using dview statistics (sum of values)
1171 - NETDATA_DOUBLE *contributions = onewayalloc_mallocz(owa, r->d * sizeof(NETDATA_DOUBLE));
1172 -
1173 - // Count queried dimensions and get their contributions from dview
1174 - size_t queried_count = 0;
1175 - for (size_t d = 0; d < r->d; d++) {
1176 - contributions[d] = 0.0;
1177 -
1178 - if (!(r->od[d] & RRDR_DIMENSION_QUERIED))
1179 - continue;
1180 -
1181 - queried_count++;
1182 -
1183 - // Use the sum from dview if available, otherwise fall back to manual calculation
1184 - if(r->dview && !isnan(r->dview[d].sum)) {
1185 - contributions[d] = fabsndd(r->dview[d].sum);
1186 - } else {
1187 - // Fallback: calculate manually from values
1188 - for(size_t i = 0; i < r->rows; i++) {
1189 - size_t idx = i * r->d + d;
1190 -
1191 - if(r->o[idx] & RRDR_VALUE_EMPTY)
1192 - continue;
1193 -
1194 - NETDATA_DOUBLE value = r->v[idx];
1195 - if(!isnan(value))
1196 - contributions[d] += fabsndd(value);
1197 - }
1198 - }
1199 - }
1200 -
1201 - // If we don't need to reduce, return original
1202 - if(queried_count <= qt->request.cardinality_limit) {
1203 - onewayalloc_freez(owa, contributions);
1204 - return r;
1205 - }
1206 -
1207 - // Create array of dimension indices sorted by contribution (descending)
1208 - struct {
1209 - size_t dim_idx;
1210 - NETDATA_DOUBLE contribution;
1211 - } *sorted_dims = onewayalloc_mallocz(owa, queried_count * sizeof(*sorted_dims));
1212 -
1213 - size_t sorted_idx = 0;
1214 - for (size_t d = 0; d < r->d; d++) {
1215 - if (r->od[d] & RRDR_DIMENSION_QUERIED) {
1216 - sorted_dims[sorted_idx].dim_idx = d;
1217 - sorted_dims[sorted_idx].contribution = contributions[d];
1218 - sorted_idx++;
1219 - }
1220 - }
1221 -
1222 - // Sort by contribution (descending)
1223 - qsort(sorted_dims, queried_count, sizeof(*sorted_dims), compare_contributions);
1224 -
1225 - // Create new RRDR with limited dimensions
1226 - size_t new_d = qt->request.cardinality_limit;
1227 - size_t remaining_count = queried_count - (qt->request.cardinality_limit - 1);
1228 - if(remaining_count > 0)
1229 - new_d = qt->request.cardinality_limit; // Keep one slot for "remaining N dimensions"
1230 - else
1231 - new_d = queried_count; // No remaining dimensions needed
1232 -
1233 - RRDR *new_r = rrdr_create(owa, qt, new_d, r->n);
1234 - if (!new_r) {
1235 - internal_error(true, "QUERY: cannot create cardinality limited RRDR");
1236 - onewayalloc_freez(owa, contributions);
1237 - onewayalloc_freez(owa, sorted_dims);
1238 - return r;
1239 - }
1240 -
1241 - // Copy basic metadata from original RRDR
1242 - new_r->view = r->view;
1243 - new_r->time_grouping = r->time_grouping;
1244 - new_r->partial_data_trimming = r->partial_data_trimming;
1245 - new_r->rows = r->rows;
1246 -
1247 - // Copy timestamps
1248 - memcpy(new_r->t, r->t, r->n * sizeof(time_t));
1249 -
1250 - // Setup arrays for new RRDR
1251 - if(new_r->d) {
1252 - new_r->dp = onewayalloc_callocz(owa, new_r->d, sizeof(*new_r->dp));
1253 - new_r->dview = onewayalloc_callocz(owa, new_r->d, sizeof(*new_r->dview));
1254 -
1255 - if(new_r->n) {
1256 - // Initialize all values as empty
1257 - for (size_t i = 0; i < new_r->n; i++) {
1258 - for (size_t d = 0; d < new_r->d; d++) {
1259 - size_t idx = i * new_r->d + d;
1260 - new_r->v[idx] = NAN;
1261 - new_r->ar[idx] = 0.0;
1262 - new_r->o[idx] = RRDR_VALUE_EMPTY;
1263 - }
1264 - }
1265 - }
1266 - }
1267 -
1268 - // Copy top dimensions
1269 - size_t kept_dimensions = (remaining_count > 0) ? qt->request.cardinality_limit - 1 : queried_count;
1270 -
1271 - for (size_t i = 0; i < kept_dimensions; i++) {
1272 - size_t src_d = sorted_dims[i].dim_idx;
1273 -
1274 - // Copy metadata
1275 - new_r->di[i] = string_dup(r->di[src_d]);
1276 - new_r->dn[i] = string_dup(r->dn[src_d]);
1277 - new_r->od[i] = r->od[src_d];
1278 - new_r->du[i] = string_dup(r->du[src_d]);
1279 - new_r->dp[i] = r->dp[src_d];
1280 -
1281 - // Copy data
1282 - for (size_t row = 0; row < r->rows; row++) {
1283 - size_t src_idx = row * r->d + src_d;
1284 - size_t dst_idx = row * new_r->d + i;
1285 -
1286 - new_r->v[dst_idx] = r->v[src_idx];
1287 - new_r->ar[dst_idx] = r->ar[src_idx];
1288 - new_r->o[dst_idx] = r->o[src_idx];
1289 - }
1290 -
1291 - // Copy dview stats
1292 - if(r->dview)
1293 - new_r->dview[i] = r->dview[src_d];
1294 - }
1295 -
1296 - // Create "remaining N dimensions" if needed
1297 - if (remaining_count > 0) {
1298 - size_t remaining_idx = kept_dimensions;
1299 -
1300 - char remaining_name[256];
1301 - snprintfz(remaining_name, sizeof(remaining_name), "remaining %zu dimension%s",
1302 - remaining_count, remaining_count == 1 ? "" : "s");
1303 -
1304 - new_r->di[remaining_idx] = string_strdupz(remaining_name);
1305 - new_r->dn[remaining_idx] = string_strdupz(remaining_name);
1306 - new_r->od[remaining_idx] = RRDR_DIMENSION_QUERIED | RRDR_DIMENSION_NONZERO;
1307 -
1308 - // Use the units from the first remaining dimension
1309 - if(kept_dimensions < queried_count) {
1310 - size_t first_remaining_d = sorted_dims[kept_dimensions].dim_idx;
1311 - new_r->du[remaining_idx] = string_dup(r->du[first_remaining_d]);
1312 - new_r->dp[remaining_idx] = r->dp[first_remaining_d];
1313 - }
1314 -
1315 - // Aggregate remaining dimensions
1316 - NETDATA_DOUBLE sum = 0.0, min = NAN, max = NAN, ars = 0.0;
1317 - size_t count = 0;
1318 -
1319 - for (size_t row = 0; row < r->rows; row++) {
1320 - size_t dst_idx = row * new_r->d + remaining_idx;
1321 - NETDATA_DOUBLE aggregated_value = 0.0;
1322 - NETDATA_DOUBLE aggregated_ar = 0.0;
1323 - RRDR_VALUE_FLAGS aggregated_flags = RRDR_VALUE_NOTHING;
1324 - bool has_values = false;
1325 -
1326 - for (size_t i = kept_dimensions; i < queried_count; i++) {
1327 - size_t src_d = sorted_dims[i].dim_idx;
1328 - size_t src_idx = row * r->d + src_d;
1329 -
1330 - if(!(r->o[src_idx] & RRDR_VALUE_EMPTY)) {
1331 - NETDATA_DOUBLE value = r->v[src_idx];
1332 - if(!isnan(value)) {
1333 - aggregated_value += value;
1334 - aggregated_ar += r->ar[src_idx];
1335 - aggregated_flags |= (r->o[src_idx] & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL));
1336 - has_values = true;
1337 - }
1338 - }
1339 - }
1340 -
1341 - if(has_values) {
1342 - new_r->v[dst_idx] = aggregated_value;
1343 - new_r->ar[dst_idx] = aggregated_ar;
1344 - new_r->o[dst_idx] = aggregated_flags & ~RRDR_VALUE_EMPTY;
1345 -
1346 - // Update statistics for dview
1347 - sum += aggregated_value;
1348 - ars += aggregated_ar;
1349 - if(count == 0) {
1350 - min = max = aggregated_value;
1351 - } else {
1352 - if(aggregated_value < min) min = aggregated_value;
1353 - if(aggregated_value > max) max = aggregated_value;
1354 - }
1355 - count++;
1356 - } else {
1357 - new_r->v[dst_idx] = NAN;
1358 - new_r->ar[dst_idx] = 0.0;
1359 - new_r->o[dst_idx] = RRDR_VALUE_EMPTY;
1360 - }
1361 - }
1362 -
1363 - // Set dview for remaining dimension
1364 - if(new_r->dview) {
1365 - new_r->dview[remaining_idx] = (STORAGE_POINT) {
1366 - .sum = sum,
1367 - .count = count,
1368 - .min = min,
1369 - .max = max,
1370 - .anomaly_count = (size_t)(ars * RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER / 100.0),
1371 - };
1372 - }
1373 - }
1374 -
1375 - // Cleanup
1376 - onewayalloc_freez(owa, contributions);
1377 - onewayalloc_freez(owa, sorted_dims);
1378 -
1379 - // Free the original RRDR
1380 - rrdr_free(owa, r);
1381 -
1382 - return new_r;
1383 -}
src/web/api/queries/query-internal.h
+11
@@ -90,12 +90,23 @@ void rrd2rrdr_query_ops_release(QUERY_ENGINE_OPS *ops);
90 time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier);
91 void rrd2rrdr_query_ops_freeall(RRDR *r);
92
93 +// query execution
94 +void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops);
95 +
96 // time aggregation
97 void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush);
98 NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush);
99 void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method);
100
101 // group by
102 +struct group_by_label_key {
103 + DICTIONARY *values;
104 +};
105 +
106 +void group_by_label_key_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data);
107 +void group_by_label_key_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused);
108 +int rrdlabels_traversal_cb_to_group_by_label_key(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data);
109 +void rrd2rrdr_set_timestamps(RRDR *r);
110 RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt);
111 void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r);
112 void rrdr2rrdr_group_by_partial_trimming(RRDR *r);
src/web/api/queries/query-window.c new
+303
@@ -0,0 +1,303 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "query-internal.h"
4 +
5 +// #define DEBUG_QUERY_LOGIC 1
6 +
7 +#ifdef DEBUG_QUERY_LOGIC
8 +#define query_debug_log_init() BUFFER *debug_log = buffer_create(1000)
9 +#define query_debug_log(args...) buffer_sprintf(debug_log, ##args)
10 +#define query_debug_log_fin() { \
11 + netdata_log_info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
12 + buffer_free(debug_log); \
13 + debug_log = NULL; \
14 + }
15 +#define query_debug_log_free() do { buffer_free(debug_log); } while(0)
16 +#else
17 +#define query_debug_log_init() debug_dummy()
18 +#define query_debug_log(args...) debug_dummy()
19 +#define query_debug_log_fin() debug_dummy()
20 +#define query_debug_log_free() debug_dummy()
21 +#endif
22 +
23 +bool query_target_calculate_window(QUERY_TARGET *qt) {
24 + if (unlikely(!qt)) return false;
25 +
26 + size_t points_requested = (long)qt->request.points;
27 + time_t after_requested = qt->request.after;
28 + time_t before_requested = qt->request.before;
29 + RRDR_TIME_GROUPING group_method = qt->request.time_group_method;
30 + time_t resampling_time_requested = qt->request.resampling_time;
31 + RRDR_OPTIONS options = qt->window.options;
32 + size_t tier = qt->request.tier;
33 + time_t update_every = qt->db.minimum_latest_update_every_s ? qt->db.minimum_latest_update_every_s : 1;
34 +
35 + // RULES
36 + // points_requested = 0
37 + // the user wants all the natural points the database has
38 + //
39 + // after_requested = 0
40 + // the user wants to start the query from the oldest point in our database
41 + //
42 + // before_requested = 0
43 + // the user wants the query to end to the latest point in our database
44 + //
45 + // when natural points are wanted, the query has to be aligned to the update_every
46 + // of the database
47 +
48 + size_t points_wanted = points_requested;
49 + time_t after_wanted = after_requested;
50 + time_t before_wanted = before_requested;
51 +
52 + bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
53 + bool automatic_natural_points = (points_wanted == 0);
54 + bool relative_period_requested = false;
55 + bool natural_points = (options & RRDR_OPTION_NATURAL_POINTS) || automatic_natural_points;
56 + bool before_is_aligned_to_db_end = false;
57 +
58 + query_debug_log_init();
59 +
60 + if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
61 + relative_period_requested = true;
62 + natural_points = true;
63 + options |= RRDR_OPTION_NATURAL_POINTS;
64 + query_debug_log(":relative+natural");
65 + }
66 +
67 + // if the user wants virtual points, make sure we do it
68 + if (options & RRDR_OPTION_VIRTUAL_POINTS)
69 + natural_points = false;
70 +
71 + // set the right flag about natural and virtual points
72 + if (natural_points) {
73 + options |= RRDR_OPTION_NATURAL_POINTS;
74 +
75 + if (options & RRDR_OPTION_VIRTUAL_POINTS)
76 + options &= ~RRDR_OPTION_VIRTUAL_POINTS;
77 + }
78 + else {
79 + options |= RRDR_OPTION_VIRTUAL_POINTS;
80 +
81 + if (options & RRDR_OPTION_NATURAL_POINTS)
82 + options &= ~RRDR_OPTION_NATURAL_POINTS;
83 + }
84 +
85 + if (after_wanted == 0 || before_wanted == 0) {
86 + relative_period_requested = true;
87 +
88 + time_t first_entry_s = qt->db.first_time_s;
89 + time_t last_entry_s = qt->db.last_time_s;
90 +
91 + if (first_entry_s == 0 || last_entry_s == 0) {
92 + internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld)", qt->id, first_entry_s, last_entry_s);
93 + after_wanted = qt->window.after;
94 + before_wanted = qt->window.before;
95 +
96 + if(after_wanted == before_wanted)
97 + after_wanted = before_wanted - update_every;
98 +
99 + if (points_wanted == 0) {
100 + points_wanted = (before_wanted - after_wanted) / update_every;
101 + query_debug_log(":zero points_wanted %zu", points_wanted);
102 + }
103 + }
104 + else {
105 + query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s);
106 +
107 + if (after_wanted == 0) {
108 + after_wanted = first_entry_s;
109 + query_debug_log(":zero after_wanted %ld", after_wanted);
110 + }
111 +
112 + if (before_wanted == 0) {
113 + before_wanted = last_entry_s;
114 + before_is_aligned_to_db_end = true;
115 + query_debug_log(":zero before_wanted %ld", before_wanted);
116 + }
117 +
118 + if (points_wanted == 0) {
119 + points_wanted = (last_entry_s - first_entry_s) / update_every;
120 + query_debug_log(":zero points_wanted %zu", points_wanted);
121 + }
122 + }
123 + }
124 +
125 + if (points_wanted == 0) {
126 + points_wanted = 600;
127 + query_debug_log(":zero600 points_wanted %zu", points_wanted);
128 + }
129 +
130 + // convert our before_wanted and after_wanted to absolute
131 + rrdr_relative_window_to_absolute_query(&after_wanted, &before_wanted, NULL, unittest_running);
132 + query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
133 +
134 + if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && nd_profile.storage_tiers > 1) {
135 + update_every = rrdset_find_natural_update_every_for_timeframe(
136 + qt, after_wanted, before_wanted, points_wanted, options, tier);
137 +
138 + if (update_every <= 0) update_every = qt->db.minimum_latest_update_every_s;
139 + query_debug_log(":natural update every %ld", update_every);
140 + }
141 +
142 + // this is the update_every of the query
143 + // it may be different to the update_every of the database
144 + time_t query_granularity = (natural_points) ? update_every : 1;
145 + if (query_granularity <= 0) query_granularity = 1;
146 + query_debug_log(":query_granularity %ld", query_granularity);
147 +
148 + // align before_wanted and after_wanted to query_granularity
149 + if (before_wanted % query_granularity) {
150 + before_wanted -= before_wanted % query_granularity;
151 + query_debug_log(":granularity align before_wanted %ld", before_wanted);
152 + }
153 +
154 + if (after_wanted % query_granularity) {
155 + after_wanted -= after_wanted % query_granularity;
156 + query_debug_log(":granularity align after_wanted %ld", after_wanted);
157 + }
158 +
159 + // automatic_natural_points is set when the user wants all the points available in the database
160 + if (automatic_natural_points) {
161 + points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
162 + if (unlikely(points_wanted <= 0)) points_wanted = 1;
163 + query_debug_log(":auto natural points_wanted %zu", points_wanted);
164 + }
165 +
166 + time_t duration = before_wanted - after_wanted;
167 +
168 + // if the resampling time is too big, extend the duration to the past
169 + if (unlikely(resampling_time_requested > duration)) {
170 + after_wanted = before_wanted - resampling_time_requested;
171 + duration = before_wanted - after_wanted;
172 + query_debug_log(":resampling after_wanted %ld", after_wanted);
173 + }
174 +
175 + // if the duration is not aligned to resampling time
176 + // extend the duration to the past, to avoid a gap at the chart
177 + // only when the missing duration is above 1/10th of a point
178 + if (resampling_time_requested > query_granularity && duration % resampling_time_requested) {
179 + time_t delta = duration % resampling_time_requested;
180 + if (delta > resampling_time_requested / 10) {
181 + after_wanted -= resampling_time_requested - delta;
182 + duration = before_wanted - after_wanted;
183 + query_debug_log(":resampling2 after_wanted %ld", after_wanted);
184 + }
185 + }
186 +
187 + // the available points of the query
188 + size_t points_available = (duration + 1) / query_granularity;
189 + if (unlikely(points_available <= 0)) points_available = 1;
190 + query_debug_log(":points_available %zu", points_available);
191 +
192 + if (points_wanted > points_available) {
193 + points_wanted = points_available;
194 + query_debug_log(":max points_wanted %zu", points_wanted);
195 + }
196 +
197 + if(points_wanted > 86400 && !unittest_running) {
198 + points_wanted = 86400;
199 + query_debug_log(":absolute max points_wanted %zu", points_wanted);
200 + }
201 +
202 + // calculate the desired grouping of source data points
203 + size_t group = points_available / points_wanted;
204 + if (group == 0) group = 1;
205 +
206 + // round "group" to the closest integer
207 + if (points_available % points_wanted > points_wanted / 2)
208 + group++;
209 +
210 + query_debug_log(":group %zu", group);
211 +
212 + if (points_wanted * group * query_granularity < (size_t)duration) {
213 + // the grouping we are going to do, is not enough
214 + // to cover the entire duration requested, so
215 + // we have to change the number of points, to make sure we will
216 + // respect the timeframe as closely as possibly
217 +
218 + // let's see how many points are the optimal
219 + points_wanted = points_available / group;
220 +
221 + if (points_wanted * group < points_available)
222 + points_wanted++;
223 +
224 + if (unlikely(points_wanted == 0))
225 + points_wanted = 1;
226 +
227 + query_debug_log(":optimal points %zu", points_wanted);
228 + }
229 +
230 + // resampling_time_requested enforces a certain grouping multiple
231 + NETDATA_DOUBLE resampling_divisor = 1.0;
232 + size_t resampling_group = 1;
233 + if (unlikely(resampling_time_requested > query_granularity)) {
234 + // the points we should group to satisfy gtime
235 + resampling_group = resampling_time_requested / query_granularity;
236 + if (unlikely(resampling_time_requested % query_granularity))
237 + resampling_group++;
238 +
239 + query_debug_log(":resampling group %zu", resampling_group);
240 +
241 + // adapt group according to resampling_group
242 + if (unlikely(group < resampling_group)) {
243 + group = resampling_group; // do not allow grouping below the desired one
244 + query_debug_log(":group less res %zu", group);
245 + }
246 + if (unlikely(group % resampling_group)) {
247 + group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
248 + query_debug_log(":group mod res %zu", group);
249 + }
250 +
251 + // resampling_divisor = group / resampling_group;
252 + resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested;
253 + query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor);
254 + }
255 +
256 + // now that we have group, align the requested timeframe to fit it.
257 + if (aligned && before_wanted % (group * query_granularity)) {
258 + if (before_is_aligned_to_db_end)
259 + before_wanted -= before_wanted % (time_t)(group * query_granularity);
260 + else
261 + before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity);
262 + query_debug_log(":align before_wanted %ld", before_wanted);
263 + }
264 +
265 + after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity;
266 + query_debug_log(":final after_wanted %ld", after_wanted);
267 +
268 + duration = before_wanted - after_wanted;
269 + query_debug_log(":final duration %ld", duration + 1);
270 +
271 + query_debug_log_fin();
272 +
273 + internal_error(points_wanted != duration / (query_granularity * group) + 1,
274 + "QUERY: points_wanted %zu is not points %zu",
275 + points_wanted, (size_t)(duration / (query_granularity * group) + 1));
276 +
277 + internal_error(group < resampling_group,
278 + "QUERY: group %zu is less than the desired group points %zu",
279 + group, resampling_group);
280 +
281 + internal_error(group > resampling_group && group % resampling_group,
282 + "QUERY: group %zu is not a multiple of the desired group points %zu",
283 + group, resampling_group);
284 +
285 + // -------------------------------------------------------------------------
286 + // update QUERY_TARGET with our calculations
287 +
288 + qt->window.after = after_wanted;
289 + qt->window.before = before_wanted;
290 + qt->window.relative = relative_period_requested;
291 + qt->window.points = points_wanted;
292 + qt->window.group = group;
293 + qt->window.time_group_method = group_method;
294 + qt->window.time_group_options = qt->request.time_group_options;
295 + qt->window.query_granularity = query_granularity;
296 + qt->window.resampling_group = resampling_group;
297 + qt->window.resampling_divisor = resampling_divisor;
298 + qt->window.options = options;
299 + qt->window.tier = tier;
300 + qt->window.aligned = aligned;
301 +
302 + return true;
303 +}
src/web/api/queries/query.c
-748
@@ -2,454 +2,6 @@
2
3 #include "query-internal.h"
4
5 -// ----------------------------------------------------------------------------
6 -// helpers to find our way in RRDR
7 -
8 -ALWAYS_INLINE
9 -static RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long rrdr_line) {
10 - return &r->o[ rrdr_line * r->d ];
11 -}
12 -
13 -ALWAYS_INLINE
14 -static NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rrdr_line) {
15 - return &r->v[ rrdr_line * r->d ];
16 -}
17 -
18 -ALWAYS_INLINE
19 -static long rrdr_line_init(RRDR *r __maybe_unused, time_t t __maybe_unused, long rrdr_line) {
20 - rrdr_line++;
21 -
22 - internal_fatal(rrdr_line >= (long)r->n,
23 - "QUERY: requested to step above RRDR size for query '%s'",
24 - r->internal.qt->id);
25 -
26 - internal_fatal(r->t[rrdr_line] != t,
27 - "QUERY: wrong timestamp at RRDR line %ld, expected %ld, got %ld, of query '%s'",
28 - rrdr_line, r->t[rrdr_line], t, r->internal.qt->id);
29 -
30 - return rrdr_line;
31 -}
32 -
33 -// ----------------------------------------------------------------------------
34 -// dimension level query engine
35 -
36 -#define query_interpolate_point(this_point, last_point, now) do { \
37 - if(likely( \
38 - /* the point to interpolate is more than 1s wide */ \
39 - (this_point).sp.end_time_s - (this_point).sp.start_time_s > 1 \
40 - \
41 - /* the two points are exactly next to each other */ \
42 - && (last_point).sp.end_time_s == (this_point).sp.start_time_s \
43 - \
44 - /* both points are valid numbers */ \
45 - && netdata_double_isnumber((this_point).value) \
46 - && netdata_double_isnumber((last_point).value) \
47 - \
48 - )) { \
49 - (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).sp.end_time_s - (now)) / (NETDATA_DOUBLE)((this_point).sp.end_time_s - (this_point).sp.start_time_s)); \
50 - (this_point).sp.end_time_s = now; \
51 - } \
52 -} while(0)
53 -
54 -#define query_add_point_to_group(r, point, ops, add_flush) do { \
55 - if(likely(netdata_double_isnumber((point).value))) { \
56 - if(likely(fpclassify((point).value) != FP_ZERO)) \
57 - (ops)->group_points_non_zero++; \
58 - \
59 - if(unlikely((point).sp.flags & SN_FLAG_RESET)) \
60 - (ops)->group_value_flags |= RRDR_VALUE_RESET; \
61 - \
62 - time_grouping_add(r, (point).value, add_flush); \
63 - \
64 - storage_point_merge_to((ops)->group_point, (point).sp); \
65 - if(!(point).added) \
66 - storage_point_merge_to((ops)->query_point, (point).sp); \
67 - } \
68 - \
69 - (ops)->group_points_added++; \
70 -} while(0)
71 -
72 -NOT_INLINE_HOT static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops) {
73 - QUERY_TARGET *qt = r->internal.qt;
74 - QUERY_METRIC *qm = ops->qm;
75 -
76 - const RRDR_TIME_GROUPING add_flush = r->time_grouping.add_flush;
77 -
78 - ops->group_point = STORAGE_POINT_UNSET;
79 - ops->query_point = STORAGE_POINT_UNSET;
80 -
81 - RRDR_OPTIONS options = qt->window.options;
82 - size_t points_wanted = qt->window.points;
83 - time_t after_wanted = qt->window.after;
84 - time_t before_wanted = qt->window.before; (void)before_wanted;
85 -
86 -// bool debug_this = false;
87 -// if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
88 -// debug_this = true;
89 -
90 - size_t points_added = 0;
91 -
92 - long rrdr_line = -1;
93 - bool use_anomaly_bit_as_value = (r->internal.qt->window.options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
94 -
95 - NETDATA_DOUBLE min = r->view.min, max = r->view.max;
96 -
97 - QUERY_POINT last2_point = QUERY_POINT_EMPTY;
98 - QUERY_POINT last1_point = QUERY_POINT_EMPTY;
99 - QUERY_POINT new_point = QUERY_POINT_EMPTY;
100 -
101 - // ONE POINT READ-AHEAD
102 - // when we switch plans, we read-ahead a point from the next plan
103 - // to join them smoothly at the exact time the next plan begins
104 - STORAGE_POINT next1_point = STORAGE_POINT_UNSET;
105 -
106 - time_t now_start_time = after_wanted - ops->query_granularity;
107 - time_t now_end_time = after_wanted + ops->view_update_every - ops->query_granularity;
108 -
109 - size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch;
110 - size_t query_is_finished_counter = 0;
111 -
112 - // The main loop, based on the query granularity we need
113 - for( ; points_added < points_wanted && query_is_finished_counter <= 10 ;
114 - now_start_time = now_end_time, now_end_time += ops->view_update_every) {
115 -
116 - if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {
117 - query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s);
118 - db_points_read_since_plan_switch = 0;
119 - }
120 -
121 - // read all the points of the db, prior to the time we need (now_end_time)
122 -
123 - size_t count_same_end_time = 0;
124 - while(count_same_end_time < 100) {
125 - if(likely(count_same_end_time == 0)) {
126 - last2_point = last1_point;
127 - last1_point = new_point;
128 - }
129 -
130 - if(unlikely(storage_engine_query_is_finished(ops->seqh))) {
131 - query_is_finished_counter++;
132 -
133 - if(count_same_end_time != 0) {
134 - last2_point = last1_point;
135 - last1_point = new_point;
136 - }
137 - new_point = QUERY_POINT_EMPTY;
138 - new_point.sp.start_time_s = last1_point.sp.end_time_s;
139 - new_point.sp.end_time_s = now_end_time;
140 -//
141 -// if(debug_this) netdata_log_info("QUERY: is finished() returned true");
142 -//
143 - break;
144 - }
145 - else
146 - query_is_finished_counter = 0;
147 -
148 - // fetch the new point
149 - {
150 - STORAGE_POINT sp;
151 - if(likely(storage_point_is_unset(next1_point))) {
152 - db_points_read_since_plan_switch++;
153 - sp = storage_engine_query_next_metric(ops->seqh);
154 - ops->db_points_read_per_tier[ops->tier]++;
155 - ops->db_total_points_read++;
156 -
157 - if(unlikely(options & RRDR_OPTION_ABSOLUTE))
158 - storage_point_make_positive(sp);
159 - }
160 - else {
161 - // ONE POINT READ-AHEAD
162 - sp = next1_point;
163 - storage_point_unset(next1_point);
164 - db_points_read_since_plan_switch = 1;
165 - }
166 -
167 - // ONE POINT READ-AHEAD
168 - if(unlikely(query_plan_should_switch_plan(ops, sp.end_time_s) &&
169 - query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s))) {
170 -
171 - // The end time of the current point, crosses our plans (tiers)
172 - // so, we switched plan (tier)
173 - //
174 - // There are 2 cases now:
175 - //
176 - // A. the entire point of the previous plan is to the future of point from the next plan
177 - // B. part of the point of the previous plan overlaps with the point from the next plan
178 -
179 - STORAGE_POINT sp2 = storage_engine_query_next_metric(ops->seqh);
180 - ops->db_points_read_per_tier[ops->tier]++;
181 - ops->db_total_points_read++;
182 -
183 - if(unlikely(options & RRDR_OPTION_ABSOLUTE))
184 - storage_point_make_positive(sp);
185 -
186 - if(sp.start_time_s > sp2.start_time_s)
187 - // the point from the previous plan is useless
188 - sp = sp2;
189 - else
190 - // let the query run from the previous plan
191 - // but setting this will also cut off the interpolation
192 - // of the point from the previous plan
193 - next1_point = sp2;
194 - }
195 -
196 - new_point.sp = sp;
197 - new_point.added = false;
198 - query_point_set_id(new_point, ops->db_total_points_read);
199 -
200 -// if(debug_this)
201 -// netdata_log_info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld",
202 -// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
203 -//
204 - // get the right value from the point we got
205 - if(likely(!storage_point_is_unset(sp) && !storage_point_is_gap(sp))) {
206 -
207 - if(unlikely(use_anomaly_bit_as_value))
208 - new_point.value = storage_point_anomaly_rate(new_point.sp);
209 -
210 - else {
211 - switch (ops->tier_query_fetch) {
212 - default:
213 - case TIER_QUERY_FETCH_AVERAGE:
214 - new_point.value = sp.sum / (NETDATA_DOUBLE)sp.count;
215 - break;
216 -
217 - case TIER_QUERY_FETCH_MIN:
218 - new_point.value = sp.min;
219 - break;
220 -
221 - case TIER_QUERY_FETCH_MAX:
222 - new_point.value = sp.max;
223 - break;
224 -
225 - case TIER_QUERY_FETCH_SUM:
226 - new_point.value = sp.sum;
227 - break;
228 - }
229 - }
230 - }
231 - else
232 - new_point.value = NAN;
233 - }
234 -
235 - // check if the db is giving us zero duration points
236 - if(unlikely(db_points_read_since_plan_switch > 1 &&
237 - new_point.sp.start_time_s == new_point.sp.end_time_s)) {
238 -
239 - internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned "
240 - "point %zu from %ld to %ld, that are both equal",
241 - qt->id, query_metric_id(qt, qm),
242 - new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s);
243 -
244 - new_point.sp.start_time_s = new_point.sp.end_time_s - ops->tier_ptr->db_update_every_s;
245 - }
246 -
247 - // check if the db is advancing the query
248 - if(unlikely(db_points_read_since_plan_switch > 1 &&
249 - new_point.sp.end_time_s <= last1_point.sp.end_time_s)) {
250 -
251 - internal_error(true,
252 - "QUERY: '%s', dimension '%s' next_metric() returned "
253 - "point %zu from %ld to %ld, before the "
254 - "last point %zu from %ld to %ld, "
255 - "now is %ld to %ld",
256 - qt->id, query_metric_id(qt, qm),
257 - new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s,
258 - last1_point.id, last1_point.sp.start_time_s, last1_point.sp.end_time_s,
259 - now_start_time, now_end_time);
260 -
261 - count_same_end_time++;
262 - continue;
263 - }
264 - count_same_end_time = 0;
265 -
266 - // decide how to use this point
267 - if(likely(new_point.sp.end_time_s < now_end_time)) { // likely to favor tier0
268 - // this db point ends before our now_end_time
269 -
270 - if(likely(new_point.sp.end_time_s >= now_start_time)) { // likely to favor tier0
271 - // this db point ends after our now_start time
272 -
273 - query_add_point_to_group(r, new_point, ops, add_flush);
274 - new_point.added = true;
275 - }
276 - else {
277 - // we don't need this db point
278 - // it is totally outside our current time-frame
279 -
280 - // this is desirable for the first point of the query
281 - // because it allows us to interpolate the next point
282 - // at exactly the time we will want
283 -
284 - // we only log if this is not point 1
285 - internal_error(new_point.sp.end_time_s < ops->plan_expanded_after &&
286 - db_points_read_since_plan_switch > 1,
287 - "QUERY: '%s', dimension '%s' next_metric() "
288 - "returned point %zu from %ld time %ld, "
289 - "which is entirely before our current timeframe %ld to %ld "
290 - "(and before the entire query, after %ld, before %ld)",
291 - qt->id, query_metric_id(qt, qm),
292 - new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s,
293 - now_start_time, now_end_time,
294 - ops->plan_expanded_after, ops->plan_expanded_before);
295 - }
296 -
297 - }
298 - else {
299 - // the point ends in the future
300 - // so, we will interpolate it below, at the inner loop
301 - break;
302 - }
303 - }
304 -
305 - if(unlikely(count_same_end_time)) {
306 - internal_error(true,
307 - "QUERY: '%s', dimension '%s', the database does not advance the query,"
308 - " it returned an end time less or equal to the end time of the last "
309 - "point we got %ld, %zu times",
310 - qt->id, query_metric_id(qt, qm),
311 - last1_point.sp.end_time_s, count_same_end_time);
312 -
313 - if(unlikely(new_point.sp.end_time_s <= last1_point.sp.end_time_s))
314 - new_point.sp.end_time_s = now_end_time;
315 - }
316 -
317 - time_t stop_time = new_point.sp.end_time_s;
318 - if(unlikely(!storage_point_is_unset(next1_point) && next1_point.start_time_s >= now_end_time)) {
319 - // ONE POINT READ-AHEAD
320 - // the point crosses the start time of the
321 - // read ahead storage point we have read
322 - stop_time = next1_point.start_time_s;
323 - }
324 -
325 - // the inner loop
326 - // we have 3 points in memory: last2, last1, new
327 - // we select the one to use based on their timestamps
328 -
329 - internal_fatal(now_end_time > stop_time || points_added >= points_wanted,
330 - "QUERY: first part of query provides invalid point to interpolate (now_end_time %ld, stop_time %ld",
331 - now_end_time, stop_time);
332 -
333 - do {
334 - // now_start_time is wrong in this loop
335 - // but, we don't need it
336 -
337 - QUERY_POINT current_point;
338 -
339 - if(likely(now_end_time > new_point.sp.start_time_s)) {
340 - // it is time for our NEW point to be used
341 - current_point = new_point;
342 - new_point.added = true; // first copy, then set it, so that new_point will not be added again
343 - query_interpolate_point(current_point, last1_point, now_end_time);
344 -
345 -// internal_error(current_point.id > 0
346 -// && last1_point.id == 0
347 -// && current_point.end_time > after_wanted
348 -// && current_point.end_time > now_end_time,
349 -// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
350 -// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
351 -// " but we could really favor by having last_point1 in this query.",
352 -// qt->id, string2str(qm->dimension.id),
353 -// after_wanted, before_wanted,
354 -// ops.view_update_every, ops.query_granularity,
355 -// current_point.id, current_point.start_time, current_point.end_time,
356 -// now_end_time);
357 - }
358 - else if(likely(now_end_time <= last1_point.sp.end_time_s)) {
359 - // our LAST point is still valid
360 - current_point = last1_point;
361 - last1_point.added = true; // first copy, then set it, so that last1_point will not be added again
362 - query_interpolate_point(current_point, last2_point, now_end_time);
363 -
364 -// internal_error(current_point.id > 0
365 -// && last2_point.id == 0
366 -// && current_point.end_time > after_wanted
367 -// && current_point.end_time > now_end_time,
368 -// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
369 -// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
370 -// " but we could really favor by having last_point2 in this query.",
371 -// qt->id, string2str(qm->dimension.id),
372 -// after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
373 -// current_point.id, current_point.start_time, current_point.end_time,
374 -// now_end_time);
375 - }
376 - else {
377 - // a GAP, we don't have a value this time
378 - current_point = QUERY_POINT_EMPTY;
379 - }
380 -
381 - query_add_point_to_group(r, current_point, ops, add_flush);
382 -
383 - rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line);
384 - size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
385 -
386 - // find the place to store our values
387 - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index];
388 -
389 - // update the dimension options
390 - if(likely(ops->group_points_non_zero))
391 - r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
392 -
393 - // store the specific point options
394 - *rrdr_value_options_ptr = ops->group_value_flags;
395 -
396 - // store the group value
397 - NETDATA_DOUBLE group_value = time_grouping_flush(r, rrdr_value_options_ptr, add_flush);
398 - r->v[rrdr_o_v_index] = group_value;
399 -
400 - r->ar[rrdr_o_v_index] = storage_point_anomaly_rate(ops->group_point);
401 -
402 - if(likely(points_added || r->internal.queries_count)) {
403 - // find the min/max across all dimensions
404 -
405 - if(unlikely(group_value < min)) min = group_value;
406 - if(unlikely(group_value > max)) max = group_value;
407 -
408 - }
409 - else {
410 - // runs only when r->internal.queries_count == 0 && points_added == 0
411 - // so, on the first point added for the query.
412 - min = max = group_value;
413 - }
414 -
415 - points_added++;
416 - ops->group_points_added = 0;
417 - ops->group_value_flags = RRDR_VALUE_NOTHING;
418 - ops->group_points_non_zero = 0;
419 - ops->group_point = STORAGE_POINT_UNSET;
420 -
421 - now_end_time += ops->view_update_every;
422 - } while(now_end_time <= stop_time && points_added < points_wanted);
423 -
424 - // the loop above increased "now" by ops->view_update_every,
425 - // but the main loop will increase it too,
426 - // so, let's undo the last iteration of this loop
427 - now_end_time -= ops->view_update_every;
428 - }
429 - query_planer_finalize_remaining_plans(ops);
430 -
431 - qm->query_points = ops->query_point;
432 -
433 - // fill the rest of the points with empty values
434 - while (points_added < points_wanted) {
435 - rrdr_line++;
436 - size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr;
437 - r->o[rrdr_o_v_index] = RRDR_VALUE_EMPTY;
438 - r->v[rrdr_o_v_index] = 0.0;
439 - r->ar[rrdr_o_v_index] = 0.0;
440 - points_added++;
441 - }
442 -
443 - r->internal.queries_count++;
444 - r->view.min = min;
445 - r->view.max = max;
446 -
447 - r->stats.result_points_generated += points_added;
448 - r->stats.db_points_read += ops->db_total_points_read;
449 - for(size_t tr = 0; tr < nd_profile.storage_tiers; tr++)
450 - qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr];
451 -}
452 -
5 // ----------------------------------------------------------------------------
6 // fill RRDR for the whole chart
7
@@ -523,306 +75,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
75 }
76 #endif // NETDATA_INTERNAL_CHECKS
77
526 -// #define DEBUG_QUERY_LOGIC 1
527 -
528 -#ifdef DEBUG_QUERY_LOGIC
529 -#define query_debug_log_init() BUFFER *debug_log = buffer_create(1000)
530 -#define query_debug_log(args...) buffer_sprintf(debug_log, ##args)
531 -#define query_debug_log_fin() { \
532 - netdata_log_info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
533 - buffer_free(debug_log); \
534 - debug_log = NULL; \
535 - }
536 -#define query_debug_log_free() do { buffer_free(debug_log); } while(0)
537 -#else
538 -#define query_debug_log_init() debug_dummy()
539 -#define query_debug_log(args...) debug_dummy()
540 -#define query_debug_log_fin() debug_dummy()
541 -#define query_debug_log_free() debug_dummy()
542 -#endif
543 -
544 -bool query_target_calculate_window(QUERY_TARGET *qt) {
545 - if (unlikely(!qt)) return false;
546 -
547 - size_t points_requested = (long)qt->request.points;
548 - time_t after_requested = qt->request.after;
549 - time_t before_requested = qt->request.before;
550 - RRDR_TIME_GROUPING group_method = qt->request.time_group_method;
551 - time_t resampling_time_requested = qt->request.resampling_time;
552 - RRDR_OPTIONS options = qt->window.options;
553 - size_t tier = qt->request.tier;
554 - time_t update_every = qt->db.minimum_latest_update_every_s ? qt->db.minimum_latest_update_every_s : 1;
555 -
556 - // RULES
557 - // points_requested = 0
558 - // the user wants all the natural points the database has
559 - //
560 - // after_requested = 0
561 - // the user wants to start the query from the oldest point in our database
562 - //
563 - // before_requested = 0
564 - // the user wants the query to end to the latest point in our database
565 - //
566 - // when natural points are wanted, the query has to be aligned to the update_every
567 - // of the database
568 -
569 - size_t points_wanted = points_requested;
570 - time_t after_wanted = after_requested;
571 - time_t before_wanted = before_requested;
572 -
573 - bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
574 - bool automatic_natural_points = (points_wanted == 0);
575 - bool relative_period_requested = false;
576 - bool natural_points = (options & RRDR_OPTION_NATURAL_POINTS) || automatic_natural_points;
577 - bool before_is_aligned_to_db_end = false;
578 -
579 - query_debug_log_init();
580 -
581 - if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
582 - relative_period_requested = true;
583 - natural_points = true;
584 - options |= RRDR_OPTION_NATURAL_POINTS;
585 - query_debug_log(":relative+natural");
586 - }
587 -
588 - // if the user wants virtual points, make sure we do it
589 - if (options & RRDR_OPTION_VIRTUAL_POINTS)
590 - natural_points = false;
591 -
592 - // set the right flag about natural and virtual points
593 - if (natural_points) {
594 - options |= RRDR_OPTION_NATURAL_POINTS;
595 -
596 - if (options & RRDR_OPTION_VIRTUAL_POINTS)
597 - options &= ~RRDR_OPTION_VIRTUAL_POINTS;
598 - }
599 - else {
600 - options |= RRDR_OPTION_VIRTUAL_POINTS;
601 -
602 - if (options & RRDR_OPTION_NATURAL_POINTS)
603 - options &= ~RRDR_OPTION_NATURAL_POINTS;
604 - }
605 -
606 - if (after_wanted == 0 || before_wanted == 0) {
607 - relative_period_requested = true;
608 -
609 - time_t first_entry_s = qt->db.first_time_s;
610 - time_t last_entry_s = qt->db.last_time_s;
611 -
612 - if (first_entry_s == 0 || last_entry_s == 0) {
613 - internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld)", qt->id, first_entry_s, last_entry_s);
614 - after_wanted = qt->window.after;
615 - before_wanted = qt->window.before;
616 -
617 - if(after_wanted == before_wanted)
618 - after_wanted = before_wanted - update_every;
619 -
620 - if (points_wanted == 0) {
621 - points_wanted = (before_wanted - after_wanted) / update_every;
622 - query_debug_log(":zero points_wanted %zu", points_wanted);
623 - }
624 - }
625 - else {
626 - query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s);
627 -
628 - if (after_wanted == 0) {
629 - after_wanted = first_entry_s;
630 - query_debug_log(":zero after_wanted %ld", after_wanted);
631 - }
632 -
633 - if (before_wanted == 0) {
634 - before_wanted = last_entry_s;
635 - before_is_aligned_to_db_end = true;
636 - query_debug_log(":zero before_wanted %ld", before_wanted);
637 - }
638 -
639 - if (points_wanted == 0) {
640 - points_wanted = (last_entry_s - first_entry_s) / update_every;
641 - query_debug_log(":zero points_wanted %zu", points_wanted);
642 - }
643 - }
644 - }
645 -
646 - if (points_wanted == 0) {
647 - points_wanted = 600;
648 - query_debug_log(":zero600 points_wanted %zu", points_wanted);
649 - }
650 -
651 - // convert our before_wanted and after_wanted to absolute
652 - rrdr_relative_window_to_absolute_query(&after_wanted, &before_wanted, NULL, unittest_running);
653 - query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
654 -
655 - if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && nd_profile.storage_tiers > 1) {
656 - update_every = rrdset_find_natural_update_every_for_timeframe(
657 - qt, after_wanted, before_wanted, points_wanted, options, tier);
658 -
659 - if (update_every <= 0) update_every = qt->db.minimum_latest_update_every_s;
660 - query_debug_log(":natural update every %ld", update_every);
661 - }
662 -
663 - // this is the update_every of the query
664 - // it may be different to the update_every of the database
665 - time_t query_granularity = (natural_points) ? update_every : 1;
666 - if (query_granularity <= 0) query_granularity = 1;
667 - query_debug_log(":query_granularity %ld", query_granularity);
668 -
669 - // align before_wanted and after_wanted to query_granularity
670 - if (before_wanted % query_granularity) {
671 - before_wanted -= before_wanted % query_granularity;
672 - query_debug_log(":granularity align before_wanted %ld", before_wanted);
673 - }
674 -
675 - if (after_wanted % query_granularity) {
676 - after_wanted -= after_wanted % query_granularity;
677 - query_debug_log(":granularity align after_wanted %ld", after_wanted);
678 - }
679 -
680 - // automatic_natural_points is set when the user wants all the points available in the database
681 - if (automatic_natural_points) {
682 - points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
683 - if (unlikely(points_wanted <= 0)) points_wanted = 1;
684 - query_debug_log(":auto natural points_wanted %zu", points_wanted);
685 - }
686 -
687 - time_t duration = before_wanted - after_wanted;
688 -
689 - // if the resampling time is too big, extend the duration to the past
690 - if (unlikely(resampling_time_requested > duration)) {
691 - after_wanted = before_wanted - resampling_time_requested;
692 - duration = before_wanted - after_wanted;
693 - query_debug_log(":resampling after_wanted %ld", after_wanted);
694 - }
695 -
696 - // if the duration is not aligned to resampling time
697 - // extend the duration to the past, to avoid a gap at the chart
698 - // only when the missing duration is above 1/10th of a point
699 - if (resampling_time_requested > query_granularity && duration % resampling_time_requested) {
700 - time_t delta = duration % resampling_time_requested;
701 - if (delta > resampling_time_requested / 10) {
702 - after_wanted -= resampling_time_requested - delta;
703 - duration = before_wanted - after_wanted;
704 - query_debug_log(":resampling2 after_wanted %ld", after_wanted);
705 - }
706 - }
707 -
708 - // the available points of the query
709 - size_t points_available = (duration + 1) / query_granularity;
710 - if (unlikely(points_available <= 0)) points_available = 1;
711 - query_debug_log(":points_available %zu", points_available);
712 -
713 - if (points_wanted > points_available) {
714 - points_wanted = points_available;
715 - query_debug_log(":max points_wanted %zu", points_wanted);
716 - }
717 -
718 - if(points_wanted > 86400 && !unittest_running) {
719 - points_wanted = 86400;
720 - query_debug_log(":absolute max points_wanted %zu", points_wanted);
721 - }
722 -
723 - // calculate the desired grouping of source data points
724 - size_t group = points_available / points_wanted;
725 - if (group == 0) group = 1;
726 -
727 - // round "group" to the closest integer
728 - if (points_available % points_wanted > points_wanted / 2)
729 - group++;
730 -
731 - query_debug_log(":group %zu", group);
732 -
733 - if (points_wanted * group * query_granularity < (size_t)duration) {
734 - // the grouping we are going to do, is not enough
735 - // to cover the entire duration requested, so
736 - // we have to change the number of points, to make sure we will
737 - // respect the timeframe as closely as possibly
738 -
739 - // let's see how many points are the optimal
740 - points_wanted = points_available / group;
741 -
742 - if (points_wanted * group < points_available)
743 - points_wanted++;
744 -
745 - if (unlikely(points_wanted == 0))
746 - points_wanted = 1;
747 -
748 - query_debug_log(":optimal points %zu", points_wanted);
749 - }
750 -
751 - // resampling_time_requested enforces a certain grouping multiple
752 - NETDATA_DOUBLE resampling_divisor = 1.0;
753 - size_t resampling_group = 1;
754 - if (unlikely(resampling_time_requested > query_granularity)) {
755 - // the points we should group to satisfy gtime
756 - resampling_group = resampling_time_requested / query_granularity;
757 - if (unlikely(resampling_time_requested % query_granularity))
758 - resampling_group++;
759 -
760 - query_debug_log(":resampling group %zu", resampling_group);
761 -
762 - // adapt group according to resampling_group
763 - if (unlikely(group < resampling_group)) {
764 - group = resampling_group; // do not allow grouping below the desired one
765 - query_debug_log(":group less res %zu", group);
766 - }
767 - if (unlikely(group % resampling_group)) {
768 - group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
769 - query_debug_log(":group mod res %zu", group);
770 - }
771 -
772 - // resampling_divisor = group / resampling_group;
773 - resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested;
774 - query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor);
775 - }
776 -
777 - // now that we have group, align the requested timeframe to fit it.
778 - if (aligned && before_wanted % (group * query_granularity)) {
779 - if (before_is_aligned_to_db_end)
780 - before_wanted -= before_wanted % (time_t)(group * query_granularity);
781 - else
782 - before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity);
783 - query_debug_log(":align before_wanted %ld", before_wanted);
784 - }
785 -
786 - after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity;
787 - query_debug_log(":final after_wanted %ld", after_wanted);
788 -
789 - duration = before_wanted - after_wanted;
790 - query_debug_log(":final duration %ld", duration + 1);
791 -
792 - query_debug_log_fin();
793 -
794 - internal_error(points_wanted != duration / (query_granularity * group) + 1,
795 - "QUERY: points_wanted %zu is not points %zu",
796 - points_wanted, (size_t)(duration / (query_granularity * group) + 1));
797 -
798 - internal_error(group < resampling_group,
799 - "QUERY: group %zu is less than the desired group points %zu",
800 - group, resampling_group);
801 -
802 - internal_error(group > resampling_group && group % resampling_group,
803 - "QUERY: group %zu is not a multiple of the desired group points %zu",
804 - group, resampling_group);
805 -
806 - // -------------------------------------------------------------------------
807 - // update QUERY_TARGET with our calculations
808 -
809 - qt->window.after = after_wanted;
810 - qt->window.before = before_wanted;
811 - qt->window.relative = relative_period_requested;
812 - qt->window.points = points_wanted;
813 - qt->window.group = group;
814 - qt->window.time_group_method = group_method;
815 - qt->window.time_group_options = qt->request.time_group_options;
816 - qt->window.query_granularity = query_granularity;
817 - qt->window.resampling_group = resampling_group;
818 - qt->window.resampling_divisor = resampling_divisor;
819 - qt->window.options = options;
820 - qt->window.tier = tier;
821 - qt->window.aligned = aligned;
822 -
823 - return true;
824 -}
825 -
78 // ----------------------------------------------------------------------------
79 // query entry point
80