percentage of group is now aggregatable at cloud across multiple nodes (#15109)
Costa Tsaousis committed
May 30, 2023 at 11:13 UTC
44b6c223b3e13774df45a96dd48588aa8a66ba42
4 files changed
+33
-14
web/api/formatters/json/json.c
+13
-4
@@ -270,8 +270,12 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
270
buffer_json_member_add_uint64(wb, "value", 0);
271
buffer_json_member_add_uint64(wb, "arp", 1);
272
buffer_json_member_add_uint64(wb, "pa", 2);
273
- if(expose_gbc)
274
- buffer_json_member_add_uint64(wb, "count", 3);
273
+ if(expose_gbc) {
274
+ if(r->vh)
275
+ buffer_json_member_add_uint64(wb, "hidden", 3);
276
+ else
277
+ buffer_json_member_add_uint64(wb, "count", 3);
278
+ }
279
buffer_json_object_close(wb);
280
281
buffer_json_member_add_array(wb, "data");
@@ -286,6 +290,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
290
// for each line in the array
291
for (i = start; i != end; i += step) {
292
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
293
+ NETDATA_DOUBLE *ch = (r->vh) ? &r->vh[ i * r->d ] : NULL;
294
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
295
NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
296
uint32_t *gbc = &r->gbc [ i * r->d ];
@@ -325,8 +330,12 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
330
buffer_json_add_array_item_uint64(wb, o);
331
332
// add the count
328
- if(expose_gbc)
329
- buffer_json_add_array_item_uint64(wb, gbc[d]);
333
+ if(expose_gbc) {
334
+ if(ch)
335
+ buffer_json_add_array_item_double(wb, ch[d]);
336
+ else
337
+ buffer_json_add_array_item_uint64(wb, gbc[d]);
338
+ }
339
340
buffer_json_array_close(wb); // point
341
}
web/api/formatters/rrd2json.h
+1
-1
@@ -87,7 +87,7 @@ int rrdset2value_api_v1(
87
);
88
89
static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) {
90
- if(unlikely(options & RRDR_OPTION_RETURN_RAW))
90
+ if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED)))
91
return true;
92
93
if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false;
web/api/netdata-swagger.yaml
+10
-1
@@ -241,6 +241,7 @@ paths:
241
A comma separated list of the groupings required.
242
All possible values can be combined together, except `selected`. If `selected` is given in the list, all others are ignored.
243
The order they are placed in the list is currently ignored.
244
+ This parameter is also accepted as `group_by[0]` and `group_by[1]` when multiple grouping passes are required.
245
required: false
246
schema:
247
type: array
@@ -261,6 +262,7 @@ paths:
262
in: query
263
description: |
264
A comma separated list of the label keys to group by their values. The order of the labels in the list is respected.
265
+ This parameter is also accepted as `group_by_label[0]` and `group_by_label[1]` when multiple grouping passes are required.
266
required: false
267
schema:
268
type: string
@@ -271,6 +273,7 @@ paths:
273
description: |
274
The aggregation function to apply when grouping metrics together.
275
When option `raw` is given, `average` and `avg` behave like `sum` and the caller is expected to calculate the average.
276
+ This parameter is also accepted as `aggregation[0]` and `aggregation[1]` when multiple grouping passes are required.
277
required: false
278
schema:
279
type: string
@@ -280,6 +283,7 @@ paths:
283
- avg
284
- average
285
- sum
286
+ - percentage
287
default: average
288
- $ref: '#/components/parameters/scopeNodes'
289
- $ref: '#/components/parameters/scopeContexts'
@@ -2741,8 +2745,13 @@ components:
2745
type: integer
2746
count:
2747
description: |
2744
- The number of metrics aggregated into this point. This exists only when the option `raw` is given to the query.
2748
+ The number of metrics aggregated into this point.
2749
+ This exists only when the option `raw` is given to the query and the final aggregation point is NOT `percentage`.
2750
type: integer
2751
+ hidden:
2752
+ description: |
2753
+ The sum of the non-selected dimensions aggregated for this group item point.
2754
+ This exists only when the option `raw` is given to the query and the final aggregation method is `percentage`.
2755
data:
2756
type: array
2757
items:
web/api/queries/query.c
+9
-8
@@ -2992,7 +2992,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2992
// initialize partial trimming
2993
r->partial_data_trimming.max_update_every = update_every_max;
2994
r->partial_data_trimming.expected_after =
2995
- (!(qt->window.options & RRDR_OPTION_RETURN_RAW) &&
2995
+ (!query_target_aggregatable(qt) &&
2996
qt->window.before >= qt->window.now - update_every_max) ?
2997
qt->window.before - update_every_max :
2998
qt->window.before;
@@ -3168,7 +3168,7 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
3168
}
3169
3170
static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
3171
- if(!r->vh)
3171
+ if(!r->vh || query_target_aggregatable(r->internal.qt))
3172
return;
3173
3174
for(size_t i = 0; i < r->n ;i++) {
@@ -3191,7 +3191,10 @@ static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
3191
}
3192
}
3193
3194
-static void rrd2rrdr_convert_to_percentage(RRDR *r) {
3194
+static void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) {
3195
+ if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt))
3196
+ return;
3197
+
3198
size_t global_min_max_values = 0;
3199
NETDATA_DOUBLE global_min = NAN, global_max = NAN;
3200
@@ -3289,8 +3292,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3292
3293
if(!r_tmp->group_by.r) {
3294
// v1 query
3292
- if(options & RRDR_OPTION_PERCENTAGE)
3293
- rrd2rrdr_convert_to_percentage(r_tmp);
3295
+ rrd2rrdr_convert_values_to_percentage_of_total(r_tmp);
3296
return r_tmp;
3297
}
3298
// v2 query
@@ -3330,7 +3332,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3332
if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE)
3333
aggregation = qt->request.group_by[g].aggregation;
3334
3333
- if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before)
3335
+ if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before)
3336
rrdr2rrdr_group_by_partial_trimming(r);
3337
3338
// apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
@@ -3422,8 +3424,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3424
qt->window.options &= ~RRDR_OPTION_NONZERO;
3425
}
3426
3425
- if(options & RRDR_OPTION_PERCENTAGE && !(options & RRDR_OPTION_RETURN_RAW))
3426
- rrd2rrdr_convert_to_percentage(r);
3427
+ rrd2rrdr_convert_values_to_percentage_of_total(r);
3428
3429
// update query instance counts in query host and query context
3430
{