Percentage of group aggregatable at cloud - fixed for backwards compatibility (#15126)
* percentage of group is now aggregatable at cloud across multiple nodes * do not break backwards compatibility with percentage-of-instance * calculate the percentage when percentage-of-instance is requested * increase capability version
Costa Tsaousis committed
May 31, 2023 at 19:06 UTC
6a629d999f3ea9728b534bb1cc1ba3c3bfe7063d
6 files changed
+57
-17
aclk/aclk_capas.c
+2
-2
@@ -13,7 +13,7 @@ const struct capability *aclk_get_agent_capas()
13
{ .name = "mc", .version = 0, .enabled = 0 },
14
{ .name = "ctx", .version = 1, .enabled = 1 },
15
{ .name = "funcs", .version = 1, .enabled = 1 },
16
- { .name = "http_api_v2", .version = 1, .enabled = 1 },
16
+ { .name = "http_api_v2", .version = 3, .enabled = 1 },
17
{ .name = "health", .version = 1, .enabled = 0 },
18
{ .name = "req_cancel", .version = 1, .enabled = 1 },
19
{ .name = NULL, .version = 0, .enabled = 0 }
@@ -39,7 +39,7 @@ struct capability *aclk_get_node_instance_capas(RRDHOST *host)
39
.enabled = enable_metric_correlations },
40
{ .name = "ctx", .version = 1, .enabled = 1 },
41
{ .name = "funcs", .version = 0, .enabled = 0 },
42
- { .name = "http_api_v2", .version = 2, .enabled = 1 },
42
+ { .name = "http_api_v2", .version = 3, .enabled = 1 },
43
{ .name = "health", .version = 1, .enabled = host->health.health_enabled },
44
{ .name = "req_cancel", .version = 1, .enabled = 1 },
45
{ .name = NULL, .version = 0, .enabled = 0 }
database/contexts/rrdcontext.h
+18
@@ -524,6 +524,24 @@ bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t b
524
525
#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW)
526
527
+static inline bool query_has_group_by_aggregation_percentage(QUERY_TARGET *qt) {
528
+
529
+ // backwards compatibility
530
+ // If the request was made with group_by = "percentage-of-instance"
531
+ // we need to send back "raw" output with "count"
532
+ // otherwise, we need to send back "raw" output with "hidden"
533
+
534
+ for(int g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
535
+ if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
536
+ return false;
537
+
538
+ if(qt->request.group_by[g].aggregation == RRDR_GROUP_BY_FUNCTION_PERCENTAGE)
539
+ return true;
540
+ }
541
+
542
+ return false;
543
+}
544
+
545
static inline bool query_target_has_percentage_of_group(QUERY_TARGET *qt) {
546
for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
547
if (qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
web/api/formatters/json/json.c
+15
-6
@@ -244,12 +244,12 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
244
//info("RRD2JSON(): %s: END", r->st->id);
245
}
246
247
-
247
void rrdr2json_v2(RRDR *r, BUFFER *wb) {
248
QUERY_TARGET *qt = r->internal.qt;
249
RRDR_OPTIONS options = qt->window.options;
250
252
- bool expose_gbc = query_target_aggregatable(qt);
251
+ bool send_fourth_number = query_target_aggregatable(qt);
252
+ bool fourth_number_is_vh = send_fourth_number && r->vh && query_has_group_by_aggregation_percentage(qt);
253
254
buffer_json_member_add_object(wb, "result");
255
@@ -270,8 +270,12 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
270
buffer_json_member_add_uint64(wb, "value", 0);
271
buffer_json_member_add_uint64(wb, "arp", 1);
272
buffer_json_member_add_uint64(wb, "pa", 2);
273
- if(expose_gbc)
274
- buffer_json_member_add_uint64(wb, "count", 3);
273
+ if(send_fourth_number) {
274
+ if(fourth_number_is_vh)
275
+ buffer_json_member_add_uint64(wb, "hidden", 3);
276
+ else
277
+ buffer_json_member_add_uint64(wb, "count", 3);
278
+ }
279
buffer_json_object_close(wb);
280
281
buffer_json_member_add_array(wb, "data");
@@ -286,6 +290,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
290
// for each line in the array
291
for (i = start; i != end; i += step) {
292
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
293
+ NETDATA_DOUBLE *ch = fourth_number_is_vh ? &r->vh[i * r->d ] : NULL;
294
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
295
NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
296
uint32_t *gbc = &r->gbc [ i * r->d ];
@@ -325,8 +330,12 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
330
buffer_json_add_array_item_uint64(wb, o);
331
332
// add the count
328
- if(expose_gbc)
329
- buffer_json_add_array_item_uint64(wb, gbc[d]);
333
+ if(send_fourth_number) {
334
+ if(fourth_number_is_vh)
335
+ buffer_json_add_array_item_double(wb, ch[d]);
336
+ else
337
+ buffer_json_add_array_item_uint64(wb, gbc[d]);
338
+ }
339
340
buffer_json_array_close(wb); // point
341
}
web/api/formatters/rrd2json.h
+1
-1
@@ -87,7 +87,7 @@ int rrdset2value_api_v1(
87
);
88
89
static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) {
90
- if(unlikely(options & RRDR_OPTION_RETURN_RAW))
90
+ if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED)))
91
return true;
92
93
if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false;
web/api/netdata-swagger.yaml
+10
-1
@@ -241,6 +241,7 @@ paths:
241
A comma separated list of the groupings required.
242
All possible values can be combined together, except `selected`. If `selected` is given in the list, all others are ignored.
243
The order they are placed in the list is currently ignored.
244
+ This parameter is also accepted as `group_by[0]` and `group_by[1]` when multiple grouping passes are required.
245
required: false
246
schema:
247
type: array
@@ -261,6 +262,7 @@ paths:
262
in: query
263
description: |
264
A comma separated list of the label keys to group by their values. The order of the labels in the list is respected.
265
+ This parameter is also accepted as `group_by_label[0]` and `group_by_label[1]` when multiple grouping passes are required.
266
required: false
267
schema:
268
type: string
@@ -271,6 +273,7 @@ paths:
273
description: |
274
The aggregation function to apply when grouping metrics together.
275
When option `raw` is given, `average` and `avg` behave like `sum` and the caller is expected to calculate the average.
276
+ This parameter is also accepted as `aggregation[0]` and `aggregation[1]` when multiple grouping passes are required.
277
required: false
278
schema:
279
type: string
@@ -280,6 +283,7 @@ paths:
283
- avg
284
- average
285
- sum
286
+ - percentage
287
default: average
288
- $ref: '#/components/parameters/scopeNodes'
289
- $ref: '#/components/parameters/scopeContexts'
@@ -2741,8 +2745,13 @@ components:
2745
type: integer
2746
count:
2747
description: |
2744
- The number of metrics aggregated into this point. This exists only when the option `raw` is given to the query.
2748
+ The number of metrics aggregated into this point.
2749
+ This exists only when the option `raw` is given to the query and the final aggregation point is NOT `percentage`.
2750
type: integer
2751
+ hidden:
2752
+ description: |
2753
+ The sum of the non-selected dimensions aggregated for this group item point.
2754
+ This exists only when the option `raw` is given to the query and the final aggregation method is `percentage`.
2755
data:
2756
type: array
2757
items:
web/api/queries/query.c
+11
-7
@@ -2992,7 +2992,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2992
// initialize partial trimming
2993
r->partial_data_trimming.max_update_every = update_every_max;
2994
r->partial_data_trimming.expected_after =
2995
- (!(qt->window.options & RRDR_OPTION_RETURN_RAW) &&
2995
+ (!query_target_aggregatable(qt) &&
2996
qt->window.before >= qt->window.now - update_every_max) ?
2997
qt->window.before - update_every_max :
2998
qt->window.before;
@@ -3171,6 +3171,9 @@ static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
3171
if(!r->vh)
3172
return;
3173
3174
+ if(query_target_aggregatable(r->internal.qt) && query_has_group_by_aggregation_percentage(r->internal.qt))
3175
+ return;
3176
+
3177
for(size_t i = 0; i < r->n ;i++) {
3178
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
3179
NETDATA_DOUBLE *ch = &r->vh[ i * r->d ];
@@ -3191,7 +3194,10 @@ static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
3194
}
3195
}
3196
3194
-static void rrd2rrdr_convert_to_percentage(RRDR *r) {
3197
+static void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) {
3198
+ if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt))
3199
+ return;
3200
+
3201
size_t global_min_max_values = 0;
3202
NETDATA_DOUBLE global_min = NAN, global_max = NAN;
3203
@@ -3289,8 +3295,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3295
3296
if(!r_tmp->group_by.r) {
3297
// v1 query
3292
- if(options & RRDR_OPTION_PERCENTAGE)
3293
- rrd2rrdr_convert_to_percentage(r_tmp);
3298
+ rrd2rrdr_convert_values_to_percentage_of_total(r_tmp);
3299
return r_tmp;
3300
}
3301
// v2 query
@@ -3330,7 +3335,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3335
if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE)
3336
aggregation = qt->request.group_by[g].aggregation;
3337
3333
- if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before)
3338
+ if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before)
3339
rrdr2rrdr_group_by_partial_trimming(r);
3340
3341
// apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
@@ -3422,8 +3427,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3427
qt->window.options &= ~RRDR_OPTION_NONZERO;
3428
}
3429
3425
- if(options & RRDR_OPTION_PERCENTAGE && !(options & RRDR_OPTION_RETURN_RAW))
3426
- rrd2rrdr_convert_to_percentage(r);
3430
+ rrd2rrdr_convert_values_to_percentage_of_total(r);
3431
3432
// update query instance counts in query host and query context
3433
{