/api/v2/data percentage calculation on grouped queries (#15100)
allow aggregation=percentage to calculate the percentage over any grouping
Costa Tsaousis committed
May 25, 2023 at 17:24 UTC
67c1b4419f41569e716ade122c0d5e6136cea128
2 files changed
+33
-21
web/api/queries/query.c
+32
-21
@@ -883,6 +883,9 @@ RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s) {
883
if(strcmp(s, "sum") == 0)
884
return RRDR_GROUP_BY_FUNCTION_SUM;
885
886
+ if(strcmp(s, "percentage") == 0)
887
+ return RRDR_GROUP_BY_FUNCTION_PERCENTAGE;
888
+
889
return RRDR_GROUP_BY_FUNCTION_AVERAGE;
890
}
891
@@ -900,6 +903,9 @@ const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_b
903
904
case RRDR_GROUP_BY_FUNCTION_SUM:
905
return "sum";
906
+
907
+ case RRDR_GROUP_BY_FUNCTION_PERCENTAGE:
908
+ return "percentage";
909
}
910
}
911
@@ -2555,9 +2561,9 @@ static void rrd2rrdr_set_timestamps(RRDR *r) {
2561
before_wanted, r->t[points_wanted - 1]);
2562
}
2563
2558
-static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) {
2564
+static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
2565
buffer_flush(key);
2560
- if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) {
2566
+ if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
2567
buffer_strcat(key, "__hidden_dimensions__");
2568
}
2569
else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
@@ -2599,9 +2605,9 @@ static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_b
2605
}
2606
}
2607
2602
-static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) {
2608
+static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
2609
buffer_flush(key);
2604
- if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) {
2610
+ if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
2611
buffer_strcat(key, "__hidden_dimensions__");
2612
}
2613
else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
@@ -2654,9 +2660,9 @@ static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by
2660
}
2661
}
2662
2657
-static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) {
2663
+static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_group) {
2664
buffer_flush(key);
2659
- if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) {
2665
+ if(unlikely(!query_has_percentage_of_group && qm->status & RRDR_DIMENSION_HIDDEN)) {
2666
buffer_strcat(key, "__hidden_dimensions__");
2667
}
2668
else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) {
@@ -2758,13 +2764,16 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2764
}
2765
2766
// make sure there are valid group-by methods
2761
- bool query_has_percentage_of_instance = false;
2767
+ bool query_has_percentage_of_group = false;
2768
for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) {
2769
if(!(qt->request.group_by[g].group_by & SUPPORTED_GROUP_BY_METHODS))
2770
qt->request.group_by[g].group_by = (g == 0) ? RRDR_GROUP_BY_DIMENSION : RRDR_GROUP_BY_NONE;
2771
2772
if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)
2767
- query_has_percentage_of_instance = true;
2773
+ query_has_percentage_of_group = true;
2774
+
2775
+ if(qt->request.group_by[g].aggregation == RRDR_GROUP_BY_FUNCTION_PERCENTAGE)
2776
+ query_has_percentage_of_group = true;
2777
}
2778
2779
// merge all group-by options to upper levels
@@ -2815,6 +2824,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2824
2825
for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) {
2826
RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by;
2827
+ RRDR_GROUP_BY_FUNCTION aggregation_method = qt->request.group_by[g].aggregation;
2828
2829
if(group_by == RRDR_GROUP_BY_NONE)
2830
break;
@@ -2855,7 +2865,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2865
// --------------------------------------------------------------------
2866
// generate the group by key
2867
2858
- query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance);
2868
+ query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
2869
2870
// lookup the key in the dictionary
2871
@@ -2869,13 +2879,13 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2879
// ----------------------------------------------------------------
2880
// generate the dimension id
2881
2872
- query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance);
2882
+ query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
2883
entries[pos].id = string_strdupz(buffer_tostring(key));
2884
2885
// ----------------------------------------------------------------
2886
// generate the dimension name
2887
2878
- query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance);
2888
+ query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_group);
2889
entries[pos].name = string_strdupz(buffer_tostring(key));
2890
2891
// add the rest of the info
@@ -2914,7 +2924,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2924
// the query target adds to it the non-zero flag
2925
qm->status |= RRDR_DIMENSION_GROUPED;
2926
2917
- if(query_has_percentage_of_instance)
2927
+ if(query_has_percentage_of_group)
2928
// when the query has percentage of instance
2929
// there will be no hidden dimensions in the final query
2930
// so we have to remove the hidden flag from all dimensions
@@ -2935,7 +2945,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2945
goto cleanup;
2946
}
2947
2938
- bool hidden_dimension_on_percentage_of_instance = hidden_dimensions && (group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE);
2948
+ bool hidden_dimension_on_percentage_of_group = hidden_dimensions && ((group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) || (aggregation_method == RRDR_GROUP_BY_FUNCTION_PERCENTAGE));
2949
2950
// prevent double cleanup in case of error
2951
added = 0;
@@ -2954,7 +2964,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2964
r->gbc = onewayalloc_callocz(owa, r->n * r->d, sizeof(*r->gbc));
2965
r->dqp = onewayalloc_callocz(owa, r->d, sizeof(STORAGE_POINT));
2966
2957
- if(hidden_dimension_on_percentage_of_instance)
2967
+ if(hidden_dimension_on_percentage_of_group)
2968
// this is where we are going to group the hidden dimensions
2969
r->vh = onewayalloc_mallocz(owa, r->n * r->d * sizeof(*r->vh));
2970
@@ -3073,9 +3083,9 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp,
3083
internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)");
3084
internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)");
3085
3076
- bool hidden_dimension_on_percentage_of_instance = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh;
3086
+ bool hidden_dimension_on_percentage_of_group = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh;
3087
3078
- if(!hidden_dimension_on_percentage_of_instance) {
3088
+ if(!hidden_dimension_on_percentage_of_group) {
3089
r_dst->od[d_dst] |= r_tmp->od[d_tmp];
3090
storage_point_merge_to(r_dst->dqp[d_dst], *query_points);
3091
}
@@ -3092,7 +3102,7 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp,
3102
continue;
3103
3104
size_t idx_dst = i * r_dst->d + d_dst;
3095
- NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_instance) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ];
3105
+ NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_group) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ];
3106
RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ];
3107
NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ];
3108
uint32_t *gbc = &r_dst->gbc[ idx_dst ];
@@ -3101,6 +3111,7 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp,
3111
default:
3112
case RRDR_GROUP_BY_FUNCTION_AVERAGE:
3113
case RRDR_GROUP_BY_FUNCTION_SUM:
3114
+ case RRDR_GROUP_BY_FUNCTION_PERCENTAGE:
3115
if(isnan(*cn))
3116
*cn = n_tmp;
3117
else
@@ -3118,7 +3129,7 @@ static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp,
3129
break;
3130
}
3131
3121
- if(!hidden_dimension_on_percentage_of_instance) {
3132
+ if(!hidden_dimension_on_percentage_of_group) {
3133
*co &= ~RRDR_VALUE_EMPTY;
3134
*co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL));
3135
*ar += ar_tmp;
@@ -3161,7 +3172,7 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
3172
}
3173
}
3174
3164
-static void rrdr2rrdr_group_by_calculate_percentage_of_instance(RRDR *r) {
3175
+static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
3176
if(!r->vh)
3177
return;
3178
@@ -3291,7 +3302,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3302
3303
// do the additional passes on RRDRs
3304
RRDR *last_r = r_tmp->group_by.r;
3294
- rrdr2rrdr_group_by_calculate_percentage_of_instance(last_r);
3305
+ rrdr2rrdr_group_by_calculate_percentage_of_group(last_r);
3306
3307
RRDR *r = last_r->group_by.r;
3308
size_t pass = 0;
@@ -3302,7 +3313,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
3313
qt->request.group_by[pass].aggregation,
3314
&last_r->dqp[d], pass);
3315
}
3305
- rrdr2rrdr_group_by_calculate_percentage_of_instance(r);
3316
+ rrdr2rrdr_group_by_calculate_percentage_of_group(r);
3317
3318
last_r = r;
3319
r = last_r->group_by.r;
web/api/queries/query.h
+1
@@ -85,6 +85,7 @@ typedef enum rrdr_group_by_function {
85
RRDR_GROUP_BY_FUNCTION_MIN,
86
RRDR_GROUP_BY_FUNCTION_MAX,
87
RRDR_GROUP_BY_FUNCTION_SUM,
88
+ RRDR_GROUP_BY_FUNCTION_PERCENTAGE,
89
} RRDR_GROUP_BY_FUNCTION;
90
91
RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s);