| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "query-internal.h" |
| 4 | |
| 5 | void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, size_t d_tmp, |
| 6 | RRDR_GROUP_BY_FUNCTION group_by_aggregate_function, |
| 7 | STORAGE_POINT *query_points, size_t pass __maybe_unused) { |
| 8 | if(!r_tmp || r_dst == r_tmp || !(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED)) |
| 9 | return; |
| 10 | |
| 11 | internal_fatal(r_dst->n != r_tmp->n, "QUERY: group-by source and destination do not have the same number of rows"); |
| 12 | internal_fatal(d_dst >= r_dst->d, "QUERY: group-by destination dimension number exceeds destination RRDR size"); |
| 13 | internal_fatal(d_tmp >= r_tmp->d, "QUERY: group-by source dimension number exceeds source RRDR size"); |
| 14 | internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)"); |
| 15 | internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)"); |
| 16 | |
| 17 | bool hidden_dimension_on_percentage_of_group = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh; |
| 18 | |
| 19 | if(!hidden_dimension_on_percentage_of_group) { |
| 20 | r_dst->od[d_dst] |= r_tmp->od[d_tmp]; |
| 21 | storage_point_merge_to(r_dst->dqp[d_dst], *query_points); |
| 22 | } |
| 23 | |
| 24 | // do the group_by |
| 25 | for(size_t i = 0; i != rrdr_rows(r_tmp) ; i++) { |
| 26 | |
| 27 | size_t idx_tmp = i * r_tmp->d + d_tmp; |
| 28 | NETDATA_DOUBLE n_tmp = r_tmp->v[ idx_tmp ]; |
| 29 | RRDR_VALUE_FLAGS o_tmp = r_tmp->o[ idx_tmp ]; |
| 30 | NETDATA_DOUBLE ar_tmp = r_tmp->ar[ idx_tmp ]; |
| 31 | |
| 32 | if(o_tmp & RRDR_VALUE_EMPTY) |
| 33 | continue; |
| 34 | |
| 35 | size_t idx_dst = i * r_dst->d + d_dst; |
| 36 | NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_group) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ]; |
| 37 | RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ]; |
| 38 | NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ]; |
| 39 | uint32_t *gbc = &r_dst->gbc[ idx_dst ]; |
| 40 | |
| 41 | switch(group_by_aggregate_function) { |
| 42 | default: |
| 43 | case RRDR_GROUP_BY_FUNCTION_AVERAGE: |
| 44 | case RRDR_GROUP_BY_FUNCTION_SUM: |
| 45 | case RRDR_GROUP_BY_FUNCTION_PERCENTAGE: |
| 46 | if(isnan(*cn)) |
| 47 | *cn = n_tmp; |
| 48 | else |
| 49 | *cn += n_tmp; |
| 50 | break; |
| 51 | |
| 52 | case RRDR_GROUP_BY_FUNCTION_MIN: |
| 53 | if(isnan(*cn) || n_tmp < *cn) |
| 54 | *cn = n_tmp; |
| 55 | break; |
| 56 | |
| 57 | case RRDR_GROUP_BY_FUNCTION_MAX: |
| 58 | if(isnan(*cn) || n_tmp > *cn) |
| 59 | *cn = n_tmp; |
| 60 | break; |
| 61 | |
| 62 | case RRDR_GROUP_BY_FUNCTION_EXTREMES: |
| 63 | // For extremes, we need to keep track of the value with the maximum absolute value |
| 64 | if(isnan(*cn) || fabsndd(n_tmp) > fabsndd(*cn)) |
| 65 | *cn = n_tmp; |
| 66 | break; |
| 67 | } |
| 68 | |
| 69 | if(!hidden_dimension_on_percentage_of_group) { |
| 70 | *co &= ~RRDR_VALUE_EMPTY; |
| 71 | *co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL)); |
| 72 | *ar += ar_tmp; |
| 73 | (*gbc)++; |
| 74 | } |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | void rrdr2rrdr_group_by_partial_trimming(RRDR *r) { |
| 79 | time_t trimmable_after = r->partial_data_trimming.expected_after; |
| 80 | |
| 81 | // find the point just before the trimmable ones |
| 82 | ssize_t i = (ssize_t)r->n - 1; |
| 83 | for( ; i >= 0 ;i--) { |
| 84 | if (r->t[i] < trimmable_after) |
| 85 | break; |
| 86 | } |
| 87 | |
| 88 | if(unlikely(i < 0)) |
| 89 | return; |
| 90 | |
| 91 | // internal_error(true, "Found trimmable index %zd (from 0 to %zu)", i, r->n - 1); |
| 92 | |
| 93 | size_t last_row_gbc = 0; |
| 94 | for (; i < (ssize_t)r->n; i++) { |
| 95 | size_t row_gbc = 0; |
| 96 | for (size_t d = 0; d < r->d; d++) { |
| 97 | if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) |
| 98 | continue; |
| 99 | |
| 100 | row_gbc += r->gbc[ i * r->d + d ]; |
| 101 | } |
| 102 | |
| 103 | // internal_error(true, "GBC of index %zd is %zu", i, row_gbc); |
| 104 | |
| 105 | if (unlikely(r->t[i] >= trimmable_after && (row_gbc < last_row_gbc || !row_gbc))) { |
| 106 | // discard the rest of the points |
| 107 | // internal_error(true, "Discarding points %zd to %zu", i, r->n - 1); |
| 108 | r->partial_data_trimming.trimmed_after = r->t[i]; |
| 109 | r->rows = i; |
| 110 | break; |
| 111 | } |
| 112 | else |
| 113 | last_row_gbc = row_gbc; |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) { |
| 118 | if(!r->vh) |
| 119 | return; |
| 120 | |
| 121 | if(query_target_aggregatable(r->internal.qt) && query_has_group_by_aggregation_percentage(r->internal.qt)) |
| 122 | return; |
| 123 | |
| 124 | for(size_t i = 0; i < r->n ;i++) { |
| 125 | NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; |
| 126 | NETDATA_DOUBLE *ch = &r->vh[ i * r->d ]; |
| 127 | |
| 128 | for(size_t d = 0; d < r->d ;d++) { |
| 129 | NETDATA_DOUBLE n = cn[d]; |
| 130 | NETDATA_DOUBLE h = ch[d]; |
| 131 | |
| 132 | if(isnan(n)) |
| 133 | cn[d] = 0.0; |
| 134 | |
| 135 | else if(isnan(h)) |
| 136 | cn[d] = 100.0; |
| 137 | |
| 138 | else |
| 139 | cn[d] = n * 100.0 / (n + h); |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | |
| 145 | void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) { |
| 146 | if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt)) |
| 147 | return; |
| 148 | |
| 149 | size_t global_min_max_values = 0; |
| 150 | NETDATA_DOUBLE global_min = NAN, global_max = NAN; |
| 151 | |
| 152 | for(size_t i = 0; i != r->n ;i++) { |
| 153 | NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; |
| 154 | RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; |
| 155 | |
| 156 | NETDATA_DOUBLE total = 0; |
| 157 | for (size_t d = 0; d < r->d; d++) { |
| 158 | if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) |
| 159 | continue; |
| 160 | |
| 161 | if(co[d] & RRDR_VALUE_EMPTY) |
| 162 | continue; |
| 163 | |
| 164 | total += cn[d]; |
| 165 | } |
| 166 | |
| 167 | if(total == 0.0) |
| 168 | total = 1.0; |
| 169 | |
| 170 | for (size_t d = 0; d < r->d; d++) { |
| 171 | if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) |
| 172 | continue; |
| 173 | |
| 174 | if(co[d] & RRDR_VALUE_EMPTY) |
| 175 | continue; |
| 176 | |
| 177 | NETDATA_DOUBLE n = cn[d]; |
| 178 | n = cn[d] = n * 100.0 / total; |
| 179 | |
| 180 | if(unlikely(!global_min_max_values++)) |
| 181 | global_min = global_max = n; |
| 182 | else { |
| 183 | if(n < global_min) |
| 184 | global_min = n; |
| 185 | if(n > global_max) |
| 186 | global_max = n; |
| 187 | } |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | r->view.min = global_min; |
| 192 | r->view.max = global_max; |
| 193 | |
| 194 | if(!r->dview) |
| 195 | // v1 query |
| 196 | return; |
| 197 | |
| 198 | // v2 query |
| 199 | |
| 200 | for (size_t d = 0; d < r->d; d++) { |
| 201 | if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) |
| 202 | continue; |
| 203 | |
| 204 | size_t count = 0; |
| 205 | NETDATA_DOUBLE min = 0.0, max = 0.0, sum = 0.0, ars = 0.0; |
| 206 | for(size_t i = 0; i != r->rows ;i++) { // we use r->rows to respect trimming |
| 207 | size_t idx = i * r->d + d; |
| 208 | |
| 209 | RRDR_VALUE_FLAGS o = r->o[ idx ]; |
| 210 | |
| 211 | if (o & RRDR_VALUE_EMPTY) |
| 212 | continue; |
| 213 | |
| 214 | NETDATA_DOUBLE ar = r->ar[ idx ]; |
| 215 | ars += ar; |
| 216 | |
| 217 | NETDATA_DOUBLE n = r->v[ idx ]; |
| 218 | sum += n; |
| 219 | |
| 220 | if(!count++) |
| 221 | min = max = n; |
| 222 | else { |
| 223 | if(n < min) |
| 224 | min = n; |
| 225 | if(n > max) |
| 226 | max = n; |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | r->dview[d] = (STORAGE_POINT) { |
| 231 | .sum = sum, |
| 232 | .count = count, |
| 233 | .min = min, |
| 234 | .max = max, |
| 235 | .anomaly_count = (size_t)(ars * (NETDATA_DOUBLE)count), |
| 236 | }; |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) { |
| 241 | QUERY_TARGET *qt = r_tmp->internal.qt; |
| 242 | |
| 243 | if(!r_tmp->group_by.r) { |
| 244 | // v1 query |
| 245 | rrd2rrdr_convert_values_to_percentage_of_total(r_tmp); |
| 246 | return r_tmp; |
| 247 | } |
| 248 | // v2 query |
| 249 | |
| 250 | // do the additional passes on RRDRs |
| 251 | RRDR *last_r = r_tmp->group_by.r; |
| 252 | rrdr2rrdr_group_by_calculate_percentage_of_group(last_r); |
| 253 | |
| 254 | RRDR *r = last_r->group_by.r; |
| 255 | size_t pass = 0; |
| 256 | while(r) { |
| 257 | pass++; |
| 258 | for(size_t d = 0; d < last_r->d ;d++) { |
| 259 | rrd2rrdr_group_by_add_metric(r, last_r->dgbs[d], last_r, d, |
| 260 | qt->request.group_by[pass].aggregation, |
| 261 | &last_r->dqp[d], pass); |
| 262 | } |
| 263 | rrdr2rrdr_group_by_calculate_percentage_of_group(r); |
| 264 | |
| 265 | last_r = r; |
| 266 | r = last_r->group_by.r; |
| 267 | } |
| 268 | |
| 269 | // free all RRDRs except the last one |
| 270 | r = r_tmp; |
| 271 | while(r != last_r) { |
| 272 | r_tmp = r->group_by.r; |
| 273 | r->group_by.r = NULL; |
| 274 | rrdr_free(r->internal.owa, r); |
| 275 | r = r_tmp; |
| 276 | } |
| 277 | r = last_r; |
| 278 | |
| 279 | // find the final aggregation |
| 280 | RRDR_GROUP_BY_FUNCTION aggregation = qt->request.group_by[0].aggregation; |
| 281 | for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) |
| 282 | if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE) |
| 283 | aggregation = qt->request.group_by[g].aggregation; |
| 284 | |
| 285 | if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before) |
| 286 | rrdr2rrdr_group_by_partial_trimming(r); |
| 287 | |
| 288 | // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max |
| 289 | size_t global_min_max_values = 0; |
| 290 | size_t dimensions_nonzero = 0; |
| 291 | NETDATA_DOUBLE global_min = NAN, global_max = NAN; |
| 292 | for (size_t d = 0; d < r->d; d++) { |
| 293 | if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) |
| 294 | continue; |
| 295 | |
| 296 | size_t points_nonzero = 0; |
| 297 | NETDATA_DOUBLE min = 0, max = 0, sum = 0, ars = 0; |
| 298 | size_t count = 0; |
| 299 | |
| 300 | for(size_t i = 0; i != r->n ;i++) { |
| 301 | size_t idx = i * r->d + d; |
| 302 | |
| 303 | NETDATA_DOUBLE *cn = &r->v[ idx ]; |
| 304 | RRDR_VALUE_FLAGS *co = &r->o[ idx ]; |
| 305 | NETDATA_DOUBLE *ar = &r->ar[ idx ]; |
| 306 | uint32_t gbc = r->gbc[ idx ]; |
| 307 | |
| 308 | if(likely(gbc)) { |
| 309 | *co &= ~RRDR_VALUE_EMPTY; |
| 310 | |
| 311 | if(gbc != r->dgbc[d]) |
| 312 | *co |= RRDR_VALUE_PARTIAL; |
| 313 | |
| 314 | NETDATA_DOUBLE n; |
| 315 | |
| 316 | sum += *cn; |
| 317 | ars += *ar; |
| 318 | |
| 319 | if(aggregation == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt)) |
| 320 | n = (*cn /= gbc); |
| 321 | else |
| 322 | n = *cn; |
| 323 | |
| 324 | if(!query_target_aggregatable(qt)) |
| 325 | *ar /= gbc; |
| 326 | |
| 327 | if(islessgreater(n, 0.0)) |
| 328 | points_nonzero++; |
| 329 | |
| 330 | if(unlikely(!count)) |
| 331 | min = max = n; |
| 332 | else { |
| 333 | if(n < min) |
| 334 | min = n; |
| 335 | |
| 336 | if(n > max) |
| 337 | max = n; |
| 338 | } |
| 339 | |
| 340 | if(unlikely(!global_min_max_values++)) |
| 341 | global_min = global_max = n; |
| 342 | else { |
| 343 | if(n < global_min) |
| 344 | global_min = n; |
| 345 | |
| 346 | if(n > global_max) |
| 347 | global_max = n; |
| 348 | } |
| 349 | |
| 350 | count += gbc; |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | if(points_nonzero) { |
| 355 | r->od[d] |= RRDR_DIMENSION_NONZERO; |
| 356 | dimensions_nonzero++; |
| 357 | } |
| 358 | |
| 359 | r->dview[d] = (STORAGE_POINT) { |
| 360 | .sum = sum, |
| 361 | .count = count, |
| 362 | .min = min, |
| 363 | .max = max, |
| 364 | .anomaly_count = (size_t)(ars * RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER / 100.0), |
| 365 | }; |
| 366 | } |
| 367 | |
| 368 | r->view.min = global_min; |
| 369 | r->view.max = global_max; |
| 370 | |
| 371 | if(!dimensions_nonzero && (qt->window.options & RRDR_OPTION_NONZERO)) { |
| 372 | // all dimensions are zero |
| 373 | // remove the nonzero option |
| 374 | qt->window.options &= ~RRDR_OPTION_NONZERO; |
| 375 | } |
| 376 | |
| 377 | rrd2rrdr_convert_values_to_percentage_of_total(r); |
| 378 | |
| 379 | // update query instance counts in query host and query context |
| 380 | { |
| 381 | size_t h = 0, c = 0, i = 0; |
| 382 | for(; h < qt->nodes.used ; h++) { |
| 383 | QUERY_NODE *qn = &qt->nodes.array[h]; |
| 384 | |
| 385 | for(; c < qt->contexts.used ;c++) { |
| 386 | QUERY_CONTEXT *qc = &qt->contexts.array[c]; |
| 387 | |
| 388 | if(!rrdcontext_acquired_belongs_to_host(qc->rca, qn->rrdhost)) |
| 389 | break; |
| 390 | |
| 391 | for(; i < qt->instances.used ;i++) { |
| 392 | QUERY_INSTANCE *qi = &qt->instances.array[i]; |
| 393 | |
| 394 | if(!rrdinstance_acquired_belongs_to_context(qi->ria, qc->rca)) |
| 395 | break; |
| 396 | |
| 397 | if(qi->metrics.queried) { |
| 398 | qc->instances.queried++; |
| 399 | qn->instances.queried++; |
| 400 | } |
| 401 | else if(qi->metrics.failed) { |
| 402 | qc->instances.failed++; |
| 403 | qn->instances.failed++; |
| 404 | } |
| 405 | } |
| 406 | } |
| 407 | } |
| 408 | } |
| 409 | |
| 410 | return r; |
| 411 | } |